From beabb6085d42cbb961e3a5dc217fdd840fee4b0d Mon Sep 17 00:00:00 2001 From: Runxi Yu Date: Thu, 5 Mar 2026 17:36:48 +0800 Subject: internal/compress: Import flate and such from klauspost/compress --- internal/compress/LICENSE | 29 + internal/compress/doc.go | 3 + internal/compress/flate/_gen/gen_inflate.go | 303 + internal/compress/flate/deflate.go | 996 +++ internal/compress/flate/deflate_test.go | 706 ++ internal/compress/flate/dict_decoder.go | 181 + internal/compress/flate/dict_decoder_test.go | 139 + internal/compress/flate/example_test.go | 245 + internal/compress/flate/fast_encoder.go | 189 + internal/compress/flate/flate_test.go | 366 + internal/compress/flate/fuzz_test.go | 174 + internal/compress/flate/huffman_bit_writer.go | 1174 +++ internal/compress/flate/huffman_bit_writer_test.go | 381 + internal/compress/flate/huffman_code.go | 417 + internal/compress/flate/huffman_sortByFreq.go | 159 + internal/compress/flate/huffman_sortByLiteral.go | 201 + internal/compress/flate/inflate.go | 865 ++ internal/compress/flate/inflate_gen.go | 1283 +++ internal/compress/flate/inflate_test.go | 302 + internal/compress/flate/level1.go | 215 + internal/compress/flate/level2.go | 214 + internal/compress/flate/level3.go | 241 + internal/compress/flate/level4.go | 221 + internal/compress/flate/level5.go | 705 ++ internal/compress/flate/level6.go | 325 + internal/compress/flate/matchlen_generic.go | 34 + internal/compress/flate/reader_test.go | 108 + internal/compress/flate/regmask_amd64.go | 37 + internal/compress/flate/regmask_other.go | 39 + internal/compress/flate/stateless.go | 325 + .../compress/flate/testdata/fuzz/FuzzEncoding.zip | Bin 0 -> 1213291 bytes .../flate/testdata/fuzz/encode-raw-corpus.zip | Bin 0 -> 683330 bytes .../flate/testdata/huffman-null-max.dyn.expect | Bin 0 -> 78 bytes .../testdata/huffman-null-max.dyn.expect-noinput | Bin 0 -> 78 bytes .../flate/testdata/huffman-null-max.golden | Bin 0 -> 8204 bytes .../compress/flate/testdata/huffman-null-max.in | Bin 0 -> 65535 bytes .../flate/testdata/huffman-null-max.sync.expect | Bin 0 -> 78 bytes .../testdata/huffman-null-max.sync.expect-noinput | Bin 0 -> 78 bytes .../flate/testdata/huffman-null-max.wb.expect | Bin 0 -> 78 bytes .../testdata/huffman-null-max.wb.expect-noinput | Bin 0 -> 78 bytes .../compress/flate/testdata/huffman-pi.dyn.expect | Bin 0 -> 1696 bytes .../flate/testdata/huffman-pi.dyn.expect-noinput | Bin 0 -> 1696 bytes internal/compress/flate/testdata/huffman-pi.golden | Bin 0 -> 1606 bytes internal/compress/flate/testdata/huffman-pi.in | 1 + .../compress/flate/testdata/huffman-pi.sync.expect | Bin 0 -> 1696 bytes .../flate/testdata/huffman-pi.sync.expect-noinput | Bin 0 -> 1696 bytes .../compress/flate/testdata/huffman-pi.wb.expect | Bin 0 -> 1696 bytes .../flate/testdata/huffman-pi.wb.expect-noinput | Bin 0 -> 1696 bytes .../flate/testdata/huffman-rand-1k.dyn.expect | Bin 0 -> 1005 bytes .../testdata/huffman-rand-1k.dyn.expect-noinput | Bin 0 -> 1054 bytes .../compress/flate/testdata/huffman-rand-1k.golden | Bin 0 -> 1005 bytes .../compress/flate/testdata/huffman-rand-1k.in | Bin 0 -> 1000 bytes .../flate/testdata/huffman-rand-1k.sync.expect | Bin 0 -> 1005 bytes .../testdata/huffman-rand-1k.sync.expect-noinput | Bin 0 -> 1054 bytes .../flate/testdata/huffman-rand-1k.wb.expect | Bin 0 -> 1005 bytes .../testdata/huffman-rand-1k.wb.expect-noinput | Bin 0 -> 1054 bytes .../flate/testdata/huffman-rand-limit.dyn.expect | Bin 0 -> 186 bytes .../testdata/huffman-rand-limit.dyn.expect-noinput | Bin 0 -> 186 bytes .../flate/testdata/huffman-rand-limit.golden | Bin 0 -> 246 bytes .../compress/flate/testdata/huffman-rand-limit.in | 4 + .../flate/testdata/huffman-rand-limit.sync.expect | Bin 0 -> 186 bytes .../huffman-rand-limit.sync.expect-noinput | Bin 0 -> 186 bytes .../flate/testdata/huffman-rand-limit.wb.expect | Bin 0 -> 186 bytes .../testdata/huffman-rand-limit.wb.expect-noinput | Bin 0 -> 186 bytes .../flate/testdata/huffman-rand-max.golden | Bin 0 -> 65540 bytes .../compress/flate/testdata/huffman-rand-max.in | Bin 0 -> 65535 bytes .../flate/testdata/huffman-shifts.dyn.expect | Bin 0 -> 32 bytes .../testdata/huffman-shifts.dyn.expect-noinput | Bin 0 -> 32 bytes .../compress/flate/testdata/huffman-shifts.golden | Bin 0 -> 1812 bytes internal/compress/flate/testdata/huffman-shifts.in | 2 + .../flate/testdata/huffman-shifts.sync.expect | Bin 0 -> 32 bytes .../testdata/huffman-shifts.sync.expect-noinput | Bin 0 -> 32 bytes .../flate/testdata/huffman-shifts.wb.expect | Bin 0 -> 32 bytes .../testdata/huffman-shifts.wb.expect-noinput | Bin 0 -> 32 bytes .../flate/testdata/huffman-text-shift.dyn.expect | Bin 0 -> 231 bytes .../testdata/huffman-text-shift.dyn.expect-noinput | Bin 0 -> 231 bytes .../flate/testdata/huffman-text-shift.golden | Bin 0 -> 231 bytes .../compress/flate/testdata/huffman-text-shift.in | 14 + .../flate/testdata/huffman-text-shift.sync.expect | Bin 0 -> 231 bytes .../huffman-text-shift.sync.expect-noinput | Bin 0 -> 231 bytes .../flate/testdata/huffman-text-shift.wb.expect | Bin 0 -> 231 bytes .../testdata/huffman-text-shift.wb.expect-noinput | Bin 0 -> 231 bytes .../flate/testdata/huffman-text.dyn.expect | 1 + .../flate/testdata/huffman-text.dyn.expect-noinput | 1 + .../compress/flate/testdata/huffman-text.golden | 3 + internal/compress/flate/testdata/huffman-text.in | 13 + .../flate/testdata/huffman-text.sync.expect | 1 + .../testdata/huffman-text.sync.expect-noinput | 1 + .../compress/flate/testdata/huffman-text.wb.expect | 1 + .../flate/testdata/huffman-text.wb.expect-noinput | 1 + .../flate/testdata/huffman-zero.dyn.expect | Bin 0 -> 6 bytes .../flate/testdata/huffman-zero.dyn.expect-noinput | Bin 0 -> 6 bytes .../compress/flate/testdata/huffman-zero.golden | Bin 0 -> 51 bytes internal/compress/flate/testdata/huffman-zero.in | 1 + .../flate/testdata/huffman-zero.sync.expect | Bin 0 -> 6 bytes .../testdata/huffman-zero.sync.expect-noinput | Bin 0 -> 6 bytes .../compress/flate/testdata/huffman-zero.wb.expect | Bin 0 -> 6 bytes .../flate/testdata/huffman-zero.wb.expect-noinput | Bin 0 -> 6 bytes .../testdata/null-long-match.dyn.expect-noinput | Bin 0 -> 206 bytes .../testdata/null-long-match.sync.expect-noinput | Bin 0 -> 206 bytes .../testdata/null-long-match.wb.expect-noinput | Bin 0 -> 206 bytes internal/compress/flate/testdata/partial-block | 1 + internal/compress/flate/testdata/regression.zip | Bin 0 -> 483763 bytes internal/compress/flate/testdata/tokens.bin | 63 + internal/compress/flate/token.go | 379 + internal/compress/flate/token_test.go | 54 + internal/compress/flate/writer_test.go | 544 ++ internal/compress/internal/fuzz/helpers.go | 218 + internal/compress/internal/le/le.go | 5 + internal/compress/internal/le/unsafe_disabled.go | 42 + internal/compress/internal/le/unsafe_enabled.go | 52 + .../compress/testdata/Mark.Twain-Tom.Sawyer.txt | 8472 ++++++++++++++++++++ internal/compress/testdata/case1.bin | Bin 0 -> 55 bytes internal/compress/testdata/case2.bin | 1 + internal/compress/testdata/case3.bin | 1 + internal/compress/testdata/crash1.bin | Bin 0 -> 5 bytes internal/compress/testdata/crash2.bin | 1 + internal/compress/testdata/crash3.bin | 1 + internal/compress/testdata/crash4.bin | 1 + internal/compress/testdata/crash5.bin | 1 + internal/compress/testdata/dec-crash6.bin | Bin 0 -> 11 bytes internal/compress/testdata/dec-hang1.bin | 1 + internal/compress/testdata/dec-hang2.bin | 1 + internal/compress/testdata/dec-hang3.bin | 1 + internal/compress/testdata/dec-symlen1.bin | Bin 0 -> 49 bytes internal/compress/testdata/e.txt | 1 + internal/compress/testdata/endnonzero.bin | Bin 0 -> 7 bytes internal/compress/testdata/endzerobits.bin | Bin 0 -> 5 bytes internal/compress/testdata/fse-artifact3.bin | Bin 0 -> 4116 bytes internal/compress/testdata/gettysburg.txt | 29 + internal/compress/testdata/html.txt | 1183 +++ internal/compress/testdata/normcount2.bin | 1 + internal/compress/testdata/pi.txt | 1 + internal/compress/testdata/pngdata.bin | Bin 0 -> 51200 bytes internal/compress/testdata/sharnd.out | Bin 0 -> 100004 bytes internal/compress/zlib/reader.go | 205 + internal/compress/zlib/reader_reset.go | 92 + internal/compress/zlib/reader_test.go | 193 + internal/compress/zlib/writer.go | 204 + internal/compress/zlib/writer_header.go | 71 + internal/compress/zlib/writer_test.go | 185 + internal/zlib/LICENSE | 27 - internal/zlib/reader.go | 205 - internal/zlib/reader_reset.go | 91 - internal/zlib/writer.go | 204 - internal/zlib/writer_header.go | 72 - objectstore/loose/parse.go | 2 +- objectstore/loose/read_header.go | 2 +- objectstore/loose/read_reader.go | 2 +- objectstore/loose/write_writer.go | 2 +- objectstore/packed/entry_inflate.go | 2 +- 151 files changed, 22630 insertions(+), 604 deletions(-) create mode 100644 internal/compress/LICENSE create mode 100644 internal/compress/doc.go create mode 100644 internal/compress/flate/_gen/gen_inflate.go create mode 100644 internal/compress/flate/deflate.go create mode 100644 internal/compress/flate/deflate_test.go create mode 100644 internal/compress/flate/dict_decoder.go create mode 100644 internal/compress/flate/dict_decoder_test.go create mode 100644 internal/compress/flate/example_test.go create mode 100644 internal/compress/flate/fast_encoder.go create mode 100644 internal/compress/flate/flate_test.go create mode 100644 internal/compress/flate/fuzz_test.go create mode 100644 internal/compress/flate/huffman_bit_writer.go create mode 100644 internal/compress/flate/huffman_bit_writer_test.go create mode 100644 internal/compress/flate/huffman_code.go create mode 100644 internal/compress/flate/huffman_sortByFreq.go create mode 100644 internal/compress/flate/huffman_sortByLiteral.go create mode 100644 internal/compress/flate/inflate.go create mode 100644 internal/compress/flate/inflate_gen.go create mode 100644 internal/compress/flate/inflate_test.go create mode 100644 internal/compress/flate/level1.go create mode 100644 internal/compress/flate/level2.go create mode 100644 internal/compress/flate/level3.go create mode 100644 internal/compress/flate/level4.go create mode 100644 internal/compress/flate/level5.go create mode 100644 internal/compress/flate/level6.go create mode 100644 internal/compress/flate/matchlen_generic.go create mode 100644 internal/compress/flate/reader_test.go create mode 100644 internal/compress/flate/regmask_amd64.go create mode 100644 internal/compress/flate/regmask_other.go create mode 100644 internal/compress/flate/stateless.go create mode 100644 internal/compress/flate/testdata/fuzz/FuzzEncoding.zip create mode 100644 internal/compress/flate/testdata/fuzz/encode-raw-corpus.zip create mode 100644 internal/compress/flate/testdata/huffman-null-max.dyn.expect create mode 100644 internal/compress/flate/testdata/huffman-null-max.dyn.expect-noinput create mode 100644 internal/compress/flate/testdata/huffman-null-max.golden create mode 100644 internal/compress/flate/testdata/huffman-null-max.in create mode 100644 internal/compress/flate/testdata/huffman-null-max.sync.expect create mode 100644 internal/compress/flate/testdata/huffman-null-max.sync.expect-noinput create mode 100644 internal/compress/flate/testdata/huffman-null-max.wb.expect create mode 100644 internal/compress/flate/testdata/huffman-null-max.wb.expect-noinput create mode 100644 internal/compress/flate/testdata/huffman-pi.dyn.expect create mode 100644 internal/compress/flate/testdata/huffman-pi.dyn.expect-noinput create mode 100644 internal/compress/flate/testdata/huffman-pi.golden create mode 100644 internal/compress/flate/testdata/huffman-pi.in create mode 100644 internal/compress/flate/testdata/huffman-pi.sync.expect create mode 100644 internal/compress/flate/testdata/huffman-pi.sync.expect-noinput create mode 100644 internal/compress/flate/testdata/huffman-pi.wb.expect create mode 100644 internal/compress/flate/testdata/huffman-pi.wb.expect-noinput create mode 100644 internal/compress/flate/testdata/huffman-rand-1k.dyn.expect create mode 100644 internal/compress/flate/testdata/huffman-rand-1k.dyn.expect-noinput create mode 100644 internal/compress/flate/testdata/huffman-rand-1k.golden create mode 100644 internal/compress/flate/testdata/huffman-rand-1k.in create mode 100644 internal/compress/flate/testdata/huffman-rand-1k.sync.expect create mode 100644 internal/compress/flate/testdata/huffman-rand-1k.sync.expect-noinput create mode 100644 internal/compress/flate/testdata/huffman-rand-1k.wb.expect create mode 100644 internal/compress/flate/testdata/huffman-rand-1k.wb.expect-noinput create mode 100644 internal/compress/flate/testdata/huffman-rand-limit.dyn.expect create mode 100644 internal/compress/flate/testdata/huffman-rand-limit.dyn.expect-noinput create mode 100644 internal/compress/flate/testdata/huffman-rand-limit.golden create mode 100644 internal/compress/flate/testdata/huffman-rand-limit.in create mode 100644 internal/compress/flate/testdata/huffman-rand-limit.sync.expect create mode 100644 internal/compress/flate/testdata/huffman-rand-limit.sync.expect-noinput create mode 100644 internal/compress/flate/testdata/huffman-rand-limit.wb.expect create mode 100644 internal/compress/flate/testdata/huffman-rand-limit.wb.expect-noinput create mode 100644 internal/compress/flate/testdata/huffman-rand-max.golden create mode 100644 internal/compress/flate/testdata/huffman-rand-max.in create mode 100644 internal/compress/flate/testdata/huffman-shifts.dyn.expect create mode 100644 internal/compress/flate/testdata/huffman-shifts.dyn.expect-noinput create mode 100644 internal/compress/flate/testdata/huffman-shifts.golden create mode 100644 internal/compress/flate/testdata/huffman-shifts.in create mode 100644 internal/compress/flate/testdata/huffman-shifts.sync.expect create mode 100644 internal/compress/flate/testdata/huffman-shifts.sync.expect-noinput create mode 100644 internal/compress/flate/testdata/huffman-shifts.wb.expect create mode 100644 internal/compress/flate/testdata/huffman-shifts.wb.expect-noinput create mode 100644 internal/compress/flate/testdata/huffman-text-shift.dyn.expect create mode 100644 internal/compress/flate/testdata/huffman-text-shift.dyn.expect-noinput create mode 100644 internal/compress/flate/testdata/huffman-text-shift.golden create mode 100644 internal/compress/flate/testdata/huffman-text-shift.in create mode 100644 internal/compress/flate/testdata/huffman-text-shift.sync.expect create mode 100644 internal/compress/flate/testdata/huffman-text-shift.sync.expect-noinput create mode 100644 internal/compress/flate/testdata/huffman-text-shift.wb.expect create mode 100644 internal/compress/flate/testdata/huffman-text-shift.wb.expect-noinput create mode 100644 internal/compress/flate/testdata/huffman-text.dyn.expect create mode 100644 internal/compress/flate/testdata/huffman-text.dyn.expect-noinput create mode 100644 internal/compress/flate/testdata/huffman-text.golden create mode 100644 internal/compress/flate/testdata/huffman-text.in create mode 100644 internal/compress/flate/testdata/huffman-text.sync.expect create mode 100644 internal/compress/flate/testdata/huffman-text.sync.expect-noinput create mode 100644 internal/compress/flate/testdata/huffman-text.wb.expect create mode 100644 internal/compress/flate/testdata/huffman-text.wb.expect-noinput create mode 100644 internal/compress/flate/testdata/huffman-zero.dyn.expect create mode 100644 internal/compress/flate/testdata/huffman-zero.dyn.expect-noinput create mode 100644 internal/compress/flate/testdata/huffman-zero.golden create mode 100644 internal/compress/flate/testdata/huffman-zero.in create mode 100644 internal/compress/flate/testdata/huffman-zero.sync.expect create mode 100644 internal/compress/flate/testdata/huffman-zero.sync.expect-noinput create mode 100644 internal/compress/flate/testdata/huffman-zero.wb.expect create mode 100644 internal/compress/flate/testdata/huffman-zero.wb.expect-noinput create mode 100644 internal/compress/flate/testdata/null-long-match.dyn.expect-noinput create mode 100644 internal/compress/flate/testdata/null-long-match.sync.expect-noinput create mode 100644 internal/compress/flate/testdata/null-long-match.wb.expect-noinput create mode 100644 internal/compress/flate/testdata/partial-block create mode 100644 internal/compress/flate/testdata/regression.zip create mode 100644 internal/compress/flate/testdata/tokens.bin create mode 100644 internal/compress/flate/token.go create mode 100644 internal/compress/flate/token_test.go create mode 100644 internal/compress/flate/writer_test.go create mode 100644 internal/compress/internal/fuzz/helpers.go create mode 100644 internal/compress/internal/le/le.go create mode 100644 internal/compress/internal/le/unsafe_disabled.go create mode 100644 internal/compress/internal/le/unsafe_enabled.go create mode 100644 internal/compress/testdata/Mark.Twain-Tom.Sawyer.txt create mode 100644 internal/compress/testdata/case1.bin create mode 100644 internal/compress/testdata/case2.bin create mode 100644 internal/compress/testdata/case3.bin create mode 100644 internal/compress/testdata/crash1.bin create mode 100644 internal/compress/testdata/crash2.bin create mode 100644 internal/compress/testdata/crash3.bin create mode 100644 internal/compress/testdata/crash4.bin create mode 100644 internal/compress/testdata/crash5.bin create mode 100644 internal/compress/testdata/dec-crash6.bin create mode 100644 internal/compress/testdata/dec-hang1.bin create mode 100644 internal/compress/testdata/dec-hang2.bin create mode 100644 internal/compress/testdata/dec-hang3.bin create mode 100644 internal/compress/testdata/dec-symlen1.bin create mode 100644 internal/compress/testdata/e.txt create mode 100644 internal/compress/testdata/endnonzero.bin create mode 100644 internal/compress/testdata/endzerobits.bin create mode 100644 internal/compress/testdata/fse-artifact3.bin create mode 100644 internal/compress/testdata/gettysburg.txt create mode 100644 internal/compress/testdata/html.txt create mode 100644 internal/compress/testdata/normcount2.bin create mode 100644 internal/compress/testdata/pi.txt create mode 100644 internal/compress/testdata/pngdata.bin create mode 100644 internal/compress/testdata/sharnd.out create mode 100644 internal/compress/zlib/reader.go create mode 100644 internal/compress/zlib/reader_reset.go create mode 100644 internal/compress/zlib/reader_test.go create mode 100644 internal/compress/zlib/writer.go create mode 100644 internal/compress/zlib/writer_header.go create mode 100644 internal/compress/zlib/writer_test.go delete mode 100644 internal/zlib/LICENSE delete mode 100644 internal/zlib/reader.go delete mode 100644 internal/zlib/reader_reset.go delete mode 100644 internal/zlib/writer.go delete mode 100644 internal/zlib/writer_header.go diff --git a/internal/compress/LICENSE b/internal/compress/LICENSE new file mode 100644 index 00000000..a013710f --- /dev/null +++ b/internal/compress/LICENSE @@ -0,0 +1,29 @@ +Copyright (c) 2012 The Go Authors. All rights reserved. +Copyright (c) 2019 Klaus Post. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + * Neither the name of Google Inc. nor the names of its +contributors may be used to endorse or promote products derived from +this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + diff --git a/internal/compress/doc.go b/internal/compress/doc.go new file mode 100644 index 00000000..dbc7fd23 --- /dev/null +++ b/internal/compress/doc.go @@ -0,0 +1,3 @@ +// Package compress encapsulates custom compression algorithms. + +package compress diff --git a/internal/compress/flate/_gen/gen_inflate.go b/internal/compress/flate/_gen/gen_inflate.go new file mode 100644 index 00000000..33f14005 --- /dev/null +++ b/internal/compress/flate/_gen/gen_inflate.go @@ -0,0 +1,303 @@ +//go:build generate +// +build generate + +//go:generate go run $GOFILE +//go:generate go fmt ../inflate_gen.go + +package main + +import ( + "os" + "strings" +) + +func main() { + f, err := os.Create("../inflate_gen.go") + if err != nil { + panic(err) + } + defer f.Close() + types := []string{"*bytes.Buffer", "*bytes.Reader", "*bufio.Reader", "*strings.Reader", "Reader"} + names := []string{"BytesBuffer", "BytesReader", "BufioReader", "StringsReader", "GenericReader"} + imports := []string{"bytes", "bufio", "fmt", "strings", "math/bits"} + f.WriteString(`// Code generated by go generate gen_inflate.go. DO NOT EDIT. + +package flate + +import ( +`) + + for _, imp := range imports { + f.WriteString("\t\"" + imp + "\"\n") + } + f.WriteString(")\n\n") + + template := ` + +// Decode a single Huffman block from f. +// hl and hd are the Huffman states for the lit/length values +// and the distance values, respectively. If hd == nil, using the +// fixed distance encoding associated with fixed Huffman blocks. +func (f *decompressor) $FUNCNAME$() { + const ( + stateInit = iota // Zero value must be stateInit + stateDict + ) + fr := f.r.($TYPE$) + + // Optimization. Compiler isn't smart enough to keep f.b,f.nb in registers, + // but is smart enough to keep local variables in registers, so use nb and b, + // inline call to moreBits and reassign b,nb back to f on return. + fnb, fb, dict := f.nb, f.b, &f.dict + + switch f.stepState { + case stateInit: + goto readLiteral + case stateDict: + goto copyHistory + } + +readLiteral: + // Read literal and/or (length, distance) according to RFC section 3.2.3. + { + var v int + { + // Inlined v, err := f.huffSym(f.hl) + // Since a huffmanDecoder can be empty or be composed of a degenerate tree + // with single element, huffSym must error on these two edge cases. In both + // cases, the chunks slice will be 0 for the invalid sequence, leading it + // satisfy the n == 0 check below. + n := uint(f.hl.maxRead) + for { + for fnb < n { + c, err := fr.ReadByte() + if err != nil { + f.b, f.nb = fb, fnb + f.err = noEOF(err) + return + } + f.roffset++ + fb |= uint32(c) << (fnb & regSizeMaskUint32) + fnb += 8 + } + chunk := f.hl.chunks[fb&(huffmanNumChunks-1)] + n = uint(chunk & huffmanCountMask) + if n > huffmanChunkBits { + chunk = f.hl.links[chunk>>huffmanValueShift][(fb>>huffmanChunkBits)&f.hl.linkMask] + n = uint(chunk & huffmanCountMask) + } + if n <= fnb { + if n == 0 { + f.b, f.nb = fb, fnb + if debugDecode { + fmt.Println("huffsym: n==0") + } + f.err = CorruptInputError(f.roffset) + return + } + fb = fb >> (n & regSizeMaskUint32) + fnb = fnb - n + v = int(chunk >> huffmanValueShift) + break + } + } + } + + var length int + switch { + case v < 256: + dict.writeByte(byte(v)) + if dict.availWrite() == 0 { + f.toRead = dict.readFlush() + f.step = $FUNCNAME$ + f.stepState = stateInit + f.b, f.nb = fb, fnb + return + } + goto readLiteral + case v == 256: + f.b, f.nb = fb, fnb + f.finishBlock() + return + // otherwise, reference to older data + case v < 265: + length = v - (257 - 3) + case v < maxNumLit: + val := decCodeToLen[(v - 257)] + length = int(val.length) + 3 + n := uint(val.extra) + for fnb < n { + c, err := fr.ReadByte() + if err != nil { + f.b, f.nb = fb, fnb + if debugDecode { + fmt.Println("morebits n>0:", err) + } + f.err = err + return + } + f.roffset++ + fb |= uint32(c) << (fnb®SizeMaskUint32) + fnb += 8 + } + length += int(fb & bitMask32[n]) + fb >>= n & regSizeMaskUint32 + fnb -= n + default: + if debugDecode { + fmt.Println(v, ">= maxNumLit") + } + f.err = CorruptInputError(f.roffset) + f.b, f.nb = fb, fnb + return + } + + var dist uint32 + if f.hd == nil { + for fnb < 5 { + c, err := fr.ReadByte() + if err != nil { + f.b, f.nb = fb, fnb + if debugDecode { + fmt.Println("morebits f.nb<5:", err) + } + f.err = err + return + } + f.roffset++ + fb |= uint32(c) << (fnb®SizeMaskUint32) + fnb += 8 + } + dist = uint32(bits.Reverse8(uint8(fb & 0x1F << 3))) + fb >>= 5 + fnb -= 5 + } else { + // Since a huffmanDecoder can be empty or be composed of a degenerate tree + // with single element, huffSym must error on these two edge cases. In both + // cases, the chunks slice will be 0 for the invalid sequence, leading it + // satisfy the n == 0 check below. + n := uint(f.hd.maxRead) + // Optimization. Compiler isn't smart enough to keep f.b,f.nb in registers, + // but is smart enough to keep local variables in registers, so use nb and b, + // inline call to moreBits and reassign b,nb back to f on return. + for { + for fnb < n { + c, err := fr.ReadByte() + if err != nil { + f.b, f.nb = fb, fnb + f.err = noEOF(err) + return + } + f.roffset++ + fb |= uint32(c) << (fnb & regSizeMaskUint32) + fnb += 8 + } + chunk := f.hd.chunks[fb&(huffmanNumChunks-1)] + n = uint(chunk & huffmanCountMask) + if n > huffmanChunkBits { + chunk = f.hd.links[chunk>>huffmanValueShift][(fb>>huffmanChunkBits)&f.hd.linkMask] + n = uint(chunk & huffmanCountMask) + } + if n <= fnb { + if n == 0 { + f.b, f.nb = fb, fnb + if debugDecode { + fmt.Println("huffsym: n==0") + } + f.err = CorruptInputError(f.roffset) + return + } + fb = fb >> (n & regSizeMaskUint32) + fnb = fnb - n + dist = uint32(chunk >> huffmanValueShift) + break + } + } + } + + switch { + case dist < 4: + dist++ + case dist < maxNumDist: + nb := uint(dist-2) >> 1 + // have 1 bit in bottom of dist, need nb more. + extra := (dist & 1) << (nb & regSizeMaskUint32) + for fnb < nb { + c, err := fr.ReadByte() + if err != nil { + f.b, f.nb = fb, fnb + if debugDecode { + fmt.Println("morebits f.nb>= nb & regSizeMaskUint32 + fnb -= nb + dist = 1<<((nb+1)®SizeMaskUint32) + 1 + extra + // slower: dist = bitMask32[nb+1] + 2 + extra + default: + f.b, f.nb = fb, fnb + if debugDecode { + fmt.Println("dist too big:", dist, maxNumDist) + } + f.err = CorruptInputError(f.roffset) + return + } + + // No check on length; encoding can be prescient. + if dist > uint32(dict.histSize()) { + f.b, f.nb = fb, fnb + if debugDecode { + fmt.Println("dist > dict.histSize():", dist, dict.histSize()) + } + f.err = CorruptInputError(f.roffset) + return + } + + f.copyLen, f.copyDist = length, int(dist) + goto copyHistory + } + +copyHistory: + // Perform a backwards copy according to RFC section 3.2.3. + { + cnt := dict.tryWriteCopy(f.copyDist, f.copyLen) + if cnt == 0 { + cnt = dict.writeCopy(f.copyDist, f.copyLen) + } + f.copyLen -= cnt + + if dict.availWrite() == 0 || f.copyLen > 0 { + f.toRead = dict.readFlush() + f.step = $FUNCNAME$ // We need to continue this work + f.stepState = stateDict + f.b, f.nb = fb, fnb + return + } + goto readLiteral + } + // Not reached +} + +` + for i, t := range types { + s := strings.Replace(template, "$FUNCNAME$", "huffman"+names[i], -1) + s = strings.Replace(s, "$TYPE$", t, -1) + f.WriteString(s) + } + f.WriteString("func (f *decompressor) huffmanBlockDecoder() {\n") + f.WriteString("\tswitch f.r.(type) {\n") + for i, t := range types { + f.WriteString("\t\tcase " + t + ":\n") + f.WriteString("\t\t\tf.huffman" + names[i] + "()\n") + } + f.WriteString("\t\tdefault:\n") + f.WriteString("\t\t\tf.huffmanGenericReader()\n") + f.WriteString("\t}\n}\n") +} diff --git a/internal/compress/flate/deflate.go b/internal/compress/flate/deflate.go new file mode 100644 index 00000000..ac4a2344 --- /dev/null +++ b/internal/compress/flate/deflate.go @@ -0,0 +1,996 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Copyright (c) 2015 Klaus Post +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package flate + +import ( + "errors" + "fmt" + "io" + "math" + + "codeberg.org/lindenii/furgit/internal/compress/internal/le" +) + +const ( + NoCompression = 0 + BestSpeed = 1 + BestCompression = 9 + DefaultCompression = -1 + + // HuffmanOnly disables Lempel-Ziv match searching and only performs Huffman + // entropy encoding. This mode is useful in compressing data that has + // already been compressed with an LZ style algorithm (e.g. Snappy or LZ4) + // that lacks an entropy encoder. Compression gains are achieved when + // certain bytes in the input stream occur more frequently than others. + // + // Note that HuffmanOnly produces a compressed output that is + // RFC 1951 compliant. That is, any valid DEFLATE decompressor will + // continue to be able to decompress this output. + HuffmanOnly = -2 + ConstantCompression = HuffmanOnly // compatibility alias. + + logWindowSize = 15 + windowSize = 1 << logWindowSize + windowMask = windowSize - 1 + logMaxOffsetSize = 15 // Standard DEFLATE + minMatchLength = 4 // The smallest match that the compressor looks for + maxMatchLength = 258 // The longest match for the compressor + minOffsetSize = 1 // The shortest offset that makes any sense + + // The maximum number of tokens we will encode at the time. + // Smaller sizes usually creates less optimal blocks. + // Bigger can make context switching slow. + // We use this for levels 7-9, so we make it big. + maxFlateBlockTokens = 1 << 15 + maxStoreBlockSize = 65535 + hashBits = 17 // After 17 performance degrades + hashSize = 1 << hashBits + hashMask = (1 << hashBits) - 1 + hashShift = (hashBits + minMatchLength - 1) / minMatchLength + maxHashOffset = 1 << 28 + + skipNever = math.MaxInt32 + + debugDeflate = false +) + +type compressionLevel struct { + good, lazy, nice, chain, fastSkipHashing, level int +} + +// Compression levels have been rebalanced from zlib deflate defaults +// to give a bigger spread in speed and compression. +// See https://blog.klauspost.com/rebalancing-deflate-compression-levels/ +var levels = []compressionLevel{ + {}, // 0 + // Level 1-6 uses specialized algorithm - values not used + {0, 0, 0, 0, 0, 1}, + {0, 0, 0, 0, 0, 2}, + {0, 0, 0, 0, 0, 3}, + {0, 0, 0, 0, 0, 4}, + {0, 0, 0, 0, 0, 5}, + {0, 0, 0, 0, 0, 6}, + // Levels 7-9 use increasingly more lazy matching + // and increasingly stringent conditions for "good enough". + {8, 12, 16, 24, skipNever, 7}, + {16, 30, 40, 64, skipNever, 8}, + {32, 258, 258, 1024, skipNever, 9}, +} + +// advancedState contains state for the advanced levels, with bigger hash tables, etc. +type advancedState struct { + // deflate state + length int + offset int + maxInsertIndex int + chainHead int + hashOffset int + + ii uint16 // position of last match, intended to overflow to reset. + + // input window: unprocessed data is window[index:windowEnd] + index int + hashMatch [maxMatchLength + minMatchLength]uint32 + + // Input hash chains + // hashHead[hashValue] contains the largest inputIndex with the specified hash value + // If hashHead[hashValue] is within the current window, then + // hashPrev[hashHead[hashValue] & windowMask] contains the previous index + // with the same hash value. + hashHead [hashSize]uint32 + hashPrev [windowSize]uint32 +} + +type compressor struct { + compressionLevel + + h *huffmanEncoder + w *huffmanBitWriter + + // compression algorithm + fill func(*compressor, []byte) int // copy data to window + step func(*compressor) // process window + + window []byte + windowEnd int + blockStart int // window index where current tokens start + err error + + // queued output tokens + tokens tokens + fast fastEnc + state *advancedState + + sync bool // requesting flush + byteAvailable bool // if true, still need to process window[index-1]. +} + +func (d *compressor) fillDeflate(b []byte) int { + s := d.state + if s.index >= 2*windowSize-(minMatchLength+maxMatchLength) { + // shift the window by windowSize + //copy(d.window[:], d.window[windowSize:2*windowSize]) + *(*[windowSize]byte)(d.window) = *(*[windowSize]byte)(d.window[windowSize:]) + s.index -= windowSize + d.windowEnd -= windowSize + if d.blockStart >= windowSize { + d.blockStart -= windowSize + } else { + d.blockStart = math.MaxInt32 + } + s.hashOffset += windowSize + if s.hashOffset > maxHashOffset { + delta := s.hashOffset - 1 + s.hashOffset -= delta + s.chainHead -= delta + // Iterate over slices instead of arrays to avoid copying + // the entire table onto the stack (Issue #18625). + for i, v := range s.hashPrev[:] { + if int(v) > delta { + s.hashPrev[i] = uint32(int(v) - delta) + } else { + s.hashPrev[i] = 0 + } + } + for i, v := range s.hashHead[:] { + if int(v) > delta { + s.hashHead[i] = uint32(int(v) - delta) + } else { + s.hashHead[i] = 0 + } + } + } + } + n := copy(d.window[d.windowEnd:], b) + d.windowEnd += n + return n +} + +func (d *compressor) writeBlock(tok *tokens, index int, eof bool) error { + if index > 0 || eof { + var window []byte + if d.blockStart <= index { + window = d.window[d.blockStart:index] + } + d.blockStart = index + //d.w.writeBlock(tok, eof, window) + d.w.writeBlockDynamic(tok, eof, window, d.sync) + return d.w.err + } + return nil +} + +// writeBlockSkip writes the current block and uses the number of tokens +// to determine if the block should be stored on no matches, or +// only huffman encoded. +func (d *compressor) writeBlockSkip(tok *tokens, index int, eof bool) error { + if index > 0 || eof { + if d.blockStart <= index { + window := d.window[d.blockStart:index] + // If we removed less than a 64th of all literals + // we huffman compress the block. + if int(tok.n) > len(window)-int(tok.n>>6) { + d.w.writeBlockHuff(eof, window, d.sync) + } else { + // Write a dynamic huffman block. + d.w.writeBlockDynamic(tok, eof, window, d.sync) + } + } else { + d.w.writeBlock(tok, eof, nil) + } + d.blockStart = index + return d.w.err + } + return nil +} + +// fillWindow will fill the current window with the supplied +// dictionary and calculate all hashes. +// This is much faster than doing a full encode. +// Should only be used after a start/reset. +func (d *compressor) fillWindow(b []byte) { + // Do not fill window if we are in store-only or huffman mode. + if d.level <= 0 && d.level > -MinCustomWindowSize { + return + } + if d.fast != nil { + // encode the last data, but discard the result + if len(b) > maxMatchOffset { + b = b[len(b)-maxMatchOffset:] + } + d.fast.Encode(&d.tokens, b) + d.tokens.Reset() + return + } + s := d.state + // If we are given too much, cut it. + if len(b) > windowSize { + b = b[len(b)-windowSize:] + } + // Add all to window. + n := copy(d.window[d.windowEnd:], b) + + // Calculate 256 hashes at the time (more L1 cache hits) + loops := (n + 256 - minMatchLength) / 256 + for j := range loops { + startindex := j * 256 + end := min(startindex+256+minMatchLength-1, n) + tocheck := d.window[startindex:end] + dstSize := len(tocheck) - minMatchLength + 1 + + if dstSize <= 0 { + continue + } + + dst := s.hashMatch[:dstSize] + bulkHash4(tocheck, dst) + var newH uint32 + for i, val := range dst { + di := i + startindex + newH = val & hashMask + // Get previous value with the same hash. + // Our chain should point to the previous value. + s.hashPrev[di&windowMask] = s.hashHead[newH] + // Set the head of the hash chain to us. + s.hashHead[newH] = uint32(di + s.hashOffset) + } + } + // Update window information. + d.windowEnd += n + s.index = n +} + +// Try to find a match starting at index whose length is greater than prevSize. +// We only look at chainCount possibilities before giving up. +// pos = s.index, prevHead = s.chainHead-s.hashOffset, prevLength=minMatchLength-1, lookahead +func (d *compressor) findMatch(pos int, prevHead int, lookahead int) (length, offset int, ok bool) { + minMatchLook := min(lookahead, maxMatchLength) + + win := d.window[0 : pos+minMatchLook] + + // We quit when we get a match that's at least nice long + nice := min(d.nice, len(win)-pos) + + // If we've got a match that's good enough, only look in 1/4 the chain. + tries := d.chain + length = minMatchLength - 1 + + wEnd := win[pos+length] + wPos := win[pos:] + minIndex := max(pos-windowSize, 0) + offset = 0 + + if d.chain < 100 { + for i := prevHead; tries > 0; tries-- { + if wEnd == win[i+length] { + n := matchLen(win[i:i+minMatchLook], wPos) + if n > length { + length = n + offset = pos - i + ok = true + if n >= nice { + // The match is good enough that we don't try to find a better one. + break + } + wEnd = win[pos+n] + } + } + if i <= minIndex { + // hashPrev[i & windowMask] has already been overwritten, so stop now. + break + } + i = int(d.state.hashPrev[i&windowMask]) - d.state.hashOffset + if i < minIndex { + break + } + } + return + } + + // Minimum gain to accept a match. + cGain := 4 + + // Some like it higher (CSV), some like it lower (JSON) + const baseCost = 3 + // Base is 4 bytes at with an additional cost. + // Matches must be better than this. + + for i := prevHead; tries > 0; tries-- { + if wEnd == win[i+length] { + n := matchLen(win[i:i+minMatchLook], wPos) + if n > length { + // Calculate gain. Estimate + newGain := d.h.bitLengthRaw(wPos[:n]) - int(offsetExtraBits[offsetCode(uint32(pos-i))]) - baseCost - int(lengthExtraBits[lengthCodes[(n-3)&255]]) + + //fmt.Println("gain:", newGain, "prev:", cGain, "raw:", d.h.bitLengthRaw(wPos[:n]), "this-len:", n, "prev-len:", length) + if newGain > cGain { + length = n + offset = pos - i + cGain = newGain + ok = true + if n >= nice { + // The match is good enough that we don't try to find a better one. + break + } + wEnd = win[pos+n] + } + } + } + if i <= minIndex { + // hashPrev[i & windowMask] has already been overwritten, so stop now. + break + } + i = int(d.state.hashPrev[i&windowMask]) - d.state.hashOffset + if i < minIndex { + break + } + } + return +} + +func (d *compressor) writeStoredBlock(buf []byte) error { + if d.w.writeStoredHeader(len(buf), false); d.w.err != nil { + return d.w.err + } + d.w.writeBytes(buf) + return d.w.err +} + +// hash4 returns a hash representation of the first 4 bytes +// of the supplied slice. +// The caller must ensure that len(b) >= 4. +func hash4(b []byte) uint32 { + return hash4u(le.Load32(b, 0), hashBits) +} + +// hash4 returns the hash of u to fit in a hash table with h bits. +// Preferably h should be a constant and should always be <32. +func hash4u(u uint32, h uint8) uint32 { + return (u * prime4bytes) >> (32 - h) +} + +// bulkHash4 will compute hashes using the same +// algorithm as hash4 +func bulkHash4(b []byte, dst []uint32) { + if len(b) < 4 { + return + } + hb := le.Load32(b, 0) + + dst[0] = hash4u(hb, hashBits) + end := len(b) - 4 + 1 + for i := 1; i < end; i++ { + hb = (hb >> 8) | uint32(b[i+3])<<24 + dst[i] = hash4u(hb, hashBits) + } +} + +func (d *compressor) initDeflate() { + d.window = make([]byte, 2*windowSize) + d.byteAvailable = false + d.err = nil + if d.state == nil { + return + } + s := d.state + s.index = 0 + s.hashOffset = 1 + s.length = minMatchLength - 1 + s.offset = 0 + s.chainHead = -1 +} + +// deflateLazy is the same as deflate, but with d.fastSkipHashing == skipNever, +// meaning it always has lazy matching on. +func (d *compressor) deflateLazy() { + s := d.state + // Sanity enables additional runtime tests. + // It's intended to be used during development + // to supplement the currently ad-hoc unit tests. + const sanity = debugDeflate + + if d.windowEnd-s.index < minMatchLength+maxMatchLength && !d.sync { + return + } + if d.windowEnd != s.index && d.chain > 100 { + // Get literal huffman coder. + if d.h == nil { + d.h = newHuffmanEncoder(maxFlateBlockTokens) + } + var tmp [256]uint16 + toIndex := d.window[s.index:d.windowEnd] + toIndex = toIndex[:min(len(toIndex), maxFlateBlockTokens)] + for _, v := range toIndex { + tmp[v]++ + } + d.h.generate(tmp[:], 15) + } + + s.maxInsertIndex = d.windowEnd - (minMatchLength - 1) + + for { + if sanity && s.index > d.windowEnd { + panic("index > windowEnd") + } + lookahead := d.windowEnd - s.index + if lookahead < minMatchLength+maxMatchLength { + if !d.sync { + return + } + if sanity && s.index > d.windowEnd { + panic("index > windowEnd") + } + if lookahead == 0 { + // Flush current output block if any. + if d.byteAvailable { + // There is still one pending token that needs to be flushed + d.tokens.AddLiteral(d.window[s.index-1]) + d.byteAvailable = false + } + if d.tokens.n > 0 { + if d.err = d.writeBlock(&d.tokens, s.index, false); d.err != nil { + return + } + d.tokens.Reset() + } + return + } + } + if s.index < s.maxInsertIndex { + // Update the hash + hash := hash4(d.window[s.index:]) + ch := s.hashHead[hash] + s.chainHead = int(ch) + s.hashPrev[s.index&windowMask] = ch + s.hashHead[hash] = uint32(s.index + s.hashOffset) + } + prevLength := s.length + prevOffset := s.offset + s.length = minMatchLength - 1 + s.offset = 0 + minIndex := max(s.index-windowSize, 0) + + if s.chainHead-s.hashOffset >= minIndex && lookahead > prevLength && prevLength < d.lazy { + if newLength, newOffset, ok := d.findMatch(s.index, s.chainHead-s.hashOffset, lookahead); ok { + s.length = newLength + s.offset = newOffset + } + } + + if prevLength >= minMatchLength && s.length <= prevLength { + // No better match, but check for better match at end... + // + // Skip forward a number of bytes. + // Offset of 2 seems to yield best results. 3 is sometimes better. + const checkOff = 2 + + // Check all, except full length + if prevLength < maxMatchLength-checkOff { + prevIndex := s.index - 1 + if prevIndex+prevLength < s.maxInsertIndex { + end := min(lookahead, maxMatchLength+checkOff) + end += prevIndex + + // Hash at match end. + h := hash4(d.window[prevIndex+prevLength:]) + ch2 := int(s.hashHead[h]) - s.hashOffset - prevLength + if prevIndex-ch2 != prevOffset && ch2 > minIndex+checkOff { + length := matchLen(d.window[prevIndex+checkOff:end], d.window[ch2+checkOff:]) + // It seems like a pure length metric is best. + if length > prevLength { + prevLength = length + prevOffset = prevIndex - ch2 + + // Extend back... + for i := checkOff - 1; i >= 0; i-- { + if prevLength >= maxMatchLength || d.window[prevIndex+i] != d.window[ch2+i] { + // Emit tokens we "owe" + for j := 0; j <= i; j++ { + d.tokens.AddLiteral(d.window[prevIndex+j]) + if d.tokens.n == maxFlateBlockTokens { + // The block includes the current character + if d.err = d.writeBlock(&d.tokens, s.index, false); d.err != nil { + return + } + d.tokens.Reset() + } + s.index++ + if s.index < s.maxInsertIndex { + h := hash4(d.window[s.index:]) + ch := s.hashHead[h] + s.chainHead = int(ch) + s.hashPrev[s.index&windowMask] = ch + s.hashHead[h] = uint32(s.index + s.hashOffset) + } + } + break + } else { + prevLength++ + } + } + } else if false { + // Check one further ahead. + // Only rarely better, disabled for now. + prevIndex++ + h := hash4(d.window[prevIndex+prevLength:]) + ch2 := int(s.hashHead[h]) - s.hashOffset - prevLength + if prevIndex-ch2 != prevOffset && ch2 > minIndex+checkOff { + length := matchLen(d.window[prevIndex+checkOff:end], d.window[ch2+checkOff:]) + // It seems like a pure length metric is best. + if length > prevLength+checkOff { + prevLength = length + prevOffset = prevIndex - ch2 + prevIndex-- + + // Extend back... + for i := checkOff; i >= 0; i-- { + if prevLength >= maxMatchLength || d.window[prevIndex+i] != d.window[ch2+i-1] { + // Emit tokens we "owe" + for j := 0; j <= i; j++ { + d.tokens.AddLiteral(d.window[prevIndex+j]) + if d.tokens.n == maxFlateBlockTokens { + // The block includes the current character + if d.err = d.writeBlock(&d.tokens, s.index, false); d.err != nil { + return + } + d.tokens.Reset() + } + s.index++ + if s.index < s.maxInsertIndex { + h := hash4(d.window[s.index:]) + ch := s.hashHead[h] + s.chainHead = int(ch) + s.hashPrev[s.index&windowMask] = ch + s.hashHead[h] = uint32(s.index + s.hashOffset) + } + } + break + } else { + prevLength++ + } + } + } + } + } + } + } + } + // There was a match at the previous step, and the current match is + // not better. Output the previous match. + d.tokens.AddMatch(uint32(prevLength-3), uint32(prevOffset-minOffsetSize)) + + // Insert in the hash table all strings up to the end of the match. + // index and index-1 are already inserted. If there is not enough + // lookahead, the last two strings are not inserted into the hash + // table. + newIndex := s.index + prevLength - 1 + // Calculate missing hashes + end := min(newIndex, s.maxInsertIndex) + end += minMatchLength - 1 + startindex := min(s.index+1, s.maxInsertIndex) + tocheck := d.window[startindex:end] + dstSize := len(tocheck) - minMatchLength + 1 + if dstSize > 0 { + dst := s.hashMatch[:dstSize] + bulkHash4(tocheck, dst) + var newH uint32 + for i, val := range dst { + di := i + startindex + newH = val & hashMask + // Get previous value with the same hash. + // Our chain should point to the previous value. + s.hashPrev[di&windowMask] = s.hashHead[newH] + // Set the head of the hash chain to us. + s.hashHead[newH] = uint32(di + s.hashOffset) + } + } + + s.index = newIndex + d.byteAvailable = false + s.length = minMatchLength - 1 + if d.tokens.n == maxFlateBlockTokens { + // The block includes the current character + if d.err = d.writeBlock(&d.tokens, s.index, false); d.err != nil { + return + } + d.tokens.Reset() + } + s.ii = 0 + } else { + // Reset, if we got a match this run. + if s.length >= minMatchLength { + s.ii = 0 + } + // We have a byte waiting. Emit it. + if d.byteAvailable { + s.ii++ + d.tokens.AddLiteral(d.window[s.index-1]) + if d.tokens.n == maxFlateBlockTokens { + if d.err = d.writeBlock(&d.tokens, s.index, false); d.err != nil { + return + } + d.tokens.Reset() + } + s.index++ + + // If we have a long run of no matches, skip additional bytes + // Resets when s.ii overflows after 64KB. + if n := int(s.ii) - d.chain; n > 0 { + n = 1 + int(n>>6) + for j := 0; j < n; j++ { + if s.index >= d.windowEnd-1 { + break + } + d.tokens.AddLiteral(d.window[s.index-1]) + if d.tokens.n == maxFlateBlockTokens { + if d.err = d.writeBlock(&d.tokens, s.index, false); d.err != nil { + return + } + d.tokens.Reset() + } + // Index... + if s.index < s.maxInsertIndex { + h := hash4(d.window[s.index:]) + ch := s.hashHead[h] + s.chainHead = int(ch) + s.hashPrev[s.index&windowMask] = ch + s.hashHead[h] = uint32(s.index + s.hashOffset) + } + s.index++ + } + // Flush last byte + d.tokens.AddLiteral(d.window[s.index-1]) + d.byteAvailable = false + // s.length = minMatchLength - 1 // not needed, since s.ii is reset above, so it should never be > minMatchLength + if d.tokens.n == maxFlateBlockTokens { + if d.err = d.writeBlock(&d.tokens, s.index, false); d.err != nil { + return + } + d.tokens.Reset() + } + } + } else { + s.index++ + d.byteAvailable = true + } + } + } +} + +func (d *compressor) store() { + if d.windowEnd > 0 && (d.windowEnd == maxStoreBlockSize || d.sync) { + d.err = d.writeStoredBlock(d.window[:d.windowEnd]) + d.windowEnd = 0 + } +} + +// fillWindow will fill the buffer with data for huffman-only compression. +// The number of bytes copied is returned. +func (d *compressor) fillBlock(b []byte) int { + n := copy(d.window[d.windowEnd:], b) + d.windowEnd += n + return n +} + +// storeHuff will compress and store the currently added data, +// if enough has been accumulated or we at the end of the stream. +// Any error that occurred will be in d.err +func (d *compressor) storeHuff() { + if d.windowEnd < len(d.window) && !d.sync || d.windowEnd == 0 { + return + } + d.w.writeBlockHuff(false, d.window[:d.windowEnd], d.sync) + d.err = d.w.err + d.windowEnd = 0 +} + +// storeFast will compress and store the currently added data, +// if enough has been accumulated or we at the end of the stream. +// Any error that occurred will be in d.err +func (d *compressor) storeFast() { + // We only compress if we have maxStoreBlockSize. + if d.windowEnd < len(d.window) { + if !d.sync { + return + } + // Handle extremely small sizes. + if d.windowEnd < 128 { + if d.windowEnd == 0 { + return + } + if d.windowEnd <= 32 { + d.err = d.writeStoredBlock(d.window[:d.windowEnd]) + } else { + d.w.writeBlockHuff(false, d.window[:d.windowEnd], true) + d.err = d.w.err + } + d.tokens.Reset() + d.windowEnd = 0 + d.fast.Reset() + return + } + } + + d.fast.Encode(&d.tokens, d.window[:d.windowEnd]) + // If we made zero matches, store the block as is. + if d.tokens.n == 0 { + d.err = d.writeStoredBlock(d.window[:d.windowEnd]) + // If we removed less than 1/16th, huffman compress the block. + } else if int(d.tokens.n) > d.windowEnd-(d.windowEnd>>4) { + d.w.writeBlockHuff(false, d.window[:d.windowEnd], d.sync) + d.err = d.w.err + } else { + d.w.writeBlockDynamic(&d.tokens, false, d.window[:d.windowEnd], d.sync) + d.err = d.w.err + } + d.tokens.Reset() + d.windowEnd = 0 +} + +// write will add input byte to the stream. +// Unless an error occurs all bytes will be consumed. +func (d *compressor) write(b []byte) (n int, err error) { + if d.err != nil { + return 0, d.err + } + n = len(b) + for len(b) > 0 { + if d.windowEnd == len(d.window) || d.sync { + d.step(d) + } + b = b[d.fill(d, b):] + if d.err != nil { + return 0, d.err + } + } + return n, d.err +} + +func (d *compressor) syncFlush() error { + d.sync = true + if d.err != nil { + return d.err + } + d.step(d) + if d.err == nil { + d.w.writeStoredHeader(0, false) + d.w.flush() + d.err = d.w.err + } + d.sync = false + return d.err +} + +func (d *compressor) init(w io.Writer, level int) (err error) { + d.w = newHuffmanBitWriter(w) + + switch { + case level == NoCompression: + d.window = make([]byte, maxStoreBlockSize) + d.fill = (*compressor).fillBlock + d.step = (*compressor).store + case level == ConstantCompression: + d.w.logNewTablePenalty = 10 + d.window = make([]byte, 32<<10) + d.fill = (*compressor).fillBlock + d.step = (*compressor).storeHuff + case level == DefaultCompression: + level = 5 + fallthrough + case level >= 1 && level <= 6: + d.w.logNewTablePenalty = 7 + d.fast = newFastEnc(level) + d.window = make([]byte, maxStoreBlockSize) + d.fill = (*compressor).fillBlock + d.step = (*compressor).storeFast + case 7 <= level && level <= 9: + d.w.logNewTablePenalty = 8 + d.state = &advancedState{} + d.compressionLevel = levels[level] + d.initDeflate() + d.fill = (*compressor).fillDeflate + d.step = (*compressor).deflateLazy + case -level >= MinCustomWindowSize && -level <= MaxCustomWindowSize: + d.w.logNewTablePenalty = 7 + d.fast = &fastEncL5Window{maxOffset: int32(-level), cur: maxStoreBlockSize} + d.window = make([]byte, maxStoreBlockSize) + d.fill = (*compressor).fillBlock + d.step = (*compressor).storeFast + default: + return fmt.Errorf("flate: invalid compression level %d: want value in range [-2, 9]", level) + } + d.level = level + return nil +} + +// reset the state of the compressor. +func (d *compressor) reset(w io.Writer) { + d.w.reset(w) + d.sync = false + d.err = nil + // We only need to reset a few things for Snappy. + if d.fast != nil { + d.fast.Reset() + d.windowEnd = 0 + d.tokens.Reset() + return + } + switch d.compressionLevel.chain { + case 0: + // level was NoCompression or ConstantCompression. + d.windowEnd = 0 + default: + s := d.state + s.chainHead = -1 + for i := range s.hashHead { + s.hashHead[i] = 0 + } + for i := range s.hashPrev { + s.hashPrev[i] = 0 + } + s.hashOffset = 1 + s.index, d.windowEnd = 0, 0 + d.blockStart, d.byteAvailable = 0, false + d.tokens.Reset() + s.length = minMatchLength - 1 + s.offset = 0 + s.ii = 0 + s.maxInsertIndex = 0 + } +} + +func (d *compressor) close() error { + if d.err != nil { + return d.err + } + d.sync = true + d.step(d) + if d.err != nil { + return d.err + } + if d.w.writeStoredHeader(0, true); d.w.err != nil { + return d.w.err + } + d.w.flush() + d.w.reset(nil) + return d.w.err +} + +// NewWriter returns a new Writer compressing data at the given level. +// Following zlib, levels range from 1 (BestSpeed) to 9 (BestCompression); +// higher levels typically run slower but compress more. +// Level 0 (NoCompression) does not attempt any compression; it only adds the +// necessary DEFLATE framing. +// Level -1 (DefaultCompression) uses the default compression level. +// Level -2 (ConstantCompression) will use Huffman compression only, giving +// a very fast compression for all types of input, but sacrificing considerable +// compression efficiency. +// +// If level is in the range [-2, 9] then the error returned will be nil. +// Otherwise the error returned will be non-nil. +func NewWriter(w io.Writer, level int) (*Writer, error) { + var dw Writer + if err := dw.d.init(w, level); err != nil { + return nil, err + } + return &dw, nil +} + +// NewWriterDict is like NewWriter but initializes the new +// Writer with a preset dictionary. The returned Writer behaves +// as if the dictionary had been written to it without producing +// any compressed output. The compressed data written to w +// can only be decompressed by a Reader initialized with the +// same dictionary. +func NewWriterDict(w io.Writer, level int, dict []byte) (*Writer, error) { + zw, err := NewWriter(w, level) + if err != nil { + return nil, err + } + zw.d.fillWindow(dict) + zw.dict = append(zw.dict, dict...) // duplicate dictionary for Reset method. + return zw, err +} + +// MinCustomWindowSize is the minimum window size that can be sent to NewWriterWindow. +const MinCustomWindowSize = 32 + +// MaxCustomWindowSize is the maximum custom window that can be sent to NewWriterWindow. +const MaxCustomWindowSize = windowSize + +// NewWriterWindow returns a new Writer compressing data with a custom window size. +// windowSize must be from MinCustomWindowSize to MaxCustomWindowSize. +func NewWriterWindow(w io.Writer, windowSize int) (*Writer, error) { + if windowSize < MinCustomWindowSize { + return nil, errors.New("flate: requested window size less than MinWindowSize") + } + if windowSize > MaxCustomWindowSize { + return nil, errors.New("flate: requested window size bigger than MaxCustomWindowSize") + } + var dw Writer + if err := dw.d.init(w, -windowSize); err != nil { + return nil, err + } + return &dw, nil +} + +// A Writer takes data written to it and writes the compressed +// form of that data to an underlying writer (see NewWriter). +type Writer struct { + d compressor + dict []byte +} + +// Write writes data to w, which will eventually write the +// compressed form of data to its underlying writer. +func (w *Writer) Write(data []byte) (n int, err error) { + return w.d.write(data) +} + +// Flush flushes any pending data to the underlying writer. +// It is useful mainly in compressed network protocols, to ensure that +// a remote reader has enough data to reconstruct a packet. +// Flush does not return until the data has been written. +// Calling Flush when there is no pending data still causes the Writer +// to emit a sync marker of at least 4 bytes. +// If the underlying writer returns an error, Flush returns that error. +// +// In the terminology of the zlib library, Flush is equivalent to Z_SYNC_FLUSH. +func (w *Writer) Flush() error { + // For more about flushing: + // http://www.bolet.org/~pornin/deflate-flush.html + return w.d.syncFlush() +} + +// Close flushes and closes the writer. +func (w *Writer) Close() error { + return w.d.close() +} + +// Reset discards the writer's state and makes it equivalent to +// the result of NewWriter or NewWriterDict called with dst +// and w's level and dictionary. +func (w *Writer) Reset(dst io.Writer) { + if len(w.dict) > 0 { + // w was created with NewWriterDict + w.d.reset(dst) + if dst != nil { + w.d.fillWindow(w.dict) + } + } else { + // w was created with NewWriter + w.d.reset(dst) + } +} + +// ResetDict discards the writer's state and makes it equivalent to +// the result of NewWriter or NewWriterDict called with dst +// and w's level, but sets a specific dictionary. +func (w *Writer) ResetDict(dst io.Writer, dict []byte) { + w.dict = dict + w.d.reset(dst) + w.d.fillWindow(w.dict) +} diff --git a/internal/compress/flate/deflate_test.go b/internal/compress/flate/deflate_test.go new file mode 100644 index 00000000..8e082968 --- /dev/null +++ b/internal/compress/flate/deflate_test.go @@ -0,0 +1,706 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Copyright (c) 2015 Klaus Post +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package flate + +import ( + "bytes" + "fmt" + "io" + "os" + "reflect" + "strings" + "sync" + "testing" +) + +type deflateTest struct { + in []byte + level int + out []byte +} + +type deflateInflateTest struct { + in []byte +} + +type reverseBitsTest struct { + in uint16 + bitCount uint8 + out uint16 +} + +var deflateTests = []*deflateTest{ + 0: {[]byte{}, 0, []byte{0x3, 0x0}}, + 1: {[]byte{0x11}, BestCompression, []byte{0x12, 0x4, 0xc, 0x0}}, + 2: {[]byte{0x11}, BestCompression, []byte{0x12, 0x4, 0xc, 0x0}}, + 3: {[]byte{0x11}, BestCompression, []byte{0x12, 0x4, 0xc, 0x0}}, + + 4: {[]byte{0x11}, 0, []byte{0x0, 0x1, 0x0, 0xfe, 0xff, 0x11, 0x3, 0x0}}, + 5: {[]byte{0x11, 0x12}, 0, []byte{0x0, 0x2, 0x0, 0xfd, 0xff, 0x11, 0x12, 0x3, 0x0}}, + 6: {[]byte{0x11, 0x11, 0x11, 0x11, 0x11, 0x11, 0x11, 0x11}, 0, + []byte{0x0, 0x8, 0x0, 0xf7, 0xff, 0x11, 0x11, 0x11, 0x11, 0x11, 0x11, 0x11, 0x11, 0x3, 0x0}, + }, + 7: {[]byte{}, 1, []byte{0x3, 0x0}}, + 8: {[]byte{0x11}, BestCompression, []byte{0x12, 0x4, 0xc, 0x0}}, + 9: {[]byte{0x11, 0x12}, BestCompression, []byte{0x12, 0x14, 0x2, 0xc, 0x0}}, + 10: {[]byte{0x11, 0x11, 0x11, 0x11, 0x11, 0x11, 0x11, 0x11, 0x11}, BestCompression, []byte{0x12, 0x84, 0x1, 0xc0, 0x0}}, + 11: {[]byte{}, 9, []byte{0x3, 0x0}}, + 12: {[]byte{0x11}, 9, []byte{0x12, 0x4, 0xc, 0x0}}, + 13: {[]byte{0x11, 0x12}, 9, []byte{0x12, 0x14, 0x2, 0xc, 0x0}}, + 14: {[]byte{0x11, 0x11, 0x11, 0x11, 0x11, 0x11, 0x11, 0x11, 0x11}, 9, []byte{0x12, 0x84, 0x1, 0xc0, 0x0}}, +} + +var deflateInflateTests = []*deflateInflateTest{ + {[]byte{}}, + {[]byte{0x11}}, + {[]byte{0x11, 0x12}}, + {[]byte{0x11, 0x11, 0x11, 0x11, 0x11, 0x11, 0x11, 0x11}}, + {[]byte{0x11, 0x10, 0x13, 0x41, 0x21, 0x21, 0x41, 0x13, 0x87, 0x78, 0x13}}, + {largeDataChunk()}, +} + +var reverseBitsTests = []*reverseBitsTest{ + {1, 1, 1}, + {1, 2, 2}, + {1, 3, 4}, + {1, 4, 8}, + {1, 5, 16}, + {17, 5, 17}, + {257, 9, 257}, + {29, 5, 23}, +} + +func largeDataChunk() []byte { + result := make([]byte, 100000) + for i := range result { + result[i] = byte(i * i & 0xFF) + } + return result +} + +func TestBulkHash4(t *testing.T) { + for _, x := range deflateTests { + y := x.out + if len(y) >= minMatchLength { + y = append(y, y...) + for j := 4; j < len(y); j++ { + y := y[:j] + dst := make([]uint32, len(y)-minMatchLength+1) + for i := range dst { + dst[i] = uint32(i + 100) + } + bulkHash4(y, dst) + for i, val := range dst { + got := val + expect := hash4(y[i:]) + if got != expect && got == uint32(i)+100 { + t.Errorf("Len:%d Index:%d, expected 0x%08x but not modified", len(y), i, expect) + } else if got != expect { + t.Errorf("Len:%d Index:%d, got 0x%08x expected:0x%08x", len(y), i, got, expect) + } else { + //t.Logf("Len:%d Index:%d OK (0x%08x)", len(y), i, got) + } + } + } + } + } +} + +func TestDeflate(t *testing.T) { + for i, h := range deflateTests { + var buf bytes.Buffer + w, err := NewWriter(&buf, h.level) + if err != nil { + t.Errorf("NewWriter: %v", err) + continue + } + w.Write(h.in) + w.Close() + if !bytes.Equal(buf.Bytes(), h.out) { + t.Errorf("%d: Deflate(%d, %x) got \n%#v, want \n%#v", i, h.level, h.in, buf.Bytes(), h.out) + } + } +} + +// A sparseReader returns a stream consisting of 0s followed by 1<<16 1s. +// This tests missing hash references in a very large input. +type sparseReader struct { + l int64 + cur int64 +} + +func (r *sparseReader) Read(b []byte) (n int, err error) { + if r.cur >= r.l { + return 0, io.EOF + } + n = len(b) + cur := r.cur + int64(n) + if cur > r.l { + n -= int(cur - r.l) + cur = r.l + } + for i := range b[0:n] { + if r.cur+int64(i) >= r.l-1<<16 { + b[i] = 1 + } else { + b[i] = 0 + } + } + r.cur = cur + return +} + +func TestVeryLongSparseChunk(t *testing.T) { + if testing.Short() { + t.Skip("skipping sparse chunk during short test") + } + var buf bytes.Buffer + w, err := NewWriter(&buf, 1) + if err != nil { + t.Errorf("NewWriter: %v", err) + return + } + if _, err = io.Copy(w, &sparseReader{l: 23e8}); err != nil { + t.Errorf("Compress failed: %v", err) + return + } + t.Log("Length:", buf.Len()) +} + +func TestOneMByte(t *testing.T) { + var input [1024 * 1024]byte + + var compressedOutput bytes.Buffer + for level := HuffmanOnly; level <= BestCompression; level++ { + compressedOutput.Reset() + compressor, err := NewWriter(&compressedOutput, level) + if err != nil { + t.Fatalf("create: %s", err) + } + // Use single write... + if _, err := compressor.Write(input[:]); err != nil { + t.Fatalf("compress: %s", err) + } + + if err := compressor.Close(); err != nil { + t.Fatalf("close: %s", err) + } + + var decompressedOutput bytes.Buffer + + decompresser := NewReader(&compressedOutput) + t.Log("level:", level, "compressed:", compressedOutput.Len()) + if _, err := io.Copy(&decompressedOutput, decompresser); err != nil { + t.Fatalf("decompress: %s", err) + } + + if !bytes.Equal(input[:], decompressedOutput.Bytes()) { + t.Fatal("input and output do not match") + } + } +} + +type syncBuffer struct { + buf bytes.Buffer + mu sync.RWMutex + closed bool + ready chan bool +} + +func newSyncBuffer() *syncBuffer { + return &syncBuffer{ready: make(chan bool, 1)} +} + +func (b *syncBuffer) Read(p []byte) (n int, err error) { + for { + b.mu.RLock() + n, err = b.buf.Read(p) + b.mu.RUnlock() + if n > 0 || b.closed { + return + } + <-b.ready + } +} + +func (b *syncBuffer) signal() { + select { + case b.ready <- true: + default: + } +} + +func (b *syncBuffer) Write(p []byte) (n int, err error) { + n, err = b.buf.Write(p) + b.signal() + return +} + +func (b *syncBuffer) WriteMode() { + b.mu.Lock() +} + +func (b *syncBuffer) ReadMode() { + b.mu.Unlock() + b.signal() +} + +func (b *syncBuffer) Close() error { + b.closed = true + b.signal() + return nil +} + +func testSync(t *testing.T, level int, input []byte, name string) { + if len(input) == 0 { + return + } + + t.Logf("--testSync %d, %d, %s", level, len(input), name) + buf := newSyncBuffer() + buf1 := new(bytes.Buffer) + buf.WriteMode() + w, err := NewWriter(io.MultiWriter(buf, buf1), level) + if err != nil { + t.Errorf("NewWriter: %v", err) + return + } + r := NewReader(buf) + + // Write half the input and read back. + for i := range 2 { + var lo, hi int + if i == 0 { + lo, hi = 0, (len(input)+1)/2 + } else { + lo, hi = (len(input)+1)/2, len(input) + } + t.Logf("#%d: write %d-%d", i, lo, hi) + if _, err := w.Write(input[lo:hi]); err != nil { + t.Errorf("testSync: write: %v", err) + return + } + if i == 0 { + if err := w.Flush(); err != nil { + t.Errorf("testSync: flush: %v", err) + return + } + } else { + if err := w.Close(); err != nil { + t.Errorf("testSync: close: %v", err) + } + } + buf.ReadMode() + out := make([]byte, hi-lo+1) + m, err := io.ReadAtLeast(r, out, hi-lo) + t.Logf("#%d: read %d", i, m) + if m != hi-lo || err != nil { + t.Errorf("testSync/%d (%d, %d, %s): read %d: %d, %v (%d left)", i, level, len(input), name, hi-lo, m, err, buf.buf.Len()) + return + } + if !bytes.Equal(input[lo:hi], out[:hi-lo]) { + t.Errorf("testSync/%d: read wrong bytes: %x vs %x", i, input[lo:hi], out[:hi-lo]) + return + } + // This test originally checked that after reading + // the first half of the input, there was nothing left + // in the read buffer (buf.buf.Len() != 0) but that is + // not necessarily the case: the write Flush may emit + // some extra framing bits that are not necessary + // to process to obtain the first half of the uncompressed + // data. The test ran correctly most of the time, because + // the background goroutine had usually read even + // those extra bits by now, but it's not a useful thing to + // check. + buf.WriteMode() + } + buf.ReadMode() + out := make([]byte, 10) + if n, err := r.Read(out); n > 0 || err != io.EOF { + t.Errorf("testSync (%d, %d, %s): final Read: %d, %v (hex: %x)", level, len(input), name, n, err, out[0:n]) + } + if buf.buf.Len() != 0 { + t.Errorf("testSync (%d, %d, %s): extra data at end", level, len(input), name) + } + r.Close() + + // stream should work for ordinary reader too + r = NewReader(buf1) + out, err = io.ReadAll(r) + if err != nil { + t.Errorf("testSync: read: %s", err) + return + } + r.Close() + if !bytes.Equal(input, out) { + t.Errorf("testSync: decompress(compress(data)) != data: level=%d input=%s", level, name) + } +} + +func testToFromWithLevelAndLimit(t *testing.T, level int, input []byte, name string, limit int) { + var buffer bytes.Buffer + w, err := NewWriter(&buffer, level) + if err != nil { + t.Errorf("NewWriter: %v", err) + return + } + w.Write(input) + w.Close() + if limit > 0 { + t.Logf("level: %d - Size:%.2f%%, %d b\n", level, float64(buffer.Len()*100)/float64(limit), buffer.Len()) + } + if limit > 0 && buffer.Len() > limit { + t.Errorf("level: %d, len(compress(data)) = %d > limit = %d", level, buffer.Len(), limit) + } + + r := NewReader(&buffer) + out, err := io.ReadAll(r) + if err != nil { + t.Errorf("read: %s", err) + return + } + r.Close() + if !bytes.Equal(input, out) { + os.WriteFile("testdata/fails/"+t.Name()+".got", out, os.ModePerm) + os.WriteFile("testdata/fails/"+t.Name()+".want", input, os.ModePerm) + t.Errorf("decompress(compress(data)) != data: level=%d input=%s", level, name) + return + } + testSync(t, level, input, name) +} + +func testToFromWithLimit(t *testing.T, input []byte, name string, limit [11]int) { + for i := range 10 { + testToFromWithLevelAndLimit(t, i, input, name, limit[i]) + } + testToFromWithLevelAndLimit(t, -2, input, name, limit[10]) +} + +func TestDeflateInflate(t *testing.T) { + for i, h := range deflateInflateTests { + testToFromWithLimit(t, h.in, fmt.Sprintf("#%d", i), [11]int{}) + } +} + +func TestReverseBits(t *testing.T) { + for _, h := range reverseBitsTests { + if v := reverseBits(h.in, h.bitCount); v != h.out { + t.Errorf("reverseBits(%v,%v) = %v, want %v", + h.in, h.bitCount, v, h.out) + } + } +} + +type deflateInflateStringTest struct { + filename string + label string + limit [11]int // Number 11 is ConstantCompression +} + +var deflateInflateStringTests = []deflateInflateStringTest{ + { + "../testdata/e.txt", + "2.718281828...", + [...]int{100018, 67900, 50960, 51150, 50930, 50790, 50790, 50790, 50790, 50790, 43683 + 100}, + }, + { + "../testdata/Mark.Twain-Tom.Sawyer.txt", + "Mark.Twain-Tom.Sawyer", + [...]int{387999, 185000, 182361, 179974, 174124, 168819, 162936, 160506, 160295, 160295, 233460 + 100}, + }, +} + +func TestDeflateInflateString(t *testing.T) { + for _, test := range deflateInflateStringTests { + gold, err := os.ReadFile(test.filename) + if err != nil { + t.Error(err) + } + // Remove returns that may be present on Windows + neutral := strings.Map(func(r rune) rune { + if r != '\r' { + return r + } + return -1 + }, string(gold)) + + testToFromWithLimit(t, []byte(neutral), test.label, test.limit) + + if testing.Short() { + break + } + } +} + +func TestReaderDict(t *testing.T) { + const ( + dict = "hello world" + text = "hello again world" + ) + var b bytes.Buffer + w, err := NewWriter(&b, 5) + if err != nil { + t.Fatalf("NewWriter: %v", err) + } + w.Write([]byte(dict)) + w.Flush() + b.Reset() + w.Write([]byte(text)) + w.Close() + + r := NewReaderDict(&b, []byte(dict)) + data, err := io.ReadAll(r) + if err != nil { + t.Fatal(err) + } + if string(data) != "hello again world" { + t.Fatalf("read returned %q want %q", string(data), text) + } +} + +func TestWriterDict(t *testing.T) { + const ( + dict = "hello world Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua." + text = "hello world Lorem ipsum dolor sit amet" + ) + // This test is sensitive to algorithm changes that skip + // data in favour of speed. Higher levels are less prone to this + // so we test level 4-9. + for l := 4; l < 9; l++ { + var b bytes.Buffer + w, err := NewWriter(&b, l) + if err != nil { + t.Fatalf("level %d, NewWriter: %v", l, err) + } + w.Write([]byte(dict)) + w.Flush() + b.Reset() + w.Write([]byte(text)) + w.Close() + + var b1 bytes.Buffer + w, _ = NewWriterDict(&b1, l, []byte(dict)) + w.Write([]byte(text)) + w.Close() + + if !bytes.Equal(b1.Bytes(), b.Bytes()) { + t.Errorf("level %d, writer wrote\n%v\n want\n%v", l, b1.Bytes(), b.Bytes()) + } + } +} + +// See http://code.google.com/p/go/issues/detail?id=2508 +func TestRegression2508(t *testing.T) { + if testing.Short() { + t.Logf("test disabled with -short") + return + } + w, err := NewWriter(io.Discard, 1) + if err != nil { + t.Fatalf("NewWriter: %v", err) + } + buf := make([]byte, 1024) + for range 131072 { + if _, err := w.Write(buf); err != nil { + t.Fatalf("writer failed: %v", err) + } + } + w.Close() +} + +func TestWriterReset(t *testing.T) { + for level := -2; level <= 9; level++ { + if level == -1 { + level++ + } + if testing.Short() && level > 1 { + break + } + w, err := NewWriter(io.Discard, level) + if err != nil { + t.Fatalf("NewWriter: %v", err) + } + buf := []byte("hello world") + for range 1024 { + w.Write(buf) + } + w.Reset(io.Discard) + + wref, err := NewWriter(io.Discard, level) + if err != nil { + t.Fatalf("NewWriter: %v", err) + } + + // DeepEqual doesn't compare functions. + w.d.fill, wref.d.fill = nil, nil + w.d.step, wref.d.step = nil, nil + w.d.state, wref.d.state = nil, nil + w.d.fast, wref.d.fast = nil, nil + + // hashMatch is always overwritten when used. + if w.d.tokens.n != 0 { + t.Errorf("level %d Writer not reset after Reset. %d tokens were present", level, w.d.tokens.n) + } + // As long as the length is 0, we don't care about the content. + w.d.tokens = wref.d.tokens + + // We don't care if there are values in the window, as long as it is at d.index is 0 + w.d.window = wref.d.window + if !reflect.DeepEqual(w, wref) { + t.Errorf("level %d Writer not reset after Reset", level) + } + } + + for i := HuffmanOnly; i <= BestCompression; i++ { + testResetOutput(t, fmt.Sprint("level-", i), func(w io.Writer) (*Writer, error) { return NewWriter(w, i) }) + } + dict := []byte(strings.Repeat("we are the world - how are you?", 3)) + for i := HuffmanOnly; i <= BestCompression; i++ { + testResetOutput(t, fmt.Sprint("dict-level-", i), func(w io.Writer) (*Writer, error) { return NewWriterDict(w, i, dict) }) + } + for i := HuffmanOnly; i <= BestCompression; i++ { + testResetOutput(t, fmt.Sprint("dict-reset-level-", i), func(w io.Writer) (*Writer, error) { + w2, err := NewWriter(nil, i) + if err != nil { + return w2, err + } + w2.ResetDict(w, dict) + return w2, nil + }) + } + testResetOutput(t, fmt.Sprint("dict-reset-window"), func(w io.Writer) (*Writer, error) { + w2, err := NewWriterWindow(nil, 1024) + if err != nil { + return w2, err + } + w2.ResetDict(w, dict) + return w2, nil + }) +} + +func testResetOutput(t *testing.T, name string, newWriter func(w io.Writer) (*Writer, error)) { + t.Run(name, func(t *testing.T) { + buf := new(bytes.Buffer) + w, err := newWriter(buf) + if err != nil { + t.Fatalf("NewWriter: %v", err) + } + b := []byte("hello world - how are you doing?") + for range 1024 { + w.Write(b) + } + w.Close() + out1 := buf.Bytes() + + buf2 := new(bytes.Buffer) + w.Reset(buf2) + for range 1024 { + w.Write(b) + } + w.Close() + out2 := buf2.Bytes() + + if len(out1) != len(out2) { + t.Errorf("got %d, expected %d bytes", len(out2), len(out1)) + } + if !bytes.Equal(out1, out2) { + mm := 0 + for i, b := range out1[:len(out2)] { + if b != out2[i] { + t.Errorf("mismatch index %d: %02x, expected %02x", i, out2[i], b) + } + mm++ + if mm == 10 { + t.Fatal("Stopping") + } + } + } + t.Logf("got %d bytes", len(out1)) + }) +} + +// TestBestSpeed tests that round-tripping through deflate and then inflate +// recovers the original input. The Write sizes are near the thresholds in the +// compressor.encSpeed method (0, 16, 128), as well as near maxStoreBlockSize +// (65535). +func TestBestSpeed(t *testing.T) { + abc := make([]byte, 128) + for i := range abc { + abc[i] = byte(i) + } + abcabc := bytes.Repeat(abc, 131072/len(abc)) + var want []byte + + testCases := [][]int{ + {65536, 0}, + {65536, 1}, + {65536, 1, 256}, + {65536, 1, 65536}, + {65536, 14}, + {65536, 15}, + {65536, 16}, + {65536, 16, 256}, + {65536, 16, 65536}, + {65536, 127}, + {65536, 128}, + {65536, 128, 256}, + {65536, 128, 65536}, + {65536, 129}, + {65536, 65536, 256}, + {65536, 65536, 65536}, + } + + for i, tc := range testCases { + if testing.Short() && i > 5 { + t.Skip() + } + for _, firstN := range []int{1, 65534, 65535, 65536, 65537, 131072} { + tc[0] = firstN + outer: + for _, flush := range []bool{false, true} { + buf := new(bytes.Buffer) + want = want[:0] + + w, err := NewWriter(buf, BestSpeed) + if err != nil { + t.Errorf("i=%d, firstN=%d, flush=%t: NewWriter: %v", i, firstN, flush, err) + continue + } + for _, n := range tc { + want = append(want, abcabc[:n]...) + if _, err := w.Write(abcabc[:n]); err != nil { + t.Errorf("i=%d, firstN=%d, flush=%t: Write: %v", i, firstN, flush, err) + continue outer + } + if !flush { + continue + } + if err := w.Flush(); err != nil { + t.Errorf("i=%d, firstN=%d, flush=%t: Flush: %v", i, firstN, flush, err) + continue outer + } + } + if err := w.Close(); err != nil { + t.Errorf("i=%d, firstN=%d, flush=%t: Close: %v", i, firstN, flush, err) + continue + } + + r := NewReader(buf) + got, err := io.ReadAll(r) + if err != nil { + t.Errorf("i=%d, firstN=%d, flush=%t: ReadAll: %v", i, firstN, flush, err) + continue + } + r.Close() + + if !bytes.Equal(got, want) { + t.Errorf("i=%d, firstN=%d, flush=%t: corruption during deflate-then-inflate", i, firstN, flush) + continue + } + } + } + } +} diff --git a/internal/compress/flate/dict_decoder.go b/internal/compress/flate/dict_decoder.go new file mode 100644 index 00000000..cb855abc --- /dev/null +++ b/internal/compress/flate/dict_decoder.go @@ -0,0 +1,181 @@ +// Copyright 2016 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package flate + +// dictDecoder implements the LZ77 sliding dictionary as used in decompression. +// LZ77 decompresses data through sequences of two forms of commands: +// +// - Literal insertions: Runs of one or more symbols are inserted into the data +// stream as is. This is accomplished through the writeByte method for a +// single symbol, or combinations of writeSlice/writeMark for multiple symbols. +// Any valid stream must start with a literal insertion if no preset dictionary +// is used. +// +// - Backward copies: Runs of one or more symbols are copied from previously +// emitted data. Backward copies come as the tuple (dist, length) where dist +// determines how far back in the stream to copy from and length determines how +// many bytes to copy. Note that it is valid for the length to be greater than +// the distance. Since LZ77 uses forward copies, that situation is used to +// perform a form of run-length encoding on repeated runs of symbols. +// The writeCopy and tryWriteCopy are used to implement this command. +// +// For performance reasons, this implementation performs little to no sanity +// checks about the arguments. As such, the invariants documented for each +// method call must be respected. +type dictDecoder struct { + hist []byte // Sliding window history + + // Invariant: 0 <= rdPos <= wrPos <= len(hist) + wrPos int // Current output position in buffer + rdPos int // Have emitted hist[:rdPos] already + full bool // Has a full window length been written yet? +} + +// init initializes dictDecoder to have a sliding window dictionary of the given +// size. If a preset dict is provided, it will initialize the dictionary with +// the contents of dict. +func (dd *dictDecoder) init(size int, dict []byte) { + *dd = dictDecoder{hist: dd.hist} + + if cap(dd.hist) < size { + dd.hist = make([]byte, size) + } + dd.hist = dd.hist[:size] + + if len(dict) > len(dd.hist) { + dict = dict[len(dict)-len(dd.hist):] + } + dd.wrPos = copy(dd.hist, dict) + if dd.wrPos == len(dd.hist) { + dd.wrPos = 0 + dd.full = true + } + dd.rdPos = dd.wrPos +} + +// histSize reports the total amount of historical data in the dictionary. +func (dd *dictDecoder) histSize() int { + if dd.full { + return len(dd.hist) + } + return dd.wrPos +} + +// availRead reports the number of bytes that can be flushed by readFlush. +func (dd *dictDecoder) availRead() int { + return dd.wrPos - dd.rdPos +} + +// availWrite reports the available amount of output buffer space. +func (dd *dictDecoder) availWrite() int { + return len(dd.hist) - dd.wrPos +} + +// writeSlice returns a slice of the available buffer to write data to. +// +// This invariant will be kept: len(s) <= availWrite() +func (dd *dictDecoder) writeSlice() []byte { + return dd.hist[dd.wrPos:] +} + +// writeMark advances the writer pointer by cnt. +// +// This invariant must be kept: 0 <= cnt <= availWrite() +func (dd *dictDecoder) writeMark(cnt int) { + dd.wrPos += cnt +} + +// writeByte writes a single byte to the dictionary. +// +// This invariant must be kept: 0 < availWrite() +func (dd *dictDecoder) writeByte(c byte) { + dd.hist[dd.wrPos] = c + dd.wrPos++ +} + +// writeCopy copies a string at a given (dist, length) to the output. +// This returns the number of bytes copied and may be less than the requested +// length if the available space in the output buffer is too small. +// +// This invariant must be kept: 0 < dist <= histSize() +func (dd *dictDecoder) writeCopy(dist, length int) int { + dstBase := dd.wrPos + dstPos := dstBase + srcPos := dstPos - dist + endPos := min(dstPos+length, len(dd.hist)) + + // Copy non-overlapping section after destination position. + // + // This section is non-overlapping in that the copy length for this section + // is always less than or equal to the backwards distance. This can occur + // if a distance refers to data that wraps-around in the buffer. + // Thus, a backwards copy is performed here; that is, the exact bytes in + // the source prior to the copy is placed in the destination. + if srcPos < 0 { + srcPos += len(dd.hist) + dstPos += copy(dd.hist[dstPos:endPos], dd.hist[srcPos:]) + srcPos = 0 + } + + // Copy possibly overlapping section before destination position. + // + // This section can overlap if the copy length for this section is larger + // than the backwards distance. This is allowed by LZ77 so that repeated + // strings can be succinctly represented using (dist, length) pairs. + // Thus, a forwards copy is performed here; that is, the bytes copied is + // possibly dependent on the resulting bytes in the destination as the copy + // progresses along. This is functionally equivalent to the following: + // + // for i := 0; i < endPos-dstPos; i++ { + // dd.hist[dstPos+i] = dd.hist[srcPos+i] + // } + // dstPos = endPos + // + for dstPos < endPos { + dstPos += copy(dd.hist[dstPos:endPos], dd.hist[srcPos:dstPos]) + } + + dd.wrPos = dstPos + return dstPos - dstBase +} + +// tryWriteCopy tries to copy a string at a given (distance, length) to the +// output. This specialized version is optimized for short distances. +// +// This method is designed to be inlined for performance reasons. +// +// This invariant must be kept: 0 < dist <= histSize() +func (dd *dictDecoder) tryWriteCopy(dist, length int) int { + dstPos := dd.wrPos + endPos := dstPos + length + if dstPos < dist || endPos > len(dd.hist) { + return 0 + } + dstBase := dstPos + srcPos := dstPos - dist + + // Copy possibly overlapping section before destination position. +loop: + dstPos += copy(dd.hist[dstPos:endPos], dd.hist[srcPos:dstPos]) + if dstPos < endPos { + goto loop // Avoid for-loop so that this function can be inlined + } + + dd.wrPos = dstPos + return dstPos - dstBase +} + +// readFlush returns a slice of the historical buffer that is ready to be +// emitted to the user. The data returned by readFlush must be fully consumed +// before calling any other dictDecoder methods. +func (dd *dictDecoder) readFlush() []byte { + toRead := dd.hist[dd.rdPos:dd.wrPos] + dd.rdPos = dd.wrPos + if dd.wrPos == len(dd.hist) { + dd.wrPos, dd.rdPos = 0, 0 + dd.full = true + } + return toRead +} diff --git a/internal/compress/flate/dict_decoder_test.go b/internal/compress/flate/dict_decoder_test.go new file mode 100644 index 00000000..9275cff7 --- /dev/null +++ b/internal/compress/flate/dict_decoder_test.go @@ -0,0 +1,139 @@ +// Copyright 2016 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package flate + +import ( + "bytes" + "strings" + "testing" +) + +func TestDictDecoder(t *testing.T) { + const ( + abc = "ABC\n" + fox = "The quick brown fox jumped over the lazy dog!\n" + poem = "The Road Not Taken\nRobert Frost\n" + + "\n" + + "Two roads diverged in a yellow wood,\n" + + "And sorry I could not travel both\n" + + "And be one traveler, long I stood\n" + + "And looked down one as far as I could\n" + + "To where it bent in the undergrowth;\n" + + "\n" + + "Then took the other, as just as fair,\n" + + "And having perhaps the better claim,\n" + + "Because it was grassy and wanted wear;\n" + + "Though as for that the passing there\n" + + "Had worn them really about the same,\n" + + "\n" + + "And both that morning equally lay\n" + + "In leaves no step had trodden black.\n" + + "Oh, I kept the first for another day!\n" + + "Yet knowing how way leads on to way,\n" + + "I doubted if I should ever come back.\n" + + "\n" + + "I shall be telling this with a sigh\n" + + "Somewhere ages and ages hence:\n" + + "Two roads diverged in a wood, and I-\n" + + "I took the one less traveled by,\n" + + "And that has made all the difference.\n" + ) + + var poemRefs = []struct { + dist int // Backward distance (0 if this is an insertion) + length int // Length of copy or insertion + }{ + {0, 38}, {33, 3}, {0, 48}, {79, 3}, {0, 11}, {34, 5}, {0, 6}, {23, 7}, + {0, 8}, {50, 3}, {0, 2}, {69, 3}, {34, 5}, {0, 4}, {97, 3}, {0, 4}, + {43, 5}, {0, 6}, {7, 4}, {88, 7}, {0, 12}, {80, 3}, {0, 2}, {141, 4}, + {0, 1}, {196, 3}, {0, 3}, {157, 3}, {0, 6}, {181, 3}, {0, 2}, {23, 3}, + {77, 3}, {28, 5}, {128, 3}, {110, 4}, {70, 3}, {0, 4}, {85, 6}, {0, 2}, + {182, 6}, {0, 4}, {133, 3}, {0, 7}, {47, 5}, {0, 20}, {112, 5}, {0, 1}, + {58, 3}, {0, 8}, {59, 3}, {0, 4}, {173, 3}, {0, 5}, {114, 3}, {0, 4}, + {92, 5}, {0, 2}, {71, 3}, {0, 2}, {76, 5}, {0, 1}, {46, 3}, {96, 4}, + {130, 4}, {0, 3}, {360, 3}, {0, 3}, {178, 5}, {0, 7}, {75, 3}, {0, 3}, + {45, 6}, {0, 6}, {299, 6}, {180, 3}, {70, 6}, {0, 1}, {48, 3}, {66, 4}, + {0, 3}, {47, 5}, {0, 9}, {325, 3}, {0, 1}, {359, 3}, {318, 3}, {0, 2}, + {199, 3}, {0, 1}, {344, 3}, {0, 3}, {248, 3}, {0, 10}, {310, 3}, {0, 3}, + {93, 6}, {0, 3}, {252, 3}, {157, 4}, {0, 2}, {273, 5}, {0, 14}, {99, 4}, + {0, 1}, {464, 4}, {0, 2}, {92, 4}, {495, 3}, {0, 1}, {322, 4}, {16, 4}, + {0, 3}, {402, 3}, {0, 2}, {237, 4}, {0, 2}, {432, 4}, {0, 1}, {483, 5}, + {0, 2}, {294, 4}, {0, 2}, {306, 3}, {113, 5}, {0, 1}, {26, 4}, {164, 3}, + {488, 4}, {0, 1}, {542, 3}, {248, 6}, {0, 5}, {205, 3}, {0, 8}, {48, 3}, + {449, 6}, {0, 2}, {192, 3}, {328, 4}, {9, 5}, {433, 3}, {0, 3}, {622, 25}, + {615, 5}, {46, 5}, {0, 2}, {104, 3}, {475, 10}, {549, 3}, {0, 4}, {597, 8}, + {314, 3}, {0, 1}, {473, 6}, {317, 5}, {0, 1}, {400, 3}, {0, 3}, {109, 3}, + {151, 3}, {48, 4}, {0, 4}, {125, 3}, {108, 3}, {0, 2}, + } + + var got, want bytes.Buffer + var dd dictDecoder + dd.init(1<<11, nil) + + var writeCopy = func(dist, length int) { + for length > 0 { + cnt := dd.tryWriteCopy(dist, length) + if cnt == 0 { + cnt = dd.writeCopy(dist, length) + } + + length -= cnt + if dd.availWrite() == 0 { + got.Write(dd.readFlush()) + } + } + } + var writeString = func(str string) { + for len(str) > 0 { + cnt := copy(dd.writeSlice(), str) + str = str[cnt:] + dd.writeMark(cnt) + if dd.availWrite() == 0 { + got.Write(dd.readFlush()) + } + } + } + + writeString(".") + want.WriteByte('.') + + str := poem + for _, ref := range poemRefs { + if ref.dist == 0 { + writeString(str[:ref.length]) + } else { + writeCopy(ref.dist, ref.length) + } + str = str[ref.length:] + } + want.WriteString(poem) + + writeCopy(dd.histSize(), 33) + want.Write(want.Bytes()[:33]) + + writeString(abc) + writeCopy(len(abc), 59*len(abc)) + want.WriteString(strings.Repeat(abc, 60)) + + writeString(fox) + writeCopy(len(fox), 9*len(fox)) + want.WriteString(strings.Repeat(fox, 10)) + + writeString(".") + writeCopy(1, 9) + want.WriteString(strings.Repeat(".", 10)) + + writeString(strings.ToUpper(poem)) + writeCopy(len(poem), 7*len(poem)) + want.WriteString(strings.Repeat(strings.ToUpper(poem), 8)) + + writeCopy(dd.histSize(), 10) + want.Write(want.Bytes()[want.Len()-dd.histSize():][:10]) + + got.Write(dd.readFlush()) + if got.String() != want.String() { + t.Errorf("final string mismatch:\ngot %q\nwant %q", got.String(), want.String()) + } +} diff --git a/internal/compress/flate/example_test.go b/internal/compress/flate/example_test.go new file mode 100644 index 00000000..d541e0e1 --- /dev/null +++ b/internal/compress/flate/example_test.go @@ -0,0 +1,245 @@ +// Copyright 2016 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package flate_test + +import ( + "bytes" + "fmt" + "io" + "log" + "os" + "strings" + "sync" + + "codeberg.org/lindenii/furgit/internal/compress/flate" +) + +// In performance critical applications, Reset can be used to discard the +// current compressor or decompressor state and reinitialize them quickly +// by taking advantage of previously allocated memory. +func Example_reset() { + proverbs := []string{ + "Don't communicate by sharing memory, share memory by communicating.\n", + "Concurrency is not parallelism.\n", + "The bigger the interface, the weaker the abstraction.\n", + "Documentation is for users.\n", + } + + var r strings.Reader + var b bytes.Buffer + buf := make([]byte, 32<<10) + + zw, err := flate.NewWriter(nil, flate.DefaultCompression) + if err != nil { + log.Fatal(err) + } + zr := flate.NewReader(nil) + + for _, s := range proverbs { + r.Reset(s) + b.Reset() + + // Reset the compressor and encode from some input stream. + zw.Reset(&b) + if _, err := io.CopyBuffer(zw, &r, buf); err != nil { + log.Fatal(err) + } + if err := zw.Close(); err != nil { + log.Fatal(err) + } + + // Reset the decompressor and decode to some output stream. + if err := zr.(flate.Resetter).Reset(&b, nil); err != nil { + log.Fatal(err) + } + if _, err := io.CopyBuffer(os.Stdout, zr, buf); err != nil { + log.Fatal(err) + } + if err := zr.Close(); err != nil { + log.Fatal(err) + } + } + + // Output: + // Don't communicate by sharing memory, share memory by communicating. + // Concurrency is not parallelism. + // The bigger the interface, the weaker the abstraction. + // Documentation is for users. +} + +// A preset dictionary can be used to improve the compression ratio. +// The downside to using a dictionary is that the compressor and decompressor +// must agree in advance what dictionary to use. +func Example_dictionary() { + // The dictionary is a string of bytes. When compressing some input data, + // the compressor will attempt to substitute substrings with matches found + // in the dictionary. As such, the dictionary should only contain substrings + // that are expected to be found in the actual data stream. + const dict = `` + `` + `` + ` + + + + + + ... + +` + + var b bytes.Buffer + + // Compress the data using the specially crafted dictionary. + zw, err := flate.NewWriterDict(&b, flate.BestCompression, []byte(dict)) + if err != nil { + log.Fatal(err) + } + if _, err := io.Copy(zw, strings.NewReader(data)); err != nil { + log.Fatal(err) + } + if err := zw.Close(); err != nil { + log.Fatal(err) + } + + // The decompressor must use the same dictionary as the compressor. + // Otherwise, the input may appear as corrupted. + fmt.Println("Decompressed output using the dictionary:") + zr := flate.NewReaderDict(bytes.NewReader(b.Bytes()), []byte(dict)) + if _, err := io.Copy(os.Stdout, zr); err != nil { + log.Fatal(err) + } + if err := zr.Close(); err != nil { + log.Fatal(err) + } + + fmt.Println() + + // Substitute all of the bytes in the dictionary with a '#' to visually + // demonstrate the approximate effectiveness of using a preset dictionary. + fmt.Println("Substrings matched by the dictionary are marked with #:") + hashDict := []byte(dict) + for i := range hashDict { + hashDict[i] = '#' + } + zr = flate.NewReaderDict(&b, hashDict) + if _, err := io.Copy(os.Stdout, zr); err != nil { + log.Fatal(err) + } + if err := zr.Close(); err != nil { + log.Fatal(err) + } + + // Output: + // Decompressed output using the dictionary: + // + // + // + // + // + // + // ... + // + // + // Substrings matched by the dictionary are marked with #: + // ##################### + // ###### + // ############title###########The Go Programming Language"/# + // ############authors###########Alan Donovan and Brian Kernighan"/# + // ############published###########2015-10-26"/# + // ############isbn###########978-0134190440"/# + // ######... cap(e.hist) { + if cap(e.hist) == 0 { + e.hist = make([]byte, 0, allocHistory) + } else { + if cap(e.hist) < maxMatchOffset*2 { + panic("unexpected buffer size") + } + // Move down + offset := int32(len(e.hist)) - maxMatchOffset + // copy(e.hist[0:maxMatchOffset], e.hist[offset:]) + *(*[maxMatchOffset]byte)(e.hist) = *(*[maxMatchOffset]byte)(e.hist[offset:]) + e.cur += offset + e.hist = e.hist[:maxMatchOffset] + } + } + s := int32(len(e.hist)) + e.hist = append(e.hist, src...) + return s +} + +type tableEntryPrev struct { + Cur tableEntry + Prev tableEntry +} + +// hash7 returns the hash of the lowest 7 bytes of u to fit in a hash table with h bits. +// Preferably h should be a constant and should always be <64. +func hash7(u uint64, h uint8) uint32 { + return uint32(((u << (64 - 56)) * prime7bytes) >> ((64 - h) & reg8SizeMask64)) +} + +// hashLen returns a hash of the lowest mls bytes of with length output bits. +// mls must be >=3 and <=8. Any other value will return hash for 4 bytes. +// length should always be < 32. +// Preferably length and mls should be a constant for inlining. +func hashLen(u uint64, length, mls uint8) uint32 { + switch mls { + case 3: + return (uint32(u<<8) * prime3bytes) >> (32 - length) + case 5: + return uint32(((u << (64 - 40)) * prime5bytes) >> (64 - length)) + case 6: + return uint32(((u << (64 - 48)) * prime6bytes) >> (64 - length)) + case 7: + return uint32(((u << (64 - 56)) * prime7bytes) >> (64 - length)) + case 8: + return uint32((u * prime8bytes) >> (64 - length)) + default: + return (uint32(u) * prime4bytes) >> (32 - length) + } +} + +// matchlen will return the match length between offsets and t in src. +// The maximum length returned is maxMatchLength - 4. +// It is assumed that s > t, that t >=0 and s < len(src). +func (e *fastGen) matchlen(s, t int, src []byte) int32 { + if debugDeflate { + if t >= s { + panic(fmt.Sprint("t >=s:", t, s)) + } + if int(s) >= len(src) { + panic(fmt.Sprint("s >= len(src):", s, len(src))) + } + if t < 0 { + panic(fmt.Sprint("t < 0:", t)) + } + if s-t > maxMatchOffset { + panic(fmt.Sprint(s, "-", t, "(", s-t, ") > maxMatchLength (", maxMatchOffset, ")")) + } + } + a := src[s:min(s+maxMatchLength-4, len(src))] + b := src[t:] + return int32(matchLen(a, b)) +} + +// matchlenLong will return the match length between offsets and t in src. +// It is assumed that s > t, that t >=0 and s < len(src). +func (e *fastGen) matchlenLong(s, t int, src []byte) int32 { + if debugDeflate { + if t >= s { + panic(fmt.Sprint("t >=s:", t, s)) + } + if int(s) >= len(src) { + panic(fmt.Sprint("s >= len(src):", s, len(src))) + } + if t < 0 { + panic(fmt.Sprint("t < 0:", t)) + } + if s-t > maxMatchOffset { + panic(fmt.Sprint(s, "-", t, "(", s-t, ") > maxMatchLength (", maxMatchOffset, ")")) + } + } + return int32(matchLen(src[s:], src[t:])) +} + +// Reset the encoding table. +func (e *fastGen) Reset() { + if cap(e.hist) < allocHistory { + e.hist = make([]byte, 0, allocHistory) + } + // We offset current position so everything will be out of reach. + // If we are above the buffer reset it will be cleared anyway since len(hist) == 0. + if e.cur <= bufferReset { + e.cur += maxMatchOffset + int32(len(e.hist)) + } + e.hist = e.hist[:0] +} diff --git a/internal/compress/flate/flate_test.go b/internal/compress/flate/flate_test.go new file mode 100644 index 00000000..9817efef --- /dev/null +++ b/internal/compress/flate/flate_test.go @@ -0,0 +1,366 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// This test tests some internals of the flate package. +// The tests in package compress/gzip serve as the +// end-to-end test of the decompressor. + +package flate + +import ( + "archive/zip" + "bytes" + "compress/flate" + "encoding/hex" + "fmt" + "io" + "os" + "testing" +) + +// The following test should not panic. +func TestIssue5915(t *testing.T) { + bits := []int{4, 0, 0, 6, 4, 3, 2, 3, 3, 4, 4, 5, 0, 0, 0, 0, 5, 5, 6, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 8, 6, 0, 11, 0, 8, 0, 6, 6, 10, 8} + var h huffmanDecoder + if h.init(bits) { + t.Fatalf("Given sequence of bits is bad, and should not succeed.") + } +} + +// The following test should not panic. +func TestIssue5962(t *testing.T) { + bits := []int{4, 0, 0, 6, 4, 3, 2, 3, 3, 4, 4, 5, 0, 0, 0, 0, + 5, 5, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11} + var h huffmanDecoder + if h.init(bits) { + t.Fatalf("Given sequence of bits is bad, and should not succeed.") + } +} + +// The following test should not panic. +func TestIssue6255(t *testing.T) { + bits1 := []int{1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 11} + bits2 := []int{11, 13} + var h huffmanDecoder + if !h.init(bits1) { + t.Fatalf("Given sequence of bits is good and should succeed.") + } + if h.init(bits2) { + t.Fatalf("Given sequence of bits is bad and should not succeed.") + } +} + +func TestInvalidEncoding(t *testing.T) { + // Initialize Huffman decoder to recognize "0". + var h huffmanDecoder + if !h.init([]int{1}) { + t.Fatal("Failed to initialize Huffman decoder") + } + + // Initialize decompressor with invalid Huffman coding. + var f decompressor + f.r = bytes.NewReader([]byte{0xff}) + + _, err := f.huffSym(&h) + if err == nil { + t.Fatal("Should have rejected invalid bit sequence") + } +} + +func TestRegressions(t *testing.T) { + // Test fuzzer regressions + data, err := os.ReadFile("testdata/regression.zip") + if err != nil { + t.Fatal(err) + } + zr, err := zip.NewReader(bytes.NewReader(data), int64(len(data))) + if err != nil { + t.Fatal(err) + } + for _, tt := range zr.File { + data, err := tt.Open() + if err != nil { + t.Fatal(err) + } + data1, err := io.ReadAll(data) + if err != nil { + t.Fatal(err) + } + t.Run(tt.Name, func(t *testing.T) { + if testing.Short() && len(data1) > 10000 { + t.SkipNow() + } + for level := 0; level <= 9; level++ { + t.Run(fmt.Sprint(tt.Name+"-level", 1), func(t *testing.T) { + buf := new(bytes.Buffer) + fw, err := NewWriter(buf, level) + if err != nil { + t.Error(err) + } + n, err := fw.Write(data1) + if n != len(data1) { + t.Error("short write") + } + if err != nil { + t.Error(err) + } + err = fw.Close() + if err != nil { + t.Error(err) + } + fr1 := NewReader(buf) + data2, err := io.ReadAll(fr1) + if err != nil { + t.Error(err) + } + if !bytes.Equal(data1, data2) { + t.Error("not equal") + } + // Do it again... + buf.Reset() + fw.Reset(buf) + n, err = fw.Write(data1) + if n != len(data1) { + t.Error("short write") + } + if err != nil { + t.Error(err) + } + err = fw.Close() + if err != nil { + t.Error(err) + } + fr1 = flate.NewReader(buf) + data2, err = io.ReadAll(fr1) + if err != nil { + t.Error(err) + } + if !bytes.Equal(data1, data2) { + t.Error("not equal") + } + }) + } + t.Run(tt.Name+"stateless", func(t *testing.T) { + // Split into two and use history... + buf := new(bytes.Buffer) + err = StatelessDeflate(buf, data1[:len(data1)/2], false, nil) + if err != nil { + t.Error(err) + } + + // Use top half as dictionary... + dict := data1[:len(data1)/2] + err = StatelessDeflate(buf, data1[len(data1)/2:], true, dict) + if err != nil { + t.Error(err) + } + t.Log(buf.Len()) + fr1 := NewReader(buf) + data2, err := io.ReadAll(fr1) + if err != nil { + t.Error(err) + } + if !bytes.Equal(data1, data2) { + //fmt.Printf("want:%x\ngot: %x\n", data1, data2) + t.Error("not equal") + } + }) + }) + } +} + +func TestInvalidBits(t *testing.T) { + oversubscribed := []int{1, 2, 3, 4, 4, 5} + incomplete := []int{1, 2, 4, 4} + var h huffmanDecoder + if h.init(oversubscribed) { + t.Fatal("Should reject oversubscribed bit-length set") + } + if h.init(incomplete) { + t.Fatal("Should reject incomplete bit-length set") + } +} + +func TestStreams(t *testing.T) { + // To verify any of these hexstrings as valid or invalid flate streams + // according to the C zlib library, you can use the Python wrapper library: + // >>> hex_string = "010100feff11" + // >>> import zlib + // >>> zlib.decompress(hex_string.decode("hex"), -15) # Negative means raw DEFLATE + // '\x11' + + testCases := []struct { + desc string // Description of the stream + stream string // Hexstring of the input DEFLATE stream + want string // Expected result. Use "fail" to expect failure + }{{ + "degenerate HCLenTree", + "05e0010000000000100000000000000000000000000000000000000000000000" + + "00000000000000000004", + "fail", + }, { + "complete HCLenTree, empty HLitTree, empty HDistTree", + "05e0010400000000000000000000000000000000000000000000000000000000" + + "00000000000000000010", + "fail", + }, { + "empty HCLenTree", + "05e0010000000000000000000000000000000000000000000000000000000000" + + "00000000000000000010", + "fail", + }, { + "complete HCLenTree, complete HLitTree, empty HDistTree, use missing HDist symbol", + "000100feff000de0010400000000100000000000000000000000000000000000" + + "0000000000000000000000000000002c", + "fail", + }, { + "complete HCLenTree, complete HLitTree, degenerate HDistTree, use missing HDist symbol", + "000100feff000de0010000000000000000000000000000000000000000000000" + + "00000000000000000610000000004070", + "fail", + }, { + "complete HCLenTree, empty HLitTree, empty HDistTree", + "05e0010400000000100400000000000000000000000000000000000000000000" + + "0000000000000000000000000008", + "fail", + }, { + "complete HCLenTree, empty HLitTree, degenerate HDistTree", + "05e0010400000000100400000000000000000000000000000000000000000000" + + "0000000000000000000800000008", + "fail", + }, { + "complete HCLenTree, degenerate HLitTree, degenerate HDistTree, use missing HLit symbol", + "05e0010400000000100000000000000000000000000000000000000000000000" + + "0000000000000000001c", + "fail", + }, { + "complete HCLenTree, complete HLitTree, too large HDistTree", + "edff870500000000200400000000000000000000000000000000000000000000" + + "000000000000000000080000000000000004", + "fail", + }, { + "complete HCLenTree, complete HLitTree, empty HDistTree, excessive repeater code", + "edfd870500000000200400000000000000000000000000000000000000000000" + + "000000000000000000e8b100", + "fail", + }, { + "complete HCLenTree, complete HLitTree, empty HDistTree of normal length 30", + "05fd01240000000000f8ffffffffffffffffffffffffffffffffffffffffffff" + + "ffffffffffffffffff07000000fe01", + "", + }, { + "complete HCLenTree, complete HLitTree, empty HDistTree of excessive length 31", + "05fe01240000000000f8ffffffffffffffffffffffffffffffffffffffffffff" + + "ffffffffffffffffff07000000fc03", + "fail", + }, { + "complete HCLenTree, over-subscribed HLitTree, empty HDistTree", + "05e001240000000000fcffffffffffffffffffffffffffffffffffffffffffff" + + "ffffffffffffffffff07f00f", + "fail", + }, { + "complete HCLenTree, under-subscribed HLitTree, empty HDistTree", + "05e001240000000000fcffffffffffffffffffffffffffffffffffffffffffff" + + "fffffffffcffffffff07f00f", + "fail", + }, { + "complete HCLenTree, complete HLitTree with single code, empty HDistTree", + "05e001240000000000f8ffffffffffffffffffffffffffffffffffffffffffff" + + "ffffffffffffffffff07f00f", + "01", + }, { + "complete HCLenTree, complete HLitTree with multiple codes, empty HDistTree", + "05e301240000000000f8ffffffffffffffffffffffffffffffffffffffffffff" + + "ffffffffffffffffff07807f", + "01", + }, { + "complete HCLenTree, complete HLitTree, degenerate HDistTree, use valid HDist symbol", + "000100feff000de0010400000000100000000000000000000000000000000000" + + "0000000000000000000000000000003c", + "00000000", + }, { + "complete HCLenTree, degenerate HLitTree, degenerate HDistTree", + "05e0010400000000100000000000000000000000000000000000000000000000" + + "0000000000000000000c", + "", + }, { + "complete HCLenTree, degenerate HLitTree, empty HDistTree", + "05e0010400000000100000000000000000000000000000000000000000000000" + + "00000000000000000004", + "", + }, { + "complete HCLenTree, complete HLitTree, empty HDistTree, spanning repeater code", + "edfd870500000000200400000000000000000000000000000000000000000000" + + "000000000000000000e8b000", + "", + }, { + "complete HCLenTree with length codes, complete HLitTree, empty HDistTree", + "ede0010400000000100000000000000000000000000000000000000000000000" + + "0000000000000000000400004000", + "", + }, { + "complete HCLenTree, complete HLitTree, degenerate HDistTree, use valid HLit symbol 284 with count 31", + "000100feff00ede0010400000000100000000000000000000000000000000000" + + "000000000000000000000000000000040000407f00", + "0000000000000000000000000000000000000000000000000000000000000000" + + "0000000000000000000000000000000000000000000000000000000000000000" + + "0000000000000000000000000000000000000000000000000000000000000000" + + "0000000000000000000000000000000000000000000000000000000000000000" + + "0000000000000000000000000000000000000000000000000000000000000000" + + "0000000000000000000000000000000000000000000000000000000000000000" + + "0000000000000000000000000000000000000000000000000000000000000000" + + "0000000000000000000000000000000000000000000000000000000000000000" + + "000000", + }, { + "complete HCLenTree, complete HLitTree, degenerate HDistTree, use valid HLit and HDist symbols", + "0cc2010d00000082b0ac4aff0eb07d27060000ffff", + "616263616263", + }, { + "fixed block, use reserved symbol 287", + "33180700", + "fail", + }, { + "raw block", + "010100feff11", + "11", + }, { + "issue 10426 - over-subscribed HCLenTree causes a hang", + "344c4a4e494d4b070000ff2e2eff2e2e2e2e2eff", + "fail", + }, { + "issue 11030 - empty HDistTree unexpectedly leads to error", + "05c0070600000080400fff37a0ca", + "", + }, { + "issue 11033 - empty HDistTree unexpectedly leads to error", + "050fb109c020cca5d017dcbca044881ee1034ec149c8980bbc413c2ab35be9dc" + + "b1473449922449922411202306ee97b0383a521b4ffdcf3217f9f7d3adb701", + "3130303634342068652e706870005d05355f7ed957ff084a90925d19e3ebc6d0" + + "c6d7", + }} + + for i, tc := range testCases { + data, err := hex.DecodeString(tc.stream) + if err != nil { + t.Fatal(err) + } + data, err = io.ReadAll(NewReader(bytes.NewReader(data))) + if tc.want == "fail" { + if err == nil { + t.Errorf("#%d (%s): got nil error, want non-nil", i, tc.desc) + } + } else { + if err != nil { + t.Errorf("#%d (%s): %v", i, tc.desc, err) + continue + } + if got := hex.EncodeToString(data); got != tc.want { + t.Errorf("#%d (%s):\ngot %q\nwant %q", i, tc.desc, got, tc.want) + } + + } + } +} diff --git a/internal/compress/flate/fuzz_test.go b/internal/compress/flate/fuzz_test.go new file mode 100644 index 00000000..b97cd055 --- /dev/null +++ b/internal/compress/flate/fuzz_test.go @@ -0,0 +1,174 @@ +//go:build go1.18 + +package flate + +import ( + "bytes" + "flag" + "io" + "os" + "strconv" + "testing" + + "codeberg.org/lindenii/furgit/internal/compress/internal/fuzz" +) + +// Fuzzing tweaks: +var fuzzStartF = flag.Int("start", HuffmanOnly, "Start fuzzing at this level") +var fuzzEndF = flag.Int("end", BestCompression, "End fuzzing at this level (inclusive)") +var fuzzMaxF = flag.Int("max", 1<<20, "Maximum input size") +var fuzzSLF = flag.Bool("sl", true, "Include stateless encodes") +var fuzzWindow = flag.Bool("windows", true, "Include windowed encodes") + +func TestMain(m *testing.M) { + flag.Parse() + os.Exit(m.Run()) +} + +func FuzzEncoding(f *testing.F) { + fuzz.AddFromZip(f, "testdata/regression.zip", fuzz.TypeRaw, false) + fuzz.AddFromZip(f, "testdata/fuzz/encode-raw-corpus.zip", fuzz.TypeRaw, testing.Short()) + fuzz.AddFromZip(f, "testdata/fuzz/FuzzEncoding.zip", fuzz.TypeGoFuzz, testing.Short()) + + startFuzz := *fuzzStartF + endFuzz := *fuzzEndF + maxSize := *fuzzMaxF + stateless := *fuzzSLF + fuzzWindow := *fuzzWindow + + decoder := NewReader(nil) + buf := new(bytes.Buffer) + encs := make([]*Writer, endFuzz-startFuzz+1) + for i := range encs { + var err error + encs[i], err = NewWriter(nil, i+startFuzz) + if err != nil { + f.Fatal(err.Error()) + } + } + + f.Fuzz(func(t *testing.T, data []byte) { + if len(data) > maxSize { + return + } + for level := startFuzz; level <= endFuzz; level++ { + msg := "level " + strconv.Itoa(level) + ":" + buf.Reset() + fw := encs[level-startFuzz] + fw.Reset(buf) + n, err := fw.Write(data) + if n != len(data) { + t.Fatal(msg + "short write") + } + if err != nil { + t.Fatal(msg + err.Error()) + } + err = fw.Close() + if err != nil { + t.Fatal(msg + err.Error()) + } + decoder.(Resetter).Reset(buf, nil) + data2, err := io.ReadAll(decoder) + if err != nil { + t.Fatal(msg + err.Error()) + } + if !bytes.Equal(data, data2) { + t.Fatal(msg + "not equal") + } + // Do it again... (also uses copy) + msg = "level " + strconv.Itoa(level) + " (reset):" + buf.Reset() + fw.Reset(buf) + _, err = io.Copy(fw, bytes.NewReader(data)) + if err != nil { + t.Fatal(msg + err.Error()) + } + err = fw.Close() + if err != nil { + t.Fatal(msg + err.Error()) + } + decoder.(Resetter).Reset(buf, nil) + data2, err = io.ReadAll(decoder) + if err != nil { + t.Fatal(msg + err.Error()) + } + if !bytes.Equal(data, data2) { + t.Fatal(msg + "not equal") + } + } + if stateless { + // Split into two and use history... + msg := "stateless:" + buf.Reset() + err := StatelessDeflate(buf, data[:len(data)/2], false, nil) + if err != nil { + t.Error(err) + } + + // Use top half as dictionary... + dict := data[:len(data)/2] + err = StatelessDeflate(buf, data[len(data)/2:], true, dict) + if err != nil { + t.Error(err) + } + + decoder.(Resetter).Reset(buf, nil) + data2, err := io.ReadAll(decoder) + if err != nil { + t.Error(err) + } + if !bytes.Equal(data, data2) { + //fmt.Printf("want:%x\ngot: %x\n", data1, data2) + t.Error(msg + "not equal") + } + } + if fuzzWindow { + msg := "windowed:" + buf.Reset() + fw, err := NewWriterWindow(buf, 1000) + if err != nil { + t.Fatal(msg + err.Error()) + } + fw.Reset(buf) + n, err := fw.Write(data) + if n != len(data) { + t.Fatal(msg + "short write") + } + if err != nil { + t.Fatal(msg + err.Error()) + } + err = fw.Close() + if err != nil { + t.Fatal(msg + err.Error()) + } + decoder.(Resetter).Reset(buf, nil) + data2, err := io.ReadAll(decoder) + if err != nil { + t.Fatal(msg + err.Error()) + } + if !bytes.Equal(data, data2) { + t.Fatal(msg + "not equal") + } + // Do it again... + msg = msg + " (reset):" + buf.Reset() + fw.Reset(buf) + n, err = fw.Write(data) + if n != len(data) { + t.Fatal(msg + "short write") + } + if err != nil { + t.Fatal(msg + err.Error()) + } + err = fw.Close() + if err != nil { + t.Fatal(msg + err.Error()) + } + decoder.(Resetter).Reset(buf, nil) + data2, err = io.ReadAll(decoder) + if err != nil { + t.Fatal(msg + err.Error()) + } + } + }) +} diff --git a/internal/compress/flate/huffman_bit_writer.go b/internal/compress/flate/huffman_bit_writer.go new file mode 100644 index 00000000..aeab2043 --- /dev/null +++ b/internal/compress/flate/huffman_bit_writer.go @@ -0,0 +1,1174 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package flate + +import ( + "fmt" + "io" + "math" + + "codeberg.org/lindenii/furgit/internal/compress/internal/le" +) + +const ( + // The largest offset code. + offsetCodeCount = 30 + + // The special code used to mark the end of a block. + endBlockMarker = 256 + + // The first length code. + lengthCodesStart = 257 + + // The number of codegen codes. + codegenCodeCount = 19 + badCode = 255 + + // maxPredefinedTokens is the maximum number of tokens + // where we check if fixed size is smaller. + maxPredefinedTokens = 250 + + // bufferFlushSize indicates the buffer size + // after which bytes are flushed to the writer. + // Should preferably be a multiple of 6, since + // we accumulate 6 bytes between writes to the buffer. + bufferFlushSize = 246 +) + +// Minimum length code that emits bits. +const lengthExtraBitsMinCode = 8 + +// The number of extra bits needed by length code X - LENGTH_CODES_START. +var lengthExtraBits = [32]uint8{ + /* 257 */ 0, 0, 0, + /* 260 */ 0, 0, 0, 0, 0, 1, 1, 1, 1, 2, + /* 270 */ 2, 2, 2, 3, 3, 3, 3, 4, 4, 4, + /* 280 */ 4, 5, 5, 5, 5, 0, +} + +// The length indicated by length code X - LENGTH_CODES_START. +var lengthBase = [32]uint8{ + 0, 1, 2, 3, 4, 5, 6, 7, 8, 10, + 12, 14, 16, 20, 24, 28, 32, 40, 48, 56, + 64, 80, 96, 112, 128, 160, 192, 224, 255, +} + +// Minimum offset code that emits bits. +const offsetExtraBitsMinCode = 4 + +// offset code word extra bits. +var offsetExtraBits = [32]int8{ + 0, 0, 0, 0, 1, 1, 2, 2, 3, 3, + 4, 4, 5, 5, 6, 6, 7, 7, 8, 8, + 9, 9, 10, 10, 11, 11, 12, 12, 13, 13, + /* extended window */ + 14, 14, +} + +var offsetCombined = [32]uint32{} + +func init() { + var offsetBase = [32]uint32{ + /* normal deflate */ + 0x000000, 0x000001, 0x000002, 0x000003, 0x000004, + 0x000006, 0x000008, 0x00000c, 0x000010, 0x000018, + 0x000020, 0x000030, 0x000040, 0x000060, 0x000080, + 0x0000c0, 0x000100, 0x000180, 0x000200, 0x000300, + 0x000400, 0x000600, 0x000800, 0x000c00, 0x001000, + 0x001800, 0x002000, 0x003000, 0x004000, 0x006000, + + /* extended window */ + 0x008000, 0x00c000, + } + + for i := range offsetCombined[:] { + // Don't use extended window values... + if offsetExtraBits[i] == 0 || offsetBase[i] > 0x006000 { + continue + } + offsetCombined[i] = uint32(offsetExtraBits[i]) | (offsetBase[i] << 8) + } +} + +// The odd order in which the codegen code sizes are written. +var codegenOrder = []uint32{16, 17, 18, 0, 8, 7, 9, 6, 10, 5, 11, 4, 12, 3, 13, 2, 14, 1, 15} + +type huffmanBitWriter struct { + // writer is the underlying writer. + // Do not use it directly; use the write method, which ensures + // that Write errors are sticky. + writer io.Writer + + // Data waiting to be written is bytes[0:nbytes] + // and then the low nbits of bits. + bits uint64 + nbits uint8 + nbytes uint8 + lastHuffMan bool + literalEncoding *huffmanEncoder + tmpLitEncoding *huffmanEncoder + offsetEncoding *huffmanEncoder + codegenEncoding *huffmanEncoder + err error + lastHeader int + // Set between 0 (reused block can be up to 2x the size) + logNewTablePenalty uint + bytes [256 + 8]byte + literalFreq [lengthCodesStart + 32]uint16 + offsetFreq [32]uint16 + codegenFreq [codegenCodeCount]uint16 + + // codegen must have an extra space for the final symbol. + codegen [literalCount + offsetCodeCount + 1]uint8 +} + +// Huffman reuse. +// +// The huffmanBitWriter supports reusing huffman tables and thereby combining block sections. +// +// This is controlled by several variables: +// +// If lastHeader is non-zero the Huffman table can be reused. +// This also indicates that a Huffman table has been generated that can output all +// possible symbols. +// It also indicates that an EOB has not yet been emitted, so if a new tabel is generated +// an EOB with the previous table must be written. +// +// If lastHuffMan is set, a table for outputting literals has been generated and offsets are invalid. +// +// An incoming block estimates the output size of a new table using a 'fresh' by calculating the +// optimal size and adding a penalty in 'logNewTablePenalty'. +// A Huffman table is not optimal, which is why we add a penalty, and generating a new table +// is slower both for compression and decompression. + +func newHuffmanBitWriter(w io.Writer) *huffmanBitWriter { + return &huffmanBitWriter{ + writer: w, + literalEncoding: newHuffmanEncoder(literalCount), + tmpLitEncoding: newHuffmanEncoder(literalCount), + codegenEncoding: newHuffmanEncoder(codegenCodeCount), + offsetEncoding: newHuffmanEncoder(offsetCodeCount), + } +} + +func (w *huffmanBitWriter) reset(writer io.Writer) { + w.writer = writer + w.bits, w.nbits, w.nbytes, w.err = 0, 0, 0, nil + w.lastHeader = 0 + w.lastHuffMan = false +} + +func (w *huffmanBitWriter) canReuse(t *tokens) (ok bool) { + a := t.offHist[:offsetCodeCount] + b := w.offsetEncoding.codes + b = b[:len(a)] + for i, v := range a { + if v != 0 && b[i].zero() { + return false + } + } + + a = t.extraHist[:literalCount-256] + b = w.literalEncoding.codes[256:literalCount] + b = b[:len(a)] + for i, v := range a { + if v != 0 && b[i].zero() { + return false + } + } + + a = t.litHist[:256] + b = w.literalEncoding.codes[:len(a)] + for i, v := range a { + if v != 0 && b[i].zero() { + return false + } + } + return true +} + +func (w *huffmanBitWriter) flush() { + if w.err != nil { + w.nbits = 0 + return + } + if w.lastHeader > 0 { + // We owe an EOB + w.writeCode(w.literalEncoding.codes[endBlockMarker]) + w.lastHeader = 0 + } + n := w.nbytes + for w.nbits != 0 { + w.bytes[n] = byte(w.bits) + w.bits >>= 8 + if w.nbits > 8 { // Avoid underflow + w.nbits -= 8 + } else { + w.nbits = 0 + } + n++ + } + w.bits = 0 + if n > 0 { + w.write(w.bytes[:n]) + } + w.nbytes = 0 +} + +func (w *huffmanBitWriter) write(b []byte) { + if w.err != nil { + return + } + _, w.err = w.writer.Write(b) +} + +func (w *huffmanBitWriter) writeBits(b int32, nb uint8) { + w.bits |= uint64(b) << (w.nbits & 63) + w.nbits += nb + if w.nbits >= 48 { + w.writeOutBits() + } +} + +func (w *huffmanBitWriter) writeBytes(bytes []byte) { + if w.err != nil { + return + } + n := w.nbytes + if w.nbits&7 != 0 { + w.err = InternalError("writeBytes with unfinished bits") + return + } + for w.nbits != 0 { + w.bytes[n] = byte(w.bits) + w.bits >>= 8 + w.nbits -= 8 + n++ + } + if n != 0 { + w.write(w.bytes[:n]) + } + w.nbytes = 0 + w.write(bytes) +} + +// RFC 1951 3.2.7 specifies a special run-length encoding for specifying +// the literal and offset lengths arrays (which are concatenated into a single +// array). This method generates that run-length encoding. +// +// The result is written into the codegen array, and the frequencies +// of each code is written into the codegenFreq array. +// Codes 0-15 are single byte codes. Codes 16-18 are followed by additional +// information. Code badCode is an end marker +// +// numLiterals The number of literals in literalEncoding +// numOffsets The number of offsets in offsetEncoding +// litenc, offenc The literal and offset encoder to use +func (w *huffmanBitWriter) generateCodegen(numLiterals int, numOffsets int, litEnc, offEnc *huffmanEncoder) { + for i := range w.codegenFreq { + w.codegenFreq[i] = 0 + } + // Note that we are using codegen both as a temporary variable for holding + // a copy of the frequencies, and as the place where we put the result. + // This is fine because the output is always shorter than the input used + // so far. + codegen := w.codegen[:] // cache + // Copy the concatenated code sizes to codegen. Put a marker at the end. + cgnl := codegen[:numLiterals] + for i := range cgnl { + cgnl[i] = litEnc.codes[i].len() + } + + cgnl = codegen[numLiterals : numLiterals+numOffsets] + for i := range cgnl { + cgnl[i] = offEnc.codes[i].len() + } + codegen[numLiterals+numOffsets] = badCode + + size := codegen[0] + count := 1 + outIndex := 0 + for inIndex := 1; size != badCode; inIndex++ { + // INVARIANT: We have seen "count" copies of size that have not yet + // had output generated for them. + nextSize := codegen[inIndex] + if nextSize == size { + count++ + continue + } + // We need to generate codegen indicating "count" of size. + if size != 0 { + codegen[outIndex] = size + outIndex++ + w.codegenFreq[size]++ + count-- + for count >= 3 { + n := min(6, count) + codegen[outIndex] = 16 + outIndex++ + codegen[outIndex] = uint8(n - 3) + outIndex++ + w.codegenFreq[16]++ + count -= n + } + } else { + for count >= 11 { + n := min(138, count) + codegen[outIndex] = 18 + outIndex++ + codegen[outIndex] = uint8(n - 11) + outIndex++ + w.codegenFreq[18]++ + count -= n + } + if count >= 3 { + // count >= 3 && count <= 10 + codegen[outIndex] = 17 + outIndex++ + codegen[outIndex] = uint8(count - 3) + outIndex++ + w.codegenFreq[17]++ + count = 0 + } + } + count-- + for ; count >= 0; count-- { + codegen[outIndex] = size + outIndex++ + w.codegenFreq[size]++ + } + // Set up invariant for next time through the loop. + size = nextSize + count = 1 + } + // Marker indicating the end of the codegen. + codegen[outIndex] = badCode +} + +func (w *huffmanBitWriter) codegens() int { + numCodegens := len(w.codegenFreq) + for numCodegens > 4 && w.codegenFreq[codegenOrder[numCodegens-1]] == 0 { + numCodegens-- + } + return numCodegens +} + +func (w *huffmanBitWriter) headerSize() (size, numCodegens int) { + numCodegens = len(w.codegenFreq) + for numCodegens > 4 && w.codegenFreq[codegenOrder[numCodegens-1]] == 0 { + numCodegens-- + } + return 3 + 5 + 5 + 4 + (3 * numCodegens) + + w.codegenEncoding.bitLength(w.codegenFreq[:]) + + int(w.codegenFreq[16])*2 + + int(w.codegenFreq[17])*3 + + int(w.codegenFreq[18])*7, numCodegens +} + +// dynamicSize returns the size of dynamically encoded data in bits. +func (w *huffmanBitWriter) dynamicReuseSize(litEnc, offEnc *huffmanEncoder) (size int) { + size = litEnc.bitLength(w.literalFreq[:]) + + offEnc.bitLength(w.offsetFreq[:]) + return size +} + +// dynamicSize returns the size of dynamically encoded data in bits. +func (w *huffmanBitWriter) dynamicSize(litEnc, offEnc *huffmanEncoder, extraBits int) (size, numCodegens int) { + header, numCodegens := w.headerSize() + size = header + + litEnc.bitLength(w.literalFreq[:]) + + offEnc.bitLength(w.offsetFreq[:]) + + extraBits + return size, numCodegens +} + +// extraBitSize will return the number of bits that will be written +// as "extra" bits on matches. +func (w *huffmanBitWriter) extraBitSize() int { + total := 0 + for i, n := range w.literalFreq[257:literalCount] { + total += int(n) * int(lengthExtraBits[i&31]) + } + for i, n := range w.offsetFreq[:offsetCodeCount] { + total += int(n) * int(offsetExtraBits[i&31]) + } + return total +} + +// fixedSize returns the size of dynamically encoded data in bits. +func (w *huffmanBitWriter) fixedSize(extraBits int) int { + return 3 + + fixedLiteralEncoding.bitLength(w.literalFreq[:]) + + fixedOffsetEncoding.bitLength(w.offsetFreq[:]) + + extraBits +} + +// storedSize calculates the stored size, including header. +// The function returns the size in bits and whether the block +// fits inside a single block. +func (w *huffmanBitWriter) storedSize(in []byte) (int, bool) { + if in == nil { + return 0, false + } + if len(in) <= maxStoreBlockSize { + return (len(in) + 5) * 8, true + } + return 0, false +} + +func (w *huffmanBitWriter) writeCode(c hcode) { + // The function does not get inlined if we "& 63" the shift. + w.bits |= c.code64() << (w.nbits & 63) + w.nbits += c.len() + if w.nbits >= 48 { + w.writeOutBits() + } +} + +// writeOutBits will write bits to the buffer. +func (w *huffmanBitWriter) writeOutBits() { + bits := w.bits + w.bits >>= 48 + w.nbits -= 48 + n := w.nbytes + + // We overwrite, but faster... + le.Store64(w.bytes[:], n, bits) + n += 6 + + if n >= bufferFlushSize { + if w.err != nil { + n = 0 + return + } + w.write(w.bytes[:n]) + n = 0 + } + + w.nbytes = n +} + +// Write the header of a dynamic Huffman block to the output stream. +// +// numLiterals The number of literals specified in codegen +// numOffsets The number of offsets specified in codegen +// numCodegens The number of codegens used in codegen +func (w *huffmanBitWriter) writeDynamicHeader(numLiterals int, numOffsets int, numCodegens int, isEof bool) { + if w.err != nil { + return + } + var firstBits int32 = 4 + if isEof { + firstBits = 5 + } + w.writeBits(firstBits, 3) + w.writeBits(int32(numLiterals-257), 5) + w.writeBits(int32(numOffsets-1), 5) + w.writeBits(int32(numCodegens-4), 4) + + for i := range numCodegens { + value := uint(w.codegenEncoding.codes[codegenOrder[i]].len()) + w.writeBits(int32(value), 3) + } + + i := 0 + for { + var codeWord = uint32(w.codegen[i]) + i++ + if codeWord == badCode { + break + } + w.writeCode(w.codegenEncoding.codes[codeWord]) + + switch codeWord { + case 16: + w.writeBits(int32(w.codegen[i]), 2) + i++ + case 17: + w.writeBits(int32(w.codegen[i]), 3) + i++ + case 18: + w.writeBits(int32(w.codegen[i]), 7) + i++ + } + } +} + +// writeStoredHeader will write a stored header. +// If the stored block is only used for EOF, +// it is replaced with a fixed huffman block. +func (w *huffmanBitWriter) writeStoredHeader(length int, isEof bool) { + if w.err != nil { + return + } + if w.lastHeader > 0 { + // We owe an EOB + w.writeCode(w.literalEncoding.codes[endBlockMarker]) + w.lastHeader = 0 + } + + // To write EOF, use a fixed encoding block. 10 bits instead of 5 bytes. + if length == 0 && isEof { + w.writeFixedHeader(isEof) + // EOB: 7 bits, value: 0 + w.writeBits(0, 7) + w.flush() + return + } + + var flag int32 + if isEof { + flag = 1 + } + w.writeBits(flag, 3) + w.flush() + w.writeBits(int32(length), 16) + w.writeBits(int32(^uint16(length)), 16) +} + +func (w *huffmanBitWriter) writeFixedHeader(isEof bool) { + if w.err != nil { + return + } + if w.lastHeader > 0 { + // We owe an EOB + w.writeCode(w.literalEncoding.codes[endBlockMarker]) + w.lastHeader = 0 + } + + // Indicate that we are a fixed Huffman block + var value int32 = 2 + if isEof { + value = 3 + } + w.writeBits(value, 3) +} + +// writeBlock will write a block of tokens with the smallest encoding. +// The original input can be supplied, and if the huffman encoded data +// is larger than the original bytes, the data will be written as a +// stored block. +// If the input is nil, the tokens will always be Huffman encoded. +func (w *huffmanBitWriter) writeBlock(tokens *tokens, eof bool, input []byte) { + if w.err != nil { + return + } + + tokens.AddEOB() + if w.lastHeader > 0 { + // We owe an EOB + w.writeCode(w.literalEncoding.codes[endBlockMarker]) + w.lastHeader = 0 + } + numLiterals, numOffsets := w.indexTokens(tokens, false) + w.generate() + var extraBits int + storedSize, storable := w.storedSize(input) + if storable { + extraBits = w.extraBitSize() + } + + // Figure out smallest code. + // Fixed Huffman baseline. + var literalEncoding = fixedLiteralEncoding + var offsetEncoding = fixedOffsetEncoding + var size = math.MaxInt32 + if tokens.n < maxPredefinedTokens { + size = w.fixedSize(extraBits) + } + + // Dynamic Huffman? + var numCodegens int + + // Generate codegen and codegenFrequencies, which indicates how to encode + // the literalEncoding and the offsetEncoding. + w.generateCodegen(numLiterals, numOffsets, w.literalEncoding, w.offsetEncoding) + w.codegenEncoding.generate(w.codegenFreq[:], 7) + dynamicSize, numCodegens := w.dynamicSize(w.literalEncoding, w.offsetEncoding, extraBits) + + if dynamicSize < size { + size = dynamicSize + literalEncoding = w.literalEncoding + offsetEncoding = w.offsetEncoding + } + + // Stored bytes? + if storable && storedSize <= size { + w.writeStoredHeader(len(input), eof) + w.writeBytes(input) + return + } + + // Huffman. + if literalEncoding == fixedLiteralEncoding { + w.writeFixedHeader(eof) + } else { + w.writeDynamicHeader(numLiterals, numOffsets, numCodegens, eof) + } + + // Write the tokens. + w.writeTokens(tokens.Slice(), literalEncoding.codes, offsetEncoding.codes) +} + +// writeBlockDynamic encodes a block using a dynamic Huffman table. +// This should be used if the symbols used have a disproportionate +// histogram distribution. +// If input is supplied and the compression savings are below 1/16th of the +// input size the block is stored. +func (w *huffmanBitWriter) writeBlockDynamic(tokens *tokens, eof bool, input []byte, sync bool) { + if w.err != nil { + return + } + + sync = sync || eof + if sync { + tokens.AddEOB() + } + + // We cannot reuse pure huffman table, and must mark as EOF. + if (w.lastHuffMan || eof) && w.lastHeader > 0 { + // We will not try to reuse. + w.writeCode(w.literalEncoding.codes[endBlockMarker]) + w.lastHeader = 0 + w.lastHuffMan = false + } + + // fillReuse enables filling of empty values. + // This will make encodings always reusable without testing. + // However, this does not appear to benefit on most cases. + const fillReuse = false + + // Check if we can reuse... + if !fillReuse && w.lastHeader > 0 && !w.canReuse(tokens) { + w.writeCode(w.literalEncoding.codes[endBlockMarker]) + w.lastHeader = 0 + } + + numLiterals, numOffsets := w.indexTokens(tokens, true) + extraBits := 0 + ssize, storable := w.storedSize(input) + + const usePrefs = true + if storable || w.lastHeader > 0 { + extraBits = w.extraBitSize() + } + + var size int + + // Check if we should reuse. + if w.lastHeader > 0 { + // Estimate size for using a new table. + // Use the previous header size as the best estimate. + newSize := w.lastHeader + tokens.EstimatedBits() + newSize += int(w.literalEncoding.codes[endBlockMarker].len()) + newSize>>w.logNewTablePenalty + + // The estimated size is calculated as an optimal table. + // We add a penalty to make it more realistic and re-use a bit more. + reuseSize := w.dynamicReuseSize(w.literalEncoding, w.offsetEncoding) + extraBits + + // Check if a new table is better. + if newSize < reuseSize { + // Write the EOB we owe. + w.writeCode(w.literalEncoding.codes[endBlockMarker]) + size = newSize + w.lastHeader = 0 + } else { + size = reuseSize + } + + if tokens.n < maxPredefinedTokens { + if preSize := w.fixedSize(extraBits) + 7; usePrefs && preSize < size { + // Check if we get a reasonable size decrease. + if storable && ssize <= size { + w.writeStoredHeader(len(input), eof) + w.writeBytes(input) + return + } + w.writeFixedHeader(eof) + if !sync { + tokens.AddEOB() + } + w.writeTokens(tokens.Slice(), fixedLiteralEncoding.codes, fixedOffsetEncoding.codes) + return + } + } + // Check if we get a reasonable size decrease. + if storable && ssize <= size { + w.writeStoredHeader(len(input), eof) + w.writeBytes(input) + return + } + } + + // We want a new block/table + if w.lastHeader == 0 { + if fillReuse && !sync { + w.fillTokens() + numLiterals, numOffsets = maxNumLit, maxNumDist + } else { + w.literalFreq[endBlockMarker] = 1 + } + + w.generate() + // Generate codegen and codegenFrequencies, which indicates how to encode + // the literalEncoding and the offsetEncoding. + w.generateCodegen(numLiterals, numOffsets, w.literalEncoding, w.offsetEncoding) + w.codegenEncoding.generate(w.codegenFreq[:], 7) + + var numCodegens int + if fillReuse && !sync { + // Reindex for accurate size... + w.indexTokens(tokens, true) + } + size, numCodegens = w.dynamicSize(w.literalEncoding, w.offsetEncoding, extraBits) + + // Store predefined, if we don't get a reasonable improvement. + if tokens.n < maxPredefinedTokens { + if preSize := w.fixedSize(extraBits); usePrefs && preSize <= size { + // Store bytes, if we don't get an improvement. + if storable && ssize <= preSize { + w.writeStoredHeader(len(input), eof) + w.writeBytes(input) + return + } + w.writeFixedHeader(eof) + if !sync { + tokens.AddEOB() + } + w.writeTokens(tokens.Slice(), fixedLiteralEncoding.codes, fixedOffsetEncoding.codes) + return + } + } + + if storable && ssize <= size { + // Store bytes, if we don't get an improvement. + w.writeStoredHeader(len(input), eof) + w.writeBytes(input) + return + } + + // Write Huffman table. + w.writeDynamicHeader(numLiterals, numOffsets, numCodegens, eof) + if !sync { + w.lastHeader, _ = w.headerSize() + } + w.lastHuffMan = false + } + + if sync { + w.lastHeader = 0 + } + // Write the tokens. + w.writeTokens(tokens.Slice(), w.literalEncoding.codes, w.offsetEncoding.codes) +} + +func (w *huffmanBitWriter) fillTokens() { + for i, v := range w.literalFreq[:literalCount] { + if v == 0 { + w.literalFreq[i] = 1 + } + } + for i, v := range w.offsetFreq[:offsetCodeCount] { + if v == 0 { + w.offsetFreq[i] = 1 + } + } +} + +// indexTokens indexes a slice of tokens, and updates +// literalFreq and offsetFreq, and generates literalEncoding +// and offsetEncoding. +// The number of literal and offset tokens is returned. +func (w *huffmanBitWriter) indexTokens(t *tokens, alwaysEOB bool) (numLiterals, numOffsets int) { + //copy(w.literalFreq[:], t.litHist[:]) + *(*[256]uint16)(w.literalFreq[:]) = t.litHist + //copy(w.literalFreq[256:], t.extraHist[:]) + *(*[32]uint16)(w.literalFreq[256:]) = t.extraHist + w.offsetFreq = t.offHist + + if t.n == 0 { + return + } + if alwaysEOB { + w.literalFreq[endBlockMarker] = 1 + } + + // get the number of literals + numLiterals = len(w.literalFreq) + for w.literalFreq[numLiterals-1] == 0 { + numLiterals-- + } + // get the number of offsets + numOffsets = len(w.offsetFreq) + for numOffsets > 0 && w.offsetFreq[numOffsets-1] == 0 { + numOffsets-- + } + if numOffsets == 0 { + // We haven't found a single match. If we want to go with the dynamic encoding, + // we should count at least one offset to be sure that the offset huffman tree could be encoded. + w.offsetFreq[0] = 1 + numOffsets = 1 + } + return +} + +func (w *huffmanBitWriter) generate() { + w.literalEncoding.generate(w.literalFreq[:literalCount], 15) + w.offsetEncoding.generate(w.offsetFreq[:offsetCodeCount], 15) +} + +// writeTokens writes a slice of tokens to the output. +// codes for literal and offset encoding must be supplied. +func (w *huffmanBitWriter) writeTokens(tokens []token, leCodes, oeCodes []hcode) { + if w.err != nil { + return + } + if len(tokens) == 0 { + return + } + + // Only last token should be endBlockMarker. + var deferEOB bool + if tokens[len(tokens)-1] == endBlockMarker { + tokens = tokens[:len(tokens)-1] + deferEOB = true + } + + // Create slices up to the next power of two to avoid bounds checks. + lits := leCodes[:256] + offs := oeCodes[:32] + lengths := leCodes[lengthCodesStart:] + lengths = lengths[:32] + + // Go 1.16 LOVES having these on stack. + bits, nbits, nbytes := w.bits, w.nbits, w.nbytes + + for _, t := range tokens { + if t < 256 { + //w.writeCode(lits[t.literal()]) + c := lits[t] + bits |= c.code64() << (nbits & 63) + nbits += c.len() + if nbits >= 48 { + le.Store64(w.bytes[:], nbytes, bits) + bits >>= 48 + nbits -= 48 + nbytes += 6 + if nbytes >= bufferFlushSize { + if w.err != nil { + nbytes = 0 + return + } + _, w.err = w.writer.Write(w.bytes[:nbytes]) + nbytes = 0 + } + } + continue + } + + // Write the length + length := t.length() + lengthCode := lengthCode(length) & 31 + if false { + w.writeCode(lengths[lengthCode]) + } else { + // inlined + c := lengths[lengthCode] + bits |= c.code64() << (nbits & 63) + nbits += c.len() + if nbits >= 48 { + le.Store64(w.bytes[:], nbytes, bits) + bits >>= 48 + nbits -= 48 + nbytes += 6 + if nbytes >= bufferFlushSize { + if w.err != nil { + nbytes = 0 + return + } + _, w.err = w.writer.Write(w.bytes[:nbytes]) + nbytes = 0 + } + } + } + + if lengthCode >= lengthExtraBitsMinCode { + extraLengthBits := lengthExtraBits[lengthCode] + //w.writeBits(extraLength, extraLengthBits) + extraLength := int32(length - lengthBase[lengthCode]) + bits |= uint64(extraLength) << (nbits & 63) + nbits += extraLengthBits + if nbits >= 48 { + le.Store64(w.bytes[:], nbytes, bits) + bits >>= 48 + nbits -= 48 + nbytes += 6 + if nbytes >= bufferFlushSize { + if w.err != nil { + nbytes = 0 + return + } + _, w.err = w.writer.Write(w.bytes[:nbytes]) + nbytes = 0 + } + } + } + // Write the offset + offset := t.offset() + offsetCode := (offset >> 16) & 31 + if false { + w.writeCode(offs[offsetCode]) + } else { + // inlined + c := offs[offsetCode] + bits |= c.code64() << (nbits & 63) + nbits += c.len() + if nbits >= 48 { + le.Store64(w.bytes[:], nbytes, bits) + bits >>= 48 + nbits -= 48 + nbytes += 6 + if nbytes >= bufferFlushSize { + if w.err != nil { + nbytes = 0 + return + } + _, w.err = w.writer.Write(w.bytes[:nbytes]) + nbytes = 0 + } + } + } + + if offsetCode >= offsetExtraBitsMinCode { + offsetComb := offsetCombined[offsetCode] + //w.writeBits(extraOffset, extraOffsetBits) + bits |= uint64((offset-(offsetComb>>8))&matchOffsetOnlyMask) << (nbits & 63) + nbits += uint8(offsetComb) + if nbits >= 48 { + le.Store64(w.bytes[:], nbytes, bits) + bits >>= 48 + nbits -= 48 + nbytes += 6 + if nbytes >= bufferFlushSize { + if w.err != nil { + nbytes = 0 + return + } + _, w.err = w.writer.Write(w.bytes[:nbytes]) + nbytes = 0 + } + } + } + } + // Restore... + w.bits, w.nbits, w.nbytes = bits, nbits, nbytes + + if deferEOB { + w.writeCode(leCodes[endBlockMarker]) + } +} + +// huffOffset is a static offset encoder used for huffman only encoding. +// It can be reused since we will not be encoding offset values. +var huffOffset *huffmanEncoder + +func init() { + w := newHuffmanBitWriter(nil) + w.offsetFreq[0] = 1 + huffOffset = newHuffmanEncoder(offsetCodeCount) + huffOffset.generate(w.offsetFreq[:offsetCodeCount], 15) +} + +// writeBlockHuff encodes a block of bytes as either +// Huffman encoded literals or uncompressed bytes if the +// results only gains very little from compression. +func (w *huffmanBitWriter) writeBlockHuff(eof bool, input []byte, sync bool) { + if w.err != nil { + return + } + + // Clear histogram + for i := range w.literalFreq[:] { + w.literalFreq[i] = 0 + } + if !w.lastHuffMan { + for i := range w.offsetFreq[:] { + w.offsetFreq[i] = 0 + } + } + + const numLiterals = endBlockMarker + 1 + const numOffsets = 1 + + // Add everything as literals + // We have to estimate the header size. + // Assume header is around 70 bytes: + // https://stackoverflow.com/a/25454430 + const guessHeaderSizeBits = 70 * 8 + histogram(input, w.literalFreq[:numLiterals]) + ssize, storable := w.storedSize(input) + if storable && len(input) > 1024 { + // Quick check for incompressible content. + abs := float64(0) + avg := float64(len(input)) / 256 + max := float64(len(input) * 2) + for _, v := range w.literalFreq[:256] { + diff := float64(v) - avg + abs += diff * diff + if abs > max { + break + } + } + if abs < max { + if debugDeflate { + fmt.Println("stored", abs, "<", max) + } + // No chance we can compress this... + w.writeStoredHeader(len(input), eof) + w.writeBytes(input) + return + } + } + w.literalFreq[endBlockMarker] = 1 + w.tmpLitEncoding.generate(w.literalFreq[:numLiterals], 15) + estBits := w.tmpLitEncoding.canReuseBits(w.literalFreq[:numLiterals]) + if estBits < math.MaxInt32 { + estBits += w.lastHeader + if w.lastHeader == 0 { + estBits += guessHeaderSizeBits + } + estBits += estBits >> w.logNewTablePenalty + } + + // Store bytes, if we don't get a reasonable improvement. + if storable && ssize <= estBits { + if debugDeflate { + fmt.Println("stored,", ssize, "<=", estBits) + } + w.writeStoredHeader(len(input), eof) + w.writeBytes(input) + return + } + + if w.lastHeader > 0 { + reuseSize := w.literalEncoding.canReuseBits(w.literalFreq[:256]) + + if estBits < reuseSize { + if debugDeflate { + fmt.Println("NOT reusing, reuse:", reuseSize/8, "> new:", estBits/8, "header est:", w.lastHeader/8, "bytes") + } + // We owe an EOB + w.writeCode(w.literalEncoding.codes[endBlockMarker]) + w.lastHeader = 0 + } else if debugDeflate { + fmt.Println("reusing, reuse:", reuseSize/8, "> new:", estBits/8, "- header est:", w.lastHeader/8) + } + } + + count := 0 + if w.lastHeader == 0 { + // Use the temp encoding, so swap. + w.literalEncoding, w.tmpLitEncoding = w.tmpLitEncoding, w.literalEncoding + // Generate codegen and codegenFrequencies, which indicates how to encode + // the literalEncoding and the offsetEncoding. + w.generateCodegen(numLiterals, numOffsets, w.literalEncoding, huffOffset) + w.codegenEncoding.generate(w.codegenFreq[:], 7) + numCodegens := w.codegens() + + // Huffman. + w.writeDynamicHeader(numLiterals, numOffsets, numCodegens, eof) + w.lastHuffMan = true + w.lastHeader, _ = w.headerSize() + if debugDeflate { + count += w.lastHeader + fmt.Println("header:", count/8) + } + } + + encoding := w.literalEncoding.codes[:256] + // Go 1.16 LOVES having these on stack. At least 1.5x the speed. + bits, nbits, nbytes := w.bits, w.nbits, w.nbytes + + if debugDeflate { + count -= int(nbytes)*8 + int(nbits) + } + // Unroll, write 3 codes/loop. + // Fastest number of unrolls. + for len(input) > 3 { + // We must have at least 48 bits free. + if nbits >= 8 { + n := nbits >> 3 + le.Store64(w.bytes[:], nbytes, bits) + bits >>= (n * 8) & 63 + nbits -= n * 8 + nbytes += n + } + if nbytes >= bufferFlushSize { + if w.err != nil { + nbytes = 0 + return + } + if debugDeflate { + count += int(nbytes) * 8 + } + _, w.err = w.writer.Write(w.bytes[:nbytes]) + nbytes = 0 + } + a, b := encoding[input[0]], encoding[input[1]] + bits |= a.code64() << (nbits & 63) + bits |= b.code64() << ((nbits + a.len()) & 63) + c := encoding[input[2]] + nbits += b.len() + a.len() + bits |= c.code64() << (nbits & 63) + nbits += c.len() + input = input[3:] + } + + // Remaining... + for _, t := range input { + if nbits >= 48 { + le.Store64(w.bytes[:], nbytes, bits) + bits >>= 48 + nbits -= 48 + nbytes += 6 + if nbytes >= bufferFlushSize { + if w.err != nil { + nbytes = 0 + return + } + if debugDeflate { + count += int(nbytes) * 8 + } + _, w.err = w.writer.Write(w.bytes[:nbytes]) + nbytes = 0 + } + } + // Bitwriting inlined, ~30% speedup + c := encoding[t] + bits |= c.code64() << (nbits & 63) + + nbits += c.len() + if debugDeflate { + count += int(c.len()) + } + } + // Restore... + w.bits, w.nbits, w.nbytes = bits, nbits, nbytes + + if debugDeflate { + nb := count + int(nbytes)*8 + int(nbits) + fmt.Println("wrote", nb, "bits,", nb/8, "bytes.") + } + // Flush if needed to have space. + if w.nbits >= 48 { + w.writeOutBits() + } + + if eof || sync { + w.writeCode(w.literalEncoding.codes[endBlockMarker]) + w.lastHeader = 0 + w.lastHuffMan = false + } +} diff --git a/internal/compress/flate/huffman_bit_writer_test.go b/internal/compress/flate/huffman_bit_writer_test.go new file mode 100644 index 00000000..dfb93e32 --- /dev/null +++ b/internal/compress/flate/huffman_bit_writer_test.go @@ -0,0 +1,381 @@ +// Copyright 2016 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package flate + +import ( + "bytes" + "flag" + "fmt" + "os" + "path/filepath" + "strings" + "testing" +) + +var update = flag.Bool("update", false, "update reference files") + +// TestBlockHuff tests huffman encoding against reference files +// to detect possible regressions. +// If encoding/bit allocation changes you can regenerate these files +// by using the -update flag. +func TestBlockHuff(t *testing.T) { + // determine input files + match, err := filepath.Glob("testdata/huffman-*.in") + if err != nil { + t.Fatal(err) + } + + for _, in := range match { + out := in // for files where input and output are identical + if strings.HasSuffix(in, ".in") { + out = in[:len(in)-len(".in")] + ".golden" + } + t.Run(in, func(t *testing.T) { + testBlockHuff(t, in, out) + }) + } +} + +func testBlockHuff(t *testing.T, in, out string) { + all, err := os.ReadFile(in) + if err != nil { + t.Error(err) + return + } + var buf bytes.Buffer + bw := newHuffmanBitWriter(&buf) + bw.logNewTablePenalty = 8 + bw.writeBlockHuff(false, all, false) + bw.flush() + got := buf.Bytes() + + want, err := os.ReadFile(out) + if err != nil && !*update { + t.Error(err) + return + } + + t.Logf("Testing %q", in) + if !bytes.Equal(got, want) { + if *update { + if in != out { + t.Logf("Updating %q", out) + if err := os.WriteFile(out, got, 0666); err != nil { + t.Error(err) + } + return + } + // in == out: don't accidentally destroy input + t.Errorf("WARNING: -update did not rewrite input file %s", in) + } + + t.Errorf("%q != %q (see %q)", in, out, in+".got") + if err := os.WriteFile(in+".got", got, 0666); err != nil { + t.Error(err) + } + return + } + t.Log("Output ok") + + // Test if the writer produces the same output after reset. + buf.Reset() + bw.reset(&buf) + bw.writeBlockHuff(false, all, false) + bw.flush() + got = buf.Bytes() + if !bytes.Equal(got, want) { + t.Errorf("after reset %q != %q (see %q)", in, out, in+".reset.got") + if err := os.WriteFile(in+".reset.got", got, 0666); err != nil { + t.Error(err) + } + return + } + t.Log("Reset ok") + testWriterEOF(t, "huff", huffTest{input: in}, true) +} + +type huffTest struct { + tokens []token + input string // File name of input data matching the tokens. + want string // File name of data with the expected output with input available. + wantNoInput string // File name of the expected output when no input is available. +} + +const ml = 0x7fc00000 // Maximum length token. Used to reduce the size of writeBlockTests + +var writeBlockTests = []huffTest{ + { + input: "testdata/huffman-null-max.in", + want: "testdata/huffman-null-max.%s.expect", + wantNoInput: "testdata/huffman-null-max.%s.expect-noinput", + tokens: []token{0x0, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, 0x0, 0x0}, + }, + { + input: "testdata/huffman-pi.in", + want: "testdata/huffman-pi.%s.expect", + wantNoInput: "testdata/huffman-pi.%s.expect-noinput", + tokens: []token{0x33, 0x2e, 0x31, 0x34, 0x31, 0x35, 0x39, 0x32, 0x36, 0x35, 0x33, 0x35, 0x38, 0x39, 0x37, 0x39, 0x33, 0x32, 0x33, 0x38, 0x34, 0x36, 0x32, 0x36, 0x34, 0x33, 0x33, 0x38, 0x33, 0x32, 0x37, 0x39, 0x35, 0x30, 0x32, 0x38, 0x38, 0x34, 0x31, 0x39, 0x37, 0x31, 0x36, 0x39, 0x33, 0x39, 0x39, 0x33, 0x37, 0x35, 0x31, 0x30, 0x35, 0x38, 0x32, 0x30, 0x39, 0x37, 0x34, 0x39, 0x34, 0x34, 0x35, 0x39, 0x32, 0x33, 0x30, 0x37, 0x38, 0x31, 0x36, 0x34, 0x30, 0x36, 0x32, 0x38, 0x36, 0x32, 0x30, 0x38, 0x39, 0x39, 0x38, 0x36, 0x32, 0x38, 0x30, 0x33, 0x34, 0x38, 0x32, 0x35, 0x33, 0x34, 0x32, 0x31, 0x31, 0x37, 0x30, 0x36, 0x37, 0x39, 0x38, 0x32, 0x31, 0x34, 0x38, 0x30, 0x38, 0x36, 0x35, 0x31, 0x33, 0x32, 0x38, 0x32, 0x33, 0x30, 0x36, 0x36, 0x34, 0x37, 0x30, 0x39, 0x33, 0x38, 0x34, 0x34, 0x36, 0x30, 0x39, 0x35, 0x35, 0x30, 0x35, 0x38, 0x32, 0x32, 0x33, 0x31, 0x37, 0x32, 0x35, 0x33, 0x35, 0x39, 0x34, 0x30, 0x38, 0x31, 0x32, 0x38, 0x34, 0x38, 0x31, 0x31, 0x31, 0x37, 0x34, 0x4040007e, 0x34, 0x31, 0x30, 0x32, 0x37, 0x30, 0x31, 0x39, 0x33, 0x38, 0x35, 0x32, 0x31, 0x31, 0x30, 0x35, 0x35, 0x35, 0x39, 0x36, 0x34, 0x34, 0x36, 0x32, 0x32, 0x39, 0x34, 0x38, 0x39, 0x35, 0x34, 0x39, 0x33, 0x30, 0x33, 0x38, 0x31, 0x40400012, 0x32, 0x38, 0x38, 0x31, 0x30, 0x39, 0x37, 0x35, 0x36, 0x36, 0x35, 0x39, 0x33, 0x33, 0x34, 0x34, 0x36, 0x40400047, 0x37, 0x35, 0x36, 0x34, 0x38, 0x32, 0x33, 0x33, 0x37, 0x38, 0x36, 0x37, 0x38, 0x33, 0x31, 0x36, 0x35, 0x32, 0x37, 0x31, 0x32, 0x30, 0x31, 0x39, 0x30, 0x39, 0x31, 0x34, 0x4040001a, 0x35, 0x36, 0x36, 0x39, 0x32, 0x33, 0x34, 0x36, 0x404000b2, 0x36, 0x31, 0x30, 0x34, 0x35, 0x34, 0x33, 0x32, 0x36, 0x40400032, 0x31, 0x33, 0x33, 0x39, 0x33, 0x36, 0x30, 0x37, 0x32, 0x36, 0x30, 0x32, 0x34, 0x39, 0x31, 0x34, 0x31, 0x32, 0x37, 0x33, 0x37, 0x32, 0x34, 0x35, 0x38, 0x37, 0x30, 0x30, 0x36, 0x36, 0x30, 0x36, 0x33, 0x31, 0x35, 0x35, 0x38, 0x38, 0x31, 0x37, 0x34, 0x38, 0x38, 0x31, 0x35, 0x32, 0x30, 0x39, 0x32, 0x30, 0x39, 0x36, 0x32, 0x38, 0x32, 0x39, 0x32, 0x35, 0x34, 0x30, 0x39, 0x31, 0x37, 0x31, 0x35, 0x33, 0x36, 0x34, 0x33, 0x36, 0x37, 0x38, 0x39, 0x32, 0x35, 0x39, 0x30, 0x33, 0x36, 0x30, 0x30, 0x31, 0x31, 0x33, 0x33, 0x30, 0x35, 0x33, 0x30, 0x35, 0x34, 0x38, 0x38, 0x32, 0x30, 0x34, 0x36, 0x36, 0x35, 0x32, 0x31, 0x33, 0x38, 0x34, 0x31, 0x34, 0x36, 0x39, 0x35, 0x31, 0x39, 0x34, 0x31, 0x35, 0x31, 0x31, 0x36, 0x30, 0x39, 0x34, 0x33, 0x33, 0x30, 0x35, 0x37, 0x32, 0x37, 0x30, 0x33, 0x36, 0x35, 0x37, 0x35, 0x39, 0x35, 0x39, 0x31, 0x39, 0x35, 0x33, 0x30, 0x39, 0x32, 0x31, 0x38, 0x36, 0x31, 0x31, 0x37, 0x404000e9, 0x33, 0x32, 0x40400009, 0x39, 0x33, 0x31, 0x30, 0x35, 0x31, 0x31, 0x38, 0x35, 0x34, 0x38, 0x30, 0x37, 0x4040010e, 0x33, 0x37, 0x39, 0x39, 0x36, 0x32, 0x37, 0x34, 0x39, 0x35, 0x36, 0x37, 0x33, 0x35, 0x31, 0x38, 0x38, 0x35, 0x37, 0x35, 0x32, 0x37, 0x32, 0x34, 0x38, 0x39, 0x31, 0x32, 0x32, 0x37, 0x39, 0x33, 0x38, 0x31, 0x38, 0x33, 0x30, 0x31, 0x31, 0x39, 0x34, 0x39, 0x31, 0x32, 0x39, 0x38, 0x33, 0x33, 0x36, 0x37, 0x33, 0x33, 0x36, 0x32, 0x34, 0x34, 0x30, 0x36, 0x35, 0x36, 0x36, 0x34, 0x33, 0x30, 0x38, 0x36, 0x30, 0x32, 0x31, 0x33, 0x39, 0x34, 0x39, 0x34, 0x36, 0x33, 0x39, 0x35, 0x32, 0x32, 0x34, 0x37, 0x33, 0x37, 0x31, 0x39, 0x30, 0x37, 0x30, 0x32, 0x31, 0x37, 0x39, 0x38, 0x40800099, 0x37, 0x30, 0x32, 0x37, 0x37, 0x30, 0x35, 0x33, 0x39, 0x32, 0x31, 0x37, 0x31, 0x37, 0x36, 0x32, 0x39, 0x33, 0x31, 0x37, 0x36, 0x37, 0x35, 0x40800232, 0x37, 0x34, 0x38, 0x31, 0x40400006, 0x36, 0x36, 0x39, 0x34, 0x30, 0x404001e7, 0x30, 0x30, 0x30, 0x35, 0x36, 0x38, 0x31, 0x32, 0x37, 0x31, 0x34, 0x35, 0x32, 0x36, 0x33, 0x35, 0x36, 0x30, 0x38, 0x32, 0x37, 0x37, 0x38, 0x35, 0x37, 0x37, 0x31, 0x33, 0x34, 0x32, 0x37, 0x35, 0x37, 0x37, 0x38, 0x39, 0x36, 0x40400129, 0x33, 0x36, 0x33, 0x37, 0x31, 0x37, 0x38, 0x37, 0x32, 0x31, 0x34, 0x36, 0x38, 0x34, 0x34, 0x30, 0x39, 0x30, 0x31, 0x32, 0x32, 0x34, 0x39, 0x35, 0x33, 0x34, 0x33, 0x30, 0x31, 0x34, 0x36, 0x35, 0x34, 0x39, 0x35, 0x38, 0x35, 0x33, 0x37, 0x31, 0x30, 0x35, 0x30, 0x37, 0x39, 0x404000ca, 0x36, 0x40400153, 0x38, 0x39, 0x32, 0x33, 0x35, 0x34, 0x404001c9, 0x39, 0x35, 0x36, 0x31, 0x31, 0x32, 0x31, 0x32, 0x39, 0x30, 0x32, 0x31, 0x39, 0x36, 0x30, 0x38, 0x36, 0x34, 0x30, 0x33, 0x34, 0x34, 0x31, 0x38, 0x31, 0x35, 0x39, 0x38, 0x31, 0x33, 0x36, 0x32, 0x39, 0x37, 0x37, 0x34, 0x40400074, 0x30, 0x39, 0x39, 0x36, 0x30, 0x35, 0x31, 0x38, 0x37, 0x30, 0x37, 0x32, 0x31, 0x31, 0x33, 0x34, 0x39, 0x40800000, 0x38, 0x33, 0x37, 0x32, 0x39, 0x37, 0x38, 0x30, 0x34, 0x39, 0x39, 0x404002da, 0x39, 0x37, 0x33, 0x31, 0x37, 0x33, 0x32, 0x38, 0x4040018a, 0x36, 0x33, 0x31, 0x38, 0x35, 0x40400301, 0x404002e8, 0x34, 0x35, 0x35, 0x33, 0x34, 0x36, 0x39, 0x30, 0x38, 0x33, 0x30, 0x32, 0x36, 0x34, 0x32, 0x35, 0x32, 0x32, 0x33, 0x30, 0x404002e3, 0x40400267, 0x38, 0x35, 0x30, 0x33, 0x35, 0x32, 0x36, 0x31, 0x39, 0x33, 0x31, 0x31, 0x40400212, 0x31, 0x30, 0x31, 0x30, 0x30, 0x30, 0x33, 0x31, 0x33, 0x37, 0x38, 0x33, 0x38, 0x37, 0x35, 0x32, 0x38, 0x38, 0x36, 0x35, 0x38, 0x37, 0x35, 0x33, 0x33, 0x32, 0x30, 0x38, 0x33, 0x38, 0x31, 0x34, 0x32, 0x30, 0x36, 0x40400140, 0x4040012b, 0x31, 0x34, 0x37, 0x33, 0x30, 0x33, 0x35, 0x39, 0x4080032e, 0x39, 0x30, 0x34, 0x32, 0x38, 0x37, 0x35, 0x35, 0x34, 0x36, 0x38, 0x37, 0x33, 0x31, 0x31, 0x35, 0x39, 0x35, 0x40400355, 0x33, 0x38, 0x38, 0x32, 0x33, 0x35, 0x33, 0x37, 0x38, 0x37, 0x35, 0x4080037f, 0x39, 0x4040013a, 0x31, 0x40400148, 0x38, 0x30, 0x35, 0x33, 0x4040018a, 0x32, 0x32, 0x36, 0x38, 0x30, 0x36, 0x36, 0x31, 0x33, 0x30, 0x30, 0x31, 0x39, 0x32, 0x37, 0x38, 0x37, 0x36, 0x36, 0x31, 0x31, 0x31, 0x39, 0x35, 0x39, 0x40400237, 0x36, 0x40800124, 0x38, 0x39, 0x33, 0x38, 0x30, 0x39, 0x35, 0x32, 0x35, 0x37, 0x32, 0x30, 0x31, 0x30, 0x36, 0x35, 0x34, 0x38, 0x35, 0x38, 0x36, 0x33, 0x32, 0x37, 0x4040009a, 0x39, 0x33, 0x36, 0x31, 0x35, 0x33, 0x40400220, 0x4080015c, 0x32, 0x33, 0x30, 0x33, 0x30, 0x31, 0x39, 0x35, 0x32, 0x30, 0x33, 0x35, 0x33, 0x30, 0x31, 0x38, 0x35, 0x32, 0x40400171, 0x40400075, 0x33, 0x36, 0x32, 0x32, 0x35, 0x39, 0x39, 0x34, 0x31, 0x33, 0x40400254, 0x34, 0x39, 0x37, 0x32, 0x31, 0x37, 0x404000de, 0x33, 0x34, 0x37, 0x39, 0x31, 0x33, 0x31, 0x35, 0x31, 0x35, 0x35, 0x37, 0x34, 0x38, 0x35, 0x37, 0x32, 0x34, 0x32, 0x34, 0x35, 0x34, 0x31, 0x35, 0x30, 0x36, 0x39, 0x4040013f, 0x38, 0x32, 0x39, 0x35, 0x33, 0x33, 0x31, 0x31, 0x36, 0x38, 0x36, 0x31, 0x37, 0x32, 0x37, 0x38, 0x40400337, 0x39, 0x30, 0x37, 0x35, 0x30, 0x39, 0x4040010d, 0x37, 0x35, 0x34, 0x36, 0x33, 0x37, 0x34, 0x36, 0x34, 0x39, 0x33, 0x39, 0x33, 0x31, 0x39, 0x32, 0x35, 0x35, 0x30, 0x36, 0x30, 0x34, 0x30, 0x30, 0x39, 0x4040026b, 0x31, 0x36, 0x37, 0x31, 0x31, 0x33, 0x39, 0x30, 0x30, 0x39, 0x38, 0x40400335, 0x34, 0x30, 0x31, 0x32, 0x38, 0x35, 0x38, 0x33, 0x36, 0x31, 0x36, 0x30, 0x33, 0x35, 0x36, 0x33, 0x37, 0x30, 0x37, 0x36, 0x36, 0x30, 0x31, 0x30, 0x34, 0x40400172, 0x38, 0x31, 0x39, 0x34, 0x32, 0x39, 0x4080041e, 0x404000ef, 0x4040028b, 0x37, 0x38, 0x33, 0x37, 0x34, 0x404004a8, 0x38, 0x32, 0x35, 0x35, 0x33, 0x37, 0x40800209, 0x32, 0x36, 0x38, 0x4040002e, 0x34, 0x30, 0x34, 0x37, 0x404001d1, 0x34, 0x404004b5, 0x4040038d, 0x38, 0x34, 0x404003a8, 0x36, 0x40c0031f, 0x33, 0x33, 0x31, 0x33, 0x36, 0x37, 0x37, 0x30, 0x32, 0x38, 0x39, 0x38, 0x39, 0x31, 0x35, 0x32, 0x40400062, 0x35, 0x32, 0x31, 0x36, 0x32, 0x30, 0x35, 0x36, 0x39, 0x36, 0x40400411, 0x30, 0x35, 0x38, 0x40400477, 0x35, 0x40400498, 0x35, 0x31, 0x31, 0x40400209, 0x38, 0x32, 0x34, 0x33, 0x30, 0x30, 0x33, 0x35, 0x35, 0x38, 0x37, 0x36, 0x34, 0x30, 0x32, 0x34, 0x37, 0x34, 0x39, 0x36, 0x34, 0x37, 0x33, 0x32, 0x36, 0x33, 0x4040043e, 0x39, 0x39, 0x32, 0x4040044b, 0x34, 0x32, 0x36, 0x39, 0x40c002c5, 0x37, 0x404001d6, 0x34, 0x4040053d, 0x4040041d, 0x39, 0x33, 0x34, 0x31, 0x37, 0x404001ad, 0x31, 0x32, 0x4040002a, 0x34, 0x4040019e, 0x31, 0x35, 0x30, 0x33, 0x30, 0x32, 0x38, 0x36, 0x31, 0x38, 0x32, 0x39, 0x37, 0x34, 0x35, 0x35, 0x35, 0x37, 0x30, 0x36, 0x37, 0x34, 0x40400135, 0x35, 0x30, 0x35, 0x34, 0x39, 0x34, 0x35, 0x38, 0x404001c5, 0x39, 0x40400051, 0x35, 0x36, 0x404001ec, 0x37, 0x32, 0x31, 0x30, 0x37, 0x39, 0x40400159, 0x33, 0x30, 0x4040010a, 0x33, 0x32, 0x31, 0x31, 0x36, 0x35, 0x33, 0x34, 0x34, 0x39, 0x38, 0x37, 0x32, 0x30, 0x32, 0x37, 0x4040011b, 0x30, 0x32, 0x33, 0x36, 0x34, 0x4040022e, 0x35, 0x34, 0x39, 0x39, 0x31, 0x31, 0x39, 0x38, 0x40400418, 0x34, 0x4040011b, 0x35, 0x33, 0x35, 0x36, 0x36, 0x33, 0x36, 0x39, 0x40400450, 0x32, 0x36, 0x35, 0x404002e4, 0x37, 0x38, 0x36, 0x32, 0x35, 0x35, 0x31, 0x404003da, 0x31, 0x37, 0x35, 0x37, 0x34, 0x36, 0x37, 0x32, 0x38, 0x39, 0x30, 0x39, 0x37, 0x37, 0x37, 0x37, 0x40800453, 0x30, 0x30, 0x30, 0x404005fd, 0x37, 0x30, 0x404004df, 0x36, 0x404003e9, 0x34, 0x39, 0x31, 0x4040041e, 0x40400297, 0x32, 0x31, 0x34, 0x37, 0x37, 0x32, 0x33, 0x35, 0x30, 0x31, 0x34, 0x31, 0x34, 0x40400643, 0x33, 0x35, 0x36, 0x404004af, 0x31, 0x36, 0x31, 0x33, 0x36, 0x31, 0x31, 0x35, 0x37, 0x33, 0x35, 0x32, 0x35, 0x40400504, 0x33, 0x34, 0x4040005b, 0x31, 0x38, 0x4040047b, 0x38, 0x34, 0x404005e7, 0x33, 0x33, 0x32, 0x33, 0x39, 0x30, 0x37, 0x33, 0x39, 0x34, 0x31, 0x34, 0x33, 0x33, 0x33, 0x34, 0x35, 0x34, 0x37, 0x37, 0x36, 0x32, 0x34, 0x40400242, 0x32, 0x35, 0x31, 0x38, 0x39, 0x38, 0x33, 0x35, 0x36, 0x39, 0x34, 0x38, 0x35, 0x35, 0x36, 0x32, 0x30, 0x39, 0x39, 0x32, 0x31, 0x39, 0x32, 0x32, 0x32, 0x31, 0x38, 0x34, 0x32, 0x37, 0x4040023e, 0x32, 0x404000ba, 0x36, 0x38, 0x38, 0x37, 0x36, 0x37, 0x31, 0x37, 0x39, 0x30, 0x40400055, 0x30, 0x40800106, 0x36, 0x36, 0x404003e7, 0x38, 0x38, 0x36, 0x32, 0x37, 0x32, 0x404006dc, 0x31, 0x37, 0x38, 0x36, 0x30, 0x38, 0x35, 0x37, 0x40400073, 0x33, 0x408002fc, 0x37, 0x39, 0x37, 0x36, 0x36, 0x38, 0x31, 0x404002bd, 0x30, 0x30, 0x39, 0x35, 0x33, 0x38, 0x38, 0x40400638, 0x33, 0x404006a5, 0x30, 0x36, 0x38, 0x30, 0x30, 0x36, 0x34, 0x32, 0x32, 0x35, 0x31, 0x32, 0x35, 0x32, 0x4040057b, 0x37, 0x33, 0x39, 0x32, 0x40400297, 0x40400474, 0x34, 0x408006b3, 0x38, 0x36, 0x32, 0x36, 0x39, 0x34, 0x35, 0x404001e5, 0x34, 0x31, 0x39, 0x36, 0x35, 0x32, 0x38, 0x35, 0x30, 0x40400099, 0x4040039c, 0x31, 0x38, 0x36, 0x33, 0x404001be, 0x34, 0x40800154, 0x32, 0x30, 0x33, 0x39, 0x4040058b, 0x34, 0x35, 0x404002bc, 0x32, 0x33, 0x37, 0x4040042c, 0x36, 0x40400510, 0x35, 0x36, 0x40400638, 0x37, 0x31, 0x39, 0x31, 0x37, 0x32, 0x38, 0x40400171, 0x37, 0x36, 0x34, 0x36, 0x35, 0x37, 0x35, 0x37, 0x33, 0x39, 0x40400101, 0x33, 0x38, 0x39, 0x40400748, 0x38, 0x33, 0x32, 0x36, 0x34, 0x35, 0x39, 0x39, 0x35, 0x38, 0x404006a7, 0x30, 0x34, 0x37, 0x38, 0x404001de, 0x40400328, 0x39, 0x4040002d, 0x36, 0x34, 0x30, 0x37, 0x38, 0x39, 0x35, 0x31, 0x4040008e, 0x36, 0x38, 0x33, 0x4040012f, 0x32, 0x35, 0x39, 0x35, 0x37, 0x30, 0x40400468, 0x38, 0x32, 0x32, 0x404002c8, 0x32, 0x4040061b, 0x34, 0x30, 0x37, 0x37, 0x32, 0x36, 0x37, 0x31, 0x39, 0x34, 0x37, 0x38, 0x40400319, 0x38, 0x32, 0x36, 0x30, 0x31, 0x34, 0x37, 0x36, 0x39, 0x39, 0x30, 0x39, 0x404004e8, 0x30, 0x31, 0x33, 0x36, 0x33, 0x39, 0x34, 0x34, 0x33, 0x4040027f, 0x33, 0x30, 0x40400105, 0x32, 0x30, 0x33, 0x34, 0x39, 0x36, 0x32, 0x35, 0x32, 0x34, 0x35, 0x31, 0x37, 0x404003b5, 0x39, 0x36, 0x35, 0x31, 0x34, 0x33, 0x31, 0x34, 0x32, 0x39, 0x38, 0x30, 0x39, 0x31, 0x39, 0x30, 0x36, 0x35, 0x39, 0x32, 0x40400282, 0x37, 0x32, 0x32, 0x31, 0x36, 0x39, 0x36, 0x34, 0x36, 0x40400419, 0x4040007a, 0x35, 0x4040050e, 0x34, 0x40800565, 0x38, 0x40400559, 0x39, 0x37, 0x4040057b, 0x35, 0x34, 0x4040049d, 0x4040023e, 0x37, 0x4040065a, 0x38, 0x34, 0x36, 0x38, 0x31, 0x33, 0x4040008c, 0x36, 0x38, 0x33, 0x38, 0x36, 0x38, 0x39, 0x34, 0x32, 0x37, 0x37, 0x34, 0x31, 0x35, 0x35, 0x39, 0x39, 0x31, 0x38, 0x35, 0x4040005a, 0x32, 0x34, 0x35, 0x39, 0x35, 0x33, 0x39, 0x35, 0x39, 0x34, 0x33, 0x31, 0x404005b7, 0x37, 0x40400012, 0x36, 0x38, 0x30, 0x38, 0x34, 0x35, 0x404002e7, 0x37, 0x33, 0x4040081e, 0x39, 0x35, 0x38, 0x34, 0x38, 0x36, 0x35, 0x33, 0x38, 0x404006e8, 0x36, 0x32, 0x404000f2, 0x36, 0x30, 0x39, 0x404004b6, 0x36, 0x30, 0x38, 0x30, 0x35, 0x31, 0x32, 0x34, 0x33, 0x38, 0x38, 0x34, 0x4040013a, 0x4040000b, 0x34, 0x31, 0x33, 0x4040030f, 0x37, 0x36, 0x32, 0x37, 0x38, 0x40400341, 0x37, 0x31, 0x35, 0x4040059b, 0x33, 0x35, 0x39, 0x39, 0x37, 0x37, 0x30, 0x30, 0x31, 0x32, 0x39, 0x40400472, 0x38, 0x39, 0x34, 0x34, 0x31, 0x40400277, 0x36, 0x38, 0x35, 0x35, 0x4040005f, 0x34, 0x30, 0x36, 0x33, 0x404008e6, 0x32, 0x30, 0x37, 0x32, 0x32, 0x40400158, 0x40800203, 0x34, 0x38, 0x31, 0x35, 0x38, 0x40400205, 0x404001fe, 0x4040027a, 0x40400298, 0x33, 0x39, 0x34, 0x35, 0x32, 0x32, 0x36, 0x37, 0x40c00496, 0x38, 0x4040058a, 0x32, 0x31, 0x404002ea, 0x32, 0x40400387, 0x35, 0x34, 0x36, 0x36, 0x36, 0x4040051b, 0x32, 0x33, 0x39, 0x38, 0x36, 0x34, 0x35, 0x36, 0x404004c4, 0x31, 0x36, 0x33, 0x35, 0x40800253, 0x40400811, 0x37, 0x404008ad, 0x39, 0x38, 0x4040045e, 0x39, 0x33, 0x36, 0x33, 0x34, 0x4040075b, 0x37, 0x34, 0x33, 0x32, 0x34, 0x4040047b, 0x31, 0x35, 0x30, 0x37, 0x36, 0x404004bb, 0x37, 0x39, 0x34, 0x35, 0x31, 0x30, 0x39, 0x4040003e, 0x30, 0x39, 0x34, 0x30, 0x404006a6, 0x38, 0x38, 0x37, 0x39, 0x37, 0x31, 0x30, 0x38, 0x39, 0x33, 0x404008f0, 0x36, 0x39, 0x31, 0x33, 0x36, 0x38, 0x36, 0x37, 0x32, 0x4040025b, 0x404001fe, 0x35, 0x4040053f, 0x40400468, 0x40400801, 0x31, 0x37, 0x39, 0x32, 0x38, 0x36, 0x38, 0x404008cc, 0x38, 0x37, 0x34, 0x37, 0x4080079e, 0x38, 0x32, 0x34, 0x4040097a, 0x38, 0x4040025b, 0x37, 0x31, 0x34, 0x39, 0x30, 0x39, 0x36, 0x37, 0x35, 0x39, 0x38, 0x404006ef, 0x33, 0x36, 0x35, 0x40400134, 0x38, 0x31, 0x4040005c, 0x40400745, 0x40400936, 0x36, 0x38, 0x32, 0x39, 0x4040057e, 0x38, 0x37, 0x32, 0x32, 0x36, 0x35, 0x38, 0x38, 0x30, 0x40400611, 0x35, 0x40400249, 0x34, 0x32, 0x37, 0x30, 0x34, 0x37, 0x37, 0x35, 0x35, 0x4040081e, 0x33, 0x37, 0x39, 0x36, 0x34, 0x31, 0x34, 0x35, 0x31, 0x35, 0x32, 0x404005fd, 0x32, 0x33, 0x34, 0x33, 0x36, 0x34, 0x35, 0x34, 0x404005de, 0x34, 0x34, 0x34, 0x37, 0x39, 0x35, 0x4040003c, 0x40400523, 0x408008e6, 0x34, 0x31, 0x4040052a, 0x33, 0x40400304, 0x35, 0x32, 0x33, 0x31, 0x40800841, 0x31, 0x36, 0x36, 0x31, 0x404008b2, 0x35, 0x39, 0x36, 0x39, 0x35, 0x33, 0x36, 0x32, 0x33, 0x31, 0x34, 0x404005ff, 0x32, 0x34, 0x38, 0x34, 0x39, 0x33, 0x37, 0x31, 0x38, 0x37, 0x31, 0x31, 0x30, 0x31, 0x34, 0x35, 0x37, 0x36, 0x35, 0x34, 0x40400761, 0x30, 0x32, 0x37, 0x39, 0x39, 0x33, 0x34, 0x34, 0x30, 0x33, 0x37, 0x34, 0x32, 0x30, 0x30, 0x37, 0x4040093f, 0x37, 0x38, 0x35, 0x33, 0x39, 0x30, 0x36, 0x32, 0x31, 0x39, 0x40800299, 0x40400345, 0x38, 0x34, 0x37, 0x408003d2, 0x38, 0x33, 0x33, 0x32, 0x31, 0x34, 0x34, 0x35, 0x37, 0x31, 0x40400284, 0x40400776, 0x34, 0x33, 0x35, 0x30, 0x40400928, 0x40400468, 0x35, 0x33, 0x31, 0x39, 0x31, 0x30, 0x34, 0x38, 0x34, 0x38, 0x31, 0x30, 0x30, 0x35, 0x33, 0x37, 0x30, 0x36, 0x404008bc, 0x4080059d, 0x40800781, 0x31, 0x40400559, 0x37, 0x4040031b, 0x35, 0x404007ec, 0x4040040c, 0x36, 0x33, 0x408007dc, 0x34, 0x40400971, 0x4080034e, 0x408003f5, 0x38, 0x4080052d, 0x40800887, 0x39, 0x40400187, 0x39, 0x31, 0x404008ce, 0x38, 0x31, 0x34, 0x36, 0x37, 0x35, 0x31, 0x4040062b, 0x31, 0x32, 0x33, 0x39, 0x40c001a9, 0x39, 0x30, 0x37, 0x31, 0x38, 0x36, 0x34, 0x39, 0x34, 0x32, 0x33, 0x31, 0x39, 0x36, 0x31, 0x35, 0x36, 0x404001ec, 0x404006bc, 0x39, 0x35, 0x40400926, 0x40400469, 0x4040011b, 0x36, 0x30, 0x33, 0x38, 0x40400a25, 0x4040016f, 0x40400384, 0x36, 0x32, 0x4040045a, 0x35, 0x4040084c, 0x36, 0x33, 0x38, 0x39, 0x33, 0x37, 0x37, 0x38, 0x37, 0x404008c5, 0x404000f8, 0x39, 0x37, 0x39, 0x32, 0x30, 0x37, 0x37, 0x33, 0x404005d7, 0x32, 0x31, 0x38, 0x32, 0x35, 0x36, 0x404007df, 0x36, 0x36, 0x404006d6, 0x34, 0x32, 0x4080067e, 0x36, 0x404006e6, 0x34, 0x34, 0x40400024, 0x35, 0x34, 0x39, 0x32, 0x30, 0x32, 0x36, 0x30, 0x35, 0x40400ab3, 0x408003e4, 0x32, 0x30, 0x31, 0x34, 0x39, 0x404004d2, 0x38, 0x35, 0x30, 0x37, 0x33, 0x40400599, 0x36, 0x36, 0x36, 0x30, 0x40400194, 0x32, 0x34, 0x33, 0x34, 0x30, 0x40400087, 0x30, 0x4040076b, 0x38, 0x36, 0x33, 0x40400956, 0x404007e4, 0x4040042b, 0x40400174, 0x35, 0x37, 0x39, 0x36, 0x32, 0x36, 0x38, 0x35, 0x36, 0x40400140, 0x35, 0x30, 0x38, 0x40400523, 0x35, 0x38, 0x37, 0x39, 0x36, 0x39, 0x39, 0x40400711, 0x35, 0x37, 0x34, 0x40400a18, 0x38, 0x34, 0x30, 0x404008b3, 0x31, 0x34, 0x35, 0x39, 0x31, 0x4040078c, 0x37, 0x30, 0x40400234, 0x30, 0x31, 0x40400be7, 0x31, 0x32, 0x40400c74, 0x30, 0x404003c3, 0x33, 0x39, 0x40400b2a, 0x40400112, 0x37, 0x31, 0x35, 0x404003b0, 0x34, 0x32, 0x30, 0x40800bf2, 0x39, 0x40400bc2, 0x30, 0x37, 0x40400341, 0x40400795, 0x40400aaf, 0x40400c62, 0x32, 0x31, 0x40400960, 0x32, 0x35, 0x31, 0x4040057b, 0x40400944, 0x39, 0x32, 0x404001b2, 0x38, 0x32, 0x36, 0x40400b66, 0x32, 0x40400278, 0x33, 0x32, 0x31, 0x35, 0x37, 0x39, 0x31, 0x39, 0x38, 0x34, 0x31, 0x34, 0x4080087b, 0x39, 0x31, 0x36, 0x34, 0x408006e8, 0x39, 0x40800b58, 0x404008db, 0x37, 0x32, 0x32, 0x40400321, 0x35, 0x404008a4, 0x40400141, 0x39, 0x31, 0x30, 0x404000bc, 0x40400c5b, 0x35, 0x32, 0x38, 0x30, 0x31, 0x37, 0x40400231, 0x37, 0x31, 0x32, 0x40400914, 0x38, 0x33, 0x32, 0x40400373, 0x31, 0x40400589, 0x30, 0x39, 0x33, 0x35, 0x33, 0x39, 0x36, 0x35, 0x37, 0x4040064b, 0x31, 0x30, 0x38, 0x33, 0x40400069, 0x35, 0x31, 0x4040077a, 0x40400d5a, 0x31, 0x34, 0x34, 0x34, 0x32, 0x31, 0x30, 0x30, 0x40400202, 0x30, 0x33, 0x4040019c, 0x31, 0x31, 0x30, 0x33, 0x40400c81, 0x40400009, 0x40400026, 0x40c00602, 0x35, 0x31, 0x36, 0x404005d9, 0x40800883, 0x4040092a, 0x35, 0x40800c42, 0x38, 0x35, 0x31, 0x37, 0x31, 0x34, 0x33, 0x37, 0x40400605, 0x4040006d, 0x31, 0x35, 0x35, 0x36, 0x35, 0x30, 0x38, 0x38, 0x404003b9, 0x39, 0x38, 0x39, 0x38, 0x35, 0x39, 0x39, 0x38, 0x32, 0x33, 0x38, 0x404001cf, 0x404009ba, 0x33, 0x4040016c, 0x4040043e, 0x404009c3, 0x38, 0x40800e05, 0x33, 0x32, 0x40400107, 0x35, 0x40400305, 0x33, 0x404001ca, 0x39, 0x4040041b, 0x39, 0x38, 0x4040087d, 0x34, 0x40400cb8, 0x37, 0x4040064b, 0x30, 0x37, 0x404000e5, 0x34, 0x38, 0x31, 0x34, 0x31, 0x40400539, 0x38, 0x35, 0x39, 0x34, 0x36, 0x31, 0x40400bc9, 0x38, 0x30}, + }, + { + input: "testdata/huffman-rand-1k.in", + want: "testdata/huffman-rand-1k.%s.expect", + wantNoInput: "testdata/huffman-rand-1k.%s.expect-noinput", + tokens: []token{0xf8, 0x8b, 0x96, 0x76, 0x48, 0xd, 0x85, 0x94, 0x25, 0x80, 0xaf, 0xc2, 0xfe, 0x8d, 0xe8, 0x20, 0xeb, 0x17, 0x86, 0xc9, 0xb7, 0xc5, 0xde, 0x6, 0xea, 0x7d, 0x18, 0x8b, 0xe7, 0x3e, 0x7, 0xda, 0xdf, 0xff, 0x6c, 0x73, 0xde, 0xcc, 0xe7, 0x6d, 0x8d, 0x4, 0x19, 0x49, 0x7f, 0x47, 0x1f, 0x48, 0x15, 0xb0, 0xe8, 0x9e, 0xf2, 0x31, 0x59, 0xde, 0x34, 0xb4, 0x5b, 0xe5, 0xe0, 0x9, 0x11, 0x30, 0xc2, 0x88, 0x5b, 0x7c, 0x5d, 0x14, 0x13, 0x6f, 0x23, 0xa9, 0xd, 0xbc, 0x2d, 0x23, 0xbe, 0xd9, 0xed, 0x75, 0x4, 0x6c, 0x99, 0xdf, 0xfd, 0x70, 0x66, 0xe6, 0xee, 0xd9, 0xb1, 0x9e, 0x6e, 0x83, 0x59, 0xd5, 0xd4, 0x80, 0x59, 0x98, 0x77, 0x89, 0x43, 0x38, 0xc9, 0xaf, 0x30, 0x32, 0x9a, 0x20, 0x1b, 0x46, 0x3d, 0x67, 0x6e, 0xd7, 0x72, 0x9e, 0x4e, 0x21, 0x4f, 0xc6, 0xe0, 0xd4, 0x7b, 0x4, 0x8d, 0xa5, 0x3, 0xf6, 0x5, 0x9b, 0x6b, 0xdc, 0x2a, 0x93, 0x77, 0x28, 0xfd, 0xb4, 0x62, 0xda, 0x20, 0xe7, 0x1f, 0xab, 0x6b, 0x51, 0x43, 0x39, 0x2f, 0xa0, 0x92, 0x1, 0x6c, 0x75, 0x3e, 0xf4, 0x35, 0xfd, 0x43, 0x2e, 0xf7, 0xa4, 0x75, 0xda, 0xea, 0x9b, 0xa, 0x64, 0xb, 0xe0, 0x23, 0x29, 0xbd, 0xf7, 0xe7, 0x83, 0x3c, 0xfb, 0xdf, 0xb3, 0xae, 0x4f, 0xa4, 0x47, 0x55, 0x99, 0xde, 0x2f, 0x96, 0x6e, 0x1c, 0x43, 0x4c, 0x87, 0xe2, 0x7c, 0xd9, 0x5f, 0x4c, 0x7c, 0xe8, 0x90, 0x3, 0xdb, 0x30, 0x95, 0xd6, 0x22, 0xc, 0x47, 0xb8, 0x4d, 0x6b, 0xbd, 0x24, 0x11, 0xab, 0x2c, 0xd7, 0xbe, 0x6e, 0x7a, 0xd6, 0x8, 0xa3, 0x98, 0xd8, 0xdd, 0x15, 0x6a, 0xfa, 0x93, 0x30, 0x1, 0x25, 0x1d, 0xa2, 0x74, 0x86, 0x4b, 0x6a, 0x95, 0xe8, 0xe1, 0x4e, 0xe, 0x76, 0xb9, 0x49, 0xa9, 0x5f, 0xa0, 0xa6, 0x63, 0x3c, 0x7e, 0x7e, 0x20, 0x13, 0x4f, 0xbb, 0x66, 0x92, 0xb8, 0x2e, 0xa4, 0xfa, 0x48, 0xcb, 0xae, 0xb9, 0x3c, 0xaf, 0xd3, 0x1f, 0xe1, 0xd5, 0x8d, 0x42, 0x6d, 0xf0, 0xfc, 0x8c, 0xc, 0x0, 0xde, 0x40, 0xab, 0x8b, 0x47, 0x97, 0x4e, 0xa8, 0xcf, 0x8e, 0xdb, 0xa6, 0x8b, 0x20, 0x9, 0x84, 0x7a, 0x66, 0xe5, 0x98, 0x29, 0x2, 0x95, 0xe6, 0x38, 0x32, 0x60, 0x3, 0xe3, 0x9a, 0x1e, 0x54, 0xe8, 0x63, 0x80, 0x48, 0x9c, 0xe7, 0x63, 0x33, 0x6e, 0xa0, 0x65, 0x83, 0xfa, 0xc6, 0xba, 0x7a, 0x43, 0x71, 0x5, 0xf5, 0x68, 0x69, 0x85, 0x9c, 0xba, 0x45, 0xcd, 0x6b, 0xb, 0x19, 0xd1, 0xbb, 0x7f, 0x70, 0x85, 0x92, 0xd1, 0xb4, 0x64, 0x82, 0xb1, 0xe4, 0x62, 0xc5, 0x3c, 0x46, 0x1f, 0x92, 0x31, 0x1c, 0x4e, 0x41, 0x77, 0xf7, 0xe7, 0x87, 0xa2, 0xf, 0x6e, 0xe8, 0x92, 0x3, 0x6b, 0xa, 0xe7, 0xa9, 0x3b, 0x11, 0xda, 0x66, 0x8a, 0x29, 0xda, 0x79, 0xe1, 0x64, 0x8d, 0xe3, 0x54, 0xd4, 0xf5, 0xef, 0x64, 0x87, 0x3b, 0xf4, 0xc2, 0xf4, 0x71, 0x13, 0xa9, 0xe9, 0xe0, 0xa2, 0x6, 0x14, 0xab, 0x5d, 0xa7, 0x96, 0x0, 0xd6, 0xc3, 0xcc, 0x57, 0xed, 0x39, 0x6a, 0x25, 0xcd, 0x76, 0xea, 0xba, 0x3a, 0xf2, 0xa1, 0x95, 0x5d, 0xe5, 0x71, 0xcf, 0x9c, 0x62, 0x9e, 0x6a, 0xfa, 0xd5, 0x31, 0xd1, 0xa8, 0x66, 0x30, 0x33, 0xaa, 0x51, 0x17, 0x13, 0x82, 0x99, 0xc8, 0x14, 0x60, 0x9f, 0x4d, 0x32, 0x6d, 0xda, 0x19, 0x26, 0x21, 0xdc, 0x7e, 0x2e, 0x25, 0x67, 0x72, 0xca, 0xf, 0x92, 0xcd, 0xf6, 0xd6, 0xcb, 0x97, 0x8a, 0x33, 0x58, 0x73, 0x70, 0x91, 0x1d, 0xbf, 0x28, 0x23, 0xa3, 0xc, 0xf1, 0x83, 0xc3, 0xc8, 0x56, 0x77, 0x68, 0xe3, 0x82, 0xba, 0xb9, 0x57, 0x56, 0x57, 0x9c, 0xc3, 0xd6, 0x14, 0x5, 0x3c, 0xb1, 0xaf, 0x93, 0xc8, 0x8a, 0x57, 0x7f, 0x53, 0xfa, 0x2f, 0xaa, 0x6e, 0x66, 0x83, 0xfa, 0x33, 0xd1, 0x21, 0xab, 0x1b, 0x71, 0xb4, 0x7c, 0xda, 0xfd, 0xfb, 0x7f, 0x20, 0xab, 0x5e, 0xd5, 0xca, 0xfd, 0xdd, 0xe0, 0xee, 0xda, 0xba, 0xa8, 0x27, 0x99, 0x97, 0x69, 0xc1, 0x3c, 0x82, 0x8c, 0xa, 0x5c, 0x2d, 0x5b, 0x88, 0x3e, 0x34, 0x35, 0x86, 0x37, 0x46, 0x79, 0xe1, 0xaa, 0x19, 0xfb, 0xaa, 0xde, 0x15, 0x9, 0xd, 0x1a, 0x57, 0xff, 0xb5, 0xf, 0xf3, 0x2b, 0x5a, 0x6a, 0x4d, 0x19, 0x77, 0x71, 0x45, 0xdf, 0x4f, 0xb3, 0xec, 0xf1, 0xeb, 0x18, 0x53, 0x3e, 0x3b, 0x47, 0x8, 0x9a, 0x73, 0xa0, 0x5c, 0x8c, 0x5f, 0xeb, 0xf, 0x3a, 0xc2, 0x43, 0x67, 0xb4, 0x66, 0x67, 0x80, 0x58, 0xe, 0xc1, 0xec, 0x40, 0xd4, 0x22, 0x94, 0xca, 0xf9, 0xe8, 0x92, 0xe4, 0x69, 0x38, 0xbe, 0x67, 0x64, 0xca, 0x50, 0xc7, 0x6, 0x67, 0x42, 0x6e, 0xa3, 0xf0, 0xb7, 0x6c, 0xf2, 0xe8, 0x5f, 0xb1, 0xaf, 0xe7, 0xdb, 0xbb, 0x77, 0xb5, 0xf8, 0xcb, 0x8, 0xc4, 0x75, 0x7e, 0xc0, 0xf9, 0x1c, 0x7f, 0x3c, 0x89, 0x2f, 0xd2, 0x58, 0x3a, 0xe2, 0xf8, 0x91, 0xb6, 0x7b, 0x24, 0x27, 0xe9, 0xae, 0x84, 0x8b, 0xde, 0x74, 0xac, 0xfd, 0xd9, 0xb7, 0x69, 0x2a, 0xec, 0x32, 0x6f, 0xf0, 0x92, 0x84, 0xf1, 0x40, 0xc, 0x8a, 0xbc, 0x39, 0x6e, 0x2e, 0x73, 0xd4, 0x6e, 0x8a, 0x74, 0x2a, 0xdc, 0x60, 0x1f, 0xa3, 0x7, 0xde, 0x75, 0x8b, 0x74, 0xc8, 0xfe, 0x63, 0x75, 0xf6, 0x3d, 0x63, 0xac, 0x33, 0x89, 0xc3, 0xf0, 0xf8, 0x2d, 0x6b, 0xb4, 0x9e, 0x74, 0x8b, 0x5c, 0x33, 0xb4, 0xca, 0xa8, 0xe4, 0x99, 0xb6, 0x90, 0xa1, 0xef, 0xf, 0xd3, 0x61, 0xb2, 0xc6, 0x1a, 0x94, 0x7c, 0x44, 0x55, 0xf4, 0x45, 0xff, 0x9e, 0xa5, 0x5a, 0xc6, 0xa0, 0xe8, 0x2a, 0xc1, 0x8d, 0x6f, 0x34, 0x11, 0xb9, 0xbe, 0x4e, 0xd9, 0x87, 0x97, 0x73, 0xcf, 0x3d, 0x23, 0xae, 0xd5, 0x1a, 0x5e, 0xae, 0x5d, 0x6a, 0x3, 0xf9, 0x22, 0xd, 0x10, 0xd9, 0x47, 0x69, 0x15, 0x3f, 0xee, 0x52, 0xa3, 0x8, 0xd2, 0x3c, 0x51, 0xf4, 0xf8, 0x9d, 0xe4, 0x98, 0x89, 0xc8, 0x67, 0x39, 0xd5, 0x5e, 0x35, 0x78, 0x27, 0xe8, 0x3c, 0x80, 0xae, 0x79, 0x71, 0xd2, 0x93, 0xf4, 0xaa, 0x51, 0x12, 0x1c, 0x4b, 0x1b, 0xe5, 0x6e, 0x15, 0x6f, 0xe4, 0xbb, 0x51, 0x9b, 0x45, 0x9f, 0xf9, 0xc4, 0x8c, 0x2a, 0xfb, 0x1a, 0xdf, 0x55, 0xd3, 0x48, 0x93, 0x27, 0x1, 0x26, 0xc2, 0x6b, 0x55, 0x6d, 0xa2, 0xfb, 0x84, 0x8b, 0xc9, 0x9e, 0x28, 0xc2, 0xef, 0x1a, 0x24, 0xec, 0x9b, 0xae, 0xbd, 0x60, 0xe9, 0x15, 0x35, 0xee, 0x42, 0xa4, 0x33, 0x5b, 0xfa, 0xf, 0xb6, 0xf7, 0x1, 0xa6, 0x2, 0x4c, 0xca, 0x90, 0x58, 0x3a, 0x96, 0x41, 0xe7, 0xcb, 0x9, 0x8c, 0xdb, 0x85, 0x4d, 0xa8, 0x89, 0xf3, 0xb5, 0x8e, 0xfd, 0x75, 0x5b, 0x4f, 0xed, 0xde, 0x3f, 0xeb, 0x38, 0xa3, 0xbe, 0xb0, 0x73, 0xfc, 0xb8, 0x54, 0xf7, 0x4c, 0x30, 0x67, 0x2e, 0x38, 0xa2, 0x54, 0x18, 0xba, 0x8, 0xbf, 0xf2, 0x39, 0xd5, 0xfe, 0xa5, 0x41, 0xc6, 0x66, 0x66, 0xba, 0x81, 0xef, 0x67, 0xe4, 0xe6, 0x3c, 0xc, 0xca, 0xa4, 0xa, 0x79, 0xb3, 0x57, 0x8b, 0x8a, 0x75, 0x98, 0x18, 0x42, 0x2f, 0x29, 0xa3, 0x82, 0xef, 0x9f, 0x86, 0x6, 0x23, 0xe1, 0x75, 0xfa, 0x8, 0xb1, 0xde, 0x17, 0x4a}, + }, + { + input: "testdata/huffman-rand-limit.in", + want: "testdata/huffman-rand-limit.%s.expect", + wantNoInput: "testdata/huffman-rand-limit.%s.expect-noinput", + tokens: []token{0x61, 0x51c00000, 0xa, 0xf8, 0x8b, 0x96, 0x76, 0x48, 0xa, 0x85, 0x94, 0x25, 0x80, 0xaf, 0xc2, 0xfe, 0x8d, 0xe8, 0x20, 0xeb, 0x17, 0x86, 0xc9, 0xb7, 0xc5, 0xde, 0x6, 0xea, 0x7d, 0x18, 0x8b, 0xe7, 0x3e, 0x7, 0xda, 0xdf, 0xff, 0x6c, 0x73, 0xde, 0xcc, 0xe7, 0x6d, 0x8d, 0x4, 0x19, 0x49, 0x7f, 0x47, 0x1f, 0x48, 0x15, 0xb0, 0xe8, 0x9e, 0xf2, 0x31, 0x59, 0xde, 0x34, 0xb4, 0x5b, 0xe5, 0xe0, 0x9, 0x11, 0x30, 0xc2, 0x88, 0x5b, 0x7c, 0x5d, 0x14, 0x13, 0x6f, 0x23, 0xa9, 0xa, 0xbc, 0x2d, 0x23, 0xbe, 0xd9, 0xed, 0x75, 0x4, 0x6c, 0x99, 0xdf, 0xfd, 0x70, 0x66, 0xe6, 0xee, 0xd9, 0xb1, 0x9e, 0x6e, 0x83, 0x59, 0xd5, 0xd4, 0x80, 0x59, 0x98, 0x77, 0x89, 0x43, 0x38, 0xc9, 0xaf, 0x30, 0x32, 0x9a, 0x20, 0x1b, 0x46, 0x3d, 0x67, 0x6e, 0xd7, 0x72, 0x9e, 0x4e, 0x21, 0x4f, 0xc6, 0xe0, 0xd4, 0x7b, 0x4, 0x8d, 0xa5, 0x3, 0xf6, 0x5, 0x9b, 0x6b, 0xdc, 0x2a, 0x93, 0x77, 0x28, 0xfd, 0xb4, 0x62, 0xda, 0x20, 0xe7, 0x1f, 0xab, 0x6b, 0x51, 0x43, 0x39, 0x2f, 0xa0, 0x92, 0x1, 0x6c, 0x75, 0x3e, 0xf4, 0x35, 0xfd, 0x43, 0x2e, 0xf7, 0xa4, 0x75, 0xda, 0xea, 0x9b, 0xa}, + }, + { + input: "testdata/huffman-shifts.in", + want: "testdata/huffman-shifts.%s.expect", + wantNoInput: "testdata/huffman-shifts.%s.expect-noinput", + tokens: []token{0x31, 0x30, 0x7fc00001, 0x7fc00001, 0x7fc00001, 0x7fc00001, 0x7fc00001, 0x7fc00001, 0x7fc00001, 0x7fc00001, 0x7fc00001, 0x7fc00001, 0x7fc00001, 0x7fc00001, 0x7fc00001, 0x7fc00001, 0x7fc00001, 0x52400001, 0xd, 0xa, 0x32, 0x33, 0x7fc00001, 0x7fc00001, 0x7fc00001, 0x7fc00001, 0x7fc00001, 0x7fc00001, 0x7fc00001, 0x7fc00001, 0x7fc00001, 0x7f400001}, + }, + { + input: "testdata/huffman-text-shift.in", + want: "testdata/huffman-text-shift.%s.expect", + wantNoInput: "testdata/huffman-text-shift.%s.expect-noinput", + tokens: []token{0x2f, 0x2f, 0x43, 0x6f, 0x70, 0x79, 0x72, 0x69, 0x67, 0x68, 0x74, 0x32, 0x30, 0x30, 0x39, 0x54, 0x68, 0x47, 0x6f, 0x41, 0x75, 0x74, 0x68, 0x6f, 0x72, 0x2e, 0x41, 0x6c, 0x6c, 0x40800016, 0x72, 0x72, 0x76, 0x64, 0x2e, 0xd, 0xa, 0x2f, 0x2f, 0x55, 0x6f, 0x66, 0x74, 0x68, 0x69, 0x6f, 0x75, 0x72, 0x63, 0x63, 0x6f, 0x64, 0x69, 0x67, 0x6f, 0x76, 0x72, 0x6e, 0x64, 0x62, 0x79, 0x42, 0x53, 0x44, 0x2d, 0x74, 0x79, 0x6c, 0x40400020, 0x6c, 0x69, 0x63, 0x6e, 0x74, 0x68, 0x74, 0x63, 0x6e, 0x62, 0x66, 0x6f, 0x75, 0x6e, 0x64, 0x69, 0x6e, 0x74, 0x68, 0x4c, 0x49, 0x43, 0x45, 0x4e, 0x53, 0x45, 0x66, 0x69, 0x6c, 0x2e, 0xd, 0xa, 0xd, 0xa, 0x70, 0x63, 0x6b, 0x67, 0x6d, 0x69, 0x6e, 0x4040000a, 0x69, 0x6d, 0x70, 0x6f, 0x72, 0x74, 0x22, 0x6f, 0x22, 0x4040000c, 0x66, 0x75, 0x6e, 0x63, 0x6d, 0x69, 0x6e, 0x28, 0x29, 0x7b, 0xd, 0xa, 0x9, 0x76, 0x72, 0x62, 0x3d, 0x6d, 0x6b, 0x28, 0x5b, 0x5d, 0x62, 0x79, 0x74, 0x2c, 0x36, 0x35, 0x35, 0x33, 0x35, 0x29, 0xd, 0xa, 0x9, 0x66, 0x2c, 0x5f, 0x3a, 0x3d, 0x6f, 0x2e, 0x43, 0x72, 0x74, 0x28, 0x22, 0x68, 0x75, 0x66, 0x66, 0x6d, 0x6e, 0x2d, 0x6e, 0x75, 0x6c, 0x6c, 0x2d, 0x6d, 0x78, 0x2e, 0x69, 0x6e, 0x22, 0x40800021, 0x2e, 0x57, 0x72, 0x69, 0x74, 0x28, 0x62, 0x29, 0xd, 0xa, 0x7d, 0xd, 0xa, 0x41, 0x42, 0x43, 0x44, 0x45, 0x46, 0x47, 0x48, 0x49, 0x4a, 0x4b, 0x4c, 0x4d, 0x4e, 0x4f, 0x50, 0x51, 0x52, 0x53, 0x54, 0x55, 0x56, 0x58, 0x78, 0x79, 0x7a, 0x21, 0x22, 0x23, 0xc2, 0xa4, 0x25, 0x26, 0x2f, 0x3f, 0x22}, + }, + { + input: "testdata/huffman-text.in", + want: "testdata/huffman-text.%s.expect", + wantNoInput: "testdata/huffman-text.%s.expect-noinput", + tokens: []token{0x2f, 0x2f, 0x20, 0x43, 0x6f, 0x70, 0x79, 0x72, 0x69, 0x67, 0x68, 0x74, 0x20, 0x32, 0x30, 0x30, 0x39, 0x20, 0x54, 0x68, 0x65, 0x20, 0x47, 0x6f, 0x20, 0x41, 0x75, 0x74, 0x68, 0x6f, 0x72, 0x73, 0x2e, 0x20, 0x41, 0x6c, 0x6c, 0x20, 0x4080001e, 0x73, 0x20, 0x72, 0x65, 0x73, 0x65, 0x72, 0x76, 0x65, 0x64, 0x2e, 0xd, 0xa, 0x2f, 0x2f, 0x20, 0x55, 0x73, 0x65, 0x20, 0x6f, 0x66, 0x20, 0x74, 0x68, 0x69, 0x73, 0x20, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x20, 0x63, 0x6f, 0x64, 0x65, 0x20, 0x69, 0x73, 0x20, 0x67, 0x6f, 0x76, 0x65, 0x72, 0x6e, 0x65, 0x64, 0x20, 0x62, 0x79, 0x20, 0x61, 0x20, 0x42, 0x53, 0x44, 0x2d, 0x73, 0x74, 0x79, 0x6c, 0x65, 0x40800036, 0x6c, 0x69, 0x63, 0x65, 0x6e, 0x73, 0x65, 0x20, 0x74, 0x68, 0x61, 0x74, 0x20, 0x63, 0x61, 0x6e, 0x20, 0x62, 0x65, 0x20, 0x66, 0x6f, 0x75, 0x6e, 0x64, 0x20, 0x69, 0x6e, 0x20, 0x74, 0x68, 0x65, 0x20, 0x4c, 0x49, 0x43, 0x45, 0x4e, 0x53, 0x45, 0x20, 0x66, 0x69, 0x6c, 0x65, 0x2e, 0xd, 0xa, 0xd, 0xa, 0x70, 0x61, 0x63, 0x6b, 0x61, 0x67, 0x65, 0x20, 0x6d, 0x61, 0x69, 0x6e, 0x4040000f, 0x69, 0x6d, 0x70, 0x6f, 0x72, 0x74, 0x20, 0x22, 0x6f, 0x73, 0x22, 0x4040000e, 0x66, 0x75, 0x6e, 0x63, 0x4080001b, 0x28, 0x29, 0x20, 0x7b, 0xd, 0xa, 0x9, 0x76, 0x61, 0x72, 0x20, 0x62, 0x20, 0x3d, 0x20, 0x6d, 0x61, 0x6b, 0x65, 0x28, 0x5b, 0x5d, 0x62, 0x79, 0x74, 0x65, 0x2c, 0x20, 0x36, 0x35, 0x35, 0x33, 0x35, 0x29, 0xd, 0xa, 0x9, 0x66, 0x2c, 0x20, 0x5f, 0x20, 0x3a, 0x3d, 0x20, 0x6f, 0x73, 0x2e, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x28, 0x22, 0x68, 0x75, 0x66, 0x66, 0x6d, 0x61, 0x6e, 0x2d, 0x6e, 0x75, 0x6c, 0x6c, 0x2d, 0x6d, 0x61, 0x78, 0x2e, 0x69, 0x6e, 0x22, 0x4080002a, 0x2e, 0x57, 0x72, 0x69, 0x74, 0x65, 0x28, 0x62, 0x29, 0xd, 0xa, 0x7d, 0xd, 0xa}, + }, + { + input: "testdata/huffman-zero.in", + want: "testdata/huffman-zero.%s.expect", + wantNoInput: "testdata/huffman-zero.%s.expect-noinput", + tokens: []token{0x30, ml, 0x4b800000}, + }, + { + input: "", + want: "", + wantNoInput: "testdata/null-long-match.%s.expect-noinput", + tokens: []token{0x0, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, 0x41400000}, + }, +} + +// TestWriteBlock tests if the writeBlock encoding has changed. +// To update the reference files use the "-update" flag on the test. +func TestWriteBlock(t *testing.T) { + for _, test := range writeBlockTests { + testBlock(t, test, "wb") + } +} + +// TestWriteBlockDynamic tests if the writeBlockDynamic encoding has changed. +// To update the reference files use the "-update" flag on the test. +func TestWriteBlockDynamic(t *testing.T) { + for _, test := range writeBlockTests { + testBlock(t, test, "dyn") + } +} + +// TestWriteBlockDynamic tests if the writeBlockDynamic encoding has changed. +// To update the reference files use the "-update" flag on the test. +func TestWriteBlockDynamicSync(t *testing.T) { + for _, test := range writeBlockTests { + testBlock(t, test, "sync") + } +} + +// testBlock tests a block against its references, +// or regenerate the references, if "-update" flag is set. +func testBlock(t *testing.T, test huffTest, ttype string) { + if test.want != "" { + test.want = fmt.Sprintf(test.want, ttype) + } + const gotSuffix = ".got" + test.wantNoInput = fmt.Sprintf(test.wantNoInput, ttype) + tokens := indexTokens(test.tokens) + if *update { + if test.input != "" { + t.Logf("Updating %q", test.want) + input, err := os.ReadFile(test.input) + if err != nil { + t.Error(err) + return + } + + f, err := os.Create(test.want) + if err != nil { + t.Error(err) + return + } + defer f.Close() + bw := newHuffmanBitWriter(f) + writeToType(t, ttype, bw, tokens, input) + } + + t.Logf("Updating %q", test.wantNoInput) + f, err := os.Create(test.wantNoInput) + if err != nil { + t.Error(err) + return + } + defer f.Close() + bw := newHuffmanBitWriter(f) + writeToType(t, ttype, bw, tokens, nil) + return + } + + if test.input != "" { + t.Logf("Testing %q", test.want) + input, err := os.ReadFile(test.input) + if err != nil { + t.Error(err) + return + } + want, err := os.ReadFile(test.want) + if err != nil { + t.Error(err) + return + } + var buf bytes.Buffer + bw := newHuffmanBitWriter(&buf) + writeToType(t, ttype, bw, tokens, input) + + got := buf.Bytes() + if !bytes.Equal(got, want) { + t.Errorf("writeBlock did not yield expected result for file %q with input. See %q", test.want, test.want+gotSuffix) + if err := os.WriteFile(test.want+gotSuffix, got, 0666); err != nil { + t.Error(err) + } + } + t.Log("Output ok") + + // Test if the writer produces the same output after reset. + buf.Reset() + bw.reset(&buf) + writeToType(t, ttype, bw, tokens, input) + bw.flush() + got = buf.Bytes() + if !bytes.Equal(got, want) { + t.Errorf("reset: writeBlock did not yield expected result for file %q with input. See %q", test.want, test.want+".reset"+gotSuffix) + if err := os.WriteFile(test.want+".reset"+gotSuffix, got, 0666); err != nil { + t.Error(err) + } + return + } + t.Log("Reset ok") + testWriterEOF(t, "wb", test, true) + } + t.Logf("Testing %q", test.wantNoInput) + wantNI, err := os.ReadFile(test.wantNoInput) + if err != nil { + t.Error(err) + return + } + var buf bytes.Buffer + bw := newHuffmanBitWriter(&buf) + writeToType(t, ttype, bw, tokens, nil) + + got := buf.Bytes() + if !bytes.Equal(got, wantNI) { + t.Errorf("writeBlock did not yield expected result for file %q with input. See %q", test.wantNoInput, test.wantNoInput+gotSuffix) + if err := os.WriteFile(test.wantNoInput+gotSuffix, got, 0666); err != nil { + t.Error(err) + } + } else if got[0]&1 == 1 { + t.Error("got unexpected EOF") + return + } + + t.Log("Output ok") + + // Test if the writer produces the same output after reset. + buf.Reset() + bw.reset(&buf) + writeToType(t, ttype, bw, tokens, nil) + bw.flush() + got = buf.Bytes() + if !bytes.Equal(got, wantNI) { + t.Errorf("reset: writeBlock did not yield expected result for file %q without input. See %q", test.wantNoInput, test.wantNoInput+".reset"+gotSuffix) + if err := os.WriteFile(test.wantNoInput+".reset"+gotSuffix, got, 0666); err != nil { + t.Error(err) + } + return + } + t.Log("Reset ok") + testWriterEOF(t, "wb", test, false) +} + +func writeToType(t *testing.T, ttype string, bw *huffmanBitWriter, tok tokens, input []byte) { + switch ttype { + case "wb": + bw.writeBlock(&tok, false, input) + case "dyn": + bw.writeBlockDynamic(&tok, false, input, false) + case "sync": + bw.writeBlockDynamic(&tok, false, input, true) + default: + panic("unknown test type") + } + + if bw.err != nil { + t.Error(bw.err) + return + } + + bw.flush() + if bw.err != nil { + t.Error(bw.err) + return + } +} + +// testWriterEOF tests if the written block contains an EOF marker. +func testWriterEOF(t *testing.T, ttype string, test huffTest, useInput bool) { + if useInput && test.input == "" { + return + } + var input []byte + if useInput { + var err error + input, err = os.ReadFile(test.input) + if err != nil { + t.Error(err) + return + } + } + var buf bytes.Buffer + bw := newHuffmanBitWriter(&buf) + tokens := indexTokens(test.tokens) + switch ttype { + case "wb": + bw.writeBlock(&tokens, true, input) + case "dyn": + bw.writeBlockDynamic(&tokens, true, input, true) + case "huff": + bw.writeBlockHuff(true, input, true) + default: + panic("unknown test type") + } + if bw.err != nil { + t.Error(bw.err) + return + } + + bw.flush() + if bw.err != nil { + t.Error(bw.err) + return + } + b := buf.Bytes() + if len(b) == 0 { + t.Error("no output received") + return + } + if b[0]&1 != 1 { + t.Errorf("block not marked with EOF for input %q", test.input) + return + } + t.Log("EOF ok") +} diff --git a/internal/compress/flate/huffman_code.go b/internal/compress/flate/huffman_code.go new file mode 100644 index 00000000..5f901bd0 --- /dev/null +++ b/internal/compress/flate/huffman_code.go @@ -0,0 +1,417 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package flate + +import ( + "math" + "math/bits" +) + +const ( + maxBitsLimit = 16 + // number of valid literals + literalCount = 286 +) + +// hcode is a huffman code with a bit code and bit length. +type hcode uint32 + +func (h hcode) len() uint8 { + return uint8(h) +} + +func (h hcode) code64() uint64 { + return uint64(h >> 8) +} + +func (h hcode) zero() bool { + return h == 0 +} + +type huffmanEncoder struct { + codes []hcode + bitCount [17]int32 + + // Allocate a reusable buffer with the longest possible frequency table. + // Possible lengths are codegenCodeCount, offsetCodeCount and literalCount. + // The largest of these is literalCount, so we allocate for that case. + freqcache [literalCount + 1]literalNode +} + +type literalNode struct { + literal uint16 + freq uint16 +} + +// A levelInfo describes the state of the constructed tree for a given depth. +type levelInfo struct { + // Our level. for better printing + level int32 + + // The frequency of the last node at this level + lastFreq int32 + + // The frequency of the next character to add to this level + nextCharFreq int32 + + // The frequency of the next pair (from level below) to add to this level. + // Only valid if the "needed" value of the next lower level is 0. + nextPairFreq int32 + + // The number of chains remaining to generate for this level before moving + // up to the next level + needed int32 +} + +// set sets the code and length of an hcode. +func (h *hcode) set(code uint16, length uint8) { + *h = hcode(length) | (hcode(code) << 8) +} + +func newhcode(code uint16, length uint8) hcode { + return hcode(length) | (hcode(code) << 8) +} + +func reverseBits(number uint16, bitLength byte) uint16 { + return bits.Reverse16(number << ((16 - bitLength) & 15)) +} + +func maxNode() literalNode { return literalNode{math.MaxUint16, math.MaxUint16} } + +func newHuffmanEncoder(size int) *huffmanEncoder { + // Make capacity to next power of two. + c := uint(bits.Len32(uint32(size - 1))) + return &huffmanEncoder{codes: make([]hcode, size, 1<= 3 +// The cases of 0, 1, and 2 literals are handled by special case code. +// +// list An array of the literals with non-zero frequencies +// +// and their associated frequencies. The array is in order of increasing +// frequency, and has as its last element a special element with frequency +// MaxInt32 +// +// maxBits The maximum number of bits that should be used to encode any literal. +// +// Must be less than 16. +// +// return An integer array in which array[i] indicates the number of literals +// +// that should be encoded in i bits. +func (h *huffmanEncoder) bitCounts(list []literalNode, maxBits int32) []int32 { + if maxBits >= maxBitsLimit { + panic("flate: maxBits too large") + } + n := int32(len(list)) + list = list[0 : n+1] + list[n] = maxNode() + + // The tree can't have greater depth than n - 1, no matter what. This + // saves a little bit of work in some small cases + if maxBits > n-1 { + maxBits = n - 1 + } + + // Create information about each of the levels. + // A bogus "Level 0" whose sole purpose is so that + // level1.prev.needed==0. This makes level1.nextPairFreq + // be a legitimate value that never gets chosen. + var levels [maxBitsLimit]levelInfo + // leafCounts[i] counts the number of literals at the left + // of ancestors of the rightmost node at level i. + // leafCounts[i][j] is the number of literals at the left + // of the level j ancestor. + var leafCounts [maxBitsLimit][maxBitsLimit]int32 + + // Descending to only have 1 bounds check. + l2f := int32(list[2].freq) + l1f := int32(list[1].freq) + l0f := int32(list[0].freq) + int32(list[1].freq) + + for level := int32(1); level <= maxBits; level++ { + // For every level, the first two items are the first two characters. + // We initialize the levels as if we had already figured this out. + levels[level] = levelInfo{ + level: level, + lastFreq: l1f, + nextCharFreq: l2f, + nextPairFreq: l0f, + } + leafCounts[level][level] = 2 + if level == 1 { + levels[level].nextPairFreq = math.MaxInt32 + } + } + + // We need a total of 2*n - 2 items at top level and have already generated 2. + levels[maxBits].needed = 2*n - 4 + + level := uint32(maxBits) + for level < 16 { + l := &levels[level] + if l.nextPairFreq == math.MaxInt32 && l.nextCharFreq == math.MaxInt32 { + // We've run out of both leafs and pairs. + // End all calculations for this level. + // To make sure we never come back to this level or any lower level, + // set nextPairFreq impossibly large. + l.needed = 0 + levels[level+1].nextPairFreq = math.MaxInt32 + level++ + continue + } + + prevFreq := l.lastFreq + if l.nextCharFreq < l.nextPairFreq { + // The next item on this row is a leaf node. + n := leafCounts[level][level] + 1 + l.lastFreq = l.nextCharFreq + // Lower leafCounts are the same of the previous node. + leafCounts[level][level] = n + e := list[n] + if e.literal < math.MaxUint16 { + l.nextCharFreq = int32(e.freq) + } else { + l.nextCharFreq = math.MaxInt32 + } + } else { + // The next item on this row is a pair from the previous row. + // nextPairFreq isn't valid until we generate two + // more values in the level below + l.lastFreq = l.nextPairFreq + // Take leaf counts from the lower level, except counts[level] remains the same. + if true { + save := leafCounts[level][level] + leafCounts[level] = leafCounts[level-1] + leafCounts[level][level] = save + } else { + copy(leafCounts[level][:level], leafCounts[level-1][:level]) + } + levels[l.level-1].needed = 2 + } + + if l.needed--; l.needed == 0 { + // We've done everything we need to do for this level. + // Continue calculating one level up. Fill in nextPairFreq + // of that level with the sum of the two nodes we've just calculated on + // this level. + if l.level == maxBits { + // All done! + break + } + levels[l.level+1].nextPairFreq = prevFreq + l.lastFreq + level++ + } else { + // If we stole from below, move down temporarily to replenish it. + for levels[level-1].needed > 0 { + level-- + } + } + } + + // Somethings is wrong if at the end, the top level is null or hasn't used + // all of the leaves. + if leafCounts[maxBits][maxBits] != n { + panic("leafCounts[maxBits][maxBits] != n") + } + + bitCount := h.bitCount[:maxBits+1] + bits := 1 + counts := &leafCounts[maxBits] + for level := maxBits; level > 0; level-- { + // chain.leafCount gives the number of literals requiring at least "bits" + // bits to encode. + bitCount[bits] = counts[level] - counts[level-1] + bits++ + } + return bitCount +} + +// Look at the leaves and assign them a bit count and an encoding as specified +// in RFC 1951 3.2.2 +func (h *huffmanEncoder) assignEncodingAndSize(bitCount []int32, list []literalNode) { + code := uint16(0) + for n, bits := range bitCount { + code <<= 1 + if n == 0 || bits == 0 { + continue + } + // The literals list[len(list)-bits] .. list[len(list)-bits] + // are encoded using "bits" bits, and get the values + // code, code + 1, .... The code values are + // assigned in literal order (not frequency order). + chunk := list[len(list)-int(bits):] + + sortByLiteral(chunk) + for _, node := range chunk { + h.codes[node.literal] = newhcode(reverseBits(code, uint8(n)), uint8(n)) + code++ + } + list = list[0 : len(list)-int(bits)] + } +} + +// Update this Huffman Code object to be the minimum code for the specified frequency count. +// +// freq An array of frequencies, in which frequency[i] gives the frequency of literal i. +// maxBits The maximum number of bits to use for any literal. +func (h *huffmanEncoder) generate(freq []uint16, maxBits int32) { + list := h.freqcache[:len(freq)+1] + codes := h.codes[:len(freq)] + // Number of non-zero literals + count := 0 + // Set list to be the set of all non-zero literals and their frequencies + for i, f := range freq { + if f != 0 { + list[count] = literalNode{uint16(i), f} + count++ + } else { + codes[i] = 0 + } + } + list[count] = literalNode{} + + list = list[:count] + if count <= 2 { + // Handle the small cases here, because they are awkward for the general case code. With + // two or fewer literals, everything has bit length 1. + for i, node := range list { + // "list" is in order of increasing literal value. + h.codes[node.literal].set(uint16(i), 1) + } + return + } + sortByFreq(list) + + // Get the number of literals for each bit count + bitCount := h.bitCounts(list, maxBits) + // And do the assignment + h.assignEncodingAndSize(bitCount, list) +} + +// atLeastOne clamps the result between 1 and 15. +func atLeastOne(v float32) float32 { + if v < 1 { + return 1 + } + if v > 15 { + return 15 + } + return v +} + +func histogram(b []byte, h []uint16) { + if true && len(b) >= 8<<10 { + // Split for bigger inputs + histogramSplit(b, h) + } else { + h = h[:256] + for _, t := range b { + h[t]++ + } + } +} + +func histogramSplit(b []byte, h []uint16) { + // Tested, and slightly faster than 2-way. + // Writing to separate arrays and combining is also slightly slower. + h = h[:256] + for len(b)&3 != 0 { + h[b[0]]++ + b = b[1:] + } + n := len(b) / 4 + x, y, z, w := b[:n], b[n:], b[n+n:], b[n+n+n:] + y, z, w = y[:len(x)], z[:len(x)], w[:len(x)] + for i, t := range x { + v0 := &h[t] + v1 := &h[y[i]] + v3 := &h[w[i]] + v2 := &h[z[i]] + *v0++ + *v1++ + *v2++ + *v3++ + } +} diff --git a/internal/compress/flate/huffman_sortByFreq.go b/internal/compress/flate/huffman_sortByFreq.go new file mode 100644 index 00000000..6c05ba8c --- /dev/null +++ b/internal/compress/flate/huffman_sortByFreq.go @@ -0,0 +1,159 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package flate + +// Sort sorts data. +// It makes one call to data.Len to determine n, and O(n*log(n)) calls to +// data.Less and data.Swap. The sort is not guaranteed to be stable. +func sortByFreq(data []literalNode) { + n := len(data) + quickSortByFreq(data, 0, n, maxDepth(n)) +} + +func quickSortByFreq(data []literalNode, a, b, maxDepth int) { + for b-a > 12 { // Use ShellSort for slices <= 12 elements + if maxDepth == 0 { + heapSort(data, a, b) + return + } + maxDepth-- + mlo, mhi := doPivotByFreq(data, a, b) + // Avoiding recursion on the larger subproblem guarantees + // a stack depth of at most lg(b-a). + if mlo-a < b-mhi { + quickSortByFreq(data, a, mlo, maxDepth) + a = mhi // i.e., quickSortByFreq(data, mhi, b) + } else { + quickSortByFreq(data, mhi, b, maxDepth) + b = mlo // i.e., quickSortByFreq(data, a, mlo) + } + } + if b-a > 1 { + // Do ShellSort pass with gap 6 + // It could be written in this simplified form cause b-a <= 12 + for i := a + 6; i < b; i++ { + if data[i].freq == data[i-6].freq && data[i].literal < data[i-6].literal || data[i].freq < data[i-6].freq { + data[i], data[i-6] = data[i-6], data[i] + } + } + insertionSortByFreq(data, a, b) + } +} + +func doPivotByFreq(data []literalNode, lo, hi int) (midlo, midhi int) { + m := int(uint(lo+hi) >> 1) // Written like this to avoid integer overflow. + if hi-lo > 40 { + // Tukey's ``Ninther,'' median of three medians of three. + s := (hi - lo) / 8 + medianOfThreeSortByFreq(data, lo, lo+s, lo+2*s) + medianOfThreeSortByFreq(data, m, m-s, m+s) + medianOfThreeSortByFreq(data, hi-1, hi-1-s, hi-1-2*s) + } + medianOfThreeSortByFreq(data, lo, m, hi-1) + + // Invariants are: + // data[lo] = pivot (set up by ChoosePivot) + // data[lo < i < a] < pivot + // data[a <= i < b] <= pivot + // data[b <= i < c] unexamined + // data[c <= i < hi-1] > pivot + // data[hi-1] >= pivot + pivot := lo + a, c := lo+1, hi-1 + + for ; a < c && (data[a].freq == data[pivot].freq && data[a].literal < data[pivot].literal || data[a].freq < data[pivot].freq); a++ { + } + b := a + for { + for ; b < c && (data[pivot].freq == data[b].freq && data[pivot].literal > data[b].literal || data[pivot].freq > data[b].freq); b++ { // data[b] <= pivot + } + for ; b < c && (data[pivot].freq == data[c-1].freq && data[pivot].literal < data[c-1].literal || data[pivot].freq < data[c-1].freq); c-- { // data[c-1] > pivot + } + if b >= c { + break + } + // data[b] > pivot; data[c-1] <= pivot + data[b], data[c-1] = data[c-1], data[b] + b++ + c-- + } + // If hi-c<3 then there are duplicates (by property of median of nine). + // Let's be a bit more conservative, and set border to 5. + protect := hi-c < 5 + if !protect && hi-c < (hi-lo)/4 { + // Lets test some points for equality to pivot + dups := 0 + if data[pivot].freq == data[hi-1].freq && data[pivot].literal > data[hi-1].literal || data[pivot].freq > data[hi-1].freq { // data[hi-1] = pivot + data[c], data[hi-1] = data[hi-1], data[c] + c++ + dups++ + } + if data[b-1].freq == data[pivot].freq && data[b-1].literal > data[pivot].literal || data[b-1].freq > data[pivot].freq { // data[b-1] = pivot + b-- + dups++ + } + // m-lo = (hi-lo)/2 > 6 + // b-lo > (hi-lo)*3/4-1 > 8 + // ==> m < b ==> data[m] <= pivot + if data[m].freq == data[pivot].freq && data[m].literal > data[pivot].literal || data[m].freq > data[pivot].freq { // data[m] = pivot + data[m], data[b-1] = data[b-1], data[m] + b-- + dups++ + } + // if at least 2 points are equal to pivot, assume skewed distribution + protect = dups > 1 + } + if protect { + // Protect against a lot of duplicates + // Add invariant: + // data[a <= i < b] unexamined + // data[b <= i < c] = pivot + for { + for ; a < b && (data[b-1].freq == data[pivot].freq && data[b-1].literal > data[pivot].literal || data[b-1].freq > data[pivot].freq); b-- { // data[b] == pivot + } + for ; a < b && (data[a].freq == data[pivot].freq && data[a].literal < data[pivot].literal || data[a].freq < data[pivot].freq); a++ { // data[a] < pivot + } + if a >= b { + break + } + // data[a] == pivot; data[b-1] < pivot + data[a], data[b-1] = data[b-1], data[a] + a++ + b-- + } + } + // Swap pivot into middle + data[pivot], data[b-1] = data[b-1], data[pivot] + return b - 1, c +} + +// Insertion sort +func insertionSortByFreq(data []literalNode, a, b int) { + for i := a + 1; i < b; i++ { + for j := i; j > a && (data[j].freq == data[j-1].freq && data[j].literal < data[j-1].literal || data[j].freq < data[j-1].freq); j-- { + data[j], data[j-1] = data[j-1], data[j] + } + } +} + +// quickSortByFreq, loosely following Bentley and McIlroy, +// ``Engineering a Sort Function,'' SP&E November 1993. + +// medianOfThreeSortByFreq moves the median of the three values data[m0], data[m1], data[m2] into data[m1]. +func medianOfThreeSortByFreq(data []literalNode, m1, m0, m2 int) { + // sort 3 elements + if data[m1].freq == data[m0].freq && data[m1].literal < data[m0].literal || data[m1].freq < data[m0].freq { + data[m1], data[m0] = data[m0], data[m1] + } + // data[m0] <= data[m1] + if data[m2].freq == data[m1].freq && data[m2].literal < data[m1].literal || data[m2].freq < data[m1].freq { + data[m2], data[m1] = data[m1], data[m2] + // data[m0] <= data[m2] && data[m1] < data[m2] + if data[m1].freq == data[m0].freq && data[m1].literal < data[m0].literal || data[m1].freq < data[m0].freq { + data[m1], data[m0] = data[m0], data[m1] + } + } + // now data[m0] <= data[m1] <= data[m2] +} diff --git a/internal/compress/flate/huffman_sortByLiteral.go b/internal/compress/flate/huffman_sortByLiteral.go new file mode 100644 index 00000000..93f1aea1 --- /dev/null +++ b/internal/compress/flate/huffman_sortByLiteral.go @@ -0,0 +1,201 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package flate + +// Sort sorts data. +// It makes one call to data.Len to determine n, and O(n*log(n)) calls to +// data.Less and data.Swap. The sort is not guaranteed to be stable. +func sortByLiteral(data []literalNode) { + n := len(data) + quickSort(data, 0, n, maxDepth(n)) +} + +func quickSort(data []literalNode, a, b, maxDepth int) { + for b-a > 12 { // Use ShellSort for slices <= 12 elements + if maxDepth == 0 { + heapSort(data, a, b) + return + } + maxDepth-- + mlo, mhi := doPivot(data, a, b) + // Avoiding recursion on the larger subproblem guarantees + // a stack depth of at most lg(b-a). + if mlo-a < b-mhi { + quickSort(data, a, mlo, maxDepth) + a = mhi // i.e., quickSort(data, mhi, b) + } else { + quickSort(data, mhi, b, maxDepth) + b = mlo // i.e., quickSort(data, a, mlo) + } + } + if b-a > 1 { + // Do ShellSort pass with gap 6 + // It could be written in this simplified form cause b-a <= 12 + for i := a + 6; i < b; i++ { + if data[i].literal < data[i-6].literal { + data[i], data[i-6] = data[i-6], data[i] + } + } + insertionSort(data, a, b) + } +} +func heapSort(data []literalNode, a, b int) { + first := a + lo := 0 + hi := b - a + + // Build heap with greatest element at top. + for i := (hi - 1) / 2; i >= 0; i-- { + siftDown(data, i, hi, first) + } + + // Pop elements, largest first, into end of data. + for i := hi - 1; i >= 0; i-- { + data[first], data[first+i] = data[first+i], data[first] + siftDown(data, lo, i, first) + } +} + +// siftDown implements the heap property on data[lo, hi). +// first is an offset into the array where the root of the heap lies. +func siftDown(data []literalNode, lo, hi, first int) { + root := lo + for { + child := 2*root + 1 + if child >= hi { + break + } + if child+1 < hi && data[first+child].literal < data[first+child+1].literal { + child++ + } + if data[first+root].literal > data[first+child].literal { + return + } + data[first+root], data[first+child] = data[first+child], data[first+root] + root = child + } +} +func doPivot(data []literalNode, lo, hi int) (midlo, midhi int) { + m := int(uint(lo+hi) >> 1) // Written like this to avoid integer overflow. + if hi-lo > 40 { + // Tukey's ``Ninther,'' median of three medians of three. + s := (hi - lo) / 8 + medianOfThree(data, lo, lo+s, lo+2*s) + medianOfThree(data, m, m-s, m+s) + medianOfThree(data, hi-1, hi-1-s, hi-1-2*s) + } + medianOfThree(data, lo, m, hi-1) + + // Invariants are: + // data[lo] = pivot (set up by ChoosePivot) + // data[lo < i < a] < pivot + // data[a <= i < b] <= pivot + // data[b <= i < c] unexamined + // data[c <= i < hi-1] > pivot + // data[hi-1] >= pivot + pivot := lo + a, c := lo+1, hi-1 + + for ; a < c && data[a].literal < data[pivot].literal; a++ { + } + b := a + for { + for ; b < c && data[pivot].literal > data[b].literal; b++ { // data[b] <= pivot + } + for ; b < c && data[pivot].literal < data[c-1].literal; c-- { // data[c-1] > pivot + } + if b >= c { + break + } + // data[b] > pivot; data[c-1] <= pivot + data[b], data[c-1] = data[c-1], data[b] + b++ + c-- + } + // If hi-c<3 then there are duplicates (by property of median of nine). + // Let's be a bit more conservative, and set border to 5. + protect := hi-c < 5 + if !protect && hi-c < (hi-lo)/4 { + // Lets test some points for equality to pivot + dups := 0 + if data[pivot].literal > data[hi-1].literal { // data[hi-1] = pivot + data[c], data[hi-1] = data[hi-1], data[c] + c++ + dups++ + } + if data[b-1].literal > data[pivot].literal { // data[b-1] = pivot + b-- + dups++ + } + // m-lo = (hi-lo)/2 > 6 + // b-lo > (hi-lo)*3/4-1 > 8 + // ==> m < b ==> data[m] <= pivot + if data[m].literal > data[pivot].literal { // data[m] = pivot + data[m], data[b-1] = data[b-1], data[m] + b-- + dups++ + } + // if at least 2 points are equal to pivot, assume skewed distribution + protect = dups > 1 + } + if protect { + // Protect against a lot of duplicates + // Add invariant: + // data[a <= i < b] unexamined + // data[b <= i < c] = pivot + for { + for ; a < b && data[b-1].literal > data[pivot].literal; b-- { // data[b] == pivot + } + for ; a < b && data[a].literal < data[pivot].literal; a++ { // data[a] < pivot + } + if a >= b { + break + } + // data[a] == pivot; data[b-1] < pivot + data[a], data[b-1] = data[b-1], data[a] + a++ + b-- + } + } + // Swap pivot into middle + data[pivot], data[b-1] = data[b-1], data[pivot] + return b - 1, c +} + +// Insertion sort +func insertionSort(data []literalNode, a, b int) { + for i := a + 1; i < b; i++ { + for j := i; j > a && data[j].literal < data[j-1].literal; j-- { + data[j], data[j-1] = data[j-1], data[j] + } + } +} + +// maxDepth returns a threshold at which quicksort should switch +// to heapsort. It returns 2*ceil(lg(n+1)). +func maxDepth(n int) int { + var depth int + for i := n; i > 0; i >>= 1 { + depth++ + } + return depth * 2 +} + +// medianOfThree moves the median of the three values data[m0], data[m1], data[m2] into data[m1]. +func medianOfThree(data []literalNode, m1, m0, m2 int) { + // sort 3 elements + if data[m1].literal < data[m0].literal { + data[m1], data[m0] = data[m0], data[m1] + } + // data[m0] <= data[m1] + if data[m2].literal < data[m1].literal { + data[m2], data[m1] = data[m1], data[m2] + // data[m0] <= data[m2] && data[m1] < data[m2] + if data[m1].literal < data[m0].literal { + data[m1], data[m0] = data[m0], data[m1] + } + } + // now data[m0] <= data[m1] <= data[m2] +} diff --git a/internal/compress/flate/inflate.go b/internal/compress/flate/inflate.go new file mode 100644 index 00000000..6e90126d --- /dev/null +++ b/internal/compress/flate/inflate.go @@ -0,0 +1,865 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package flate implements the DEFLATE compressed data format, described in +// RFC 1951. The gzip and zlib packages implement access to DEFLATE-based file +// formats. +package flate + +import ( + "bufio" + "compress/flate" + "fmt" + "io" + "math/bits" + "sync" +) + +const ( + maxCodeLen = 16 // max length of Huffman code + maxCodeLenMask = 15 // mask for max length of Huffman code + // The next three numbers come from the RFC section 3.2.7, with the + // additional proviso in section 3.2.5 which implies that distance codes + // 30 and 31 should never occur in compressed data. + maxNumLit = 286 + maxNumDist = 30 + numCodes = 19 // number of codes in Huffman meta-code + + debugDecode = false +) + +// Value of length - 3 and extra bits. +type lengthExtra struct { + length, extra uint8 +} + +var decCodeToLen = [32]lengthExtra{{length: 0x0, extra: 0x0}, {length: 0x1, extra: 0x0}, {length: 0x2, extra: 0x0}, {length: 0x3, extra: 0x0}, {length: 0x4, extra: 0x0}, {length: 0x5, extra: 0x0}, {length: 0x6, extra: 0x0}, {length: 0x7, extra: 0x0}, {length: 0x8, extra: 0x1}, {length: 0xa, extra: 0x1}, {length: 0xc, extra: 0x1}, {length: 0xe, extra: 0x1}, {length: 0x10, extra: 0x2}, {length: 0x14, extra: 0x2}, {length: 0x18, extra: 0x2}, {length: 0x1c, extra: 0x2}, {length: 0x20, extra: 0x3}, {length: 0x28, extra: 0x3}, {length: 0x30, extra: 0x3}, {length: 0x38, extra: 0x3}, {length: 0x40, extra: 0x4}, {length: 0x50, extra: 0x4}, {length: 0x60, extra: 0x4}, {length: 0x70, extra: 0x4}, {length: 0x80, extra: 0x5}, {length: 0xa0, extra: 0x5}, {length: 0xc0, extra: 0x5}, {length: 0xe0, extra: 0x5}, {length: 0xff, extra: 0x0}, {length: 0x0, extra: 0x0}, {length: 0x0, extra: 0x0}, {length: 0x0, extra: 0x0}} + +var bitMask32 = [32]uint32{ + 0, 1, 3, 7, 0xF, 0x1F, 0x3F, 0x7F, 0xFF, + 0x1FF, 0x3FF, 0x7FF, 0xFFF, 0x1FFF, 0x3FFF, 0x7FFF, 0xFFFF, + 0x1ffff, 0x3ffff, 0x7FFFF, 0xfFFFF, 0x1fFFFF, 0x3fFFFF, 0x7fFFFF, 0xffFFFF, + 0x1ffFFFF, 0x3ffFFFF, 0x7ffFFFF, 0xfffFFFF, 0x1fffFFFF, 0x3fffFFFF, 0x7fffFFFF, +} // up to 32 bits + +// Initialize the fixedHuffmanDecoder only once upon first use. +var fixedOnce sync.Once +var fixedHuffmanDecoder huffmanDecoder + +// A CorruptInputError reports the presence of corrupt input at a given offset. +type CorruptInputError = flate.CorruptInputError + +// An InternalError reports an error in the flate code itself. +type InternalError string + +func (e InternalError) Error() string { return "flate: internal error: " + string(e) } + +// A ReadError reports an error encountered while reading input. +// +// Deprecated: No longer returned. +type ReadError = flate.ReadError + +// A WriteError reports an error encountered while writing output. +// +// Deprecated: No longer returned. +type WriteError = flate.WriteError + +// Resetter resets a ReadCloser returned by NewReader or NewReaderDict to +// to switch to a new underlying Reader. This permits reusing a ReadCloser +// instead of allocating a new one. +type Resetter interface { + // Reset discards any buffered data and resets the Resetter as if it was + // newly initialized with the given reader. + Reset(r io.Reader, dict []byte) error +} + +// The data structure for decoding Huffman tables is based on that of +// zlib. There is a lookup table of a fixed bit width (huffmanChunkBits), +// For codes smaller than the table width, there are multiple entries +// (each combination of trailing bits has the same value). For codes +// larger than the table width, the table contains a link to an overflow +// table. The width of each entry in the link table is the maximum code +// size minus the chunk width. +// +// Note that you can do a lookup in the table even without all bits +// filled. Since the extra bits are zero, and the DEFLATE Huffman codes +// have the property that shorter codes come before longer ones, the +// bit length estimate in the result is a lower bound on the actual +// number of bits. +// +// See the following: +// http://www.gzip.org/algorithm.txt + +// chunk & 15 is number of bits +// chunk >> 4 is value, including table link + +const ( + huffmanChunkBits = 9 + huffmanNumChunks = 1 << huffmanChunkBits + huffmanCountMask = 15 + huffmanValueShift = 4 +) + +type huffmanDecoder struct { + maxRead int // the maximum number of bits we can read and not overread + chunks *[huffmanNumChunks]uint16 // chunks as described above + links [][]uint16 // overflow links + linkMask uint32 // mask the width of the link table +} + +// Initialize Huffman decoding tables from array of code lengths. +// Following this function, h is guaranteed to be initialized into a complete +// tree (i.e., neither over-subscribed nor under-subscribed). The exception is a +// degenerate case where the tree has only a single symbol with length 1. Empty +// trees are permitted. +func (h *huffmanDecoder) init(lengths []int) bool { + // Sanity enables additional runtime tests during Huffman + // table construction. It's intended to be used during + // development to supplement the currently ad-hoc unit tests. + const sanity = false + + if h.chunks == nil { + h.chunks = new([huffmanNumChunks]uint16) + } + + if h.maxRead != 0 { + *h = huffmanDecoder{chunks: h.chunks, links: h.links} + } + + // Count number of codes of each length, + // compute maxRead and max length. + var count [maxCodeLen]int + var min, max int + for _, n := range lengths { + if n == 0 { + continue + } + if min == 0 || n < min { + min = n + } + if n > max { + max = n + } + count[n&maxCodeLenMask]++ + } + + // Empty tree. The decompressor.huffSym function will fail later if the tree + // is used. Technically, an empty tree is only valid for the HDIST tree and + // not the HCLEN and HLIT tree. However, a stream with an empty HCLEN tree + // is guaranteed to fail since it will attempt to use the tree to decode the + // codes for the HLIT and HDIST trees. Similarly, an empty HLIT tree is + // guaranteed to fail later since the compressed data section must be + // composed of at least one symbol (the end-of-block marker). + if max == 0 { + return true + } + + code := 0 + var nextcode [maxCodeLen]int + for i := min; i <= max; i++ { + code <<= 1 + nextcode[i&maxCodeLenMask] = code + code += count[i&maxCodeLenMask] + } + + // Check that the coding is complete (i.e., that we've + // assigned all 2-to-the-max possible bit sequences). + // Exception: To be compatible with zlib, we also need to + // accept degenerate single-code codings. See also + // TestDegenerateHuffmanCoding. + if code != 1< huffmanChunkBits { + numLinks := 1 << (uint(max) - huffmanChunkBits) + h.linkMask = uint32(numLinks - 1) + + // create link tables + link := nextcode[huffmanChunkBits+1] >> 1 + if cap(h.links) < huffmanNumChunks-link { + h.links = make([][]uint16, huffmanNumChunks-link) + } else { + h.links = h.links[:huffmanNumChunks-link] + } + for j := uint(link); j < huffmanNumChunks; j++ { + reverse := int(bits.Reverse16(uint16(j))) + reverse >>= uint(16 - huffmanChunkBits) + off := j - uint(link) + if sanity && h.chunks[reverse] != 0 { + panic("impossible: overwriting existing chunk") + } + h.chunks[reverse] = uint16(off<>= uint(16 - n) + if n <= huffmanChunkBits { + for off := reverse; off < len(h.chunks); off += 1 << uint(n) { + // We should never need to overwrite + // an existing chunk. Also, 0 is + // never a valid chunk, because the + // lower 4 "count" bits should be + // between 1 and 15. + if sanity && h.chunks[off] != 0 { + panic("impossible: overwriting existing chunk") + } + h.chunks[off] = chunk + } + } else { + j := reverse & (huffmanNumChunks - 1) + if sanity && h.chunks[j]&huffmanCountMask != huffmanChunkBits+1 { + // Longer codes should have been + // associated with a link table above. + panic("impossible: not an indirect chunk") + } + value := h.chunks[j] >> huffmanValueShift + linktab := h.links[value] + reverse >>= huffmanChunkBits + for off := reverse; off < len(linktab); off += 1 << uint(n-huffmanChunkBits) { + if sanity && linktab[off] != 0 { + panic("impossible: overwriting existing chunk") + } + linktab[off] = chunk + } + } + } + + if sanity { + // Above we've sanity checked that we never overwrote + // an existing entry. Here we additionally check that + // we filled the tables completely. + for i, chunk := range h.chunks { + if chunk == 0 { + // As an exception, in the degenerate + // single-code case, we allow odd + // chunks to be missing. + if code == 1 && i%2 == 1 { + continue + } + panic("impossible: missing chunk") + } + } + for _, linktab := range h.links { + for _, chunk := range linktab { + if chunk == 0 { + panic("impossible: missing chunk") + } + } + } + } + + return true +} + +// Reader is the actual read interface needed by NewReader. +// If the passed in io.Reader does not also have ReadByte, +// the NewReader will introduce its own buffering. +type Reader interface { + io.Reader + io.ByteReader +} + +type step uint8 + +const ( + copyData step = iota + 1 + nextBlock + huffmanBytesBuffer + huffmanBytesReader + huffmanBufioReader + huffmanStringsReader + huffmanGenericReader +) + +// flushMode tells decompressor when to return data +type flushMode uint8 + +const ( + syncFlush flushMode = iota // return data after sync flush block + partialFlush // return data after each block +) + +// Decompress state. +type decompressor struct { + // Input source. + r Reader + roffset int64 + + // Huffman decoders for literal/length, distance. + h1, h2 huffmanDecoder + + // Length arrays used to define Huffman codes. + bits *[maxNumLit + maxNumDist]int + codebits *[numCodes]int + + // Output history, buffer. + dict dictDecoder + + // Next step in the decompression, + // and decompression state. + step step + stepState int + err error + toRead []byte + hl, hd *huffmanDecoder + copyLen int + copyDist int + + // Temporary buffer (avoids repeated allocation). + buf [4]byte + + // Input bits, in top of b. + b uint32 + + nb uint + final bool + + flushMode flushMode +} + +func (f *decompressor) nextBlock() { + for f.nb < 1+2 { + if f.err = f.moreBits(); f.err != nil { + return + } + } + f.final = f.b&1 == 1 + f.b >>= 1 + typ := f.b & 3 + f.b >>= 2 + f.nb -= 1 + 2 + switch typ { + case 0: + f.dataBlock() + if debugDecode { + fmt.Println("stored block") + } + case 1: + // compressed, fixed Huffman tables + f.hl = &fixedHuffmanDecoder + f.hd = nil + f.huffmanBlockDecoder() + if debugDecode { + fmt.Println("predefinied huffman block") + } + case 2: + // compressed, dynamic Huffman tables + if f.err = f.readHuffman(); f.err != nil { + break + } + f.hl = &f.h1 + f.hd = &f.h2 + f.huffmanBlockDecoder() + if debugDecode { + fmt.Println("dynamic huffman block") + } + default: + // 3 is reserved. + if debugDecode { + fmt.Println("reserved data block encountered") + } + f.err = CorruptInputError(f.roffset) + } +} + +func (f *decompressor) Read(b []byte) (int, error) { + for { + if len(f.toRead) > 0 { + n := copy(b, f.toRead) + f.toRead = f.toRead[n:] + if len(f.toRead) == 0 { + return n, f.err + } + return n, nil + } + if f.err != nil { + return 0, f.err + } + + f.doStep() + + if f.err != nil && len(f.toRead) == 0 { + f.toRead = f.dict.readFlush() // Flush what's left in case of error + } + } +} + +// WriteTo implements the io.WriteTo interface for io.Copy and friends. +func (f *decompressor) WriteTo(w io.Writer) (int64, error) { + total := int64(0) + flushed := false + for { + if len(f.toRead) > 0 { + n, err := w.Write(f.toRead) + total += int64(n) + if err != nil { + f.err = err + return total, err + } + if n != len(f.toRead) { + return total, io.ErrShortWrite + } + f.toRead = f.toRead[:0] + } + if f.err != nil && flushed { + if f.err == io.EOF { + return total, nil + } + return total, f.err + } + if f.err == nil { + f.doStep() + } + if len(f.toRead) == 0 && f.err != nil && !flushed { + f.toRead = f.dict.readFlush() // Flush what's left in case of error + flushed = true + } + } +} + +func (f *decompressor) Close() error { + if f.err == io.EOF { + return nil + } + return f.err +} + +// RFC 1951 section 3.2.7. +// Compression with dynamic Huffman codes + +var codeOrder = [...]int{16, 17, 18, 0, 8, 7, 9, 6, 10, 5, 11, 4, 12, 3, 13, 2, 14, 1, 15} + +func (f *decompressor) readHuffman() error { + // HLIT[5], HDIST[5], HCLEN[4]. + for f.nb < 5+5+4 { + if err := f.moreBits(); err != nil { + return err + } + } + nlit := int(f.b&0x1F) + 257 + if nlit > maxNumLit { + if debugDecode { + fmt.Println("nlit > maxNumLit", nlit) + } + return CorruptInputError(f.roffset) + } + f.b >>= 5 + ndist := int(f.b&0x1F) + 1 + if ndist > maxNumDist { + if debugDecode { + fmt.Println("ndist > maxNumDist", ndist) + } + return CorruptInputError(f.roffset) + } + f.b >>= 5 + nclen := int(f.b&0xF) + 4 + // numCodes is 19, so nclen is always valid. + f.b >>= 4 + f.nb -= 5 + 5 + 4 + + // (HCLEN+4)*3 bits: code lengths in the magic codeOrder order. + for i := range nclen { + for f.nb < 3 { + if err := f.moreBits(); err != nil { + return err + } + } + f.codebits[codeOrder[i]] = int(f.b & 0x7) + f.b >>= 3 + f.nb -= 3 + } + for i := nclen; i < len(codeOrder); i++ { + f.codebits[codeOrder[i]] = 0 + } + if !f.h1.init(f.codebits[0:]) { + if debugDecode { + fmt.Println("init codebits failed") + } + return CorruptInputError(f.roffset) + } + + // HLIT + 257 code lengths, HDIST + 1 code lengths, + // using the code length Huffman code. + for i, n := 0, nlit+ndist; i < n; { + x, err := f.huffSym(&f.h1) + if err != nil { + return err + } + if x < 16 { + // Actual length. + f.bits[i] = x + i++ + continue + } + // Repeat previous length or zero. + var rep int + var nb uint + var b int + switch x { + default: + return InternalError("unexpected length code") + case 16: + rep = 3 + nb = 2 + if i == 0 { + if debugDecode { + fmt.Println("i==0") + } + return CorruptInputError(f.roffset) + } + b = f.bits[i-1] + case 17: + rep = 3 + nb = 3 + b = 0 + case 18: + rep = 11 + nb = 7 + b = 0 + } + for f.nb < nb { + if err := f.moreBits(); err != nil { + if debugDecode { + fmt.Println("morebits:", err) + } + return err + } + } + rep += int(f.b & uint32(1<<(nb®SizeMaskUint32)-1)) + f.b >>= nb & regSizeMaskUint32 + f.nb -= nb + if i+rep > n { + if debugDecode { + fmt.Println("i+rep > n", i, rep, n) + } + return CorruptInputError(f.roffset) + } + for j := 0; j < rep; j++ { + f.bits[i] = b + i++ + } + } + + if !f.h1.init(f.bits[0:nlit]) || !f.h2.init(f.bits[nlit:nlit+ndist]) { + if debugDecode { + fmt.Println("init2 failed") + } + return CorruptInputError(f.roffset) + } + + // As an optimization, we can initialize the maxRead bits to read at a time + // for the HLIT tree to the length of the EOB marker since we know that + // every block must terminate with one. This preserves the property that + // we never read any extra bytes after the end of the DEFLATE stream. + if f.h1.maxRead < f.bits[endBlockMarker] { + f.h1.maxRead = f.bits[endBlockMarker] + } + if !f.final { + // If not the final block, the smallest block possible is + // a predefined table, BTYPE=01, with a single EOB marker. + // This will take up 3 + 7 bits. + f.h1.maxRead += 10 + } + + return nil +} + +// Copy a single uncompressed data block from input to output. +func (f *decompressor) dataBlock() { + // Uncompressed. + // Discard current half-byte. + left := (f.nb) & 7 + f.nb -= left + f.b >>= left + + offBytes := f.nb >> 3 + // Unfilled values will be overwritten. + f.buf[0] = uint8(f.b) + f.buf[1] = uint8(f.b >> 8) + f.buf[2] = uint8(f.b >> 16) + f.buf[3] = uint8(f.b >> 24) + + f.roffset += int64(offBytes) + f.nb, f.b = 0, 0 + + // Length then ones-complement of length. + nr, err := io.ReadFull(f.r, f.buf[offBytes:4]) + f.roffset += int64(nr) + if err != nil { + f.err = noEOF(err) + return + } + n := uint16(f.buf[0]) | uint16(f.buf[1])<<8 + nn := uint16(f.buf[2]) | uint16(f.buf[3])<<8 + if nn != ^n { + if debugDecode { + ncomp := ^n + fmt.Println("uint16(nn) != uint16(^n)", nn, ncomp) + } + f.err = CorruptInputError(f.roffset) + return + } + + if n == 0 { + if f.flushMode == syncFlush { + f.toRead = f.dict.readFlush() + } + + f.finishBlock() + return + } + + f.copyLen = int(n) + f.copyData() +} + +// copyData copies f.copyLen bytes from the underlying reader into f.hist. +// It pauses for reads when f.hist is full. +func (f *decompressor) copyData() { + buf := f.dict.writeSlice() + if len(buf) > f.copyLen { + buf = buf[:f.copyLen] + } + + cnt, err := io.ReadFull(f.r, buf) + f.roffset += int64(cnt) + f.copyLen -= cnt + f.dict.writeMark(cnt) + if err != nil { + f.err = noEOF(err) + return + } + + if f.dict.availWrite() == 0 || f.copyLen > 0 { + f.toRead = f.dict.readFlush() + f.step = copyData + return + } + f.finishBlock() +} + +func (f *decompressor) finishBlock() { + if f.final { + if f.dict.availRead() > 0 { + f.toRead = f.dict.readFlush() + } + + f.err = io.EOF + } else if f.flushMode == partialFlush && f.dict.availRead() > 0 { + f.toRead = f.dict.readFlush() + } + + f.step = nextBlock +} + +func (f *decompressor) doStep() { + switch f.step { + case copyData: + f.copyData() + case nextBlock: + f.nextBlock() + case huffmanBytesBuffer: + f.huffmanBytesBuffer() + case huffmanBytesReader: + f.huffmanBytesReader() + case huffmanBufioReader: + f.huffmanBufioReader() + case huffmanStringsReader: + f.huffmanStringsReader() + case huffmanGenericReader: + f.huffmanGenericReader() + default: + panic("BUG: unexpected step state") + } +} + +// noEOF returns err, unless err == io.EOF, in which case it returns io.ErrUnexpectedEOF. +func noEOF(e error) error { + if e == io.EOF { + return io.ErrUnexpectedEOF + } + return e +} + +func (f *decompressor) moreBits() error { + c, err := f.r.ReadByte() + if err != nil { + return noEOF(err) + } + f.roffset++ + f.b |= uint32(c) << (f.nb & regSizeMaskUint32) + f.nb += 8 + return nil +} + +// Read the next Huffman-encoded symbol from f according to h. +func (f *decompressor) huffSym(h *huffmanDecoder) (int, error) { + // Since a huffmanDecoder can be empty or be composed of a degenerate tree + // with single element, huffSym must error on these two edge cases. In both + // cases, the chunks slice will be 0 for the invalid sequence, leading it + // satisfy the n == 0 check below. + n := uint(h.maxRead) + // Optimization. Compiler isn't smart enough to keep f.b,f.nb in registers, + // but is smart enough to keep local variables in registers, so use nb and b, + // inline call to moreBits and reassign b,nb back to f on return. + nb, b := f.nb, f.b + for { + for nb < n { + c, err := f.r.ReadByte() + if err != nil { + f.b = b + f.nb = nb + return 0, noEOF(err) + } + f.roffset++ + b |= uint32(c) << (nb & regSizeMaskUint32) + nb += 8 + } + chunk := h.chunks[b&(huffmanNumChunks-1)] + n = uint(chunk & huffmanCountMask) + if n > huffmanChunkBits { + chunk = h.links[chunk>>huffmanValueShift][(b>>huffmanChunkBits)&h.linkMask] + n = uint(chunk & huffmanCountMask) + } + if n <= nb { + if n == 0 { + f.b = b + f.nb = nb + if debugDecode { + fmt.Println("huffsym: n==0") + } + f.err = CorruptInputError(f.roffset) + return 0, f.err + } + f.b = b >> (n & regSizeMaskUint32) + f.nb = nb - n + return int(chunk >> huffmanValueShift), nil + } + } +} + +func makeReader(r io.Reader) Reader { + if rr, ok := r.(Reader); ok { + return rr + } + return bufio.NewReader(r) +} + +func fixedHuffmanDecoderInit() { + fixedOnce.Do(func() { + // These come from the RFC section 3.2.6. + var bits [288]int + for i := range 144 { + bits[i] = 8 + } + for i := 144; i < 256; i++ { + bits[i] = 9 + } + for i := 256; i < 280; i++ { + bits[i] = 7 + } + for i := 280; i < 288; i++ { + bits[i] = 8 + } + fixedHuffmanDecoder.init(bits[:]) + }) +} + +func (f *decompressor) Reset(r io.Reader, dict []byte) error { + *f = decompressor{ + r: makeReader(r), + bits: f.bits, + codebits: f.codebits, + h1: f.h1, + h2: f.h2, + dict: f.dict, + step: nextBlock, + } + f.dict.init(maxMatchOffset, dict) + return nil +} + +type ReaderOpt func(*decompressor) + +// WithPartialBlock tells decompressor to return after each block, +// so it can read data written with partial flush +func WithPartialBlock() ReaderOpt { + return func(f *decompressor) { + f.flushMode = partialFlush + } +} + +// WithDict initializes the reader with a preset dictionary +func WithDict(dict []byte) ReaderOpt { + return func(f *decompressor) { + f.dict.init(maxMatchOffset, dict) + } +} + +// NewReaderOpts returns new reader with provided options +func NewReaderOpts(r io.Reader, opts ...ReaderOpt) io.ReadCloser { + fixedHuffmanDecoderInit() + + var f decompressor + f.r = makeReader(r) + f.bits = new([maxNumLit + maxNumDist]int) + f.codebits = new([numCodes]int) + f.step = nextBlock + f.dict.init(maxMatchOffset, nil) + + for _, opt := range opts { + opt(&f) + } + + return &f +} + +// NewReader returns a new ReadCloser that can be used +// to read the uncompressed version of r. +// If r does not also implement io.ByteReader, +// the decompressor may read more data than necessary from r. +// It is the caller's responsibility to call Close on the ReadCloser +// when finished reading. +// +// The ReadCloser returned by NewReader also implements Resetter. +func NewReader(r io.Reader) io.ReadCloser { + return NewReaderOpts(r) +} + +// NewReaderDict is like NewReader but initializes the reader +// with a preset dictionary. The returned Reader behaves as if +// the uncompressed data stream started with the given dictionary, +// which has already been read. NewReaderDict is typically used +// to read data compressed by NewWriterDict. +// +// The ReadCloser returned by NewReader also implements Resetter. +func NewReaderDict(r io.Reader, dict []byte) io.ReadCloser { + return NewReaderOpts(r, WithDict(dict)) +} diff --git a/internal/compress/flate/inflate_gen.go b/internal/compress/flate/inflate_gen.go new file mode 100644 index 00000000..2b2f993f --- /dev/null +++ b/internal/compress/flate/inflate_gen.go @@ -0,0 +1,1283 @@ +// Code generated by go generate gen_inflate.go. DO NOT EDIT. + +package flate + +import ( + "bufio" + "bytes" + "fmt" + "math/bits" + "strings" +) + +// Decode a single Huffman block from f. +// hl and hd are the Huffman states for the lit/length values +// and the distance values, respectively. If hd == nil, using the +// fixed distance encoding associated with fixed Huffman blocks. +func (f *decompressor) huffmanBytesBuffer() { + const ( + stateInit = iota // Zero value must be stateInit + stateDict + ) + fr := f.r.(*bytes.Buffer) + + // Optimization. Compiler isn't smart enough to keep f.b,f.nb in registers, + // but is smart enough to keep local variables in registers, so use nb and b, + // inline call to moreBits and reassign b,nb back to f on return. + fnb, fb, dict := f.nb, f.b, &f.dict + + switch f.stepState { + case stateInit: + goto readLiteral + case stateDict: + goto copyHistory + } + +readLiteral: + // Read literal and/or (length, distance) according to RFC section 3.2.3. + { + var v int + { + // Inlined v, err := f.huffSym(f.hl) + // Since a huffmanDecoder can be empty or be composed of a degenerate tree + // with single element, huffSym must error on these two edge cases. In both + // cases, the chunks slice will be 0 for the invalid sequence, leading it + // satisfy the n == 0 check below. + n := uint(f.hl.maxRead) + for { + for fnb < n { + c, err := fr.ReadByte() + if err != nil { + f.b, f.nb = fb, fnb + f.err = noEOF(err) + return + } + f.roffset++ + fb |= uint32(c) << (fnb & regSizeMaskUint32) + fnb += 8 + } + chunk := f.hl.chunks[fb&(huffmanNumChunks-1)] + n = uint(chunk & huffmanCountMask) + if n > huffmanChunkBits { + chunk = f.hl.links[chunk>>huffmanValueShift][(fb>>huffmanChunkBits)&f.hl.linkMask] + n = uint(chunk & huffmanCountMask) + } + if n <= fnb { + if n == 0 { + f.b, f.nb = fb, fnb + if debugDecode { + fmt.Println("huffsym: n==0") + } + f.err = CorruptInputError(f.roffset) + return + } + fb = fb >> (n & regSizeMaskUint32) + fnb = fnb - n + v = int(chunk >> huffmanValueShift) + break + } + } + } + + var length int + switch { + case v < 256: + dict.writeByte(byte(v)) + if dict.availWrite() == 0 { + f.toRead = dict.readFlush() + f.step = huffmanBytesBuffer + f.stepState = stateInit + f.b, f.nb = fb, fnb + return + } + goto readLiteral + case v == 256: + f.b, f.nb = fb, fnb + f.finishBlock() + return + // otherwise, reference to older data + case v < 265: + length = v - (257 - 3) + case v < maxNumLit: + val := decCodeToLen[(v - 257)] + length = int(val.length) + 3 + n := uint(val.extra) + for fnb < n { + c, err := fr.ReadByte() + if err != nil { + f.b, f.nb = fb, fnb + if debugDecode { + fmt.Println("morebits n>0:", err) + } + f.err = err + return + } + f.roffset++ + fb |= uint32(c) << (fnb & regSizeMaskUint32) + fnb += 8 + } + length += int(fb & bitMask32[n]) + fb >>= n & regSizeMaskUint32 + fnb -= n + default: + if debugDecode { + fmt.Println(v, ">= maxNumLit") + } + f.err = CorruptInputError(f.roffset) + f.b, f.nb = fb, fnb + return + } + + var dist uint32 + if f.hd == nil { + for fnb < 5 { + c, err := fr.ReadByte() + if err != nil { + f.b, f.nb = fb, fnb + if debugDecode { + fmt.Println("morebits f.nb<5:", err) + } + f.err = err + return + } + f.roffset++ + fb |= uint32(c) << (fnb & regSizeMaskUint32) + fnb += 8 + } + dist = uint32(bits.Reverse8(uint8(fb & 0x1F << 3))) + fb >>= 5 + fnb -= 5 + } else { + // Since a huffmanDecoder can be empty or be composed of a degenerate tree + // with single element, huffSym must error on these two edge cases. In both + // cases, the chunks slice will be 0 for the invalid sequence, leading it + // satisfy the n == 0 check below. + n := uint(f.hd.maxRead) + // Optimization. Compiler isn't smart enough to keep f.b,f.nb in registers, + // but is smart enough to keep local variables in registers, so use nb and b, + // inline call to moreBits and reassign b,nb back to f on return. + for { + for fnb < n { + c, err := fr.ReadByte() + if err != nil { + f.b, f.nb = fb, fnb + f.err = noEOF(err) + return + } + f.roffset++ + fb |= uint32(c) << (fnb & regSizeMaskUint32) + fnb += 8 + } + chunk := f.hd.chunks[fb&(huffmanNumChunks-1)] + n = uint(chunk & huffmanCountMask) + if n > huffmanChunkBits { + chunk = f.hd.links[chunk>>huffmanValueShift][(fb>>huffmanChunkBits)&f.hd.linkMask] + n = uint(chunk & huffmanCountMask) + } + if n <= fnb { + if n == 0 { + f.b, f.nb = fb, fnb + if debugDecode { + fmt.Println("huffsym: n==0") + } + f.err = CorruptInputError(f.roffset) + return + } + fb = fb >> (n & regSizeMaskUint32) + fnb = fnb - n + dist = uint32(chunk >> huffmanValueShift) + break + } + } + } + + switch { + case dist < 4: + dist++ + case dist < maxNumDist: + nb := uint(dist-2) >> 1 + // have 1 bit in bottom of dist, need nb more. + extra := (dist & 1) << (nb & regSizeMaskUint32) + for fnb < nb { + c, err := fr.ReadByte() + if err != nil { + f.b, f.nb = fb, fnb + if debugDecode { + fmt.Println("morebits f.nb>= nb & regSizeMaskUint32 + fnb -= nb + dist = 1<<((nb+1)®SizeMaskUint32) + 1 + extra + // slower: dist = bitMask32[nb+1] + 2 + extra + default: + f.b, f.nb = fb, fnb + if debugDecode { + fmt.Println("dist too big:", dist, maxNumDist) + } + f.err = CorruptInputError(f.roffset) + return + } + + // No check on length; encoding can be prescient. + if dist > uint32(dict.histSize()) { + f.b, f.nb = fb, fnb + if debugDecode { + fmt.Println("dist > dict.histSize():", dist, dict.histSize()) + } + f.err = CorruptInputError(f.roffset) + return + } + + f.copyLen, f.copyDist = length, int(dist) + goto copyHistory + } + +copyHistory: + // Perform a backwards copy according to RFC section 3.2.3. + { + cnt := dict.tryWriteCopy(f.copyDist, f.copyLen) + if cnt == 0 { + cnt = dict.writeCopy(f.copyDist, f.copyLen) + } + f.copyLen -= cnt + + if dict.availWrite() == 0 || f.copyLen > 0 { + f.toRead = dict.readFlush() + f.step = huffmanBytesBuffer // We need to continue this work + f.stepState = stateDict + f.b, f.nb = fb, fnb + return + } + goto readLiteral + } + // Not reached +} + +// Decode a single Huffman block from f. +// hl and hd are the Huffman states for the lit/length values +// and the distance values, respectively. If hd == nil, using the +// fixed distance encoding associated with fixed Huffman blocks. +func (f *decompressor) huffmanBytesReader() { + const ( + stateInit = iota // Zero value must be stateInit + stateDict + ) + fr := f.r.(*bytes.Reader) + + // Optimization. Compiler isn't smart enough to keep f.b,f.nb in registers, + // but is smart enough to keep local variables in registers, so use nb and b, + // inline call to moreBits and reassign b,nb back to f on return. + fnb, fb, dict := f.nb, f.b, &f.dict + + switch f.stepState { + case stateInit: + goto readLiteral + case stateDict: + goto copyHistory + } + +readLiteral: + // Read literal and/or (length, distance) according to RFC section 3.2.3. + { + var v int + { + // Inlined v, err := f.huffSym(f.hl) + // Since a huffmanDecoder can be empty or be composed of a degenerate tree + // with single element, huffSym must error on these two edge cases. In both + // cases, the chunks slice will be 0 for the invalid sequence, leading it + // satisfy the n == 0 check below. + n := uint(f.hl.maxRead) + for { + for fnb < n { + c, err := fr.ReadByte() + if err != nil { + f.b, f.nb = fb, fnb + f.err = noEOF(err) + return + } + f.roffset++ + fb |= uint32(c) << (fnb & regSizeMaskUint32) + fnb += 8 + } + chunk := f.hl.chunks[fb&(huffmanNumChunks-1)] + n = uint(chunk & huffmanCountMask) + if n > huffmanChunkBits { + chunk = f.hl.links[chunk>>huffmanValueShift][(fb>>huffmanChunkBits)&f.hl.linkMask] + n = uint(chunk & huffmanCountMask) + } + if n <= fnb { + if n == 0 { + f.b, f.nb = fb, fnb + if debugDecode { + fmt.Println("huffsym: n==0") + } + f.err = CorruptInputError(f.roffset) + return + } + fb = fb >> (n & regSizeMaskUint32) + fnb = fnb - n + v = int(chunk >> huffmanValueShift) + break + } + } + } + + var length int + switch { + case v < 256: + dict.writeByte(byte(v)) + if dict.availWrite() == 0 { + f.toRead = dict.readFlush() + f.step = huffmanBytesReader + f.stepState = stateInit + f.b, f.nb = fb, fnb + return + } + goto readLiteral + case v == 256: + f.b, f.nb = fb, fnb + f.finishBlock() + return + // otherwise, reference to older data + case v < 265: + length = v - (257 - 3) + case v < maxNumLit: + val := decCodeToLen[(v - 257)] + length = int(val.length) + 3 + n := uint(val.extra) + for fnb < n { + c, err := fr.ReadByte() + if err != nil { + f.b, f.nb = fb, fnb + if debugDecode { + fmt.Println("morebits n>0:", err) + } + f.err = err + return + } + f.roffset++ + fb |= uint32(c) << (fnb & regSizeMaskUint32) + fnb += 8 + } + length += int(fb & bitMask32[n]) + fb >>= n & regSizeMaskUint32 + fnb -= n + default: + if debugDecode { + fmt.Println(v, ">= maxNumLit") + } + f.err = CorruptInputError(f.roffset) + f.b, f.nb = fb, fnb + return + } + + var dist uint32 + if f.hd == nil { + for fnb < 5 { + c, err := fr.ReadByte() + if err != nil { + f.b, f.nb = fb, fnb + if debugDecode { + fmt.Println("morebits f.nb<5:", err) + } + f.err = err + return + } + f.roffset++ + fb |= uint32(c) << (fnb & regSizeMaskUint32) + fnb += 8 + } + dist = uint32(bits.Reverse8(uint8(fb & 0x1F << 3))) + fb >>= 5 + fnb -= 5 + } else { + // Since a huffmanDecoder can be empty or be composed of a degenerate tree + // with single element, huffSym must error on these two edge cases. In both + // cases, the chunks slice will be 0 for the invalid sequence, leading it + // satisfy the n == 0 check below. + n := uint(f.hd.maxRead) + // Optimization. Compiler isn't smart enough to keep f.b,f.nb in registers, + // but is smart enough to keep local variables in registers, so use nb and b, + // inline call to moreBits and reassign b,nb back to f on return. + for { + for fnb < n { + c, err := fr.ReadByte() + if err != nil { + f.b, f.nb = fb, fnb + f.err = noEOF(err) + return + } + f.roffset++ + fb |= uint32(c) << (fnb & regSizeMaskUint32) + fnb += 8 + } + chunk := f.hd.chunks[fb&(huffmanNumChunks-1)] + n = uint(chunk & huffmanCountMask) + if n > huffmanChunkBits { + chunk = f.hd.links[chunk>>huffmanValueShift][(fb>>huffmanChunkBits)&f.hd.linkMask] + n = uint(chunk & huffmanCountMask) + } + if n <= fnb { + if n == 0 { + f.b, f.nb = fb, fnb + if debugDecode { + fmt.Println("huffsym: n==0") + } + f.err = CorruptInputError(f.roffset) + return + } + fb = fb >> (n & regSizeMaskUint32) + fnb = fnb - n + dist = uint32(chunk >> huffmanValueShift) + break + } + } + } + + switch { + case dist < 4: + dist++ + case dist < maxNumDist: + nb := uint(dist-2) >> 1 + // have 1 bit in bottom of dist, need nb more. + extra := (dist & 1) << (nb & regSizeMaskUint32) + for fnb < nb { + c, err := fr.ReadByte() + if err != nil { + f.b, f.nb = fb, fnb + if debugDecode { + fmt.Println("morebits f.nb>= nb & regSizeMaskUint32 + fnb -= nb + dist = 1<<((nb+1)®SizeMaskUint32) + 1 + extra + // slower: dist = bitMask32[nb+1] + 2 + extra + default: + f.b, f.nb = fb, fnb + if debugDecode { + fmt.Println("dist too big:", dist, maxNumDist) + } + f.err = CorruptInputError(f.roffset) + return + } + + // No check on length; encoding can be prescient. + if dist > uint32(dict.histSize()) { + f.b, f.nb = fb, fnb + if debugDecode { + fmt.Println("dist > dict.histSize():", dist, dict.histSize()) + } + f.err = CorruptInputError(f.roffset) + return + } + + f.copyLen, f.copyDist = length, int(dist) + goto copyHistory + } + +copyHistory: + // Perform a backwards copy according to RFC section 3.2.3. + { + cnt := dict.tryWriteCopy(f.copyDist, f.copyLen) + if cnt == 0 { + cnt = dict.writeCopy(f.copyDist, f.copyLen) + } + f.copyLen -= cnt + + if dict.availWrite() == 0 || f.copyLen > 0 { + f.toRead = dict.readFlush() + f.step = huffmanBytesReader // We need to continue this work + f.stepState = stateDict + f.b, f.nb = fb, fnb + return + } + goto readLiteral + } + // Not reached +} + +// Decode a single Huffman block from f. +// hl and hd are the Huffman states for the lit/length values +// and the distance values, respectively. If hd == nil, using the +// fixed distance encoding associated with fixed Huffman blocks. +func (f *decompressor) huffmanBufioReader() { + const ( + stateInit = iota // Zero value must be stateInit + stateDict + ) + fr := f.r.(*bufio.Reader) + + // Optimization. Compiler isn't smart enough to keep f.b,f.nb in registers, + // but is smart enough to keep local variables in registers, so use nb and b, + // inline call to moreBits and reassign b,nb back to f on return. + fnb, fb, dict := f.nb, f.b, &f.dict + + switch f.stepState { + case stateInit: + goto readLiteral + case stateDict: + goto copyHistory + } + +readLiteral: + // Read literal and/or (length, distance) according to RFC section 3.2.3. + { + var v int + { + // Inlined v, err := f.huffSym(f.hl) + // Since a huffmanDecoder can be empty or be composed of a degenerate tree + // with single element, huffSym must error on these two edge cases. In both + // cases, the chunks slice will be 0 for the invalid sequence, leading it + // satisfy the n == 0 check below. + n := uint(f.hl.maxRead) + for { + for fnb < n { + c, err := fr.ReadByte() + if err != nil { + f.b, f.nb = fb, fnb + f.err = noEOF(err) + return + } + f.roffset++ + fb |= uint32(c) << (fnb & regSizeMaskUint32) + fnb += 8 + } + chunk := f.hl.chunks[fb&(huffmanNumChunks-1)] + n = uint(chunk & huffmanCountMask) + if n > huffmanChunkBits { + chunk = f.hl.links[chunk>>huffmanValueShift][(fb>>huffmanChunkBits)&f.hl.linkMask] + n = uint(chunk & huffmanCountMask) + } + if n <= fnb { + if n == 0 { + f.b, f.nb = fb, fnb + if debugDecode { + fmt.Println("huffsym: n==0") + } + f.err = CorruptInputError(f.roffset) + return + } + fb = fb >> (n & regSizeMaskUint32) + fnb = fnb - n + v = int(chunk >> huffmanValueShift) + break + } + } + } + + var length int + switch { + case v < 256: + dict.writeByte(byte(v)) + if dict.availWrite() == 0 { + f.toRead = dict.readFlush() + f.step = huffmanBufioReader + f.stepState = stateInit + f.b, f.nb = fb, fnb + return + } + goto readLiteral + case v == 256: + f.b, f.nb = fb, fnb + f.finishBlock() + return + // otherwise, reference to older data + case v < 265: + length = v - (257 - 3) + case v < maxNumLit: + val := decCodeToLen[(v - 257)] + length = int(val.length) + 3 + n := uint(val.extra) + for fnb < n { + c, err := fr.ReadByte() + if err != nil { + f.b, f.nb = fb, fnb + if debugDecode { + fmt.Println("morebits n>0:", err) + } + f.err = err + return + } + f.roffset++ + fb |= uint32(c) << (fnb & regSizeMaskUint32) + fnb += 8 + } + length += int(fb & bitMask32[n]) + fb >>= n & regSizeMaskUint32 + fnb -= n + default: + if debugDecode { + fmt.Println(v, ">= maxNumLit") + } + f.err = CorruptInputError(f.roffset) + f.b, f.nb = fb, fnb + return + } + + var dist uint32 + if f.hd == nil { + for fnb < 5 { + c, err := fr.ReadByte() + if err != nil { + f.b, f.nb = fb, fnb + if debugDecode { + fmt.Println("morebits f.nb<5:", err) + } + f.err = err + return + } + f.roffset++ + fb |= uint32(c) << (fnb & regSizeMaskUint32) + fnb += 8 + } + dist = uint32(bits.Reverse8(uint8(fb & 0x1F << 3))) + fb >>= 5 + fnb -= 5 + } else { + // Since a huffmanDecoder can be empty or be composed of a degenerate tree + // with single element, huffSym must error on these two edge cases. In both + // cases, the chunks slice will be 0 for the invalid sequence, leading it + // satisfy the n == 0 check below. + n := uint(f.hd.maxRead) + // Optimization. Compiler isn't smart enough to keep f.b,f.nb in registers, + // but is smart enough to keep local variables in registers, so use nb and b, + // inline call to moreBits and reassign b,nb back to f on return. + for { + for fnb < n { + c, err := fr.ReadByte() + if err != nil { + f.b, f.nb = fb, fnb + f.err = noEOF(err) + return + } + f.roffset++ + fb |= uint32(c) << (fnb & regSizeMaskUint32) + fnb += 8 + } + chunk := f.hd.chunks[fb&(huffmanNumChunks-1)] + n = uint(chunk & huffmanCountMask) + if n > huffmanChunkBits { + chunk = f.hd.links[chunk>>huffmanValueShift][(fb>>huffmanChunkBits)&f.hd.linkMask] + n = uint(chunk & huffmanCountMask) + } + if n <= fnb { + if n == 0 { + f.b, f.nb = fb, fnb + if debugDecode { + fmt.Println("huffsym: n==0") + } + f.err = CorruptInputError(f.roffset) + return + } + fb = fb >> (n & regSizeMaskUint32) + fnb = fnb - n + dist = uint32(chunk >> huffmanValueShift) + break + } + } + } + + switch { + case dist < 4: + dist++ + case dist < maxNumDist: + nb := uint(dist-2) >> 1 + // have 1 bit in bottom of dist, need nb more. + extra := (dist & 1) << (nb & regSizeMaskUint32) + for fnb < nb { + c, err := fr.ReadByte() + if err != nil { + f.b, f.nb = fb, fnb + if debugDecode { + fmt.Println("morebits f.nb>= nb & regSizeMaskUint32 + fnb -= nb + dist = 1<<((nb+1)®SizeMaskUint32) + 1 + extra + // slower: dist = bitMask32[nb+1] + 2 + extra + default: + f.b, f.nb = fb, fnb + if debugDecode { + fmt.Println("dist too big:", dist, maxNumDist) + } + f.err = CorruptInputError(f.roffset) + return + } + + // No check on length; encoding can be prescient. + if dist > uint32(dict.histSize()) { + f.b, f.nb = fb, fnb + if debugDecode { + fmt.Println("dist > dict.histSize():", dist, dict.histSize()) + } + f.err = CorruptInputError(f.roffset) + return + } + + f.copyLen, f.copyDist = length, int(dist) + goto copyHistory + } + +copyHistory: + // Perform a backwards copy according to RFC section 3.2.3. + { + cnt := dict.tryWriteCopy(f.copyDist, f.copyLen) + if cnt == 0 { + cnt = dict.writeCopy(f.copyDist, f.copyLen) + } + f.copyLen -= cnt + + if dict.availWrite() == 0 || f.copyLen > 0 { + f.toRead = dict.readFlush() + f.step = huffmanBufioReader // We need to continue this work + f.stepState = stateDict + f.b, f.nb = fb, fnb + return + } + goto readLiteral + } + // Not reached +} + +// Decode a single Huffman block from f. +// hl and hd are the Huffman states for the lit/length values +// and the distance values, respectively. If hd == nil, using the +// fixed distance encoding associated with fixed Huffman blocks. +func (f *decompressor) huffmanStringsReader() { + const ( + stateInit = iota // Zero value must be stateInit + stateDict + ) + fr := f.r.(*strings.Reader) + + // Optimization. Compiler isn't smart enough to keep f.b,f.nb in registers, + // but is smart enough to keep local variables in registers, so use nb and b, + // inline call to moreBits and reassign b,nb back to f on return. + fnb, fb, dict := f.nb, f.b, &f.dict + + switch f.stepState { + case stateInit: + goto readLiteral + case stateDict: + goto copyHistory + } + +readLiteral: + // Read literal and/or (length, distance) according to RFC section 3.2.3. + { + var v int + { + // Inlined v, err := f.huffSym(f.hl) + // Since a huffmanDecoder can be empty or be composed of a degenerate tree + // with single element, huffSym must error on these two edge cases. In both + // cases, the chunks slice will be 0 for the invalid sequence, leading it + // satisfy the n == 0 check below. + n := uint(f.hl.maxRead) + for { + for fnb < n { + c, err := fr.ReadByte() + if err != nil { + f.b, f.nb = fb, fnb + f.err = noEOF(err) + return + } + f.roffset++ + fb |= uint32(c) << (fnb & regSizeMaskUint32) + fnb += 8 + } + chunk := f.hl.chunks[fb&(huffmanNumChunks-1)] + n = uint(chunk & huffmanCountMask) + if n > huffmanChunkBits { + chunk = f.hl.links[chunk>>huffmanValueShift][(fb>>huffmanChunkBits)&f.hl.linkMask] + n = uint(chunk & huffmanCountMask) + } + if n <= fnb { + if n == 0 { + f.b, f.nb = fb, fnb + if debugDecode { + fmt.Println("huffsym: n==0") + } + f.err = CorruptInputError(f.roffset) + return + } + fb = fb >> (n & regSizeMaskUint32) + fnb = fnb - n + v = int(chunk >> huffmanValueShift) + break + } + } + } + + var length int + switch { + case v < 256: + dict.writeByte(byte(v)) + if dict.availWrite() == 0 { + f.toRead = dict.readFlush() + f.step = huffmanStringsReader + f.stepState = stateInit + f.b, f.nb = fb, fnb + return + } + goto readLiteral + case v == 256: + f.b, f.nb = fb, fnb + f.finishBlock() + return + // otherwise, reference to older data + case v < 265: + length = v - (257 - 3) + case v < maxNumLit: + val := decCodeToLen[(v - 257)] + length = int(val.length) + 3 + n := uint(val.extra) + for fnb < n { + c, err := fr.ReadByte() + if err != nil { + f.b, f.nb = fb, fnb + if debugDecode { + fmt.Println("morebits n>0:", err) + } + f.err = err + return + } + f.roffset++ + fb |= uint32(c) << (fnb & regSizeMaskUint32) + fnb += 8 + } + length += int(fb & bitMask32[n]) + fb >>= n & regSizeMaskUint32 + fnb -= n + default: + if debugDecode { + fmt.Println(v, ">= maxNumLit") + } + f.err = CorruptInputError(f.roffset) + f.b, f.nb = fb, fnb + return + } + + var dist uint32 + if f.hd == nil { + for fnb < 5 { + c, err := fr.ReadByte() + if err != nil { + f.b, f.nb = fb, fnb + if debugDecode { + fmt.Println("morebits f.nb<5:", err) + } + f.err = err + return + } + f.roffset++ + fb |= uint32(c) << (fnb & regSizeMaskUint32) + fnb += 8 + } + dist = uint32(bits.Reverse8(uint8(fb & 0x1F << 3))) + fb >>= 5 + fnb -= 5 + } else { + // Since a huffmanDecoder can be empty or be composed of a degenerate tree + // with single element, huffSym must error on these two edge cases. In both + // cases, the chunks slice will be 0 for the invalid sequence, leading it + // satisfy the n == 0 check below. + n := uint(f.hd.maxRead) + // Optimization. Compiler isn't smart enough to keep f.b,f.nb in registers, + // but is smart enough to keep local variables in registers, so use nb and b, + // inline call to moreBits and reassign b,nb back to f on return. + for { + for fnb < n { + c, err := fr.ReadByte() + if err != nil { + f.b, f.nb = fb, fnb + f.err = noEOF(err) + return + } + f.roffset++ + fb |= uint32(c) << (fnb & regSizeMaskUint32) + fnb += 8 + } + chunk := f.hd.chunks[fb&(huffmanNumChunks-1)] + n = uint(chunk & huffmanCountMask) + if n > huffmanChunkBits { + chunk = f.hd.links[chunk>>huffmanValueShift][(fb>>huffmanChunkBits)&f.hd.linkMask] + n = uint(chunk & huffmanCountMask) + } + if n <= fnb { + if n == 0 { + f.b, f.nb = fb, fnb + if debugDecode { + fmt.Println("huffsym: n==0") + } + f.err = CorruptInputError(f.roffset) + return + } + fb = fb >> (n & regSizeMaskUint32) + fnb = fnb - n + dist = uint32(chunk >> huffmanValueShift) + break + } + } + } + + switch { + case dist < 4: + dist++ + case dist < maxNumDist: + nb := uint(dist-2) >> 1 + // have 1 bit in bottom of dist, need nb more. + extra := (dist & 1) << (nb & regSizeMaskUint32) + for fnb < nb { + c, err := fr.ReadByte() + if err != nil { + f.b, f.nb = fb, fnb + if debugDecode { + fmt.Println("morebits f.nb>= nb & regSizeMaskUint32 + fnb -= nb + dist = 1<<((nb+1)®SizeMaskUint32) + 1 + extra + // slower: dist = bitMask32[nb+1] + 2 + extra + default: + f.b, f.nb = fb, fnb + if debugDecode { + fmt.Println("dist too big:", dist, maxNumDist) + } + f.err = CorruptInputError(f.roffset) + return + } + + // No check on length; encoding can be prescient. + if dist > uint32(dict.histSize()) { + f.b, f.nb = fb, fnb + if debugDecode { + fmt.Println("dist > dict.histSize():", dist, dict.histSize()) + } + f.err = CorruptInputError(f.roffset) + return + } + + f.copyLen, f.copyDist = length, int(dist) + goto copyHistory + } + +copyHistory: + // Perform a backwards copy according to RFC section 3.2.3. + { + cnt := dict.tryWriteCopy(f.copyDist, f.copyLen) + if cnt == 0 { + cnt = dict.writeCopy(f.copyDist, f.copyLen) + } + f.copyLen -= cnt + + if dict.availWrite() == 0 || f.copyLen > 0 { + f.toRead = dict.readFlush() + f.step = huffmanStringsReader // We need to continue this work + f.stepState = stateDict + f.b, f.nb = fb, fnb + return + } + goto readLiteral + } + // Not reached +} + +// Decode a single Huffman block from f. +// hl and hd are the Huffman states for the lit/length values +// and the distance values, respectively. If hd == nil, using the +// fixed distance encoding associated with fixed Huffman blocks. +func (f *decompressor) huffmanGenericReader() { + const ( + stateInit = iota // Zero value must be stateInit + stateDict + ) + fr := f.r.(Reader) + + // Optimization. Compiler isn't smart enough to keep f.b,f.nb in registers, + // but is smart enough to keep local variables in registers, so use nb and b, + // inline call to moreBits and reassign b,nb back to f on return. + fnb, fb, dict := f.nb, f.b, &f.dict + + switch f.stepState { + case stateInit: + goto readLiteral + case stateDict: + goto copyHistory + } + +readLiteral: + // Read literal and/or (length, distance) according to RFC section 3.2.3. + { + var v int + { + // Inlined v, err := f.huffSym(f.hl) + // Since a huffmanDecoder can be empty or be composed of a degenerate tree + // with single element, huffSym must error on these two edge cases. In both + // cases, the chunks slice will be 0 for the invalid sequence, leading it + // satisfy the n == 0 check below. + n := uint(f.hl.maxRead) + for { + for fnb < n { + c, err := fr.ReadByte() + if err != nil { + f.b, f.nb = fb, fnb + f.err = noEOF(err) + return + } + f.roffset++ + fb |= uint32(c) << (fnb & regSizeMaskUint32) + fnb += 8 + } + chunk := f.hl.chunks[fb&(huffmanNumChunks-1)] + n = uint(chunk & huffmanCountMask) + if n > huffmanChunkBits { + chunk = f.hl.links[chunk>>huffmanValueShift][(fb>>huffmanChunkBits)&f.hl.linkMask] + n = uint(chunk & huffmanCountMask) + } + if n <= fnb { + if n == 0 { + f.b, f.nb = fb, fnb + if debugDecode { + fmt.Println("huffsym: n==0") + } + f.err = CorruptInputError(f.roffset) + return + } + fb = fb >> (n & regSizeMaskUint32) + fnb = fnb - n + v = int(chunk >> huffmanValueShift) + break + } + } + } + + var length int + switch { + case v < 256: + dict.writeByte(byte(v)) + if dict.availWrite() == 0 { + f.toRead = dict.readFlush() + f.step = huffmanGenericReader + f.stepState = stateInit + f.b, f.nb = fb, fnb + return + } + goto readLiteral + case v == 256: + f.b, f.nb = fb, fnb + f.finishBlock() + return + // otherwise, reference to older data + case v < 265: + length = v - (257 - 3) + case v < maxNumLit: + val := decCodeToLen[(v - 257)] + length = int(val.length) + 3 + n := uint(val.extra) + for fnb < n { + c, err := fr.ReadByte() + if err != nil { + f.b, f.nb = fb, fnb + if debugDecode { + fmt.Println("morebits n>0:", err) + } + f.err = err + return + } + f.roffset++ + fb |= uint32(c) << (fnb & regSizeMaskUint32) + fnb += 8 + } + length += int(fb & bitMask32[n]) + fb >>= n & regSizeMaskUint32 + fnb -= n + default: + if debugDecode { + fmt.Println(v, ">= maxNumLit") + } + f.err = CorruptInputError(f.roffset) + f.b, f.nb = fb, fnb + return + } + + var dist uint32 + if f.hd == nil { + for fnb < 5 { + c, err := fr.ReadByte() + if err != nil { + f.b, f.nb = fb, fnb + if debugDecode { + fmt.Println("morebits f.nb<5:", err) + } + f.err = err + return + } + f.roffset++ + fb |= uint32(c) << (fnb & regSizeMaskUint32) + fnb += 8 + } + dist = uint32(bits.Reverse8(uint8(fb & 0x1F << 3))) + fb >>= 5 + fnb -= 5 + } else { + // Since a huffmanDecoder can be empty or be composed of a degenerate tree + // with single element, huffSym must error on these two edge cases. In both + // cases, the chunks slice will be 0 for the invalid sequence, leading it + // satisfy the n == 0 check below. + n := uint(f.hd.maxRead) + // Optimization. Compiler isn't smart enough to keep f.b,f.nb in registers, + // but is smart enough to keep local variables in registers, so use nb and b, + // inline call to moreBits and reassign b,nb back to f on return. + for { + for fnb < n { + c, err := fr.ReadByte() + if err != nil { + f.b, f.nb = fb, fnb + f.err = noEOF(err) + return + } + f.roffset++ + fb |= uint32(c) << (fnb & regSizeMaskUint32) + fnb += 8 + } + chunk := f.hd.chunks[fb&(huffmanNumChunks-1)] + n = uint(chunk & huffmanCountMask) + if n > huffmanChunkBits { + chunk = f.hd.links[chunk>>huffmanValueShift][(fb>>huffmanChunkBits)&f.hd.linkMask] + n = uint(chunk & huffmanCountMask) + } + if n <= fnb { + if n == 0 { + f.b, f.nb = fb, fnb + if debugDecode { + fmt.Println("huffsym: n==0") + } + f.err = CorruptInputError(f.roffset) + return + } + fb = fb >> (n & regSizeMaskUint32) + fnb = fnb - n + dist = uint32(chunk >> huffmanValueShift) + break + } + } + } + + switch { + case dist < 4: + dist++ + case dist < maxNumDist: + nb := uint(dist-2) >> 1 + // have 1 bit in bottom of dist, need nb more. + extra := (dist & 1) << (nb & regSizeMaskUint32) + for fnb < nb { + c, err := fr.ReadByte() + if err != nil { + f.b, f.nb = fb, fnb + if debugDecode { + fmt.Println("morebits f.nb>= nb & regSizeMaskUint32 + fnb -= nb + dist = 1<<((nb+1)®SizeMaskUint32) + 1 + extra + // slower: dist = bitMask32[nb+1] + 2 + extra + default: + f.b, f.nb = fb, fnb + if debugDecode { + fmt.Println("dist too big:", dist, maxNumDist) + } + f.err = CorruptInputError(f.roffset) + return + } + + // No check on length; encoding can be prescient. + if dist > uint32(dict.histSize()) { + f.b, f.nb = fb, fnb + if debugDecode { + fmt.Println("dist > dict.histSize():", dist, dict.histSize()) + } + f.err = CorruptInputError(f.roffset) + return + } + + f.copyLen, f.copyDist = length, int(dist) + goto copyHistory + } + +copyHistory: + // Perform a backwards copy according to RFC section 3.2.3. + { + cnt := dict.tryWriteCopy(f.copyDist, f.copyLen) + if cnt == 0 { + cnt = dict.writeCopy(f.copyDist, f.copyLen) + } + f.copyLen -= cnt + + if dict.availWrite() == 0 || f.copyLen > 0 { + f.toRead = dict.readFlush() + f.step = huffmanGenericReader // We need to continue this work + f.stepState = stateDict + f.b, f.nb = fb, fnb + return + } + goto readLiteral + } + // Not reached +} + +func (f *decompressor) huffmanBlockDecoder() { + switch f.r.(type) { + case *bytes.Buffer: + f.huffmanBytesBuffer() + case *bytes.Reader: + f.huffmanBytesReader() + case *bufio.Reader: + f.huffmanBufioReader() + case *strings.Reader: + f.huffmanStringsReader() + case Reader: + f.huffmanGenericReader() + default: + f.huffmanGenericReader() + } +} diff --git a/internal/compress/flate/inflate_test.go b/internal/compress/flate/inflate_test.go new file mode 100644 index 00000000..d018991c --- /dev/null +++ b/internal/compress/flate/inflate_test.go @@ -0,0 +1,302 @@ +// Copyright 2014 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package flate + +import ( + "bytes" + "crypto/rand" + "io" + "os" + "strconv" + "strings" + "testing" +) + +func TestReset(t *testing.T) { + ss := []string{ + "lorem ipsum izzle fo rizzle", + "the quick brown fox jumped over", + } + + deflated := make([]bytes.Buffer, 2) + for i, s := range ss { + w, _ := NewWriter(&deflated[i], 1) + w.Write([]byte(s)) + w.Close() + } + + inflated := make([]bytes.Buffer, 2) + + f := NewReader(&deflated[0]) + io.Copy(&inflated[0], f) + f.(Resetter).Reset(&deflated[1], nil) + io.Copy(&inflated[1], f) + f.Close() + + for i, s := range ss { + if s != inflated[i].String() { + t.Errorf("inflated[%d]:\ngot %q\nwant %q", i, inflated[i], s) + } + } +} + +func TestReaderTruncated(t *testing.T) { + vectors := []struct{ input, output string }{ + {"\x00", ""}, + {"\x00\f", ""}, + {"\x00\f\x00", ""}, + {"\x00\f\x00\xf3\xff", ""}, + {"\x00\f\x00\xf3\xffhello", "hello"}, + {"\x00\f\x00\xf3\xffhello, world", "hello, world"}, + {"\x02", ""}, + {"\xf2H\xcd", "He"}, + {"\xf2HÍ™0a\u0084\t", "Hel\x90\x90\x90\x90\x90"}, + {"\xf2HÍ™0a\u0084\t\x00", "Hel\x90\x90\x90\x90\x90"}, + } + + for i, v := range vectors { + r := strings.NewReader(v.input) + zr := NewReader(r) + b, err := io.ReadAll(zr) + if err != io.ErrUnexpectedEOF { + t.Errorf("test %d, error mismatch: got %v, want io.ErrUnexpectedEOF", i, err) + } + if string(b) != v.output { + t.Errorf("test %d, output mismatch: got %q, want %q", i, b, v.output) + } + } +} + +func TestResetDict(t *testing.T) { + dict := []byte("the lorem fox") + ss := []string{ + "lorem ipsum izzle fo rizzle", + "the quick brown fox jumped over", + } + + deflated := make([]bytes.Buffer, len(ss)) + for i, s := range ss { + w, _ := NewWriterDict(&deflated[i], DefaultCompression, dict) + w.Write([]byte(s)) + w.Close() + } + + inflated := make([]bytes.Buffer, len(ss)) + + f := NewReader(nil) + for i := range inflated { + f.(Resetter).Reset(&deflated[i], dict) + io.Copy(&inflated[i], f) + } + f.Close() + + for i, s := range ss { + if s != inflated[i].String() { + t.Errorf("inflated[%d]:\ngot %q\nwant %q", i, inflated[i], s) + } + } +} + +// Tests ported from zlib/test/infcover.c +type infTest struct { + hex string + id string + n int +} + +var infTests = []infTest{ + {"0 0 0 0 0", "invalid stored block lengths", 1}, + {"3 0", "fixed", 0}, + {"6", "invalid block type", 1}, + {"1 1 0 fe ff 0", "stored", 0}, + {"fc 0 0", "too many length or distance symbols", 1}, + {"4 0 fe ff", "invalid code lengths set", 1}, + {"4 0 24 49 0", "invalid bit length repeat", 1}, + {"4 0 24 e9 ff ff", "invalid bit length repeat", 1}, + {"4 0 24 e9 ff 6d", "invalid code -- missing end-of-block", 1}, + {"4 80 49 92 24 49 92 24 71 ff ff 93 11 0", "invalid literal/lengths set", 1}, + {"4 80 49 92 24 49 92 24 f b4 ff ff c3 84", "invalid distances set", 1}, + {"4 c0 81 8 0 0 0 0 20 7f eb b 0 0", "invalid literal/length code", 1}, + {"2 7e ff ff", "invalid distance code", 1}, + {"c c0 81 0 0 0 0 0 90 ff 6b 4 0", "invalid distance too far back", 1}, + + // also trailer mismatch just in inflate() + {"1f 8b 8 0 0 0 0 0 0 0 3 0 0 0 0 1", "incorrect data check", -1}, + {"1f 8b 8 0 0 0 0 0 0 0 3 0 0 0 0 0 0 0 0 1", "incorrect length check", -1}, + {"5 c0 21 d 0 0 0 80 b0 fe 6d 2f 91 6c", "pull 17", 0}, + {"5 e0 81 91 24 cb b2 2c 49 e2 f 2e 8b 9a 47 56 9f fb fe ec d2 ff 1f", "long code", 0}, + {"ed c0 1 1 0 0 0 40 20 ff 57 1b 42 2c 4f", "length extra", 0}, + {"ed cf c1 b1 2c 47 10 c4 30 fa 6f 35 1d 1 82 59 3d fb be 2e 2a fc f c", "long distance and extra", 0}, + {"ed c0 81 0 0 0 0 80 a0 fd a9 17 a9 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 6", "window end", 0}, +} + +func TestInflate(t *testing.T) { + for _, test := range infTests { + hex := strings.Split(test.hex, " ") + data := make([]byte, len(hex)) + for i, h := range hex { + b, _ := strconv.ParseInt(h, 16, 32) + data[i] = byte(b) + } + buf := bytes.NewReader(data) + r := NewReader(buf) + + _, err := io.Copy(io.Discard, r) + if (test.n == 0 && err == nil) || (test.n != 0 && err != nil) { + t.Logf("%q: OK:", test.id) + t.Logf(" - got %v", err) + continue + } + + if test.n == 0 && err != nil { + t.Errorf("%q: Expected no error, but got %v", test.id, err) + continue + } + + if test.n != 0 && err == nil { + t.Errorf("%q:Expected an error, but got none", test.id) + continue + } + t.Fatal(test.n, err) + } + + for _, test := range infOutTests { + hex := strings.Split(test.hex, " ") + data := make([]byte, len(hex)) + for i, h := range hex { + b, _ := strconv.ParseInt(h, 16, 32) + data[i] = byte(b) + } + buf := bytes.NewReader(data) + r := NewReader(buf) + + _, err := io.Copy(io.Discard, r) + if test.err == (err != nil) { + t.Logf("%q: OK:", test.id) + t.Logf(" - got %v", err) + continue + } + + if test.err == false && err != nil { + t.Errorf("%q: Expected no error, but got %v", test.id, err) + continue + } + + if test.err && err == nil { + t.Errorf("%q: Expected an error, but got none", test.id) + continue + } + t.Fatal(test.err, err) + } + +} + +// Tests ported from zlib/test/infcover.c +// Since zlib inflate is push (writer) instead of pull (reader) +// some of the window size tests have been removed, since they +// are irrelevant. +type infOutTest struct { + hex string + id string + step int + win int + length int + err bool +} + +var infOutTests = []infOutTest{ + {"2 8 20 80 0 3 0", "inflate_fast TYPE return", 0, -15, 258, false}, + {"63 18 5 40 c 0", "window wrap", 3, -8, 300, false}, + {"e5 e0 81 ad 6d cb b2 2c c9 01 1e 59 63 ae 7d ee fb 4d fd b5 35 41 68 ff 7f 0f 0 0 0", "fast length extra bits", 0, -8, 258, true}, + {"25 fd 81 b5 6d 59 b6 6a 49 ea af 35 6 34 eb 8c b9 f6 b9 1e ef 67 49 50 fe ff ff 3f 0 0", "fast distance extra bits", 0, -8, 258, true}, + {"3 7e 0 0 0 0 0", "fast invalid distance code", 0, -8, 258, true}, + {"1b 7 0 0 0 0 0", "fast invalid literal/length code", 0, -8, 258, true}, + {"d c7 1 ae eb 38 c 4 41 a0 87 72 de df fb 1f b8 36 b1 38 5d ff ff 0", "fast 2nd level codes and too far back", 0, -8, 258, true}, + {"63 18 5 8c 10 8 0 0 0 0", "very common case", 0, -8, 259, false}, + {"63 60 60 18 c9 0 8 18 18 18 26 c0 28 0 29 0 0 0", "contiguous and wrap around window", 6, -8, 259, false}, + {"63 0 3 0 0 0 0 0", "copy direct from output", 0, -8, 259, false}, + {"1f 8b 0 0", "bad gzip method", 0, 31, 0, true}, + {"1f 8b 8 80", "bad gzip flags", 0, 31, 0, true}, + {"77 85", "bad zlib method", 0, 15, 0, true}, + {"78 9c", "bad zlib window size", 0, 8, 0, true}, + {"1f 8b 8 1e 0 0 0 0 0 0 1 0 0 0 0 0 0", "bad header crc", 0, 47, 1, true}, + {"1f 8b 8 2 0 0 0 0 0 0 1d 26 3 0 0 0 0 0 0 0 0 0", "check gzip length", 0, 47, 0, true}, + {"78 90", "bad zlib header check", 0, 47, 0, true}, + {"8 b8 0 0 0 1", "need dictionary", 0, 8, 0, true}, + {"63 18 68 30 d0 0 0", "force split window update", 4, -8, 259, false}, + {"3 0", "use fixed blocks", 0, -15, 1, false}, + {"", "bad window size", 0, 1, 0, true}, +} + +func TestWriteTo(t *testing.T) { + input := make([]byte, 100000) + n, err := rand.Read(input) + if err != nil { + t.Fatal(err) + } + if n != len(input) { + t.Fatal("did not fill buffer") + } + compressed := &bytes.Buffer{} + w, err := NewWriter(compressed, -2) + if err != nil { + t.Fatal(err) + } + n, err = w.Write(input) + if err != nil { + t.Fatal(err) + } + if n != len(input) { + t.Fatal("did not fill buffer") + } + w.Close() + buf := compressed.Bytes() + + dec := NewReader(bytes.NewBuffer(buf)) + // ReadAll does not use WriteTo, but we wrap it in a NopCloser to be sure. + readall, err := io.ReadAll(io.NopCloser(dec)) + if err != nil { + t.Fatal(err) + } + if len(readall) != len(input) { + t.Fatal("did not decompress everything") + } + + dec = NewReader(bytes.NewBuffer(buf)) + wtbuf := &bytes.Buffer{} + written, err := dec.(io.WriterTo).WriteTo(wtbuf) + if err != nil { + t.Fatal(err) + } + if written != int64(len(input)) { + t.Error("Returned length did not match, expected", len(input), "got", written) + } + if wtbuf.Len() != len(input) { + t.Error("Actual Length did not match, expected", len(input), "got", wtbuf.Len()) + } + if !bytes.Equal(wtbuf.Bytes(), input) { + t.Fatal("output did not match input") + } +} + +func TestReaderPartialBlock(t *testing.T) { + data, err := os.ReadFile("testdata/partial-block") + if err != nil { + t.Error(err) + } + + r := NewReaderOpts(bytes.NewReader(data), WithPartialBlock()) + rb := make([]byte, 32) + n, err := r.Read(rb) + if err != nil { + t.Fatalf("Read: %v", err) + } + + expected := "hello, world" + actual := string(rb[:n]) + if expected != actual { + t.Fatalf("expected: %v, got: %v", expected, actual) + } +} diff --git a/internal/compress/flate/level1.go b/internal/compress/flate/level1.go new file mode 100644 index 00000000..41c312e8 --- /dev/null +++ b/internal/compress/flate/level1.go @@ -0,0 +1,215 @@ +package flate + +import ( + "fmt" + + "codeberg.org/lindenii/furgit/internal/compress/internal/le" +) + +// fastGen maintains the table for matches, +// and the previous byte block for level 2. +// This is the generic implementation. +type fastEncL1 struct { + fastGen + table [tableSize]tableEntry +} + +// EncodeL1 uses a similar algorithm to level 1 +func (e *fastEncL1) Encode(dst *tokens, src []byte) { + const ( + inputMargin = 12 - 1 + minNonLiteralBlockSize = 1 + 1 + inputMargin + hashBytes = 5 + ) + if debugDeflate && e.cur < 0 { + panic(fmt.Sprint("e.cur < 0: ", e.cur)) + } + + // Protect against e.cur wraparound. + for e.cur >= bufferReset { + if len(e.hist) == 0 { + for i := range e.table[:] { + e.table[i] = tableEntry{} + } + e.cur = maxMatchOffset + break + } + // Shift down everything in the table that isn't already too far away. + minOff := e.cur + int32(len(e.hist)) - maxMatchOffset + for i := range e.table[:] { + v := e.table[i].offset + if v <= minOff { + v = 0 + } else { + v = v - e.cur + maxMatchOffset + } + e.table[i].offset = v + } + e.cur = maxMatchOffset + } + + s := e.addBlock(src) + + // This check isn't in the Snappy implementation, but there, the caller + // instead of the callee handles this case. + if len(src) < minNonLiteralBlockSize { + // We do not fill the token table. + // This will be picked up by caller. + dst.n = uint16(len(src)) + return + } + + // Override src + src = e.hist + nextEmit := s + + // sLimit is when to stop looking for offset/length copies. The inputMargin + // lets us use a fast path for emitLiteral in the main loop, while we are + // looking for copies. + sLimit := int32(len(src) - inputMargin) + + // nextEmit is where in src the next emitLiteral should start from. + cv := load6432(src, s) + + for { + const skipLog = 5 + const doEvery = 2 + + nextS := s + var candidate tableEntry + var t int32 + for { + nextHash := hashLen(cv, tableBits, hashBytes) + candidate = e.table[nextHash] + nextS = s + doEvery + (s-nextEmit)>>skipLog + if nextS > sLimit { + goto emitRemainder + } + + now := load6432(src, nextS) + e.table[nextHash] = tableEntry{offset: s + e.cur} + nextHash = hashLen(now, tableBits, hashBytes) + t = candidate.offset - e.cur + if s-t < maxMatchOffset && uint32(cv) == load3232(src, t) { + e.table[nextHash] = tableEntry{offset: nextS + e.cur} + break + } + + // Do one right away... + cv = now + s = nextS + nextS++ + candidate = e.table[nextHash] + now >>= 8 + e.table[nextHash] = tableEntry{offset: s + e.cur} + + t = candidate.offset - e.cur + if s-t < maxMatchOffset && uint32(cv) == load3232(src, t) { + e.table[nextHash] = tableEntry{offset: nextS + e.cur} + break + } + cv = now + s = nextS + } + + // A 4-byte match has been found. We'll later see if more than 4 bytes + // match. But, prior to the match, src[nextEmit:s] are unmatched. Emit + // them as literal bytes. + for { + // Invariant: we have a 4-byte match at s, and no need to emit any + // literal bytes prior to s. + + // Extend the 4-byte match as long as possible. + l := e.matchlenLong(int(s+4), int(t+4), src) + 4 + + // Extend backwards + for t > 0 && s > nextEmit && le.Load8(src, t-1) == le.Load8(src, s-1) { + s-- + t-- + l++ + } + if nextEmit < s { + if false { + emitLiteral(dst, src[nextEmit:s]) + } else { + for _, v := range src[nextEmit:s] { + dst.tokens[dst.n] = token(v) + dst.litHist[v]++ + dst.n++ + } + } + } + + // Save the match found + if false { + dst.AddMatchLong(l, uint32(s-t-baseMatchOffset)) + } else { + // Inlined... + xoffset := uint32(s - t - baseMatchOffset) + xlength := l + oc := offsetCode(xoffset) + xoffset |= oc << 16 + for xlength > 0 { + xl := xlength + if xl > 258 { + if xl > 258+baseMatchLength { + xl = 258 + } else { + xl = 258 - baseMatchLength + } + } + xlength -= xl + xl -= baseMatchLength + dst.extraHist[lengthCodes1[uint8(xl)]]++ + dst.offHist[oc]++ + dst.tokens[dst.n] = token(matchType | uint32(xl)<= s { + s = nextS + 1 + } + if s >= sLimit { + // Index first pair after match end. + if int(s+l+8) < len(src) { + cv := load6432(src, s) + e.table[hashLen(cv, tableBits, hashBytes)] = tableEntry{offset: s + e.cur} + } + goto emitRemainder + } + + // We could immediately start working at s now, but to improve + // compression we first update the hash table at s-2 and at s. If + // another emitCopy is not our next move, also calculate nextHash + // at s+1. At least on GOARCH=amd64, these three hash calculations + // are faster as one load64 call (with some shifts) instead of + // three load32 calls. + x := load6432(src, s-2) + o := e.cur + s - 2 + prevHash := hashLen(x, tableBits, hashBytes) + e.table[prevHash] = tableEntry{offset: o} + x >>= 16 + currHash := hashLen(x, tableBits, hashBytes) + candidate = e.table[currHash] + e.table[currHash] = tableEntry{offset: o + 2} + + t = candidate.offset - e.cur + if s-t > maxMatchOffset || uint32(x) != load3232(src, t) { + cv = x >> 8 + s++ + break + } + } + } + +emitRemainder: + if int(nextEmit) < len(src) { + // If nothing was added, don't encode literals. + if dst.n == 0 { + return + } + emitLiteral(dst, src[nextEmit:]) + } +} diff --git a/internal/compress/flate/level2.go b/internal/compress/flate/level2.go new file mode 100644 index 00000000..c8d047f2 --- /dev/null +++ b/internal/compress/flate/level2.go @@ -0,0 +1,214 @@ +package flate + +import "fmt" + +// fastGen maintains the table for matches, +// and the previous byte block for level 2. +// This is the generic implementation. +type fastEncL2 struct { + fastGen + table [bTableSize]tableEntry +} + +// EncodeL2 uses a similar algorithm to level 1, but is capable +// of matching across blocks giving better compression at a small slowdown. +func (e *fastEncL2) Encode(dst *tokens, src []byte) { + const ( + inputMargin = 12 - 1 + minNonLiteralBlockSize = 1 + 1 + inputMargin + hashBytes = 5 + ) + + if debugDeflate && e.cur < 0 { + panic(fmt.Sprint("e.cur < 0: ", e.cur)) + } + + // Protect against e.cur wraparound. + for e.cur >= bufferReset { + if len(e.hist) == 0 { + for i := range e.table[:] { + e.table[i] = tableEntry{} + } + e.cur = maxMatchOffset + break + } + // Shift down everything in the table that isn't already too far away. + minOff := e.cur + int32(len(e.hist)) - maxMatchOffset + for i := range e.table[:] { + v := e.table[i].offset + if v <= minOff { + v = 0 + } else { + v = v - e.cur + maxMatchOffset + } + e.table[i].offset = v + } + e.cur = maxMatchOffset + } + + s := e.addBlock(src) + + // This check isn't in the Snappy implementation, but there, the caller + // instead of the callee handles this case. + if len(src) < minNonLiteralBlockSize { + // We do not fill the token table. + // This will be picked up by caller. + dst.n = uint16(len(src)) + return + } + + // Override src + src = e.hist + nextEmit := s + + // sLimit is when to stop looking for offset/length copies. The inputMargin + // lets us use a fast path for emitLiteral in the main loop, while we are + // looking for copies. + sLimit := int32(len(src) - inputMargin) + + // nextEmit is where in src the next emitLiteral should start from. + cv := load6432(src, s) + for { + // When should we start skipping if we haven't found matches in a long while. + const skipLog = 5 + const doEvery = 2 + + nextS := s + var candidate tableEntry + for { + nextHash := hashLen(cv, bTableBits, hashBytes) + s = nextS + nextS = s + doEvery + (s-nextEmit)>>skipLog + if nextS > sLimit { + goto emitRemainder + } + candidate = e.table[nextHash] + now := load6432(src, nextS) + e.table[nextHash] = tableEntry{offset: s + e.cur} + nextHash = hashLen(now, bTableBits, hashBytes) + + offset := s - (candidate.offset - e.cur) + if offset < maxMatchOffset && uint32(cv) == load3232(src, candidate.offset-e.cur) { + e.table[nextHash] = tableEntry{offset: nextS + e.cur} + break + } + + // Do one right away... + cv = now + s = nextS + nextS++ + candidate = e.table[nextHash] + now >>= 8 + e.table[nextHash] = tableEntry{offset: s + e.cur} + + offset = s - (candidate.offset - e.cur) + if offset < maxMatchOffset && uint32(cv) == load3232(src, candidate.offset-e.cur) { + break + } + cv = now + } + + // A 4-byte match has been found. We'll later see if more than 4 bytes + // match. But, prior to the match, src[nextEmit:s] are unmatched. Emit + // them as literal bytes. + + // Call emitCopy, and then see if another emitCopy could be our next + // move. Repeat until we find no match for the input immediately after + // what was consumed by the last emitCopy call. + // + // If we exit this loop normally then we need to call emitLiteral next, + // though we don't yet know how big the literal will be. We handle that + // by proceeding to the next iteration of the main loop. We also can + // exit this loop via goto if we get close to exhausting the input. + for { + // Invariant: we have a 4-byte match at s, and no need to emit any + // literal bytes prior to s. + + // Extend the 4-byte match as long as possible. + t := candidate.offset - e.cur + l := e.matchlenLong(int(s+4), int(t+4), src) + 4 + + // Extend backwards + for t > 0 && s > nextEmit && src[t-1] == src[s-1] { + s-- + t-- + l++ + } + if nextEmit < s { + if false { + emitLiteral(dst, src[nextEmit:s]) + } else { + for _, v := range src[nextEmit:s] { + dst.tokens[dst.n] = token(v) + dst.litHist[v]++ + dst.n++ + } + } + } + + dst.AddMatchLong(l, uint32(s-t-baseMatchOffset)) + s += l + nextEmit = s + if nextS >= s { + s = nextS + 1 + } + + if s >= sLimit { + // Index first pair after match end. + if int(s+l+8) < len(src) { + cv := load6432(src, s) + e.table[hashLen(cv, bTableBits, hashBytes)] = tableEntry{offset: s + e.cur} + } + goto emitRemainder + } + + // Store every second hash in-between, but offset by 1. + for i := s - l + 2; i < s-5; i += 7 { + x := load6432(src, i) + nextHash := hashLen(x, bTableBits, hashBytes) + e.table[nextHash] = tableEntry{offset: e.cur + i} + // Skip one + x >>= 16 + nextHash = hashLen(x, bTableBits, hashBytes) + e.table[nextHash] = tableEntry{offset: e.cur + i + 2} + // Skip one + x >>= 16 + nextHash = hashLen(x, bTableBits, hashBytes) + e.table[nextHash] = tableEntry{offset: e.cur + i + 4} + } + + // We could immediately start working at s now, but to improve + // compression we first update the hash table at s-2 to s. If + // another emitCopy is not our next move, also calculate nextHash + // at s+1. At least on GOARCH=amd64, these three hash calculations + // are faster as one load64 call (with some shifts) instead of + // three load32 calls. + x := load6432(src, s-2) + o := e.cur + s - 2 + prevHash := hashLen(x, bTableBits, hashBytes) + prevHash2 := hashLen(x>>8, bTableBits, hashBytes) + e.table[prevHash] = tableEntry{offset: o} + e.table[prevHash2] = tableEntry{offset: o + 1} + currHash := hashLen(x>>16, bTableBits, hashBytes) + candidate = e.table[currHash] + e.table[currHash] = tableEntry{offset: o + 2} + + offset := s - (candidate.offset - e.cur) + if offset > maxMatchOffset || uint32(x>>16) != load3232(src, candidate.offset-e.cur) { + cv = x >> 24 + s++ + break + } + } + } + +emitRemainder: + if int(nextEmit) < len(src) { + // If nothing was added, don't encode literals. + if dst.n == 0 { + return + } + + emitLiteral(dst, src[nextEmit:]) + } +} diff --git a/internal/compress/flate/level3.go b/internal/compress/flate/level3.go new file mode 100644 index 00000000..33f9fb15 --- /dev/null +++ b/internal/compress/flate/level3.go @@ -0,0 +1,241 @@ +package flate + +import "fmt" + +// fastEncL3 +type fastEncL3 struct { + fastGen + table [1 << 16]tableEntryPrev +} + +// Encode uses a similar algorithm to level 2, will check up to two candidates. +func (e *fastEncL3) Encode(dst *tokens, src []byte) { + const ( + inputMargin = 12 - 1 + minNonLiteralBlockSize = 1 + 1 + inputMargin + tableBits = 16 + tableSize = 1 << tableBits + hashBytes = 5 + ) + + if debugDeflate && e.cur < 0 { + panic(fmt.Sprint("e.cur < 0: ", e.cur)) + } + + // Protect against e.cur wraparound. + for e.cur >= bufferReset { + if len(e.hist) == 0 { + for i := range e.table[:] { + e.table[i] = tableEntryPrev{} + } + e.cur = maxMatchOffset + break + } + // Shift down everything in the table that isn't already too far away. + minOff := e.cur + int32(len(e.hist)) - maxMatchOffset + for i := range e.table[:] { + v := e.table[i] + if v.Cur.offset <= minOff { + v.Cur.offset = 0 + } else { + v.Cur.offset = v.Cur.offset - e.cur + maxMatchOffset + } + if v.Prev.offset <= minOff { + v.Prev.offset = 0 + } else { + v.Prev.offset = v.Prev.offset - e.cur + maxMatchOffset + } + e.table[i] = v + } + e.cur = maxMatchOffset + } + + s := e.addBlock(src) + + // Skip if too small. + if len(src) < minNonLiteralBlockSize { + // We do not fill the token table. + // This will be picked up by caller. + dst.n = uint16(len(src)) + return + } + + // Override src + src = e.hist + nextEmit := s + + // sLimit is when to stop looking for offset/length copies. The inputMargin + // lets us use a fast path for emitLiteral in the main loop, while we are + // looking for copies. + sLimit := int32(len(src) - inputMargin) + + // nextEmit is where in src the next emitLiteral should start from. + cv := load6432(src, s) + for { + const skipLog = 7 + nextS := s + var candidate tableEntry + for { + nextHash := hashLen(cv, tableBits, hashBytes) + s = nextS + nextS = s + 1 + (s-nextEmit)>>skipLog + if nextS > sLimit { + goto emitRemainder + } + candidates := e.table[nextHash] + now := load6432(src, nextS) + + // Safe offset distance until s + 4... + minOffset := e.cur + s - (maxMatchOffset - 4) + e.table[nextHash] = tableEntryPrev{Prev: candidates.Cur, Cur: tableEntry{offset: s + e.cur}} + + // Check both candidates + candidate = candidates.Cur + if candidate.offset < minOffset { + cv = now + // Previous will also be invalid, we have nothing. + continue + } + + if uint32(cv) == load3232(src, candidate.offset-e.cur) { + if candidates.Prev.offset < minOffset || uint32(cv) != load3232(src, candidates.Prev.offset-e.cur) { + break + } + // Both match and are valid, pick longest. + offset := s - (candidate.offset - e.cur) + o2 := s - (candidates.Prev.offset - e.cur) + l1, l2 := matchLen(src[s+4:], src[s-offset+4:]), matchLen(src[s+4:], src[s-o2+4:]) + if l2 > l1 { + candidate = candidates.Prev + } + break + } else { + // We only check if value mismatches. + // Offset will always be invalid in other cases. + candidate = candidates.Prev + if candidate.offset > minOffset && uint32(cv) == load3232(src, candidate.offset-e.cur) { + break + } + } + cv = now + } + + // Call emitCopy, and then see if another emitCopy could be our next + // move. Repeat until we find no match for the input immediately after + // what was consumed by the last emitCopy call. + // + // If we exit this loop normally then we need to call emitLiteral next, + // though we don't yet know how big the literal will be. We handle that + // by proceeding to the next iteration of the main loop. We also can + // exit this loop via goto if we get close to exhausting the input. + for { + // Invariant: we have a 4-byte match at s, and no need to emit any + // literal bytes prior to s. + + // Extend the 4-byte match as long as possible. + // + t := candidate.offset - e.cur + l := e.matchlenLong(int(s+4), int(t+4), src) + 4 + + // Extend backwards + for t > 0 && s > nextEmit && src[t-1] == src[s-1] { + s-- + t-- + l++ + } + if nextEmit < s { + if false { + emitLiteral(dst, src[nextEmit:s]) + } else { + for _, v := range src[nextEmit:s] { + dst.tokens[dst.n] = token(v) + dst.litHist[v]++ + dst.n++ + } + } + } + + dst.AddMatchLong(l, uint32(s-t-baseMatchOffset)) + s += l + nextEmit = s + if nextS >= s { + s = nextS + 1 + } + + if s >= sLimit { + t += l + // Index first pair after match end. + if int(t+8) < len(src) && t > 0 { + cv = load6432(src, t) + nextHash := hashLen(cv, tableBits, hashBytes) + e.table[nextHash] = tableEntryPrev{ + Prev: e.table[nextHash].Cur, + Cur: tableEntry{offset: e.cur + t}, + } + } + goto emitRemainder + } + + // Store every 5th hash in-between. + for i := s - l + 2; i < s-5; i += 6 { + nextHash := hashLen(load6432(src, i), tableBits, hashBytes) + e.table[nextHash] = tableEntryPrev{ + Prev: e.table[nextHash].Cur, + Cur: tableEntry{offset: e.cur + i}} + } + // We could immediately start working at s now, but to improve + // compression we first update the hash table at s-2 to s. + x := load6432(src, s-2) + prevHash := hashLen(x, tableBits, hashBytes) + + e.table[prevHash] = tableEntryPrev{ + Prev: e.table[prevHash].Cur, + Cur: tableEntry{offset: e.cur + s - 2}, + } + x >>= 8 + prevHash = hashLen(x, tableBits, hashBytes) + + e.table[prevHash] = tableEntryPrev{ + Prev: e.table[prevHash].Cur, + Cur: tableEntry{offset: e.cur + s - 1}, + } + x >>= 8 + currHash := hashLen(x, tableBits, hashBytes) + candidates := e.table[currHash] + cv = x + e.table[currHash] = tableEntryPrev{ + Prev: candidates.Cur, + Cur: tableEntry{offset: s + e.cur}, + } + + // Check both candidates + candidate = candidates.Cur + minOffset := e.cur + s - (maxMatchOffset - 4) + + if candidate.offset > minOffset { + if uint32(cv) == load3232(src, candidate.offset-e.cur) { + // Found a match... + continue + } + candidate = candidates.Prev + if candidate.offset > minOffset && uint32(cv) == load3232(src, candidate.offset-e.cur) { + // Match at prev... + continue + } + } + cv = x >> 8 + s++ + break + } + } + +emitRemainder: + if int(nextEmit) < len(src) { + // If nothing was added, don't encode literals. + if dst.n == 0 { + return + } + + emitLiteral(dst, src[nextEmit:]) + } +} diff --git a/internal/compress/flate/level4.go b/internal/compress/flate/level4.go new file mode 100644 index 00000000..88509e19 --- /dev/null +++ b/internal/compress/flate/level4.go @@ -0,0 +1,221 @@ +package flate + +import "fmt" + +type fastEncL4 struct { + fastGen + table [tableSize]tableEntry + bTable [tableSize]tableEntry +} + +func (e *fastEncL4) Encode(dst *tokens, src []byte) { + const ( + inputMargin = 12 - 1 + minNonLiteralBlockSize = 1 + 1 + inputMargin + hashShortBytes = 4 + ) + if debugDeflate && e.cur < 0 { + panic(fmt.Sprint("e.cur < 0: ", e.cur)) + } + // Protect against e.cur wraparound. + for e.cur >= bufferReset { + if len(e.hist) == 0 { + for i := range e.table[:] { + e.table[i] = tableEntry{} + } + for i := range e.bTable[:] { + e.bTable[i] = tableEntry{} + } + e.cur = maxMatchOffset + break + } + // Shift down everything in the table that isn't already too far away. + minOff := e.cur + int32(len(e.hist)) - maxMatchOffset + for i := range e.table[:] { + v := e.table[i].offset + if v <= minOff { + v = 0 + } else { + v = v - e.cur + maxMatchOffset + } + e.table[i].offset = v + } + for i := range e.bTable[:] { + v := e.bTable[i].offset + if v <= minOff { + v = 0 + } else { + v = v - e.cur + maxMatchOffset + } + e.bTable[i].offset = v + } + e.cur = maxMatchOffset + } + + s := e.addBlock(src) + + // This check isn't in the Snappy implementation, but there, the caller + // instead of the callee handles this case. + if len(src) < minNonLiteralBlockSize { + // We do not fill the token table. + // This will be picked up by caller. + dst.n = uint16(len(src)) + return + } + + // Override src + src = e.hist + nextEmit := s + + // sLimit is when to stop looking for offset/length copies. The inputMargin + // lets us use a fast path for emitLiteral in the main loop, while we are + // looking for copies. + sLimit := int32(len(src) - inputMargin) + + // nextEmit is where in src the next emitLiteral should start from. + cv := load6432(src, s) + for { + const skipLog = 6 + const doEvery = 1 + + nextS := s + var t int32 + for { + nextHashS := hashLen(cv, tableBits, hashShortBytes) + nextHashL := hash7(cv, tableBits) + + s = nextS + nextS = s + doEvery + (s-nextEmit)>>skipLog + if nextS > sLimit { + goto emitRemainder + } + // Fetch a short+long candidate + sCandidate := e.table[nextHashS] + lCandidate := e.bTable[nextHashL] + next := load6432(src, nextS) + entry := tableEntry{offset: s + e.cur} + e.table[nextHashS] = entry + e.bTable[nextHashL] = entry + + t = lCandidate.offset - e.cur + if s-t < maxMatchOffset && uint32(cv) == load3232(src, t) { + // We got a long match. Use that. + break + } + + t = sCandidate.offset - e.cur + if s-t < maxMatchOffset && uint32(cv) == load3232(src, t) { + // Found a 4 match... + lCandidate = e.bTable[hash7(next, tableBits)] + + // If the next long is a candidate, check if we should use that instead... + lOff := lCandidate.offset - e.cur + if nextS-lOff < maxMatchOffset && load3232(src, lOff) == uint32(next) { + l1, l2 := matchLen(src[s+4:], src[t+4:]), matchLen(src[nextS+4:], src[nextS-lOff+4:]) + if l2 > l1 { + s = nextS + t = lCandidate.offset - e.cur + } + } + break + } + cv = next + } + + // A 4-byte match has been found. We'll later see if more than 4 bytes + // match. But, prior to the match, src[nextEmit:s] are unmatched. Emit + // them as literal bytes. + + // Extend the 4-byte match as long as possible. + l := e.matchlenLong(int(s+4), int(t+4), src) + 4 + + // Extend backwards + for t > 0 && s > nextEmit && src[t-1] == src[s-1] { + s-- + t-- + l++ + } + if nextEmit < s { + if false { + emitLiteral(dst, src[nextEmit:s]) + } else { + for _, v := range src[nextEmit:s] { + dst.tokens[dst.n] = token(v) + dst.litHist[v]++ + dst.n++ + } + } + } + if debugDeflate { + if t >= s { + panic("s-t") + } + if (s - t) > maxMatchOffset { + panic(fmt.Sprintln("mmo", t)) + } + if l < baseMatchLength { + panic("bml") + } + } + + dst.AddMatchLong(l, uint32(s-t-baseMatchOffset)) + s += l + nextEmit = s + if nextS >= s { + s = nextS + 1 + } + + if s >= sLimit { + // Index first pair after match end. + if int(s+8) < len(src) { + cv := load6432(src, s) + e.table[hashLen(cv, tableBits, hashShortBytes)] = tableEntry{offset: s + e.cur} + e.bTable[hash7(cv, tableBits)] = tableEntry{offset: s + e.cur} + } + goto emitRemainder + } + + // Store every 3rd hash in-between + if true { + i := nextS + if i < s-1 { + cv := load6432(src, i) + t := tableEntry{offset: i + e.cur} + t2 := tableEntry{offset: t.offset + 1} + e.bTable[hash7(cv, tableBits)] = t + e.bTable[hash7(cv>>8, tableBits)] = t2 + e.table[hashLen(cv>>8, tableBits, hashShortBytes)] = t2 + + i += 3 + for ; i < s-1; i += 3 { + cv := load6432(src, i) + t := tableEntry{offset: i + e.cur} + t2 := tableEntry{offset: t.offset + 1} + e.bTable[hash7(cv, tableBits)] = t + e.bTable[hash7(cv>>8, tableBits)] = t2 + e.table[hashLen(cv>>8, tableBits, hashShortBytes)] = t2 + } + } + } + + // We could immediately start working at s now, but to improve + // compression we first update the hash table at s-1 and at s. + x := load6432(src, s-1) + o := e.cur + s - 1 + prevHashS := hashLen(x, tableBits, hashShortBytes) + prevHashL := hash7(x, tableBits) + e.table[prevHashS] = tableEntry{offset: o} + e.bTable[prevHashL] = tableEntry{offset: o} + cv = x >> 8 + } + +emitRemainder: + if int(nextEmit) < len(src) { + // If nothing was added, don't encode literals. + if dst.n == 0 { + return + } + + emitLiteral(dst, src[nextEmit:]) + } +} diff --git a/internal/compress/flate/level5.go b/internal/compress/flate/level5.go new file mode 100644 index 00000000..a22ad7d1 --- /dev/null +++ b/internal/compress/flate/level5.go @@ -0,0 +1,705 @@ +package flate + +import "fmt" + +type fastEncL5 struct { + fastGen + table [tableSize]tableEntry + bTable [tableSize]tableEntryPrev +} + +func (e *fastEncL5) Encode(dst *tokens, src []byte) { + const ( + inputMargin = 12 - 1 + minNonLiteralBlockSize = 1 + 1 + inputMargin + hashShortBytes = 4 + ) + if debugDeflate && e.cur < 0 { + panic(fmt.Sprint("e.cur < 0: ", e.cur)) + } + + // Protect against e.cur wraparound. + for e.cur >= bufferReset { + if len(e.hist) == 0 { + for i := range e.table[:] { + e.table[i] = tableEntry{} + } + for i := range e.bTable[:] { + e.bTable[i] = tableEntryPrev{} + } + e.cur = maxMatchOffset + break + } + // Shift down everything in the table that isn't already too far away. + minOff := e.cur + int32(len(e.hist)) - maxMatchOffset + for i := range e.table[:] { + v := e.table[i].offset + if v <= minOff { + v = 0 + } else { + v = v - e.cur + maxMatchOffset + } + e.table[i].offset = v + } + for i := range e.bTable[:] { + v := e.bTable[i] + if v.Cur.offset <= minOff { + v.Cur.offset = 0 + v.Prev.offset = 0 + } else { + v.Cur.offset = v.Cur.offset - e.cur + maxMatchOffset + if v.Prev.offset <= minOff { + v.Prev.offset = 0 + } else { + v.Prev.offset = v.Prev.offset - e.cur + maxMatchOffset + } + } + e.bTable[i] = v + } + e.cur = maxMatchOffset + } + + s := e.addBlock(src) + + // This check isn't in the Snappy implementation, but there, the caller + // instead of the callee handles this case. + if len(src) < minNonLiteralBlockSize { + // We do not fill the token table. + // This will be picked up by caller. + dst.n = uint16(len(src)) + return + } + + // Override src + src = e.hist + nextEmit := s + + // sLimit is when to stop looking for offset/length copies. The inputMargin + // lets us use a fast path for emitLiteral in the main loop, while we are + // looking for copies. + sLimit := int32(len(src) - inputMargin) + + // nextEmit is where in src the next emitLiteral should start from. + cv := load6432(src, s) + for { + const skipLog = 6 + const doEvery = 1 + + nextS := s + var l int32 + var t int32 + for { + nextHashS := hashLen(cv, tableBits, hashShortBytes) + nextHashL := hash7(cv, tableBits) + + s = nextS + nextS = s + doEvery + (s-nextEmit)>>skipLog + if nextS > sLimit { + goto emitRemainder + } + // Fetch a short+long candidate + sCandidate := e.table[nextHashS] + lCandidate := e.bTable[nextHashL] + next := load6432(src, nextS) + entry := tableEntry{offset: s + e.cur} + e.table[nextHashS] = entry + eLong := &e.bTable[nextHashL] + eLong.Cur, eLong.Prev = entry, eLong.Cur + + nextHashS = hashLen(next, tableBits, hashShortBytes) + nextHashL = hash7(next, tableBits) + + t = lCandidate.Cur.offset - e.cur + if s-t < maxMatchOffset { + if uint32(cv) == load3232(src, t) { + // Store the next match + e.table[nextHashS] = tableEntry{offset: nextS + e.cur} + eLong := &e.bTable[nextHashL] + eLong.Cur, eLong.Prev = tableEntry{offset: nextS + e.cur}, eLong.Cur + + t2 := lCandidate.Prev.offset - e.cur + if s-t2 < maxMatchOffset && uint32(cv) == load3232(src, t2) { + l = e.matchlen(int(s+4), int(t+4), src) + 4 + ml1 := e.matchlen(int(s+4), int(t2+4), src) + 4 + if ml1 > l { + t = t2 + l = ml1 + break + } + } + break + } + t = lCandidate.Prev.offset - e.cur + if s-t < maxMatchOffset && uint32(cv) == load3232(src, t) { + // Store the next match + e.table[nextHashS] = tableEntry{offset: nextS + e.cur} + eLong := &e.bTable[nextHashL] + eLong.Cur, eLong.Prev = tableEntry{offset: nextS + e.cur}, eLong.Cur + break + } + } + + t = sCandidate.offset - e.cur + if s-t < maxMatchOffset && uint32(cv) == load3232(src, t) { + // Found a 4 match... + l = e.matchlen(int(s+4), int(t+4), src) + 4 + lCandidate = e.bTable[nextHashL] + // Store the next match + + e.table[nextHashS] = tableEntry{offset: nextS + e.cur} + eLong := &e.bTable[nextHashL] + eLong.Cur, eLong.Prev = tableEntry{offset: nextS + e.cur}, eLong.Cur + + // If the next long is a candidate, use that... + t2 := lCandidate.Cur.offset - e.cur + if nextS-t2 < maxMatchOffset { + if load3232(src, t2) == uint32(next) { + ml := e.matchlen(int(nextS+4), int(t2+4), src) + 4 + if ml > l { + t = t2 + s = nextS + l = ml + break + } + } + // If the previous long is a candidate, use that... + t2 = lCandidate.Prev.offset - e.cur + if nextS-t2 < maxMatchOffset && load3232(src, t2) == uint32(next) { + ml := e.matchlen(int(nextS+4), int(t2+4), src) + 4 + if ml > l { + t = t2 + s = nextS + l = ml + break + } + } + } + break + } + cv = next + } + + // A 4-byte match has been found. We'll later see if more than 4 bytes + // match. But, prior to the match, src[nextEmit:s] are unmatched. Emit + // them as literal bytes. + + if l == 0 { + // Extend the 4-byte match as long as possible. + l = e.matchlenLong(int(s+4), int(t+4), src) + 4 + } else if l == maxMatchLength { + l += e.matchlenLong(int(s+l), int(t+l), src) + } + + // Try to locate a better match by checking the end of best match... + if sAt := s + l; l < 30 && sAt < sLimit { + // Allow some bytes at the beginning to mismatch. + // Sweet spot is 2/3 bytes depending on input. + // 3 is only a little better when it is but sometimes a lot worse. + // The skipped bytes are tested in Extend backwards, + // and still picked up as part of the match if they do. + const skipBeginning = 2 + eLong := e.bTable[hash7(load6432(src, sAt), tableBits)].Cur.offset + t2 := eLong - e.cur - l + skipBeginning + s2 := s + skipBeginning + off := s2 - t2 + if t2 >= 0 && off < maxMatchOffset && off > 0 { + if l2 := e.matchlenLong(int(s2), int(t2), src); l2 > l { + t = t2 + l = l2 + s = s2 + } + } + } + + // Extend backwards + for t > 0 && s > nextEmit && src[t-1] == src[s-1] { + s-- + t-- + l++ + } + if nextEmit < s { + if false { + emitLiteral(dst, src[nextEmit:s]) + } else { + for _, v := range src[nextEmit:s] { + dst.tokens[dst.n] = token(v) + dst.litHist[v]++ + dst.n++ + } + } + } + if debugDeflate { + if t >= s { + panic(fmt.Sprintln("s-t", s, t)) + } + if (s - t) > maxMatchOffset { + panic(fmt.Sprintln("mmo", s-t)) + } + if l < baseMatchLength { + panic("bml") + } + } + + dst.AddMatchLong(l, uint32(s-t-baseMatchOffset)) + s += l + nextEmit = s + if nextS >= s { + s = nextS + 1 + } + + if s >= sLimit { + goto emitRemainder + } + + // Store every 3rd hash in-between. + if true { + const hashEvery = 3 + i := s - l + 1 + if i < s-1 { + cv := load6432(src, i) + t := tableEntry{offset: i + e.cur} + e.table[hashLen(cv, tableBits, hashShortBytes)] = t + eLong := &e.bTable[hash7(cv, tableBits)] + eLong.Cur, eLong.Prev = t, eLong.Cur + + // Do an long at i+1 + cv >>= 8 + t = tableEntry{offset: t.offset + 1} + eLong = &e.bTable[hash7(cv, tableBits)] + eLong.Cur, eLong.Prev = t, eLong.Cur + + // We only have enough bits for a short entry at i+2 + cv >>= 8 + t = tableEntry{offset: t.offset + 1} + e.table[hashLen(cv, tableBits, hashShortBytes)] = t + + // Skip one - otherwise we risk hitting 's' + i += 4 + for ; i < s-1; i += hashEvery { + cv := load6432(src, i) + t := tableEntry{offset: i + e.cur} + t2 := tableEntry{offset: t.offset + 1} + eLong := &e.bTable[hash7(cv, tableBits)] + eLong.Cur, eLong.Prev = t, eLong.Cur + e.table[hashLen(cv>>8, tableBits, hashShortBytes)] = t2 + } + } + } + + // We could immediately start working at s now, but to improve + // compression we first update the hash table at s-1 and at s. + x := load6432(src, s-1) + o := e.cur + s - 1 + prevHashS := hashLen(x, tableBits, hashShortBytes) + prevHashL := hash7(x, tableBits) + e.table[prevHashS] = tableEntry{offset: o} + eLong := &e.bTable[prevHashL] + eLong.Cur, eLong.Prev = tableEntry{offset: o}, eLong.Cur + cv = x >> 8 + } + +emitRemainder: + if int(nextEmit) < len(src) { + // If nothing was added, don't encode literals. + if dst.n == 0 { + return + } + + emitLiteral(dst, src[nextEmit:]) + } +} + +// fastEncL5Window is a level 5 encoder, +// but with a custom window size. +type fastEncL5Window struct { + hist []byte + cur int32 + maxOffset int32 + table [tableSize]tableEntry + bTable [tableSize]tableEntryPrev +} + +func (e *fastEncL5Window) Encode(dst *tokens, src []byte) { + const ( + inputMargin = 12 - 1 + minNonLiteralBlockSize = 1 + 1 + inputMargin + hashShortBytes = 4 + ) + maxMatchOffset := e.maxOffset + if debugDeflate && e.cur < 0 { + panic(fmt.Sprint("e.cur < 0: ", e.cur)) + } + + // Protect against e.cur wraparound. + for e.cur >= bufferReset { + if len(e.hist) == 0 { + for i := range e.table[:] { + e.table[i] = tableEntry{} + } + for i := range e.bTable[:] { + e.bTable[i] = tableEntryPrev{} + } + e.cur = maxMatchOffset + break + } + // Shift down everything in the table that isn't already too far away. + minOff := e.cur + int32(len(e.hist)) - maxMatchOffset + for i := range e.table[:] { + v := e.table[i].offset + if v <= minOff { + v = 0 + } else { + v = v - e.cur + maxMatchOffset + } + e.table[i].offset = v + } + for i := range e.bTable[:] { + v := e.bTable[i] + if v.Cur.offset <= minOff { + v.Cur.offset = 0 + v.Prev.offset = 0 + } else { + v.Cur.offset = v.Cur.offset - e.cur + maxMatchOffset + if v.Prev.offset <= minOff { + v.Prev.offset = 0 + } else { + v.Prev.offset = v.Prev.offset - e.cur + maxMatchOffset + } + } + e.bTable[i] = v + } + e.cur = maxMatchOffset + } + + s := e.addBlock(src) + + // This check isn't in the Snappy implementation, but there, the caller + // instead of the callee handles this case. + if len(src) < minNonLiteralBlockSize { + // We do not fill the token table. + // This will be picked up by caller. + dst.n = uint16(len(src)) + return + } + + // Override src + src = e.hist + nextEmit := s + + // sLimit is when to stop looking for offset/length copies. The inputMargin + // lets us use a fast path for emitLiteral in the main loop, while we are + // looking for copies. + sLimit := int32(len(src) - inputMargin) + + // nextEmit is where in src the next emitLiteral should start from. + cv := load6432(src, s) + for { + const skipLog = 6 + const doEvery = 1 + + nextS := s + var l int32 + var t int32 + for { + nextHashS := hashLen(cv, tableBits, hashShortBytes) + nextHashL := hash7(cv, tableBits) + + s = nextS + nextS = s + doEvery + (s-nextEmit)>>skipLog + if nextS > sLimit { + goto emitRemainder + } + // Fetch a short+long candidate + sCandidate := e.table[nextHashS] + lCandidate := e.bTable[nextHashL] + next := load6432(src, nextS) + entry := tableEntry{offset: s + e.cur} + e.table[nextHashS] = entry + eLong := &e.bTable[nextHashL] + eLong.Cur, eLong.Prev = entry, eLong.Cur + + nextHashS = hashLen(next, tableBits, hashShortBytes) + nextHashL = hash7(next, tableBits) + + t = lCandidate.Cur.offset - e.cur + if s-t < maxMatchOffset { + if uint32(cv) == load3232(src, t) { + // Store the next match + e.table[nextHashS] = tableEntry{offset: nextS + e.cur} + eLong := &e.bTable[nextHashL] + eLong.Cur, eLong.Prev = tableEntry{offset: nextS + e.cur}, eLong.Cur + + t2 := lCandidate.Prev.offset - e.cur + if s-t2 < maxMatchOffset && uint32(cv) == load3232(src, t2) { + l = e.matchlen(s+4, t+4, src) + 4 + ml1 := e.matchlen(s+4, t2+4, src) + 4 + if ml1 > l { + t = t2 + l = ml1 + break + } + } + break + } + t = lCandidate.Prev.offset - e.cur + if s-t < maxMatchOffset && uint32(cv) == load3232(src, t) { + // Store the next match + e.table[nextHashS] = tableEntry{offset: nextS + e.cur} + eLong := &e.bTable[nextHashL] + eLong.Cur, eLong.Prev = tableEntry{offset: nextS + e.cur}, eLong.Cur + break + } + } + + t = sCandidate.offset - e.cur + if s-t < maxMatchOffset && uint32(cv) == load3232(src, t) { + // Found a 4 match... + l = e.matchlen(s+4, t+4, src) + 4 + lCandidate = e.bTable[nextHashL] + // Store the next match + + e.table[nextHashS] = tableEntry{offset: nextS + e.cur} + eLong := &e.bTable[nextHashL] + eLong.Cur, eLong.Prev = tableEntry{offset: nextS + e.cur}, eLong.Cur + + // If the next long is a candidate, use that... + t2 := lCandidate.Cur.offset - e.cur + if nextS-t2 < maxMatchOffset { + if load3232(src, t2) == uint32(next) { + ml := e.matchlen(nextS+4, t2+4, src) + 4 + if ml > l { + t = t2 + s = nextS + l = ml + break + } + } + // If the previous long is a candidate, use that... + t2 = lCandidate.Prev.offset - e.cur + if nextS-t2 < maxMatchOffset && load3232(src, t2) == uint32(next) { + ml := e.matchlen(nextS+4, t2+4, src) + 4 + if ml > l { + t = t2 + s = nextS + l = ml + break + } + } + } + break + } + cv = next + } + + // A 4-byte match has been found. We'll later see if more than 4 bytes + // match. But, prior to the match, src[nextEmit:s] are unmatched. Emit + // them as literal bytes. + + if l == 0 { + // Extend the 4-byte match as long as possible. + l = e.matchlenLong(s+4, t+4, src) + 4 + } else if l == maxMatchLength { + l += e.matchlenLong(s+l, t+l, src) + } + + // Try to locate a better match by checking the end of best match... + if sAt := s + l; l < 30 && sAt < sLimit { + // Allow some bytes at the beginning to mismatch. + // Sweet spot is 2/3 bytes depending on input. + // 3 is only a little better when it is but sometimes a lot worse. + // The skipped bytes are tested in Extend backwards, + // and still picked up as part of the match if they do. + const skipBeginning = 2 + eLong := e.bTable[hash7(load6432(src, sAt), tableBits)].Cur.offset + t2 := eLong - e.cur - l + skipBeginning + s2 := s + skipBeginning + off := s2 - t2 + if t2 >= 0 && off < maxMatchOffset && off > 0 { + if l2 := e.matchlenLong(s2, t2, src); l2 > l { + t = t2 + l = l2 + s = s2 + } + } + } + + // Extend backwards + for t > 0 && s > nextEmit && src[t-1] == src[s-1] { + s-- + t-- + l++ + } + if nextEmit < s { + if false { + emitLiteral(dst, src[nextEmit:s]) + } else { + for _, v := range src[nextEmit:s] { + dst.tokens[dst.n] = token(v) + dst.litHist[v]++ + dst.n++ + } + } + } + if debugDeflate { + if t >= s { + panic(fmt.Sprintln("s-t", s, t)) + } + if (s - t) > maxMatchOffset { + panic(fmt.Sprintln("mmo", s-t)) + } + if l < baseMatchLength { + panic("bml") + } + } + + dst.AddMatchLong(l, uint32(s-t-baseMatchOffset)) + s += l + nextEmit = s + if nextS >= s { + s = nextS + 1 + } + + if s >= sLimit { + goto emitRemainder + } + + // Store every 3rd hash in-between. + if true { + const hashEvery = 3 + i := s - l + 1 + if i < s-1 { + cv := load6432(src, i) + t := tableEntry{offset: i + e.cur} + e.table[hashLen(cv, tableBits, hashShortBytes)] = t + eLong := &e.bTable[hash7(cv, tableBits)] + eLong.Cur, eLong.Prev = t, eLong.Cur + + // Do an long at i+1 + cv >>= 8 + t = tableEntry{offset: t.offset + 1} + eLong = &e.bTable[hash7(cv, tableBits)] + eLong.Cur, eLong.Prev = t, eLong.Cur + + // We only have enough bits for a short entry at i+2 + cv >>= 8 + t = tableEntry{offset: t.offset + 1} + e.table[hashLen(cv, tableBits, hashShortBytes)] = t + + // Skip one - otherwise we risk hitting 's' + i += 4 + for ; i < s-1; i += hashEvery { + cv := load6432(src, i) + t := tableEntry{offset: i + e.cur} + t2 := tableEntry{offset: t.offset + 1} + eLong := &e.bTable[hash7(cv, tableBits)] + eLong.Cur, eLong.Prev = t, eLong.Cur + e.table[hashLen(cv>>8, tableBits, hashShortBytes)] = t2 + } + } + } + + // We could immediately start working at s now, but to improve + // compression we first update the hash table at s-1 and at s. + x := load6432(src, s-1) + o := e.cur + s - 1 + prevHashS := hashLen(x, tableBits, hashShortBytes) + prevHashL := hash7(x, tableBits) + e.table[prevHashS] = tableEntry{offset: o} + eLong := &e.bTable[prevHashL] + eLong.Cur, eLong.Prev = tableEntry{offset: o}, eLong.Cur + cv = x >> 8 + } + +emitRemainder: + if int(nextEmit) < len(src) { + // If nothing was added, don't encode literals. + if dst.n == 0 { + return + } + + emitLiteral(dst, src[nextEmit:]) + } +} + +// Reset the encoding table. +func (e *fastEncL5Window) Reset() { + // We keep the same allocs, since we are compressing the same block sizes. + if cap(e.hist) < allocHistory { + e.hist = make([]byte, 0, allocHistory) + } + + // We offset current position so everything will be out of reach. + // If we are above the buffer reset it will be cleared anyway since len(hist) == 0. + if e.cur <= int32(bufferReset) { + e.cur += e.maxOffset + int32(len(e.hist)) + } + e.hist = e.hist[:0] +} + +func (e *fastEncL5Window) addBlock(src []byte) int32 { + // check if we have space already + maxMatchOffset := e.maxOffset + + if len(e.hist)+len(src) > cap(e.hist) { + if cap(e.hist) == 0 { + e.hist = make([]byte, 0, allocHistory) + } else { + if cap(e.hist) < int(maxMatchOffset*2) { + panic("unexpected buffer size") + } + // Move down + offset := int32(len(e.hist)) - maxMatchOffset + copy(e.hist[0:maxMatchOffset], e.hist[offset:]) + e.cur += offset + e.hist = e.hist[:maxMatchOffset] + } + } + s := int32(len(e.hist)) + e.hist = append(e.hist, src...) + return s +} + +// matchlen will return the match length between offsets and t in src. +// The maximum length returned is maxMatchLength - 4. +// It is assumed that s > t, that t >=0 and s < len(src). +func (e *fastEncL5Window) matchlen(s, t int32, src []byte) int32 { + if debugDecode { + if t >= s { + panic(fmt.Sprint("t >=s:", t, s)) + } + if int(s) >= len(src) { + panic(fmt.Sprint("s >= len(src):", s, len(src))) + } + if t < 0 { + panic(fmt.Sprint("t < 0:", t)) + } + if s-t > e.maxOffset { + panic(fmt.Sprint(s, "-", t, "(", s-t, ") > maxMatchLength (", maxMatchOffset, ")")) + } + } + s1 := min(int(s)+maxMatchLength-4, len(src)) + + // Extend the match to be as long as possible. + return int32(matchLen(src[s:s1], src[t:])) +} + +// matchlenLong will return the match length between offsets and t in src. +// It is assumed that s > t, that t >=0 and s < len(src). +func (e *fastEncL5Window) matchlenLong(s, t int32, src []byte) int32 { + if debugDeflate { + if t >= s { + panic(fmt.Sprint("t >=s:", t, s)) + } + if int(s) >= len(src) { + panic(fmt.Sprint("s >= len(src):", s, len(src))) + } + if t < 0 { + panic(fmt.Sprint("t < 0:", t)) + } + if s-t > e.maxOffset { + panic(fmt.Sprint(s, "-", t, "(", s-t, ") > maxMatchLength (", maxMatchOffset, ")")) + } + } + // Extend the match to be as long as possible. + return int32(matchLen(src[s:], src[t:])) +} diff --git a/internal/compress/flate/level6.go b/internal/compress/flate/level6.go new file mode 100644 index 00000000..96f5bb43 --- /dev/null +++ b/internal/compress/flate/level6.go @@ -0,0 +1,325 @@ +package flate + +import "fmt" + +type fastEncL6 struct { + fastGen + table [tableSize]tableEntry + bTable [tableSize]tableEntryPrev +} + +func (e *fastEncL6) Encode(dst *tokens, src []byte) { + const ( + inputMargin = 12 - 1 + minNonLiteralBlockSize = 1 + 1 + inputMargin + hashShortBytes = 4 + ) + if debugDeflate && e.cur < 0 { + panic(fmt.Sprint("e.cur < 0: ", e.cur)) + } + + // Protect against e.cur wraparound. + for e.cur >= bufferReset { + if len(e.hist) == 0 { + for i := range e.table[:] { + e.table[i] = tableEntry{} + } + for i := range e.bTable[:] { + e.bTable[i] = tableEntryPrev{} + } + e.cur = maxMatchOffset + break + } + // Shift down everything in the table that isn't already too far away. + minOff := e.cur + int32(len(e.hist)) - maxMatchOffset + for i := range e.table[:] { + v := e.table[i].offset + if v <= minOff { + v = 0 + } else { + v = v - e.cur + maxMatchOffset + } + e.table[i].offset = v + } + for i := range e.bTable[:] { + v := e.bTable[i] + if v.Cur.offset <= minOff { + v.Cur.offset = 0 + v.Prev.offset = 0 + } else { + v.Cur.offset = v.Cur.offset - e.cur + maxMatchOffset + if v.Prev.offset <= minOff { + v.Prev.offset = 0 + } else { + v.Prev.offset = v.Prev.offset - e.cur + maxMatchOffset + } + } + e.bTable[i] = v + } + e.cur = maxMatchOffset + } + + s := e.addBlock(src) + + // This check isn't in the Snappy implementation, but there, the caller + // instead of the callee handles this case. + if len(src) < minNonLiteralBlockSize { + // We do not fill the token table. + // This will be picked up by caller. + dst.n = uint16(len(src)) + return + } + + // Override src + src = e.hist + nextEmit := s + + // sLimit is when to stop looking for offset/length copies. The inputMargin + // lets us use a fast path for emitLiteral in the main loop, while we are + // looking for copies. + sLimit := int32(len(src) - inputMargin) + + // nextEmit is where in src the next emitLiteral should start from. + cv := load6432(src, s) + // Repeat MUST be > 1 and within range + repeat := int32(1) + for { + const skipLog = 7 + const doEvery = 1 + + nextS := s + var l int32 + var t int32 + for { + nextHashS := hashLen(cv, tableBits, hashShortBytes) + nextHashL := hash7(cv, tableBits) + s = nextS + nextS = s + doEvery + (s-nextEmit)>>skipLog + if nextS > sLimit { + goto emitRemainder + } + // Fetch a short+long candidate + sCandidate := e.table[nextHashS] + lCandidate := e.bTable[nextHashL] + next := load6432(src, nextS) + entry := tableEntry{offset: s + e.cur} + e.table[nextHashS] = entry + eLong := &e.bTable[nextHashL] + eLong.Cur, eLong.Prev = entry, eLong.Cur + + // Calculate hashes of 'next' + nextHashS = hashLen(next, tableBits, hashShortBytes) + nextHashL = hash7(next, tableBits) + + t = lCandidate.Cur.offset - e.cur + if s-t < maxMatchOffset { + if uint32(cv) == load3232(src, t) { + // Long candidate matches at least 4 bytes. + + // Store the next match + e.table[nextHashS] = tableEntry{offset: nextS + e.cur} + eLong := &e.bTable[nextHashL] + eLong.Cur, eLong.Prev = tableEntry{offset: nextS + e.cur}, eLong.Cur + + // Check the previous long candidate as well. + t2 := lCandidate.Prev.offset - e.cur + if s-t2 < maxMatchOffset && uint32(cv) == load3232(src, t2) { + l = e.matchlen(int(s+4), int(t+4), src) + 4 + ml1 := e.matchlen(int(s+4), int(t2+4), src) + 4 + if ml1 > l { + t = t2 + l = ml1 + break + } + } + break + } + // Current value did not match, but check if previous long value does. + t = lCandidate.Prev.offset - e.cur + if s-t < maxMatchOffset && uint32(cv) == load3232(src, t) { + // Store the next match + e.table[nextHashS] = tableEntry{offset: nextS + e.cur} + eLong := &e.bTable[nextHashL] + eLong.Cur, eLong.Prev = tableEntry{offset: nextS + e.cur}, eLong.Cur + break + } + } + + t = sCandidate.offset - e.cur + if s-t < maxMatchOffset && uint32(cv) == load3232(src, t) { + // Found a 4 match... + l = e.matchlen(int(s+4), int(t+4), src) + 4 + + // Look up next long candidate (at nextS) + lCandidate = e.bTable[nextHashL] + + // Store the next match + e.table[nextHashS] = tableEntry{offset: nextS + e.cur} + eLong := &e.bTable[nextHashL] + eLong.Cur, eLong.Prev = tableEntry{offset: nextS + e.cur}, eLong.Cur + + // Check repeat at s + repOff + const repOff = 1 + t2 := s - repeat + repOff + if load3232(src, t2) == uint32(cv>>(8*repOff)) { + ml := e.matchlen(int(s+4+repOff), int(t2+4), src) + 4 + if ml > l { + t = t2 + l = ml + s += repOff + // Not worth checking more. + break + } + } + + // If the next long is a candidate, use that... + t2 = lCandidate.Cur.offset - e.cur + if nextS-t2 < maxMatchOffset { + if load3232(src, t2) == uint32(next) { + ml := e.matchlen(int(nextS+4), int(t2+4), src) + 4 + if ml > l { + t = t2 + s = nextS + l = ml + // This is ok, but check previous as well. + } + } + // If the previous long is a candidate, use that... + t2 = lCandidate.Prev.offset - e.cur + if nextS-t2 < maxMatchOffset && load3232(src, t2) == uint32(next) { + ml := e.matchlen(int(nextS+4), int(t2+4), src) + 4 + if ml > l { + t = t2 + s = nextS + l = ml + break + } + } + } + break + } + cv = next + } + + // A 4-byte match has been found. We'll later see if more than 4 bytes + // match. But, prior to the match, src[nextEmit:s] are unmatched. Emit + // them as literal bytes. + + // Extend the 4-byte match as long as possible. + if l == 0 { + l = e.matchlenLong(int(s+4), int(t+4), src) + 4 + } else if l == maxMatchLength { + l += e.matchlenLong(int(s+l), int(t+l), src) + } + + // Try to locate a better match by checking the end-of-match... + if sAt := s + l; sAt < sLimit { + // Allow some bytes at the beginning to mismatch. + // Sweet spot is 2/3 bytes depending on input. + // 3 is only a little better when it is but sometimes a lot worse. + // The skipped bytes are tested in Extend backwards, + // and still picked up as part of the match if they do. + const skipBeginning = 2 + eLong := &e.bTable[hash7(load6432(src, sAt), tableBits)] + // Test current + t2 := eLong.Cur.offset - e.cur - l + skipBeginning + s2 := s + skipBeginning + off := s2 - t2 + if off < maxMatchOffset { + if off > 0 && t2 >= 0 { + if l2 := e.matchlenLong(int(s2), int(t2), src); l2 > l { + t = t2 + l = l2 + s = s2 + } + } + // Test next: + t2 = eLong.Prev.offset - e.cur - l + skipBeginning + off := s2 - t2 + if off > 0 && off < maxMatchOffset && t2 >= 0 { + if l2 := e.matchlenLong(int(s2), int(t2), src); l2 > l { + t = t2 + l = l2 + s = s2 + } + } + } + } + + // Extend backwards + for t > 0 && s > nextEmit && src[t-1] == src[s-1] { + s-- + t-- + l++ + } + if nextEmit < s { + if false { + emitLiteral(dst, src[nextEmit:s]) + } else { + for _, v := range src[nextEmit:s] { + dst.tokens[dst.n] = token(v) + dst.litHist[v]++ + dst.n++ + } + } + } + if false { + if t >= s { + panic(fmt.Sprintln("s-t", s, t)) + } + if (s - t) > maxMatchOffset { + panic(fmt.Sprintln("mmo", s-t)) + } + if l < baseMatchLength { + panic("bml") + } + } + + dst.AddMatchLong(l, uint32(s-t-baseMatchOffset)) + repeat = s - t + s += l + nextEmit = s + if nextS >= s { + s = nextS + 1 + } + + if s >= sLimit { + // Index after match end. + for i := nextS + 1; i < int32(len(src))-8; i += 2 { + cv := load6432(src, i) + e.table[hashLen(cv, tableBits, hashShortBytes)] = tableEntry{offset: i + e.cur} + eLong := &e.bTable[hash7(cv, tableBits)] + eLong.Cur, eLong.Prev = tableEntry{offset: i + e.cur}, eLong.Cur + } + goto emitRemainder + } + + // Store every long hash in-between and every second short. + if true { + for i := nextS + 1; i < s-1; i += 2 { + cv := load6432(src, i) + t := tableEntry{offset: i + e.cur} + t2 := tableEntry{offset: t.offset + 1} + eLong := &e.bTable[hash7(cv, tableBits)] + eLong2 := &e.bTable[hash7(cv>>8, tableBits)] + e.table[hashLen(cv, tableBits, hashShortBytes)] = t + eLong.Cur, eLong.Prev = t, eLong.Cur + eLong2.Cur, eLong2.Prev = t2, eLong2.Cur + } + } + + // We could immediately start working at s now, but to improve + // compression we first update the hash table at s-1 and at s. + cv = load6432(src, s) + } + +emitRemainder: + if int(nextEmit) < len(src) { + // If nothing was added, don't encode literals. + if dst.n == 0 { + return + } + + emitLiteral(dst, src[nextEmit:]) + } +} diff --git a/internal/compress/flate/matchlen_generic.go b/internal/compress/flate/matchlen_generic.go new file mode 100644 index 00000000..63c0637d --- /dev/null +++ b/internal/compress/flate/matchlen_generic.go @@ -0,0 +1,34 @@ +// Copyright 2019+ Klaus Post. All rights reserved. +// License information can be found in the LICENSE file. + +package flate + +import ( + "math/bits" + + "codeberg.org/lindenii/furgit/internal/compress/internal/le" +) + +// matchLen returns the maximum common prefix length of a and b. +// a must be the shortest of the two. +func matchLen(a, b []byte) (n int) { + left := len(a) + for left >= 8 { + diff := le.Load64(a, n) ^ le.Load64(b, n) + if diff != 0 { + return n + bits.TrailingZeros64(diff)>>3 + } + n += 8 + left -= 8 + } + + a = a[n:] + b = b[n:] + for i := range a { + if a[i] != b[i] { + break + } + n++ + } + return n +} diff --git a/internal/compress/flate/reader_test.go b/internal/compress/flate/reader_test.go new file mode 100644 index 00000000..6eedfb9b --- /dev/null +++ b/internal/compress/flate/reader_test.go @@ -0,0 +1,108 @@ +// Copyright 2012 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package flate + +import ( + "bytes" + "io" + "os" + "runtime" + "strings" + "testing" +) + +func TestNlitOutOfRange(t *testing.T) { + // Trying to decode this bogus flate data, which has a Huffman table + // with nlit=288, should not panic. + io.Copy(io.Discard, NewReader(strings.NewReader( + "\xfc\xfe\x36\xe7\x5e\x1c\xef\xb3\x55\x58\x77\xb6\x56\xb5\x43\xf4"+ + "\x6f\xf2\xd2\xe6\x3d\x99\xa0\x85\x8c\x48\xeb\xf8\xda\x83\x04\x2a"+ + "\x75\xc4\xf8\x0f\x12\x11\xb9\xb4\x4b\x09\xa0\xbe\x8b\x91\x4c"))) +} + +const ( + digits = iota + twain + random +) + +var testfiles = []string{ + // Digits is the digits of the irrational number e. Its decimal representation + // does not repeat, but there are only 10 possible digits, so it should be + // reasonably compressible. + digits: "../testdata/e.txt", + // Twain is Project Gutenberg's edition of Mark Twain's classic English novel. + twain: "../testdata/Mark.Twain-Tom.Sawyer.txt", + // Random bytes + random: "../testdata/sharnd.out", +} + +func benchmarkDecode(b *testing.B, testfile, level, n int) { + b.ReportAllocs() + b.StopTimer() + b.SetBytes(int64(n)) + buf0, err := os.ReadFile(testfiles[testfile]) + if err != nil { + b.Fatal(err) + } + if len(buf0) == 0 { + b.Fatalf("test file %q has no data", testfiles[testfile]) + } + compressed := new(bytes.Buffer) + w, err := NewWriter(compressed, level) + if err != nil { + b.Fatal(err) + } + for i := 0; i < n; i += len(buf0) { + if len(buf0) > n-i { + buf0 = buf0[:n-i] + } + io.Copy(w, bytes.NewReader(buf0)) + } + w.Close() + buf1 := compressed.Bytes() + buf0, compressed, w = nil, nil, nil + r := NewReader(bytes.NewReader(buf1)) + res := r.(Resetter) + runtime.GC() + b.StartTimer() + + for i := 0; i < b.N; i++ { + _ = res.Reset(bytes.NewReader(buf1), nil) + _, _ = io.Copy(io.Discard, r) + } +} + +// These short names are so that gofmt doesn't break the BenchmarkXxx function +// bodies below over multiple lines. +const ( + constant = ConstantCompression + speed = BestSpeed + default_ = DefaultCompression + compress = BestCompression + oneK = -1024 +) + +func BenchmarkDecodeDigitsSpeed1e4(b *testing.B) { benchmarkDecode(b, digits, speed, 1e4) } +func BenchmarkDecodeDigitsSpeed1e5(b *testing.B) { benchmarkDecode(b, digits, speed, 1e5) } +func BenchmarkDecodeDigitsSpeed1e6(b *testing.B) { benchmarkDecode(b, digits, speed, 1e6) } +func BenchmarkDecodeDigitsDefault1e4(b *testing.B) { benchmarkDecode(b, digits, default_, 1e4) } +func BenchmarkDecodeDigitsDefault1e5(b *testing.B) { benchmarkDecode(b, digits, default_, 1e5) } +func BenchmarkDecodeDigitsDefault1e6(b *testing.B) { benchmarkDecode(b, digits, default_, 1e6) } +func BenchmarkDecodeDigitsCompress1e4(b *testing.B) { benchmarkDecode(b, digits, compress, 1e4) } +func BenchmarkDecodeDigitsCompress1e5(b *testing.B) { benchmarkDecode(b, digits, compress, 1e5) } +func BenchmarkDecodeDigitsCompress1e6(b *testing.B) { benchmarkDecode(b, digits, compress, 1e6) } +func BenchmarkDecodeTwainSpeed1e4(b *testing.B) { benchmarkDecode(b, twain, speed, 1e4) } +func BenchmarkDecodeTwainSpeed1e5(b *testing.B) { benchmarkDecode(b, twain, speed, 1e5) } +func BenchmarkDecodeTwainSpeed1e6(b *testing.B) { benchmarkDecode(b, twain, speed, 1e6) } +func BenchmarkDecodeTwainDefault1e4(b *testing.B) { benchmarkDecode(b, twain, default_, 1e4) } +func BenchmarkDecodeTwainDefault1e5(b *testing.B) { benchmarkDecode(b, twain, default_, 1e5) } +func BenchmarkDecodeTwainDefault1e6(b *testing.B) { benchmarkDecode(b, twain, default_, 1e6) } +func BenchmarkDecodeTwainCompress1e4(b *testing.B) { benchmarkDecode(b, twain, compress, 1e4) } +func BenchmarkDecodeTwainCompress1e5(b *testing.B) { benchmarkDecode(b, twain, compress, 1e5) } +func BenchmarkDecodeTwainCompress1e6(b *testing.B) { benchmarkDecode(b, twain, compress, 1e6) } +func BenchmarkDecodeRandomSpeed1e4(b *testing.B) { benchmarkDecode(b, random, speed, 1e4) } +func BenchmarkDecodeRandomSpeed1e5(b *testing.B) { benchmarkDecode(b, random, speed, 1e5) } +func BenchmarkDecodeRandomSpeed1e6(b *testing.B) { benchmarkDecode(b, random, speed, 1e6) } diff --git a/internal/compress/flate/regmask_amd64.go b/internal/compress/flate/regmask_amd64.go new file mode 100644 index 00000000..6ed28061 --- /dev/null +++ b/internal/compress/flate/regmask_amd64.go @@ -0,0 +1,37 @@ +package flate + +const ( + // Masks for shifts with register sizes of the shift value. + // This can be used to work around the x86 design of shifting by mod register size. + // It can be used when a variable shift is always smaller than the register size. + + // reg8SizeMaskX - shift value is 8 bits, shifted is X + reg8SizeMask8 = 7 + reg8SizeMask16 = 15 + reg8SizeMask32 = 31 + reg8SizeMask64 = 63 + + // reg16SizeMaskX - shift value is 16 bits, shifted is X + reg16SizeMask8 = reg8SizeMask8 + reg16SizeMask16 = reg8SizeMask16 + reg16SizeMask32 = reg8SizeMask32 + reg16SizeMask64 = reg8SizeMask64 + + // reg32SizeMaskX - shift value is 32 bits, shifted is X + reg32SizeMask8 = reg8SizeMask8 + reg32SizeMask16 = reg8SizeMask16 + reg32SizeMask32 = reg8SizeMask32 + reg32SizeMask64 = reg8SizeMask64 + + // reg64SizeMaskX - shift value is 64 bits, shifted is X + reg64SizeMask8 = reg8SizeMask8 + reg64SizeMask16 = reg8SizeMask16 + reg64SizeMask32 = reg8SizeMask32 + reg64SizeMask64 = reg8SizeMask64 + + // regSizeMaskUintX - shift value is uint, shifted is X + regSizeMaskUint8 = reg8SizeMask8 + regSizeMaskUint16 = reg8SizeMask16 + regSizeMaskUint32 = reg8SizeMask32 + regSizeMaskUint64 = reg8SizeMask64 +) diff --git a/internal/compress/flate/regmask_other.go b/internal/compress/flate/regmask_other.go new file mode 100644 index 00000000..e62caf71 --- /dev/null +++ b/internal/compress/flate/regmask_other.go @@ -0,0 +1,39 @@ +//go:build !amd64 + +package flate + +const ( + // Masks for shifts with register sizes of the shift value. + // This can be used to work around the x86 design of shifting by mod register size. + // It can be used when a variable shift is always smaller than the register size. + + // reg8SizeMaskX - shift value is 8 bits, shifted is X + reg8SizeMask8 = 0xff + reg8SizeMask16 = 0xff + reg8SizeMask32 = 0xff + reg8SizeMask64 = 0xff + + // reg16SizeMaskX - shift value is 16 bits, shifted is X + reg16SizeMask8 = 0xffff + reg16SizeMask16 = 0xffff + reg16SizeMask32 = 0xffff + reg16SizeMask64 = 0xffff + + // reg32SizeMaskX - shift value is 32 bits, shifted is X + reg32SizeMask8 = 0xffffffff + reg32SizeMask16 = 0xffffffff + reg32SizeMask32 = 0xffffffff + reg32SizeMask64 = 0xffffffff + + // reg64SizeMaskX - shift value is 64 bits, shifted is X + reg64SizeMask8 = 0xffffffffffffffff + reg64SizeMask16 = 0xffffffffffffffff + reg64SizeMask32 = 0xffffffffffffffff + reg64SizeMask64 = 0xffffffffffffffff + + // regSizeMaskUintX - shift value is uint, shifted is X + regSizeMaskUint8 = ^uint(0) + regSizeMaskUint16 = ^uint(0) + regSizeMaskUint32 = ^uint(0) + regSizeMaskUint64 = ^uint(0) +) diff --git a/internal/compress/flate/stateless.go b/internal/compress/flate/stateless.go new file mode 100644 index 00000000..7e944bfb --- /dev/null +++ b/internal/compress/flate/stateless.go @@ -0,0 +1,325 @@ +package flate + +import ( + "io" + "math" + "sync" + + "codeberg.org/lindenii/furgit/internal/compress/internal/le" +) + +const ( + maxStatelessBlock = math.MaxInt16 + // dictionary will be taken from maxStatelessBlock, so limit it. + maxStatelessDict = 8 << 10 + + slTableBits = 13 + slTableSize = 1 << slTableBits + slTableShift = 32 - slTableBits +) + +type statelessWriter struct { + dst io.Writer + closed bool +} + +func (s *statelessWriter) Close() error { + if s.closed { + return nil + } + s.closed = true + // Emit EOF block + return StatelessDeflate(s.dst, nil, true, nil) +} + +func (s *statelessWriter) Write(p []byte) (n int, err error) { + err = StatelessDeflate(s.dst, p, false, nil) + if err != nil { + return 0, err + } + return len(p), nil +} + +func (s *statelessWriter) Reset(w io.Writer) { + s.dst = w + s.closed = false +} + +// NewStatelessWriter will do compression but without maintaining any state +// between Write calls. +// There will be no memory kept between Write calls, +// but compression and speed will be suboptimal. +// Because of this, the size of actual Write calls will affect output size. +func NewStatelessWriter(dst io.Writer) io.WriteCloser { + return &statelessWriter{dst: dst} +} + +// bitWriterPool contains bit writers that can be reused. +var bitWriterPool = sync.Pool{ + New: func() any { + return newHuffmanBitWriter(nil) + }, +} + +// tokensPool contains tokens struct objects that can be reused +var tokensPool = sync.Pool{ + New: func() any { + return &tokens{} + }, +} + +// StatelessDeflate allows compressing directly to a Writer without retaining state. +// When returning everything will be flushed. +// Up to 8KB of an optional dictionary can be given which is presumed to precede the block. +// Longer dictionaries will be truncated and will still produce valid output. +// Sending nil dictionary is perfectly fine. +func StatelessDeflate(out io.Writer, in []byte, eof bool, dict []byte) error { + bw := bitWriterPool.Get().(*huffmanBitWriter) + bw.reset(out) + defer func() { + // don't keep a reference to our output + bw.reset(nil) + bitWriterPool.Put(bw) + }() + if eof && len(in) == 0 { + // Just write an EOF block. + // Could be faster... + bw.writeStoredHeader(0, true) + bw.flush() + return bw.err + } + + // Truncate dict + if len(dict) > maxStatelessDict { + dict = dict[len(dict)-maxStatelessDict:] + } + + // For subsequent loops, keep shallow dict reference to avoid alloc+copy. + var inDict []byte + + dst := tokensPool.Get().(*tokens) + dst.Reset() + defer func() { + tokensPool.Put(dst) + }() + + for len(in) > 0 { + todo := in + if len(inDict) > 0 { + if len(todo) > maxStatelessBlock-maxStatelessDict { + todo = todo[:maxStatelessBlock-maxStatelessDict] + } + } else if len(todo) > maxStatelessBlock-len(dict) { + todo = todo[:maxStatelessBlock-len(dict)] + } + inOrg := in + in = in[len(todo):] + uncompressed := todo + if len(dict) > 0 { + // combine dict and source + bufLen := len(todo) + len(dict) + combined := make([]byte, bufLen) + copy(combined, dict) + copy(combined[len(dict):], todo) + todo = combined + } + // Compress + if len(inDict) == 0 { + statelessEnc(dst, todo, int16(len(dict))) + } else { + statelessEnc(dst, inDict[:maxStatelessDict+len(todo)], maxStatelessDict) + } + isEof := eof && len(in) == 0 + + if dst.n == 0 { + bw.writeStoredHeader(len(uncompressed), isEof) + if bw.err != nil { + return bw.err + } + bw.writeBytes(uncompressed) + } else if int(dst.n) > len(uncompressed)-len(uncompressed)>>4 { + // If we removed less than 1/16th, huffman compress the block. + bw.writeBlockHuff(isEof, uncompressed, len(in) == 0) + } else { + bw.writeBlockDynamic(dst, isEof, uncompressed, len(in) == 0) + } + if len(in) > 0 { + // Retain a dict if we have more + inDict = inOrg[len(uncompressed)-maxStatelessDict:] + dict = nil + dst.Reset() + } + if bw.err != nil { + return bw.err + } + } + if !eof { + // Align, only a stored block can do that. + bw.writeStoredHeader(0, false) + } + bw.flush() + return bw.err +} + +func hashSL(u uint32) uint32 { + return (u * 0x1e35a7bd) >> slTableShift +} + +func load3216(b []byte, i int16) uint32 { + return le.Load32(b, i) +} + +func load6416(b []byte, i int16) uint64 { + return le.Load64(b, i) +} + +func statelessEnc(dst *tokens, src []byte, startAt int16) { + const ( + inputMargin = 12 - 1 + minNonLiteralBlockSize = 1 + 1 + inputMargin + ) + + type tableEntry struct { + offset int16 + } + + var table [slTableSize]tableEntry + + // This check isn't in the Snappy implementation, but there, the caller + // instead of the callee handles this case. + if len(src)-int(startAt) < minNonLiteralBlockSize { + // We do not fill the token table. + // This will be picked up by caller. + dst.n = 0 + return + } + // Index until startAt + if startAt > 0 { + cv := load3232(src, 0) + for i := range startAt { + table[hashSL(cv)] = tableEntry{offset: i} + cv = (cv >> 8) | (uint32(src[i+4]) << 24) + } + } + + s := startAt + 1 + nextEmit := startAt + // sLimit is when to stop looking for offset/length copies. The inputMargin + // lets us use a fast path for emitLiteral in the main loop, while we are + // looking for copies. + sLimit := int16(len(src) - inputMargin) + + // nextEmit is where in src the next emitLiteral should start from. + cv := load3216(src, s) + + for { + const skipLog = 5 + const doEvery = 2 + + nextS := s + var candidate tableEntry + for { + nextHash := hashSL(cv) + candidate = table[nextHash] + nextS = s + doEvery + (s-nextEmit)>>skipLog + if nextS > sLimit || nextS <= 0 { + goto emitRemainder + } + + now := load6416(src, nextS) + table[nextHash] = tableEntry{offset: s} + nextHash = hashSL(uint32(now)) + + if cv == load3216(src, candidate.offset) { + table[nextHash] = tableEntry{offset: nextS} + break + } + + // Do one right away... + cv = uint32(now) + s = nextS + nextS++ + candidate = table[nextHash] + now >>= 8 + table[nextHash] = tableEntry{offset: s} + + if cv == load3216(src, candidate.offset) { + table[nextHash] = tableEntry{offset: nextS} + break + } + cv = uint32(now) + s = nextS + } + + // A 4-byte match has been found. We'll later see if more than 4 bytes + // match. But, prior to the match, src[nextEmit:s] are unmatched. Emit + // them as literal bytes. + for { + // Invariant: we have a 4-byte match at s, and no need to emit any + // literal bytes prior to s. + + // Extend the 4-byte match as long as possible. + t := candidate.offset + l := int16(matchLen(src[s+4:], src[t+4:]) + 4) + + // Extend backwards + for t > 0 && s > nextEmit && src[t-1] == src[s-1] { + s-- + t-- + l++ + } + if nextEmit < s { + if false { + emitLiteral(dst, src[nextEmit:s]) + } else { + for _, v := range src[nextEmit:s] { + dst.tokens[dst.n] = token(v) + dst.litHist[v]++ + dst.n++ + } + } + } + + // Save the match found + dst.AddMatchLong(int32(l), uint32(s-t-baseMatchOffset)) + s += l + nextEmit = s + if nextS >= s { + s = nextS + 1 + } + if s >= sLimit { + goto emitRemainder + } + + // We could immediately start working at s now, but to improve + // compression we first update the hash table at s-2 and at s. If + // another emitCopy is not our next move, also calculate nextHash + // at s+1. At least on GOARCH=amd64, these three hash calculations + // are faster as one load64 call (with some shifts) instead of + // three load32 calls. + x := load6416(src, s-2) + o := s - 2 + prevHash := hashSL(uint32(x)) + table[prevHash] = tableEntry{offset: o} + x >>= 16 + currHash := hashSL(uint32(x)) + candidate = table[currHash] + table[currHash] = tableEntry{offset: o + 2} + + if uint32(x) != load3216(src, candidate.offset) { + cv = uint32(x >> 8) + s++ + break + } + } + } + +emitRemainder: + if int(nextEmit) < len(src) { + // If nothing was added, don't encode literals. + if dst.n == 0 { + return + } + emitLiteral(dst, src[nextEmit:]) + } +} diff --git a/internal/compress/flate/testdata/fuzz/FuzzEncoding.zip b/internal/compress/flate/testdata/fuzz/FuzzEncoding.zip new file mode 100644 index 00000000..feae35f1 Binary files /dev/null and b/internal/compress/flate/testdata/fuzz/FuzzEncoding.zip differ diff --git a/internal/compress/flate/testdata/fuzz/encode-raw-corpus.zip b/internal/compress/flate/testdata/fuzz/encode-raw-corpus.zip new file mode 100644 index 00000000..7b33f54f Binary files /dev/null and b/internal/compress/flate/testdata/fuzz/encode-raw-corpus.zip differ diff --git a/internal/compress/flate/testdata/huffman-null-max.dyn.expect b/internal/compress/flate/testdata/huffman-null-max.dyn.expect new file mode 100644 index 00000000..c0816514 Binary files /dev/null and b/internal/compress/flate/testdata/huffman-null-max.dyn.expect differ diff --git a/internal/compress/flate/testdata/huffman-null-max.dyn.expect-noinput b/internal/compress/flate/testdata/huffman-null-max.dyn.expect-noinput new file mode 100644 index 00000000..c0816514 Binary files /dev/null and b/internal/compress/flate/testdata/huffman-null-max.dyn.expect-noinput differ diff --git a/internal/compress/flate/testdata/huffman-null-max.golden b/internal/compress/flate/testdata/huffman-null-max.golden new file mode 100644 index 00000000..db422ca3 Binary files /dev/null and b/internal/compress/flate/testdata/huffman-null-max.golden differ diff --git a/internal/compress/flate/testdata/huffman-null-max.in b/internal/compress/flate/testdata/huffman-null-max.in new file mode 100644 index 00000000..5dfddf07 Binary files /dev/null and b/internal/compress/flate/testdata/huffman-null-max.in differ diff --git a/internal/compress/flate/testdata/huffman-null-max.sync.expect b/internal/compress/flate/testdata/huffman-null-max.sync.expect new file mode 100644 index 00000000..c0816514 Binary files /dev/null and b/internal/compress/flate/testdata/huffman-null-max.sync.expect differ diff --git a/internal/compress/flate/testdata/huffman-null-max.sync.expect-noinput b/internal/compress/flate/testdata/huffman-null-max.sync.expect-noinput new file mode 100644 index 00000000..c0816514 Binary files /dev/null and b/internal/compress/flate/testdata/huffman-null-max.sync.expect-noinput differ diff --git a/internal/compress/flate/testdata/huffman-null-max.wb.expect b/internal/compress/flate/testdata/huffman-null-max.wb.expect new file mode 100644 index 00000000..c0816514 Binary files /dev/null and b/internal/compress/flate/testdata/huffman-null-max.wb.expect differ diff --git a/internal/compress/flate/testdata/huffman-null-max.wb.expect-noinput b/internal/compress/flate/testdata/huffman-null-max.wb.expect-noinput new file mode 100644 index 00000000..c0816514 Binary files /dev/null and b/internal/compress/flate/testdata/huffman-null-max.wb.expect-noinput differ diff --git a/internal/compress/flate/testdata/huffman-pi.dyn.expect b/internal/compress/flate/testdata/huffman-pi.dyn.expect new file mode 100644 index 00000000..e4396ac6 Binary files /dev/null and b/internal/compress/flate/testdata/huffman-pi.dyn.expect differ diff --git a/internal/compress/flate/testdata/huffman-pi.dyn.expect-noinput b/internal/compress/flate/testdata/huffman-pi.dyn.expect-noinput new file mode 100644 index 00000000..e4396ac6 Binary files /dev/null and b/internal/compress/flate/testdata/huffman-pi.dyn.expect-noinput differ diff --git a/internal/compress/flate/testdata/huffman-pi.golden b/internal/compress/flate/testdata/huffman-pi.golden new file mode 100644 index 00000000..23d8f7f9 Binary files /dev/null and b/internal/compress/flate/testdata/huffman-pi.golden differ diff --git a/internal/compress/flate/testdata/huffman-pi.in b/internal/compress/flate/testdata/huffman-pi.in new file mode 100644 index 00000000..efaed434 --- /dev/null +++ b/internal/compress/flate/testdata/huffman-pi.in @@ -0,0 +1 @@ +3.141592653589793238462643383279502884197169399375105820974944592307816406286208998628034825342117067982148086513282306647093844609550582231725359408128481117450284102701938521105559644622948954930381964428810975665933446128475648233786783165271201909145648566923460348610454326648213393607260249141273724587006606315588174881520920962829254091715364367892590360011330530548820466521384146951941511609433057270365759591953092186117381932611793105118548074462379962749567351885752724891227938183011949129833673362440656643086021394946395224737190702179860943702770539217176293176752384674818467669405132000568127145263560827785771342757789609173637178721468440901224953430146549585371050792279689258923542019956112129021960864034418159813629774771309960518707211349999998372978049951059731732816096318595024459455346908302642522308253344685035261931188171010003137838752886587533208381420617177669147303598253490428755468731159562863882353787593751957781857780532171226806613001927876611195909216420198938095257201065485863278865936153381827968230301952035301852968995773622599413891249721775283479131515574857242454150695950829533116861727855889075098381754637464939319255060400927701671139009848824012858361603563707660104710181942955596198946767837449448255379774726847104047534646208046684259069491293313677028989152104752162056966024058038150193511253382430035587640247496473263914199272604269922796782354781636009341721641219924586315030286182974555706749838505494588586926995690927210797509302955321165344987202755960236480665499119881834797753566369807426542527862551818417574672890977772793800081647060016145249192173217214772350141441973568548161361157352552133475741849468438523323907394143334547762416862518983569485562099219222184272550254256887671790494601653466804988627232791786085784383827967976681454100953883786360950680064225125205117392984896084128488626945604241965285022210661186306744278622039194945047123713786960956364371917287467764657573962413890865832645995813390478027590099465764078951269468398352595709825822620522489407726719478268482601476990902640136394437455305068203496252451749399651431429809190659250937221696461515709858387410597885959772975498930161753928468138268683868942774155991855925245953959431049972524680845987273644695848653836736222626099124608051243884390451244136549762780797715691435997700129616089441694868555848406353422072225828488648158456028506016842739452267467678895252138522549954666727823986456596116354886230577456498035593634568174324112515076069479451096596094025228879710893145669136867228748940560101503308617928680920874760917824938589009714909675985261365549781893129784821682998948722658804857564014270477555132379641451523746234364542858444795265867821051141354735739523113427166102135969536231442952484937187110145765403590279934403742007310578539062198387447808478489683321445713868751943506430218453191048481005370614680674919278191197939952061419663428754440643745123718192179998391015919561814675142691239748940907186494231961567945208095146550225231603881930142093762137855956638937787083039069792077346722182562599661501421503068038447734549202605414665925201497442850732518666002132434088190710486331734649651453905796268561005508106658796998163574736384052571459102897064140110971206280439039759515677157700420337869936007230558763176359421873125147120532928191826186125867321579198414848829164470609575270695722091756711672291098169091528017350671274858322287183520935396572512108357915136988209144421006751033467110314126711136990865851639831501970165151168517143765761835155650884909989859982387345528331635507647918535893226185489632132933089857064204675259070915481416549859461637180 \ No newline at end of file diff --git a/internal/compress/flate/testdata/huffman-pi.sync.expect b/internal/compress/flate/testdata/huffman-pi.sync.expect new file mode 100644 index 00000000..e4396ac6 Binary files /dev/null and b/internal/compress/flate/testdata/huffman-pi.sync.expect differ diff --git a/internal/compress/flate/testdata/huffman-pi.sync.expect-noinput b/internal/compress/flate/testdata/huffman-pi.sync.expect-noinput new file mode 100644 index 00000000..e4396ac6 Binary files /dev/null and b/internal/compress/flate/testdata/huffman-pi.sync.expect-noinput differ diff --git a/internal/compress/flate/testdata/huffman-pi.wb.expect b/internal/compress/flate/testdata/huffman-pi.wb.expect new file mode 100644 index 00000000..e4396ac6 Binary files /dev/null and b/internal/compress/flate/testdata/huffman-pi.wb.expect differ diff --git a/internal/compress/flate/testdata/huffman-pi.wb.expect-noinput b/internal/compress/flate/testdata/huffman-pi.wb.expect-noinput new file mode 100644 index 00000000..e4396ac6 Binary files /dev/null and b/internal/compress/flate/testdata/huffman-pi.wb.expect-noinput differ diff --git a/internal/compress/flate/testdata/huffman-rand-1k.dyn.expect b/internal/compress/flate/testdata/huffman-rand-1k.dyn.expect new file mode 100644 index 00000000..09dc798e Binary files /dev/null and b/internal/compress/flate/testdata/huffman-rand-1k.dyn.expect differ diff --git a/internal/compress/flate/testdata/huffman-rand-1k.dyn.expect-noinput b/internal/compress/flate/testdata/huffman-rand-1k.dyn.expect-noinput new file mode 100644 index 00000000..0c24742f Binary files /dev/null and b/internal/compress/flate/testdata/huffman-rand-1k.dyn.expect-noinput differ diff --git a/internal/compress/flate/testdata/huffman-rand-1k.golden b/internal/compress/flate/testdata/huffman-rand-1k.golden new file mode 100644 index 00000000..09dc798e Binary files /dev/null and b/internal/compress/flate/testdata/huffman-rand-1k.golden differ diff --git a/internal/compress/flate/testdata/huffman-rand-1k.in b/internal/compress/flate/testdata/huffman-rand-1k.in new file mode 100644 index 00000000..ce038ebb Binary files /dev/null and b/internal/compress/flate/testdata/huffman-rand-1k.in differ diff --git a/internal/compress/flate/testdata/huffman-rand-1k.sync.expect b/internal/compress/flate/testdata/huffman-rand-1k.sync.expect new file mode 100644 index 00000000..09dc798e Binary files /dev/null and b/internal/compress/flate/testdata/huffman-rand-1k.sync.expect differ diff --git a/internal/compress/flate/testdata/huffman-rand-1k.sync.expect-noinput b/internal/compress/flate/testdata/huffman-rand-1k.sync.expect-noinput new file mode 100644 index 00000000..0c24742f Binary files /dev/null and b/internal/compress/flate/testdata/huffman-rand-1k.sync.expect-noinput differ diff --git a/internal/compress/flate/testdata/huffman-rand-1k.wb.expect b/internal/compress/flate/testdata/huffman-rand-1k.wb.expect new file mode 100644 index 00000000..09dc798e Binary files /dev/null and b/internal/compress/flate/testdata/huffman-rand-1k.wb.expect differ diff --git a/internal/compress/flate/testdata/huffman-rand-1k.wb.expect-noinput b/internal/compress/flate/testdata/huffman-rand-1k.wb.expect-noinput new file mode 100644 index 00000000..0c24742f Binary files /dev/null and b/internal/compress/flate/testdata/huffman-rand-1k.wb.expect-noinput differ diff --git a/internal/compress/flate/testdata/huffman-rand-limit.dyn.expect b/internal/compress/flate/testdata/huffman-rand-limit.dyn.expect new file mode 100644 index 00000000..881e59c9 Binary files /dev/null and b/internal/compress/flate/testdata/huffman-rand-limit.dyn.expect differ diff --git a/internal/compress/flate/testdata/huffman-rand-limit.dyn.expect-noinput b/internal/compress/flate/testdata/huffman-rand-limit.dyn.expect-noinput new file mode 100644 index 00000000..881e59c9 Binary files /dev/null and b/internal/compress/flate/testdata/huffman-rand-limit.dyn.expect-noinput differ diff --git a/internal/compress/flate/testdata/huffman-rand-limit.golden b/internal/compress/flate/testdata/huffman-rand-limit.golden new file mode 100644 index 00000000..9ca0eb1c Binary files /dev/null and b/internal/compress/flate/testdata/huffman-rand-limit.golden differ diff --git a/internal/compress/flate/testdata/huffman-rand-limit.in b/internal/compress/flate/testdata/huffman-rand-limit.in new file mode 100644 index 00000000..fb5b1be6 --- /dev/null +++ b/internal/compress/flate/testdata/huffman-rand-limit.in @@ -0,0 +1,4 @@ +aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa +ø‹–vH +…”%€¯Âþè ë†É·ÅÞê}‹ç>ÚßÿlsÞÌçmIGH°èžò1YÞ4´[åà 0ˆ[|]o#© +¼-#¾Ùíul™ßýpfæîÙ±žnƒYÕÔ€Y˜w‰C8ɯ02š F=gn×ržN!OÆàÔ{¥ö›kÜ*“w(ý´bÚ ç«kQC9/ ’lu>ô5ýC.÷¤uÚê› diff --git a/internal/compress/flate/testdata/huffman-rand-limit.sync.expect b/internal/compress/flate/testdata/huffman-rand-limit.sync.expect new file mode 100644 index 00000000..881e59c9 Binary files /dev/null and b/internal/compress/flate/testdata/huffman-rand-limit.sync.expect differ diff --git a/internal/compress/flate/testdata/huffman-rand-limit.sync.expect-noinput b/internal/compress/flate/testdata/huffman-rand-limit.sync.expect-noinput new file mode 100644 index 00000000..881e59c9 Binary files /dev/null and b/internal/compress/flate/testdata/huffman-rand-limit.sync.expect-noinput differ diff --git a/internal/compress/flate/testdata/huffman-rand-limit.wb.expect b/internal/compress/flate/testdata/huffman-rand-limit.wb.expect new file mode 100644 index 00000000..881e59c9 Binary files /dev/null and b/internal/compress/flate/testdata/huffman-rand-limit.wb.expect differ diff --git a/internal/compress/flate/testdata/huffman-rand-limit.wb.expect-noinput b/internal/compress/flate/testdata/huffman-rand-limit.wb.expect-noinput new file mode 100644 index 00000000..881e59c9 Binary files /dev/null and b/internal/compress/flate/testdata/huffman-rand-limit.wb.expect-noinput differ diff --git a/internal/compress/flate/testdata/huffman-rand-max.golden b/internal/compress/flate/testdata/huffman-rand-max.golden new file mode 100644 index 00000000..47d53c89 Binary files /dev/null and b/internal/compress/flate/testdata/huffman-rand-max.golden differ diff --git a/internal/compress/flate/testdata/huffman-rand-max.in b/internal/compress/flate/testdata/huffman-rand-max.in new file mode 100644 index 00000000..8418633d Binary files /dev/null and b/internal/compress/flate/testdata/huffman-rand-max.in differ diff --git a/internal/compress/flate/testdata/huffman-shifts.dyn.expect b/internal/compress/flate/testdata/huffman-shifts.dyn.expect new file mode 100644 index 00000000..7812c1c6 Binary files /dev/null and b/internal/compress/flate/testdata/huffman-shifts.dyn.expect differ diff --git a/internal/compress/flate/testdata/huffman-shifts.dyn.expect-noinput b/internal/compress/flate/testdata/huffman-shifts.dyn.expect-noinput new file mode 100644 index 00000000..7812c1c6 Binary files /dev/null and b/internal/compress/flate/testdata/huffman-shifts.dyn.expect-noinput differ diff --git a/internal/compress/flate/testdata/huffman-shifts.golden b/internal/compress/flate/testdata/huffman-shifts.golden new file mode 100644 index 00000000..f5133778 Binary files /dev/null and b/internal/compress/flate/testdata/huffman-shifts.golden differ diff --git a/internal/compress/flate/testdata/huffman-shifts.in b/internal/compress/flate/testdata/huffman-shifts.in new file mode 100644 index 00000000..7c7a50d1 --- /dev/null +++ b/internal/compress/flate/testdata/huffman-shifts.in @@ -0,0 +1,2 @@ +101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010 +232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323 \ No newline at end of file diff --git a/internal/compress/flate/testdata/huffman-shifts.sync.expect b/internal/compress/flate/testdata/huffman-shifts.sync.expect new file mode 100644 index 00000000..7812c1c6 Binary files /dev/null and b/internal/compress/flate/testdata/huffman-shifts.sync.expect differ diff --git a/internal/compress/flate/testdata/huffman-shifts.sync.expect-noinput b/internal/compress/flate/testdata/huffman-shifts.sync.expect-noinput new file mode 100644 index 00000000..7812c1c6 Binary files /dev/null and b/internal/compress/flate/testdata/huffman-shifts.sync.expect-noinput differ diff --git a/internal/compress/flate/testdata/huffman-shifts.wb.expect b/internal/compress/flate/testdata/huffman-shifts.wb.expect new file mode 100644 index 00000000..7812c1c6 Binary files /dev/null and b/internal/compress/flate/testdata/huffman-shifts.wb.expect differ diff --git a/internal/compress/flate/testdata/huffman-shifts.wb.expect-noinput b/internal/compress/flate/testdata/huffman-shifts.wb.expect-noinput new file mode 100644 index 00000000..7812c1c6 Binary files /dev/null and b/internal/compress/flate/testdata/huffman-shifts.wb.expect-noinput differ diff --git a/internal/compress/flate/testdata/huffman-text-shift.dyn.expect b/internal/compress/flate/testdata/huffman-text-shift.dyn.expect new file mode 100644 index 00000000..71ce3aeb Binary files /dev/null and b/internal/compress/flate/testdata/huffman-text-shift.dyn.expect differ diff --git a/internal/compress/flate/testdata/huffman-text-shift.dyn.expect-noinput b/internal/compress/flate/testdata/huffman-text-shift.dyn.expect-noinput new file mode 100644 index 00000000..71ce3aeb Binary files /dev/null and b/internal/compress/flate/testdata/huffman-text-shift.dyn.expect-noinput differ diff --git a/internal/compress/flate/testdata/huffman-text-shift.golden b/internal/compress/flate/testdata/huffman-text-shift.golden new file mode 100644 index 00000000..ff023114 Binary files /dev/null and b/internal/compress/flate/testdata/huffman-text-shift.golden differ diff --git a/internal/compress/flate/testdata/huffman-text-shift.in b/internal/compress/flate/testdata/huffman-text-shift.in new file mode 100644 index 00000000..cc5c3ad6 --- /dev/null +++ b/internal/compress/flate/testdata/huffman-text-shift.in @@ -0,0 +1,14 @@ +//Copyright2009ThGoAuthor.Allrightrrvd. +//UofthiourccodigovrndbyBSD-tyl +//licnthtcnbfoundinthLICENSEfil. + +pckgmin + +import"o" + +funcmin(){ + vrb=mk([]byt,65535) + f,_:=o.Crt("huffmn-null-mx.in") + f.Writ(b) +} +ABCDEFGHIJKLMNOPQRSTUVXxyz!"#¤%&/?" \ No newline at end of file diff --git a/internal/compress/flate/testdata/huffman-text-shift.sync.expect b/internal/compress/flate/testdata/huffman-text-shift.sync.expect new file mode 100644 index 00000000..71ce3aeb Binary files /dev/null and b/internal/compress/flate/testdata/huffman-text-shift.sync.expect differ diff --git a/internal/compress/flate/testdata/huffman-text-shift.sync.expect-noinput b/internal/compress/flate/testdata/huffman-text-shift.sync.expect-noinput new file mode 100644 index 00000000..71ce3aeb Binary files /dev/null and b/internal/compress/flate/testdata/huffman-text-shift.sync.expect-noinput differ diff --git a/internal/compress/flate/testdata/huffman-text-shift.wb.expect b/internal/compress/flate/testdata/huffman-text-shift.wb.expect new file mode 100644 index 00000000..71ce3aeb Binary files /dev/null and b/internal/compress/flate/testdata/huffman-text-shift.wb.expect differ diff --git a/internal/compress/flate/testdata/huffman-text-shift.wb.expect-noinput b/internal/compress/flate/testdata/huffman-text-shift.wb.expect-noinput new file mode 100644 index 00000000..71ce3aeb Binary files /dev/null and b/internal/compress/flate/testdata/huffman-text-shift.wb.expect-noinput differ diff --git a/internal/compress/flate/testdata/huffman-text.dyn.expect b/internal/compress/flate/testdata/huffman-text.dyn.expect new file mode 100644 index 00000000..d448727c --- /dev/null +++ b/internal/compress/flate/testdata/huffman-text.dyn.expect @@ -0,0 +1 @@ +Ë_Kó0Åñëò½ê`KÇó0AasÄ›)^ˆHšþ²„¥IÉŸbß»¬—_>ç4 a˜¢=›Œ›Í-^ á1`_² 1 ìÃÌ ‘Å‘:ÁYÓà-‚F66!…A…Ž`Îa¤è©C;Aâþô°Nyr4ßœUä!™¡¤GKСøÖ#ÂóÓáør:B[G‚3Ω.òLè¥õ׶ýbFRuM]¼š­^⇳Å(#ZìÐË ÕŸí”i…›íöÿvÉÙB¯ð…»B‡H2S]™¢u/ýÚçÖ½üÖWóT¼G›©n—œýrö \ No newline at end of file diff --git a/internal/compress/flate/testdata/huffman-text.dyn.expect-noinput b/internal/compress/flate/testdata/huffman-text.dyn.expect-noinput new file mode 100644 index 00000000..d448727c --- /dev/null +++ b/internal/compress/flate/testdata/huffman-text.dyn.expect-noinput @@ -0,0 +1 @@ +Ë_Kó0Åñëò½ê`KÇó0AasÄ›)^ˆHšþ²„¥IÉŸbß»¬—_>ç4 a˜¢=›Œ›Í-^ á1`_² 1 ìÃÌ ‘Å‘:ÁYÓà-‚F66!…A…Ž`Îa¤è©C;Aâþô°Nyr4ßœUä!™¡¤GKСøÖ#ÂóÓáør:B[G‚3Ω.òLè¥õ׶ýbFRuM]¼š­^⇳Å(#ZìÐË ÕŸí”i…›íöÿvÉÙB¯ð…»B‡H2S]™¢u/ýÚçÖ½üÖWóT¼G›©n—œýrö \ No newline at end of file diff --git a/internal/compress/flate/testdata/huffman-text.golden b/internal/compress/flate/testdata/huffman-text.golden new file mode 100644 index 00000000..6d34c61f --- /dev/null +++ b/internal/compress/flate/testdata/huffman-text.golden @@ -0,0 +1,3 @@ +ÀAKó0ðóx¾ÃŸžZØÚñ¾LPØaÎ!‚x™âADÒöI–&#I‹EüîþšÇp]¢LÆ¿íö¯Fðp˜² 1Õ88‡h“¢$‰³ô5SÓà- ‚F66!…)v‚.ô›0„Y¢—í…ûóÃ&åÅ SÓÀÙN|d£2:åÑ +t˜|ë‘àùéxz9Ÿ ­“š‰éªº‹£²ž‰ÉŽ×3Š +&&=ù£²¾¬ðôšUD‹=Fu‘òã³]²¬q³ÛýßUL+½Æîö©>FQYÊÂLZÊoüäÜfTßµõEÅ´Òõ{´Yʶbúeú \ No newline at end of file diff --git a/internal/compress/flate/testdata/huffman-text.in b/internal/compress/flate/testdata/huffman-text.in new file mode 100644 index 00000000..73398b98 --- /dev/null +++ b/internal/compress/flate/testdata/huffman-text.in @@ -0,0 +1,13 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package main + +import "os" + +func main() { + var b = make([]byte, 65535) + f, _ := os.Create("huffman-null-max.in") + f.Write(b) +} diff --git a/internal/compress/flate/testdata/huffman-text.sync.expect b/internal/compress/flate/testdata/huffman-text.sync.expect new file mode 100644 index 00000000..d448727c --- /dev/null +++ b/internal/compress/flate/testdata/huffman-text.sync.expect @@ -0,0 +1 @@ +Ë_Kó0Åñëò½ê`KÇó0AasÄ›)^ˆHšþ²„¥IÉŸbß»¬—_>ç4 a˜¢=›Œ›Í-^ á1`_² 1 ìÃÌ ‘Å‘:ÁYÓà-‚F66!…A…Ž`Îa¤è©C;Aâþô°Nyr4ßœUä!™¡¤GKСøÖ#ÂóÓáør:B[G‚3Ω.òLè¥õ׶ýbFRuM]¼š­^⇳Å(#ZìÐË ÕŸí”i…›íöÿvÉÙB¯ð…»B‡H2S]™¢u/ýÚçÖ½üÖWóT¼G›©n—œýrö \ No newline at end of file diff --git a/internal/compress/flate/testdata/huffman-text.sync.expect-noinput b/internal/compress/flate/testdata/huffman-text.sync.expect-noinput new file mode 100644 index 00000000..d448727c --- /dev/null +++ b/internal/compress/flate/testdata/huffman-text.sync.expect-noinput @@ -0,0 +1 @@ +Ë_Kó0Åñëò½ê`KÇó0AasÄ›)^ˆHšþ²„¥IÉŸbß»¬—_>ç4 a˜¢=›Œ›Í-^ á1`_² 1 ìÃÌ ‘Å‘:ÁYÓà-‚F66!…A…Ž`Îa¤è©C;Aâþô°Nyr4ßœUä!™¡¤GKСøÖ#ÂóÓáør:B[G‚3Ω.òLè¥õ׶ýbFRuM]¼š­^⇳Å(#ZìÐË ÕŸí”i…›íöÿvÉÙB¯ð…»B‡H2S]™¢u/ýÚçÖ½üÖWóT¼G›©n—œýrö \ No newline at end of file diff --git a/internal/compress/flate/testdata/huffman-text.wb.expect b/internal/compress/flate/testdata/huffman-text.wb.expect new file mode 100644 index 00000000..d448727c --- /dev/null +++ b/internal/compress/flate/testdata/huffman-text.wb.expect @@ -0,0 +1 @@ +Ë_Kó0Åñëò½ê`KÇó0AasÄ›)^ˆHšþ²„¥IÉŸbß»¬—_>ç4 a˜¢=›Œ›Í-^ á1`_² 1 ìÃÌ ‘Å‘:ÁYÓà-‚F66!…A…Ž`Îa¤è©C;Aâþô°Nyr4ßœUä!™¡¤GKСøÖ#ÂóÓáør:B[G‚3Ω.òLè¥õ׶ýbFRuM]¼š­^⇳Å(#ZìÐË ÕŸí”i…›íöÿvÉÙB¯ð…»B‡H2S]™¢u/ýÚçÖ½üÖWóT¼G›©n—œýrö \ No newline at end of file diff --git a/internal/compress/flate/testdata/huffman-text.wb.expect-noinput b/internal/compress/flate/testdata/huffman-text.wb.expect-noinput new file mode 100644 index 00000000..d448727c --- /dev/null +++ b/internal/compress/flate/testdata/huffman-text.wb.expect-noinput @@ -0,0 +1 @@ +Ë_Kó0Åñëò½ê`KÇó0AasÄ›)^ˆHšþ²„¥IÉŸbß»¬—_>ç4 a˜¢=›Œ›Í-^ á1`_² 1 ìÃÌ ‘Å‘:ÁYÓà-‚F66!…A…Ž`Îa¤è©C;Aâþô°Nyr4ßœUä!™¡¤GKСøÖ#ÂóÓáør:B[G‚3Ω.òLè¥õ׶ýbFRuM]¼š­^⇳Å(#ZìÐË ÕŸí”i…›íöÿvÉÙB¯ð…»B‡H2S]™¢u/ýÚçÖ½üÖWóT¼G›©n—œýrö \ No newline at end of file diff --git a/internal/compress/flate/testdata/huffman-zero.dyn.expect b/internal/compress/flate/testdata/huffman-zero.dyn.expect new file mode 100644 index 00000000..dbe401c5 Binary files /dev/null and b/internal/compress/flate/testdata/huffman-zero.dyn.expect differ diff --git a/internal/compress/flate/testdata/huffman-zero.dyn.expect-noinput b/internal/compress/flate/testdata/huffman-zero.dyn.expect-noinput new file mode 100644 index 00000000..dbe401c5 Binary files /dev/null and b/internal/compress/flate/testdata/huffman-zero.dyn.expect-noinput differ diff --git a/internal/compress/flate/testdata/huffman-zero.golden b/internal/compress/flate/testdata/huffman-zero.golden new file mode 100644 index 00000000..5abdbaff Binary files /dev/null and b/internal/compress/flate/testdata/huffman-zero.golden differ diff --git a/internal/compress/flate/testdata/huffman-zero.in b/internal/compress/flate/testdata/huffman-zero.in new file mode 100644 index 00000000..349be0e6 --- /dev/null +++ b/internal/compress/flate/testdata/huffman-zero.in @@ -0,0 +1 @@ +00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000 \ No newline at end of file diff --git a/internal/compress/flate/testdata/huffman-zero.sync.expect b/internal/compress/flate/testdata/huffman-zero.sync.expect new file mode 100644 index 00000000..dbe401c5 Binary files /dev/null and b/internal/compress/flate/testdata/huffman-zero.sync.expect differ diff --git a/internal/compress/flate/testdata/huffman-zero.sync.expect-noinput b/internal/compress/flate/testdata/huffman-zero.sync.expect-noinput new file mode 100644 index 00000000..dbe401c5 Binary files /dev/null and b/internal/compress/flate/testdata/huffman-zero.sync.expect-noinput differ diff --git a/internal/compress/flate/testdata/huffman-zero.wb.expect b/internal/compress/flate/testdata/huffman-zero.wb.expect new file mode 100644 index 00000000..dbe401c5 Binary files /dev/null and b/internal/compress/flate/testdata/huffman-zero.wb.expect differ diff --git a/internal/compress/flate/testdata/huffman-zero.wb.expect-noinput b/internal/compress/flate/testdata/huffman-zero.wb.expect-noinput new file mode 100644 index 00000000..dbe401c5 Binary files /dev/null and b/internal/compress/flate/testdata/huffman-zero.wb.expect-noinput differ diff --git a/internal/compress/flate/testdata/null-long-match.dyn.expect-noinput b/internal/compress/flate/testdata/null-long-match.dyn.expect-noinput new file mode 100644 index 00000000..8b92d9fc Binary files /dev/null and b/internal/compress/flate/testdata/null-long-match.dyn.expect-noinput differ diff --git a/internal/compress/flate/testdata/null-long-match.sync.expect-noinput b/internal/compress/flate/testdata/null-long-match.sync.expect-noinput new file mode 100644 index 00000000..8b92d9fc Binary files /dev/null and b/internal/compress/flate/testdata/null-long-match.sync.expect-noinput differ diff --git a/internal/compress/flate/testdata/null-long-match.wb.expect-noinput b/internal/compress/flate/testdata/null-long-match.wb.expect-noinput new file mode 100644 index 00000000..8b92d9fc Binary files /dev/null and b/internal/compress/flate/testdata/null-long-match.wb.expect-noinput differ diff --git a/internal/compress/flate/testdata/partial-block b/internal/compress/flate/testdata/partial-block new file mode 100644 index 00000000..b14e816a --- /dev/null +++ b/internal/compress/flate/testdata/partial-block @@ -0,0 +1 @@ +ÊHÍÉÉ×Q(Ï/ÊI \ No newline at end of file diff --git a/internal/compress/flate/testdata/regression.zip b/internal/compress/flate/testdata/regression.zip new file mode 100644 index 00000000..73cf8403 Binary files /dev/null and b/internal/compress/flate/testdata/regression.zip differ diff --git a/internal/compress/flate/testdata/tokens.bin b/internal/compress/flate/testdata/tokens.bin new file mode 100644 index 00000000..b93c6968 --- /dev/null +++ b/internal/compress/flate/testdata/tokens.bin @@ -0,0 +1,63 @@ + + + Ž€€ˆname>Wikipú€‚뀈en.¶‚€‚¨€€„Ÿ‚€„€€‚/Main_PageMediaWiki 1.6alphaSpecial­€€À0" /É€€ª1">TalkÅ€€À2">User©€€À3©€€† tØ€€Æ4">Ü‚€‚”ƒ€„Ý€€À5®€€ Ê6">Image€À7ª€€ˆÚ€€Ê8µƒ€ˆ“€‚½€À9®€€ Ê10">Template À1Óƒ€‚®€€ˆ Ì2">HelpÞ€€Â3ª€€†Ú€€Ì4">Category½€Â5®€€Ž Ì00">PortŽ†€Ä101­€€Šà€€¦À€s·†€‚Òˆ€†õˆ€ŠAaA12005-12-27T18:46:47Z　‚Ÿ€€ŠØ€€„ 614213«€Ž쀂Ó€€ #REDIRECT [[AAA]]adding cur_id=5: {{R from CamelCase}}Ò€€‚®€€†—‚€„ƒ€ÎÔ‚€„a]]Ô€€¦§ƒ€ämericanSamoaÉ‚€‚­ƒ€”6ˆ‚€Ž­ƒ€°9Ôƒ€Ë€‚ƒ€Œ­ƒ€˜4:1õ…€ä­ƒ€¼to°ƒ€Š6 Û‚€¦°ƒ€èÝ‚€ˆ Þ‚€„·ƒ€”ppliedEthics·ƒ€œ8·ƒ€º5898943¦€€Ž·ƒ€š2-02-25T15:43:11·ƒ€Ðip>Conø“€ˆ script + <™”€‚áö€„€€‚Afgha’¡€‚anª£€†y132002-08-27T03:07:44ZMagnus¹÷€‚skewhoops³€€‚€€†×€€ˆ<Þ€‚ xml:space="õ¥€„rve">#REDIRECT [[¼‚€ˆ·‹€‚Ò‚€•ƒ€€Geo£È€„—ƒ€ ì€—ƒ€²9“‚€”÷‚€Ž—ƒ€†2-25T15:43:11—ƒ€Ðip>Con꽄 ¼‚criptmaxnumlit + offHist [32]uint16 // offset codes + litHist [256]uint16 // codes 0->255 + nFilled int + n uint16 // Must be able to contain maxStoreBlockSize + tokens [maxStoreBlockSize + 1]token +} + +func (t *tokens) Reset() { + if t.n == 0 { + return + } + t.n = 0 + t.nFilled = 0 + for i := range t.litHist[:] { + t.litHist[i] = 0 + } + for i := range t.extraHist[:] { + t.extraHist[i] = 0 + } + for i := range t.offHist[:] { + t.offHist[i] = 0 + } +} + +func (t *tokens) Fill() { + if t.n == 0 { + return + } + for i, v := range t.litHist[:] { + if v == 0 { + t.litHist[i] = 1 + t.nFilled++ + } + } + for i, v := range t.extraHist[:literalCount-256] { + if v == 0 { + t.nFilled++ + t.extraHist[i] = 1 + } + } + for i, v := range t.offHist[:offsetCodeCount] { + if v == 0 { + t.offHist[i] = 1 + } + } +} + +func indexTokens(in []token) tokens { + var t tokens + t.indexTokens(in) + return t +} + +func (t *tokens) indexTokens(in []token) { + t.Reset() + for _, tok := range in { + if tok < matchType { + t.AddLiteral(tok.literal()) + continue + } + t.AddMatch(uint32(tok.length()), tok.offset()&matchOffsetOnlyMask) + } +} + +// emitLiteral writes a literal chunk and returns the number of bytes written. +func emitLiteral(dst *tokens, lit []byte) { + for _, v := range lit { + dst.tokens[dst.n] = token(v) + dst.litHist[v]++ + dst.n++ + } +} + +func (t *tokens) AddLiteral(lit byte) { + t.tokens[t.n] = token(lit) + t.litHist[lit]++ + t.n++ +} + +// from https://stackoverflow.com/a/28730362 +func mFastLog2(val float32) float32 { + ux := int32(math.Float32bits(val)) + log2 := (float32)(((ux >> 23) & 255) - 128) + ux &= -0x7f800001 + ux += 127 << 23 + uval := math.Float32frombits(uint32(ux)) + log2 += ((-0.34484843)*uval+2.02466578)*uval - 0.67487759 + return log2 +} + +// EstimatedBits will return an minimum size estimated by an *optimal* +// compression of the block. +// The size of the block +func (t *tokens) EstimatedBits() int { + shannon := float32(0) + bits := int(0) + nMatches := 0 + total := int(t.n) + t.nFilled + if total > 0 { + invTotal := 1.0 / float32(total) + for _, v := range t.litHist[:] { + if v > 0 { + n := float32(v) + shannon += atLeastOne(-mFastLog2(n*invTotal)) * n + } + } + // Just add 15 for EOB + shannon += 15 + for i, v := range t.extraHist[1 : literalCount-256] { + if v > 0 { + n := float32(v) + shannon += atLeastOne(-mFastLog2(n*invTotal)) * n + bits += int(lengthExtraBits[i&31]) * int(v) + nMatches += int(v) + } + } + } + if nMatches > 0 { + invTotal := 1.0 / float32(nMatches) + for i, v := range t.offHist[:offsetCodeCount] { + if v > 0 { + n := float32(v) + shannon += atLeastOne(-mFastLog2(n*invTotal)) * n + bits += int(offsetExtraBits[i&31]) * int(v) + } + } + } + return int(shannon) + bits +} + +// AddMatch adds a match to the tokens. +// This function is very sensitive to inlining and right on the border. +func (t *tokens) AddMatch(xlength uint32, xoffset uint32) { + if debugDeflate { + if xlength >= maxMatchLength+baseMatchLength { + panic(fmt.Errorf("invalid length: %v", xlength)) + } + if xoffset >= maxMatchOffset+baseMatchOffset { + panic(fmt.Errorf("invalid offset: %v", xoffset)) + } + } + oCode := offsetCode(xoffset) + xoffset |= oCode << 16 + + t.extraHist[lengthCodes1[uint8(xlength)]]++ + t.offHist[oCode&31]++ + t.tokens[t.n] = token(matchType | xlength<= maxMatchOffset+baseMatchOffset { + panic(fmt.Errorf("invalid offset: %v", xoffset)) + } + } + oc := offsetCode(xoffset) + xoffset |= oc << 16 + for xlength > 0 { + xl := xlength + if xl > 258 { + // We need to have at least baseMatchLength left over for next loop. + if xl > 258+baseMatchLength { + xl = 258 + } else { + xl = 258 - baseMatchLength + } + } + xlength -= xl + xl -= baseMatchLength + t.extraHist[lengthCodes1[uint8(xl)]]++ + t.offHist[oc&31]++ + t.tokens[t.n] = token(matchType | uint32(xl)<> lengthShift) } + +// Convert length to code. +func lengthCode(len uint8) uint8 { return lengthCodes[len] } + +// Returns the offset code corresponding to a specific offset +func offsetCode(off uint32) uint32 { + if false { + if off < uint32(len(offsetCodes)) { + return offsetCodes[off&255] + } else if off>>7 < uint32(len(offsetCodes)) { + return offsetCodes[(off>>7)&255] + 14 + } else { + return offsetCodes[(off>>14)&255] + 28 + } + } + if off < uint32(len(offsetCodes)) { + return offsetCodes[uint8(off)] + } + return offsetCodes14[uint8(off>>7)] +} diff --git a/internal/compress/flate/token_test.go b/internal/compress/flate/token_test.go new file mode 100644 index 00000000..9070c341 --- /dev/null +++ b/internal/compress/flate/token_test.go @@ -0,0 +1,54 @@ +package flate + +import ( + "bytes" + "os" + "testing" +) + +type testFatal interface { + Fatal(args ...any) +} + +// loadTestTokens will load test tokens. +// First block from enwik9, varint encoded. +func loadTestTokens(t testFatal) *tokens { + b, err := os.ReadFile("testdata/tokens.bin") + if err != nil { + t.Fatal(err) + } + var tokens tokens + err = tokens.FromVarInt(b) + if err != nil { + t.Fatal(err) + } + return &tokens +} + +func Test_tokens_EstimatedBits(t *testing.T) { + tok := loadTestTokens(t) + // The estimated size, update if method changes. + const expect = 221057 + n := tok.EstimatedBits() + var buf bytes.Buffer + wr := newHuffmanBitWriter(&buf) + wr.writeBlockDynamic(tok, true, nil, true) + if wr.err != nil { + t.Fatal(wr.err) + } + wr.flush() + t.Log("got:", n, "actual:", buf.Len()*8, "(header not part of estimate)") + if n != expect { + t.Error("want:", expect, "bits, got:", n) + } +} + +func Benchmark_tokens_EstimatedBits(b *testing.B) { + tok := loadTestTokens(b) + b.ResetTimer() + // One "byte", one token iteration. + b.SetBytes(1) + for i := 0; i < b.N; i++ { + _ = tok.EstimatedBits() + } +} diff --git a/internal/compress/flate/writer_test.go b/internal/compress/flate/writer_test.go new file mode 100644 index 00000000..ea448298 --- /dev/null +++ b/internal/compress/flate/writer_test.go @@ -0,0 +1,544 @@ +// Copyright 2012 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package flate + +import ( + "archive/zip" + "bytes" + "compress/flate" + "fmt" + "io" + "math" + "math/rand" + "os" + "runtime" + "strconv" + "strings" + "testing" +) + +func TestWriterMemUsage(t *testing.T) { + testMem := func(t *testing.T, fn func()) { + var before, after runtime.MemStats + runtime.GC() + runtime.ReadMemStats(&before) + fn() + runtime.GC() + runtime.ReadMemStats(&after) + t.Logf("%s: Memory Used: %dKB, %d allocs", t.Name(), (after.HeapInuse-before.HeapInuse)/1024, after.HeapObjects-before.HeapObjects) + } + data := make([]byte, 100000) + t.Run("stateless", func(t *testing.T) { + testMem(t, func() { + StatelessDeflate(io.Discard, data, false, nil) + }) + }) + for level := HuffmanOnly; level <= BestCompression; level++ { + t.Run(fmt.Sprint("level-", level), func(t *testing.T) { + var zr *Writer + var err error + testMem(t, func() { + zr, err = NewWriter(io.Discard, level) + if err != nil { + t.Fatal(err) + } + zr.Write(data) + }) + zr.Close() + }) + } + for level := HuffmanOnly; level <= BestCompression; level++ { + t.Run(fmt.Sprint("stdlib-", level), func(t *testing.T) { + var zr *flate.Writer + var err error + testMem(t, func() { + zr, err = flate.NewWriter(io.Discard, level) + if err != nil { + t.Fatal(err) + } + zr.Write(data) + }) + zr.Close() + }) + } +} + +func TestWriterRegression(t *testing.T) { + data, err := os.ReadFile("testdata/regression.zip") + if err != nil { + t.Fatal(err) + } + for level := HuffmanOnly; level <= BestCompression; level++ { + t.Run(fmt.Sprint("level_", level), func(t *testing.T) { + zr, err := zip.NewReader(bytes.NewReader(data), int64(len(data))) + if err != nil { + t.Fatal(err) + } + + for _, tt := range zr.File { + if !strings.HasSuffix(t.Name(), "") { + continue + } + + t.Run(tt.Name, func(t *testing.T) { + if testing.Short() && tt.FileInfo().Size() > 10000 { + t.SkipNow() + } + r, err := tt.Open() + if err != nil { + t.Error(err) + return + } + in, err := io.ReadAll(r) + if err != nil { + t.Error(err) + } + msg := "level " + strconv.Itoa(level) + ":" + buf := new(bytes.Buffer) + fw, err := NewWriter(buf, level) + if err != nil { + t.Fatal(msg + err.Error()) + } + n, err := fw.Write(in) + if n != len(in) { + t.Fatal(msg + "short write") + } + if err != nil { + t.Fatal(msg + err.Error()) + } + err = fw.Close() + if err != nil { + t.Fatal(msg + err.Error()) + } + fr1 := NewReader(buf) + data2, err := io.ReadAll(fr1) + if err != nil { + t.Fatal(msg + err.Error()) + } + if !bytes.Equal(in, data2) { + t.Fatal(msg + "not equal") + } + // Do it again... + msg = "level " + strconv.Itoa(level) + " (reset):" + buf.Reset() + fw.Reset(buf) + n, err = fw.Write(in) + if n != len(in) { + t.Fatal(msg + "short write") + } + if err != nil { + t.Fatal(msg + err.Error()) + } + err = fw.Close() + if err != nil { + t.Fatal(msg + err.Error()) + } + fr1 = NewReader(buf) + data2, err = io.ReadAll(fr1) + if err != nil { + t.Fatal(msg + err.Error()) + } + if !bytes.Equal(in, data2) { + t.Fatal(msg + "not equal") + } + }) + } + }) + } +} + +func benchmarkEncoder(b *testing.B, testfile, level, n int) { + b.SetBytes(int64(n)) + buf0, err := os.ReadFile(testfiles[testfile]) + if err != nil { + b.Fatal(err) + } + if len(buf0) == 0 { + b.Fatalf("test file %q has no data", testfiles[testfile]) + } + buf1 := make([]byte, n) + for i := 0; i < n; i += len(buf0) { + if len(buf0) > n-i { + buf0 = buf0[:n-i] + } + copy(buf1[i:], buf0) + } + buf0 = nil + runtime.GC() + w, err := NewWriter(io.Discard, level) + if err != nil { + b.Fatal(err) + } + b.ResetTimer() + b.ReportAllocs() + for i := 0; i < b.N; i++ { + w.Reset(io.Discard) + _, err = w.Write(buf1) + if err != nil { + b.Fatal(err) + } + err = w.Close() + if err != nil { + b.Fatal(err) + } + } +} + +func BenchmarkEncodeDigitsConstant1e4(b *testing.B) { benchmarkEncoder(b, digits, constant, 1e4) } +func BenchmarkEncodeDigitsConstant1e5(b *testing.B) { benchmarkEncoder(b, digits, constant, 1e5) } +func BenchmarkEncodeDigitsConstant1e6(b *testing.B) { benchmarkEncoder(b, digits, constant, 1e6) } +func BenchmarkEncodeDigitsSpeed1e4(b *testing.B) { benchmarkEncoder(b, digits, speed, 1e4) } +func BenchmarkEncodeDigitsSpeed1e5(b *testing.B) { benchmarkEncoder(b, digits, speed, 1e5) } +func BenchmarkEncodeDigitsSpeed1e6(b *testing.B) { benchmarkEncoder(b, digits, speed, 1e6) } +func BenchmarkEncodeDigitsDefault1e4(b *testing.B) { benchmarkEncoder(b, digits, default_, 1e4) } +func BenchmarkEncodeDigitsDefault1e5(b *testing.B) { benchmarkEncoder(b, digits, default_, 1e5) } +func BenchmarkEncodeDigitsDefault1e6(b *testing.B) { benchmarkEncoder(b, digits, default_, 1e6) } +func BenchmarkEncodeDigitsCompress1e4(b *testing.B) { benchmarkEncoder(b, digits, compress, 1e4) } +func BenchmarkEncodeDigitsCompress1e5(b *testing.B) { benchmarkEncoder(b, digits, compress, 1e5) } +func BenchmarkEncodeDigitsCompress1e6(b *testing.B) { benchmarkEncoder(b, digits, compress, 1e6) } +func BenchmarkEncodeDigitsSL1e4(b *testing.B) { benchmarkStatelessEncoder(b, digits, 1e4) } +func BenchmarkEncodeDigitsSL1e5(b *testing.B) { benchmarkStatelessEncoder(b, digits, 1e5) } +func BenchmarkEncodeDigitsSL1e6(b *testing.B) { benchmarkStatelessEncoder(b, digits, 1e6) } +func BenchmarkEncodeTwainConstant1e4(b *testing.B) { benchmarkEncoder(b, twain, constant, 1e4) } +func BenchmarkEncodeTwainConstant1e5(b *testing.B) { benchmarkEncoder(b, twain, constant, 1e5) } +func BenchmarkEncodeTwainConstant1e6(b *testing.B) { benchmarkEncoder(b, twain, constant, 1e6) } +func BenchmarkEncodeTwainSpeed1e4(b *testing.B) { benchmarkEncoder(b, twain, speed, 1e4) } +func BenchmarkEncodeTwainSpeed1e5(b *testing.B) { benchmarkEncoder(b, twain, speed, 1e5) } +func BenchmarkEncodeTwainSpeed1e6(b *testing.B) { benchmarkEncoder(b, twain, speed, 1e6) } +func BenchmarkEncodeTwainDefault1e4(b *testing.B) { benchmarkEncoder(b, twain, default_, 1e4) } +func BenchmarkEncodeTwainDefault1e5(b *testing.B) { benchmarkEncoder(b, twain, default_, 1e5) } +func BenchmarkEncodeTwainDefault1e6(b *testing.B) { benchmarkEncoder(b, twain, default_, 1e6) } +func BenchmarkEncodeTwainCompress1e4(b *testing.B) { benchmarkEncoder(b, twain, compress, 1e4) } +func BenchmarkEncodeTwainCompress1e5(b *testing.B) { benchmarkEncoder(b, twain, compress, 1e5) } +func BenchmarkEncodeTwainCompress1e6(b *testing.B) { benchmarkEncoder(b, twain, compress, 1e6) } +func BenchmarkEncodeTwainSL1e4(b *testing.B) { benchmarkStatelessEncoder(b, twain, 1e4) } +func BenchmarkEncodeTwainSL1e5(b *testing.B) { benchmarkStatelessEncoder(b, twain, 1e5) } +func BenchmarkEncodeTwainSL1e6(b *testing.B) { benchmarkStatelessEncoder(b, twain, 1e6) } + +func BenchmarkEncodeTwain1024Win1e4(b *testing.B) { benchmarkEncoder(b, twain, oneK, 1e4) } +func BenchmarkEncodeTwain1024Win1e5(b *testing.B) { benchmarkEncoder(b, twain, oneK, 1e5) } +func BenchmarkEncodeTwain1024Win1e6(b *testing.B) { benchmarkEncoder(b, twain, oneK, 1e6) } + +func benchmarkStatelessEncoder(b *testing.B, testfile, n int) { + b.SetBytes(int64(n)) + buf0, err := os.ReadFile(testfiles[testfile]) + if err != nil { + b.Fatal(err) + } + if len(buf0) == 0 { + b.Fatalf("test file %q has no data", testfiles[testfile]) + } + buf1 := make([]byte, n) + for i := 0; i < n; i += len(buf0) { + if len(buf0) > n-i { + buf0 = buf0[:n-i] + } + copy(buf1[i:], buf0) + } + buf0 = nil + runtime.GC() + b.ResetTimer() + b.ReportAllocs() + for i := 0; i < b.N; i++ { + w := NewStatelessWriter(io.Discard) + _, err = w.Write(buf1) + if err != nil { + b.Fatal(err) + } + err = w.Close() + if err != nil { + b.Fatal(err) + } + } +} + +// A writer that fails after N writes. +type errorWriter struct { + N int +} + +func (e *errorWriter) Write(b []byte) (int, error) { + if e.N <= 0 { + return 0, io.ErrClosedPipe + } + e.N-- + return len(b), nil +} + +// Test if errors from the underlying writer is passed upwards. +func TestWriteError(t *testing.T) { + buf := new(bytes.Buffer) + n := 65536 + if !testing.Short() { + n *= 4 + } + for i := 0; i < n; i++ { + fmt.Fprintf(buf, "asdasfasf%d%dfghfgujyut%dyutyu\n", i, i, i) + } + in := buf.Bytes() + // We create our own buffer to control number of writes. + copyBuf := make([]byte, 128) + for l := range 10 { + for fail := 1; fail <= 256; fail *= 2 { + // Fail after 'fail' writes + ew := &errorWriter{N: fail} + w, err := NewWriter(ew, l) + if err != nil { + t.Fatalf("NewWriter: level %d: %v", l, err) + } + n, err := copyBuffer(w, bytes.NewBuffer(in), copyBuf) + if err == nil { + t.Fatalf("Level %d: Expected an error, writer was %#v", l, ew) + } + n2, err := w.Write([]byte{1, 2, 2, 3, 4, 5}) + if n2 != 0 { + t.Fatal("Level", l, "Expected 0 length write, got", n) + } + if err == nil { + t.Fatal("Level", l, "Expected an error") + } + err = w.Flush() + if err == nil { + t.Fatal("Level", l, "Expected an error on flush") + } + err = w.Close() + if err == nil { + t.Fatal("Level", l, "Expected an error on close") + } + + w.Reset(io.Discard) + n2, err = w.Write([]byte{1, 2, 3, 4, 5, 6}) + if err != nil { + t.Fatal("Level", l, "Got unexpected error after reset:", err) + } + if n2 == 0 { + t.Fatal("Level", l, "Got 0 length write, expected > 0") + } + if testing.Short() { + return + } + } + } +} + +// Test if errors from the underlying writer is passed upwards. +func TestWriter_Reset(t *testing.T) { + buf := new(bytes.Buffer) + n := 65536 + if !testing.Short() { + n *= 4 + } + for i := 0; i < n; i++ { + fmt.Fprintf(buf, "asdasfasf%d%dfghfgujyut%dyutyu\n", i, i, i) + } + in := buf.Bytes() + for l := range 10 { + if testing.Short() && l > 1 { + continue + } + t.Run(fmt.Sprintf("level-%d", l), func(t *testing.T) { + t.Parallel() + offset := 1 + if testing.Short() { + offset = 256 + } + for ; offset <= 256; offset *= 2 { + // Fail after 'fail' writes + w, err := NewWriter(io.Discard, l) + if err != nil { + t.Fatalf("NewWriter: level %d: %v", l, err) + } + if w.d.fast == nil { + t.Skip("Not Fast...") + return + } + for i := 0; i < (bufferReset-len(in)-offset-maxMatchOffset)/maxMatchOffset; i++ { + // skip ahead to where we are close to wrap around... + w.d.fast.Reset() + } + w.d.fast.Reset() + _, err = w.Write(in) + if err != nil { + t.Fatal(err) + } + for range 50 { + // skip ahead again... This should wrap around... + w.d.fast.Reset() + } + w.d.fast.Reset() + + _, err = w.Write(in) + if err != nil { + t.Fatal(err) + } + for range (math.MaxUint32 - bufferReset) / maxMatchOffset { + // skip ahead to where we are close to wrap around... + w.d.fast.Reset() + } + + _, err = w.Write(in) + if err != nil { + t.Fatal(err) + } + err = w.Close() + if err != nil { + t.Fatal(err) + } + } + }) + } +} + +func TestDeterministicL1(t *testing.T) { testDeterministic(1, t) } +func TestDeterministicL2(t *testing.T) { testDeterministic(2, t) } +func TestDeterministicL3(t *testing.T) { testDeterministic(3, t) } +func TestDeterministicL4(t *testing.T) { testDeterministic(4, t) } +func TestDeterministicL5(t *testing.T) { testDeterministic(5, t) } +func TestDeterministicL6(t *testing.T) { testDeterministic(6, t) } +func TestDeterministicL7(t *testing.T) { testDeterministic(7, t) } +func TestDeterministicL8(t *testing.T) { testDeterministic(8, t) } +func TestDeterministicL9(t *testing.T) { testDeterministic(9, t) } +func TestDeterministicL0(t *testing.T) { testDeterministic(0, t) } +func TestDeterministicLM2(t *testing.T) { testDeterministic(-2, t) } + +func testDeterministic(i int, t *testing.T) { + // Test so much we cross a good number of block boundaries. + var length = maxStoreBlockSize*30 + 500 + if testing.Short() { + length /= 10 + } + + // Create a random, but compressible stream. + rng := rand.New(rand.NewSource(1)) + t1 := make([]byte, length) + for i := range t1 { + t1[i] = byte(rng.Int63() & 7) + } + + // Do our first encode. + var b1 bytes.Buffer + br := bytes.NewBuffer(t1) + w, err := NewWriter(&b1, i) + if err != nil { + t.Fatal(err) + } + // Use a very small prime sized buffer. + cbuf := make([]byte, 787) + _, err = copyBuffer(w, br, cbuf) + if err != nil { + t.Fatal(err) + } + w.Close() + + // We choose a different buffer size, + // bigger than a maximum block, and also a prime. + var b2 bytes.Buffer + cbuf = make([]byte, 81761) + br2 := bytes.NewBuffer(t1) + w2, err := NewWriter(&b2, i) + if err != nil { + t.Fatal(err) + } + _, err = copyBuffer(w2, br2, cbuf) + if err != nil { + t.Fatal(err) + } + w2.Close() + + b1b := b1.Bytes() + b2b := b2.Bytes() + + if !bytes.Equal(b1b, b2b) { + t.Errorf("level %d did not produce deterministic result, result mismatch, len(a) = %d, len(b) = %d", i, len(b1b), len(b2b)) + } + + // Test using io.WriterTo interface. + var b3 bytes.Buffer + br = bytes.NewBuffer(t1) + w, err = NewWriter(&b3, i) + if err != nil { + t.Fatal(err) + } + _, err = br.WriteTo(w) + if err != nil { + t.Fatal(err) + } + w.Close() + + b3b := b3.Bytes() + if !bytes.Equal(b1b, b3b) { + t.Errorf("level %d (io.WriterTo) did not produce deterministic result, result mismatch, len(a) = %d, len(b) = %d", i, len(b1b), len(b3b)) + } +} + +// copyBuffer is a copy of io.CopyBuffer, since we want to support older go versions. +// This is modified to never use io.WriterTo or io.ReaderFrom interfaces. +func copyBuffer(dst io.Writer, src io.Reader, buf []byte) (written int64, err error) { + if buf == nil { + buf = make([]byte, 32*1024) + } + for { + nr, er := src.Read(buf) + if nr > 0 { + nw, ew := dst.Write(buf[0:nr]) + if nw > 0 { + written += int64(nw) + } + if ew != nil { + err = ew + break + } + if nr != nw { + err = io.ErrShortWrite + break + } + } + if er == io.EOF { + break + } + if er != nil { + err = er + break + } + } + return written, err +} + +func BenchmarkCompressAllocations(b *testing.B) { + payload := []byte(strings.Repeat("Tiny payload", 20)) + for j := -2; j <= 9; j++ { + b.Run("level("+strconv.Itoa(j)+")", func(b *testing.B) { + b.Run("flate", func(b *testing.B) { + b.ReportAllocs() + + for i := 0; i < b.N; i++ { + w, err := NewWriter(io.Discard, j) + if err != nil { + b.Fatal(err) + } + w.Write(payload) + w.Close() + } + }) + }) + } +} + +func BenchmarkCompressAllocationsSingle(b *testing.B) { + payload := []byte(strings.Repeat("Tiny payload", 20)) + const level = 2 + b.Run("flate", func(b *testing.B) { + b.ReportAllocs() + + for i := 0; i < b.N; i++ { + w, err := NewWriter(io.Discard, level) + if err != nil { + b.Fatal(err) + } + w.Write(payload) + w.Close() + } + }) +} diff --git a/internal/compress/internal/fuzz/helpers.go b/internal/compress/internal/fuzz/helpers.go new file mode 100644 index 00000000..ff32ce49 --- /dev/null +++ b/internal/compress/internal/fuzz/helpers.go @@ -0,0 +1,218 @@ +// Copyright (c) 2024+ Klaus Post. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package fuzz provides a way to add test cases to a testing.F instance from a zip file. +package fuzz + +import ( + "archive/zip" + "bytes" + "encoding/binary" + "fmt" + "go/ast" + "go/parser" + "go/token" + "io" + "os" + "strconv" + "testing" +) + +type InputType uint8 + +const ( + // TypeRaw indicates that files are raw bytes. + TypeRaw InputType = iota + // TypeGoFuzz indicates files are from Go Fuzzer. + TypeGoFuzz + // TypeOSSFuzz indicates that files are from OSS fuzzer with size before data. + TypeOSSFuzz +) + +// AddFromZip will read the supplied zip and add all as corpus for f. +// Byte slices only. +func AddFromZip(f *testing.F, filename string, t InputType, short bool) { + file, err := os.Open(filename) + if err != nil { + f.Fatal(err) + } + fi, err := file.Stat() + if fi == nil { + return + } + + if err != nil { + f.Fatal(err) + } + zr, err := zip.NewReader(file, fi.Size()) + if err != nil { + f.Fatal(err) + } + for i, file := range zr.File { + if short && i%10 != 0 { + continue + } + rc, err := file.Open() + if err != nil { + f.Fatal(err) + } + + b, err := io.ReadAll(rc) + if err != nil { + f.Fatal(err) + } + rc.Close() + t := t + if t == TypeOSSFuzz { + t = TypeRaw // Fallback + if len(b) >= 4 { + sz := binary.BigEndian.Uint32(b) + if sz <= uint32(len(b))-4 { + f.Add(b[4 : 4+sz]) + continue + } + } + } + + if bytes.HasPrefix(b, []byte("go test fuzz")) { + t = TypeGoFuzz + } else { + t = TypeRaw + } + + if t == TypeRaw { + f.Add(b) + continue + } + vals, err := unmarshalCorpusFile(b) + if err != nil { + f.Fatal(err) + } + for _, v := range vals { + f.Add(v) + } + } +} + +// ReturnFromZip will read the supplied zip and add all as corpus for f. +// Byte slices only. +func ReturnFromZip(tb testing.TB, filename string, t InputType, fn func([]byte)) { + file, err := os.Open(filename) + if err != nil { + tb.Fatal(err) + } + fi, err := file.Stat() + if fi == nil { + return + } + if err != nil { + tb.Fatal(err) + } + zr, err := zip.NewReader(file, fi.Size()) + if err != nil { + tb.Fatal(err) + } + for _, file := range zr.File { + rc, err := file.Open() + if err != nil { + tb.Fatal(err) + } + + b, err := io.ReadAll(rc) + if err != nil { + tb.Fatal(err) + } + rc.Close() + t := t + if t == TypeOSSFuzz { + t = TypeRaw // Fallback + if len(b) >= 4 { + sz := binary.BigEndian.Uint32(b) + if sz <= uint32(len(b))-4 { + fn(b[4 : 4+sz]) + continue + } + } + } + + if bytes.HasPrefix(b, []byte("go test fuzz")) { + t = TypeGoFuzz + } else { + t = TypeRaw + } + + if t == TypeRaw { + fn(b) + continue + } + vals, err := unmarshalCorpusFile(b) + if err != nil { + tb.Fatal(err) + } + for _, v := range vals { + fn(v) + } + } +} + +// unmarshalCorpusFile decodes corpus bytes into their respective values. +func unmarshalCorpusFile(b []byte) ([][]byte, error) { + if len(b) == 0 { + return nil, fmt.Errorf("cannot unmarshal empty string") + } + lines := bytes.Split(b, []byte("\n")) + if len(lines) < 2 { + return nil, fmt.Errorf("must include version and at least one value") + } + var vals = make([][]byte, 0, len(lines)-1) + for _, line := range lines[1:] { + line = bytes.TrimSpace(line) + if len(line) == 0 { + continue + } + v, err := parseCorpusValue(line) + if err != nil { + return nil, fmt.Errorf("malformed line %q: %v", line, err) + } + vals = append(vals, v) + } + return vals, nil +} + +// parseCorpusValue +func parseCorpusValue(line []byte) ([]byte, error) { + fs := token.NewFileSet() + expr, err := parser.ParseExprFrom(fs, "(test)", line, 0) + if err != nil { + return nil, err + } + call, ok := expr.(*ast.CallExpr) + if !ok { + return nil, fmt.Errorf("expected call expression") + } + if len(call.Args) != 1 { + return nil, fmt.Errorf("expected call expression with 1 argument; got %d", len(call.Args)) + } + arg := call.Args[0] + + if arrayType, ok := call.Fun.(*ast.ArrayType); ok { + if arrayType.Len != nil { + return nil, fmt.Errorf("expected []byte or primitive type") + } + elt, ok := arrayType.Elt.(*ast.Ident) + if !ok || elt.Name != "byte" { + return nil, fmt.Errorf("expected []byte") + } + lit, ok := arg.(*ast.BasicLit) + if !ok || lit.Kind != token.STRING { + return nil, fmt.Errorf("string literal required for type []byte") + } + s, err := strconv.Unquote(lit.Value) + if err != nil { + return nil, err + } + return []byte(s), nil + } + return nil, fmt.Errorf("expected []byte") +} diff --git a/internal/compress/internal/le/le.go b/internal/compress/internal/le/le.go new file mode 100644 index 00000000..e54909e1 --- /dev/null +++ b/internal/compress/internal/le/le.go @@ -0,0 +1,5 @@ +package le + +type Indexer interface { + int | int8 | int16 | int32 | int64 | uint | uint8 | uint16 | uint32 | uint64 +} diff --git a/internal/compress/internal/le/unsafe_disabled.go b/internal/compress/internal/le/unsafe_disabled.go new file mode 100644 index 00000000..4f2a0d8c --- /dev/null +++ b/internal/compress/internal/le/unsafe_disabled.go @@ -0,0 +1,42 @@ +//go:build !(amd64 || arm64 || ppc64le || riscv64) || nounsafe || purego || appengine + +package le + +import ( + "encoding/binary" +) + +// Load8 will load from b at index i. +func Load8[I Indexer](b []byte, i I) byte { + return b[i] +} + +// Load16 will load from b at index i. +func Load16[I Indexer](b []byte, i I) uint16 { + return binary.LittleEndian.Uint16(b[i:]) +} + +// Load32 will load from b at index i. +func Load32[I Indexer](b []byte, i I) uint32 { + return binary.LittleEndian.Uint32(b[i:]) +} + +// Load64 will load from b at index i. +func Load64[I Indexer](b []byte, i I) uint64 { + return binary.LittleEndian.Uint64(b[i:]) +} + +// Store16 will store v at b. +func Store16(b []byte, v uint16) { + binary.LittleEndian.PutUint16(b, v) +} + +// Store32 will store v at b. +func Store32(b []byte, v uint32) { + binary.LittleEndian.PutUint32(b, v) +} + +// Store64 will store v at b. +func Store64[I Indexer](b []byte, i I, v uint64) { + binary.LittleEndian.PutUint64(b[i:], v) +} diff --git a/internal/compress/internal/le/unsafe_enabled.go b/internal/compress/internal/le/unsafe_enabled.go new file mode 100644 index 00000000..218a38bc --- /dev/null +++ b/internal/compress/internal/le/unsafe_enabled.go @@ -0,0 +1,52 @@ +// We enable 64 bit LE platforms: + +//go:build (amd64 || arm64 || ppc64le || riscv64) && !nounsafe && !purego && !appengine + +package le + +import ( + "unsafe" +) + +// Load8 will load from b at index i. +func Load8[I Indexer](b []byte, i I) byte { + //return binary.LittleEndian.Uint16(b[i:]) + //return *(*uint16)(unsafe.Pointer(&b[i])) + return *(*byte)(unsafe.Add(unsafe.Pointer(unsafe.SliceData(b)), i)) +} + +// Load16 will load from b at index i. +func Load16[I Indexer](b []byte, i I) uint16 { + //return binary.LittleEndian.Uint16(b[i:]) + //return *(*uint16)(unsafe.Pointer(&b[i])) + return *(*uint16)(unsafe.Add(unsafe.Pointer(unsafe.SliceData(b)), i)) +} + +// Load32 will load from b at index i. +func Load32[I Indexer](b []byte, i I) uint32 { + //return binary.LittleEndian.Uint32(b[i:]) + //return *(*uint32)(unsafe.Pointer(&b[i])) + return *(*uint32)(unsafe.Add(unsafe.Pointer(unsafe.SliceData(b)), i)) +} + +// Load64 will load from b at index i. +func Load64[I Indexer](b []byte, i I) uint64 { + //return binary.LittleEndian.Uint64(b[i:]) + //return *(*uint64)(unsafe.Pointer(&b[i])) + return *(*uint64)(unsafe.Add(unsafe.Pointer(unsafe.SliceData(b)), i)) +} + +// Store16 will store v at b. +func Store16(b []byte, v uint16) { + *(*uint16)(unsafe.Pointer(unsafe.SliceData(b))) = v +} + +// Store32 will store v at b. +func Store32(b []byte, v uint32) { + *(*uint32)(unsafe.Pointer(unsafe.SliceData(b))) = v +} + +// Store64 will store v at b[i:]. +func Store64[I Indexer](b []byte, i I, v uint64) { + *(*uint64)(unsafe.Add(unsafe.Pointer(unsafe.SliceData(b)), i)) = v +} diff --git a/internal/compress/testdata/Mark.Twain-Tom.Sawyer.txt b/internal/compress/testdata/Mark.Twain-Tom.Sawyer.txt new file mode 100644 index 00000000..565627a9 --- /dev/null +++ b/internal/compress/testdata/Mark.Twain-Tom.Sawyer.txt @@ -0,0 +1,8472 @@ +Produced by David Widger. The previous edition was updated by Jose +Menendez. + + + + + + THE ADVENTURES OF TOM SAWYER + BY + MARK TWAIN + (Samuel Langhorne Clemens) + + + + + P R E F A C E + +MOST of the adventures recorded in this book really occurred; one or +two were experiences of my own, the rest those of boys who were +schoolmates of mine. Huck Finn is drawn from life; Tom Sawyer also, but +not from an individual--he is a combination of the characteristics of +three boys whom I knew, and therefore belongs to the composite order of +architecture. + +The odd superstitions touched upon were all prevalent among children +and slaves in the West at the period of this story--that is to say, +thirty or forty years ago. + +Although my book is intended mainly for the entertainment of boys and +girls, I hope it will not be shunned by men and women on that account, +for part of my plan has been to try to pleasantly remind adults of what +they once were themselves, and of how they felt and thought and talked, +and what queer enterprises they sometimes engaged in. + + THE AUTHOR. + +HARTFORD, 1876. + + + + T O M S A W Y E R + + + +CHAPTER I + +"TOM!" + +No answer. + +"TOM!" + +No answer. + +"What's gone with that boy, I wonder? You TOM!" + +No answer. + +The old lady pulled her spectacles down and looked over them about the +room; then she put them up and looked out under them. She seldom or +never looked THROUGH them for so small a thing as a boy; they were her +state pair, the pride of her heart, and were built for "style," not +service--she could have seen through a pair of stove-lids just as well. +She looked perplexed for a moment, and then said, not fiercely, but +still loud enough for the furniture to hear: + +"Well, I lay if I get hold of you I'll--" + +She did not finish, for by this time she was bending down and punching +under the bed with the broom, and so she needed breath to punctuate the +punches with. She resurrected nothing but the cat. + +"I never did see the beat of that boy!" + +She went to the open door and stood in it and looked out among the +tomato vines and "jimpson" weeds that constituted the garden. No Tom. +So she lifted up her voice at an angle calculated for distance and +shouted: + +"Y-o-u-u TOM!" + +There was a slight noise behind her and she turned just in time to +seize a small boy by the slack of his roundabout and arrest his flight. + +"There! I might 'a' thought of that closet. What you been doing in +there?" + +"Nothing." + +"Nothing! Look at your hands. And look at your mouth. What IS that +truck?" + +"I don't know, aunt." + +"Well, I know. It's jam--that's what it is. Forty times I've said if +you didn't let that jam alone I'd skin you. Hand me that switch." + +The switch hovered in the air--the peril was desperate-- + +"My! Look behind you, aunt!" + +The old lady whirled round, and snatched her skirts out of danger. The +lad fled on the instant, scrambled up the high board-fence, and +disappeared over it. + +His aunt Polly stood surprised a moment, and then broke into a gentle +laugh. + +"Hang the boy, can't I never learn anything? Ain't he played me tricks +enough like that for me to be looking out for him by this time? But old +fools is the biggest fools there is. Can't learn an old dog new tricks, +as the saying is. But my goodness, he never plays them alike, two days, +and how is a body to know what's coming? He 'pears to know just how +long he can torment me before I get my dander up, and he knows if he +can make out to put me off for a minute or make me laugh, it's all down +again and I can't hit him a lick. I ain't doing my duty by that boy, +and that's the Lord's truth, goodness knows. Spare the rod and spile +the child, as the Good Book says. I'm a laying up sin and suffering for +us both, I know. He's full of the Old Scratch, but laws-a-me! he's my +own dead sister's boy, poor thing, and I ain't got the heart to lash +him, somehow. Every time I let him off, my conscience does hurt me so, +and every time I hit him my old heart most breaks. Well-a-well, man +that is born of woman is of few days and full of trouble, as the +Scripture says, and I reckon it's so. He'll play hookey this evening, * +and [* Southwestern for "afternoon"] I'll just be obleeged to make him +work, to-morrow, to punish him. It's mighty hard to make him work +Saturdays, when all the boys is having holiday, but he hates work more +than he hates anything else, and I've GOT to do some of my duty by him, +or I'll be the ruination of the child." + +Tom did play hookey, and he had a very good time. He got back home +barely in season to help Jim, the small colored boy, saw next-day's +wood and split the kindlings before supper--at least he was there in +time to tell his adventures to Jim while Jim did three-fourths of the +work. Tom's younger brother (or rather half-brother) Sid was already +through with his part of the work (picking up chips), for he was a +quiet boy, and had no adventurous, troublesome ways. + +While Tom was eating his supper, and stealing sugar as opportunity +offered, Aunt Polly asked him questions that were full of guile, and +very deep--for she wanted to trap him into damaging revealments. Like +many other simple-hearted souls, it was her pet vanity to believe she +was endowed with a talent for dark and mysterious diplomacy, and she +loved to contemplate her most transparent devices as marvels of low +cunning. Said she: + +"Tom, it was middling warm in school, warn't it?" + +"Yes'm." + +"Powerful warm, warn't it?" + +"Yes'm." + +"Didn't you want to go in a-swimming, Tom?" + +A bit of a scare shot through Tom--a touch of uncomfortable suspicion. +He searched Aunt Polly's face, but it told him nothing. So he said: + +"No'm--well, not very much." + +The old lady reached out her hand and felt Tom's shirt, and said: + +"But you ain't too warm now, though." And it flattered her to reflect +that she had discovered that the shirt was dry without anybody knowing +that that was what she had in her mind. But in spite of her, Tom knew +where the wind lay, now. So he forestalled what might be the next move: + +"Some of us pumped on our heads--mine's damp yet. See?" + +Aunt Polly was vexed to think she had overlooked that bit of +circumstantial evidence, and missed a trick. Then she had a new +inspiration: + +"Tom, you didn't have to undo your shirt collar where I sewed it, to +pump on your head, did you? Unbutton your jacket!" + +The trouble vanished out of Tom's face. He opened his jacket. His +shirt collar was securely sewed. + +"Bother! Well, go 'long with you. I'd made sure you'd played hookey +and been a-swimming. But I forgive ye, Tom. I reckon you're a kind of a +singed cat, as the saying is--better'n you look. THIS time." + +She was half sorry her sagacity had miscarried, and half glad that Tom +had stumbled into obedient conduct for once. + +But Sidney said: + +"Well, now, if I didn't think you sewed his collar with white thread, +but it's black." + +"Why, I did sew it with white! Tom!" + +But Tom did not wait for the rest. As he went out at the door he said: + +"Siddy, I'll lick you for that." + +In a safe place Tom examined two large needles which were thrust into +the lapels of his jacket, and had thread bound about them--one needle +carried white thread and the other black. He said: + +"She'd never noticed if it hadn't been for Sid. Confound it! sometimes +she sews it with white, and sometimes she sews it with black. I wish to +geeminy she'd stick to one or t'other--I can't keep the run of 'em. But +I bet you I'll lam Sid for that. I'll learn him!" + +He was not the Model Boy of the village. He knew the model boy very +well though--and loathed him. + +Within two minutes, or even less, he had forgotten all his troubles. +Not because his troubles were one whit less heavy and bitter to him +than a man's are to a man, but because a new and powerful interest bore +them down and drove them out of his mind for the time--just as men's +misfortunes are forgotten in the excitement of new enterprises. This +new interest was a valued novelty in whistling, which he had just +acquired from a negro, and he was suffering to practise it undisturbed. +It consisted in a peculiar bird-like turn, a sort of liquid warble, +produced by touching the tongue to the roof of the mouth at short +intervals in the midst of the music--the reader probably remembers how +to do it, if he has ever been a boy. Diligence and attention soon gave +him the knack of it, and he strode down the street with his mouth full +of harmony and his soul full of gratitude. He felt much as an +astronomer feels who has discovered a new planet--no doubt, as far as +strong, deep, unalloyed pleasure is concerned, the advantage was with +the boy, not the astronomer. + +The summer evenings were long. It was not dark, yet. Presently Tom +checked his whistle. A stranger was before him--a boy a shade larger +than himself. A new-comer of any age or either sex was an impressive +curiosity in the poor little shabby village of St. Petersburg. This boy +was well dressed, too--well dressed on a week-day. This was simply +astounding. His cap was a dainty thing, his close-buttoned blue cloth +roundabout was new and natty, and so were his pantaloons. He had shoes +on--and it was only Friday. He even wore a necktie, a bright bit of +ribbon. He had a citified air about him that ate into Tom's vitals. The +more Tom stared at the splendid marvel, the higher he turned up his +nose at his finery and the shabbier and shabbier his own outfit seemed +to him to grow. Neither boy spoke. If one moved, the other moved--but +only sidewise, in a circle; they kept face to face and eye to eye all +the time. Finally Tom said: + +"I can lick you!" + +"I'd like to see you try it." + +"Well, I can do it." + +"No you can't, either." + +"Yes I can." + +"No you can't." + +"I can." + +"You can't." + +"Can!" + +"Can't!" + +An uncomfortable pause. Then Tom said: + +"What's your name?" + +"'Tisn't any of your business, maybe." + +"Well I 'low I'll MAKE it my business." + +"Well why don't you?" + +"If you say much, I will." + +"Much--much--MUCH. There now." + +"Oh, you think you're mighty smart, DON'T you? I could lick you with +one hand tied behind me, if I wanted to." + +"Well why don't you DO it? You SAY you can do it." + +"Well I WILL, if you fool with me." + +"Oh yes--I've seen whole families in the same fix." + +"Smarty! You think you're SOME, now, DON'T you? Oh, what a hat!" + +"You can lump that hat if you don't like it. I dare you to knock it +off--and anybody that'll take a dare will suck eggs." + +"You're a liar!" + +"You're another." + +"You're a fighting liar and dasn't take it up." + +"Aw--take a walk!" + +"Say--if you give me much more of your sass I'll take and bounce a +rock off'n your head." + +"Oh, of COURSE you will." + +"Well I WILL." + +"Well why don't you DO it then? What do you keep SAYING you will for? +Why don't you DO it? It's because you're afraid." + +"I AIN'T afraid." + +"You are." + +"I ain't." + +"You are." + +Another pause, and more eying and sidling around each other. Presently +they were shoulder to shoulder. Tom said: + +"Get away from here!" + +"Go away yourself!" + +"I won't." + +"I won't either." + +So they stood, each with a foot placed at an angle as a brace, and +both shoving with might and main, and glowering at each other with +hate. But neither could get an advantage. After struggling till both +were hot and flushed, each relaxed his strain with watchful caution, +and Tom said: + +"You're a coward and a pup. I'll tell my big brother on you, and he +can thrash you with his little finger, and I'll make him do it, too." + +"What do I care for your big brother? I've got a brother that's bigger +than he is--and what's more, he can throw him over that fence, too." +[Both brothers were imaginary.] + +"That's a lie." + +"YOUR saying so don't make it so." + +Tom drew a line in the dust with his big toe, and said: + +"I dare you to step over that, and I'll lick you till you can't stand +up. Anybody that'll take a dare will steal sheep." + +The new boy stepped over promptly, and said: + +"Now you said you'd do it, now let's see you do it." + +"Don't you crowd me now; you better look out." + +"Well, you SAID you'd do it--why don't you do it?" + +"By jingo! for two cents I WILL do it." + +The new boy took two broad coppers out of his pocket and held them out +with derision. Tom struck them to the ground. In an instant both boys +were rolling and tumbling in the dirt, gripped together like cats; and +for the space of a minute they tugged and tore at each other's hair and +clothes, punched and scratched each other's nose, and covered +themselves with dust and glory. Presently the confusion took form, and +through the fog of battle Tom appeared, seated astride the new boy, and +pounding him with his fists. "Holler 'nuff!" said he. + +The boy only struggled to free himself. He was crying--mainly from rage. + +"Holler 'nuff!"--and the pounding went on. + +At last the stranger got out a smothered "'Nuff!" and Tom let him up +and said: + +"Now that'll learn you. Better look out who you're fooling with next +time." + +The new boy went off brushing the dust from his clothes, sobbing, +snuffling, and occasionally looking back and shaking his head and +threatening what he would do to Tom the "next time he caught him out." +To which Tom responded with jeers, and started off in high feather, and +as soon as his back was turned the new boy snatched up a stone, threw +it and hit him between the shoulders and then turned tail and ran like +an antelope. Tom chased the traitor home, and thus found out where he +lived. He then held a position at the gate for some time, daring the +enemy to come outside, but the enemy only made faces at him through the +window and declined. At last the enemy's mother appeared, and called +Tom a bad, vicious, vulgar child, and ordered him away. So he went +away; but he said he "'lowed" to "lay" for that boy. + +He got home pretty late that night, and when he climbed cautiously in +at the window, he uncovered an ambuscade, in the person of his aunt; +and when she saw the state his clothes were in her resolution to turn +his Saturday holiday into captivity at hard labor became adamantine in +its firmness. + + + +CHAPTER II + +SATURDAY morning was come, and all the summer world was bright and +fresh, and brimming with life. There was a song in every heart; and if +the heart was young the music issued at the lips. There was cheer in +every face and a spring in every step. The locust-trees were in bloom +and the fragrance of the blossoms filled the air. Cardiff Hill, beyond +the village and above it, was green with vegetation and it lay just far +enough away to seem a Delectable Land, dreamy, reposeful, and inviting. + +Tom appeared on the sidewalk with a bucket of whitewash and a +long-handled brush. He surveyed the fence, and all gladness left him and +a deep melancholy settled down upon his spirit. Thirty yards of board +fence nine feet high. Life to him seemed hollow, and existence but a +burden. Sighing, he dipped his brush and passed it along the topmost +plank; repeated the operation; did it again; compared the insignificant +whitewashed streak with the far-reaching continent of unwhitewashed +fence, and sat down on a tree-box discouraged. Jim came skipping out at +the gate with a tin pail, and singing Buffalo Gals. Bringing water from +the town pump had always been hateful work in Tom's eyes, before, but +now it did not strike him so. He remembered that there was company at +the pump. White, mulatto, and negro boys and girls were always there +waiting their turns, resting, trading playthings, quarrelling, +fighting, skylarking. And he remembered that although the pump was only +a hundred and fifty yards off, Jim never got back with a bucket of +water under an hour--and even then somebody generally had to go after +him. Tom said: + +"Say, Jim, I'll fetch the water if you'll whitewash some." + +Jim shook his head and said: + +"Can't, Mars Tom. Ole missis, she tole me I got to go an' git dis +water an' not stop foolin' roun' wid anybody. She say she spec' Mars +Tom gwine to ax me to whitewash, an' so she tole me go 'long an' 'tend +to my own business--she 'lowed SHE'D 'tend to de whitewashin'." + +"Oh, never you mind what she said, Jim. That's the way she always +talks. Gimme the bucket--I won't be gone only a a minute. SHE won't +ever know." + +"Oh, I dasn't, Mars Tom. Ole missis she'd take an' tar de head off'n +me. 'Deed she would." + +"SHE! She never licks anybody--whacks 'em over the head with her +thimble--and who cares for that, I'd like to know. She talks awful, but +talk don't hurt--anyways it don't if she don't cry. Jim, I'll give you +a marvel. I'll give you a white alley!" + +Jim began to waver. + +"White alley, Jim! And it's a bully taw." + +"My! Dat's a mighty gay marvel, I tell you! But Mars Tom I's powerful +'fraid ole missis--" + +"And besides, if you will I'll show you my sore toe." + +Jim was only human--this attraction was too much for him. He put down +his pail, took the white alley, and bent over the toe with absorbing +interest while the bandage was being unwound. In another moment he was +flying down the street with his pail and a tingling rear, Tom was +whitewashing with vigor, and Aunt Polly was retiring from the field +with a slipper in her hand and triumph in her eye. + +But Tom's energy did not last. He began to think of the fun he had +planned for this day, and his sorrows multiplied. Soon the free boys +would come tripping along on all sorts of delicious expeditions, and +they would make a world of fun of him for having to work--the very +thought of it burnt him like fire. He got out his worldly wealth and +examined it--bits of toys, marbles, and trash; enough to buy an +exchange of WORK, maybe, but not half enough to buy so much as half an +hour of pure freedom. So he returned his straitened means to his +pocket, and gave up the idea of trying to buy the boys. At this dark +and hopeless moment an inspiration burst upon him! Nothing less than a +great, magnificent inspiration. + +He took up his brush and went tranquilly to work. Ben Rogers hove in +sight presently--the very boy, of all boys, whose ridicule he had been +dreading. Ben's gait was the hop-skip-and-jump--proof enough that his +heart was light and his anticipations high. He was eating an apple, and +giving a long, melodious whoop, at intervals, followed by a deep-toned +ding-dong-dong, ding-dong-dong, for he was personating a steamboat. As +he drew near, he slackened speed, took the middle of the street, leaned +far over to starboard and rounded to ponderously and with laborious +pomp and circumstance--for he was personating the Big Missouri, and +considered himself to be drawing nine feet of water. He was boat and +captain and engine-bells combined, so he had to imagine himself +standing on his own hurricane-deck giving the orders and executing them: + +"Stop her, sir! Ting-a-ling-ling!" The headway ran almost out, and he +drew up slowly toward the sidewalk. + +"Ship up to back! Ting-a-ling-ling!" His arms straightened and +stiffened down his sides. + +"Set her back on the stabboard! Ting-a-ling-ling! Chow! ch-chow-wow! +Chow!" His right hand, meantime, describing stately circles--for it was +representing a forty-foot wheel. + +"Let her go back on the labboard! Ting-a-lingling! Chow-ch-chow-chow!" +The left hand began to describe circles. + +"Stop the stabboard! Ting-a-ling-ling! Stop the labboard! Come ahead +on the stabboard! Stop her! Let your outside turn over slow! +Ting-a-ling-ling! Chow-ow-ow! Get out that head-line! LIVELY now! +Come--out with your spring-line--what're you about there! Take a turn +round that stump with the bight of it! Stand by that stage, now--let her +go! Done with the engines, sir! Ting-a-ling-ling! SH'T! S'H'T! SH'T!" +(trying the gauge-cocks). + +Tom went on whitewashing--paid no attention to the steamboat. Ben +stared a moment and then said: "Hi-YI! YOU'RE up a stump, ain't you!" + +No answer. Tom surveyed his last touch with the eye of an artist, then +he gave his brush another gentle sweep and surveyed the result, as +before. Ben ranged up alongside of him. Tom's mouth watered for the +apple, but he stuck to his work. Ben said: + +"Hello, old chap, you got to work, hey?" + +Tom wheeled suddenly and said: + +"Why, it's you, Ben! I warn't noticing." + +"Say--I'm going in a-swimming, I am. Don't you wish you could? But of +course you'd druther WORK--wouldn't you? Course you would!" + +Tom contemplated the boy a bit, and said: + +"What do you call work?" + +"Why, ain't THAT work?" + +Tom resumed his whitewashing, and answered carelessly: + +"Well, maybe it is, and maybe it ain't. All I know, is, it suits Tom +Sawyer." + +"Oh come, now, you don't mean to let on that you LIKE it?" + +The brush continued to move. + +"Like it? Well, I don't see why I oughtn't to like it. Does a boy get +a chance to whitewash a fence every day?" + +That put the thing in a new light. Ben stopped nibbling his apple. Tom +swept his brush daintily back and forth--stepped back to note the +effect--added a touch here and there--criticised the effect again--Ben +watching every move and getting more and more interested, more and more +absorbed. Presently he said: + +"Say, Tom, let ME whitewash a little." + +Tom considered, was about to consent; but he altered his mind: + +"No--no--I reckon it wouldn't hardly do, Ben. You see, Aunt Polly's +awful particular about this fence--right here on the street, you know +--but if it was the back fence I wouldn't mind and SHE wouldn't. Yes, +she's awful particular about this fence; it's got to be done very +careful; I reckon there ain't one boy in a thousand, maybe two +thousand, that can do it the way it's got to be done." + +"No--is that so? Oh come, now--lemme just try. Only just a little--I'd +let YOU, if you was me, Tom." + +"Ben, I'd like to, honest injun; but Aunt Polly--well, Jim wanted to +do it, but she wouldn't let him; Sid wanted to do it, and she wouldn't +let Sid. Now don't you see how I'm fixed? If you was to tackle this +fence and anything was to happen to it--" + +"Oh, shucks, I'll be just as careful. Now lemme try. Say--I'll give +you the core of my apple." + +"Well, here--No, Ben, now don't. I'm afeard--" + +"I'll give you ALL of it!" + +Tom gave up the brush with reluctance in his face, but alacrity in his +heart. And while the late steamer Big Missouri worked and sweated in +the sun, the retired artist sat on a barrel in the shade close by, +dangled his legs, munched his apple, and planned the slaughter of more +innocents. There was no lack of material; boys happened along every +little while; they came to jeer, but remained to whitewash. By the time +Ben was fagged out, Tom had traded the next chance to Billy Fisher for +a kite, in good repair; and when he played out, Johnny Miller bought in +for a dead rat and a string to swing it with--and so on, and so on, +hour after hour. And when the middle of the afternoon came, from being +a poor poverty-stricken boy in the morning, Tom was literally rolling +in wealth. He had besides the things before mentioned, twelve marbles, +part of a jews-harp, a piece of blue bottle-glass to look through, a +spool cannon, a key that wouldn't unlock anything, a fragment of chalk, +a glass stopper of a decanter, a tin soldier, a couple of tadpoles, six +fire-crackers, a kitten with only one eye, a brass doorknob, a +dog-collar--but no dog--the handle of a knife, four pieces of +orange-peel, and a dilapidated old window sash. + +He had had a nice, good, idle time all the while--plenty of company +--and the fence had three coats of whitewash on it! If he hadn't run out +of whitewash he would have bankrupted every boy in the village. + +Tom said to himself that it was not such a hollow world, after all. He +had discovered a great law of human action, without knowing it--namely, +that in order to make a man or a boy covet a thing, it is only +necessary to make the thing difficult to attain. If he had been a great +and wise philosopher, like the writer of this book, he would now have +comprehended that Work consists of whatever a body is OBLIGED to do, +and that Play consists of whatever a body is not obliged to do. And +this would help him to understand why constructing artificial flowers +or performing on a tread-mill is work, while rolling ten-pins or +climbing Mont Blanc is only amusement. There are wealthy gentlemen in +England who drive four-horse passenger-coaches twenty or thirty miles +on a daily line, in the summer, because the privilege costs them +considerable money; but if they were offered wages for the service, +that would turn it into work and then they would resign. + +The boy mused awhile over the substantial change which had taken place +in his worldly circumstances, and then wended toward headquarters to +report. + + + +CHAPTER III + +TOM presented himself before Aunt Polly, who was sitting by an open +window in a pleasant rearward apartment, which was bedroom, +breakfast-room, dining-room, and library, combined. The balmy summer +air, the restful quiet, the odor of the flowers, and the drowsing murmur +of the bees had had their effect, and she was nodding over her knitting +--for she had no company but the cat, and it was asleep in her lap. Her +spectacles were propped up on her gray head for safety. She had thought +that of course Tom had deserted long ago, and she wondered at seeing him +place himself in her power again in this intrepid way. He said: "Mayn't +I go and play now, aunt?" + +"What, a'ready? How much have you done?" + +"It's all done, aunt." + +"Tom, don't lie to me--I can't bear it." + +"I ain't, aunt; it IS all done." + +Aunt Polly placed small trust in such evidence. She went out to see +for herself; and she would have been content to find twenty per cent. +of Tom's statement true. When she found the entire fence whitewashed, +and not only whitewashed but elaborately coated and recoated, and even +a streak added to the ground, her astonishment was almost unspeakable. +She said: + +"Well, I never! There's no getting round it, you can work when you're +a mind to, Tom." And then she diluted the compliment by adding, "But +it's powerful seldom you're a mind to, I'm bound to say. Well, go 'long +and play; but mind you get back some time in a week, or I'll tan you." + +She was so overcome by the splendor of his achievement that she took +him into the closet and selected a choice apple and delivered it to +him, along with an improving lecture upon the added value and flavor a +treat took to itself when it came without sin through virtuous effort. +And while she closed with a happy Scriptural flourish, he "hooked" a +doughnut. + +Then he skipped out, and saw Sid just starting up the outside stairway +that led to the back rooms on the second floor. Clods were handy and +the air was full of them in a twinkling. They raged around Sid like a +hail-storm; and before Aunt Polly could collect her surprised faculties +and sally to the rescue, six or seven clods had taken personal effect, +and Tom was over the fence and gone. There was a gate, but as a general +thing he was too crowded for time to make use of it. His soul was at +peace, now that he had settled with Sid for calling attention to his +black thread and getting him into trouble. + +Tom skirted the block, and came round into a muddy alley that led by +the back of his aunt's cow-stable. He presently got safely beyond the +reach of capture and punishment, and hastened toward the public square +of the village, where two "military" companies of boys had met for +conflict, according to previous appointment. Tom was General of one of +these armies, Joe Harper (a bosom friend) General of the other. These +two great commanders did not condescend to fight in person--that being +better suited to the still smaller fry--but sat together on an eminence +and conducted the field operations by orders delivered through +aides-de-camp. Tom's army won a great victory, after a long and +hard-fought battle. Then the dead were counted, prisoners exchanged, +the terms of the next disagreement agreed upon, and the day for the +necessary battle appointed; after which the armies fell into line and +marched away, and Tom turned homeward alone. + +As he was passing by the house where Jeff Thatcher lived, he saw a new +girl in the garden--a lovely little blue-eyed creature with yellow hair +plaited into two long-tails, white summer frock and embroidered +pantalettes. The fresh-crowned hero fell without firing a shot. A +certain Amy Lawrence vanished out of his heart and left not even a +memory of herself behind. He had thought he loved her to distraction; +he had regarded his passion as adoration; and behold it was only a poor +little evanescent partiality. He had been months winning her; she had +confessed hardly a week ago; he had been the happiest and the proudest +boy in the world only seven short days, and here in one instant of time +she had gone out of his heart like a casual stranger whose visit is +done. + +He worshipped this new angel with furtive eye, till he saw that she +had discovered him; then he pretended he did not know she was present, +and began to "show off" in all sorts of absurd boyish ways, in order to +win her admiration. He kept up this grotesque foolishness for some +time; but by-and-by, while he was in the midst of some dangerous +gymnastic performances, he glanced aside and saw that the little girl +was wending her way toward the house. Tom came up to the fence and +leaned on it, grieving, and hoping she would tarry yet awhile longer. +She halted a moment on the steps and then moved toward the door. Tom +heaved a great sigh as she put her foot on the threshold. But his face +lit up, right away, for she tossed a pansy over the fence a moment +before she disappeared. + +The boy ran around and stopped within a foot or two of the flower, and +then shaded his eyes with his hand and began to look down street as if +he had discovered something of interest going on in that direction. +Presently he picked up a straw and began trying to balance it on his +nose, with his head tilted far back; and as he moved from side to side, +in his efforts, he edged nearer and nearer toward the pansy; finally +his bare foot rested upon it, his pliant toes closed upon it, and he +hopped away with the treasure and disappeared round the corner. But +only for a minute--only while he could button the flower inside his +jacket, next his heart--or next his stomach, possibly, for he was not +much posted in anatomy, and not hypercritical, anyway. + +He returned, now, and hung about the fence till nightfall, "showing +off," as before; but the girl never exhibited herself again, though Tom +comforted himself a little with the hope that she had been near some +window, meantime, and been aware of his attentions. Finally he strode +home reluctantly, with his poor head full of visions. + +All through supper his spirits were so high that his aunt wondered +"what had got into the child." He took a good scolding about clodding +Sid, and did not seem to mind it in the least. He tried to steal sugar +under his aunt's very nose, and got his knuckles rapped for it. He said: + +"Aunt, you don't whack Sid when he takes it." + +"Well, Sid don't torment a body the way you do. You'd be always into +that sugar if I warn't watching you." + +Presently she stepped into the kitchen, and Sid, happy in his +immunity, reached for the sugar-bowl--a sort of glorying over Tom which +was wellnigh unbearable. But Sid's fingers slipped and the bowl dropped +and broke. Tom was in ecstasies. In such ecstasies that he even +controlled his tongue and was silent. He said to himself that he would +not speak a word, even when his aunt came in, but would sit perfectly +still till she asked who did the mischief; and then he would tell, and +there would be nothing so good in the world as to see that pet model +"catch it." He was so brimful of exultation that he could hardly hold +himself when the old lady came back and stood above the wreck +discharging lightnings of wrath from over her spectacles. He said to +himself, "Now it's coming!" And the next instant he was sprawling on +the floor! The potent palm was uplifted to strike again when Tom cried +out: + +"Hold on, now, what 'er you belting ME for?--Sid broke it!" + +Aunt Polly paused, perplexed, and Tom looked for healing pity. But +when she got her tongue again, she only said: + +"Umf! Well, you didn't get a lick amiss, I reckon. You been into some +other audacious mischief when I wasn't around, like enough." + +Then her conscience reproached her, and she yearned to say something +kind and loving; but she judged that this would be construed into a +confession that she had been in the wrong, and discipline forbade that. +So she kept silence, and went about her affairs with a troubled heart. +Tom sulked in a corner and exalted his woes. He knew that in her heart +his aunt was on her knees to him, and he was morosely gratified by the +consciousness of it. He would hang out no signals, he would take notice +of none. He knew that a yearning glance fell upon him, now and then, +through a film of tears, but he refused recognition of it. He pictured +himself lying sick unto death and his aunt bending over him beseeching +one little forgiving word, but he would turn his face to the wall, and +die with that word unsaid. Ah, how would she feel then? And he pictured +himself brought home from the river, dead, with his curls all wet, and +his sore heart at rest. How she would throw herself upon him, and how +her tears would fall like rain, and her lips pray God to give her back +her boy and she would never, never abuse him any more! But he would lie +there cold and white and make no sign--a poor little sufferer, whose +griefs were at an end. He so worked upon his feelings with the pathos +of these dreams, that he had to keep swallowing, he was so like to +choke; and his eyes swam in a blur of water, which overflowed when he +winked, and ran down and trickled from the end of his nose. And such a +luxury to him was this petting of his sorrows, that he could not bear +to have any worldly cheeriness or any grating delight intrude upon it; +it was too sacred for such contact; and so, presently, when his cousin +Mary danced in, all alive with the joy of seeing home again after an +age-long visit of one week to the country, he got up and moved in +clouds and darkness out at one door as she brought song and sunshine in +at the other. + +He wandered far from the accustomed haunts of boys, and sought +desolate places that were in harmony with his spirit. A log raft in the +river invited him, and he seated himself on its outer edge and +contemplated the dreary vastness of the stream, wishing, the while, +that he could only be drowned, all at once and unconsciously, without +undergoing the uncomfortable routine devised by nature. Then he thought +of his flower. He got it out, rumpled and wilted, and it mightily +increased his dismal felicity. He wondered if she would pity him if she +knew? Would she cry, and wish that she had a right to put her arms +around his neck and comfort him? Or would she turn coldly away like all +the hollow world? This picture brought such an agony of pleasurable +suffering that he worked it over and over again in his mind and set it +up in new and varied lights, till he wore it threadbare. At last he +rose up sighing and departed in the darkness. + +About half-past nine or ten o'clock he came along the deserted street +to where the Adored Unknown lived; he paused a moment; no sound fell +upon his listening ear; a candle was casting a dull glow upon the +curtain of a second-story window. Was the sacred presence there? He +climbed the fence, threaded his stealthy way through the plants, till +he stood under that window; he looked up at it long, and with emotion; +then he laid him down on the ground under it, disposing himself upon +his back, with his hands clasped upon his breast and holding his poor +wilted flower. And thus he would die--out in the cold world, with no +shelter over his homeless head, no friendly hand to wipe the +death-damps from his brow, no loving face to bend pityingly over him +when the great agony came. And thus SHE would see him when she looked +out upon the glad morning, and oh! would she drop one little tear upon +his poor, lifeless form, would she heave one little sigh to see a bright +young life so rudely blighted, so untimely cut down? + +The window went up, a maid-servant's discordant voice profaned the +holy calm, and a deluge of water drenched the prone martyr's remains! + +The strangling hero sprang up with a relieving snort. There was a whiz +as of a missile in the air, mingled with the murmur of a curse, a sound +as of shivering glass followed, and a small, vague form went over the +fence and shot away in the gloom. + +Not long after, as Tom, all undressed for bed, was surveying his +drenched garments by the light of a tallow dip, Sid woke up; but if he +had any dim idea of making any "references to allusions," he thought +better of it and held his peace, for there was danger in Tom's eye. + +Tom turned in without the added vexation of prayers, and Sid made +mental note of the omission. + + + +CHAPTER IV + +THE sun rose upon a tranquil world, and beamed down upon the peaceful +village like a benediction. Breakfast over, Aunt Polly had family +worship: it began with a prayer built from the ground up of solid +courses of Scriptural quotations, welded together with a thin mortar of +originality; and from the summit of this she delivered a grim chapter +of the Mosaic Law, as from Sinai. + +Then Tom girded up his loins, so to speak, and went to work to "get +his verses." Sid had learned his lesson days before. Tom bent all his +energies to the memorizing of five verses, and he chose part of the +Sermon on the Mount, because he could find no verses that were shorter. +At the end of half an hour Tom had a vague general idea of his lesson, +but no more, for his mind was traversing the whole field of human +thought, and his hands were busy with distracting recreations. Mary +took his book to hear him recite, and he tried to find his way through +the fog: + +"Blessed are the--a--a--" + +"Poor"-- + +"Yes--poor; blessed are the poor--a--a--" + +"In spirit--" + +"In spirit; blessed are the poor in spirit, for they--they--" + +"THEIRS--" + +"For THEIRS. Blessed are the poor in spirit, for theirs is the kingdom +of heaven. Blessed are they that mourn, for they--they--" + +"Sh--" + +"For they--a--" + +"S, H, A--" + +"For they S, H--Oh, I don't know what it is!" + +"SHALL!" + +"Oh, SHALL! for they shall--for they shall--a--a--shall mourn--a--a-- +blessed are they that shall--they that--a--they that shall mourn, for +they shall--a--shall WHAT? Why don't you tell me, Mary?--what do you +want to be so mean for?" + +"Oh, Tom, you poor thick-headed thing, I'm not teasing you. I wouldn't +do that. You must go and learn it again. Don't you be discouraged, Tom, +you'll manage it--and if you do, I'll give you something ever so nice. +There, now, that's a good boy." + +"All right! What is it, Mary, tell me what it is." + +"Never you mind, Tom. You know if I say it's nice, it is nice." + +"You bet you that's so, Mary. All right, I'll tackle it again." + +And he did "tackle it again"--and under the double pressure of +curiosity and prospective gain he did it with such spirit that he +accomplished a shining success. Mary gave him a brand-new "Barlow" +knife worth twelve and a half cents; and the convulsion of delight that +swept his system shook him to his foundations. True, the knife would +not cut anything, but it was a "sure-enough" Barlow, and there was +inconceivable grandeur in that--though where the Western boys ever got +the idea that such a weapon could possibly be counterfeited to its +injury is an imposing mystery and will always remain so, perhaps. Tom +contrived to scarify the cupboard with it, and was arranging to begin +on the bureau, when he was called off to dress for Sunday-school. + +Mary gave him a tin basin of water and a piece of soap, and he went +outside the door and set the basin on a little bench there; then he +dipped the soap in the water and laid it down; turned up his sleeves; +poured out the water on the ground, gently, and then entered the +kitchen and began to wipe his face diligently on the towel behind the +door. But Mary removed the towel and said: + +"Now ain't you ashamed, Tom. You mustn't be so bad. Water won't hurt +you." + +Tom was a trifle disconcerted. The basin was refilled, and this time +he stood over it a little while, gathering resolution; took in a big +breath and began. When he entered the kitchen presently, with both eyes +shut and groping for the towel with his hands, an honorable testimony +of suds and water was dripping from his face. But when he emerged from +the towel, he was not yet satisfactory, for the clean territory stopped +short at his chin and his jaws, like a mask; below and beyond this line +there was a dark expanse of unirrigated soil that spread downward in +front and backward around his neck. Mary took him in hand, and when she +was done with him he was a man and a brother, without distinction of +color, and his saturated hair was neatly brushed, and its short curls +wrought into a dainty and symmetrical general effect. [He privately +smoothed out the curls, with labor and difficulty, and plastered his +hair close down to his head; for he held curls to be effeminate, and +his own filled his life with bitterness.] Then Mary got out a suit of +his clothing that had been used only on Sundays during two years--they +were simply called his "other clothes"--and so by that we know the +size of his wardrobe. The girl "put him to rights" after he had dressed +himself; she buttoned his neat roundabout up to his chin, turned his +vast shirt collar down over his shoulders, brushed him off and crowned +him with his speckled straw hat. He now looked exceedingly improved and +uncomfortable. He was fully as uncomfortable as he looked; for there +was a restraint about whole clothes and cleanliness that galled him. He +hoped that Mary would forget his shoes, but the hope was blighted; she +coated them thoroughly with tallow, as was the custom, and brought them +out. He lost his temper and said he was always being made to do +everything he didn't want to do. But Mary said, persuasively: + +"Please, Tom--that's a good boy." + +So he got into the shoes snarling. Mary was soon ready, and the three +children set out for Sunday-school--a place that Tom hated with his +whole heart; but Sid and Mary were fond of it. + +Sabbath-school hours were from nine to half-past ten; and then church +service. Two of the children always remained for the sermon +voluntarily, and the other always remained too--for stronger reasons. +The church's high-backed, uncushioned pews would seat about three +hundred persons; the edifice was but a small, plain affair, with a sort +of pine board tree-box on top of it for a steeple. At the door Tom +dropped back a step and accosted a Sunday-dressed comrade: + +"Say, Billy, got a yaller ticket?" + +"Yes." + +"What'll you take for her?" + +"What'll you give?" + +"Piece of lickrish and a fish-hook." + +"Less see 'em." + +Tom exhibited. They were satisfactory, and the property changed hands. +Then Tom traded a couple of white alleys for three red tickets, and +some small trifle or other for a couple of blue ones. He waylaid other +boys as they came, and went on buying tickets of various colors ten or +fifteen minutes longer. He entered the church, now, with a swarm of +clean and noisy boys and girls, proceeded to his seat and started a +quarrel with the first boy that came handy. The teacher, a grave, +elderly man, interfered; then turned his back a moment and Tom pulled a +boy's hair in the next bench, and was absorbed in his book when the boy +turned around; stuck a pin in another boy, presently, in order to hear +him say "Ouch!" and got a new reprimand from his teacher. Tom's whole +class were of a pattern--restless, noisy, and troublesome. When they +came to recite their lessons, not one of them knew his verses +perfectly, but had to be prompted all along. However, they worried +through, and each got his reward--in small blue tickets, each with a +passage of Scripture on it; each blue ticket was pay for two verses of +the recitation. Ten blue tickets equalled a red one, and could be +exchanged for it; ten red tickets equalled a yellow one; for ten yellow +tickets the superintendent gave a very plainly bound Bible (worth forty +cents in those easy times) to the pupil. How many of my readers would +have the industry and application to memorize two thousand verses, even +for a Dore Bible? And yet Mary had acquired two Bibles in this way--it +was the patient work of two years--and a boy of German parentage had +won four or five. He once recited three thousand verses without +stopping; but the strain upon his mental faculties was too great, and +he was little better than an idiot from that day forth--a grievous +misfortune for the school, for on great occasions, before company, the +superintendent (as Tom expressed it) had always made this boy come out +and "spread himself." Only the older pupils managed to keep their +tickets and stick to their tedious work long enough to get a Bible, and +so the delivery of one of these prizes was a rare and noteworthy +circumstance; the successful pupil was so great and conspicuous for +that day that on the spot every scholar's heart was fired with a fresh +ambition that often lasted a couple of weeks. It is possible that Tom's +mental stomach had never really hungered for one of those prizes, but +unquestionably his entire being had for many a day longed for the glory +and the eclat that came with it. + +In due course the superintendent stood up in front of the pulpit, with +a closed hymn-book in his hand and his forefinger inserted between its +leaves, and commanded attention. When a Sunday-school superintendent +makes his customary little speech, a hymn-book in the hand is as +necessary as is the inevitable sheet of music in the hand of a singer +who stands forward on the platform and sings a solo at a concert +--though why, is a mystery: for neither the hymn-book nor the sheet of +music is ever referred to by the sufferer. This superintendent was a +slim creature of thirty-five, with a sandy goatee and short sandy hair; +he wore a stiff standing-collar whose upper edge almost reached his +ears and whose sharp points curved forward abreast the corners of his +mouth--a fence that compelled a straight lookout ahead, and a turning +of the whole body when a side view was required; his chin was propped +on a spreading cravat which was as broad and as long as a bank-note, +and had fringed ends; his boot toes were turned sharply up, in the +fashion of the day, like sleigh-runners--an effect patiently and +laboriously produced by the young men by sitting with their toes +pressed against a wall for hours together. Mr. Walters was very earnest +of mien, and very sincere and honest at heart; and he held sacred +things and places in such reverence, and so separated them from worldly +matters, that unconsciously to himself his Sunday-school voice had +acquired a peculiar intonation which was wholly absent on week-days. He +began after this fashion: + +"Now, children, I want you all to sit up just as straight and pretty +as you can and give me all your attention for a minute or two. There +--that is it. That is the way good little boys and girls should do. I see +one little girl who is looking out of the window--I am afraid she +thinks I am out there somewhere--perhaps up in one of the trees making +a speech to the little birds. [Applausive titter.] I want to tell you +how good it makes me feel to see so many bright, clean little faces +assembled in a place like this, learning to do right and be good." And +so forth and so on. It is not necessary to set down the rest of the +oration. It was of a pattern which does not vary, and so it is familiar +to us all. + +The latter third of the speech was marred by the resumption of fights +and other recreations among certain of the bad boys, and by fidgetings +and whisperings that extended far and wide, washing even to the bases +of isolated and incorruptible rocks like Sid and Mary. But now every +sound ceased suddenly, with the subsidence of Mr. Walters' voice, and +the conclusion of the speech was received with a burst of silent +gratitude. + +A good part of the whispering had been occasioned by an event which +was more or less rare--the entrance of visitors: lawyer Thatcher, +accompanied by a very feeble and aged man; a fine, portly, middle-aged +gentleman with iron-gray hair; and a dignified lady who was doubtless +the latter's wife. The lady was leading a child. Tom had been restless +and full of chafings and repinings; conscience-smitten, too--he could +not meet Amy Lawrence's eye, he could not brook her loving gaze. But +when he saw this small new-comer his soul was all ablaze with bliss in +a moment. The next moment he was "showing off" with all his might +--cuffing boys, pulling hair, making faces--in a word, using every art +that seemed likely to fascinate a girl and win her applause. His +exaltation had but one alloy--the memory of his humiliation in this +angel's garden--and that record in sand was fast washing out, under +the waves of happiness that were sweeping over it now. + +The visitors were given the highest seat of honor, and as soon as Mr. +Walters' speech was finished, he introduced them to the school. The +middle-aged man turned out to be a prodigious personage--no less a one +than the county judge--altogether the most august creation these +children had ever looked upon--and they wondered what kind of material +he was made of--and they half wanted to hear him roar, and were half +afraid he might, too. He was from Constantinople, twelve miles away--so +he had travelled, and seen the world--these very eyes had looked upon +the county court-house--which was said to have a tin roof. The awe +which these reflections inspired was attested by the impressive silence +and the ranks of staring eyes. This was the great Judge Thatcher, +brother of their own lawyer. Jeff Thatcher immediately went forward, to +be familiar with the great man and be envied by the school. It would +have been music to his soul to hear the whisperings: + +"Look at him, Jim! He's a going up there. Say--look! he's a going to +shake hands with him--he IS shaking hands with him! By jings, don't you +wish you was Jeff?" + +Mr. Walters fell to "showing off," with all sorts of official +bustlings and activities, giving orders, delivering judgments, +discharging directions here, there, everywhere that he could find a +target. The librarian "showed off"--running hither and thither with his +arms full of books and making a deal of the splutter and fuss that +insect authority delights in. The young lady teachers "showed off" +--bending sweetly over pupils that were lately being boxed, lifting +pretty warning fingers at bad little boys and patting good ones +lovingly. The young gentlemen teachers "showed off" with small +scoldings and other little displays of authority and fine attention to +discipline--and most of the teachers, of both sexes, found business up +at the library, by the pulpit; and it was business that frequently had +to be done over again two or three times (with much seeming vexation). +The little girls "showed off" in various ways, and the little boys +"showed off" with such diligence that the air was thick with paper wads +and the murmur of scufflings. And above it all the great man sat and +beamed a majestic judicial smile upon all the house, and warmed himself +in the sun of his own grandeur--for he was "showing off," too. + +There was only one thing wanting to make Mr. Walters' ecstasy +complete, and that was a chance to deliver a Bible-prize and exhibit a +prodigy. Several pupils had a few yellow tickets, but none had enough +--he had been around among the star pupils inquiring. He would have given +worlds, now, to have that German lad back again with a sound mind. + +And now at this moment, when hope was dead, Tom Sawyer came forward +with nine yellow tickets, nine red tickets, and ten blue ones, and +demanded a Bible. This was a thunderbolt out of a clear sky. Walters +was not expecting an application from this source for the next ten +years. But there was no getting around it--here were the certified +checks, and they were good for their face. Tom was therefore elevated +to a place with the Judge and the other elect, and the great news was +announced from headquarters. It was the most stunning surprise of the +decade, and so profound was the sensation that it lifted the new hero +up to the judicial one's altitude, and the school had two marvels to +gaze upon in place of one. The boys were all eaten up with envy--but +those that suffered the bitterest pangs were those who perceived too +late that they themselves had contributed to this hated splendor by +trading tickets to Tom for the wealth he had amassed in selling +whitewashing privileges. These despised themselves, as being the dupes +of a wily fraud, a guileful snake in the grass. + +The prize was delivered to Tom with as much effusion as the +superintendent could pump up under the circumstances; but it lacked +somewhat of the true gush, for the poor fellow's instinct taught him +that there was a mystery here that could not well bear the light, +perhaps; it was simply preposterous that this boy had warehoused two +thousand sheaves of Scriptural wisdom on his premises--a dozen would +strain his capacity, without a doubt. + +Amy Lawrence was proud and glad, and she tried to make Tom see it in +her face--but he wouldn't look. She wondered; then she was just a grain +troubled; next a dim suspicion came and went--came again; she watched; +a furtive glance told her worlds--and then her heart broke, and she was +jealous, and angry, and the tears came and she hated everybody. Tom +most of all (she thought). + +Tom was introduced to the Judge; but his tongue was tied, his breath +would hardly come, his heart quaked--partly because of the awful +greatness of the man, but mainly because he was her parent. He would +have liked to fall down and worship him, if it were in the dark. The +Judge put his hand on Tom's head and called him a fine little man, and +asked him what his name was. The boy stammered, gasped, and got it out: + +"Tom." + +"Oh, no, not Tom--it is--" + +"Thomas." + +"Ah, that's it. I thought there was more to it, maybe. That's very +well. But you've another one I daresay, and you'll tell it to me, won't +you?" + +"Tell the gentleman your other name, Thomas," said Walters, "and say +sir. You mustn't forget your manners." + +"Thomas Sawyer--sir." + +"That's it! That's a good boy. Fine boy. Fine, manly little fellow. +Two thousand verses is a great many--very, very great many. And you +never can be sorry for the trouble you took to learn them; for +knowledge is worth more than anything there is in the world; it's what +makes great men and good men; you'll be a great man and a good man +yourself, some day, Thomas, and then you'll look back and say, It's all +owing to the precious Sunday-school privileges of my boyhood--it's all +owing to my dear teachers that taught me to learn--it's all owing to +the good superintendent, who encouraged me, and watched over me, and +gave me a beautiful Bible--a splendid elegant Bible--to keep and have +it all for my own, always--it's all owing to right bringing up! That is +what you will say, Thomas--and you wouldn't take any money for those +two thousand verses--no indeed you wouldn't. And now you wouldn't mind +telling me and this lady some of the things you've learned--no, I know +you wouldn't--for we are proud of little boys that learn. Now, no +doubt you know the names of all the twelve disciples. Won't you tell us +the names of the first two that were appointed?" + +Tom was tugging at a button-hole and looking sheepish. He blushed, +now, and his eyes fell. Mr. Walters' heart sank within him. He said to +himself, it is not possible that the boy can answer the simplest +question--why DID the Judge ask him? Yet he felt obliged to speak up +and say: + +"Answer the gentleman, Thomas--don't be afraid." + +Tom still hung fire. + +"Now I know you'll tell me," said the lady. "The names of the first +two disciples were--" + +"DAVID AND GOLIAH!" + +Let us draw the curtain of charity over the rest of the scene. + + + +CHAPTER V + +ABOUT half-past ten the cracked bell of the small church began to +ring, and presently the people began to gather for the morning sermon. +The Sunday-school children distributed themselves about the house and +occupied pews with their parents, so as to be under supervision. Aunt +Polly came, and Tom and Sid and Mary sat with her--Tom being placed +next the aisle, in order that he might be as far away from the open +window and the seductive outside summer scenes as possible. The crowd +filed up the aisles: the aged and needy postmaster, who had seen better +days; the mayor and his wife--for they had a mayor there, among other +unnecessaries; the justice of the peace; the widow Douglass, fair, +smart, and forty, a generous, good-hearted soul and well-to-do, her +hill mansion the only palace in the town, and the most hospitable and +much the most lavish in the matter of festivities that St. Petersburg +could boast; the bent and venerable Major and Mrs. Ward; lawyer +Riverson, the new notable from a distance; next the belle of the +village, followed by a troop of lawn-clad and ribbon-decked young +heart-breakers; then all the young clerks in town in a body--for they +had stood in the vestibule sucking their cane-heads, a circling wall of +oiled and simpering admirers, till the last girl had run their gantlet; +and last of all came the Model Boy, Willie Mufferson, taking as heedful +care of his mother as if she were cut glass. He always brought his +mother to church, and was the pride of all the matrons. The boys all +hated him, he was so good. And besides, he had been "thrown up to them" +so much. His white handkerchief was hanging out of his pocket behind, as +usual on Sundays--accidentally. Tom had no handkerchief, and he looked +upon boys who had as snobs. + +The congregation being fully assembled, now, the bell rang once more, +to warn laggards and stragglers, and then a solemn hush fell upon the +church which was only broken by the tittering and whispering of the +choir in the gallery. The choir always tittered and whispered all +through service. There was once a church choir that was not ill-bred, +but I have forgotten where it was, now. It was a great many years ago, +and I can scarcely remember anything about it, but I think it was in +some foreign country. + +The minister gave out the hymn, and read it through with a relish, in +a peculiar style which was much admired in that part of the country. +His voice began on a medium key and climbed steadily up till it reached +a certain point, where it bore with strong emphasis upon the topmost +word and then plunged down as if from a spring-board: + + Shall I be car-ri-ed toe the skies, on flow'ry BEDS of ease, + + Whilst others fight to win the prize, and sail thro' BLOODY seas? + +He was regarded as a wonderful reader. At church "sociables" he was +always called upon to read poetry; and when he was through, the ladies +would lift up their hands and let them fall helplessly in their laps, +and "wall" their eyes, and shake their heads, as much as to say, "Words +cannot express it; it is too beautiful, TOO beautiful for this mortal +earth." + +After the hymn had been sung, the Rev. Mr. Sprague turned himself into +a bulletin-board, and read off "notices" of meetings and societies and +things till it seemed that the list would stretch out to the crack of +doom--a queer custom which is still kept up in America, even in cities, +away here in this age of abundant newspapers. Often, the less there is +to justify a traditional custom, the harder it is to get rid of it. + +And now the minister prayed. A good, generous prayer it was, and went +into details: it pleaded for the church, and the little children of the +church; for the other churches of the village; for the village itself; +for the county; for the State; for the State officers; for the United +States; for the churches of the United States; for Congress; for the +President; for the officers of the Government; for poor sailors, tossed +by stormy seas; for the oppressed millions groaning under the heel of +European monarchies and Oriental despotisms; for such as have the light +and the good tidings, and yet have not eyes to see nor ears to hear +withal; for the heathen in the far islands of the sea; and closed with +a supplication that the words he was about to speak might find grace +and favor, and be as seed sown in fertile ground, yielding in time a +grateful harvest of good. Amen. + +There was a rustling of dresses, and the standing congregation sat +down. The boy whose history this book relates did not enjoy the prayer, +he only endured it--if he even did that much. He was restive all +through it; he kept tally of the details of the prayer, unconsciously +--for he was not listening, but he knew the ground of old, and the +clergyman's regular route over it--and when a little trifle of new +matter was interlarded, his ear detected it and his whole nature +resented it; he considered additions unfair, and scoundrelly. In the +midst of the prayer a fly had lit on the back of the pew in front of +him and tortured his spirit by calmly rubbing its hands together, +embracing its head with its arms, and polishing it so vigorously that +it seemed to almost part company with the body, and the slender thread +of a neck was exposed to view; scraping its wings with its hind legs +and smoothing them to its body as if they had been coat-tails; going +through its whole toilet as tranquilly as if it knew it was perfectly +safe. As indeed it was; for as sorely as Tom's hands itched to grab for +it they did not dare--he believed his soul would be instantly destroyed +if he did such a thing while the prayer was going on. But with the +closing sentence his hand began to curve and steal forward; and the +instant the "Amen" was out the fly was a prisoner of war. His aunt +detected the act and made him let it go. + +The minister gave out his text and droned along monotonously through +an argument that was so prosy that many a head by and by began to nod +--and yet it was an argument that dealt in limitless fire and brimstone +and thinned the predestined elect down to a company so small as to be +hardly worth the saving. Tom counted the pages of the sermon; after +church he always knew how many pages there had been, but he seldom knew +anything else about the discourse. However, this time he was really +interested for a little while. The minister made a grand and moving +picture of the assembling together of the world's hosts at the +millennium when the lion and the lamb should lie down together and a +little child should lead them. But the pathos, the lesson, the moral of +the great spectacle were lost upon the boy; he only thought of the +conspicuousness of the principal character before the on-looking +nations; his face lit with the thought, and he said to himself that he +wished he could be that child, if it was a tame lion. + +Now he lapsed into suffering again, as the dry argument was resumed. +Presently he bethought him of a treasure he had and got it out. It was +a large black beetle with formidable jaws--a "pinchbug," he called it. +It was in a percussion-cap box. The first thing the beetle did was to +take him by the finger. A natural fillip followed, the beetle went +floundering into the aisle and lit on its back, and the hurt finger +went into the boy's mouth. The beetle lay there working its helpless +legs, unable to turn over. Tom eyed it, and longed for it; but it was +safe out of his reach. Other people uninterested in the sermon found +relief in the beetle, and they eyed it too. Presently a vagrant poodle +dog came idling along, sad at heart, lazy with the summer softness and +the quiet, weary of captivity, sighing for change. He spied the beetle; +the drooping tail lifted and wagged. He surveyed the prize; walked +around it; smelt at it from a safe distance; walked around it again; +grew bolder, and took a closer smell; then lifted his lip and made a +gingerly snatch at it, just missing it; made another, and another; +began to enjoy the diversion; subsided to his stomach with the beetle +between his paws, and continued his experiments; grew weary at last, +and then indifferent and absent-minded. His head nodded, and little by +little his chin descended and touched the enemy, who seized it. There +was a sharp yelp, a flirt of the poodle's head, and the beetle fell a +couple of yards away, and lit on its back once more. The neighboring +spectators shook with a gentle inward joy, several faces went behind +fans and handkerchiefs, and Tom was entirely happy. The dog looked +foolish, and probably felt so; but there was resentment in his heart, +too, and a craving for revenge. So he went to the beetle and began a +wary attack on it again; jumping at it from every point of a circle, +lighting with his fore-paws within an inch of the creature, making even +closer snatches at it with his teeth, and jerking his head till his +ears flapped again. But he grew tired once more, after a while; tried +to amuse himself with a fly but found no relief; followed an ant +around, with his nose close to the floor, and quickly wearied of that; +yawned, sighed, forgot the beetle entirely, and sat down on it. Then +there was a wild yelp of agony and the poodle went sailing up the +aisle; the yelps continued, and so did the dog; he crossed the house in +front of the altar; he flew down the other aisle; he crossed before the +doors; he clamored up the home-stretch; his anguish grew with his +progress, till presently he was but a woolly comet moving in its orbit +with the gleam and the speed of light. At last the frantic sufferer +sheered from its course, and sprang into its master's lap; he flung it +out of the window, and the voice of distress quickly thinned away and +died in the distance. + +By this time the whole church was red-faced and suffocating with +suppressed laughter, and the sermon had come to a dead standstill. The +discourse was resumed presently, but it went lame and halting, all +possibility of impressiveness being at an end; for even the gravest +sentiments were constantly being received with a smothered burst of +unholy mirth, under cover of some remote pew-back, as if the poor +parson had said a rarely facetious thing. It was a genuine relief to +the whole congregation when the ordeal was over and the benediction +pronounced. + +Tom Sawyer went home quite cheerful, thinking to himself that there +was some satisfaction about divine service when there was a bit of +variety in it. He had but one marring thought; he was willing that the +dog should play with his pinchbug, but he did not think it was upright +in him to carry it off. + + + +CHAPTER VI + +MONDAY morning found Tom Sawyer miserable. Monday morning always found +him so--because it began another week's slow suffering in school. He +generally began that day with wishing he had had no intervening +holiday, it made the going into captivity and fetters again so much +more odious. + +Tom lay thinking. Presently it occurred to him that he wished he was +sick; then he could stay home from school. Here was a vague +possibility. He canvassed his system. No ailment was found, and he +investigated again. This time he thought he could detect colicky +symptoms, and he began to encourage them with considerable hope. But +they soon grew feeble, and presently died wholly away. He reflected +further. Suddenly he discovered something. One of his upper front teeth +was loose. This was lucky; he was about to begin to groan, as a +"starter," as he called it, when it occurred to him that if he came +into court with that argument, his aunt would pull it out, and that +would hurt. So he thought he would hold the tooth in reserve for the +present, and seek further. Nothing offered for some little time, and +then he remembered hearing the doctor tell about a certain thing that +laid up a patient for two or three weeks and threatened to make him +lose a finger. So the boy eagerly drew his sore toe from under the +sheet and held it up for inspection. But now he did not know the +necessary symptoms. However, it seemed well worth while to chance it, +so he fell to groaning with considerable spirit. + +But Sid slept on unconscious. + +Tom groaned louder, and fancied that he began to feel pain in the toe. + +No result from Sid. + +Tom was panting with his exertions by this time. He took a rest and +then swelled himself up and fetched a succession of admirable groans. + +Sid snored on. + +Tom was aggravated. He said, "Sid, Sid!" and shook him. This course +worked well, and Tom began to groan again. Sid yawned, stretched, then +brought himself up on his elbow with a snort, and began to stare at +Tom. Tom went on groaning. Sid said: + +"Tom! Say, Tom!" [No response.] "Here, Tom! TOM! What is the matter, +Tom?" And he shook him and looked in his face anxiously. + +Tom moaned out: + +"Oh, don't, Sid. Don't joggle me." + +"Why, what's the matter, Tom? I must call auntie." + +"No--never mind. It'll be over by and by, maybe. Don't call anybody." + +"But I must! DON'T groan so, Tom, it's awful. How long you been this +way?" + +"Hours. Ouch! Oh, don't stir so, Sid, you'll kill me." + +"Tom, why didn't you wake me sooner? Oh, Tom, DON'T! It makes my +flesh crawl to hear you. Tom, what is the matter?" + +"I forgive you everything, Sid. [Groan.] Everything you've ever done +to me. When I'm gone--" + +"Oh, Tom, you ain't dying, are you? Don't, Tom--oh, don't. Maybe--" + +"I forgive everybody, Sid. [Groan.] Tell 'em so, Sid. And Sid, you +give my window-sash and my cat with one eye to that new girl that's +come to town, and tell her--" + +But Sid had snatched his clothes and gone. Tom was suffering in +reality, now, so handsomely was his imagination working, and so his +groans had gathered quite a genuine tone. + +Sid flew down-stairs and said: + +"Oh, Aunt Polly, come! Tom's dying!" + +"Dying!" + +"Yes'm. Don't wait--come quick!" + +"Rubbage! I don't believe it!" + +But she fled up-stairs, nevertheless, with Sid and Mary at her heels. +And her face grew white, too, and her lip trembled. When she reached +the bedside she gasped out: + +"You, Tom! Tom, what's the matter with you?" + +"Oh, auntie, I'm--" + +"What's the matter with you--what is the matter with you, child?" + +"Oh, auntie, my sore toe's mortified!" + +The old lady sank down into a chair and laughed a little, then cried a +little, then did both together. This restored her and she said: + +"Tom, what a turn you did give me. Now you shut up that nonsense and +climb out of this." + +The groans ceased and the pain vanished from the toe. The boy felt a +little foolish, and he said: + +"Aunt Polly, it SEEMED mortified, and it hurt so I never minded my +tooth at all." + +"Your tooth, indeed! What's the matter with your tooth?" + +"One of them's loose, and it aches perfectly awful." + +"There, there, now, don't begin that groaning again. Open your mouth. +Well--your tooth IS loose, but you're not going to die about that. +Mary, get me a silk thread, and a chunk of fire out of the kitchen." + +Tom said: + +"Oh, please, auntie, don't pull it out. It don't hurt any more. I wish +I may never stir if it does. Please don't, auntie. I don't want to stay +home from school." + +"Oh, you don't, don't you? So all this row was because you thought +you'd get to stay home from school and go a-fishing? Tom, Tom, I love +you so, and you seem to try every way you can to break my old heart +with your outrageousness." By this time the dental instruments were +ready. The old lady made one end of the silk thread fast to Tom's tooth +with a loop and tied the other to the bedpost. Then she seized the +chunk of fire and suddenly thrust it almost into the boy's face. The +tooth hung dangling by the bedpost, now. + +But all trials bring their compensations. As Tom wended to school +after breakfast, he was the envy of every boy he met because the gap in +his upper row of teeth enabled him to expectorate in a new and +admirable way. He gathered quite a following of lads interested in the +exhibition; and one that had cut his finger and had been a centre of +fascination and homage up to this time, now found himself suddenly +without an adherent, and shorn of his glory. His heart was heavy, and +he said with a disdain which he did not feel that it wasn't anything to +spit like Tom Sawyer; but another boy said, "Sour grapes!" and he +wandered away a dismantled hero. + +Shortly Tom came upon the juvenile pariah of the village, Huckleberry +Finn, son of the town drunkard. Huckleberry was cordially hated and +dreaded by all the mothers of the town, because he was idle and lawless +and vulgar and bad--and because all their children admired him so, and +delighted in his forbidden society, and wished they dared to be like +him. Tom was like the rest of the respectable boys, in that he envied +Huckleberry his gaudy outcast condition, and was under strict orders +not to play with him. So he played with him every time he got a chance. +Huckleberry was always dressed in the cast-off clothes of full-grown +men, and they were in perennial bloom and fluttering with rags. His hat +was a vast ruin with a wide crescent lopped out of its brim; his coat, +when he wore one, hung nearly to his heels and had the rearward buttons +far down the back; but one suspender supported his trousers; the seat +of the trousers bagged low and contained nothing, the fringed legs +dragged in the dirt when not rolled up. + +Huckleberry came and went, at his own free will. He slept on doorsteps +in fine weather and in empty hogsheads in wet; he did not have to go to +school or to church, or call any being master or obey anybody; he could +go fishing or swimming when and where he chose, and stay as long as it +suited him; nobody forbade him to fight; he could sit up as late as he +pleased; he was always the first boy that went barefoot in the spring +and the last to resume leather in the fall; he never had to wash, nor +put on clean clothes; he could swear wonderfully. In a word, everything +that goes to make life precious that boy had. So thought every +harassed, hampered, respectable boy in St. Petersburg. + +Tom hailed the romantic outcast: + +"Hello, Huckleberry!" + +"Hello yourself, and see how you like it." + +"What's that you got?" + +"Dead cat." + +"Lemme see him, Huck. My, he's pretty stiff. Where'd you get him?" + +"Bought him off'n a boy." + +"What did you give?" + +"I give a blue ticket and a bladder that I got at the slaughter-house." + +"Where'd you get the blue ticket?" + +"Bought it off'n Ben Rogers two weeks ago for a hoop-stick." + +"Say--what is dead cats good for, Huck?" + +"Good for? Cure warts with." + +"No! Is that so? I know something that's better." + +"I bet you don't. What is it?" + +"Why, spunk-water." + +"Spunk-water! I wouldn't give a dern for spunk-water." + +"You wouldn't, wouldn't you? D'you ever try it?" + +"No, I hain't. But Bob Tanner did." + +"Who told you so!" + +"Why, he told Jeff Thatcher, and Jeff told Johnny Baker, and Johnny +told Jim Hollis, and Jim told Ben Rogers, and Ben told a nigger, and +the nigger told me. There now!" + +"Well, what of it? They'll all lie. Leastways all but the nigger. I +don't know HIM. But I never see a nigger that WOULDN'T lie. Shucks! Now +you tell me how Bob Tanner done it, Huck." + +"Why, he took and dipped his hand in a rotten stump where the +rain-water was." + +"In the daytime?" + +"Certainly." + +"With his face to the stump?" + +"Yes. Least I reckon so." + +"Did he say anything?" + +"I don't reckon he did. I don't know." + +"Aha! Talk about trying to cure warts with spunk-water such a blame +fool way as that! Why, that ain't a-going to do any good. You got to go +all by yourself, to the middle of the woods, where you know there's a +spunk-water stump, and just as it's midnight you back up against the +stump and jam your hand in and say: + + 'Barley-corn, barley-corn, injun-meal shorts, + Spunk-water, spunk-water, swaller these warts,' + +and then walk away quick, eleven steps, with your eyes shut, and then +turn around three times and walk home without speaking to anybody. +Because if you speak the charm's busted." + +"Well, that sounds like a good way; but that ain't the way Bob Tanner +done." + +"No, sir, you can bet he didn't, becuz he's the wartiest boy in this +town; and he wouldn't have a wart on him if he'd knowed how to work +spunk-water. I've took off thousands of warts off of my hands that way, +Huck. I play with frogs so much that I've always got considerable many +warts. Sometimes I take 'em off with a bean." + +"Yes, bean's good. I've done that." + +"Have you? What's your way?" + +"You take and split the bean, and cut the wart so as to get some +blood, and then you put the blood on one piece of the bean and take and +dig a hole and bury it 'bout midnight at the crossroads in the dark of +the moon, and then you burn up the rest of the bean. You see that piece +that's got the blood on it will keep drawing and drawing, trying to +fetch the other piece to it, and so that helps the blood to draw the +wart, and pretty soon off she comes." + +"Yes, that's it, Huck--that's it; though when you're burying it if you +say 'Down bean; off wart; come no more to bother me!' it's better. +That's the way Joe Harper does, and he's been nearly to Coonville and +most everywheres. But say--how do you cure 'em with dead cats?" + +"Why, you take your cat and go and get in the graveyard 'long about +midnight when somebody that was wicked has been buried; and when it's +midnight a devil will come, or maybe two or three, but you can't see +'em, you can only hear something like the wind, or maybe hear 'em talk; +and when they're taking that feller away, you heave your cat after 'em +and say, 'Devil follow corpse, cat follow devil, warts follow cat, I'm +done with ye!' That'll fetch ANY wart." + +"Sounds right. D'you ever try it, Huck?" + +"No, but old Mother Hopkins told me." + +"Well, I reckon it's so, then. Becuz they say she's a witch." + +"Say! Why, Tom, I KNOW she is. She witched pap. Pap says so his own +self. He come along one day, and he see she was a-witching him, so he +took up a rock, and if she hadn't dodged, he'd a got her. Well, that +very night he rolled off'n a shed wher' he was a layin drunk, and broke +his arm." + +"Why, that's awful. How did he know she was a-witching him?" + +"Lord, pap can tell, easy. Pap says when they keep looking at you +right stiddy, they're a-witching you. Specially if they mumble. Becuz +when they mumble they're saying the Lord's Prayer backards." + +"Say, Hucky, when you going to try the cat?" + +"To-night. I reckon they'll come after old Hoss Williams to-night." + +"But they buried him Saturday. Didn't they get him Saturday night?" + +"Why, how you talk! How could their charms work till midnight?--and +THEN it's Sunday. Devils don't slosh around much of a Sunday, I don't +reckon." + +"I never thought of that. That's so. Lemme go with you?" + +"Of course--if you ain't afeard." + +"Afeard! 'Tain't likely. Will you meow?" + +"Yes--and you meow back, if you get a chance. Last time, you kep' me +a-meowing around till old Hays went to throwing rocks at me and says +'Dern that cat!' and so I hove a brick through his window--but don't +you tell." + +"I won't. I couldn't meow that night, becuz auntie was watching me, +but I'll meow this time. Say--what's that?" + +"Nothing but a tick." + +"Where'd you get him?" + +"Out in the woods." + +"What'll you take for him?" + +"I don't know. I don't want to sell him." + +"All right. It's a mighty small tick, anyway." + +"Oh, anybody can run a tick down that don't belong to them. I'm +satisfied with it. It's a good enough tick for me." + +"Sho, there's ticks a plenty. I could have a thousand of 'em if I +wanted to." + +"Well, why don't you? Becuz you know mighty well you can't. This is a +pretty early tick, I reckon. It's the first one I've seen this year." + +"Say, Huck--I'll give you my tooth for him." + +"Less see it." + +Tom got out a bit of paper and carefully unrolled it. Huckleberry +viewed it wistfully. The temptation was very strong. At last he said: + +"Is it genuwyne?" + +Tom lifted his lip and showed the vacancy. + +"Well, all right," said Huckleberry, "it's a trade." + +Tom enclosed the tick in the percussion-cap box that had lately been +the pinchbug's prison, and the boys separated, each feeling wealthier +than before. + +When Tom reached the little isolated frame schoolhouse, he strode in +briskly, with the manner of one who had come with all honest speed. +He hung his hat on a peg and flung himself into his seat with +business-like alacrity. The master, throned on high in his great +splint-bottom arm-chair, was dozing, lulled by the drowsy hum of study. +The interruption roused him. + +"Thomas Sawyer!" + +Tom knew that when his name was pronounced in full, it meant trouble. + +"Sir!" + +"Come up here. Now, sir, why are you late again, as usual?" + +Tom was about to take refuge in a lie, when he saw two long tails of +yellow hair hanging down a back that he recognized by the electric +sympathy of love; and by that form was THE ONLY VACANT PLACE on the +girls' side of the schoolhouse. He instantly said: + +"I STOPPED TO TALK WITH HUCKLEBERRY FINN!" + +The master's pulse stood still, and he stared helplessly. The buzz of +study ceased. The pupils wondered if this foolhardy boy had lost his +mind. The master said: + +"You--you did what?" + +"Stopped to talk with Huckleberry Finn." + +There was no mistaking the words. + +"Thomas Sawyer, this is the most astounding confession I have ever +listened to. No mere ferule will answer for this offence. Take off your +jacket." + +The master's arm performed until it was tired and the stock of +switches notably diminished. Then the order followed: + +"Now, sir, go and sit with the girls! And let this be a warning to you." + +The titter that rippled around the room appeared to abash the boy, but +in reality that result was caused rather more by his worshipful awe of +his unknown idol and the dread pleasure that lay in his high good +fortune. He sat down upon the end of the pine bench and the girl +hitched herself away from him with a toss of her head. Nudges and winks +and whispers traversed the room, but Tom sat still, with his arms upon +the long, low desk before him, and seemed to study his book. + +By and by attention ceased from him, and the accustomed school murmur +rose upon the dull air once more. Presently the boy began to steal +furtive glances at the girl. She observed it, "made a mouth" at him and +gave him the back of her head for the space of a minute. When she +cautiously faced around again, a peach lay before her. She thrust it +away. Tom gently put it back. She thrust it away again, but with less +animosity. Tom patiently returned it to its place. Then she let it +remain. Tom scrawled on his slate, "Please take it--I got more." The +girl glanced at the words, but made no sign. Now the boy began to draw +something on the slate, hiding his work with his left hand. For a time +the girl refused to notice; but her human curiosity presently began to +manifest itself by hardly perceptible signs. The boy worked on, +apparently unconscious. The girl made a sort of noncommittal attempt to +see, but the boy did not betray that he was aware of it. At last she +gave in and hesitatingly whispered: + +"Let me see it." + +Tom partly uncovered a dismal caricature of a house with two gable +ends to it and a corkscrew of smoke issuing from the chimney. Then the +girl's interest began to fasten itself upon the work and she forgot +everything else. When it was finished, she gazed a moment, then +whispered: + +"It's nice--make a man." + +The artist erected a man in the front yard, that resembled a derrick. +He could have stepped over the house; but the girl was not +hypercritical; she was satisfied with the monster, and whispered: + +"It's a beautiful man--now make me coming along." + +Tom drew an hour-glass with a full moon and straw limbs to it and +armed the spreading fingers with a portentous fan. The girl said: + +"It's ever so nice--I wish I could draw." + +"It's easy," whispered Tom, "I'll learn you." + +"Oh, will you? When?" + +"At noon. Do you go home to dinner?" + +"I'll stay if you will." + +"Good--that's a whack. What's your name?" + +"Becky Thatcher. What's yours? Oh, I know. It's Thomas Sawyer." + +"That's the name they lick me by. I'm Tom when I'm good. You call me +Tom, will you?" + +"Yes." + +Now Tom began to scrawl something on the slate, hiding the words from +the girl. But she was not backward this time. She begged to see. Tom +said: + +"Oh, it ain't anything." + +"Yes it is." + +"No it ain't. You don't want to see." + +"Yes I do, indeed I do. Please let me." + +"You'll tell." + +"No I won't--deed and deed and double deed won't." + +"You won't tell anybody at all? Ever, as long as you live?" + +"No, I won't ever tell ANYbody. Now let me." + +"Oh, YOU don't want to see!" + +"Now that you treat me so, I WILL see." And she put her small hand +upon his and a little scuffle ensued, Tom pretending to resist in +earnest but letting his hand slip by degrees till these words were +revealed: "I LOVE YOU." + +"Oh, you bad thing!" And she hit his hand a smart rap, but reddened +and looked pleased, nevertheless. + +Just at this juncture the boy felt a slow, fateful grip closing on his +ear, and a steady lifting impulse. In that wise he was borne across the +house and deposited in his own seat, under a peppering fire of giggles +from the whole school. Then the master stood over him during a few +awful moments, and finally moved away to his throne without saying a +word. But although Tom's ear tingled, his heart was jubilant. + +As the school quieted down Tom made an honest effort to study, but the +turmoil within him was too great. In turn he took his place in the +reading class and made a botch of it; then in the geography class and +turned lakes into mountains, mountains into rivers, and rivers into +continents, till chaos was come again; then in the spelling class, and +got "turned down," by a succession of mere baby words, till he brought +up at the foot and yielded up the pewter medal which he had worn with +ostentation for months. + + + +CHAPTER VII + +THE harder Tom tried to fasten his mind on his book, the more his +ideas wandered. So at last, with a sigh and a yawn, he gave it up. It +seemed to him that the noon recess would never come. The air was +utterly dead. There was not a breath stirring. It was the sleepiest of +sleepy days. The drowsing murmur of the five and twenty studying +scholars soothed the soul like the spell that is in the murmur of bees. +Away off in the flaming sunshine, Cardiff Hill lifted its soft green +sides through a shimmering veil of heat, tinted with the purple of +distance; a few birds floated on lazy wing high in the air; no other +living thing was visible but some cows, and they were asleep. Tom's +heart ached to be free, or else to have something of interest to do to +pass the dreary time. His hand wandered into his pocket and his face +lit up with a glow of gratitude that was prayer, though he did not know +it. Then furtively the percussion-cap box came out. He released the +tick and put him on the long flat desk. The creature probably glowed +with a gratitude that amounted to prayer, too, at this moment, but it +was premature: for when he started thankfully to travel off, Tom turned +him aside with a pin and made him take a new direction. + +Tom's bosom friend sat next him, suffering just as Tom had been, and +now he was deeply and gratefully interested in this entertainment in an +instant. This bosom friend was Joe Harper. The two boys were sworn +friends all the week, and embattled enemies on Saturdays. Joe took a +pin out of his lapel and began to assist in exercising the prisoner. +The sport grew in interest momently. Soon Tom said that they were +interfering with each other, and neither getting the fullest benefit of +the tick. So he put Joe's slate on the desk and drew a line down the +middle of it from top to bottom. + +"Now," said he, "as long as he is on your side you can stir him up and +I'll let him alone; but if you let him get away and get on my side, +you're to leave him alone as long as I can keep him from crossing over." + +"All right, go ahead; start him up." + +The tick escaped from Tom, presently, and crossed the equator. Joe +harassed him awhile, and then he got away and crossed back again. This +change of base occurred often. While one boy was worrying the tick with +absorbing interest, the other would look on with interest as strong, +the two heads bowed together over the slate, and the two souls dead to +all things else. At last luck seemed to settle and abide with Joe. The +tick tried this, that, and the other course, and got as excited and as +anxious as the boys themselves, but time and again just as he would +have victory in his very grasp, so to speak, and Tom's fingers would be +twitching to begin, Joe's pin would deftly head him off, and keep +possession. At last Tom could stand it no longer. The temptation was +too strong. So he reached out and lent a hand with his pin. Joe was +angry in a moment. Said he: + +"Tom, you let him alone." + +"I only just want to stir him up a little, Joe." + +"No, sir, it ain't fair; you just let him alone." + +"Blame it, I ain't going to stir him much." + +"Let him alone, I tell you." + +"I won't!" + +"You shall--he's on my side of the line." + +"Look here, Joe Harper, whose is that tick?" + +"I don't care whose tick he is--he's on my side of the line, and you +sha'n't touch him." + +"Well, I'll just bet I will, though. He's my tick and I'll do what I +blame please with him, or die!" + +A tremendous whack came down on Tom's shoulders, and its duplicate on +Joe's; and for the space of two minutes the dust continued to fly from +the two jackets and the whole school to enjoy it. The boys had been too +absorbed to notice the hush that had stolen upon the school awhile +before when the master came tiptoeing down the room and stood over +them. He had contemplated a good part of the performance before he +contributed his bit of variety to it. + +When school broke up at noon, Tom flew to Becky Thatcher, and +whispered in her ear: + +"Put on your bonnet and let on you're going home; and when you get to +the corner, give the rest of 'em the slip, and turn down through the +lane and come back. I'll go the other way and come it over 'em the same +way." + +So the one went off with one group of scholars, and the other with +another. In a little while the two met at the bottom of the lane, and +when they reached the school they had it all to themselves. Then they +sat together, with a slate before them, and Tom gave Becky the pencil +and held her hand in his, guiding it, and so created another surprising +house. When the interest in art began to wane, the two fell to talking. +Tom was swimming in bliss. He said: + +"Do you love rats?" + +"No! I hate them!" + +"Well, I do, too--LIVE ones. But I mean dead ones, to swing round your +head with a string." + +"No, I don't care for rats much, anyway. What I like is chewing-gum." + +"Oh, I should say so! I wish I had some now." + +"Do you? I've got some. I'll let you chew it awhile, but you must give +it back to me." + +That was agreeable, so they chewed it turn about, and dangled their +legs against the bench in excess of contentment. + +"Was you ever at a circus?" said Tom. + +"Yes, and my pa's going to take me again some time, if I'm good." + +"I been to the circus three or four times--lots of times. Church ain't +shucks to a circus. There's things going on at a circus all the time. +I'm going to be a clown in a circus when I grow up." + +"Oh, are you! That will be nice. They're so lovely, all spotted up." + +"Yes, that's so. And they get slathers of money--most a dollar a day, +Ben Rogers says. Say, Becky, was you ever engaged?" + +"What's that?" + +"Why, engaged to be married." + +"No." + +"Would you like to?" + +"I reckon so. I don't know. What is it like?" + +"Like? Why it ain't like anything. You only just tell a boy you won't +ever have anybody but him, ever ever ever, and then you kiss and that's +all. Anybody can do it." + +"Kiss? What do you kiss for?" + +"Why, that, you know, is to--well, they always do that." + +"Everybody?" + +"Why, yes, everybody that's in love with each other. Do you remember +what I wrote on the slate?" + +"Ye--yes." + +"What was it?" + +"I sha'n't tell you." + +"Shall I tell YOU?" + +"Ye--yes--but some other time." + +"No, now." + +"No, not now--to-morrow." + +"Oh, no, NOW. Please, Becky--I'll whisper it, I'll whisper it ever so +easy." + +Becky hesitating, Tom took silence for consent, and passed his arm +about her waist and whispered the tale ever so softly, with his mouth +close to her ear. And then he added: + +"Now you whisper it to me--just the same." + +She resisted, for a while, and then said: + +"You turn your face away so you can't see, and then I will. But you +mustn't ever tell anybody--WILL you, Tom? Now you won't, WILL you?" + +"No, indeed, indeed I won't. Now, Becky." + +He turned his face away. She bent timidly around till her breath +stirred his curls and whispered, "I--love--you!" + +Then she sprang away and ran around and around the desks and benches, +with Tom after her, and took refuge in a corner at last, with her +little white apron to her face. Tom clasped her about her neck and +pleaded: + +"Now, Becky, it's all done--all over but the kiss. Don't you be afraid +of that--it ain't anything at all. Please, Becky." And he tugged at her +apron and the hands. + +By and by she gave up, and let her hands drop; her face, all glowing +with the struggle, came up and submitted. Tom kissed the red lips and +said: + +"Now it's all done, Becky. And always after this, you know, you ain't +ever to love anybody but me, and you ain't ever to marry anybody but +me, ever never and forever. Will you?" + +"No, I'll never love anybody but you, Tom, and I'll never marry +anybody but you--and you ain't to ever marry anybody but me, either." + +"Certainly. Of course. That's PART of it. And always coming to school +or when we're going home, you're to walk with me, when there ain't +anybody looking--and you choose me and I choose you at parties, because +that's the way you do when you're engaged." + +"It's so nice. I never heard of it before." + +"Oh, it's ever so gay! Why, me and Amy Lawrence--" + +The big eyes told Tom his blunder and he stopped, confused. + +"Oh, Tom! Then I ain't the first you've ever been engaged to!" + +The child began to cry. Tom said: + +"Oh, don't cry, Becky, I don't care for her any more." + +"Yes, you do, Tom--you know you do." + +Tom tried to put his arm about her neck, but she pushed him away and +turned her face to the wall, and went on crying. Tom tried again, with +soothing words in his mouth, and was repulsed again. Then his pride was +up, and he strode away and went outside. He stood about, restless and +uneasy, for a while, glancing at the door, every now and then, hoping +she would repent and come to find him. But she did not. Then he began +to feel badly and fear that he was in the wrong. It was a hard struggle +with him to make new advances, now, but he nerved himself to it and +entered. She was still standing back there in the corner, sobbing, with +her face to the wall. Tom's heart smote him. He went to her and stood a +moment, not knowing exactly how to proceed. Then he said hesitatingly: + +"Becky, I--I don't care for anybody but you." + +No reply--but sobs. + +"Becky"--pleadingly. "Becky, won't you say something?" + +More sobs. + +Tom got out his chiefest jewel, a brass knob from the top of an +andiron, and passed it around her so that she could see it, and said: + +"Please, Becky, won't you take it?" + +She struck it to the floor. Then Tom marched out of the house and over +the hills and far away, to return to school no more that day. Presently +Becky began to suspect. She ran to the door; he was not in sight; she +flew around to the play-yard; he was not there. Then she called: + +"Tom! Come back, Tom!" + +She listened intently, but there was no answer. She had no companions +but silence and loneliness. So she sat down to cry again and upbraid +herself; and by this time the scholars began to gather again, and she +had to hide her griefs and still her broken heart and take up the cross +of a long, dreary, aching afternoon, with none among the strangers +about her to exchange sorrows with. + + + +CHAPTER VIII + +TOM dodged hither and thither through lanes until he was well out of +the track of returning scholars, and then fell into a moody jog. He +crossed a small "branch" two or three times, because of a prevailing +juvenile superstition that to cross water baffled pursuit. Half an hour +later he was disappearing behind the Douglas mansion on the summit of +Cardiff Hill, and the schoolhouse was hardly distinguishable away off +in the valley behind him. He entered a dense wood, picked his pathless +way to the centre of it, and sat down on a mossy spot under a spreading +oak. There was not even a zephyr stirring; the dead noonday heat had +even stilled the songs of the birds; nature lay in a trance that was +broken by no sound but the occasional far-off hammering of a +woodpecker, and this seemed to render the pervading silence and sense +of loneliness the more profound. The boy's soul was steeped in +melancholy; his feelings were in happy accord with his surroundings. He +sat long with his elbows on his knees and his chin in his hands, +meditating. It seemed to him that life was but a trouble, at best, and +he more than half envied Jimmy Hodges, so lately released; it must be +very peaceful, he thought, to lie and slumber and dream forever and +ever, with the wind whispering through the trees and caressing the +grass and the flowers over the grave, and nothing to bother and grieve +about, ever any more. If he only had a clean Sunday-school record he +could be willing to go, and be done with it all. Now as to this girl. +What had he done? Nothing. He had meant the best in the world, and been +treated like a dog--like a very dog. She would be sorry some day--maybe +when it was too late. Ah, if he could only die TEMPORARILY! + +But the elastic heart of youth cannot be compressed into one +constrained shape long at a time. Tom presently began to drift +insensibly back into the concerns of this life again. What if he turned +his back, now, and disappeared mysteriously? What if he went away--ever +so far away, into unknown countries beyond the seas--and never came +back any more! How would she feel then! The idea of being a clown +recurred to him now, only to fill him with disgust. For frivolity and +jokes and spotted tights were an offense, when they intruded themselves +upon a spirit that was exalted into the vague august realm of the +romantic. No, he would be a soldier, and return after long years, all +war-worn and illustrious. No--better still, he would join the Indians, +and hunt buffaloes and go on the warpath in the mountain ranges and the +trackless great plains of the Far West, and away in the future come +back a great chief, bristling with feathers, hideous with paint, and +prance into Sunday-school, some drowsy summer morning, with a +bloodcurdling war-whoop, and sear the eyeballs of all his companions +with unappeasable envy. But no, there was something gaudier even than +this. He would be a pirate! That was it! NOW his future lay plain +before him, and glowing with unimaginable splendor. How his name would +fill the world, and make people shudder! How gloriously he would go +plowing the dancing seas, in his long, low, black-hulled racer, the +Spirit of the Storm, with his grisly flag flying at the fore! And at +the zenith of his fame, how he would suddenly appear at the old village +and stalk into church, brown and weather-beaten, in his black velvet +doublet and trunks, his great jack-boots, his crimson sash, his belt +bristling with horse-pistols, his crime-rusted cutlass at his side, his +slouch hat with waving plumes, his black flag unfurled, with the skull +and crossbones on it, and hear with swelling ecstasy the whisperings, +"It's Tom Sawyer the Pirate!--the Black Avenger of the Spanish Main!" + +Yes, it was settled; his career was determined. He would run away from +home and enter upon it. He would start the very next morning. Therefore +he must now begin to get ready. He would collect his resources +together. He went to a rotten log near at hand and began to dig under +one end of it with his Barlow knife. He soon struck wood that sounded +hollow. He put his hand there and uttered this incantation impressively: + +"What hasn't come here, come! What's here, stay here!" + +Then he scraped away the dirt, and exposed a pine shingle. He took it +up and disclosed a shapely little treasure-house whose bottom and sides +were of shingles. In it lay a marble. Tom's astonishment was boundless! +He scratched his head with a perplexed air, and said: + +"Well, that beats anything!" + +Then he tossed the marble away pettishly, and stood cogitating. The +truth was, that a superstition of his had failed, here, which he and +all his comrades had always looked upon as infallible. If you buried a +marble with certain necessary incantations, and left it alone a +fortnight, and then opened the place with the incantation he had just +used, you would find that all the marbles you had ever lost had +gathered themselves together there, meantime, no matter how widely they +had been separated. But now, this thing had actually and unquestionably +failed. Tom's whole structure of faith was shaken to its foundations. +He had many a time heard of this thing succeeding but never of its +failing before. It did not occur to him that he had tried it several +times before, himself, but could never find the hiding-places +afterward. He puzzled over the matter some time, and finally decided +that some witch had interfered and broken the charm. He thought he +would satisfy himself on that point; so he searched around till he +found a small sandy spot with a little funnel-shaped depression in it. +He laid himself down and put his mouth close to this depression and +called-- + +"Doodle-bug, doodle-bug, tell me what I want to know! Doodle-bug, +doodle-bug, tell me what I want to know!" + +The sand began to work, and presently a small black bug appeared for a +second and then darted under again in a fright. + +"He dasn't tell! So it WAS a witch that done it. I just knowed it." + +He well knew the futility of trying to contend against witches, so he +gave up discouraged. But it occurred to him that he might as well have +the marble he had just thrown away, and therefore he went and made a +patient search for it. But he could not find it. Now he went back to +his treasure-house and carefully placed himself just as he had been +standing when he tossed the marble away; then he took another marble +from his pocket and tossed it in the same way, saying: + +"Brother, go find your brother!" + +He watched where it stopped, and went there and looked. But it must +have fallen short or gone too far; so he tried twice more. The last +repetition was successful. The two marbles lay within a foot of each +other. + +Just here the blast of a toy tin trumpet came faintly down the green +aisles of the forest. Tom flung off his jacket and trousers, turned a +suspender into a belt, raked away some brush behind the rotten log, +disclosing a rude bow and arrow, a lath sword and a tin trumpet, and in +a moment had seized these things and bounded away, barelegged, with +fluttering shirt. He presently halted under a great elm, blew an +answering blast, and then began to tiptoe and look warily out, this way +and that. He said cautiously--to an imaginary company: + +"Hold, my merry men! Keep hid till I blow." + +Now appeared Joe Harper, as airily clad and elaborately armed as Tom. +Tom called: + +"Hold! Who comes here into Sherwood Forest without my pass?" + +"Guy of Guisborne wants no man's pass. Who art thou that--that--" + +"Dares to hold such language," said Tom, prompting--for they talked +"by the book," from memory. + +"Who art thou that dares to hold such language?" + +"I, indeed! I am Robin Hood, as thy caitiff carcase soon shall know." + +"Then art thou indeed that famous outlaw? Right gladly will I dispute +with thee the passes of the merry wood. Have at thee!" + +They took their lath swords, dumped their other traps on the ground, +struck a fencing attitude, foot to foot, and began a grave, careful +combat, "two up and two down." Presently Tom said: + +"Now, if you've got the hang, go it lively!" + +So they "went it lively," panting and perspiring with the work. By and +by Tom shouted: + +"Fall! fall! Why don't you fall?" + +"I sha'n't! Why don't you fall yourself? You're getting the worst of +it." + +"Why, that ain't anything. I can't fall; that ain't the way it is in +the book. The book says, 'Then with one back-handed stroke he slew poor +Guy of Guisborne.' You're to turn around and let me hit you in the +back." + +There was no getting around the authorities, so Joe turned, received +the whack and fell. + +"Now," said Joe, getting up, "you got to let me kill YOU. That's fair." + +"Why, I can't do that, it ain't in the book." + +"Well, it's blamed mean--that's all." + +"Well, say, Joe, you can be Friar Tuck or Much the miller's son, and +lam me with a quarter-staff; or I'll be the Sheriff of Nottingham and +you be Robin Hood a little while and kill me." + +This was satisfactory, and so these adventures were carried out. Then +Tom became Robin Hood again, and was allowed by the treacherous nun to +bleed his strength away through his neglected wound. And at last Joe, +representing a whole tribe of weeping outlaws, dragged him sadly forth, +gave his bow into his feeble hands, and Tom said, "Where this arrow +falls, there bury poor Robin Hood under the greenwood tree." Then he +shot the arrow and fell back and would have died, but he lit on a +nettle and sprang up too gaily for a corpse. + +The boys dressed themselves, hid their accoutrements, and went off +grieving that there were no outlaws any more, and wondering what modern +civilization could claim to have done to compensate for their loss. +They said they would rather be outlaws a year in Sherwood Forest than +President of the United States forever. + + + +CHAPTER IX + +AT half-past nine, that night, Tom and Sid were sent to bed, as usual. +They said their prayers, and Sid was soon asleep. Tom lay awake and +waited, in restless impatience. When it seemed to him that it must be +nearly daylight, he heard the clock strike ten! This was despair. He +would have tossed and fidgeted, as his nerves demanded, but he was +afraid he might wake Sid. So he lay still, and stared up into the dark. +Everything was dismally still. By and by, out of the stillness, little, +scarcely perceptible noises began to emphasize themselves. The ticking +of the clock began to bring itself into notice. Old beams began to +crack mysteriously. The stairs creaked faintly. Evidently spirits were +abroad. A measured, muffled snore issued from Aunt Polly's chamber. And +now the tiresome chirping of a cricket that no human ingenuity could +locate, began. Next the ghastly ticking of a deathwatch in the wall at +the bed's head made Tom shudder--it meant that somebody's days were +numbered. Then the howl of a far-off dog rose on the night air, and was +answered by a fainter howl from a remoter distance. Tom was in an +agony. At last he was satisfied that time had ceased and eternity +begun; he began to doze, in spite of himself; the clock chimed eleven, +but he did not hear it. And then there came, mingling with his +half-formed dreams, a most melancholy caterwauling. The raising of a +neighboring window disturbed him. A cry of "Scat! you devil!" and the +crash of an empty bottle against the back of his aunt's woodshed +brought him wide awake, and a single minute later he was dressed and +out of the window and creeping along the roof of the "ell" on all +fours. He "meow'd" with caution once or twice, as he went; then jumped +to the roof of the woodshed and thence to the ground. Huckleberry Finn +was there, with his dead cat. The boys moved off and disappeared in the +gloom. At the end of half an hour they were wading through the tall +grass of the graveyard. + +It was a graveyard of the old-fashioned Western kind. It was on a +hill, about a mile and a half from the village. It had a crazy board +fence around it, which leaned inward in places, and outward the rest of +the time, but stood upright nowhere. Grass and weeds grew rank over the +whole cemetery. All the old graves were sunken in, there was not a +tombstone on the place; round-topped, worm-eaten boards staggered over +the graves, leaning for support and finding none. "Sacred to the memory +of" So-and-So had been painted on them once, but it could no longer +have been read, on the most of them, now, even if there had been light. + +A faint wind moaned through the trees, and Tom feared it might be the +spirits of the dead, complaining at being disturbed. The boys talked +little, and only under their breath, for the time and the place and the +pervading solemnity and silence oppressed their spirits. They found the +sharp new heap they were seeking, and ensconced themselves within the +protection of three great elms that grew in a bunch within a few feet +of the grave. + +Then they waited in silence for what seemed a long time. The hooting +of a distant owl was all the sound that troubled the dead stillness. +Tom's reflections grew oppressive. He must force some talk. So he said +in a whisper: + +"Hucky, do you believe the dead people like it for us to be here?" + +Huckleberry whispered: + +"I wisht I knowed. It's awful solemn like, AIN'T it?" + +"I bet it is." + +There was a considerable pause, while the boys canvassed this matter +inwardly. Then Tom whispered: + +"Say, Hucky--do you reckon Hoss Williams hears us talking?" + +"O' course he does. Least his sperrit does." + +Tom, after a pause: + +"I wish I'd said Mister Williams. But I never meant any harm. +Everybody calls him Hoss." + +"A body can't be too partic'lar how they talk 'bout these-yer dead +people, Tom." + +This was a damper, and conversation died again. + +Presently Tom seized his comrade's arm and said: + +"Sh!" + +"What is it, Tom?" And the two clung together with beating hearts. + +"Sh! There 'tis again! Didn't you hear it?" + +"I--" + +"There! Now you hear it." + +"Lord, Tom, they're coming! They're coming, sure. What'll we do?" + +"I dono. Think they'll see us?" + +"Oh, Tom, they can see in the dark, same as cats. I wisht I hadn't +come." + +"Oh, don't be afeard. I don't believe they'll bother us. We ain't +doing any harm. If we keep perfectly still, maybe they won't notice us +at all." + +"I'll try to, Tom, but, Lord, I'm all of a shiver." + +"Listen!" + +The boys bent their heads together and scarcely breathed. A muffled +sound of voices floated up from the far end of the graveyard. + +"Look! See there!" whispered Tom. "What is it?" + +"It's devil-fire. Oh, Tom, this is awful." + +Some vague figures approached through the gloom, swinging an +old-fashioned tin lantern that freckled the ground with innumerable +little spangles of light. Presently Huckleberry whispered with a +shudder: + +"It's the devils sure enough. Three of 'em! Lordy, Tom, we're goners! +Can you pray?" + +"I'll try, but don't you be afeard. They ain't going to hurt us. 'Now +I lay me down to sleep, I--'" + +"Sh!" + +"What is it, Huck?" + +"They're HUMANS! One of 'em is, anyway. One of 'em's old Muff Potter's +voice." + +"No--'tain't so, is it?" + +"I bet I know it. Don't you stir nor budge. He ain't sharp enough to +notice us. Drunk, the same as usual, likely--blamed old rip!" + +"All right, I'll keep still. Now they're stuck. Can't find it. Here +they come again. Now they're hot. Cold again. Hot again. Red hot! +They're p'inted right, this time. Say, Huck, I know another o' them +voices; it's Injun Joe." + +"That's so--that murderin' half-breed! I'd druther they was devils a +dern sight. What kin they be up to?" + +The whisper died wholly out, now, for the three men had reached the +grave and stood within a few feet of the boys' hiding-place. + +"Here it is," said the third voice; and the owner of it held the +lantern up and revealed the face of young Doctor Robinson. + +Potter and Injun Joe were carrying a handbarrow with a rope and a +couple of shovels on it. They cast down their load and began to open +the grave. The doctor put the lantern at the head of the grave and came +and sat down with his back against one of the elm trees. He was so +close the boys could have touched him. + +"Hurry, men!" he said, in a low voice; "the moon might come out at any +moment." + +They growled a response and went on digging. For some time there was +no noise but the grating sound of the spades discharging their freight +of mould and gravel. It was very monotonous. Finally a spade struck +upon the coffin with a dull woody accent, and within another minute or +two the men had hoisted it out on the ground. They pried off the lid +with their shovels, got out the body and dumped it rudely on the +ground. The moon drifted from behind the clouds and exposed the pallid +face. The barrow was got ready and the corpse placed on it, covered +with a blanket, and bound to its place with the rope. Potter took out a +large spring-knife and cut off the dangling end of the rope and then +said: + +"Now the cussed thing's ready, Sawbones, and you'll just out with +another five, or here she stays." + +"That's the talk!" said Injun Joe. + +"Look here, what does this mean?" said the doctor. "You required your +pay in advance, and I've paid you." + +"Yes, and you done more than that," said Injun Joe, approaching the +doctor, who was now standing. "Five years ago you drove me away from +your father's kitchen one night, when I come to ask for something to +eat, and you said I warn't there for any good; and when I swore I'd get +even with you if it took a hundred years, your father had me jailed for +a vagrant. Did you think I'd forget? The Injun blood ain't in me for +nothing. And now I've GOT you, and you got to SETTLE, you know!" + +He was threatening the doctor, with his fist in his face, by this +time. The doctor struck out suddenly and stretched the ruffian on the +ground. Potter dropped his knife, and exclaimed: + +"Here, now, don't you hit my pard!" and the next moment he had +grappled with the doctor and the two were struggling with might and +main, trampling the grass and tearing the ground with their heels. +Injun Joe sprang to his feet, his eyes flaming with passion, snatched +up Potter's knife, and went creeping, catlike and stooping, round and +round about the combatants, seeking an opportunity. All at once the +doctor flung himself free, seized the heavy headboard of Williams' +grave and felled Potter to the earth with it--and in the same instant +the half-breed saw his chance and drove the knife to the hilt in the +young man's breast. He reeled and fell partly upon Potter, flooding him +with his blood, and in the same moment the clouds blotted out the +dreadful spectacle and the two frightened boys went speeding away in +the dark. + +Presently, when the moon emerged again, Injun Joe was standing over +the two forms, contemplating them. The doctor murmured inarticulately, +gave a long gasp or two and was still. The half-breed muttered: + +"THAT score is settled--damn you." + +Then he robbed the body. After which he put the fatal knife in +Potter's open right hand, and sat down on the dismantled coffin. Three +--four--five minutes passed, and then Potter began to stir and moan. His +hand closed upon the knife; he raised it, glanced at it, and let it +fall, with a shudder. Then he sat up, pushing the body from him, and +gazed at it, and then around him, confusedly. His eyes met Joe's. + +"Lord, how is this, Joe?" he said. + +"It's a dirty business," said Joe, without moving. + +"What did you do it for?" + +"I! I never done it!" + +"Look here! That kind of talk won't wash." + +Potter trembled and grew white. + +"I thought I'd got sober. I'd no business to drink to-night. But it's +in my head yet--worse'n when we started here. I'm all in a muddle; +can't recollect anything of it, hardly. Tell me, Joe--HONEST, now, old +feller--did I do it? Joe, I never meant to--'pon my soul and honor, I +never meant to, Joe. Tell me how it was, Joe. Oh, it's awful--and him +so young and promising." + +"Why, you two was scuffling, and he fetched you one with the headboard +and you fell flat; and then up you come, all reeling and staggering +like, and snatched the knife and jammed it into him, just as he fetched +you another awful clip--and here you've laid, as dead as a wedge til +now." + +"Oh, I didn't know what I was a-doing. I wish I may die this minute if +I did. It was all on account of the whiskey and the excitement, I +reckon. I never used a weepon in my life before, Joe. I've fought, but +never with weepons. They'll all say that. Joe, don't tell! Say you +won't tell, Joe--that's a good feller. I always liked you, Joe, and +stood up for you, too. Don't you remember? You WON'T tell, WILL you, +Joe?" And the poor creature dropped on his knees before the stolid +murderer, and clasped his appealing hands. + +"No, you've always been fair and square with me, Muff Potter, and I +won't go back on you. There, now, that's as fair as a man can say." + +"Oh, Joe, you're an angel. I'll bless you for this the longest day I +live." And Potter began to cry. + +"Come, now, that's enough of that. This ain't any time for blubbering. +You be off yonder way and I'll go this. Move, now, and don't leave any +tracks behind you." + +Potter started on a trot that quickly increased to a run. The +half-breed stood looking after him. He muttered: + +"If he's as much stunned with the lick and fuddled with the rum as he +had the look of being, he won't think of the knife till he's gone so +far he'll be afraid to come back after it to such a place by himself +--chicken-heart!" + +Two or three minutes later the murdered man, the blanketed corpse, the +lidless coffin, and the open grave were under no inspection but the +moon's. The stillness was complete again, too. + + + +CHAPTER X + +THE two boys flew on and on, toward the village, speechless with +horror. They glanced backward over their shoulders from time to time, +apprehensively, as if they feared they might be followed. Every stump +that started up in their path seemed a man and an enemy, and made them +catch their breath; and as they sped by some outlying cottages that lay +near the village, the barking of the aroused watch-dogs seemed to give +wings to their feet. + +"If we can only get to the old tannery before we break down!" +whispered Tom, in short catches between breaths. "I can't stand it much +longer." + +Huckleberry's hard pantings were his only reply, and the boys fixed +their eyes on the goal of their hopes and bent to their work to win it. +They gained steadily on it, and at last, breast to breast, they burst +through the open door and fell grateful and exhausted in the sheltering +shadows beyond. By and by their pulses slowed down, and Tom whispered: + +"Huckleberry, what do you reckon'll come of this?" + +"If Doctor Robinson dies, I reckon hanging'll come of it." + +"Do you though?" + +"Why, I KNOW it, Tom." + +Tom thought a while, then he said: + +"Who'll tell? We?" + +"What are you talking about? S'pose something happened and Injun Joe +DIDN'T hang? Why, he'd kill us some time or other, just as dead sure as +we're a laying here." + +"That's just what I was thinking to myself, Huck." + +"If anybody tells, let Muff Potter do it, if he's fool enough. He's +generally drunk enough." + +Tom said nothing--went on thinking. Presently he whispered: + +"Huck, Muff Potter don't know it. How can he tell?" + +"What's the reason he don't know it?" + +"Because he'd just got that whack when Injun Joe done it. D'you reckon +he could see anything? D'you reckon he knowed anything?" + +"By hokey, that's so, Tom!" + +"And besides, look-a-here--maybe that whack done for HIM!" + +"No, 'taint likely, Tom. He had liquor in him; I could see that; and +besides, he always has. Well, when pap's full, you might take and belt +him over the head with a church and you couldn't phase him. He says so, +his own self. So it's the same with Muff Potter, of course. But if a +man was dead sober, I reckon maybe that whack might fetch him; I dono." + +After another reflective silence, Tom said: + +"Hucky, you sure you can keep mum?" + +"Tom, we GOT to keep mum. You know that. That Injun devil wouldn't +make any more of drownding us than a couple of cats, if we was to +squeak 'bout this and they didn't hang him. Now, look-a-here, Tom, less +take and swear to one another--that's what we got to do--swear to keep +mum." + +"I'm agreed. It's the best thing. Would you just hold hands and swear +that we--" + +"Oh no, that wouldn't do for this. That's good enough for little +rubbishy common things--specially with gals, cuz THEY go back on you +anyway, and blab if they get in a huff--but there orter be writing +'bout a big thing like this. And blood." + +Tom's whole being applauded this idea. It was deep, and dark, and +awful; the hour, the circumstances, the surroundings, were in keeping +with it. He picked up a clean pine shingle that lay in the moonlight, +took a little fragment of "red keel" out of his pocket, got the moon on +his work, and painfully scrawled these lines, emphasizing each slow +down-stroke by clamping his tongue between his teeth, and letting up +the pressure on the up-strokes. [See next page.] + + "Huck Finn and + Tom Sawyer swears + they will keep mum + about This and They + wish They may Drop + down dead in Their + Tracks if They ever + Tell and Rot." + +Huckleberry was filled with admiration of Tom's facility in writing, +and the sublimity of his language. He at once took a pin from his lapel +and was going to prick his flesh, but Tom said: + +"Hold on! Don't do that. A pin's brass. It might have verdigrease on +it." + +"What's verdigrease?" + +"It's p'ison. That's what it is. You just swaller some of it once +--you'll see." + +So Tom unwound the thread from one of his needles, and each boy +pricked the ball of his thumb and squeezed out a drop of blood. In +time, after many squeezes, Tom managed to sign his initials, using the +ball of his little finger for a pen. Then he showed Huckleberry how to +make an H and an F, and the oath was complete. They buried the shingle +close to the wall, with some dismal ceremonies and incantations, and +the fetters that bound their tongues were considered to be locked and +the key thrown away. + +A figure crept stealthily through a break in the other end of the +ruined building, now, but they did not notice it. + +"Tom," whispered Huckleberry, "does this keep us from EVER telling +--ALWAYS?" + +"Of course it does. It don't make any difference WHAT happens, we got +to keep mum. We'd drop down dead--don't YOU know that?" + +"Yes, I reckon that's so." + +They continued to whisper for some little time. Presently a dog set up +a long, lugubrious howl just outside--within ten feet of them. The boys +clasped each other suddenly, in an agony of fright. + +"Which of us does he mean?" gasped Huckleberry. + +"I dono--peep through the crack. Quick!" + +"No, YOU, Tom!" + +"I can't--I can't DO it, Huck!" + +"Please, Tom. There 'tis again!" + +"Oh, lordy, I'm thankful!" whispered Tom. "I know his voice. It's Bull +Harbison." * + +[* If Mr. Harbison owned a slave named Bull, Tom would have spoken of +him as "Harbison's Bull," but a son or a dog of that name was "Bull +Harbison."] + +"Oh, that's good--I tell you, Tom, I was most scared to death; I'd a +bet anything it was a STRAY dog." + +The dog howled again. The boys' hearts sank once more. + +"Oh, my! that ain't no Bull Harbison!" whispered Huckleberry. "DO, Tom!" + +Tom, quaking with fear, yielded, and put his eye to the crack. His +whisper was hardly audible when he said: + +"Oh, Huck, IT S A STRAY DOG!" + +"Quick, Tom, quick! Who does he mean?" + +"Huck, he must mean us both--we're right together." + +"Oh, Tom, I reckon we're goners. I reckon there ain't no mistake 'bout +where I'LL go to. I been so wicked." + +"Dad fetch it! This comes of playing hookey and doing everything a +feller's told NOT to do. I might a been good, like Sid, if I'd a tried +--but no, I wouldn't, of course. But if ever I get off this time, I lay +I'll just WALLER in Sunday-schools!" And Tom began to snuffle a little. + +"YOU bad!" and Huckleberry began to snuffle too. "Consound it, Tom +Sawyer, you're just old pie, 'longside o' what I am. Oh, LORDY, lordy, +lordy, I wisht I only had half your chance." + +Tom choked off and whispered: + +"Look, Hucky, look! He's got his BACK to us!" + +Hucky looked, with joy in his heart. + +"Well, he has, by jingoes! Did he before?" + +"Yes, he did. But I, like a fool, never thought. Oh, this is bully, +you know. NOW who can he mean?" + +The howling stopped. Tom pricked up his ears. + +"Sh! What's that?" he whispered. + +"Sounds like--like hogs grunting. No--it's somebody snoring, Tom." + +"That IS it! Where 'bouts is it, Huck?" + +"I bleeve it's down at 'tother end. Sounds so, anyway. Pap used to +sleep there, sometimes, 'long with the hogs, but laws bless you, he +just lifts things when HE snores. Besides, I reckon he ain't ever +coming back to this town any more." + +The spirit of adventure rose in the boys' souls once more. + +"Hucky, do you das't to go if I lead?" + +"I don't like to, much. Tom, s'pose it's Injun Joe!" + +Tom quailed. But presently the temptation rose up strong again and the +boys agreed to try, with the understanding that they would take to +their heels if the snoring stopped. So they went tiptoeing stealthily +down, the one behind the other. When they had got to within five steps +of the snorer, Tom stepped on a stick, and it broke with a sharp snap. +The man moaned, writhed a little, and his face came into the moonlight. +It was Muff Potter. The boys' hearts had stood still, and their hopes +too, when the man moved, but their fears passed away now. They tiptoed +out, through the broken weather-boarding, and stopped at a little +distance to exchange a parting word. That long, lugubrious howl rose on +the night air again! They turned and saw the strange dog standing +within a few feet of where Potter was lying, and FACING Potter, with +his nose pointing heavenward. + +"Oh, geeminy, it's HIM!" exclaimed both boys, in a breath. + +"Say, Tom--they say a stray dog come howling around Johnny Miller's +house, 'bout midnight, as much as two weeks ago; and a whippoorwill +come in and lit on the banisters and sung, the very same evening; and +there ain't anybody dead there yet." + +"Well, I know that. And suppose there ain't. Didn't Gracie Miller fall +in the kitchen fire and burn herself terrible the very next Saturday?" + +"Yes, but she ain't DEAD. And what's more, she's getting better, too." + +"All right, you wait and see. She's a goner, just as dead sure as Muff +Potter's a goner. That's what the niggers say, and they know all about +these kind of things, Huck." + +Then they separated, cogitating. When Tom crept in at his bedroom +window the night was almost spent. He undressed with excessive caution, +and fell asleep congratulating himself that nobody knew of his +escapade. He was not aware that the gently-snoring Sid was awake, and +had been so for an hour. + +When Tom awoke, Sid was dressed and gone. There was a late look in the +light, a late sense in the atmosphere. He was startled. Why had he not +been called--persecuted till he was up, as usual? The thought filled +him with bodings. Within five minutes he was dressed and down-stairs, +feeling sore and drowsy. The family were still at table, but they had +finished breakfast. There was no voice of rebuke; but there were +averted eyes; there was a silence and an air of solemnity that struck a +chill to the culprit's heart. He sat down and tried to seem gay, but it +was up-hill work; it roused no smile, no response, and he lapsed into +silence and let his heart sink down to the depths. + +After breakfast his aunt took him aside, and Tom almost brightened in +the hope that he was going to be flogged; but it was not so. His aunt +wept over him and asked him how he could go and break her old heart so; +and finally told him to go on, and ruin himself and bring her gray +hairs with sorrow to the grave, for it was no use for her to try any +more. This was worse than a thousand whippings, and Tom's heart was +sorer now than his body. He cried, he pleaded for forgiveness, promised +to reform over and over again, and then received his dismissal, feeling +that he had won but an imperfect forgiveness and established but a +feeble confidence. + +He left the presence too miserable to even feel revengeful toward Sid; +and so the latter's prompt retreat through the back gate was +unnecessary. He moped to school gloomy and sad, and took his flogging, +along with Joe Harper, for playing hookey the day before, with the air +of one whose heart was busy with heavier woes and wholly dead to +trifles. Then he betook himself to his seat, rested his elbows on his +desk and his jaws in his hands, and stared at the wall with the stony +stare of suffering that has reached the limit and can no further go. +His elbow was pressing against some hard substance. After a long time +he slowly and sadly changed his position, and took up this object with +a sigh. It was in a paper. He unrolled it. A long, lingering, colossal +sigh followed, and his heart broke. It was his brass andiron knob! + +This final feather broke the camel's back. + + + +CHAPTER XI + +CLOSE upon the hour of noon the whole village was suddenly electrified +with the ghastly news. No need of the as yet undreamed-of telegraph; +the tale flew from man to man, from group to group, from house to +house, with little less than telegraphic speed. Of course the +schoolmaster gave holiday for that afternoon; the town would have +thought strangely of him if he had not. + +A gory knife had been found close to the murdered man, and it had been +recognized by somebody as belonging to Muff Potter--so the story ran. +And it was said that a belated citizen had come upon Potter washing +himself in the "branch" about one or two o'clock in the morning, and +that Potter had at once sneaked off--suspicious circumstances, +especially the washing which was not a habit with Potter. It was also +said that the town had been ransacked for this "murderer" (the public +are not slow in the matter of sifting evidence and arriving at a +verdict), but that he could not be found. Horsemen had departed down +all the roads in every direction, and the Sheriff "was confident" that +he would be captured before night. + +All the town was drifting toward the graveyard. Tom's heartbreak +vanished and he joined the procession, not because he would not a +thousand times rather go anywhere else, but because an awful, +unaccountable fascination drew him on. Arrived at the dreadful place, +he wormed his small body through the crowd and saw the dismal +spectacle. It seemed to him an age since he was there before. Somebody +pinched his arm. He turned, and his eyes met Huckleberry's. Then both +looked elsewhere at once, and wondered if anybody had noticed anything +in their mutual glance. But everybody was talking, and intent upon the +grisly spectacle before them. + +"Poor fellow!" "Poor young fellow!" "This ought to be a lesson to +grave robbers!" "Muff Potter'll hang for this if they catch him!" This +was the drift of remark; and the minister said, "It was a judgment; His +hand is here." + +Now Tom shivered from head to heel; for his eye fell upon the stolid +face of Injun Joe. At this moment the crowd began to sway and struggle, +and voices shouted, "It's him! it's him! he's coming himself!" + +"Who? Who?" from twenty voices. + +"Muff Potter!" + +"Hallo, he's stopped!--Look out, he's turning! Don't let him get away!" + +People in the branches of the trees over Tom's head said he wasn't +trying to get away--he only looked doubtful and perplexed. + +"Infernal impudence!" said a bystander; "wanted to come and take a +quiet look at his work, I reckon--didn't expect any company." + +The crowd fell apart, now, and the Sheriff came through, +ostentatiously leading Potter by the arm. The poor fellow's face was +haggard, and his eyes showed the fear that was upon him. When he stood +before the murdered man, he shook as with a palsy, and he put his face +in his hands and burst into tears. + +"I didn't do it, friends," he sobbed; "'pon my word and honor I never +done it." + +"Who's accused you?" shouted a voice. + +This shot seemed to carry home. Potter lifted his face and looked +around him with a pathetic hopelessness in his eyes. He saw Injun Joe, +and exclaimed: + +"Oh, Injun Joe, you promised me you'd never--" + +"Is that your knife?" and it was thrust before him by the Sheriff. + +Potter would have fallen if they had not caught him and eased him to +the ground. Then he said: + +"Something told me 't if I didn't come back and get--" He shuddered; +then waved his nerveless hand with a vanquished gesture and said, "Tell +'em, Joe, tell 'em--it ain't any use any more." + +Then Huckleberry and Tom stood dumb and staring, and heard the +stony-hearted liar reel off his serene statement, they expecting every +moment that the clear sky would deliver God's lightnings upon his head, +and wondering to see how long the stroke was delayed. And when he had +finished and still stood alive and whole, their wavering impulse to +break their oath and save the poor betrayed prisoner's life faded and +vanished away, for plainly this miscreant had sold himself to Satan and +it would be fatal to meddle with the property of such a power as that. + +"Why didn't you leave? What did you want to come here for?" somebody +said. + +"I couldn't help it--I couldn't help it," Potter moaned. "I wanted to +run away, but I couldn't seem to come anywhere but here." And he fell +to sobbing again. + +Injun Joe repeated his statement, just as calmly, a few minutes +afterward on the inquest, under oath; and the boys, seeing that the +lightnings were still withheld, were confirmed in their belief that Joe +had sold himself to the devil. He was now become, to them, the most +balefully interesting object they had ever looked upon, and they could +not take their fascinated eyes from his face. + +They inwardly resolved to watch him nights, when opportunity should +offer, in the hope of getting a glimpse of his dread master. + +Injun Joe helped to raise the body of the murdered man and put it in a +wagon for removal; and it was whispered through the shuddering crowd +that the wound bled a little! The boys thought that this happy +circumstance would turn suspicion in the right direction; but they were +disappointed, for more than one villager remarked: + +"It was within three feet of Muff Potter when it done it." + +Tom's fearful secret and gnawing conscience disturbed his sleep for as +much as a week after this; and at breakfast one morning Sid said: + +"Tom, you pitch around and talk in your sleep so much that you keep me +awake half the time." + +Tom blanched and dropped his eyes. + +"It's a bad sign," said Aunt Polly, gravely. "What you got on your +mind, Tom?" + +"Nothing. Nothing 't I know of." But the boy's hand shook so that he +spilled his coffee. + +"And you do talk such stuff," Sid said. "Last night you said, 'It's +blood, it's blood, that's what it is!' You said that over and over. And +you said, 'Don't torment me so--I'll tell!' Tell WHAT? What is it +you'll tell?" + +Everything was swimming before Tom. There is no telling what might +have happened, now, but luckily the concern passed out of Aunt Polly's +face and she came to Tom's relief without knowing it. She said: + +"Sho! It's that dreadful murder. I dream about it most every night +myself. Sometimes I dream it's me that done it." + +Mary said she had been affected much the same way. Sid seemed +satisfied. Tom got out of the presence as quick as he plausibly could, +and after that he complained of toothache for a week, and tied up his +jaws every night. He never knew that Sid lay nightly watching, and +frequently slipped the bandage free and then leaned on his elbow +listening a good while at a time, and afterward slipped the bandage +back to its place again. Tom's distress of mind wore off gradually and +the toothache grew irksome and was discarded. If Sid really managed to +make anything out of Tom's disjointed mutterings, he kept it to himself. + +It seemed to Tom that his schoolmates never would get done holding +inquests on dead cats, and thus keeping his trouble present to his +mind. Sid noticed that Tom never was coroner at one of these inquiries, +though it had been his habit to take the lead in all new enterprises; +he noticed, too, that Tom never acted as a witness--and that was +strange; and Sid did not overlook the fact that Tom even showed a +marked aversion to these inquests, and always avoided them when he +could. Sid marvelled, but said nothing. However, even inquests went out +of vogue at last, and ceased to torture Tom's conscience. + +Every day or two, during this time of sorrow, Tom watched his +opportunity and went to the little grated jail-window and smuggled such +small comforts through to the "murderer" as he could get hold of. The +jail was a trifling little brick den that stood in a marsh at the edge +of the village, and no guards were afforded for it; indeed, it was +seldom occupied. These offerings greatly helped to ease Tom's +conscience. + +The villagers had a strong desire to tar-and-feather Injun Joe and +ride him on a rail, for body-snatching, but so formidable was his +character that nobody could be found who was willing to take the lead +in the matter, so it was dropped. He had been careful to begin both of +his inquest-statements with the fight, without confessing the +grave-robbery that preceded it; therefore it was deemed wisest not +to try the case in the courts at present. + + + +CHAPTER XII + +ONE of the reasons why Tom's mind had drifted away from its secret +troubles was, that it had found a new and weighty matter to interest +itself about. Becky Thatcher had stopped coming to school. Tom had +struggled with his pride a few days, and tried to "whistle her down the +wind," but failed. He began to find himself hanging around her father's +house, nights, and feeling very miserable. She was ill. What if she +should die! There was distraction in the thought. He no longer took an +interest in war, nor even in piracy. The charm of life was gone; there +was nothing but dreariness left. He put his hoop away, and his bat; +there was no joy in them any more. His aunt was concerned. She began to +try all manner of remedies on him. She was one of those people who are +infatuated with patent medicines and all new-fangled methods of +producing health or mending it. She was an inveterate experimenter in +these things. When something fresh in this line came out she was in a +fever, right away, to try it; not on herself, for she was never ailing, +but on anybody else that came handy. She was a subscriber for all the +"Health" periodicals and phrenological frauds; and the solemn ignorance +they were inflated with was breath to her nostrils. All the "rot" they +contained about ventilation, and how to go to bed, and how to get up, +and what to eat, and what to drink, and how much exercise to take, and +what frame of mind to keep one's self in, and what sort of clothing to +wear, was all gospel to her, and she never observed that her +health-journals of the current month customarily upset everything they +had recommended the month before. She was as simple-hearted and honest +as the day was long, and so she was an easy victim. She gathered +together her quack periodicals and her quack medicines, and thus armed +with death, went about on her pale horse, metaphorically speaking, with +"hell following after." But she never suspected that she was not an +angel of healing and the balm of Gilead in disguise, to the suffering +neighbors. + +The water treatment was new, now, and Tom's low condition was a +windfall to her. She had him out at daylight every morning, stood him +up in the woodshed and drowned him with a deluge of cold water; then +she scrubbed him down with a towel like a file, and so brought him to; +then she rolled him up in a wet sheet and put him away under blankets +till she sweated his soul clean and "the yellow stains of it came +through his pores"--as Tom said. + +Yet notwithstanding all this, the boy grew more and more melancholy +and pale and dejected. She added hot baths, sitz baths, shower baths, +and plunges. The boy remained as dismal as a hearse. She began to +assist the water with a slim oatmeal diet and blister-plasters. She +calculated his capacity as she would a jug's, and filled him up every +day with quack cure-alls. + +Tom had become indifferent to persecution by this time. This phase +filled the old lady's heart with consternation. This indifference must +be broken up at any cost. Now she heard of Pain-killer for the first +time. She ordered a lot at once. She tasted it and was filled with +gratitude. It was simply fire in a liquid form. She dropped the water +treatment and everything else, and pinned her faith to Pain-killer. She +gave Tom a teaspoonful and watched with the deepest anxiety for the +result. Her troubles were instantly at rest, her soul at peace again; +for the "indifference" was broken up. The boy could not have shown a +wilder, heartier interest, if she had built a fire under him. + +Tom felt that it was time to wake up; this sort of life might be +romantic enough, in his blighted condition, but it was getting to have +too little sentiment and too much distracting variety about it. So he +thought over various plans for relief, and finally hit pon that of +professing to be fond of Pain-killer. He asked for it so often that he +became a nuisance, and his aunt ended by telling him to help himself +and quit bothering her. If it had been Sid, she would have had no +misgivings to alloy her delight; but since it was Tom, she watched the +bottle clandestinely. She found that the medicine did really diminish, +but it did not occur to her that the boy was mending the health of a +crack in the sitting-room floor with it. + +One day Tom was in the act of dosing the crack when his aunt's yellow +cat came along, purring, eying the teaspoon avariciously, and begging +for a taste. Tom said: + +"Don't ask for it unless you want it, Peter." + +But Peter signified that he did want it. + +"You better make sure." + +Peter was sure. + +"Now you've asked for it, and I'll give it to you, because there ain't +anything mean about me; but if you find you don't like it, you mustn't +blame anybody but your own self." + +Peter was agreeable. So Tom pried his mouth open and poured down the +Pain-killer. Peter sprang a couple of yards in the air, and then +delivered a war-whoop and set off round and round the room, banging +against furniture, upsetting flower-pots, and making general havoc. +Next he rose on his hind feet and pranced around, in a frenzy of +enjoyment, with his head over his shoulder and his voice proclaiming +his unappeasable happiness. Then he went tearing around the house again +spreading chaos and destruction in his path. Aunt Polly entered in time +to see him throw a few double summersets, deliver a final mighty +hurrah, and sail through the open window, carrying the rest of the +flower-pots with him. The old lady stood petrified with astonishment, +peering over her glasses; Tom lay on the floor expiring with laughter. + +"Tom, what on earth ails that cat?" + +"I don't know, aunt," gasped the boy. + +"Why, I never see anything like it. What did make him act so?" + +"Deed I don't know, Aunt Polly; cats always act so when they're having +a good time." + +"They do, do they?" There was something in the tone that made Tom +apprehensive. + +"Yes'm. That is, I believe they do." + +"You DO?" + +"Yes'm." + +The old lady was bending down, Tom watching, with interest emphasized +by anxiety. Too late he divined her "drift." The handle of the telltale +teaspoon was visible under the bed-valance. Aunt Polly took it, held it +up. Tom winced, and dropped his eyes. Aunt Polly raised him by the +usual handle--his ear--and cracked his head soundly with her thimble. + +"Now, sir, what did you want to treat that poor dumb beast so, for?" + +"I done it out of pity for him--because he hadn't any aunt." + +"Hadn't any aunt!--you numskull. What has that got to do with it?" + +"Heaps. Because if he'd had one she'd a burnt him out herself! She'd a +roasted his bowels out of him 'thout any more feeling than if he was a +human!" + +Aunt Polly felt a sudden pang of remorse. This was putting the thing +in a new light; what was cruelty to a cat MIGHT be cruelty to a boy, +too. She began to soften; she felt sorry. Her eyes watered a little, +and she put her hand on Tom's head and said gently: + +"I was meaning for the best, Tom. And, Tom, it DID do you good." + +Tom looked up in her face with just a perceptible twinkle peeping +through his gravity. + +"I know you was meaning for the best, aunty, and so was I with Peter. +It done HIM good, too. I never see him get around so since--" + +"Oh, go 'long with you, Tom, before you aggravate me again. And you +try and see if you can't be a good boy, for once, and you needn't take +any more medicine." + +Tom reached school ahead of time. It was noticed that this strange +thing had been occurring every day latterly. And now, as usual of late, +he hung about the gate of the schoolyard instead of playing with his +comrades. He was sick, he said, and he looked it. He tried to seem to +be looking everywhere but whither he really was looking--down the road. +Presently Jeff Thatcher hove in sight, and Tom's face lighted; he gazed +a moment, and then turned sorrowfully away. When Jeff arrived, Tom +accosted him; and "led up" warily to opportunities for remark about +Becky, but the giddy lad never could see the bait. Tom watched and +watched, hoping whenever a frisking frock came in sight, and hating the +owner of it as soon as he saw she was not the right one. At last frocks +ceased to appear, and he dropped hopelessly into the dumps; he entered +the empty schoolhouse and sat down to suffer. Then one more frock +passed in at the gate, and Tom's heart gave a great bound. The next +instant he was out, and "going on" like an Indian; yelling, laughing, +chasing boys, jumping over the fence at risk of life and limb, throwing +handsprings, standing on his head--doing all the heroic things he could +conceive of, and keeping a furtive eye out, all the while, to see if +Becky Thatcher was noticing. But she seemed to be unconscious of it +all; she never looked. Could it be possible that she was not aware that +he was there? He carried his exploits to her immediate vicinity; came +war-whooping around, snatched a boy's cap, hurled it to the roof of the +schoolhouse, broke through a group of boys, tumbling them in every +direction, and fell sprawling, himself, under Becky's nose, almost +upsetting her--and she turned, with her nose in the air, and he heard +her say: "Mf! some people think they're mighty smart--always showing +off!" + +Tom's cheeks burned. He gathered himself up and sneaked off, crushed +and crestfallen. + + + +CHAPTER XIII + +TOM'S mind was made up now. He was gloomy and desperate. He was a +forsaken, friendless boy, he said; nobody loved him; when they found +out what they had driven him to, perhaps they would be sorry; he had +tried to do right and get along, but they would not let him; since +nothing would do them but to be rid of him, let it be so; and let them +blame HIM for the consequences--why shouldn't they? What right had the +friendless to complain? Yes, they had forced him to it at last: he +would lead a life of crime. There was no choice. + +By this time he was far down Meadow Lane, and the bell for school to +"take up" tinkled faintly upon his ear. He sobbed, now, to think he +should never, never hear that old familiar sound any more--it was very +hard, but it was forced on him; since he was driven out into the cold +world, he must submit--but he forgave them. Then the sobs came thick +and fast. + +Just at this point he met his soul's sworn comrade, Joe Harper +--hard-eyed, and with evidently a great and dismal purpose in his heart. +Plainly here were "two souls with but a single thought." Tom, wiping +his eyes with his sleeve, began to blubber out something about a +resolution to escape from hard usage and lack of sympathy at home by +roaming abroad into the great world never to return; and ended by +hoping that Joe would not forget him. + +But it transpired that this was a request which Joe had just been +going to make of Tom, and had come to hunt him up for that purpose. His +mother had whipped him for drinking some cream which he had never +tasted and knew nothing about; it was plain that she was tired of him +and wished him to go; if she felt that way, there was nothing for him +to do but succumb; he hoped she would be happy, and never regret having +driven her poor boy out into the unfeeling world to suffer and die. + +As the two boys walked sorrowing along, they made a new compact to +stand by each other and be brothers and never separate till death +relieved them of their troubles. Then they began to lay their plans. +Joe was for being a hermit, and living on crusts in a remote cave, and +dying, some time, of cold and want and grief; but after listening to +Tom, he conceded that there were some conspicuous advantages about a +life of crime, and so he consented to be a pirate. + +Three miles below St. Petersburg, at a point where the Mississippi +River was a trifle over a mile wide, there was a long, narrow, wooded +island, with a shallow bar at the head of it, and this offered well as +a rendezvous. It was not inhabited; it lay far over toward the further +shore, abreast a dense and almost wholly unpeopled forest. So Jackson's +Island was chosen. Who were to be the subjects of their piracies was a +matter that did not occur to them. Then they hunted up Huckleberry +Finn, and he joined them promptly, for all careers were one to him; he +was indifferent. They presently separated to meet at a lonely spot on +the river-bank two miles above the village at the favorite hour--which +was midnight. There was a small log raft there which they meant to +capture. Each would bring hooks and lines, and such provision as he +could steal in the most dark and mysterious way--as became outlaws. And +before the afternoon was done, they had all managed to enjoy the sweet +glory of spreading the fact that pretty soon the town would "hear +something." All who got this vague hint were cautioned to "be mum and +wait." + +About midnight Tom arrived with a boiled ham and a few trifles, +and stopped in a dense undergrowth on a small bluff overlooking the +meeting-place. It was starlight, and very still. The mighty river lay +like an ocean at rest. Tom listened a moment, but no sound disturbed the +quiet. Then he gave a low, distinct whistle. It was answered from under +the bluff. Tom whistled twice more; these signals were answered in the +same way. Then a guarded voice said: + +"Who goes there?" + +"Tom Sawyer, the Black Avenger of the Spanish Main. Name your names." + +"Huck Finn the Red-Handed, and Joe Harper the Terror of the Seas." Tom +had furnished these titles, from his favorite literature. + +"'Tis well. Give the countersign." + +Two hoarse whispers delivered the same awful word simultaneously to +the brooding night: + +"BLOOD!" + +Then Tom tumbled his ham over the bluff and let himself down after it, +tearing both skin and clothes to some extent in the effort. There was +an easy, comfortable path along the shore under the bluff, but it +lacked the advantages of difficulty and danger so valued by a pirate. + +The Terror of the Seas had brought a side of bacon, and had about worn +himself out with getting it there. Finn the Red-Handed had stolen a +skillet and a quantity of half-cured leaf tobacco, and had also brought +a few corn-cobs to make pipes with. But none of the pirates smoked or +"chewed" but himself. The Black Avenger of the Spanish Main said it +would never do to start without some fire. That was a wise thought; +matches were hardly known there in that day. They saw a fire +smouldering upon a great raft a hundred yards above, and they went +stealthily thither and helped themselves to a chunk. They made an +imposing adventure of it, saying, "Hist!" every now and then, and +suddenly halting with finger on lip; moving with hands on imaginary +dagger-hilts; and giving orders in dismal whispers that if "the foe" +stirred, to "let him have it to the hilt," because "dead men tell no +tales." They knew well enough that the raftsmen were all down at the +village laying in stores or having a spree, but still that was no +excuse for their conducting this thing in an unpiratical way. + +They shoved off, presently, Tom in command, Huck at the after oar and +Joe at the forward. Tom stood amidships, gloomy-browed, and with folded +arms, and gave his orders in a low, stern whisper: + +"Luff, and bring her to the wind!" + +"Aye-aye, sir!" + +"Steady, steady-y-y-y!" + +"Steady it is, sir!" + +"Let her go off a point!" + +"Point it is, sir!" + +As the boys steadily and monotonously drove the raft toward mid-stream +it was no doubt understood that these orders were given only for +"style," and were not intended to mean anything in particular. + +"What sail's she carrying?" + +"Courses, tops'ls, and flying-jib, sir." + +"Send the r'yals up! Lay out aloft, there, half a dozen of ye +--foretopmaststuns'l! Lively, now!" + +"Aye-aye, sir!" + +"Shake out that maintogalans'l! Sheets and braces! NOW my hearties!" + +"Aye-aye, sir!" + +"Hellum-a-lee--hard a port! Stand by to meet her when she comes! Port, +port! NOW, men! With a will! Stead-y-y-y!" + +"Steady it is, sir!" + +The raft drew beyond the middle of the river; the boys pointed her +head right, and then lay on their oars. The river was not high, so +there was not more than a two or three mile current. Hardly a word was +said during the next three-quarters of an hour. Now the raft was +passing before the distant town. Two or three glimmering lights showed +where it lay, peacefully sleeping, beyond the vague vast sweep of +star-gemmed water, unconscious of the tremendous event that was happening. +The Black Avenger stood still with folded arms, "looking his last" upon +the scene of his former joys and his later sufferings, and wishing +"she" could see him now, abroad on the wild sea, facing peril and death +with dauntless heart, going to his doom with a grim smile on his lips. +It was but a small strain on his imagination to remove Jackson's Island +beyond eyeshot of the village, and so he "looked his last" with a +broken and satisfied heart. The other pirates were looking their last, +too; and they all looked so long that they came near letting the +current drift them out of the range of the island. But they discovered +the danger in time, and made shift to avert it. About two o'clock in +the morning the raft grounded on the bar two hundred yards above the +head of the island, and they waded back and forth until they had landed +their freight. Part of the little raft's belongings consisted of an old +sail, and this they spread over a nook in the bushes for a tent to +shelter their provisions; but they themselves would sleep in the open +air in good weather, as became outlaws. + +They built a fire against the side of a great log twenty or thirty +steps within the sombre depths of the forest, and then cooked some +bacon in the frying-pan for supper, and used up half of the corn "pone" +stock they had brought. It seemed glorious sport to be feasting in that +wild, free way in the virgin forest of an unexplored and uninhabited +island, far from the haunts of men, and they said they never would +return to civilization. The climbing fire lit up their faces and threw +its ruddy glare upon the pillared tree-trunks of their forest temple, +and upon the varnished foliage and festooning vines. + +When the last crisp slice of bacon was gone, and the last allowance of +corn pone devoured, the boys stretched themselves out on the grass, +filled with contentment. They could have found a cooler place, but they +would not deny themselves such a romantic feature as the roasting +camp-fire. + +"AIN'T it gay?" said Joe. + +"It's NUTS!" said Tom. "What would the boys say if they could see us?" + +"Say? Well, they'd just die to be here--hey, Hucky!" + +"I reckon so," said Huckleberry; "anyways, I'm suited. I don't want +nothing better'n this. I don't ever get enough to eat, gen'ally--and +here they can't come and pick at a feller and bullyrag him so." + +"It's just the life for me," said Tom. "You don't have to get up, +mornings, and you don't have to go to school, and wash, and all that +blame foolishness. You see a pirate don't have to do ANYTHING, Joe, +when he's ashore, but a hermit HE has to be praying considerable, and +then he don't have any fun, anyway, all by himself that way." + +"Oh yes, that's so," said Joe, "but I hadn't thought much about it, +you know. I'd a good deal rather be a pirate, now that I've tried it." + +"You see," said Tom, "people don't go much on hermits, nowadays, like +they used to in old times, but a pirate's always respected. And a +hermit's got to sleep on the hardest place he can find, and put +sackcloth and ashes on his head, and stand out in the rain, and--" + +"What does he put sackcloth and ashes on his head for?" inquired Huck. + +"I dono. But they've GOT to do it. Hermits always do. You'd have to do +that if you was a hermit." + +"Dern'd if I would," said Huck. + +"Well, what would you do?" + +"I dono. But I wouldn't do that." + +"Why, Huck, you'd HAVE to. How'd you get around it?" + +"Why, I just wouldn't stand it. I'd run away." + +"Run away! Well, you WOULD be a nice old slouch of a hermit. You'd be +a disgrace." + +The Red-Handed made no response, being better employed. He had +finished gouging out a cob, and now he fitted a weed stem to it, loaded +it with tobacco, and was pressing a coal to the charge and blowing a +cloud of fragrant smoke--he was in the full bloom of luxurious +contentment. The other pirates envied him this majestic vice, and +secretly resolved to acquire it shortly. Presently Huck said: + +"What does pirates have to do?" + +Tom said: + +"Oh, they have just a bully time--take ships and burn them, and get +the money and bury it in awful places in their island where there's +ghosts and things to watch it, and kill everybody in the ships--make +'em walk a plank." + +"And they carry the women to the island," said Joe; "they don't kill +the women." + +"No," assented Tom, "they don't kill the women--they're too noble. And +the women's always beautiful, too. + +"And don't they wear the bulliest clothes! Oh no! All gold and silver +and di'monds," said Joe, with enthusiasm. + +"Who?" said Huck. + +"Why, the pirates." + +Huck scanned his own clothing forlornly. + +"I reckon I ain't dressed fitten for a pirate," said he, with a +regretful pathos in his voice; "but I ain't got none but these." + +But the other boys told him the fine clothes would come fast enough, +after they should have begun their adventures. They made him understand +that his poor rags would do to begin with, though it was customary for +wealthy pirates to start with a proper wardrobe. + +Gradually their talk died out and drowsiness began to steal upon the +eyelids of the little waifs. The pipe dropped from the fingers of the +Red-Handed, and he slept the sleep of the conscience-free and the +weary. The Terror of the Seas and the Black Avenger of the Spanish Main +had more difficulty in getting to sleep. They said their prayers +inwardly, and lying down, since there was nobody there with authority +to make them kneel and recite aloud; in truth, they had a mind not to +say them at all, but they were afraid to proceed to such lengths as +that, lest they might call down a sudden and special thunderbolt from +heaven. Then at once they reached and hovered upon the imminent verge +of sleep--but an intruder came, now, that would not "down." It was +conscience. They began to feel a vague fear that they had been doing +wrong to run away; and next they thought of the stolen meat, and then +the real torture came. They tried to argue it away by reminding +conscience that they had purloined sweetmeats and apples scores of +times; but conscience was not to be appeased by such thin +plausibilities; it seemed to them, in the end, that there was no +getting around the stubborn fact that taking sweetmeats was only +"hooking," while taking bacon and hams and such valuables was plain +simple stealing--and there was a command against that in the Bible. So +they inwardly resolved that so long as they remained in the business, +their piracies should not again be sullied with the crime of stealing. +Then conscience granted a truce, and these curiously inconsistent +pirates fell peacefully to sleep. + + + +CHAPTER XIV + +WHEN Tom awoke in the morning, he wondered where he was. He sat up and +rubbed his eyes and looked around. Then he comprehended. It was the +cool gray dawn, and there was a delicious sense of repose and peace in +the deep pervading calm and silence of the woods. Not a leaf stirred; +not a sound obtruded upon great Nature's meditation. Beaded dewdrops +stood upon the leaves and grasses. A white layer of ashes covered the +fire, and a thin blue breath of smoke rose straight into the air. Joe +and Huck still slept. + +Now, far away in the woods a bird called; another answered; presently +the hammering of a woodpecker was heard. Gradually the cool dim gray of +the morning whitened, and as gradually sounds multiplied and life +manifested itself. The marvel of Nature shaking off sleep and going to +work unfolded itself to the musing boy. A little green worm came +crawling over a dewy leaf, lifting two-thirds of his body into the air +from time to time and "sniffing around," then proceeding again--for he +was measuring, Tom said; and when the worm approached him, of its own +accord, he sat as still as a stone, with his hopes rising and falling, +by turns, as the creature still came toward him or seemed inclined to +go elsewhere; and when at last it considered a painful moment with its +curved body in the air and then came decisively down upon Tom's leg and +began a journey over him, his whole heart was glad--for that meant that +he was going to have a new suit of clothes--without the shadow of a +doubt a gaudy piratical uniform. Now a procession of ants appeared, +from nowhere in particular, and went about their labors; one struggled +manfully by with a dead spider five times as big as itself in its arms, +and lugged it straight up a tree-trunk. A brown spotted lady-bug +climbed the dizzy height of a grass blade, and Tom bent down close to +it and said, "Lady-bug, lady-bug, fly away home, your house is on fire, +your children's alone," and she took wing and went off to see about it +--which did not surprise the boy, for he knew of old that this insect was +credulous about conflagrations, and he had practised upon its +simplicity more than once. A tumblebug came next, heaving sturdily at +its ball, and Tom touched the creature, to see it shut its legs against +its body and pretend to be dead. The birds were fairly rioting by this +time. A catbird, the Northern mocker, lit in a tree over Tom's head, +and trilled out her imitations of her neighbors in a rapture of +enjoyment; then a shrill jay swept down, a flash of blue flame, and +stopped on a twig almost within the boy's reach, cocked his head to one +side and eyed the strangers with a consuming curiosity; a gray squirrel +and a big fellow of the "fox" kind came skurrying along, sitting up at +intervals to inspect and chatter at the boys, for the wild things had +probably never seen a human being before and scarcely knew whether to +be afraid or not. All Nature was wide awake and stirring, now; long +lances of sunlight pierced down through the dense foliage far and near, +and a few butterflies came fluttering upon the scene. + +Tom stirred up the other pirates and they all clattered away with a +shout, and in a minute or two were stripped and chasing after and +tumbling over each other in the shallow limpid water of the white +sandbar. They felt no longing for the little village sleeping in the +distance beyond the majestic waste of water. A vagrant current or a +slight rise in the river had carried off their raft, but this only +gratified them, since its going was something like burning the bridge +between them and civilization. + +They came back to camp wonderfully refreshed, glad-hearted, and +ravenous; and they soon had the camp-fire blazing up again. Huck found +a spring of clear cold water close by, and the boys made cups of broad +oak or hickory leaves, and felt that water, sweetened with such a +wildwood charm as that, would be a good enough substitute for coffee. +While Joe was slicing bacon for breakfast, Tom and Huck asked him to +hold on a minute; they stepped to a promising nook in the river-bank +and threw in their lines; almost immediately they had reward. Joe had +not had time to get impatient before they were back again with some +handsome bass, a couple of sun-perch and a small catfish--provisions +enough for quite a family. They fried the fish with the bacon, and were +astonished; for no fish had ever seemed so delicious before. They did +not know that the quicker a fresh-water fish is on the fire after he is +caught the better he is; and they reflected little upon what a sauce +open-air sleeping, open-air exercise, bathing, and a large ingredient +of hunger make, too. + +They lay around in the shade, after breakfast, while Huck had a smoke, +and then went off through the woods on an exploring expedition. They +tramped gayly along, over decaying logs, through tangled underbrush, +among solemn monarchs of the forest, hung from their crowns to the +ground with a drooping regalia of grape-vines. Now and then they came +upon snug nooks carpeted with grass and jeweled with flowers. + +They found plenty of things to be delighted with, but nothing to be +astonished at. They discovered that the island was about three miles +long and a quarter of a mile wide, and that the shore it lay closest to +was only separated from it by a narrow channel hardly two hundred yards +wide. They took a swim about every hour, so it was close upon the +middle of the afternoon when they got back to camp. They were too +hungry to stop to fish, but they fared sumptuously upon cold ham, and +then threw themselves down in the shade to talk. But the talk soon +began to drag, and then died. The stillness, the solemnity that brooded +in the woods, and the sense of loneliness, began to tell upon the +spirits of the boys. They fell to thinking. A sort of undefined longing +crept upon them. This took dim shape, presently--it was budding +homesickness. Even Finn the Red-Handed was dreaming of his doorsteps +and empty hogsheads. But they were all ashamed of their weakness, and +none was brave enough to speak his thought. + +For some time, now, the boys had been dully conscious of a peculiar +sound in the distance, just as one sometimes is of the ticking of a +clock which he takes no distinct note of. But now this mysterious sound +became more pronounced, and forced a recognition. The boys started, +glanced at each other, and then each assumed a listening attitude. +There was a long silence, profound and unbroken; then a deep, sullen +boom came floating down out of the distance. + +"What is it!" exclaimed Joe, under his breath. + +"I wonder," said Tom in a whisper. + +"'Tain't thunder," said Huckleberry, in an awed tone, "becuz thunder--" + +"Hark!" said Tom. "Listen--don't talk." + +They waited a time that seemed an age, and then the same muffled boom +troubled the solemn hush. + +"Let's go and see." + +They sprang to their feet and hurried to the shore toward the town. +They parted the bushes on the bank and peered out over the water. The +little steam ferryboat was about a mile below the village, drifting +with the current. Her broad deck seemed crowded with people. There were +a great many skiffs rowing about or floating with the stream in the +neighborhood of the ferryboat, but the boys could not determine what +the men in them were doing. Presently a great jet of white smoke burst +from the ferryboat's side, and as it expanded and rose in a lazy cloud, +that same dull throb of sound was borne to the listeners again. + +"I know now!" exclaimed Tom; "somebody's drownded!" + +"That's it!" said Huck; "they done that last summer, when Bill Turner +got drownded; they shoot a cannon over the water, and that makes him +come up to the top. Yes, and they take loaves of bread and put +quicksilver in 'em and set 'em afloat, and wherever there's anybody +that's drownded, they'll float right there and stop." + +"Yes, I've heard about that," said Joe. "I wonder what makes the bread +do that." + +"Oh, it ain't the bread, so much," said Tom; "I reckon it's mostly +what they SAY over it before they start it out." + +"But they don't say anything over it," said Huck. "I've seen 'em and +they don't." + +"Well, that's funny," said Tom. "But maybe they say it to themselves. +Of COURSE they do. Anybody might know that." + +The other boys agreed that there was reason in what Tom said, because +an ignorant lump of bread, uninstructed by an incantation, could not be +expected to act very intelligently when set upon an errand of such +gravity. + +"By jings, I wish I was over there, now," said Joe. + +"I do too" said Huck "I'd give heaps to know who it is." + +The boys still listened and watched. Presently a revealing thought +flashed through Tom's mind, and he exclaimed: + +"Boys, I know who's drownded--it's us!" + +They felt like heroes in an instant. Here was a gorgeous triumph; they +were missed; they were mourned; hearts were breaking on their account; +tears were being shed; accusing memories of unkindness to these poor +lost lads were rising up, and unavailing regrets and remorse were being +indulged; and best of all, the departed were the talk of the whole +town, and the envy of all the boys, as far as this dazzling notoriety +was concerned. This was fine. It was worth while to be a pirate, after +all. + +As twilight drew on, the ferryboat went back to her accustomed +business and the skiffs disappeared. The pirates returned to camp. They +were jubilant with vanity over their new grandeur and the illustrious +trouble they were making. They caught fish, cooked supper and ate it, +and then fell to guessing at what the village was thinking and saying +about them; and the pictures they drew of the public distress on their +account were gratifying to look upon--from their point of view. But +when the shadows of night closed them in, they gradually ceased to +talk, and sat gazing into the fire, with their minds evidently +wandering elsewhere. The excitement was gone, now, and Tom and Joe +could not keep back thoughts of certain persons at home who were not +enjoying this fine frolic as much as they were. Misgivings came; they +grew troubled and unhappy; a sigh or two escaped, unawares. By and by +Joe timidly ventured upon a roundabout "feeler" as to how the others +might look upon a return to civilization--not right now, but-- + +Tom withered him with derision! Huck, being uncommitted as yet, joined +in with Tom, and the waverer quickly "explained," and was glad to get +out of the scrape with as little taint of chicken-hearted homesickness +clinging to his garments as he could. Mutiny was effectually laid to +rest for the moment. + +As the night deepened, Huck began to nod, and presently to snore. Joe +followed next. Tom lay upon his elbow motionless, for some time, +watching the two intently. At last he got up cautiously, on his knees, +and went searching among the grass and the flickering reflections flung +by the camp-fire. He picked up and inspected several large +semi-cylinders of the thin white bark of a sycamore, and finally chose +two which seemed to suit him. Then he knelt by the fire and painfully +wrote something upon each of these with his "red keel"; one he rolled up +and put in his jacket pocket, and the other he put in Joe's hat and +removed it to a little distance from the owner. And he also put into the +hat certain schoolboy treasures of almost inestimable value--among them +a lump of chalk, an India-rubber ball, three fishhooks, and one of that +kind of marbles known as a "sure 'nough crystal." Then he tiptoed his +way cautiously among the trees till he felt that he was out of hearing, +and straightway broke into a keen run in the direction of the sandbar. + + + +CHAPTER XV + +A FEW minutes later Tom was in the shoal water of the bar, wading +toward the Illinois shore. Before the depth reached his middle he was +half-way over; the current would permit no more wading, now, so he +struck out confidently to swim the remaining hundred yards. He swam +quartering upstream, but still was swept downward rather faster than he +had expected. However, he reached the shore finally, and drifted along +till he found a low place and drew himself out. He put his hand on his +jacket pocket, found his piece of bark safe, and then struck through +the woods, following the shore, with streaming garments. Shortly before +ten o'clock he came out into an open place opposite the village, and +saw the ferryboat lying in the shadow of the trees and the high bank. +Everything was quiet under the blinking stars. He crept down the bank, +watching with all his eyes, slipped into the water, swam three or four +strokes and climbed into the skiff that did "yawl" duty at the boat's +stern. He laid himself down under the thwarts and waited, panting. + +Presently the cracked bell tapped and a voice gave the order to "cast +off." A minute or two later the skiff's head was standing high up, +against the boat's swell, and the voyage was begun. Tom felt happy in +his success, for he knew it was the boat's last trip for the night. At +the end of a long twelve or fifteen minutes the wheels stopped, and Tom +slipped overboard and swam ashore in the dusk, landing fifty yards +downstream, out of danger of possible stragglers. + +He flew along unfrequented alleys, and shortly found himself at his +aunt's back fence. He climbed over, approached the "ell," and looked in +at the sitting-room window, for a light was burning there. There sat +Aunt Polly, Sid, Mary, and Joe Harper's mother, grouped together, +talking. They were by the bed, and the bed was between them and the +door. Tom went to the door and began to softly lift the latch; then he +pressed gently and the door yielded a crack; he continued pushing +cautiously, and quaking every time it creaked, till he judged he might +squeeze through on his knees; so he put his head through and began, +warily. + +"What makes the candle blow so?" said Aunt Polly. Tom hurried up. +"Why, that door's open, I believe. Why, of course it is. No end of +strange things now. Go 'long and shut it, Sid." + +Tom disappeared under the bed just in time. He lay and "breathed" +himself for a time, and then crept to where he could almost touch his +aunt's foot. + +"But as I was saying," said Aunt Polly, "he warn't BAD, so to say +--only mischEEvous. Only just giddy, and harum-scarum, you know. He +warn't any more responsible than a colt. HE never meant any harm, and +he was the best-hearted boy that ever was"--and she began to cry. + +"It was just so with my Joe--always full of his devilment, and up to +every kind of mischief, but he was just as unselfish and kind as he +could be--and laws bless me, to think I went and whipped him for taking +that cream, never once recollecting that I throwed it out myself +because it was sour, and I never to see him again in this world, never, +never, never, poor abused boy!" And Mrs. Harper sobbed as if her heart +would break. + +"I hope Tom's better off where he is," said Sid, "but if he'd been +better in some ways--" + +"SID!" Tom felt the glare of the old lady's eye, though he could not +see it. "Not a word against my Tom, now that he's gone! God'll take +care of HIM--never you trouble YOURself, sir! Oh, Mrs. Harper, I don't +know how to give him up! I don't know how to give him up! He was such a +comfort to me, although he tormented my old heart out of me, 'most." + +"The Lord giveth and the Lord hath taken away--Blessed be the name of +the Lord! But it's so hard--Oh, it's so hard! Only last Saturday my +Joe busted a firecracker right under my nose and I knocked him +sprawling. Little did I know then, how soon--Oh, if it was to do over +again I'd hug him and bless him for it." + +"Yes, yes, yes, I know just how you feel, Mrs. Harper, I know just +exactly how you feel. No longer ago than yesterday noon, my Tom took +and filled the cat full of Pain-killer, and I did think the cretur +would tear the house down. And God forgive me, I cracked Tom's head +with my thimble, poor boy, poor dead boy. But he's out of all his +troubles now. And the last words I ever heard him say was to reproach--" + +But this memory was too much for the old lady, and she broke entirely +down. Tom was snuffling, now, himself--and more in pity of himself than +anybody else. He could hear Mary crying, and putting in a kindly word +for him from time to time. He began to have a nobler opinion of himself +than ever before. Still, he was sufficiently touched by his aunt's +grief to long to rush out from under the bed and overwhelm her with +joy--and the theatrical gorgeousness of the thing appealed strongly to +his nature, too, but he resisted and lay still. + +He went on listening, and gathered by odds and ends that it was +conjectured at first that the boys had got drowned while taking a swim; +then the small raft had been missed; next, certain boys said the +missing lads had promised that the village should "hear something" +soon; the wise-heads had "put this and that together" and decided that +the lads had gone off on that raft and would turn up at the next town +below, presently; but toward noon the raft had been found, lodged +against the Missouri shore some five or six miles below the village +--and then hope perished; they must be drowned, else hunger would have +driven them home by nightfall if not sooner. It was believed that the +search for the bodies had been a fruitless effort merely because the +drowning must have occurred in mid-channel, since the boys, being good +swimmers, would otherwise have escaped to shore. This was Wednesday +night. If the bodies continued missing until Sunday, all hope would be +given over, and the funerals would be preached on that morning. Tom +shuddered. + +Mrs. Harper gave a sobbing good-night and turned to go. Then with a +mutual impulse the two bereaved women flung themselves into each +other's arms and had a good, consoling cry, and then parted. Aunt Polly +was tender far beyond her wont, in her good-night to Sid and Mary. Sid +snuffled a bit and Mary went off crying with all her heart. + +Aunt Polly knelt down and prayed for Tom so touchingly, so +appealingly, and with such measureless love in her words and her old +trembling voice, that he was weltering in tears again, long before she +was through. + +He had to keep still long after she went to bed, for she kept making +broken-hearted ejaculations from time to time, tossing unrestfully, and +turning over. But at last she was still, only moaning a little in her +sleep. Now the boy stole out, rose gradually by the bedside, shaded the +candle-light with his hand, and stood regarding her. His heart was full +of pity for her. He took out his sycamore scroll and placed it by the +candle. But something occurred to him, and he lingered considering. His +face lighted with a happy solution of his thought; he put the bark +hastily in his pocket. Then he bent over and kissed the faded lips, and +straightway made his stealthy exit, latching the door behind him. + +He threaded his way back to the ferry landing, found nobody at large +there, and walked boldly on board the boat, for he knew she was +tenantless except that there was a watchman, who always turned in and +slept like a graven image. He untied the skiff at the stern, slipped +into it, and was soon rowing cautiously upstream. When he had pulled a +mile above the village, he started quartering across and bent himself +stoutly to his work. He hit the landing on the other side neatly, for +this was a familiar bit of work to him. He was moved to capture the +skiff, arguing that it might be considered a ship and therefore +legitimate prey for a pirate, but he knew a thorough search would be +made for it and that might end in revelations. So he stepped ashore and +entered the woods. + +He sat down and took a long rest, torturing himself meanwhile to keep +awake, and then started warily down the home-stretch. The night was far +spent. It was broad daylight before he found himself fairly abreast the +island bar. He rested again until the sun was well up and gilding the +great river with its splendor, and then he plunged into the stream. A +little later he paused, dripping, upon the threshold of the camp, and +heard Joe say: + +"No, Tom's true-blue, Huck, and he'll come back. He won't desert. He +knows that would be a disgrace to a pirate, and Tom's too proud for +that sort of thing. He's up to something or other. Now I wonder what?" + +"Well, the things is ours, anyway, ain't they?" + +"Pretty near, but not yet, Huck. The writing says they are if he ain't +back here to breakfast." + +"Which he is!" exclaimed Tom, with fine dramatic effect, stepping +grandly into camp. + +A sumptuous breakfast of bacon and fish was shortly provided, and as +the boys set to work upon it, Tom recounted (and adorned) his +adventures. They were a vain and boastful company of heroes when the +tale was done. Then Tom hid himself away in a shady nook to sleep till +noon, and the other pirates got ready to fish and explore. + + + +CHAPTER XVI + +AFTER dinner all the gang turned out to hunt for turtle eggs on the +bar. They went about poking sticks into the sand, and when they found a +soft place they went down on their knees and dug with their hands. +Sometimes they would take fifty or sixty eggs out of one hole. They +were perfectly round white things a trifle smaller than an English +walnut. They had a famous fried-egg feast that night, and another on +Friday morning. + +After breakfast they went whooping and prancing out on the bar, and +chased each other round and round, shedding clothes as they went, until +they were naked, and then continued the frolic far away up the shoal +water of the bar, against the stiff current, which latter tripped their +legs from under them from time to time and greatly increased the fun. +And now and then they stooped in a group and splashed water in each +other's faces with their palms, gradually approaching each other, with +averted faces to avoid the strangling sprays, and finally gripping and +struggling till the best man ducked his neighbor, and then they all +went under in a tangle of white legs and arms and came up blowing, +sputtering, laughing, and gasping for breath at one and the same time. + +When they were well exhausted, they would run out and sprawl on the +dry, hot sand, and lie there and cover themselves up with it, and by +and by break for the water again and go through the original +performance once more. Finally it occurred to them that their naked +skin represented flesh-colored "tights" very fairly; so they drew a +ring in the sand and had a circus--with three clowns in it, for none +would yield this proudest post to his neighbor. + +Next they got their marbles and played "knucks" and "ring-taw" and +"keeps" till that amusement grew stale. Then Joe and Huck had another +swim, but Tom would not venture, because he found that in kicking off +his trousers he had kicked his string of rattlesnake rattles off his +ankle, and he wondered how he had escaped cramp so long without the +protection of this mysterious charm. He did not venture again until he +had found it, and by that time the other boys were tired and ready to +rest. They gradually wandered apart, dropped into the "dumps," and fell +to gazing longingly across the wide river to where the village lay +drowsing in the sun. Tom found himself writing "BECKY" in the sand with +his big toe; he scratched it out, and was angry with himself for his +weakness. But he wrote it again, nevertheless; he could not help it. He +erased it once more and then took himself out of temptation by driving +the other boys together and joining them. + +But Joe's spirits had gone down almost beyond resurrection. He was so +homesick that he could hardly endure the misery of it. The tears lay +very near the surface. Huck was melancholy, too. Tom was downhearted, +but tried hard not to show it. He had a secret which he was not ready +to tell, yet, but if this mutinous depression was not broken up soon, +he would have to bring it out. He said, with a great show of +cheerfulness: + +"I bet there's been pirates on this island before, boys. We'll explore +it again. They've hid treasures here somewhere. How'd you feel to light +on a rotten chest full of gold and silver--hey?" + +But it roused only faint enthusiasm, which faded out, with no reply. +Tom tried one or two other seductions; but they failed, too. It was +discouraging work. Joe sat poking up the sand with a stick and looking +very gloomy. Finally he said: + +"Oh, boys, let's give it up. I want to go home. It's so lonesome." + +"Oh no, Joe, you'll feel better by and by," said Tom. "Just think of +the fishing that's here." + +"I don't care for fishing. I want to go home." + +"But, Joe, there ain't such another swimming-place anywhere." + +"Swimming's no good. I don't seem to care for it, somehow, when there +ain't anybody to say I sha'n't go in. I mean to go home." + +"Oh, shucks! Baby! You want to see your mother, I reckon." + +"Yes, I DO want to see my mother--and you would, too, if you had one. +I ain't any more baby than you are." And Joe snuffled a little. + +"Well, we'll let the cry-baby go home to his mother, won't we, Huck? +Poor thing--does it want to see its mother? And so it shall. You like +it here, don't you, Huck? We'll stay, won't we?" + +Huck said, "Y-e-s"--without any heart in it. + +"I'll never speak to you again as long as I live," said Joe, rising. +"There now!" And he moved moodily away and began to dress himself. + +"Who cares!" said Tom. "Nobody wants you to. Go 'long home and get +laughed at. Oh, you're a nice pirate. Huck and me ain't cry-babies. +We'll stay, won't we, Huck? Let him go if he wants to. I reckon we can +get along without him, per'aps." + +But Tom was uneasy, nevertheless, and was alarmed to see Joe go +sullenly on with his dressing. And then it was discomforting to see +Huck eying Joe's preparations so wistfully, and keeping up such an +ominous silence. Presently, without a parting word, Joe began to wade +off toward the Illinois shore. Tom's heart began to sink. He glanced at +Huck. Huck could not bear the look, and dropped his eyes. Then he said: + +"I want to go, too, Tom. It was getting so lonesome anyway, and now +it'll be worse. Let's us go, too, Tom." + +"I won't! You can all go, if you want to. I mean to stay." + +"Tom, I better go." + +"Well, go 'long--who's hendering you." + +Huck began to pick up his scattered clothes. He said: + +"Tom, I wisht you'd come, too. Now you think it over. We'll wait for +you when we get to shore." + +"Well, you'll wait a blame long time, that's all." + +Huck started sorrowfully away, and Tom stood looking after him, with a +strong desire tugging at his heart to yield his pride and go along too. +He hoped the boys would stop, but they still waded slowly on. It +suddenly dawned on Tom that it was become very lonely and still. He +made one final struggle with his pride, and then darted after his +comrades, yelling: + +"Wait! Wait! I want to tell you something!" + +They presently stopped and turned around. When he got to where they +were, he began unfolding his secret, and they listened moodily till at +last they saw the "point" he was driving at, and then they set up a +war-whoop of applause and said it was "splendid!" and said if he had +told them at first, they wouldn't have started away. He made a plausible +excuse; but his real reason had been the fear that not even the secret +would keep them with him any very great length of time, and so he had +meant to hold it in reserve as a last seduction. + +The lads came gayly back and went at their sports again with a will, +chattering all the time about Tom's stupendous plan and admiring the +genius of it. After a dainty egg and fish dinner, Tom said he wanted to +learn to smoke, now. Joe caught at the idea and said he would like to +try, too. So Huck made pipes and filled them. These novices had never +smoked anything before but cigars made of grape-vine, and they "bit" +the tongue, and were not considered manly anyway. + +Now they stretched themselves out on their elbows and began to puff, +charily, and with slender confidence. The smoke had an unpleasant +taste, and they gagged a little, but Tom said: + +"Why, it's just as easy! If I'd a knowed this was all, I'd a learnt +long ago." + +"So would I," said Joe. "It's just nothing." + +"Why, many a time I've looked at people smoking, and thought well I +wish I could do that; but I never thought I could," said Tom. + +"That's just the way with me, hain't it, Huck? You've heard me talk +just that way--haven't you, Huck? I'll leave it to Huck if I haven't." + +"Yes--heaps of times," said Huck. + +"Well, I have too," said Tom; "oh, hundreds of times. Once down by the +slaughter-house. Don't you remember, Huck? Bob Tanner was there, and +Johnny Miller, and Jeff Thatcher, when I said it. Don't you remember, +Huck, 'bout me saying that?" + +"Yes, that's so," said Huck. "That was the day after I lost a white +alley. No, 'twas the day before." + +"There--I told you so," said Tom. "Huck recollects it." + +"I bleeve I could smoke this pipe all day," said Joe. "I don't feel +sick." + +"Neither do I," said Tom. "I could smoke it all day. But I bet you +Jeff Thatcher couldn't." + +"Jeff Thatcher! Why, he'd keel over just with two draws. Just let him +try it once. HE'D see!" + +"I bet he would. And Johnny Miller--I wish could see Johnny Miller +tackle it once." + +"Oh, don't I!" said Joe. "Why, I bet you Johnny Miller couldn't any +more do this than nothing. Just one little snifter would fetch HIM." + +"'Deed it would, Joe. Say--I wish the boys could see us now." + +"So do I." + +"Say--boys, don't say anything about it, and some time when they're +around, I'll come up to you and say, 'Joe, got a pipe? I want a smoke.' +And you'll say, kind of careless like, as if it warn't anything, you'll +say, 'Yes, I got my OLD pipe, and another one, but my tobacker ain't +very good.' And I'll say, 'Oh, that's all right, if it's STRONG +enough.' And then you'll out with the pipes, and we'll light up just as +ca'm, and then just see 'em look!" + +"By jings, that'll be gay, Tom! I wish it was NOW!" + +"So do I! And when we tell 'em we learned when we was off pirating, +won't they wish they'd been along?" + +"Oh, I reckon not! I'll just BET they will!" + +So the talk ran on. But presently it began to flag a trifle, and grow +disjointed. The silences widened; the expectoration marvellously +increased. Every pore inside the boys' cheeks became a spouting +fountain; they could scarcely bail out the cellars under their tongues +fast enough to prevent an inundation; little overflowings down their +throats occurred in spite of all they could do, and sudden retchings +followed every time. Both boys were looking very pale and miserable, +now. Joe's pipe dropped from his nerveless fingers. Tom's followed. +Both fountains were going furiously and both pumps bailing with might +and main. Joe said feebly: + +"I've lost my knife. I reckon I better go and find it." + +Tom said, with quivering lips and halting utterance: + +"I'll help you. You go over that way and I'll hunt around by the +spring. No, you needn't come, Huck--we can find it." + +So Huck sat down again, and waited an hour. Then he found it lonesome, +and went to find his comrades. They were wide apart in the woods, both +very pale, both fast asleep. But something informed him that if they +had had any trouble they had got rid of it. + +They were not talkative at supper that night. They had a humble look, +and when Huck prepared his pipe after the meal and was going to prepare +theirs, they said no, they were not feeling very well--something they +ate at dinner had disagreed with them. + +About midnight Joe awoke, and called the boys. There was a brooding +oppressiveness in the air that seemed to bode something. The boys +huddled themselves together and sought the friendly companionship of +the fire, though the dull dead heat of the breathless atmosphere was +stifling. They sat still, intent and waiting. The solemn hush +continued. Beyond the light of the fire everything was swallowed up in +the blackness of darkness. Presently there came a quivering glow that +vaguely revealed the foliage for a moment and then vanished. By and by +another came, a little stronger. Then another. Then a faint moan came +sighing through the branches of the forest and the boys felt a fleeting +breath upon their cheeks, and shuddered with the fancy that the Spirit +of the Night had gone by. There was a pause. Now a weird flash turned +night into day and showed every little grass-blade, separate and +distinct, that grew about their feet. And it showed three white, +startled faces, too. A deep peal of thunder went rolling and tumbling +down the heavens and lost itself in sullen rumblings in the distance. A +sweep of chilly air passed by, rustling all the leaves and snowing the +flaky ashes broadcast about the fire. Another fierce glare lit up the +forest and an instant crash followed that seemed to rend the tree-tops +right over the boys' heads. They clung together in terror, in the thick +gloom that followed. A few big rain-drops fell pattering upon the +leaves. + +"Quick! boys, go for the tent!" exclaimed Tom. + +They sprang away, stumbling over roots and among vines in the dark, no +two plunging in the same direction. A furious blast roared through the +trees, making everything sing as it went. One blinding flash after +another came, and peal on peal of deafening thunder. And now a +drenching rain poured down and the rising hurricane drove it in sheets +along the ground. The boys cried out to each other, but the roaring +wind and the booming thunder-blasts drowned their voices utterly. +However, one by one they straggled in at last and took shelter under +the tent, cold, scared, and streaming with water; but to have company +in misery seemed something to be grateful for. They could not talk, the +old sail flapped so furiously, even if the other noises would have +allowed them. The tempest rose higher and higher, and presently the +sail tore loose from its fastenings and went winging away on the blast. +The boys seized each others' hands and fled, with many tumblings and +bruises, to the shelter of a great oak that stood upon the river-bank. +Now the battle was at its highest. Under the ceaseless conflagration of +lightning that flamed in the skies, everything below stood out in +clean-cut and shadowless distinctness: the bending trees, the billowy +river, white with foam, the driving spray of spume-flakes, the dim +outlines of the high bluffs on the other side, glimpsed through the +drifting cloud-rack and the slanting veil of rain. Every little while +some giant tree yielded the fight and fell crashing through the younger +growth; and the unflagging thunder-peals came now in ear-splitting +explosive bursts, keen and sharp, and unspeakably appalling. The storm +culminated in one matchless effort that seemed likely to tear the island +to pieces, burn it up, drown it to the tree-tops, blow it away, and +deafen every creature in it, all at one and the same moment. It was a +wild night for homeless young heads to be out in. + +But at last the battle was done, and the forces retired with weaker +and weaker threatenings and grumblings, and peace resumed her sway. The +boys went back to camp, a good deal awed; but they found there was +still something to be thankful for, because the great sycamore, the +shelter of their beds, was a ruin, now, blasted by the lightnings, and +they were not under it when the catastrophe happened. + +Everything in camp was drenched, the camp-fire as well; for they were +but heedless lads, like their generation, and had made no provision +against rain. Here was matter for dismay, for they were soaked through +and chilled. They were eloquent in their distress; but they presently +discovered that the fire had eaten so far up under the great log it had +been built against (where it curved upward and separated itself from +the ground), that a handbreadth or so of it had escaped wetting; so +they patiently wrought until, with shreds and bark gathered from the +under sides of sheltered logs, they coaxed the fire to burn again. Then +they piled on great dead boughs till they had a roaring furnace, and +were glad-hearted once more. They dried their boiled ham and had a +feast, and after that they sat by the fire and expanded and glorified +their midnight adventure until morning, for there was not a dry spot to +sleep on, anywhere around. + +As the sun began to steal in upon the boys, drowsiness came over them, +and they went out on the sandbar and lay down to sleep. They got +scorched out by and by, and drearily set about getting breakfast. After +the meal they felt rusty, and stiff-jointed, and a little homesick once +more. Tom saw the signs, and fell to cheering up the pirates as well as +he could. But they cared nothing for marbles, or circus, or swimming, +or anything. He reminded them of the imposing secret, and raised a ray +of cheer. While it lasted, he got them interested in a new device. This +was to knock off being pirates, for a while, and be Indians for a +change. They were attracted by this idea; so it was not long before +they were stripped, and striped from head to heel with black mud, like +so many zebras--all of them chiefs, of course--and then they went +tearing through the woods to attack an English settlement. + +By and by they separated into three hostile tribes, and darted upon +each other from ambush with dreadful war-whoops, and killed and scalped +each other by thousands. It was a gory day. Consequently it was an +extremely satisfactory one. + +They assembled in camp toward supper-time, hungry and happy; but now a +difficulty arose--hostile Indians could not break the bread of +hospitality together without first making peace, and this was a simple +impossibility without smoking a pipe of peace. There was no other +process that ever they had heard of. Two of the savages almost wished +they had remained pirates. However, there was no other way; so with +such show of cheerfulness as they could muster they called for the pipe +and took their whiff as it passed, in due form. + +And behold, they were glad they had gone into savagery, for they had +gained something; they found that they could now smoke a little without +having to go and hunt for a lost knife; they did not get sick enough to +be seriously uncomfortable. They were not likely to fool away this high +promise for lack of effort. No, they practised cautiously, after +supper, with right fair success, and so they spent a jubilant evening. +They were prouder and happier in their new acquirement than they would +have been in the scalping and skinning of the Six Nations. We will +leave them to smoke and chatter and brag, since we have no further use +for them at present. + + + +CHAPTER XVII + +BUT there was no hilarity in the little town that same tranquil +Saturday afternoon. The Harpers, and Aunt Polly's family, were being +put into mourning, with great grief and many tears. An unusual quiet +possessed the village, although it was ordinarily quiet enough, in all +conscience. The villagers conducted their concerns with an absent air, +and talked little; but they sighed often. The Saturday holiday seemed a +burden to the children. They had no heart in their sports, and +gradually gave them up. + +In the afternoon Becky Thatcher found herself moping about the +deserted schoolhouse yard, and feeling very melancholy. But she found +nothing there to comfort her. She soliloquized: + +"Oh, if I only had a brass andiron-knob again! But I haven't got +anything now to remember him by." And she choked back a little sob. + +Presently she stopped, and said to herself: + +"It was right here. Oh, if it was to do over again, I wouldn't say +that--I wouldn't say it for the whole world. But he's gone now; I'll +never, never, never see him any more." + +This thought broke her down, and she wandered away, with tears rolling +down her cheeks. Then quite a group of boys and girls--playmates of +Tom's and Joe's--came by, and stood looking over the paling fence and +talking in reverent tones of how Tom did so-and-so the last time they +saw him, and how Joe said this and that small trifle (pregnant with +awful prophecy, as they could easily see now!)--and each speaker +pointed out the exact spot where the lost lads stood at the time, and +then added something like "and I was a-standing just so--just as I am +now, and as if you was him--I was as close as that--and he smiled, just +this way--and then something seemed to go all over me, like--awful, you +know--and I never thought what it meant, of course, but I can see now!" + +Then there was a dispute about who saw the dead boys last in life, and +many claimed that dismal distinction, and offered evidences, more or +less tampered with by the witness; and when it was ultimately decided +who DID see the departed last, and exchanged the last words with them, +the lucky parties took upon themselves a sort of sacred importance, and +were gaped at and envied by all the rest. One poor chap, who had no +other grandeur to offer, said with tolerably manifest pride in the +remembrance: + +"Well, Tom Sawyer he licked me once." + +But that bid for glory was a failure. Most of the boys could say that, +and so that cheapened the distinction too much. The group loitered +away, still recalling memories of the lost heroes, in awed voices. + +When the Sunday-school hour was finished, the next morning, the bell +began to toll, instead of ringing in the usual way. It was a very still +Sabbath, and the mournful sound seemed in keeping with the musing hush +that lay upon nature. The villagers began to gather, loitering a moment +in the vestibule to converse in whispers about the sad event. But there +was no whispering in the house; only the funereal rustling of dresses +as the women gathered to their seats disturbed the silence there. None +could remember when the little church had been so full before. There +was finally a waiting pause, an expectant dumbness, and then Aunt Polly +entered, followed by Sid and Mary, and they by the Harper family, all +in deep black, and the whole congregation, the old minister as well, +rose reverently and stood until the mourners were seated in the front +pew. There was another communing silence, broken at intervals by +muffled sobs, and then the minister spread his hands abroad and prayed. +A moving hymn was sung, and the text followed: "I am the Resurrection +and the Life." + +As the service proceeded, the clergyman drew such pictures of the +graces, the winning ways, and the rare promise of the lost lads that +every soul there, thinking he recognized these pictures, felt a pang in +remembering that he had persistently blinded himself to them always +before, and had as persistently seen only faults and flaws in the poor +boys. The minister related many a touching incident in the lives of the +departed, too, which illustrated their sweet, generous natures, and the +people could easily see, now, how noble and beautiful those episodes +were, and remembered with grief that at the time they occurred they had +seemed rank rascalities, well deserving of the cowhide. The +congregation became more and more moved, as the pathetic tale went on, +till at last the whole company broke down and joined the weeping +mourners in a chorus of anguished sobs, the preacher himself giving way +to his feelings, and crying in the pulpit. + +There was a rustle in the gallery, which nobody noticed; a moment +later the church door creaked; the minister raised his streaming eyes +above his handkerchief, and stood transfixed! First one and then +another pair of eyes followed the minister's, and then almost with one +impulse the congregation rose and stared while the three dead boys came +marching up the aisle, Tom in the lead, Joe next, and Huck, a ruin of +drooping rags, sneaking sheepishly in the rear! They had been hid in +the unused gallery listening to their own funeral sermon! + +Aunt Polly, Mary, and the Harpers threw themselves upon their restored +ones, smothered them with kisses and poured out thanksgivings, while +poor Huck stood abashed and uncomfortable, not knowing exactly what to +do or where to hide from so many unwelcoming eyes. He wavered, and +started to slink away, but Tom seized him and said: + +"Aunt Polly, it ain't fair. Somebody's got to be glad to see Huck." + +"And so they shall. I'm glad to see him, poor motherless thing!" And +the loving attentions Aunt Polly lavished upon him were the one thing +capable of making him more uncomfortable than he was before. + +Suddenly the minister shouted at the top of his voice: "Praise God +from whom all blessings flow--SING!--and put your hearts in it!" + +And they did. Old Hundred swelled up with a triumphant burst, and +while it shook the rafters Tom Sawyer the Pirate looked around upon the +envying juveniles about him and confessed in his heart that this was +the proudest moment of his life. + +As the "sold" congregation trooped out they said they would almost be +willing to be made ridiculous again to hear Old Hundred sung like that +once more. + +Tom got more cuffs and kisses that day--according to Aunt Polly's +varying moods--than he had earned before in a year; and he hardly knew +which expressed the most gratefulness to God and affection for himself. + + + +CHAPTER XVIII + +THAT was Tom's great secret--the scheme to return home with his +brother pirates and attend their own funerals. They had paddled over to +the Missouri shore on a log, at dusk on Saturday, landing five or six +miles below the village; they had slept in the woods at the edge of the +town till nearly daylight, and had then crept through back lanes and +alleys and finished their sleep in the gallery of the church among a +chaos of invalided benches. + +At breakfast, Monday morning, Aunt Polly and Mary were very loving to +Tom, and very attentive to his wants. There was an unusual amount of +talk. In the course of it Aunt Polly said: + +"Well, I don't say it wasn't a fine joke, Tom, to keep everybody +suffering 'most a week so you boys had a good time, but it is a pity +you could be so hard-hearted as to let me suffer so. If you could come +over on a log to go to your funeral, you could have come over and give +me a hint some way that you warn't dead, but only run off." + +"Yes, you could have done that, Tom," said Mary; "and I believe you +would if you had thought of it." + +"Would you, Tom?" said Aunt Polly, her face lighting wistfully. "Say, +now, would you, if you'd thought of it?" + +"I--well, I don't know. 'Twould 'a' spoiled everything." + +"Tom, I hoped you loved me that much," said Aunt Polly, with a grieved +tone that discomforted the boy. "It would have been something if you'd +cared enough to THINK of it, even if you didn't DO it." + +"Now, auntie, that ain't any harm," pleaded Mary; "it's only Tom's +giddy way--he is always in such a rush that he never thinks of +anything." + +"More's the pity. Sid would have thought. And Sid would have come and +DONE it, too. Tom, you'll look back, some day, when it's too late, and +wish you'd cared a little more for me when it would have cost you so +little." + +"Now, auntie, you know I do care for you," said Tom. + +"I'd know it better if you acted more like it." + +"I wish now I'd thought," said Tom, with a repentant tone; "but I +dreamt about you, anyway. That's something, ain't it?" + +"It ain't much--a cat does that much--but it's better than nothing. +What did you dream?" + +"Why, Wednesday night I dreamt that you was sitting over there by the +bed, and Sid was sitting by the woodbox, and Mary next to him." + +"Well, so we did. So we always do. I'm glad your dreams could take +even that much trouble about us." + +"And I dreamt that Joe Harper's mother was here." + +"Why, she was here! Did you dream any more?" + +"Oh, lots. But it's so dim, now." + +"Well, try to recollect--can't you?" + +"Somehow it seems to me that the wind--the wind blowed the--the--" + +"Try harder, Tom! The wind did blow something. Come!" + +Tom pressed his fingers on his forehead an anxious minute, and then +said: + +"I've got it now! I've got it now! It blowed the candle!" + +"Mercy on us! Go on, Tom--go on!" + +"And it seems to me that you said, 'Why, I believe that that door--'" + +"Go ON, Tom!" + +"Just let me study a moment--just a moment. Oh, yes--you said you +believed the door was open." + +"As I'm sitting here, I did! Didn't I, Mary! Go on!" + +"And then--and then--well I won't be certain, but it seems like as if +you made Sid go and--and--" + +"Well? Well? What did I make him do, Tom? What did I make him do?" + +"You made him--you--Oh, you made him shut it." + +"Well, for the land's sake! I never heard the beat of that in all my +days! Don't tell ME there ain't anything in dreams, any more. Sereny +Harper shall know of this before I'm an hour older. I'd like to see her +get around THIS with her rubbage 'bout superstition. Go on, Tom!" + +"Oh, it's all getting just as bright as day, now. Next you said I +warn't BAD, only mischeevous and harum-scarum, and not any more +responsible than--than--I think it was a colt, or something." + +"And so it was! Well, goodness gracious! Go on, Tom!" + +"And then you began to cry." + +"So I did. So I did. Not the first time, neither. And then--" + +"Then Mrs. Harper she began to cry, and said Joe was just the same, +and she wished she hadn't whipped him for taking cream when she'd +throwed it out her own self--" + +"Tom! The sperrit was upon you! You was a prophesying--that's what you +was doing! Land alive, go on, Tom!" + +"Then Sid he said--he said--" + +"I don't think I said anything," said Sid. + +"Yes you did, Sid," said Mary. + +"Shut your heads and let Tom go on! What did he say, Tom?" + +"He said--I THINK he said he hoped I was better off where I was gone +to, but if I'd been better sometimes--" + +"THERE, d'you hear that! It was his very words!" + +"And you shut him up sharp." + +"I lay I did! There must 'a' been an angel there. There WAS an angel +there, somewheres!" + +"And Mrs. Harper told about Joe scaring her with a firecracker, and +you told about Peter and the Painkiller--" + +"Just as true as I live!" + +"And then there was a whole lot of talk 'bout dragging the river for +us, and 'bout having the funeral Sunday, and then you and old Miss +Harper hugged and cried, and she went." + +"It happened just so! It happened just so, as sure as I'm a-sitting in +these very tracks. Tom, you couldn't told it more like if you'd 'a' +seen it! And then what? Go on, Tom!" + +"Then I thought you prayed for me--and I could see you and hear every +word you said. And you went to bed, and I was so sorry that I took and +wrote on a piece of sycamore bark, 'We ain't dead--we are only off +being pirates,' and put it on the table by the candle; and then you +looked so good, laying there asleep, that I thought I went and leaned +over and kissed you on the lips." + +"Did you, Tom, DID you! I just forgive you everything for that!" And +she seized the boy in a crushing embrace that made him feel like the +guiltiest of villains. + +"It was very kind, even though it was only a--dream," Sid soliloquized +just audibly. + +"Shut up, Sid! A body does just the same in a dream as he'd do if he +was awake. Here's a big Milum apple I've been saving for you, Tom, if +you was ever found again--now go 'long to school. I'm thankful to the +good God and Father of us all I've got you back, that's long-suffering +and merciful to them that believe on Him and keep His word, though +goodness knows I'm unworthy of it, but if only the worthy ones got His +blessings and had His hand to help them over the rough places, there's +few enough would smile here or ever enter into His rest when the long +night comes. Go 'long Sid, Mary, Tom--take yourselves off--you've +hendered me long enough." + +The children left for school, and the old lady to call on Mrs. Harper +and vanquish her realism with Tom's marvellous dream. Sid had better +judgment than to utter the thought that was in his mind as he left the +house. It was this: "Pretty thin--as long a dream as that, without any +mistakes in it!" + +What a hero Tom was become, now! He did not go skipping and prancing, +but moved with a dignified swagger as became a pirate who felt that the +public eye was on him. And indeed it was; he tried not to seem to see +the looks or hear the remarks as he passed along, but they were food +and drink to him. Smaller boys than himself flocked at his heels, as +proud to be seen with him, and tolerated by him, as if he had been the +drummer at the head of a procession or the elephant leading a menagerie +into town. Boys of his own size pretended not to know he had been away +at all; but they were consuming with envy, nevertheless. They would +have given anything to have that swarthy suntanned skin of his, and his +glittering notoriety; and Tom would not have parted with either for a +circus. + +At school the children made so much of him and of Joe, and delivered +such eloquent admiration from their eyes, that the two heroes were not +long in becoming insufferably "stuck-up." They began to tell their +adventures to hungry listeners--but they only began; it was not a thing +likely to have an end, with imaginations like theirs to furnish +material. And finally, when they got out their pipes and went serenely +puffing around, the very summit of glory was reached. + +Tom decided that he could be independent of Becky Thatcher now. Glory +was sufficient. He would live for glory. Now that he was distinguished, +maybe she would be wanting to "make up." Well, let her--she should see +that he could be as indifferent as some other people. Presently she +arrived. Tom pretended not to see her. He moved away and joined a group +of boys and girls and began to talk. Soon he observed that she was +tripping gayly back and forth with flushed face and dancing eyes, +pretending to be busy chasing schoolmates, and screaming with laughter +when she made a capture; but he noticed that she always made her +captures in his vicinity, and that she seemed to cast a conscious eye +in his direction at such times, too. It gratified all the vicious +vanity that was in him; and so, instead of winning him, it only "set +him up" the more and made him the more diligent to avoid betraying that +he knew she was about. Presently she gave over skylarking, and moved +irresolutely about, sighing once or twice and glancing furtively and +wistfully toward Tom. Then she observed that now Tom was talking more +particularly to Amy Lawrence than to any one else. She felt a sharp +pang and grew disturbed and uneasy at once. She tried to go away, but +her feet were treacherous, and carried her to the group instead. She +said to a girl almost at Tom's elbow--with sham vivacity: + +"Why, Mary Austin! you bad girl, why didn't you come to Sunday-school?" + +"I did come--didn't you see me?" + +"Why, no! Did you? Where did you sit?" + +"I was in Miss Peters' class, where I always go. I saw YOU." + +"Did you? Why, it's funny I didn't see you. I wanted to tell you about +the picnic." + +"Oh, that's jolly. Who's going to give it?" + +"My ma's going to let me have one." + +"Oh, goody; I hope she'll let ME come." + +"Well, she will. The picnic's for me. She'll let anybody come that I +want, and I want you." + +"That's ever so nice. When is it going to be?" + +"By and by. Maybe about vacation." + +"Oh, won't it be fun! You going to have all the girls and boys?" + +"Yes, every one that's friends to me--or wants to be"; and she glanced +ever so furtively at Tom, but he talked right along to Amy Lawrence +about the terrible storm on the island, and how the lightning tore the +great sycamore tree "all to flinders" while he was "standing within +three feet of it." + +"Oh, may I come?" said Grace Miller. + +"Yes." + +"And me?" said Sally Rogers. + +"Yes." + +"And me, too?" said Susy Harper. "And Joe?" + +"Yes." + +And so on, with clapping of joyful hands till all the group had begged +for invitations but Tom and Amy. Then Tom turned coolly away, still +talking, and took Amy with him. Becky's lips trembled and the tears +came to her eyes; she hid these signs with a forced gayety and went on +chattering, but the life had gone out of the picnic, now, and out of +everything else; she got away as soon as she could and hid herself and +had what her sex call "a good cry." Then she sat moody, with wounded +pride, till the bell rang. She roused up, now, with a vindictive cast +in her eye, and gave her plaited tails a shake and said she knew what +SHE'D do. + +At recess Tom continued his flirtation with Amy with jubilant +self-satisfaction. And he kept drifting about to find Becky and lacerate +her with the performance. At last he spied her, but there was a sudden +falling of his mercury. She was sitting cosily on a little bench behind +the schoolhouse looking at a picture-book with Alfred Temple--and so +absorbed were they, and their heads so close together over the book, +that they did not seem to be conscious of anything in the world besides. +Jealousy ran red-hot through Tom's veins. He began to hate himself for +throwing away the chance Becky had offered for a reconciliation. He +called himself a fool, and all the hard names he could think of. He +wanted to cry with vexation. Amy chatted happily along, as they walked, +for her heart was singing, but Tom's tongue had lost its function. He +did not hear what Amy was saying, and whenever she paused expectantly he +could only stammer an awkward assent, which was as often misplaced as +otherwise. He kept drifting to the rear of the schoolhouse, again and +again, to sear his eyeballs with the hateful spectacle there. He could +not help it. And it maddened him to see, as he thought he saw, that +Becky Thatcher never once suspected that he was even in the land of the +living. But she did see, nevertheless; and she knew she was winning her +fight, too, and was glad to see him suffer as she had suffered. + +Amy's happy prattle became intolerable. Tom hinted at things he had to +attend to; things that must be done; and time was fleeting. But in +vain--the girl chirped on. Tom thought, "Oh, hang her, ain't I ever +going to get rid of her?" At last he must be attending to those +things--and she said artlessly that she would be "around" when school +let out. And he hastened away, hating her for it. + +"Any other boy!" Tom thought, grating his teeth. "Any boy in the whole +town but that Saint Louis smarty that thinks he dresses so fine and is +aristocracy! Oh, all right, I licked you the first day you ever saw +this town, mister, and I'll lick you again! You just wait till I catch +you out! I'll just take and--" + +And he went through the motions of thrashing an imaginary boy +--pummelling the air, and kicking and gouging. "Oh, you do, do you? You +holler 'nough, do you? Now, then, let that learn you!" And so the +imaginary flogging was finished to his satisfaction. + +Tom fled home at noon. His conscience could not endure any more of +Amy's grateful happiness, and his jealousy could bear no more of the +other distress. Becky resumed her picture inspections with Alfred, but +as the minutes dragged along and no Tom came to suffer, her triumph +began to cloud and she lost interest; gravity and absent-mindedness +followed, and then melancholy; two or three times she pricked up her +ear at a footstep, but it was a false hope; no Tom came. At last she +grew entirely miserable and wished she hadn't carried it so far. When +poor Alfred, seeing that he was losing her, he did not know how, kept +exclaiming: "Oh, here's a jolly one! look at this!" she lost patience +at last, and said, "Oh, don't bother me! I don't care for them!" and +burst into tears, and got up and walked away. + +Alfred dropped alongside and was going to try to comfort her, but she +said: + +"Go away and leave me alone, can't you! I hate you!" + +So the boy halted, wondering what he could have done--for she had said +she would look at pictures all through the nooning--and she walked on, +crying. Then Alfred went musing into the deserted schoolhouse. He was +humiliated and angry. He easily guessed his way to the truth--the girl +had simply made a convenience of him to vent her spite upon Tom Sawyer. +He was far from hating Tom the less when this thought occurred to him. +He wished there was some way to get that boy into trouble without much +risk to himself. Tom's spelling-book fell under his eye. Here was his +opportunity. He gratefully opened to the lesson for the afternoon and +poured ink upon the page. + +Becky, glancing in at a window behind him at the moment, saw the act, +and moved on, without discovering herself. She started homeward, now, +intending to find Tom and tell him; Tom would be thankful and their +troubles would be healed. Before she was half way home, however, she +had changed her mind. The thought of Tom's treatment of her when she +was talking about her picnic came scorching back and filled her with +shame. She resolved to let him get whipped on the damaged +spelling-book's account, and to hate him forever, into the bargain. + + + +CHAPTER XIX + +TOM arrived at home in a dreary mood, and the first thing his aunt +said to him showed him that he had brought his sorrows to an +unpromising market: + +"Tom, I've a notion to skin you alive!" + +"Auntie, what have I done?" + +"Well, you've done enough. Here I go over to Sereny Harper, like an +old softy, expecting I'm going to make her believe all that rubbage +about that dream, when lo and behold you she'd found out from Joe that +you was over here and heard all the talk we had that night. Tom, I +don't know what is to become of a boy that will act like that. It makes +me feel so bad to think you could let me go to Sereny Harper and make +such a fool of myself and never say a word." + +This was a new aspect of the thing. His smartness of the morning had +seemed to Tom a good joke before, and very ingenious. It merely looked +mean and shabby now. He hung his head and could not think of anything +to say for a moment. Then he said: + +"Auntie, I wish I hadn't done it--but I didn't think." + +"Oh, child, you never think. You never think of anything but your own +selfishness. You could think to come all the way over here from +Jackson's Island in the night to laugh at our troubles, and you could +think to fool me with a lie about a dream; but you couldn't ever think +to pity us and save us from sorrow." + +"Auntie, I know now it was mean, but I didn't mean to be mean. I +didn't, honest. And besides, I didn't come over here to laugh at you +that night." + +"What did you come for, then?" + +"It was to tell you not to be uneasy about us, because we hadn't got +drownded." + +"Tom, Tom, I would be the thankfullest soul in this world if I could +believe you ever had as good a thought as that, but you know you never +did--and I know it, Tom." + +"Indeed and 'deed I did, auntie--I wish I may never stir if I didn't." + +"Oh, Tom, don't lie--don't do it. It only makes things a hundred times +worse." + +"It ain't a lie, auntie; it's the truth. I wanted to keep you from +grieving--that was all that made me come." + +"I'd give the whole world to believe that--it would cover up a power +of sins, Tom. I'd 'most be glad you'd run off and acted so bad. But it +ain't reasonable; because, why didn't you tell me, child?" + +"Why, you see, when you got to talking about the funeral, I just got +all full of the idea of our coming and hiding in the church, and I +couldn't somehow bear to spoil it. So I just put the bark back in my +pocket and kept mum." + +"What bark?" + +"The bark I had wrote on to tell you we'd gone pirating. I wish, now, +you'd waked up when I kissed you--I do, honest." + +The hard lines in his aunt's face relaxed and a sudden tenderness +dawned in her eyes. + +"DID you kiss me, Tom?" + +"Why, yes, I did." + +"Are you sure you did, Tom?" + +"Why, yes, I did, auntie--certain sure." + +"What did you kiss me for, Tom?" + +"Because I loved you so, and you laid there moaning and I was so sorry." + +The words sounded like truth. The old lady could not hide a tremor in +her voice when she said: + +"Kiss me again, Tom!--and be off with you to school, now, and don't +bother me any more." + +The moment he was gone, she ran to a closet and got out the ruin of a +jacket which Tom had gone pirating in. Then she stopped, with it in her +hand, and said to herself: + +"No, I don't dare. Poor boy, I reckon he's lied about it--but it's a +blessed, blessed lie, there's such a comfort come from it. I hope the +Lord--I KNOW the Lord will forgive him, because it was such +goodheartedness in him to tell it. But I don't want to find out it's a +lie. I won't look." + +She put the jacket away, and stood by musing a minute. Twice she put +out her hand to take the garment again, and twice she refrained. Once +more she ventured, and this time she fortified herself with the +thought: "It's a good lie--it's a good lie--I won't let it grieve me." +So she sought the jacket pocket. A moment later she was reading Tom's +piece of bark through flowing tears and saying: "I could forgive the +boy, now, if he'd committed a million sins!" + + + +CHAPTER XX + +THERE was something about Aunt Polly's manner, when she kissed Tom, +that swept away his low spirits and made him lighthearted and happy +again. He started to school and had the luck of coming upon Becky +Thatcher at the head of Meadow Lane. His mood always determined his +manner. Without a moment's hesitation he ran to her and said: + +"I acted mighty mean to-day, Becky, and I'm so sorry. I won't ever, +ever do that way again, as long as ever I live--please make up, won't +you?" + +The girl stopped and looked him scornfully in the face: + +"I'll thank you to keep yourself TO yourself, Mr. Thomas Sawyer. I'll +never speak to you again." + +She tossed her head and passed on. Tom was so stunned that he had not +even presence of mind enough to say "Who cares, Miss Smarty?" until the +right time to say it had gone by. So he said nothing. But he was in a +fine rage, nevertheless. He moped into the schoolyard wishing she were +a boy, and imagining how he would trounce her if she were. He presently +encountered her and delivered a stinging remark as he passed. She +hurled one in return, and the angry breach was complete. It seemed to +Becky, in her hot resentment, that she could hardly wait for school to +"take in," she was so impatient to see Tom flogged for the injured +spelling-book. If she had had any lingering notion of exposing Alfred +Temple, Tom's offensive fling had driven it entirely away. + +Poor girl, she did not know how fast she was nearing trouble herself. +The master, Mr. Dobbins, had reached middle age with an unsatisfied +ambition. The darling of his desires was, to be a doctor, but poverty +had decreed that he should be nothing higher than a village +schoolmaster. Every day he took a mysterious book out of his desk and +absorbed himself in it at times when no classes were reciting. He kept +that book under lock and key. There was not an urchin in school but was +perishing to have a glimpse of it, but the chance never came. Every boy +and girl had a theory about the nature of that book; but no two +theories were alike, and there was no way of getting at the facts in +the case. Now, as Becky was passing by the desk, which stood near the +door, she noticed that the key was in the lock! It was a precious +moment. She glanced around; found herself alone, and the next instant +she had the book in her hands. The title-page--Professor Somebody's +ANATOMY--carried no information to her mind; so she began to turn the +leaves. She came at once upon a handsomely engraved and colored +frontispiece--a human figure, stark naked. At that moment a shadow fell +on the page and Tom Sawyer stepped in at the door and caught a glimpse +of the picture. Becky snatched at the book to close it, and had the +hard luck to tear the pictured page half down the middle. She thrust +the volume into the desk, turned the key, and burst out crying with +shame and vexation. + +"Tom Sawyer, you are just as mean as you can be, to sneak up on a +person and look at what they're looking at." + +"How could I know you was looking at anything?" + +"You ought to be ashamed of yourself, Tom Sawyer; you know you're +going to tell on me, and oh, what shall I do, what shall I do! I'll be +whipped, and I never was whipped in school." + +Then she stamped her little foot and said: + +"BE so mean if you want to! I know something that's going to happen. +You just wait and you'll see! Hateful, hateful, hateful!"--and she +flung out of the house with a new explosion of crying. + +Tom stood still, rather flustered by this onslaught. Presently he said +to himself: + +"What a curious kind of a fool a girl is! Never been licked in school! +Shucks! What's a licking! That's just like a girl--they're so +thin-skinned and chicken-hearted. Well, of course I ain't going to tell +old Dobbins on this little fool, because there's other ways of getting +even on her, that ain't so mean; but what of it? Old Dobbins will ask +who it was tore his book. Nobody'll answer. Then he'll do just the way +he always does--ask first one and then t'other, and when he comes to the +right girl he'll know it, without any telling. Girls' faces always tell +on them. They ain't got any backbone. She'll get licked. Well, it's a +kind of a tight place for Becky Thatcher, because there ain't any way +out of it." Tom conned the thing a moment longer, and then added: "All +right, though; she'd like to see me in just such a fix--let her sweat it +out!" + +Tom joined the mob of skylarking scholars outside. In a few moments +the master arrived and school "took in." Tom did not feel a strong +interest in his studies. Every time he stole a glance at the girls' +side of the room Becky's face troubled him. Considering all things, he +did not want to pity her, and yet it was all he could do to help it. He +could get up no exultation that was really worthy the name. Presently +the spelling-book discovery was made, and Tom's mind was entirely full +of his own matters for a while after that. Becky roused up from her +lethargy of distress and showed good interest in the proceedings. She +did not expect that Tom could get out of his trouble by denying that he +spilt the ink on the book himself; and she was right. The denial only +seemed to make the thing worse for Tom. Becky supposed she would be +glad of that, and she tried to believe she was glad of it, but she +found she was not certain. When the worst came to the worst, she had an +impulse to get up and tell on Alfred Temple, but she made an effort and +forced herself to keep still--because, said she to herself, "he'll tell +about me tearing the picture sure. I wouldn't say a word, not to save +his life!" + +Tom took his whipping and went back to his seat not at all +broken-hearted, for he thought it was possible that he had unknowingly +upset the ink on the spelling-book himself, in some skylarking bout--he +had denied it for form's sake and because it was custom, and had stuck +to the denial from principle. + +A whole hour drifted by, the master sat nodding in his throne, the air +was drowsy with the hum of study. By and by, Mr. Dobbins straightened +himself up, yawned, then unlocked his desk, and reached for his book, +but seemed undecided whether to take it out or leave it. Most of the +pupils glanced up languidly, but there were two among them that watched +his movements with intent eyes. Mr. Dobbins fingered his book absently +for a while, then took it out and settled himself in his chair to read! +Tom shot a glance at Becky. He had seen a hunted and helpless rabbit +look as she did, with a gun levelled at its head. Instantly he forgot +his quarrel with her. Quick--something must be done! done in a flash, +too! But the very imminence of the emergency paralyzed his invention. +Good!--he had an inspiration! He would run and snatch the book, spring +through the door and fly. But his resolution shook for one little +instant, and the chance was lost--the master opened the volume. If Tom +only had the wasted opportunity back again! Too late. There was no help +for Becky now, he said. The next moment the master faced the school. +Every eye sank under his gaze. There was that in it which smote even +the innocent with fear. There was silence while one might count ten +--the master was gathering his wrath. Then he spoke: "Who tore this book?" + +There was not a sound. One could have heard a pin drop. The stillness +continued; the master searched face after face for signs of guilt. + +"Benjamin Rogers, did you tear this book?" + +A denial. Another pause. + +"Joseph Harper, did you?" + +Another denial. Tom's uneasiness grew more and more intense under the +slow torture of these proceedings. The master scanned the ranks of +boys--considered a while, then turned to the girls: + +"Amy Lawrence?" + +A shake of the head. + +"Gracie Miller?" + +The same sign. + +"Susan Harper, did you do this?" + +Another negative. The next girl was Becky Thatcher. Tom was trembling +from head to foot with excitement and a sense of the hopelessness of +the situation. + +"Rebecca Thatcher" [Tom glanced at her face--it was white with terror] +--"did you tear--no, look me in the face" [her hands rose in appeal] +--"did you tear this book?" + +A thought shot like lightning through Tom's brain. He sprang to his +feet and shouted--"I done it!" + +The school stared in perplexity at this incredible folly. Tom stood a +moment, to gather his dismembered faculties; and when he stepped +forward to go to his punishment the surprise, the gratitude, the +adoration that shone upon him out of poor Becky's eyes seemed pay +enough for a hundred floggings. Inspired by the splendor of his own +act, he took without an outcry the most merciless flaying that even Mr. +Dobbins had ever administered; and also received with indifference the +added cruelty of a command to remain two hours after school should be +dismissed--for he knew who would wait for him outside till his +captivity was done, and not count the tedious time as loss, either. + +Tom went to bed that night planning vengeance against Alfred Temple; +for with shame and repentance Becky had told him all, not forgetting +her own treachery; but even the longing for vengeance had to give way, +soon, to pleasanter musings, and he fell asleep at last with Becky's +latest words lingering dreamily in his ear-- + +"Tom, how COULD you be so noble!" + + + +CHAPTER XXI + +VACATION was approaching. The schoolmaster, always severe, grew +severer and more exacting than ever, for he wanted the school to make a +good showing on "Examination" day. His rod and his ferule were seldom +idle now--at least among the smaller pupils. Only the biggest boys, and +young ladies of eighteen and twenty, escaped lashing. Mr. Dobbins' +lashings were very vigorous ones, too; for although he carried, under +his wig, a perfectly bald and shiny head, he had only reached middle +age, and there was no sign of feebleness in his muscle. As the great +day approached, all the tyranny that was in him came to the surface; he +seemed to take a vindictive pleasure in punishing the least +shortcomings. The consequence was, that the smaller boys spent their +days in terror and suffering and their nights in plotting revenge. They +threw away no opportunity to do the master a mischief. But he kept +ahead all the time. The retribution that followed every vengeful +success was so sweeping and majestic that the boys always retired from +the field badly worsted. At last they conspired together and hit upon a +plan that promised a dazzling victory. They swore in the sign-painter's +boy, told him the scheme, and asked his help. He had his own reasons +for being delighted, for the master boarded in his father's family and +had given the boy ample cause to hate him. The master's wife would go +on a visit to the country in a few days, and there would be nothing to +interfere with the plan; the master always prepared himself for great +occasions by getting pretty well fuddled, and the sign-painter's boy +said that when the dominie had reached the proper condition on +Examination Evening he would "manage the thing" while he napped in his +chair; then he would have him awakened at the right time and hurried +away to school. + +In the fulness of time the interesting occasion arrived. At eight in +the evening the schoolhouse was brilliantly lighted, and adorned with +wreaths and festoons of foliage and flowers. The master sat throned in +his great chair upon a raised platform, with his blackboard behind him. +He was looking tolerably mellow. Three rows of benches on each side and +six rows in front of him were occupied by the dignitaries of the town +and by the parents of the pupils. To his left, back of the rows of +citizens, was a spacious temporary platform upon which were seated the +scholars who were to take part in the exercises of the evening; rows of +small boys, washed and dressed to an intolerable state of discomfort; +rows of gawky big boys; snowbanks of girls and young ladies clad in +lawn and muslin and conspicuously conscious of their bare arms, their +grandmothers' ancient trinkets, their bits of pink and blue ribbon and +the flowers in their hair. All the rest of the house was filled with +non-participating scholars. + +The exercises began. A very little boy stood up and sheepishly +recited, "You'd scarce expect one of my age to speak in public on the +stage," etc.--accompanying himself with the painfully exact and +spasmodic gestures which a machine might have used--supposing the +machine to be a trifle out of order. But he got through safely, though +cruelly scared, and got a fine round of applause when he made his +manufactured bow and retired. + +A little shamefaced girl lisped, "Mary had a little lamb," etc., +performed a compassion-inspiring curtsy, got her meed of applause, and +sat down flushed and happy. + +Tom Sawyer stepped forward with conceited confidence and soared into +the unquenchable and indestructible "Give me liberty or give me death" +speech, with fine fury and frantic gesticulation, and broke down in the +middle of it. A ghastly stage-fright seized him, his legs quaked under +him and he was like to choke. True, he had the manifest sympathy of the +house but he had the house's silence, too, which was even worse than +its sympathy. The master frowned, and this completed the disaster. Tom +struggled awhile and then retired, utterly defeated. There was a weak +attempt at applause, but it died early. + +"The Boy Stood on the Burning Deck" followed; also "The Assyrian Came +Down," and other declamatory gems. Then there were reading exercises, +and a spelling fight. The meagre Latin class recited with honor. The +prime feature of the evening was in order, now--original "compositions" +by the young ladies. Each in her turn stepped forward to the edge of +the platform, cleared her throat, held up her manuscript (tied with +dainty ribbon), and proceeded to read, with labored attention to +"expression" and punctuation. The themes were the same that had been +illuminated upon similar occasions by their mothers before them, their +grandmothers, and doubtless all their ancestors in the female line +clear back to the Crusades. "Friendship" was one; "Memories of Other +Days"; "Religion in History"; "Dream Land"; "The Advantages of +Culture"; "Forms of Political Government Compared and Contrasted"; +"Melancholy"; "Filial Love"; "Heart Longings," etc., etc. + +A prevalent feature in these compositions was a nursed and petted +melancholy; another was a wasteful and opulent gush of "fine language"; +another was a tendency to lug in by the ears particularly prized words +and phrases until they were worn entirely out; and a peculiarity that +conspicuously marked and marred them was the inveterate and intolerable +sermon that wagged its crippled tail at the end of each and every one +of them. No matter what the subject might be, a brain-racking effort +was made to squirm it into some aspect or other that the moral and +religious mind could contemplate with edification. The glaring +insincerity of these sermons was not sufficient to compass the +banishment of the fashion from the schools, and it is not sufficient +to-day; it never will be sufficient while the world stands, perhaps. +There is no school in all our land where the young ladies do not feel +obliged to close their compositions with a sermon; and you will find +that the sermon of the most frivolous and the least religious girl in +the school is always the longest and the most relentlessly pious. But +enough of this. Homely truth is unpalatable. + +Let us return to the "Examination." The first composition that was +read was one entitled "Is this, then, Life?" Perhaps the reader can +endure an extract from it: + + "In the common walks of life, with what delightful + emotions does the youthful mind look forward to some + anticipated scene of festivity! Imagination is busy + sketching rose-tinted pictures of joy. In fancy, the + voluptuous votary of fashion sees herself amid the + festive throng, 'the observed of all observers.' Her + graceful form, arrayed in snowy robes, is whirling + through the mazes of the joyous dance; her eye is + brightest, her step is lightest in the gay assembly. + + "In such delicious fancies time quickly glides by, + and the welcome hour arrives for her entrance into + the Elysian world, of which she has had such bright + dreams. How fairy-like does everything appear to + her enchanted vision! Each new scene is more charming + than the last. But after a while she finds that + beneath this goodly exterior, all is vanity, the + flattery which once charmed her soul, now grates + harshly upon her ear; the ball-room has lost its + charms; and with wasted health and imbittered heart, + she turns away with the conviction that earthly + pleasures cannot satisfy the longings of the soul!" + +And so forth and so on. There was a buzz of gratification from time to +time during the reading, accompanied by whispered ejaculations of "How +sweet!" "How eloquent!" "So true!" etc., and after the thing had closed +with a peculiarly afflicting sermon the applause was enthusiastic. + +Then arose a slim, melancholy girl, whose face had the "interesting" +paleness that comes of pills and indigestion, and read a "poem." Two +stanzas of it will do: + + "A MISSOURI MAIDEN'S FAREWELL TO ALABAMA + + "Alabama, good-bye! I love thee well! + But yet for a while do I leave thee now! + Sad, yes, sad thoughts of thee my heart doth swell, + And burning recollections throng my brow! + For I have wandered through thy flowery woods; + Have roamed and read near Tallapoosa's stream; + Have listened to Tallassee's warring floods, + And wooed on Coosa's side Aurora's beam. + + "Yet shame I not to bear an o'er-full heart, + Nor blush to turn behind my tearful eyes; + 'Tis from no stranger land I now must part, + 'Tis to no strangers left I yield these sighs. + Welcome and home were mine within this State, + Whose vales I leave--whose spires fade fast from me + And cold must be mine eyes, and heart, and tete, + When, dear Alabama! they turn cold on thee!" + +There were very few there who knew what "tete" meant, but the poem was +very satisfactory, nevertheless. + +Next appeared a dark-complexioned, black-eyed, black-haired young +lady, who paused an impressive moment, assumed a tragic expression, and +began to read in a measured, solemn tone: + + "A VISION + + "Dark and tempestuous was night. Around the + throne on high not a single star quivered; but + the deep intonations of the heavy thunder + constantly vibrated upon the ear; whilst the + terrific lightning revelled in angry mood + through the cloudy chambers of heaven, seeming + to scorn the power exerted over its terror by + the illustrious Franklin! Even the boisterous + winds unanimously came forth from their mystic + homes, and blustered about as if to enhance by + their aid the wildness of the scene. + + "At such a time, so dark, so dreary, for human + sympathy my very spirit sighed; but instead thereof, + + "'My dearest friend, my counsellor, my comforter + and guide--My joy in grief, my second bliss + in joy,' came to my side. She moved like one of + those bright beings pictured in the sunny walks + of fancy's Eden by the romantic and young, a + queen of beauty unadorned save by her own + transcendent loveliness. So soft was her step, it + failed to make even a sound, and but for the + magical thrill imparted by her genial touch, as + other unobtrusive beauties, she would have glided + away un-perceived--unsought. A strange sadness + rested upon her features, like icy tears upon + the robe of December, as she pointed to the + contending elements without, and bade me contemplate + the two beings presented." + +This nightmare occupied some ten pages of manuscript and wound up with +a sermon so destructive of all hope to non-Presbyterians that it took +the first prize. This composition was considered to be the very finest +effort of the evening. The mayor of the village, in delivering the +prize to the author of it, made a warm speech in which he said that it +was by far the most "eloquent" thing he had ever listened to, and that +Daniel Webster himself might well be proud of it. + +It may be remarked, in passing, that the number of compositions in +which the word "beauteous" was over-fondled, and human experience +referred to as "life's page," was up to the usual average. + +Now the master, mellow almost to the verge of geniality, put his chair +aside, turned his back to the audience, and began to draw a map of +America on the blackboard, to exercise the geography class upon. But he +made a sad business of it with his unsteady hand, and a smothered +titter rippled over the house. He knew what the matter was, and set +himself to right it. He sponged out lines and remade them; but he only +distorted them more than ever, and the tittering was more pronounced. +He threw his entire attention upon his work, now, as if determined not +to be put down by the mirth. He felt that all eyes were fastened upon +him; he imagined he was succeeding, and yet the tittering continued; it +even manifestly increased. And well it might. There was a garret above, +pierced with a scuttle over his head; and down through this scuttle +came a cat, suspended around the haunches by a string; she had a rag +tied about her head and jaws to keep her from mewing; as she slowly +descended she curved upward and clawed at the string, she swung +downward and clawed at the intangible air. The tittering rose higher +and higher--the cat was within six inches of the absorbed teacher's +head--down, down, a little lower, and she grabbed his wig with her +desperate claws, clung to it, and was snatched up into the garret in an +instant with her trophy still in her possession! And how the light did +blaze abroad from the master's bald pate--for the sign-painter's boy +had GILDED it! + +That broke up the meeting. The boys were avenged. Vacation had come. + + NOTE:--The pretended "compositions" quoted in + this chapter are taken without alteration from a + volume entitled "Prose and Poetry, by a Western + Lady"--but they are exactly and precisely after + the schoolgirl pattern, and hence are much + happier than any mere imitations could be. + + + +CHAPTER XXII + +TOM joined the new order of Cadets of Temperance, being attracted by +the showy character of their "regalia." He promised to abstain from +smoking, chewing, and profanity as long as he remained a member. Now he +found out a new thing--namely, that to promise not to do a thing is the +surest way in the world to make a body want to go and do that very +thing. Tom soon found himself tormented with a desire to drink and +swear; the desire grew to be so intense that nothing but the hope of a +chance to display himself in his red sash kept him from withdrawing +from the order. Fourth of July was coming; but he soon gave that up +--gave it up before he had worn his shackles over forty-eight hours--and +fixed his hopes upon old Judge Frazer, justice of the peace, who was +apparently on his deathbed and would have a big public funeral, since +he was so high an official. During three days Tom was deeply concerned +about the Judge's condition and hungry for news of it. Sometimes his +hopes ran high--so high that he would venture to get out his regalia +and practise before the looking-glass. But the Judge had a most +discouraging way of fluctuating. At last he was pronounced upon the +mend--and then convalescent. Tom was disgusted; and felt a sense of +injury, too. He handed in his resignation at once--and that night the +Judge suffered a relapse and died. Tom resolved that he would never +trust a man like that again. + +The funeral was a fine thing. The Cadets paraded in a style calculated +to kill the late member with envy. Tom was a free boy again, however +--there was something in that. He could drink and swear, now--but found +to his surprise that he did not want to. The simple fact that he could, +took the desire away, and the charm of it. + +Tom presently wondered to find that his coveted vacation was beginning +to hang a little heavily on his hands. + +He attempted a diary--but nothing happened during three days, and so +he abandoned it. + +The first of all the negro minstrel shows came to town, and made a +sensation. Tom and Joe Harper got up a band of performers and were +happy for two days. + +Even the Glorious Fourth was in some sense a failure, for it rained +hard, there was no procession in consequence, and the greatest man in +the world (as Tom supposed), Mr. Benton, an actual United States +Senator, proved an overwhelming disappointment--for he was not +twenty-five feet high, nor even anywhere in the neighborhood of it. + +A circus came. The boys played circus for three days afterward in +tents made of rag carpeting--admission, three pins for boys, two for +girls--and then circusing was abandoned. + +A phrenologist and a mesmerizer came--and went again and left the +village duller and drearier than ever. + +There were some boys-and-girls' parties, but they were so few and so +delightful that they only made the aching voids between ache the harder. + +Becky Thatcher was gone to her Constantinople home to stay with her +parents during vacation--so there was no bright side to life anywhere. + +The dreadful secret of the murder was a chronic misery. It was a very +cancer for permanency and pain. + +Then came the measles. + +During two long weeks Tom lay a prisoner, dead to the world and its +happenings. He was very ill, he was interested in nothing. When he got +upon his feet at last and moved feebly down-town, a melancholy change +had come over everything and every creature. There had been a +"revival," and everybody had "got religion," not only the adults, but +even the boys and girls. Tom went about, hoping against hope for the +sight of one blessed sinful face, but disappointment crossed him +everywhere. He found Joe Harper studying a Testament, and turned sadly +away from the depressing spectacle. He sought Ben Rogers, and found him +visiting the poor with a basket of tracts. He hunted up Jim Hollis, who +called his attention to the precious blessing of his late measles as a +warning. Every boy he encountered added another ton to his depression; +and when, in desperation, he flew for refuge at last to the bosom of +Huckleberry Finn and was received with a Scriptural quotation, his +heart broke and he crept home and to bed realizing that he alone of all +the town was lost, forever and forever. + +And that night there came on a terrific storm, with driving rain, +awful claps of thunder and blinding sheets of lightning. He covered his +head with the bedclothes and waited in a horror of suspense for his +doom; for he had not the shadow of a doubt that all this hubbub was +about him. He believed he had taxed the forbearance of the powers above +to the extremity of endurance and that this was the result. It might +have seemed to him a waste of pomp and ammunition to kill a bug with a +battery of artillery, but there seemed nothing incongruous about the +getting up such an expensive thunderstorm as this to knock the turf +from under an insect like himself. + +By and by the tempest spent itself and died without accomplishing its +object. The boy's first impulse was to be grateful, and reform. His +second was to wait--for there might not be any more storms. + +The next day the doctors were back; Tom had relapsed. The three weeks +he spent on his back this time seemed an entire age. When he got abroad +at last he was hardly grateful that he had been spared, remembering how +lonely was his estate, how companionless and forlorn he was. He drifted +listlessly down the street and found Jim Hollis acting as judge in a +juvenile court that was trying a cat for murder, in the presence of her +victim, a bird. He found Joe Harper and Huck Finn up an alley eating a +stolen melon. Poor lads! they--like Tom--had suffered a relapse. + + + +CHAPTER XXIII + +AT last the sleepy atmosphere was stirred--and vigorously: the murder +trial came on in the court. It became the absorbing topic of village +talk immediately. Tom could not get away from it. Every reference to +the murder sent a shudder to his heart, for his troubled conscience and +fears almost persuaded him that these remarks were put forth in his +hearing as "feelers"; he did not see how he could be suspected of +knowing anything about the murder, but still he could not be +comfortable in the midst of this gossip. It kept him in a cold shiver +all the time. He took Huck to a lonely place to have a talk with him. +It would be some relief to unseal his tongue for a little while; to +divide his burden of distress with another sufferer. Moreover, he +wanted to assure himself that Huck had remained discreet. + +"Huck, have you ever told anybody about--that?" + +"'Bout what?" + +"You know what." + +"Oh--'course I haven't." + +"Never a word?" + +"Never a solitary word, so help me. What makes you ask?" + +"Well, I was afeard." + +"Why, Tom Sawyer, we wouldn't be alive two days if that got found out. +YOU know that." + +Tom felt more comfortable. After a pause: + +"Huck, they couldn't anybody get you to tell, could they?" + +"Get me to tell? Why, if I wanted that half-breed devil to drownd me +they could get me to tell. They ain't no different way." + +"Well, that's all right, then. I reckon we're safe as long as we keep +mum. But let's swear again, anyway. It's more surer." + +"I'm agreed." + +So they swore again with dread solemnities. + +"What is the talk around, Huck? I've heard a power of it." + +"Talk? Well, it's just Muff Potter, Muff Potter, Muff Potter all the +time. It keeps me in a sweat, constant, so's I want to hide som'ers." + +"That's just the same way they go on round me. I reckon he's a goner. +Don't you feel sorry for him, sometimes?" + +"Most always--most always. He ain't no account; but then he hain't +ever done anything to hurt anybody. Just fishes a little, to get money +to get drunk on--and loafs around considerable; but lord, we all do +that--leastways most of us--preachers and such like. But he's kind of +good--he give me half a fish, once, when there warn't enough for two; +and lots of times he's kind of stood by me when I was out of luck." + +"Well, he's mended kites for me, Huck, and knitted hooks on to my +line. I wish we could get him out of there." + +"My! we couldn't get him out, Tom. And besides, 'twouldn't do any +good; they'd ketch him again." + +"Yes--so they would. But I hate to hear 'em abuse him so like the +dickens when he never done--that." + +"I do too, Tom. Lord, I hear 'em say he's the bloodiest looking +villain in this country, and they wonder he wasn't ever hung before." + +"Yes, they talk like that, all the time. I've heard 'em say that if he +was to get free they'd lynch him." + +"And they'd do it, too." + +The boys had a long talk, but it brought them little comfort. As the +twilight drew on, they found themselves hanging about the neighborhood +of the little isolated jail, perhaps with an undefined hope that +something would happen that might clear away their difficulties. But +nothing happened; there seemed to be no angels or fairies interested in +this luckless captive. + +The boys did as they had often done before--went to the cell grating +and gave Potter some tobacco and matches. He was on the ground floor +and there were no guards. + +His gratitude for their gifts had always smote their consciences +before--it cut deeper than ever, this time. They felt cowardly and +treacherous to the last degree when Potter said: + +"You've been mighty good to me, boys--better'n anybody else in this +town. And I don't forget it, I don't. Often I says to myself, says I, +'I used to mend all the boys' kites and things, and show 'em where the +good fishin' places was, and befriend 'em what I could, and now they've +all forgot old Muff when he's in trouble; but Tom don't, and Huck +don't--THEY don't forget him, says I, 'and I don't forget them.' Well, +boys, I done an awful thing--drunk and crazy at the time--that's the +only way I account for it--and now I got to swing for it, and it's +right. Right, and BEST, too, I reckon--hope so, anyway. Well, we won't +talk about that. I don't want to make YOU feel bad; you've befriended +me. But what I want to say, is, don't YOU ever get drunk--then you won't +ever get here. Stand a litter furder west--so--that's it; it's a prime +comfort to see faces that's friendly when a body's in such a muck of +trouble, and there don't none come here but yourn. Good friendly +faces--good friendly faces. Git up on one another's backs and let me +touch 'em. That's it. Shake hands--yourn'll come through the bars, but +mine's too big. Little hands, and weak--but they've helped Muff Potter +a power, and they'd help him more if they could." + +Tom went home miserable, and his dreams that night were full of +horrors. The next day and the day after, he hung about the court-room, +drawn by an almost irresistible impulse to go in, but forcing himself +to stay out. Huck was having the same experience. They studiously +avoided each other. Each wandered away, from time to time, but the same +dismal fascination always brought them back presently. Tom kept his +ears open when idlers sauntered out of the court-room, but invariably +heard distressing news--the toils were closing more and more +relentlessly around poor Potter. At the end of the second day the +village talk was to the effect that Injun Joe's evidence stood firm and +unshaken, and that there was not the slightest question as to what the +jury's verdict would be. + +Tom was out late, that night, and came to bed through the window. He +was in a tremendous state of excitement. It was hours before he got to +sleep. All the village flocked to the court-house the next morning, for +this was to be the great day. Both sexes were about equally represented +in the packed audience. After a long wait the jury filed in and took +their places; shortly afterward, Potter, pale and haggard, timid and +hopeless, was brought in, with chains upon him, and seated where all +the curious eyes could stare at him; no less conspicuous was Injun Joe, +stolid as ever. There was another pause, and then the judge arrived and +the sheriff proclaimed the opening of the court. The usual whisperings +among the lawyers and gathering together of papers followed. These +details and accompanying delays worked up an atmosphere of preparation +that was as impressive as it was fascinating. + +Now a witness was called who testified that he found Muff Potter +washing in the brook, at an early hour of the morning that the murder +was discovered, and that he immediately sneaked away. After some +further questioning, counsel for the prosecution said: + +"Take the witness." + +The prisoner raised his eyes for a moment, but dropped them again when +his own counsel said: + +"I have no questions to ask him." + +The next witness proved the finding of the knife near the corpse. +Counsel for the prosecution said: + +"Take the witness." + +"I have no questions to ask him," Potter's lawyer replied. + +A third witness swore he had often seen the knife in Potter's +possession. + +"Take the witness." + +Counsel for Potter declined to question him. The faces of the audience +began to betray annoyance. Did this attorney mean to throw away his +client's life without an effort? + +Several witnesses deposed concerning Potter's guilty behavior when +brought to the scene of the murder. They were allowed to leave the +stand without being cross-questioned. + +Every detail of the damaging circumstances that occurred in the +graveyard upon that morning which all present remembered so well was +brought out by credible witnesses, but none of them were cross-examined +by Potter's lawyer. The perplexity and dissatisfaction of the house +expressed itself in murmurs and provoked a reproof from the bench. +Counsel for the prosecution now said: + +"By the oaths of citizens whose simple word is above suspicion, we +have fastened this awful crime, beyond all possibility of question, +upon the unhappy prisoner at the bar. We rest our case here." + +A groan escaped from poor Potter, and he put his face in his hands and +rocked his body softly to and fro, while a painful silence reigned in +the court-room. Many men were moved, and many women's compassion +testified itself in tears. Counsel for the defence rose and said: + +"Your honor, in our remarks at the opening of this trial, we +foreshadowed our purpose to prove that our client did this fearful deed +while under the influence of a blind and irresponsible delirium +produced by drink. We have changed our mind. We shall not offer that +plea." [Then to the clerk:] "Call Thomas Sawyer!" + +A puzzled amazement awoke in every face in the house, not even +excepting Potter's. Every eye fastened itself with wondering interest +upon Tom as he rose and took his place upon the stand. The boy looked +wild enough, for he was badly scared. The oath was administered. + +"Thomas Sawyer, where were you on the seventeenth of June, about the +hour of midnight?" + +Tom glanced at Injun Joe's iron face and his tongue failed him. The +audience listened breathless, but the words refused to come. After a +few moments, however, the boy got a little of his strength back, and +managed to put enough of it into his voice to make part of the house +hear: + +"In the graveyard!" + +"A little bit louder, please. Don't be afraid. You were--" + +"In the graveyard." + +A contemptuous smile flitted across Injun Joe's face. + +"Were you anywhere near Horse Williams' grave?" + +"Yes, sir." + +"Speak up--just a trifle louder. How near were you?" + +"Near as I am to you." + +"Were you hidden, or not?" + +"I was hid." + +"Where?" + +"Behind the elms that's on the edge of the grave." + +Injun Joe gave a barely perceptible start. + +"Any one with you?" + +"Yes, sir. I went there with--" + +"Wait--wait a moment. Never mind mentioning your companion's name. We +will produce him at the proper time. Did you carry anything there with +you." + +Tom hesitated and looked confused. + +"Speak out, my boy--don't be diffident. The truth is always +respectable. What did you take there?" + +"Only a--a--dead cat." + +There was a ripple of mirth, which the court checked. + +"We will produce the skeleton of that cat. Now, my boy, tell us +everything that occurred--tell it in your own way--don't skip anything, +and don't be afraid." + +Tom began--hesitatingly at first, but as he warmed to his subject his +words flowed more and more easily; in a little while every sound ceased +but his own voice; every eye fixed itself upon him; with parted lips +and bated breath the audience hung upon his words, taking no note of +time, rapt in the ghastly fascinations of the tale. The strain upon +pent emotion reached its climax when the boy said: + +"--and as the doctor fetched the board around and Muff Potter fell, +Injun Joe jumped with the knife and--" + +Crash! Quick as lightning the half-breed sprang for a window, tore his +way through all opposers, and was gone! + + + +CHAPTER XXIV + +TOM was a glittering hero once more--the pet of the old, the envy of +the young. His name even went into immortal print, for the village +paper magnified him. There were some that believed he would be +President, yet, if he escaped hanging. + +As usual, the fickle, unreasoning world took Muff Potter to its bosom +and fondled him as lavishly as it had abused him before. But that sort +of conduct is to the world's credit; therefore it is not well to find +fault with it. + +Tom's days were days of splendor and exultation to him, but his nights +were seasons of horror. Injun Joe infested all his dreams, and always +with doom in his eye. Hardly any temptation could persuade the boy to +stir abroad after nightfall. Poor Huck was in the same state of +wretchedness and terror, for Tom had told the whole story to the lawyer +the night before the great day of the trial, and Huck was sore afraid +that his share in the business might leak out, yet, notwithstanding +Injun Joe's flight had saved him the suffering of testifying in court. +The poor fellow had got the attorney to promise secrecy, but what of +that? Since Tom's harassed conscience had managed to drive him to the +lawyer's house by night and wring a dread tale from lips that had been +sealed with the dismalest and most formidable of oaths, Huck's +confidence in the human race was well-nigh obliterated. + +Daily Muff Potter's gratitude made Tom glad he had spoken; but nightly +he wished he had sealed up his tongue. + +Half the time Tom was afraid Injun Joe would never be captured; the +other half he was afraid he would be. He felt sure he never could draw +a safe breath again until that man was dead and he had seen the corpse. + +Rewards had been offered, the country had been scoured, but no Injun +Joe was found. One of those omniscient and awe-inspiring marvels, a +detective, came up from St. Louis, moused around, shook his head, +looked wise, and made that sort of astounding success which members of +that craft usually achieve. That is to say, he "found a clew." But you +can't hang a "clew" for murder, and so after that detective had got +through and gone home, Tom felt just as insecure as he was before. + +The slow days drifted on, and each left behind it a slightly lightened +weight of apprehension. + + + +CHAPTER XXV + +THERE comes a time in every rightly-constructed boy's life when he has +a raging desire to go somewhere and dig for hidden treasure. This +desire suddenly came upon Tom one day. He sallied out to find Joe +Harper, but failed of success. Next he sought Ben Rogers; he had gone +fishing. Presently he stumbled upon Huck Finn the Red-Handed. Huck +would answer. Tom took him to a private place and opened the matter to +him confidentially. Huck was willing. Huck was always willing to take a +hand in any enterprise that offered entertainment and required no +capital, for he had a troublesome superabundance of that sort of time +which is not money. "Where'll we dig?" said Huck. + +"Oh, most anywhere." + +"Why, is it hid all around?" + +"No, indeed it ain't. It's hid in mighty particular places, Huck +--sometimes on islands, sometimes in rotten chests under the end of a +limb of an old dead tree, just where the shadow falls at midnight; but +mostly under the floor in ha'nted houses." + +"Who hides it?" + +"Why, robbers, of course--who'd you reckon? Sunday-school +sup'rintendents?" + +"I don't know. If 'twas mine I wouldn't hide it; I'd spend it and have +a good time." + +"So would I. But robbers don't do that way. They always hide it and +leave it there." + +"Don't they come after it any more?" + +"No, they think they will, but they generally forget the marks, or +else they die. Anyway, it lays there a long time and gets rusty; and by +and by somebody finds an old yellow paper that tells how to find the +marks--a paper that's got to be ciphered over about a week because it's +mostly signs and hy'roglyphics." + +"Hyro--which?" + +"Hy'roglyphics--pictures and things, you know, that don't seem to mean +anything." + +"Have you got one of them papers, Tom?" + +"No." + +"Well then, how you going to find the marks?" + +"I don't want any marks. They always bury it under a ha'nted house or +on an island, or under a dead tree that's got one limb sticking out. +Well, we've tried Jackson's Island a little, and we can try it again +some time; and there's the old ha'nted house up the Still-House branch, +and there's lots of dead-limb trees--dead loads of 'em." + +"Is it under all of them?" + +"How you talk! No!" + +"Then how you going to know which one to go for?" + +"Go for all of 'em!" + +"Why, Tom, it'll take all summer." + +"Well, what of that? Suppose you find a brass pot with a hundred +dollars in it, all rusty and gray, or rotten chest full of di'monds. +How's that?" + +Huck's eyes glowed. + +"That's bully. Plenty bully enough for me. Just you gimme the hundred +dollars and I don't want no di'monds." + +"All right. But I bet you I ain't going to throw off on di'monds. Some +of 'em's worth twenty dollars apiece--there ain't any, hardly, but's +worth six bits or a dollar." + +"No! Is that so?" + +"Cert'nly--anybody'll tell you so. Hain't you ever seen one, Huck?" + +"Not as I remember." + +"Oh, kings have slathers of them." + +"Well, I don' know no kings, Tom." + +"I reckon you don't. But if you was to go to Europe you'd see a raft +of 'em hopping around." + +"Do they hop?" + +"Hop?--your granny! No!" + +"Well, what did you say they did, for?" + +"Shucks, I only meant you'd SEE 'em--not hopping, of course--what do +they want to hop for?--but I mean you'd just see 'em--scattered around, +you know, in a kind of a general way. Like that old humpbacked Richard." + +"Richard? What's his other name?" + +"He didn't have any other name. Kings don't have any but a given name." + +"No?" + +"But they don't." + +"Well, if they like it, Tom, all right; but I don't want to be a king +and have only just a given name, like a nigger. But say--where you +going to dig first?" + +"Well, I don't know. S'pose we tackle that old dead-limb tree on the +hill t'other side of Still-House branch?" + +"I'm agreed." + +So they got a crippled pick and a shovel, and set out on their +three-mile tramp. They arrived hot and panting, and threw themselves +down in the shade of a neighboring elm to rest and have a smoke. + +"I like this," said Tom. + +"So do I." + +"Say, Huck, if we find a treasure here, what you going to do with your +share?" + +"Well, I'll have pie and a glass of soda every day, and I'll go to +every circus that comes along. I bet I'll have a gay time." + +"Well, ain't you going to save any of it?" + +"Save it? What for?" + +"Why, so as to have something to live on, by and by." + +"Oh, that ain't any use. Pap would come back to thish-yer town some +day and get his claws on it if I didn't hurry up, and I tell you he'd +clean it out pretty quick. What you going to do with yourn, Tom?" + +"I'm going to buy a new drum, and a sure-'nough sword, and a red +necktie and a bull pup, and get married." + +"Married!" + +"That's it." + +"Tom, you--why, you ain't in your right mind." + +"Wait--you'll see." + +"Well, that's the foolishest thing you could do. Look at pap and my +mother. Fight! Why, they used to fight all the time. I remember, mighty +well." + +"That ain't anything. The girl I'm going to marry won't fight." + +"Tom, I reckon they're all alike. They'll all comb a body. Now you +better think 'bout this awhile. I tell you you better. What's the name +of the gal?" + +"It ain't a gal at all--it's a girl." + +"It's all the same, I reckon; some says gal, some says girl--both's +right, like enough. Anyway, what's her name, Tom?" + +"I'll tell you some time--not now." + +"All right--that'll do. Only if you get married I'll be more lonesomer +than ever." + +"No you won't. You'll come and live with me. Now stir out of this and +we'll go to digging." + +They worked and sweated for half an hour. No result. They toiled +another half-hour. Still no result. Huck said: + +"Do they always bury it as deep as this?" + +"Sometimes--not always. Not generally. I reckon we haven't got the +right place." + +So they chose a new spot and began again. The labor dragged a little, +but still they made progress. They pegged away in silence for some +time. Finally Huck leaned on his shovel, swabbed the beaded drops from +his brow with his sleeve, and said: + +"Where you going to dig next, after we get this one?" + +"I reckon maybe we'll tackle the old tree that's over yonder on +Cardiff Hill back of the widow's." + +"I reckon that'll be a good one. But won't the widow take it away from +us, Tom? It's on her land." + +"SHE take it away! Maybe she'd like to try it once. Whoever finds one +of these hid treasures, it belongs to him. It don't make any difference +whose land it's on." + +That was satisfactory. The work went on. By and by Huck said: + +"Blame it, we must be in the wrong place again. What do you think?" + +"It is mighty curious, Huck. I don't understand it. Sometimes witches +interfere. I reckon maybe that's what's the trouble now." + +"Shucks! Witches ain't got no power in the daytime." + +"Well, that's so. I didn't think of that. Oh, I know what the matter +is! What a blamed lot of fools we are! You got to find out where the +shadow of the limb falls at midnight, and that's where you dig!" + +"Then consound it, we've fooled away all this work for nothing. Now +hang it all, we got to come back in the night. It's an awful long way. +Can you get out?" + +"I bet I will. We've got to do it to-night, too, because if somebody +sees these holes they'll know in a minute what's here and they'll go +for it." + +"Well, I'll come around and maow to-night." + +"All right. Let's hide the tools in the bushes." + +The boys were there that night, about the appointed time. They sat in +the shadow waiting. It was a lonely place, and an hour made solemn by +old traditions. Spirits whispered in the rustling leaves, ghosts lurked +in the murky nooks, the deep baying of a hound floated up out of the +distance, an owl answered with his sepulchral note. The boys were +subdued by these solemnities, and talked little. By and by they judged +that twelve had come; they marked where the shadow fell, and began to +dig. Their hopes commenced to rise. Their interest grew stronger, and +their industry kept pace with it. The hole deepened and still deepened, +but every time their hearts jumped to hear the pick strike upon +something, they only suffered a new disappointment. It was only a stone +or a chunk. At last Tom said: + +"It ain't any use, Huck, we're wrong again." + +"Well, but we CAN'T be wrong. We spotted the shadder to a dot." + +"I know it, but then there's another thing." + +"What's that?". + +"Why, we only guessed at the time. Like enough it was too late or too +early." + +Huck dropped his shovel. + +"That's it," said he. "That's the very trouble. We got to give this +one up. We can't ever tell the right time, and besides this kind of +thing's too awful, here this time of night with witches and ghosts +a-fluttering around so. I feel as if something's behind me all the time; +and I'm afeard to turn around, becuz maybe there's others in front +a-waiting for a chance. I been creeping all over, ever since I got here." + +"Well, I've been pretty much so, too, Huck. They most always put in a +dead man when they bury a treasure under a tree, to look out for it." + +"Lordy!" + +"Yes, they do. I've always heard that." + +"Tom, I don't like to fool around much where there's dead people. A +body's bound to get into trouble with 'em, sure." + +"I don't like to stir 'em up, either. S'pose this one here was to +stick his skull out and say something!" + +"Don't Tom! It's awful." + +"Well, it just is. Huck, I don't feel comfortable a bit." + +"Say, Tom, let's give this place up, and try somewheres else." + +"All right, I reckon we better." + +"What'll it be?" + +Tom considered awhile; and then said: + +"The ha'nted house. That's it!" + +"Blame it, I don't like ha'nted houses, Tom. Why, they're a dern sight +worse'n dead people. Dead people might talk, maybe, but they don't come +sliding around in a shroud, when you ain't noticing, and peep over your +shoulder all of a sudden and grit their teeth, the way a ghost does. I +couldn't stand such a thing as that, Tom--nobody could." + +"Yes, but, Huck, ghosts don't travel around only at night. They won't +hender us from digging there in the daytime." + +"Well, that's so. But you know mighty well people don't go about that +ha'nted house in the day nor the night." + +"Well, that's mostly because they don't like to go where a man's been +murdered, anyway--but nothing's ever been seen around that house except +in the night--just some blue lights slipping by the windows--no regular +ghosts." + +"Well, where you see one of them blue lights flickering around, Tom, +you can bet there's a ghost mighty close behind it. It stands to +reason. Becuz you know that they don't anybody but ghosts use 'em." + +"Yes, that's so. But anyway they don't come around in the daytime, so +what's the use of our being afeard?" + +"Well, all right. We'll tackle the ha'nted house if you say so--but I +reckon it's taking chances." + +They had started down the hill by this time. There in the middle of +the moonlit valley below them stood the "ha'nted" house, utterly +isolated, its fences gone long ago, rank weeds smothering the very +doorsteps, the chimney crumbled to ruin, the window-sashes vacant, a +corner of the roof caved in. The boys gazed awhile, half expecting to +see a blue light flit past a window; then talking in a low tone, as +befitted the time and the circumstances, they struck far off to the +right, to give the haunted house a wide berth, and took their way +homeward through the woods that adorned the rearward side of Cardiff +Hill. + + + +CHAPTER XXVI + +ABOUT noon the next day the boys arrived at the dead tree; they had +come for their tools. Tom was impatient to go to the haunted house; +Huck was measurably so, also--but suddenly said: + +"Lookyhere, Tom, do you know what day it is?" + +Tom mentally ran over the days of the week, and then quickly lifted +his eyes with a startled look in them-- + +"My! I never once thought of it, Huck!" + +"Well, I didn't neither, but all at once it popped onto me that it was +Friday." + +"Blame it, a body can't be too careful, Huck. We might 'a' got into an +awful scrape, tackling such a thing on a Friday." + +"MIGHT! Better say we WOULD! There's some lucky days, maybe, but +Friday ain't." + +"Any fool knows that. I don't reckon YOU was the first that found it +out, Huck." + +"Well, I never said I was, did I? And Friday ain't all, neither. I had +a rotten bad dream last night--dreampt about rats." + +"No! Sure sign of trouble. Did they fight?" + +"No." + +"Well, that's good, Huck. When they don't fight it's only a sign that +there's trouble around, you know. All we got to do is to look mighty +sharp and keep out of it. We'll drop this thing for to-day, and play. +Do you know Robin Hood, Huck?" + +"No. Who's Robin Hood?" + +"Why, he was one of the greatest men that was ever in England--and the +best. He was a robber." + +"Cracky, I wisht I was. Who did he rob?" + +"Only sheriffs and bishops and rich people and kings, and such like. +But he never bothered the poor. He loved 'em. He always divided up with +'em perfectly square." + +"Well, he must 'a' been a brick." + +"I bet you he was, Huck. Oh, he was the noblest man that ever was. +They ain't any such men now, I can tell you. He could lick any man in +England, with one hand tied behind him; and he could take his yew bow +and plug a ten-cent piece every time, a mile and a half." + +"What's a YEW bow?" + +"I don't know. It's some kind of a bow, of course. And if he hit that +dime only on the edge he would set down and cry--and curse. But we'll +play Robin Hood--it's nobby fun. I'll learn you." + +"I'm agreed." + +So they played Robin Hood all the afternoon, now and then casting a +yearning eye down upon the haunted house and passing a remark about the +morrow's prospects and possibilities there. As the sun began to sink +into the west they took their way homeward athwart the long shadows of +the trees and soon were buried from sight in the forests of Cardiff +Hill. + +On Saturday, shortly after noon, the boys were at the dead tree again. +They had a smoke and a chat in the shade, and then dug a little in +their last hole, not with great hope, but merely because Tom said there +were so many cases where people had given up a treasure after getting +down within six inches of it, and then somebody else had come along and +turned it up with a single thrust of a shovel. The thing failed this +time, however, so the boys shouldered their tools and went away feeling +that they had not trifled with fortune, but had fulfilled all the +requirements that belong to the business of treasure-hunting. + +When they reached the haunted house there was something so weird and +grisly about the dead silence that reigned there under the baking sun, +and something so depressing about the loneliness and desolation of the +place, that they were afraid, for a moment, to venture in. Then they +crept to the door and took a trembling peep. They saw a weed-grown, +floorless room, unplastered, an ancient fireplace, vacant windows, a +ruinous staircase; and here, there, and everywhere hung ragged and +abandoned cobwebs. They presently entered, softly, with quickened +pulses, talking in whispers, ears alert to catch the slightest sound, +and muscles tense and ready for instant retreat. + +In a little while familiarity modified their fears and they gave the +place a critical and interested examination, rather admiring their own +boldness, and wondering at it, too. Next they wanted to look up-stairs. +This was something like cutting off retreat, but they got to daring +each other, and of course there could be but one result--they threw +their tools into a corner and made the ascent. Up there were the same +signs of decay. In one corner they found a closet that promised +mystery, but the promise was a fraud--there was nothing in it. Their +courage was up now and well in hand. They were about to go down and +begin work when-- + +"Sh!" said Tom. + +"What is it?" whispered Huck, blanching with fright. + +"Sh!... There!... Hear it?" + +"Yes!... Oh, my! Let's run!" + +"Keep still! Don't you budge! They're coming right toward the door." + +The boys stretched themselves upon the floor with their eyes to +knot-holes in the planking, and lay waiting, in a misery of fear. + +"They've stopped.... No--coming.... Here they are. Don't whisper +another word, Huck. My goodness, I wish I was out of this!" + +Two men entered. Each boy said to himself: "There's the old deaf and +dumb Spaniard that's been about town once or twice lately--never saw +t'other man before." + +"T'other" was a ragged, unkempt creature, with nothing very pleasant +in his face. The Spaniard was wrapped in a serape; he had bushy white +whiskers; long white hair flowed from under his sombrero, and he wore +green goggles. When they came in, "t'other" was talking in a low voice; +they sat down on the ground, facing the door, with their backs to the +wall, and the speaker continued his remarks. His manner became less +guarded and his words more distinct as he proceeded: + +"No," said he, "I've thought it all over, and I don't like it. It's +dangerous." + +"Dangerous!" grunted the "deaf and dumb" Spaniard--to the vast +surprise of the boys. "Milksop!" + +This voice made the boys gasp and quake. It was Injun Joe's! There was +silence for some time. Then Joe said: + +"What's any more dangerous than that job up yonder--but nothing's come +of it." + +"That's different. Away up the river so, and not another house about. +'Twon't ever be known that we tried, anyway, long as we didn't succeed." + +"Well, what's more dangerous than coming here in the daytime!--anybody +would suspicion us that saw us." + +"I know that. But there warn't any other place as handy after that +fool of a job. I want to quit this shanty. I wanted to yesterday, only +it warn't any use trying to stir out of here, with those infernal boys +playing over there on the hill right in full view." + +"Those infernal boys" quaked again under the inspiration of this +remark, and thought how lucky it was that they had remembered it was +Friday and concluded to wait a day. They wished in their hearts they +had waited a year. + +The two men got out some food and made a luncheon. After a long and +thoughtful silence, Injun Joe said: + +"Look here, lad--you go back up the river where you belong. Wait there +till you hear from me. I'll take the chances on dropping into this town +just once more, for a look. We'll do that 'dangerous' job after I've +spied around a little and think things look well for it. Then for +Texas! We'll leg it together!" + +This was satisfactory. Both men presently fell to yawning, and Injun +Joe said: + +"I'm dead for sleep! It's your turn to watch." + +He curled down in the weeds and soon began to snore. His comrade +stirred him once or twice and he became quiet. Presently the watcher +began to nod; his head drooped lower and lower, both men began to snore +now. + +The boys drew a long, grateful breath. Tom whispered: + +"Now's our chance--come!" + +Huck said: + +"I can't--I'd die if they was to wake." + +Tom urged--Huck held back. At last Tom rose slowly and softly, and +started alone. But the first step he made wrung such a hideous creak +from the crazy floor that he sank down almost dead with fright. He +never made a second attempt. The boys lay there counting the dragging +moments till it seemed to them that time must be done and eternity +growing gray; and then they were grateful to note that at last the sun +was setting. + +Now one snore ceased. Injun Joe sat up, stared around--smiled grimly +upon his comrade, whose head was drooping upon his knees--stirred him +up with his foot and said: + +"Here! YOU'RE a watchman, ain't you! All right, though--nothing's +happened." + +"My! have I been asleep?" + +"Oh, partly, partly. Nearly time for us to be moving, pard. What'll we +do with what little swag we've got left?" + +"I don't know--leave it here as we've always done, I reckon. No use to +take it away till we start south. Six hundred and fifty in silver's +something to carry." + +"Well--all right--it won't matter to come here once more." + +"No--but I'd say come in the night as we used to do--it's better." + +"Yes: but look here; it may be a good while before I get the right +chance at that job; accidents might happen; 'tain't in such a very good +place; we'll just regularly bury it--and bury it deep." + +"Good idea," said the comrade, who walked across the room, knelt down, +raised one of the rearward hearth-stones and took out a bag that +jingled pleasantly. He subtracted from it twenty or thirty dollars for +himself and as much for Injun Joe, and passed the bag to the latter, +who was on his knees in the corner, now, digging with his bowie-knife. + +The boys forgot all their fears, all their miseries in an instant. +With gloating eyes they watched every movement. Luck!--the splendor of +it was beyond all imagination! Six hundred dollars was money enough to +make half a dozen boys rich! Here was treasure-hunting under the +happiest auspices--there would not be any bothersome uncertainty as to +where to dig. They nudged each other every moment--eloquent nudges and +easily understood, for they simply meant--"Oh, but ain't you glad NOW +we're here!" + +Joe's knife struck upon something. + +"Hello!" said he. + +"What is it?" said his comrade. + +"Half-rotten plank--no, it's a box, I believe. Here--bear a hand and +we'll see what it's here for. Never mind, I've broke a hole." + +He reached his hand in and drew it out-- + +"Man, it's money!" + +The two men examined the handful of coins. They were gold. The boys +above were as excited as themselves, and as delighted. + +Joe's comrade said: + +"We'll make quick work of this. There's an old rusty pick over amongst +the weeds in the corner the other side of the fireplace--I saw it a +minute ago." + +He ran and brought the boys' pick and shovel. Injun Joe took the pick, +looked it over critically, shook his head, muttered something to +himself, and then began to use it. The box was soon unearthed. It was +not very large; it was iron bound and had been very strong before the +slow years had injured it. The men contemplated the treasure awhile in +blissful silence. + +"Pard, there's thousands of dollars here," said Injun Joe. + +"'Twas always said that Murrel's gang used to be around here one +summer," the stranger observed. + +"I know it," said Injun Joe; "and this looks like it, I should say." + +"Now you won't need to do that job." + +The half-breed frowned. Said he: + +"You don't know me. Least you don't know all about that thing. 'Tain't +robbery altogether--it's REVENGE!" and a wicked light flamed in his +eyes. "I'll need your help in it. When it's finished--then Texas. Go +home to your Nance and your kids, and stand by till you hear from me." + +"Well--if you say so; what'll we do with this--bury it again?" + +"Yes. [Ravishing delight overhead.] NO! by the great Sachem, no! +[Profound distress overhead.] I'd nearly forgot. That pick had fresh +earth on it! [The boys were sick with terror in a moment.] What +business has a pick and a shovel here? What business with fresh earth +on them? Who brought them here--and where are they gone? Have you heard +anybody?--seen anybody? What! bury it again and leave them to come and +see the ground disturbed? Not exactly--not exactly. We'll take it to my +den." + +"Why, of course! Might have thought of that before. You mean Number +One?" + +"No--Number Two--under the cross. The other place is bad--too common." + +"All right. It's nearly dark enough to start." + +Injun Joe got up and went about from window to window cautiously +peeping out. Presently he said: + +"Who could have brought those tools here? Do you reckon they can be +up-stairs?" + +The boys' breath forsook them. Injun Joe put his hand on his knife, +halted a moment, undecided, and then turned toward the stairway. The +boys thought of the closet, but their strength was gone. The steps came +creaking up the stairs--the intolerable distress of the situation woke +the stricken resolution of the lads--they were about to spring for the +closet, when there was a crash of rotten timbers and Injun Joe landed +on the ground amid the debris of the ruined stairway. He gathered +himself up cursing, and his comrade said: + +"Now what's the use of all that? If it's anybody, and they're up +there, let them STAY there--who cares? If they want to jump down, now, +and get into trouble, who objects? It will be dark in fifteen minutes +--and then let them follow us if they want to. I'm willing. In my +opinion, whoever hove those things in here caught a sight of us and +took us for ghosts or devils or something. I'll bet they're running +yet." + +Joe grumbled awhile; then he agreed with his friend that what daylight +was left ought to be economized in getting things ready for leaving. +Shortly afterward they slipped out of the house in the deepening +twilight, and moved toward the river with their precious box. + +Tom and Huck rose up, weak but vastly relieved, and stared after them +through the chinks between the logs of the house. Follow? Not they. +They were content to reach ground again without broken necks, and take +the townward track over the hill. They did not talk much. They were too +much absorbed in hating themselves--hating the ill luck that made them +take the spade and the pick there. But for that, Injun Joe never would +have suspected. He would have hidden the silver with the gold to wait +there till his "revenge" was satisfied, and then he would have had the +misfortune to find that money turn up missing. Bitter, bitter luck that +the tools were ever brought there! + +They resolved to keep a lookout for that Spaniard when he should come +to town spying out for chances to do his revengeful job, and follow him +to "Number Two," wherever that might be. Then a ghastly thought +occurred to Tom. + +"Revenge? What if he means US, Huck!" + +"Oh, don't!" said Huck, nearly fainting. + +They talked it all over, and as they entered town they agreed to +believe that he might possibly mean somebody else--at least that he +might at least mean nobody but Tom, since only Tom had testified. + +Very, very small comfort it was to Tom to be alone in danger! Company +would be a palpable improvement, he thought. + + + +CHAPTER XXVII + +THE adventure of the day mightily tormented Tom's dreams that night. +Four times he had his hands on that rich treasure and four times it +wasted to nothingness in his fingers as sleep forsook him and +wakefulness brought back the hard reality of his misfortune. As he lay +in the early morning recalling the incidents of his great adventure, he +noticed that they seemed curiously subdued and far away--somewhat as if +they had happened in another world, or in a time long gone by. Then it +occurred to him that the great adventure itself must be a dream! There +was one very strong argument in favor of this idea--namely, that the +quantity of coin he had seen was too vast to be real. He had never seen +as much as fifty dollars in one mass before, and he was like all boys +of his age and station in life, in that he imagined that all references +to "hundreds" and "thousands" were mere fanciful forms of speech, and +that no such sums really existed in the world. He never had supposed +for a moment that so large a sum as a hundred dollars was to be found +in actual money in any one's possession. If his notions of hidden +treasure had been analyzed, they would have been found to consist of a +handful of real dimes and a bushel of vague, splendid, ungraspable +dollars. + +But the incidents of his adventure grew sensibly sharper and clearer +under the attrition of thinking them over, and so he presently found +himself leaning to the impression that the thing might not have been a +dream, after all. This uncertainty must be swept away. He would snatch +a hurried breakfast and go and find Huck. Huck was sitting on the +gunwale of a flatboat, listlessly dangling his feet in the water and +looking very melancholy. Tom concluded to let Huck lead up to the +subject. If he did not do it, then the adventure would be proved to +have been only a dream. + +"Hello, Huck!" + +"Hello, yourself." + +Silence, for a minute. + +"Tom, if we'd 'a' left the blame tools at the dead tree, we'd 'a' got +the money. Oh, ain't it awful!" + +"'Tain't a dream, then, 'tain't a dream! Somehow I most wish it was. +Dog'd if I don't, Huck." + +"What ain't a dream?" + +"Oh, that thing yesterday. I been half thinking it was." + +"Dream! If them stairs hadn't broke down you'd 'a' seen how much dream +it was! I've had dreams enough all night--with that patch-eyed Spanish +devil going for me all through 'em--rot him!" + +"No, not rot him. FIND him! Track the money!" + +"Tom, we'll never find him. A feller don't have only one chance for +such a pile--and that one's lost. I'd feel mighty shaky if I was to see +him, anyway." + +"Well, so'd I; but I'd like to see him, anyway--and track him out--to +his Number Two." + +"Number Two--yes, that's it. I been thinking 'bout that. But I can't +make nothing out of it. What do you reckon it is?" + +"I dono. It's too deep. Say, Huck--maybe it's the number of a house!" + +"Goody!... No, Tom, that ain't it. If it is, it ain't in this +one-horse town. They ain't no numbers here." + +"Well, that's so. Lemme think a minute. Here--it's the number of a +room--in a tavern, you know!" + +"Oh, that's the trick! They ain't only two taverns. We can find out +quick." + +"You stay here, Huck, till I come." + +Tom was off at once. He did not care to have Huck's company in public +places. He was gone half an hour. He found that in the best tavern, No. +2 had long been occupied by a young lawyer, and was still so occupied. +In the less ostentatious house, No. 2 was a mystery. The +tavern-keeper's young son said it was kept locked all the time, and he +never saw anybody go into it or come out of it except at night; he did +not know any particular reason for this state of things; had had some +little curiosity, but it was rather feeble; had made the most of the +mystery by entertaining himself with the idea that that room was +"ha'nted"; had noticed that there was a light in there the night before. + +"That's what I've found out, Huck. I reckon that's the very No. 2 +we're after." + +"I reckon it is, Tom. Now what you going to do?" + +"Lemme think." + +Tom thought a long time. Then he said: + +"I'll tell you. The back door of that No. 2 is the door that comes out +into that little close alley between the tavern and the old rattle trap +of a brick store. Now you get hold of all the door-keys you can find, +and I'll nip all of auntie's, and the first dark night we'll go there +and try 'em. And mind you, keep a lookout for Injun Joe, because he +said he was going to drop into town and spy around once more for a +chance to get his revenge. If you see him, you just follow him; and if +he don't go to that No. 2, that ain't the place." + +"Lordy, I don't want to foller him by myself!" + +"Why, it'll be night, sure. He mightn't ever see you--and if he did, +maybe he'd never think anything." + +"Well, if it's pretty dark I reckon I'll track him. I dono--I dono. +I'll try." + +"You bet I'll follow him, if it's dark, Huck. Why, he might 'a' found +out he couldn't get his revenge, and be going right after that money." + +"It's so, Tom, it's so. I'll foller him; I will, by jingoes!" + +"Now you're TALKING! Don't you ever weaken, Huck, and I won't." + + + +CHAPTER XXVIII + +THAT night Tom and Huck were ready for their adventure. They hung +about the neighborhood of the tavern until after nine, one watching the +alley at a distance and the other the tavern door. Nobody entered the +alley or left it; nobody resembling the Spaniard entered or left the +tavern door. The night promised to be a fair one; so Tom went home with +the understanding that if a considerable degree of darkness came on, +Huck was to come and "maow," whereupon he would slip out and try the +keys. But the night remained clear, and Huck closed his watch and +retired to bed in an empty sugar hogshead about twelve. + +Tuesday the boys had the same ill luck. Also Wednesday. But Thursday +night promised better. Tom slipped out in good season with his aunt's +old tin lantern, and a large towel to blindfold it with. He hid the +lantern in Huck's sugar hogshead and the watch began. An hour before +midnight the tavern closed up and its lights (the only ones +thereabouts) were put out. No Spaniard had been seen. Nobody had +entered or left the alley. Everything was auspicious. The blackness of +darkness reigned, the perfect stillness was interrupted only by +occasional mutterings of distant thunder. + +Tom got his lantern, lit it in the hogshead, wrapped it closely in the +towel, and the two adventurers crept in the gloom toward the tavern. +Huck stood sentry and Tom felt his way into the alley. Then there was a +season of waiting anxiety that weighed upon Huck's spirits like a +mountain. He began to wish he could see a flash from the lantern--it +would frighten him, but it would at least tell him that Tom was alive +yet. It seemed hours since Tom had disappeared. Surely he must have +fainted; maybe he was dead; maybe his heart had burst under terror and +excitement. In his uneasiness Huck found himself drawing closer and +closer to the alley; fearing all sorts of dreadful things, and +momentarily expecting some catastrophe to happen that would take away +his breath. There was not much to take away, for he seemed only able to +inhale it by thimblefuls, and his heart would soon wear itself out, the +way it was beating. Suddenly there was a flash of light and Tom came +tearing by him: "Run!" said he; "run, for your life!" + +He needn't have repeated it; once was enough; Huck was making thirty +or forty miles an hour before the repetition was uttered. The boys +never stopped till they reached the shed of a deserted slaughter-house +at the lower end of the village. Just as they got within its shelter +the storm burst and the rain poured down. As soon as Tom got his breath +he said: + +"Huck, it was awful! I tried two of the keys, just as soft as I could; +but they seemed to make such a power of racket that I couldn't hardly +get my breath I was so scared. They wouldn't turn in the lock, either. +Well, without noticing what I was doing, I took hold of the knob, and +open comes the door! It warn't locked! I hopped in, and shook off the +towel, and, GREAT CAESAR'S GHOST!" + +"What!--what'd you see, Tom?" + +"Huck, I most stepped onto Injun Joe's hand!" + +"No!" + +"Yes! He was lying there, sound asleep on the floor, with his old +patch on his eye and his arms spread out." + +"Lordy, what did you do? Did he wake up?" + +"No, never budged. Drunk, I reckon. I just grabbed that towel and +started!" + +"I'd never 'a' thought of the towel, I bet!" + +"Well, I would. My aunt would make me mighty sick if I lost it." + +"Say, Tom, did you see that box?" + +"Huck, I didn't wait to look around. I didn't see the box, I didn't +see the cross. I didn't see anything but a bottle and a tin cup on the +floor by Injun Joe; yes, I saw two barrels and lots more bottles in the +room. Don't you see, now, what's the matter with that ha'nted room?" + +"How?" + +"Why, it's ha'nted with whiskey! Maybe ALL the Temperance Taverns have +got a ha'nted room, hey, Huck?" + +"Well, I reckon maybe that's so. Who'd 'a' thought such a thing? But +say, Tom, now's a mighty good time to get that box, if Injun Joe's +drunk." + +"It is, that! You try it!" + +Huck shuddered. + +"Well, no--I reckon not." + +"And I reckon not, Huck. Only one bottle alongside of Injun Joe ain't +enough. If there'd been three, he'd be drunk enough and I'd do it." + +There was a long pause for reflection, and then Tom said: + +"Lookyhere, Huck, less not try that thing any more till we know Injun +Joe's not in there. It's too scary. Now, if we watch every night, we'll +be dead sure to see him go out, some time or other, and then we'll +snatch that box quicker'n lightning." + +"Well, I'm agreed. I'll watch the whole night long, and I'll do it +every night, too, if you'll do the other part of the job." + +"All right, I will. All you got to do is to trot up Hooper Street a +block and maow--and if I'm asleep, you throw some gravel at the window +and that'll fetch me." + +"Agreed, and good as wheat!" + +"Now, Huck, the storm's over, and I'll go home. It'll begin to be +daylight in a couple of hours. You go back and watch that long, will +you?" + +"I said I would, Tom, and I will. I'll ha'nt that tavern every night +for a year! I'll sleep all day and I'll stand watch all night." + +"That's all right. Now, where you going to sleep?" + +"In Ben Rogers' hayloft. He lets me, and so does his pap's nigger man, +Uncle Jake. I tote water for Uncle Jake whenever he wants me to, and +any time I ask him he gives me a little something to eat if he can +spare it. That's a mighty good nigger, Tom. He likes me, becuz I don't +ever act as if I was above him. Sometime I've set right down and eat +WITH him. But you needn't tell that. A body's got to do things when +he's awful hungry he wouldn't want to do as a steady thing." + +"Well, if I don't want you in the daytime, I'll let you sleep. I won't +come bothering around. Any time you see something's up, in the night, +just skip right around and maow." + + + +CHAPTER XXIX + +THE first thing Tom heard on Friday morning was a glad piece of news +--Judge Thatcher's family had come back to town the night before. Both +Injun Joe and the treasure sunk into secondary importance for a moment, +and Becky took the chief place in the boy's interest. He saw her and +they had an exhausting good time playing "hi-spy" and "gully-keeper" +with a crowd of their school-mates. The day was completed and crowned +in a peculiarly satisfactory way: Becky teased her mother to appoint +the next day for the long-promised and long-delayed picnic, and she +consented. The child's delight was boundless; and Tom's not more +moderate. The invitations were sent out before sunset, and straightway +the young folks of the village were thrown into a fever of preparation +and pleasurable anticipation. Tom's excitement enabled him to keep +awake until a pretty late hour, and he had good hopes of hearing Huck's +"maow," and of having his treasure to astonish Becky and the picnickers +with, next day; but he was disappointed. No signal came that night. + +Morning came, eventually, and by ten or eleven o'clock a giddy and +rollicking company were gathered at Judge Thatcher's, and everything +was ready for a start. It was not the custom for elderly people to mar +the picnics with their presence. The children were considered safe +enough under the wings of a few young ladies of eighteen and a few +young gentlemen of twenty-three or thereabouts. The old steam ferryboat +was chartered for the occasion; presently the gay throng filed up the +main street laden with provision-baskets. Sid was sick and had to miss +the fun; Mary remained at home to entertain him. The last thing Mrs. +Thatcher said to Becky, was: + +"You'll not get back till late. Perhaps you'd better stay all night +with some of the girls that live near the ferry-landing, child." + +"Then I'll stay with Susy Harper, mamma." + +"Very well. And mind and behave yourself and don't be any trouble." + +Presently, as they tripped along, Tom said to Becky: + +"Say--I'll tell you what we'll do. 'Stead of going to Joe Harper's +we'll climb right up the hill and stop at the Widow Douglas'. She'll +have ice-cream! She has it most every day--dead loads of it. And she'll +be awful glad to have us." + +"Oh, that will be fun!" + +Then Becky reflected a moment and said: + +"But what will mamma say?" + +"How'll she ever know?" + +The girl turned the idea over in her mind, and said reluctantly: + +"I reckon it's wrong--but--" + +"But shucks! Your mother won't know, and so what's the harm? All she +wants is that you'll be safe; and I bet you she'd 'a' said go there if +she'd 'a' thought of it. I know she would!" + +The Widow Douglas' splendid hospitality was a tempting bait. It and +Tom's persuasions presently carried the day. So it was decided to say +nothing anybody about the night's programme. Presently it occurred to +Tom that maybe Huck might come this very night and give the signal. The +thought took a deal of the spirit out of his anticipations. Still he +could not bear to give up the fun at Widow Douglas'. And why should he +give it up, he reasoned--the signal did not come the night before, so +why should it be any more likely to come to-night? The sure fun of the +evening outweighed the uncertain treasure; and, boy-like, he determined +to yield to the stronger inclination and not allow himself to think of +the box of money another time that day. + +Three miles below town the ferryboat stopped at the mouth of a woody +hollow and tied up. The crowd swarmed ashore and soon the forest +distances and craggy heights echoed far and near with shoutings and +laughter. All the different ways of getting hot and tired were gone +through with, and by-and-by the rovers straggled back to camp fortified +with responsible appetites, and then the destruction of the good things +began. After the feast there was a refreshing season of rest and chat +in the shade of spreading oaks. By-and-by somebody shouted: + +"Who's ready for the cave?" + +Everybody was. Bundles of candles were procured, and straightway there +was a general scamper up the hill. The mouth of the cave was up the +hillside--an opening shaped like a letter A. Its massive oaken door +stood unbarred. Within was a small chamber, chilly as an ice-house, and +walled by Nature with solid limestone that was dewy with a cold sweat. +It was romantic and mysterious to stand here in the deep gloom and look +out upon the green valley shining in the sun. But the impressiveness of +the situation quickly wore off, and the romping began again. The moment +a candle was lighted there was a general rush upon the owner of it; a +struggle and a gallant defence followed, but the candle was soon +knocked down or blown out, and then there was a glad clamor of laughter +and a new chase. But all things have an end. By-and-by the procession +went filing down the steep descent of the main avenue, the flickering +rank of lights dimly revealing the lofty walls of rock almost to their +point of junction sixty feet overhead. This main avenue was not more +than eight or ten feet wide. Every few steps other lofty and still +narrower crevices branched from it on either hand--for McDougal's cave +was but a vast labyrinth of crooked aisles that ran into each other and +out again and led nowhere. It was said that one might wander days and +nights together through its intricate tangle of rifts and chasms, and +never find the end of the cave; and that he might go down, and down, +and still down, into the earth, and it was just the same--labyrinth +under labyrinth, and no end to any of them. No man "knew" the cave. +That was an impossible thing. Most of the young men knew a portion of +it, and it was not customary to venture much beyond this known portion. +Tom Sawyer knew as much of the cave as any one. + +The procession moved along the main avenue some three-quarters of a +mile, and then groups and couples began to slip aside into branch +avenues, fly along the dismal corridors, and take each other by +surprise at points where the corridors joined again. Parties were able +to elude each other for the space of half an hour without going beyond +the "known" ground. + +By-and-by, one group after another came straggling back to the mouth +of the cave, panting, hilarious, smeared from head to foot with tallow +drippings, daubed with clay, and entirely delighted with the success of +the day. Then they were astonished to find that they had been taking no +note of time and that night was about at hand. The clanging bell had +been calling for half an hour. However, this sort of close to the day's +adventures was romantic and therefore satisfactory. When the ferryboat +with her wild freight pushed into the stream, nobody cared sixpence for +the wasted time but the captain of the craft. + +Huck was already upon his watch when the ferryboat's lights went +glinting past the wharf. He heard no noise on board, for the young +people were as subdued and still as people usually are who are nearly +tired to death. He wondered what boat it was, and why she did not stop +at the wharf--and then he dropped her out of his mind and put his +attention upon his business. The night was growing cloudy and dark. Ten +o'clock came, and the noise of vehicles ceased, scattered lights began +to wink out, all straggling foot-passengers disappeared, the village +betook itself to its slumbers and left the small watcher alone with the +silence and the ghosts. Eleven o'clock came, and the tavern lights were +put out; darkness everywhere, now. Huck waited what seemed a weary long +time, but nothing happened. His faith was weakening. Was there any use? +Was there really any use? Why not give it up and turn in? + +A noise fell upon his ear. He was all attention in an instant. The +alley door closed softly. He sprang to the corner of the brick store. +The next moment two men brushed by him, and one seemed to have +something under his arm. It must be that box! So they were going to +remove the treasure. Why call Tom now? It would be absurd--the men +would get away with the box and never be found again. No, he would +stick to their wake and follow them; he would trust to the darkness for +security from discovery. So communing with himself, Huck stepped out +and glided along behind the men, cat-like, with bare feet, allowing +them to keep just far enough ahead not to be invisible. + +They moved up the river street three blocks, then turned to the left +up a cross-street. They went straight ahead, then, until they came to +the path that led up Cardiff Hill; this they took. They passed by the +old Welshman's house, half-way up the hill, without hesitating, and +still climbed upward. Good, thought Huck, they will bury it in the old +quarry. But they never stopped at the quarry. They passed on, up the +summit. They plunged into the narrow path between the tall sumach +bushes, and were at once hidden in the gloom. Huck closed up and +shortened his distance, now, for they would never be able to see him. +He trotted along awhile; then slackened his pace, fearing he was +gaining too fast; moved on a piece, then stopped altogether; listened; +no sound; none, save that he seemed to hear the beating of his own +heart. The hooting of an owl came over the hill--ominous sound! But no +footsteps. Heavens, was everything lost! He was about to spring with +winged feet, when a man cleared his throat not four feet from him! +Huck's heart shot into his throat, but he swallowed it again; and then +he stood there shaking as if a dozen agues had taken charge of him at +once, and so weak that he thought he must surely fall to the ground. He +knew where he was. He knew he was within five steps of the stile +leading into Widow Douglas' grounds. Very well, he thought, let them +bury it there; it won't be hard to find. + +Now there was a voice--a very low voice--Injun Joe's: + +"Damn her, maybe she's got company--there's lights, late as it is." + +"I can't see any." + +This was that stranger's voice--the stranger of the haunted house. A +deadly chill went to Huck's heart--this, then, was the "revenge" job! +His thought was, to fly. Then he remembered that the Widow Douglas had +been kind to him more than once, and maybe these men were going to +murder her. He wished he dared venture to warn her; but he knew he +didn't dare--they might come and catch him. He thought all this and +more in the moment that elapsed between the stranger's remark and Injun +Joe's next--which was-- + +"Because the bush is in your way. Now--this way--now you see, don't +you?" + +"Yes. Well, there IS company there, I reckon. Better give it up." + +"Give it up, and I just leaving this country forever! Give it up and +maybe never have another chance. I tell you again, as I've told you +before, I don't care for her swag--you may have it. But her husband was +rough on me--many times he was rough on me--and mainly he was the +justice of the peace that jugged me for a vagrant. And that ain't all. +It ain't a millionth part of it! He had me HORSEWHIPPED!--horsewhipped +in front of the jail, like a nigger!--with all the town looking on! +HORSEWHIPPED!--do you understand? He took advantage of me and died. But +I'll take it out of HER." + +"Oh, don't kill her! Don't do that!" + +"Kill? Who said anything about killing? I would kill HIM if he was +here; but not her. When you want to get revenge on a woman you don't +kill her--bosh! you go for her looks. You slit her nostrils--you notch +her ears like a sow!" + +"By God, that's--" + +"Keep your opinion to yourself! It will be safest for you. I'll tie +her to the bed. If she bleeds to death, is that my fault? I'll not cry, +if she does. My friend, you'll help me in this thing--for MY sake +--that's why you're here--I mightn't be able alone. If you flinch, I'll +kill you. Do you understand that? And if I have to kill you, I'll kill +her--and then I reckon nobody'll ever know much about who done this +business." + +"Well, if it's got to be done, let's get at it. The quicker the +better--I'm all in a shiver." + +"Do it NOW? And company there? Look here--I'll get suspicious of you, +first thing you know. No--we'll wait till the lights are out--there's +no hurry." + +Huck felt that a silence was going to ensue--a thing still more awful +than any amount of murderous talk; so he held his breath and stepped +gingerly back; planted his foot carefully and firmly, after balancing, +one-legged, in a precarious way and almost toppling over, first on one +side and then on the other. He took another step back, with the same +elaboration and the same risks; then another and another, and--a twig +snapped under his foot! His breath stopped and he listened. There was +no sound--the stillness was perfect. His gratitude was measureless. Now +he turned in his tracks, between the walls of sumach bushes--turned +himself as carefully as if he were a ship--and then stepped quickly but +cautiously along. When he emerged at the quarry he felt secure, and so +he picked up his nimble heels and flew. Down, down he sped, till he +reached the Welshman's. He banged at the door, and presently the heads +of the old man and his two stalwart sons were thrust from windows. + +"What's the row there? Who's banging? What do you want?" + +"Let me in--quick! I'll tell everything." + +"Why, who are you?" + +"Huckleberry Finn--quick, let me in!" + +"Huckleberry Finn, indeed! It ain't a name to open many doors, I +judge! But let him in, lads, and let's see what's the trouble." + +"Please don't ever tell I told you," were Huck's first words when he +got in. "Please don't--I'd be killed, sure--but the widow's been good +friends to me sometimes, and I want to tell--I WILL tell if you'll +promise you won't ever say it was me." + +"By George, he HAS got something to tell, or he wouldn't act so!" +exclaimed the old man; "out with it and nobody here'll ever tell, lad." + +Three minutes later the old man and his sons, well armed, were up the +hill, and just entering the sumach path on tiptoe, their weapons in +their hands. Huck accompanied them no further. He hid behind a great +bowlder and fell to listening. There was a lagging, anxious silence, +and then all of a sudden there was an explosion of firearms and a cry. + +Huck waited for no particulars. He sprang away and sped down the hill +as fast as his legs could carry him. + + + +CHAPTER XXX + +AS the earliest suspicion of dawn appeared on Sunday morning, Huck +came groping up the hill and rapped gently at the old Welshman's door. +The inmates were asleep, but it was a sleep that was set on a +hair-trigger, on account of the exciting episode of the night. A call +came from a window: + +"Who's there!" + +Huck's scared voice answered in a low tone: + +"Please let me in! It's only Huck Finn!" + +"It's a name that can open this door night or day, lad!--and welcome!" + +These were strange words to the vagabond boy's ears, and the +pleasantest he had ever heard. He could not recollect that the closing +word had ever been applied in his case before. The door was quickly +unlocked, and he entered. Huck was given a seat and the old man and his +brace of tall sons speedily dressed themselves. + +"Now, my boy, I hope you're good and hungry, because breakfast will be +ready as soon as the sun's up, and we'll have a piping hot one, too +--make yourself easy about that! I and the boys hoped you'd turn up and +stop here last night." + +"I was awful scared," said Huck, "and I run. I took out when the +pistols went off, and I didn't stop for three mile. I've come now becuz +I wanted to know about it, you know; and I come before daylight becuz I +didn't want to run across them devils, even if they was dead." + +"Well, poor chap, you do look as if you'd had a hard night of it--but +there's a bed here for you when you've had your breakfast. No, they +ain't dead, lad--we are sorry enough for that. You see we knew right +where to put our hands on them, by your description; so we crept along +on tiptoe till we got within fifteen feet of them--dark as a cellar +that sumach path was--and just then I found I was going to sneeze. It +was the meanest kind of luck! I tried to keep it back, but no use +--'twas bound to come, and it did come! I was in the lead with my pistol +raised, and when the sneeze started those scoundrels a-rustling to get +out of the path, I sung out, 'Fire boys!' and blazed away at the place +where the rustling was. So did the boys. But they were off in a jiffy, +those villains, and we after them, down through the woods. I judge we +never touched them. They fired a shot apiece as they started, but their +bullets whizzed by and didn't do us any harm. As soon as we lost the +sound of their feet we quit chasing, and went down and stirred up the +constables. They got a posse together, and went off to guard the river +bank, and as soon as it is light the sheriff and a gang are going to +beat up the woods. My boys will be with them presently. I wish we had +some sort of description of those rascals--'twould help a good deal. +But you couldn't see what they were like, in the dark, lad, I suppose?" + +"Oh yes; I saw them down-town and follered them." + +"Splendid! Describe them--describe them, my boy!" + +"One's the old deaf and dumb Spaniard that's ben around here once or +twice, and t'other's a mean-looking, ragged--" + +"That's enough, lad, we know the men! Happened on them in the woods +back of the widow's one day, and they slunk away. Off with you, boys, +and tell the sheriff--get your breakfast to-morrow morning!" + +The Welshman's sons departed at once. As they were leaving the room +Huck sprang up and exclaimed: + +"Oh, please don't tell ANYbody it was me that blowed on them! Oh, +please!" + +"All right if you say it, Huck, but you ought to have the credit of +what you did." + +"Oh no, no! Please don't tell!" + +When the young men were gone, the old Welshman said: + +"They won't tell--and I won't. But why don't you want it known?" + +Huck would not explain, further than to say that he already knew too +much about one of those men and would not have the man know that he +knew anything against him for the whole world--he would be killed for +knowing it, sure. + +The old man promised secrecy once more, and said: + +"How did you come to follow these fellows, lad? Were they looking +suspicious?" + +Huck was silent while he framed a duly cautious reply. Then he said: + +"Well, you see, I'm a kind of a hard lot,--least everybody says so, +and I don't see nothing agin it--and sometimes I can't sleep much, on +account of thinking about it and sort of trying to strike out a new way +of doing. That was the way of it last night. I couldn't sleep, and so I +come along up-street 'bout midnight, a-turning it all over, and when I +got to that old shackly brick store by the Temperance Tavern, I backed +up agin the wall to have another think. Well, just then along comes +these two chaps slipping along close by me, with something under their +arm, and I reckoned they'd stole it. One was a-smoking, and t'other one +wanted a light; so they stopped right before me and the cigars lit up +their faces and I see that the big one was the deaf and dumb Spaniard, +by his white whiskers and the patch on his eye, and t'other one was a +rusty, ragged-looking devil." + +"Could you see the rags by the light of the cigars?" + +This staggered Huck for a moment. Then he said: + +"Well, I don't know--but somehow it seems as if I did." + +"Then they went on, and you--" + +"Follered 'em--yes. That was it. I wanted to see what was up--they +sneaked along so. I dogged 'em to the widder's stile, and stood in the +dark and heard the ragged one beg for the widder, and the Spaniard +swear he'd spile her looks just as I told you and your two--" + +"What! The DEAF AND DUMB man said all that!" + +Huck had made another terrible mistake! He was trying his best to keep +the old man from getting the faintest hint of who the Spaniard might +be, and yet his tongue seemed determined to get him into trouble in +spite of all he could do. He made several efforts to creep out of his +scrape, but the old man's eye was upon him and he made blunder after +blunder. Presently the Welshman said: + +"My boy, don't be afraid of me. I wouldn't hurt a hair of your head +for all the world. No--I'd protect you--I'd protect you. This Spaniard +is not deaf and dumb; you've let that slip without intending it; you +can't cover that up now. You know something about that Spaniard that +you want to keep dark. Now trust me--tell me what it is, and trust me +--I won't betray you." + +Huck looked into the old man's honest eyes a moment, then bent over +and whispered in his ear: + +"'Tain't a Spaniard--it's Injun Joe!" + +The Welshman almost jumped out of his chair. In a moment he said: + +"It's all plain enough, now. When you talked about notching ears and +slitting noses I judged that that was your own embellishment, because +white men don't take that sort of revenge. But an Injun! That's a +different matter altogether." + +During breakfast the talk went on, and in the course of it the old man +said that the last thing which he and his sons had done, before going +to bed, was to get a lantern and examine the stile and its vicinity for +marks of blood. They found none, but captured a bulky bundle of-- + +"Of WHAT?" + +If the words had been lightning they could not have leaped with a more +stunning suddenness from Huck's blanched lips. His eyes were staring +wide, now, and his breath suspended--waiting for the answer. The +Welshman started--stared in return--three seconds--five seconds--ten +--then replied: + +"Of burglar's tools. Why, what's the MATTER with you?" + +Huck sank back, panting gently, but deeply, unutterably grateful. The +Welshman eyed him gravely, curiously--and presently said: + +"Yes, burglar's tools. That appears to relieve you a good deal. But +what did give you that turn? What were YOU expecting we'd found?" + +Huck was in a close place--the inquiring eye was upon him--he would +have given anything for material for a plausible answer--nothing +suggested itself--the inquiring eye was boring deeper and deeper--a +senseless reply offered--there was no time to weigh it, so at a venture +he uttered it--feebly: + +"Sunday-school books, maybe." + +Poor Huck was too distressed to smile, but the old man laughed loud +and joyously, shook up the details of his anatomy from head to foot, +and ended by saying that such a laugh was money in a-man's pocket, +because it cut down the doctor's bill like everything. Then he added: + +"Poor old chap, you're white and jaded--you ain't well a bit--no +wonder you're a little flighty and off your balance. But you'll come +out of it. Rest and sleep will fetch you out all right, I hope." + +Huck was irritated to think he had been such a goose and betrayed such +a suspicious excitement, for he had dropped the idea that the parcel +brought from the tavern was the treasure, as soon as he had heard the +talk at the widow's stile. He had only thought it was not the treasure, +however--he had not known that it wasn't--and so the suggestion of a +captured bundle was too much for his self-possession. But on the whole +he felt glad the little episode had happened, for now he knew beyond +all question that that bundle was not THE bundle, and so his mind was +at rest and exceedingly comfortable. In fact, everything seemed to be +drifting just in the right direction, now; the treasure must be still +in No. 2, the men would be captured and jailed that day, and he and Tom +could seize the gold that night without any trouble or any fear of +interruption. + +Just as breakfast was completed there was a knock at the door. Huck +jumped for a hiding-place, for he had no mind to be connected even +remotely with the late event. The Welshman admitted several ladies and +gentlemen, among them the Widow Douglas, and noticed that groups of +citizens were climbing up the hill--to stare at the stile. So the news +had spread. The Welshman had to tell the story of the night to the +visitors. The widow's gratitude for her preservation was outspoken. + +"Don't say a word about it, madam. There's another that you're more +beholden to than you are to me and my boys, maybe, but he don't allow +me to tell his name. We wouldn't have been there but for him." + +Of course this excited a curiosity so vast that it almost belittled +the main matter--but the Welshman allowed it to eat into the vitals of +his visitors, and through them be transmitted to the whole town, for he +refused to part with his secret. When all else had been learned, the +widow said: + +"I went to sleep reading in bed and slept straight through all that +noise. Why didn't you come and wake me?" + +"We judged it warn't worth while. Those fellows warn't likely to come +again--they hadn't any tools left to work with, and what was the use of +waking you up and scaring you to death? My three negro men stood guard +at your house all the rest of the night. They've just come back." + +More visitors came, and the story had to be told and retold for a +couple of hours more. + +There was no Sabbath-school during day-school vacation, but everybody +was early at church. The stirring event was well canvassed. News came +that not a sign of the two villains had been yet discovered. When the +sermon was finished, Judge Thatcher's wife dropped alongside of Mrs. +Harper as she moved down the aisle with the crowd and said: + +"Is my Becky going to sleep all day? I just expected she would be +tired to death." + +"Your Becky?" + +"Yes," with a startled look--"didn't she stay with you last night?" + +"Why, no." + +Mrs. Thatcher turned pale, and sank into a pew, just as Aunt Polly, +talking briskly with a friend, passed by. Aunt Polly said: + +"Good-morning, Mrs. Thatcher. Good-morning, Mrs. Harper. I've got a +boy that's turned up missing. I reckon my Tom stayed at your house last +night--one of you. And now he's afraid to come to church. I've got to +settle with him." + +Mrs. Thatcher shook her head feebly and turned paler than ever. + +"He didn't stay with us," said Mrs. Harper, beginning to look uneasy. +A marked anxiety came into Aunt Polly's face. + +"Joe Harper, have you seen my Tom this morning?" + +"No'm." + +"When did you see him last?" + +Joe tried to remember, but was not sure he could say. The people had +stopped moving out of church. Whispers passed along, and a boding +uneasiness took possession of every countenance. Children were +anxiously questioned, and young teachers. They all said they had not +noticed whether Tom and Becky were on board the ferryboat on the +homeward trip; it was dark; no one thought of inquiring if any one was +missing. One young man finally blurted out his fear that they were +still in the cave! Mrs. Thatcher swooned away. Aunt Polly fell to +crying and wringing her hands. + +The alarm swept from lip to lip, from group to group, from street to +street, and within five minutes the bells were wildly clanging and the +whole town was up! The Cardiff Hill episode sank into instant +insignificance, the burglars were forgotten, horses were saddled, +skiffs were manned, the ferryboat ordered out, and before the horror +was half an hour old, two hundred men were pouring down highroad and +river toward the cave. + +All the long afternoon the village seemed empty and dead. Many women +visited Aunt Polly and Mrs. Thatcher and tried to comfort them. They +cried with them, too, and that was still better than words. All the +tedious night the town waited for news; but when the morning dawned at +last, all the word that came was, "Send more candles--and send food." +Mrs. Thatcher was almost crazed; and Aunt Polly, also. Judge Thatcher +sent messages of hope and encouragement from the cave, but they +conveyed no real cheer. + +The old Welshman came home toward daylight, spattered with +candle-grease, smeared with clay, and almost worn out. He found Huck +still in the bed that had been provided for him, and delirious with +fever. The physicians were all at the cave, so the Widow Douglas came +and took charge of the patient. She said she would do her best by him, +because, whether he was good, bad, or indifferent, he was the Lord's, +and nothing that was the Lord's was a thing to be neglected. The +Welshman said Huck had good spots in him, and the widow said: + +"You can depend on it. That's the Lord's mark. He don't leave it off. +He never does. Puts it somewhere on every creature that comes from his +hands." + +Early in the forenoon parties of jaded men began to straggle into the +village, but the strongest of the citizens continued searching. All the +news that could be gained was that remotenesses of the cavern were +being ransacked that had never been visited before; that every corner +and crevice was going to be thoroughly searched; that wherever one +wandered through the maze of passages, lights were to be seen flitting +hither and thither in the distance, and shoutings and pistol-shots sent +their hollow reverberations to the ear down the sombre aisles. In one +place, far from the section usually traversed by tourists, the names +"BECKY & TOM" had been found traced upon the rocky wall with +candle-smoke, and near at hand a grease-soiled bit of ribbon. Mrs. +Thatcher recognized the ribbon and cried over it. She said it was the +last relic she should ever have of her child; and that no other memorial +of her could ever be so precious, because this one parted latest from +the living body before the awful death came. Some said that now and +then, in the cave, a far-away speck of light would glimmer, and then a +glorious shout would burst forth and a score of men go trooping down the +echoing aisle--and then a sickening disappointment always followed; the +children were not there; it was only a searcher's light. + +Three dreadful days and nights dragged their tedious hours along, and +the village sank into a hopeless stupor. No one had heart for anything. +The accidental discovery, just made, that the proprietor of the +Temperance Tavern kept liquor on his premises, scarcely fluttered the +public pulse, tremendous as the fact was. In a lucid interval, Huck +feebly led up to the subject of taverns, and finally asked--dimly +dreading the worst--if anything had been discovered at the Temperance +Tavern since he had been ill. + +"Yes," said the widow. + +Huck started up in bed, wild-eyed: + +"What? What was it?" + +"Liquor!--and the place has been shut up. Lie down, child--what a turn +you did give me!" + +"Only tell me just one thing--only just one--please! Was it Tom Sawyer +that found it?" + +The widow burst into tears. "Hush, hush, child, hush! I've told you +before, you must NOT talk. You are very, very sick!" + +Then nothing but liquor had been found; there would have been a great +powwow if it had been the gold. So the treasure was gone forever--gone +forever! But what could she be crying about? Curious that she should +cry. + +These thoughts worked their dim way through Huck's mind, and under the +weariness they gave him he fell asleep. The widow said to herself: + +"There--he's asleep, poor wreck. Tom Sawyer find it! Pity but somebody +could find Tom Sawyer! Ah, there ain't many left, now, that's got hope +enough, or strength enough, either, to go on searching." + + + +CHAPTER XXXI + +NOW to return to Tom and Becky's share in the picnic. They tripped +along the murky aisles with the rest of the company, visiting the +familiar wonders of the cave--wonders dubbed with rather +over-descriptive names, such as "The Drawing-Room," "The Cathedral," +"Aladdin's Palace," and so on. Presently the hide-and-seek frolicking +began, and Tom and Becky engaged in it with zeal until the exertion +began to grow a trifle wearisome; then they wandered down a sinuous +avenue holding their candles aloft and reading the tangled web-work of +names, dates, post-office addresses, and mottoes with which the rocky +walls had been frescoed (in candle-smoke). Still drifting along and +talking, they scarcely noticed that they were now in a part of the cave +whose walls were not frescoed. They smoked their own names under an +overhanging shelf and moved on. Presently they came to a place where a +little stream of water, trickling over a ledge and carrying a limestone +sediment with it, had, in the slow-dragging ages, formed a laced and +ruffled Niagara in gleaming and imperishable stone. Tom squeezed his +small body behind it in order to illuminate it for Becky's +gratification. He found that it curtained a sort of steep natural +stairway which was enclosed between narrow walls, and at once the +ambition to be a discoverer seized him. Becky responded to his call, +and they made a smoke-mark for future guidance, and started upon their +quest. They wound this way and that, far down into the secret depths of +the cave, made another mark, and branched off in search of novelties to +tell the upper world about. In one place they found a spacious cavern, +from whose ceiling depended a multitude of shining stalactites of the +length and circumference of a man's leg; they walked all about it, +wondering and admiring, and presently left it by one of the numerous +passages that opened into it. This shortly brought them to a bewitching +spring, whose basin was incrusted with a frostwork of glittering +crystals; it was in the midst of a cavern whose walls were supported by +many fantastic pillars which had been formed by the joining of great +stalactites and stalagmites together, the result of the ceaseless +water-drip of centuries. Under the roof vast knots of bats had packed +themselves together, thousands in a bunch; the lights disturbed the +creatures and they came flocking down by hundreds, squeaking and +darting furiously at the candles. Tom knew their ways and the danger of +this sort of conduct. He seized Becky's hand and hurried her into the +first corridor that offered; and none too soon, for a bat struck +Becky's light out with its wing while she was passing out of the +cavern. The bats chased the children a good distance; but the fugitives +plunged into every new passage that offered, and at last got rid of the +perilous things. Tom found a subterranean lake, shortly, which +stretched its dim length away until its shape was lost in the shadows. +He wanted to explore its borders, but concluded that it would be best +to sit down and rest awhile, first. Now, for the first time, the deep +stillness of the place laid a clammy hand upon the spirits of the +children. Becky said: + +"Why, I didn't notice, but it seems ever so long since I heard any of +the others." + +"Come to think, Becky, we are away down below them--and I don't know +how far away north, or south, or east, or whichever it is. We couldn't +hear them here." + +Becky grew apprehensive. + +"I wonder how long we've been down here, Tom? We better start back." + +"Yes, I reckon we better. P'raps we better." + +"Can you find the way, Tom? It's all a mixed-up crookedness to me." + +"I reckon I could find it--but then the bats. If they put our candles +out it will be an awful fix. Let's try some other way, so as not to go +through there." + +"Well. But I hope we won't get lost. It would be so awful!" and the +girl shuddered at the thought of the dreadful possibilities. + +They started through a corridor, and traversed it in silence a long +way, glancing at each new opening, to see if there was anything +familiar about the look of it; but they were all strange. Every time +Tom made an examination, Becky would watch his face for an encouraging +sign, and he would say cheerily: + +"Oh, it's all right. This ain't the one, but we'll come to it right +away!" + +But he felt less and less hopeful with each failure, and presently +began to turn off into diverging avenues at sheer random, in desperate +hope of finding the one that was wanted. He still said it was "all +right," but there was such a leaden dread at his heart that the words +had lost their ring and sounded just as if he had said, "All is lost!" +Becky clung to his side in an anguish of fear, and tried hard to keep +back the tears, but they would come. At last she said: + +"Oh, Tom, never mind the bats, let's go back that way! We seem to get +worse and worse off all the time." + +"Listen!" said he. + +Profound silence; silence so deep that even their breathings were +conspicuous in the hush. Tom shouted. The call went echoing down the +empty aisles and died out in the distance in a faint sound that +resembled a ripple of mocking laughter. + +"Oh, don't do it again, Tom, it is too horrid," said Becky. + +"It is horrid, but I better, Becky; they might hear us, you know," and +he shouted again. + +The "might" was even a chillier horror than the ghostly laughter, it +so confessed a perishing hope. The children stood still and listened; +but there was no result. Tom turned upon the back track at once, and +hurried his steps. It was but a little while before a certain +indecision in his manner revealed another fearful fact to Becky--he +could not find his way back! + +"Oh, Tom, you didn't make any marks!" + +"Becky, I was such a fool! Such a fool! I never thought we might want +to come back! No--I can't find the way. It's all mixed up." + +"Tom, Tom, we're lost! we're lost! We never can get out of this awful +place! Oh, why DID we ever leave the others!" + +She sank to the ground and burst into such a frenzy of crying that Tom +was appalled with the idea that she might die, or lose her reason. He +sat down by her and put his arms around her; she buried her face in his +bosom, she clung to him, she poured out her terrors, her unavailing +regrets, and the far echoes turned them all to jeering laughter. Tom +begged her to pluck up hope again, and she said she could not. He fell +to blaming and abusing himself for getting her into this miserable +situation; this had a better effect. She said she would try to hope +again, she would get up and follow wherever he might lead if only he +would not talk like that any more. For he was no more to blame than +she, she said. + +So they moved on again--aimlessly--simply at random--all they could do +was to move, keep moving. For a little while, hope made a show of +reviving--not with any reason to back it, but only because it is its +nature to revive when the spring has not been taken out of it by age +and familiarity with failure. + +By-and-by Tom took Becky's candle and blew it out. This economy meant +so much! Words were not needed. Becky understood, and her hope died +again. She knew that Tom had a whole candle and three or four pieces in +his pockets--yet he must economize. + +By-and-by, fatigue began to assert its claims; the children tried to +pay attention, for it was dreadful to think of sitting down when time +was grown to be so precious, moving, in some direction, in any +direction, was at least progress and might bear fruit; but to sit down +was to invite death and shorten its pursuit. + +At last Becky's frail limbs refused to carry her farther. She sat +down. Tom rested with her, and they talked of home, and the friends +there, and the comfortable beds and, above all, the light! Becky cried, +and Tom tried to think of some way of comforting her, but all his +encouragements were grown threadbare with use, and sounded like +sarcasms. Fatigue bore so heavily upon Becky that she drowsed off to +sleep. Tom was grateful. He sat looking into her drawn face and saw it +grow smooth and natural under the influence of pleasant dreams; and +by-and-by a smile dawned and rested there. The peaceful face reflected +somewhat of peace and healing into his own spirit, and his thoughts +wandered away to bygone times and dreamy memories. While he was deep in +his musings, Becky woke up with a breezy little laugh--but it was +stricken dead upon her lips, and a groan followed it. + +"Oh, how COULD I sleep! I wish I never, never had waked! No! No, I +don't, Tom! Don't look so! I won't say it again." + +"I'm glad you've slept, Becky; you'll feel rested, now, and we'll find +the way out." + +"We can try, Tom; but I've seen such a beautiful country in my dream. +I reckon we are going there." + +"Maybe not, maybe not. Cheer up, Becky, and let's go on trying." + +They rose up and wandered along, hand in hand and hopeless. They tried +to estimate how long they had been in the cave, but all they knew was +that it seemed days and weeks, and yet it was plain that this could not +be, for their candles were not gone yet. A long time after this--they +could not tell how long--Tom said they must go softly and listen for +dripping water--they must find a spring. They found one presently, and +Tom said it was time to rest again. Both were cruelly tired, yet Becky +said she thought she could go a little farther. She was surprised to +hear Tom dissent. She could not understand it. They sat down, and Tom +fastened his candle to the wall in front of them with some clay. +Thought was soon busy; nothing was said for some time. Then Becky broke +the silence: + +"Tom, I am so hungry!" + +Tom took something out of his pocket. + +"Do you remember this?" said he. + +Becky almost smiled. + +"It's our wedding-cake, Tom." + +"Yes--I wish it was as big as a barrel, for it's all we've got." + +"I saved it from the picnic for us to dream on, Tom, the way grown-up +people do with wedding-cake--but it'll be our--" + +She dropped the sentence where it was. Tom divided the cake and Becky +ate with good appetite, while Tom nibbled at his moiety. There was +abundance of cold water to finish the feast with. By-and-by Becky +suggested that they move on again. Tom was silent a moment. Then he +said: + +"Becky, can you bear it if I tell you something?" + +Becky's face paled, but she thought she could. + +"Well, then, Becky, we must stay here, where there's water to drink. +That little piece is our last candle!" + +Becky gave loose to tears and wailings. Tom did what he could to +comfort her, but with little effect. At length Becky said: + +"Tom!" + +"Well, Becky?" + +"They'll miss us and hunt for us!" + +"Yes, they will! Certainly they will!" + +"Maybe they're hunting for us now, Tom." + +"Why, I reckon maybe they are. I hope they are." + +"When would they miss us, Tom?" + +"When they get back to the boat, I reckon." + +"Tom, it might be dark then--would they notice we hadn't come?" + +"I don't know. But anyway, your mother would miss you as soon as they +got home." + +A frightened look in Becky's face brought Tom to his senses and he saw +that he had made a blunder. Becky was not to have gone home that night! +The children became silent and thoughtful. In a moment a new burst of +grief from Becky showed Tom that the thing in his mind had struck hers +also--that the Sabbath morning might be half spent before Mrs. Thatcher +discovered that Becky was not at Mrs. Harper's. + +The children fastened their eyes upon their bit of candle and watched +it melt slowly and pitilessly away; saw the half inch of wick stand +alone at last; saw the feeble flame rise and fall, climb the thin +column of smoke, linger at its top a moment, and then--the horror of +utter darkness reigned! + +How long afterward it was that Becky came to a slow consciousness that +she was crying in Tom's arms, neither could tell. All that they knew +was, that after what seemed a mighty stretch of time, both awoke out of +a dead stupor of sleep and resumed their miseries once more. Tom said +it might be Sunday, now--maybe Monday. He tried to get Becky to talk, +but her sorrows were too oppressive, all her hopes were gone. Tom said +that they must have been missed long ago, and no doubt the search was +going on. He would shout and maybe some one would come. He tried it; +but in the darkness the distant echoes sounded so hideously that he +tried it no more. + +The hours wasted away, and hunger came to torment the captives again. +A portion of Tom's half of the cake was left; they divided and ate it. +But they seemed hungrier than before. The poor morsel of food only +whetted desire. + +By-and-by Tom said: + +"SH! Did you hear that?" + +Both held their breath and listened. There was a sound like the +faintest, far-off shout. Instantly Tom answered it, and leading Becky +by the hand, started groping down the corridor in its direction. +Presently he listened again; again the sound was heard, and apparently +a little nearer. + +"It's them!" said Tom; "they're coming! Come along, Becky--we're all +right now!" + +The joy of the prisoners was almost overwhelming. Their speed was +slow, however, because pitfalls were somewhat common, and had to be +guarded against. They shortly came to one and had to stop. It might be +three feet deep, it might be a hundred--there was no passing it at any +rate. Tom got down on his breast and reached as far down as he could. +No bottom. They must stay there and wait until the searchers came. They +listened; evidently the distant shoutings were growing more distant! a +moment or two more and they had gone altogether. The heart-sinking +misery of it! Tom whooped until he was hoarse, but it was of no use. He +talked hopefully to Becky; but an age of anxious waiting passed and no +sounds came again. + +The children groped their way back to the spring. The weary time +dragged on; they slept again, and awoke famished and woe-stricken. Tom +believed it must be Tuesday by this time. + +Now an idea struck him. There were some side passages near at hand. It +would be better to explore some of these than bear the weight of the +heavy time in idleness. He took a kite-line from his pocket, tied it to +a projection, and he and Becky started, Tom in the lead, unwinding the +line as he groped along. At the end of twenty steps the corridor ended +in a "jumping-off place." Tom got down on his knees and felt below, and +then as far around the corner as he could reach with his hands +conveniently; he made an effort to stretch yet a little farther to the +right, and at that moment, not twenty yards away, a human hand, holding +a candle, appeared from behind a rock! Tom lifted up a glorious shout, +and instantly that hand was followed by the body it belonged to--Injun +Joe's! Tom was paralyzed; he could not move. He was vastly gratified +the next moment, to see the "Spaniard" take to his heels and get +himself out of sight. Tom wondered that Joe had not recognized his +voice and come over and killed him for testifying in court. But the +echoes must have disguised the voice. Without doubt, that was it, he +reasoned. Tom's fright weakened every muscle in his body. He said to +himself that if he had strength enough to get back to the spring he +would stay there, and nothing should tempt him to run the risk of +meeting Injun Joe again. He was careful to keep from Becky what it was +he had seen. He told her he had only shouted "for luck." + +But hunger and wretchedness rise superior to fears in the long run. +Another tedious wait at the spring and another long sleep brought +changes. The children awoke tortured with a raging hunger. Tom believed +that it must be Wednesday or Thursday or even Friday or Saturday, now, +and that the search had been given over. He proposed to explore another +passage. He felt willing to risk Injun Joe and all other terrors. But +Becky was very weak. She had sunk into a dreary apathy and would not be +roused. She said she would wait, now, where she was, and die--it would +not be long. She told Tom to go with the kite-line and explore if he +chose; but she implored him to come back every little while and speak +to her; and she made him promise that when the awful time came, he +would stay by her and hold her hand until all was over. + +Tom kissed her, with a choking sensation in his throat, and made a +show of being confident of finding the searchers or an escape from the +cave; then he took the kite-line in his hand and went groping down one +of the passages on his hands and knees, distressed with hunger and sick +with bodings of coming doom. + + + +CHAPTER XXXII + +TUESDAY afternoon came, and waned to the twilight. The village of St. +Petersburg still mourned. The lost children had not been found. Public +prayers had been offered up for them, and many and many a private +prayer that had the petitioner's whole heart in it; but still no good +news came from the cave. The majority of the searchers had given up the +quest and gone back to their daily avocations, saying that it was plain +the children could never be found. Mrs. Thatcher was very ill, and a +great part of the time delirious. People said it was heartbreaking to +hear her call her child, and raise her head and listen a whole minute +at a time, then lay it wearily down again with a moan. Aunt Polly had +drooped into a settled melancholy, and her gray hair had grown almost +white. The village went to its rest on Tuesday night, sad and forlorn. + +Away in the middle of the night a wild peal burst from the village +bells, and in a moment the streets were swarming with frantic half-clad +people, who shouted, "Turn out! turn out! they're found! they're +found!" Tin pans and horns were added to the din, the population massed +itself and moved toward the river, met the children coming in an open +carriage drawn by shouting citizens, thronged around it, joined its +homeward march, and swept magnificently up the main street roaring +huzzah after huzzah! + +The village was illuminated; nobody went to bed again; it was the +greatest night the little town had ever seen. During the first half-hour +a procession of villagers filed through Judge Thatcher's house, seized +the saved ones and kissed them, squeezed Mrs. Thatcher's hand, tried to +speak but couldn't--and drifted out raining tears all over the place. + +Aunt Polly's happiness was complete, and Mrs. Thatcher's nearly so. It +would be complete, however, as soon as the messenger dispatched with +the great news to the cave should get the word to her husband. Tom lay +upon a sofa with an eager auditory about him and told the history of +the wonderful adventure, putting in many striking additions to adorn it +withal; and closed with a description of how he left Becky and went on +an exploring expedition; how he followed two avenues as far as his +kite-line would reach; how he followed a third to the fullest stretch of +the kite-line, and was about to turn back when he glimpsed a far-off +speck that looked like daylight; dropped the line and groped toward it, +pushed his head and shoulders through a small hole, and saw the broad +Mississippi rolling by! And if it had only happened to be night he would +not have seen that speck of daylight and would not have explored that +passage any more! He told how he went back for Becky and broke the good +news and she told him not to fret her with such stuff, for she was +tired, and knew she was going to die, and wanted to. He described how he +labored with her and convinced her; and how she almost died for joy when +she had groped to where she actually saw the blue speck of daylight; how +he pushed his way out at the hole and then helped her out; how they sat +there and cried for gladness; how some men came along in a skiff and Tom +hailed them and told them their situation and their famished condition; +how the men didn't believe the wild tale at first, "because," said they, +"you are five miles down the river below the valley the cave is in" +--then took them aboard, rowed to a house, gave them supper, made them +rest till two or three hours after dark and then brought them home. + +Before day-dawn, Judge Thatcher and the handful of searchers with him +were tracked out, in the cave, by the twine clews they had strung +behind them, and informed of the great news. + +Three days and nights of toil and hunger in the cave were not to be +shaken off at once, as Tom and Becky soon discovered. They were +bedridden all of Wednesday and Thursday, and seemed to grow more and +more tired and worn, all the time. Tom got about, a little, on +Thursday, was down-town Friday, and nearly as whole as ever Saturday; +but Becky did not leave her room until Sunday, and then she looked as +if she had passed through a wasting illness. + +Tom learned of Huck's sickness and went to see him on Friday, but +could not be admitted to the bedroom; neither could he on Saturday or +Sunday. He was admitted daily after that, but was warned to keep still +about his adventure and introduce no exciting topic. The Widow Douglas +stayed by to see that he obeyed. At home Tom learned of the Cardiff +Hill event; also that the "ragged man's" body had eventually been found +in the river near the ferry-landing; he had been drowned while trying +to escape, perhaps. + +About a fortnight after Tom's rescue from the cave, he started off to +visit Huck, who had grown plenty strong enough, now, to hear exciting +talk, and Tom had some that would interest him, he thought. Judge +Thatcher's house was on Tom's way, and he stopped to see Becky. The +Judge and some friends set Tom to talking, and some one asked him +ironically if he wouldn't like to go to the cave again. Tom said he +thought he wouldn't mind it. The Judge said: + +"Well, there are others just like you, Tom, I've not the least doubt. +But we have taken care of that. Nobody will get lost in that cave any +more." + +"Why?" + +"Because I had its big door sheathed with boiler iron two weeks ago, +and triple-locked--and I've got the keys." + +Tom turned as white as a sheet. + +"What's the matter, boy! Here, run, somebody! Fetch a glass of water!" + +The water was brought and thrown into Tom's face. + +"Ah, now you're all right. What was the matter with you, Tom?" + +"Oh, Judge, Injun Joe's in the cave!" + + + +CHAPTER XXXIII + +WITHIN a few minutes the news had spread, and a dozen skiff-loads of +men were on their way to McDougal's cave, and the ferryboat, well +filled with passengers, soon followed. Tom Sawyer was in the skiff that +bore Judge Thatcher. + +When the cave door was unlocked, a sorrowful sight presented itself in +the dim twilight of the place. Injun Joe lay stretched upon the ground, +dead, with his face close to the crack of the door, as if his longing +eyes had been fixed, to the latest moment, upon the light and the cheer +of the free world outside. Tom was touched, for he knew by his own +experience how this wretch had suffered. His pity was moved, but +nevertheless he felt an abounding sense of relief and security, now, +which revealed to him in a degree which he had not fully appreciated +before how vast a weight of dread had been lying upon him since the day +he lifted his voice against this bloody-minded outcast. + +Injun Joe's bowie-knife lay close by, its blade broken in two. The +great foundation-beam of the door had been chipped and hacked through, +with tedious labor; useless labor, too, it was, for the native rock +formed a sill outside it, and upon that stubborn material the knife had +wrought no effect; the only damage done was to the knife itself. But if +there had been no stony obstruction there the labor would have been +useless still, for if the beam had been wholly cut away Injun Joe could +not have squeezed his body under the door, and he knew it. So he had +only hacked that place in order to be doing something--in order to pass +the weary time--in order to employ his tortured faculties. Ordinarily +one could find half a dozen bits of candle stuck around in the crevices +of this vestibule, left there by tourists; but there were none now. The +prisoner had searched them out and eaten them. He had also contrived to +catch a few bats, and these, also, he had eaten, leaving only their +claws. The poor unfortunate had starved to death. In one place, near at +hand, a stalagmite had been slowly growing up from the ground for ages, +builded by the water-drip from a stalactite overhead. The captive had +broken off the stalagmite, and upon the stump had placed a stone, +wherein he had scooped a shallow hollow to catch the precious drop +that fell once in every three minutes with the dreary regularity of a +clock-tick--a dessertspoonful once in four and twenty hours. That drop +was falling when the Pyramids were new; when Troy fell; when the +foundations of Rome were laid; when Christ was crucified; when the +Conqueror created the British empire; when Columbus sailed; when the +massacre at Lexington was "news." It is falling now; it will still be +falling when all these things shall have sunk down the afternoon of +history, and the twilight of tradition, and been swallowed up in the +thick night of oblivion. Has everything a purpose and a mission? Did +this drop fall patiently during five thousand years to be ready for +this flitting human insect's need? and has it another important object +to accomplish ten thousand years to come? No matter. It is many and +many a year since the hapless half-breed scooped out the stone to catch +the priceless drops, but to this day the tourist stares longest at that +pathetic stone and that slow-dropping water when he comes to see the +wonders of McDougal's cave. Injun Joe's cup stands first in the list of +the cavern's marvels; even "Aladdin's Palace" cannot rival it. + +Injun Joe was buried near the mouth of the cave; and people flocked +there in boats and wagons from the towns and from all the farms and +hamlets for seven miles around; they brought their children, and all +sorts of provisions, and confessed that they had had almost as +satisfactory a time at the funeral as they could have had at the +hanging. + +This funeral stopped the further growth of one thing--the petition to +the governor for Injun Joe's pardon. The petition had been largely +signed; many tearful and eloquent meetings had been held, and a +committee of sappy women been appointed to go in deep mourning and wail +around the governor, and implore him to be a merciful ass and trample +his duty under foot. Injun Joe was believed to have killed five +citizens of the village, but what of that? If he had been Satan himself +there would have been plenty of weaklings ready to scribble their names +to a pardon-petition, and drip a tear on it from their permanently +impaired and leaky water-works. + +The morning after the funeral Tom took Huck to a private place to have +an important talk. Huck had learned all about Tom's adventure from the +Welshman and the Widow Douglas, by this time, but Tom said he reckoned +there was one thing they had not told him; that thing was what he +wanted to talk about now. Huck's face saddened. He said: + +"I know what it is. You got into No. 2 and never found anything but +whiskey. Nobody told me it was you; but I just knowed it must 'a' ben +you, soon as I heard 'bout that whiskey business; and I knowed you +hadn't got the money becuz you'd 'a' got at me some way or other and +told me even if you was mum to everybody else. Tom, something's always +told me we'd never get holt of that swag." + +"Why, Huck, I never told on that tavern-keeper. YOU know his tavern +was all right the Saturday I went to the picnic. Don't you remember you +was to watch there that night?" + +"Oh yes! Why, it seems 'bout a year ago. It was that very night that I +follered Injun Joe to the widder's." + +"YOU followed him?" + +"Yes--but you keep mum. I reckon Injun Joe's left friends behind him, +and I don't want 'em souring on me and doing me mean tricks. If it +hadn't ben for me he'd be down in Texas now, all right." + +Then Huck told his entire adventure in confidence to Tom, who had only +heard of the Welshman's part of it before. + +"Well," said Huck, presently, coming back to the main question, +"whoever nipped the whiskey in No. 2, nipped the money, too, I reckon +--anyways it's a goner for us, Tom." + +"Huck, that money wasn't ever in No. 2!" + +"What!" Huck searched his comrade's face keenly. "Tom, have you got on +the track of that money again?" + +"Huck, it's in the cave!" + +Huck's eyes blazed. + +"Say it again, Tom." + +"The money's in the cave!" + +"Tom--honest injun, now--is it fun, or earnest?" + +"Earnest, Huck--just as earnest as ever I was in my life. Will you go +in there with me and help get it out?" + +"I bet I will! I will if it's where we can blaze our way to it and not +get lost." + +"Huck, we can do that without the least little bit of trouble in the +world." + +"Good as wheat! What makes you think the money's--" + +"Huck, you just wait till we get in there. If we don't find it I'll +agree to give you my drum and every thing I've got in the world. I +will, by jings." + +"All right--it's a whiz. When do you say?" + +"Right now, if you say it. Are you strong enough?" + +"Is it far in the cave? I ben on my pins a little, three or four days, +now, but I can't walk more'n a mile, Tom--least I don't think I could." + +"It's about five mile into there the way anybody but me would go, +Huck, but there's a mighty short cut that they don't anybody but me +know about. Huck, I'll take you right to it in a skiff. I'll float the +skiff down there, and I'll pull it back again all by myself. You +needn't ever turn your hand over." + +"Less start right off, Tom." + +"All right. We want some bread and meat, and our pipes, and a little +bag or two, and two or three kite-strings, and some of these +new-fangled things they call lucifer matches. I tell you, many's +the time I wished I had some when I was in there before." + +A trifle after noon the boys borrowed a small skiff from a citizen who +was absent, and got under way at once. When they were several miles +below "Cave Hollow," Tom said: + +"Now you see this bluff here looks all alike all the way down from the +cave hollow--no houses, no wood-yards, bushes all alike. But do you see +that white place up yonder where there's been a landslide? Well, that's +one of my marks. We'll get ashore, now." + +They landed. + +"Now, Huck, where we're a-standing you could touch that hole I got out +of with a fishing-pole. See if you can find it." + +Huck searched all the place about, and found nothing. Tom proudly +marched into a thick clump of sumach bushes and said: + +"Here you are! Look at it, Huck; it's the snuggest hole in this +country. You just keep mum about it. All along I've been wanting to be +a robber, but I knew I'd got to have a thing like this, and where to +run across it was the bother. We've got it now, and we'll keep it +quiet, only we'll let Joe Harper and Ben Rogers in--because of course +there's got to be a Gang, or else there wouldn't be any style about it. +Tom Sawyer's Gang--it sounds splendid, don't it, Huck?" + +"Well, it just does, Tom. And who'll we rob?" + +"Oh, most anybody. Waylay people--that's mostly the way." + +"And kill them?" + +"No, not always. Hive them in the cave till they raise a ransom." + +"What's a ransom?" + +"Money. You make them raise all they can, off'n their friends; and +after you've kept them a year, if it ain't raised then you kill them. +That's the general way. Only you don't kill the women. You shut up the +women, but you don't kill them. They're always beautiful and rich, and +awfully scared. You take their watches and things, but you always take +your hat off and talk polite. They ain't anybody as polite as robbers +--you'll see that in any book. Well, the women get to loving you, and +after they've been in the cave a week or two weeks they stop crying and +after that you couldn't get them to leave. If you drove them out they'd +turn right around and come back. It's so in all the books." + +"Why, it's real bully, Tom. I believe it's better'n to be a pirate." + +"Yes, it's better in some ways, because it's close to home and +circuses and all that." + +By this time everything was ready and the boys entered the hole, Tom +in the lead. They toiled their way to the farther end of the tunnel, +then made their spliced kite-strings fast and moved on. A few steps +brought them to the spring, and Tom felt a shudder quiver all through +him. He showed Huck the fragment of candle-wick perched on a lump of +clay against the wall, and described how he and Becky had watched the +flame struggle and expire. + +The boys began to quiet down to whispers, now, for the stillness and +gloom of the place oppressed their spirits. They went on, and presently +entered and followed Tom's other corridor until they reached the +"jumping-off place." The candles revealed the fact that it was not +really a precipice, but only a steep clay hill twenty or thirty feet +high. Tom whispered: + +"Now I'll show you something, Huck." + +He held his candle aloft and said: + +"Look as far around the corner as you can. Do you see that? There--on +the big rock over yonder--done with candle-smoke." + +"Tom, it's a CROSS!" + +"NOW where's your Number Two? 'UNDER THE CROSS,' hey? Right yonder's +where I saw Injun Joe poke up his candle, Huck!" + +Huck stared at the mystic sign awhile, and then said with a shaky voice: + +"Tom, less git out of here!" + +"What! and leave the treasure?" + +"Yes--leave it. Injun Joe's ghost is round about there, certain." + +"No it ain't, Huck, no it ain't. It would ha'nt the place where he +died--away out at the mouth of the cave--five mile from here." + +"No, Tom, it wouldn't. It would hang round the money. I know the ways +of ghosts, and so do you." + +Tom began to fear that Huck was right. Misgivings gathered in his +mind. But presently an idea occurred to him-- + +"Lookyhere, Huck, what fools we're making of ourselves! Injun Joe's +ghost ain't a going to come around where there's a cross!" + +The point was well taken. It had its effect. + +"Tom, I didn't think of that. But that's so. It's luck for us, that +cross is. I reckon we'll climb down there and have a hunt for that box." + +Tom went first, cutting rude steps in the clay hill as he descended. +Huck followed. Four avenues opened out of the small cavern which the +great rock stood in. The boys examined three of them with no result. +They found a small recess in the one nearest the base of the rock, with +a pallet of blankets spread down in it; also an old suspender, some +bacon rind, and the well-gnawed bones of two or three fowls. But there +was no money-box. The lads searched and researched this place, but in +vain. Tom said: + +"He said UNDER the cross. Well, this comes nearest to being under the +cross. It can't be under the rock itself, because that sets solid on +the ground." + +They searched everywhere once more, and then sat down discouraged. +Huck could suggest nothing. By-and-by Tom said: + +"Lookyhere, Huck, there's footprints and some candle-grease on the +clay about one side of this rock, but not on the other sides. Now, +what's that for? I bet you the money IS under the rock. I'm going to +dig in the clay." + +"That ain't no bad notion, Tom!" said Huck with animation. + +Tom's "real Barlow" was out at once, and he had not dug four inches +before he struck wood. + +"Hey, Huck!--you hear that?" + +Huck began to dig and scratch now. Some boards were soon uncovered and +removed. They had concealed a natural chasm which led under the rock. +Tom got into this and held his candle as far under the rock as he +could, but said he could not see to the end of the rift. He proposed to +explore. He stooped and passed under; the narrow way descended +gradually. He followed its winding course, first to the right, then to +the left, Huck at his heels. Tom turned a short curve, by-and-by, and +exclaimed: + +"My goodness, Huck, lookyhere!" + +It was the treasure-box, sure enough, occupying a snug little cavern, +along with an empty powder-keg, a couple of guns in leather cases, two +or three pairs of old moccasins, a leather belt, and some other rubbish +well soaked with the water-drip. + +"Got it at last!" said Huck, ploughing among the tarnished coins with +his hand. "My, but we're rich, Tom!" + +"Huck, I always reckoned we'd get it. It's just too good to believe, +but we HAVE got it, sure! Say--let's not fool around here. Let's snake +it out. Lemme see if I can lift the box." + +It weighed about fifty pounds. Tom could lift it, after an awkward +fashion, but could not carry it conveniently. + +"I thought so," he said; "THEY carried it like it was heavy, that day +at the ha'nted house. I noticed that. I reckon I was right to think of +fetching the little bags along." + +The money was soon in the bags and the boys took it up to the cross +rock. + +"Now less fetch the guns and things," said Huck. + +"No, Huck--leave them there. They're just the tricks to have when we +go to robbing. We'll keep them there all the time, and we'll hold our +orgies there, too. It's an awful snug place for orgies." + +"What orgies?" + +"I dono. But robbers always have orgies, and of course we've got to +have them, too. Come along, Huck, we've been in here a long time. It's +getting late, I reckon. I'm hungry, too. We'll eat and smoke when we +get to the skiff." + +They presently emerged into the clump of sumach bushes, looked warily +out, found the coast clear, and were soon lunching and smoking in the +skiff. As the sun dipped toward the horizon they pushed out and got +under way. Tom skimmed up the shore through the long twilight, chatting +cheerily with Huck, and landed shortly after dark. + +"Now, Huck," said Tom, "we'll hide the money in the loft of the +widow's woodshed, and I'll come up in the morning and we'll count it +and divide, and then we'll hunt up a place out in the woods for it +where it will be safe. Just you lay quiet here and watch the stuff till +I run and hook Benny Taylor's little wagon; I won't be gone a minute." + +He disappeared, and presently returned with the wagon, put the two +small sacks into it, threw some old rags on top of them, and started +off, dragging his cargo behind him. When the boys reached the +Welshman's house, they stopped to rest. Just as they were about to move +on, the Welshman stepped out and said: + +"Hallo, who's that?" + +"Huck and Tom Sawyer." + +"Good! Come along with me, boys, you are keeping everybody waiting. +Here--hurry up, trot ahead--I'll haul the wagon for you. Why, it's not +as light as it might be. Got bricks in it?--or old metal?" + +"Old metal," said Tom. + +"I judged so; the boys in this town will take more trouble and fool +away more time hunting up six bits' worth of old iron to sell to the +foundry than they would to make twice the money at regular work. But +that's human nature--hurry along, hurry along!" + +The boys wanted to know what the hurry was about. + +"Never mind; you'll see, when we get to the Widow Douglas'." + +Huck said with some apprehension--for he was long used to being +falsely accused: + +"Mr. Jones, we haven't been doing nothing." + +The Welshman laughed. + +"Well, I don't know, Huck, my boy. I don't know about that. Ain't you +and the widow good friends?" + +"Yes. Well, she's ben good friends to me, anyway." + +"All right, then. What do you want to be afraid for?" + +This question was not entirely answered in Huck's slow mind before he +found himself pushed, along with Tom, into Mrs. Douglas' drawing-room. +Mr. Jones left the wagon near the door and followed. + +The place was grandly lighted, and everybody that was of any +consequence in the village was there. The Thatchers were there, the +Harpers, the Rogerses, Aunt Polly, Sid, Mary, the minister, the editor, +and a great many more, and all dressed in their best. The widow +received the boys as heartily as any one could well receive two such +looking beings. They were covered with clay and candle-grease. Aunt +Polly blushed crimson with humiliation, and frowned and shook her head +at Tom. Nobody suffered half as much as the two boys did, however. Mr. +Jones said: + +"Tom wasn't at home, yet, so I gave him up; but I stumbled on him and +Huck right at my door, and so I just brought them along in a hurry." + +"And you did just right," said the widow. "Come with me, boys." + +She took them to a bedchamber and said: + +"Now wash and dress yourselves. Here are two new suits of clothes +--shirts, socks, everything complete. They're Huck's--no, no thanks, +Huck--Mr. Jones bought one and I the other. But they'll fit both of you. +Get into them. We'll wait--come down when you are slicked up enough." + +Then she left. + + + +CHAPTER XXXIV + +HUCK said: "Tom, we can slope, if we can find a rope. The window ain't +high from the ground." + +"Shucks! what do you want to slope for?" + +"Well, I ain't used to that kind of a crowd. I can't stand it. I ain't +going down there, Tom." + +"Oh, bother! It ain't anything. I don't mind it a bit. I'll take care +of you." + +Sid appeared. + +"Tom," said he, "auntie has been waiting for you all the afternoon. +Mary got your Sunday clothes ready, and everybody's been fretting about +you. Say--ain't this grease and clay, on your clothes?" + +"Now, Mr. Siddy, you jist 'tend to your own business. What's all this +blow-out about, anyway?" + +"It's one of the widow's parties that she's always having. This time +it's for the Welshman and his sons, on account of that scrape they +helped her out of the other night. And say--I can tell you something, +if you want to know." + +"Well, what?" + +"Why, old Mr. Jones is going to try to spring something on the people +here to-night, but I overheard him tell auntie to-day about it, as a +secret, but I reckon it's not much of a secret now. Everybody knows +--the widow, too, for all she tries to let on she don't. Mr. Jones was +bound Huck should be here--couldn't get along with his grand secret +without Huck, you know!" + +"Secret about what, Sid?" + +"About Huck tracking the robbers to the widow's. I reckon Mr. Jones +was going to make a grand time over his surprise, but I bet you it will +drop pretty flat." + +Sid chuckled in a very contented and satisfied way. + +"Sid, was it you that told?" + +"Oh, never mind who it was. SOMEBODY told--that's enough." + +"Sid, there's only one person in this town mean enough to do that, and +that's you. If you had been in Huck's place you'd 'a' sneaked down the +hill and never told anybody on the robbers. You can't do any but mean +things, and you can't bear to see anybody praised for doing good ones. +There--no thanks, as the widow says"--and Tom cuffed Sid's ears and +helped him to the door with several kicks. "Now go and tell auntie if +you dare--and to-morrow you'll catch it!" + +Some minutes later the widow's guests were at the supper-table, and a +dozen children were propped up at little side-tables in the same room, +after the fashion of that country and that day. At the proper time Mr. +Jones made his little speech, in which he thanked the widow for the +honor she was doing himself and his sons, but said that there was +another person whose modesty-- + +And so forth and so on. He sprung his secret about Huck's share in the +adventure in the finest dramatic manner he was master of, but the +surprise it occasioned was largely counterfeit and not as clamorous and +effusive as it might have been under happier circumstances. However, +the widow made a pretty fair show of astonishment, and heaped so many +compliments and so much gratitude upon Huck that he almost forgot the +nearly intolerable discomfort of his new clothes in the entirely +intolerable discomfort of being set up as a target for everybody's gaze +and everybody's laudations. + +The widow said she meant to give Huck a home under her roof and have +him educated; and that when she could spare the money she would start +him in business in a modest way. Tom's chance was come. He said: + +"Huck don't need it. Huck's rich." + +Nothing but a heavy strain upon the good manners of the company kept +back the due and proper complimentary laugh at this pleasant joke. But +the silence was a little awkward. Tom broke it: + +"Huck's got money. Maybe you don't believe it, but he's got lots of +it. Oh, you needn't smile--I reckon I can show you. You just wait a +minute." + +Tom ran out of doors. The company looked at each other with a +perplexed interest--and inquiringly at Huck, who was tongue-tied. + +"Sid, what ails Tom?" said Aunt Polly. "He--well, there ain't ever any +making of that boy out. I never--" + +Tom entered, struggling with the weight of his sacks, and Aunt Polly +did not finish her sentence. Tom poured the mass of yellow coin upon +the table and said: + +"There--what did I tell you? Half of it's Huck's and half of it's mine!" + +The spectacle took the general breath away. All gazed, nobody spoke +for a moment. Then there was a unanimous call for an explanation. Tom +said he could furnish it, and he did. The tale was long, but brimful of +interest. There was scarcely an interruption from any one to break the +charm of its flow. When he had finished, Mr. Jones said: + +"I thought I had fixed up a little surprise for this occasion, but it +don't amount to anything now. This one makes it sing mighty small, I'm +willing to allow." + +The money was counted. The sum amounted to a little over twelve +thousand dollars. It was more than any one present had ever seen at one +time before, though several persons were there who were worth +considerably more than that in property. + + + +CHAPTER XXXV + +THE reader may rest satisfied that Tom's and Huck's windfall made a +mighty stir in the poor little village of St. Petersburg. So vast a +sum, all in actual cash, seemed next to incredible. It was talked +about, gloated over, glorified, until the reason of many of the +citizens tottered under the strain of the unhealthy excitement. Every +"haunted" house in St. Petersburg and the neighboring villages was +dissected, plank by plank, and its foundations dug up and ransacked for +hidden treasure--and not by boys, but men--pretty grave, unromantic +men, too, some of them. Wherever Tom and Huck appeared they were +courted, admired, stared at. The boys were not able to remember that +their remarks had possessed weight before; but now their sayings were +treasured and repeated; everything they did seemed somehow to be +regarded as remarkable; they had evidently lost the power of doing and +saying commonplace things; moreover, their past history was raked up +and discovered to bear marks of conspicuous originality. The village +paper published biographical sketches of the boys. + +The Widow Douglas put Huck's money out at six per cent., and Judge +Thatcher did the same with Tom's at Aunt Polly's request. Each lad had +an income, now, that was simply prodigious--a dollar for every week-day +in the year and half of the Sundays. It was just what the minister got +--no, it was what he was promised--he generally couldn't collect it. A +dollar and a quarter a week would board, lodge, and school a boy in +those old simple days--and clothe him and wash him, too, for that +matter. + +Judge Thatcher had conceived a great opinion of Tom. He said that no +commonplace boy would ever have got his daughter out of the cave. When +Becky told her father, in strict confidence, how Tom had taken her +whipping at school, the Judge was visibly moved; and when she pleaded +grace for the mighty lie which Tom had told in order to shift that +whipping from her shoulders to his own, the Judge said with a fine +outburst that it was a noble, a generous, a magnanimous lie--a lie that +was worthy to hold up its head and march down through history breast to +breast with George Washington's lauded Truth about the hatchet! Becky +thought her father had never looked so tall and so superb as when he +walked the floor and stamped his foot and said that. She went straight +off and told Tom about it. + +Judge Thatcher hoped to see Tom a great lawyer or a great soldier some +day. He said he meant to look to it that Tom should be admitted to the +National Military Academy and afterward trained in the best law school +in the country, in order that he might be ready for either career or +both. + +Huck Finn's wealth and the fact that he was now under the Widow +Douglas' protection introduced him into society--no, dragged him into +it, hurled him into it--and his sufferings were almost more than he +could bear. The widow's servants kept him clean and neat, combed and +brushed, and they bedded him nightly in unsympathetic sheets that had +not one little spot or stain which he could press to his heart and know +for a friend. He had to eat with a knife and fork; he had to use +napkin, cup, and plate; he had to learn his book, he had to go to +church; he had to talk so properly that speech was become insipid in +his mouth; whithersoever he turned, the bars and shackles of +civilization shut him in and bound him hand and foot. + +He bravely bore his miseries three weeks, and then one day turned up +missing. For forty-eight hours the widow hunted for him everywhere in +great distress. The public were profoundly concerned; they searched +high and low, they dragged the river for his body. Early the third +morning Tom Sawyer wisely went poking among some old empty hogsheads +down behind the abandoned slaughter-house, and in one of them he found +the refugee. Huck had slept there; he had just breakfasted upon some +stolen odds and ends of food, and was lying off, now, in comfort, with +his pipe. He was unkempt, uncombed, and clad in the same old ruin of +rags that had made him picturesque in the days when he was free and +happy. Tom routed him out, told him the trouble he had been causing, +and urged him to go home. Huck's face lost its tranquil content, and +took a melancholy cast. He said: + +"Don't talk about it, Tom. I've tried it, and it don't work; it don't +work, Tom. It ain't for me; I ain't used to it. The widder's good to +me, and friendly; but I can't stand them ways. She makes me get up just +at the same time every morning; she makes me wash, they comb me all to +thunder; she won't let me sleep in the woodshed; I got to wear them +blamed clothes that just smothers me, Tom; they don't seem to any air +git through 'em, somehow; and they're so rotten nice that I can't set +down, nor lay down, nor roll around anywher's; I hain't slid on a +cellar-door for--well, it 'pears to be years; I got to go to church and +sweat and sweat--I hate them ornery sermons! I can't ketch a fly in +there, I can't chaw. I got to wear shoes all Sunday. The widder eats by +a bell; she goes to bed by a bell; she gits up by a bell--everything's +so awful reg'lar a body can't stand it." + +"Well, everybody does that way, Huck." + +"Tom, it don't make no difference. I ain't everybody, and I can't +STAND it. It's awful to be tied up so. And grub comes too easy--I don't +take no interest in vittles, that way. I got to ask to go a-fishing; I +got to ask to go in a-swimming--dern'd if I hain't got to ask to do +everything. Well, I'd got to talk so nice it wasn't no comfort--I'd got +to go up in the attic and rip out awhile, every day, to git a taste in +my mouth, or I'd a died, Tom. The widder wouldn't let me smoke; she +wouldn't let me yell, she wouldn't let me gape, nor stretch, nor +scratch, before folks--" [Then with a spasm of special irritation and +injury]--"And dad fetch it, she prayed all the time! I never see such a +woman! I HAD to shove, Tom--I just had to. And besides, that school's +going to open, and I'd a had to go to it--well, I wouldn't stand THAT, +Tom. Looky here, Tom, being rich ain't what it's cracked up to be. It's +just worry and worry, and sweat and sweat, and a-wishing you was dead +all the time. Now these clothes suits me, and this bar'l suits me, and +I ain't ever going to shake 'em any more. Tom, I wouldn't ever got into +all this trouble if it hadn't 'a' ben for that money; now you just take +my sheer of it along with your'n, and gimme a ten-center sometimes--not +many times, becuz I don't give a dern for a thing 'thout it's tollable +hard to git--and you go and beg off for me with the widder." + +"Oh, Huck, you know I can't do that. 'Tain't fair; and besides if +you'll try this thing just a while longer you'll come to like it." + +"Like it! Yes--the way I'd like a hot stove if I was to set on it long +enough. No, Tom, I won't be rich, and I won't live in them cussed +smothery houses. I like the woods, and the river, and hogsheads, and +I'll stick to 'em, too. Blame it all! just as we'd got guns, and a +cave, and all just fixed to rob, here this dern foolishness has got to +come up and spile it all!" + +Tom saw his opportunity-- + +"Lookyhere, Huck, being rich ain't going to keep me back from turning +robber." + +"No! Oh, good-licks; are you in real dead-wood earnest, Tom?" + +"Just as dead earnest as I'm sitting here. But Huck, we can't let you +into the gang if you ain't respectable, you know." + +Huck's joy was quenched. + +"Can't let me in, Tom? Didn't you let me go for a pirate?" + +"Yes, but that's different. A robber is more high-toned than what a +pirate is--as a general thing. In most countries they're awful high up +in the nobility--dukes and such." + +"Now, Tom, hain't you always ben friendly to me? You wouldn't shet me +out, would you, Tom? You wouldn't do that, now, WOULD you, Tom?" + +"Huck, I wouldn't want to, and I DON'T want to--but what would people +say? Why, they'd say, 'Mph! Tom Sawyer's Gang! pretty low characters in +it!' They'd mean you, Huck. You wouldn't like that, and I wouldn't." + +Huck was silent for some time, engaged in a mental struggle. Finally +he said: + +"Well, I'll go back to the widder for a month and tackle it and see if +I can come to stand it, if you'll let me b'long to the gang, Tom." + +"All right, Huck, it's a whiz! Come along, old chap, and I'll ask the +widow to let up on you a little, Huck." + +"Will you, Tom--now will you? That's good. If she'll let up on some of +the roughest things, I'll smoke private and cuss private, and crowd +through or bust. When you going to start the gang and turn robbers?" + +"Oh, right off. We'll get the boys together and have the initiation +to-night, maybe." + +"Have the which?" + +"Have the initiation." + +"What's that?" + +"It's to swear to stand by one another, and never tell the gang's +secrets, even if you're chopped all to flinders, and kill anybody and +all his family that hurts one of the gang." + +"That's gay--that's mighty gay, Tom, I tell you." + +"Well, I bet it is. And all that swearing's got to be done at +midnight, in the lonesomest, awfulest place you can find--a ha'nted +house is the best, but they're all ripped up now." + +"Well, midnight's good, anyway, Tom." + +"Yes, so it is. And you've got to swear on a coffin, and sign it with +blood." + +"Now, that's something LIKE! Why, it's a million times bullier than +pirating. I'll stick to the widder till I rot, Tom; and if I git to be +a reg'lar ripper of a robber, and everybody talking 'bout it, I reckon +she'll be proud she snaked me in out of the wet." + + + +CONCLUSION + +SO endeth this chronicle. It being strictly a history of a BOY, it +must stop here; the story could not go much further without becoming +the history of a MAN. When one writes a novel about grown people, he +knows exactly where to stop--that is, with a marriage; but when he +writes of juveniles, he must stop where he best can. + +Most of the characters that perform in this book still live, and are +prosperous and happy. Some day it may seem worth while to take up the +story of the younger ones again and see what sort of men and women they +turned out to be; therefore it will be wisest not to reveal any of that +part of their lives at present. + + + + + +End of the Project Gutenberg EBook of The Adventures of Tom Sawyer, Complete +by Mark Twain (Samuel Clemens) diff --git a/internal/compress/testdata/case1.bin b/internal/compress/testdata/case1.bin new file mode 100644 index 00000000..723b4bc2 Binary files /dev/null and b/internal/compress/testdata/case1.bin differ diff --git a/internal/compress/testdata/case2.bin b/internal/compress/testdata/case2.bin new file mode 100644 index 00000000..c34928f3 --- /dev/null +++ b/internal/compress/testdata/case2.bin @@ -0,0 +1 @@ +55555555555550540723072774072245322006ÿ00565 \ No newline at end of file diff --git a/internal/compress/testdata/case3.bin b/internal/compress/testdata/case3.bin new file mode 100644 index 00000000..c44f88d4 --- /dev/null +++ b/internal/compress/testdata/case3.bin @@ -0,0 +1 @@ +ÿ150230257800001061083404ÿ4101581512311545562525 \ No newline at end of file diff --git a/internal/compress/testdata/crash1.bin b/internal/compress/testdata/crash1.bin new file mode 100644 index 00000000..d483846c Binary files /dev/null and b/internal/compress/testdata/crash1.bin differ diff --git a/internal/compress/testdata/crash2.bin b/internal/compress/testdata/crash2.bin new file mode 100644 index 00000000..721a6531 --- /dev/null +++ b/internal/compress/testdata/crash2.bin @@ -0,0 +1 @@ +313254615470505 \ No newline at end of file diff --git a/internal/compress/testdata/crash3.bin b/internal/compress/testdata/crash3.bin new file mode 100644 index 00000000..3e3d493a --- /dev/null +++ b/internal/compress/testdata/crash3.bin @@ -0,0 +1 @@ +‡‡û \ No newline at end of file diff --git a/internal/compress/testdata/crash4.bin b/internal/compress/testdata/crash4.bin new file mode 100644 index 00000000..725b093f --- /dev/null +++ b/internal/compress/testdata/crash4.bin @@ -0,0 +1 @@ +úÿÿÿÏïÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿ® \ No newline at end of file diff --git a/internal/compress/testdata/crash5.bin b/internal/compress/testdata/crash5.bin new file mode 100644 index 00000000..a521217b --- /dev/null +++ b/internal/compress/testdata/crash5.bin @@ -0,0 +1 @@ +úÿÿ55511151231257827021181583404541015625ß \ No newline at end of file diff --git a/internal/compress/testdata/dec-crash6.bin b/internal/compress/testdata/dec-crash6.bin new file mode 100644 index 00000000..d7832124 Binary files /dev/null and b/internal/compress/testdata/dec-crash6.bin differ diff --git a/internal/compress/testdata/dec-hang1.bin b/internal/compress/testdata/dec-hang1.bin new file mode 100644 index 00000000..e07926fb --- /dev/null +++ b/internal/compress/testdata/dec-hang1.bin @@ -0,0 +1 @@ +$ä \ No newline at end of file diff --git a/internal/compress/testdata/dec-hang2.bin b/internal/compress/testdata/dec-hang2.bin new file mode 100644 index 00000000..c1484ef8 --- /dev/null +++ b/internal/compress/testdata/dec-hang2.bin @@ -0,0 +1 @@ +þÿŠ|C ðO \ No newline at end of file diff --git a/internal/compress/testdata/dec-hang3.bin b/internal/compress/testdata/dec-hang3.bin new file mode 100644 index 00000000..3bd97e9c --- /dev/null +++ b/internal/compress/testdata/dec-hang3.bin @@ -0,0 +1 @@ +þÿÿVV \ No newline at end of file diff --git a/internal/compress/testdata/dec-symlen1.bin b/internal/compress/testdata/dec-symlen1.bin new file mode 100644 index 00000000..c5589c99 Binary files /dev/null and b/internal/compress/testdata/dec-symlen1.bin differ diff --git a/internal/compress/testdata/e.txt b/internal/compress/testdata/e.txt new file mode 100644 index 00000000..5ca186f1 --- /dev/null +++ b/internal/compress/testdata/e.txt @@ -0,0 +1 @@ +2.7182818284590452353602874713526624977572470936999595749669676277240766303535475945713821785251664274274663919320030599218174135966290435729003342952605956307381323286279434907632338298807531952510190115738341879307021540891499348841675092447614606680822648001684774118537423454424371075390777449920695517027618386062613313845830007520449338265602976067371132007093287091274437470472306969772093101416928368190255151086574637721112523897844250569536967707854499699679468644549059879316368892300987931277361782154249992295763514822082698951936680331825288693984964651058209392398294887933203625094431173012381970684161403970198376793206832823764648042953118023287825098194558153017567173613320698112509961818815930416903515988885193458072738667385894228792284998920868058257492796104841984443634632449684875602336248270419786232090021609902353043699418491463140934317381436405462531520961836908887070167683964243781405927145635490613031072085103837505101157477041718986106873969655212671546889570350354021234078498193343210681701210056278802351930332247450158539047304199577770935036604169973297250886876966403555707162268447162560798826517871341951246652010305921236677194325278675398558944896970964097545918569563802363701621120477427228364896134225164450781824423529486363721417402388934412479635743702637552944483379980161254922785092577825620926226483262779333865664816277251640191059004916449982893150566047258027786318641551956532442586982946959308019152987211725563475463964479101459040905862984967912874068705048958586717479854667757573205681288459205413340539220001137863009455606881667400169842055804033637953764520304024322566135278369511778838638744396625322498506549958862342818997077332761717839280349465014345588970719425863987727547109629537415211151368350627526023264847287039207643100595841166120545297030236472549296669381151373227536450988890313602057248176585118063036442812314965507047510254465011727211555194866850800368532281831521960037356252794495158284188294787610852639813955990067376482922443752871846245780361929819713991475644882626039033814418232625150974827987779964373089970388867782271383605772978824125611907176639465070633045279546618550966661856647097113444740160704626215680717481877844371436988218559670959102596862002353718588748569652200050311734392073211390803293634479727355955277349071783793421637012050054513263835440001863239914907054797780566978533580489669062951194324730995876552368128590413832411607226029983305353708761389396391779574540161372236187893652605381558415871869255386061647798340254351284396129460352913325942794904337299085731580290958631382683291477116396337092400316894586360606458459251269946557248391865642097526850823075442545993769170419777800853627309417101634349076964237222943523661255725088147792231519747780605696725380171807763603462459278778465850656050780844211529697521890874019660906651803516501792504619501366585436632712549639908549144200014574760819302212066024330096412704894390397177195180699086998606636583232278709376502260149291011517177635944602023249300280401867723910288097866605651183260043688508817157238669842242201024950551881694803221002515426494639812873677658927688163598312477886520141174110913601164995076629077943646005851941998560162647907615321038727557126992518275687989302761761146162549356495903798045838182323368612016243736569846703785853305275833337939907521660692380533698879565137285593883499894707416181550125397064648171946708348197214488898790676503795903669672494992545279033729636162658976039498576741397359441023744329709355477982629614591442936451428617158587339746791897571211956187385783644758448423555581050025611492391518893099463428413936080383091662818811503715284967059741625628236092168075150177725387402564253470879089137291722828611515915683725241630772254406337875931059826760944203261924285317018781772960235413060672136046000389661093647095141417185777014180606443636815464440053316087783143174440811949422975599314011888683314832802706553833004693290115744147563139997221703804617092894579096271662260740718749975359212756084414737823303270330168237193648002173285734935947564334129943024850235732214597843282641421684878721673367010615094243456984401873312810107945127223737886126058165668053714396127888732527373890392890506865324138062796025930387727697783792868409325365880733988457218746021005311483351323850047827169376218004904795597959290591655470505777514308175112698985188408718564026035305583737832422924185625644255022672155980274012617971928047139600689163828665277009752767069777036439260224372841840883251848770472638440379530166905465937461619323840363893131364327137688841026811219891275223056256756254701725086349765367288605966752740868627407912856576996313789753034660616669804218267724560530660773899624218340859882071864682623215080288286359746839654358856685503773131296587975810501214916207656769950659715344763470320853215603674828608378656803073062657633469774295634643716709397193060876963495328846833613038829431040800296873869117066666146800015121143442256023874474325250769387077775193299942137277211258843608715834835626961661980572526612206797540621062080649882918454395301529982092503005498257043390553570168653120526495614857249257386206917403695213533732531666345466588597286659451136441370331393672118569553952108458407244323835586063106806964924851232632699514603596037297253198368423363904632136710116192821711150282801604488058802382031981493096369596735832742024988245684941273860566491352526706046234450549227581151709314921879592718001940968866986837037302200475314338181092708030017205935530520700706072233999463990571311587099635777359027196285061146514837526209565346713290025994397663114545902685898979115837093419370441155121920117164880566945938131183843765620627846310490346293950029458341164824114969758326011800731699437393506966295712410273239138741754923071862454543222039552735295240245903805744502892246886285336542213815722131163288112052146489805180092024719391710555390113943316681515828843687606961102505171007392762385553386272553538830960671644662370922646809671254061869502143176211668140097595281493907222601112681153108387317617323235263605838173151034595736538223534992935822836851007810884634349983518404451704270189381994243410090575376257767571118090088164183319201962623416288166521374717325477727783488774366518828752156685719506371936565390389449366421764003121527870222366463635755503565576948886549500270853923617105502131147413744106134445544192101336172996285694899193369184729478580729156088510396781959429833186480756083679551496636448965592948187851784038773326247051945050419847742014183947731202815886845707290544057510601285258056594703046836344592652552137008068752009593453607316226118728173928074623094685367823106097921599360019946237993434210687813497346959246469752506246958616909178573976595199392993995567542714654910456860702099012606818704984178079173924071945996323060254707901774527513186809982284730860766536866855516467702911336827563107223346726113705490795365834538637196235856312618387156774118738527722922594743373785695538456246801013905727871016512966636764451872465653730402443684140814488732957847348490003019477888020460324660842875351848364959195082888323206522128104190448047247949291342284951970022601310430062410717971502793433263407995960531446053230488528972917659876016667811937932372453857209607582277178483361613582612896226118129455927462767137794487586753657544861407611931125958512655759734573015333642630767985443385761715333462325270572005303988289499034259566232975782488735029259166825894456894655992658454762694528780516501720674785417887982276806536650641910973434528878338621726156269582654478205672987756426325321594294418039943217000090542650763095588465895171709147607437136893319469090981904501290307099566226620303182649365733698419555776963787624918852865686607600566025605445711337286840205574416030837052312242587223438854123179481388550075689381124935386318635287083799845692619981794523364087429591180747453419551420351726184200845509170845682368200897739455842679214273477560879644279202708312150156406341341617166448069815483764491573900121217041547872591998943825364950514771379399147205219529079396137621107238494290616357604596231253506068537651423115349665683715116604220796394466621163255157729070978473156278277598788136491951257483328793771571459091064841642678309949723674420175862269402159407924480541255360431317992696739157542419296607312393763542139230617876753958711436104089409966089471418340698362993675362621545247298464213752891079884381306095552622720837518629837066787224430195793793786072107254277289071732854874374355781966511716618330881129120245204048682200072344035025448202834254187884653602591506445271657700044521097735585897622655484941621714989532383421600114062950718490427789258552743035221396835679018076406042138307308774460170842688272261177180842664333651780002171903449234264266292261456004337383868335555343453004264818473989215627086095650629340405264943244261445665921291225648893569655009154306426134252668472594914314239398845432486327461842846655985332312210466259890141712103446084271616619001257195870793217569698544013397622096749454185407118446433946990162698351607848924514058940946395267807354579700307051163682519487701189764002827648414160587206184185297189154019688253289309149665345753571427318482016384644832499037886069008072709327673127581966563941148961716832980455139729506687604740915420428429993541025829113502241690769431668574242522509026939034814856451303069925199590436384028429267412573422447765584177886171737265462085498294498946787350929581652632072258992368768457017823038096567883112289305809140572610865884845873101658151167533327674887014829167419701512559782572707406431808601428149024146780472327597684269633935773542930186739439716388611764209004068663398856841681003872389214483176070116684503887212364367043314091155733280182977988736590916659612402021778558854876176161989370794380056663364884365089144805571039765214696027662583599051987042300179465536788567430285974600143785483237068701190078499404930918919181649327259774030074879681484882342932023012128032327460392219687528340516906974194257614673978110715464186273369091584973185011183960482533518748438923177292613543024932562896371361977285456622924461644497284597867711574125670307871885109336344480149675240618536569532074170533486782754827815415561966911055101472799040386897220465550833170782394808785990501947563108984124144672821865459971596639015641941751820935932616316888380132758752601460507676098392625726411120135288591317848299475682472564885533357279772205543568126302535748216585414000805314820697137262149755576051890481622376790414926742600071045922695314835188137463887104273544767623577933993970632396604969145303273887874557905934937772320142954803345000695256980935282887783710670585567749481373858630385762823040694005665340584887527005308832459182183494318049834199639981458773435863115940570443683515285383609442955964360676090221741896883548131643997437764158365242234642619597390455450680695232850751868719449064767791886720306418630751053512149851051207313846648717547518382979990189317751550639981016466414592102406838294603208535554058147159273220677567669213664081505900806952540610628536408293276621931939933861623836069111767785448236129326858199965239275488427435414402884536455595124735546139403154952097397051896240157976832639450633230452192645049651735466775699295718989690470902730288544945416699791992948038254980285946029052763145580316514066229171223429375806143993484914362107993576737317948964252488813720435579287511385856973381976083524423240466778020948399639946684833774706725483618848273000648319163826022110555221246733323184463005504481849916996622087746140216157021029603318588727333298779352570182393861244026868339555870607758169954398469568540671174444932479519572159419645863736126915526457574786985964242176592896862383506370433939811671397544736228625506803682664135541448048997721373174119199970017293907303350869020922519124447393278376156321810842898207706974138707053266117683698647741787180202729412982310888796831880854367327806879771659111654224453806625861711729498038248879986504061563975629936962809358189761491017145343556659542757064194408833816841111166200759787244137082333917886114708228657531078536674695018462140736493917366254937783014074302668422150335117736471853872324040421037907750266020114814935482228916663640782450166815341213505278578539332606110249802273093636740213515386431693015267460536064351732154701091440650878823636764236831187390937464232609021646365627553976834019482932795750624399645272578624400375983422050808935129023122475970644105678361870877172333555465482598906861201410107222465904008553798235253885171623518256518482203125214950700378300411216212126052726059944320443056274522916128891766814160639131235975350390320077529587392412476451850809163911459296071156344204347133544720981178461451077872399140606290228276664309264900592249810291068759434533858330391178747575977065953570979640012224092199031158229259667913153991561438070129260780197022589662923368154312499412259460023399472228171056603931877226800493833148980338548909468685130789292064242819174795866199944411196208730498064385006852620258432842085582338566936649849720817046135376163584015342840674118587581546514598270228676671855309311923340191286170613364873183197560812569460089402953094429119590295968563923037689976327462283900735457144596414108229285922239332836210192822937243590283003884445701383771632056518351970100115722010956997890484964453434612129224964732356126321951155701565824427661599326463155806672053127596948538057364208384918887095176052287817339462747644656858900936266123311152910816041524100214195937349786431661556732702792109593543055579732660554677963552005378304619540636971842916168582734122217145885870814274090248185446421774876925093328785670674677381226752831653559245204578070541352576903253522738963847495646255940378924925007624386893776475310102323746733771474581625530698032499033676455430305274561512961214585944432150749051491453950981001388737926379964873728396416897555132275962011838248650746985492038097691932606437608743209385602815642849756549307909733854185583515789409814007691892389063090542534883896831762904120212949167195811935791203162514344096503132835216728021372415947344095498316138322505486708172221475138425166790445416617303200820330902895488808516797258495813407132180533988828139346049850532340472595097214331492586604248511405819579711564191458842833000525684776874305916390494306871343118796189637475503362820939949343690321031976898112055595369465424704173323895394046035325396758354395350516720261647961347790912327995264929045151148307923369382166010702872651938143844844532639517394110131152502750465749343063766541866128915264446926222884366299462732467958736383501937142786471398054038215513463223702071533134887083174146591492406359493020921122052610312390682941345696785958518393491382340884274312419099152870804332809132993078936867127413922890033069995875921815297612482409116951587789964090352577345938248232053055567238095022266790439614231852991989181065554412477204508510210071522352342792531266930108270633942321762570076323139159349709946933241013908779161651226804414809765618979735043151396066913258379033748620836695475083280318786707751177525663963479259219733577949555498655214193398170268639987388347010255262052312317215254062571636771270010760912281528326508984359568975961038372157726831170734552250194121701541318793651818502020877326906133592182000762327269503283827391243828198170871168108951187896746707073377869592565542713340052326706040004348843432902760360498027862160749469654989210474443927871934536701798673920803845633723311983855862638008516345597194441994344624761123844617615736242015935078520825600604101556889899501732554337298073561699861101908472096600708320280569917042590103876928658336557728758684250492690370934262028022399861803400211320742198642917383679176232826444645756330336556777374808644109969141827774253417010988435853189339175934511574023847292909015468559163792696196841000676598399744972047287881831200233383298030567865480871476464512824264478216644266616732096012564794514827125671326697067367144617795643752391742928503987022583734069852309190464967260243411270345611114149835783901793499713790913696706497637127248466613279908254305449295528594932793818341607827091326680865655921102733746700132583428715240835661522165574998431236278287106649401564670141943713823863454729606978693335973109537126499416282656463708490580151538205338326511289504938566468752921135932220265681856418260827538790002407915892646028490894922299966167437731347776134150965262448332709343898412056926145108857812249139616912534202918139898683901335795857624435194008943955180554746554000051766240202825944828833811886381749594284892013520090951007864941868256009273977667585642598378587497776669563350170748579027248701370264203283965756348010818356182372177082236423186591595883669487322411726504487268392328453010991677518376831599821263237123854357312681202445175401852132663740538802901249728180895021553100673598184430429105288459323064725590442355960551978839325930339572934663055160430923785677229293537208416693134575284011873746854691620648991164726909428982971065606801805807843600461866223562874591385185904416250663222249561448724413813849763797102676020845531824111963927941069619465426480006761727618115630063644321116224837379105623611358836334550102286170517890440570419577859833348463317921904494652923021469259756566389965893747728751393377105569802455757436190501772466214587592374418657530064998056688376964229825501195065837843125232135309371235243969149662310110328243570065781487677299160941153954063362752423712935549926713485031578238899567545287915578420483105749330060197958207739558522807307048950936235550769837881926357141779338750216344391014187576711938914416277109602859415809719913429313295145924373636456473035037374538503489286113141638094752301745088784885645741275003353303416138096560043105860548355773946625033230034341587814634602169235079216111013148948281895391028916816328709309713184139815427678818067628650978085718262117003140003377301581536334149093237034703637513354537634521050370995452942055232078817449370937677056009306353645510913481627378204985657055608784211964039972344556458607689515569686899384896439195225232309703301037277227710870564912966121061494072782442033414057441446459968236966118878411656290355117839944070961772567164919790168195234523807446299877664824873753313018142763910519234685081979001796519907050490865237442841652776611425351538665162781316090964802801234493372427866930894827913465443931965254154829494577875758599482099181824522449312077768250830768282335001597040419199560509705364696473142448453825888112602753909548852639708652339052941829691802357120545328231809270356491743371932080628731303589640570873779967845174740515317401384878082881006046388936711640477755985481263907504747295012609419990373721246201677030517790352952793168766305099837441859803498821239340919805055103821539827677291373138006715339240126954586376422065097810852907639079727841301764553247527073788764069366420012194745702358295481365781809867944020220280822637957006755393575808086318932075864444206644691649334467698180811716568665213389686173592450920801465312529777966137198695916451869432324246404401672381978020728394418264502183131483366019384891972317817154372192103946638473715630226701801343515930442853848941825678870721238520597263859224934763623122188113706307506918260109689069251417142514218153491532129077723748506635489170892850760234351768218355008829647410655814882049239533702270536705630750317499788187009989251020178015601042277836283644323729779929935160925884515772055232896978333126427671291093993103773425910592303277652667641874842441076564447767097790392324958416348527735171981064673837142742974468992320406932506062834468937543016787815320616009057693404906146176607094380110915443261929000745209895959201159412324102274845482605404361871836330268992858623582145643879695210235266673372434423091577183277565800211928270391042391966426911155333594569685782817020325495552528875464466074620294766116004435551604735044292127916358748473501590215522120388281168021413865865168464569964810015633741255098479730138656275460161279246359783661480163871602794405482710196290774543628092612567507181773641749763254436773503632580004042919906963117397787875081560227368824967077635559869284901628768699628053790181848148810833946900016380791075960745504688912686792812391148880036720729730801354431325347713094186717178607522981373539126772812593958220524289991371690685650421575056729991274177149279608831502358697816190894908487717722503860872618384947939757440664912760518878124233683125467278331513186758915668300679210215947336858591201395360301678110413444411030903388761520488296909104689167671555373346622545575975202624771242796225983278405833585897671474205724047439720232895903726148688388003174146490203843590358527993123871042845981608996101945691646983837718267264685264869172948414153004604004299585035164101899027529366867431834955447458124140190754681607770977920579383895378192128847409929537040546962226547278807248685508046571043123854873351653070570784584243335550958221912862797205455466267099131902370311779690892786623112661337671178512943059323281605826535623848164192144732543731002062738466812351691016359252588256806438946389880872735284406462208149513862275239938938734905082625472417781702582044129853760499827899020083498387362992498125742354568439023012261733665820546785671147973065077035475620567428300187473019197310881157516777005071432012726354601912460800451608108641835539669946936947322271670748972850464195392966434725254724357659192969949061670189061433616907056148280980363243454128229968275980226694045642181328624517549652147221620839824594576613342710564957193564431561774500828376935700995419541839029151033187933907614207467028867968594985439789457300768939890070073924697461812855764662265412913204052279071212820653775058280040897163467163709024906774736309136904002615646432159560910851092445162454420141442641660181385990017417408244245378610158433361777292580611159192008414091888191208858207627011483671760749046980914443057262211104583300789331698191603917150622792986282709446275915009683226345073725451366858172483498470080840163868209726371345205439802277866337293290829914010645589761697455978409211409167684020269370229231743334499986901841510888993165125090001163719114994852024821586396216294981753094623047604832399379391002142532996476235163569009445086058091202459904612118623318278614464727795523218635916551883057930657703331498510068357135624341881884405780028844018129031378653794869614630467726914552953690154167025838032477842272417994513653582260971652588356712133519546838335349801503269359798167463231847628306340588324731228951257944267639877946713121042763380872695738609314631539148548792514028885025189788076023838995615684850391995855029256054176767663145354058496296796781349420116003325874431438746248313850214980401681940795687219268462617287403480967931949965604299190281810597603263251746405016454606266765529010639868703668263299050577706266397868453584384057673298268163448646707439990917504018892319267557518354054956017732907127219134577524905771512773358423314008356080926962298894163047287780054743798498545562870729968407382937218623831766524716090967192007237658894226186550487552614557855898773008703234726418384831040394818743616224455286163287628541175946460497027724490799275146445792982549802258601001772437840167723166802004162547244179415547810554178036773553354467030326469619447560812831933095679685582771932031205941616693902049665352189672822671972640029493307384717544753761937017882976382487233361813499414541694736549254840633793674361541081593464960431603544354737728802361047743115330785159902977771499610274627769759612488879448609863349422852847651310277926279743981957617505591300993377368240510902583759345170015340522266144077237050890044496613295859536020556034009492820943862994618834790932894161098856594954213114335608810239423706087108026465913203560121875933791639666437282836752328391688865373751335794859860107569374889645657187292540448508624449947816273842517229343960137212406286783636675845331904743954740664015260871940915743955282773904303868772728262065663129387459875317749973799293043294371763801856280061141619563942414312254397099163565102848315765427037906837175764870230052388197498746636856292655058222887713221781440489538099681072143012394693530931524054081215705402274414521876541901428386744260011889041724570537470755550581632831687247110220353727166112304857340460879272501694701067831178927095527253222125224361673343366384756590949728221809418684074238351567868893421148203905824224324264643630201441787982022116248471657468291146315407563770222740135841109076078464780070182766336227978104546331131294044833570134869585165267459515187680033395522410548181767867772152798270250117195816577603549732923724732067853690257536233971216884390878879262188202305529937132397194333083536231248870386416194361506529551267334207198502259771408638122015980894363561808597010080081622557455039101321981979045520049618583777721048046635533806616517023595097133203631578945644487800945620369784973459902004606886572701865867757842758530645706617127194967371083950603267501532435909029491516973738110897934782297684100117657987098185725131372267749706609250481876835516003714638685918913011736805218743265426063700710595364425062760458252336880552521181566417553430681181548267844169315284408461087588214317641649835663127518728182948655658524206852221830755306118393326934164459415342651778653397980580828158806300749952897558204686612590853678738603318442905510689778698417735603118111677563872589911516803236547002987989628986181014596471307916144369564690909518788574398821730583884980809523077569358851616027719521488998358632323127308909861560777386006984035267826785387215920936255817889813416247486456433211043194821421299793188104636399541496539441501383868748384870224681829391860319598667962363489309283087840712400431022706137591368056518861313458307990705003607588327248867879324093380071864152853317943535073401891193638546730000660453783784472469288830546979000131248952100446949032058838294923613919284305249167833012980192255157050378521810552961623637523647962685751660066539364142273063001648652613891842243501797455993616794063303522111829071597538821839777552812981538570168702202620274678647916644030729018445497956399844836807851997088201407769199261674991148329821854382718946282165387064858588646221611410343570342878862979083418871606214430014533275029715104673156021000043869510583773779766003460887624861640938645252177935289947578496255243925598620521409052346250847830487046492688313289470553891357290706967599556298586669559721686506052072801342104355762779184021797626656484580261591407173477009039475168017709900129391137881248534255949312866653465033728846390649968460644741907524313323903404908195233044389559060547854954620263256676813262435925020249516275607080900436460421497025691488555265022810327762115842282433269528629137662675481993546118143913367579700141255870143319434764035725376914388899683088262844616425575034001428982557620386364384137906519612917777354183694676232982904981261717676191554292570438432239918482261744350470199171258214687683172646078959690569981353264435973965173473319484798758064137926885413552523275720457329477215706850016950046959758389373527538622664943456437071610511521617176237598050900553232154896062817794302268640579555845730600598376482703339859420098582351400179507104569019191359062304102336798080907240196312675268916362136351032648077232914950859151265812143823371072949148088472355286394195993455684156344577951727033374238129903260198160571971183950662758220321837136059718025940870615534713104482272716848395524105913605919812444978458110854511231668173534838253724825347636777581712867205865148285317273569069839935110763432091319780314031658897379628301178409806410175016511072932907832177487566289310650383806093372841399226733384778203302020700517188941706465146238366720632742644336612174011766914919235570905644803016342294301837655263108450172510307540942604409687066288066265900569082451407632599158164499361455172452057020443093722305550217222299706209749268609762787409626448772056043078634808885709143464793241536214303199965695610753570417207285334250171325558818113295504095217830139465216436594262960768570585698507157151317262928960072587601564840556088613165411835958628710665496282599535127193244635791046554389165150954187306071015034430609582302257455974944275067630926322529966338219395202927917973247094559691016402983683080426309910481567503623509654924302589575273521412445149542462972258510120707802110188106722347972579330653187713438466713807546383471635428854957610942841898601794658721444495198801550804042506452191484989920400007310672369944655246020908767882300064337725657385010969899058191290957079866699453765080407917852438222041070599278889267745752084287526377986730360561230710723922581504781379172731261234878334034473833573601973235946604273704635201327182592410906040097638585857716958419563109577748529579836844756803121874818202833941887076311731615289811756429711334181497218078040465077657204457082859417475114926179367379999220181789399433337731146911970737861041963986422166045588965683206701337505745038872111332436739840284188639147633491695114032583475841514170325690161784931455706904169858050217798497637014758914810543205854914100662201721719726878930012101267481270235940855162601689425111458499658315589660460091525797881670384625905383256920520425791378948827579603278877535466861441826827797651258953563761485994485049706638406266121957141911063246061774180577212381659872472432252969098533628440799030007594546281549235506086481557928961969617060715201589825299772803520002610888814176506636216905928021516429198484077446143617891415191517976537848282687018750030264867608433204658525470555882410254654806040437372771834769014720664234434374255514129178503032471263418076525187802925534774001104853996960549926508093910691337614841834884596365621526610332239417467064368340504749943339802285610313083038484571294767389856293937641914407036507544622061186499127249643799875806537850203753189972618014404667793050140301580709266213229273649718653952866567538572115133606114457222800851183757899219543063413692302293139751143702404830227357629039911794499248480915071002444078482866598579406525539141041497342780203520135419925977628178182825372022920108186449448349255421793982723279357095828748597126780783134286180750497175747373730296280477376908932558914598141724852658299510882230055223242218586191394795184220131553319634363922684259164168669438122537135960710031743651959027712571604588486044820674410935215327906816032054215967959066411120187618531256710150212239401285668608469435937408158536481912528004920724042172170913983123118054043277015835629513656274610248827706488865037765175678806872498861657094846665770674577000207144332525555736557083150320019082992096545498737419756608619533492312940263904930982014700371161829485939931199955070455381196711289367735249958182011774799788636393286405807810818657337668157893827656450642917396685579555053188715314552353070355994740186225988149854660737787698781542360397080977412361518245964026869979609564523828584235953564615185448165799966460648261396618720304839119560250381111550938420209894591555760083897989949964566262540514195610780090298667014635238532066032574466820259430618801773091109212741138269148784355679352572808875543164693077235363768226036080174040660997151176880434927489197133087822951123746632635635328517394189466510943745768270782209928468034684157443127739811044186762032954475468077511126663685479944460934809992951875666499902261686019672053749149951226823637895865245462813439289338365156536992413109638102559114643923805213907862893561660998836479175633176725856523591069520326895990054884753424160586689820067483163174286329119633399132709086065074595260357157323069712106423424081597068328707624437165532750228797802598690981111226558888151520837482450034463046505984569690276166958278982913613535306291331427881888249342136442417833519319786543940201465328083410341785272489879050919932369270996567133507711905899945951923990615156165480300145359212550696405345263823452155999210578191371030188979206408883974767667144727314254467923500524618849237455307575734902707342496298879996942094595961008702501329453325358045689285707241207965919809225550560061971283541270202072583994171175520920820151096509526685113897577150810849443508285458749912943857563115668324566827992991861539009255871716840495663991959154034218364537212023678608655364745175654879318925644085274489190918193411667583563439758886046349413111875241038425467937999203546910411935443113219136068129657568583611774564654674861061988591414805799318725367531243470335482637527081353105570818049642498584646147973467599315946514787025065271083508782350656532331797738656666181652390017664988485456054961300215776115255813396184027067814900350252876823607822107397102339146870159735868589015297010347780503292154014359595298683404657471756232196640515401477953167461726208727304820634652469109953327375561090578378455945469160223687689641425960164689647106348074109928546482353083540132332924864037318003195202317476206537726163717445360549726690601711176761047774971666890152163838974311714180622222345718567941507299526201086205084783127474791909996889937275229053674785020500038630036526218800670926674104806027341997756660029427941090400064654281074454007616429525362460261476180471744322889953285828397762184600967669267581270302806519535452053173536808954589902180783145775891280203970053633193821100095443241244197949192916205234421346395653840781209416214835001155883618421164283992454027590719621537570187067083731012246141362048926555668109467076386536083015847614512581588569610030337081197058344452874666198891534664244887911940711423940115986970795745946337170243268484864632018986352827092313047089215684758207753034387689978702323438584381125011714013265769320554911860153519551654627941175593967947958810333935413289702528893533748106257875620364294270257512121137330213811951395756419122685155962476203282038726342066227347868223036522019655729325905068134849292299647248229359787842720945578267329975853818536442370617353517653060396801087899490506654491544577952166038552398013798104340564182403396162494910454712104839439200945914647542424785991096900046541371091630096785951563947332190934511838669964622788855817353221326876634958059123761251203010983867841195725887799206041260049865895027247133146763722204388398558347770112599424691208308595666787531942465131444389971195968105937957532155524204659410081418351120174196853432672343271868099625045432475688702055341969199545300952644398446384346598830418262932239295612610045884644244285011551557765935780379565026806130721758672048541797157896401554276881090475899564605488362989140226580026134158039480357971019004151547655018391755772677897148793477372747525743898158705040701968215101218826088040084551332795162841280679678965570163917067779841529149397403158167896865448841319046368332179115059107813898261026271979696826411179918656038993895418928488851750122504754778999508544083983800725431468842988412616042682248823097788556495765424017114510393927980290997604904428832198976751320535115230545666467143795931915272680278210241540629795828828466355623580986725638200565215519951793551069127710538552661926903526081367717666435071213453983711357500975854405939558661737828297120544693182260401670308530911657973113259516101749193468250063285777004686987177255226525708428745733039859744230639751837209975339055095883623642814493247460522424051972825153787541962759327436278819283740253185668545040893929401040561666867664402868211607294830305236465560955351079987185041352121321534713770667681396211443891632403235741573773787908838267618458756361026435182951815392455211729022985278518025598478407179607904114472041476091765804302984501746867981277584971731733287305281134969591668387877072315968334322509070204019030503595891994666652037530271923764252552910347950343816357721698115464329245608951158732012675424975710520894362639501382962152214033621065422821876739580121286442788547491928976959315766891987305176388698461503354594898541849550251690616888419122873385522699976822609645007504500096116866129171093180282355042553653997166054753907348915189650027442328981181709248273610863801576007240601649547082331349361582435128299050405405333992577071321011503713898695076713447940748097845416328110406350804863393555238405735580863718763530261867971725608155328716436111474875107033512913923595452951407437943144900950809932872153235195999616750297532475931909938012968640379783553559071355708369947311923538531051736669154087312467233440702525006918026747725078958903448856673081487299464807786497709361969389290891718228134002845552513917355978456150353144603409441211512001738697261466786933733154341007587514908295822756919350542184106448264951943804240543255345965248373785310657979037977505031436474651422484768831323479762673689855474944277949916560108528257618964374464656819789319422077536824661110427671936481836360534108748971066866318805026555929568123959680449295166615409802610781691689418764353363449482900125929366840591370059526914934421861891742142561071896846626335874414976973921566392767687720145153302241853125308442727245771161505550519076276250016522166274796257424425420546785767478190959486500575711016264847833741198041625940813327229905891486422127968042984725356237202887830051788539737909455265135144073130049869453403245984236934627060242579432563660640597549471239092372458126154582526667304702319359866523378856244229188278436440434628094888288712101968642736370461639297485616780079779959696843367730352483047478240669928277140069031660709951473154191919911453182543906294573298686613524886500574780251977607442660798300291573030523199052185718628543687577860915726925232573171665625274275808460620177046433101212443409281314659760221360416223031167750085960128475289259463348312408766740128170543067985261868949895004918275008304998926472034986965363326210919830621495095877228260815566702155693484634079776879525038204442326697479264829899016938511552124688935873289878336267819361764023681714606495185508780596635354698788205094762016350757090024201498400967867845405354130050482404996646978558002628931826518708714613909521454987992300431779500489569529280112698632533646737179519363094399609176354568799002814515169743717518330632232942199132137614506411391269837128970829395360832883050256072727563548374205497856659895469089938558918441085605111510354367477810778500572718180809661542709143010161515013086522842238721618109043183163796046431523184434669799904865336375319295967726080853457652274714047941973192220960296582500937408249714373040087376988068797038047223488825819819025644086847749767508999164153502160223967816357097637814023962825054332801828798160046910336602415904504637333597488119998663995617171089911809851197616486499233594328274275983382931099806461605360243604040848379619072542165869409486682092396143083817303621520642297839982533698027039931804024928814430649614747600087654305571672697259114631990688823893005380061568007730984416061355843701277573463708822073792921409548717956947854414951731561828176343929570234710460088230637509877521391223419548471196982303169544468045517922669260631327498272520906329003279972932906827204647650366969765227673645419031639887433042226322021325368176044169612053532174352764937901877252263626883107879345194133825996368795020985033021472307603375442346871647223795507794130304865403488955400210765171630884759704098331306109510294140865574071074640401937347718815339902047036749084359309086354777210564861918603858715882024476138160390378532660185842568914109194464566162667753712365992832481865739251429498555141512136758288423285957759412684479036912662015308418041737698963759002546999454131659341985624780714434977201991702665380714107259910648709897259362243300706760476097690456341576573395549588448948093604077155688747288451838106069038026528318275560395905381507241627615047252487759578650784894547389096573312763852962664517004459626327934637721151028545472312880039058405918498833810711366073657536918428084655898982349219315205257478363855266205400703561310260405145079325925798227406012199249391735122145336707913500607486561657301854049217477162051678486507913573336334257685988361252720250944019430674728667983441293018131344299088234006652915385763779110955708000600143579956351811596764725075668367726052352939773016348235753572874236648294604770429166438403558846422370760111774821079625901180265548868995181239470625954254584491340203400196442965370643088660925268811549596291166168612036195319253262662271108142149856132646467211954801142455133946382385908540917878668826947602781853283155445565265933912487885639504644196022475186011405239187543742526581685003052301877096152411653980646785444273124462179491306502631062903402737260479940181929954454297256377507172705659271779285537195547433852182309492703218343678206382655341157162788603990157495208065443409462446634653253581574814022471260618973060860559065082163068709634119751925774318683671722139063093061019303182326666420628155129647685313861018672921889347039342072245556791239578260248978371473556820782675452142687314252252601795889759116238720807580527221031327444754083319215135934526961397220564699247718289310588394769170851420631557192703636345039529604362885088555160008371973526383838996789184600327073682083234847108471706160879195227388252347506380811606090840124222431476103563328940609282430125462013806032608121942876847907192546246309055749298781661271916548229644317263587524548607563020667656942355342774617635549231817456159185668061686428714964129290560130053913469569829490891003991259088290348791943368696942620662946948514931472688923571615032405542263391673583102728579723061998175868700492227418629077079508809336215346303842967525604369606110193842723883107587771653594778681499030978765900869583480043137176832954871752604714113064847270887246697164585218774442100900090916189819413456305028950484575822161887397443918833085509908566008543102796375247476265353031558684515120283396640547496946343986288291957510384781539068343717740714095628337554413567955424664601335663617305811711646062717854078898495334329100315985673932305693426085376230981047171826940937686754301837015557540822371538037838383342702379535934403549452173960327095407712107332936507766465603712364707109272580867897181182493799540477008369348889220963814281561595610931815183701135104790176383595168144627670903450457460997444500166918675661035889313483800512736411157304599205955471122443903196476642761038164285918037488354360663299436899730090925177601162043761411616688128178292382311221745850238080733727204908880095181889576314103157447684338100457385008523652069340710078955916549813037292944462306371284357984809871964143085146878525033128989319500645722582281175483887671061073178169281242483613796475692482076321356427357261609825142445262515952514875273805633150964052552659776922077806644338105562443538136258941809788015677378951310313157361136026047890761945591820289365770116416881703644242694283057457471567494391573593353763114830246668754727566653059819746822346578699972291792416156043557665183382167059157867799311835820189855730344883681934418305987021880502259192818047775223884407167894780414701414651073580452021499197980812095692195622632313741870979731320870864552236740416185590793816745658234353037283309503729022429802768451559528656923189798000383061378732434546500582722712325031420712488100290697226311129067629080951145758060270806092801504406139446350643069742785469477459876821004441453438033759717384777232052065301037861326418823586036569054773343070911759152582503029410738914441818378779490613137536794654893375260322906277631983337976816641721083140551864133302224787118511817036598365960493964571491686005656771360533192423185262166760222073368844844409234470948568027905894191829969467724456269443308241243846160408284006424867072583661011433404214473683453638496544701067827313169538435919120440283949541956874453676459875488726170687163109591315801609722382049772577307454562979127906177531663252857205858766376754282917933549923678212008601904369428956102301731743150352204665675088491593025926618816581008701658499456495586855628208747248318351516339189292646558880593601275151838235485893426165223086697314511412035659916934103076974774451947043836739600076578628245472064617380804602903639144493859012422380173377038154675297645596518492676039300171943042511794045679862114630138402371099347243455794730048929825402680821621522346560274258486595687074510352794291633405915025075992398611224340312056999780516223878772230396359709132856830486160362127579561601328561866388146004722200580017580282279272167842720649966956840905752590774886105493806116954293569077377792821084159737469613143291808510446953973485067590503662391722108732333169909603363771705474725026941732982890400239372879549386540463828596742216318201530139629734398479588628632934746650690284066719018081265539973675916799759010867483920062877888531102781695087545740384607594616919584610655963327283485609570305572502494416337066573150237126843581984154103154401008430380631442183776750349813408169325201240813452285974626715177152223063741359255747513535160669108359443999692315898156732033027129284241219651936303734407981204656795322986357374589031654007016472204989445629050395873788912680565516464274460174738175296313458739390484560414203426465560422112239134631023161290836446988901247285192778589195228773637440432659264672239982186452797664826673070168802722052338600372842903155828454593854349099449420750911108532138744823216151007808922516285123275724355101999038195993350032641446053470357293073912578481757987468353429629749652545426864234949270336399427519354240001973125098882419600095766257217621860474573769577649582201796258392376391717855799468922496750179251915218219624653575570564228220399546682648329822996167217080156801080799777126517156274295763666959661983507435667132218383358509536665806605597148376773866922551603463644386269977295750658468929599809168949981898588529537874489519527097766262684177088590284321676352132630838812766335363319004134332844347630067982023716933653652880580156390360562722752187272454764258840995216482554453662083811789117725225682611478014242896970967121967502094421226279437073328703410646312100557376727450271638975234111426287828736758358819056742163061523416789476056879277154789714326222041069587947186435439940738639948986836168919377836648327137363654676901173760246643082285362494712605173293777247276797635865806019396287718060679122426813922872134061694882029506831654589707623668302556167559477498715183426989208952182644710514911419441192277010977616645850068963849426165593473112961064282379048216056210094265076173838082479030510998790719611852832556787472942907151041468948104916751035295897242381802288151276582257190705537652455285511598636421244284176256230139538669970308943645907600684938040875210854159851278070333207779865635907968462191534944587677170063778573171211036517486371634098385626541555573292664616402279791195975248525300376741774056125700303625811704838385391207273191845064713669122576415213769896260940351804147432053600369234179035440735703058314741623452840188940808983125191307741823338981880316339159565954543405777784331681162551898060409183018907512170192983622897099598983405484962284289398469847938668614293324543983592637036699355184231661615244505980576745765335552338715678211466689996845227042954589710922163652573965950289645637766038988037941517917867910675199009966139206238732318786758420544279396366759104126821843375015743069045967947046685602358283919759975285865384338189120042853787549302768972168199113340697282255535300044743958830079799736518459131437946494086272149669719100359399974735262764126125995350902609540048669398955899487421379590802893196914845826873123710180229775301190684280440780938156598081694611679374425663244656799606363751546304833112722231812338371779800439731087402647536582575657351059978314264831879619843765495877803685261751835391844920488198629786329743136948511780579298636452193232481339393090754566368038513630619718033957979522539508697432546502659123585049283028832934489284591373621624852528877442891851104093746333590660233239711922814450735588373324057814862662207486215513375036775585494138678352928273109003823116855374520901095101174796663003330352534143230024288248051396631446632656081582045216883922312025671065388459503224002320453633895521539919011035217362720909565500846486605368975498478995875596103167696587161281951919668893326641203784750417081752273735270989343717167642329956935697166213782736138899530515711822960896394055380431939398453970864418654291655853168697537052760701061488025700785387150835779480952313152747735711713643356413242974208137266896149109564214803567792270566625834289773407718710649866150447478726164249976671481383053947984958938064202886667951943482750168192023591633247099185942520392818083953020434979919361853380201407072481627304313418985942503858404365993281651941497377286729589582881907490040331593436076189609669494800067194371424058105327517721952474344983414191979918179909864631583246021516575531754156198940698289315745851842783390581029411600498699307751428513021286202539508732388779357409781288187000829944831476678183644656510024467827445695591845768068704978044824105799710771577579093525803824227377612436908709875189149049904225568041463131309240101049368241449253427992201346380538342369643767428862595140146178201810734100565466708236854312816339049676558789901487477972479202502227218169405159042170892104287552188658308608452708423928652597536146290037780167001654671681605343292907573031466562485809639550080023347676187068086526878722783177420214068980703410506200235273632267291964034093571225623659496432076928058165514428643204955256838543079254299909353199329432966018220787933122323225928276556048763399988478426451731890365879756498207607478270258861409976050788036706732268192473513646356758611212953074644777149423343867876705824452296605797007134458987594126654609414211447540007211790607458330686866231309155780005966522736183536340439991445294960728379007338249976020630448806064574892740547730693971337007962746135534442514745423654662752252624869916077111131569725392943756732215758704952417232428206555322808868670153681482911738542735797154157943689491063759749151524510096986573825654899585216747260540468342338610760823605782941948009334370046866568258579827323875158302566720152604684361412652956519894291184887986819088277339147282063794512260294515707367105637720023427811802621502691790400488001808901847311751199425460594416773315777951735444490965752131026306836047140331442314298077895617051256930051804287472368435536402764392777908638966566390166776625678575354239947427919442544664643315554138265543388487778859972063679660692327601733858843763144148113561693030468420017434061395220072403658812798249143261731617813894970955038369479594617979829257740992171922783223006387384996138434398468502234780438733784470928703890536420557474836284616809363650973790900204118525835525201575239280826462555785658190226958376345342663420946214426672453987171047721482128157607275305173330963455909323664528978019175132987747952929099598069790148515839540444283988381797511245355548426126784217797728268989735007954505834273726937288386902125284843370917479603207479554080911491866208687184899550445210616155437083299502854903659617362726552868081324793106686855857401668022408227992433394360936223390321499357262507480617409173636062365464458476384647869520547719533384203403990244761056010612777546471464177412625548519830144627405538601855708359981544891286863480720710061787059669365218674805943569985859699554089329219507269337550235821561424994538234781138316591662683103065194730233419384164076823699357668723462219641322516076261161976034708844046473083172682611277723613381938490606534404043904909864126903479263503943531836741051762565704797064478004684323069430241749029731181951132935746854550484711078742905499870600373983113761544808189067620753424526993443755719446665453524088287267537759197074526286322840219629557247932987132852479994638938924943286917770190128914220188747760484939855471168524810559991574441551507431214406120333762869533792439547155394213121021954430556748370425907553004950664994802614794524739012802842646689229455664958621308118913500279654910344806150170407268010067948926855360944990373928383520627992820181576427054962997401900837493444950600754365525758905546552402103412862124809003162941975876195941956592556732874237856112669741771367104424821916671499611728903944393665340294226514575682907490402153401026923964977275904729573320027982816062130523130658731513076913832317193626664465502290735017347656293033318520949298475227462534564256702254695786484819977513326393221579478212493307051107367474918016345667888810782101151826314878755138027101379868751299375133303843885631415175908928986956197561123025310875057188962535763225834275763348421016668109884514141469311719314272028007223449941999003964948245457520704922091620614222912795322688239046498239081592961111003756999529251250673688233852648213896986384052437049402152187547825163347082430303521036927849762517317825860862215614519165573478940019558704784741658847364803865995119651409542615026615147651220820245816010801218275982577477652393859159165067449846149161165153821266726927461290533753163055654440793427876550267301214578324885948736899073512166118397877342715872870912311383472485146035661382188014840560716074652441118841800734067898587159273982452147328317214621907330492060817440914125388918087968538960627860118193099489240811702350413554126823863744341209267781729790694714759018264824761112414556423937732224538665992861551475342773370683344173073150805440138894084087253197595538897613986400165639906934600670780501058567196636796167140097031535132386972899001749862948883362389858632127176571330142071330179992326381982094042993377790345261665892577931395405145369730429462079488033141099249907113241694504241391265397274078984953073730364134893688060340009640631540701820289244667315059736321311926231179142794944897281477264038321021720718017561601025111179022163703476297572233435788863537030535008357679180120653016668316780269873860755423748298548246360981608957670421903145684942967286646362305101773132268579232832164818921732941553151386988781837232271364011755881332524294135348699384658137175857614330952147617551708342432434174779579226338663454959438736807839569911987059388085500837507984051126658973018149321061950769007587519836861526164087252594820126991923916722273718430385263107266000047367872474915828601694439920041571102706081507270147619679971490141639274282889578424398001497985658130305740620028554097382687819891158955487586486645709231721825870342960508203415938806006561845735081804032347750084214100574577342802985404049555529215986404933246481040773076611691605586804857302606467764258503301836174306413323887707999698641372275526317649662882467901094531117120243890323410259937511584651917675138077575448307953064925086002835629697045016137935696266759775923436166369375035368699454550392874449940328328128905560530091416446608691247256021455381248285307613556149618444364923014290938289373215312818797541139219415606631622784836152140668972661027123715779503062132916001988806369127647416567067485490795342762338253943990022498972883660263920518704790601584084302914787302246651371144395418253441269003331181914268070735159284180415100555199146564934872796969351992963117195821262627236458009708099166752820365818699111948365866102758375863322993225541477479210421324166848264953111826527351008031659958888814809945737293785681411438021523876706455063233067233939551964260397443829874822322662036352861302543796600943104500158604854027036789711934695579989189112302233381602302236277726084846296189550730850698061500281436425336666311433321645213882557346329366870956708432252564333895997812402164189946978348320376011613913855499933990786652305860332060641949298931012423081105800169745975038516887112037747631577311831360002742502722451570906304496369230938382329175076469684003556425503797106891999812319602533733677437970687713814747552190142928586781724044248049323750330957002929126630316970587409214456472022710796484778657310660832173093768033821742156446602190335203981531618935787083561603302255162155107179460621892674335641960083663483835896703409115513087820138723494714321400450513941428998350576038799343355677628023346565854351219361896876831439866735726040869511136649881229957801618882834124004126142251475184552502502640896823664946401177803776799157180146386554733265278569418005501363433953502870836220605121839418516239153709790768084909674194289061134979961034672077354959593868862427986411437928435620575955500144308051267664432183688321434583708549082240014585748228606859593502657405750939203135881722442164955416889785558265198046245527898343289578416968890756237467281044803018524217706136533236073856228166664597654076844715963930782091017090763377917711485205493367936868430832404126789220929930411890501756484917499452393770674524578019171841679541825554377930299249277892416277257788147974770446005423669346157135208417428211847353652367573702352791459837645712257646122605628127852169580892808988394594406165340521932514843306105322700231133680378433377389724881307874325614952744243584753011150345103737688223837573804282007358586938044331529253129961025096113761670187568525921208929131354473196308440066835155160913925692912175784379179004808848023029304392630921342768601226558630456913133560978156776098711809238440656353136182676923761613389237802972720736243967239854144480757286813436768000573823963610796223140429490728058551444771338682314499547929338131259971996894072233847404542592316639781608209399269744676323921370773991899853301483814622364299493902073285072098040905300059160091641710175605409814301906444379905831277826625762288108104414704097708248077905168225857235732665234414956169007985520848841886027352780861218049418060017941147110410688703738674378147161236141950474056521041002268987858525470689031657094677131822113205505046579701869337769278257145248837213394613987859786320048011792814546859096532616616068403160077901584946840224344163938313618742275417712170336151163782359059685168880561304838542087505126933144171705880517278127917564053282929427357971823360842784676292324980318169828654166132873909074116734612367109059236155113860447246378721244612580406931724769152219217409096880209008801535633471775664392125733993165330324425899852598966724744126503608416484160724482125980550754851232313331300621490042708542735985913041306918279258584509440150719217604794274047740253314305451367710311947544521321732225875550489799267468541529538871443696399406391099267018219539890685186755868574434469213792094590683677929528246795437302263472495359466300235998990248299853826140395410812427393530207575128774273992824866921285637240069184859771126480352376025469714309316636539718514623865421671429236191647402172547787238964043145364190541101514371773797752463632741619269990461595895793940622986041489302535678633503526382069821487003578061101552210224486633247184367035502326672749787730470216165019711937442505629639916559369593557640005236360445141148916155147776301876302136068825296274460238077523189646894043033182148655637014692476427395401909403584437251915352134557610698046469739424511797999048754951422010043090235713636892619493763602673645872492900162675597083797995647487354531686531900176427222751039446099641439322672532108666047912598938351926694497553568096931962642014042788365702610390456105151611792018698900673027082384103280213487456720062839744828713298223957579105420819286308176631987048287388639069922461848323992902685392499812367091421613488781501234093387999776097433615750910992585468475923085725368613605356762146929424264323906626708602846163376051573599050869800314239735368928435294958099434465414316189806451480849292695749412903363373410480943579407321266012450796613789442208485840536446021616517885568969302685188950832476793300404851688934411125834396590422211152736276278672366665845757559585409486248261694480201791748223085835007862255216359325125768382924978090431102048708975715033330963651576804501966025215527080352103848176167004443740572131294252820989545456276344353575741673638980108310579931697917916718271145837435222026387771805250290791645414791173616253155840768495583288190293564201219633684854080865928095131505012602919562576032932512847250469881908146475324342363863860247943921015193235101390117789997483527186469346024554247028375300033725403910085997650987642832802908445662021678362267272292737780213652404028817217012490974899454430826861772239385250883760749742195942655217301733355851389407457348144161511380845358039740277795072051893487170722955427683655826706766313911972211811528466502223383490906676554168336907959409404576472940901354356409277969379842065738891481990225399022315913388145851487225126560927576795873759207013915029216513720851137197522734365458411622066281660256333632074449918511469174455062297146086578736313585389023662557285424516018080487167823688885575325066254262367702604215835160174851981885460860036597606743233346410471991027562358645341748631726556391320606407754779439671383653877377610828300019937359760370467245737880967939894493795829602910746901609451288456550071458091887879542641820145369659962842686882363495879277007025298960996798975941955735253914237782443302746708282008722602053415292735847582937522487377937899136764642153727843553986244015856488692101644781661602962113570056638347990334049623875941092886778920270077504951511405782565295015024484968204744379710872943108541684540513016310902267112951959140520827546866418137305837933236150599142045255880213558474751516267815309465541240524091663857551298894834797423322854504140527354235070335984964593699534959698554244978249586929179182415068053002553370412778703476446244329205906832901886692400222391918714603175399666877477960121790688623311002908668305431787009355066944389131913333586368037447530664502418437136030852288582121720231274167009740351431532131803978033680228154223490183737494117973254478594157962104378787072154814091725163615415163381388912588517924237727229603497305533840942889918919161186249580560073570527227874940321250645426206304469470804277945973817146810395192821550688079136701210109944220737024613687196031491162370967939354636396448139025711768057799751751298979667073292674886430097398814873780767363792886767781170520534367705731566895899181530825761606591843760505051704242093231358724816618683821026679970982966436224723644898648976857100173643547336955619347638598187756855912376232580849341570570863450733443976604780386678461711520325115528237161469200634713570383377229877321365028868868859434051205798386937002783312365427450532283462669786446920780944052138528653384627970748017872477988461146015077617116261800781557915472305214759943058006652042710117125674185860274188801377931279938153727692612114066810156521441903567333926116697140453812010040811760123270513163743154487571768761575554916236601762880220601068655524141619314312671535587154866747899398685510873576261006923021359580838145290642217792987748784161516349497309700794368305080955621264592795333690631936594413261117944256602433064619312002953123619348034504503004315096798588111896950537335671086336886944665564112662287921812114121425167348136472449021275252555647623248505638391391630760976364990288930588053406631352470996993362568102360392264043588787550723319888417590521211390376609272658409023873553418516426444865247805763826160023858280693148922231457758783791564902227590699346481624734399733206013058796068136378152964615963260698744961105368384203105364183675373594176373955988088591188920114871545460924735613515979992999722298041707112256996310945945097765566409972722824015293663094891067963296735505830412258608050740410916678539569261234499102819759563955711753011823480304181029089719655278245770283085321733741593938595853203645590564229716679900322284081259569032886928291260139267587858284765599075828016611120063145411315144108875767081854894287737618991537664505164279985451077400771946398046265077776614053524831090497899859510873112620613018757108643735744708366215377470972660188656210681516328000908086198554303597948479869789466434027029290899143432223920333487108261968698934611177160561910681226015874410833093070377506876977485840324132474643763087889666151972556180371472590029550718424245405129246729039791532535999005557334600111693557020225722442772950263840538309433999383388018839553821540371447394465152512354603526742382254148328248990134023054550811390236768038649723899924257800315803725555410178461863478690646045865826036072306952576113184134225274786464852363324759102670562466350802553058142201552282050989197818420425028259521880098846231828512448393059455162005455907776121981297954040150653985341579053629101777939776957892084510979265382905626736402636703151957650493344879513766262192237185642999150828898080904189181015450813145034385734032579549707819385285699926238835221520814478940626889936085239827537174490903769904145555260249190126341431327373827075950390882531223536876389814182564965563294518709637484074360669912550026080424160562533591856230955376566866124027875883101021495284600804805028045254063691285010599912421270508133194975917146762267305044225075915290251742774636494555052325186322411388406191257012917881384181566918237215400893603475101448554254698937834239606460813666829750019379115061709452680984785152862123171377897417492087541064556959508967969794980679770961683057941674310519254486327358885118436597143583348756027405400165571178309126113117314169066606067613797690123141099672013123730329707678988740099317309687380126740538923612230370779727025191340850390101739924877352408881040807749924412635346413181858792480760553268122881584307471326768283097203149049868884456187976015468233715478415429742230166504759393312132256510189175368566338139736836336126010908419590215582111816677413843969205870515074254852744810154541079359513596653630049188769523677579147319184225806802539818418929888943038224766186405856591859943091324575886587044653095332668532261321209825839180538360814144791320319699276037194760191286674308615217243049852806380129834255379486287824758850820609389214668693729881191560115633701248675404205911464930888219050248857645752083363921499441937170268576222251074166230901665867067714568862793343153513505688216165112807318529333124070912343832502302341169501745502360505475824093175657701604884577017762183184615567978427541088499501610912720817913532406784267161792013428902861583277304794830971705537485109380418091491750245433432217445924133037928381694330975012918544596923388733288616144238100112755828623259628572648121538348900698511503485369544461542161283241700533583180520082915722904696365553178152398468725451306350506984981006205514844020769539324155096762680887603572463913955278222246439122592651921288446961107463586148252820017348957533954255019475442643148903233373926763409115527189768429887783617346613535388507656327107814312435018965109238453660236940276060642119384227665755210663671879603217527184404651560427289869560206997012906367847161654793068868305846508082886614111979138822898112498261434559408961813509226857611474609406147937240008842153535862052780125014270055274468359151840373309373580494342483940467505708347927948338133276237937844629209323999417593374917899786484958148818865149169302451512835579818112344900827168644548306546633975256079615935830821400021951611342337058359111545217293721664061708131602078213341260356852013161345136871600980378712556766143923146458085652084039744217352744813741215277475202259244561520365608268890193913957991844109971588312780020898275935898106482117936157951837937026741451400902833064466209280549839169261068975151083963132117128513257434964510681479694782619701483204392206140109523453209269311762298139422044308117317394338867965739135764377642819353621467837436136161591167926578700137748127848510041447845416464568496606699139509524527949914769441031612575776863713634644477006787131066832417871556281779122339077841275184193161188155887229676749605752053192594847679397486414128879475647133049543555044790277128690095643357913405127375570391806822344718167939329121448449553897728696601037841520390662890781218240141299368590465146519209198605347788576842696538459445700169758422531241268031418456268722581132040056433413524302102739213788415250475704533878002467378571470021087314693254557923134757243640544448132093266582986850659125571745568328831440322798049274104403921761438405750750288608423536966715191668510428001748971774811216784160854454400190449242294333666338347684438072624307319019363571067447363413698467328522605570126450123348367412135721830146848071241856625742852208909104583727386227300781566668914250733456373259567253354316171586533339843321723688126003809020585719930855573100508771533737446465211874481748868710652311198691114058503492239156755462142467550498676710264926176510110766876596258810039163948397811986615585196216487695936398904500383258041054420595482859955239065758108017936807080830518996468540836412752905182813744878769639548306385089756146421874889271294890398025623046812175145502330254086076115859321603465240763923593699949180470780496764486889980902123735780457040380820770357387588525976042434608851075199334470112741787878845674656640471901619633546770714090590826954225196409446319547658653032104723804625249971910690110456227579220926904132753699634145768795242244563973018311291451151322757841320376225862458224784696669785947914981610522628786944136373683125108310682898766123782697506343047263278453719024447970975017396831214493357290791648779915089163278018852504558488782722376705263811803792477835540018117452957747339714012352011459901984753358434861297092928529424139865507522507808919352104173963493428604871342370429572757862549365917805401652536330410692033704691093097588782938291296447890613200063096560747882082122140978472301680600835812336957051454650181292694364578357815608503303392466039553797630836137289498678842851139853615593352782103740733076818433040893624460576706096188294529171362940967592507631348636606011346115980434147450705511490716640635688739020690279453438236930531133440901381392849163507484449076828386687476663619303412376248380175840467851210698290605196112357188811150723607303158506622574566366740720668999061320627793994112805759798332878792144188725498543014546662945079670707688135022230580562225942983096887732856788971494623888272184647618153045844390967248232348259587963698908456664795754200195991919240707615823002328977439748112690476546256873684352229063217889227643289360535947903046811114130586348244566489159211382258867880972564351646404364328416076247766114349880319792230537889671148058968061594279189647401954989466232962162567264739015818692956765601444248501821713300527995551312539849919933907083138030214072556753022600033565715934283182650908979350869698950542635843046765145668997627989606295925119763672907762567862769469947280606094290314917493590511523235698715397127866718077578671910380368991445381484562682604003456798248689847811138328054940490519768008320299631757043011485087384048591850157264392187414592464617404735275250506783992273121600117160338604710710015235631159734711153198198710616109850375758965576728904060387168114313084172893710817412764581206119054145955378853200366615264923610030157044627231777788649806700723598889528747481372190175074700005571108178930354895017924552067329003818814068686247959272205591627902292600592107710510448103392878991286820705448979977319695574374529708195463942431669050083984398993036790655541596099324867822475424361758944371791403787168166189093900243862038610001362193667280872414291108080291896093127526202667881902085595708111853836166128848729527875143202956393295910508349687029060692838441522579419764824996318479414814660898281725690484184326061946254276693688953540732363428302189694947766126078346328490315128061501009539164530614554234923393806214007779256337619373052025699319099789404390847443596972052065999017828537676265683558625452697455260991024576619614037537859594506363227095122489241931813728141668427013096050734578659047904243852086508154491350136491698639048125666610843702294730266721499164849610746803261583352580352858275799038584091667618877199539888680431991650866887781701439663176815592262016991396613153738021294160006906947533431677802632207226265881842757216055461439677336258462997385077307751473833315101468395296411397329672457933540390136107395245686243008096720460995545708974893048753897955544443791303790422346037768729236001386569593952300768091377768847789746299699489949016141866131552200856673695770822720338936659590666350594330040363762591189195691561626122704788696510356062748423100605472091437069471661080277379848576543481249822444235828329813543645124092220896643987201997945619030397327254617823136363375927622656301565813545578319730419339269008282952718252138855126583037630477490625995514925943105307478901043009876580816508144862607975129633326675259272351611791836777128931053144471668835182920514343609292493191180249366051791485330421043899773019267686085347768149502299280938065840007311767895491286098112311307002535600347898600653805084532572431553654422067661352337408211307834360326940015926958459588297845649462271300855594293344520727007718206398887404742186697709349647758173683580193168322111365547392288184271373843690526638607662451284299368435082612881367358536293873792369928837047900484722240370919885912556341130849457067599032002751632513926694249485692320904596897775676762684224768120033279577059394613185252356456291805905295974791266162882381429824622654141067246487216174351317397697122228010100668178786776119825961537643641828573481088089988571570279722274734750248439022607880448075724807701621064670166965100202654371260046641935546165838945950143502160890185703558173661823437491622669077311800121188299737319891006060966841193266075165452741829459541189277264192546108246351931647783837078295218389645376236304858042774417907169146356546201215125418664885396161542055152375000426794253417764590821513675258479774465114750438460596325820468809667795709044645884673847481638045635188183210386594798204376334738389017759714236223057776395541011294523488098341476645559342209402059733452337956309441446698222457026367119493286653989491344225517746402732596722993581333110831711807234044326813737231209669052411856734897392234152750707954137453460386506786693396236535556479102508529284294227710593056660625152290924148057080971159783458351173168204129645967070633303569271821496292272073250126955216172649821895790908865085382490848904421755530946832055636316431893917626269931034289485184392539670922412565933079102365485294162132200251193795272480340133135247014182195618419055761030190199521647459734401211601239235679307823190770288415814605647291481745105388060109787505925537152356112290181284710137917215124667428500061818271276125025241876177485994084521492727902567005925854431027704636911098800554312457229683836980470864041706010966962231877065395275783874454229129966623016408054769705821417128636329650130416501278156397799631957412627634011130135082721772287129164002237230234809031485343677016544959380750634285293053131127965945266651960426350406454862543383772209428482543536823186182982713182489884498260285705690699045790998144649193654563259496570044689011049923939218088155626191834404362264965506449848521612498442375928443642612004256628602157801140467879662339228190804577624109076487087406157070486658398144845855803277997327929143195789110373530019873110486895656281917362036703039179710646309906285483702836118486672219457621775034511770110458001291255925462680537427727378863726783016568351092332280649908459179620305691566806180826586923920561895421631986004793961133953226395999749526798801074576466538377400437463695133685671362553184054638475191646737948743270916620098057717103475575333102702706317395612448413745782734376330101853438497450236265733191742446567787499665000938706441886733491099877926005340862442833450486907338279348425305698737469497333364267191968992849534561045719338665222471536681145666596959735075972188416698767321649331898967182978657974612216573922404856900225324160367805329990925438960169901664189038843548375648056012628830409421321300206164540821986138099462721214327234457806819925823202851398237118926541234460723597174777907172041523181575194793527456442984630888846385381068621715274531612303165705848974316209831401326306699896632888532682145204083110738032052784669279984003137878996525635126885368435559620598057278951754498694219326972133205286374577983487319388899574634252048213337552584571056619586932031563299451502519194559691231437579991138301656117185508816658756751184338145761060365142858427872190232598107834593970738225147111878311540875777560020664124562293239116606733386480367086953749244898068000217666674827426925968686433731916548717750106343608307376281613984107392410037196754833838054369880310983922140260514297591221159148505938770679068701351029862207502287721123345624421024715163941251258954337788492834236361124473822814504596821452253550035968325337489186278678359443979041598043992124889848660795045011701169092519383155609441705397900600291315024253848282782826223304151370929502192196508374714697845805550615914539506437316401173317807741497557116733034632008408954066541694665746735785483133770133628948904397670025863002540635264006601631712883920305576358989492412827022489373848906764385339931878608019223108328847459816417701264089078551777830131616162049792779670521847212730327970738223860581986744668610994383049960437407323195784473254857416239738852016202384784256163512597161783106850156299135559874758848151014815490937380933394074455700842090155903853444962128368313687375166780513082594599771257467939781491953642874321122421579851584491669362551569370916855252644720786527971466476760328471332985501945689772758983450586004316822658631176606237201721007922216410188299330808409384014213759697185976897042759041500946595252763487628135867117352364964121058854934496645898651826545634382851159137631569519895230262881794959971545221250667461174394884433312659432286710965281109501693028351496524082850120190831078678067061851145740970787563117610746428835593915985421673115153096948758378955979586132649569817205284291038172721213138681565524428109871168862743968021885581515367531218374119972919471325465199144188500672036481975944167950887487934416759598361960010994838744709079104099785974656112459851972157558134628546189728615020774374529539536929655449012953097288963767713353842429715394179547179095580120134210175150931491664699052366350233024087218654727629639065723341455005903913890253699317155917179823065162679744711857951506573868504088229934804445549850597823297898617029498418376255258757455303112991914341109413088238114443068843062655305601658801408561023324210300218460588586954418502977463085858496130037238190325162225570729975710727306066072916922978033647048840958711228045188511908718588299514331534128549297173849768523136276076868494780364948299904475715771141080958058141208956059471668626290036145602625334863284986816039463372436667112964460292915746181117789169695839947080954788863503281129626899231110099889317815313946681882028368363373822281414974006917942192888817139116283910295684918233358930813360131488748366464224381776081007739183393749346933644748150564933649323157235306109385796839902153381449126925350768211098738352197507736653475499431740580563099143218212547336281359488317681489194306530426029773885492974570569448783077945878865062970895499843760181694031056909587141386804846359853684034105948341788438963179956468815791937174656705047441528027712541569401365862097760735632832966564135817028088013546326104892768731829917950379944446328158595181380144716817284996793061814177131912099236282922612543236071226270324572637946863533391758737446552006008819975294017572421299723542069630427857950608911113416534893431149175314953530067419744979017235181671568754163484949491289001739377451431928382431183263265079530371177806185851153508809998200482761808307209649636476943066172549186143700971387567940218696710148540307471561091358933165600167252126542502898612259306484105898847129649230941215144563947889999327145875969555737090855150648002321476443037232466147111552578583071024936898814562568786834745518893385181791667579054210421036349316257870476543126790661216644142285017446278477132740595579600648343288827864837043456066966456899746910373987712891593313271266247505582258634928427718355831641593667712218537642376222104779338956378722902509543014182257180331300148113377736941508488867501893156994849838936052666818012783912005801431596441910546663236810148207799356523056490420711364192200177189107935243234322761787712568251126481332974354926568682748715986654943041648468220593921673359485057849622807932422649812705271398407720995707236227009245067665680069149966555737866411877079767754867028786431817941521796178310655030287157272282250812017060713380339641841211253856248920130010782462165136989511064611133562443838185366273563783436921279354709230119655914915800561707258518503167289370411936374780625824298250726464801821523430268081486978164824349353456855843696378384153838051184406043696871666416514036129729992912630842812149152469877429332305214999981829046119471676727503742221367186614654042534463141660649871499001000660041544868437352208483059495953182872280520828676300361091734508632133033647289584176588755345227938480297724485711815574893561311524926772006362198369980664159549388683836411891430443767715498026544959061738265591178545999378510861446014967645550103653971251138583505085112442517772923814396233043724036032603181442991365750246012787514117944901305803452199992701148071712847770301254994886841867572975189214295652512486943983729047410363121899124217339550688778643130750024823361832738729697376598820053895902935486054979802320400472236873557411858132734337978931582039412878989728973298812553514507641535360519462112217000676321611195841029252568536561813138784086477147099724553013170761712163186600291464501378587854802096244703771373587720086738054108140042311418525803293267396324596914044834665722042880679280616029884043400536534009706581694636096660911110968789751801325224478246957913251892122653056085866541115373584912790254654369020869419871125588453729063224423222287139122012248769976837147645598526739225904997885514250047585260297929306159913444898341973583316070107516452301310796620382579278533125161760789984630103493496981494261055367836366022561213767081421091373531780682420175737470287189310207606953355721704357535177461573524838432101571399813798596607129664438314791296359275429627129436142685922138993054980645399144588692472767598544271527788443836760149912897358259961869729756588978741082189422337344547375227693199222635973520722998387368484349176841191020246627479579564349615012657433845758638834735832242535328142047826934473129971189346354502994681747128179298167439644524956655532311649920677163664580318205849626132234652606175413532444702007661807418914040158148560001030119994109595492321434406067634769713089513389171050503856336503545166431774489640061738861761193622676890576955693918707703942304940038440622614449572516631017080642923345170422426679607075404028551182398361531383751432493056398381877995594942545196756559181968690885283434886050828529642437578712929439366177362830136595872723080969468398938676366226456791132977469812675226595621009318322081754694778878755356188335083870248295346078597023609865656376722755704495258739871812593441903785275571333409842450127258596692434317689018966145404453679047136294238156127656824247864736176671770647002431119711090007474065945650315375044177982192306323700872039212085499569681061379189029961178936752146022386905665481382858280449537530160921422195940638787074787991194920898374091788534417523064715030278397979864517336625329511775105559014160459873338186887977858817291976604516353353556047648420520888811722831990044504284486852338334530105533929637308039738230604714104525470094899407601215247602819963846343554852932377161410869591950786873276075400085220065031871239272857835807010762542769655355964789450166013816295177908531139811092831583216931563867459747449584385282701658246192092219529134323496779345585613140207765996142546463288677356891785576835169608392864188830094883324700447958316931533832382377876344426323456301679513671047510469669001217777128065522453689371871451567394733440447280450959433090683667110655953338602938000999949010642769859623260401863733572846679531229683156358145420890540651226419162015504500430562136991850941034609601030543816694795964585804425194905110733387679946734471718615647723811737035654917628707589456035519195603962301157866323750234725054461073979402475184415558178087962822231972692984516683306919505079993357259165675557294585962182052650473353712351623662770479333289322136141858785972771685682725303734836891911847197133753088446777943274857148827821608844765700041403499921376794209627560883081509438030705666022764678117533361028187800710219794428777313146387857817205661409023041499923248268982477222109852189758140879763486146763606368674611966620347304608917277240045953051376938375381543486981101990651706961774052218247422657652138152740612699012706880875386408669901461740890540981877671880076124151967064152117653084325544261017536348281196837493395825742541244634247233586360777980960199745187758845459645895956779558869098404768259253477849930457883128541747079059795909431627722327844578918694214929451540174214623240300841907975296782445969183509474202123617940309048634960534054931299919496087957952586977170236680033862505764938088740994009589948109397983231108838769236490221499111120870639202892490698435333152727991330986335454324971441378059132240814960156485679843966464780280409057580889190254236606774500413415794312112501275232250148067232979652230488493751166084976116412777395311302041566848265531411348993243747890268935173904043294851610659785832253168204202834993641595980197343889883020994152152288611175126686173051956249367180053845637855129171848417841594797435580617856680758491080185805695567990185198397660693358224779136504562705766735170961550493338390452612404395517449136885115987454340932040102218982707539212403241042424451570052968378815749468441508011138612561164102477190903050040240662278945607061512108266146098662040425010583978098192019726759010749924884966139441184159734610382401178556739080566483321039073867083298691078093495828888707110651559651222542929154212923108071159723275797510859911398076844732639426419452063138217862260999160086752446265457028969067192282283045169111363652774517975842147102219099906257373383472726498678244401048998507631630668050267115944636293525120269424810854530602810627264236538250773340575475701704367039596467715959261029438313074897245505729085688496091346323165819468660587092144653716755655531962091865952628448253731353698162517351930115341581171353292035873164168839107994000677266031617527582917398395852606454113318985505747847121053505795649095931672167565624818782002769963734155880000867852567422461511406015760115910256449002264980039498403358091309140197877843650167960167465370287466062584346329708303725980494653589318912163976013193079476972058034710553111117215859219066231028099212084069283091906017370764654655683413207556315315006453462321007133584907633048328153458698497332599801187479664273140279381289961720524540674695271948079930396730194274036466594154400092799908634806622334906695224044652158992864203435098858422692019340575496840904812955522654754650713532842543496616084954788090727649930252702815067862810825243222979985391759845188868387004477101866772159439708514664612871148749531862180941719676843144666435175837688436786081446319641912566574047718699160915550910878919431253671945651261878486910876729910565595155159739659034383628124629118117760949411880105946336671039049777312004243578115790429823045072038322781246413671297959415082918378213212876890545963586369344879749784841123274921331663162812456388238288715648447883142417650147980187858215768793063001153788998014623690135803753306246148576074932567807682651045738059018831237617271889933790487113395588485234240255002352200613574914318259142479829367775490496399350755839668967578364316618369307625603528602940662803255416535431518013714821941772672244005268401996533334184004345525296592918502940131600651124395297874364222806977720437363717873457948420238745151249157913139411148608416429347958793681868609689684640858334131017858142710955416293375915178392341303110543328703526599993904966822112768158316511246866451167351378214345336650598328347443536290312393672084593164394941881138607974670134709640378534907149089842317891739783650654751982883367395714360000003439863363212091718954899055748693397700245632475954504411422582410783866837655467400137324322809113692670682805397549111166171102397437749479335174036135005397581475520834285772800986189401984375446435081498218360112577632447389452051636938585136484259964518361856989088721789764694721246807900330925083496645841656554261294195108847197209106605105540933731954888406444080280579549008076040034154662137669606444293774985897353625591959618552448187940317374508256072895120945456562159540405425814886929842786582357673195799285293120866275922366115137445767916063621675267440451221051052090834707443986137829082352772895849625656881972792768694795806100573787084121444815034797422312103295359297822377134077549545477791813823542607184617108389097825964406170543546968567030745411634244134486308676327949177682923093183221341455482591367202823284396549001805653203960795517074496039006696990334199278212696767771835209083959545341866777944872740383733381985235884202840150981579594685874537989503257362809837592216229258598599123843993575573285028613155970362934249814178056461615863415338635077223269996508860870999964899373049307170967888740149746147542880387421250689212155876692242387434701120990859082164073576380817386959755176083877600277517253037133445654852635661720197563001580049790223419586738061442401502436288957503206533690825756785507020555105572381878574650371086308158185862815883054564662297694803970618265491385181326737485227188267917919091354407852685476254126683398240534022469989966652573155637645862251862823092085424412805997628505488913098331761884983352975136073772030571342739638126588567405013841074788943393996603591853934198416322617654857376671943132840050626295140357877264680649549355746326408186979718630218760025813995719923601345374229758918285167511358171472625828596940798518571870075823122317068134867930884899275181661399609753105295773584618525865211893339375771859916335112163441037910451845019023066893064178977808158101360449495409665363660370075881004450265734935127707426742578608784898185628869980851665713320835842613381142623855420315774246613108873106318111989880289722849790551075148403702290580483052731884959994156606537314021296702220821915862905952604040620011815269664910068587592655660567562963361434230232810747488395040380984981860056164646099819257616235478710913832967563761506732550860683433720438748186791668975746563456020002562889601191100980453350423842063824039434163502977688802779835087481178298349417211674919425601608685332435385951152061809031241698182079314615062073826097180458265687043623935757495737332781578904386011378078508110273049446611821957450170106059384336519458628360682108585130499820420578458577175933849015564447305834515291412561679970569657426139901681932056241927977282026714297258700193234337873153939403115411184101414292741703537542003698760608765500109345299007034032401334806388514095769557147190364152027721127070187421548123931953220997506553022646844227700020589045922742423904937051507367764629844971682121994198274794049092601715727439368569721862936007387077810797440975556627807371228030350048829843919546433753355787895064018998685060281902452191177018634505171087023903398550540704454189088472042376499749035038518949505897971286631644699407490959473411581934618336692169573605081585080837952036335619947691937965065016808710250735070825260046821242820434367245824478859256555487861614478717581068572356895150707602217433511627331709472765932413249132702425519391509083601346239612335001086614623850633127072987745618984384288764099836164964775714638573247333226653894523588365972955159905187411779288608760239306160016168434070611663449248395156319152882728822831375458678269830696691220130954815935450754923554167766876455212545681242936427474153815692219503331560151614492247512488957534835926226263545406704767033866410025277276800886383266629488582740369655329362236090572479794734434077704284318507901973469071141230364111729224929307731939309795452877412451183953480382210373644697046967493042810911797232448615413264031578430955396671061468083815548947146733652483679138566431084747848676243012018489329109615281108087617422779131629345494425395422727309645057976122885347393189600810965202090151104579377602529543130188938184010247010134929317443562883578609861545691161669857388024973756940558138630581099823372565164920155443216861690537054630176154809626620800633059320775897175589925862195462096455464624399535391743228225433267174308492508396461328929584567927365409119947616225155964704061297047759818551878441419948614013153859322060745185909608884280218943358691959604936409651570327527570641500776261323783648149005245481413195989296398441371781402764122087644989688629798910870164270169014007825748311598976330612951195680427485317886333041169767175063822135213839779138443325644288490872919067009802496281560626258636942322658490628628035057282983101266919109637258378149363774960594515216932644945188292639525772348420077356021656909077097264985642831778694777804964343991762549216500608626285329471055602670413384500507827390640287529864161287496473708235188892189612641279553536442286955430551308700009878557534223100547153412810957024870812654319123261956462149376527526356402127388765103883255007364899937167183280028398832319373301564123277185395654932422977953016534830128490677845037490891749347389015649588574802194996722621185874361039774946338633057887487405540005440439344888192044102134790034598411927024921557026873700970995205391930979319495883265922171508324621942300185974396706491149559411733728199869021311629886680267446443489233020607003821262841723679627307191405008084085703978151998148822390059948911946474438682533745889962375133378280532928272016815977970066488394482446332210928320504045983008943565954267256879714918703447338237767914829203283196838105907715727191903042365315650957464549643425328069510396558733549803850995143463506175361480050195045201350200180281506933241918267855737764414097080945745624854867704904368368717590918057269794010465019484853146726642978667687697789291431128505043098192949736165944259471754765135205245072597538577958372797702972231435199958499522344049394502115428867244188717409524554771867484911475031801773304689909317974472957035192387686405544278134169807249382219749124257510162187439772902147704638010731470653154201300583810458905006764557332998149945854655105526374914354195867992595981412218735238407957416123372264063860431988936249867649693592569592128495906254446474331759999685163660305216426770428154681777589339252115538590526823311608302751194384823861552852465010329467297198112105314125898165100120742688143577590825227466863206188376830450921784582526239594189673003640808624233657620979111641766331328852352062487922978959456450333733139422384778582717195412347860434376165241568717943562570215636666680088531006728947033079540804583324192188488870712275670333173939262509073556164513677064199539111948881240659821685787131385056850623094155206877987539740658484250135205615103489821873770245063583314243624807432542464195984647411575625441010389671576677263196442524931941806472423789334668561083789808830313571333157729435664956078125304917594015895146954965223118559669048559467607968190167266634650186182955669893965019614544401768162810604465068448139561667220729261210164692339016793399632833013163850830967942792934551268435760356901970523138364640961311774904600772840862214747547653221505518116489887879087780918009050706040061220010051271575991225725282523378026809030528461581739558198122397010092017202251606352922464781615533532275453264543087093320924631855976580561717446840450048285353396546862678852330044967795580761661801833668792312510460809773895565488962815089519622093675058841609752282328250433712970186608193748968699961301486924694482420723632912367052542145464162968910442981633373266871675946715392611950649224725627254543274193495995569590243279097174392258098103601486364409101491734183079646345064833303404765711827040276868271418084574998493392039317445402616663674646668754385093967129918067471909885312710726724428584870694307099756567949198418996425748884764622030325637751112534060087936904565779272035205921345924272965206683338510673615276261016026647772485083344719891986802656197236420847504962661607797092906844757798251795569758235084371746103310387911789239441630112634077535773520558040066982523191225570519133631407211349723226549151062961739050617857127509403623146700931176133132018631158730886798239298009805089491510788371194099750375473674305745187265414016446924576792185753680363289139664155342066705623272936001177781498886100830877849571709880858667023104043242526785955562077310543072298032125941107957349146684680220501816192150766649106862033378713826058987655210423668198670177861672671972374156917880001690656659046965316154923604061891820982414006103779407166342002735828911994182647812782659666207030384795881442790246669264032799404016800137293477301530941805070587421153284642203006550763966756168318897005152026656649929417382840327305940740147117478464839241225676523593418554066440983706083636457657081801664285044258224551650808864421212113914352453935225522162483791737330329812349528984098613273709957407786789349311975204237925022851375880436791854547836416773151821457226504640800104202100410766027807729152555503218182387221708112766208665317651926458452495269685376314437998340336947124447247796973890514941120010934140073794061859447165516612674930799374705772930521750426383798367668159183589049652163726492960837147204067428996276720315410211504333742057182854090136325721437592054640471894328548696883599785122262130812989581571391597464534806099601555877223193450760315411663112963843719400333736013305526352571490454327925190794007111504785378036370897340146753465517470747096935814912797188187854376797751675927822300312945518595042883902735494672667647506072643698761394806879080593531793001711000214417701504495496412454361656210150919997862972495905809191825255486358703529320142005857057855419217730505342687533799076038746689684283402648733290888881745453047194740939258407362058242849349024756883352446212456101562729065130618520732925434179252299417447855189995098959999877410951464170076989305620163502192692653166599093238118295411937545448509428621839424186218067457128099385258842631930670182098008050900019819621758458932516877698594110522845465835679362969619219080897536813210484518784516230623911878024604050824909336069998094776253792973597037759066145994638578378211017122446355845171941670344732162722443265914858595797823752976323442911242311368603724514438765801271594060878788638511089680883165505046309006148832545452819908256238805872042843941834687865142541377686054291079721004271658 diff --git a/internal/compress/testdata/endnonzero.bin b/internal/compress/testdata/endnonzero.bin new file mode 100644 index 00000000..cf08368a Binary files /dev/null and b/internal/compress/testdata/endnonzero.bin differ diff --git a/internal/compress/testdata/endzerobits.bin b/internal/compress/testdata/endzerobits.bin new file mode 100644 index 00000000..d9952f92 Binary files /dev/null and b/internal/compress/testdata/endzerobits.bin differ diff --git a/internal/compress/testdata/fse-artifact3.bin b/internal/compress/testdata/fse-artifact3.bin new file mode 100644 index 00000000..0607a9e7 Binary files /dev/null and b/internal/compress/testdata/fse-artifact3.bin differ diff --git a/internal/compress/testdata/gettysburg.txt b/internal/compress/testdata/gettysburg.txt new file mode 100644 index 00000000..2c9bcde3 --- /dev/null +++ b/internal/compress/testdata/gettysburg.txt @@ -0,0 +1,29 @@ + Four score and seven years ago our fathers brought forth on +this continent, a new nation, conceived in Liberty, and dedicated +to the proposition that all men are created equal. + Now we are engaged in a great Civil War, testing whether that +nation, or any nation so conceived and so dedicated, can long +endure. + We are met on a great battle-field of that war. + We have come to dedicate a portion of that field, as a final +resting place for those who here gave their lives that that +nation might live. It is altogether fitting and proper that +we should do this. + But, in a larger sense, we can not dedicate - we can not +consecrate - we can not hallow - this ground. + The brave men, living and dead, who struggled here, have +consecrated it, far above our poor power to add or detract. +The world will little note, nor long remember what we say here, +but it can never forget what they did here. + It is for us the living, rather, to be dedicated here to the +unfinished work which they who fought here have thus far so +nobly advanced. It is rather for us to be here dedicated to +the great task remaining before us - that from these honored +dead we take increased devotion to that cause for which they +gave the last full measure of devotion - + that we here highly resolve that these dead shall not have +died in vain - that this nation, under God, shall have a new +birth of freedom - and that government of the people, by the +people, for the people, shall not perish from this earth. + +Abraham Lincoln, November 19, 1863, Gettysburg, Pennsylvania diff --git a/internal/compress/testdata/html.txt b/internal/compress/testdata/html.txt new file mode 100644 index 00000000..a8603543 --- /dev/null +++ b/internal/compress/testdata/html.txt @@ -0,0 +1,1183 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +Crédit Agricole Alpes Provence (Bouches-du-Rhône, Hautes Alpes et Vaucluse) - Crédit Agricole Alpes Provence (Bouches-du-Rhône, Hautes Alpes et Vaucluse) + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ + + + + + + + +
+ + + + + + + + + + + + + + +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ + + + + + +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ + + + + + + + + +
+ © Crédit Agricole 2011 +
+ +
+ +
+ + + + + \ No newline at end of file diff --git a/internal/compress/testdata/normcount2.bin b/internal/compress/testdata/normcount2.bin new file mode 100644 index 00000000..39050dae --- /dev/null +++ b/internal/compress/testdata/normcount2.bin @@ -0,0 +1 @@ +868000113000000fd9F055125272181835410155551151-0_0040Y2BW4K_0x_j4L___e__331ms__QSlz_I__ \ No newline at end of file diff --git a/internal/compress/testdata/pi.txt b/internal/compress/testdata/pi.txt new file mode 100644 index 00000000..ca99bbc2 --- /dev/null +++ b/internal/compress/testdata/pi.txt @@ -0,0 +1 @@ +3.1415926535897932384626433832795028841971693993751058209749445923078164062862089986280348253421170679821480865132823066470938446095505822317253594081284811174502841027019385211055596446229489549303819644288109756659334461284756482337867831652712019091456485669234603486104543266482133936072602491412737245870066063155881748815209209628292540917153643678925903600113305305488204665213841469519415116094330572703657595919530921861173819326117931051185480744623799627495673518857527248912279381830119491298336733624406566430860213949463952247371907021798609437027705392171762931767523846748184676694051320005681271452635608277857713427577896091736371787214684409012249534301465495853710507922796892589235420199561121290219608640344181598136297747713099605187072113499999983729780499510597317328160963185950244594553469083026425223082533446850352619311881710100031378387528865875332083814206171776691473035982534904287554687311595628638823537875937519577818577805321712268066130019278766111959092164201989380952572010654858632788659361533818279682303019520353018529689957736225994138912497217752834791315155748572424541506959508295331168617278558890750983817546374649393192550604009277016711390098488240128583616035637076601047101819429555961989467678374494482553797747268471040475346462080466842590694912933136770289891521047521620569660240580381501935112533824300355876402474964732639141992726042699227967823547816360093417216412199245863150302861829745557067498385054945885869269956909272107975093029553211653449872027559602364806654991198818347977535663698074265425278625518184175746728909777727938000816470600161452491921732172147723501414419735685481613611573525521334757418494684385233239073941433345477624168625189835694855620992192221842725502542568876717904946016534668049886272327917860857843838279679766814541009538837863609506800642251252051173929848960841284886269456042419652850222106611863067442786220391949450471237137869609563643719172874677646575739624138908658326459958133904780275900994657640789512694683983525957098258226205224894077267194782684826014769909026401363944374553050682034962524517493996514314298091906592509372216964615157098583874105978859597729754989301617539284681382686838689427741559918559252459539594310499725246808459872736446958486538367362226260991246080512438843904512441365497627807977156914359977001296160894416948685558484063534220722258284886481584560285060168427394522674676788952521385225499546667278239864565961163548862305774564980355936345681743241125150760694794510965960940252288797108931456691368672287489405601015033086179286809208747609178249385890097149096759852613655497818931297848216829989487226588048575640142704775551323796414515237462343645428584447952658678210511413547357395231134271661021359695362314429524849371871101457654035902799344037420073105785390621983874478084784896833214457138687519435064302184531910484810053706146806749192781911979399520614196634287544406437451237181921799983910159195618146751426912397489409071864942319615679452080951465502252316038819301420937621378559566389377870830390697920773467221825625996615014215030680384477345492026054146659252014974428507325186660021324340881907104863317346496514539057962685610055081066587969981635747363840525714591028970641401109712062804390397595156771577004203378699360072305587631763594218731251471205329281918261861258673215791984148488291644706095752706957220917567116722910981690915280173506712748583222871835209353965725121083579151369882091444210067510334671103141267111369908658516398315019701651511685171437657618351556508849099898599823873455283316355076479185358932261854896321329330898570642046752590709154814165498594616371802709819943099244889575712828905923233260972997120844335732654893823911932597463667305836041428138830320382490375898524374417029132765618093773444030707469211201913020330380197621101100449293215160842444859637669838952286847831235526582131449576857262433441893039686426243410773226978028073189154411010446823252716201052652272111660396665573092547110557853763466820653109896526918620564769312570586356620185581007293606598764861179104533488503461136576867532494416680396265797877185560845529654126654085306143444318586769751456614068007002378776591344017127494704205622305389945613140711270004078547332699390814546646458807972708266830634328587856983052358089330657574067954571637752542021149557615814002501262285941302164715509792592309907965473761255176567513575178296664547791745011299614890304639947132962107340437518957359614589019389713111790429782856475032031986915140287080859904801094121472213179476477726224142548545403321571853061422881375850430633217518297986622371721591607716692547487389866549494501146540628433663937900397692656721463853067360965712091807638327166416274888800786925602902284721040317211860820419000422966171196377921337575114959501566049631862947265473642523081770367515906735023507283540567040386743513622224771589150495309844489333096340878076932599397805419341447377441842631298608099888687413260472156951623965864573021631598193195167353812974167729478672422924654366800980676928238280689964004824354037014163149658979409243237896907069779422362508221688957383798623001593776471651228935786015881617557829735233446042815126272037343146531977774160319906655418763979293344195215413418994854447345673831624993419131814809277771038638773431772075456545322077709212019051660962804909263601975988281613323166636528619326686336062735676303544776280350450777235547105859548702790814356240145171806246436267945612753181340783303362542327839449753824372058353114771199260638133467768796959703098339130771098704085913374641442822772634659470474587847787201927715280731767907707157213444730605700733492436931138350493163128404251219256517980694113528013147013047816437885185290928545201165839341965621349143415956258658655705526904965209858033850722426482939728584783163057777560688876446248246857926039535277348030480290058760758251047470916439613626760449256274204208320856611906254543372131535958450687724602901618766795240616342522577195429162991930645537799140373404328752628889639958794757291746426357455254079091451357111369410911939325191076020825202618798531887705842972591677813149699009019211697173727847684726860849003377024242916513005005168323364350389517029893922334517220138128069650117844087451960121228599371623130171144484640903890644954440061986907548516026327505298349187407866808818338510228334508504860825039302133219715518430635455007668282949304137765527939751754613953984683393638304746119966538581538420568533862186725233402830871123282789212507712629463229563989898935821167456270102183564622013496715188190973038119800497340723961036854066431939509790190699639552453005450580685501956730229219139339185680344903982059551002263535361920419947455385938102343955449597783779023742161727111723643435439478221818528624085140066604433258885698670543154706965747458550332323342107301545940516553790686627333799585115625784322988273723198987571415957811196358330059408730681216028764962867446047746491599505497374256269010490377819868359381465741268049256487985561453723478673303904688383436346553794986419270563872931748723320837601123029911367938627089438799362016295154133714248928307220126901475466847653576164773794675200490757155527819653621323926406160136358155907422020203187277605277219005561484255518792530343513984425322341576233610642506390497500865627109535919465897514131034822769306247435363256916078154781811528436679570611086153315044521274739245449454236828860613408414863776700961207151249140430272538607648236341433462351897576645216413767969031495019108575984423919862916421939949072362346468441173940326591840443780513338945257423995082965912285085558215725031071257012668302402929525220118726767562204154205161841634847565169998116141010029960783869092916030288400269104140792886215078424516709087000699282120660418371806535567252532567532861291042487761825829765157959847035622262934860034158722980534989650226291748788202734209222245339856264766914905562842503912757710284027998066365825488926488025456610172967026640765590429099456815065265305371829412703369313785178609040708667114965583434347693385781711386455873678123014587687126603489139095620099393610310291616152881384379099042317473363948045759314931405297634757481193567091101377517210080315590248530906692037671922033229094334676851422144773793937517034436619910403375111735471918550464490263655128162288244625759163330391072253837421821408835086573917715096828874782656995995744906617583441375223970968340800535598491754173818839994469748676265516582765848358845314277568790029095170283529716344562129640435231176006651012412006597558512761785838292041974844236080071930457618932349229279650198751872127267507981255470958904556357921221033346697499235630254947802490114195212382815309114079073860251522742995818072471625916685451333123948049470791191532673430282441860414263639548000448002670496248201792896476697583183271314251702969234889627668440323260927524960357996469256504936818360900323809293459588970695365349406034021665443755890045632882250545255640564482465151875471196218443965825337543885690941130315095261793780029741207665147939425902989695946995565761218656196733786236256125216320862869222103274889218654364802296780705765615144632046927906821207388377814233562823608963208068222468012248261177185896381409183903673672220888321513755600372798394004152970028783076670944474560134556417254370906979396122571429894671543578468788614445812314593571984922528471605049221242470141214780573455105008019086996033027634787081081754501193071412233908663938339529425786905076431006383519834389341596131854347546495569781038293097164651438407007073604112373599843452251610507027056235266012764848308407611830130527932054274628654036036745328651057065874882256981579367897669742205750596834408697350201410206723585020072452256326513410559240190274216248439140359989535394590944070469120914093870012645600162374288021092764579310657922955249887275846101264836999892256959688159205600101655256375678566722796619885782794848855834397518744545512965634434803966420557982936804352202770984294232533022576341807039476994159791594530069752148293366555661567873640053666564165473217043903521329543529169414599041608753201868379370234888689479151071637852902345292440773659495630510074210871426134974595615138498713757047101787957310422969066670214498637464595280824369445789772330048764765241339075920434019634039114732023380715095222010682563427471646024335440051521266932493419673977041595683753555166730273900749729736354964533288869844061196496162773449518273695588220757355176651589855190986665393549481068873206859907540792342402300925900701731960362254756478940647548346647760411463233905651343306844953979070903023460461470961696886885014083470405460742958699138296682468185710318879065287036650832431974404771855678934823089431068287027228097362480939962706074726455399253994428081137369433887294063079261595995462624629707062594845569034711972996409089418059534393251236235508134949004364278527138315912568989295196427287573946914272534366941532361004537304881985517065941217352462589548730167600298865925786628561249665523533829428785425340483083307016537228563559152534784459818313411290019992059813522051173365856407826484942764411376393866924803118364453698589175442647399882284621844900877769776312795722672655562596282542765318300134070922334365779160128093179401718598599933849235495640057099558561134980252499066984233017350358044081168552653117099570899427328709258487894436460050410892266917835258707859512983441729535195378855345737426085902908176515578039059464087350612322611200937310804854852635722825768203416050484662775045003126200800799804925485346941469775164932709504934639382432227188515974054702148289711177792376122578873477188196825462981268685817050740272550263329044976277894423621674119186269439650671515779586756482399391760426017633870454990176143641204692182370764887834196896861181558158736062938603810171215855272668300823834046564758804051380801633638874216371406435495561868964112282140753302655100424104896783528588290243670904887118190909494533144218287661810310073547705498159680772009474696134360928614849417850171807793068108546900094458995279424398139213505586422196483491512639012803832001097738680662877923971801461343244572640097374257007359210031541508936793008169980536520276007277496745840028362405346037263416554259027601834840306811381855105979705664007509426087885735796037324514146786703688098806097164258497595138069309449401515422221943291302173912538355915031003330325111749156969174502714943315155885403922164097229101129035521815762823283182342548326111912800928252561902052630163911477247331485739107775874425387611746578671169414776421441111263583553871361011023267987756410246824032264834641766369806637857681349204530224081972785647198396308781543221166912246415911776732253264335686146186545222681268872684459684424161078540167681420808850280054143613146230821025941737562389942075713627516745731891894562835257044133543758575342698699472547031656613991999682628247270641336222178923903176085428943733935618891651250424404008952719837873864805847268954624388234375178852014395600571048119498842390606136957342315590796703461491434478863604103182350736502778590897578272731305048893989009923913503373250855982655867089242612429473670193907727130706869170926462548423240748550366080136046689511840093668609546325002145852930950000907151058236267293264537382104938724996699339424685516483261134146110680267446637334375340764294026682973865220935701626384648528514903629320199199688285171839536691345222444708045923966028171565515656661113598231122506289058549145097157553900243931535190902107119457300243880176615035270862602537881797519478061013715004489917210022201335013106016391541589578037117792775225978742891917915522417189585361680594741234193398420218745649256443462392531953135103311476394911995072858430658361935369329699289837914941939406085724863968836903265564364216644257607914710869984315733749648835292769328220762947282381537409961545598798259891093717126218283025848112389011968221429457667580718653806506487026133892822994972574530332838963818439447707794022843598834100358385423897354243956475556840952248445541392394100016207693636846776413017819659379971557468541946334893748439129742391433659360410035234377706588867781139498616478747140793263858738624732889645643598774667638479466504074111825658378878454858148962961273998413442726086061872455452360643153710112746809778704464094758280348769758948328241239292960582948619196670918958089833201210318430340128495116203534280144127617285830243559830032042024512072872535581195840149180969253395075778400067465526031446167050827682772223534191102634163157147406123850425845988419907611287258059113935689601431668283176323567325417073420817332230462987992804908514094790368878687894930546955703072619009502076433493359106024545086453628935456862958531315337183868265617862273637169757741830239860065914816164049449650117321313895747062088474802365371031150898427992754426853277974311395143574172219759799359685252285745263796289612691572357986620573408375766873884266405990993505000813375432454635967504844235284874701443545419576258473564216198134073468541117668831186544893776979566517279662326714810338643913751865946730024434500544995399742372328712494834706044063471606325830649829795510109541836235030309453097335834462839476304775645015008507578949548931393944899216125525597701436858943585877526379625597081677643800125436502371412783467926101995585224717220177723700417808419423948725406801556035998390548985723546745642390585850216719031395262944554391316631345308939062046784387785054239390524731362012947691874975191011472315289326772533918146607300089027768963114810902209724520759167297007850580717186381054967973100167870850694207092232908070383263453452038027860990556900134137182368370991949516489600755049341267876436746384902063964019766685592335654639138363185745698147196210841080961884605456039038455343729141446513474940784884423772175154334260306698831768331001133108690421939031080143784334151370924353013677631084913516156422698475074303297167469640666531527035325467112667522460551199581831963763707617991919203579582007595605302346267757943936307463056901080114942714100939136913810725813781357894005599500183542511841721360557275221035268037357265279224173736057511278872181908449006178013889710770822931002797665935838758909395688148560263224393726562472776037890814458837855019702843779362407825052704875816470324581290878395232453237896029841669225489649715606981192186584926770403956481278102179913217416305810554598801300484562997651121241536374515005635070127815926714241342103301566165356024733807843028655257222753049998837015348793008062601809623815161366903341111386538510919367393835229345888322550887064507539473952043968079067086806445096986548801682874343786126453815834280753061845485903798217994599681154419742536344399602902510015888272164745006820704193761584547123183460072629339550548239557137256840232268213012476794522644820910235647752723082081063518899152692889108455571126603965034397896278250016110153235160519655904211844949907789992007329476905868577878720982901352956613978884860509786085957017731298155314951681467176959760994210036183559138777817698458758104466283998806006162298486169353373865787735983361613384133853684211978938900185295691967804554482858483701170967212535338758621582310133103877668272115726949518179589754693992642197915523385766231676275475703546994148929041301863861194391962838870543677743224276809132365449485366768000001065262485473055861598999140170769838548318875014293890899506854530765116803337322265175662207526951791442252808165171667766727930354851542040238174608923283917032754257508676551178593950027933895920576682789677644531840404185540104351348389531201326378369283580827193783126549617459970567450718332065034556644034490453627560011250184335607361222765949278393706478426456763388188075656121689605041611390390639601620221536849410926053876887148379895599991120991646464411918568277004574243434021672276445589330127781586869525069499364610175685060167145354315814801054588605645501332037586454858403240298717093480910556211671546848477803944756979804263180991756422809873998766973237695737015808068229045992123661689025962730430679316531149401764737693873514093361833216142802149763399189835484875625298752423873077559555955465196394401821840998412489826236737714672260616336432964063357281070788758164043814850188411431885988276944901193212968271588841338694346828590066640806314077757725705630729400492940302420498416565479736705485580445865720227637840466823379852827105784319753541795011347273625774080213476826045022851579795797647467022840999561601569108903845824502679265942055503958792298185264800706837650418365620945554346135134152570065974881916341359556719649654032187271602648593049039787489589066127250794828276938953521753621850796297785146188432719223223810158744450528665238022532843891375273845892384422535472653098171578447834215822327020690287232330053862163479885094695472004795231120150432932266282727632177908840087861480221475376578105819702226309717495072127248479478169572961423658595782090830733233560348465318730293026659645013718375428897557971449924654038681799213893469244741985097334626793321072686870768062639919361965044099542167627840914669856925715074315740793805323925239477557441591845821562518192155233709607483329234921034514626437449805596103307994145347784574699992128599999399612281615219314888769388022281083001986016549416542616968586788372609587745676182507275992950893180521872924610867639958916145855058397274209809097817293239301067663868240401113040247007350857828724627134946368531815469690466968693925472519413992914652423857762550047485295476814795467007050347999588867695016124972282040303995463278830695976249361510102436555352230690612949388599015734661023712235478911292547696176005047974928060721268039226911027772261025441492215765045081206771735712027180242968106203776578837166909109418074487814049075517820385653909910477594141321543284406250301802757169650820964273484146957263978842560084531214065935809041271135920041975985136254796160632288736181367373244506079244117639975974619383584574915988097667447093006546342423460634237474666080431701260052055928493695941434081468529815053947178900451835755154125223590590687264878635752541911288877371766374860276606349603536794702692322971868327717393236192007774522126247518698334951510198642698878471719396649769070825217423365662725928440620430214113719922785269984698847702323823840055655517889087661360130477098438611687052310553149162517283732728676007248172987637569816335415074608838663640693470437206688651275688266149730788657015685016918647488541679154596507234287730699853713904300266530783987763850323818215535597323530686043010675760838908627049841888595138091030423595782495143988590113185835840667472370297149785084145853085781339156270760356390763947311455495832266945702494139831634332378975955680856836297253867913275055542524491943589128405045226953812179131914513500993846311774017971512283785460116035955402864405902496466930707769055481028850208085800878115773817191741776017330738554758006056014337743299012728677253043182519757916792969965041460706645712588834697979642931622965520168797300035646304579308840327480771811555330909887025505207680463034608658165394876951960044084820659673794731680864156456505300498816164905788311543454850526600698230931577765003780704661264706021457505793270962047825615247145918965223608396645624105195510522357239739512881816405978591427914816542632892004281609136937773722299983327082082969955737727375667615527113922588055201898876201141680054687365580633471603734291703907986396522961312801782679717289822936070288069087768660593252746378405397691848082041021944719713869256084162451123980620113184541244782050110798760717155683154078865439041210873032402010685341947230476666721749869868547076781205124736792479193150856444775379853799732234456122785843296846647513336573692387201464723679427870042503255589926884349592876124007558756946413705625140011797133166207153715436006876477318675587148783989081074295309410605969443158477539700943988394914432353668539209946879645066533985738887866147629443414010498889931600512076781035886116602029611936396821349607501116498327856353161451684576956871090029997698412632665023477167286573785790857466460772283415403114415294188047825438761770790430001566986776795760909966936075594965152736349811896413043311662774712338817406037317439705406703109676765748695358789670031925866259410510533584384656023391796749267844763708474978333655579007384191473198862713525954625181604342253729962863267496824058060296421146386436864224724887283434170441573482481833301640566959668866769563491416328426414974533349999480002669987588815935073578151958899005395120853510357261373640343675347141048360175464883004078464167452167371904831096767113443494819262681110739948250607394950735031690197318521195526356325843390998224986240670310768318446607291248747540316179699411397387765899868554170318847788675929026070043212666179192235209382278788809886335991160819235355570464634911320859189796132791319756490976000139962344455350143464268604644958624769094347048293294140411146540923988344435159133201077394411184074107684981066347241048239358274019449356651610884631256785297769734684303061462418035852933159734583038455410337010916767763742762102137013548544509263071901147318485749233181672072137279355679528443925481560913728128406333039373562420016045664557414588166052166608738748047243391212955877763906969037078828527753894052460758496231574369171131761347838827194168606625721036851321566478001476752310393578606896111259960281839309548709059073861351914591819510297327875571049729011487171897180046961697770017913919613791417162707018958469214343696762927459109940060084983568425201915593703701011049747339493877885989417433031785348707603221982970579751191440510994235883034546353492349826883624043327267415540301619505680654180939409982020609994140216890900708213307230896621197755306659188141191577836272927461561857103721724710095214236964830864102592887457999322374955191221951903424452307535133806856807354464995127203174487195403976107308060269906258076020292731455252078079914184290638844373499681458273372072663917670201183004648190002413083508846584152148991276106513741539435657211390328574918769094413702090517031487773461652879848235338297260136110984514841823808120540996125274580881099486972216128524897425555516076371675054896173016809613803811914361143992106380050832140987604599309324851025168294467260666138151745712559754953580239983146982203613380828499356705575524712902745397762140493182014658008021566536067765508783804304134310591804606800834591136640834887408005741272586704792258319127415739080914383138456424150940849133918096840251163991936853225557338966953749026620923261318855891580832455571948453875628786128859004106006073746501402627824027346962528217174941582331749239683530136178653673760642166778137739951006589528877427662636841830680190804609849809469763667335662282915132352788806157768278159588669180238940333076441912403412022316368577860357276941541778826435238131905028087018575047046312933353757285386605888904583111450773942935201994321971171642235005644042979892081594307167019857469273848653833436145794634175922573898588001698014757420542995801242958105456510831046297282937584161162532562516572498078492099897990620035936509934721582965174135798491047111660791587436986541222348341887722929446335178653856731962559852026072947674072616767145573649812105677716893484917660771705277187601199908144113058645577910525684304811440261938402322470939249802933550731845890355397133088446174107959162511714864874468611247605428673436709046678468670274091881014249711149657817724279347070216688295610877794405048437528443375108828264771978540006509704033021862556147332117771174413350281608840351781452541964320309576018694649088681545285621346988355444560249556668436602922195124830910605377201980218310103270417838665447181260397190688462370857518080035327047185659499476124248110999288679158969049563947624608424065930948621507690314987020673533848349550836366017848771060809804269247132410009464014373603265645184566792456669551001502298330798496079949882497061723674493612262229617908143114146609412341593593095854079139087208322733549572080757165171876599449856937956238755516175754380917805280294642004472153962807463602113294255916002570735628126387331060058910652457080244749375431841494014821199962764531068006631183823761639663180931444671298615527598201451410275600689297502463040173514891945763607893528555053173314164570504996443890936308438744847839616840518452732884032345202470568516465716477139323775517294795126132398229602394548579754586517458787713318138752959809412174227300352296508089177705068259248822322154938048371454781647213976820963320508305647920482085920475499857320388876391601995240918938945576768749730856955958010659526503036266159750662225084067428898265907510637563569968211510949669744580547288693631020367823250182323708459790111548472087618212477813266330412076216587312970811230758159821248639807212407868878114501655825136178903070860870198975889807456643955157415363193191981070575336633738038272152798849350397480015890519420879711308051233933221903466249917169150948541401871060354603794643379005890957721180804465743962806186717861017156740967662080295766577051291209907944304632892947306159510430902221439371849560634056189342513057268291465783293340524635028929175470872564842600349629611654138230077313327298305001602567240141851520418907011542885799208121984493156999059182011819733500126187728036812481995877070207532406361259313438595542547781961142935163561223496661522614735399674051584998603552953329245752388810136202347624669055816438967863097627365504724348643071218494373485300606387644566272186661701238127715621379746149861328744117714552444708997144522885662942440230184791205478498574521634696448973892062401943518310088283480249249085403077863875165911302873958787098100772718271874529013972836614842142871705531796543076504534324600536361472618180969976933486264077435199928686323835088756683595097265574815431940195576850437248001020413749831872259677387154958399718444907279141965845930083942637020875635398216962055324803212267498911402678528599673405242031091797899905718821949391320753431707980023736590985375520238911643467185582906853711897952626234492483392496342449714656846591248918556629589329909035239233333647435203707701010843880032907598342170185542283861617210417603011645918780539367447472059985023582891833692922337323999480437108419659473162654825748099482509991833006976569367159689364493348864744213500840700660883597235039532340179582557036016936990988671132109798897070517280755855191269930673099250704070245568507786790694766126298082251633136399521170984528092630375922426742575599892892783704744452189363203489415521044597261883800300677617931381399162058062701651024458869247649246891924612125310275731390840470007143561362316992371694848132554200914530410371354532966206392105479824392125172540132314902740585892063217589494345489068463993137570910346332714153162232805522972979538018801628590735729554162788676498274186164218789885741071649069191851162815285486794173638906653885764229158342500673612453849160674137340173572779956341043326883569507814931378007362354180070619180267328551191942676091221035987469241172837493126163395001239599240508454375698507957046222664619000103500490183034153545842833764378111988556318777792537201166718539541835984438305203762819440761594106820716970302285152250573126093046898423433152732131361216582808075212631547730604423774753505952287174402666389148817173086436111389069420279088143119448799417154042103412190847094080254023932942945493878640230512927119097513536000921971105412096683111516328705423028470073120658032626417116165957613272351566662536672718998534199895236884830999302757419916463841427077988708874229277053891227172486322028898425125287217826030500994510824783572905691988555467886079462805371227042466543192145281760741482403827835829719301017888345674167811398954750448339314689630763396657226727043393216745421824557062524797219978668542798977992339579057581890622525473582205236424850783407110144980478726691990186438822932305382318559732869780922253529591017341407334884761005564018242392192695062083183814546983923664613639891012102177095976704908305081854704194664371312299692358895384930136356576186106062228705599423371631021278457446463989738188566746260879482018647487672727222062676465338099801966883680994159075776852639865146253336312450536402610569605513183813174261184420189088853196356986962795036738424313011331753305329802016688817481342988681585577810343231753064784983210629718425184385534427620128234570716988530518326179641178579608888150329602290705614476220915094739035946646916235396809201394578175891088931992112260073928149169481615273842736264298098234063200244024495894456129167049508235812487391799648641133480324757775219708932772262349486015046652681439877051615317026696929704928316285504212898146706195331970269507214378230476875280287354126166391708245925170010714180854800636923259462019002278087409859771921805158532147392653251559035410209284665925299914353791825314545290598415817637058927906909896911164381187809435371521332261443625314490127454772695739393481546916311624928873574718824071503995009446731954316193855485207665738825139639163576723151005556037263394867208207808653734942440115799667507360711159351331959197120948964717553024531364770942094635696982226673775209945168450643623824211853534887989395673187806606107885440005508276570305587448541805778891719207881423351138662929667179643468760077047999537883387870348718021842437342112273940255717690819603092018240188427057046092622564178375265263358324240661253311529423457965569502506810018310900411245379015332966156970522379210325706937051090830789479999004999395322153622748476603613677697978567386584670936679588583788795625946464891376652199588286933801836011932368578558558195556042156250883650203322024513762158204618106705195330653060606501054887167245377942831338871631395596905832083416898476065607118347136218123246227258841990286142087284956879639325464285343075301105285713829643709990356948885285190402956047346131138263878897551788560424998748316382804046848618938189590542039889872650697620201995548412650005394428203930127481638158530396439925470201672759328574366661644110962566337305409219519675148328734808957477775278344221091073111351828046036347198185655572957144747682552857863349342858423118749440003229690697758315903858039353521358860079600342097547392296733310649395601812237812854584317605561733861126734780745850676063048229409653041118306671081893031108871728167519579675347188537229309616143204006381322465841111157758358581135018569047815368938137718472814751998350504781297718599084707621974605887423256995828892535041937958260616211842368768511418316068315867994601652057740529423053601780313357263267054790338401257305912339601880137825421927094767337191987287385248057421248921183470876629667207272325650565129333126059505777727542471241648312832982072361750574673870128209575544305968395555686861188397135522084452852640081252027665557677495969626612604565245684086139238265768583384698499778726706555191854468698469478495734622606294219624557085371272776523098955450193037732166649182578154677292005212667143463209637891852323215018976126034373684067194193037746880999296877582441047878123266253181845960453853543839114496775312864260925211537673258866722604042523491087026958099647595805794663973419064010036361904042033113579336542426303561457009011244800890020801478056603710154122328891465722393145076071670643556827437743965789067972687438473076346451677562103098604092717090951280863090297385044527182892749689212106670081648583395537735919136950153162018908887484210798706899114804669270650940762046502772528650728905328548561433160812693005693785417861096969202538865034577183176686885923681488475276498468821949739729707737187188400414323127636504814531122850990020742409255859252926103021067368154347015252348786351643976235860419194129697690405264832347009911154242601273438022089331096686367898694977994001260164227609260823493041180643829138347354679725399262338791582998486459271734059225620749105308531537182911681637219395188700957788181586850464507699343940987433514431626330317247747486897918209239480833143970840673084079589358108966564775859905563769525232653614424780230826811831037735887089240613031336477371011628214614661679404090518615260360092521947218890918107335871964142144478654899528582343947050079830388538860831035719306002771194558021911942899922722353458707566246926177663178855144350218287026685610665003531050216318206017609217984684936863161293727951873078972637353717150256378733579771808184878458866504335824377004147710414934927438457587107159731559439426412570270965125108115548247939403597681188117282472158250109496096625393395380922195591918188552678062149923172763163218339896938075616855911752998450132067129392404144593862398809381240452191484831646210147389182510109096773869066404158973610476436500068077105656718486281496371118832192445663945814491486165500495676982690308911185687986929470513524816091743243015383684707292898982846022237301452655679898627767968091469798378268764311598832109043715611299766521539635464420869197567370005738764978437686287681792497469438427465256316323005551304174227341646455127812784577772457520386543754282825671412885834544435132562054464241011037955464190581168623059644769587054072141985212106734332410756767575818456990693046047522770167005684543969234041711089888993416350585157887353430815520811772071880379104046983069578685473937656433631979786803671873079693924236321448450354776315670255390065423117920153464977929066241508328858395290542637687668968805033317227800185885069736232403894700471897619347344308437443759925034178807972235859134245813144049847701732361694719765715353197754997162785663119046912609182591249890367654176979903623755286526375733763526969344354400473067198868901968147428767790866979688522501636949856730217523132529265375896415171479559538784278499866456302878831962099830494519874396369070682762657485810439112232618794059941554063270131989895703761105323606298674803779153767511583043208498720920280929752649812569163425000522908872646925284666104665392171482080130502298052637836426959733707053922789153510568883938113249757071331029504430346715989448786847116438328050692507766274500122003526203709466023414648998390252588830148678162196775194583167718762757200505439794412459900771152051546199305098386982542846407255540927403132571632640792934183342147090412542533523248021932277075355546795871638358750181593387174236061551171013123525633485820365146141870049205704372018261733194715700867578539336078622739558185797587258744102542077105475361294047460100094095444959662881486915903899071865980563617137692227290764197755177720104276496949611056220592502420217704269622154958726453989227697660310524980855759471631075870133208861463266412591148633881220284440694169488261529577625325019870359870674380469821942056381255833436421949232275937221289056420943082352544084110864545369404969271494003319782861318186188811118408257865928757426384450059944229568586460481033015388911499486935436030221810943466764000022362550573631294626296096198760564259963946138692330837196265954739234624134597795748524647837980795693198650815977675350553918991151335252298736112779182748542008689539658359421963331502869561192012298889887006079992795411188269023078913107603617634779489432032102773359416908650071932804017163840644987871753756781185321328408216571107549528294974936214608215583205687232185574065161096274874375098092230211609982633033915469494644491004515280925089745074896760324090768983652940657920198315265410658136823791984090645712468948470209357761193139980246813405200394781949866202624008902150166163813538381515037735022966074627952910384068685569070157516624192987244482719429331004854824454580718897633003232525821581280327467962002814762431828622171054352898348208273451680186131719593324711074662228508710666117703465352839577625997744672185715816126411143271794347885990892808486694914139097716736900277758502686646540565950394867841110790116104008572744562938425494167594605487117235946429105850909950214958793112196135908315882620682332156153086833730838173279328196983875087083483880463884784418840031847126974543709373298362402875197920802321878744882872843727378017827008058782410749357514889978911739746129320351081432703251409030487462262942344327571260086642508333187688650756429271605525289544921537651751492196367181049435317858383453865255656640657251363575064353236508936790431702597878177190314867963840828810209461490079715137717099061954969640070867667102330048672631475510537231757114322317411411680622864206388906210192355223546711662137499693269321737043105987225039456574924616978260970253359475020913836673772894438696400028110344026084712899000746807764844088711341352503367877316797709372778682166117865344231732264637847697875144332095340001650692130546476890985050203015044880834261845208730530973189492916425322933612431514306578264070283898409841602950309241897120971601649265613413433422298827909921786042679812457285345801338260995877178113102167340256562744007296834066198480676615805021691833723680399027931606420436812079900316264449146190219458229690992122788553948783538305646864881655562294315673128274390826450611628942803501661336697824051770155219626522725455850738640585299830379180350432876703809252167907571204061237596327685674845079151147313440001832570344920909712435809447900462494313455028900680648704293534037436032625820535790118395649089354345101342969617545249573960621490288728932792520696535386396443225388327522499605986974759882329916263545973324445163755334377492928990581175786355555626937426910947117002165411718219750519831787137106051063795558588905568852887989084750915764639074693619881507814685262133252473837651192990156109189777922008705793396463827490680698769168197492365624226087154176100430608904377976678519661891404144925270480881971498801542057787006521594009289777601330756847966992955433656139847738060394368895887646054983871478968482805384701730871117761159663505039979343869339119789887109156541709133082607647406305711411098839388095481437828474528838368079418884342666222070438722887413947801017721392281911992365405516395893474263953824829609036900288359327745855060801317988407162446563997948275783650195514221551339281978226984278638391679715091262410548725700924070045488485692950448110738087996547481568913935380943474556972128919827177020766613602489581468119133614121258783895577357194986317210844398901423948496659251731388171602663261931065366535041473070804414939169363262373767777095850313255990095762731957308648042467701212327020533742667053142448208168130306397378736642483672539837487690980602182785786216512738563513290148903509883270617258932575363993979055729175160097615459044771692265806315111028038436017374742152476085152099016158582312571590733421736576267142390478279587281505095633092802668458937649649770232973641319060982740633531089792464242134583740901169391964250459128813403498810635400887596820054408364386516617880557608956896727531538081942077332597917278437625661184319891025007491829086475149794003160703845549465385946027452447466812314687943441610993338908992638411847425257044572517459325738989565185716575961481266020310797628254165590506042479114016957900338356574869252800743025623419498286467914476322774005529460903940177536335655471931000175430047504719144899841040015867946179241610016454716551337074073950260442769538553834397550548871099785205401175169747581344926079433689543783221172450687344231989878844128542064742809735625807066983106979935260693392135685881391214807354728463227784908087002467776303605551232386656295178853719673034634701222939581606792509153217489030840886516061119011498443412350124646928028805996134283511884715449771278473361766285062169778717743824362565711779450064477718370221999106695021656757644044997940765037999954845002710665987813603802314126836905783190460792765297277694043613023051787080546511542469395265127101052927070306673024447125973939950514628404767431363739978259184541176413327906460636584152927019030276017339474866960348694976541752429306040727005059039503148522921392575594845078867977925253931765156416197168443524369794447355964260633391055126826061595726217036698506473281266724521989060549880280782881429796336696744124805982192146339565745722102298677599746738126069367069134081559412016115960190237753525556300606247983261249881288192937343476862689219239777833910733106588256813777172328315329082525092733047850724977139448333892552081175608452966590553940965568541706001179857293813998258319293679100391844099286575605993598910002969864460974714718470101531283762631146774209145574041815908800064943237855839308530828305476076799524357391631221886057549673832243195650655460852881201902363644712703748634421727257879503428486312944916318475347531435041392096108796057730987201352484075057637199253650470908582513936863463863368042891767107602111159828875539940120076013947033661793715396306139863655492213741597905119083588290097656647300733879314678913181465109316761575821351424860442292445304113160652700974330088499034675405518640677342603583409608605533747362760935658853109760994238347382222087292464497684560579562516765574088410321731345627735856052358236389532038534024842273371639123973215995440828421666636023296545694703577184873442034227706653837387506169212768015766181095420097708363604361110592409117889540338021426523948929686439808926114635414571535194342850721353453018315875628275733898268898523557799295727645229391567477566676051087887648453493636068278050564622813598885879259940946446041705204470046315137975431737187756039815962647501410906658866162180038266989961965580587208639721176995219466789857011798332440601811575658074284182910615193917630059194314434605154047710570054339000182453117733718955857603607182860506356479979004139761808955363669603162193113250223851791672055180659263518036251214575926238369348222665895576994660491938112486609099798128571823494006615552196112207203092277646200999315244273589488710576623894693889446495093960330454340842102462401048723328750081749179875543879387381439894238011762700837196053094383940063756116458560943129517597713935396074322792489221267045808183313764165818269562105872892447740035947009268662659651422050630078592002488291860839743732353849083964326147000532423540647042089499210250404726781059083644007466380020870126664209457181702946752278540074508552377720890581683918446592829417018288233014971554235235911774818628592967605048203864343108779562892925405638946621948268711042828163893975711757786915430165058602965217459581988878680408110328432739867198621306205559855266036405046282152306154594474489908839081999738747452969810776201487134000122535522246695409315213115337915798026979555710508507473874750758068765376445782524432638046143042889235934852961058269382103498000405248407084403561167817170512813378805705643450616119330424440798260377951198548694559152051960093041271007277849301555038895360338261929343797081874320949914159593396368110627557295278004254863060054523839151068998913578820019411786535682149118528207852130125518518493711503422159542244511900207393539627400208110465530207932867254740543652717595893500716336076321614725815407642053020045340183572338292661915308354095120226329165054426123619197051613839357326693760156914429944943744856809775696303129588719161129294681884936338647392747601226964158848900965717086160598147204467428664208765334799858222090619802173211614230419477754990738738567941189824660913091691772274207233367635032678340586301930193242996397204445179288122854478211953530898910125342975524727635730226281382091807439748671453590778633530160821559911314144205091447293535022230817193663509346865858656314855575862447818620108711889760652969899269328178705576435143382060141077329261063431525337182243385263520217735440715281898137698755157574546939727150488469793619500477720970561793913828989845327426227288647108883270173723258818244658436249580592560338105215606206155713299156084892064340303395262263451454283678698288074251422567451806184149564686111635404971897682154227722479474033571527436819409892050113653400123846714296551867344153741615042563256713430247655125219218035780169240326699541746087592409207004669340396510178134857835694440760470232540755557764728450751826890418293966113310160131119077398632462778219023650660374041606724962490137433217246454097412995570529142438208076098364823465973886691349919784013108015581343979194852830436739012482082444814128095443773898320059864909159505322857914576884962578665885999179867520554558099004556461178755249370124553217170194282884617402736649978475508294228020232901221630102309772151569446427909802190826689868834263071609207914085197695235553488657743425277531197247430873043619511396119080030255878387644206085044730631299277888942729189727169890575925244679660189707482960949190648764693702750773866432391919042254290235318923377293166736086996228032557185308919284403805071030064776847863243191000223929785255372375566213644740096760539439838235764606992465260089090624105904215453927904411529580345334500256244101006359530039598864466169595626351878060688513723462707997327233134693971456285542615467650632465676620279245208581347717608521691340946520307673391841147504140168924121319826881568664561485380287539331160232292555618941042995335640095786495340935115266454024418775949316930560448686420862757201172319526405023099774567647838488973464317215980626787671838005247696884084989185086149003432403476742686245952395890358582135006450998178244636087317754378859677672919526111213859194725451400301180503437875277664402762618941017576872680428176623860680477885242887430259145247073950546525135339459598789619778911041890292943818567205070964606263541732944649576612651953495701860015412623962286413897796733329070567376962156498184506842263690367849555970026079867996261019039331263768556968767029295371162528005543100786408728939225714512481135778627664902425161990277471090335933309304948380597856628844787441469841499067123764789582263294904679812089984857163571087831191848630254501620929805829208334813638405421720056121989353669371336733392464416125223196943471206417375491216357008573694397305979709719726666642267431117762176403068681310351899112271339724036887000996862922546465006385288620393800504778276912835603372548255793912985251506829969107754257647488325341412132800626717094009098223529657957997803018282428490221470748111124018607613415150387569830918652780658896682362523937845272634530420418802508442363190383318384550522367992357752929106925043261446950109861088899914658551881873582528164302520939285258077969737620845637482114433988162710031703151334402309526351929588680690821355853680161000213740851154484912685841268695899174149133820578492800698255195740201818105641297250836070356851055331787840829000041552511865779453963317538532092149720526607831260281961164858098684587525129997404092797683176639914655386108937587952214971731728131517932904431121815871023518740757222100123768721944747209349312324107065080618562372526732540733324875754482967573450019321902199119960797989373383673242576103938985349278777473980508080015544764061053522202325409443567718794565430406735896491017610775948364540823486130254718476485189575836674399791508512858020607820554462991723202028222914886959399729974297471155371858924238493855858595407438104882624648788053304271463011941589896328792678327322456103852197011130466587100500083285177311776489735230926661234588873102883515626446023671996644554727608310118788389151149340939344750073025855814756190881398752357812331342279866503522725367171230756861045004548970360079569827626392344107146584895780241408158405229536937499710665594894459246286619963556350652623405339439142111271810691052290024657423604130093691889255865784668461215679554256605416005071276641766056874274200329577160643448606201239821698271723197826816628249938714995449137302051843669076723577400053932662622760323659751718925901801104290384274185507894887438832703063283279963007200698012244365116394086922220745320244624121155804354542064215121585056896157356414313068883443185280853975927734433655384188340303517822946253702015782157373265523185763554098954033236382319219892171177449469403678296185920803403867575834111518824177439145077366384071880489358256868542011645031357633355509440319236720348651010561049872726472131986543435450409131859513145181276437310438972507004981987052176272494065214619959232142314439776546708351714749367986186552791715824080651063799500184295938799158350171580759883784962257398512129810326379376218322456594236685376799113140108043139732335449090824910499143325843298821033984698141715756010829706583065211347076803680695322971990599904451209087275776225351040902392888779424630483280319132710495478599180196967835321464441189260631526618167443193550817081875477050802654025294109218264858213857526688155584113198560022135158887210365696087515063187533002942118682221893775546027227291290504292259787710667873840000616772154638441292371193521828499824350920891801685572798156421858191197490985730570332667646460728757430565372602768982373259745084479649545648030771598153955827779139373601717422996027353102768719449444917939785144631597314435351850491413941557329382048542123508173912549749819308714396615132942045919380106231421774199184060180347949887691051557905554806953878540066453375981862846419905220452803306263695626490910827627115903856995051246529996062855443838330327638599800792922846659503551211245284087516229060262011857775313747949362055496401073001348853150735487353905602908933526400713274732621960311773433943673385759124508149335736911664541281788171454023054750667136518258284898099512139193995633241336556777098003081910272040997148687418134667006094051021462690280449159646545330107754695413088714165312544813061192407821188690056027781824235022696189344352547633573536485619363254417756613981703930632872166905722259745209192917262199844409646158269456380239502837121686446561785235565164127712826918688615572716201474934052276946595712198314943381622114006936307430444173284786101777743837977037231795255434107223445512555589998646183876764903972461167959018100035098928641204195163551108763204267612979826529425882951141275841262732790798807559751851576841264742209479721843309352972665210015662514552994745127631550917636730259462132930190402837954246323258550301096706922720227074863419005438302650681214142135057154175057508639907673946335146209082888934938376439399256900604067311422093312195936202982972351163259386772241477911629572780752395056251581603133359382311500518626890530658368129988108663263271980611271548858798093487912913707498230575929091862939195014721197586067270092547718025750337730799397134539532646195269996596385654917590458333585799102012713204583903200853878881633637685182083727885131175227769609787962142372162545214591281831798216044111311671406914827170981015457781939202311563871950805024679725792497605772625913328559726371211201905720771409148645074094926718035815157571514050397610963846755569298970383547314100223802583468767350129775413279532060971154506484212185936490997917766874774481882870632315515865032898164228288232746866106592732197907162384642153489852476216789050260998045266483929542357287343977680495774091449538391575565485459058976495198513801007958010783759945775299196700547602252552034453988712538780171960718164078124847847257912407824544361682345239570689514272269750431873633263011103053423335821609333191218806608268341428910415173247216053355849993224548730778822905252324234861531520976938461042582849714963475341837562003014915703279685301868631572488401526639835689563634657435321783493199825542117308467745297085839507616458229630324424328237737450517028560698067889521768198156710781633405266759539424926280756968326107495323390536223090807081455919837355377748742029039018142937311529334644468151212945097596534306284215319445727118614900017650558177095302468875263250119705209476159416768727784472000192789137251841622857783792284439084301181121496366424659033634194540657183544771912446621259392656620306888520055599121235363718226922531781458792593750441448933981608657900876165024635197045828895481793756681046474614105142498870252139936870509372305447734112641354892806841059107716677821238332810262185587751312721179344448201440425745083063944738363793906283008973306241380614589414227694747931665717623182472168350678076487573420491557628217583972975134478990696589532548940335615613167403276472469212505759116251529654568544633498114317670257295661844775487469378464233737238981920662048511894378868224807279352022501796545343757274163910791972952950812942922205347717304184477915673991738418311710362524395716152714669005814700002633010452643547865903290733205468338872078735444762647925297690170912007874183736735087713376977683496344252419949951388315074877537433849458259765560996555954318040920178497184685497370696212088524377013853757681416632722412634423982152941645378000492507262765150789085071265997036708726692764308377229685985169122305037462744310852934305273078865283977335246017463527703205938179125396915621063637625882937571373840754406468964783100704580613446731271591194608435935825987782835266531151065041623295329047772174083559349723758552138048305090009646676088301540612824308740645594431853413755220166305812111033453120745086824339432159043594430312431227471385842030390106070940315235556172767994160020393975099897629335325855575624808996691829864222677502360193257974726742578211119734709402357457222271212526852384295874273501563660093188045493338989741571490544182559738080871565281430102670460284316819230392535297795765862414392701549740879273131051636119137577008929564823323648298263024607975875767745377160102490804624301856524161756655600160859121534556267602192689982855377872583145144082654583484409478463178777374794653580169960779405568701192328608041130904629350871827125934668712766694873899824598527786499569165464029458935064964335809824765965165142090986755203808309203230487342703468288751604071546653834619611223013759451579252696743642531927390036038608236450762698827497618723575476762889950752114804852527950845033958570838130476937881321123674281319487950228066320170022460331989671970649163741175854851878484012054844672588851401562725019821719066960812627785485964818369621410721714214986361918774754509650308957099470934337856981674465828267911940611956037845397855839240761276344105766751024307559814552786167815949657062559755074306521085301597908073343736079432866757890533483669555486803913433720156498834220893399971641479746938696905480089193067138057171505857307148815649920714086758259602876056459782423770242469805328056632787041926768467116266879463486950464507420219373945259262668613552940624781361206202636498199999498405143868285258956342264328707663299304891723400725471764188685351372332667877921738347541480022803392997357936152412755829569276837231234798989446274330454566790062032420516396282588443085438307201495672106460533238537203143242112607424485845094580494081820927639140008540422023556260218564348994145439950410980591817948882628052066441086319001688568155169229486203010738897181007709290590480749092427141018933542818429995988169660993836961644381528877214085268088757488293258735809905670755817017949161906114001908553744882726200936685604475596557476485674008177381703307380305476973609786543859382187220583902344443508867499866506040645874346005331827436296177862518081893144363251205107094690813586440519229512932450078833398788429339342435126343365204385812912834345297308652909783300671261798130316794385535726296998740359570458452230856390098913179475948752126397078375944861139451960286751210561638976008880092746115860800207803341591451797073036835196977766076373785333012024120112046988609209339085365773222392412449051532780950955866459477634482269986074813297302630975028812103517723124465095349653693090018637764094094349837313251321862080214809922685502948454661814715557444709669530177690434272031892770604717784527939160472281534379803539679861424370956683221491465438014593829277393396032754048009552231816667380357183932757077142046723838624617803976292377131209580789363841447929802588065522129262093623930637313496640186619510811583471173312025805866727639992763579078063818813069156366274125431259589936119647626101405563503399523140323113819656236327198961837254845333702062563464223952766943568376761368711962921818754576081617053031590728828700712313666308722754918661395773730546065997437810987649802414011242142773668082751390959313404155826266789510846776118665957660165998178089414985754976284387856100263796543178313634025135814161151902096499133548733131115022700681930135929595971640197196053625033558479980963488718039111612813595968565478868325856437896173159762002419621552896297904819822199462269487137462444729093456470028537694958859591606789282491054412515996300781368367490209374915732896270028656829344431342347351239298259166739503425995868970697267332582735903121288746660451461487850346142827765991608090398652575717263081833494441820193533385071292345774375579344062178711330063106003324053991693682603746176638565758877580201229366353270267100681261825172914608202541892885935244491070138206211553827793565296914576502048643282865557934707209634807372692141186895467322767751335690190153723669036865389161291688887876407525493494249733427181178892759931596719354758988097924525262363659036320070854440784544797348291802082044926670634420437555325050527522833778887040804033531923407685630109347772125639088640413101073817853338316038135280828119040832564401842053746792992622037698718018061122624490909242641985820861751177113789051609140381575003366424156095216328197122335023167422600567941281406217219641842705784328959802882335059828208196666249035857789940333152274817776952843681630088531769694783690580671064828083598046698841098135158654906933319522394363287923990534810987830274500172065433699066117784554364687723631844464768069142828004551074686645392805399409108754939166095731619715033166968309929466349142798780842257220697148875580637480308862995118473187124777291910070227588893486939456289515802965372150409603107761289831263589964893410247036036645058687287589051406841238124247386385427908282733827973326885504935874303160274749063129572349742611221517417153133618622410913869500688835898962349276317316478340077460886655598733382113829928776911495492184192087771606068472874673681886167507221017261103830671787856694812948785048943063086169948798703160515884108282351274153538513365895332948629494495061868514779105804696039069372662670386512905201137810858616188886947957607413585534585151768051973334433495230120395770739623771316030242887200537320998253008977618973129817881944671731160647231476248457551928732782825127182446807824215216469567819294098238926284943760248852279003620219386696482215628093605373178040863727268426696421929946819214908701707533361094791381804063287387593848269535583077395761447997270003472880182785281389503217986345216111066608839314053226944905455527867894417579202440021450780192099804461382547805858048442416404775031536054906591430078158372430123137511562284015838644270890718284816757527123846782459534334449622010096071051370608461801187543120725491334994247617115633321408934609156561550600317384218701570226103101916603887064661438897736318780940711527528174689576401581047016965247557740891644568677717158500583269943401677202156767724068128366565264122982439465133197359199709403275938502669557470231813203243716420586141033606524536939160050644953060161267822648942437397166717661231048975031885732165554988342121802846912529086101485527815277625623750456375769497734336846015607727035509629049392487088406281067943622418704747008368842671022558302403599841645951122485272633632645114017395248086194635840783753556885622317115520947223065437092606797351000565549381224575483728545711797393615756167641692895805257297522338558611388322171107362265816218842443178857488798109026653793426664216990914056536432249301334867988154886628665052346997235574738424830590423677143278792316422403877764330192600192284778313837632536121025336935812624086866699738275977365682227907215832478888642369346396164363308730139814211430306008730666164803678984091335926293402304324974926887831643602681011309570716141912830686577323532639653677390317661361315965553584999398600565155921936759977717933019744688148371103206503693192894521402650915465184309936553493337183425298433679915939417466223900389527673813330617747629574943868716978453767219493506590875711917720875477107189937960894774512654757501871194870738736785890200617373321075693302216320628432065671192096950585761173961632326217708945426214609858410237813215817727602222738133495410481003073275107799948991977963883530734443457532975914263768405442264784216063122769646967156473999043715903323906560726644116438605404838847161912109008701019130726071044114143241976796828547885524779476481802959736049439700479596040292746299203572099761950140348315380947714601056333446998820822120587281510729182971211917876424880354672316916541852256729234429187128163232596965413548589577133208339911288775917226115273379010341362085614577992398778325083550730199818459025958355989260553299673770491722454935329683300002230181517226575787524058832249085821280089747909326100762578770428656006996176212176845478996440705066241710213327486796237430229155358200780141165348065647488230615003392068983794766255036549822805329662862117930628430170492402301985719978948836897183043805182174419147660429752437251683435411217038631379411422095295885798060152938752753799030938871683572095760715221900279379292786303637268765822681241993384808166021603722154710143007377537792699069587121289288019052031601285861825494413353820784883465311632650407642428390870121015194231961652268422003711230464300673442064747718021353070124098860353399152667923871101706221865883573781210935179775604425634694999787251125440854522274810914874307259869602040275941178942581281882159952359658979181144077653354321757595255536158128001163846720319346507296807990793963714961774312119402021297573125165253768017359101557338153772001952444543620071848475663415407442328621060997613243487548847434539665981338717466093020535070271952983943271425371155766600025784423031073429551533945060486222764966687624079324353192992639253731076892135352572321080889819339168668278948281170472624501948409700975760920983724090074717973340788141825195842598096241747610138252643955135259311885045636264188300338539652435997416931322894719878308427600401368074703904097238473945834896186539790594118599310356168436869219485382055780395773881360679549900085123259442529724486666766834641402189915944565309423440650667851948417766779470472041958822043295380326310537494883122180391279678446100139726753892195119117836587662528083690053249004597410947068772912328214304635337283519953648274325833119144459017809607782883583730111857543659958982724531925310588115026307542571493943024453931870179923608166611305426253995833897942971602070338767815033010280120095997252222280801423571094760351925544434929986767817891045559063015953809761875920358937341978962358931125983902598310267193304189215109689156225069659119828323455503059081730735195503721665870288053992138576037035377105178021280129566841984140362872725623214428754302210909472721073474134975514190737043318276626177275996888826027225247133683353452816692779591328861381766349857728936900965749562287103024362590772412219094300871755692625758065709912016659622436080242870024547362036394841255954881727272473653467783647201918303998717627037515724649922289467932322693619177641614618795613956699567783068290316589699430767333508234990790624100202506134057344300695745474682175690441651540636584680463692621274211075399042188716127617787014258864825775223889184599523376292377915585744549477361295525952226578636462118377598473700347971408206994145580719080213590732269233100831759510659019121294795408603640757358750205890208704579670007055262505811420663907459215273309406823649441590891009220296680523325266198911311842016291631076894084723564366808182168657219688268358402785500782804043453710183651096951782335743030504852653738073531074185917705610397395062640355442275156101107261779370634723804990666922161971194259120445084641746383589938239946517395509000859479990136026674261494290066467115067175422177038774507673563742154782905911012619157555870238957001405117822646989944917908301795475876760168094100135837613578591356924455647764464178667115391951357696104864922490083446715486383054477914330097680486878348184672733758436892724310447406807685278625585165092088263813233623148733336714764520450876627614950389949504809560460989604329123358348859990294526400284994280878624039811814884767301216754161106629995553668193123287425702063738352020086863691311733469731741219153633246745325630871347302792174956227014687325867891734558379964351358800959350877556356248810493852999007675135513527792412429277488565888566513247302514710210575352516511814850902750476845518252096331899068527614435138213662152368890578786699432288816028377482035506016029894009119713850179871683633744139275973644017007014763706655703504338121113576415018451821413619823495159601064752712575935185304332875537783057509567425442684712219618709178560783936144511383335649103256405733898667178123972237519316430617013859539474367843392670986712452211189690840236327411496601243483098929941738030588417166613073040067588380432111555379440605497721705942821514886165672771240903387727745629097110134885184374118695655449745736845218066982911045058004299887953899027804383596282409421860556287788428802127553884803728640019441614257499904272009595204654170598104989967504511936471172772220436102614079750809686975176600237187748348016120310234680567112644766123747627852190241202569943534716226660893675219833111813511146503854895025120655772636145473604426859498074396932331297127377157347099713952291182653485155587137336629120242714302503763269501350911612952993785864681307226486008270881333538193703682598867893321238327053297625857382790097826460545598555131836688844628265133798491667839409761353766251798258249663458771950124384040359140849209733754642474488176184070023569580177410177696925077814893386672557898564589851056891960924398841569280696983352240225634570497312245269354193837004843183357196516626721575524193401933099018319309196582920969656247667683659647019595754739345514337413708761517323677204227385674279170698204549953095918872434939524094441678998846319845504852393662972079777452814399418256789457795712552426826089940863317371538896262889629402112108884427376568624527612130371017300785135715404533041507959447776143597437803742436646973247138410492124314138903579092416036406314038149831481905251720937103964026808994832572297954564042701757722904173234796073618787889913318305843069394825961318713816423467218730845133877219086975104942843769325024981656673816260615941768252509993741672883951744066932549653403101452225316189009235376486378482881344209870048096227171226407489571939002918573307460104360729190945767994614929290427981687729426487729952858434647775386906950148984133924540394144680263625402118614317031251117577642829914644533408920976961699098372652361768745605894704968170136974909523072082682887890730190018253425805343421705928713931737993142410852647390948284596418093614138475831136130576108462366837237695913492615824516221552134879244145041756848064120636520170386330129532777699023118648020067556905682295016354931992305914246396217025329747573114094220180199368035026495636955866425906762685687372110339156793839895765565193177883000241613539562437777840801748819373095020699900890899328088397430367736595524891300156633294077907139615464534088791510300651321934486673248275907946807879819425019582622320395131252014109960531260696555404248670549986786923021746989009547850725672978794769888831093487464426400718183160331655511534276155622405474473378049246214952133258527698847336269182649174338987824789278468918828054669982303689939783413747587025805716349413568433929396068192061773331791738208562436433635359863494496890781064019674074436583667071586924521182997893804077137501290858646578905771426833582768978554717687184427726120509266486102051535642840632368481807287940717127966820060727559555904040233178749447346454760628189541512139162918444297651066947969354016866010055196077687335396511614930937570968554559381513789569039251014953265628147011998326992200066392875374713135236421589265126204072887716578358405219646054105435443642166562244565042999010256586927279142752931172082793937751326106052881235373451068372939893580871243869385934389175713376300720319760816604464683937725806909237297523486702916910426369262090199605204121024077648190316014085863558427609537086558164273995349346546314504040199528537252004957805254656251154109252437991326262713609099402902262062836752132305065183934057450112099341464918433323646569371725914489324159006242020612885732926133596808726500045628284557574596592120530341310111827501306961509835515632004310784601906565493806542525229161991819959602752327702249855738824899882707465936355768582560518068964285376850772012220347920993936179268206590142165615925306737944568949070853263568196831861772268249911472615732035807646298116244013316737892788689229032593349861797021994981925739617673075834417098559222170171825712777534491508205278430904619460835217402005838672849709411023266953921445461066215006410674740207009189911951376466904481267253691537162290791385403937560077835153374167747942100384002308951850994548779039346122220865060160500351776264831611153325587705073541279249909859373473787081194253055121436979749914951860535920403830235716352727630874693219622190064260886183676103346002255477477813641012691906569686495012688376296907233961276287223041141813610060264044030035996988919945827397624114613744804059697062576764723766065541618574690527229238228275186799156983390747671146103022776606020061246876477728819096791613354019881402757992174167678799231603963569492851513633647219540611171767387372555728522940054361785176502307544693869307873499110352182532929726044553210797887711449898870911511237250604238753734841257086064069052058452122754533848008205302450456517669518576913200042816758054924811780519832646032445792829730129105318385636821206215531288668564956512613892261367064093953334570526986959692350353094224543865278677673027540402702246384483553239914751363441044050092330361271496081355490531539021002299595756583705381261965683144286057956696622154721695620870013727768536960840704833325132793112232507148630206951245395003735723346807094656483089209801534878705633491092366057554050864111521441481434630437273271045027768661953107858323334857840297160925215326092558932655600672124359464255065996771770388445396181632879614460817789272171836908880126778207430106422524634807454300476492885553409062185153654355474125476152769772667769772777058315801412185688011705028365275543214803488004442979998062157904564161957212784508928489806426497427090579129069217807298769477975112447305991406050629946894280931034216416629935614828130998870745292716048433630818404126469637925843094185442216359084576146078558562473814931427078266215185541603870206876980461747400808324343665382354555109449498431093494759944672673665352517662706772194183191977196378015702169933675083760057163454643671776723387588643405644871566964321041282595645349841388412890420682047007615596916843038999348366793542549210328113363184722592305554383058206941675629992013373175489122037230349072681068534454035993561823576312837767640631013125335212141994611869350833176587852047112364331226765129964171325217513553261867681942338790365468908001827135283584888444111761234101179918709236507184857856221021104009776994453121795022479578069506532965940383987369907240797679040826794007618729547835963492793904576973661643405359792219285870574957481696694062334272619733518136626063735982575552496509807260123668283605928341855848026958413772558970883789942910549800331113884603401939166122186696058491571485733568286149500019097591125218800396419762163559375743718011480559442298730418196808085647265713547612831629200449880315402105530597076666362749328308916880932359290081787411985738317192616728834918402429721290434965526942726402559641463525914348400675867690350382320572934132981593533044446496829441367323442158380761694831219333119819061096142952201536170298575105594326461468505452684975764807808009221335811378197749271768545075538328768874474591593731162470601091244609829424841287520224462594477638749491997840446829257360968534549843266536862844489365704111817793806441616531223600214918768769467398407517176307516849856359201486892943105940202457969622924566644881967576294349535326382171613395757790766370764569570259738800438415805894336137106551859987600754924187211714889295221737721146081154344982665479872580056674724051122007383459271575727715218589946948117940644466399432370044291140747218180224825837736017346685300744985564715420036123593397312914458591522887408719508708632218837288262822884631843717261903305777147651564143822306791847386039147683108141358275755853643597721650028277803713422869688787349795096031108899196143386664068450697420787700280509367203387232629637856038653216432348815557557018469089074647879122436375556668678067610544955017260791142930831285761254481944449473244819093795369008206384631678225064809531810406570254327604385703505922818919878065865412184299217273720955103242251079718077833042609086794273428955735559252723805511440438001239041687716445180226491681641927401106451622431101700056691121733189423400547959684669804298017362570406733282129962153684881404102194463424646220745575643960452985313071409084608499653767803793201899140865814662175319337665970114330608625009829566917638846056762972931464911493704624469351984039534449135141193667933301936617663652555149174982307987072280860859626112660504289296966535652516688885572112276802772743708917389639772257564890533401038855931125679991516589025016486961427207005916056166159702451989051832969278935550303934681219761582183980483960562523091462638447386296039848924386187298507775928792722068554807210497817653286210187476766897248841139560349480376727036316921007350834073865261684507482496448597428134936480372426116704266870831925040997615319076855770327421785010006441984124207396400139603601583810565928413684574119102736420274163723488214524101347716529603128408658419787951116511529827814620379139855006399960326591248525308493690313130100799977191362230866011099929142871249388541612038020411340188887219693477904497527454288072803509305828754420755134816660927879353566521255620139988249628478726214432362853676502591450468377635282587652139156480972141929675549384375582600253168536356731379262475878049445944183429172756988376226261846365452743497662411138451305481449836311789784489732076719508784158618879692955819733250699951402601511675529750575437810242238957925786562128432731202200716730574069286869363930186765958251326499145950260917069347519408975357464016830811798846452473618956056479426358070562563281189269663026479535951097127659136233180866921535788607812759910537171402204506186075374866306350591483916467656723205714516886170790984695932236724946737583099607042589220481550799132752088583781117685214269334786921895240622657921043620348852926267984013953216458791151579050460579710838983371864038024417511347226472547010794793996953554669619726763255229914654933499663234185951450360980344092212206712567698723427940708857070474293173329188523896721971353924492426178641188637790962814486917869468177591717150669111480020759432012061969637795103227089029566085562225452602610460736131368869009281721068198618553780982018471154163630326265699283424155023600978046417108525537612728905335045506135684143775854429677977014660294387687225115363801191758154028120818255606485410787933598921064427244898618961629413418001295130683638609294100083136673372153008352696235737175330738653338204842190308186449184093723944033405244909554558016406460761581010301767488475017661908692946098769201691202181688291040870709560951470416921147027413390052253340834812870353031023919699978597413908593605433599697075604460134242453682496098772581311024732798562072126572499003468293886872304895562253204463602639854225258416464324271611419817802482595563544907219226583863662663750835944314877635156145710745528016159677048442714194435183275698407552677926411261765250615965235457187956673170913319358761628255920783080185206890151504713340386100310055914817852110384754542933389188444120517943969970194112695119526564919594189975418393234647424290702718875223534393673633663200307232747037407123982562024662651974090199762452056198557625760008708173083288344381831070054514493545885422678578551915372292379555494333410174420169600090696415612732297770221217951868376359082255128816470021992348864043959153018464004714321186360622527011541122283802778538911098490201342741014121559769965438877197485376431158229838533123071751132961904559007938064276695819014842627991221792947987348901868471676503827328552059082984529806259250352128451925927986593506132961946796252373972565584157853744567558998032405492186962888490332560851455344391660226257775512916200772796852629387937530454181080729285891989715381797343496187232927614747850192611450413274873242970583408471112333746274617274626582415324271059322506255302314738759251724787322881491455915605036334575424233779160374952502493022351481961381162563911415610326844958072508273431765944054098269765269344579863479709743124498271933113863873159636361218623497261409556079920628316999420072054811525353393946076850019909886553861433495781650089961649079678142901148387645682174914075623767618453775144031475411206760160726460556859257799322070337333398916369504346690694828436629980037414527627716547623825546170883189810868806847853705536480469350958818025360529740793538676511195079373282083146268960071075175520614433784114549950136432446328193346389050936545714506900864483440180428363390513578157273973334537284263372174065775771079830517555721036795976901889958494130195999573017901240193908681356585539661941371794487632079868800371607303220547423572266896801882123424391885984168972277652194032493227314793669234004848976059037958094696041754279613782553781223947646147832926976545162290281701100437846038756544151739433960048915318817576650500951697402415644771293656614253949368884230517400129920556854289853897942669956777027089146513736892206104415481662156804219838476730871787590279209175900695273456682026513373111518000181434120962601658629821076663523361774007837783423709152644063054071807843358061072961105550020415131696373046849213356837265400307509829089364612047891114753037049893952833457824082817386441322710002968311940203323456420826473276233830294639378998375836554559919340866235090967961134004867027123176526663710778725111860354037554487418693519733656621772359229396776463251562023487570113795712096237723431370212031004965152111976013176419408203437348512852602913334915125083119802850177855710725373149139215709105130965059885999931560863655477403551898166733535880048214665099741433761182777723351910741217572841592580872591315074606025634903777263373914461377038021318347447301113032670296917335047701632106616227830027269283365584011791419447808748253360714403296252285775009808599609040936312635621328162071453406104224112083010008587264252112262480142647519426184325853386753874054743491072710049754281159466017136122590440158991600229827801796035194080046513534752698777609527839984368086908989197839693532179980139135442552717910225397010810632143048511378291498511381969143043497500189980681644412123273328307192824362406733196554692677851193152775113446468905504248113361434984604849051258345683266441528489713972376040328212660253516693914082049947320486021627759791771234751097502403078935759937715095021751693555827072533911892334070223832077585802137174778378778391015234132098489423459613692340497998279304144463162707214796117456975719681239291913740982925805561955207434243295982898980529233366415419256367380689494201471241340525072204061794355252555225008748790086568314542835167750542294803274783044056438581591952666758282929705226127628711040134801787224801789684052407924360582742467443076721645270313451354167649668901274786801010295133862698649748212118629040337691568576240699296372493097201628707200189835423690364149270236961938547372480329855045112089192879829874467864129159417531675602533435310626745254507114181483239880607297140234725520713490798398982355268723950909365667878992383712578976248755990443228895388377317348941122757071410959790047919301046740750411435381782464630795989555638991884773781341347070246747362112048986226991888517456251732519341352038115863350123913054441910073628447567514161050410973505852762044489190978901984315485280533985777844313933883994310444465669244550885946314081751220331390681596592510546858013133838152176418210433429788826119630443111388796258746090226130900849975430395771243230616906262919403921439740270894777663702488155499322458825979020631257436910946393252806241642476868495455324938017639371615636847859823715902385421265840615367228607131702674740131145261063765383390315921943469817605358380310612887852051546933639241088467632009567089718367490578163085158138161966882222047570437590614338040725853862083565176998426774523195824182683698270160237414938363496629351576854061397342746470899685618170160551104880971554859118617189668025973541705423985135560018720335079060946421271143993196046527424050882225359773481519135438571253258540493946010865793798058620143366078825219717809025817370870916460452727977153509910340736425020386386718220522879694458387652947951048660717390229327455426785669776865939923416834122274663015062155320502655341460995249356050854921756549134830958906536175693817637473644183378974229700703545206663170929607591989627732423090252397443861014263098687733913882518684316501027964911497737582888913450341148865948670215492101084328080783428089417298008983297536940644969903125399863919581601468995220880662285408414864274786281975546629278814621607171381880180840572084715868906836919393381864278454537956719272397972364651667592011057995663962598535512763558768140213409829016296873429850792471846056874828331381259161962476156902875901072733103299140623864608333378638257926302391590003557609032477281338887339178096966601469615031754226751125993315529674213336300222964906480934582008181061802100227664580400278213336758573019011371754672763059044353131319036092489097246427928455549913490005180295707082919052556781889913899625138662319380053611346224294610248954072404857123256628888931722116432947816190554868054943441034090680716088028227959686950133643814268252170472870863010137301155236861416908375675747637239763185757038109443390564564468524183028148107998376918512127201935044041804604721626939445788377090105974693219720558114078775989772072009689382249303236830515862657281114637996983137517937623215111252349734305240622105244234353732905655163406669506165892878218707756794176080712973781335187117931650033155523822487730653444179453415395202424449703410120874072188109388268167512042299404948179449472732894770111574139441228455521828424922240658752689172272780607116754046973008037039618787796694882555614674384392570115829546661358678671897661297311267200072971553613027503556167817765442287442114729881614802705243806817653573275578602505847084013208837932816008769081300492491473682517035382219619039014999523495387105997351143478292339499187936608692301375596368532373806703591144243268561512109404259582639301678017128669239283231057658851714020211196957064799814031505633045141564414623163763809904402816256917576489142569714163598439317433270237812336938043012892626375382667795034169334323607500248175741808750388475094939454896209740485442635637164995949920980884294790363666297526003243856352945844728944547166209297495496616877414120882130477022816116456044007236351581149729739218966737382647204722642221242016560150284971306332795814302516013694825567014780935790889657134926158161346901806965089556310121218491805847922720691871696316330044858020102860657858591269974637661741463934159569539554203314628026518951167938074573315759846086173702687867602943677780500244673391332431669880354073232388281847501051641331189537036488422690270478052742490603492082954755054003457160184072574536938145531175354210726557835615499874447480427323457880061873149341566046352979779455075359304795687209316724536547208381685855606043801977030764246083489876101345709394877002946175792061952549255757109038525171488525265671045349813419803390641529876343695420256080277614421914318921393908834543131769685101840103844472348948869520981943531906506555354617335814045544837884752526253949665869992058417652780125341033896469818642430034146791380619028059607854888010789705516946215228773090104467462497979992627120951684779568482583341402266477210843362437593741610536734041954738964197895425335036301861400951534766961476255651873823292468547356935802896011536791787303553159378363082248615177770541577576561759358512016692943111138863582159667618830326104164651714846979385422621687161400122378213779774131268977266712992025922017408770076956283473932201088159356286281928563571893384958850603853158179760679479840878360975960149733420572704603521790605647603285569276273495182203236144112584182426247712012035776388895974318232827871314608053533574494297621796789034568169889553518504478325616380709476951699086247100019748809205009521943632378719764870339223811540363475488626845956159755193765410115014067001226927474393888589943859730245414801061235908036274585288493563251585384383242493252666087588908318700709100237377106576985056433928854337658342596750653715005333514489908293887737352051459333049626531415141386124437935885070944688045486975358170212908490787347806814366323322819415827345671356443171537967818058195852464840084032909981943781718177302317003989733050495387356116261023999433259780126893432605584710278764901070923443884634011735556865903585244919370181041626208504299258697435817098133894045934471937493877624232409852832762266604942385129709453245586252103600829286649724174919141988966129558076770979594795306013119159011773943104209049079424448868513086844493705909026006120649425744710353547657859242708130410618546219881830090634588187038755856274911587375421064667951346487586771543838018521348281915812462599335160198935595167968932852205824799421034512715877163345222995418839680448835529753361286837225935390079201666941339091168758803988828869216002373257361588207163516271332810518187602104852180675526648673908900907195138058626735124312215691637902277328705410842037841525683288718046987952513073266340278519059417338920358540395677035611329354482585628287610610698229721420961993509331312171187891078766872044548876089410174798647137882462153955933333275562009439580434537919782280590395959927436913793778664940964048777841748336432684026282932406260081908081804390914556351936856063045089142289645219987798849347477729132797266027658401667890136490508741142126861969862044126965282981087045479861559545338021201155646979976785738920186243599326777689454060508218838227909833627167124490026761178498264377033002081844590009717235204331994708242098771514449751017055643029542821819670009202515615844174205933658148134902693111517093872260026458630561325605792560927332265579346280805683443921373688405650434307396574061017779370141424615493070741360805442100295600095663588977899267630517718781943706761498217564186590116160865408635391513039201316805769034172596453692350806417446562351523929050409479953184074862151210561833854566176652606393713658802521666223576132201941701372664966073252010771947931265282763302413805164907174565964853748354669194523580315301969160480994606814904037819829732360930087135760798621425422096419004367905479049930078372421581954535418371129368658430553842717628035279128821129308351575656599944741788438381565148434229858704245592434693295232821803508333726283791830216591836181554217157448465778420134329982594566884558266171979012180849480332448787258183774805522268151011371745368417870280274452442905474518234674919564188551244421337783521423865979925988203287085109338386829906571994614906290257427686038850511032638544540419184958866538545040571323629681069146814847869659166861842756798460041868762298055562963045953227923051616721591968675849523635298935788507746081537321454642984792310511676357749494622952569497660359473962430995343310404994209677883827002714478494069037073249106444151696053256560586778757417472110827435774315194060757983563629143326397812218946287447798119807225646714664054850131009656786314880090303749338875364183165134982546694673316118123364854397649325026179549357204305402182974871251107404011611405899911093062492312813116340549262571356721818628932786138833718028535056503591952741400869510926167541476792668032109237467087213606278332922386413619594121339278036118276324106004740971111048140003623342714514483334641675466354699731494756643423659493496845884551524150756376605086632827424794136062876041290644913828519456402643153225858624043141838669590633245063000392213192647625962691510904457695301444054618037857503036686212462278639752746667870121003392984873375014475600322100622358029343774955032037012738468163061026570300872275462966796880890587127676361066225722352229739206443093524327228100859973095132528630601105497915644791845004618046762408928925680912930592960642357021061524646205023248966593987324933967376952023991760898474571843531936646529125848064480196520162838795189499336759241485626136995945307287254532463291529110128763770605570609531377527751867923292134955245133089867969165129073841302167573238637575820080363575728002754490327953079900799442541108725693188014667935595834676432868876966610097395749967836593397846346959948950610490383647409504695226063858046758073069912290474089879166872117147527644711604401952718169508289733537148530928937046384420893299771125856840846608339934045689026787516008775461267988015465856522061210953490796707365539702576199431376639960606061106406959330828171876426043573425361756943784848495250108266488395159700490598380812105221111091943323951136051446459834210799058082093716464523127704023160072138543723461267260997870385657091998507595634613248460188409850194287687902268734556500519121546544063829253851276317663922050938345204300773017029940362615434001322763910912988327863920412300445551684054889809080779174636092439334912641164240093880746356607262336695842764583698268734815881961058571835767462009650526065929263548291499045768307210893245857073701660717398194485028842603963660746031184786225831056580870870305567595861341700745402965687634774176431051751036732869245558582082372038601781739405175130437994868822320044378043103170921034261674998000073016094814586374488778522273076330495383944345382770608760763542098445008306247630253572781032783461766970544287155315340016497076657195985041748199087201490875686037783591994719343352772947285537925787684832301101859365800717291186967617655053775030293033830706448912811412025506150896411007623824574488655182581058140345320124754723269087547507078577659732542844459353044992070014538748948226556442223696365544194225441338212225477497535494624827680533336983284156138692363443358553868471111430498248398991803165458638289353799130535222833430137953372954016257623228081138499491876144141322933767106563492528814528239506209022357876684650116660097382753660405446941653422239052108314585847035529352219928272760574821266065291385530345549744551470344939486863429459658431024190785923680224560763936784166270518555178702904073557304620639692453307795782245949710420188043000183881429008173039450507342787013124466860092778581811040911511729374873627887874907465285565434748886831064110051023020875107768918781525622735251550379532444857787277617001964853703555167655209119339343762866284619844026295252183678522367475108809781507098978413086245881522660963551401874495836926917799047120726494905737264286005211403581231076006699518536124862746756375896225299116496066876508261734178484789337295056739007878617925351440621045366250640463728815698232317500596261080921955211150859302955654967538862612972339914628358476048627627027309739202001432248707582337354915246085608210328882974183906478869923273691360048837436615223517058437705545210815513361262142911815615301758882573594892507108879262128641392443309383797333867806131795237315266773820858024701433527009243803266951742119507670884326346442749127558907746863582162166042741315170212458586056233631493164646913946562497471741958354218607748711057338458433689939645913740603382159352243594751626239188685307822821763983237306180204246560477527943104796189724299533029792497481684052893791044947004590864991872727345413508101983881864673609392571930511968645601855782450218231065889437986522432050677379966196955472440585922417953006820451795370043472451762893566770508490213107736625751697335527462302943031203596260953423574397249659211010657817826108745318874803187430823573699195156340957162700992444929749105489851519658664740148225106335367949737142510229341882585117371994499115097583746130105505064197721531929354875371191630262030328588658528480193509225875775597425276584011721342323648084027143356367542046375182552524944329657043861387865901965738802868401894087672816714137033661732650120578653915780703088714261519075001492576112927675193096728453971160213606303090542243966320674323582797889332324405779199278484633339777737655901870574806828678347965624146102899508487399692970750432753029972872297327934442988646412725348160603779707298299173029296308695801996312413304939350493325412355071054461182591141116454534710329881047844067780138077131465400099386306481266614330858206811395838319169545558259426895769841428893743467084107946318932539106963955780706021245974898293564613560788983472419979478564362042094613412387613198865352358312996862268948608408456655606876954501274486631405054735351746873009806322780468912246821460806727627708402402266155485024008952891657117617439020337584877842911289623247059191874691042005848326140677333751027195653994697162517248312230633919328707983800748485726516123434933273356664473358556430235280883924348278760886164943289399166399210488307847777048045728491456303353265070029588906265915498509407972767567129795010098229476228961891591441520032283878773485130979081019129267227103778898053964156362364169154985768408398468861684375407065121039062506128107663799047908879674778069738473170475253442156390387201238806323688037017949308954900776331523063548374256816653361606641980030188287123767481898330246836371488309259283375902278942588060087286038859168849730693948020511221766359138251524278670094406942355120201568377778851824670025651708509249623747726813694284350062938814429987905301056217375459182679973217735029368928065210025396268807498092643458011655715886700443503976505323478287327368840863540002740676783821963522226539290939807367391364082898722017776747168118195856133721583119054682936083236976113450281757830202934845982925000895682630271263295866292147653142233351793093387951357095346377183684092444422096319331295620305575517340067973740614162107923633423805646850092037167152642556371853889571416419772387422610596667396997173168169415435095283193556417705668622215217991151355639707143312893657553844648326201206424338016955862698561022460646069330793847858814367407000599769703649019273328826135329363112403650698652160638987250267238087403396744397830258296894256896741864336134979475245526291426522842419243083388103580053787023999542172113686550275341362211693140694669513186928102574795985605145005021715913317751609957865551981886193211282110709442287240442481153406055895958355815232012184605820563592699303478851132068626627588771446035996656108430725696500563064489187599466596772847171539573612108180841547273142661748933134174632662354222072600146012701206934639520564445543291662986660783089068118790090815295063626782075614388815781351134695366303878412092346942868730839320432333872775496805210302821544324723388845215343727250128589747691460808314404125868181540049187772287869801853454537006526655649170915429522756709222217474112062720656622989806032891672068743654948246108697367225547404812889242471854323605753411672850757552057131156697954584887398742228135887985840783135060548290551482785294891121905383195624228719484759407859398047901094194070671764439032730712135887385049993638838205501683402777496070276844880281912220636888636811043569529300652195528261526991271637277388418993287130563464688227398288763198645709836308917786487086676185485680047672552675414742851028145807403152992197814557756843681110185317498167016426647884090262682824448258027532094549915104518517716546311804904567985713257528117913656278158111288816562285876030875974963849435275676612168959261485030785362045274507752950631012480341804584059432926079854435620093708091821523920371790678121992280496069738238743312626730306795943960954957189577217915597300588693646845576676092450906088202212235719254536715191834872587423919410890444115959932760044506556206461164655665487594247369252336955993030355095817626176231849561906494839673002037763874369343999829430209147073618947932692762445186560239559053705128978163455423320114975994896278424327483788032701418676952621180975006405149755889650293004867605208010491537885413909424531691719987628941277221129464568294860281493181560249677887949813777216229359437811004448060797672429276249510784153446429150842764520002042769470698041775832209097020291657347251582904630910359037842977572651720877244740952267166306005469716387943171196873484688738186656751279298575016363411314627530499019135646823804329970695770150789337728658035712790913767420805655493624646 diff --git a/internal/compress/testdata/pngdata.bin b/internal/compress/testdata/pngdata.bin new file mode 100644 index 00000000..f48a75c9 Binary files /dev/null and b/internal/compress/testdata/pngdata.bin differ diff --git a/internal/compress/testdata/sharnd.out b/internal/compress/testdata/sharnd.out new file mode 100644 index 00000000..b474465e Binary files /dev/null and b/internal/compress/testdata/sharnd.out differ diff --git a/internal/compress/zlib/reader.go b/internal/compress/zlib/reader.go new file mode 100644 index 00000000..2d009887 --- /dev/null +++ b/internal/compress/zlib/reader.go @@ -0,0 +1,205 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +/* +Package zlib implements reading and writing of zlib format compressed data, +as specified in RFC 1950. + +This package differs from the standard library's compress/zlib package +in that it pools readers and writers to reduce allocations. + +Note that closing a reader or writer causes it to be returned to a pool +for reuse. Therefore, the caller must not retain references to a +reader or writer after closing it; in the standard library's +compress/zlib package, it is legal to Reset a closed reader or writer +and continue using it; that is not allowed here, so there is simply no +Resetter interface. + +The implementation provides filters that uncompress during reading +and compress during writing. For example, to write compressed data +to a buffer: + + var b bytes.Buffer + w := zlib.NewWriter(&b) + w.Write([]byte("hello, world\n")) + w.Close() + +and to read that data back: + + r, err := zlib.NewReader(&b) + io.Copy(os.Stdout, r) + r.Close() +*/ +package zlib + +import ( + "encoding/binary" + "errors" + "hash" + "io" + "sync" + + "codeberg.org/lindenii/furgit/internal/compress/flate" +) + +const ( + zlibDeflate = 8 + zlibMaxWindow = 7 +) + +var ( + // ErrChecksum is returned when reading ZLIB data that has an invalid checksum. + ErrChecksum = errors.New("zlib: invalid checksum") + // ErrDictionary is returned when reading ZLIB data that has an invalid dictionary. + ErrDictionary = errors.New("zlib: invalid dictionary") + // ErrHeader is returned when reading ZLIB data that has an invalid header. + ErrHeader = errors.New("zlib: invalid header") +) + +var readerPool = sync.Pool{ + New: func() any { + r := new(Reader) + + return r + }, +} + +// Reader reads and verifies one zlib stream. +// +// Reader implements io.ReadCloser. +type Reader struct { + r flate.Reader + decompressor io.ReadCloser + digest hash.Hash32 + counter *countingFlateReader + err error + scratch [4]byte +} + +// countingFlateReader wraps flate input and tracks consumed bytes. +type countingFlateReader struct { + inner flate.Reader + read uint64 +} + +// Read implements io.Reader. +func (reader *countingFlateReader) Read(dst []byte) (int, error) { + n, err := reader.inner.Read(dst) + reader.read += uint64(n) + + return n, err +} + +// ReadByte implements io.ByteReader. +func (reader *countingFlateReader) ReadByte() (byte, error) { + b, err := reader.inner.ReadByte() + if err == nil { + reader.read++ + } + + return b, err +} + +// NewReader creates a new ReadCloser. +// Reads from the returned ReadCloser read and decompress data from r. +// If r does not implement [io.ByteReader], the decompressor may read more +// data than necessary from r. +// It is the caller's responsibility to call Close on the ReadCloser when done. +func NewReader(r io.Reader) (*Reader, error) { + return NewReaderDict(r, nil) +} + +// NewReaderDict is like [NewReader] but uses a preset dictionary. +// NewReaderDict ignores the dictionary if the compressed data does not refer to it. +// If the compressed data refers to a different dictionary, NewReaderDict returns [ErrDictionary]. +func NewReaderDict(r io.Reader, dict []byte) (*Reader, error) { + v := readerPool.Get() + + z, ok := v.(*Reader) + if !ok { + panic("zlib: pool returned unexpected type") + } + + err := z.reset(r, dict) + if err != nil { + return nil, err + } + + return z, nil +} + +// Read decompresses bytes from receiver into p. +func (z *Reader) Read(p []byte) (int, error) { + if z.err != nil { + return 0, z.err + } + + var n int + + n, z.err = z.decompressor.Read(p) + + _, err := z.digest.Write(p[0:n]) + if err != nil { + z.err = err + + return n, z.err + } + + if !errors.Is(z.err, io.EOF) { + // In the normal case we return here. + return n, z.err + } + + // Finished file; check checksum. + _, err = io.ReadFull(z.r, z.scratch[0:4]) + if err != nil { + if errors.Is(err, io.EOF) { + err = io.ErrUnexpectedEOF + } + + z.err = err + + return n, z.err + } + // ZLIB (RFC 1950) is big-endian, unlike GZIP (RFC 1952). + checksum := binary.BigEndian.Uint32(z.scratch[:4]) + if checksum != z.digest.Sum32() { + z.err = ErrChecksum + + return n, z.err + } + + return n, io.EOF +} + +// InputConsumed returns compressed bytes consumed from stream input. +// +// This count includes the zlib header, deflate payload, and zlib checksum +// trailer bytes read by the reader. +func (z *Reader) InputConsumed() uint64 { + if z.counter == nil { + return 0 + } + + return z.counter.read +} + +// Close does not close the wrapped [io.Reader] originally passed to [NewReader]. +// In order for the ZLIB checksum to be verified, the reader must be +// fully consumed until the [io.EOF]. +// Close returns the instance to a global pool; you MUST NOT keep references after Close. +func (z *Reader) Close() error { + if z.err != nil && !errors.Is(z.err, io.EOF) { + return z.err + } + + z.err = z.decompressor.Close() + if z.err != nil { + return z.err + } + + readerPool.Put(z) + + return nil +} diff --git a/internal/compress/zlib/reader_reset.go b/internal/compress/zlib/reader_reset.go new file mode 100644 index 00000000..f374111c --- /dev/null +++ b/internal/compress/zlib/reader_reset.go @@ -0,0 +1,92 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package zlib + +import ( + "bufio" + "encoding/binary" + "errors" + "io" + + "codeberg.org/lindenii/furgit/internal/adler32" + "codeberg.org/lindenii/furgit/internal/compress/flate" +) + +// reset resets receiver to read a new zlib stream. +func (z *Reader) reset(r io.Reader, dict []byte) error { + *z = Reader{decompressor: z.decompressor} + + var input flate.Reader + if fr, ok := r.(flate.Reader); ok { + input = fr + } else { + input = bufio.NewReader(r) + } + + z.counter = &countingFlateReader{inner: input} + z.r = z.counter + + // Read the header (RFC 1950 section 2.2.). + _, z.err = io.ReadFull(z.r, z.scratch[0:2]) + if z.err != nil { + if errors.Is(z.err, io.EOF) { + z.err = io.ErrUnexpectedEOF + } + + return z.err + } + + h := binary.BigEndian.Uint16(z.scratch[:2]) + if (z.scratch[0]&0x0f != zlibDeflate) || (z.scratch[0]>>4 > zlibMaxWindow) || (h%31 != 0) { + z.err = ErrHeader + + return z.err + } + + haveDict := z.scratch[1]&0x20 != 0 + if haveDict { + _, z.err = io.ReadFull(z.r, z.scratch[0:4]) + if z.err != nil { + if errors.Is(z.err, io.EOF) { + z.err = io.ErrUnexpectedEOF + } + + return z.err + } + + checksum := binary.BigEndian.Uint32(z.scratch[:4]) + if checksum != adler32.Checksum(dict) { + z.err = ErrDictionary + + return z.err + } + } + + if z.decompressor != nil { + resetter, ok := z.decompressor.(flate.Resetter) + if !ok { + panic("zlib: pooled decompressor does not implement flate.Resetter") + } + + z.err = resetter.Reset(z.r, dict) + if z.err != nil { + return z.err + } + + z.digest = adler32.New() + + return nil + } + + if haveDict { + z.decompressor = flate.NewReaderDict(z.r, dict) + } else { + z.decompressor = flate.NewReader(z.r) + } + + z.digest = adler32.New() + + return nil +} diff --git a/internal/compress/zlib/reader_test.go b/internal/compress/zlib/reader_test.go new file mode 100644 index 00000000..9b534b1e --- /dev/null +++ b/internal/compress/zlib/reader_test.go @@ -0,0 +1,193 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package zlib + +import ( + "bytes" + "errors" + "io" + "testing" +) + +type zlibTest struct { + desc string + raw string + compressed []byte + dict []byte + err error +} + +// Compare-to-golden test data was generated by the ZLIB example program at +// https://www.zlib.net/zpipe.c + +var zlibTests = []zlibTest{ + { + "truncated empty", + "", + []byte{}, + nil, + io.ErrUnexpectedEOF, + }, + { + "truncated dict", + "", + []byte{0x78, 0xbb}, + []byte{0x00}, + io.ErrUnexpectedEOF, + }, + { + "truncated checksum", + "", + []byte{0x78, 0xbb, 0x00, 0x01, 0x00, 0x01, 0xca, 0x48, + 0xcd, 0xc9, 0xc9, 0xd7, 0x51, 0x28, 0xcf, 0x2f, + 0xca, 0x49, 0x01, 0x04, 0x00, 0x00, 0xff, 0xff, + }, + []byte{0x00}, + io.ErrUnexpectedEOF, + }, + { + "empty", + "", + []byte{0x78, 0x9c, 0x03, 0x00, 0x00, 0x00, 0x00, 0x01}, + nil, + nil, + }, + { + "goodbye", + "goodbye, world", + []byte{ + 0x78, 0x9c, 0x4b, 0xcf, 0xcf, 0x4f, 0x49, 0xaa, + 0x4c, 0xd5, 0x51, 0x28, 0xcf, 0x2f, 0xca, 0x49, + 0x01, 0x00, 0x28, 0xa5, 0x05, 0x5e, + }, + nil, + nil, + }, + { + "bad header (CINFO)", + "", + []byte{0x88, 0x98, 0x03, 0x00, 0x00, 0x00, 0x00, 0x01}, + nil, + ErrHeader, + }, + { + "bad header (FCHECK)", + "", + []byte{0x78, 0x9f, 0x03, 0x00, 0x00, 0x00, 0x00, 0x01}, + nil, + ErrHeader, + }, + { + "bad checksum", + "", + []byte{0x78, 0x9c, 0x03, 0x00, 0x00, 0x00, 0x00, 0xff}, + nil, + ErrChecksum, + }, + { + "not enough data", + "", + []byte{0x78, 0x9c, 0x03, 0x00, 0x00, 0x00}, + nil, + io.ErrUnexpectedEOF, + }, + { + "excess data is silently ignored", + "", + []byte{ + 0x78, 0x9c, 0x03, 0x00, 0x00, 0x00, 0x00, 0x01, + 0x78, 0x9c, 0xff, + }, + nil, + nil, + }, + { + "dictionary", + "Hello, World!\n", + []byte{ + 0x78, 0xbb, 0x1c, 0x32, 0x04, 0x27, 0xf3, 0x00, + 0xb1, 0x75, 0x20, 0x1c, 0x45, 0x2e, 0x00, 0x24, + 0x12, 0x04, 0x74, + }, + []byte{ + 0x48, 0x65, 0x6c, 0x6c, 0x6f, 0x20, 0x57, 0x6f, 0x72, 0x6c, 0x64, 0x0a, + }, + nil, + }, + { + "wrong dictionary", + "", + []byte{ + 0x78, 0xbb, 0x1c, 0x32, 0x04, 0x27, 0xf3, 0x00, + 0xb1, 0x75, 0x20, 0x1c, 0x45, 0x2e, 0x00, 0x24, + 0x12, 0x04, 0x74, + }, + []byte{ + 0x48, 0x65, 0x6c, 0x6c, + }, + ErrDictionary, + }, + { + "truncated zlib stream amid raw-block", + "hello", + []byte{ + 0x78, 0x9c, 0x00, 0x0c, 0x00, 0xf3, 0xff, 0x68, 0x65, 0x6c, 0x6c, 0x6f, + }, + nil, + io.ErrUnexpectedEOF, + }, + { + "truncated zlib stream amid fixed-block", + "He", + []byte{ + 0x78, 0x9c, 0xf2, 0x48, 0xcd, + }, + nil, + io.ErrUnexpectedEOF, + }, +} + +func TestDecompressor(t *testing.T) { + b := new(bytes.Buffer) + for _, tt := range zlibTests { + in := bytes.NewReader(tt.compressed) + + zr, err := NewReaderDict(in, tt.dict) + if err != nil { + if !errors.Is(err, tt.err) { + t.Errorf("%s: NewReader: %s", tt.desc, err) + } + + continue + } + defer zr.Close() + + // Read and verify correctness of data. + b.Reset() + + n, err := io.Copy(b, zr) + if err != nil { + if !errors.Is(err, tt.err) { + t.Errorf("%s: io.Copy: %v want %v", tt.desc, err, tt.err) + } + + continue + } + + s := b.String() + if s != tt.raw { + t.Errorf("%s: got %d-byte %q want %d-byte %q", tt.desc, n, s, len(tt.raw), tt.raw) + } + + // Check for sticky errors. + if n, err := zr.Read([]byte{0}); n != 0 || !errors.Is(err, io.EOF) { + t.Errorf("%s: Read() = (%d, %v), want (0, io.EOF)", tt.desc, n, err) + } + + if err := zr.Close(); err != nil { + t.Errorf("%s: Close() = %v, want nil", tt.desc, err) + } + } +} diff --git a/internal/compress/zlib/writer.go b/internal/compress/zlib/writer.go new file mode 100644 index 00000000..8a5562fb --- /dev/null +++ b/internal/compress/zlib/writer.go @@ -0,0 +1,204 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package zlib + +import ( + "encoding/binary" + "fmt" + "hash" + "io" + "sync" + + "codeberg.org/lindenii/furgit/internal/compress/flate" +) + +// These constants are copied from the [flate] package, so that code that imports +// [compress/zlib] does not also have to import [compress/flate]. +const ( + NoCompression = flate.NoCompression + BestSpeed = flate.BestSpeed + BestCompression = flate.BestCompression + DefaultCompression = flate.DefaultCompression + HuffmanOnly = flate.HuffmanOnly +) + +// A Writer takes data written to it and writes the compressed +// form of that data to an underlying writer (see [NewWriter]). +type Writer struct { + w io.Writer + level int + dict []byte + compressor *flate.Writer + digest hash.Hash32 + err error + scratch [4]byte + wroteHeader bool +} + +var writerPool = sync.Pool{ + New: func() any { + return new(Writer) + }, +} + +// NewWriter creates a new [Writer]. +// Writes to the returned Writer are compressed and written to w. +// +// It is the caller's responsibility to call Close on the Writer when done. +// Writes may be buffered and not flushed until Close. +func NewWriter(w io.Writer) *Writer { + z, _ := NewWriterLevelDict(w, DefaultCompression, nil) + + return z +} + +// NewWriterLevel is like [NewWriter] but specifies the compression level instead +// of assuming [DefaultCompression]. +// +// The compression level can be [DefaultCompression], [NoCompression], [HuffmanOnly] +// or any integer value between [BestSpeed] and [BestCompression] inclusive. +// The error returned will be nil if the level is valid. +func NewWriterLevel(w io.Writer, level int) (*Writer, error) { + return NewWriterLevelDict(w, level, nil) +} + +// NewWriterLevelDict is like [NewWriterLevel] but specifies a dictionary to +// compress with. +// +// The dictionary may be nil. If not, its contents should not be modified until +// the Writer is closed. +func NewWriterLevelDict(w io.Writer, level int, dict []byte) (*Writer, error) { + if level < HuffmanOnly || level > BestCompression { + return nil, fmt.Errorf("zlib: invalid compression level: %d", level) + } + + v := writerPool.Get() + + z, ok := v.(*Writer) + if !ok { + panic("zlib: pool returned unexpected type") + } + + // flate.Writer can only be Reset with the same level/dictionary mode. + // Reuse it only when the configuration is unchanged and dictionary-free. + reuseCompressor := z.compressor != nil && z.level == level && z.dict == nil && dict == nil + if !reuseCompressor { + z.compressor = nil + } + + if z.digest != nil { + z.digest.Reset() + } + + *z = Writer{ + w: w, + level: level, + dict: dict, + compressor: z.compressor, + digest: z.digest, + } + if z.compressor != nil { + z.compressor.Reset(w) + } + + return z, nil +} + +// Reset clears the state of the [Writer] z such that it is equivalent to its +// initial state from [NewWriterLevel] or [NewWriterLevelDict], but instead writing +// to w. +func (z *Writer) Reset(w io.Writer) { + z.w = w + // z.level and z.dict left unchanged. + if z.compressor != nil { + z.compressor.Reset(w) + } + + if z.digest != nil { + z.digest.Reset() + } + + z.err = nil + z.scratch = [4]byte{} + z.wroteHeader = false +} + +// Write writes a compressed form of p to the underlying [io.Writer]. The +// compressed bytes are not necessarily flushed until the [Writer] is closed or +// explicitly flushed. +func (z *Writer) Write(p []byte) (n int, err error) { + if !z.wroteHeader { + z.err = z.writeHeader() + } + + if z.err != nil { + return 0, z.err + } + + if len(p) == 0 { + return 0, nil + } + + n, err = z.compressor.Write(p) + if err != nil { + z.err = err + + return n, err + } + + _, err = z.digest.Write(p) + if err != nil { + z.err = err + + return 0, z.err + } + + return n, err +} + +// Flush flushes the Writer to its underlying [io.Writer]. +func (z *Writer) Flush() error { + if !z.wroteHeader { + z.err = z.writeHeader() + } + + if z.err != nil { + return z.err + } + + z.err = z.compressor.Flush() + + return z.err +} + +// Close closes the Writer, flushing any unwritten data to the underlying +// [io.Writer], but does not close the underlying io.Writer. +func (z *Writer) Close() error { + if !z.wroteHeader { + z.err = z.writeHeader() + } + + if z.err != nil { + return z.err + } + + z.err = z.compressor.Close() + if z.err != nil { + return z.err + } + + checksum := z.digest.Sum32() + // ZLIB (RFC 1950) is big-endian, unlike GZIP (RFC 1952). + binary.BigEndian.PutUint32(z.scratch[:], checksum) + + _, z.err = z.w.Write(z.scratch[0:4]) + if z.err != nil { + return z.err + } + + writerPool.Put(z) + + return nil +} diff --git a/internal/compress/zlib/writer_header.go b/internal/compress/zlib/writer_header.go new file mode 100644 index 00000000..43d3bdf5 --- /dev/null +++ b/internal/compress/zlib/writer_header.go @@ -0,0 +1,71 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package zlib + +import ( + "encoding/binary" + + "codeberg.org/lindenii/furgit/internal/adler32" + "codeberg.org/lindenii/furgit/internal/compress/flate" +) + +// writeHeader writes the ZLIB header. +func (z *Writer) writeHeader() (err error) { + z.wroteHeader = true + // ZLIB has a two-byte header (as documented in RFC 1950). + // The first four bits is the CINFO (compression info), which is 7 for the default deflate window size. + // The next four bits is the CM (compression method), which is 8 for deflate. + z.scratch[0] = 0x78 + // The next two bits is the FLEVEL (compression level). The four values are: + // 0=fastest, 1=fast, 2=default, 3=best. + // The next bit, FDICT, is set if a dictionary is given. + // The final five FCHECK bits form a mod-31 checksum. + switch z.level { + case -2, 0, 1: + z.scratch[1] = 0 << 6 + case 2, 3, 4, 5: + z.scratch[1] = 1 << 6 + case 6, -1: + z.scratch[1] = 2 << 6 + case 7, 8, 9: + z.scratch[1] = 3 << 6 + default: + panic("unreachable") + } + + if z.dict != nil { + z.scratch[1] |= 1 << 5 + } + + z.scratch[1] += uint8(31 - binary.BigEndian.Uint16(z.scratch[:2])%31) //#nosec G115 + + _, err = z.w.Write(z.scratch[0:2]) + if err != nil { + return err + } + + if z.dict != nil { + // The next four bytes are the Adler-32 checksum of the dictionary. + binary.BigEndian.PutUint32(z.scratch[:], adler32.Checksum(z.dict)) + + _, err = z.w.Write(z.scratch[0:4]) + if err != nil { + return err + } + } + + if z.compressor == nil { + // Initialize deflater unless the Writer is being reused + // after a Reset call. + z.compressor, err = flate.NewWriterDict(z.w, z.level, z.dict) + if err != nil { + return err + } + + z.digest = adler32.New() + } + + return nil +} diff --git a/internal/compress/zlib/writer_test.go b/internal/compress/zlib/writer_test.go new file mode 100644 index 00000000..6deffdb8 --- /dev/null +++ b/internal/compress/zlib/writer_test.go @@ -0,0 +1,185 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package zlib + +import ( + "bytes" + "fmt" + "io" + "os" + "testing" +) + +var filenames = []string{ + "../testdata/gettysburg.txt", + "../testdata/e.txt", + "../testdata/pi.txt", +} + +var data = []string{ + "test a reasonable sized string that can be compressed", +} + +// Tests that compressing and then decompressing the given file at the given compression level and dictionary +// yields equivalent bytes to the original file. +func testFileLevelDict(t *testing.T, fn string, level int, d string) { + // Read the file, as golden output. + golden, err := os.Open(fn) + if err != nil { + t.Errorf("%s (level=%d, dict=%q): %v", fn, level, d, err) + + return + } + defer golden.Close() + + b0, err0 := io.ReadAll(golden) + if err0 != nil { + t.Errorf("%s (level=%d, dict=%q): %v", fn, level, d, err0) + + return + } + + testLevelDict(t, fn, b0, level, d) +} + +func testLevelDict(t *testing.T, fn string, b0 []byte, level int, d string) { + // Make dictionary, if given. + var dict []byte + if d != "" { + dict = []byte(d) + } + + // Push data through a pipe that compresses at the write end, and decompresses at the read end. + piper, pipew := io.Pipe() + defer piper.Close() + + go func() { + defer pipew.Close() + + zlibw, err := NewWriterLevelDict(pipew, level, dict) + if err != nil { + t.Errorf("%s (level=%d, dict=%q): %v", fn, level, d, err) + + return + } + defer zlibw.Close() + + _, err = zlibw.Write(b0) + if err != nil { + t.Errorf("%s (level=%d, dict=%q): %v", fn, level, d, err) + + return + } + }() + + zlibr, err := NewReaderDict(piper, dict) + if err != nil { + t.Errorf("%s (level=%d, dict=%q): %v", fn, level, d, err) + + return + } + defer zlibr.Close() + + // Compare the decompressed data. + b1, err1 := io.ReadAll(zlibr) + if err1 != nil { + t.Errorf("%s (level=%d, dict=%q): %v", fn, level, d, err1) + + return + } + + if len(b0) != len(b1) { + t.Errorf("%s (level=%d, dict=%q): length mismatch %d versus %d", fn, level, d, len(b0), len(b1)) + + return + } + + for i := range b0 { + if b0[i] != b1[i] { + t.Errorf("%s (level=%d, dict=%q): mismatch at %d, 0x%02x versus 0x%02x\n", fn, level, d, i, b0[i], b1[i]) + + return + } + } +} + +func TestWriter(t *testing.T) { + for i, s := range data { + b := []byte(s) + tag := fmt.Sprintf("#%d", i) + testLevelDict(t, tag, b, DefaultCompression, "") + testLevelDict(t, tag, b, NoCompression, "") + testLevelDict(t, tag, b, HuffmanOnly, "") + + for level := BestSpeed; level <= BestCompression; level++ { + testLevelDict(t, tag, b, level, "") + } + } +} + +func TestWriterBig(t *testing.T) { + for i, fn := range filenames { + testFileLevelDict(t, fn, DefaultCompression, "") + testFileLevelDict(t, fn, NoCompression, "") + testFileLevelDict(t, fn, HuffmanOnly, "") + + for level := BestSpeed; level <= BestCompression; level++ { + testFileLevelDict(t, fn, level, "") + + if level >= 1 && testing.Short() { + break + } + } + + if i == 0 && testing.Short() { + break + } + } +} + +func TestWriterDict(t *testing.T) { + const dictionary = "0123456789." + for i, fn := range filenames { + testFileLevelDict(t, fn, DefaultCompression, dictionary) + testFileLevelDict(t, fn, NoCompression, dictionary) + testFileLevelDict(t, fn, HuffmanOnly, dictionary) + + for level := BestSpeed; level <= BestCompression; level++ { + testFileLevelDict(t, fn, level, dictionary) + + if level >= 1 && testing.Short() { + break + } + } + + if i == 0 && testing.Short() { + break + } + } +} + +func TestWriterDictIsUsed(t *testing.T) { + var ( + input = []byte("Lorem ipsum dolor sit amet, consectetur adipisicing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua.") + buf bytes.Buffer + ) + + compressor, err := NewWriterLevelDict(&buf, BestCompression, input) + if err != nil { + t.Errorf("error in NewWriterLevelDict: %s", err) + + return + } + + compressor.Write(input) + compressor.Close() + + const expectedMaxSize = 25 + + output := buf.Bytes() + if len(output) > expectedMaxSize { + t.Errorf("result too large (got %d, want <= %d bytes). Is the dictionary being used?", len(output), expectedMaxSize) + } +} diff --git a/internal/zlib/LICENSE b/internal/zlib/LICENSE deleted file mode 100644 index 2a7cf70d..00000000 --- a/internal/zlib/LICENSE +++ /dev/null @@ -1,27 +0,0 @@ -Copyright 2009 The Go Authors. - -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are -met: - - * Redistributions of source code must retain the above copyright -notice, this list of conditions and the following disclaimer. - * Redistributions in binary form must reproduce the above -copyright notice, this list of conditions and the following disclaimer -in the documentation and/or other materials provided with the -distribution. - * Neither the name of Google LLC nor the names of its -contributors may be used to endorse or promote products derived from -this software without specific prior written permission. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/internal/zlib/reader.go b/internal/zlib/reader.go deleted file mode 100644 index c1e26aae..00000000 --- a/internal/zlib/reader.go +++ /dev/null @@ -1,205 +0,0 @@ -// Copyright 2009 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -/* -Package zlib implements reading and writing of zlib format compressed data, -as specified in RFC 1950. - -This package differs from the standard library's compress/zlib package -in that it pools readers and writers to reduce allocations. - -Note that closing a reader or writer causes it to be returned to a pool -for reuse. Therefore, the caller must not retain references to a -reader or writer after closing it; in the standard library's -compress/zlib package, it is legal to Reset a closed reader or writer -and continue using it; that is not allowed here, so there is simply no -Resetter interface. - -The implementation provides filters that uncompress during reading -and compress during writing. For example, to write compressed data -to a buffer: - - var b bytes.Buffer - w := zlib.NewWriter(&b) - w.Write([]byte("hello, world\n")) - w.Close() - -and to read that data back: - - r, err := zlib.NewReader(&b) - io.Copy(os.Stdout, r) - r.Close() -*/ -package zlib - -import ( - "encoding/binary" - "errors" - "hash" - "io" - "sync" - - "github.com/klauspost/compress/flate" -) - -const ( - zlibDeflate = 8 - zlibMaxWindow = 7 -) - -var ( - // ErrChecksum is returned when reading ZLIB data that has an invalid checksum. - ErrChecksum = errors.New("zlib: invalid checksum") - // ErrDictionary is returned when reading ZLIB data that has an invalid dictionary. - ErrDictionary = errors.New("zlib: invalid dictionary") - // ErrHeader is returned when reading ZLIB data that has an invalid header. - ErrHeader = errors.New("zlib: invalid header") -) - -var readerPool = sync.Pool{ - New: func() any { - r := new(Reader) - - return r - }, -} - -// Reader reads and verifies one zlib stream. -// -// Reader implements io.ReadCloser. -type Reader struct { - r flate.Reader - decompressor io.ReadCloser - digest hash.Hash32 - counter *countingFlateReader - err error - scratch [4]byte -} - -// countingFlateReader wraps flate input and tracks consumed bytes. -type countingFlateReader struct { - inner flate.Reader - read uint64 -} - -// Read implements io.Reader. -func (reader *countingFlateReader) Read(dst []byte) (int, error) { - n, err := reader.inner.Read(dst) - reader.read += uint64(n) - - return n, err -} - -// ReadByte implements io.ByteReader. -func (reader *countingFlateReader) ReadByte() (byte, error) { - b, err := reader.inner.ReadByte() - if err == nil { - reader.read++ - } - - return b, err -} - -// NewReader creates a new ReadCloser. -// Reads from the returned ReadCloser read and decompress data from r. -// If r does not implement [io.ByteReader], the decompressor may read more -// data than necessary from r. -// It is the caller's responsibility to call Close on the ReadCloser when done. -func NewReader(r io.Reader) (*Reader, error) { - return NewReaderDict(r, nil) -} - -// NewReaderDict is like [NewReader] but uses a preset dictionary. -// NewReaderDict ignores the dictionary if the compressed data does not refer to it. -// If the compressed data refers to a different dictionary, NewReaderDict returns [ErrDictionary]. -func NewReaderDict(r io.Reader, dict []byte) (*Reader, error) { - v := readerPool.Get() - - z, ok := v.(*Reader) - if !ok { - panic("zlib: pool returned unexpected type") - } - - err := z.reset(r, dict) - if err != nil { - return nil, err - } - - return z, nil -} - -// Read decompresses bytes from receiver into p. -func (z *Reader) Read(p []byte) (int, error) { - if z.err != nil { - return 0, z.err - } - - var n int - - n, z.err = z.decompressor.Read(p) - - _, err := z.digest.Write(p[0:n]) - if err != nil { - z.err = err - - return n, z.err - } - - if !errors.Is(z.err, io.EOF) { - // In the normal case we return here. - return n, z.err - } - - // Finished file; check checksum. - _, err = io.ReadFull(z.r, z.scratch[0:4]) - if err != nil { - if errors.Is(err, io.EOF) { - err = io.ErrUnexpectedEOF - } - - z.err = err - - return n, z.err - } - // ZLIB (RFC 1950) is big-endian, unlike GZIP (RFC 1952). - checksum := binary.BigEndian.Uint32(z.scratch[:4]) - if checksum != z.digest.Sum32() { - z.err = ErrChecksum - - return n, z.err - } - - return n, io.EOF -} - -// InputConsumed returns compressed bytes consumed from stream input. -// -// This count includes the zlib header, deflate payload, and zlib checksum -// trailer bytes read by the reader. -func (z *Reader) InputConsumed() uint64 { - if z.counter == nil { - return 0 - } - - return z.counter.read -} - -// Close does not close the wrapped [io.Reader] originally passed to [NewReader]. -// In order for the ZLIB checksum to be verified, the reader must be -// fully consumed until the [io.EOF]. -// Close returns the instance to a global pool; you MUST NOT keep references after Close. -func (z *Reader) Close() error { - if z.err != nil && !errors.Is(z.err, io.EOF) { - return z.err - } - - z.err = z.decompressor.Close() - if z.err != nil { - return z.err - } - - readerPool.Put(z) - - return nil -} diff --git a/internal/zlib/reader_reset.go b/internal/zlib/reader_reset.go deleted file mode 100644 index 4d9d4471..00000000 --- a/internal/zlib/reader_reset.go +++ /dev/null @@ -1,91 +0,0 @@ -// Copyright 2009 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package zlib - -import ( - "bufio" - "encoding/binary" - "errors" - "io" - - "codeberg.org/lindenii/furgit/internal/adler32" - - "github.com/klauspost/compress/flate" -) - -// reset resets receiver to read a new zlib stream. -func (z *Reader) reset(r io.Reader, dict []byte) error { - *z = Reader{decompressor: z.decompressor} - var input flate.Reader - if fr, ok := r.(flate.Reader); ok { - input = fr - } else { - input = bufio.NewReader(r) - } - z.counter = &countingFlateReader{inner: input} - z.r = z.counter - - // Read the header (RFC 1950 section 2.2.). - _, z.err = io.ReadFull(z.r, z.scratch[0:2]) - if z.err != nil { - if errors.Is(z.err, io.EOF) { - z.err = io.ErrUnexpectedEOF - } - - return z.err - } - - h := binary.BigEndian.Uint16(z.scratch[:2]) - if (z.scratch[0]&0x0f != zlibDeflate) || (z.scratch[0]>>4 > zlibMaxWindow) || (h%31 != 0) { - z.err = ErrHeader - - return z.err - } - - haveDict := z.scratch[1]&0x20 != 0 - if haveDict { - _, z.err = io.ReadFull(z.r, z.scratch[0:4]) - if z.err != nil { - if errors.Is(z.err, io.EOF) { - z.err = io.ErrUnexpectedEOF - } - - return z.err - } - - checksum := binary.BigEndian.Uint32(z.scratch[:4]) - if checksum != adler32.Checksum(dict) { - z.err = ErrDictionary - - return z.err - } - } - - if z.decompressor != nil { - resetter, ok := z.decompressor.(flate.Resetter) - if !ok { - panic("zlib: pooled decompressor does not implement flate.Resetter") - } - - z.err = resetter.Reset(z.r, dict) - if z.err != nil { - return z.err - } - - z.digest = adler32.New() - - return nil - } - - if haveDict { - z.decompressor = flate.NewReaderDict(z.r, dict) - } else { - z.decompressor = flate.NewReader(z.r) - } - - z.digest = adler32.New() - - return nil -} diff --git a/internal/zlib/writer.go b/internal/zlib/writer.go deleted file mode 100644 index ba0a1f17..00000000 --- a/internal/zlib/writer.go +++ /dev/null @@ -1,204 +0,0 @@ -// Copyright 2009 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package zlib - -import ( - "encoding/binary" - "fmt" - "hash" - "io" - "sync" - - "github.com/klauspost/compress/flate" -) - -// These constants are copied from the [flate] package, so that code that imports -// [compress/zlib] does not also have to import [compress/flate]. -const ( - NoCompression = flate.NoCompression - BestSpeed = flate.BestSpeed - BestCompression = flate.BestCompression - DefaultCompression = flate.DefaultCompression - HuffmanOnly = flate.HuffmanOnly -) - -// A Writer takes data written to it and writes the compressed -// form of that data to an underlying writer (see [NewWriter]). -type Writer struct { - w io.Writer - level int - dict []byte - compressor *flate.Writer - digest hash.Hash32 - err error - scratch [4]byte - wroteHeader bool -} - -var writerPool = sync.Pool{ - New: func() any { - return new(Writer) - }, -} - -// NewWriter creates a new [Writer]. -// Writes to the returned Writer are compressed and written to w. -// -// It is the caller's responsibility to call Close on the Writer when done. -// Writes may be buffered and not flushed until Close. -func NewWriter(w io.Writer) *Writer { - z, _ := NewWriterLevelDict(w, DefaultCompression, nil) - - return z -} - -// NewWriterLevel is like [NewWriter] but specifies the compression level instead -// of assuming [DefaultCompression]. -// -// The compression level can be [DefaultCompression], [NoCompression], [HuffmanOnly] -// or any integer value between [BestSpeed] and [BestCompression] inclusive. -// The error returned will be nil if the level is valid. -func NewWriterLevel(w io.Writer, level int) (*Writer, error) { - return NewWriterLevelDict(w, level, nil) -} - -// NewWriterLevelDict is like [NewWriterLevel] but specifies a dictionary to -// compress with. -// -// The dictionary may be nil. If not, its contents should not be modified until -// the Writer is closed. -func NewWriterLevelDict(w io.Writer, level int, dict []byte) (*Writer, error) { - if level < HuffmanOnly || level > BestCompression { - return nil, fmt.Errorf("zlib: invalid compression level: %d", level) - } - - v := writerPool.Get() - - z, ok := v.(*Writer) - if !ok { - panic("zlib: pool returned unexpected type") - } - - // flate.Writer can only be Reset with the same level/dictionary mode. - // Reuse it only when the configuration is unchanged and dictionary-free. - reuseCompressor := z.compressor != nil && z.level == level && z.dict == nil && dict == nil - if !reuseCompressor { - z.compressor = nil - } - - if z.digest != nil { - z.digest.Reset() - } - - *z = Writer{ - w: w, - level: level, - dict: dict, - compressor: z.compressor, - digest: z.digest, - } - if z.compressor != nil { - z.compressor.Reset(w) - } - - return z, nil -} - -// Reset clears the state of the [Writer] z such that it is equivalent to its -// initial state from [NewWriterLevel] or [NewWriterLevelDict], but instead writing -// to w. -func (z *Writer) Reset(w io.Writer) { - z.w = w - // z.level and z.dict left unchanged. - if z.compressor != nil { - z.compressor.Reset(w) - } - - if z.digest != nil { - z.digest.Reset() - } - - z.err = nil - z.scratch = [4]byte{} - z.wroteHeader = false -} - -// Write writes a compressed form of p to the underlying [io.Writer]. The -// compressed bytes are not necessarily flushed until the [Writer] is closed or -// explicitly flushed. -func (z *Writer) Write(p []byte) (n int, err error) { - if !z.wroteHeader { - z.err = z.writeHeader() - } - - if z.err != nil { - return 0, z.err - } - - if len(p) == 0 { - return 0, nil - } - - n, err = z.compressor.Write(p) - if err != nil { - z.err = err - - return n, err - } - - _, err = z.digest.Write(p) - if err != nil { - z.err = err - - return 0, z.err - } - - return n, err -} - -// Flush flushes the Writer to its underlying [io.Writer]. -func (z *Writer) Flush() error { - if !z.wroteHeader { - z.err = z.writeHeader() - } - - if z.err != nil { - return z.err - } - - z.err = z.compressor.Flush() - - return z.err -} - -// Close closes the Writer, flushing any unwritten data to the underlying -// [io.Writer], but does not close the underlying io.Writer. -func (z *Writer) Close() error { - if !z.wroteHeader { - z.err = z.writeHeader() - } - - if z.err != nil { - return z.err - } - - z.err = z.compressor.Close() - if z.err != nil { - return z.err - } - - checksum := z.digest.Sum32() - // ZLIB (RFC 1950) is big-endian, unlike GZIP (RFC 1952). - binary.BigEndian.PutUint32(z.scratch[:], checksum) - - _, z.err = z.w.Write(z.scratch[0:4]) - if z.err != nil { - return z.err - } - - writerPool.Put(z) - - return nil -} diff --git a/internal/zlib/writer_header.go b/internal/zlib/writer_header.go deleted file mode 100644 index 29b02fb2..00000000 --- a/internal/zlib/writer_header.go +++ /dev/null @@ -1,72 +0,0 @@ -// Copyright 2009 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package zlib - -import ( - "encoding/binary" - - "codeberg.org/lindenii/furgit/internal/adler32" - - "github.com/klauspost/compress/flate" -) - -// writeHeader writes the ZLIB header. -func (z *Writer) writeHeader() (err error) { - z.wroteHeader = true - // ZLIB has a two-byte header (as documented in RFC 1950). - // The first four bits is the CINFO (compression info), which is 7 for the default deflate window size. - // The next four bits is the CM (compression method), which is 8 for deflate. - z.scratch[0] = 0x78 - // The next two bits is the FLEVEL (compression level). The four values are: - // 0=fastest, 1=fast, 2=default, 3=best. - // The next bit, FDICT, is set if a dictionary is given. - // The final five FCHECK bits form a mod-31 checksum. - switch z.level { - case -2, 0, 1: - z.scratch[1] = 0 << 6 - case 2, 3, 4, 5: - z.scratch[1] = 1 << 6 - case 6, -1: - z.scratch[1] = 2 << 6 - case 7, 8, 9: - z.scratch[1] = 3 << 6 - default: - panic("unreachable") - } - - if z.dict != nil { - z.scratch[1] |= 1 << 5 - } - - z.scratch[1] += uint8(31 - binary.BigEndian.Uint16(z.scratch[:2])%31) //#nosec G115 - - _, err = z.w.Write(z.scratch[0:2]) - if err != nil { - return err - } - - if z.dict != nil { - // The next four bytes are the Adler-32 checksum of the dictionary. - binary.BigEndian.PutUint32(z.scratch[:], adler32.Checksum(z.dict)) - - _, err = z.w.Write(z.scratch[0:4]) - if err != nil { - return err - } - } - - if z.compressor == nil { - // Initialize deflater unless the Writer is being reused - // after a Reset call. - z.compressor, err = flate.NewWriterDict(z.w, z.level, z.dict) - if err != nil { - return err - } - - z.digest = adler32.New() - } - - return nil -} diff --git a/objectstore/loose/parse.go b/objectstore/loose/parse.go index e88d7c6c..2a41e359 100644 --- a/objectstore/loose/parse.go +++ b/objectstore/loose/parse.go @@ -6,7 +6,7 @@ import ( "io" "os" - "codeberg.org/lindenii/furgit/internal/zlib" + "codeberg.org/lindenii/furgit/internal/compress/zlib" "codeberg.org/lindenii/furgit/objectheader" "codeberg.org/lindenii/furgit/objecttype" ) diff --git a/objectstore/loose/read_header.go b/objectstore/loose/read_header.go index abfb1a02..494acfc4 100644 --- a/objectstore/loose/read_header.go +++ b/objectstore/loose/read_header.go @@ -3,7 +3,7 @@ package loose import ( "bufio" - "codeberg.org/lindenii/furgit/internal/zlib" + "codeberg.org/lindenii/furgit/internal/compress/zlib" "codeberg.org/lindenii/furgit/objectid" "codeberg.org/lindenii/furgit/objecttype" ) diff --git a/objectstore/loose/read_reader.go b/objectstore/loose/read_reader.go index a0a51cc1..6f434840 100644 --- a/objectstore/loose/read_reader.go +++ b/objectstore/loose/read_reader.go @@ -8,7 +8,7 @@ import ( "os" "codeberg.org/lindenii/furgit/internal/iolimit" - "codeberg.org/lindenii/furgit/internal/zlib" + "codeberg.org/lindenii/furgit/internal/compress/zlib" "codeberg.org/lindenii/furgit/objectid" "codeberg.org/lindenii/furgit/objecttype" ) diff --git a/objectstore/loose/write_writer.go b/objectstore/loose/write_writer.go index f76c882e..04a93134 100644 --- a/objectstore/loose/write_writer.go +++ b/objectstore/loose/write_writer.go @@ -5,7 +5,7 @@ import ( "hash" "os" - "codeberg.org/lindenii/furgit/internal/zlib" + "codeberg.org/lindenii/furgit/internal/compress/zlib" "codeberg.org/lindenii/furgit/objectid" ) diff --git a/objectstore/packed/entry_inflate.go b/objectstore/packed/entry_inflate.go index cbdb6a89..1c3943e9 100644 --- a/objectstore/packed/entry_inflate.go +++ b/objectstore/packed/entry_inflate.go @@ -6,7 +6,7 @@ import ( "io" "math" - "codeberg.org/lindenii/furgit/internal/zlib" + "codeberg.org/lindenii/furgit/internal/compress/zlib" ) // zlibReaderAt opens a zlib reader starting at data offset within pack. -- cgit v1.3.1-10-gc9f91