diff options
Diffstat (limited to 'internal')
188 files changed, 5740 insertions, 12176 deletions
diff --git a/internal/cache/clock/bench_test.go b/internal/cache/clock/bench_test.go new file mode 100644 index 00000000..48bdff99 --- /dev/null +++ b/internal/cache/clock/bench_test.go @@ -0,0 +1,117 @@ +package clock //nolint:testpackage + +import ( + "sync/atomic" + "testing" + + "lindenii.org/go/lgo/intconv" +) + +func benchWeight(_, _ int) uint64 { return 1 } + +// goroutineSeq hands each parallel worker a distinct starting offset +// so workers don't go in lockstep over identical keys. +// +//nolint:gochecknoglobals +var goroutineSeq atomic.Uint64 + +func workerOffset() int { + result, err := intconv.Uint64ToInt(goroutineSeq.Add(1) * 0x9E3779B1) + if err != nil { + panic(err) + } + + return result +} + +func BenchmarkReadHeavy(b *testing.B) { + // ≈95% Get hits, ≈5% Add, and fits budget. + const n = 100_000 + + clock := New(n, benchWeight) + for k := range n { + clock.Add(k, k) + } + + b.ResetTimer() + b.RunParallel(func(pb *testing.PB) { + i := workerOffset() + for pb.Next() { + i++ + + key := i % n + if i%20 == 0 { + clock.Add(key, key) + } else { + _, _ = clock.Get(key) + } + } + }) +} + +func BenchmarkHotKey(b *testing.B) { + // Every worker does Get over a relatively small hot set. + const ( + hot = 64 + maxWeight = 4096 + ) + + clock := New(maxWeight, benchWeight) + for k := range hot { + clock.Add(k, k) + } + + b.ResetTimer() + b.RunParallel(func(pb *testing.PB) { + i := workerOffset() + for pb.Next() { + i++ + _, _ = clock.Get(i % hot) + } + }) +} + +func BenchmarkMixed(b *testing.B) { + // Even split of Get and Add over a working set that fits. + const n = 100_000 + + clock := New(n, benchWeight) + for k := range n { + clock.Add(k, k) + } + + b.ResetTimer() + b.RunParallel(func(pb *testing.PB) { + i := workerOffset() + for pb.Next() { + i++ + + key := i % n + if i%2 == 0 { + clock.Add(key, key) + } else { + _, _ = clock.Get(key) + } + } + }) +} + +func BenchmarkChurn(b *testing.B) { + // Every op inserts a fresh key into a small budget. + // I don't think this is likely to happen for git delta clocks, + // but this may matter if we use this for other workloads later. + const maxWeight = 4096 + + clock := New(maxWeight, benchWeight) + + b.ResetTimer() + b.RunParallel(func(pb *testing.PB) { + base := workerOffset() + + i := 0 + for pb.Next() { + i++ + clock.Add(base+i, i) + } + }) +} diff --git a/internal/cache/clock/clock.go b/internal/cache/clock/clock.go new file mode 100644 index 00000000..27d83c7b --- /dev/null +++ b/internal/cache/clock/clock.go @@ -0,0 +1,86 @@ +package clock + +import ( + "hash/maphash" + "runtime" + + "lindenii.org/go/lgo/intconv" +) + +// maxShards bounds the shard count. +// +// Keep it relatively modest +// so the per-shard budget +// stays large enough to admit sizable values. +const maxShards = 16 + +// WeightFunc reports one entry's weight, used for eviction budgeting. +type WeightFunc[K comparable, V any] func(key K, value V) uint64 + +// Clock is a concurrent, weight-bounded cache +// with CLOCK eviction. +// +// Reads are lock-free; +// writes lock only the shard that owns the key. +// +// Labels: MT-Safe. +type Clock[K comparable, V any] struct { + shards []*shard[K, V] + seed maphash.Seed + mask uint64 + weightFn WeightFunc[K, V] +} + +// New returns a cache bounded to maxWeight total weight, +// weighing entries with weightFn. +// +// New panics if weightFn is nil. +func New[K comparable, V any](maxWeight uint64, weightFn WeightFunc[K, V]) *Clock[K, V] { + if weightFn == nil { + panic("internal/clock: nil weight function") + } + + count, mask := shardLayout(maxWeight) + perShard := maxWeight / (mask + 1) + + shards := make([]*shard[K, V], count) + for i := range shards { + shards[i] = newShard[K, V](perShard) + } + + return &Clock[K, V]{ + shards: shards, + seed: maphash.MakeSeed(), + mask: mask, + weightFn: weightFn, + } +} + +// shardLayout picks a power-of-two shard count and its address mask. +// +// Tracks GOMAXPROCS, capped at maxShards, +// and is shrunk so the per-shard budget +// stays at least one while maxWeight is nonzero. +func shardLayout(maxWeight uint64) (int, uint64) { + count := 1 + for count < runtime.GOMAXPROCS(0) && count < maxShards { + count *= 2 + } + + countU, err := intconv.IntToUint64(count) + if err != nil { + return 1, 0 + } + + for countU > maxWeight && countU > 1 { + count /= 2 + countU /= 2 + } + + return count, countU - 1 +} + +// shardFor returns the shard that owns key. +func (clock *Clock[K, V]) shardFor(key K) *shard[K, V] { + return clock.shards[maphash.Comparable(clock.seed, key)&clock.mask] +} diff --git a/internal/cache/clock/clock_ops.go b/internal/cache/clock/clock_ops.go new file mode 100644 index 00000000..a21f44c3 --- /dev/null +++ b/internal/cache/clock/clock_ops.go @@ -0,0 +1,51 @@ +package clock + +// Add inserts or replaces key, marking it recently used. +// +// It reports whether the entry was admitted; +// an entry heavier than the per-shard budget is rejected +// and leaves the cache unchanged. +func (clock *Clock[K, V]) Add(key K, value V) bool { + return clock.shardFor(key).add(key, value, clock.weightFn(key, value)) +} + +// Get returns the value for key and marks it recently used. +// +//nolint:ireturn +func (clock *Clock[K, V]) Get(key K) (V, bool) { + return clock.shardFor(key).get(key) +} + +// Peek returns the value for key without changing its recency. +// +//nolint:ireturn +func (clock *Clock[K, V]) Peek(key K) (V, bool) { + return clock.shardFor(key).peek(key) +} + +// Len returns the number of cached entries. +func (clock *Clock[K, V]) Len() int { + total := 0 + for _, shard := range clock.shards { + total += shard.len() + } + + return total +} + +// Weight returns the current total weight across all shards. +func (clock *Clock[K, V]) Weight() uint64 { + var total uint64 + for _, shard := range clock.shards { + total += shard.loadWeight() + } + + return total +} + +// Clear removes all entries. +func (clock *Clock[K, V]) Clear() { + for _, shard := range clock.shards { + shard.clear() + } +} diff --git a/internal/cache/clock/clock_test.go b/internal/cache/clock/clock_test.go new file mode 100644 index 00000000..36622818 --- /dev/null +++ b/internal/cache/clock/clock_test.go @@ -0,0 +1,100 @@ +package clock_test + +import ( + "fmt" + "strings" + "testing" + + "lindenii.org/go/furgit/internal/cache/clock" + "lindenii.org/go/lgo/intconv" +) + +func byteWeight(_ string, value string) uint64 { + weight, err := intconv.IntToUint64(len(value)) + if err != nil { + return 0 + } + + return weight +} + +func TestCacheAddGetPeek(t *testing.T) { + t.Parallel() + + clock := clock.New(1<<20, byteWeight) + + if !clock.Add("a", "alpha") { + t.Fatalf("Add(a) should succeed") + } + + if got, ok := clock.Get("a"); !ok || got != "alpha" { + t.Fatalf("Get(a) = (%q, %v), want (alpha, true)", got, ok) + } + + if got, ok := clock.Peek("a"); !ok || got != "alpha" { + t.Fatalf("Peek(a) = (%q, %v), want (alpha, true)", got, ok) + } + + if _, ok := clock.Get("missing"); ok { + t.Fatalf("Get(missing) should miss") + } +} + +func TestCacheWeightStaysBounded(t *testing.T) { + t.Parallel() + + const maxWeight = 4096 + + clock := clock.New(maxWeight, byteWeight) + value := strings.Repeat("x", 64) + + for i := range 1000 { + clock.Add(fmt.Sprintf("key-%d", i), value) + } + + if got := clock.Weight(); got > maxWeight { + t.Fatalf("weight = %d, exceeds max %d", got, maxWeight) + } +} + +func TestCacheLenAndClear(t *testing.T) { + t.Parallel() + + clock := clock.New(1<<20, byteWeight) + + for i := range 10 { + clock.Add(fmt.Sprintf("key-%d", i), "v") + } + + if got := clock.Len(); got != 10 { + t.Fatalf("Len = %d, want 10", got) + } + + clock.Clear() + + if got := clock.Len(); got != 0 { + t.Fatalf("Len after Clear = %d, want 0", got) + } + + if got := clock.Weight(); got != 0 { + t.Fatalf("Weight after Clear = %d, want 0", got) + } +} + +func TestCacheRejectsOversized(t *testing.T) { + t.Parallel() + + clock := clock.New(4, byteWeight) + + if clock.Add("a", "xxxxx") { + t.Fatalf("oversized Add should report false") + } + + if _, ok := clock.Get("a"); ok { + t.Fatalf("oversized entry must not be clockd") + } + + if got := clock.Weight(); got != 0 { + t.Fatalf("weight = %d, want 0", got) + } +} diff --git a/internal/cache/clock/concurrent_test.go b/internal/cache/clock/concurrent_test.go new file mode 100644 index 00000000..4b7da162 --- /dev/null +++ b/internal/cache/clock/concurrent_test.go @@ -0,0 +1,103 @@ +package clock //nolint:testpackage + +import ( + "sync" + "testing" +) + +func keyValue(key int) int { + return key*1000003 + 7 +} + +func TestConcurrentStress(t *testing.T) { + t.Parallel() + + const ( + maxWeight = 512 + keys = 400 + workers = 8 + rounds = 5000 + ) + + clock := New(maxWeight, func(_ int, _ int) uint64 { return 1 }) + + var wg sync.WaitGroup + + for worker := range workers { + wg.Go(func() { + for i := range rounds { + key := (worker*7 + i) % keys + + switch i % 4 { + case 0, 1: + clock.Add(key, keyValue(key)) + case 2: + if got, ok := clock.Get(key); ok && got != keyValue(key) { + t.Errorf("Get(%d) = %d, want %d", key, got, keyValue(key)) + } + case 3: + if got, ok := clock.Peek(key); ok && got != keyValue(key) { + t.Errorf("Peek(%d) = %d, want %d", key, got, keyValue(key)) + } + } + } + }) + } + + wg.Wait() + + checkCache(t, clock) + + if got := clock.Weight(); got > maxWeight { + t.Fatalf("weight %d exceeds max %d", got, maxWeight) + } +} + +func TestReadDuringEviction(t *testing.T) { + t.Parallel() + + const ( + maxWeight = 8 + hot = 64 + writers = 2 + readers = 6 + rounds = 20000 + ) + + clock := New(maxWeight, func(_ int, _ int) uint64 { return 1 }) + + var wg sync.WaitGroup + + for range writers { + wg.Go(func() { + for i := range rounds { + key := i % hot + clock.Add(key, keyValue(key)) + } + }) + } + + for range readers { + wg.Go(func() { + for i := range rounds { + key := i % hot + + if got, ok := clock.Get(key); ok && got != keyValue(key) { + t.Errorf("Get(%d) = %d, want %d", key, got, keyValue(key)) + } + + if got, ok := clock.Peek(key); ok && got != keyValue(key) { + t.Errorf("Peek(%d) = %d, want %d", key, got, keyValue(key)) + } + } + }) + } + + wg.Wait() + + checkCache(t, clock) + + if got := clock.Weight(); got > maxWeight { + t.Fatalf("weight %d exceeds max %d", got, maxWeight) + } +} diff --git a/internal/cache/clock/doc.go b/internal/cache/clock/doc.go new file mode 100644 index 00000000..6f28805b --- /dev/null +++ b/internal/cache/clock/doc.go @@ -0,0 +1,9 @@ +// Package clock provides a concurrent, weight-bounded object cache. +// +// The cache is sharded by key, +// and each shard owns an independent fraction of the total budget. +// An entry heavier than that per-shard fraction is never admitted, +// so callers should keep the total budget well above the largest value. +// +// Labels: MT-Safe. +package clock diff --git a/internal/cache/clock/fuzz_test.go b/internal/cache/clock/fuzz_test.go new file mode 100644 index 00000000..af0d4024 --- /dev/null +++ b/internal/cache/clock/fuzz_test.go @@ -0,0 +1,53 @@ +package clock //nolint:testpackage + +import "testing" + +// FuzzShard replays a decoded op stream against one shard, +// checking the value oracle and invariants after every op. +func FuzzShard(f *testing.F) { + f.Add([]byte{}) + f.Add([]byte{0, 1, 10, 0, 2, 10, 0, 3, 10, 0, 4, 10, 1, 1, 0, 0, 5, 10}) + f.Add([]byte{0, 7, 200, 0, 7, 5, 2, 7, 0, 3, 0, 0, 0, 8, 8}) + + f.Fuzz(func(t *testing.T, program []byte) { + const maxWeight = 32 + + shard := newShard[uint8, uint64](maxWeight) + shadow := make(map[uint8]uint64) + + var nonce uint64 + + for i := 0; i+2 < len(program); i += 3 { + key := program[i+1] + weight := uint64(program[i+2]) + + switch program[i] % 4 { + case 0: // add + nonce++ + value := nonce + + admitted := shard.add(key, value, weight) + if admitted != (weight <= maxWeight) { + t.Fatalf("add(%d, w=%d) admitted=%v, want %v", key, weight, admitted, weight <= maxWeight) + } + + if admitted { + shadow[key] = value + } + case 1: // get + if got, ok := shard.get(key); ok && got != shadow[key] { + t.Fatalf("get(%d) = %d, want %d", key, got, shadow[key]) + } + case 2: // peek + if got, ok := shard.peek(key); ok && got != shadow[key] { + t.Fatalf("peek(%d) = %d, want %d", key, got, shadow[key]) + } + case 3: // clear + shard.clear() + clear(shadow) + } + + checkShard(t, shard) + } + }) +} diff --git a/internal/cache/clock/invariant_test.go b/internal/cache/clock/invariant_test.go new file mode 100644 index 00000000..04896ca1 --- /dev/null +++ b/internal/cache/clock/invariant_test.go @@ -0,0 +1,88 @@ +package clock //nolint:testpackage + +import "testing" + +// checkShard verifies a shard's structural invariants at a quiescent point. +// +// It must be called with no concurrent operations in flight. +func checkShard[K comparable, V any](t *testing.T, shard *shard[K, V]) { + t.Helper() + + shard.mu.Lock() + defer shard.mu.Unlock() + + ringLen := 0 + + var ringWeight uint64 + + seen := make(map[*entry[K, V]]struct{}) + + if shard.hand != nil { //nolint:nestif + for e := shard.hand; ; e = e.next { + if e.prev == nil || e.next == nil { + t.Fatalf("nil ring link at key %v", e.key) + } + + if e.next.prev != e || e.prev.next != e { + t.Fatalf("ring links not reciprocal at key %v", e.key) + } + + if _, dup := seen[e]; dup { + t.Fatalf("ring revisits a node before returning to the hand") + } + + seen[e] = struct{}{} + ringLen++ + ringWeight += e.weight + + if got, ok := shard.items.Load(e.key); !ok || got != e { + t.Fatalf("ring node %v is not mapped to itself", e.key) + } + + if e.next == shard.hand { + break + } + } + } + + if ringLen != shard.count { + t.Fatalf("ring length %d != count %d", ringLen, shard.count) + } + + if ringWeight != shard.weight { + t.Fatalf("ring weight %d != shard weight %d", ringWeight, shard.weight) + } + + if shard.weight > shard.maxWeight { + t.Fatalf("weight %d exceeds budget %d", shard.weight, shard.maxWeight) + } + + if (shard.hand == nil) != (shard.count == 0) { + t.Fatalf("hand/count disagree: hand=%v count=%d", shard.hand, shard.count) + } + + mapLen := 0 + + shard.items.Range(func(_ K, e *entry[K, V]) bool { + mapLen++ + + if _, ok := seen[e]; !ok { + t.Fatalf("mapped entry %v missing from ring", e.key) + } + + return true + }) + + if mapLen != shard.count { + t.Fatalf("map size %d != count %d", mapLen, shard.count) + } +} + +// checkCache verifies every shard's invariants. +func checkCache[K comparable, V any](t *testing.T, clock *Clock[K, V]) { + t.Helper() + + for _, shard := range clock.shards { + checkShard(t, shard) + } +} diff --git a/internal/cache/clock/shard.go b/internal/cache/clock/shard.go new file mode 100644 index 00000000..22e58b2f --- /dev/null +++ b/internal/cache/clock/shard.go @@ -0,0 +1,67 @@ +package clock + +import ( + "sync" + "sync/atomic" + + lsync "lindenii.org/go/lgo/sync" +) + +// entry is one cached key/value with CLOCK. +type entry[K comparable, V any] struct { + key K + value V + weight uint64 + prev, next *entry[K, V] + + // referenced is set on access and cleared by the eviction sweep; + // prev and next link the entry into its shard's ring. + referenced atomic.Bool +} + +// shard is an independently locked CLOCK cache. +type shard[K comparable, V any] struct { + items lsync.Map[K, *entry[K, V]] + + hand *entry[K, V] + weight uint64 + count int + maxWeight uint64 + + // mu protects the ring, hand, totals, and writes. + mu sync.Mutex +} + +// newShard returns an empty shard with the given weight budget. +func newShard[K comparable, V any](maxWeight uint64) *shard[K, V] { + return &shard[K, V]{ //nolint:exhaustruct + maxWeight: maxWeight, + } +} + +// len returns the shard's entry count. +func (shard *shard[K, V]) len() int { + shard.mu.Lock() + defer shard.mu.Unlock() + + return shard.count +} + +// loadWeight returns the shard's current total weight. +func (shard *shard[K, V]) loadWeight() uint64 { + shard.mu.Lock() + defer shard.mu.Unlock() + + return shard.weight +} + +// clear removes every entry. +func (shard *shard[K, V]) clear() { + shard.mu.Lock() + defer shard.mu.Unlock() + + shard.items.Clear() + shard.hand = nil + shard.weight = 0 + shard.count = 0 +} diff --git a/internal/cache/clock/shard_read.go b/internal/cache/clock/shard_read.go new file mode 100644 index 00000000..624e3409 --- /dev/null +++ b/internal/cache/clock/shard_read.go @@ -0,0 +1,33 @@ +package clock + +// get returns the value for key and marks it referenced. +// +//nolint:ireturn +func (shard *shard[K, V]) get(key K) (V, bool) { + e, ok := shard.items.Load(key) + if !ok { + var zero V + + return zero, false + } + + if !e.referenced.Load() { + e.referenced.Store(true) + } + + return e.value, true +} + +// peek returns the value for key without affecting eviction. +// +//nolint:ireturn +func (shard *shard[K, V]) peek(key K) (V, bool) { + e, ok := shard.items.Load(key) + if !ok { + var zero V + + return zero, false + } + + return e.value, true +} diff --git a/internal/cache/clock/shard_test.go b/internal/cache/clock/shard_test.go new file mode 100644 index 00000000..d974a30e --- /dev/null +++ b/internal/cache/clock/shard_test.go @@ -0,0 +1,149 @@ +package clock //nolint:testpackage + +import "testing" + +func TestShardEvictsOldest(t *testing.T) { + t.Parallel() + + shard := newShard[string, string](8) + shard.add("a", "a", 4) + shard.add("b", "b", 4) + shard.add("c", "c", 4) // overflows, evicts + + if _, ok := shard.peek("a"); ok { + t.Fatalf("a should have been evicted") + } + + for _, key := range []string{"b", "c"} { + if _, ok := shard.peek(key); !ok { + t.Fatalf("%s should remain present", key) + } + } + + if got := shard.loadWeight(); got != 8 { + t.Fatalf("weight = %d, want 8", got) + } + + if got := shard.len(); got != 2 { + t.Fatalf("len = %d, want 2", got) + } +} + +func TestShardGetGivesSecondChance(t *testing.T) { + t.Parallel() + + shard := newShard[string, string](8) + shard.add("a", "a", 4) + shard.add("b", "b", 4) + shard.add("c", "c", 4) // a evicted; b and c survive + + if _, ok := shard.get("b"); !ok { + t.Fatalf("get(b) should hit") + } + + shard.add("d", "d", 4) // b was just touched, so c is evicted instead + + if _, ok := shard.peek("c"); ok { + t.Fatalf("c should have been evicted after b was touched") + } + + for _, key := range []string{"b", "d"} { + if _, ok := shard.peek(key); !ok { + t.Fatalf("%s should remain present", key) + } + } +} + +func TestShardPeekGivesNoSecondChance(t *testing.T) { + t.Parallel() + + shard := newShard[string, string](8) + shard.add("a", "a", 4) + shard.add("b", "b", 4) + shard.add("c", "c", 4) // a is evicted; b and c survive with cleared bits + + if _, ok := shard.peek("b"); !ok { + t.Fatalf("peek(b) should hit") + } + + shard.add("d", "d", 4) // peek did not refresh b, so b is evicted + + if _, ok := shard.peek("b"); ok { + t.Fatalf("b should have been evicted; peek must not grant a second chance") + } + + for _, key := range []string{"c", "d"} { + if _, ok := shard.peek(key); !ok { + t.Fatalf("%s should remain present", key) + } + } +} + +func TestShardReplaceUpdatesWeight(t *testing.T) { + t.Parallel() + + shard := newShard[string, string](100) + shard.add("a", "old", 4) + shard.add("a", "new", 6) + + if got, ok := shard.peek("a"); !ok || got != "new" { + t.Fatalf("peek(a) = (%q, %v), want (new, true)", got, ok) + } + + if got := shard.loadWeight(); got != 6 { + t.Fatalf("weight = %d, want 6", got) + } + + if got := shard.len(); got != 1 { + t.Fatalf("len = %d, want 1", got) + } +} + +func TestShardRejectsOversized(t *testing.T) { + t.Parallel() + + shard := newShard[string, string](5) + shard.add("a", "x", 3) + + if shard.add("b", "big", 6) { + t.Fatalf("oversized add should report false") + } + + if shard.add("a", "huge", 6) { + t.Fatalf("oversized replace should report false") + } + + if got, ok := shard.peek("a"); !ok || got != "x" { + t.Fatalf("peek(a) = (%q, %v), want (x, true); cache must be unchanged", got, ok) + } + + if _, ok := shard.peek("b"); ok { + t.Fatalf("b must not have been admitted") + } + + if got := shard.loadWeight(); got != 3 { + t.Fatalf("weight = %d, want 3", got) + } +} + +func TestShardClear(t *testing.T) { + t.Parallel() + + shard := newShard[string, string](100) + shard.add("a", "a", 4) + shard.add("b", "b", 4) + + shard.clear() + + if got := shard.loadWeight(); got != 0 { + t.Fatalf("weight = %d, want 0", got) + } + + if got := shard.len(); got != 0 { + t.Fatalf("len = %d, want 0", got) + } + + if _, ok := shard.peek("a"); ok { + t.Fatalf("a should be gone after clear") + } +} diff --git a/internal/cache/clock/shard_write.go b/internal/cache/clock/shard_write.go new file mode 100644 index 00000000..40ddabd0 --- /dev/null +++ b/internal/cache/clock/shard_write.go @@ -0,0 +1,105 @@ +package clock + +// add inserts or replaces key, then evicts down to budget. +// +// It reports whether the entry was admitted; +// an entry heavier than the shard budget is rejected +// and leaves the shard unchanged. +func (shard *shard[K, V]) add(key K, value V, weight uint64) bool { + if weight > shard.maxWeight { + return false + } + + shard.mu.Lock() + defer shard.mu.Unlock() + + if old, ok := shard.items.Load(key); ok { + shard.unlink(old) + shard.items.Delete(key) + shard.weight -= old.weight + shard.count-- + } + + e := &entry[K, V]{ //nolint:exhaustruct + key: key, + value: value, + weight: weight, + } + e.referenced.Store(true) + + shard.linkBeforeHand(e) + shard.items.Store(key, e) + shard.weight += weight + shard.count++ + + shard.evict() + + return true +} + +// evict advances the clock hand until the shard is within budget. +// +// A referenced entry is spared once, its bit cleared; +// after a full rotation of spared entries the next is evicted regardless, +// so a flood of concurrent reads cannot stall progress. +func (shard *shard[K, V]) evict() { + skips := 0 + + for shard.weight > shard.maxWeight && shard.hand != nil { + victim := shard.hand + + if victim.referenced.Load() && skips < shard.count { + victim.referenced.Store(false) + shard.hand = victim.next + skips++ + + continue + } + + shard.unlink(victim) + shard.items.Delete(victim.key) + shard.weight -= victim.weight + shard.count-- + skips = 0 + } +} + +// linkBeforeHand inserts e just behind the hand, +// so a full rotation passes before the sweep examines it. +func (shard *shard[K, V]) linkBeforeHand(e *entry[K, V]) { + if shard.hand == nil { + e.prev = e + e.next = e + shard.hand = e + + return + } + + tail := shard.hand.prev + tail.next = e + e.prev = tail + e.next = shard.hand + shard.hand.prev = e +} + +// unlink removes e from the ring, +// moving the hand off e when it points at it. +func (shard *shard[K, V]) unlink(e *entry[K, V]) { + if e.next == e { + shard.hand = nil + e.prev = nil + e.next = nil + + return + } + + e.prev.next = e.next + e.next.prev = e.prev + + if shard.hand == e { + shard.hand = e.next + } + + e.prev = nil + e.next = nil +} diff --git a/internal/cache/doc.go b/internal/cache/doc.go new file mode 100644 index 00000000..d0376ed9 --- /dev/null +++ b/internal/cache/doc.go @@ -0,0 +1,2 @@ +// Package cache provides caches for a few different access patterns. +package cache diff --git a/internal/compress/flate/_gen/gen_inflate.go b/internal/compress/flate/_gen/gen_inflate.go deleted file mode 100644 index 33f14005..00000000 --- a/internal/compress/flate/_gen/gen_inflate.go +++ /dev/null @@ -1,303 +0,0 @@ -//go:build generate -// +build generate - -//go:generate go run $GOFILE -//go:generate go fmt ../inflate_gen.go - -package main - -import ( - "os" - "strings" -) - -func main() { - f, err := os.Create("../inflate_gen.go") - if err != nil { - panic(err) - } - defer f.Close() - types := []string{"*bytes.Buffer", "*bytes.Reader", "*bufio.Reader", "*strings.Reader", "Reader"} - names := []string{"BytesBuffer", "BytesReader", "BufioReader", "StringsReader", "GenericReader"} - imports := []string{"bytes", "bufio", "fmt", "strings", "math/bits"} - f.WriteString(`// Code generated by go generate gen_inflate.go. DO NOT EDIT. - -package flate - -import ( -`) - - for _, imp := range imports { - f.WriteString("\t\"" + imp + "\"\n") - } - f.WriteString(")\n\n") - - template := ` - -// Decode a single Huffman block from f. -// hl and hd are the Huffman states for the lit/length values -// and the distance values, respectively. If hd == nil, using the -// fixed distance encoding associated with fixed Huffman blocks. -func (f *decompressor) $FUNCNAME$() { - const ( - stateInit = iota // Zero value must be stateInit - stateDict - ) - fr := f.r.($TYPE$) - - // Optimization. Compiler isn't smart enough to keep f.b,f.nb in registers, - // but is smart enough to keep local variables in registers, so use nb and b, - // inline call to moreBits and reassign b,nb back to f on return. - fnb, fb, dict := f.nb, f.b, &f.dict - - switch f.stepState { - case stateInit: - goto readLiteral - case stateDict: - goto copyHistory - } - -readLiteral: - // Read literal and/or (length, distance) according to RFC section 3.2.3. - { - var v int - { - // Inlined v, err := f.huffSym(f.hl) - // Since a huffmanDecoder can be empty or be composed of a degenerate tree - // with single element, huffSym must error on these two edge cases. In both - // cases, the chunks slice will be 0 for the invalid sequence, leading it - // satisfy the n == 0 check below. - n := uint(f.hl.maxRead) - for { - for fnb < n { - c, err := fr.ReadByte() - if err != nil { - f.b, f.nb = fb, fnb - f.err = noEOF(err) - return - } - f.roffset++ - fb |= uint32(c) << (fnb & regSizeMaskUint32) - fnb += 8 - } - chunk := f.hl.chunks[fb&(huffmanNumChunks-1)] - n = uint(chunk & huffmanCountMask) - if n > huffmanChunkBits { - chunk = f.hl.links[chunk>>huffmanValueShift][(fb>>huffmanChunkBits)&f.hl.linkMask] - n = uint(chunk & huffmanCountMask) - } - if n <= fnb { - if n == 0 { - f.b, f.nb = fb, fnb - if debugDecode { - fmt.Println("huffsym: n==0") - } - f.err = CorruptInputError(f.roffset) - return - } - fb = fb >> (n & regSizeMaskUint32) - fnb = fnb - n - v = int(chunk >> huffmanValueShift) - break - } - } - } - - var length int - switch { - case v < 256: - dict.writeByte(byte(v)) - if dict.availWrite() == 0 { - f.toRead = dict.readFlush() - f.step = $FUNCNAME$ - f.stepState = stateInit - f.b, f.nb = fb, fnb - return - } - goto readLiteral - case v == 256: - f.b, f.nb = fb, fnb - f.finishBlock() - return - // otherwise, reference to older data - case v < 265: - length = v - (257 - 3) - case v < maxNumLit: - val := decCodeToLen[(v - 257)] - length = int(val.length) + 3 - n := uint(val.extra) - for fnb < n { - c, err := fr.ReadByte() - if err != nil { - f.b, f.nb = fb, fnb - if debugDecode { - fmt.Println("morebits n>0:", err) - } - f.err = err - return - } - f.roffset++ - fb |= uint32(c) << (fnb®SizeMaskUint32) - fnb += 8 - } - length += int(fb & bitMask32[n]) - fb >>= n & regSizeMaskUint32 - fnb -= n - default: - if debugDecode { - fmt.Println(v, ">= maxNumLit") - } - f.err = CorruptInputError(f.roffset) - f.b, f.nb = fb, fnb - return - } - - var dist uint32 - if f.hd == nil { - for fnb < 5 { - c, err := fr.ReadByte() - if err != nil { - f.b, f.nb = fb, fnb - if debugDecode { - fmt.Println("morebits f.nb<5:", err) - } - f.err = err - return - } - f.roffset++ - fb |= uint32(c) << (fnb®SizeMaskUint32) - fnb += 8 - } - dist = uint32(bits.Reverse8(uint8(fb & 0x1F << 3))) - fb >>= 5 - fnb -= 5 - } else { - // Since a huffmanDecoder can be empty or be composed of a degenerate tree - // with single element, huffSym must error on these two edge cases. In both - // cases, the chunks slice will be 0 for the invalid sequence, leading it - // satisfy the n == 0 check below. - n := uint(f.hd.maxRead) - // Optimization. Compiler isn't smart enough to keep f.b,f.nb in registers, - // but is smart enough to keep local variables in registers, so use nb and b, - // inline call to moreBits and reassign b,nb back to f on return. - for { - for fnb < n { - c, err := fr.ReadByte() - if err != nil { - f.b, f.nb = fb, fnb - f.err = noEOF(err) - return - } - f.roffset++ - fb |= uint32(c) << (fnb & regSizeMaskUint32) - fnb += 8 - } - chunk := f.hd.chunks[fb&(huffmanNumChunks-1)] - n = uint(chunk & huffmanCountMask) - if n > huffmanChunkBits { - chunk = f.hd.links[chunk>>huffmanValueShift][(fb>>huffmanChunkBits)&f.hd.linkMask] - n = uint(chunk & huffmanCountMask) - } - if n <= fnb { - if n == 0 { - f.b, f.nb = fb, fnb - if debugDecode { - fmt.Println("huffsym: n==0") - } - f.err = CorruptInputError(f.roffset) - return - } - fb = fb >> (n & regSizeMaskUint32) - fnb = fnb - n - dist = uint32(chunk >> huffmanValueShift) - break - } - } - } - - switch { - case dist < 4: - dist++ - case dist < maxNumDist: - nb := uint(dist-2) >> 1 - // have 1 bit in bottom of dist, need nb more. - extra := (dist & 1) << (nb & regSizeMaskUint32) - for fnb < nb { - c, err := fr.ReadByte() - if err != nil { - f.b, f.nb = fb, fnb - if debugDecode { - fmt.Println("morebits f.nb<nb:", err) - } - f.err = err - return - } - f.roffset++ - fb |= uint32(c) << (fnb®SizeMaskUint32) - fnb += 8 - } - extra |= fb & bitMask32[nb] - fb >>= nb & regSizeMaskUint32 - fnb -= nb - dist = 1<<((nb+1)®SizeMaskUint32) + 1 + extra - // slower: dist = bitMask32[nb+1] + 2 + extra - default: - f.b, f.nb = fb, fnb - if debugDecode { - fmt.Println("dist too big:", dist, maxNumDist) - } - f.err = CorruptInputError(f.roffset) - return - } - - // No check on length; encoding can be prescient. - if dist > uint32(dict.histSize()) { - f.b, f.nb = fb, fnb - if debugDecode { - fmt.Println("dist > dict.histSize():", dist, dict.histSize()) - } - f.err = CorruptInputError(f.roffset) - return - } - - f.copyLen, f.copyDist = length, int(dist) - goto copyHistory - } - -copyHistory: - // Perform a backwards copy according to RFC section 3.2.3. - { - cnt := dict.tryWriteCopy(f.copyDist, f.copyLen) - if cnt == 0 { - cnt = dict.writeCopy(f.copyDist, f.copyLen) - } - f.copyLen -= cnt - - if dict.availWrite() == 0 || f.copyLen > 0 { - f.toRead = dict.readFlush() - f.step = $FUNCNAME$ // We need to continue this work - f.stepState = stateDict - f.b, f.nb = fb, fnb - return - } - goto readLiteral - } - // Not reached -} - -` - for i, t := range types { - s := strings.Replace(template, "$FUNCNAME$", "huffman"+names[i], -1) - s = strings.Replace(s, "$TYPE$", t, -1) - f.WriteString(s) - } - f.WriteString("func (f *decompressor) huffmanBlockDecoder() {\n") - f.WriteString("\tswitch f.r.(type) {\n") - for i, t := range types { - f.WriteString("\t\tcase " + t + ":\n") - f.WriteString("\t\t\tf.huffman" + names[i] + "()\n") - } - f.WriteString("\t\tdefault:\n") - f.WriteString("\t\t\tf.huffmanGenericReader()\n") - f.WriteString("\t}\n}\n") -} diff --git a/internal/compress/flate/deflate.go b/internal/compress/flate/deflate.go deleted file mode 100644 index 8c8457e2..00000000 --- a/internal/compress/flate/deflate.go +++ /dev/null @@ -1,996 +0,0 @@ -// Copyright 2009 The Go Authors. All rights reserved. -// Copyright (c) 2015 Klaus Post -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package flate - -import ( - "errors" - "fmt" - "io" - "math" - - "lindenii.org/go/furgit/internal/compress/internal/le" -) - -const ( - NoCompression = 0 - BestSpeed = 1 - BestCompression = 9 - DefaultCompression = -1 - - // HuffmanOnly disables Lempel-Ziv match searching and only performs Huffman - // entropy encoding. This mode is useful in compressing data that has - // already been compressed with an LZ style algorithm (e.g. Snappy or LZ4) - // that lacks an entropy encoder. Compression gains are achieved when - // certain bytes in the input stream occur more frequently than others. - // - // Note that HuffmanOnly produces a compressed output that is - // RFC 1951 compliant. That is, any valid DEFLATE decompressor will - // continue to be able to decompress this output. - HuffmanOnly = -2 - ConstantCompression = HuffmanOnly // compatibility alias. - - logWindowSize = 15 - windowSize = 1 << logWindowSize - windowMask = windowSize - 1 - logMaxOffsetSize = 15 // Standard DEFLATE - minMatchLength = 4 // The smallest match that the compressor looks for - maxMatchLength = 258 // The longest match for the compressor - minOffsetSize = 1 // The shortest offset that makes any sense - - // The maximum number of tokens we will encode at the time. - // Smaller sizes usually creates less optimal blocks. - // Bigger can make context switching slow. - // We use this for levels 7-9, so we make it big. - maxFlateBlockTokens = 1 << 15 - maxStoreBlockSize = 65535 - hashBits = 17 // After 17 performance degrades - hashSize = 1 << hashBits - hashMask = (1 << hashBits) - 1 - hashShift = (hashBits + minMatchLength - 1) / minMatchLength - maxHashOffset = 1 << 28 - - skipNever = math.MaxInt32 - - debugDeflate = false -) - -type compressionLevel struct { - good, lazy, nice, chain, fastSkipHashing, level int -} - -// Compression levels have been rebalanced from zlib deflate defaults -// to give a bigger spread in speed and compression. -// See https://blog.klauspost.com/rebalancing-deflate-compression-levels/ -var levels = []compressionLevel{ - {}, // 0 - // Level 1-6 uses specialized algorithm - values not used - {0, 0, 0, 0, 0, 1}, - {0, 0, 0, 0, 0, 2}, - {0, 0, 0, 0, 0, 3}, - {0, 0, 0, 0, 0, 4}, - {0, 0, 0, 0, 0, 5}, - {0, 0, 0, 0, 0, 6}, - // Levels 7-9 use increasingly more lazy matching - // and increasingly stringent conditions for "good enough". - {8, 12, 16, 24, skipNever, 7}, - {16, 30, 40, 64, skipNever, 8}, - {32, 258, 258, 1024, skipNever, 9}, -} - -// advancedState contains state for the advanced levels, with bigger hash tables, etc. -type advancedState struct { - // deflate state - length int - offset int - maxInsertIndex int - chainHead int - hashOffset int - - ii uint16 // position of last match, intended to overflow to reset. - - // input window: unprocessed data is window[index:windowEnd] - index int - hashMatch [maxMatchLength + minMatchLength]uint32 - - // Input hash chains - // hashHead[hashValue] contains the largest inputIndex with the specified hash value - // If hashHead[hashValue] is within the current window, then - // hashPrev[hashHead[hashValue] & windowMask] contains the previous index - // with the same hash value. - hashHead [hashSize]uint32 - hashPrev [windowSize]uint32 -} - -type compressor struct { - compressionLevel - - h *huffmanEncoder - w *huffmanBitWriter - - // compression algorithm - fill func(*compressor, []byte) int // copy data to window - step func(*compressor) // process window - - window []byte - windowEnd int - blockStart int // window index where current tokens start - err error - - // queued output tokens - tokens tokens - fast fastEnc - state *advancedState - - sync bool // requesting flush - byteAvailable bool // if true, still need to process window[index-1]. -} - -func (d *compressor) fillDeflate(b []byte) int { - s := d.state - if s.index >= 2*windowSize-(minMatchLength+maxMatchLength) { - // shift the window by windowSize - // copy(d.window[:], d.window[windowSize:2*windowSize]) - *(*[windowSize]byte)(d.window) = *(*[windowSize]byte)(d.window[windowSize:]) - s.index -= windowSize - d.windowEnd -= windowSize - if d.blockStart >= windowSize { - d.blockStart -= windowSize - } else { - d.blockStart = math.MaxInt32 - } - s.hashOffset += windowSize - if s.hashOffset > maxHashOffset { - delta := s.hashOffset - 1 - s.hashOffset -= delta - s.chainHead -= delta - // Iterate over slices instead of arrays to avoid copying - // the entire table onto the stack (Issue #18625). - for i, v := range s.hashPrev[:] { - if int(v) > delta { - s.hashPrev[i] = uint32(int(v) - delta) - } else { - s.hashPrev[i] = 0 - } - } - for i, v := range s.hashHead[:] { - if int(v) > delta { - s.hashHead[i] = uint32(int(v) - delta) - } else { - s.hashHead[i] = 0 - } - } - } - } - n := copy(d.window[d.windowEnd:], b) - d.windowEnd += n - return n -} - -func (d *compressor) writeBlock(tok *tokens, index int, eof bool) error { - if index > 0 || eof { - var window []byte - if d.blockStart <= index { - window = d.window[d.blockStart:index] - } - d.blockStart = index - // d.w.writeBlock(tok, eof, window) - d.w.writeBlockDynamic(tok, eof, window, d.sync) - return d.w.err - } - return nil -} - -// writeBlockSkip writes the current block and uses the number of tokens -// to determine if the block should be stored on no matches, or -// only huffman encoded. -func (d *compressor) writeBlockSkip(tok *tokens, index int, eof bool) error { - if index > 0 || eof { - if d.blockStart <= index { - window := d.window[d.blockStart:index] - // If we removed less than a 64th of all literals - // we huffman compress the block. - if int(tok.n) > len(window)-int(tok.n>>6) { - d.w.writeBlockHuff(eof, window, d.sync) - } else { - // Write a dynamic huffman block. - d.w.writeBlockDynamic(tok, eof, window, d.sync) - } - } else { - d.w.writeBlock(tok, eof, nil) - } - d.blockStart = index - return d.w.err - } - return nil -} - -// fillWindow will fill the current window with the supplied -// dictionary and calculate all hashes. -// This is much faster than doing a full encode. -// Should only be used after a start/reset. -func (d *compressor) fillWindow(b []byte) { - // Do not fill window if we are in store-only or huffman mode. - if d.level <= 0 && d.level > -MinCustomWindowSize { - return - } - if d.fast != nil { - // encode the last data, but discard the result - if len(b) > maxMatchOffset { - b = b[len(b)-maxMatchOffset:] - } - d.fast.Encode(&d.tokens, b) - d.tokens.Reset() - return - } - s := d.state - // If we are given too much, cut it. - if len(b) > windowSize { - b = b[len(b)-windowSize:] - } - // Add all to window. - n := copy(d.window[d.windowEnd:], b) - - // Calculate 256 hashes at the time (more L1 cache hits) - loops := (n + 256 - minMatchLength) / 256 - for j := range loops { - startindex := j * 256 - end := min(startindex+256+minMatchLength-1, n) - tocheck := d.window[startindex:end] - dstSize := len(tocheck) - minMatchLength + 1 - - if dstSize <= 0 { - continue - } - - dst := s.hashMatch[:dstSize] - bulkHash4(tocheck, dst) - var newH uint32 - for i, val := range dst { - di := i + startindex - newH = val & hashMask - // Get previous value with the same hash. - // Our chain should point to the previous value. - s.hashPrev[di&windowMask] = s.hashHead[newH] - // Set the head of the hash chain to us. - s.hashHead[newH] = uint32(di + s.hashOffset) - } - } - // Update window information. - d.windowEnd += n - s.index = n -} - -// Try to find a match starting at index whose length is greater than prevSize. -// We only look at chainCount possibilities before giving up. -// pos = s.index, prevHead = s.chainHead-s.hashOffset, prevLength=minMatchLength-1, lookahead -func (d *compressor) findMatch(pos int, prevHead int, lookahead int) (length, offset int, ok bool) { - minMatchLook := min(lookahead, maxMatchLength) - - win := d.window[0 : pos+minMatchLook] - - // We quit when we get a match that's at least nice long - nice := min(d.nice, len(win)-pos) - - // If we've got a match that's good enough, only look in 1/4 the chain. - tries := d.chain - length = minMatchLength - 1 - - wEnd := win[pos+length] - wPos := win[pos:] - minIndex := max(pos-windowSize, 0) - offset = 0 - - if d.chain < 100 { - for i := prevHead; tries > 0; tries-- { - if wEnd == win[i+length] { - n := matchLen(win[i:i+minMatchLook], wPos) - if n > length { - length = n - offset = pos - i - ok = true - if n >= nice { - // The match is good enough that we don't try to find a better one. - break - } - wEnd = win[pos+n] - } - } - if i <= minIndex { - // hashPrev[i & windowMask] has already been overwritten, so stop now. - break - } - i = int(d.state.hashPrev[i&windowMask]) - d.state.hashOffset - if i < minIndex { - break - } - } - return - } - - // Minimum gain to accept a match. - cGain := 4 - - // Some like it higher (CSV), some like it lower (JSON) - const baseCost = 3 - // Base is 4 bytes at with an additional cost. - // Matches must be better than this. - - for i := prevHead; tries > 0; tries-- { - if wEnd == win[i+length] { - n := matchLen(win[i:i+minMatchLook], wPos) - if n > length { - // Calculate gain. Estimate - newGain := d.h.bitLengthRaw(wPos[:n]) - int(offsetExtraBits[offsetCode(uint32(pos-i))]) - baseCost - int(lengthExtraBits[lengthCodes[(n-3)&255]]) - - // fmt.Println("gain:", newGain, "prev:", cGain, "raw:", d.h.bitLengthRaw(wPos[:n]), "this-len:", n, "prev-len:", length) - if newGain > cGain { - length = n - offset = pos - i - cGain = newGain - ok = true - if n >= nice { - // The match is good enough that we don't try to find a better one. - break - } - wEnd = win[pos+n] - } - } - } - if i <= minIndex { - // hashPrev[i & windowMask] has already been overwritten, so stop now. - break - } - i = int(d.state.hashPrev[i&windowMask]) - d.state.hashOffset - if i < minIndex { - break - } - } - return -} - -func (d *compressor) writeStoredBlock(buf []byte) error { - if d.w.writeStoredHeader(len(buf), false); d.w.err != nil { - return d.w.err - } - d.w.writeBytes(buf) - return d.w.err -} - -// hash4 returns a hash representation of the first 4 bytes -// of the supplied slice. -// The caller must ensure that len(b) >= 4. -func hash4(b []byte) uint32 { - return hash4u(le.Load32(b, 0), hashBits) -} - -// hash4 returns the hash of u to fit in a hash table with h bits. -// Preferably h should be a constant and should always be <32. -func hash4u(u uint32, h uint8) uint32 { - return (u * prime4bytes) >> (32 - h) -} - -// bulkHash4 will compute hashes using the same -// algorithm as hash4 -func bulkHash4(b []byte, dst []uint32) { - if len(b) < 4 { - return - } - hb := le.Load32(b, 0) - - dst[0] = hash4u(hb, hashBits) - end := len(b) - 4 + 1 - for i := 1; i < end; i++ { - hb = (hb >> 8) | uint32(b[i+3])<<24 - dst[i] = hash4u(hb, hashBits) - } -} - -func (d *compressor) initDeflate() { - d.window = make([]byte, 2*windowSize) - d.byteAvailable = false - d.err = nil - if d.state == nil { - return - } - s := d.state - s.index = 0 - s.hashOffset = 1 - s.length = minMatchLength - 1 - s.offset = 0 - s.chainHead = -1 -} - -// deflateLazy is the same as deflate, but with d.fastSkipHashing == skipNever, -// meaning it always has lazy matching on. -func (d *compressor) deflateLazy() { - s := d.state - // Sanity enables additional runtime tests. - // It's intended to be used during development - // to supplement the currently ad-hoc unit tests. - const sanity = debugDeflate - - if d.windowEnd-s.index < minMatchLength+maxMatchLength && !d.sync { - return - } - if d.windowEnd != s.index && d.chain > 100 { - // Get literal huffman coder. - if d.h == nil { - d.h = newHuffmanEncoder(maxFlateBlockTokens) - } - var tmp [256]uint16 - toIndex := d.window[s.index:d.windowEnd] - toIndex = toIndex[:min(len(toIndex), maxFlateBlockTokens)] - for _, v := range toIndex { - tmp[v]++ - } - d.h.generate(tmp[:], 15) - } - - s.maxInsertIndex = d.windowEnd - (minMatchLength - 1) - - for { - if sanity && s.index > d.windowEnd { - panic("index > windowEnd") - } - lookahead := d.windowEnd - s.index - if lookahead < minMatchLength+maxMatchLength { - if !d.sync { - return - } - if sanity && s.index > d.windowEnd { - panic("index > windowEnd") - } - if lookahead == 0 { - // Flush current output block if any. - if d.byteAvailable { - // There is still one pending token that needs to be flushed - d.tokens.AddLiteral(d.window[s.index-1]) - d.byteAvailable = false - } - if d.tokens.n > 0 { - if d.err = d.writeBlock(&d.tokens, s.index, false); d.err != nil { - return - } - d.tokens.Reset() - } - return - } - } - if s.index < s.maxInsertIndex { - // Update the hash - hash := hash4(d.window[s.index:]) - ch := s.hashHead[hash] - s.chainHead = int(ch) - s.hashPrev[s.index&windowMask] = ch - s.hashHead[hash] = uint32(s.index + s.hashOffset) - } - prevLength := s.length - prevOffset := s.offset - s.length = minMatchLength - 1 - s.offset = 0 - minIndex := max(s.index-windowSize, 0) - - if s.chainHead-s.hashOffset >= minIndex && lookahead > prevLength && prevLength < d.lazy { - if newLength, newOffset, ok := d.findMatch(s.index, s.chainHead-s.hashOffset, lookahead); ok { - s.length = newLength - s.offset = newOffset - } - } - - if prevLength >= minMatchLength && s.length <= prevLength { - // No better match, but check for better match at end... - // - // Skip forward a number of bytes. - // Offset of 2 seems to yield best results. 3 is sometimes better. - const checkOff = 2 - - // Check all, except full length - if prevLength < maxMatchLength-checkOff { - prevIndex := s.index - 1 - if prevIndex+prevLength < s.maxInsertIndex { - end := min(lookahead, maxMatchLength+checkOff) - end += prevIndex - - // Hash at match end. - h := hash4(d.window[prevIndex+prevLength:]) - ch2 := int(s.hashHead[h]) - s.hashOffset - prevLength - if prevIndex-ch2 != prevOffset && ch2 > minIndex+checkOff { - length := matchLen(d.window[prevIndex+checkOff:end], d.window[ch2+checkOff:]) - // It seems like a pure length metric is best. - if length > prevLength { - prevLength = length - prevOffset = prevIndex - ch2 - - // Extend back... - for i := checkOff - 1; i >= 0; i-- { - if prevLength >= maxMatchLength || d.window[prevIndex+i] != d.window[ch2+i] { - // Emit tokens we "owe" - for j := 0; j <= i; j++ { - d.tokens.AddLiteral(d.window[prevIndex+j]) - if d.tokens.n == maxFlateBlockTokens { - // The block includes the current character - if d.err = d.writeBlock(&d.tokens, s.index, false); d.err != nil { - return - } - d.tokens.Reset() - } - s.index++ - if s.index < s.maxInsertIndex { - h := hash4(d.window[s.index:]) - ch := s.hashHead[h] - s.chainHead = int(ch) - s.hashPrev[s.index&windowMask] = ch - s.hashHead[h] = uint32(s.index + s.hashOffset) - } - } - break - } else { - prevLength++ - } - } - } else if false { - // Check one further ahead. - // Only rarely better, disabled for now. - prevIndex++ - h := hash4(d.window[prevIndex+prevLength:]) - ch2 := int(s.hashHead[h]) - s.hashOffset - prevLength - if prevIndex-ch2 != prevOffset && ch2 > minIndex+checkOff { - length := matchLen(d.window[prevIndex+checkOff:end], d.window[ch2+checkOff:]) - // It seems like a pure length metric is best. - if length > prevLength+checkOff { - prevLength = length - prevOffset = prevIndex - ch2 - prevIndex-- - - // Extend back... - for i := checkOff; i >= 0; i-- { - if prevLength >= maxMatchLength || d.window[prevIndex+i] != d.window[ch2+i-1] { - // Emit tokens we "owe" - for j := 0; j <= i; j++ { - d.tokens.AddLiteral(d.window[prevIndex+j]) - if d.tokens.n == maxFlateBlockTokens { - // The block includes the current character - if d.err = d.writeBlock(&d.tokens, s.index, false); d.err != nil { - return - } - d.tokens.Reset() - } - s.index++ - if s.index < s.maxInsertIndex { - h := hash4(d.window[s.index:]) - ch := s.hashHead[h] - s.chainHead = int(ch) - s.hashPrev[s.index&windowMask] = ch - s.hashHead[h] = uint32(s.index + s.hashOffset) - } - } - break - } else { - prevLength++ - } - } - } - } - } - } - } - } - // There was a match at the previous step, and the current match is - // not better. Output the previous match. - d.tokens.AddMatch(uint32(prevLength-3), uint32(prevOffset-minOffsetSize)) - - // Insert in the hash table all strings up to the end of the match. - // index and index-1 are already inserted. If there is not enough - // lookahead, the last two strings are not inserted into the hash - // table. - newIndex := s.index + prevLength - 1 - // Calculate missing hashes - end := min(newIndex, s.maxInsertIndex) - end += minMatchLength - 1 - startindex := min(s.index+1, s.maxInsertIndex) - tocheck := d.window[startindex:end] - dstSize := len(tocheck) - minMatchLength + 1 - if dstSize > 0 { - dst := s.hashMatch[:dstSize] - bulkHash4(tocheck, dst) - var newH uint32 - for i, val := range dst { - di := i + startindex - newH = val & hashMask - // Get previous value with the same hash. - // Our chain should point to the previous value. - s.hashPrev[di&windowMask] = s.hashHead[newH] - // Set the head of the hash chain to us. - s.hashHead[newH] = uint32(di + s.hashOffset) - } - } - - s.index = newIndex - d.byteAvailable = false - s.length = minMatchLength - 1 - if d.tokens.n == maxFlateBlockTokens { - // The block includes the current character - if d.err = d.writeBlock(&d.tokens, s.index, false); d.err != nil { - return - } - d.tokens.Reset() - } - s.ii = 0 - } else { - // Reset, if we got a match this run. - if s.length >= minMatchLength { - s.ii = 0 - } - // We have a byte waiting. Emit it. - if d.byteAvailable { - s.ii++ - d.tokens.AddLiteral(d.window[s.index-1]) - if d.tokens.n == maxFlateBlockTokens { - if d.err = d.writeBlock(&d.tokens, s.index, false); d.err != nil { - return - } - d.tokens.Reset() - } - s.index++ - - // If we have a long run of no matches, skip additional bytes - // Resets when s.ii overflows after 64KB. - if n := int(s.ii) - d.chain; n > 0 { - n = 1 + int(n>>6) - for j := 0; j < n; j++ { - if s.index >= d.windowEnd-1 { - break - } - d.tokens.AddLiteral(d.window[s.index-1]) - if d.tokens.n == maxFlateBlockTokens { - if d.err = d.writeBlock(&d.tokens, s.index, false); d.err != nil { - return - } - d.tokens.Reset() - } - // Index... - if s.index < s.maxInsertIndex { - h := hash4(d.window[s.index:]) - ch := s.hashHead[h] - s.chainHead = int(ch) - s.hashPrev[s.index&windowMask] = ch - s.hashHead[h] = uint32(s.index + s.hashOffset) - } - s.index++ - } - // Flush last byte - d.tokens.AddLiteral(d.window[s.index-1]) - d.byteAvailable = false - // s.length = minMatchLength - 1 // not needed, since s.ii is reset above, so it should never be > minMatchLength - if d.tokens.n == maxFlateBlockTokens { - if d.err = d.writeBlock(&d.tokens, s.index, false); d.err != nil { - return - } - d.tokens.Reset() - } - } - } else { - s.index++ - d.byteAvailable = true - } - } - } -} - -func (d *compressor) store() { - if d.windowEnd > 0 && (d.windowEnd == maxStoreBlockSize || d.sync) { - d.err = d.writeStoredBlock(d.window[:d.windowEnd]) - d.windowEnd = 0 - } -} - -// fillWindow will fill the buffer with data for huffman-only compression. -// The number of bytes copied is returned. -func (d *compressor) fillBlock(b []byte) int { - n := copy(d.window[d.windowEnd:], b) - d.windowEnd += n - return n -} - -// storeHuff will compress and store the currently added data, -// if enough has been accumulated or we at the end of the stream. -// Any error that occurred will be in d.err -func (d *compressor) storeHuff() { - if d.windowEnd < len(d.window) && !d.sync || d.windowEnd == 0 { - return - } - d.w.writeBlockHuff(false, d.window[:d.windowEnd], d.sync) - d.err = d.w.err - d.windowEnd = 0 -} - -// storeFast will compress and store the currently added data, -// if enough has been accumulated or we at the end of the stream. -// Any error that occurred will be in d.err -func (d *compressor) storeFast() { - // We only compress if we have maxStoreBlockSize. - if d.windowEnd < len(d.window) { - if !d.sync { - return - } - // Handle extremely small sizes. - if d.windowEnd < 128 { - if d.windowEnd == 0 { - return - } - if d.windowEnd <= 32 { - d.err = d.writeStoredBlock(d.window[:d.windowEnd]) - } else { - d.w.writeBlockHuff(false, d.window[:d.windowEnd], true) - d.err = d.w.err - } - d.tokens.Reset() - d.windowEnd = 0 - d.fast.Reset() - return - } - } - - d.fast.Encode(&d.tokens, d.window[:d.windowEnd]) - // If we made zero matches, store the block as is. - if d.tokens.n == 0 { - d.err = d.writeStoredBlock(d.window[:d.windowEnd]) - // If we removed less than 1/16th, huffman compress the block. - } else if int(d.tokens.n) > d.windowEnd-(d.windowEnd>>4) { - d.w.writeBlockHuff(false, d.window[:d.windowEnd], d.sync) - d.err = d.w.err - } else { - d.w.writeBlockDynamic(&d.tokens, false, d.window[:d.windowEnd], d.sync) - d.err = d.w.err - } - d.tokens.Reset() - d.windowEnd = 0 -} - -// write will add input byte to the stream. -// Unless an error occurs all bytes will be consumed. -func (d *compressor) write(b []byte) (n int, err error) { - if d.err != nil { - return 0, d.err - } - n = len(b) - for len(b) > 0 { - if d.windowEnd == len(d.window) || d.sync { - d.step(d) - } - b = b[d.fill(d, b):] - if d.err != nil { - return 0, d.err - } - } - return n, d.err -} - -func (d *compressor) syncFlush() error { - d.sync = true - if d.err != nil { - return d.err - } - d.step(d) - if d.err == nil { - d.w.writeStoredHeader(0, false) - d.w.flush() - d.err = d.w.err - } - d.sync = false - return d.err -} - -func (d *compressor) init(w io.Writer, level int) (err error) { - d.w = newHuffmanBitWriter(w) - - switch { - case level == NoCompression: - d.window = make([]byte, maxStoreBlockSize) - d.fill = (*compressor).fillBlock - d.step = (*compressor).store - case level == ConstantCompression: - d.w.logNewTablePenalty = 10 - d.window = make([]byte, 32<<10) - d.fill = (*compressor).fillBlock - d.step = (*compressor).storeHuff - case level == DefaultCompression: - level = 5 - fallthrough - case level >= 1 && level <= 6: - d.w.logNewTablePenalty = 7 - d.fast = newFastEnc(level) - d.window = make([]byte, maxStoreBlockSize) - d.fill = (*compressor).fillBlock - d.step = (*compressor).storeFast - case 7 <= level && level <= 9: - d.w.logNewTablePenalty = 8 - d.state = &advancedState{} - d.compressionLevel = levels[level] - d.initDeflate() - d.fill = (*compressor).fillDeflate - d.step = (*compressor).deflateLazy - case -level >= MinCustomWindowSize && -level <= MaxCustomWindowSize: - d.w.logNewTablePenalty = 7 - d.fast = &fastEncL5Window{maxOffset: int32(-level), cur: maxStoreBlockSize} - d.window = make([]byte, maxStoreBlockSize) - d.fill = (*compressor).fillBlock - d.step = (*compressor).storeFast - default: - return fmt.Errorf("flate: invalid compression level %d: want value in range [-2, 9]", level) - } - d.level = level - return nil -} - -// reset the state of the compressor. -func (d *compressor) reset(w io.Writer) { - d.w.reset(w) - d.sync = false - d.err = nil - // We only need to reset a few things for Snappy. - if d.fast != nil { - d.fast.Reset() - d.windowEnd = 0 - d.tokens.Reset() - return - } - switch d.compressionLevel.chain { - case 0: - // level was NoCompression or ConstantCompression. - d.windowEnd = 0 - default: - s := d.state - s.chainHead = -1 - for i := range s.hashHead { - s.hashHead[i] = 0 - } - for i := range s.hashPrev { - s.hashPrev[i] = 0 - } - s.hashOffset = 1 - s.index, d.windowEnd = 0, 0 - d.blockStart, d.byteAvailable = 0, false - d.tokens.Reset() - s.length = minMatchLength - 1 - s.offset = 0 - s.ii = 0 - s.maxInsertIndex = 0 - } -} - -func (d *compressor) close() error { - if d.err != nil { - return d.err - } - d.sync = true - d.step(d) - if d.err != nil { - return d.err - } - if d.w.writeStoredHeader(0, true); d.w.err != nil { - return d.w.err - } - d.w.flush() - d.w.reset(nil) - return d.w.err -} - -// NewWriter returns a new Writer compressing data at the given level. -// Following zlib, levels range from 1 (BestSpeed) to 9 (BestCompression); -// higher levels typically run slower but compress more. -// Level 0 (NoCompression) does not attempt any compression; it only adds the -// necessary DEFLATE framing. -// Level -1 (DefaultCompression) uses the default compression level. -// Level -2 (ConstantCompression) will use Huffman compression only, giving -// a very fast compression for all types of input, but sacrificing considerable -// compression efficiency. -// -// If level is in the range [-2, 9] then the error returned will be nil. -// Otherwise the error returned will be non-nil. -func NewWriter(w io.Writer, level int) (*Writer, error) { - var dw Writer - if err := dw.d.init(w, level); err != nil { - return nil, err - } - return &dw, nil -} - -// NewWriterDict is like NewWriter but initializes the new -// Writer with a preset dictionary. The returned Writer behaves -// as if the dictionary had been written to it without producing -// any compressed output. The compressed data written to w -// can only be decompressed by a Reader initialized with the -// same dictionary. -func NewWriterDict(w io.Writer, level int, dict []byte) (*Writer, error) { - zw, err := NewWriter(w, level) - if err != nil { - return nil, err - } - zw.d.fillWindow(dict) - zw.dict = append(zw.dict, dict...) // duplicate dictionary for Reset method. - return zw, err -} - -// MinCustomWindowSize is the minimum window size that can be sent to NewWriterWindow. -const MinCustomWindowSize = 32 - -// MaxCustomWindowSize is the maximum custom window that can be sent to NewWriterWindow. -const MaxCustomWindowSize = windowSize - -// NewWriterWindow returns a new Writer compressing data with a custom window size. -// windowSize must be from MinCustomWindowSize to MaxCustomWindowSize. -func NewWriterWindow(w io.Writer, windowSize int) (*Writer, error) { - if windowSize < MinCustomWindowSize { - return nil, errors.New("flate: requested window size less than MinWindowSize") - } - if windowSize > MaxCustomWindowSize { - return nil, errors.New("flate: requested window size bigger than MaxCustomWindowSize") - } - var dw Writer - if err := dw.d.init(w, -windowSize); err != nil { - return nil, err - } - return &dw, nil -} - -// A Writer takes data written to it and writes the compressed -// form of that data to an underlying writer (see NewWriter). -type Writer struct { - d compressor - dict []byte -} - -// Write writes data to w, which will eventually write the -// compressed form of data to its underlying writer. -func (w *Writer) Write(data []byte) (n int, err error) { - return w.d.write(data) -} - -// Flush flushes any pending data to the underlying writer. -// It is useful mainly in compressed network protocols, to ensure that -// a remote reader has enough data to reconstruct a packet. -// Flush does not return until the data has been written. -// Calling Flush when there is no pending data still causes the Writer -// to emit a sync marker of at least 4 bytes. -// If the underlying writer returns an error, Flush returns that error. -// -// In the terminology of the zlib library, Flush is equivalent to Z_SYNC_FLUSH. -func (w *Writer) Flush() error { - // For more about flushing: - // http://www.bolet.org/~pornin/deflate-flush.html - return w.d.syncFlush() -} - -// Close flushes and closes the writer. -func (w *Writer) Close() error { - return w.d.close() -} - -// Reset discards the writer's state and makes it equivalent to -// the result of NewWriter or NewWriterDict called with dst -// and w's level and dictionary. -func (w *Writer) Reset(dst io.Writer) { - if len(w.dict) > 0 { - // w was created with NewWriterDict - w.d.reset(dst) - if dst != nil { - w.d.fillWindow(w.dict) - } - } else { - // w was created with NewWriter - w.d.reset(dst) - } -} - -// ResetDict discards the writer's state and makes it equivalent to -// the result of NewWriter or NewWriterDict called with dst -// and w's level, but sets a specific dictionary. -func (w *Writer) ResetDict(dst io.Writer, dict []byte) { - w.dict = dict - w.d.reset(dst) - w.d.fillWindow(w.dict) -} diff --git a/internal/compress/flate/deflate_test.go b/internal/compress/flate/deflate_test.go deleted file mode 100644 index 9ac3da1f..00000000 --- a/internal/compress/flate/deflate_test.go +++ /dev/null @@ -1,708 +0,0 @@ -// Copyright 2009 The Go Authors. All rights reserved. -// Copyright (c) 2015 Klaus Post -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package flate - -import ( - "bytes" - "fmt" - "io" - "os" - "reflect" - "strings" - "sync" - "testing" -) - -type deflateTest struct { - in []byte - level int - out []byte -} - -type deflateInflateTest struct { - in []byte -} - -type reverseBitsTest struct { - in uint16 - bitCount uint8 - out uint16 -} - -var deflateTests = []*deflateTest{ - 0: {[]byte{}, 0, []byte{0x3, 0x0}}, - 1: {[]byte{0x11}, BestCompression, []byte{0x12, 0x4, 0xc, 0x0}}, - 2: {[]byte{0x11}, BestCompression, []byte{0x12, 0x4, 0xc, 0x0}}, - 3: {[]byte{0x11}, BestCompression, []byte{0x12, 0x4, 0xc, 0x0}}, - - 4: {[]byte{0x11}, 0, []byte{0x0, 0x1, 0x0, 0xfe, 0xff, 0x11, 0x3, 0x0}}, - 5: {[]byte{0x11, 0x12}, 0, []byte{0x0, 0x2, 0x0, 0xfd, 0xff, 0x11, 0x12, 0x3, 0x0}}, - 6: { - []byte{0x11, 0x11, 0x11, 0x11, 0x11, 0x11, 0x11, 0x11}, - 0, - []byte{0x0, 0x8, 0x0, 0xf7, 0xff, 0x11, 0x11, 0x11, 0x11, 0x11, 0x11, 0x11, 0x11, 0x3, 0x0}, - }, - 7: {[]byte{}, 1, []byte{0x3, 0x0}}, - 8: {[]byte{0x11}, BestCompression, []byte{0x12, 0x4, 0xc, 0x0}}, - 9: {[]byte{0x11, 0x12}, BestCompression, []byte{0x12, 0x14, 0x2, 0xc, 0x0}}, - 10: {[]byte{0x11, 0x11, 0x11, 0x11, 0x11, 0x11, 0x11, 0x11, 0x11}, BestCompression, []byte{0x12, 0x84, 0x1, 0xc0, 0x0}}, - 11: {[]byte{}, 9, []byte{0x3, 0x0}}, - 12: {[]byte{0x11}, 9, []byte{0x12, 0x4, 0xc, 0x0}}, - 13: {[]byte{0x11, 0x12}, 9, []byte{0x12, 0x14, 0x2, 0xc, 0x0}}, - 14: {[]byte{0x11, 0x11, 0x11, 0x11, 0x11, 0x11, 0x11, 0x11, 0x11}, 9, []byte{0x12, 0x84, 0x1, 0xc0, 0x0}}, -} - -var deflateInflateTests = []*deflateInflateTest{ - {[]byte{}}, - {[]byte{0x11}}, - {[]byte{0x11, 0x12}}, - {[]byte{0x11, 0x11, 0x11, 0x11, 0x11, 0x11, 0x11, 0x11}}, - {[]byte{0x11, 0x10, 0x13, 0x41, 0x21, 0x21, 0x41, 0x13, 0x87, 0x78, 0x13}}, - {largeDataChunk()}, -} - -var reverseBitsTests = []*reverseBitsTest{ - {1, 1, 1}, - {1, 2, 2}, - {1, 3, 4}, - {1, 4, 8}, - {1, 5, 16}, - {17, 5, 17}, - {257, 9, 257}, - {29, 5, 23}, -} - -func largeDataChunk() []byte { - result := make([]byte, 100000) - for i := range result { - result[i] = byte(i * i & 0xFF) - } - return result -} - -func TestBulkHash4(t *testing.T) { - for _, x := range deflateTests { - y := x.out - if len(y) >= minMatchLength { - y = append(y, y...) - for j := 4; j < len(y); j++ { - y := y[:j] - dst := make([]uint32, len(y)-minMatchLength+1) - for i := range dst { - dst[i] = uint32(i + 100) - } - bulkHash4(y, dst) - for i, val := range dst { - got := val - expect := hash4(y[i:]) - if got != expect && got == uint32(i)+100 { - t.Errorf("Len:%d Index:%d, expected 0x%08x but not modified", len(y), i, expect) - } else if got != expect { - t.Errorf("Len:%d Index:%d, got 0x%08x expected:0x%08x", len(y), i, got, expect) - } else { - // t.Logf("Len:%d Index:%d OK (0x%08x)", len(y), i, got) - } - } - } - } - } -} - -func TestDeflate(t *testing.T) { - for i, h := range deflateTests { - var buf bytes.Buffer - w, err := NewWriter(&buf, h.level) - if err != nil { - t.Errorf("NewWriter: %v", err) - continue - } - w.Write(h.in) - w.Close() - if !bytes.Equal(buf.Bytes(), h.out) { - t.Errorf("%d: Deflate(%d, %x) got \n%#v, want \n%#v", i, h.level, h.in, buf.Bytes(), h.out) - } - } -} - -// A sparseReader returns a stream consisting of 0s followed by 1<<16 1s. -// This tests missing hash references in a very large input. -type sparseReader struct { - l int64 - cur int64 -} - -func (r *sparseReader) Read(b []byte) (n int, err error) { - if r.cur >= r.l { - return 0, io.EOF - } - n = len(b) - cur := r.cur + int64(n) - if cur > r.l { - n -= int(cur - r.l) - cur = r.l - } - for i := range b[0:n] { - if r.cur+int64(i) >= r.l-1<<16 { - b[i] = 1 - } else { - b[i] = 0 - } - } - r.cur = cur - return -} - -func TestVeryLongSparseChunk(t *testing.T) { - if testing.Short() { - t.Skip("skipping sparse chunk during short test") - } - var buf bytes.Buffer - w, err := NewWriter(&buf, 1) - if err != nil { - t.Errorf("NewWriter: %v", err) - return - } - if _, err = io.Copy(w, &sparseReader{l: 23e8}); err != nil { - t.Errorf("Compress failed: %v", err) - return - } - t.Log("Length:", buf.Len()) -} - -func TestOneMByte(t *testing.T) { - var input [1024 * 1024]byte - - var compressedOutput bytes.Buffer - for level := HuffmanOnly; level <= BestCompression; level++ { - compressedOutput.Reset() - compressor, err := NewWriter(&compressedOutput, level) - if err != nil { - t.Fatalf("create: %s", err) - } - // Use single write... - if _, err := compressor.Write(input[:]); err != nil { - t.Fatalf("compress: %s", err) - } - - if err := compressor.Close(); err != nil { - t.Fatalf("close: %s", err) - } - - var decompressedOutput bytes.Buffer - - decompresser := NewReader(&compressedOutput) - t.Log("level:", level, "compressed:", compressedOutput.Len()) - if _, err := io.Copy(&decompressedOutput, decompresser); err != nil { - t.Fatalf("decompress: %s", err) - } - - if !bytes.Equal(input[:], decompressedOutput.Bytes()) { - t.Fatal("input and output do not match") - } - } -} - -type syncBuffer struct { - buf bytes.Buffer - mu sync.RWMutex - closed bool - ready chan bool -} - -func newSyncBuffer() *syncBuffer { - return &syncBuffer{ready: make(chan bool, 1)} -} - -func (b *syncBuffer) Read(p []byte) (n int, err error) { - for { - b.mu.RLock() - n, err = b.buf.Read(p) - b.mu.RUnlock() - if n > 0 || b.closed { - return - } - <-b.ready - } -} - -func (b *syncBuffer) signal() { - select { - case b.ready <- true: - default: - } -} - -func (b *syncBuffer) Write(p []byte) (n int, err error) { - n, err = b.buf.Write(p) - b.signal() - return -} - -func (b *syncBuffer) WriteMode() { - b.mu.Lock() -} - -func (b *syncBuffer) ReadMode() { - b.mu.Unlock() - b.signal() -} - -func (b *syncBuffer) Close() error { - b.closed = true - b.signal() - return nil -} - -func testSync(t *testing.T, level int, input []byte, name string) { - if len(input) == 0 { - return - } - - t.Logf("--testSync %d, %d, %s", level, len(input), name) - buf := newSyncBuffer() - buf1 := new(bytes.Buffer) - buf.WriteMode() - w, err := NewWriter(io.MultiWriter(buf, buf1), level) - if err != nil { - t.Errorf("NewWriter: %v", err) - return - } - r := NewReader(buf) - - // Write half the input and read back. - for i := range 2 { - var lo, hi int - if i == 0 { - lo, hi = 0, (len(input)+1)/2 - } else { - lo, hi = (len(input)+1)/2, len(input) - } - t.Logf("#%d: write %d-%d", i, lo, hi) - if _, err := w.Write(input[lo:hi]); err != nil { - t.Errorf("testSync: write: %v", err) - return - } - if i == 0 { - if err := w.Flush(); err != nil { - t.Errorf("testSync: flush: %v", err) - return - } - } else { - if err := w.Close(); err != nil { - t.Errorf("testSync: close: %v", err) - } - } - buf.ReadMode() - out := make([]byte, hi-lo+1) - m, err := io.ReadAtLeast(r, out, hi-lo) - t.Logf("#%d: read %d", i, m) - if m != hi-lo || err != nil { - t.Errorf("testSync/%d (%d, %d, %s): read %d: %d, %v (%d left)", i, level, len(input), name, hi-lo, m, err, buf.buf.Len()) - return - } - if !bytes.Equal(input[lo:hi], out[:hi-lo]) { - t.Errorf("testSync/%d: read wrong bytes: %x vs %x", i, input[lo:hi], out[:hi-lo]) - return - } - // This test originally checked that after reading - // the first half of the input, there was nothing left - // in the read buffer (buf.buf.Len() != 0) but that is - // not necessarily the case: the write Flush may emit - // some extra framing bits that are not necessary - // to process to obtain the first half of the uncompressed - // data. The test ran correctly most of the time, because - // the background goroutine had usually read even - // those extra bits by now, but it's not a useful thing to - // check. - buf.WriteMode() - } - buf.ReadMode() - out := make([]byte, 10) - if n, err := r.Read(out); n > 0 || err != io.EOF { - t.Errorf("testSync (%d, %d, %s): final Read: %d, %v (hex: %x)", level, len(input), name, n, err, out[0:n]) - } - if buf.buf.Len() != 0 { - t.Errorf("testSync (%d, %d, %s): extra data at end", level, len(input), name) - } - r.Close() - - // stream should work for ordinary reader too - r = NewReader(buf1) - out, err = io.ReadAll(r) - if err != nil { - t.Errorf("testSync: read: %s", err) - return - } - r.Close() - if !bytes.Equal(input, out) { - t.Errorf("testSync: decompress(compress(data)) != data: level=%d input=%s", level, name) - } -} - -func testToFromWithLevelAndLimit(t *testing.T, level int, input []byte, name string, limit int) { - var buffer bytes.Buffer - w, err := NewWriter(&buffer, level) - if err != nil { - t.Errorf("NewWriter: %v", err) - return - } - w.Write(input) - w.Close() - if limit > 0 { - t.Logf("level: %d - Size:%.2f%%, %d b\n", level, float64(buffer.Len()*100)/float64(limit), buffer.Len()) - } - if limit > 0 && buffer.Len() > limit { - t.Errorf("level: %d, len(compress(data)) = %d > limit = %d", level, buffer.Len(), limit) - } - - r := NewReader(&buffer) - out, err := io.ReadAll(r) - if err != nil { - t.Errorf("read: %s", err) - return - } - r.Close() - if !bytes.Equal(input, out) { - os.WriteFile("testdata/fails/"+t.Name()+".got", out, os.ModePerm) - os.WriteFile("testdata/fails/"+t.Name()+".want", input, os.ModePerm) - t.Errorf("decompress(compress(data)) != data: level=%d input=%s", level, name) - return - } - testSync(t, level, input, name) -} - -func testToFromWithLimit(t *testing.T, input []byte, name string, limit [11]int) { - for i := range 10 { - testToFromWithLevelAndLimit(t, i, input, name, limit[i]) - } - testToFromWithLevelAndLimit(t, -2, input, name, limit[10]) -} - -func TestDeflateInflate(t *testing.T) { - for i, h := range deflateInflateTests { - testToFromWithLimit(t, h.in, fmt.Sprintf("#%d", i), [11]int{}) - } -} - -func TestReverseBits(t *testing.T) { - for _, h := range reverseBitsTests { - if v := reverseBits(h.in, h.bitCount); v != h.out { - t.Errorf("reverseBits(%v,%v) = %v, want %v", - h.in, h.bitCount, v, h.out) - } - } -} - -type deflateInflateStringTest struct { - filename string - label string - limit [11]int // Number 11 is ConstantCompression -} - -var deflateInflateStringTests = []deflateInflateStringTest{ - { - "../testdata/e.txt", - "2.718281828...", - [...]int{100018, 67900, 50960, 51150, 50930, 50790, 50790, 50790, 50790, 50790, 43683 + 100}, - }, - { - "../testdata/Mark.Twain-Tom.Sawyer.txt", - "Mark.Twain-Tom.Sawyer", - [...]int{387999, 185000, 182361, 179974, 174124, 168819, 162936, 160506, 160295, 160295, 233460 + 100}, - }, -} - -func TestDeflateInflateString(t *testing.T) { - for _, test := range deflateInflateStringTests { - gold, err := os.ReadFile(test.filename) - if err != nil { - t.Error(err) - } - // Remove returns that may be present on Windows - neutral := strings.Map(func(r rune) rune { - if r != '\r' { - return r - } - return -1 - }, string(gold)) - - testToFromWithLimit(t, []byte(neutral), test.label, test.limit) - - if testing.Short() { - break - } - } -} - -func TestReaderDict(t *testing.T) { - const ( - dict = "hello world" - text = "hello again world" - ) - var b bytes.Buffer - w, err := NewWriter(&b, 5) - if err != nil { - t.Fatalf("NewWriter: %v", err) - } - w.Write([]byte(dict)) - w.Flush() - b.Reset() - w.Write([]byte(text)) - w.Close() - - r := NewReaderDict(&b, []byte(dict)) - data, err := io.ReadAll(r) - if err != nil { - t.Fatal(err) - } - if string(data) != "hello again world" { - t.Fatalf("read returned %q want %q", string(data), text) - } -} - -func TestWriterDict(t *testing.T) { - const ( - dict = "hello world Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua." - text = "hello world Lorem ipsum dolor sit amet" - ) - // This test is sensitive to algorithm changes that skip - // data in favour of speed. Higher levels are less prone to this - // so we test level 4-9. - for l := 4; l < 9; l++ { - var b bytes.Buffer - w, err := NewWriter(&b, l) - if err != nil { - t.Fatalf("level %d, NewWriter: %v", l, err) - } - w.Write([]byte(dict)) - w.Flush() - b.Reset() - w.Write([]byte(text)) - w.Close() - - var b1 bytes.Buffer - w, _ = NewWriterDict(&b1, l, []byte(dict)) - w.Write([]byte(text)) - w.Close() - - if !bytes.Equal(b1.Bytes(), b.Bytes()) { - t.Errorf("level %d, writer wrote\n%v\n want\n%v", l, b1.Bytes(), b.Bytes()) - } - } -} - -// See http://code.google.com/p/go/issues/detail?id=2508 -func TestRegression2508(t *testing.T) { - if testing.Short() { - t.Logf("test disabled with -short") - return - } - w, err := NewWriter(io.Discard, 1) - if err != nil { - t.Fatalf("NewWriter: %v", err) - } - buf := make([]byte, 1024) - for range 131072 { - if _, err := w.Write(buf); err != nil { - t.Fatalf("writer failed: %v", err) - } - } - w.Close() -} - -func TestWriterReset(t *testing.T) { - for level := -2; level <= 9; level++ { - if level == -1 { - level++ - } - if testing.Short() && level > 1 { - break - } - w, err := NewWriter(io.Discard, level) - if err != nil { - t.Fatalf("NewWriter: %v", err) - } - buf := []byte("hello world") - for range 1024 { - w.Write(buf) - } - w.Reset(io.Discard) - - wref, err := NewWriter(io.Discard, level) - if err != nil { - t.Fatalf("NewWriter: %v", err) - } - - // DeepEqual doesn't compare functions. - w.d.fill, wref.d.fill = nil, nil - w.d.step, wref.d.step = nil, nil - w.d.state, wref.d.state = nil, nil - w.d.fast, wref.d.fast = nil, nil - - // hashMatch is always overwritten when used. - if w.d.tokens.n != 0 { - t.Errorf("level %d Writer not reset after Reset. %d tokens were present", level, w.d.tokens.n) - } - // As long as the length is 0, we don't care about the content. - w.d.tokens = wref.d.tokens - - // We don't care if there are values in the window, as long as it is at d.index is 0 - w.d.window = wref.d.window - if !reflect.DeepEqual(w, wref) { - t.Errorf("level %d Writer not reset after Reset", level) - } - } - - for i := HuffmanOnly; i <= BestCompression; i++ { - testResetOutput(t, fmt.Sprint("level-", i), func(w io.Writer) (*Writer, error) { return NewWriter(w, i) }) - } - dict := []byte(strings.Repeat("we are the world - how are you?", 3)) - for i := HuffmanOnly; i <= BestCompression; i++ { - testResetOutput(t, fmt.Sprint("dict-level-", i), func(w io.Writer) (*Writer, error) { return NewWriterDict(w, i, dict) }) - } - for i := HuffmanOnly; i <= BestCompression; i++ { - testResetOutput(t, fmt.Sprint("dict-reset-level-", i), func(w io.Writer) (*Writer, error) { - w2, err := NewWriter(nil, i) - if err != nil { - return w2, err - } - w2.ResetDict(w, dict) - return w2, nil - }) - } - testResetOutput(t, fmt.Sprint("dict-reset-window"), func(w io.Writer) (*Writer, error) { - w2, err := NewWriterWindow(nil, 1024) - if err != nil { - return w2, err - } - w2.ResetDict(w, dict) - return w2, nil - }) -} - -func testResetOutput(t *testing.T, name string, newWriter func(w io.Writer) (*Writer, error)) { - t.Run(name, func(t *testing.T) { - buf := new(bytes.Buffer) - w, err := newWriter(buf) - if err != nil { - t.Fatalf("NewWriter: %v", err) - } - b := []byte("hello world - how are you doing?") - for range 1024 { - w.Write(b) - } - w.Close() - out1 := buf.Bytes() - - buf2 := new(bytes.Buffer) - w.Reset(buf2) - for range 1024 { - w.Write(b) - } - w.Close() - out2 := buf2.Bytes() - - if len(out1) != len(out2) { - t.Errorf("got %d, expected %d bytes", len(out2), len(out1)) - } - if !bytes.Equal(out1, out2) { - mm := 0 - for i, b := range out1[:len(out2)] { - if b != out2[i] { - t.Errorf("mismatch index %d: %02x, expected %02x", i, out2[i], b) - } - mm++ - if mm == 10 { - t.Fatal("Stopping") - } - } - } - t.Logf("got %d bytes", len(out1)) - }) -} - -// TestBestSpeed tests that round-tripping through deflate and then inflate -// recovers the original input. The Write sizes are near the thresholds in the -// compressor.encSpeed method (0, 16, 128), as well as near maxStoreBlockSize -// (65535). -func TestBestSpeed(t *testing.T) { - abc := make([]byte, 128) - for i := range abc { - abc[i] = byte(i) - } - abcabc := bytes.Repeat(abc, 131072/len(abc)) - var want []byte - - testCases := [][]int{ - {65536, 0}, - {65536, 1}, - {65536, 1, 256}, - {65536, 1, 65536}, - {65536, 14}, - {65536, 15}, - {65536, 16}, - {65536, 16, 256}, - {65536, 16, 65536}, - {65536, 127}, - {65536, 128}, - {65536, 128, 256}, - {65536, 128, 65536}, - {65536, 129}, - {65536, 65536, 256}, - {65536, 65536, 65536}, - } - - for i, tc := range testCases { - if testing.Short() && i > 5 { - t.Skip() - } - for _, firstN := range []int{1, 65534, 65535, 65536, 65537, 131072} { - tc[0] = firstN - outer: - for _, flush := range []bool{false, true} { - buf := new(bytes.Buffer) - want = want[:0] - - w, err := NewWriter(buf, BestSpeed) - if err != nil { - t.Errorf("i=%d, firstN=%d, flush=%t: NewWriter: %v", i, firstN, flush, err) - continue - } - for _, n := range tc { - want = append(want, abcabc[:n]...) - if _, err := w.Write(abcabc[:n]); err != nil { - t.Errorf("i=%d, firstN=%d, flush=%t: Write: %v", i, firstN, flush, err) - continue outer - } - if !flush { - continue - } - if err := w.Flush(); err != nil { - t.Errorf("i=%d, firstN=%d, flush=%t: Flush: %v", i, firstN, flush, err) - continue outer - } - } - if err := w.Close(); err != nil { - t.Errorf("i=%d, firstN=%d, flush=%t: Close: %v", i, firstN, flush, err) - continue - } - - r := NewReader(buf) - got, err := io.ReadAll(r) - if err != nil { - t.Errorf("i=%d, firstN=%d, flush=%t: ReadAll: %v", i, firstN, flush, err) - continue - } - r.Close() - - if !bytes.Equal(got, want) { - t.Errorf("i=%d, firstN=%d, flush=%t: corruption during deflate-then-inflate", i, firstN, flush) - continue - } - } - } - } -} diff --git a/internal/compress/flate/dict_decoder.go b/internal/compress/flate/dict_decoder.go deleted file mode 100644 index cb855abc..00000000 --- a/internal/compress/flate/dict_decoder.go +++ /dev/null @@ -1,181 +0,0 @@ -// Copyright 2016 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package flate - -// dictDecoder implements the LZ77 sliding dictionary as used in decompression. -// LZ77 decompresses data through sequences of two forms of commands: -// -// - Literal insertions: Runs of one or more symbols are inserted into the data -// stream as is. This is accomplished through the writeByte method for a -// single symbol, or combinations of writeSlice/writeMark for multiple symbols. -// Any valid stream must start with a literal insertion if no preset dictionary -// is used. -// -// - Backward copies: Runs of one or more symbols are copied from previously -// emitted data. Backward copies come as the tuple (dist, length) where dist -// determines how far back in the stream to copy from and length determines how -// many bytes to copy. Note that it is valid for the length to be greater than -// the distance. Since LZ77 uses forward copies, that situation is used to -// perform a form of run-length encoding on repeated runs of symbols. -// The writeCopy and tryWriteCopy are used to implement this command. -// -// For performance reasons, this implementation performs little to no sanity -// checks about the arguments. As such, the invariants documented for each -// method call must be respected. -type dictDecoder struct { - hist []byte // Sliding window history - - // Invariant: 0 <= rdPos <= wrPos <= len(hist) - wrPos int // Current output position in buffer - rdPos int // Have emitted hist[:rdPos] already - full bool // Has a full window length been written yet? -} - -// init initializes dictDecoder to have a sliding window dictionary of the given -// size. If a preset dict is provided, it will initialize the dictionary with -// the contents of dict. -func (dd *dictDecoder) init(size int, dict []byte) { - *dd = dictDecoder{hist: dd.hist} - - if cap(dd.hist) < size { - dd.hist = make([]byte, size) - } - dd.hist = dd.hist[:size] - - if len(dict) > len(dd.hist) { - dict = dict[len(dict)-len(dd.hist):] - } - dd.wrPos = copy(dd.hist, dict) - if dd.wrPos == len(dd.hist) { - dd.wrPos = 0 - dd.full = true - } - dd.rdPos = dd.wrPos -} - -// histSize reports the total amount of historical data in the dictionary. -func (dd *dictDecoder) histSize() int { - if dd.full { - return len(dd.hist) - } - return dd.wrPos -} - -// availRead reports the number of bytes that can be flushed by readFlush. -func (dd *dictDecoder) availRead() int { - return dd.wrPos - dd.rdPos -} - -// availWrite reports the available amount of output buffer space. -func (dd *dictDecoder) availWrite() int { - return len(dd.hist) - dd.wrPos -} - -// writeSlice returns a slice of the available buffer to write data to. -// -// This invariant will be kept: len(s) <= availWrite() -func (dd *dictDecoder) writeSlice() []byte { - return dd.hist[dd.wrPos:] -} - -// writeMark advances the writer pointer by cnt. -// -// This invariant must be kept: 0 <= cnt <= availWrite() -func (dd *dictDecoder) writeMark(cnt int) { - dd.wrPos += cnt -} - -// writeByte writes a single byte to the dictionary. -// -// This invariant must be kept: 0 < availWrite() -func (dd *dictDecoder) writeByte(c byte) { - dd.hist[dd.wrPos] = c - dd.wrPos++ -} - -// writeCopy copies a string at a given (dist, length) to the output. -// This returns the number of bytes copied and may be less than the requested -// length if the available space in the output buffer is too small. -// -// This invariant must be kept: 0 < dist <= histSize() -func (dd *dictDecoder) writeCopy(dist, length int) int { - dstBase := dd.wrPos - dstPos := dstBase - srcPos := dstPos - dist - endPos := min(dstPos+length, len(dd.hist)) - - // Copy non-overlapping section after destination position. - // - // This section is non-overlapping in that the copy length for this section - // is always less than or equal to the backwards distance. This can occur - // if a distance refers to data that wraps-around in the buffer. - // Thus, a backwards copy is performed here; that is, the exact bytes in - // the source prior to the copy is placed in the destination. - if srcPos < 0 { - srcPos += len(dd.hist) - dstPos += copy(dd.hist[dstPos:endPos], dd.hist[srcPos:]) - srcPos = 0 - } - - // Copy possibly overlapping section before destination position. - // - // This section can overlap if the copy length for this section is larger - // than the backwards distance. This is allowed by LZ77 so that repeated - // strings can be succinctly represented using (dist, length) pairs. - // Thus, a forwards copy is performed here; that is, the bytes copied is - // possibly dependent on the resulting bytes in the destination as the copy - // progresses along. This is functionally equivalent to the following: - // - // for i := 0; i < endPos-dstPos; i++ { - // dd.hist[dstPos+i] = dd.hist[srcPos+i] - // } - // dstPos = endPos - // - for dstPos < endPos { - dstPos += copy(dd.hist[dstPos:endPos], dd.hist[srcPos:dstPos]) - } - - dd.wrPos = dstPos - return dstPos - dstBase -} - -// tryWriteCopy tries to copy a string at a given (distance, length) to the -// output. This specialized version is optimized for short distances. -// -// This method is designed to be inlined for performance reasons. -// -// This invariant must be kept: 0 < dist <= histSize() -func (dd *dictDecoder) tryWriteCopy(dist, length int) int { - dstPos := dd.wrPos - endPos := dstPos + length - if dstPos < dist || endPos > len(dd.hist) { - return 0 - } - dstBase := dstPos - srcPos := dstPos - dist - - // Copy possibly overlapping section before destination position. -loop: - dstPos += copy(dd.hist[dstPos:endPos], dd.hist[srcPos:dstPos]) - if dstPos < endPos { - goto loop // Avoid for-loop so that this function can be inlined - } - - dd.wrPos = dstPos - return dstPos - dstBase -} - -// readFlush returns a slice of the historical buffer that is ready to be -// emitted to the user. The data returned by readFlush must be fully consumed -// before calling any other dictDecoder methods. -func (dd *dictDecoder) readFlush() []byte { - toRead := dd.hist[dd.rdPos:dd.wrPos] - dd.rdPos = dd.wrPos - if dd.wrPos == len(dd.hist) { - dd.wrPos, dd.rdPos = 0, 0 - dd.full = true - } - return toRead -} diff --git a/internal/compress/flate/dict_decoder_test.go b/internal/compress/flate/dict_decoder_test.go deleted file mode 100644 index 8bc48a3e..00000000 --- a/internal/compress/flate/dict_decoder_test.go +++ /dev/null @@ -1,284 +0,0 @@ -// Copyright 2016 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package flate - -import ( - "bytes" - "strings" - "testing" -) - -func TestDictDecoder(t *testing.T) { - const ( - abc = "ABC\n" - fox = "The quick brown fox jumped over the lazy dog!\n" - poem = "The Road Not Taken\nRobert Frost\n" + - "\n" + - "Two roads diverged in a yellow wood,\n" + - "And sorry I could not travel both\n" + - "And be one traveler, long I stood\n" + - "And looked down one as far as I could\n" + - "To where it bent in the undergrowth;\n" + - "\n" + - "Then took the other, as just as fair,\n" + - "And having perhaps the better claim,\n" + - "Because it was grassy and wanted wear;\n" + - "Though as for that the passing there\n" + - "Had worn them really about the same,\n" + - "\n" + - "And both that morning equally lay\n" + - "In leaves no step had trodden black.\n" + - "Oh, I kept the first for another day!\n" + - "Yet knowing how way leads on to way,\n" + - "I doubted if I should ever come back.\n" + - "\n" + - "I shall be telling this with a sigh\n" + - "Somewhere ages and ages hence:\n" + - "Two roads diverged in a wood, and I-\n" + - "I took the one less traveled by,\n" + - "And that has made all the difference.\n" - ) - - poemRefs := []struct { - dist int // Backward distance (0 if this is an insertion) - length int // Length of copy or insertion - }{ - {0, 38}, - {33, 3}, - {0, 48}, - {79, 3}, - {0, 11}, - {34, 5}, - {0, 6}, - {23, 7}, - {0, 8}, - {50, 3}, - {0, 2}, - {69, 3}, - {34, 5}, - {0, 4}, - {97, 3}, - {0, 4}, - {43, 5}, - {0, 6}, - {7, 4}, - {88, 7}, - {0, 12}, - {80, 3}, - {0, 2}, - {141, 4}, - {0, 1}, - {196, 3}, - {0, 3}, - {157, 3}, - {0, 6}, - {181, 3}, - {0, 2}, - {23, 3}, - {77, 3}, - {28, 5}, - {128, 3}, - {110, 4}, - {70, 3}, - {0, 4}, - {85, 6}, - {0, 2}, - {182, 6}, - {0, 4}, - {133, 3}, - {0, 7}, - {47, 5}, - {0, 20}, - {112, 5}, - {0, 1}, - {58, 3}, - {0, 8}, - {59, 3}, - {0, 4}, - {173, 3}, - {0, 5}, - {114, 3}, - {0, 4}, - {92, 5}, - {0, 2}, - {71, 3}, - {0, 2}, - {76, 5}, - {0, 1}, - {46, 3}, - {96, 4}, - {130, 4}, - {0, 3}, - {360, 3}, - {0, 3}, - {178, 5}, - {0, 7}, - {75, 3}, - {0, 3}, - {45, 6}, - {0, 6}, - {299, 6}, - {180, 3}, - {70, 6}, - {0, 1}, - {48, 3}, - {66, 4}, - {0, 3}, - {47, 5}, - {0, 9}, - {325, 3}, - {0, 1}, - {359, 3}, - {318, 3}, - {0, 2}, - {199, 3}, - {0, 1}, - {344, 3}, - {0, 3}, - {248, 3}, - {0, 10}, - {310, 3}, - {0, 3}, - {93, 6}, - {0, 3}, - {252, 3}, - {157, 4}, - {0, 2}, - {273, 5}, - {0, 14}, - {99, 4}, - {0, 1}, - {464, 4}, - {0, 2}, - {92, 4}, - {495, 3}, - {0, 1}, - {322, 4}, - {16, 4}, - {0, 3}, - {402, 3}, - {0, 2}, - {237, 4}, - {0, 2}, - {432, 4}, - {0, 1}, - {483, 5}, - {0, 2}, - {294, 4}, - {0, 2}, - {306, 3}, - {113, 5}, - {0, 1}, - {26, 4}, - {164, 3}, - {488, 4}, - {0, 1}, - {542, 3}, - {248, 6}, - {0, 5}, - {205, 3}, - {0, 8}, - {48, 3}, - {449, 6}, - {0, 2}, - {192, 3}, - {328, 4}, - {9, 5}, - {433, 3}, - {0, 3}, - {622, 25}, - {615, 5}, - {46, 5}, - {0, 2}, - {104, 3}, - {475, 10}, - {549, 3}, - {0, 4}, - {597, 8}, - {314, 3}, - {0, 1}, - {473, 6}, - {317, 5}, - {0, 1}, - {400, 3}, - {0, 3}, - {109, 3}, - {151, 3}, - {48, 4}, - {0, 4}, - {125, 3}, - {108, 3}, - {0, 2}, - } - - var got, want bytes.Buffer - var dd dictDecoder - dd.init(1<<11, nil) - - writeCopy := func(dist, length int) { - for length > 0 { - cnt := dd.tryWriteCopy(dist, length) - if cnt == 0 { - cnt = dd.writeCopy(dist, length) - } - - length -= cnt - if dd.availWrite() == 0 { - got.Write(dd.readFlush()) - } - } - } - writeString := func(str string) { - for len(str) > 0 { - cnt := copy(dd.writeSlice(), str) - str = str[cnt:] - dd.writeMark(cnt) - if dd.availWrite() == 0 { - got.Write(dd.readFlush()) - } - } - } - - writeString(".") - want.WriteByte('.') - - str := poem - for _, ref := range poemRefs { - if ref.dist == 0 { - writeString(str[:ref.length]) - } else { - writeCopy(ref.dist, ref.length) - } - str = str[ref.length:] - } - want.WriteString(poem) - - writeCopy(dd.histSize(), 33) - want.Write(want.Bytes()[:33]) - - writeString(abc) - writeCopy(len(abc), 59*len(abc)) - want.WriteString(strings.Repeat(abc, 60)) - - writeString(fox) - writeCopy(len(fox), 9*len(fox)) - want.WriteString(strings.Repeat(fox, 10)) - - writeString(".") - writeCopy(1, 9) - want.WriteString(strings.Repeat(".", 10)) - - writeString(strings.ToUpper(poem)) - writeCopy(len(poem), 7*len(poem)) - want.WriteString(strings.Repeat(strings.ToUpper(poem), 8)) - - writeCopy(dd.histSize(), 10) - want.Write(want.Bytes()[want.Len()-dd.histSize():][:10]) - - got.Write(dd.readFlush()) - if got.String() != want.String() { - t.Errorf("final string mismatch:\ngot %q\nwant %q", got.String(), want.String()) - } -} diff --git a/internal/compress/flate/example_test.go b/internal/compress/flate/example_test.go deleted file mode 100644 index 45f44edf..00000000 --- a/internal/compress/flate/example_test.go +++ /dev/null @@ -1,240 +0,0 @@ -// Copyright 2016 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package flate_test - -import ( - "bytes" - "fmt" - "io" - "log" - "os" - "strings" - "sync" - - "lindenii.org/go/furgit/internal/compress/flate" -) - -// In performance critical applications, Reset can be used to discard the -// current compressor or decompressor state and reinitialize them quickly -// by taking advantage of previously allocated memory. -func Example_reset() { - proverbs := []string{ - "Don't communicate by sharing memory, share memory by communicating.\n", - "Concurrency is not parallelism.\n", - "The bigger the interface, the weaker the abstraction.\n", - "Documentation is for users.\n", - } - - var r strings.Reader - var b bytes.Buffer - buf := make([]byte, 32<<10) - - zw, err := flate.NewWriter(nil, flate.DefaultCompression) - if err != nil { - log.Fatal(err) - } - zr := flate.NewReader(nil) - - for _, s := range proverbs { - r.Reset(s) - b.Reset() - - // Reset the compressor and encode from some input stream. - zw.Reset(&b) - if _, err := io.CopyBuffer(zw, &r, buf); err != nil { - log.Fatal(err) - } - if err := zw.Close(); err != nil { - log.Fatal(err) - } - - // Reset the decompressor and decode to some output stream. - if err := zr.(flate.Resetter).Reset(&b, nil); err != nil { - log.Fatal(err) - } - if _, err := io.CopyBuffer(os.Stdout, zr, buf); err != nil { - log.Fatal(err) - } - if err := zr.Close(); err != nil { - log.Fatal(err) - } - } - - // Output: - // Don't communicate by sharing memory, share memory by communicating. - // Concurrency is not parallelism. - // The bigger the interface, the weaker the abstraction. - // Documentation is for users. -} - -// A preset dictionary can be used to improve the compression ratio. -// The downside to using a dictionary is that the compressor and decompressor -// must agree in advance what dictionary to use. -func Example_dictionary() { - // The dictionary is a string of bytes. When compressing some input data, - // the compressor will attempt to substitute substrings with matches found - // in the dictionary. As such, the dictionary should only contain substrings - // that are expected to be found in the actual data stream. - const dict = `<?xml version="1.0"?>` + `<book>` + `<data>` + `<meta name="` + `" content="` - - // The data to compress should (but is not required to) contain frequent - // substrings that match those in the dictionary. - const data = `<?xml version="1.0"?> -<book> - <meta name="title" content="The Go Programming Language"/> - <meta name="authors" content="Alan Donovan and Brian Kernighan"/> - <meta name="published" content="2015-10-26"/> - <meta name="isbn" content="978-0134190440"/> - <data>...</data> -</book> -` - - var b bytes.Buffer - - // Compress the data using the specially crafted dictionary. - zw, err := flate.NewWriterDict(&b, flate.BestCompression, []byte(dict)) - if err != nil { - log.Fatal(err) - } - if _, err := io.Copy(zw, strings.NewReader(data)); err != nil { - log.Fatal(err) - } - if err := zw.Close(); err != nil { - log.Fatal(err) - } - - // The decompressor must use the same dictionary as the compressor. - // Otherwise, the input may appear as corrupted. - fmt.Println("Decompressed output using the dictionary:") - zr := flate.NewReaderDict(bytes.NewReader(b.Bytes()), []byte(dict)) - if _, err := io.Copy(os.Stdout, zr); err != nil { - log.Fatal(err) - } - if err := zr.Close(); err != nil { - log.Fatal(err) - } - - fmt.Println() - - // Substitute all of the bytes in the dictionary with a '#' to visually - // demonstrate the approximate effectiveness of using a preset dictionary. - fmt.Println("Substrings matched by the dictionary are marked with #:") - hashDict := []byte(dict) - for i := range hashDict { - hashDict[i] = '#' - } - zr = flate.NewReaderDict(&b, hashDict) - if _, err := io.Copy(os.Stdout, zr); err != nil { - log.Fatal(err) - } - if err := zr.Close(); err != nil { - log.Fatal(err) - } - - // Output: - // Decompressed output using the dictionary: - // <?xml version="1.0"?> - // <book> - // <meta name="title" content="The Go Programming Language"/> - // <meta name="authors" content="Alan Donovan and Brian Kernighan"/> - // <meta name="published" content="2015-10-26"/> - // <meta name="isbn" content="978-0134190440"/> - // <data>...</data> - // </book> - // - // Substrings matched by the dictionary are marked with #: - // ##################### - // ###### - // ############title###########The Go Programming Language"/# - // ############authors###########Alan Donovan and Brian Kernighan"/# - // ############published###########2015-10-26"/# - // ############isbn###########978-0134190440"/# - // ######...</##### - // </##### -} - -// DEFLATE is suitable for transmitting compressed data across the network. -func Example_synchronization() { - var wg sync.WaitGroup - defer wg.Wait() - - // Use io.Pipe to simulate a network connection. - // A real network application should take care to properly close the - // underlying connection. - rp, wp := io.Pipe() - - // Start a goroutine to act as the transmitter. - wg.Go(func() { - defer wp.Close() - - zw, err := flate.NewWriter(wp, flate.BestSpeed) - if err != nil { - log.Fatal(err) - } - - b := make([]byte, 256) - for m := range strings.FieldsSeq("A long time ago in a galaxy far, far away...") { - // We use a simple framing format where the first byte is the - // message length, followed the message itself. - b[0] = uint8(copy(b[1:], m)) - - if _, err := zw.Write(b[:1+len(m)]); err != nil { - log.Fatal(err) - } - - // Flush ensures that the receiver can read all data sent so far. - if err := zw.Flush(); err != nil { - log.Fatal(err) - } - } - - if err := zw.Close(); err != nil { - log.Fatal(err) - } - }) - - // Start a goroutine to act as the receiver. - wg.Go(func() { - zr := flate.NewReader(rp) - - b := make([]byte, 256) - for { - // Read the message length. - // This is guaranteed to return for every corresponding - // Flush and Close on the transmitter side. - if _, err := io.ReadFull(zr, b[:1]); err != nil { - if err == io.EOF { - break // The transmitter closed the stream - } - log.Fatal(err) - } - - // Read the message content. - n := int(b[0]) - if _, err := io.ReadFull(zr, b[:n]); err != nil { - log.Fatal(err) - } - - fmt.Printf("Received %d bytes: %s\n", n, b[:n]) - } - fmt.Println() - - if err := zr.Close(); err != nil { - log.Fatal(err) - } - }) - - // Output: - // Received 1 bytes: A - // Received 4 bytes: long - // Received 4 bytes: time - // Received 3 bytes: ago - // Received 2 bytes: in - // Received 1 bytes: a - // Received 6 bytes: galaxy - // Received 4 bytes: far, - // Received 3 bytes: far - // Received 7 bytes: away... -} diff --git a/internal/compress/flate/fast_encoder.go b/internal/compress/flate/fast_encoder.go deleted file mode 100644 index 7af14349..00000000 --- a/internal/compress/flate/fast_encoder.go +++ /dev/null @@ -1,189 +0,0 @@ -// Copyright 2011 The Snappy-Go Authors. All rights reserved. -// Modified for deflate by Klaus Post (c) 2015. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package flate - -import ( - "fmt" - - "lindenii.org/go/furgit/internal/compress/internal/le" -) - -type fastEnc interface { - Encode(dst *tokens, src []byte) - Reset() -} - -func newFastEnc(level int) fastEnc { - switch level { - case 1: - return &fastEncL1{fastGen: fastGen{cur: maxStoreBlockSize}} - case 2: - return &fastEncL2{fastGen: fastGen{cur: maxStoreBlockSize}} - case 3: - return &fastEncL3{fastGen: fastGen{cur: maxStoreBlockSize}} - case 4: - return &fastEncL4{fastGen: fastGen{cur: maxStoreBlockSize}} - case 5: - return &fastEncL5{fastGen: fastGen{cur: maxStoreBlockSize}} - case 6: - return &fastEncL6{fastGen: fastGen{cur: maxStoreBlockSize}} - default: - panic("invalid level specified") - } -} - -const ( - tableBits = 15 // Bits used in the table - tableSize = 1 << tableBits // Size of the table - tableShift = 32 - tableBits // Right-shift to get the tableBits most significant bits of a uint32. - baseMatchOffset = 1 // The smallest match offset - baseMatchLength = 3 // The smallest match length per the RFC section 3.2.5 - maxMatchOffset = 1 << 15 // The largest match offset - - bTableBits = 17 // Bits used in the big tables - bTableSize = 1 << bTableBits // Size of the table - allocHistory = maxStoreBlockSize * 5 // Size to preallocate for history. - bufferReset = (1 << 31) - allocHistory - maxStoreBlockSize - 1 // Reset the buffer offset when reaching this. -) - -const ( - prime3bytes = 506832829 - prime4bytes = 2654435761 - prime5bytes = 889523592379 - prime6bytes = 227718039650203 - prime7bytes = 58295818150454627 - prime8bytes = 0xcf1bbcdcb7a56463 -) - -func load3232(b []byte, i int32) uint32 { - return le.Load32(b, i) -} - -func load6432(b []byte, i int32) uint64 { - return le.Load64(b, i) -} - -type tableEntry struct { - offset int32 -} - -// fastGen maintains the table for matches, -// and the previous byte block for level 2. -// This is the generic implementation. -type fastGen struct { - hist []byte - cur int32 -} - -func (e *fastGen) addBlock(src []byte) int32 { - // check if we have space already - if len(e.hist)+len(src) > cap(e.hist) { - if cap(e.hist) == 0 { - e.hist = make([]byte, 0, allocHistory) - } else { - if cap(e.hist) < maxMatchOffset*2 { - panic("unexpected buffer size") - } - // Move down - offset := int32(len(e.hist)) - maxMatchOffset - // copy(e.hist[0:maxMatchOffset], e.hist[offset:]) - *(*[maxMatchOffset]byte)(e.hist) = *(*[maxMatchOffset]byte)(e.hist[offset:]) - e.cur += offset - e.hist = e.hist[:maxMatchOffset] - } - } - s := int32(len(e.hist)) - e.hist = append(e.hist, src...) - return s -} - -type tableEntryPrev struct { - Cur tableEntry - Prev tableEntry -} - -// hash7 returns the hash of the lowest 7 bytes of u to fit in a hash table with h bits. -// Preferably h should be a constant and should always be <64. -func hash7(u uint64, h uint8) uint32 { - return uint32(((u << (64 - 56)) * prime7bytes) >> ((64 - h) & reg8SizeMask64)) -} - -// hashLen returns a hash of the lowest mls bytes of with length output bits. -// mls must be >=3 and <=8. Any other value will return hash for 4 bytes. -// length should always be < 32. -// Preferably length and mls should be a constant for inlining. -func hashLen(u uint64, length, mls uint8) uint32 { - switch mls { - case 3: - return (uint32(u<<8) * prime3bytes) >> (32 - length) - case 5: - return uint32(((u << (64 - 40)) * prime5bytes) >> (64 - length)) - case 6: - return uint32(((u << (64 - 48)) * prime6bytes) >> (64 - length)) - case 7: - return uint32(((u << (64 - 56)) * prime7bytes) >> (64 - length)) - case 8: - return uint32((u * prime8bytes) >> (64 - length)) - default: - return (uint32(u) * prime4bytes) >> (32 - length) - } -} - -// matchlen will return the match length between offsets and t in src. -// The maximum length returned is maxMatchLength - 4. -// It is assumed that s > t, that t >=0 and s < len(src). -func (e *fastGen) matchlen(s, t int, src []byte) int32 { - if debugDeflate { - if t >= s { - panic(fmt.Sprint("t >=s:", t, s)) - } - if int(s) >= len(src) { - panic(fmt.Sprint("s >= len(src):", s, len(src))) - } - if t < 0 { - panic(fmt.Sprint("t < 0:", t)) - } - if s-t > maxMatchOffset { - panic(fmt.Sprint(s, "-", t, "(", s-t, ") > maxMatchLength (", maxMatchOffset, ")")) - } - } - a := src[s:min(s+maxMatchLength-4, len(src))] - b := src[t:] - return int32(matchLen(a, b)) -} - -// matchlenLong will return the match length between offsets and t in src. -// It is assumed that s > t, that t >=0 and s < len(src). -func (e *fastGen) matchlenLong(s, t int, src []byte) int32 { - if debugDeflate { - if t >= s { - panic(fmt.Sprint("t >=s:", t, s)) - } - if int(s) >= len(src) { - panic(fmt.Sprint("s >= len(src):", s, len(src))) - } - if t < 0 { - panic(fmt.Sprint("t < 0:", t)) - } - if s-t > maxMatchOffset { - panic(fmt.Sprint(s, "-", t, "(", s-t, ") > maxMatchLength (", maxMatchOffset, ")")) - } - } - return int32(matchLen(src[s:], src[t:])) -} - -// Reset the encoding table. -func (e *fastGen) Reset() { - if cap(e.hist) < allocHistory { - e.hist = make([]byte, 0, allocHistory) - } - // We offset current position so everything will be out of reach. - // If we are above the buffer reset it will be cleared anyway since len(hist) == 0. - if e.cur <= bufferReset { - e.cur += maxMatchOffset + int32(len(e.hist)) - } - e.hist = e.hist[:0] -} diff --git a/internal/compress/flate/flate_test.go b/internal/compress/flate/flate_test.go deleted file mode 100644 index 7b019548..00000000 --- a/internal/compress/flate/flate_test.go +++ /dev/null @@ -1,370 +0,0 @@ -// Copyright 2009 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -// This test tests some internals of the flate package. -// The tests in package compress/gzip serve as the -// end-to-end test of the decompressor. - -package flate - -import ( - "archive/zip" - "bytes" - "compress/flate" - "encoding/hex" - "fmt" - "io" - "os" - "testing" -) - -// The following test should not panic. -func TestIssue5915(t *testing.T) { - bits := []int{ - 4, 0, 0, 6, 4, 3, 2, 3, 3, 4, 4, 5, 0, 0, 0, 0, 5, 5, 6, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 8, 6, 0, 11, 0, 8, 0, 6, 6, 10, 8, - } - var h huffmanDecoder - if h.init(bits) { - t.Fatalf("Given sequence of bits is bad, and should not succeed.") - } -} - -// The following test should not panic. -func TestIssue5962(t *testing.T) { - bits := []int{ - 4, 0, 0, 6, 4, 3, 2, 3, 3, 4, 4, 5, 0, 0, 0, 0, - 5, 5, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, - } - var h huffmanDecoder - if h.init(bits) { - t.Fatalf("Given sequence of bits is bad, and should not succeed.") - } -} - -// The following test should not panic. -func TestIssue6255(t *testing.T) { - bits1 := []int{1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 11} - bits2 := []int{11, 13} - var h huffmanDecoder - if !h.init(bits1) { - t.Fatalf("Given sequence of bits is good and should succeed.") - } - if h.init(bits2) { - t.Fatalf("Given sequence of bits is bad and should not succeed.") - } -} - -func TestInvalidEncoding(t *testing.T) { - // Initialize Huffman decoder to recognize "0". - var h huffmanDecoder - if !h.init([]int{1}) { - t.Fatal("Failed to initialize Huffman decoder") - } - - // Initialize decompressor with invalid Huffman coding. - var f decompressor - f.r = bytes.NewReader([]byte{0xff}) - - _, err := f.huffSym(&h) - if err == nil { - t.Fatal("Should have rejected invalid bit sequence") - } -} - -func TestRegressions(t *testing.T) { - // Test fuzzer regressions - data, err := os.ReadFile("testdata/regression.zip") - if err != nil { - t.Fatal(err) - } - zr, err := zip.NewReader(bytes.NewReader(data), int64(len(data))) - if err != nil { - t.Fatal(err) - } - for _, tt := range zr.File { - data, err := tt.Open() - if err != nil { - t.Fatal(err) - } - data1, err := io.ReadAll(data) - if err != nil { - t.Fatal(err) - } - t.Run(tt.Name, func(t *testing.T) { - if testing.Short() && len(data1) > 10000 { - t.SkipNow() - } - for level := 0; level <= 9; level++ { - t.Run(fmt.Sprint(tt.Name+"-level", 1), func(t *testing.T) { - buf := new(bytes.Buffer) - fw, err := NewWriter(buf, level) - if err != nil { - t.Error(err) - } - n, err := fw.Write(data1) - if n != len(data1) { - t.Error("short write") - } - if err != nil { - t.Error(err) - } - err = fw.Close() - if err != nil { - t.Error(err) - } - fr1 := NewReader(buf) - data2, err := io.ReadAll(fr1) - if err != nil { - t.Error(err) - } - if !bytes.Equal(data1, data2) { - t.Error("not equal") - } - // Do it again... - buf.Reset() - fw.Reset(buf) - n, err = fw.Write(data1) - if n != len(data1) { - t.Error("short write") - } - if err != nil { - t.Error(err) - } - err = fw.Close() - if err != nil { - t.Error(err) - } - fr1 = flate.NewReader(buf) - data2, err = io.ReadAll(fr1) - if err != nil { - t.Error(err) - } - if !bytes.Equal(data1, data2) { - t.Error("not equal") - } - }) - } - t.Run(tt.Name+"stateless", func(t *testing.T) { - // Split into two and use history... - buf := new(bytes.Buffer) - err = StatelessDeflate(buf, data1[:len(data1)/2], false, nil) - if err != nil { - t.Error(err) - } - - // Use top half as dictionary... - dict := data1[:len(data1)/2] - err = StatelessDeflate(buf, data1[len(data1)/2:], true, dict) - if err != nil { - t.Error(err) - } - t.Log(buf.Len()) - fr1 := NewReader(buf) - data2, err := io.ReadAll(fr1) - if err != nil { - t.Error(err) - } - if !bytes.Equal(data1, data2) { - // fmt.Printf("want:%x\ngot: %x\n", data1, data2) - t.Error("not equal") - } - }) - }) - } -} - -func TestInvalidBits(t *testing.T) { - oversubscribed := []int{1, 2, 3, 4, 4, 5} - incomplete := []int{1, 2, 4, 4} - var h huffmanDecoder - if h.init(oversubscribed) { - t.Fatal("Should reject oversubscribed bit-length set") - } - if h.init(incomplete) { - t.Fatal("Should reject incomplete bit-length set") - } -} - -func TestStreams(t *testing.T) { - // To verify any of these hexstrings as valid or invalid flate streams - // according to the C zlib library, you can use the Python wrapper library: - // >>> hex_string = "010100feff11" - // >>> import zlib - // >>> zlib.decompress(hex_string.decode("hex"), -15) # Negative means raw DEFLATE - // '\x11' - - testCases := []struct { - desc string // Description of the stream - stream string // Hexstring of the input DEFLATE stream - want string // Expected result. Use "fail" to expect failure - }{{ - "degenerate HCLenTree", - "05e0010000000000100000000000000000000000000000000000000000000000" + - "00000000000000000004", - "fail", - }, { - "complete HCLenTree, empty HLitTree, empty HDistTree", - "05e0010400000000000000000000000000000000000000000000000000000000" + - "00000000000000000010", - "fail", - }, { - "empty HCLenTree", - "05e0010000000000000000000000000000000000000000000000000000000000" + - "00000000000000000010", - "fail", - }, { - "complete HCLenTree, complete HLitTree, empty HDistTree, use missing HDist symbol", - "000100feff000de0010400000000100000000000000000000000000000000000" + - "0000000000000000000000000000002c", - "fail", - }, { - "complete HCLenTree, complete HLitTree, degenerate HDistTree, use missing HDist symbol", - "000100feff000de0010000000000000000000000000000000000000000000000" + - "00000000000000000610000000004070", - "fail", - }, { - "complete HCLenTree, empty HLitTree, empty HDistTree", - "05e0010400000000100400000000000000000000000000000000000000000000" + - "0000000000000000000000000008", - "fail", - }, { - "complete HCLenTree, empty HLitTree, degenerate HDistTree", - "05e0010400000000100400000000000000000000000000000000000000000000" + - "0000000000000000000800000008", - "fail", - }, { - "complete HCLenTree, degenerate HLitTree, degenerate HDistTree, use missing HLit symbol", - "05e0010400000000100000000000000000000000000000000000000000000000" + - "0000000000000000001c", - "fail", - }, { - "complete HCLenTree, complete HLitTree, too large HDistTree", - "edff870500000000200400000000000000000000000000000000000000000000" + - "000000000000000000080000000000000004", - "fail", - }, { - "complete HCLenTree, complete HLitTree, empty HDistTree, excessive repeater code", - "edfd870500000000200400000000000000000000000000000000000000000000" + - "000000000000000000e8b100", - "fail", - }, { - "complete HCLenTree, complete HLitTree, empty HDistTree of normal length 30", - "05fd01240000000000f8ffffffffffffffffffffffffffffffffffffffffffff" + - "ffffffffffffffffff07000000fe01", - "", - }, { - "complete HCLenTree, complete HLitTree, empty HDistTree of excessive length 31", - "05fe01240000000000f8ffffffffffffffffffffffffffffffffffffffffffff" + - "ffffffffffffffffff07000000fc03", - "fail", - }, { - "complete HCLenTree, over-subscribed HLitTree, empty HDistTree", - "05e001240000000000fcffffffffffffffffffffffffffffffffffffffffffff" + - "ffffffffffffffffff07f00f", - "fail", - }, { - "complete HCLenTree, under-subscribed HLitTree, empty HDistTree", - "05e001240000000000fcffffffffffffffffffffffffffffffffffffffffffff" + - "fffffffffcffffffff07f00f", - "fail", - }, { - "complete HCLenTree, complete HLitTree with single code, empty HDistTree", - "05e001240000000000f8ffffffffffffffffffffffffffffffffffffffffffff" + - "ffffffffffffffffff07f00f", - "01", - }, { - "complete HCLenTree, complete HLitTree with multiple codes, empty HDistTree", - "05e301240000000000f8ffffffffffffffffffffffffffffffffffffffffffff" + - "ffffffffffffffffff07807f", - "01", - }, { - "complete HCLenTree, complete HLitTree, degenerate HDistTree, use valid HDist symbol", - "000100feff000de0010400000000100000000000000000000000000000000000" + - "0000000000000000000000000000003c", - "00000000", - }, { - "complete HCLenTree, degenerate HLitTree, degenerate HDistTree", - "05e0010400000000100000000000000000000000000000000000000000000000" + - "0000000000000000000c", - "", - }, { - "complete HCLenTree, degenerate HLitTree, empty HDistTree", - "05e0010400000000100000000000000000000000000000000000000000000000" + - "00000000000000000004", - "", - }, { - "complete HCLenTree, complete HLitTree, empty HDistTree, spanning repeater code", - "edfd870500000000200400000000000000000000000000000000000000000000" + - "000000000000000000e8b000", - "", - }, { - "complete HCLenTree with length codes, complete HLitTree, empty HDistTree", - "ede0010400000000100000000000000000000000000000000000000000000000" + - "0000000000000000000400004000", - "", - }, { - "complete HCLenTree, complete HLitTree, degenerate HDistTree, use valid HLit symbol 284 with count 31", - "000100feff00ede0010400000000100000000000000000000000000000000000" + - "000000000000000000000000000000040000407f00", - "0000000000000000000000000000000000000000000000000000000000000000" + - "0000000000000000000000000000000000000000000000000000000000000000" + - "0000000000000000000000000000000000000000000000000000000000000000" + - "0000000000000000000000000000000000000000000000000000000000000000" + - "0000000000000000000000000000000000000000000000000000000000000000" + - "0000000000000000000000000000000000000000000000000000000000000000" + - "0000000000000000000000000000000000000000000000000000000000000000" + - "0000000000000000000000000000000000000000000000000000000000000000" + - "000000", - }, { - "complete HCLenTree, complete HLitTree, degenerate HDistTree, use valid HLit and HDist symbols", - "0cc2010d00000082b0ac4aff0eb07d27060000ffff", - "616263616263", - }, { - "fixed block, use reserved symbol 287", - "33180700", - "fail", - }, { - "raw block", - "010100feff11", - "11", - }, { - "issue 10426 - over-subscribed HCLenTree causes a hang", - "344c4a4e494d4b070000ff2e2eff2e2e2e2e2eff", - "fail", - }, { - "issue 11030 - empty HDistTree unexpectedly leads to error", - "05c0070600000080400fff37a0ca", - "", - }, { - "issue 11033 - empty HDistTree unexpectedly leads to error", - "050fb109c020cca5d017dcbca044881ee1034ec149c8980bbc413c2ab35be9dc" + - "b1473449922449922411202306ee97b0383a521b4ffdcf3217f9f7d3adb701", - "3130303634342068652e706870005d05355f7ed957ff084a90925d19e3ebc6d0" + - "c6d7", - }} - - for i, tc := range testCases { - data, err := hex.DecodeString(tc.stream) - if err != nil { - t.Fatal(err) - } - data, err = io.ReadAll(NewReader(bytes.NewReader(data))) - if tc.want == "fail" { - if err == nil { - t.Errorf("#%d (%s): got nil error, want non-nil", i, tc.desc) - } - } else { - if err != nil { - t.Errorf("#%d (%s): %v", i, tc.desc, err) - continue - } - if got := hex.EncodeToString(data); got != tc.want { - t.Errorf("#%d (%s):\ngot %q\nwant %q", i, tc.desc, got, tc.want) - } - - } - } -} diff --git a/internal/compress/flate/fuzz_test.go b/internal/compress/flate/fuzz_test.go deleted file mode 100644 index b3d0098f..00000000 --- a/internal/compress/flate/fuzz_test.go +++ /dev/null @@ -1,176 +0,0 @@ -//go:build go1.18 - -package flate - -import ( - "bytes" - "flag" - "io" - "os" - "strconv" - "testing" - - "lindenii.org/go/furgit/internal/compress/internal/fuzz" -) - -// Fuzzing tweaks: -var ( - fuzzStartF = flag.Int("start", HuffmanOnly, "Start fuzzing at this level") - fuzzEndF = flag.Int("end", BestCompression, "End fuzzing at this level (inclusive)") - fuzzMaxF = flag.Int("max", 1<<20, "Maximum input size") - fuzzSLF = flag.Bool("sl", true, "Include stateless encodes") - fuzzWindow = flag.Bool("windows", true, "Include windowed encodes") -) - -func TestMain(m *testing.M) { - flag.Parse() - os.Exit(m.Run()) -} - -func FuzzEncoding(f *testing.F) { - fuzz.AddFromZip(f, "testdata/regression.zip", fuzz.TypeRaw, false) - fuzz.AddFromZip(f, "testdata/fuzz/encode-raw-corpus.zip", fuzz.TypeRaw, testing.Short()) - fuzz.AddFromZip(f, "testdata/fuzz/FuzzEncoding.zip", fuzz.TypeGoFuzz, testing.Short()) - - startFuzz := *fuzzStartF - endFuzz := *fuzzEndF - maxSize := *fuzzMaxF - stateless := *fuzzSLF - fuzzWindow := *fuzzWindow - - decoder := NewReader(nil) - buf := new(bytes.Buffer) - encs := make([]*Writer, endFuzz-startFuzz+1) - for i := range encs { - var err error - encs[i], err = NewWriter(nil, i+startFuzz) - if err != nil { - f.Fatal(err.Error()) - } - } - - f.Fuzz(func(t *testing.T, data []byte) { - if len(data) > maxSize { - return - } - for level := startFuzz; level <= endFuzz; level++ { - msg := "level " + strconv.Itoa(level) + ":" - buf.Reset() - fw := encs[level-startFuzz] - fw.Reset(buf) - n, err := fw.Write(data) - if n != len(data) { - t.Fatal(msg + "short write") - } - if err != nil { - t.Fatal(msg + err.Error()) - } - err = fw.Close() - if err != nil { - t.Fatal(msg + err.Error()) - } - decoder.(Resetter).Reset(buf, nil) - data2, err := io.ReadAll(decoder) - if err != nil { - t.Fatal(msg + err.Error()) - } - if !bytes.Equal(data, data2) { - t.Fatal(msg + "not equal") - } - // Do it again... (also uses copy) - msg = "level " + strconv.Itoa(level) + " (reset):" - buf.Reset() - fw.Reset(buf) - _, err = io.Copy(fw, bytes.NewReader(data)) - if err != nil { - t.Fatal(msg + err.Error()) - } - err = fw.Close() - if err != nil { - t.Fatal(msg + err.Error()) - } - decoder.(Resetter).Reset(buf, nil) - data2, err = io.ReadAll(decoder) - if err != nil { - t.Fatal(msg + err.Error()) - } - if !bytes.Equal(data, data2) { - t.Fatal(msg + "not equal") - } - } - if stateless { - // Split into two and use history... - msg := "stateless:" - buf.Reset() - err := StatelessDeflate(buf, data[:len(data)/2], false, nil) - if err != nil { - t.Error(err) - } - - // Use top half as dictionary... - dict := data[:len(data)/2] - err = StatelessDeflate(buf, data[len(data)/2:], true, dict) - if err != nil { - t.Error(err) - } - - decoder.(Resetter).Reset(buf, nil) - data2, err := io.ReadAll(decoder) - if err != nil { - t.Error(err) - } - if !bytes.Equal(data, data2) { - // fmt.Printf("want:%x\ngot: %x\n", data1, data2) - t.Error(msg + "not equal") - } - } - if fuzzWindow { - msg := "windowed:" - buf.Reset() - fw, err := NewWriterWindow(buf, 1000) - if err != nil { - t.Fatal(msg + err.Error()) - } - fw.Reset(buf) - n, err := fw.Write(data) - if n != len(data) { - t.Fatal(msg + "short write") - } - if err != nil { - t.Fatal(msg + err.Error()) - } - err = fw.Close() - if err != nil { - t.Fatal(msg + err.Error()) - } - decoder.(Resetter).Reset(buf, nil) - data2, err := io.ReadAll(decoder) - if err != nil { - t.Fatal(msg + err.Error()) - } - if !bytes.Equal(data, data2) { - t.Fatal(msg + "not equal") - } - // Do it again... - msg = msg + " (reset):" - buf.Reset() - fw.Reset(buf) - n, err = fw.Write(data) - if n != len(data) { - t.Fatal(msg + "short write") - } - if err != nil { - t.Fatal(msg + err.Error()) - } - err = fw.Close() - if err != nil { - t.Fatal(msg + err.Error()) - } - decoder.(Resetter).Reset(buf, nil) - data2, err = io.ReadAll(decoder) - if err != nil { - t.Fatal(msg + err.Error()) - } - } - }) -} diff --git a/internal/compress/flate/huffman_bit_writer.go b/internal/compress/flate/huffman_bit_writer.go deleted file mode 100644 index 18dff811..00000000 --- a/internal/compress/flate/huffman_bit_writer.go +++ /dev/null @@ -1,1174 +0,0 @@ -// Copyright 2009 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package flate - -import ( - "fmt" - "io" - "math" - - "lindenii.org/go/furgit/internal/compress/internal/le" -) - -const ( - // The largest offset code. - offsetCodeCount = 30 - - // The special code used to mark the end of a block. - endBlockMarker = 256 - - // The first length code. - lengthCodesStart = 257 - - // The number of codegen codes. - codegenCodeCount = 19 - badCode = 255 - - // maxPredefinedTokens is the maximum number of tokens - // where we check if fixed size is smaller. - maxPredefinedTokens = 250 - - // bufferFlushSize indicates the buffer size - // after which bytes are flushed to the writer. - // Should preferably be a multiple of 6, since - // we accumulate 6 bytes between writes to the buffer. - bufferFlushSize = 246 -) - -// Minimum length code that emits bits. -const lengthExtraBitsMinCode = 8 - -// The number of extra bits needed by length code X - LENGTH_CODES_START. -var lengthExtraBits = [32]uint8{ - /* 257 */ 0, 0, 0, - /* 260 */ 0, 0, 0, 0, 0, 1, 1, 1, 1, 2, - /* 270 */ 2, 2, 2, 3, 3, 3, 3, 4, 4, 4, - /* 280 */ 4, 5, 5, 5, 5, 0, -} - -// The length indicated by length code X - LENGTH_CODES_START. -var lengthBase = [32]uint8{ - 0, 1, 2, 3, 4, 5, 6, 7, 8, 10, - 12, 14, 16, 20, 24, 28, 32, 40, 48, 56, - 64, 80, 96, 112, 128, 160, 192, 224, 255, -} - -// Minimum offset code that emits bits. -const offsetExtraBitsMinCode = 4 - -// offset code word extra bits. -var offsetExtraBits = [32]int8{ - 0, 0, 0, 0, 1, 1, 2, 2, 3, 3, - 4, 4, 5, 5, 6, 6, 7, 7, 8, 8, - 9, 9, 10, 10, 11, 11, 12, 12, 13, 13, - /* extended window */ - 14, 14, -} - -var offsetCombined = [32]uint32{} - -func init() { - offsetBase := [32]uint32{ - /* normal deflate */ - 0x000000, 0x000001, 0x000002, 0x000003, 0x000004, - 0x000006, 0x000008, 0x00000c, 0x000010, 0x000018, - 0x000020, 0x000030, 0x000040, 0x000060, 0x000080, - 0x0000c0, 0x000100, 0x000180, 0x000200, 0x000300, - 0x000400, 0x000600, 0x000800, 0x000c00, 0x001000, - 0x001800, 0x002000, 0x003000, 0x004000, 0x006000, - - /* extended window */ - 0x008000, 0x00c000, - } - - for i := range offsetCombined[:] { - // Don't use extended window values... - if offsetExtraBits[i] == 0 || offsetBase[i] > 0x006000 { - continue - } - offsetCombined[i] = uint32(offsetExtraBits[i]) | (offsetBase[i] << 8) - } -} - -// The odd order in which the codegen code sizes are written. -var codegenOrder = []uint32{16, 17, 18, 0, 8, 7, 9, 6, 10, 5, 11, 4, 12, 3, 13, 2, 14, 1, 15} - -type huffmanBitWriter struct { - // writer is the underlying writer. - // Do not use it directly; use the write method, which ensures - // that Write errors are sticky. - writer io.Writer - - // Data waiting to be written is bytes[0:nbytes] - // and then the low nbits of bits. - bits uint64 - nbits uint8 - nbytes uint8 - lastHuffMan bool - literalEncoding *huffmanEncoder - tmpLitEncoding *huffmanEncoder - offsetEncoding *huffmanEncoder - codegenEncoding *huffmanEncoder - err error - lastHeader int - // Set between 0 (reused block can be up to 2x the size) - logNewTablePenalty uint - bytes [256 + 8]byte - literalFreq [lengthCodesStart + 32]uint16 - offsetFreq [32]uint16 - codegenFreq [codegenCodeCount]uint16 - - // codegen must have an extra space for the final symbol. - codegen [literalCount + offsetCodeCount + 1]uint8 -} - -// Huffman reuse. -// -// The huffmanBitWriter supports reusing huffman tables and thereby combining block sections. -// -// This is controlled by several variables: -// -// If lastHeader is non-zero the Huffman table can be reused. -// This also indicates that a Huffman table has been generated that can output all -// possible symbols. -// It also indicates that an EOB has not yet been emitted, so if a new tabel is generated -// an EOB with the previous table must be written. -// -// If lastHuffMan is set, a table for outputting literals has been generated and offsets are invalid. -// -// An incoming block estimates the output size of a new table using a 'fresh' by calculating the -// optimal size and adding a penalty in 'logNewTablePenalty'. -// A Huffman table is not optimal, which is why we add a penalty, and generating a new table -// is slower both for compression and decompression. - -func newHuffmanBitWriter(w io.Writer) *huffmanBitWriter { - return &huffmanBitWriter{ - writer: w, - literalEncoding: newHuffmanEncoder(literalCount), - tmpLitEncoding: newHuffmanEncoder(literalCount), - codegenEncoding: newHuffmanEncoder(codegenCodeCount), - offsetEncoding: newHuffmanEncoder(offsetCodeCount), - } -} - -func (w *huffmanBitWriter) reset(writer io.Writer) { - w.writer = writer - w.bits, w.nbits, w.nbytes, w.err = 0, 0, 0, nil - w.lastHeader = 0 - w.lastHuffMan = false -} - -func (w *huffmanBitWriter) canReuse(t *tokens) (ok bool) { - a := t.offHist[:offsetCodeCount] - b := w.offsetEncoding.codes - b = b[:len(a)] - for i, v := range a { - if v != 0 && b[i].zero() { - return false - } - } - - a = t.extraHist[:literalCount-256] - b = w.literalEncoding.codes[256:literalCount] - b = b[:len(a)] - for i, v := range a { - if v != 0 && b[i].zero() { - return false - } - } - - a = t.litHist[:256] - b = w.literalEncoding.codes[:len(a)] - for i, v := range a { - if v != 0 && b[i].zero() { - return false - } - } - return true -} - -func (w *huffmanBitWriter) flush() { - if w.err != nil { - w.nbits = 0 - return - } - if w.lastHeader > 0 { - // We owe an EOB - w.writeCode(w.literalEncoding.codes[endBlockMarker]) - w.lastHeader = 0 - } - n := w.nbytes - for w.nbits != 0 { - w.bytes[n] = byte(w.bits) - w.bits >>= 8 - if w.nbits > 8 { // Avoid underflow - w.nbits -= 8 - } else { - w.nbits = 0 - } - n++ - } - w.bits = 0 - if n > 0 { - w.write(w.bytes[:n]) - } - w.nbytes = 0 -} - -func (w *huffmanBitWriter) write(b []byte) { - if w.err != nil { - return - } - _, w.err = w.writer.Write(b) -} - -func (w *huffmanBitWriter) writeBits(b int32, nb uint8) { - w.bits |= uint64(b) << (w.nbits & 63) - w.nbits += nb - if w.nbits >= 48 { - w.writeOutBits() - } -} - -func (w *huffmanBitWriter) writeBytes(bytes []byte) { - if w.err != nil { - return - } - n := w.nbytes - if w.nbits&7 != 0 { - w.err = InternalError("writeBytes with unfinished bits") - return - } - for w.nbits != 0 { - w.bytes[n] = byte(w.bits) - w.bits >>= 8 - w.nbits -= 8 - n++ - } - if n != 0 { - w.write(w.bytes[:n]) - } - w.nbytes = 0 - w.write(bytes) -} - -// RFC 1951 3.2.7 specifies a special run-length encoding for specifying -// the literal and offset lengths arrays (which are concatenated into a single -// array). This method generates that run-length encoding. -// -// The result is written into the codegen array, and the frequencies -// of each code is written into the codegenFreq array. -// Codes 0-15 are single byte codes. Codes 16-18 are followed by additional -// information. Code badCode is an end marker -// -// numLiterals The number of literals in literalEncoding -// numOffsets The number of offsets in offsetEncoding -// litenc, offenc The literal and offset encoder to use -func (w *huffmanBitWriter) generateCodegen(numLiterals int, numOffsets int, litEnc, offEnc *huffmanEncoder) { - for i := range w.codegenFreq { - w.codegenFreq[i] = 0 - } - // Note that we are using codegen both as a temporary variable for holding - // a copy of the frequencies, and as the place where we put the result. - // This is fine because the output is always shorter than the input used - // so far. - codegen := w.codegen[:] // cache - // Copy the concatenated code sizes to codegen. Put a marker at the end. - cgnl := codegen[:numLiterals] - for i := range cgnl { - cgnl[i] = litEnc.codes[i].len() - } - - cgnl = codegen[numLiterals : numLiterals+numOffsets] - for i := range cgnl { - cgnl[i] = offEnc.codes[i].len() - } - codegen[numLiterals+numOffsets] = badCode - - size := codegen[0] - count := 1 - outIndex := 0 - for inIndex := 1; size != badCode; inIndex++ { - // INVARIANT: We have seen "count" copies of size that have not yet - // had output generated for them. - nextSize := codegen[inIndex] - if nextSize == size { - count++ - continue - } - // We need to generate codegen indicating "count" of size. - if size != 0 { - codegen[outIndex] = size - outIndex++ - w.codegenFreq[size]++ - count-- - for count >= 3 { - n := min(6, count) - codegen[outIndex] = 16 - outIndex++ - codegen[outIndex] = uint8(n - 3) - outIndex++ - w.codegenFreq[16]++ - count -= n - } - } else { - for count >= 11 { - n := min(138, count) - codegen[outIndex] = 18 - outIndex++ - codegen[outIndex] = uint8(n - 11) - outIndex++ - w.codegenFreq[18]++ - count -= n - } - if count >= 3 { - // count >= 3 && count <= 10 - codegen[outIndex] = 17 - outIndex++ - codegen[outIndex] = uint8(count - 3) - outIndex++ - w.codegenFreq[17]++ - count = 0 - } - } - count-- - for ; count >= 0; count-- { - codegen[outIndex] = size - outIndex++ - w.codegenFreq[size]++ - } - // Set up invariant for next time through the loop. - size = nextSize - count = 1 - } - // Marker indicating the end of the codegen. - codegen[outIndex] = badCode -} - -func (w *huffmanBitWriter) codegens() int { - numCodegens := len(w.codegenFreq) - for numCodegens > 4 && w.codegenFreq[codegenOrder[numCodegens-1]] == 0 { - numCodegens-- - } - return numCodegens -} - -func (w *huffmanBitWriter) headerSize() (size, numCodegens int) { - numCodegens = len(w.codegenFreq) - for numCodegens > 4 && w.codegenFreq[codegenOrder[numCodegens-1]] == 0 { - numCodegens-- - } - return 3 + 5 + 5 + 4 + (3 * numCodegens) + - w.codegenEncoding.bitLength(w.codegenFreq[:]) + - int(w.codegenFreq[16])*2 + - int(w.codegenFreq[17])*3 + - int(w.codegenFreq[18])*7, numCodegens -} - -// dynamicSize returns the size of dynamically encoded data in bits. -func (w *huffmanBitWriter) dynamicReuseSize(litEnc, offEnc *huffmanEncoder) (size int) { - size = litEnc.bitLength(w.literalFreq[:]) + - offEnc.bitLength(w.offsetFreq[:]) - return size -} - -// dynamicSize returns the size of dynamically encoded data in bits. -func (w *huffmanBitWriter) dynamicSize(litEnc, offEnc *huffmanEncoder, extraBits int) (size, numCodegens int) { - header, numCodegens := w.headerSize() - size = header + - litEnc.bitLength(w.literalFreq[:]) + - offEnc.bitLength(w.offsetFreq[:]) + - extraBits - return size, numCodegens -} - -// extraBitSize will return the number of bits that will be written -// as "extra" bits on matches. -func (w *huffmanBitWriter) extraBitSize() int { - total := 0 - for i, n := range w.literalFreq[257:literalCount] { - total += int(n) * int(lengthExtraBits[i&31]) - } - for i, n := range w.offsetFreq[:offsetCodeCount] { - total += int(n) * int(offsetExtraBits[i&31]) - } - return total -} - -// fixedSize returns the size of dynamically encoded data in bits. -func (w *huffmanBitWriter) fixedSize(extraBits int) int { - return 3 + - fixedLiteralEncoding.bitLength(w.literalFreq[:]) + - fixedOffsetEncoding.bitLength(w.offsetFreq[:]) + - extraBits -} - -// storedSize calculates the stored size, including header. -// The function returns the size in bits and whether the block -// fits inside a single block. -func (w *huffmanBitWriter) storedSize(in []byte) (int, bool) { - if in == nil { - return 0, false - } - if len(in) <= maxStoreBlockSize { - return (len(in) + 5) * 8, true - } - return 0, false -} - -func (w *huffmanBitWriter) writeCode(c hcode) { - // The function does not get inlined if we "& 63" the shift. - w.bits |= c.code64() << (w.nbits & 63) - w.nbits += c.len() - if w.nbits >= 48 { - w.writeOutBits() - } -} - -// writeOutBits will write bits to the buffer. -func (w *huffmanBitWriter) writeOutBits() { - bits := w.bits - w.bits >>= 48 - w.nbits -= 48 - n := w.nbytes - - // We overwrite, but faster... - le.Store64(w.bytes[:], n, bits) - n += 6 - - if n >= bufferFlushSize { - if w.err != nil { - n = 0 - return - } - w.write(w.bytes[:n]) - n = 0 - } - - w.nbytes = n -} - -// Write the header of a dynamic Huffman block to the output stream. -// -// numLiterals The number of literals specified in codegen -// numOffsets The number of offsets specified in codegen -// numCodegens The number of codegens used in codegen -func (w *huffmanBitWriter) writeDynamicHeader(numLiterals int, numOffsets int, numCodegens int, isEof bool) { - if w.err != nil { - return - } - var firstBits int32 = 4 - if isEof { - firstBits = 5 - } - w.writeBits(firstBits, 3) - w.writeBits(int32(numLiterals-257), 5) - w.writeBits(int32(numOffsets-1), 5) - w.writeBits(int32(numCodegens-4), 4) - - for i := range numCodegens { - value := uint(w.codegenEncoding.codes[codegenOrder[i]].len()) - w.writeBits(int32(value), 3) - } - - i := 0 - for { - codeWord := uint32(w.codegen[i]) - i++ - if codeWord == badCode { - break - } - w.writeCode(w.codegenEncoding.codes[codeWord]) - - switch codeWord { - case 16: - w.writeBits(int32(w.codegen[i]), 2) - i++ - case 17: - w.writeBits(int32(w.codegen[i]), 3) - i++ - case 18: - w.writeBits(int32(w.codegen[i]), 7) - i++ - } - } -} - -// writeStoredHeader will write a stored header. -// If the stored block is only used for EOF, -// it is replaced with a fixed huffman block. -func (w *huffmanBitWriter) writeStoredHeader(length int, isEof bool) { - if w.err != nil { - return - } - if w.lastHeader > 0 { - // We owe an EOB - w.writeCode(w.literalEncoding.codes[endBlockMarker]) - w.lastHeader = 0 - } - - // To write EOF, use a fixed encoding block. 10 bits instead of 5 bytes. - if length == 0 && isEof { - w.writeFixedHeader(isEof) - // EOB: 7 bits, value: 0 - w.writeBits(0, 7) - w.flush() - return - } - - var flag int32 - if isEof { - flag = 1 - } - w.writeBits(flag, 3) - w.flush() - w.writeBits(int32(length), 16) - w.writeBits(int32(^uint16(length)), 16) -} - -func (w *huffmanBitWriter) writeFixedHeader(isEof bool) { - if w.err != nil { - return - } - if w.lastHeader > 0 { - // We owe an EOB - w.writeCode(w.literalEncoding.codes[endBlockMarker]) - w.lastHeader = 0 - } - - // Indicate that we are a fixed Huffman block - var value int32 = 2 - if isEof { - value = 3 - } - w.writeBits(value, 3) -} - -// writeBlock will write a block of tokens with the smallest encoding. -// The original input can be supplied, and if the huffman encoded data -// is larger than the original bytes, the data will be written as a -// stored block. -// If the input is nil, the tokens will always be Huffman encoded. -func (w *huffmanBitWriter) writeBlock(tokens *tokens, eof bool, input []byte) { - if w.err != nil { - return - } - - tokens.AddEOB() - if w.lastHeader > 0 { - // We owe an EOB - w.writeCode(w.literalEncoding.codes[endBlockMarker]) - w.lastHeader = 0 - } - numLiterals, numOffsets := w.indexTokens(tokens, false) - w.generate() - var extraBits int - storedSize, storable := w.storedSize(input) - if storable { - extraBits = w.extraBitSize() - } - - // Figure out smallest code. - // Fixed Huffman baseline. - literalEncoding := fixedLiteralEncoding - offsetEncoding := fixedOffsetEncoding - size := math.MaxInt32 - if tokens.n < maxPredefinedTokens { - size = w.fixedSize(extraBits) - } - - // Dynamic Huffman? - var numCodegens int - - // Generate codegen and codegenFrequencies, which indicates how to encode - // the literalEncoding and the offsetEncoding. - w.generateCodegen(numLiterals, numOffsets, w.literalEncoding, w.offsetEncoding) - w.codegenEncoding.generate(w.codegenFreq[:], 7) - dynamicSize, numCodegens := w.dynamicSize(w.literalEncoding, w.offsetEncoding, extraBits) - - if dynamicSize < size { - size = dynamicSize - literalEncoding = w.literalEncoding - offsetEncoding = w.offsetEncoding - } - - // Stored bytes? - if storable && storedSize <= size { - w.writeStoredHeader(len(input), eof) - w.writeBytes(input) - return - } - - // Huffman. - if literalEncoding == fixedLiteralEncoding { - w.writeFixedHeader(eof) - } else { - w.writeDynamicHeader(numLiterals, numOffsets, numCodegens, eof) - } - - // Write the tokens. - w.writeTokens(tokens.Slice(), literalEncoding.codes, offsetEncoding.codes) -} - -// writeBlockDynamic encodes a block using a dynamic Huffman table. -// This should be used if the symbols used have a disproportionate -// histogram distribution. -// If input is supplied and the compression savings are below 1/16th of the -// input size the block is stored. -func (w *huffmanBitWriter) writeBlockDynamic(tokens *tokens, eof bool, input []byte, sync bool) { - if w.err != nil { - return - } - - sync = sync || eof - if sync { - tokens.AddEOB() - } - - // We cannot reuse pure huffman table, and must mark as EOF. - if (w.lastHuffMan || eof) && w.lastHeader > 0 { - // We will not try to reuse. - w.writeCode(w.literalEncoding.codes[endBlockMarker]) - w.lastHeader = 0 - w.lastHuffMan = false - } - - // fillReuse enables filling of empty values. - // This will make encodings always reusable without testing. - // However, this does not appear to benefit on most cases. - const fillReuse = false - - // Check if we can reuse... - if !fillReuse && w.lastHeader > 0 && !w.canReuse(tokens) { - w.writeCode(w.literalEncoding.codes[endBlockMarker]) - w.lastHeader = 0 - } - - numLiterals, numOffsets := w.indexTokens(tokens, true) - extraBits := 0 - ssize, storable := w.storedSize(input) - - const usePrefs = true - if storable || w.lastHeader > 0 { - extraBits = w.extraBitSize() - } - - var size int - - // Check if we should reuse. - if w.lastHeader > 0 { - // Estimate size for using a new table. - // Use the previous header size as the best estimate. - newSize := w.lastHeader + tokens.EstimatedBits() - newSize += int(w.literalEncoding.codes[endBlockMarker].len()) + newSize>>w.logNewTablePenalty - - // The estimated size is calculated as an optimal table. - // We add a penalty to make it more realistic and re-use a bit more. - reuseSize := w.dynamicReuseSize(w.literalEncoding, w.offsetEncoding) + extraBits - - // Check if a new table is better. - if newSize < reuseSize { - // Write the EOB we owe. - w.writeCode(w.literalEncoding.codes[endBlockMarker]) - size = newSize - w.lastHeader = 0 - } else { - size = reuseSize - } - - if tokens.n < maxPredefinedTokens { - if preSize := w.fixedSize(extraBits) + 7; usePrefs && preSize < size { - // Check if we get a reasonable size decrease. - if storable && ssize <= size { - w.writeStoredHeader(len(input), eof) - w.writeBytes(input) - return - } - w.writeFixedHeader(eof) - if !sync { - tokens.AddEOB() - } - w.writeTokens(tokens.Slice(), fixedLiteralEncoding.codes, fixedOffsetEncoding.codes) - return - } - } - // Check if we get a reasonable size decrease. - if storable && ssize <= size { - w.writeStoredHeader(len(input), eof) - w.writeBytes(input) - return - } - } - - // We want a new block/table - if w.lastHeader == 0 { - if fillReuse && !sync { - w.fillTokens() - numLiterals, numOffsets = maxNumLit, maxNumDist - } else { - w.literalFreq[endBlockMarker] = 1 - } - - w.generate() - // Generate codegen and codegenFrequencies, which indicates how to encode - // the literalEncoding and the offsetEncoding. - w.generateCodegen(numLiterals, numOffsets, w.literalEncoding, w.offsetEncoding) - w.codegenEncoding.generate(w.codegenFreq[:], 7) - - var numCodegens int - if fillReuse && !sync { - // Reindex for accurate size... - w.indexTokens(tokens, true) - } - size, numCodegens = w.dynamicSize(w.literalEncoding, w.offsetEncoding, extraBits) - - // Store predefined, if we don't get a reasonable improvement. - if tokens.n < maxPredefinedTokens { - if preSize := w.fixedSize(extraBits); usePrefs && preSize <= size { - // Store bytes, if we don't get an improvement. - if storable && ssize <= preSize { - w.writeStoredHeader(len(input), eof) - w.writeBytes(input) - return - } - w.writeFixedHeader(eof) - if !sync { - tokens.AddEOB() - } - w.writeTokens(tokens.Slice(), fixedLiteralEncoding.codes, fixedOffsetEncoding.codes) - return - } - } - - if storable && ssize <= size { - // Store bytes, if we don't get an improvement. - w.writeStoredHeader(len(input), eof) - w.writeBytes(input) - return - } - - // Write Huffman table. - w.writeDynamicHeader(numLiterals, numOffsets, numCodegens, eof) - if !sync { - w.lastHeader, _ = w.headerSize() - } - w.lastHuffMan = false - } - - if sync { - w.lastHeader = 0 - } - // Write the tokens. - w.writeTokens(tokens.Slice(), w.literalEncoding.codes, w.offsetEncoding.codes) -} - -func (w *huffmanBitWriter) fillTokens() { - for i, v := range w.literalFreq[:literalCount] { - if v == 0 { - w.literalFreq[i] = 1 - } - } - for i, v := range w.offsetFreq[:offsetCodeCount] { - if v == 0 { - w.offsetFreq[i] = 1 - } - } -} - -// indexTokens indexes a slice of tokens, and updates -// literalFreq and offsetFreq, and generates literalEncoding -// and offsetEncoding. -// The number of literal and offset tokens is returned. -func (w *huffmanBitWriter) indexTokens(t *tokens, alwaysEOB bool) (numLiterals, numOffsets int) { - // copy(w.literalFreq[:], t.litHist[:]) - *(*[256]uint16)(w.literalFreq[:]) = t.litHist - // copy(w.literalFreq[256:], t.extraHist[:]) - *(*[32]uint16)(w.literalFreq[256:]) = t.extraHist - w.offsetFreq = t.offHist - - if t.n == 0 { - return - } - if alwaysEOB { - w.literalFreq[endBlockMarker] = 1 - } - - // get the number of literals - numLiterals = len(w.literalFreq) - for w.literalFreq[numLiterals-1] == 0 { - numLiterals-- - } - // get the number of offsets - numOffsets = len(w.offsetFreq) - for numOffsets > 0 && w.offsetFreq[numOffsets-1] == 0 { - numOffsets-- - } - if numOffsets == 0 { - // We haven't found a single match. If we want to go with the dynamic encoding, - // we should count at least one offset to be sure that the offset huffman tree could be encoded. - w.offsetFreq[0] = 1 - numOffsets = 1 - } - return -} - -func (w *huffmanBitWriter) generate() { - w.literalEncoding.generate(w.literalFreq[:literalCount], 15) - w.offsetEncoding.generate(w.offsetFreq[:offsetCodeCount], 15) -} - -// writeTokens writes a slice of tokens to the output. -// codes for literal and offset encoding must be supplied. -func (w *huffmanBitWriter) writeTokens(tokens []token, leCodes, oeCodes []hcode) { - if w.err != nil { - return - } - if len(tokens) == 0 { - return - } - - // Only last token should be endBlockMarker. - var deferEOB bool - if tokens[len(tokens)-1] == endBlockMarker { - tokens = tokens[:len(tokens)-1] - deferEOB = true - } - - // Create slices up to the next power of two to avoid bounds checks. - lits := leCodes[:256] - offs := oeCodes[:32] - lengths := leCodes[lengthCodesStart:] - lengths = lengths[:32] - - // Go 1.16 LOVES having these on stack. - bits, nbits, nbytes := w.bits, w.nbits, w.nbytes - - for _, t := range tokens { - if t < 256 { - // w.writeCode(lits[t.literal()]) - c := lits[t] - bits |= c.code64() << (nbits & 63) - nbits += c.len() - if nbits >= 48 { - le.Store64(w.bytes[:], nbytes, bits) - bits >>= 48 - nbits -= 48 - nbytes += 6 - if nbytes >= bufferFlushSize { - if w.err != nil { - nbytes = 0 - return - } - _, w.err = w.writer.Write(w.bytes[:nbytes]) - nbytes = 0 - } - } - continue - } - - // Write the length - length := t.length() - lengthCode := lengthCode(length) & 31 - if false { - w.writeCode(lengths[lengthCode]) - } else { - // inlined - c := lengths[lengthCode] - bits |= c.code64() << (nbits & 63) - nbits += c.len() - if nbits >= 48 { - le.Store64(w.bytes[:], nbytes, bits) - bits >>= 48 - nbits -= 48 - nbytes += 6 - if nbytes >= bufferFlushSize { - if w.err != nil { - nbytes = 0 - return - } - _, w.err = w.writer.Write(w.bytes[:nbytes]) - nbytes = 0 - } - } - } - - if lengthCode >= lengthExtraBitsMinCode { - extraLengthBits := lengthExtraBits[lengthCode] - // w.writeBits(extraLength, extraLengthBits) - extraLength := int32(length - lengthBase[lengthCode]) - bits |= uint64(extraLength) << (nbits & 63) - nbits += extraLengthBits - if nbits >= 48 { - le.Store64(w.bytes[:], nbytes, bits) - bits >>= 48 - nbits -= 48 - nbytes += 6 - if nbytes >= bufferFlushSize { - if w.err != nil { - nbytes = 0 - return - } - _, w.err = w.writer.Write(w.bytes[:nbytes]) - nbytes = 0 - } - } - } - // Write the offset - offset := t.offset() - offsetCode := (offset >> 16) & 31 - if false { - w.writeCode(offs[offsetCode]) - } else { - // inlined - c := offs[offsetCode] - bits |= c.code64() << (nbits & 63) - nbits += c.len() - if nbits >= 48 { - le.Store64(w.bytes[:], nbytes, bits) - bits >>= 48 - nbits -= 48 - nbytes += 6 - if nbytes >= bufferFlushSize { - if w.err != nil { - nbytes = 0 - return - } - _, w.err = w.writer.Write(w.bytes[:nbytes]) - nbytes = 0 - } - } - } - - if offsetCode >= offsetExtraBitsMinCode { - offsetComb := offsetCombined[offsetCode] - // w.writeBits(extraOffset, extraOffsetBits) - bits |= uint64((offset-(offsetComb>>8))&matchOffsetOnlyMask) << (nbits & 63) - nbits += uint8(offsetComb) - if nbits >= 48 { - le.Store64(w.bytes[:], nbytes, bits) - bits >>= 48 - nbits -= 48 - nbytes += 6 - if nbytes >= bufferFlushSize { - if w.err != nil { - nbytes = 0 - return - } - _, w.err = w.writer.Write(w.bytes[:nbytes]) - nbytes = 0 - } - } - } - } - // Restore... - w.bits, w.nbits, w.nbytes = bits, nbits, nbytes - - if deferEOB { - w.writeCode(leCodes[endBlockMarker]) - } -} - -// huffOffset is a static offset encoder used for huffman only encoding. -// It can be reused since we will not be encoding offset values. -var huffOffset *huffmanEncoder - -func init() { - w := newHuffmanBitWriter(nil) - w.offsetFreq[0] = 1 - huffOffset = newHuffmanEncoder(offsetCodeCount) - huffOffset.generate(w.offsetFreq[:offsetCodeCount], 15) -} - -// writeBlockHuff encodes a block of bytes as either -// Huffman encoded literals or uncompressed bytes if the -// results only gains very little from compression. -func (w *huffmanBitWriter) writeBlockHuff(eof bool, input []byte, sync bool) { - if w.err != nil { - return - } - - // Clear histogram - for i := range w.literalFreq[:] { - w.literalFreq[i] = 0 - } - if !w.lastHuffMan { - for i := range w.offsetFreq[:] { - w.offsetFreq[i] = 0 - } - } - - const numLiterals = endBlockMarker + 1 - const numOffsets = 1 - - // Add everything as literals - // We have to estimate the header size. - // Assume header is around 70 bytes: - // https://stackoverflow.com/a/25454430 - const guessHeaderSizeBits = 70 * 8 - histogram(input, w.literalFreq[:numLiterals]) - ssize, storable := w.storedSize(input) - if storable && len(input) > 1024 { - // Quick check for incompressible content. - abs := float64(0) - avg := float64(len(input)) / 256 - max := float64(len(input) * 2) - for _, v := range w.literalFreq[:256] { - diff := float64(v) - avg - abs += diff * diff - if abs > max { - break - } - } - if abs < max { - if debugDeflate { - fmt.Println("stored", abs, "<", max) - } - // No chance we can compress this... - w.writeStoredHeader(len(input), eof) - w.writeBytes(input) - return - } - } - w.literalFreq[endBlockMarker] = 1 - w.tmpLitEncoding.generate(w.literalFreq[:numLiterals], 15) - estBits := w.tmpLitEncoding.canReuseBits(w.literalFreq[:numLiterals]) - if estBits < math.MaxInt32 { - estBits += w.lastHeader - if w.lastHeader == 0 { - estBits += guessHeaderSizeBits - } - estBits += estBits >> w.logNewTablePenalty - } - - // Store bytes, if we don't get a reasonable improvement. - if storable && ssize <= estBits { - if debugDeflate { - fmt.Println("stored,", ssize, "<=", estBits) - } - w.writeStoredHeader(len(input), eof) - w.writeBytes(input) - return - } - - if w.lastHeader > 0 { - reuseSize := w.literalEncoding.canReuseBits(w.literalFreq[:256]) - - if estBits < reuseSize { - if debugDeflate { - fmt.Println("NOT reusing, reuse:", reuseSize/8, "> new:", estBits/8, "header est:", w.lastHeader/8, "bytes") - } - // We owe an EOB - w.writeCode(w.literalEncoding.codes[endBlockMarker]) - w.lastHeader = 0 - } else if debugDeflate { - fmt.Println("reusing, reuse:", reuseSize/8, "> new:", estBits/8, "- header est:", w.lastHeader/8) - } - } - - count := 0 - if w.lastHeader == 0 { - // Use the temp encoding, so swap. - w.literalEncoding, w.tmpLitEncoding = w.tmpLitEncoding, w.literalEncoding - // Generate codegen and codegenFrequencies, which indicates how to encode - // the literalEncoding and the offsetEncoding. - w.generateCodegen(numLiterals, numOffsets, w.literalEncoding, huffOffset) - w.codegenEncoding.generate(w.codegenFreq[:], 7) - numCodegens := w.codegens() - - // Huffman. - w.writeDynamicHeader(numLiterals, numOffsets, numCodegens, eof) - w.lastHuffMan = true - w.lastHeader, _ = w.headerSize() - if debugDeflate { - count += w.lastHeader - fmt.Println("header:", count/8) - } - } - - encoding := w.literalEncoding.codes[:256] - // Go 1.16 LOVES having these on stack. At least 1.5x the speed. - bits, nbits, nbytes := w.bits, w.nbits, w.nbytes - - if debugDeflate { - count -= int(nbytes)*8 + int(nbits) - } - // Unroll, write 3 codes/loop. - // Fastest number of unrolls. - for len(input) > 3 { - // We must have at least 48 bits free. - if nbits >= 8 { - n := nbits >> 3 - le.Store64(w.bytes[:], nbytes, bits) - bits >>= (n * 8) & 63 - nbits -= n * 8 - nbytes += n - } - if nbytes >= bufferFlushSize { - if w.err != nil { - nbytes = 0 - return - } - if debugDeflate { - count += int(nbytes) * 8 - } - _, w.err = w.writer.Write(w.bytes[:nbytes]) - nbytes = 0 - } - a, b := encoding[input[0]], encoding[input[1]] - bits |= a.code64() << (nbits & 63) - bits |= b.code64() << ((nbits + a.len()) & 63) - c := encoding[input[2]] - nbits += b.len() + a.len() - bits |= c.code64() << (nbits & 63) - nbits += c.len() - input = input[3:] - } - - // Remaining... - for _, t := range input { - if nbits >= 48 { - le.Store64(w.bytes[:], nbytes, bits) - bits >>= 48 - nbits -= 48 - nbytes += 6 - if nbytes >= bufferFlushSize { - if w.err != nil { - nbytes = 0 - return - } - if debugDeflate { - count += int(nbytes) * 8 - } - _, w.err = w.writer.Write(w.bytes[:nbytes]) - nbytes = 0 - } - } - // Bitwriting inlined, ~30% speedup - c := encoding[t] - bits |= c.code64() << (nbits & 63) - - nbits += c.len() - if debugDeflate { - count += int(c.len()) - } - } - // Restore... - w.bits, w.nbits, w.nbytes = bits, nbits, nbytes - - if debugDeflate { - nb := count + int(nbytes)*8 + int(nbits) - fmt.Println("wrote", nb, "bits,", nb/8, "bytes.") - } - // Flush if needed to have space. - if w.nbits >= 48 { - w.writeOutBits() - } - - if eof || sync { - w.writeCode(w.literalEncoding.codes[endBlockMarker]) - w.lastHeader = 0 - w.lastHuffMan = false - } -} diff --git a/internal/compress/flate/huffman_bit_writer_test.go b/internal/compress/flate/huffman_bit_writer_test.go deleted file mode 100644 index 3fc414e2..00000000 --- a/internal/compress/flate/huffman_bit_writer_test.go +++ /dev/null @@ -1,381 +0,0 @@ -// Copyright 2016 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package flate - -import ( - "bytes" - "flag" - "fmt" - "os" - "path/filepath" - "strings" - "testing" -) - -var update = flag.Bool("update", false, "update reference files") - -// TestBlockHuff tests huffman encoding against reference files -// to detect possible regressions. -// If encoding/bit allocation changes you can regenerate these files -// by using the -update flag. -func TestBlockHuff(t *testing.T) { - // determine input files - match, err := filepath.Glob("testdata/huffman-*.in") - if err != nil { - t.Fatal(err) - } - - for _, in := range match { - out := in // for files where input and output are identical - if strings.HasSuffix(in, ".in") { - out = in[:len(in)-len(".in")] + ".golden" - } - t.Run(in, func(t *testing.T) { - testBlockHuff(t, in, out) - }) - } -} - -func testBlockHuff(t *testing.T, in, out string) { - all, err := os.ReadFile(in) - if err != nil { - t.Error(err) - return - } - var buf bytes.Buffer - bw := newHuffmanBitWriter(&buf) - bw.logNewTablePenalty = 8 - bw.writeBlockHuff(false, all, false) - bw.flush() - got := buf.Bytes() - - want, err := os.ReadFile(out) - if err != nil && !*update { - t.Error(err) - return - } - - t.Logf("Testing %q", in) - if !bytes.Equal(got, want) { - if *update { - if in != out { - t.Logf("Updating %q", out) - if err := os.WriteFile(out, got, 0o666); err != nil { - t.Error(err) - } - return - } - // in == out: don't accidentally destroy input - t.Errorf("WARNING: -update did not rewrite input file %s", in) - } - - t.Errorf("%q != %q (see %q)", in, out, in+".got") - if err := os.WriteFile(in+".got", got, 0o666); err != nil { - t.Error(err) - } - return - } - t.Log("Output ok") - - // Test if the writer produces the same output after reset. - buf.Reset() - bw.reset(&buf) - bw.writeBlockHuff(false, all, false) - bw.flush() - got = buf.Bytes() - if !bytes.Equal(got, want) { - t.Errorf("after reset %q != %q (see %q)", in, out, in+".reset.got") - if err := os.WriteFile(in+".reset.got", got, 0o666); err != nil { - t.Error(err) - } - return - } - t.Log("Reset ok") - testWriterEOF(t, "huff", huffTest{input: in}, true) -} - -type huffTest struct { - tokens []token - input string // File name of input data matching the tokens. - want string // File name of data with the expected output with input available. - wantNoInput string // File name of the expected output when no input is available. -} - -const ml = 0x7fc00000 // Maximum length token. Used to reduce the size of writeBlockTests - -var writeBlockTests = []huffTest{ - { - input: "testdata/huffman-null-max.in", - want: "testdata/huffman-null-max.%s.expect", - wantNoInput: "testdata/huffman-null-max.%s.expect-noinput", - tokens: []token{0x0, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, 0x0, 0x0}, - }, - { - input: "testdata/huffman-pi.in", - want: "testdata/huffman-pi.%s.expect", - wantNoInput: "testdata/huffman-pi.%s.expect-noinput", - tokens: []token{0x33, 0x2e, 0x31, 0x34, 0x31, 0x35, 0x39, 0x32, 0x36, 0x35, 0x33, 0x35, 0x38, 0x39, 0x37, 0x39, 0x33, 0x32, 0x33, 0x38, 0x34, 0x36, 0x32, 0x36, 0x34, 0x33, 0x33, 0x38, 0x33, 0x32, 0x37, 0x39, 0x35, 0x30, 0x32, 0x38, 0x38, 0x34, 0x31, 0x39, 0x37, 0x31, 0x36, 0x39, 0x33, 0x39, 0x39, 0x33, 0x37, 0x35, 0x31, 0x30, 0x35, 0x38, 0x32, 0x30, 0x39, 0x37, 0x34, 0x39, 0x34, 0x34, 0x35, 0x39, 0x32, 0x33, 0x30, 0x37, 0x38, 0x31, 0x36, 0x34, 0x30, 0x36, 0x32, 0x38, 0x36, 0x32, 0x30, 0x38, 0x39, 0x39, 0x38, 0x36, 0x32, 0x38, 0x30, 0x33, 0x34, 0x38, 0x32, 0x35, 0x33, 0x34, 0x32, 0x31, 0x31, 0x37, 0x30, 0x36, 0x37, 0x39, 0x38, 0x32, 0x31, 0x34, 0x38, 0x30, 0x38, 0x36, 0x35, 0x31, 0x33, 0x32, 0x38, 0x32, 0x33, 0x30, 0x36, 0x36, 0x34, 0x37, 0x30, 0x39, 0x33, 0x38, 0x34, 0x34, 0x36, 0x30, 0x39, 0x35, 0x35, 0x30, 0x35, 0x38, 0x32, 0x32, 0x33, 0x31, 0x37, 0x32, 0x35, 0x33, 0x35, 0x39, 0x34, 0x30, 0x38, 0x31, 0x32, 0x38, 0x34, 0x38, 0x31, 0x31, 0x31, 0x37, 0x34, 0x4040007e, 0x34, 0x31, 0x30, 0x32, 0x37, 0x30, 0x31, 0x39, 0x33, 0x38, 0x35, 0x32, 0x31, 0x31, 0x30, 0x35, 0x35, 0x35, 0x39, 0x36, 0x34, 0x34, 0x36, 0x32, 0x32, 0x39, 0x34, 0x38, 0x39, 0x35, 0x34, 0x39, 0x33, 0x30, 0x33, 0x38, 0x31, 0x40400012, 0x32, 0x38, 0x38, 0x31, 0x30, 0x39, 0x37, 0x35, 0x36, 0x36, 0x35, 0x39, 0x33, 0x33, 0x34, 0x34, 0x36, 0x40400047, 0x37, 0x35, 0x36, 0x34, 0x38, 0x32, 0x33, 0x33, 0x37, 0x38, 0x36, 0x37, 0x38, 0x33, 0x31, 0x36, 0x35, 0x32, 0x37, 0x31, 0x32, 0x30, 0x31, 0x39, 0x30, 0x39, 0x31, 0x34, 0x4040001a, 0x35, 0x36, 0x36, 0x39, 0x32, 0x33, 0x34, 0x36, 0x404000b2, 0x36, 0x31, 0x30, 0x34, 0x35, 0x34, 0x33, 0x32, 0x36, 0x40400032, 0x31, 0x33, 0x33, 0x39, 0x33, 0x36, 0x30, 0x37, 0x32, 0x36, 0x30, 0x32, 0x34, 0x39, 0x31, 0x34, 0x31, 0x32, 0x37, 0x33, 0x37, 0x32, 0x34, 0x35, 0x38, 0x37, 0x30, 0x30, 0x36, 0x36, 0x30, 0x36, 0x33, 0x31, 0x35, 0x35, 0x38, 0x38, 0x31, 0x37, 0x34, 0x38, 0x38, 0x31, 0x35, 0x32, 0x30, 0x39, 0x32, 0x30, 0x39, 0x36, 0x32, 0x38, 0x32, 0x39, 0x32, 0x35, 0x34, 0x30, 0x39, 0x31, 0x37, 0x31, 0x35, 0x33, 0x36, 0x34, 0x33, 0x36, 0x37, 0x38, 0x39, 0x32, 0x35, 0x39, 0x30, 0x33, 0x36, 0x30, 0x30, 0x31, 0x31, 0x33, 0x33, 0x30, 0x35, 0x33, 0x30, 0x35, 0x34, 0x38, 0x38, 0x32, 0x30, 0x34, 0x36, 0x36, 0x35, 0x32, 0x31, 0x33, 0x38, 0x34, 0x31, 0x34, 0x36, 0x39, 0x35, 0x31, 0x39, 0x34, 0x31, 0x35, 0x31, 0x31, 0x36, 0x30, 0x39, 0x34, 0x33, 0x33, 0x30, 0x35, 0x37, 0x32, 0x37, 0x30, 0x33, 0x36, 0x35, 0x37, 0x35, 0x39, 0x35, 0x39, 0x31, 0x39, 0x35, 0x33, 0x30, 0x39, 0x32, 0x31, 0x38, 0x36, 0x31, 0x31, 0x37, 0x404000e9, 0x33, 0x32, 0x40400009, 0x39, 0x33, 0x31, 0x30, 0x35, 0x31, 0x31, 0x38, 0x35, 0x34, 0x38, 0x30, 0x37, 0x4040010e, 0x33, 0x37, 0x39, 0x39, 0x36, 0x32, 0x37, 0x34, 0x39, 0x35, 0x36, 0x37, 0x33, 0x35, 0x31, 0x38, 0x38, 0x35, 0x37, 0x35, 0x32, 0x37, 0x32, 0x34, 0x38, 0x39, 0x31, 0x32, 0x32, 0x37, 0x39, 0x33, 0x38, 0x31, 0x38, 0x33, 0x30, 0x31, 0x31, 0x39, 0x34, 0x39, 0x31, 0x32, 0x39, 0x38, 0x33, 0x33, 0x36, 0x37, 0x33, 0x33, 0x36, 0x32, 0x34, 0x34, 0x30, 0x36, 0x35, 0x36, 0x36, 0x34, 0x33, 0x30, 0x38, 0x36, 0x30, 0x32, 0x31, 0x33, 0x39, 0x34, 0x39, 0x34, 0x36, 0x33, 0x39, 0x35, 0x32, 0x32, 0x34, 0x37, 0x33, 0x37, 0x31, 0x39, 0x30, 0x37, 0x30, 0x32, 0x31, 0x37, 0x39, 0x38, 0x40800099, 0x37, 0x30, 0x32, 0x37, 0x37, 0x30, 0x35, 0x33, 0x39, 0x32, 0x31, 0x37, 0x31, 0x37, 0x36, 0x32, 0x39, 0x33, 0x31, 0x37, 0x36, 0x37, 0x35, 0x40800232, 0x37, 0x34, 0x38, 0x31, 0x40400006, 0x36, 0x36, 0x39, 0x34, 0x30, 0x404001e7, 0x30, 0x30, 0x30, 0x35, 0x36, 0x38, 0x31, 0x32, 0x37, 0x31, 0x34, 0x35, 0x32, 0x36, 0x33, 0x35, 0x36, 0x30, 0x38, 0x32, 0x37, 0x37, 0x38, 0x35, 0x37, 0x37, 0x31, 0x33, 0x34, 0x32, 0x37, 0x35, 0x37, 0x37, 0x38, 0x39, 0x36, 0x40400129, 0x33, 0x36, 0x33, 0x37, 0x31, 0x37, 0x38, 0x37, 0x32, 0x31, 0x34, 0x36, 0x38, 0x34, 0x34, 0x30, 0x39, 0x30, 0x31, 0x32, 0x32, 0x34, 0x39, 0x35, 0x33, 0x34, 0x33, 0x30, 0x31, 0x34, 0x36, 0x35, 0x34, 0x39, 0x35, 0x38, 0x35, 0x33, 0x37, 0x31, 0x30, 0x35, 0x30, 0x37, 0x39, 0x404000ca, 0x36, 0x40400153, 0x38, 0x39, 0x32, 0x33, 0x35, 0x34, 0x404001c9, 0x39, 0x35, 0x36, 0x31, 0x31, 0x32, 0x31, 0x32, 0x39, 0x30, 0x32, 0x31, 0x39, 0x36, 0x30, 0x38, 0x36, 0x34, 0x30, 0x33, 0x34, 0x34, 0x31, 0x38, 0x31, 0x35, 0x39, 0x38, 0x31, 0x33, 0x36, 0x32, 0x39, 0x37, 0x37, 0x34, 0x40400074, 0x30, 0x39, 0x39, 0x36, 0x30, 0x35, 0x31, 0x38, 0x37, 0x30, 0x37, 0x32, 0x31, 0x31, 0x33, 0x34, 0x39, 0x40800000, 0x38, 0x33, 0x37, 0x32, 0x39, 0x37, 0x38, 0x30, 0x34, 0x39, 0x39, 0x404002da, 0x39, 0x37, 0x33, 0x31, 0x37, 0x33, 0x32, 0x38, 0x4040018a, 0x36, 0x33, 0x31, 0x38, 0x35, 0x40400301, 0x404002e8, 0x34, 0x35, 0x35, 0x33, 0x34, 0x36, 0x39, 0x30, 0x38, 0x33, 0x30, 0x32, 0x36, 0x34, 0x32, 0x35, 0x32, 0x32, 0x33, 0x30, 0x404002e3, 0x40400267, 0x38, 0x35, 0x30, 0x33, 0x35, 0x32, 0x36, 0x31, 0x39, 0x33, 0x31, 0x31, 0x40400212, 0x31, 0x30, 0x31, 0x30, 0x30, 0x30, 0x33, 0x31, 0x33, 0x37, 0x38, 0x33, 0x38, 0x37, 0x35, 0x32, 0x38, 0x38, 0x36, 0x35, 0x38, 0x37, 0x35, 0x33, 0x33, 0x32, 0x30, 0x38, 0x33, 0x38, 0x31, 0x34, 0x32, 0x30, 0x36, 0x40400140, 0x4040012b, 0x31, 0x34, 0x37, 0x33, 0x30, 0x33, 0x35, 0x39, 0x4080032e, 0x39, 0x30, 0x34, 0x32, 0x38, 0x37, 0x35, 0x35, 0x34, 0x36, 0x38, 0x37, 0x33, 0x31, 0x31, 0x35, 0x39, 0x35, 0x40400355, 0x33, 0x38, 0x38, 0x32, 0x33, 0x35, 0x33, 0x37, 0x38, 0x37, 0x35, 0x4080037f, 0x39, 0x4040013a, 0x31, 0x40400148, 0x38, 0x30, 0x35, 0x33, 0x4040018a, 0x32, 0x32, 0x36, 0x38, 0x30, 0x36, 0x36, 0x31, 0x33, 0x30, 0x30, 0x31, 0x39, 0x32, 0x37, 0x38, 0x37, 0x36, 0x36, 0x31, 0x31, 0x31, 0x39, 0x35, 0x39, 0x40400237, 0x36, 0x40800124, 0x38, 0x39, 0x33, 0x38, 0x30, 0x39, 0x35, 0x32, 0x35, 0x37, 0x32, 0x30, 0x31, 0x30, 0x36, 0x35, 0x34, 0x38, 0x35, 0x38, 0x36, 0x33, 0x32, 0x37, 0x4040009a, 0x39, 0x33, 0x36, 0x31, 0x35, 0x33, 0x40400220, 0x4080015c, 0x32, 0x33, 0x30, 0x33, 0x30, 0x31, 0x39, 0x35, 0x32, 0x30, 0x33, 0x35, 0x33, 0x30, 0x31, 0x38, 0x35, 0x32, 0x40400171, 0x40400075, 0x33, 0x36, 0x32, 0x32, 0x35, 0x39, 0x39, 0x34, 0x31, 0x33, 0x40400254, 0x34, 0x39, 0x37, 0x32, 0x31, 0x37, 0x404000de, 0x33, 0x34, 0x37, 0x39, 0x31, 0x33, 0x31, 0x35, 0x31, 0x35, 0x35, 0x37, 0x34, 0x38, 0x35, 0x37, 0x32, 0x34, 0x32, 0x34, 0x35, 0x34, 0x31, 0x35, 0x30, 0x36, 0x39, 0x4040013f, 0x38, 0x32, 0x39, 0x35, 0x33, 0x33, 0x31, 0x31, 0x36, 0x38, 0x36, 0x31, 0x37, 0x32, 0x37, 0x38, 0x40400337, 0x39, 0x30, 0x37, 0x35, 0x30, 0x39, 0x4040010d, 0x37, 0x35, 0x34, 0x36, 0x33, 0x37, 0x34, 0x36, 0x34, 0x39, 0x33, 0x39, 0x33, 0x31, 0x39, 0x32, 0x35, 0x35, 0x30, 0x36, 0x30, 0x34, 0x30, 0x30, 0x39, 0x4040026b, 0x31, 0x36, 0x37, 0x31, 0x31, 0x33, 0x39, 0x30, 0x30, 0x39, 0x38, 0x40400335, 0x34, 0x30, 0x31, 0x32, 0x38, 0x35, 0x38, 0x33, 0x36, 0x31, 0x36, 0x30, 0x33, 0x35, 0x36, 0x33, 0x37, 0x30, 0x37, 0x36, 0x36, 0x30, 0x31, 0x30, 0x34, 0x40400172, 0x38, 0x31, 0x39, 0x34, 0x32, 0x39, 0x4080041e, 0x404000ef, 0x4040028b, 0x37, 0x38, 0x33, 0x37, 0x34, 0x404004a8, 0x38, 0x32, 0x35, 0x35, 0x33, 0x37, 0x40800209, 0x32, 0x36, 0x38, 0x4040002e, 0x34, 0x30, 0x34, 0x37, 0x404001d1, 0x34, 0x404004b5, 0x4040038d, 0x38, 0x34, 0x404003a8, 0x36, 0x40c0031f, 0x33, 0x33, 0x31, 0x33, 0x36, 0x37, 0x37, 0x30, 0x32, 0x38, 0x39, 0x38, 0x39, 0x31, 0x35, 0x32, 0x40400062, 0x35, 0x32, 0x31, 0x36, 0x32, 0x30, 0x35, 0x36, 0x39, 0x36, 0x40400411, 0x30, 0x35, 0x38, 0x40400477, 0x35, 0x40400498, 0x35, 0x31, 0x31, 0x40400209, 0x38, 0x32, 0x34, 0x33, 0x30, 0x30, 0x33, 0x35, 0x35, 0x38, 0x37, 0x36, 0x34, 0x30, 0x32, 0x34, 0x37, 0x34, 0x39, 0x36, 0x34, 0x37, 0x33, 0x32, 0x36, 0x33, 0x4040043e, 0x39, 0x39, 0x32, 0x4040044b, 0x34, 0x32, 0x36, 0x39, 0x40c002c5, 0x37, 0x404001d6, 0x34, 0x4040053d, 0x4040041d, 0x39, 0x33, 0x34, 0x31, 0x37, 0x404001ad, 0x31, 0x32, 0x4040002a, 0x34, 0x4040019e, 0x31, 0x35, 0x30, 0x33, 0x30, 0x32, 0x38, 0x36, 0x31, 0x38, 0x32, 0x39, 0x37, 0x34, 0x35, 0x35, 0x35, 0x37, 0x30, 0x36, 0x37, 0x34, 0x40400135, 0x35, 0x30, 0x35, 0x34, 0x39, 0x34, 0x35, 0x38, 0x404001c5, 0x39, 0x40400051, 0x35, 0x36, 0x404001ec, 0x37, 0x32, 0x31, 0x30, 0x37, 0x39, 0x40400159, 0x33, 0x30, 0x4040010a, 0x33, 0x32, 0x31, 0x31, 0x36, 0x35, 0x33, 0x34, 0x34, 0x39, 0x38, 0x37, 0x32, 0x30, 0x32, 0x37, 0x4040011b, 0x30, 0x32, 0x33, 0x36, 0x34, 0x4040022e, 0x35, 0x34, 0x39, 0x39, 0x31, 0x31, 0x39, 0x38, 0x40400418, 0x34, 0x4040011b, 0x35, 0x33, 0x35, 0x36, 0x36, 0x33, 0x36, 0x39, 0x40400450, 0x32, 0x36, 0x35, 0x404002e4, 0x37, 0x38, 0x36, 0x32, 0x35, 0x35, 0x31, 0x404003da, 0x31, 0x37, 0x35, 0x37, 0x34, 0x36, 0x37, 0x32, 0x38, 0x39, 0x30, 0x39, 0x37, 0x37, 0x37, 0x37, 0x40800453, 0x30, 0x30, 0x30, 0x404005fd, 0x37, 0x30, 0x404004df, 0x36, 0x404003e9, 0x34, 0x39, 0x31, 0x4040041e, 0x40400297, 0x32, 0x31, 0x34, 0x37, 0x37, 0x32, 0x33, 0x35, 0x30, 0x31, 0x34, 0x31, 0x34, 0x40400643, 0x33, 0x35, 0x36, 0x404004af, 0x31, 0x36, 0x31, 0x33, 0x36, 0x31, 0x31, 0x35, 0x37, 0x33, 0x35, 0x32, 0x35, 0x40400504, 0x33, 0x34, 0x4040005b, 0x31, 0x38, 0x4040047b, 0x38, 0x34, 0x404005e7, 0x33, 0x33, 0x32, 0x33, 0x39, 0x30, 0x37, 0x33, 0x39, 0x34, 0x31, 0x34, 0x33, 0x33, 0x33, 0x34, 0x35, 0x34, 0x37, 0x37, 0x36, 0x32, 0x34, 0x40400242, 0x32, 0x35, 0x31, 0x38, 0x39, 0x38, 0x33, 0x35, 0x36, 0x39, 0x34, 0x38, 0x35, 0x35, 0x36, 0x32, 0x30, 0x39, 0x39, 0x32, 0x31, 0x39, 0x32, 0x32, 0x32, 0x31, 0x38, 0x34, 0x32, 0x37, 0x4040023e, 0x32, 0x404000ba, 0x36, 0x38, 0x38, 0x37, 0x36, 0x37, 0x31, 0x37, 0x39, 0x30, 0x40400055, 0x30, 0x40800106, 0x36, 0x36, 0x404003e7, 0x38, 0x38, 0x36, 0x32, 0x37, 0x32, 0x404006dc, 0x31, 0x37, 0x38, 0x36, 0x30, 0x38, 0x35, 0x37, 0x40400073, 0x33, 0x408002fc, 0x37, 0x39, 0x37, 0x36, 0x36, 0x38, 0x31, 0x404002bd, 0x30, 0x30, 0x39, 0x35, 0x33, 0x38, 0x38, 0x40400638, 0x33, 0x404006a5, 0x30, 0x36, 0x38, 0x30, 0x30, 0x36, 0x34, 0x32, 0x32, 0x35, 0x31, 0x32, 0x35, 0x32, 0x4040057b, 0x37, 0x33, 0x39, 0x32, 0x40400297, 0x40400474, 0x34, 0x408006b3, 0x38, 0x36, 0x32, 0x36, 0x39, 0x34, 0x35, 0x404001e5, 0x34, 0x31, 0x39, 0x36, 0x35, 0x32, 0x38, 0x35, 0x30, 0x40400099, 0x4040039c, 0x31, 0x38, 0x36, 0x33, 0x404001be, 0x34, 0x40800154, 0x32, 0x30, 0x33, 0x39, 0x4040058b, 0x34, 0x35, 0x404002bc, 0x32, 0x33, 0x37, 0x4040042c, 0x36, 0x40400510, 0x35, 0x36, 0x40400638, 0x37, 0x31, 0x39, 0x31, 0x37, 0x32, 0x38, 0x40400171, 0x37, 0x36, 0x34, 0x36, 0x35, 0x37, 0x35, 0x37, 0x33, 0x39, 0x40400101, 0x33, 0x38, 0x39, 0x40400748, 0x38, 0x33, 0x32, 0x36, 0x34, 0x35, 0x39, 0x39, 0x35, 0x38, 0x404006a7, 0x30, 0x34, 0x37, 0x38, 0x404001de, 0x40400328, 0x39, 0x4040002d, 0x36, 0x34, 0x30, 0x37, 0x38, 0x39, 0x35, 0x31, 0x4040008e, 0x36, 0x38, 0x33, 0x4040012f, 0x32, 0x35, 0x39, 0x35, 0x37, 0x30, 0x40400468, 0x38, 0x32, 0x32, 0x404002c8, 0x32, 0x4040061b, 0x34, 0x30, 0x37, 0x37, 0x32, 0x36, 0x37, 0x31, 0x39, 0x34, 0x37, 0x38, 0x40400319, 0x38, 0x32, 0x36, 0x30, 0x31, 0x34, 0x37, 0x36, 0x39, 0x39, 0x30, 0x39, 0x404004e8, 0x30, 0x31, 0x33, 0x36, 0x33, 0x39, 0x34, 0x34, 0x33, 0x4040027f, 0x33, 0x30, 0x40400105, 0x32, 0x30, 0x33, 0x34, 0x39, 0x36, 0x32, 0x35, 0x32, 0x34, 0x35, 0x31, 0x37, 0x404003b5, 0x39, 0x36, 0x35, 0x31, 0x34, 0x33, 0x31, 0x34, 0x32, 0x39, 0x38, 0x30, 0x39, 0x31, 0x39, 0x30, 0x36, 0x35, 0x39, 0x32, 0x40400282, 0x37, 0x32, 0x32, 0x31, 0x36, 0x39, 0x36, 0x34, 0x36, 0x40400419, 0x4040007a, 0x35, 0x4040050e, 0x34, 0x40800565, 0x38, 0x40400559, 0x39, 0x37, 0x4040057b, 0x35, 0x34, 0x4040049d, 0x4040023e, 0x37, 0x4040065a, 0x38, 0x34, 0x36, 0x38, 0x31, 0x33, 0x4040008c, 0x36, 0x38, 0x33, 0x38, 0x36, 0x38, 0x39, 0x34, 0x32, 0x37, 0x37, 0x34, 0x31, 0x35, 0x35, 0x39, 0x39, 0x31, 0x38, 0x35, 0x4040005a, 0x32, 0x34, 0x35, 0x39, 0x35, 0x33, 0x39, 0x35, 0x39, 0x34, 0x33, 0x31, 0x404005b7, 0x37, 0x40400012, 0x36, 0x38, 0x30, 0x38, 0x34, 0x35, 0x404002e7, 0x37, 0x33, 0x4040081e, 0x39, 0x35, 0x38, 0x34, 0x38, 0x36, 0x35, 0x33, 0x38, 0x404006e8, 0x36, 0x32, 0x404000f2, 0x36, 0x30, 0x39, 0x404004b6, 0x36, 0x30, 0x38, 0x30, 0x35, 0x31, 0x32, 0x34, 0x33, 0x38, 0x38, 0x34, 0x4040013a, 0x4040000b, 0x34, 0x31, 0x33, 0x4040030f, 0x37, 0x36, 0x32, 0x37, 0x38, 0x40400341, 0x37, 0x31, 0x35, 0x4040059b, 0x33, 0x35, 0x39, 0x39, 0x37, 0x37, 0x30, 0x30, 0x31, 0x32, 0x39, 0x40400472, 0x38, 0x39, 0x34, 0x34, 0x31, 0x40400277, 0x36, 0x38, 0x35, 0x35, 0x4040005f, 0x34, 0x30, 0x36, 0x33, 0x404008e6, 0x32, 0x30, 0x37, 0x32, 0x32, 0x40400158, 0x40800203, 0x34, 0x38, 0x31, 0x35, 0x38, 0x40400205, 0x404001fe, 0x4040027a, 0x40400298, 0x33, 0x39, 0x34, 0x35, 0x32, 0x32, 0x36, 0x37, 0x40c00496, 0x38, 0x4040058a, 0x32, 0x31, 0x404002ea, 0x32, 0x40400387, 0x35, 0x34, 0x36, 0x36, 0x36, 0x4040051b, 0x32, 0x33, 0x39, 0x38, 0x36, 0x34, 0x35, 0x36, 0x404004c4, 0x31, 0x36, 0x33, 0x35, 0x40800253, 0x40400811, 0x37, 0x404008ad, 0x39, 0x38, 0x4040045e, 0x39, 0x33, 0x36, 0x33, 0x34, 0x4040075b, 0x37, 0x34, 0x33, 0x32, 0x34, 0x4040047b, 0x31, 0x35, 0x30, 0x37, 0x36, 0x404004bb, 0x37, 0x39, 0x34, 0x35, 0x31, 0x30, 0x39, 0x4040003e, 0x30, 0x39, 0x34, 0x30, 0x404006a6, 0x38, 0x38, 0x37, 0x39, 0x37, 0x31, 0x30, 0x38, 0x39, 0x33, 0x404008f0, 0x36, 0x39, 0x31, 0x33, 0x36, 0x38, 0x36, 0x37, 0x32, 0x4040025b, 0x404001fe, 0x35, 0x4040053f, 0x40400468, 0x40400801, 0x31, 0x37, 0x39, 0x32, 0x38, 0x36, 0x38, 0x404008cc, 0x38, 0x37, 0x34, 0x37, 0x4080079e, 0x38, 0x32, 0x34, 0x4040097a, 0x38, 0x4040025b, 0x37, 0x31, 0x34, 0x39, 0x30, 0x39, 0x36, 0x37, 0x35, 0x39, 0x38, 0x404006ef, 0x33, 0x36, 0x35, 0x40400134, 0x38, 0x31, 0x4040005c, 0x40400745, 0x40400936, 0x36, 0x38, 0x32, 0x39, 0x4040057e, 0x38, 0x37, 0x32, 0x32, 0x36, 0x35, 0x38, 0x38, 0x30, 0x40400611, 0x35, 0x40400249, 0x34, 0x32, 0x37, 0x30, 0x34, 0x37, 0x37, 0x35, 0x35, 0x4040081e, 0x33, 0x37, 0x39, 0x36, 0x34, 0x31, 0x34, 0x35, 0x31, 0x35, 0x32, 0x404005fd, 0x32, 0x33, 0x34, 0x33, 0x36, 0x34, 0x35, 0x34, 0x404005de, 0x34, 0x34, 0x34, 0x37, 0x39, 0x35, 0x4040003c, 0x40400523, 0x408008e6, 0x34, 0x31, 0x4040052a, 0x33, 0x40400304, 0x35, 0x32, 0x33, 0x31, 0x40800841, 0x31, 0x36, 0x36, 0x31, 0x404008b2, 0x35, 0x39, 0x36, 0x39, 0x35, 0x33, 0x36, 0x32, 0x33, 0x31, 0x34, 0x404005ff, 0x32, 0x34, 0x38, 0x34, 0x39, 0x33, 0x37, 0x31, 0x38, 0x37, 0x31, 0x31, 0x30, 0x31, 0x34, 0x35, 0x37, 0x36, 0x35, 0x34, 0x40400761, 0x30, 0x32, 0x37, 0x39, 0x39, 0x33, 0x34, 0x34, 0x30, 0x33, 0x37, 0x34, 0x32, 0x30, 0x30, 0x37, 0x4040093f, 0x37, 0x38, 0x35, 0x33, 0x39, 0x30, 0x36, 0x32, 0x31, 0x39, 0x40800299, 0x40400345, 0x38, 0x34, 0x37, 0x408003d2, 0x38, 0x33, 0x33, 0x32, 0x31, 0x34, 0x34, 0x35, 0x37, 0x31, 0x40400284, 0x40400776, 0x34, 0x33, 0x35, 0x30, 0x40400928, 0x40400468, 0x35, 0x33, 0x31, 0x39, 0x31, 0x30, 0x34, 0x38, 0x34, 0x38, 0x31, 0x30, 0x30, 0x35, 0x33, 0x37, 0x30, 0x36, 0x404008bc, 0x4080059d, 0x40800781, 0x31, 0x40400559, 0x37, 0x4040031b, 0x35, 0x404007ec, 0x4040040c, 0x36, 0x33, 0x408007dc, 0x34, 0x40400971, 0x4080034e, 0x408003f5, 0x38, 0x4080052d, 0x40800887, 0x39, 0x40400187, 0x39, 0x31, 0x404008ce, 0x38, 0x31, 0x34, 0x36, 0x37, 0x35, 0x31, 0x4040062b, 0x31, 0x32, 0x33, 0x39, 0x40c001a9, 0x39, 0x30, 0x37, 0x31, 0x38, 0x36, 0x34, 0x39, 0x34, 0x32, 0x33, 0x31, 0x39, 0x36, 0x31, 0x35, 0x36, 0x404001ec, 0x404006bc, 0x39, 0x35, 0x40400926, 0x40400469, 0x4040011b, 0x36, 0x30, 0x33, 0x38, 0x40400a25, 0x4040016f, 0x40400384, 0x36, 0x32, 0x4040045a, 0x35, 0x4040084c, 0x36, 0x33, 0x38, 0x39, 0x33, 0x37, 0x37, 0x38, 0x37, 0x404008c5, 0x404000f8, 0x39, 0x37, 0x39, 0x32, 0x30, 0x37, 0x37, 0x33, 0x404005d7, 0x32, 0x31, 0x38, 0x32, 0x35, 0x36, 0x404007df, 0x36, 0x36, 0x404006d6, 0x34, 0x32, 0x4080067e, 0x36, 0x404006e6, 0x34, 0x34, 0x40400024, 0x35, 0x34, 0x39, 0x32, 0x30, 0x32, 0x36, 0x30, 0x35, 0x40400ab3, 0x408003e4, 0x32, 0x30, 0x31, 0x34, 0x39, 0x404004d2, 0x38, 0x35, 0x30, 0x37, 0x33, 0x40400599, 0x36, 0x36, 0x36, 0x30, 0x40400194, 0x32, 0x34, 0x33, 0x34, 0x30, 0x40400087, 0x30, 0x4040076b, 0x38, 0x36, 0x33, 0x40400956, 0x404007e4, 0x4040042b, 0x40400174, 0x35, 0x37, 0x39, 0x36, 0x32, 0x36, 0x38, 0x35, 0x36, 0x40400140, 0x35, 0x30, 0x38, 0x40400523, 0x35, 0x38, 0x37, 0x39, 0x36, 0x39, 0x39, 0x40400711, 0x35, 0x37, 0x34, 0x40400a18, 0x38, 0x34, 0x30, 0x404008b3, 0x31, 0x34, 0x35, 0x39, 0x31, 0x4040078c, 0x37, 0x30, 0x40400234, 0x30, 0x31, 0x40400be7, 0x31, 0x32, 0x40400c74, 0x30, 0x404003c3, 0x33, 0x39, 0x40400b2a, 0x40400112, 0x37, 0x31, 0x35, 0x404003b0, 0x34, 0x32, 0x30, 0x40800bf2, 0x39, 0x40400bc2, 0x30, 0x37, 0x40400341, 0x40400795, 0x40400aaf, 0x40400c62, 0x32, 0x31, 0x40400960, 0x32, 0x35, 0x31, 0x4040057b, 0x40400944, 0x39, 0x32, 0x404001b2, 0x38, 0x32, 0x36, 0x40400b66, 0x32, 0x40400278, 0x33, 0x32, 0x31, 0x35, 0x37, 0x39, 0x31, 0x39, 0x38, 0x34, 0x31, 0x34, 0x4080087b, 0x39, 0x31, 0x36, 0x34, 0x408006e8, 0x39, 0x40800b58, 0x404008db, 0x37, 0x32, 0x32, 0x40400321, 0x35, 0x404008a4, 0x40400141, 0x39, 0x31, 0x30, 0x404000bc, 0x40400c5b, 0x35, 0x32, 0x38, 0x30, 0x31, 0x37, 0x40400231, 0x37, 0x31, 0x32, 0x40400914, 0x38, 0x33, 0x32, 0x40400373, 0x31, 0x40400589, 0x30, 0x39, 0x33, 0x35, 0x33, 0x39, 0x36, 0x35, 0x37, 0x4040064b, 0x31, 0x30, 0x38, 0x33, 0x40400069, 0x35, 0x31, 0x4040077a, 0x40400d5a, 0x31, 0x34, 0x34, 0x34, 0x32, 0x31, 0x30, 0x30, 0x40400202, 0x30, 0x33, 0x4040019c, 0x31, 0x31, 0x30, 0x33, 0x40400c81, 0x40400009, 0x40400026, 0x40c00602, 0x35, 0x31, 0x36, 0x404005d9, 0x40800883, 0x4040092a, 0x35, 0x40800c42, 0x38, 0x35, 0x31, 0x37, 0x31, 0x34, 0x33, 0x37, 0x40400605, 0x4040006d, 0x31, 0x35, 0x35, 0x36, 0x35, 0x30, 0x38, 0x38, 0x404003b9, 0x39, 0x38, 0x39, 0x38, 0x35, 0x39, 0x39, 0x38, 0x32, 0x33, 0x38, 0x404001cf, 0x404009ba, 0x33, 0x4040016c, 0x4040043e, 0x404009c3, 0x38, 0x40800e05, 0x33, 0x32, 0x40400107, 0x35, 0x40400305, 0x33, 0x404001ca, 0x39, 0x4040041b, 0x39, 0x38, 0x4040087d, 0x34, 0x40400cb8, 0x37, 0x4040064b, 0x30, 0x37, 0x404000e5, 0x34, 0x38, 0x31, 0x34, 0x31, 0x40400539, 0x38, 0x35, 0x39, 0x34, 0x36, 0x31, 0x40400bc9, 0x38, 0x30}, - }, - { - input: "testdata/huffman-rand-1k.in", - want: "testdata/huffman-rand-1k.%s.expect", - wantNoInput: "testdata/huffman-rand-1k.%s.expect-noinput", - tokens: []token{0xf8, 0x8b, 0x96, 0x76, 0x48, 0xd, 0x85, 0x94, 0x25, 0x80, 0xaf, 0xc2, 0xfe, 0x8d, 0xe8, 0x20, 0xeb, 0x17, 0x86, 0xc9, 0xb7, 0xc5, 0xde, 0x6, 0xea, 0x7d, 0x18, 0x8b, 0xe7, 0x3e, 0x7, 0xda, 0xdf, 0xff, 0x6c, 0x73, 0xde, 0xcc, 0xe7, 0x6d, 0x8d, 0x4, 0x19, 0x49, 0x7f, 0x47, 0x1f, 0x48, 0x15, 0xb0, 0xe8, 0x9e, 0xf2, 0x31, 0x59, 0xde, 0x34, 0xb4, 0x5b, 0xe5, 0xe0, 0x9, 0x11, 0x30, 0xc2, 0x88, 0x5b, 0x7c, 0x5d, 0x14, 0x13, 0x6f, 0x23, 0xa9, 0xd, 0xbc, 0x2d, 0x23, 0xbe, 0xd9, 0xed, 0x75, 0x4, 0x6c, 0x99, 0xdf, 0xfd, 0x70, 0x66, 0xe6, 0xee, 0xd9, 0xb1, 0x9e, 0x6e, 0x83, 0x59, 0xd5, 0xd4, 0x80, 0x59, 0x98, 0x77, 0x89, 0x43, 0x38, 0xc9, 0xaf, 0x30, 0x32, 0x9a, 0x20, 0x1b, 0x46, 0x3d, 0x67, 0x6e, 0xd7, 0x72, 0x9e, 0x4e, 0x21, 0x4f, 0xc6, 0xe0, 0xd4, 0x7b, 0x4, 0x8d, 0xa5, 0x3, 0xf6, 0x5, 0x9b, 0x6b, 0xdc, 0x2a, 0x93, 0x77, 0x28, 0xfd, 0xb4, 0x62, 0xda, 0x20, 0xe7, 0x1f, 0xab, 0x6b, 0x51, 0x43, 0x39, 0x2f, 0xa0, 0x92, 0x1, 0x6c, 0x75, 0x3e, 0xf4, 0x35, 0xfd, 0x43, 0x2e, 0xf7, 0xa4, 0x75, 0xda, 0xea, 0x9b, 0xa, 0x64, 0xb, 0xe0, 0x23, 0x29, 0xbd, 0xf7, 0xe7, 0x83, 0x3c, 0xfb, 0xdf, 0xb3, 0xae, 0x4f, 0xa4, 0x47, 0x55, 0x99, 0xde, 0x2f, 0x96, 0x6e, 0x1c, 0x43, 0x4c, 0x87, 0xe2, 0x7c, 0xd9, 0x5f, 0x4c, 0x7c, 0xe8, 0x90, 0x3, 0xdb, 0x30, 0x95, 0xd6, 0x22, 0xc, 0x47, 0xb8, 0x4d, 0x6b, 0xbd, 0x24, 0x11, 0xab, 0x2c, 0xd7, 0xbe, 0x6e, 0x7a, 0xd6, 0x8, 0xa3, 0x98, 0xd8, 0xdd, 0x15, 0x6a, 0xfa, 0x93, 0x30, 0x1, 0x25, 0x1d, 0xa2, 0x74, 0x86, 0x4b, 0x6a, 0x95, 0xe8, 0xe1, 0x4e, 0xe, 0x76, 0xb9, 0x49, 0xa9, 0x5f, 0xa0, 0xa6, 0x63, 0x3c, 0x7e, 0x7e, 0x20, 0x13, 0x4f, 0xbb, 0x66, 0x92, 0xb8, 0x2e, 0xa4, 0xfa, 0x48, 0xcb, 0xae, 0xb9, 0x3c, 0xaf, 0xd3, 0x1f, 0xe1, 0xd5, 0x8d, 0x42, 0x6d, 0xf0, 0xfc, 0x8c, 0xc, 0x0, 0xde, 0x40, 0xab, 0x8b, 0x47, 0x97, 0x4e, 0xa8, 0xcf, 0x8e, 0xdb, 0xa6, 0x8b, 0x20, 0x9, 0x84, 0x7a, 0x66, 0xe5, 0x98, 0x29, 0x2, 0x95, 0xe6, 0x38, 0x32, 0x60, 0x3, 0xe3, 0x9a, 0x1e, 0x54, 0xe8, 0x63, 0x80, 0x48, 0x9c, 0xe7, 0x63, 0x33, 0x6e, 0xa0, 0x65, 0x83, 0xfa, 0xc6, 0xba, 0x7a, 0x43, 0x71, 0x5, 0xf5, 0x68, 0x69, 0x85, 0x9c, 0xba, 0x45, 0xcd, 0x6b, 0xb, 0x19, 0xd1, 0xbb, 0x7f, 0x70, 0x85, 0x92, 0xd1, 0xb4, 0x64, 0x82, 0xb1, 0xe4, 0x62, 0xc5, 0x3c, 0x46, 0x1f, 0x92, 0x31, 0x1c, 0x4e, 0x41, 0x77, 0xf7, 0xe7, 0x87, 0xa2, 0xf, 0x6e, 0xe8, 0x92, 0x3, 0x6b, 0xa, 0xe7, 0xa9, 0x3b, 0x11, 0xda, 0x66, 0x8a, 0x29, 0xda, 0x79, 0xe1, 0x64, 0x8d, 0xe3, 0x54, 0xd4, 0xf5, 0xef, 0x64, 0x87, 0x3b, 0xf4, 0xc2, 0xf4, 0x71, 0x13, 0xa9, 0xe9, 0xe0, 0xa2, 0x6, 0x14, 0xab, 0x5d, 0xa7, 0x96, 0x0, 0xd6, 0xc3, 0xcc, 0x57, 0xed, 0x39, 0x6a, 0x25, 0xcd, 0x76, 0xea, 0xba, 0x3a, 0xf2, 0xa1, 0x95, 0x5d, 0xe5, 0x71, 0xcf, 0x9c, 0x62, 0x9e, 0x6a, 0xfa, 0xd5, 0x31, 0xd1, 0xa8, 0x66, 0x30, 0x33, 0xaa, 0x51, 0x17, 0x13, 0x82, 0x99, 0xc8, 0x14, 0x60, 0x9f, 0x4d, 0x32, 0x6d, 0xda, 0x19, 0x26, 0x21, 0xdc, 0x7e, 0x2e, 0x25, 0x67, 0x72, 0xca, 0xf, 0x92, 0xcd, 0xf6, 0xd6, 0xcb, 0x97, 0x8a, 0x33, 0x58, 0x73, 0x70, 0x91, 0x1d, 0xbf, 0x28, 0x23, 0xa3, 0xc, 0xf1, 0x83, 0xc3, 0xc8, 0x56, 0x77, 0x68, 0xe3, 0x82, 0xba, 0xb9, 0x57, 0x56, 0x57, 0x9c, 0xc3, 0xd6, 0x14, 0x5, 0x3c, 0xb1, 0xaf, 0x93, 0xc8, 0x8a, 0x57, 0x7f, 0x53, 0xfa, 0x2f, 0xaa, 0x6e, 0x66, 0x83, 0xfa, 0x33, 0xd1, 0x21, 0xab, 0x1b, 0x71, 0xb4, 0x7c, 0xda, 0xfd, 0xfb, 0x7f, 0x20, 0xab, 0x5e, 0xd5, 0xca, 0xfd, 0xdd, 0xe0, 0xee, 0xda, 0xba, 0xa8, 0x27, 0x99, 0x97, 0x69, 0xc1, 0x3c, 0x82, 0x8c, 0xa, 0x5c, 0x2d, 0x5b, 0x88, 0x3e, 0x34, 0x35, 0x86, 0x37, 0x46, 0x79, 0xe1, 0xaa, 0x19, 0xfb, 0xaa, 0xde, 0x15, 0x9, 0xd, 0x1a, 0x57, 0xff, 0xb5, 0xf, 0xf3, 0x2b, 0x5a, 0x6a, 0x4d, 0x19, 0x77, 0x71, 0x45, 0xdf, 0x4f, 0xb3, 0xec, 0xf1, 0xeb, 0x18, 0x53, 0x3e, 0x3b, 0x47, 0x8, 0x9a, 0x73, 0xa0, 0x5c, 0x8c, 0x5f, 0xeb, 0xf, 0x3a, 0xc2, 0x43, 0x67, 0xb4, 0x66, 0x67, 0x80, 0x58, 0xe, 0xc1, 0xec, 0x40, 0xd4, 0x22, 0x94, 0xca, 0xf9, 0xe8, 0x92, 0xe4, 0x69, 0x38, 0xbe, 0x67, 0x64, 0xca, 0x50, 0xc7, 0x6, 0x67, 0x42, 0x6e, 0xa3, 0xf0, 0xb7, 0x6c, 0xf2, 0xe8, 0x5f, 0xb1, 0xaf, 0xe7, 0xdb, 0xbb, 0x77, 0xb5, 0xf8, 0xcb, 0x8, 0xc4, 0x75, 0x7e, 0xc0, 0xf9, 0x1c, 0x7f, 0x3c, 0x89, 0x2f, 0xd2, 0x58, 0x3a, 0xe2, 0xf8, 0x91, 0xb6, 0x7b, 0x24, 0x27, 0xe9, 0xae, 0x84, 0x8b, 0xde, 0x74, 0xac, 0xfd, 0xd9, 0xb7, 0x69, 0x2a, 0xec, 0x32, 0x6f, 0xf0, 0x92, 0x84, 0xf1, 0x40, 0xc, 0x8a, 0xbc, 0x39, 0x6e, 0x2e, 0x73, 0xd4, 0x6e, 0x8a, 0x74, 0x2a, 0xdc, 0x60, 0x1f, 0xa3, 0x7, 0xde, 0x75, 0x8b, 0x74, 0xc8, 0xfe, 0x63, 0x75, 0xf6, 0x3d, 0x63, 0xac, 0x33, 0x89, 0xc3, 0xf0, 0xf8, 0x2d, 0x6b, 0xb4, 0x9e, 0x74, 0x8b, 0x5c, 0x33, 0xb4, 0xca, 0xa8, 0xe4, 0x99, 0xb6, 0x90, 0xa1, 0xef, 0xf, 0xd3, 0x61, 0xb2, 0xc6, 0x1a, 0x94, 0x7c, 0x44, 0x55, 0xf4, 0x45, 0xff, 0x9e, 0xa5, 0x5a, 0xc6, 0xa0, 0xe8, 0x2a, 0xc1, 0x8d, 0x6f, 0x34, 0x11, 0xb9, 0xbe, 0x4e, 0xd9, 0x87, 0x97, 0x73, 0xcf, 0x3d, 0x23, 0xae, 0xd5, 0x1a, 0x5e, 0xae, 0x5d, 0x6a, 0x3, 0xf9, 0x22, 0xd, 0x10, 0xd9, 0x47, 0x69, 0x15, 0x3f, 0xee, 0x52, 0xa3, 0x8, 0xd2, 0x3c, 0x51, 0xf4, 0xf8, 0x9d, 0xe4, 0x98, 0x89, 0xc8, 0x67, 0x39, 0xd5, 0x5e, 0x35, 0x78, 0x27, 0xe8, 0x3c, 0x80, 0xae, 0x79, 0x71, 0xd2, 0x93, 0xf4, 0xaa, 0x51, 0x12, 0x1c, 0x4b, 0x1b, 0xe5, 0x6e, 0x15, 0x6f, 0xe4, 0xbb, 0x51, 0x9b, 0x45, 0x9f, 0xf9, 0xc4, 0x8c, 0x2a, 0xfb, 0x1a, 0xdf, 0x55, 0xd3, 0x48, 0x93, 0x27, 0x1, 0x26, 0xc2, 0x6b, 0x55, 0x6d, 0xa2, 0xfb, 0x84, 0x8b, 0xc9, 0x9e, 0x28, 0xc2, 0xef, 0x1a, 0x24, 0xec, 0x9b, 0xae, 0xbd, 0x60, 0xe9, 0x15, 0x35, 0xee, 0x42, 0xa4, 0x33, 0x5b, 0xfa, 0xf, 0xb6, 0xf7, 0x1, 0xa6, 0x2, 0x4c, 0xca, 0x90, 0x58, 0x3a, 0x96, 0x41, 0xe7, 0xcb, 0x9, 0x8c, 0xdb, 0x85, 0x4d, 0xa8, 0x89, 0xf3, 0xb5, 0x8e, 0xfd, 0x75, 0x5b, 0x4f, 0xed, 0xde, 0x3f, 0xeb, 0x38, 0xa3, 0xbe, 0xb0, 0x73, 0xfc, 0xb8, 0x54, 0xf7, 0x4c, 0x30, 0x67, 0x2e, 0x38, 0xa2, 0x54, 0x18, 0xba, 0x8, 0xbf, 0xf2, 0x39, 0xd5, 0xfe, 0xa5, 0x41, 0xc6, 0x66, 0x66, 0xba, 0x81, 0xef, 0x67, 0xe4, 0xe6, 0x3c, 0xc, 0xca, 0xa4, 0xa, 0x79, 0xb3, 0x57, 0x8b, 0x8a, 0x75, 0x98, 0x18, 0x42, 0x2f, 0x29, 0xa3, 0x82, 0xef, 0x9f, 0x86, 0x6, 0x23, 0xe1, 0x75, 0xfa, 0x8, 0xb1, 0xde, 0x17, 0x4a}, - }, - { - input: "testdata/huffman-rand-limit.in", - want: "testdata/huffman-rand-limit.%s.expect", - wantNoInput: "testdata/huffman-rand-limit.%s.expect-noinput", - tokens: []token{0x61, 0x51c00000, 0xa, 0xf8, 0x8b, 0x96, 0x76, 0x48, 0xa, 0x85, 0x94, 0x25, 0x80, 0xaf, 0xc2, 0xfe, 0x8d, 0xe8, 0x20, 0xeb, 0x17, 0x86, 0xc9, 0xb7, 0xc5, 0xde, 0x6, 0xea, 0x7d, 0x18, 0x8b, 0xe7, 0x3e, 0x7, 0xda, 0xdf, 0xff, 0x6c, 0x73, 0xde, 0xcc, 0xe7, 0x6d, 0x8d, 0x4, 0x19, 0x49, 0x7f, 0x47, 0x1f, 0x48, 0x15, 0xb0, 0xe8, 0x9e, 0xf2, 0x31, 0x59, 0xde, 0x34, 0xb4, 0x5b, 0xe5, 0xe0, 0x9, 0x11, 0x30, 0xc2, 0x88, 0x5b, 0x7c, 0x5d, 0x14, 0x13, 0x6f, 0x23, 0xa9, 0xa, 0xbc, 0x2d, 0x23, 0xbe, 0xd9, 0xed, 0x75, 0x4, 0x6c, 0x99, 0xdf, 0xfd, 0x70, 0x66, 0xe6, 0xee, 0xd9, 0xb1, 0x9e, 0x6e, 0x83, 0x59, 0xd5, 0xd4, 0x80, 0x59, 0x98, 0x77, 0x89, 0x43, 0x38, 0xc9, 0xaf, 0x30, 0x32, 0x9a, 0x20, 0x1b, 0x46, 0x3d, 0x67, 0x6e, 0xd7, 0x72, 0x9e, 0x4e, 0x21, 0x4f, 0xc6, 0xe0, 0xd4, 0x7b, 0x4, 0x8d, 0xa5, 0x3, 0xf6, 0x5, 0x9b, 0x6b, 0xdc, 0x2a, 0x93, 0x77, 0x28, 0xfd, 0xb4, 0x62, 0xda, 0x20, 0xe7, 0x1f, 0xab, 0x6b, 0x51, 0x43, 0x39, 0x2f, 0xa0, 0x92, 0x1, 0x6c, 0x75, 0x3e, 0xf4, 0x35, 0xfd, 0x43, 0x2e, 0xf7, 0xa4, 0x75, 0xda, 0xea, 0x9b, 0xa}, - }, - { - input: "testdata/huffman-shifts.in", - want: "testdata/huffman-shifts.%s.expect", - wantNoInput: "testdata/huffman-shifts.%s.expect-noinput", - tokens: []token{0x31, 0x30, 0x7fc00001, 0x7fc00001, 0x7fc00001, 0x7fc00001, 0x7fc00001, 0x7fc00001, 0x7fc00001, 0x7fc00001, 0x7fc00001, 0x7fc00001, 0x7fc00001, 0x7fc00001, 0x7fc00001, 0x7fc00001, 0x7fc00001, 0x52400001, 0xd, 0xa, 0x32, 0x33, 0x7fc00001, 0x7fc00001, 0x7fc00001, 0x7fc00001, 0x7fc00001, 0x7fc00001, 0x7fc00001, 0x7fc00001, 0x7fc00001, 0x7f400001}, - }, - { - input: "testdata/huffman-text-shift.in", - want: "testdata/huffman-text-shift.%s.expect", - wantNoInput: "testdata/huffman-text-shift.%s.expect-noinput", - tokens: []token{0x2f, 0x2f, 0x43, 0x6f, 0x70, 0x79, 0x72, 0x69, 0x67, 0x68, 0x74, 0x32, 0x30, 0x30, 0x39, 0x54, 0x68, 0x47, 0x6f, 0x41, 0x75, 0x74, 0x68, 0x6f, 0x72, 0x2e, 0x41, 0x6c, 0x6c, 0x40800016, 0x72, 0x72, 0x76, 0x64, 0x2e, 0xd, 0xa, 0x2f, 0x2f, 0x55, 0x6f, 0x66, 0x74, 0x68, 0x69, 0x6f, 0x75, 0x72, 0x63, 0x63, 0x6f, 0x64, 0x69, 0x67, 0x6f, 0x76, 0x72, 0x6e, 0x64, 0x62, 0x79, 0x42, 0x53, 0x44, 0x2d, 0x74, 0x79, 0x6c, 0x40400020, 0x6c, 0x69, 0x63, 0x6e, 0x74, 0x68, 0x74, 0x63, 0x6e, 0x62, 0x66, 0x6f, 0x75, 0x6e, 0x64, 0x69, 0x6e, 0x74, 0x68, 0x4c, 0x49, 0x43, 0x45, 0x4e, 0x53, 0x45, 0x66, 0x69, 0x6c, 0x2e, 0xd, 0xa, 0xd, 0xa, 0x70, 0x63, 0x6b, 0x67, 0x6d, 0x69, 0x6e, 0x4040000a, 0x69, 0x6d, 0x70, 0x6f, 0x72, 0x74, 0x22, 0x6f, 0x22, 0x4040000c, 0x66, 0x75, 0x6e, 0x63, 0x6d, 0x69, 0x6e, 0x28, 0x29, 0x7b, 0xd, 0xa, 0x9, 0x76, 0x72, 0x62, 0x3d, 0x6d, 0x6b, 0x28, 0x5b, 0x5d, 0x62, 0x79, 0x74, 0x2c, 0x36, 0x35, 0x35, 0x33, 0x35, 0x29, 0xd, 0xa, 0x9, 0x66, 0x2c, 0x5f, 0x3a, 0x3d, 0x6f, 0x2e, 0x43, 0x72, 0x74, 0x28, 0x22, 0x68, 0x75, 0x66, 0x66, 0x6d, 0x6e, 0x2d, 0x6e, 0x75, 0x6c, 0x6c, 0x2d, 0x6d, 0x78, 0x2e, 0x69, 0x6e, 0x22, 0x40800021, 0x2e, 0x57, 0x72, 0x69, 0x74, 0x28, 0x62, 0x29, 0xd, 0xa, 0x7d, 0xd, 0xa, 0x41, 0x42, 0x43, 0x44, 0x45, 0x46, 0x47, 0x48, 0x49, 0x4a, 0x4b, 0x4c, 0x4d, 0x4e, 0x4f, 0x50, 0x51, 0x52, 0x53, 0x54, 0x55, 0x56, 0x58, 0x78, 0x79, 0x7a, 0x21, 0x22, 0x23, 0xc2, 0xa4, 0x25, 0x26, 0x2f, 0x3f, 0x22}, - }, - { - input: "testdata/huffman-text.in", - want: "testdata/huffman-text.%s.expect", - wantNoInput: "testdata/huffman-text.%s.expect-noinput", - tokens: []token{0x2f, 0x2f, 0x20, 0x43, 0x6f, 0x70, 0x79, 0x72, 0x69, 0x67, 0x68, 0x74, 0x20, 0x32, 0x30, 0x30, 0x39, 0x20, 0x54, 0x68, 0x65, 0x20, 0x47, 0x6f, 0x20, 0x41, 0x75, 0x74, 0x68, 0x6f, 0x72, 0x73, 0x2e, 0x20, 0x41, 0x6c, 0x6c, 0x20, 0x4080001e, 0x73, 0x20, 0x72, 0x65, 0x73, 0x65, 0x72, 0x76, 0x65, 0x64, 0x2e, 0xd, 0xa, 0x2f, 0x2f, 0x20, 0x55, 0x73, 0x65, 0x20, 0x6f, 0x66, 0x20, 0x74, 0x68, 0x69, 0x73, 0x20, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x20, 0x63, 0x6f, 0x64, 0x65, 0x20, 0x69, 0x73, 0x20, 0x67, 0x6f, 0x76, 0x65, 0x72, 0x6e, 0x65, 0x64, 0x20, 0x62, 0x79, 0x20, 0x61, 0x20, 0x42, 0x53, 0x44, 0x2d, 0x73, 0x74, 0x79, 0x6c, 0x65, 0x40800036, 0x6c, 0x69, 0x63, 0x65, 0x6e, 0x73, 0x65, 0x20, 0x74, 0x68, 0x61, 0x74, 0x20, 0x63, 0x61, 0x6e, 0x20, 0x62, 0x65, 0x20, 0x66, 0x6f, 0x75, 0x6e, 0x64, 0x20, 0x69, 0x6e, 0x20, 0x74, 0x68, 0x65, 0x20, 0x4c, 0x49, 0x43, 0x45, 0x4e, 0x53, 0x45, 0x20, 0x66, 0x69, 0x6c, 0x65, 0x2e, 0xd, 0xa, 0xd, 0xa, 0x70, 0x61, 0x63, 0x6b, 0x61, 0x67, 0x65, 0x20, 0x6d, 0x61, 0x69, 0x6e, 0x4040000f, 0x69, 0x6d, 0x70, 0x6f, 0x72, 0x74, 0x20, 0x22, 0x6f, 0x73, 0x22, 0x4040000e, 0x66, 0x75, 0x6e, 0x63, 0x4080001b, 0x28, 0x29, 0x20, 0x7b, 0xd, 0xa, 0x9, 0x76, 0x61, 0x72, 0x20, 0x62, 0x20, 0x3d, 0x20, 0x6d, 0x61, 0x6b, 0x65, 0x28, 0x5b, 0x5d, 0x62, 0x79, 0x74, 0x65, 0x2c, 0x20, 0x36, 0x35, 0x35, 0x33, 0x35, 0x29, 0xd, 0xa, 0x9, 0x66, 0x2c, 0x20, 0x5f, 0x20, 0x3a, 0x3d, 0x20, 0x6f, 0x73, 0x2e, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x28, 0x22, 0x68, 0x75, 0x66, 0x66, 0x6d, 0x61, 0x6e, 0x2d, 0x6e, 0x75, 0x6c, 0x6c, 0x2d, 0x6d, 0x61, 0x78, 0x2e, 0x69, 0x6e, 0x22, 0x4080002a, 0x2e, 0x57, 0x72, 0x69, 0x74, 0x65, 0x28, 0x62, 0x29, 0xd, 0xa, 0x7d, 0xd, 0xa}, - }, - { - input: "testdata/huffman-zero.in", - want: "testdata/huffman-zero.%s.expect", - wantNoInput: "testdata/huffman-zero.%s.expect-noinput", - tokens: []token{0x30, ml, 0x4b800000}, - }, - { - input: "", - want: "", - wantNoInput: "testdata/null-long-match.%s.expect-noinput", - tokens: []token{0x0, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, 0x41400000}, - }, -} - -// TestWriteBlock tests if the writeBlock encoding has changed. -// To update the reference files use the "-update" flag on the test. -func TestWriteBlock(t *testing.T) { - for _, test := range writeBlockTests { - testBlock(t, test, "wb") - } -} - -// TestWriteBlockDynamic tests if the writeBlockDynamic encoding has changed. -// To update the reference files use the "-update" flag on the test. -func TestWriteBlockDynamic(t *testing.T) { - for _, test := range writeBlockTests { - testBlock(t, test, "dyn") - } -} - -// TestWriteBlockDynamic tests if the writeBlockDynamic encoding has changed. -// To update the reference files use the "-update" flag on the test. -func TestWriteBlockDynamicSync(t *testing.T) { - for _, test := range writeBlockTests { - testBlock(t, test, "sync") - } -} - -// testBlock tests a block against its references, -// or regenerate the references, if "-update" flag is set. -func testBlock(t *testing.T, test huffTest, ttype string) { - if test.want != "" { - test.want = fmt.Sprintf(test.want, ttype) - } - const gotSuffix = ".got" - test.wantNoInput = fmt.Sprintf(test.wantNoInput, ttype) - tokens := indexTokens(test.tokens) - if *update { - if test.input != "" { - t.Logf("Updating %q", test.want) - input, err := os.ReadFile(test.input) - if err != nil { - t.Error(err) - return - } - - f, err := os.Create(test.want) - if err != nil { - t.Error(err) - return - } - defer f.Close() - bw := newHuffmanBitWriter(f) - writeToType(t, ttype, bw, tokens, input) - } - - t.Logf("Updating %q", test.wantNoInput) - f, err := os.Create(test.wantNoInput) - if err != nil { - t.Error(err) - return - } - defer f.Close() - bw := newHuffmanBitWriter(f) - writeToType(t, ttype, bw, tokens, nil) - return - } - - if test.input != "" { - t.Logf("Testing %q", test.want) - input, err := os.ReadFile(test.input) - if err != nil { - t.Error(err) - return - } - want, err := os.ReadFile(test.want) - if err != nil { - t.Error(err) - return - } - var buf bytes.Buffer - bw := newHuffmanBitWriter(&buf) - writeToType(t, ttype, bw, tokens, input) - - got := buf.Bytes() - if !bytes.Equal(got, want) { - t.Errorf("writeBlock did not yield expected result for file %q with input. See %q", test.want, test.want+gotSuffix) - if err := os.WriteFile(test.want+gotSuffix, got, 0o666); err != nil { - t.Error(err) - } - } - t.Log("Output ok") - - // Test if the writer produces the same output after reset. - buf.Reset() - bw.reset(&buf) - writeToType(t, ttype, bw, tokens, input) - bw.flush() - got = buf.Bytes() - if !bytes.Equal(got, want) { - t.Errorf("reset: writeBlock did not yield expected result for file %q with input. See %q", test.want, test.want+".reset"+gotSuffix) - if err := os.WriteFile(test.want+".reset"+gotSuffix, got, 0o666); err != nil { - t.Error(err) - } - return - } - t.Log("Reset ok") - testWriterEOF(t, "wb", test, true) - } - t.Logf("Testing %q", test.wantNoInput) - wantNI, err := os.ReadFile(test.wantNoInput) - if err != nil { - t.Error(err) - return - } - var buf bytes.Buffer - bw := newHuffmanBitWriter(&buf) - writeToType(t, ttype, bw, tokens, nil) - - got := buf.Bytes() - if !bytes.Equal(got, wantNI) { - t.Errorf("writeBlock did not yield expected result for file %q with input. See %q", test.wantNoInput, test.wantNoInput+gotSuffix) - if err := os.WriteFile(test.wantNoInput+gotSuffix, got, 0o666); err != nil { - t.Error(err) - } - } else if got[0]&1 == 1 { - t.Error("got unexpected EOF") - return - } - - t.Log("Output ok") - - // Test if the writer produces the same output after reset. - buf.Reset() - bw.reset(&buf) - writeToType(t, ttype, bw, tokens, nil) - bw.flush() - got = buf.Bytes() - if !bytes.Equal(got, wantNI) { - t.Errorf("reset: writeBlock did not yield expected result for file %q without input. See %q", test.wantNoInput, test.wantNoInput+".reset"+gotSuffix) - if err := os.WriteFile(test.wantNoInput+".reset"+gotSuffix, got, 0o666); err != nil { - t.Error(err) - } - return - } - t.Log("Reset ok") - testWriterEOF(t, "wb", test, false) -} - -func writeToType(t *testing.T, ttype string, bw *huffmanBitWriter, tok tokens, input []byte) { - switch ttype { - case "wb": - bw.writeBlock(&tok, false, input) - case "dyn": - bw.writeBlockDynamic(&tok, false, input, false) - case "sync": - bw.writeBlockDynamic(&tok, false, input, true) - default: - panic("unknown test type") - } - - if bw.err != nil { - t.Error(bw.err) - return - } - - bw.flush() - if bw.err != nil { - t.Error(bw.err) - return - } -} - -// testWriterEOF tests if the written block contains an EOF marker. -func testWriterEOF(t *testing.T, ttype string, test huffTest, useInput bool) { - if useInput && test.input == "" { - return - } - var input []byte - if useInput { - var err error - input, err = os.ReadFile(test.input) - if err != nil { - t.Error(err) - return - } - } - var buf bytes.Buffer - bw := newHuffmanBitWriter(&buf) - tokens := indexTokens(test.tokens) - switch ttype { - case "wb": - bw.writeBlock(&tokens, true, input) - case "dyn": - bw.writeBlockDynamic(&tokens, true, input, true) - case "huff": - bw.writeBlockHuff(true, input, true) - default: - panic("unknown test type") - } - if bw.err != nil { - t.Error(bw.err) - return - } - - bw.flush() - if bw.err != nil { - t.Error(bw.err) - return - } - b := buf.Bytes() - if len(b) == 0 { - t.Error("no output received") - return - } - if b[0]&1 != 1 { - t.Errorf("block not marked with EOF for input %q", test.input) - return - } - t.Log("EOF ok") -} diff --git a/internal/compress/flate/huffman_code.go b/internal/compress/flate/huffman_code.go deleted file mode 100644 index 42da87e8..00000000 --- a/internal/compress/flate/huffman_code.go +++ /dev/null @@ -1,419 +0,0 @@ -// Copyright 2009 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package flate - -import ( - "math" - "math/bits" -) - -const ( - maxBitsLimit = 16 - // number of valid literals - literalCount = 286 -) - -// hcode is a huffman code with a bit code and bit length. -type hcode uint32 - -func (h hcode) len() uint8 { - return uint8(h) -} - -func (h hcode) code64() uint64 { - return uint64(h >> 8) -} - -func (h hcode) zero() bool { - return h == 0 -} - -type huffmanEncoder struct { - codes []hcode - bitCount [17]int32 - - // Allocate a reusable buffer with the longest possible frequency table. - // Possible lengths are codegenCodeCount, offsetCodeCount and literalCount. - // The largest of these is literalCount, so we allocate for that case. - freqcache [literalCount + 1]literalNode -} - -type literalNode struct { - literal uint16 - freq uint16 -} - -// A levelInfo describes the state of the constructed tree for a given depth. -type levelInfo struct { - // Our level. for better printing - level int32 - - // The frequency of the last node at this level - lastFreq int32 - - // The frequency of the next character to add to this level - nextCharFreq int32 - - // The frequency of the next pair (from level below) to add to this level. - // Only valid if the "needed" value of the next lower level is 0. - nextPairFreq int32 - - // The number of chains remaining to generate for this level before moving - // up to the next level - needed int32 -} - -// set sets the code and length of an hcode. -func (h *hcode) set(code uint16, length uint8) { - *h = hcode(length) | (hcode(code) << 8) -} - -func newhcode(code uint16, length uint8) hcode { - return hcode(length) | (hcode(code) << 8) -} - -func reverseBits(number uint16, bitLength byte) uint16 { - return bits.Reverse16(number << ((16 - bitLength) & 15)) -} - -func maxNode() literalNode { return literalNode{math.MaxUint16, math.MaxUint16} } - -func newHuffmanEncoder(size int) *huffmanEncoder { - // Make capacity to next power of two. - c := uint(bits.Len32(uint32(size - 1))) - return &huffmanEncoder{codes: make([]hcode, size, 1<<c)} -} - -// Generates a HuffmanCode corresponding to the fixed literal table -func generateFixedLiteralEncoding() *huffmanEncoder { - h := newHuffmanEncoder(literalCount) - codes := h.codes - var ch uint16 - for ch = range uint16(literalCount) { - var bits uint16 - var size uint8 - switch { - case ch < 144: - // size 8, 000110000 .. 10111111 - bits = ch + 48 - size = 8 - case ch < 256: - // size 9, 110010000 .. 111111111 - bits = ch + 400 - 144 - size = 9 - case ch < 280: - // size 7, 0000000 .. 0010111 - bits = ch - 256 - size = 7 - default: - // size 8, 11000000 .. 11000111 - bits = ch + 192 - 280 - size = 8 - } - codes[ch] = newhcode(reverseBits(bits, size), size) - } - return h -} - -func generateFixedOffsetEncoding() *huffmanEncoder { - h := newHuffmanEncoder(30) - codes := h.codes - for ch := range codes { - codes[ch] = newhcode(reverseBits(uint16(ch), 5), 5) - } - return h -} - -var ( - fixedLiteralEncoding = generateFixedLiteralEncoding() - fixedOffsetEncoding = generateFixedOffsetEncoding() -) - -func (h *huffmanEncoder) bitLength(freq []uint16) int { - var total int - for i, f := range freq { - if f != 0 { - total += int(f) * int(h.codes[i].len()) - } - } - return total -} - -func (h *huffmanEncoder) bitLengthRaw(b []byte) int { - var total int - for _, f := range b { - total += int(h.codes[f].len()) - } - return total -} - -// canReuseBits returns the number of bits or math.MaxInt32 if the encoder cannot be reused. -func (h *huffmanEncoder) canReuseBits(freq []uint16) int { - var total int - for i, f := range freq { - if f != 0 { - code := h.codes[i] - if code.zero() { - return math.MaxInt32 - } - total += int(f) * int(code.len()) - } - } - return total -} - -// Return the number of literals assigned to each bit size in the Huffman encoding -// -// This method is only called when list.length >= 3 -// The cases of 0, 1, and 2 literals are handled by special case code. -// -// list An array of the literals with non-zero frequencies -// -// and their associated frequencies. The array is in order of increasing -// frequency, and has as its last element a special element with frequency -// MaxInt32 -// -// maxBits The maximum number of bits that should be used to encode any literal. -// -// Must be less than 16. -// -// return An integer array in which array[i] indicates the number of literals -// -// that should be encoded in i bits. -func (h *huffmanEncoder) bitCounts(list []literalNode, maxBits int32) []int32 { - if maxBits >= maxBitsLimit { - panic("flate: maxBits too large") - } - n := int32(len(list)) - list = list[0 : n+1] - list[n] = maxNode() - - // The tree can't have greater depth than n - 1, no matter what. This - // saves a little bit of work in some small cases - if maxBits > n-1 { - maxBits = n - 1 - } - - // Create information about each of the levels. - // A bogus "Level 0" whose sole purpose is so that - // level1.prev.needed==0. This makes level1.nextPairFreq - // be a legitimate value that never gets chosen. - var levels [maxBitsLimit]levelInfo - // leafCounts[i] counts the number of literals at the left - // of ancestors of the rightmost node at level i. - // leafCounts[i][j] is the number of literals at the left - // of the level j ancestor. - var leafCounts [maxBitsLimit][maxBitsLimit]int32 - - // Descending to only have 1 bounds check. - l2f := int32(list[2].freq) - l1f := int32(list[1].freq) - l0f := int32(list[0].freq) + int32(list[1].freq) - - for level := int32(1); level <= maxBits; level++ { - // For every level, the first two items are the first two characters. - // We initialize the levels as if we had already figured this out. - levels[level] = levelInfo{ - level: level, - lastFreq: l1f, - nextCharFreq: l2f, - nextPairFreq: l0f, - } - leafCounts[level][level] = 2 - if level == 1 { - levels[level].nextPairFreq = math.MaxInt32 - } - } - - // We need a total of 2*n - 2 items at top level and have already generated 2. - levels[maxBits].needed = 2*n - 4 - - level := uint32(maxBits) - for level < 16 { - l := &levels[level] - if l.nextPairFreq == math.MaxInt32 && l.nextCharFreq == math.MaxInt32 { - // We've run out of both leafs and pairs. - // End all calculations for this level. - // To make sure we never come back to this level or any lower level, - // set nextPairFreq impossibly large. - l.needed = 0 - levels[level+1].nextPairFreq = math.MaxInt32 - level++ - continue - } - - prevFreq := l.lastFreq - if l.nextCharFreq < l.nextPairFreq { - // The next item on this row is a leaf node. - n := leafCounts[level][level] + 1 - l.lastFreq = l.nextCharFreq - // Lower leafCounts are the same of the previous node. - leafCounts[level][level] = n - e := list[n] - if e.literal < math.MaxUint16 { - l.nextCharFreq = int32(e.freq) - } else { - l.nextCharFreq = math.MaxInt32 - } - } else { - // The next item on this row is a pair from the previous row. - // nextPairFreq isn't valid until we generate two - // more values in the level below - l.lastFreq = l.nextPairFreq - // Take leaf counts from the lower level, except counts[level] remains the same. - if true { - save := leafCounts[level][level] - leafCounts[level] = leafCounts[level-1] - leafCounts[level][level] = save - } else { - copy(leafCounts[level][:level], leafCounts[level-1][:level]) - } - levels[l.level-1].needed = 2 - } - - if l.needed--; l.needed == 0 { - // We've done everything we need to do for this level. - // Continue calculating one level up. Fill in nextPairFreq - // of that level with the sum of the two nodes we've just calculated on - // this level. - if l.level == maxBits { - // All done! - break - } - levels[l.level+1].nextPairFreq = prevFreq + l.lastFreq - level++ - } else { - // If we stole from below, move down temporarily to replenish it. - for levels[level-1].needed > 0 { - level-- - } - } - } - - // Somethings is wrong if at the end, the top level is null or hasn't used - // all of the leaves. - if leafCounts[maxBits][maxBits] != n { - panic("leafCounts[maxBits][maxBits] != n") - } - - bitCount := h.bitCount[:maxBits+1] - bits := 1 - counts := &leafCounts[maxBits] - for level := maxBits; level > 0; level-- { - // chain.leafCount gives the number of literals requiring at least "bits" - // bits to encode. - bitCount[bits] = counts[level] - counts[level-1] - bits++ - } - return bitCount -} - -// Look at the leaves and assign them a bit count and an encoding as specified -// in RFC 1951 3.2.2 -func (h *huffmanEncoder) assignEncodingAndSize(bitCount []int32, list []literalNode) { - code := uint16(0) - for n, bits := range bitCount { - code <<= 1 - if n == 0 || bits == 0 { - continue - } - // The literals list[len(list)-bits] .. list[len(list)-bits] - // are encoded using "bits" bits, and get the values - // code, code + 1, .... The code values are - // assigned in literal order (not frequency order). - chunk := list[len(list)-int(bits):] - - sortByLiteral(chunk) - for _, node := range chunk { - h.codes[node.literal] = newhcode(reverseBits(code, uint8(n)), uint8(n)) - code++ - } - list = list[0 : len(list)-int(bits)] - } -} - -// Update this Huffman Code object to be the minimum code for the specified frequency count. -// -// freq An array of frequencies, in which frequency[i] gives the frequency of literal i. -// maxBits The maximum number of bits to use for any literal. -func (h *huffmanEncoder) generate(freq []uint16, maxBits int32) { - list := h.freqcache[:len(freq)+1] - codes := h.codes[:len(freq)] - // Number of non-zero literals - count := 0 - // Set list to be the set of all non-zero literals and their frequencies - for i, f := range freq { - if f != 0 { - list[count] = literalNode{uint16(i), f} - count++ - } else { - codes[i] = 0 - } - } - list[count] = literalNode{} - - list = list[:count] - if count <= 2 { - // Handle the small cases here, because they are awkward for the general case code. With - // two or fewer literals, everything has bit length 1. - for i, node := range list { - // "list" is in order of increasing literal value. - h.codes[node.literal].set(uint16(i), 1) - } - return - } - sortByFreq(list) - - // Get the number of literals for each bit count - bitCount := h.bitCounts(list, maxBits) - // And do the assignment - h.assignEncodingAndSize(bitCount, list) -} - -// atLeastOne clamps the result between 1 and 15. -func atLeastOne(v float32) float32 { - if v < 1 { - return 1 - } - if v > 15 { - return 15 - } - return v -} - -func histogram(b []byte, h []uint16) { - if true && len(b) >= 8<<10 { - // Split for bigger inputs - histogramSplit(b, h) - } else { - h = h[:256] - for _, t := range b { - h[t]++ - } - } -} - -func histogramSplit(b []byte, h []uint16) { - // Tested, and slightly faster than 2-way. - // Writing to separate arrays and combining is also slightly slower. - h = h[:256] - for len(b)&3 != 0 { - h[b[0]]++ - b = b[1:] - } - n := len(b) / 4 - x, y, z, w := b[:n], b[n:], b[n+n:], b[n+n+n:] - y, z, w = y[:len(x)], z[:len(x)], w[:len(x)] - for i, t := range x { - v0 := &h[t] - v1 := &h[y[i]] - v3 := &h[w[i]] - v2 := &h[z[i]] - *v0++ - *v1++ - *v2++ - *v3++ - } -} diff --git a/internal/compress/flate/huffman_sortByFreq.go b/internal/compress/flate/huffman_sortByFreq.go deleted file mode 100644 index 6c05ba8c..00000000 --- a/internal/compress/flate/huffman_sortByFreq.go +++ /dev/null @@ -1,159 +0,0 @@ -// Copyright 2009 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package flate - -// Sort sorts data. -// It makes one call to data.Len to determine n, and O(n*log(n)) calls to -// data.Less and data.Swap. The sort is not guaranteed to be stable. -func sortByFreq(data []literalNode) { - n := len(data) - quickSortByFreq(data, 0, n, maxDepth(n)) -} - -func quickSortByFreq(data []literalNode, a, b, maxDepth int) { - for b-a > 12 { // Use ShellSort for slices <= 12 elements - if maxDepth == 0 { - heapSort(data, a, b) - return - } - maxDepth-- - mlo, mhi := doPivotByFreq(data, a, b) - // Avoiding recursion on the larger subproblem guarantees - // a stack depth of at most lg(b-a). - if mlo-a < b-mhi { - quickSortByFreq(data, a, mlo, maxDepth) - a = mhi // i.e., quickSortByFreq(data, mhi, b) - } else { - quickSortByFreq(data, mhi, b, maxDepth) - b = mlo // i.e., quickSortByFreq(data, a, mlo) - } - } - if b-a > 1 { - // Do ShellSort pass with gap 6 - // It could be written in this simplified form cause b-a <= 12 - for i := a + 6; i < b; i++ { - if data[i].freq == data[i-6].freq && data[i].literal < data[i-6].literal || data[i].freq < data[i-6].freq { - data[i], data[i-6] = data[i-6], data[i] - } - } - insertionSortByFreq(data, a, b) - } -} - -func doPivotByFreq(data []literalNode, lo, hi int) (midlo, midhi int) { - m := int(uint(lo+hi) >> 1) // Written like this to avoid integer overflow. - if hi-lo > 40 { - // Tukey's ``Ninther,'' median of three medians of three. - s := (hi - lo) / 8 - medianOfThreeSortByFreq(data, lo, lo+s, lo+2*s) - medianOfThreeSortByFreq(data, m, m-s, m+s) - medianOfThreeSortByFreq(data, hi-1, hi-1-s, hi-1-2*s) - } - medianOfThreeSortByFreq(data, lo, m, hi-1) - - // Invariants are: - // data[lo] = pivot (set up by ChoosePivot) - // data[lo < i < a] < pivot - // data[a <= i < b] <= pivot - // data[b <= i < c] unexamined - // data[c <= i < hi-1] > pivot - // data[hi-1] >= pivot - pivot := lo - a, c := lo+1, hi-1 - - for ; a < c && (data[a].freq == data[pivot].freq && data[a].literal < data[pivot].literal || data[a].freq < data[pivot].freq); a++ { - } - b := a - for { - for ; b < c && (data[pivot].freq == data[b].freq && data[pivot].literal > data[b].literal || data[pivot].freq > data[b].freq); b++ { // data[b] <= pivot - } - for ; b < c && (data[pivot].freq == data[c-1].freq && data[pivot].literal < data[c-1].literal || data[pivot].freq < data[c-1].freq); c-- { // data[c-1] > pivot - } - if b >= c { - break - } - // data[b] > pivot; data[c-1] <= pivot - data[b], data[c-1] = data[c-1], data[b] - b++ - c-- - } - // If hi-c<3 then there are duplicates (by property of median of nine). - // Let's be a bit more conservative, and set border to 5. - protect := hi-c < 5 - if !protect && hi-c < (hi-lo)/4 { - // Lets test some points for equality to pivot - dups := 0 - if data[pivot].freq == data[hi-1].freq && data[pivot].literal > data[hi-1].literal || data[pivot].freq > data[hi-1].freq { // data[hi-1] = pivot - data[c], data[hi-1] = data[hi-1], data[c] - c++ - dups++ - } - if data[b-1].freq == data[pivot].freq && data[b-1].literal > data[pivot].literal || data[b-1].freq > data[pivot].freq { // data[b-1] = pivot - b-- - dups++ - } - // m-lo = (hi-lo)/2 > 6 - // b-lo > (hi-lo)*3/4-1 > 8 - // ==> m < b ==> data[m] <= pivot - if data[m].freq == data[pivot].freq && data[m].literal > data[pivot].literal || data[m].freq > data[pivot].freq { // data[m] = pivot - data[m], data[b-1] = data[b-1], data[m] - b-- - dups++ - } - // if at least 2 points are equal to pivot, assume skewed distribution - protect = dups > 1 - } - if protect { - // Protect against a lot of duplicates - // Add invariant: - // data[a <= i < b] unexamined - // data[b <= i < c] = pivot - for { - for ; a < b && (data[b-1].freq == data[pivot].freq && data[b-1].literal > data[pivot].literal || data[b-1].freq > data[pivot].freq); b-- { // data[b] == pivot - } - for ; a < b && (data[a].freq == data[pivot].freq && data[a].literal < data[pivot].literal || data[a].freq < data[pivot].freq); a++ { // data[a] < pivot - } - if a >= b { - break - } - // data[a] == pivot; data[b-1] < pivot - data[a], data[b-1] = data[b-1], data[a] - a++ - b-- - } - } - // Swap pivot into middle - data[pivot], data[b-1] = data[b-1], data[pivot] - return b - 1, c -} - -// Insertion sort -func insertionSortByFreq(data []literalNode, a, b int) { - for i := a + 1; i < b; i++ { - for j := i; j > a && (data[j].freq == data[j-1].freq && data[j].literal < data[j-1].literal || data[j].freq < data[j-1].freq); j-- { - data[j], data[j-1] = data[j-1], data[j] - } - } -} - -// quickSortByFreq, loosely following Bentley and McIlroy, -// ``Engineering a Sort Function,'' SP&E November 1993. - -// medianOfThreeSortByFreq moves the median of the three values data[m0], data[m1], data[m2] into data[m1]. -func medianOfThreeSortByFreq(data []literalNode, m1, m0, m2 int) { - // sort 3 elements - if data[m1].freq == data[m0].freq && data[m1].literal < data[m0].literal || data[m1].freq < data[m0].freq { - data[m1], data[m0] = data[m0], data[m1] - } - // data[m0] <= data[m1] - if data[m2].freq == data[m1].freq && data[m2].literal < data[m1].literal || data[m2].freq < data[m1].freq { - data[m2], data[m1] = data[m1], data[m2] - // data[m0] <= data[m2] && data[m1] < data[m2] - if data[m1].freq == data[m0].freq && data[m1].literal < data[m0].literal || data[m1].freq < data[m0].freq { - data[m1], data[m0] = data[m0], data[m1] - } - } - // now data[m0] <= data[m1] <= data[m2] -} diff --git a/internal/compress/flate/huffman_sortByLiteral.go b/internal/compress/flate/huffman_sortByLiteral.go deleted file mode 100644 index f6d0a404..00000000 --- a/internal/compress/flate/huffman_sortByLiteral.go +++ /dev/null @@ -1,203 +0,0 @@ -// Copyright 2009 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package flate - -// Sort sorts data. -// It makes one call to data.Len to determine n, and O(n*log(n)) calls to -// data.Less and data.Swap. The sort is not guaranteed to be stable. -func sortByLiteral(data []literalNode) { - n := len(data) - quickSort(data, 0, n, maxDepth(n)) -} - -func quickSort(data []literalNode, a, b, maxDepth int) { - for b-a > 12 { // Use ShellSort for slices <= 12 elements - if maxDepth == 0 { - heapSort(data, a, b) - return - } - maxDepth-- - mlo, mhi := doPivot(data, a, b) - // Avoiding recursion on the larger subproblem guarantees - // a stack depth of at most lg(b-a). - if mlo-a < b-mhi { - quickSort(data, a, mlo, maxDepth) - a = mhi // i.e., quickSort(data, mhi, b) - } else { - quickSort(data, mhi, b, maxDepth) - b = mlo // i.e., quickSort(data, a, mlo) - } - } - if b-a > 1 { - // Do ShellSort pass with gap 6 - // It could be written in this simplified form cause b-a <= 12 - for i := a + 6; i < b; i++ { - if data[i].literal < data[i-6].literal { - data[i], data[i-6] = data[i-6], data[i] - } - } - insertionSort(data, a, b) - } -} - -func heapSort(data []literalNode, a, b int) { - first := a - lo := 0 - hi := b - a - - // Build heap with greatest element at top. - for i := (hi - 1) / 2; i >= 0; i-- { - siftDown(data, i, hi, first) - } - - // Pop elements, largest first, into end of data. - for i := hi - 1; i >= 0; i-- { - data[first], data[first+i] = data[first+i], data[first] - siftDown(data, lo, i, first) - } -} - -// siftDown implements the heap property on data[lo, hi). -// first is an offset into the array where the root of the heap lies. -func siftDown(data []literalNode, lo, hi, first int) { - root := lo - for { - child := 2*root + 1 - if child >= hi { - break - } - if child+1 < hi && data[first+child].literal < data[first+child+1].literal { - child++ - } - if data[first+root].literal > data[first+child].literal { - return - } - data[first+root], data[first+child] = data[first+child], data[first+root] - root = child - } -} - -func doPivot(data []literalNode, lo, hi int) (midlo, midhi int) { - m := int(uint(lo+hi) >> 1) // Written like this to avoid integer overflow. - if hi-lo > 40 { - // Tukey's ``Ninther,'' median of three medians of three. - s := (hi - lo) / 8 - medianOfThree(data, lo, lo+s, lo+2*s) - medianOfThree(data, m, m-s, m+s) - medianOfThree(data, hi-1, hi-1-s, hi-1-2*s) - } - medianOfThree(data, lo, m, hi-1) - - // Invariants are: - // data[lo] = pivot (set up by ChoosePivot) - // data[lo < i < a] < pivot - // data[a <= i < b] <= pivot - // data[b <= i < c] unexamined - // data[c <= i < hi-1] > pivot - // data[hi-1] >= pivot - pivot := lo - a, c := lo+1, hi-1 - - for ; a < c && data[a].literal < data[pivot].literal; a++ { - } - b := a - for { - for ; b < c && data[pivot].literal > data[b].literal; b++ { // data[b] <= pivot - } - for ; b < c && data[pivot].literal < data[c-1].literal; c-- { // data[c-1] > pivot - } - if b >= c { - break - } - // data[b] > pivot; data[c-1] <= pivot - data[b], data[c-1] = data[c-1], data[b] - b++ - c-- - } - // If hi-c<3 then there are duplicates (by property of median of nine). - // Let's be a bit more conservative, and set border to 5. - protect := hi-c < 5 - if !protect && hi-c < (hi-lo)/4 { - // Lets test some points for equality to pivot - dups := 0 - if data[pivot].literal > data[hi-1].literal { // data[hi-1] = pivot - data[c], data[hi-1] = data[hi-1], data[c] - c++ - dups++ - } - if data[b-1].literal > data[pivot].literal { // data[b-1] = pivot - b-- - dups++ - } - // m-lo = (hi-lo)/2 > 6 - // b-lo > (hi-lo)*3/4-1 > 8 - // ==> m < b ==> data[m] <= pivot - if data[m].literal > data[pivot].literal { // data[m] = pivot - data[m], data[b-1] = data[b-1], data[m] - b-- - dups++ - } - // if at least 2 points are equal to pivot, assume skewed distribution - protect = dups > 1 - } - if protect { - // Protect against a lot of duplicates - // Add invariant: - // data[a <= i < b] unexamined - // data[b <= i < c] = pivot - for { - for ; a < b && data[b-1].literal > data[pivot].literal; b-- { // data[b] == pivot - } - for ; a < b && data[a].literal < data[pivot].literal; a++ { // data[a] < pivot - } - if a >= b { - break - } - // data[a] == pivot; data[b-1] < pivot - data[a], data[b-1] = data[b-1], data[a] - a++ - b-- - } - } - // Swap pivot into middle - data[pivot], data[b-1] = data[b-1], data[pivot] - return b - 1, c -} - -// Insertion sort -func insertionSort(data []literalNode, a, b int) { - for i := a + 1; i < b; i++ { - for j := i; j > a && data[j].literal < data[j-1].literal; j-- { - data[j], data[j-1] = data[j-1], data[j] - } - } -} - -// maxDepth returns a threshold at which quicksort should switch -// to heapsort. It returns 2*ceil(lg(n+1)). -func maxDepth(n int) int { - var depth int - for i := n; i > 0; i >>= 1 { - depth++ - } - return depth * 2 -} - -// medianOfThree moves the median of the three values data[m0], data[m1], data[m2] into data[m1]. -func medianOfThree(data []literalNode, m1, m0, m2 int) { - // sort 3 elements - if data[m1].literal < data[m0].literal { - data[m1], data[m0] = data[m0], data[m1] - } - // data[m0] <= data[m1] - if data[m2].literal < data[m1].literal { - data[m2], data[m1] = data[m1], data[m2] - // data[m0] <= data[m2] && data[m1] < data[m2] - if data[m1].literal < data[m0].literal { - data[m1], data[m0] = data[m0], data[m1] - } - } - // now data[m0] <= data[m1] <= data[m2] -} diff --git a/internal/compress/flate/inflate.go b/internal/compress/flate/inflate.go deleted file mode 100644 index f12f1e77..00000000 --- a/internal/compress/flate/inflate.go +++ /dev/null @@ -1,867 +0,0 @@ -// Copyright 2009 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -// Package flate implements the DEFLATE compressed data format, described in -// RFC 1951. The gzip and zlib packages implement access to DEFLATE-based file -// formats. -package flate - -import ( - "bufio" - "compress/flate" - "fmt" - "io" - "math/bits" - "sync" -) - -const ( - maxCodeLen = 16 // max length of Huffman code - maxCodeLenMask = 15 // mask for max length of Huffman code - // The next three numbers come from the RFC section 3.2.7, with the - // additional proviso in section 3.2.5 which implies that distance codes - // 30 and 31 should never occur in compressed data. - maxNumLit = 286 - maxNumDist = 30 - numCodes = 19 // number of codes in Huffman meta-code - - debugDecode = false -) - -// Value of length - 3 and extra bits. -type lengthExtra struct { - length, extra uint8 -} - -var decCodeToLen = [32]lengthExtra{{length: 0x0, extra: 0x0}, {length: 0x1, extra: 0x0}, {length: 0x2, extra: 0x0}, {length: 0x3, extra: 0x0}, {length: 0x4, extra: 0x0}, {length: 0x5, extra: 0x0}, {length: 0x6, extra: 0x0}, {length: 0x7, extra: 0x0}, {length: 0x8, extra: 0x1}, {length: 0xa, extra: 0x1}, {length: 0xc, extra: 0x1}, {length: 0xe, extra: 0x1}, {length: 0x10, extra: 0x2}, {length: 0x14, extra: 0x2}, {length: 0x18, extra: 0x2}, {length: 0x1c, extra: 0x2}, {length: 0x20, extra: 0x3}, {length: 0x28, extra: 0x3}, {length: 0x30, extra: 0x3}, {length: 0x38, extra: 0x3}, {length: 0x40, extra: 0x4}, {length: 0x50, extra: 0x4}, {length: 0x60, extra: 0x4}, {length: 0x70, extra: 0x4}, {length: 0x80, extra: 0x5}, {length: 0xa0, extra: 0x5}, {length: 0xc0, extra: 0x5}, {length: 0xe0, extra: 0x5}, {length: 0xff, extra: 0x0}, {length: 0x0, extra: 0x0}, {length: 0x0, extra: 0x0}, {length: 0x0, extra: 0x0}} - -var bitMask32 = [32]uint32{ - 0, 1, 3, 7, 0xF, 0x1F, 0x3F, 0x7F, 0xFF, - 0x1FF, 0x3FF, 0x7FF, 0xFFF, 0x1FFF, 0x3FFF, 0x7FFF, 0xFFFF, - 0x1ffff, 0x3ffff, 0x7FFFF, 0xfFFFF, 0x1fFFFF, 0x3fFFFF, 0x7fFFFF, 0xffFFFF, - 0x1ffFFFF, 0x3ffFFFF, 0x7ffFFFF, 0xfffFFFF, 0x1fffFFFF, 0x3fffFFFF, 0x7fffFFFF, -} // up to 32 bits - -// Initialize the fixedHuffmanDecoder only once upon first use. -var ( - fixedOnce sync.Once - fixedHuffmanDecoder huffmanDecoder -) - -// A CorruptInputError reports the presence of corrupt input at a given offset. -type CorruptInputError = flate.CorruptInputError - -// An InternalError reports an error in the flate code itself. -type InternalError string - -func (e InternalError) Error() string { return "flate: internal error: " + string(e) } - -// A ReadError reports an error encountered while reading input. -// -// Deprecated: No longer returned. -type ReadError = flate.ReadError - -// A WriteError reports an error encountered while writing output. -// -// Deprecated: No longer returned. -type WriteError = flate.WriteError - -// Resetter resets a ReadCloser returned by NewReader or NewReaderDict to -// to switch to a new underlying Reader. This permits reusing a ReadCloser -// instead of allocating a new one. -type Resetter interface { - // Reset discards any buffered data and resets the Resetter as if it was - // newly initialized with the given reader. - Reset(r io.Reader, dict []byte) error -} - -// The data structure for decoding Huffman tables is based on that of -// zlib. There is a lookup table of a fixed bit width (huffmanChunkBits), -// For codes smaller than the table width, there are multiple entries -// (each combination of trailing bits has the same value). For codes -// larger than the table width, the table contains a link to an overflow -// table. The width of each entry in the link table is the maximum code -// size minus the chunk width. -// -// Note that you can do a lookup in the table even without all bits -// filled. Since the extra bits are zero, and the DEFLATE Huffman codes -// have the property that shorter codes come before longer ones, the -// bit length estimate in the result is a lower bound on the actual -// number of bits. -// -// See the following: -// http://www.gzip.org/algorithm.txt - -// chunk & 15 is number of bits -// chunk >> 4 is value, including table link - -const ( - huffmanChunkBits = 9 - huffmanNumChunks = 1 << huffmanChunkBits - huffmanCountMask = 15 - huffmanValueShift = 4 -) - -type huffmanDecoder struct { - maxRead int // the maximum number of bits we can read and not overread - chunks *[huffmanNumChunks]uint16 // chunks as described above - links [][]uint16 // overflow links - linkMask uint32 // mask the width of the link table -} - -// Initialize Huffman decoding tables from array of code lengths. -// Following this function, h is guaranteed to be initialized into a complete -// tree (i.e., neither over-subscribed nor under-subscribed). The exception is a -// degenerate case where the tree has only a single symbol with length 1. Empty -// trees are permitted. -func (h *huffmanDecoder) init(lengths []int) bool { - // Sanity enables additional runtime tests during Huffman - // table construction. It's intended to be used during - // development to supplement the currently ad-hoc unit tests. - const sanity = false - - if h.chunks == nil { - h.chunks = new([huffmanNumChunks]uint16) - } - - if h.maxRead != 0 { - *h = huffmanDecoder{chunks: h.chunks, links: h.links} - } - - // Count number of codes of each length, - // compute maxRead and max length. - var count [maxCodeLen]int - var min, max int - for _, n := range lengths { - if n == 0 { - continue - } - if min == 0 || n < min { - min = n - } - if n > max { - max = n - } - count[n&maxCodeLenMask]++ - } - - // Empty tree. The decompressor.huffSym function will fail later if the tree - // is used. Technically, an empty tree is only valid for the HDIST tree and - // not the HCLEN and HLIT tree. However, a stream with an empty HCLEN tree - // is guaranteed to fail since it will attempt to use the tree to decode the - // codes for the HLIT and HDIST trees. Similarly, an empty HLIT tree is - // guaranteed to fail later since the compressed data section must be - // composed of at least one symbol (the end-of-block marker). - if max == 0 { - return true - } - - code := 0 - var nextcode [maxCodeLen]int - for i := min; i <= max; i++ { - code <<= 1 - nextcode[i&maxCodeLenMask] = code - code += count[i&maxCodeLenMask] - } - - // Check that the coding is complete (i.e., that we've - // assigned all 2-to-the-max possible bit sequences). - // Exception: To be compatible with zlib, we also need to - // accept degenerate single-code codings. See also - // TestDegenerateHuffmanCoding. - if code != 1<<uint(max) && !(code == 1 && max == 1) { - if debugDecode { - fmt.Println("coding failed, code, max:", code, max, code == 1<<uint(max), code == 1 && max == 1, "(one should be true)") - } - return false - } - - h.maxRead = min - - chunks := h.chunks[:] - for i := range chunks { - chunks[i] = 0 - } - - if max > huffmanChunkBits { - numLinks := 1 << (uint(max) - huffmanChunkBits) - h.linkMask = uint32(numLinks - 1) - - // create link tables - link := nextcode[huffmanChunkBits+1] >> 1 - if cap(h.links) < huffmanNumChunks-link { - h.links = make([][]uint16, huffmanNumChunks-link) - } else { - h.links = h.links[:huffmanNumChunks-link] - } - for j := uint(link); j < huffmanNumChunks; j++ { - reverse := int(bits.Reverse16(uint16(j))) - reverse >>= uint(16 - huffmanChunkBits) - off := j - uint(link) - if sanity && h.chunks[reverse] != 0 { - panic("impossible: overwriting existing chunk") - } - h.chunks[reverse] = uint16(off<<huffmanValueShift | (huffmanChunkBits + 1)) - if cap(h.links[off]) < numLinks { - h.links[off] = make([]uint16, numLinks) - } else { - h.links[off] = h.links[off][:numLinks] - } - } - } else { - h.links = h.links[:0] - } - - for i, n := range lengths { - if n == 0 { - continue - } - code := nextcode[n] - nextcode[n]++ - chunk := uint16(i<<huffmanValueShift | n) - reverse := int(bits.Reverse16(uint16(code))) - reverse >>= uint(16 - n) - if n <= huffmanChunkBits { - for off := reverse; off < len(h.chunks); off += 1 << uint(n) { - // We should never need to overwrite - // an existing chunk. Also, 0 is - // never a valid chunk, because the - // lower 4 "count" bits should be - // between 1 and 15. - if sanity && h.chunks[off] != 0 { - panic("impossible: overwriting existing chunk") - } - h.chunks[off] = chunk - } - } else { - j := reverse & (huffmanNumChunks - 1) - if sanity && h.chunks[j]&huffmanCountMask != huffmanChunkBits+1 { - // Longer codes should have been - // associated with a link table above. - panic("impossible: not an indirect chunk") - } - value := h.chunks[j] >> huffmanValueShift - linktab := h.links[value] - reverse >>= huffmanChunkBits - for off := reverse; off < len(linktab); off += 1 << uint(n-huffmanChunkBits) { - if sanity && linktab[off] != 0 { - panic("impossible: overwriting existing chunk") - } - linktab[off] = chunk - } - } - } - - if sanity { - // Above we've sanity checked that we never overwrote - // an existing entry. Here we additionally check that - // we filled the tables completely. - for i, chunk := range h.chunks { - if chunk == 0 { - // As an exception, in the degenerate - // single-code case, we allow odd - // chunks to be missing. - if code == 1 && i%2 == 1 { - continue - } - panic("impossible: missing chunk") - } - } - for _, linktab := range h.links { - for _, chunk := range linktab { - if chunk == 0 { - panic("impossible: missing chunk") - } - } - } - } - - return true -} - -// Reader is the actual read interface needed by NewReader. -// If the passed in io.Reader does not also have ReadByte, -// the NewReader will introduce its own buffering. -type Reader interface { - io.Reader - io.ByteReader -} - -type step uint8 - -const ( - copyData step = iota + 1 - nextBlock - huffmanBytesBuffer - huffmanBytesReader - huffmanBufioReader - huffmanStringsReader - huffmanGenericReader -) - -// flushMode tells decompressor when to return data -type flushMode uint8 - -const ( - syncFlush flushMode = iota // return data after sync flush block - partialFlush // return data after each block -) - -// Decompress state. -type decompressor struct { - // Input source. - r Reader - roffset int64 - - // Huffman decoders for literal/length, distance. - h1, h2 huffmanDecoder - - // Length arrays used to define Huffman codes. - bits *[maxNumLit + maxNumDist]int - codebits *[numCodes]int - - // Output history, buffer. - dict dictDecoder - - // Next step in the decompression, - // and decompression state. - step step - stepState int - err error - toRead []byte - hl, hd *huffmanDecoder - copyLen int - copyDist int - - // Temporary buffer (avoids repeated allocation). - buf [4]byte - - // Input bits, in top of b. - b uint32 - - nb uint - final bool - - flushMode flushMode -} - -func (f *decompressor) nextBlock() { - for f.nb < 1+2 { - if f.err = f.moreBits(); f.err != nil { - return - } - } - f.final = f.b&1 == 1 - f.b >>= 1 - typ := f.b & 3 - f.b >>= 2 - f.nb -= 1 + 2 - switch typ { - case 0: - f.dataBlock() - if debugDecode { - fmt.Println("stored block") - } - case 1: - // compressed, fixed Huffman tables - f.hl = &fixedHuffmanDecoder - f.hd = nil - f.huffmanBlockDecoder() - if debugDecode { - fmt.Println("predefinied huffman block") - } - case 2: - // compressed, dynamic Huffman tables - if f.err = f.readHuffman(); f.err != nil { - break - } - f.hl = &f.h1 - f.hd = &f.h2 - f.huffmanBlockDecoder() - if debugDecode { - fmt.Println("dynamic huffman block") - } - default: - // 3 is reserved. - if debugDecode { - fmt.Println("reserved data block encountered") - } - f.err = CorruptInputError(f.roffset) - } -} - -func (f *decompressor) Read(b []byte) (int, error) { - for { - if len(f.toRead) > 0 { - n := copy(b, f.toRead) - f.toRead = f.toRead[n:] - if len(f.toRead) == 0 { - return n, f.err - } - return n, nil - } - if f.err != nil { - return 0, f.err - } - - f.doStep() - - if f.err != nil && len(f.toRead) == 0 { - f.toRead = f.dict.readFlush() // Flush what's left in case of error - } - } -} - -// WriteTo implements the io.WriteTo interface for io.Copy and friends. -func (f *decompressor) WriteTo(w io.Writer) (int64, error) { - total := int64(0) - flushed := false - for { - if len(f.toRead) > 0 { - n, err := w.Write(f.toRead) - total += int64(n) - if err != nil { - f.err = err - return total, err - } - if n != len(f.toRead) { - return total, io.ErrShortWrite - } - f.toRead = f.toRead[:0] - } - if f.err != nil && flushed { - if f.err == io.EOF { - return total, nil - } - return total, f.err - } - if f.err == nil { - f.doStep() - } - if len(f.toRead) == 0 && f.err != nil && !flushed { - f.toRead = f.dict.readFlush() // Flush what's left in case of error - flushed = true - } - } -} - -func (f *decompressor) Close() error { - if f.err == io.EOF { - return nil - } - return f.err -} - -// RFC 1951 section 3.2.7. -// Compression with dynamic Huffman codes - -var codeOrder = [...]int{16, 17, 18, 0, 8, 7, 9, 6, 10, 5, 11, 4, 12, 3, 13, 2, 14, 1, 15} - -func (f *decompressor) readHuffman() error { - // HLIT[5], HDIST[5], HCLEN[4]. - for f.nb < 5+5+4 { - if err := f.moreBits(); err != nil { - return err - } - } - nlit := int(f.b&0x1F) + 257 - if nlit > maxNumLit { - if debugDecode { - fmt.Println("nlit > maxNumLit", nlit) - } - return CorruptInputError(f.roffset) - } - f.b >>= 5 - ndist := int(f.b&0x1F) + 1 - if ndist > maxNumDist { - if debugDecode { - fmt.Println("ndist > maxNumDist", ndist) - } - return CorruptInputError(f.roffset) - } - f.b >>= 5 - nclen := int(f.b&0xF) + 4 - // numCodes is 19, so nclen is always valid. - f.b >>= 4 - f.nb -= 5 + 5 + 4 - - // (HCLEN+4)*3 bits: code lengths in the magic codeOrder order. - for i := range nclen { - for f.nb < 3 { - if err := f.moreBits(); err != nil { - return err - } - } - f.codebits[codeOrder[i]] = int(f.b & 0x7) - f.b >>= 3 - f.nb -= 3 - } - for i := nclen; i < len(codeOrder); i++ { - f.codebits[codeOrder[i]] = 0 - } - if !f.h1.init(f.codebits[0:]) { - if debugDecode { - fmt.Println("init codebits failed") - } - return CorruptInputError(f.roffset) - } - - // HLIT + 257 code lengths, HDIST + 1 code lengths, - // using the code length Huffman code. - for i, n := 0, nlit+ndist; i < n; { - x, err := f.huffSym(&f.h1) - if err != nil { - return err - } - if x < 16 { - // Actual length. - f.bits[i] = x - i++ - continue - } - // Repeat previous length or zero. - var rep int - var nb uint - var b int - switch x { - default: - return InternalError("unexpected length code") - case 16: - rep = 3 - nb = 2 - if i == 0 { - if debugDecode { - fmt.Println("i==0") - } - return CorruptInputError(f.roffset) - } - b = f.bits[i-1] - case 17: - rep = 3 - nb = 3 - b = 0 - case 18: - rep = 11 - nb = 7 - b = 0 - } - for f.nb < nb { - if err := f.moreBits(); err != nil { - if debugDecode { - fmt.Println("morebits:", err) - } - return err - } - } - rep += int(f.b & uint32(1<<(nb®SizeMaskUint32)-1)) - f.b >>= nb & regSizeMaskUint32 - f.nb -= nb - if i+rep > n { - if debugDecode { - fmt.Println("i+rep > n", i, rep, n) - } - return CorruptInputError(f.roffset) - } - for j := 0; j < rep; j++ { - f.bits[i] = b - i++ - } - } - - if !f.h1.init(f.bits[0:nlit]) || !f.h2.init(f.bits[nlit:nlit+ndist]) { - if debugDecode { - fmt.Println("init2 failed") - } - return CorruptInputError(f.roffset) - } - - // As an optimization, we can initialize the maxRead bits to read at a time - // for the HLIT tree to the length of the EOB marker since we know that - // every block must terminate with one. This preserves the property that - // we never read any extra bytes after the end of the DEFLATE stream. - if f.h1.maxRead < f.bits[endBlockMarker] { - f.h1.maxRead = f.bits[endBlockMarker] - } - if !f.final { - // If not the final block, the smallest block possible is - // a predefined table, BTYPE=01, with a single EOB marker. - // This will take up 3 + 7 bits. - f.h1.maxRead += 10 - } - - return nil -} - -// Copy a single uncompressed data block from input to output. -func (f *decompressor) dataBlock() { - // Uncompressed. - // Discard current half-byte. - left := (f.nb) & 7 - f.nb -= left - f.b >>= left - - offBytes := f.nb >> 3 - // Unfilled values will be overwritten. - f.buf[0] = uint8(f.b) - f.buf[1] = uint8(f.b >> 8) - f.buf[2] = uint8(f.b >> 16) - f.buf[3] = uint8(f.b >> 24) - - f.roffset += int64(offBytes) - f.nb, f.b = 0, 0 - - // Length then ones-complement of length. - nr, err := io.ReadFull(f.r, f.buf[offBytes:4]) - f.roffset += int64(nr) - if err != nil { - f.err = noEOF(err) - return - } - n := uint16(f.buf[0]) | uint16(f.buf[1])<<8 - nn := uint16(f.buf[2]) | uint16(f.buf[3])<<8 - if nn != ^n { - if debugDecode { - ncomp := ^n - fmt.Println("uint16(nn) != uint16(^n)", nn, ncomp) - } - f.err = CorruptInputError(f.roffset) - return - } - - if n == 0 { - if f.flushMode == syncFlush { - f.toRead = f.dict.readFlush() - } - - f.finishBlock() - return - } - - f.copyLen = int(n) - f.copyData() -} - -// copyData copies f.copyLen bytes from the underlying reader into f.hist. -// It pauses for reads when f.hist is full. -func (f *decompressor) copyData() { - buf := f.dict.writeSlice() - if len(buf) > f.copyLen { - buf = buf[:f.copyLen] - } - - cnt, err := io.ReadFull(f.r, buf) - f.roffset += int64(cnt) - f.copyLen -= cnt - f.dict.writeMark(cnt) - if err != nil { - f.err = noEOF(err) - return - } - - if f.dict.availWrite() == 0 || f.copyLen > 0 { - f.toRead = f.dict.readFlush() - f.step = copyData - return - } - f.finishBlock() -} - -func (f *decompressor) finishBlock() { - if f.final { - if f.dict.availRead() > 0 { - f.toRead = f.dict.readFlush() - } - - f.err = io.EOF - } else if f.flushMode == partialFlush && f.dict.availRead() > 0 { - f.toRead = f.dict.readFlush() - } - - f.step = nextBlock -} - -func (f *decompressor) doStep() { - switch f.step { - case copyData: - f.copyData() - case nextBlock: - f.nextBlock() - case huffmanBytesBuffer: - f.huffmanBytesBuffer() - case huffmanBytesReader: - f.huffmanBytesReader() - case huffmanBufioReader: - f.huffmanBufioReader() - case huffmanStringsReader: - f.huffmanStringsReader() - case huffmanGenericReader: - f.huffmanGenericReader() - default: - panic("BUG: unexpected step state") - } -} - -// noEOF returns err, unless err == io.EOF, in which case it returns io.ErrUnexpectedEOF. -func noEOF(e error) error { - if e == io.EOF { - return io.ErrUnexpectedEOF - } - return e -} - -func (f *decompressor) moreBits() error { - c, err := f.r.ReadByte() - if err != nil { - return noEOF(err) - } - f.roffset++ - f.b |= uint32(c) << (f.nb & regSizeMaskUint32) - f.nb += 8 - return nil -} - -// Read the next Huffman-encoded symbol from f according to h. -func (f *decompressor) huffSym(h *huffmanDecoder) (int, error) { - // Since a huffmanDecoder can be empty or be composed of a degenerate tree - // with single element, huffSym must error on these two edge cases. In both - // cases, the chunks slice will be 0 for the invalid sequence, leading it - // satisfy the n == 0 check below. - n := uint(h.maxRead) - // Optimization. Compiler isn't smart enough to keep f.b,f.nb in registers, - // but is smart enough to keep local variables in registers, so use nb and b, - // inline call to moreBits and reassign b,nb back to f on return. - nb, b := f.nb, f.b - for { - for nb < n { - c, err := f.r.ReadByte() - if err != nil { - f.b = b - f.nb = nb - return 0, noEOF(err) - } - f.roffset++ - b |= uint32(c) << (nb & regSizeMaskUint32) - nb += 8 - } - chunk := h.chunks[b&(huffmanNumChunks-1)] - n = uint(chunk & huffmanCountMask) - if n > huffmanChunkBits { - chunk = h.links[chunk>>huffmanValueShift][(b>>huffmanChunkBits)&h.linkMask] - n = uint(chunk & huffmanCountMask) - } - if n <= nb { - if n == 0 { - f.b = b - f.nb = nb - if debugDecode { - fmt.Println("huffsym: n==0") - } - f.err = CorruptInputError(f.roffset) - return 0, f.err - } - f.b = b >> (n & regSizeMaskUint32) - f.nb = nb - n - return int(chunk >> huffmanValueShift), nil - } - } -} - -func makeReader(r io.Reader) Reader { - if rr, ok := r.(Reader); ok { - return rr - } - return bufio.NewReader(r) -} - -func fixedHuffmanDecoderInit() { - fixedOnce.Do(func() { - // These come from the RFC section 3.2.6. - var bits [288]int - for i := range 144 { - bits[i] = 8 - } - for i := 144; i < 256; i++ { - bits[i] = 9 - } - for i := 256; i < 280; i++ { - bits[i] = 7 - } - for i := 280; i < 288; i++ { - bits[i] = 8 - } - fixedHuffmanDecoder.init(bits[:]) - }) -} - -func (f *decompressor) Reset(r io.Reader, dict []byte) error { - *f = decompressor{ - r: makeReader(r), - bits: f.bits, - codebits: f.codebits, - h1: f.h1, - h2: f.h2, - dict: f.dict, - step: nextBlock, - } - f.dict.init(maxMatchOffset, dict) - return nil -} - -type ReaderOpt func(*decompressor) - -// WithPartialBlock tells decompressor to return after each block, -// so it can read data written with partial flush -func WithPartialBlock() ReaderOpt { - return func(f *decompressor) { - f.flushMode = partialFlush - } -} - -// WithDict initializes the reader with a preset dictionary -func WithDict(dict []byte) ReaderOpt { - return func(f *decompressor) { - f.dict.init(maxMatchOffset, dict) - } -} - -// NewReaderOpts returns new reader with provided options -func NewReaderOpts(r io.Reader, opts ...ReaderOpt) io.ReadCloser { - fixedHuffmanDecoderInit() - - var f decompressor - f.r = makeReader(r) - f.bits = new([maxNumLit + maxNumDist]int) - f.codebits = new([numCodes]int) - f.step = nextBlock - f.dict.init(maxMatchOffset, nil) - - for _, opt := range opts { - opt(&f) - } - - return &f -} - -// NewReader returns a new ReadCloser that can be used -// to read the uncompressed version of r. -// If r does not also implement io.ByteReader, -// the decompressor may read more data than necessary from r. -// It is the caller's responsibility to call Close on the ReadCloser -// when finished reading. -// -// The ReadCloser returned by NewReader also implements Resetter. -func NewReader(r io.Reader) io.ReadCloser { - return NewReaderOpts(r) -} - -// NewReaderDict is like NewReader but initializes the reader -// with a preset dictionary. The returned Reader behaves as if -// the uncompressed data stream started with the given dictionary, -// which has already been read. NewReaderDict is typically used -// to read data compressed by NewWriterDict. -// -// The ReadCloser returned by NewReader also implements Resetter. -func NewReaderDict(r io.Reader, dict []byte) io.ReadCloser { - return NewReaderOpts(r, WithDict(dict)) -} diff --git a/internal/compress/flate/inflate_gen.go b/internal/compress/flate/inflate_gen.go deleted file mode 100644 index 2b2f993f..00000000 --- a/internal/compress/flate/inflate_gen.go +++ /dev/null @@ -1,1283 +0,0 @@ -// Code generated by go generate gen_inflate.go. DO NOT EDIT. - -package flate - -import ( - "bufio" - "bytes" - "fmt" - "math/bits" - "strings" -) - -// Decode a single Huffman block from f. -// hl and hd are the Huffman states for the lit/length values -// and the distance values, respectively. If hd == nil, using the -// fixed distance encoding associated with fixed Huffman blocks. -func (f *decompressor) huffmanBytesBuffer() { - const ( - stateInit = iota // Zero value must be stateInit - stateDict - ) - fr := f.r.(*bytes.Buffer) - - // Optimization. Compiler isn't smart enough to keep f.b,f.nb in registers, - // but is smart enough to keep local variables in registers, so use nb and b, - // inline call to moreBits and reassign b,nb back to f on return. - fnb, fb, dict := f.nb, f.b, &f.dict - - switch f.stepState { - case stateInit: - goto readLiteral - case stateDict: - goto copyHistory - } - -readLiteral: - // Read literal and/or (length, distance) according to RFC section 3.2.3. - { - var v int - { - // Inlined v, err := f.huffSym(f.hl) - // Since a huffmanDecoder can be empty or be composed of a degenerate tree - // with single element, huffSym must error on these two edge cases. In both - // cases, the chunks slice will be 0 for the invalid sequence, leading it - // satisfy the n == 0 check below. - n := uint(f.hl.maxRead) - for { - for fnb < n { - c, err := fr.ReadByte() - if err != nil { - f.b, f.nb = fb, fnb - f.err = noEOF(err) - return - } - f.roffset++ - fb |= uint32(c) << (fnb & regSizeMaskUint32) - fnb += 8 - } - chunk := f.hl.chunks[fb&(huffmanNumChunks-1)] - n = uint(chunk & huffmanCountMask) - if n > huffmanChunkBits { - chunk = f.hl.links[chunk>>huffmanValueShift][(fb>>huffmanChunkBits)&f.hl.linkMask] - n = uint(chunk & huffmanCountMask) - } - if n <= fnb { - if n == 0 { - f.b, f.nb = fb, fnb - if debugDecode { - fmt.Println("huffsym: n==0") - } - f.err = CorruptInputError(f.roffset) - return - } - fb = fb >> (n & regSizeMaskUint32) - fnb = fnb - n - v = int(chunk >> huffmanValueShift) - break - } - } - } - - var length int - switch { - case v < 256: - dict.writeByte(byte(v)) - if dict.availWrite() == 0 { - f.toRead = dict.readFlush() - f.step = huffmanBytesBuffer - f.stepState = stateInit - f.b, f.nb = fb, fnb - return - } - goto readLiteral - case v == 256: - f.b, f.nb = fb, fnb - f.finishBlock() - return - // otherwise, reference to older data - case v < 265: - length = v - (257 - 3) - case v < maxNumLit: - val := decCodeToLen[(v - 257)] - length = int(val.length) + 3 - n := uint(val.extra) - for fnb < n { - c, err := fr.ReadByte() - if err != nil { - f.b, f.nb = fb, fnb - if debugDecode { - fmt.Println("morebits n>0:", err) - } - f.err = err - return - } - f.roffset++ - fb |= uint32(c) << (fnb & regSizeMaskUint32) - fnb += 8 - } - length += int(fb & bitMask32[n]) - fb >>= n & regSizeMaskUint32 - fnb -= n - default: - if debugDecode { - fmt.Println(v, ">= maxNumLit") - } - f.err = CorruptInputError(f.roffset) - f.b, f.nb = fb, fnb - return - } - - var dist uint32 - if f.hd == nil { - for fnb < 5 { - c, err := fr.ReadByte() - if err != nil { - f.b, f.nb = fb, fnb - if debugDecode { - fmt.Println("morebits f.nb<5:", err) - } - f.err = err - return - } - f.roffset++ - fb |= uint32(c) << (fnb & regSizeMaskUint32) - fnb += 8 - } - dist = uint32(bits.Reverse8(uint8(fb & 0x1F << 3))) - fb >>= 5 - fnb -= 5 - } else { - // Since a huffmanDecoder can be empty or be composed of a degenerate tree - // with single element, huffSym must error on these two edge cases. In both - // cases, the chunks slice will be 0 for the invalid sequence, leading it - // satisfy the n == 0 check below. - n := uint(f.hd.maxRead) - // Optimization. Compiler isn't smart enough to keep f.b,f.nb in registers, - // but is smart enough to keep local variables in registers, so use nb and b, - // inline call to moreBits and reassign b,nb back to f on return. - for { - for fnb < n { - c, err := fr.ReadByte() - if err != nil { - f.b, f.nb = fb, fnb - f.err = noEOF(err) - return - } - f.roffset++ - fb |= uint32(c) << (fnb & regSizeMaskUint32) - fnb += 8 - } - chunk := f.hd.chunks[fb&(huffmanNumChunks-1)] - n = uint(chunk & huffmanCountMask) - if n > huffmanChunkBits { - chunk = f.hd.links[chunk>>huffmanValueShift][(fb>>huffmanChunkBits)&f.hd.linkMask] - n = uint(chunk & huffmanCountMask) - } - if n <= fnb { - if n == 0 { - f.b, f.nb = fb, fnb - if debugDecode { - fmt.Println("huffsym: n==0") - } - f.err = CorruptInputError(f.roffset) - return - } - fb = fb >> (n & regSizeMaskUint32) - fnb = fnb - n - dist = uint32(chunk >> huffmanValueShift) - break - } - } - } - - switch { - case dist < 4: - dist++ - case dist < maxNumDist: - nb := uint(dist-2) >> 1 - // have 1 bit in bottom of dist, need nb more. - extra := (dist & 1) << (nb & regSizeMaskUint32) - for fnb < nb { - c, err := fr.ReadByte() - if err != nil { - f.b, f.nb = fb, fnb - if debugDecode { - fmt.Println("morebits f.nb<nb:", err) - } - f.err = err - return - } - f.roffset++ - fb |= uint32(c) << (fnb & regSizeMaskUint32) - fnb += 8 - } - extra |= fb & bitMask32[nb] - fb >>= nb & regSizeMaskUint32 - fnb -= nb - dist = 1<<((nb+1)®SizeMaskUint32) + 1 + extra - // slower: dist = bitMask32[nb+1] + 2 + extra - default: - f.b, f.nb = fb, fnb - if debugDecode { - fmt.Println("dist too big:", dist, maxNumDist) - } - f.err = CorruptInputError(f.roffset) - return - } - - // No check on length; encoding can be prescient. - if dist > uint32(dict.histSize()) { - f.b, f.nb = fb, fnb - if debugDecode { - fmt.Println("dist > dict.histSize():", dist, dict.histSize()) - } - f.err = CorruptInputError(f.roffset) - return - } - - f.copyLen, f.copyDist = length, int(dist) - goto copyHistory - } - -copyHistory: - // Perform a backwards copy according to RFC section 3.2.3. - { - cnt := dict.tryWriteCopy(f.copyDist, f.copyLen) - if cnt == 0 { - cnt = dict.writeCopy(f.copyDist, f.copyLen) - } - f.copyLen -= cnt - - if dict.availWrite() == 0 || f.copyLen > 0 { - f.toRead = dict.readFlush() - f.step = huffmanBytesBuffer // We need to continue this work - f.stepState = stateDict - f.b, f.nb = fb, fnb - return - } - goto readLiteral - } - // Not reached -} - -// Decode a single Huffman block from f. -// hl and hd are the Huffman states for the lit/length values -// and the distance values, respectively. If hd == nil, using the -// fixed distance encoding associated with fixed Huffman blocks. -func (f *decompressor) huffmanBytesReader() { - const ( - stateInit = iota // Zero value must be stateInit - stateDict - ) - fr := f.r.(*bytes.Reader) - - // Optimization. Compiler isn't smart enough to keep f.b,f.nb in registers, - // but is smart enough to keep local variables in registers, so use nb and b, - // inline call to moreBits and reassign b,nb back to f on return. - fnb, fb, dict := f.nb, f.b, &f.dict - - switch f.stepState { - case stateInit: - goto readLiteral - case stateDict: - goto copyHistory - } - -readLiteral: - // Read literal and/or (length, distance) according to RFC section 3.2.3. - { - var v int - { - // Inlined v, err := f.huffSym(f.hl) - // Since a huffmanDecoder can be empty or be composed of a degenerate tree - // with single element, huffSym must error on these two edge cases. In both - // cases, the chunks slice will be 0 for the invalid sequence, leading it - // satisfy the n == 0 check below. - n := uint(f.hl.maxRead) - for { - for fnb < n { - c, err := fr.ReadByte() - if err != nil { - f.b, f.nb = fb, fnb - f.err = noEOF(err) - return - } - f.roffset++ - fb |= uint32(c) << (fnb & regSizeMaskUint32) - fnb += 8 - } - chunk := f.hl.chunks[fb&(huffmanNumChunks-1)] - n = uint(chunk & huffmanCountMask) - if n > huffmanChunkBits { - chunk = f.hl.links[chunk>>huffmanValueShift][(fb>>huffmanChunkBits)&f.hl.linkMask] - n = uint(chunk & huffmanCountMask) - } - if n <= fnb { - if n == 0 { - f.b, f.nb = fb, fnb - if debugDecode { - fmt.Println("huffsym: n==0") - } - f.err = CorruptInputError(f.roffset) - return - } - fb = fb >> (n & regSizeMaskUint32) - fnb = fnb - n - v = int(chunk >> huffmanValueShift) - break - } - } - } - - var length int - switch { - case v < 256: - dict.writeByte(byte(v)) - if dict.availWrite() == 0 { - f.toRead = dict.readFlush() - f.step = huffmanBytesReader - f.stepState = stateInit - f.b, f.nb = fb, fnb - return - } - goto readLiteral - case v == 256: - f.b, f.nb = fb, fnb - f.finishBlock() - return - // otherwise, reference to older data - case v < 265: - length = v - (257 - 3) - case v < maxNumLit: - val := decCodeToLen[(v - 257)] - length = int(val.length) + 3 - n := uint(val.extra) - for fnb < n { - c, err := fr.ReadByte() - if err != nil { - f.b, f.nb = fb, fnb - if debugDecode { - fmt.Println("morebits n>0:", err) - } - f.err = err - return - } - f.roffset++ - fb |= uint32(c) << (fnb & regSizeMaskUint32) - fnb += 8 - } - length += int(fb & bitMask32[n]) - fb >>= n & regSizeMaskUint32 - fnb -= n - default: - if debugDecode { - fmt.Println(v, ">= maxNumLit") - } - f.err = CorruptInputError(f.roffset) - f.b, f.nb = fb, fnb - return - } - - var dist uint32 - if f.hd == nil { - for fnb < 5 { - c, err := fr.ReadByte() - if err != nil { - f.b, f.nb = fb, fnb - if debugDecode { - fmt.Println("morebits f.nb<5:", err) - } - f.err = err - return - } - f.roffset++ - fb |= uint32(c) << (fnb & regSizeMaskUint32) - fnb += 8 - } - dist = uint32(bits.Reverse8(uint8(fb & 0x1F << 3))) - fb >>= 5 - fnb -= 5 - } else { - // Since a huffmanDecoder can be empty or be composed of a degenerate tree - // with single element, huffSym must error on these two edge cases. In both - // cases, the chunks slice will be 0 for the invalid sequence, leading it - // satisfy the n == 0 check below. - n := uint(f.hd.maxRead) - // Optimization. Compiler isn't smart enough to keep f.b,f.nb in registers, - // but is smart enough to keep local variables in registers, so use nb and b, - // inline call to moreBits and reassign b,nb back to f on return. - for { - for fnb < n { - c, err := fr.ReadByte() - if err != nil { - f.b, f.nb = fb, fnb - f.err = noEOF(err) - return - } - f.roffset++ - fb |= uint32(c) << (fnb & regSizeMaskUint32) - fnb += 8 - } - chunk := f.hd.chunks[fb&(huffmanNumChunks-1)] - n = uint(chunk & huffmanCountMask) - if n > huffmanChunkBits { - chunk = f.hd.links[chunk>>huffmanValueShift][(fb>>huffmanChunkBits)&f.hd.linkMask] - n = uint(chunk & huffmanCountMask) - } - if n <= fnb { - if n == 0 { - f.b, f.nb = fb, fnb - if debugDecode { - fmt.Println("huffsym: n==0") - } - f.err = CorruptInputError(f.roffset) - return - } - fb = fb >> (n & regSizeMaskUint32) - fnb = fnb - n - dist = uint32(chunk >> huffmanValueShift) - break - } - } - } - - switch { - case dist < 4: - dist++ - case dist < maxNumDist: - nb := uint(dist-2) >> 1 - // have 1 bit in bottom of dist, need nb more. - extra := (dist & 1) << (nb & regSizeMaskUint32) - for fnb < nb { - c, err := fr.ReadByte() - if err != nil { - f.b, f.nb = fb, fnb - if debugDecode { - fmt.Println("morebits f.nb<nb:", err) - } - f.err = err - return - } - f.roffset++ - fb |= uint32(c) << (fnb & regSizeMaskUint32) - fnb += 8 - } - extra |= fb & bitMask32[nb] - fb >>= nb & regSizeMaskUint32 - fnb -= nb - dist = 1<<((nb+1)®SizeMaskUint32) + 1 + extra - // slower: dist = bitMask32[nb+1] + 2 + extra - default: - f.b, f.nb = fb, fnb - if debugDecode { - fmt.Println("dist too big:", dist, maxNumDist) - } - f.err = CorruptInputError(f.roffset) - return - } - - // No check on length; encoding can be prescient. - if dist > uint32(dict.histSize()) { - f.b, f.nb = fb, fnb - if debugDecode { - fmt.Println("dist > dict.histSize():", dist, dict.histSize()) - } - f.err = CorruptInputError(f.roffset) - return - } - - f.copyLen, f.copyDist = length, int(dist) - goto copyHistory - } - -copyHistory: - // Perform a backwards copy according to RFC section 3.2.3. - { - cnt := dict.tryWriteCopy(f.copyDist, f.copyLen) - if cnt == 0 { - cnt = dict.writeCopy(f.copyDist, f.copyLen) - } - f.copyLen -= cnt - - if dict.availWrite() == 0 || f.copyLen > 0 { - f.toRead = dict.readFlush() - f.step = huffmanBytesReader // We need to continue this work - f.stepState = stateDict - f.b, f.nb = fb, fnb - return - } - goto readLiteral - } - // Not reached -} - -// Decode a single Huffman block from f. -// hl and hd are the Huffman states for the lit/length values -// and the distance values, respectively. If hd == nil, using the -// fixed distance encoding associated with fixed Huffman blocks. -func (f *decompressor) huffmanBufioReader() { - const ( - stateInit = iota // Zero value must be stateInit - stateDict - ) - fr := f.r.(*bufio.Reader) - - // Optimization. Compiler isn't smart enough to keep f.b,f.nb in registers, - // but is smart enough to keep local variables in registers, so use nb and b, - // inline call to moreBits and reassign b,nb back to f on return. - fnb, fb, dict := f.nb, f.b, &f.dict - - switch f.stepState { - case stateInit: - goto readLiteral - case stateDict: - goto copyHistory - } - -readLiteral: - // Read literal and/or (length, distance) according to RFC section 3.2.3. - { - var v int - { - // Inlined v, err := f.huffSym(f.hl) - // Since a huffmanDecoder can be empty or be composed of a degenerate tree - // with single element, huffSym must error on these two edge cases. In both - // cases, the chunks slice will be 0 for the invalid sequence, leading it - // satisfy the n == 0 check below. - n := uint(f.hl.maxRead) - for { - for fnb < n { - c, err := fr.ReadByte() - if err != nil { - f.b, f.nb = fb, fnb - f.err = noEOF(err) - return - } - f.roffset++ - fb |= uint32(c) << (fnb & regSizeMaskUint32) - fnb += 8 - } - chunk := f.hl.chunks[fb&(huffmanNumChunks-1)] - n = uint(chunk & huffmanCountMask) - if n > huffmanChunkBits { - chunk = f.hl.links[chunk>>huffmanValueShift][(fb>>huffmanChunkBits)&f.hl.linkMask] - n = uint(chunk & huffmanCountMask) - } - if n <= fnb { - if n == 0 { - f.b, f.nb = fb, fnb - if debugDecode { - fmt.Println("huffsym: n==0") - } - f.err = CorruptInputError(f.roffset) - return - } - fb = fb >> (n & regSizeMaskUint32) - fnb = fnb - n - v = int(chunk >> huffmanValueShift) - break - } - } - } - - var length int - switch { - case v < 256: - dict.writeByte(byte(v)) - if dict.availWrite() == 0 { - f.toRead = dict.readFlush() - f.step = huffmanBufioReader - f.stepState = stateInit - f.b, f.nb = fb, fnb - return - } - goto readLiteral - case v == 256: - f.b, f.nb = fb, fnb - f.finishBlock() - return - // otherwise, reference to older data - case v < 265: - length = v - (257 - 3) - case v < maxNumLit: - val := decCodeToLen[(v - 257)] - length = int(val.length) + 3 - n := uint(val.extra) - for fnb < n { - c, err := fr.ReadByte() - if err != nil { - f.b, f.nb = fb, fnb - if debugDecode { - fmt.Println("morebits n>0:", err) - } - f.err = err - return - } - f.roffset++ - fb |= uint32(c) << (fnb & regSizeMaskUint32) - fnb += 8 - } - length += int(fb & bitMask32[n]) - fb >>= n & regSizeMaskUint32 - fnb -= n - default: - if debugDecode { - fmt.Println(v, ">= maxNumLit") - } - f.err = CorruptInputError(f.roffset) - f.b, f.nb = fb, fnb - return - } - - var dist uint32 - if f.hd == nil { - for fnb < 5 { - c, err := fr.ReadByte() - if err != nil { - f.b, f.nb = fb, fnb - if debugDecode { - fmt.Println("morebits f.nb<5:", err) - } - f.err = err - return - } - f.roffset++ - fb |= uint32(c) << (fnb & regSizeMaskUint32) - fnb += 8 - } - dist = uint32(bits.Reverse8(uint8(fb & 0x1F << 3))) - fb >>= 5 - fnb -= 5 - } else { - // Since a huffmanDecoder can be empty or be composed of a degenerate tree - // with single element, huffSym must error on these two edge cases. In both - // cases, the chunks slice will be 0 for the invalid sequence, leading it - // satisfy the n == 0 check below. - n := uint(f.hd.maxRead) - // Optimization. Compiler isn't smart enough to keep f.b,f.nb in registers, - // but is smart enough to keep local variables in registers, so use nb and b, - // inline call to moreBits and reassign b,nb back to f on return. - for { - for fnb < n { - c, err := fr.ReadByte() - if err != nil { - f.b, f.nb = fb, fnb - f.err = noEOF(err) - return - } - f.roffset++ - fb |= uint32(c) << (fnb & regSizeMaskUint32) - fnb += 8 - } - chunk := f.hd.chunks[fb&(huffmanNumChunks-1)] - n = uint(chunk & huffmanCountMask) - if n > huffmanChunkBits { - chunk = f.hd.links[chunk>>huffmanValueShift][(fb>>huffmanChunkBits)&f.hd.linkMask] - n = uint(chunk & huffmanCountMask) - } - if n <= fnb { - if n == 0 { - f.b, f.nb = fb, fnb - if debugDecode { - fmt.Println("huffsym: n==0") - } - f.err = CorruptInputError(f.roffset) - return - } - fb = fb >> (n & regSizeMaskUint32) - fnb = fnb - n - dist = uint32(chunk >> huffmanValueShift) - break - } - } - } - - switch { - case dist < 4: - dist++ - case dist < maxNumDist: - nb := uint(dist-2) >> 1 - // have 1 bit in bottom of dist, need nb more. - extra := (dist & 1) << (nb & regSizeMaskUint32) - for fnb < nb { - c, err := fr.ReadByte() - if err != nil { - f.b, f.nb = fb, fnb - if debugDecode { - fmt.Println("morebits f.nb<nb:", err) - } - f.err = err - return - } - f.roffset++ - fb |= uint32(c) << (fnb & regSizeMaskUint32) - fnb += 8 - } - extra |= fb & bitMask32[nb] - fb >>= nb & regSizeMaskUint32 - fnb -= nb - dist = 1<<((nb+1)®SizeMaskUint32) + 1 + extra - // slower: dist = bitMask32[nb+1] + 2 + extra - default: - f.b, f.nb = fb, fnb - if debugDecode { - fmt.Println("dist too big:", dist, maxNumDist) - } - f.err = CorruptInputError(f.roffset) - return - } - - // No check on length; encoding can be prescient. - if dist > uint32(dict.histSize()) { - f.b, f.nb = fb, fnb - if debugDecode { - fmt.Println("dist > dict.histSize():", dist, dict.histSize()) - } - f.err = CorruptInputError(f.roffset) - return - } - - f.copyLen, f.copyDist = length, int(dist) - goto copyHistory - } - -copyHistory: - // Perform a backwards copy according to RFC section 3.2.3. - { - cnt := dict.tryWriteCopy(f.copyDist, f.copyLen) - if cnt == 0 { - cnt = dict.writeCopy(f.copyDist, f.copyLen) - } - f.copyLen -= cnt - - if dict.availWrite() == 0 || f.copyLen > 0 { - f.toRead = dict.readFlush() - f.step = huffmanBufioReader // We need to continue this work - f.stepState = stateDict - f.b, f.nb = fb, fnb - return - } - goto readLiteral - } - // Not reached -} - -// Decode a single Huffman block from f. -// hl and hd are the Huffman states for the lit/length values -// and the distance values, respectively. If hd == nil, using the -// fixed distance encoding associated with fixed Huffman blocks. -func (f *decompressor) huffmanStringsReader() { - const ( - stateInit = iota // Zero value must be stateInit - stateDict - ) - fr := f.r.(*strings.Reader) - - // Optimization. Compiler isn't smart enough to keep f.b,f.nb in registers, - // but is smart enough to keep local variables in registers, so use nb and b, - // inline call to moreBits and reassign b,nb back to f on return. - fnb, fb, dict := f.nb, f.b, &f.dict - - switch f.stepState { - case stateInit: - goto readLiteral - case stateDict: - goto copyHistory - } - -readLiteral: - // Read literal and/or (length, distance) according to RFC section 3.2.3. - { - var v int - { - // Inlined v, err := f.huffSym(f.hl) - // Since a huffmanDecoder can be empty or be composed of a degenerate tree - // with single element, huffSym must error on these two edge cases. In both - // cases, the chunks slice will be 0 for the invalid sequence, leading it - // satisfy the n == 0 check below. - n := uint(f.hl.maxRead) - for { - for fnb < n { - c, err := fr.ReadByte() - if err != nil { - f.b, f.nb = fb, fnb - f.err = noEOF(err) - return - } - f.roffset++ - fb |= uint32(c) << (fnb & regSizeMaskUint32) - fnb += 8 - } - chunk := f.hl.chunks[fb&(huffmanNumChunks-1)] - n = uint(chunk & huffmanCountMask) - if n > huffmanChunkBits { - chunk = f.hl.links[chunk>>huffmanValueShift][(fb>>huffmanChunkBits)&f.hl.linkMask] - n = uint(chunk & huffmanCountMask) - } - if n <= fnb { - if n == 0 { - f.b, f.nb = fb, fnb - if debugDecode { - fmt.Println("huffsym: n==0") - } - f.err = CorruptInputError(f.roffset) - return - } - fb = fb >> (n & regSizeMaskUint32) - fnb = fnb - n - v = int(chunk >> huffmanValueShift) - break - } - } - } - - var length int - switch { - case v < 256: - dict.writeByte(byte(v)) - if dict.availWrite() == 0 { - f.toRead = dict.readFlush() - f.step = huffmanStringsReader - f.stepState = stateInit - f.b, f.nb = fb, fnb - return - } - goto readLiteral - case v == 256: - f.b, f.nb = fb, fnb - f.finishBlock() - return - // otherwise, reference to older data - case v < 265: - length = v - (257 - 3) - case v < maxNumLit: - val := decCodeToLen[(v - 257)] - length = int(val.length) + 3 - n := uint(val.extra) - for fnb < n { - c, err := fr.ReadByte() - if err != nil { - f.b, f.nb = fb, fnb - if debugDecode { - fmt.Println("morebits n>0:", err) - } - f.err = err - return - } - f.roffset++ - fb |= uint32(c) << (fnb & regSizeMaskUint32) - fnb += 8 - } - length += int(fb & bitMask32[n]) - fb >>= n & regSizeMaskUint32 - fnb -= n - default: - if debugDecode { - fmt.Println(v, ">= maxNumLit") - } - f.err = CorruptInputError(f.roffset) - f.b, f.nb = fb, fnb - return - } - - var dist uint32 - if f.hd == nil { - for fnb < 5 { - c, err := fr.ReadByte() - if err != nil { - f.b, f.nb = fb, fnb - if debugDecode { - fmt.Println("morebits f.nb<5:", err) - } - f.err = err - return - } - f.roffset++ - fb |= uint32(c) << (fnb & regSizeMaskUint32) - fnb += 8 - } - dist = uint32(bits.Reverse8(uint8(fb & 0x1F << 3))) - fb >>= 5 - fnb -= 5 - } else { - // Since a huffmanDecoder can be empty or be composed of a degenerate tree - // with single element, huffSym must error on these two edge cases. In both - // cases, the chunks slice will be 0 for the invalid sequence, leading it - // satisfy the n == 0 check below. - n := uint(f.hd.maxRead) - // Optimization. Compiler isn't smart enough to keep f.b,f.nb in registers, - // but is smart enough to keep local variables in registers, so use nb and b, - // inline call to moreBits and reassign b,nb back to f on return. - for { - for fnb < n { - c, err := fr.ReadByte() - if err != nil { - f.b, f.nb = fb, fnb - f.err = noEOF(err) - return - } - f.roffset++ - fb |= uint32(c) << (fnb & regSizeMaskUint32) - fnb += 8 - } - chunk := f.hd.chunks[fb&(huffmanNumChunks-1)] - n = uint(chunk & huffmanCountMask) - if n > huffmanChunkBits { - chunk = f.hd.links[chunk>>huffmanValueShift][(fb>>huffmanChunkBits)&f.hd.linkMask] - n = uint(chunk & huffmanCountMask) - } - if n <= fnb { - if n == 0 { - f.b, f.nb = fb, fnb - if debugDecode { - fmt.Println("huffsym: n==0") - } - f.err = CorruptInputError(f.roffset) - return - } - fb = fb >> (n & regSizeMaskUint32) - fnb = fnb - n - dist = uint32(chunk >> huffmanValueShift) - break - } - } - } - - switch { - case dist < 4: - dist++ - case dist < maxNumDist: - nb := uint(dist-2) >> 1 - // have 1 bit in bottom of dist, need nb more. - extra := (dist & 1) << (nb & regSizeMaskUint32) - for fnb < nb { - c, err := fr.ReadByte() - if err != nil { - f.b, f.nb = fb, fnb - if debugDecode { - fmt.Println("morebits f.nb<nb:", err) - } - f.err = err - return - } - f.roffset++ - fb |= uint32(c) << (fnb & regSizeMaskUint32) - fnb += 8 - } - extra |= fb & bitMask32[nb] - fb >>= nb & regSizeMaskUint32 - fnb -= nb - dist = 1<<((nb+1)®SizeMaskUint32) + 1 + extra - // slower: dist = bitMask32[nb+1] + 2 + extra - default: - f.b, f.nb = fb, fnb - if debugDecode { - fmt.Println("dist too big:", dist, maxNumDist) - } - f.err = CorruptInputError(f.roffset) - return - } - - // No check on length; encoding can be prescient. - if dist > uint32(dict.histSize()) { - f.b, f.nb = fb, fnb - if debugDecode { - fmt.Println("dist > dict.histSize():", dist, dict.histSize()) - } - f.err = CorruptInputError(f.roffset) - return - } - - f.copyLen, f.copyDist = length, int(dist) - goto copyHistory - } - -copyHistory: - // Perform a backwards copy according to RFC section 3.2.3. - { - cnt := dict.tryWriteCopy(f.copyDist, f.copyLen) - if cnt == 0 { - cnt = dict.writeCopy(f.copyDist, f.copyLen) - } - f.copyLen -= cnt - - if dict.availWrite() == 0 || f.copyLen > 0 { - f.toRead = dict.readFlush() - f.step = huffmanStringsReader // We need to continue this work - f.stepState = stateDict - f.b, f.nb = fb, fnb - return - } - goto readLiteral - } - // Not reached -} - -// Decode a single Huffman block from f. -// hl and hd are the Huffman states for the lit/length values -// and the distance values, respectively. If hd == nil, using the -// fixed distance encoding associated with fixed Huffman blocks. -func (f *decompressor) huffmanGenericReader() { - const ( - stateInit = iota // Zero value must be stateInit - stateDict - ) - fr := f.r.(Reader) - - // Optimization. Compiler isn't smart enough to keep f.b,f.nb in registers, - // but is smart enough to keep local variables in registers, so use nb and b, - // inline call to moreBits and reassign b,nb back to f on return. - fnb, fb, dict := f.nb, f.b, &f.dict - - switch f.stepState { - case stateInit: - goto readLiteral - case stateDict: - goto copyHistory - } - -readLiteral: - // Read literal and/or (length, distance) according to RFC section 3.2.3. - { - var v int - { - // Inlined v, err := f.huffSym(f.hl) - // Since a huffmanDecoder can be empty or be composed of a degenerate tree - // with single element, huffSym must error on these two edge cases. In both - // cases, the chunks slice will be 0 for the invalid sequence, leading it - // satisfy the n == 0 check below. - n := uint(f.hl.maxRead) - for { - for fnb < n { - c, err := fr.ReadByte() - if err != nil { - f.b, f.nb = fb, fnb - f.err = noEOF(err) - return - } - f.roffset++ - fb |= uint32(c) << (fnb & regSizeMaskUint32) - fnb += 8 - } - chunk := f.hl.chunks[fb&(huffmanNumChunks-1)] - n = uint(chunk & huffmanCountMask) - if n > huffmanChunkBits { - chunk = f.hl.links[chunk>>huffmanValueShift][(fb>>huffmanChunkBits)&f.hl.linkMask] - n = uint(chunk & huffmanCountMask) - } - if n <= fnb { - if n == 0 { - f.b, f.nb = fb, fnb - if debugDecode { - fmt.Println("huffsym: n==0") - } - f.err = CorruptInputError(f.roffset) - return - } - fb = fb >> (n & regSizeMaskUint32) - fnb = fnb - n - v = int(chunk >> huffmanValueShift) - break - } - } - } - - var length int - switch { - case v < 256: - dict.writeByte(byte(v)) - if dict.availWrite() == 0 { - f.toRead = dict.readFlush() - f.step = huffmanGenericReader - f.stepState = stateInit - f.b, f.nb = fb, fnb - return - } - goto readLiteral - case v == 256: - f.b, f.nb = fb, fnb - f.finishBlock() - return - // otherwise, reference to older data - case v < 265: - length = v - (257 - 3) - case v < maxNumLit: - val := decCodeToLen[(v - 257)] - length = int(val.length) + 3 - n := uint(val.extra) - for fnb < n { - c, err := fr.ReadByte() - if err != nil { - f.b, f.nb = fb, fnb - if debugDecode { - fmt.Println("morebits n>0:", err) - } - f.err = err - return - } - f.roffset++ - fb |= uint32(c) << (fnb & regSizeMaskUint32) - fnb += 8 - } - length += int(fb & bitMask32[n]) - fb >>= n & regSizeMaskUint32 - fnb -= n - default: - if debugDecode { - fmt.Println(v, ">= maxNumLit") - } - f.err = CorruptInputError(f.roffset) - f.b, f.nb = fb, fnb - return - } - - var dist uint32 - if f.hd == nil { - for fnb < 5 { - c, err := fr.ReadByte() - if err != nil { - f.b, f.nb = fb, fnb - if debugDecode { - fmt.Println("morebits f.nb<5:", err) - } - f.err = err - return - } - f.roffset++ - fb |= uint32(c) << (fnb & regSizeMaskUint32) - fnb += 8 - } - dist = uint32(bits.Reverse8(uint8(fb & 0x1F << 3))) - fb >>= 5 - fnb -= 5 - } else { - // Since a huffmanDecoder can be empty or be composed of a degenerate tree - // with single element, huffSym must error on these two edge cases. In both - // cases, the chunks slice will be 0 for the invalid sequence, leading it - // satisfy the n == 0 check below. - n := uint(f.hd.maxRead) - // Optimization. Compiler isn't smart enough to keep f.b,f.nb in registers, - // but is smart enough to keep local variables in registers, so use nb and b, - // inline call to moreBits and reassign b,nb back to f on return. - for { - for fnb < n { - c, err := fr.ReadByte() - if err != nil { - f.b, f.nb = fb, fnb - f.err = noEOF(err) - return - } - f.roffset++ - fb |= uint32(c) << (fnb & regSizeMaskUint32) - fnb += 8 - } - chunk := f.hd.chunks[fb&(huffmanNumChunks-1)] - n = uint(chunk & huffmanCountMask) - if n > huffmanChunkBits { - chunk = f.hd.links[chunk>>huffmanValueShift][(fb>>huffmanChunkBits)&f.hd.linkMask] - n = uint(chunk & huffmanCountMask) - } - if n <= fnb { - if n == 0 { - f.b, f.nb = fb, fnb - if debugDecode { - fmt.Println("huffsym: n==0") - } - f.err = CorruptInputError(f.roffset) - return - } - fb = fb >> (n & regSizeMaskUint32) - fnb = fnb - n - dist = uint32(chunk >> huffmanValueShift) - break - } - } - } - - switch { - case dist < 4: - dist++ - case dist < maxNumDist: - nb := uint(dist-2) >> 1 - // have 1 bit in bottom of dist, need nb more. - extra := (dist & 1) << (nb & regSizeMaskUint32) - for fnb < nb { - c, err := fr.ReadByte() - if err != nil { - f.b, f.nb = fb, fnb - if debugDecode { - fmt.Println("morebits f.nb<nb:", err) - } - f.err = err - return - } - f.roffset++ - fb |= uint32(c) << (fnb & regSizeMaskUint32) - fnb += 8 - } - extra |= fb & bitMask32[nb] - fb >>= nb & regSizeMaskUint32 - fnb -= nb - dist = 1<<((nb+1)®SizeMaskUint32) + 1 + extra - // slower: dist = bitMask32[nb+1] + 2 + extra - default: - f.b, f.nb = fb, fnb - if debugDecode { - fmt.Println("dist too big:", dist, maxNumDist) - } - f.err = CorruptInputError(f.roffset) - return - } - - // No check on length; encoding can be prescient. - if dist > uint32(dict.histSize()) { - f.b, f.nb = fb, fnb - if debugDecode { - fmt.Println("dist > dict.histSize():", dist, dict.histSize()) - } - f.err = CorruptInputError(f.roffset) - return - } - - f.copyLen, f.copyDist = length, int(dist) - goto copyHistory - } - -copyHistory: - // Perform a backwards copy according to RFC section 3.2.3. - { - cnt := dict.tryWriteCopy(f.copyDist, f.copyLen) - if cnt == 0 { - cnt = dict.writeCopy(f.copyDist, f.copyLen) - } - f.copyLen -= cnt - - if dict.availWrite() == 0 || f.copyLen > 0 { - f.toRead = dict.readFlush() - f.step = huffmanGenericReader // We need to continue this work - f.stepState = stateDict - f.b, f.nb = fb, fnb - return - } - goto readLiteral - } - // Not reached -} - -func (f *decompressor) huffmanBlockDecoder() { - switch f.r.(type) { - case *bytes.Buffer: - f.huffmanBytesBuffer() - case *bytes.Reader: - f.huffmanBytesReader() - case *bufio.Reader: - f.huffmanBufioReader() - case *strings.Reader: - f.huffmanStringsReader() - case Reader: - f.huffmanGenericReader() - default: - f.huffmanGenericReader() - } -} diff --git a/internal/compress/flate/inflate_test.go b/internal/compress/flate/inflate_test.go deleted file mode 100644 index f163695f..00000000 --- a/internal/compress/flate/inflate_test.go +++ /dev/null @@ -1,301 +0,0 @@ -// Copyright 2014 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package flate - -import ( - "bytes" - "crypto/rand" - "io" - "os" - "strconv" - "strings" - "testing" -) - -func TestReset(t *testing.T) { - ss := []string{ - "lorem ipsum izzle fo rizzle", - "the quick brown fox jumped over", - } - - deflated := make([]bytes.Buffer, 2) - for i, s := range ss { - w, _ := NewWriter(&deflated[i], 1) - w.Write([]byte(s)) - w.Close() - } - - inflated := make([]bytes.Buffer, 2) - - f := NewReader(&deflated[0]) - io.Copy(&inflated[0], f) - f.(Resetter).Reset(&deflated[1], nil) - io.Copy(&inflated[1], f) - f.Close() - - for i, s := range ss { - if s != inflated[i].String() { - t.Errorf("inflated[%d]:\ngot %q\nwant %q", i, inflated[i].String(), s) - } - } -} - -func TestReaderTruncated(t *testing.T) { - vectors := []struct{ input, output string }{ - {"\x00", ""}, - {"\x00\f", ""}, - {"\x00\f\x00", ""}, - {"\x00\f\x00\xf3\xff", ""}, - {"\x00\f\x00\xf3\xffhello", "hello"}, - {"\x00\f\x00\xf3\xffhello, world", "hello, world"}, - {"\x02", ""}, - {"\xf2H\xcd", "He"}, - {"\xf2HÍ™0a\u0084\t", "Hel\x90\x90\x90\x90\x90"}, - {"\xf2HÍ™0a\u0084\t\x00", "Hel\x90\x90\x90\x90\x90"}, - } - - for i, v := range vectors { - r := strings.NewReader(v.input) - zr := NewReader(r) - b, err := io.ReadAll(zr) - if err != io.ErrUnexpectedEOF { - t.Errorf("test %d, error mismatch: got %v, want io.ErrUnexpectedEOF", i, err) - } - if string(b) != v.output { - t.Errorf("test %d, output mismatch: got %q, want %q", i, b, v.output) - } - } -} - -func TestResetDict(t *testing.T) { - dict := []byte("the lorem fox") - ss := []string{ - "lorem ipsum izzle fo rizzle", - "the quick brown fox jumped over", - } - - deflated := make([]bytes.Buffer, len(ss)) - for i, s := range ss { - w, _ := NewWriterDict(&deflated[i], DefaultCompression, dict) - w.Write([]byte(s)) - w.Close() - } - - inflated := make([]bytes.Buffer, len(ss)) - - f := NewReader(nil) - for i := range inflated { - f.(Resetter).Reset(&deflated[i], dict) - io.Copy(&inflated[i], f) - } - f.Close() - - for i, s := range ss { - if s != inflated[i].String() { - t.Errorf("inflated[%d]:\ngot %q\nwant %q", i, inflated[i].String(), s) - } - } -} - -// Tests ported from zlib/test/infcover.c -type infTest struct { - hex string - id string - n int -} - -var infTests = []infTest{ - {"0 0 0 0 0", "invalid stored block lengths", 1}, - {"3 0", "fixed", 0}, - {"6", "invalid block type", 1}, - {"1 1 0 fe ff 0", "stored", 0}, - {"fc 0 0", "too many length or distance symbols", 1}, - {"4 0 fe ff", "invalid code lengths set", 1}, - {"4 0 24 49 0", "invalid bit length repeat", 1}, - {"4 0 24 e9 ff ff", "invalid bit length repeat", 1}, - {"4 0 24 e9 ff 6d", "invalid code -- missing end-of-block", 1}, - {"4 80 49 92 24 49 92 24 71 ff ff 93 11 0", "invalid literal/lengths set", 1}, - {"4 80 49 92 24 49 92 24 f b4 ff ff c3 84", "invalid distances set", 1}, - {"4 c0 81 8 0 0 0 0 20 7f eb b 0 0", "invalid literal/length code", 1}, - {"2 7e ff ff", "invalid distance code", 1}, - {"c c0 81 0 0 0 0 0 90 ff 6b 4 0", "invalid distance too far back", 1}, - - // also trailer mismatch just in inflate() - {"1f 8b 8 0 0 0 0 0 0 0 3 0 0 0 0 1", "incorrect data check", -1}, - {"1f 8b 8 0 0 0 0 0 0 0 3 0 0 0 0 0 0 0 0 1", "incorrect length check", -1}, - {"5 c0 21 d 0 0 0 80 b0 fe 6d 2f 91 6c", "pull 17", 0}, - {"5 e0 81 91 24 cb b2 2c 49 e2 f 2e 8b 9a 47 56 9f fb fe ec d2 ff 1f", "long code", 0}, - {"ed c0 1 1 0 0 0 40 20 ff 57 1b 42 2c 4f", "length extra", 0}, - {"ed cf c1 b1 2c 47 10 c4 30 fa 6f 35 1d 1 82 59 3d fb be 2e 2a fc f c", "long distance and extra", 0}, - {"ed c0 81 0 0 0 0 80 a0 fd a9 17 a9 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 6", "window end", 0}, -} - -func TestInflate(t *testing.T) { - for _, test := range infTests { - hex := strings.Split(test.hex, " ") - data := make([]byte, len(hex)) - for i, h := range hex { - b, _ := strconv.ParseInt(h, 16, 32) - data[i] = byte(b) - } - buf := bytes.NewReader(data) - r := NewReader(buf) - - _, err := io.Copy(io.Discard, r) - if (test.n == 0 && err == nil) || (test.n != 0 && err != nil) { - t.Logf("%q: OK:", test.id) - t.Logf(" - got %v", err) - continue - } - - if test.n == 0 && err != nil { - t.Errorf("%q: Expected no error, but got %v", test.id, err) - continue - } - - if test.n != 0 && err == nil { - t.Errorf("%q:Expected an error, but got none", test.id) - continue - } - t.Fatal(test.n, err) - } - - for _, test := range infOutTests { - hex := strings.Split(test.hex, " ") - data := make([]byte, len(hex)) - for i, h := range hex { - b, _ := strconv.ParseInt(h, 16, 32) - data[i] = byte(b) - } - buf := bytes.NewReader(data) - r := NewReader(buf) - - _, err := io.Copy(io.Discard, r) - if test.err == (err != nil) { - t.Logf("%q: OK:", test.id) - t.Logf(" - got %v", err) - continue - } - - if test.err == false && err != nil { - t.Errorf("%q: Expected no error, but got %v", test.id, err) - continue - } - - if test.err && err == nil { - t.Errorf("%q: Expected an error, but got none", test.id) - continue - } - t.Fatal(test.err, err) - } -} - -// Tests ported from zlib/test/infcover.c -// Since zlib inflate is push (writer) instead of pull (reader) -// some of the window size tests have been removed, since they -// are irrelevant. -type infOutTest struct { - hex string - id string - step int - win int - length int - err bool -} - -var infOutTests = []infOutTest{ - {"2 8 20 80 0 3 0", "inflate_fast TYPE return", 0, -15, 258, false}, - {"63 18 5 40 c 0", "window wrap", 3, -8, 300, false}, - {"e5 e0 81 ad 6d cb b2 2c c9 01 1e 59 63 ae 7d ee fb 4d fd b5 35 41 68 ff 7f 0f 0 0 0", "fast length extra bits", 0, -8, 258, true}, - {"25 fd 81 b5 6d 59 b6 6a 49 ea af 35 6 34 eb 8c b9 f6 b9 1e ef 67 49 50 fe ff ff 3f 0 0", "fast distance extra bits", 0, -8, 258, true}, - {"3 7e 0 0 0 0 0", "fast invalid distance code", 0, -8, 258, true}, - {"1b 7 0 0 0 0 0", "fast invalid literal/length code", 0, -8, 258, true}, - {"d c7 1 ae eb 38 c 4 41 a0 87 72 de df fb 1f b8 36 b1 38 5d ff ff 0", "fast 2nd level codes and too far back", 0, -8, 258, true}, - {"63 18 5 8c 10 8 0 0 0 0", "very common case", 0, -8, 259, false}, - {"63 60 60 18 c9 0 8 18 18 18 26 c0 28 0 29 0 0 0", "contiguous and wrap around window", 6, -8, 259, false}, - {"63 0 3 0 0 0 0 0", "copy direct from output", 0, -8, 259, false}, - {"1f 8b 0 0", "bad gzip method", 0, 31, 0, true}, - {"1f 8b 8 80", "bad gzip flags", 0, 31, 0, true}, - {"77 85", "bad zlib method", 0, 15, 0, true}, - {"78 9c", "bad zlib window size", 0, 8, 0, true}, - {"1f 8b 8 1e 0 0 0 0 0 0 1 0 0 0 0 0 0", "bad header crc", 0, 47, 1, true}, - {"1f 8b 8 2 0 0 0 0 0 0 1d 26 3 0 0 0 0 0 0 0 0 0", "check gzip length", 0, 47, 0, true}, - {"78 90", "bad zlib header check", 0, 47, 0, true}, - {"8 b8 0 0 0 1", "need dictionary", 0, 8, 0, true}, - {"63 18 68 30 d0 0 0", "force split window update", 4, -8, 259, false}, - {"3 0", "use fixed blocks", 0, -15, 1, false}, - {"", "bad window size", 0, 1, 0, true}, -} - -func TestWriteTo(t *testing.T) { - input := make([]byte, 100000) - n, err := rand.Read(input) - if err != nil { - t.Fatal(err) - } - if n != len(input) { - t.Fatal("did not fill buffer") - } - compressed := &bytes.Buffer{} - w, err := NewWriter(compressed, -2) - if err != nil { - t.Fatal(err) - } - n, err = w.Write(input) - if err != nil { - t.Fatal(err) - } - if n != len(input) { - t.Fatal("did not fill buffer") - } - w.Close() - buf := compressed.Bytes() - - dec := NewReader(bytes.NewBuffer(buf)) - // ReadAll does not use WriteTo, but we wrap it in a NopCloser to be sure. - readall, err := io.ReadAll(io.NopCloser(dec)) - if err != nil { - t.Fatal(err) - } - if len(readall) != len(input) { - t.Fatal("did not decompress everything") - } - - dec = NewReader(bytes.NewBuffer(buf)) - wtbuf := &bytes.Buffer{} - written, err := dec.(io.WriterTo).WriteTo(wtbuf) - if err != nil { - t.Fatal(err) - } - if written != int64(len(input)) { - t.Error("Returned length did not match, expected", len(input), "got", written) - } - if wtbuf.Len() != len(input) { - t.Error("Actual Length did not match, expected", len(input), "got", wtbuf.Len()) - } - if !bytes.Equal(wtbuf.Bytes(), input) { - t.Fatal("output did not match input") - } -} - -func TestReaderPartialBlock(t *testing.T) { - data, err := os.ReadFile("testdata/partial-block") - if err != nil { - t.Error(err) - } - - r := NewReaderOpts(bytes.NewReader(data), WithPartialBlock()) - rb := make([]byte, 32) - n, err := r.Read(rb) - if err != nil { - t.Fatalf("Read: %v", err) - } - - expected := "hello, world" - actual := string(rb[:n]) - if expected != actual { - t.Fatalf("expected: %v, got: %v", expected, actual) - } -} diff --git a/internal/compress/flate/level1.go b/internal/compress/flate/level1.go deleted file mode 100644 index d7bad88d..00000000 --- a/internal/compress/flate/level1.go +++ /dev/null @@ -1,215 +0,0 @@ -package flate - -import ( - "fmt" - - "lindenii.org/go/furgit/internal/compress/internal/le" -) - -// fastGen maintains the table for matches, -// and the previous byte block for level 2. -// This is the generic implementation. -type fastEncL1 struct { - fastGen - table [tableSize]tableEntry -} - -// EncodeL1 uses a similar algorithm to level 1 -func (e *fastEncL1) Encode(dst *tokens, src []byte) { - const ( - inputMargin = 12 - 1 - minNonLiteralBlockSize = 1 + 1 + inputMargin - hashBytes = 5 - ) - if debugDeflate && e.cur < 0 { - panic(fmt.Sprint("e.cur < 0: ", e.cur)) - } - - // Protect against e.cur wraparound. - for e.cur >= bufferReset { - if len(e.hist) == 0 { - for i := range e.table[:] { - e.table[i] = tableEntry{} - } - e.cur = maxMatchOffset - break - } - // Shift down everything in the table that isn't already too far away. - minOff := e.cur + int32(len(e.hist)) - maxMatchOffset - for i := range e.table[:] { - v := e.table[i].offset - if v <= minOff { - v = 0 - } else { - v = v - e.cur + maxMatchOffset - } - e.table[i].offset = v - } - e.cur = maxMatchOffset - } - - s := e.addBlock(src) - - // This check isn't in the Snappy implementation, but there, the caller - // instead of the callee handles this case. - if len(src) < minNonLiteralBlockSize { - // We do not fill the token table. - // This will be picked up by caller. - dst.n = uint16(len(src)) - return - } - - // Override src - src = e.hist - nextEmit := s - - // sLimit is when to stop looking for offset/length copies. The inputMargin - // lets us use a fast path for emitLiteral in the main loop, while we are - // looking for copies. - sLimit := int32(len(src) - inputMargin) - - // nextEmit is where in src the next emitLiteral should start from. - cv := load6432(src, s) - - for { - const skipLog = 5 - const doEvery = 2 - - nextS := s - var candidate tableEntry - var t int32 - for { - nextHash := hashLen(cv, tableBits, hashBytes) - candidate = e.table[nextHash] - nextS = s + doEvery + (s-nextEmit)>>skipLog - if nextS > sLimit { - goto emitRemainder - } - - now := load6432(src, nextS) - e.table[nextHash] = tableEntry{offset: s + e.cur} - nextHash = hashLen(now, tableBits, hashBytes) - t = candidate.offset - e.cur - if s-t < maxMatchOffset && uint32(cv) == load3232(src, t) { - e.table[nextHash] = tableEntry{offset: nextS + e.cur} - break - } - - // Do one right away... - cv = now - s = nextS - nextS++ - candidate = e.table[nextHash] - now >>= 8 - e.table[nextHash] = tableEntry{offset: s + e.cur} - - t = candidate.offset - e.cur - if s-t < maxMatchOffset && uint32(cv) == load3232(src, t) { - e.table[nextHash] = tableEntry{offset: nextS + e.cur} - break - } - cv = now - s = nextS - } - - // A 4-byte match has been found. We'll later see if more than 4 bytes - // match. But, prior to the match, src[nextEmit:s] are unmatched. Emit - // them as literal bytes. - for { - // Invariant: we have a 4-byte match at s, and no need to emit any - // literal bytes prior to s. - - // Extend the 4-byte match as long as possible. - l := e.matchlenLong(int(s+4), int(t+4), src) + 4 - - // Extend backwards - for t > 0 && s > nextEmit && le.Load8(src, t-1) == le.Load8(src, s-1) { - s-- - t-- - l++ - } - if nextEmit < s { - if false { - emitLiteral(dst, src[nextEmit:s]) - } else { - for _, v := range src[nextEmit:s] { - dst.tokens[dst.n] = token(v) - dst.litHist[v]++ - dst.n++ - } - } - } - - // Save the match found - if false { - dst.AddMatchLong(l, uint32(s-t-baseMatchOffset)) - } else { - // Inlined... - xoffset := uint32(s - t - baseMatchOffset) - xlength := l - oc := offsetCode(xoffset) - xoffset |= oc << 16 - for xlength > 0 { - xl := xlength - if xl > 258 { - if xl > 258+baseMatchLength { - xl = 258 - } else { - xl = 258 - baseMatchLength - } - } - xlength -= xl - xl -= baseMatchLength - dst.extraHist[lengthCodes1[uint8(xl)]]++ - dst.offHist[oc]++ - dst.tokens[dst.n] = token(matchType | uint32(xl)<<lengthShift | xoffset) - dst.n++ - } - } - s += l - nextEmit = s - if nextS >= s { - s = nextS + 1 - } - if s >= sLimit { - // Index first pair after match end. - if int(s+l+8) < len(src) { - cv := load6432(src, s) - e.table[hashLen(cv, tableBits, hashBytes)] = tableEntry{offset: s + e.cur} - } - goto emitRemainder - } - - // We could immediately start working at s now, but to improve - // compression we first update the hash table at s-2 and at s. If - // another emitCopy is not our next move, also calculate nextHash - // at s+1. At least on GOARCH=amd64, these three hash calculations - // are faster as one load64 call (with some shifts) instead of - // three load32 calls. - x := load6432(src, s-2) - o := e.cur + s - 2 - prevHash := hashLen(x, tableBits, hashBytes) - e.table[prevHash] = tableEntry{offset: o} - x >>= 16 - currHash := hashLen(x, tableBits, hashBytes) - candidate = e.table[currHash] - e.table[currHash] = tableEntry{offset: o + 2} - - t = candidate.offset - e.cur - if s-t > maxMatchOffset || uint32(x) != load3232(src, t) { - cv = x >> 8 - s++ - break - } - } - } - -emitRemainder: - if int(nextEmit) < len(src) { - // If nothing was added, don't encode literals. - if dst.n == 0 { - return - } - emitLiteral(dst, src[nextEmit:]) - } -} diff --git a/internal/compress/flate/level2.go b/internal/compress/flate/level2.go deleted file mode 100644 index c8d047f2..00000000 --- a/internal/compress/flate/level2.go +++ /dev/null @@ -1,214 +0,0 @@ -package flate - -import "fmt" - -// fastGen maintains the table for matches, -// and the previous byte block for level 2. -// This is the generic implementation. -type fastEncL2 struct { - fastGen - table [bTableSize]tableEntry -} - -// EncodeL2 uses a similar algorithm to level 1, but is capable -// of matching across blocks giving better compression at a small slowdown. -func (e *fastEncL2) Encode(dst *tokens, src []byte) { - const ( - inputMargin = 12 - 1 - minNonLiteralBlockSize = 1 + 1 + inputMargin - hashBytes = 5 - ) - - if debugDeflate && e.cur < 0 { - panic(fmt.Sprint("e.cur < 0: ", e.cur)) - } - - // Protect against e.cur wraparound. - for e.cur >= bufferReset { - if len(e.hist) == 0 { - for i := range e.table[:] { - e.table[i] = tableEntry{} - } - e.cur = maxMatchOffset - break - } - // Shift down everything in the table that isn't already too far away. - minOff := e.cur + int32(len(e.hist)) - maxMatchOffset - for i := range e.table[:] { - v := e.table[i].offset - if v <= minOff { - v = 0 - } else { - v = v - e.cur + maxMatchOffset - } - e.table[i].offset = v - } - e.cur = maxMatchOffset - } - - s := e.addBlock(src) - - // This check isn't in the Snappy implementation, but there, the caller - // instead of the callee handles this case. - if len(src) < minNonLiteralBlockSize { - // We do not fill the token table. - // This will be picked up by caller. - dst.n = uint16(len(src)) - return - } - - // Override src - src = e.hist - nextEmit := s - - // sLimit is when to stop looking for offset/length copies. The inputMargin - // lets us use a fast path for emitLiteral in the main loop, while we are - // looking for copies. - sLimit := int32(len(src) - inputMargin) - - // nextEmit is where in src the next emitLiteral should start from. - cv := load6432(src, s) - for { - // When should we start skipping if we haven't found matches in a long while. - const skipLog = 5 - const doEvery = 2 - - nextS := s - var candidate tableEntry - for { - nextHash := hashLen(cv, bTableBits, hashBytes) - s = nextS - nextS = s + doEvery + (s-nextEmit)>>skipLog - if nextS > sLimit { - goto emitRemainder - } - candidate = e.table[nextHash] - now := load6432(src, nextS) - e.table[nextHash] = tableEntry{offset: s + e.cur} - nextHash = hashLen(now, bTableBits, hashBytes) - - offset := s - (candidate.offset - e.cur) - if offset < maxMatchOffset && uint32(cv) == load3232(src, candidate.offset-e.cur) { - e.table[nextHash] = tableEntry{offset: nextS + e.cur} - break - } - - // Do one right away... - cv = now - s = nextS - nextS++ - candidate = e.table[nextHash] - now >>= 8 - e.table[nextHash] = tableEntry{offset: s + e.cur} - - offset = s - (candidate.offset - e.cur) - if offset < maxMatchOffset && uint32(cv) == load3232(src, candidate.offset-e.cur) { - break - } - cv = now - } - - // A 4-byte match has been found. We'll later see if more than 4 bytes - // match. But, prior to the match, src[nextEmit:s] are unmatched. Emit - // them as literal bytes. - - // Call emitCopy, and then see if another emitCopy could be our next - // move. Repeat until we find no match for the input immediately after - // what was consumed by the last emitCopy call. - // - // If we exit this loop normally then we need to call emitLiteral next, - // though we don't yet know how big the literal will be. We handle that - // by proceeding to the next iteration of the main loop. We also can - // exit this loop via goto if we get close to exhausting the input. - for { - // Invariant: we have a 4-byte match at s, and no need to emit any - // literal bytes prior to s. - - // Extend the 4-byte match as long as possible. - t := candidate.offset - e.cur - l := e.matchlenLong(int(s+4), int(t+4), src) + 4 - - // Extend backwards - for t > 0 && s > nextEmit && src[t-1] == src[s-1] { - s-- - t-- - l++ - } - if nextEmit < s { - if false { - emitLiteral(dst, src[nextEmit:s]) - } else { - for _, v := range src[nextEmit:s] { - dst.tokens[dst.n] = token(v) - dst.litHist[v]++ - dst.n++ - } - } - } - - dst.AddMatchLong(l, uint32(s-t-baseMatchOffset)) - s += l - nextEmit = s - if nextS >= s { - s = nextS + 1 - } - - if s >= sLimit { - // Index first pair after match end. - if int(s+l+8) < len(src) { - cv := load6432(src, s) - e.table[hashLen(cv, bTableBits, hashBytes)] = tableEntry{offset: s + e.cur} - } - goto emitRemainder - } - - // Store every second hash in-between, but offset by 1. - for i := s - l + 2; i < s-5; i += 7 { - x := load6432(src, i) - nextHash := hashLen(x, bTableBits, hashBytes) - e.table[nextHash] = tableEntry{offset: e.cur + i} - // Skip one - x >>= 16 - nextHash = hashLen(x, bTableBits, hashBytes) - e.table[nextHash] = tableEntry{offset: e.cur + i + 2} - // Skip one - x >>= 16 - nextHash = hashLen(x, bTableBits, hashBytes) - e.table[nextHash] = tableEntry{offset: e.cur + i + 4} - } - - // We could immediately start working at s now, but to improve - // compression we first update the hash table at s-2 to s. If - // another emitCopy is not our next move, also calculate nextHash - // at s+1. At least on GOARCH=amd64, these three hash calculations - // are faster as one load64 call (with some shifts) instead of - // three load32 calls. - x := load6432(src, s-2) - o := e.cur + s - 2 - prevHash := hashLen(x, bTableBits, hashBytes) - prevHash2 := hashLen(x>>8, bTableBits, hashBytes) - e.table[prevHash] = tableEntry{offset: o} - e.table[prevHash2] = tableEntry{offset: o + 1} - currHash := hashLen(x>>16, bTableBits, hashBytes) - candidate = e.table[currHash] - e.table[currHash] = tableEntry{offset: o + 2} - - offset := s - (candidate.offset - e.cur) - if offset > maxMatchOffset || uint32(x>>16) != load3232(src, candidate.offset-e.cur) { - cv = x >> 24 - s++ - break - } - } - } - -emitRemainder: - if int(nextEmit) < len(src) { - // If nothing was added, don't encode literals. - if dst.n == 0 { - return - } - - emitLiteral(dst, src[nextEmit:]) - } -} diff --git a/internal/compress/flate/level3.go b/internal/compress/flate/level3.go deleted file mode 100644 index 2cef0290..00000000 --- a/internal/compress/flate/level3.go +++ /dev/null @@ -1,242 +0,0 @@ -package flate - -import "fmt" - -// fastEncL3 -type fastEncL3 struct { - fastGen - table [1 << 16]tableEntryPrev -} - -// Encode uses a similar algorithm to level 2, will check up to two candidates. -func (e *fastEncL3) Encode(dst *tokens, src []byte) { - const ( - inputMargin = 12 - 1 - minNonLiteralBlockSize = 1 + 1 + inputMargin - tableBits = 16 - tableSize = 1 << tableBits - hashBytes = 5 - ) - - if debugDeflate && e.cur < 0 { - panic(fmt.Sprint("e.cur < 0: ", e.cur)) - } - - // Protect against e.cur wraparound. - for e.cur >= bufferReset { - if len(e.hist) == 0 { - for i := range e.table[:] { - e.table[i] = tableEntryPrev{} - } - e.cur = maxMatchOffset - break - } - // Shift down everything in the table that isn't already too far away. - minOff := e.cur + int32(len(e.hist)) - maxMatchOffset - for i := range e.table[:] { - v := e.table[i] - if v.Cur.offset <= minOff { - v.Cur.offset = 0 - } else { - v.Cur.offset = v.Cur.offset - e.cur + maxMatchOffset - } - if v.Prev.offset <= minOff { - v.Prev.offset = 0 - } else { - v.Prev.offset = v.Prev.offset - e.cur + maxMatchOffset - } - e.table[i] = v - } - e.cur = maxMatchOffset - } - - s := e.addBlock(src) - - // Skip if too small. - if len(src) < minNonLiteralBlockSize { - // We do not fill the token table. - // This will be picked up by caller. - dst.n = uint16(len(src)) - return - } - - // Override src - src = e.hist - nextEmit := s - - // sLimit is when to stop looking for offset/length copies. The inputMargin - // lets us use a fast path for emitLiteral in the main loop, while we are - // looking for copies. - sLimit := int32(len(src) - inputMargin) - - // nextEmit is where in src the next emitLiteral should start from. - cv := load6432(src, s) - for { - const skipLog = 7 - nextS := s - var candidate tableEntry - for { - nextHash := hashLen(cv, tableBits, hashBytes) - s = nextS - nextS = s + 1 + (s-nextEmit)>>skipLog - if nextS > sLimit { - goto emitRemainder - } - candidates := e.table[nextHash] - now := load6432(src, nextS) - - // Safe offset distance until s + 4... - minOffset := e.cur + s - (maxMatchOffset - 4) - e.table[nextHash] = tableEntryPrev{Prev: candidates.Cur, Cur: tableEntry{offset: s + e.cur}} - - // Check both candidates - candidate = candidates.Cur - if candidate.offset < minOffset { - cv = now - // Previous will also be invalid, we have nothing. - continue - } - - if uint32(cv) == load3232(src, candidate.offset-e.cur) { - if candidates.Prev.offset < minOffset || uint32(cv) != load3232(src, candidates.Prev.offset-e.cur) { - break - } - // Both match and are valid, pick longest. - offset := s - (candidate.offset - e.cur) - o2 := s - (candidates.Prev.offset - e.cur) - l1, l2 := matchLen(src[s+4:], src[s-offset+4:]), matchLen(src[s+4:], src[s-o2+4:]) - if l2 > l1 { - candidate = candidates.Prev - } - break - } else { - // We only check if value mismatches. - // Offset will always be invalid in other cases. - candidate = candidates.Prev - if candidate.offset > minOffset && uint32(cv) == load3232(src, candidate.offset-e.cur) { - break - } - } - cv = now - } - - // Call emitCopy, and then see if another emitCopy could be our next - // move. Repeat until we find no match for the input immediately after - // what was consumed by the last emitCopy call. - // - // If we exit this loop normally then we need to call emitLiteral next, - // though we don't yet know how big the literal will be. We handle that - // by proceeding to the next iteration of the main loop. We also can - // exit this loop via goto if we get close to exhausting the input. - for { - // Invariant: we have a 4-byte match at s, and no need to emit any - // literal bytes prior to s. - - // Extend the 4-byte match as long as possible. - // - t := candidate.offset - e.cur - l := e.matchlenLong(int(s+4), int(t+4), src) + 4 - - // Extend backwards - for t > 0 && s > nextEmit && src[t-1] == src[s-1] { - s-- - t-- - l++ - } - if nextEmit < s { - if false { - emitLiteral(dst, src[nextEmit:s]) - } else { - for _, v := range src[nextEmit:s] { - dst.tokens[dst.n] = token(v) - dst.litHist[v]++ - dst.n++ - } - } - } - - dst.AddMatchLong(l, uint32(s-t-baseMatchOffset)) - s += l - nextEmit = s - if nextS >= s { - s = nextS + 1 - } - - if s >= sLimit { - t += l - // Index first pair after match end. - if int(t+8) < len(src) && t > 0 { - cv = load6432(src, t) - nextHash := hashLen(cv, tableBits, hashBytes) - e.table[nextHash] = tableEntryPrev{ - Prev: e.table[nextHash].Cur, - Cur: tableEntry{offset: e.cur + t}, - } - } - goto emitRemainder - } - - // Store every 5th hash in-between. - for i := s - l + 2; i < s-5; i += 6 { - nextHash := hashLen(load6432(src, i), tableBits, hashBytes) - e.table[nextHash] = tableEntryPrev{ - Prev: e.table[nextHash].Cur, - Cur: tableEntry{offset: e.cur + i}, - } - } - // We could immediately start working at s now, but to improve - // compression we first update the hash table at s-2 to s. - x := load6432(src, s-2) - prevHash := hashLen(x, tableBits, hashBytes) - - e.table[prevHash] = tableEntryPrev{ - Prev: e.table[prevHash].Cur, - Cur: tableEntry{offset: e.cur + s - 2}, - } - x >>= 8 - prevHash = hashLen(x, tableBits, hashBytes) - - e.table[prevHash] = tableEntryPrev{ - Prev: e.table[prevHash].Cur, - Cur: tableEntry{offset: e.cur + s - 1}, - } - x >>= 8 - currHash := hashLen(x, tableBits, hashBytes) - candidates := e.table[currHash] - cv = x - e.table[currHash] = tableEntryPrev{ - Prev: candidates.Cur, - Cur: tableEntry{offset: s + e.cur}, - } - - // Check both candidates - candidate = candidates.Cur - minOffset := e.cur + s - (maxMatchOffset - 4) - - if candidate.offset > minOffset { - if uint32(cv) == load3232(src, candidate.offset-e.cur) { - // Found a match... - continue - } - candidate = candidates.Prev - if candidate.offset > minOffset && uint32(cv) == load3232(src, candidate.offset-e.cur) { - // Match at prev... - continue - } - } - cv = x >> 8 - s++ - break - } - } - -emitRemainder: - if int(nextEmit) < len(src) { - // If nothing was added, don't encode literals. - if dst.n == 0 { - return - } - - emitLiteral(dst, src[nextEmit:]) - } -} diff --git a/internal/compress/flate/level4.go b/internal/compress/flate/level4.go deleted file mode 100644 index 88509e19..00000000 --- a/internal/compress/flate/level4.go +++ /dev/null @@ -1,221 +0,0 @@ -package flate - -import "fmt" - -type fastEncL4 struct { - fastGen - table [tableSize]tableEntry - bTable [tableSize]tableEntry -} - -func (e *fastEncL4) Encode(dst *tokens, src []byte) { - const ( - inputMargin = 12 - 1 - minNonLiteralBlockSize = 1 + 1 + inputMargin - hashShortBytes = 4 - ) - if debugDeflate && e.cur < 0 { - panic(fmt.Sprint("e.cur < 0: ", e.cur)) - } - // Protect against e.cur wraparound. - for e.cur >= bufferReset { - if len(e.hist) == 0 { - for i := range e.table[:] { - e.table[i] = tableEntry{} - } - for i := range e.bTable[:] { - e.bTable[i] = tableEntry{} - } - e.cur = maxMatchOffset - break - } - // Shift down everything in the table that isn't already too far away. - minOff := e.cur + int32(len(e.hist)) - maxMatchOffset - for i := range e.table[:] { - v := e.table[i].offset - if v <= minOff { - v = 0 - } else { - v = v - e.cur + maxMatchOffset - } - e.table[i].offset = v - } - for i := range e.bTable[:] { - v := e.bTable[i].offset - if v <= minOff { - v = 0 - } else { - v = v - e.cur + maxMatchOffset - } - e.bTable[i].offset = v - } - e.cur = maxMatchOffset - } - - s := e.addBlock(src) - - // This check isn't in the Snappy implementation, but there, the caller - // instead of the callee handles this case. - if len(src) < minNonLiteralBlockSize { - // We do not fill the token table. - // This will be picked up by caller. - dst.n = uint16(len(src)) - return - } - - // Override src - src = e.hist - nextEmit := s - - // sLimit is when to stop looking for offset/length copies. The inputMargin - // lets us use a fast path for emitLiteral in the main loop, while we are - // looking for copies. - sLimit := int32(len(src) - inputMargin) - - // nextEmit is where in src the next emitLiteral should start from. - cv := load6432(src, s) - for { - const skipLog = 6 - const doEvery = 1 - - nextS := s - var t int32 - for { - nextHashS := hashLen(cv, tableBits, hashShortBytes) - nextHashL := hash7(cv, tableBits) - - s = nextS - nextS = s + doEvery + (s-nextEmit)>>skipLog - if nextS > sLimit { - goto emitRemainder - } - // Fetch a short+long candidate - sCandidate := e.table[nextHashS] - lCandidate := e.bTable[nextHashL] - next := load6432(src, nextS) - entry := tableEntry{offset: s + e.cur} - e.table[nextHashS] = entry - e.bTable[nextHashL] = entry - - t = lCandidate.offset - e.cur - if s-t < maxMatchOffset && uint32(cv) == load3232(src, t) { - // We got a long match. Use that. - break - } - - t = sCandidate.offset - e.cur - if s-t < maxMatchOffset && uint32(cv) == load3232(src, t) { - // Found a 4 match... - lCandidate = e.bTable[hash7(next, tableBits)] - - // If the next long is a candidate, check if we should use that instead... - lOff := lCandidate.offset - e.cur - if nextS-lOff < maxMatchOffset && load3232(src, lOff) == uint32(next) { - l1, l2 := matchLen(src[s+4:], src[t+4:]), matchLen(src[nextS+4:], src[nextS-lOff+4:]) - if l2 > l1 { - s = nextS - t = lCandidate.offset - e.cur - } - } - break - } - cv = next - } - - // A 4-byte match has been found. We'll later see if more than 4 bytes - // match. But, prior to the match, src[nextEmit:s] are unmatched. Emit - // them as literal bytes. - - // Extend the 4-byte match as long as possible. - l := e.matchlenLong(int(s+4), int(t+4), src) + 4 - - // Extend backwards - for t > 0 && s > nextEmit && src[t-1] == src[s-1] { - s-- - t-- - l++ - } - if nextEmit < s { - if false { - emitLiteral(dst, src[nextEmit:s]) - } else { - for _, v := range src[nextEmit:s] { - dst.tokens[dst.n] = token(v) - dst.litHist[v]++ - dst.n++ - } - } - } - if debugDeflate { - if t >= s { - panic("s-t") - } - if (s - t) > maxMatchOffset { - panic(fmt.Sprintln("mmo", t)) - } - if l < baseMatchLength { - panic("bml") - } - } - - dst.AddMatchLong(l, uint32(s-t-baseMatchOffset)) - s += l - nextEmit = s - if nextS >= s { - s = nextS + 1 - } - - if s >= sLimit { - // Index first pair after match end. - if int(s+8) < len(src) { - cv := load6432(src, s) - e.table[hashLen(cv, tableBits, hashShortBytes)] = tableEntry{offset: s + e.cur} - e.bTable[hash7(cv, tableBits)] = tableEntry{offset: s + e.cur} - } - goto emitRemainder - } - - // Store every 3rd hash in-between - if true { - i := nextS - if i < s-1 { - cv := load6432(src, i) - t := tableEntry{offset: i + e.cur} - t2 := tableEntry{offset: t.offset + 1} - e.bTable[hash7(cv, tableBits)] = t - e.bTable[hash7(cv>>8, tableBits)] = t2 - e.table[hashLen(cv>>8, tableBits, hashShortBytes)] = t2 - - i += 3 - for ; i < s-1; i += 3 { - cv := load6432(src, i) - t := tableEntry{offset: i + e.cur} - t2 := tableEntry{offset: t.offset + 1} - e.bTable[hash7(cv, tableBits)] = t - e.bTable[hash7(cv>>8, tableBits)] = t2 - e.table[hashLen(cv>>8, tableBits, hashShortBytes)] = t2 - } - } - } - - // We could immediately start working at s now, but to improve - // compression we first update the hash table at s-1 and at s. - x := load6432(src, s-1) - o := e.cur + s - 1 - prevHashS := hashLen(x, tableBits, hashShortBytes) - prevHashL := hash7(x, tableBits) - e.table[prevHashS] = tableEntry{offset: o} - e.bTable[prevHashL] = tableEntry{offset: o} - cv = x >> 8 - } - -emitRemainder: - if int(nextEmit) < len(src) { - // If nothing was added, don't encode literals. - if dst.n == 0 { - return - } - - emitLiteral(dst, src[nextEmit:]) - } -} diff --git a/internal/compress/flate/level5.go b/internal/compress/flate/level5.go deleted file mode 100644 index a22ad7d1..00000000 --- a/internal/compress/flate/level5.go +++ /dev/null @@ -1,705 +0,0 @@ -package flate - -import "fmt" - -type fastEncL5 struct { - fastGen - table [tableSize]tableEntry - bTable [tableSize]tableEntryPrev -} - -func (e *fastEncL5) Encode(dst *tokens, src []byte) { - const ( - inputMargin = 12 - 1 - minNonLiteralBlockSize = 1 + 1 + inputMargin - hashShortBytes = 4 - ) - if debugDeflate && e.cur < 0 { - panic(fmt.Sprint("e.cur < 0: ", e.cur)) - } - - // Protect against e.cur wraparound. - for e.cur >= bufferReset { - if len(e.hist) == 0 { - for i := range e.table[:] { - e.table[i] = tableEntry{} - } - for i := range e.bTable[:] { - e.bTable[i] = tableEntryPrev{} - } - e.cur = maxMatchOffset - break - } - // Shift down everything in the table that isn't already too far away. - minOff := e.cur + int32(len(e.hist)) - maxMatchOffset - for i := range e.table[:] { - v := e.table[i].offset - if v <= minOff { - v = 0 - } else { - v = v - e.cur + maxMatchOffset - } - e.table[i].offset = v - } - for i := range e.bTable[:] { - v := e.bTable[i] - if v.Cur.offset <= minOff { - v.Cur.offset = 0 - v.Prev.offset = 0 - } else { - v.Cur.offset = v.Cur.offset - e.cur + maxMatchOffset - if v.Prev.offset <= minOff { - v.Prev.offset = 0 - } else { - v.Prev.offset = v.Prev.offset - e.cur + maxMatchOffset - } - } - e.bTable[i] = v - } - e.cur = maxMatchOffset - } - - s := e.addBlock(src) - - // This check isn't in the Snappy implementation, but there, the caller - // instead of the callee handles this case. - if len(src) < minNonLiteralBlockSize { - // We do not fill the token table. - // This will be picked up by caller. - dst.n = uint16(len(src)) - return - } - - // Override src - src = e.hist - nextEmit := s - - // sLimit is when to stop looking for offset/length copies. The inputMargin - // lets us use a fast path for emitLiteral in the main loop, while we are - // looking for copies. - sLimit := int32(len(src) - inputMargin) - - // nextEmit is where in src the next emitLiteral should start from. - cv := load6432(src, s) - for { - const skipLog = 6 - const doEvery = 1 - - nextS := s - var l int32 - var t int32 - for { - nextHashS := hashLen(cv, tableBits, hashShortBytes) - nextHashL := hash7(cv, tableBits) - - s = nextS - nextS = s + doEvery + (s-nextEmit)>>skipLog - if nextS > sLimit { - goto emitRemainder - } - // Fetch a short+long candidate - sCandidate := e.table[nextHashS] - lCandidate := e.bTable[nextHashL] - next := load6432(src, nextS) - entry := tableEntry{offset: s + e.cur} - e.table[nextHashS] = entry - eLong := &e.bTable[nextHashL] - eLong.Cur, eLong.Prev = entry, eLong.Cur - - nextHashS = hashLen(next, tableBits, hashShortBytes) - nextHashL = hash7(next, tableBits) - - t = lCandidate.Cur.offset - e.cur - if s-t < maxMatchOffset { - if uint32(cv) == load3232(src, t) { - // Store the next match - e.table[nextHashS] = tableEntry{offset: nextS + e.cur} - eLong := &e.bTable[nextHashL] - eLong.Cur, eLong.Prev = tableEntry{offset: nextS + e.cur}, eLong.Cur - - t2 := lCandidate.Prev.offset - e.cur - if s-t2 < maxMatchOffset && uint32(cv) == load3232(src, t2) { - l = e.matchlen(int(s+4), int(t+4), src) + 4 - ml1 := e.matchlen(int(s+4), int(t2+4), src) + 4 - if ml1 > l { - t = t2 - l = ml1 - break - } - } - break - } - t = lCandidate.Prev.offset - e.cur - if s-t < maxMatchOffset && uint32(cv) == load3232(src, t) { - // Store the next match - e.table[nextHashS] = tableEntry{offset: nextS + e.cur} - eLong := &e.bTable[nextHashL] - eLong.Cur, eLong.Prev = tableEntry{offset: nextS + e.cur}, eLong.Cur - break - } - } - - t = sCandidate.offset - e.cur - if s-t < maxMatchOffset && uint32(cv) == load3232(src, t) { - // Found a 4 match... - l = e.matchlen(int(s+4), int(t+4), src) + 4 - lCandidate = e.bTable[nextHashL] - // Store the next match - - e.table[nextHashS] = tableEntry{offset: nextS + e.cur} - eLong := &e.bTable[nextHashL] - eLong.Cur, eLong.Prev = tableEntry{offset: nextS + e.cur}, eLong.Cur - - // If the next long is a candidate, use that... - t2 := lCandidate.Cur.offset - e.cur - if nextS-t2 < maxMatchOffset { - if load3232(src, t2) == uint32(next) { - ml := e.matchlen(int(nextS+4), int(t2+4), src) + 4 - if ml > l { - t = t2 - s = nextS - l = ml - break - } - } - // If the previous long is a candidate, use that... - t2 = lCandidate.Prev.offset - e.cur - if nextS-t2 < maxMatchOffset && load3232(src, t2) == uint32(next) { - ml := e.matchlen(int(nextS+4), int(t2+4), src) + 4 - if ml > l { - t = t2 - s = nextS - l = ml - break - } - } - } - break - } - cv = next - } - - // A 4-byte match has been found. We'll later see if more than 4 bytes - // match. But, prior to the match, src[nextEmit:s] are unmatched. Emit - // them as literal bytes. - - if l == 0 { - // Extend the 4-byte match as long as possible. - l = e.matchlenLong(int(s+4), int(t+4), src) + 4 - } else if l == maxMatchLength { - l += e.matchlenLong(int(s+l), int(t+l), src) - } - - // Try to locate a better match by checking the end of best match... - if sAt := s + l; l < 30 && sAt < sLimit { - // Allow some bytes at the beginning to mismatch. - // Sweet spot is 2/3 bytes depending on input. - // 3 is only a little better when it is but sometimes a lot worse. - // The skipped bytes are tested in Extend backwards, - // and still picked up as part of the match if they do. - const skipBeginning = 2 - eLong := e.bTable[hash7(load6432(src, sAt), tableBits)].Cur.offset - t2 := eLong - e.cur - l + skipBeginning - s2 := s + skipBeginning - off := s2 - t2 - if t2 >= 0 && off < maxMatchOffset && off > 0 { - if l2 := e.matchlenLong(int(s2), int(t2), src); l2 > l { - t = t2 - l = l2 - s = s2 - } - } - } - - // Extend backwards - for t > 0 && s > nextEmit && src[t-1] == src[s-1] { - s-- - t-- - l++ - } - if nextEmit < s { - if false { - emitLiteral(dst, src[nextEmit:s]) - } else { - for _, v := range src[nextEmit:s] { - dst.tokens[dst.n] = token(v) - dst.litHist[v]++ - dst.n++ - } - } - } - if debugDeflate { - if t >= s { - panic(fmt.Sprintln("s-t", s, t)) - } - if (s - t) > maxMatchOffset { - panic(fmt.Sprintln("mmo", s-t)) - } - if l < baseMatchLength { - panic("bml") - } - } - - dst.AddMatchLong(l, uint32(s-t-baseMatchOffset)) - s += l - nextEmit = s - if nextS >= s { - s = nextS + 1 - } - - if s >= sLimit { - goto emitRemainder - } - - // Store every 3rd hash in-between. - if true { - const hashEvery = 3 - i := s - l + 1 - if i < s-1 { - cv := load6432(src, i) - t := tableEntry{offset: i + e.cur} - e.table[hashLen(cv, tableBits, hashShortBytes)] = t - eLong := &e.bTable[hash7(cv, tableBits)] - eLong.Cur, eLong.Prev = t, eLong.Cur - - // Do an long at i+1 - cv >>= 8 - t = tableEntry{offset: t.offset + 1} - eLong = &e.bTable[hash7(cv, tableBits)] - eLong.Cur, eLong.Prev = t, eLong.Cur - - // We only have enough bits for a short entry at i+2 - cv >>= 8 - t = tableEntry{offset: t.offset + 1} - e.table[hashLen(cv, tableBits, hashShortBytes)] = t - - // Skip one - otherwise we risk hitting 's' - i += 4 - for ; i < s-1; i += hashEvery { - cv := load6432(src, i) - t := tableEntry{offset: i + e.cur} - t2 := tableEntry{offset: t.offset + 1} - eLong := &e.bTable[hash7(cv, tableBits)] - eLong.Cur, eLong.Prev = t, eLong.Cur - e.table[hashLen(cv>>8, tableBits, hashShortBytes)] = t2 - } - } - } - - // We could immediately start working at s now, but to improve - // compression we first update the hash table at s-1 and at s. - x := load6432(src, s-1) - o := e.cur + s - 1 - prevHashS := hashLen(x, tableBits, hashShortBytes) - prevHashL := hash7(x, tableBits) - e.table[prevHashS] = tableEntry{offset: o} - eLong := &e.bTable[prevHashL] - eLong.Cur, eLong.Prev = tableEntry{offset: o}, eLong.Cur - cv = x >> 8 - } - -emitRemainder: - if int(nextEmit) < len(src) { - // If nothing was added, don't encode literals. - if dst.n == 0 { - return - } - - emitLiteral(dst, src[nextEmit:]) - } -} - -// fastEncL5Window is a level 5 encoder, -// but with a custom window size. -type fastEncL5Window struct { - hist []byte - cur int32 - maxOffset int32 - table [tableSize]tableEntry - bTable [tableSize]tableEntryPrev -} - -func (e *fastEncL5Window) Encode(dst *tokens, src []byte) { - const ( - inputMargin = 12 - 1 - minNonLiteralBlockSize = 1 + 1 + inputMargin - hashShortBytes = 4 - ) - maxMatchOffset := e.maxOffset - if debugDeflate && e.cur < 0 { - panic(fmt.Sprint("e.cur < 0: ", e.cur)) - } - - // Protect against e.cur wraparound. - for e.cur >= bufferReset { - if len(e.hist) == 0 { - for i := range e.table[:] { - e.table[i] = tableEntry{} - } - for i := range e.bTable[:] { - e.bTable[i] = tableEntryPrev{} - } - e.cur = maxMatchOffset - break - } - // Shift down everything in the table that isn't already too far away. - minOff := e.cur + int32(len(e.hist)) - maxMatchOffset - for i := range e.table[:] { - v := e.table[i].offset - if v <= minOff { - v = 0 - } else { - v = v - e.cur + maxMatchOffset - } - e.table[i].offset = v - } - for i := range e.bTable[:] { - v := e.bTable[i] - if v.Cur.offset <= minOff { - v.Cur.offset = 0 - v.Prev.offset = 0 - } else { - v.Cur.offset = v.Cur.offset - e.cur + maxMatchOffset - if v.Prev.offset <= minOff { - v.Prev.offset = 0 - } else { - v.Prev.offset = v.Prev.offset - e.cur + maxMatchOffset - } - } - e.bTable[i] = v - } - e.cur = maxMatchOffset - } - - s := e.addBlock(src) - - // This check isn't in the Snappy implementation, but there, the caller - // instead of the callee handles this case. - if len(src) < minNonLiteralBlockSize { - // We do not fill the token table. - // This will be picked up by caller. - dst.n = uint16(len(src)) - return - } - - // Override src - src = e.hist - nextEmit := s - - // sLimit is when to stop looking for offset/length copies. The inputMargin - // lets us use a fast path for emitLiteral in the main loop, while we are - // looking for copies. - sLimit := int32(len(src) - inputMargin) - - // nextEmit is where in src the next emitLiteral should start from. - cv := load6432(src, s) - for { - const skipLog = 6 - const doEvery = 1 - - nextS := s - var l int32 - var t int32 - for { - nextHashS := hashLen(cv, tableBits, hashShortBytes) - nextHashL := hash7(cv, tableBits) - - s = nextS - nextS = s + doEvery + (s-nextEmit)>>skipLog - if nextS > sLimit { - goto emitRemainder - } - // Fetch a short+long candidate - sCandidate := e.table[nextHashS] - lCandidate := e.bTable[nextHashL] - next := load6432(src, nextS) - entry := tableEntry{offset: s + e.cur} - e.table[nextHashS] = entry - eLong := &e.bTable[nextHashL] - eLong.Cur, eLong.Prev = entry, eLong.Cur - - nextHashS = hashLen(next, tableBits, hashShortBytes) - nextHashL = hash7(next, tableBits) - - t = lCandidate.Cur.offset - e.cur - if s-t < maxMatchOffset { - if uint32(cv) == load3232(src, t) { - // Store the next match - e.table[nextHashS] = tableEntry{offset: nextS + e.cur} - eLong := &e.bTable[nextHashL] - eLong.Cur, eLong.Prev = tableEntry{offset: nextS + e.cur}, eLong.Cur - - t2 := lCandidate.Prev.offset - e.cur - if s-t2 < maxMatchOffset && uint32(cv) == load3232(src, t2) { - l = e.matchlen(s+4, t+4, src) + 4 - ml1 := e.matchlen(s+4, t2+4, src) + 4 - if ml1 > l { - t = t2 - l = ml1 - break - } - } - break - } - t = lCandidate.Prev.offset - e.cur - if s-t < maxMatchOffset && uint32(cv) == load3232(src, t) { - // Store the next match - e.table[nextHashS] = tableEntry{offset: nextS + e.cur} - eLong := &e.bTable[nextHashL] - eLong.Cur, eLong.Prev = tableEntry{offset: nextS + e.cur}, eLong.Cur - break - } - } - - t = sCandidate.offset - e.cur - if s-t < maxMatchOffset && uint32(cv) == load3232(src, t) { - // Found a 4 match... - l = e.matchlen(s+4, t+4, src) + 4 - lCandidate = e.bTable[nextHashL] - // Store the next match - - e.table[nextHashS] = tableEntry{offset: nextS + e.cur} - eLong := &e.bTable[nextHashL] - eLong.Cur, eLong.Prev = tableEntry{offset: nextS + e.cur}, eLong.Cur - - // If the next long is a candidate, use that... - t2 := lCandidate.Cur.offset - e.cur - if nextS-t2 < maxMatchOffset { - if load3232(src, t2) == uint32(next) { - ml := e.matchlen(nextS+4, t2+4, src) + 4 - if ml > l { - t = t2 - s = nextS - l = ml - break - } - } - // If the previous long is a candidate, use that... - t2 = lCandidate.Prev.offset - e.cur - if nextS-t2 < maxMatchOffset && load3232(src, t2) == uint32(next) { - ml := e.matchlen(nextS+4, t2+4, src) + 4 - if ml > l { - t = t2 - s = nextS - l = ml - break - } - } - } - break - } - cv = next - } - - // A 4-byte match has been found. We'll later see if more than 4 bytes - // match. But, prior to the match, src[nextEmit:s] are unmatched. Emit - // them as literal bytes. - - if l == 0 { - // Extend the 4-byte match as long as possible. - l = e.matchlenLong(s+4, t+4, src) + 4 - } else if l == maxMatchLength { - l += e.matchlenLong(s+l, t+l, src) - } - - // Try to locate a better match by checking the end of best match... - if sAt := s + l; l < 30 && sAt < sLimit { - // Allow some bytes at the beginning to mismatch. - // Sweet spot is 2/3 bytes depending on input. - // 3 is only a little better when it is but sometimes a lot worse. - // The skipped bytes are tested in Extend backwards, - // and still picked up as part of the match if they do. - const skipBeginning = 2 - eLong := e.bTable[hash7(load6432(src, sAt), tableBits)].Cur.offset - t2 := eLong - e.cur - l + skipBeginning - s2 := s + skipBeginning - off := s2 - t2 - if t2 >= 0 && off < maxMatchOffset && off > 0 { - if l2 := e.matchlenLong(s2, t2, src); l2 > l { - t = t2 - l = l2 - s = s2 - } - } - } - - // Extend backwards - for t > 0 && s > nextEmit && src[t-1] == src[s-1] { - s-- - t-- - l++ - } - if nextEmit < s { - if false { - emitLiteral(dst, src[nextEmit:s]) - } else { - for _, v := range src[nextEmit:s] { - dst.tokens[dst.n] = token(v) - dst.litHist[v]++ - dst.n++ - } - } - } - if debugDeflate { - if t >= s { - panic(fmt.Sprintln("s-t", s, t)) - } - if (s - t) > maxMatchOffset { - panic(fmt.Sprintln("mmo", s-t)) - } - if l < baseMatchLength { - panic("bml") - } - } - - dst.AddMatchLong(l, uint32(s-t-baseMatchOffset)) - s += l - nextEmit = s - if nextS >= s { - s = nextS + 1 - } - - if s >= sLimit { - goto emitRemainder - } - - // Store every 3rd hash in-between. - if true { - const hashEvery = 3 - i := s - l + 1 - if i < s-1 { - cv := load6432(src, i) - t := tableEntry{offset: i + e.cur} - e.table[hashLen(cv, tableBits, hashShortBytes)] = t - eLong := &e.bTable[hash7(cv, tableBits)] - eLong.Cur, eLong.Prev = t, eLong.Cur - - // Do an long at i+1 - cv >>= 8 - t = tableEntry{offset: t.offset + 1} - eLong = &e.bTable[hash7(cv, tableBits)] - eLong.Cur, eLong.Prev = t, eLong.Cur - - // We only have enough bits for a short entry at i+2 - cv >>= 8 - t = tableEntry{offset: t.offset + 1} - e.table[hashLen(cv, tableBits, hashShortBytes)] = t - - // Skip one - otherwise we risk hitting 's' - i += 4 - for ; i < s-1; i += hashEvery { - cv := load6432(src, i) - t := tableEntry{offset: i + e.cur} - t2 := tableEntry{offset: t.offset + 1} - eLong := &e.bTable[hash7(cv, tableBits)] - eLong.Cur, eLong.Prev = t, eLong.Cur - e.table[hashLen(cv>>8, tableBits, hashShortBytes)] = t2 - } - } - } - - // We could immediately start working at s now, but to improve - // compression we first update the hash table at s-1 and at s. - x := load6432(src, s-1) - o := e.cur + s - 1 - prevHashS := hashLen(x, tableBits, hashShortBytes) - prevHashL := hash7(x, tableBits) - e.table[prevHashS] = tableEntry{offset: o} - eLong := &e.bTable[prevHashL] - eLong.Cur, eLong.Prev = tableEntry{offset: o}, eLong.Cur - cv = x >> 8 - } - -emitRemainder: - if int(nextEmit) < len(src) { - // If nothing was added, don't encode literals. - if dst.n == 0 { - return - } - - emitLiteral(dst, src[nextEmit:]) - } -} - -// Reset the encoding table. -func (e *fastEncL5Window) Reset() { - // We keep the same allocs, since we are compressing the same block sizes. - if cap(e.hist) < allocHistory { - e.hist = make([]byte, 0, allocHistory) - } - - // We offset current position so everything will be out of reach. - // If we are above the buffer reset it will be cleared anyway since len(hist) == 0. - if e.cur <= int32(bufferReset) { - e.cur += e.maxOffset + int32(len(e.hist)) - } - e.hist = e.hist[:0] -} - -func (e *fastEncL5Window) addBlock(src []byte) int32 { - // check if we have space already - maxMatchOffset := e.maxOffset - - if len(e.hist)+len(src) > cap(e.hist) { - if cap(e.hist) == 0 { - e.hist = make([]byte, 0, allocHistory) - } else { - if cap(e.hist) < int(maxMatchOffset*2) { - panic("unexpected buffer size") - } - // Move down - offset := int32(len(e.hist)) - maxMatchOffset - copy(e.hist[0:maxMatchOffset], e.hist[offset:]) - e.cur += offset - e.hist = e.hist[:maxMatchOffset] - } - } - s := int32(len(e.hist)) - e.hist = append(e.hist, src...) - return s -} - -// matchlen will return the match length between offsets and t in src. -// The maximum length returned is maxMatchLength - 4. -// It is assumed that s > t, that t >=0 and s < len(src). -func (e *fastEncL5Window) matchlen(s, t int32, src []byte) int32 { - if debugDecode { - if t >= s { - panic(fmt.Sprint("t >=s:", t, s)) - } - if int(s) >= len(src) { - panic(fmt.Sprint("s >= len(src):", s, len(src))) - } - if t < 0 { - panic(fmt.Sprint("t < 0:", t)) - } - if s-t > e.maxOffset { - panic(fmt.Sprint(s, "-", t, "(", s-t, ") > maxMatchLength (", maxMatchOffset, ")")) - } - } - s1 := min(int(s)+maxMatchLength-4, len(src)) - - // Extend the match to be as long as possible. - return int32(matchLen(src[s:s1], src[t:])) -} - -// matchlenLong will return the match length between offsets and t in src. -// It is assumed that s > t, that t >=0 and s < len(src). -func (e *fastEncL5Window) matchlenLong(s, t int32, src []byte) int32 { - if debugDeflate { - if t >= s { - panic(fmt.Sprint("t >=s:", t, s)) - } - if int(s) >= len(src) { - panic(fmt.Sprint("s >= len(src):", s, len(src))) - } - if t < 0 { - panic(fmt.Sprint("t < 0:", t)) - } - if s-t > e.maxOffset { - panic(fmt.Sprint(s, "-", t, "(", s-t, ") > maxMatchLength (", maxMatchOffset, ")")) - } - } - // Extend the match to be as long as possible. - return int32(matchLen(src[s:], src[t:])) -} diff --git a/internal/compress/flate/level6.go b/internal/compress/flate/level6.go deleted file mode 100644 index 96f5bb43..00000000 --- a/internal/compress/flate/level6.go +++ /dev/null @@ -1,325 +0,0 @@ -package flate - -import "fmt" - -type fastEncL6 struct { - fastGen - table [tableSize]tableEntry - bTable [tableSize]tableEntryPrev -} - -func (e *fastEncL6) Encode(dst *tokens, src []byte) { - const ( - inputMargin = 12 - 1 - minNonLiteralBlockSize = 1 + 1 + inputMargin - hashShortBytes = 4 - ) - if debugDeflate && e.cur < 0 { - panic(fmt.Sprint("e.cur < 0: ", e.cur)) - } - - // Protect against e.cur wraparound. - for e.cur >= bufferReset { - if len(e.hist) == 0 { - for i := range e.table[:] { - e.table[i] = tableEntry{} - } - for i := range e.bTable[:] { - e.bTable[i] = tableEntryPrev{} - } - e.cur = maxMatchOffset - break - } - // Shift down everything in the table that isn't already too far away. - minOff := e.cur + int32(len(e.hist)) - maxMatchOffset - for i := range e.table[:] { - v := e.table[i].offset - if v <= minOff { - v = 0 - } else { - v = v - e.cur + maxMatchOffset - } - e.table[i].offset = v - } - for i := range e.bTable[:] { - v := e.bTable[i] - if v.Cur.offset <= minOff { - v.Cur.offset = 0 - v.Prev.offset = 0 - } else { - v.Cur.offset = v.Cur.offset - e.cur + maxMatchOffset - if v.Prev.offset <= minOff { - v.Prev.offset = 0 - } else { - v.Prev.offset = v.Prev.offset - e.cur + maxMatchOffset - } - } - e.bTable[i] = v - } - e.cur = maxMatchOffset - } - - s := e.addBlock(src) - - // This check isn't in the Snappy implementation, but there, the caller - // instead of the callee handles this case. - if len(src) < minNonLiteralBlockSize { - // We do not fill the token table. - // This will be picked up by caller. - dst.n = uint16(len(src)) - return - } - - // Override src - src = e.hist - nextEmit := s - - // sLimit is when to stop looking for offset/length copies. The inputMargin - // lets us use a fast path for emitLiteral in the main loop, while we are - // looking for copies. - sLimit := int32(len(src) - inputMargin) - - // nextEmit is where in src the next emitLiteral should start from. - cv := load6432(src, s) - // Repeat MUST be > 1 and within range - repeat := int32(1) - for { - const skipLog = 7 - const doEvery = 1 - - nextS := s - var l int32 - var t int32 - for { - nextHashS := hashLen(cv, tableBits, hashShortBytes) - nextHashL := hash7(cv, tableBits) - s = nextS - nextS = s + doEvery + (s-nextEmit)>>skipLog - if nextS > sLimit { - goto emitRemainder - } - // Fetch a short+long candidate - sCandidate := e.table[nextHashS] - lCandidate := e.bTable[nextHashL] - next := load6432(src, nextS) - entry := tableEntry{offset: s + e.cur} - e.table[nextHashS] = entry - eLong := &e.bTable[nextHashL] - eLong.Cur, eLong.Prev = entry, eLong.Cur - - // Calculate hashes of 'next' - nextHashS = hashLen(next, tableBits, hashShortBytes) - nextHashL = hash7(next, tableBits) - - t = lCandidate.Cur.offset - e.cur - if s-t < maxMatchOffset { - if uint32(cv) == load3232(src, t) { - // Long candidate matches at least 4 bytes. - - // Store the next match - e.table[nextHashS] = tableEntry{offset: nextS + e.cur} - eLong := &e.bTable[nextHashL] - eLong.Cur, eLong.Prev = tableEntry{offset: nextS + e.cur}, eLong.Cur - - // Check the previous long candidate as well. - t2 := lCandidate.Prev.offset - e.cur - if s-t2 < maxMatchOffset && uint32(cv) == load3232(src, t2) { - l = e.matchlen(int(s+4), int(t+4), src) + 4 - ml1 := e.matchlen(int(s+4), int(t2+4), src) + 4 - if ml1 > l { - t = t2 - l = ml1 - break - } - } - break - } - // Current value did not match, but check if previous long value does. - t = lCandidate.Prev.offset - e.cur - if s-t < maxMatchOffset && uint32(cv) == load3232(src, t) { - // Store the next match - e.table[nextHashS] = tableEntry{offset: nextS + e.cur} - eLong := &e.bTable[nextHashL] - eLong.Cur, eLong.Prev = tableEntry{offset: nextS + e.cur}, eLong.Cur - break - } - } - - t = sCandidate.offset - e.cur - if s-t < maxMatchOffset && uint32(cv) == load3232(src, t) { - // Found a 4 match... - l = e.matchlen(int(s+4), int(t+4), src) + 4 - - // Look up next long candidate (at nextS) - lCandidate = e.bTable[nextHashL] - - // Store the next match - e.table[nextHashS] = tableEntry{offset: nextS + e.cur} - eLong := &e.bTable[nextHashL] - eLong.Cur, eLong.Prev = tableEntry{offset: nextS + e.cur}, eLong.Cur - - // Check repeat at s + repOff - const repOff = 1 - t2 := s - repeat + repOff - if load3232(src, t2) == uint32(cv>>(8*repOff)) { - ml := e.matchlen(int(s+4+repOff), int(t2+4), src) + 4 - if ml > l { - t = t2 - l = ml - s += repOff - // Not worth checking more. - break - } - } - - // If the next long is a candidate, use that... - t2 = lCandidate.Cur.offset - e.cur - if nextS-t2 < maxMatchOffset { - if load3232(src, t2) == uint32(next) { - ml := e.matchlen(int(nextS+4), int(t2+4), src) + 4 - if ml > l { - t = t2 - s = nextS - l = ml - // This is ok, but check previous as well. - } - } - // If the previous long is a candidate, use that... - t2 = lCandidate.Prev.offset - e.cur - if nextS-t2 < maxMatchOffset && load3232(src, t2) == uint32(next) { - ml := e.matchlen(int(nextS+4), int(t2+4), src) + 4 - if ml > l { - t = t2 - s = nextS - l = ml - break - } - } - } - break - } - cv = next - } - - // A 4-byte match has been found. We'll later see if more than 4 bytes - // match. But, prior to the match, src[nextEmit:s] are unmatched. Emit - // them as literal bytes. - - // Extend the 4-byte match as long as possible. - if l == 0 { - l = e.matchlenLong(int(s+4), int(t+4), src) + 4 - } else if l == maxMatchLength { - l += e.matchlenLong(int(s+l), int(t+l), src) - } - - // Try to locate a better match by checking the end-of-match... - if sAt := s + l; sAt < sLimit { - // Allow some bytes at the beginning to mismatch. - // Sweet spot is 2/3 bytes depending on input. - // 3 is only a little better when it is but sometimes a lot worse. - // The skipped bytes are tested in Extend backwards, - // and still picked up as part of the match if they do. - const skipBeginning = 2 - eLong := &e.bTable[hash7(load6432(src, sAt), tableBits)] - // Test current - t2 := eLong.Cur.offset - e.cur - l + skipBeginning - s2 := s + skipBeginning - off := s2 - t2 - if off < maxMatchOffset { - if off > 0 && t2 >= 0 { - if l2 := e.matchlenLong(int(s2), int(t2), src); l2 > l { - t = t2 - l = l2 - s = s2 - } - } - // Test next: - t2 = eLong.Prev.offset - e.cur - l + skipBeginning - off := s2 - t2 - if off > 0 && off < maxMatchOffset && t2 >= 0 { - if l2 := e.matchlenLong(int(s2), int(t2), src); l2 > l { - t = t2 - l = l2 - s = s2 - } - } - } - } - - // Extend backwards - for t > 0 && s > nextEmit && src[t-1] == src[s-1] { - s-- - t-- - l++ - } - if nextEmit < s { - if false { - emitLiteral(dst, src[nextEmit:s]) - } else { - for _, v := range src[nextEmit:s] { - dst.tokens[dst.n] = token(v) - dst.litHist[v]++ - dst.n++ - } - } - } - if false { - if t >= s { - panic(fmt.Sprintln("s-t", s, t)) - } - if (s - t) > maxMatchOffset { - panic(fmt.Sprintln("mmo", s-t)) - } - if l < baseMatchLength { - panic("bml") - } - } - - dst.AddMatchLong(l, uint32(s-t-baseMatchOffset)) - repeat = s - t - s += l - nextEmit = s - if nextS >= s { - s = nextS + 1 - } - - if s >= sLimit { - // Index after match end. - for i := nextS + 1; i < int32(len(src))-8; i += 2 { - cv := load6432(src, i) - e.table[hashLen(cv, tableBits, hashShortBytes)] = tableEntry{offset: i + e.cur} - eLong := &e.bTable[hash7(cv, tableBits)] - eLong.Cur, eLong.Prev = tableEntry{offset: i + e.cur}, eLong.Cur - } - goto emitRemainder - } - - // Store every long hash in-between and every second short. - if true { - for i := nextS + 1; i < s-1; i += 2 { - cv := load6432(src, i) - t := tableEntry{offset: i + e.cur} - t2 := tableEntry{offset: t.offset + 1} - eLong := &e.bTable[hash7(cv, tableBits)] - eLong2 := &e.bTable[hash7(cv>>8, tableBits)] - e.table[hashLen(cv, tableBits, hashShortBytes)] = t - eLong.Cur, eLong.Prev = t, eLong.Cur - eLong2.Cur, eLong2.Prev = t2, eLong2.Cur - } - } - - // We could immediately start working at s now, but to improve - // compression we first update the hash table at s-1 and at s. - cv = load6432(src, s) - } - -emitRemainder: - if int(nextEmit) < len(src) { - // If nothing was added, don't encode literals. - if dst.n == 0 { - return - } - - emitLiteral(dst, src[nextEmit:]) - } -} diff --git a/internal/compress/flate/matchlen_generic.go b/internal/compress/flate/matchlen_generic.go deleted file mode 100644 index 0ccaeb93..00000000 --- a/internal/compress/flate/matchlen_generic.go +++ /dev/null @@ -1,34 +0,0 @@ -// Copyright 2019+ Klaus Post. All rights reserved. -// License information can be found in the LICENSE file. - -package flate - -import ( - "math/bits" - - "lindenii.org/go/furgit/internal/compress/internal/le" -) - -// matchLen returns the maximum common prefix length of a and b. -// a must be the shortest of the two. -func matchLen(a, b []byte) (n int) { - left := len(a) - for left >= 8 { - diff := le.Load64(a, n) ^ le.Load64(b, n) - if diff != 0 { - return n + bits.TrailingZeros64(diff)>>3 - } - n += 8 - left -= 8 - } - - a = a[n:] - b = b[n:] - for i := range a { - if a[i] != b[i] { - break - } - n++ - } - return n -} diff --git a/internal/compress/flate/reader_test.go b/internal/compress/flate/reader_test.go deleted file mode 100644 index 6eedfb9b..00000000 --- a/internal/compress/flate/reader_test.go +++ /dev/null @@ -1,108 +0,0 @@ -// Copyright 2012 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package flate - -import ( - "bytes" - "io" - "os" - "runtime" - "strings" - "testing" -) - -func TestNlitOutOfRange(t *testing.T) { - // Trying to decode this bogus flate data, which has a Huffman table - // with nlit=288, should not panic. - io.Copy(io.Discard, NewReader(strings.NewReader( - "\xfc\xfe\x36\xe7\x5e\x1c\xef\xb3\x55\x58\x77\xb6\x56\xb5\x43\xf4"+ - "\x6f\xf2\xd2\xe6\x3d\x99\xa0\x85\x8c\x48\xeb\xf8\xda\x83\x04\x2a"+ - "\x75\xc4\xf8\x0f\x12\x11\xb9\xb4\x4b\x09\xa0\xbe\x8b\x91\x4c"))) -} - -const ( - digits = iota - twain - random -) - -var testfiles = []string{ - // Digits is the digits of the irrational number e. Its decimal representation - // does not repeat, but there are only 10 possible digits, so it should be - // reasonably compressible. - digits: "../testdata/e.txt", - // Twain is Project Gutenberg's edition of Mark Twain's classic English novel. - twain: "../testdata/Mark.Twain-Tom.Sawyer.txt", - // Random bytes - random: "../testdata/sharnd.out", -} - -func benchmarkDecode(b *testing.B, testfile, level, n int) { - b.ReportAllocs() - b.StopTimer() - b.SetBytes(int64(n)) - buf0, err := os.ReadFile(testfiles[testfile]) - if err != nil { - b.Fatal(err) - } - if len(buf0) == 0 { - b.Fatalf("test file %q has no data", testfiles[testfile]) - } - compressed := new(bytes.Buffer) - w, err := NewWriter(compressed, level) - if err != nil { - b.Fatal(err) - } - for i := 0; i < n; i += len(buf0) { - if len(buf0) > n-i { - buf0 = buf0[:n-i] - } - io.Copy(w, bytes.NewReader(buf0)) - } - w.Close() - buf1 := compressed.Bytes() - buf0, compressed, w = nil, nil, nil - r := NewReader(bytes.NewReader(buf1)) - res := r.(Resetter) - runtime.GC() - b.StartTimer() - - for i := 0; i < b.N; i++ { - _ = res.Reset(bytes.NewReader(buf1), nil) - _, _ = io.Copy(io.Discard, r) - } -} - -// These short names are so that gofmt doesn't break the BenchmarkXxx function -// bodies below over multiple lines. -const ( - constant = ConstantCompression - speed = BestSpeed - default_ = DefaultCompression - compress = BestCompression - oneK = -1024 -) - -func BenchmarkDecodeDigitsSpeed1e4(b *testing.B) { benchmarkDecode(b, digits, speed, 1e4) } -func BenchmarkDecodeDigitsSpeed1e5(b *testing.B) { benchmarkDecode(b, digits, speed, 1e5) } -func BenchmarkDecodeDigitsSpeed1e6(b *testing.B) { benchmarkDecode(b, digits, speed, 1e6) } -func BenchmarkDecodeDigitsDefault1e4(b *testing.B) { benchmarkDecode(b, digits, default_, 1e4) } -func BenchmarkDecodeDigitsDefault1e5(b *testing.B) { benchmarkDecode(b, digits, default_, 1e5) } -func BenchmarkDecodeDigitsDefault1e6(b *testing.B) { benchmarkDecode(b, digits, default_, 1e6) } -func BenchmarkDecodeDigitsCompress1e4(b *testing.B) { benchmarkDecode(b, digits, compress, 1e4) } -func BenchmarkDecodeDigitsCompress1e5(b *testing.B) { benchmarkDecode(b, digits, compress, 1e5) } -func BenchmarkDecodeDigitsCompress1e6(b *testing.B) { benchmarkDecode(b, digits, compress, 1e6) } -func BenchmarkDecodeTwainSpeed1e4(b *testing.B) { benchmarkDecode(b, twain, speed, 1e4) } -func BenchmarkDecodeTwainSpeed1e5(b *testing.B) { benchmarkDecode(b, twain, speed, 1e5) } -func BenchmarkDecodeTwainSpeed1e6(b *testing.B) { benchmarkDecode(b, twain, speed, 1e6) } -func BenchmarkDecodeTwainDefault1e4(b *testing.B) { benchmarkDecode(b, twain, default_, 1e4) } -func BenchmarkDecodeTwainDefault1e5(b *testing.B) { benchmarkDecode(b, twain, default_, 1e5) } -func BenchmarkDecodeTwainDefault1e6(b *testing.B) { benchmarkDecode(b, twain, default_, 1e6) } -func BenchmarkDecodeTwainCompress1e4(b *testing.B) { benchmarkDecode(b, twain, compress, 1e4) } -func BenchmarkDecodeTwainCompress1e5(b *testing.B) { benchmarkDecode(b, twain, compress, 1e5) } -func BenchmarkDecodeTwainCompress1e6(b *testing.B) { benchmarkDecode(b, twain, compress, 1e6) } -func BenchmarkDecodeRandomSpeed1e4(b *testing.B) { benchmarkDecode(b, random, speed, 1e4) } -func BenchmarkDecodeRandomSpeed1e5(b *testing.B) { benchmarkDecode(b, random, speed, 1e5) } -func BenchmarkDecodeRandomSpeed1e6(b *testing.B) { benchmarkDecode(b, random, speed, 1e6) } diff --git a/internal/compress/flate/regmask_amd64.go b/internal/compress/flate/regmask_amd64.go deleted file mode 100644 index 6ed28061..00000000 --- a/internal/compress/flate/regmask_amd64.go +++ /dev/null @@ -1,37 +0,0 @@ -package flate - -const ( - // Masks for shifts with register sizes of the shift value. - // This can be used to work around the x86 design of shifting by mod register size. - // It can be used when a variable shift is always smaller than the register size. - - // reg8SizeMaskX - shift value is 8 bits, shifted is X - reg8SizeMask8 = 7 - reg8SizeMask16 = 15 - reg8SizeMask32 = 31 - reg8SizeMask64 = 63 - - // reg16SizeMaskX - shift value is 16 bits, shifted is X - reg16SizeMask8 = reg8SizeMask8 - reg16SizeMask16 = reg8SizeMask16 - reg16SizeMask32 = reg8SizeMask32 - reg16SizeMask64 = reg8SizeMask64 - - // reg32SizeMaskX - shift value is 32 bits, shifted is X - reg32SizeMask8 = reg8SizeMask8 - reg32SizeMask16 = reg8SizeMask16 - reg32SizeMask32 = reg8SizeMask32 - reg32SizeMask64 = reg8SizeMask64 - - // reg64SizeMaskX - shift value is 64 bits, shifted is X - reg64SizeMask8 = reg8SizeMask8 - reg64SizeMask16 = reg8SizeMask16 - reg64SizeMask32 = reg8SizeMask32 - reg64SizeMask64 = reg8SizeMask64 - - // regSizeMaskUintX - shift value is uint, shifted is X - regSizeMaskUint8 = reg8SizeMask8 - regSizeMaskUint16 = reg8SizeMask16 - regSizeMaskUint32 = reg8SizeMask32 - regSizeMaskUint64 = reg8SizeMask64 -) diff --git a/internal/compress/flate/regmask_other.go b/internal/compress/flate/regmask_other.go deleted file mode 100644 index e62caf71..00000000 --- a/internal/compress/flate/regmask_other.go +++ /dev/null @@ -1,39 +0,0 @@ -//go:build !amd64 - -package flate - -const ( - // Masks for shifts with register sizes of the shift value. - // This can be used to work around the x86 design of shifting by mod register size. - // It can be used when a variable shift is always smaller than the register size. - - // reg8SizeMaskX - shift value is 8 bits, shifted is X - reg8SizeMask8 = 0xff - reg8SizeMask16 = 0xff - reg8SizeMask32 = 0xff - reg8SizeMask64 = 0xff - - // reg16SizeMaskX - shift value is 16 bits, shifted is X - reg16SizeMask8 = 0xffff - reg16SizeMask16 = 0xffff - reg16SizeMask32 = 0xffff - reg16SizeMask64 = 0xffff - - // reg32SizeMaskX - shift value is 32 bits, shifted is X - reg32SizeMask8 = 0xffffffff - reg32SizeMask16 = 0xffffffff - reg32SizeMask32 = 0xffffffff - reg32SizeMask64 = 0xffffffff - - // reg64SizeMaskX - shift value is 64 bits, shifted is X - reg64SizeMask8 = 0xffffffffffffffff - reg64SizeMask16 = 0xffffffffffffffff - reg64SizeMask32 = 0xffffffffffffffff - reg64SizeMask64 = 0xffffffffffffffff - - // regSizeMaskUintX - shift value is uint, shifted is X - regSizeMaskUint8 = ^uint(0) - regSizeMaskUint16 = ^uint(0) - regSizeMaskUint32 = ^uint(0) - regSizeMaskUint64 = ^uint(0) -) diff --git a/internal/compress/flate/stateless.go b/internal/compress/flate/stateless.go deleted file mode 100644 index 8f86e2e6..00000000 --- a/internal/compress/flate/stateless.go +++ /dev/null @@ -1,325 +0,0 @@ -package flate - -import ( - "io" - "math" - "sync" - - "lindenii.org/go/furgit/internal/compress/internal/le" -) - -const ( - maxStatelessBlock = math.MaxInt16 - // dictionary will be taken from maxStatelessBlock, so limit it. - maxStatelessDict = 8 << 10 - - slTableBits = 13 - slTableSize = 1 << slTableBits - slTableShift = 32 - slTableBits -) - -type statelessWriter struct { - dst io.Writer - closed bool -} - -func (s *statelessWriter) Close() error { - if s.closed { - return nil - } - s.closed = true - // Emit EOF block - return StatelessDeflate(s.dst, nil, true, nil) -} - -func (s *statelessWriter) Write(p []byte) (n int, err error) { - err = StatelessDeflate(s.dst, p, false, nil) - if err != nil { - return 0, err - } - return len(p), nil -} - -func (s *statelessWriter) Reset(w io.Writer) { - s.dst = w - s.closed = false -} - -// NewStatelessWriter will do compression but without maintaining any state -// between Write calls. -// There will be no memory kept between Write calls, -// but compression and speed will be suboptimal. -// Because of this, the size of actual Write calls will affect output size. -func NewStatelessWriter(dst io.Writer) io.WriteCloser { - return &statelessWriter{dst: dst} -} - -// bitWriterPool contains bit writers that can be reused. -var bitWriterPool = sync.Pool{ - New: func() any { - return newHuffmanBitWriter(nil) - }, -} - -// tokensPool contains tokens struct objects that can be reused -var tokensPool = sync.Pool{ - New: func() any { - return &tokens{} - }, -} - -// StatelessDeflate allows compressing directly to a Writer without retaining state. -// When returning everything will be flushed. -// Up to 8KB of an optional dictionary can be given which is presumed to precede the block. -// Longer dictionaries will be truncated and will still produce valid output. -// Sending nil dictionary is perfectly fine. -func StatelessDeflate(out io.Writer, in []byte, eof bool, dict []byte) error { - bw := bitWriterPool.Get().(*huffmanBitWriter) - bw.reset(out) - defer func() { - // don't keep a reference to our output - bw.reset(nil) - bitWriterPool.Put(bw) - }() - if eof && len(in) == 0 { - // Just write an EOF block. - // Could be faster... - bw.writeStoredHeader(0, true) - bw.flush() - return bw.err - } - - // Truncate dict - if len(dict) > maxStatelessDict { - dict = dict[len(dict)-maxStatelessDict:] - } - - // For subsequent loops, keep shallow dict reference to avoid alloc+copy. - var inDict []byte - - dst := tokensPool.Get().(*tokens) - dst.Reset() - defer func() { - tokensPool.Put(dst) - }() - - for len(in) > 0 { - todo := in - if len(inDict) > 0 { - if len(todo) > maxStatelessBlock-maxStatelessDict { - todo = todo[:maxStatelessBlock-maxStatelessDict] - } - } else if len(todo) > maxStatelessBlock-len(dict) { - todo = todo[:maxStatelessBlock-len(dict)] - } - inOrg := in - in = in[len(todo):] - uncompressed := todo - if len(dict) > 0 { - // combine dict and source - bufLen := len(todo) + len(dict) - combined := make([]byte, bufLen) - copy(combined, dict) - copy(combined[len(dict):], todo) - todo = combined - } - // Compress - if len(inDict) == 0 { - statelessEnc(dst, todo, int16(len(dict))) - } else { - statelessEnc(dst, inDict[:maxStatelessDict+len(todo)], maxStatelessDict) - } - isEof := eof && len(in) == 0 - - if dst.n == 0 { - bw.writeStoredHeader(len(uncompressed), isEof) - if bw.err != nil { - return bw.err - } - bw.writeBytes(uncompressed) - } else if int(dst.n) > len(uncompressed)-len(uncompressed)>>4 { - // If we removed less than 1/16th, huffman compress the block. - bw.writeBlockHuff(isEof, uncompressed, len(in) == 0) - } else { - bw.writeBlockDynamic(dst, isEof, uncompressed, len(in) == 0) - } - if len(in) > 0 { - // Retain a dict if we have more - inDict = inOrg[len(uncompressed)-maxStatelessDict:] - dict = nil - dst.Reset() - } - if bw.err != nil { - return bw.err - } - } - if !eof { - // Align, only a stored block can do that. - bw.writeStoredHeader(0, false) - } - bw.flush() - return bw.err -} - -func hashSL(u uint32) uint32 { - return (u * 0x1e35a7bd) >> slTableShift -} - -func load3216(b []byte, i int16) uint32 { - return le.Load32(b, i) -} - -func load6416(b []byte, i int16) uint64 { - return le.Load64(b, i) -} - -func statelessEnc(dst *tokens, src []byte, startAt int16) { - const ( - inputMargin = 12 - 1 - minNonLiteralBlockSize = 1 + 1 + inputMargin - ) - - type tableEntry struct { - offset int16 - } - - var table [slTableSize]tableEntry - - // This check isn't in the Snappy implementation, but there, the caller - // instead of the callee handles this case. - if len(src)-int(startAt) < minNonLiteralBlockSize { - // We do not fill the token table. - // This will be picked up by caller. - dst.n = 0 - return - } - // Index until startAt - if startAt > 0 { - cv := load3232(src, 0) - for i := range startAt { - table[hashSL(cv)] = tableEntry{offset: i} - cv = (cv >> 8) | (uint32(src[i+4]) << 24) - } - } - - s := startAt + 1 - nextEmit := startAt - // sLimit is when to stop looking for offset/length copies. The inputMargin - // lets us use a fast path for emitLiteral in the main loop, while we are - // looking for copies. - sLimit := int16(len(src) - inputMargin) - - // nextEmit is where in src the next emitLiteral should start from. - cv := load3216(src, s) - - for { - const skipLog = 5 - const doEvery = 2 - - nextS := s - var candidate tableEntry - for { - nextHash := hashSL(cv) - candidate = table[nextHash] - nextS = s + doEvery + (s-nextEmit)>>skipLog - if nextS > sLimit || nextS <= 0 { - goto emitRemainder - } - - now := load6416(src, nextS) - table[nextHash] = tableEntry{offset: s} - nextHash = hashSL(uint32(now)) - - if cv == load3216(src, candidate.offset) { - table[nextHash] = tableEntry{offset: nextS} - break - } - - // Do one right away... - cv = uint32(now) - s = nextS - nextS++ - candidate = table[nextHash] - now >>= 8 - table[nextHash] = tableEntry{offset: s} - - if cv == load3216(src, candidate.offset) { - table[nextHash] = tableEntry{offset: nextS} - break - } - cv = uint32(now) - s = nextS - } - - // A 4-byte match has been found. We'll later see if more than 4 bytes - // match. But, prior to the match, src[nextEmit:s] are unmatched. Emit - // them as literal bytes. - for { - // Invariant: we have a 4-byte match at s, and no need to emit any - // literal bytes prior to s. - - // Extend the 4-byte match as long as possible. - t := candidate.offset - l := int16(matchLen(src[s+4:], src[t+4:]) + 4) - - // Extend backwards - for t > 0 && s > nextEmit && src[t-1] == src[s-1] { - s-- - t-- - l++ - } - if nextEmit < s { - if false { - emitLiteral(dst, src[nextEmit:s]) - } else { - for _, v := range src[nextEmit:s] { - dst.tokens[dst.n] = token(v) - dst.litHist[v]++ - dst.n++ - } - } - } - - // Save the match found - dst.AddMatchLong(int32(l), uint32(s-t-baseMatchOffset)) - s += l - nextEmit = s - if nextS >= s { - s = nextS + 1 - } - if s >= sLimit { - goto emitRemainder - } - - // We could immediately start working at s now, but to improve - // compression we first update the hash table at s-2 and at s. If - // another emitCopy is not our next move, also calculate nextHash - // at s+1. At least on GOARCH=amd64, these three hash calculations - // are faster as one load64 call (with some shifts) instead of - // three load32 calls. - x := load6416(src, s-2) - o := s - 2 - prevHash := hashSL(uint32(x)) - table[prevHash] = tableEntry{offset: o} - x >>= 16 - currHash := hashSL(uint32(x)) - candidate = table[currHash] - table[currHash] = tableEntry{offset: o + 2} - - if uint32(x) != load3216(src, candidate.offset) { - cv = uint32(x >> 8) - s++ - break - } - } - } - -emitRemainder: - if int(nextEmit) < len(src) { - // If nothing was added, don't encode literals. - if dst.n == 0 { - return - } - emitLiteral(dst, src[nextEmit:]) - } -} diff --git a/internal/compress/flate/testdata/fuzz/FuzzEncoding.zip b/internal/compress/flate/testdata/fuzz/FuzzEncoding.zip Binary files differdeleted file mode 100644 index feae35f1..00000000 --- a/internal/compress/flate/testdata/fuzz/FuzzEncoding.zip +++ /dev/null diff --git a/internal/compress/flate/testdata/fuzz/encode-raw-corpus.zip b/internal/compress/flate/testdata/fuzz/encode-raw-corpus.zip Binary files differdeleted file mode 100644 index 7b33f54f..00000000 --- a/internal/compress/flate/testdata/fuzz/encode-raw-corpus.zip +++ /dev/null diff --git a/internal/compress/flate/testdata/huffman-null-max.dyn.expect b/internal/compress/flate/testdata/huffman-null-max.dyn.expect Binary files differdeleted file mode 100644 index c0816514..00000000 --- a/internal/compress/flate/testdata/huffman-null-max.dyn.expect +++ /dev/null diff --git a/internal/compress/flate/testdata/huffman-null-max.dyn.expect-noinput b/internal/compress/flate/testdata/huffman-null-max.dyn.expect-noinput Binary files differdeleted file mode 100644 index c0816514..00000000 --- a/internal/compress/flate/testdata/huffman-null-max.dyn.expect-noinput +++ /dev/null diff --git a/internal/compress/flate/testdata/huffman-null-max.golden b/internal/compress/flate/testdata/huffman-null-max.golden Binary files differdeleted file mode 100644 index db422ca3..00000000 --- a/internal/compress/flate/testdata/huffman-null-max.golden +++ /dev/null diff --git a/internal/compress/flate/testdata/huffman-null-max.in b/internal/compress/flate/testdata/huffman-null-max.in Binary files differdeleted file mode 100644 index 5dfddf07..00000000 --- a/internal/compress/flate/testdata/huffman-null-max.in +++ /dev/null diff --git a/internal/compress/flate/testdata/huffman-null-max.sync.expect b/internal/compress/flate/testdata/huffman-null-max.sync.expect Binary files differdeleted file mode 100644 index c0816514..00000000 --- a/internal/compress/flate/testdata/huffman-null-max.sync.expect +++ /dev/null diff --git a/internal/compress/flate/testdata/huffman-null-max.sync.expect-noinput b/internal/compress/flate/testdata/huffman-null-max.sync.expect-noinput Binary files differdeleted file mode 100644 index c0816514..00000000 --- a/internal/compress/flate/testdata/huffman-null-max.sync.expect-noinput +++ /dev/null diff --git a/internal/compress/flate/testdata/huffman-null-max.wb.expect b/internal/compress/flate/testdata/huffman-null-max.wb.expect Binary files differdeleted file mode 100644 index c0816514..00000000 --- a/internal/compress/flate/testdata/huffman-null-max.wb.expect +++ /dev/null diff --git a/internal/compress/flate/testdata/huffman-null-max.wb.expect-noinput b/internal/compress/flate/testdata/huffman-null-max.wb.expect-noinput Binary files differdeleted file mode 100644 index c0816514..00000000 --- a/internal/compress/flate/testdata/huffman-null-max.wb.expect-noinput +++ /dev/null diff --git a/internal/compress/flate/testdata/huffman-pi.dyn.expect b/internal/compress/flate/testdata/huffman-pi.dyn.expect Binary files differdeleted file mode 100644 index e4396ac6..00000000 --- a/internal/compress/flate/testdata/huffman-pi.dyn.expect +++ /dev/null diff --git a/internal/compress/flate/testdata/huffman-pi.dyn.expect-noinput b/internal/compress/flate/testdata/huffman-pi.dyn.expect-noinput Binary files differdeleted file mode 100644 index e4396ac6..00000000 --- a/internal/compress/flate/testdata/huffman-pi.dyn.expect-noinput +++ /dev/null diff --git a/internal/compress/flate/testdata/huffman-pi.golden b/internal/compress/flate/testdata/huffman-pi.golden Binary files differdeleted file mode 100644 index 23d8f7f9..00000000 --- a/internal/compress/flate/testdata/huffman-pi.golden +++ /dev/null diff --git a/internal/compress/flate/testdata/huffman-pi.in b/internal/compress/flate/testdata/huffman-pi.in deleted file mode 100644 index efaed434..00000000 --- a/internal/compress/flate/testdata/huffman-pi.in +++ /dev/null @@ -1 +0,0 @@ -3.141592653589793238462643383279502884197169399375105820974944592307816406286208998628034825342117067982148086513282306647093844609550582231725359408128481117450284102701938521105559644622948954930381964428810975665933446128475648233786783165271201909145648566923460348610454326648213393607260249141273724587006606315588174881520920962829254091715364367892590360011330530548820466521384146951941511609433057270365759591953092186117381932611793105118548074462379962749567351885752724891227938183011949129833673362440656643086021394946395224737190702179860943702770539217176293176752384674818467669405132000568127145263560827785771342757789609173637178721468440901224953430146549585371050792279689258923542019956112129021960864034418159813629774771309960518707211349999998372978049951059731732816096318595024459455346908302642522308253344685035261931188171010003137838752886587533208381420617177669147303598253490428755468731159562863882353787593751957781857780532171226806613001927876611195909216420198938095257201065485863278865936153381827968230301952035301852968995773622599413891249721775283479131515574857242454150695950829533116861727855889075098381754637464939319255060400927701671139009848824012858361603563707660104710181942955596198946767837449448255379774726847104047534646208046684259069491293313677028989152104752162056966024058038150193511253382430035587640247496473263914199272604269922796782354781636009341721641219924586315030286182974555706749838505494588586926995690927210797509302955321165344987202755960236480665499119881834797753566369807426542527862551818417574672890977772793800081647060016145249192173217214772350141441973568548161361157352552133475741849468438523323907394143334547762416862518983569485562099219222184272550254256887671790494601653466804988627232791786085784383827967976681454100953883786360950680064225125205117392984896084128488626945604241965285022210661186306744278622039194945047123713786960956364371917287467764657573962413890865832645995813390478027590099465764078951269468398352595709825822620522489407726719478268482601476990902640136394437455305068203496252451749399651431429809190659250937221696461515709858387410597885959772975498930161753928468138268683868942774155991855925245953959431049972524680845987273644695848653836736222626099124608051243884390451244136549762780797715691435997700129616089441694868555848406353422072225828488648158456028506016842739452267467678895252138522549954666727823986456596116354886230577456498035593634568174324112515076069479451096596094025228879710893145669136867228748940560101503308617928680920874760917824938589009714909675985261365549781893129784821682998948722658804857564014270477555132379641451523746234364542858444795265867821051141354735739523113427166102135969536231442952484937187110145765403590279934403742007310578539062198387447808478489683321445713868751943506430218453191048481005370614680674919278191197939952061419663428754440643745123718192179998391015919561814675142691239748940907186494231961567945208095146550225231603881930142093762137855956638937787083039069792077346722182562599661501421503068038447734549202605414665925201497442850732518666002132434088190710486331734649651453905796268561005508106658796998163574736384052571459102897064140110971206280439039759515677157700420337869936007230558763176359421873125147120532928191826186125867321579198414848829164470609575270695722091756711672291098169091528017350671274858322287183520935396572512108357915136988209144421006751033467110314126711136990865851639831501970165151168517143765761835155650884909989859982387345528331635507647918535893226185489632132933089857064204675259070915481416549859461637180
\ No newline at end of file diff --git a/internal/compress/flate/testdata/huffman-pi.sync.expect b/internal/compress/flate/testdata/huffman-pi.sync.expect Binary files differdeleted file mode 100644 index e4396ac6..00000000 --- a/internal/compress/flate/testdata/huffman-pi.sync.expect +++ /dev/null diff --git a/internal/compress/flate/testdata/huffman-pi.sync.expect-noinput b/internal/compress/flate/testdata/huffman-pi.sync.expect-noinput Binary files differdeleted file mode 100644 index e4396ac6..00000000 --- a/internal/compress/flate/testdata/huffman-pi.sync.expect-noinput +++ /dev/null diff --git a/internal/compress/flate/testdata/huffman-pi.wb.expect b/internal/compress/flate/testdata/huffman-pi.wb.expect Binary files differdeleted file mode 100644 index e4396ac6..00000000 --- a/internal/compress/flate/testdata/huffman-pi.wb.expect +++ /dev/null diff --git a/internal/compress/flate/testdata/huffman-pi.wb.expect-noinput b/internal/compress/flate/testdata/huffman-pi.wb.expect-noinput Binary files differdeleted file mode 100644 index e4396ac6..00000000 --- a/internal/compress/flate/testdata/huffman-pi.wb.expect-noinput +++ /dev/null diff --git a/internal/compress/flate/testdata/huffman-rand-1k.dyn.expect b/internal/compress/flate/testdata/huffman-rand-1k.dyn.expect Binary files differdeleted file mode 100644 index 09dc798e..00000000 --- a/internal/compress/flate/testdata/huffman-rand-1k.dyn.expect +++ /dev/null diff --git a/internal/compress/flate/testdata/huffman-rand-1k.dyn.expect-noinput b/internal/compress/flate/testdata/huffman-rand-1k.dyn.expect-noinput Binary files differdeleted file mode 100644 index 0c24742f..00000000 --- a/internal/compress/flate/testdata/huffman-rand-1k.dyn.expect-noinput +++ /dev/null diff --git a/internal/compress/flate/testdata/huffman-rand-1k.golden b/internal/compress/flate/testdata/huffman-rand-1k.golden Binary files differdeleted file mode 100644 index 09dc798e..00000000 --- a/internal/compress/flate/testdata/huffman-rand-1k.golden +++ /dev/null diff --git a/internal/compress/flate/testdata/huffman-rand-1k.in b/internal/compress/flate/testdata/huffman-rand-1k.in Binary files differdeleted file mode 100644 index ce038ebb..00000000 --- a/internal/compress/flate/testdata/huffman-rand-1k.in +++ /dev/null diff --git a/internal/compress/flate/testdata/huffman-rand-1k.sync.expect b/internal/compress/flate/testdata/huffman-rand-1k.sync.expect Binary files differdeleted file mode 100644 index 09dc798e..00000000 --- a/internal/compress/flate/testdata/huffman-rand-1k.sync.expect +++ /dev/null diff --git a/internal/compress/flate/testdata/huffman-rand-1k.sync.expect-noinput b/internal/compress/flate/testdata/huffman-rand-1k.sync.expect-noinput Binary files differdeleted file mode 100644 index 0c24742f..00000000 --- a/internal/compress/flate/testdata/huffman-rand-1k.sync.expect-noinput +++ /dev/null diff --git a/internal/compress/flate/testdata/huffman-rand-1k.wb.expect b/internal/compress/flate/testdata/huffman-rand-1k.wb.expect Binary files differdeleted file mode 100644 index 09dc798e..00000000 --- a/internal/compress/flate/testdata/huffman-rand-1k.wb.expect +++ /dev/null diff --git a/internal/compress/flate/testdata/huffman-rand-1k.wb.expect-noinput b/internal/compress/flate/testdata/huffman-rand-1k.wb.expect-noinput Binary files differdeleted file mode 100644 index 0c24742f..00000000 --- a/internal/compress/flate/testdata/huffman-rand-1k.wb.expect-noinput +++ /dev/null diff --git a/internal/compress/flate/testdata/huffman-rand-limit.dyn.expect b/internal/compress/flate/testdata/huffman-rand-limit.dyn.expect Binary files differdeleted file mode 100644 index 881e59c9..00000000 --- a/internal/compress/flate/testdata/huffman-rand-limit.dyn.expect +++ /dev/null diff --git a/internal/compress/flate/testdata/huffman-rand-limit.dyn.expect-noinput b/internal/compress/flate/testdata/huffman-rand-limit.dyn.expect-noinput Binary files differdeleted file mode 100644 index 881e59c9..00000000 --- a/internal/compress/flate/testdata/huffman-rand-limit.dyn.expect-noinput +++ /dev/null diff --git a/internal/compress/flate/testdata/huffman-rand-limit.golden b/internal/compress/flate/testdata/huffman-rand-limit.golden Binary files differdeleted file mode 100644 index 9ca0eb1c..00000000 --- a/internal/compress/flate/testdata/huffman-rand-limit.golden +++ /dev/null diff --git a/internal/compress/flate/testdata/huffman-rand-limit.in b/internal/compress/flate/testdata/huffman-rand-limit.in deleted file mode 100644 index fb5b1be6..00000000 --- a/internal/compress/flate/testdata/huffman-rand-limit.in +++ /dev/null @@ -1,4 +0,0 @@ -aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa -ø‹–vH -…”%€¯Âþè ë†É·ÅÞê}‹ç>ÚßÿlsÞÌçmIG H°èžò1YÞ4´[åà 0ˆ[|]o#© -¼-#¾Ùíul™ßýpfæîÙ±žnƒYÕÔ€Y˜w‰C8ɯ02š F=gn×ržN!OÆàÔ{¥ö›kÜ*“w(ý´bÚ ç «kQC9/ ’lu>ô5ýC.÷¤uÚê› diff --git a/internal/compress/flate/testdata/huffman-rand-limit.sync.expect b/internal/compress/flate/testdata/huffman-rand-limit.sync.expect Binary files differdeleted file mode 100644 index 881e59c9..00000000 --- a/internal/compress/flate/testdata/huffman-rand-limit.sync.expect +++ /dev/null diff --git a/internal/compress/flate/testdata/huffman-rand-limit.sync.expect-noinput b/internal/compress/flate/testdata/huffman-rand-limit.sync.expect-noinput Binary files differdeleted file mode 100644 index 881e59c9..00000000 --- a/internal/compress/flate/testdata/huffman-rand-limit.sync.expect-noinput +++ /dev/null diff --git a/internal/compress/flate/testdata/huffman-rand-limit.wb.expect b/internal/compress/flate/testdata/huffman-rand-limit.wb.expect Binary files differdeleted file mode 100644 index 881e59c9..00000000 --- a/internal/compress/flate/testdata/huffman-rand-limit.wb.expect +++ /dev/null diff --git a/internal/compress/flate/testdata/huffman-rand-limit.wb.expect-noinput b/internal/compress/flate/testdata/huffman-rand-limit.wb.expect-noinput Binary files differdeleted file mode 100644 index 881e59c9..00000000 --- a/internal/compress/flate/testdata/huffman-rand-limit.wb.expect-noinput +++ /dev/null diff --git a/internal/compress/flate/testdata/huffman-rand-max.golden b/internal/compress/flate/testdata/huffman-rand-max.golden Binary files differdeleted file mode 100644 index 47d53c89..00000000 --- a/internal/compress/flate/testdata/huffman-rand-max.golden +++ /dev/null diff --git a/internal/compress/flate/testdata/huffman-rand-max.in b/internal/compress/flate/testdata/huffman-rand-max.in Binary files differdeleted file mode 100644 index 8418633d..00000000 --- a/internal/compress/flate/testdata/huffman-rand-max.in +++ /dev/null diff --git a/internal/compress/flate/testdata/huffman-shifts.dyn.expect b/internal/compress/flate/testdata/huffman-shifts.dyn.expect Binary files differdeleted file mode 100644 index 7812c1c6..00000000 --- a/internal/compress/flate/testdata/huffman-shifts.dyn.expect +++ /dev/null diff --git a/internal/compress/flate/testdata/huffman-shifts.dyn.expect-noinput b/internal/compress/flate/testdata/huffman-shifts.dyn.expect-noinput Binary files differdeleted file mode 100644 index 7812c1c6..00000000 --- a/internal/compress/flate/testdata/huffman-shifts.dyn.expect-noinput +++ /dev/null diff --git a/internal/compress/flate/testdata/huffman-shifts.golden b/internal/compress/flate/testdata/huffman-shifts.golden Binary files differdeleted file mode 100644 index f5133778..00000000 --- a/internal/compress/flate/testdata/huffman-shifts.golden +++ /dev/null diff --git a/internal/compress/flate/testdata/huffman-shifts.in b/internal/compress/flate/testdata/huffman-shifts.in deleted file mode 100644 index 7c7a50d1..00000000 --- a/internal/compress/flate/testdata/huffman-shifts.in +++ /dev/null @@ -1,2 +0,0 @@ -101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010
-232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323
\ No newline at end of file diff --git a/internal/compress/flate/testdata/huffman-shifts.sync.expect b/internal/compress/flate/testdata/huffman-shifts.sync.expect Binary files differdeleted file mode 100644 index 7812c1c6..00000000 --- a/internal/compress/flate/testdata/huffman-shifts.sync.expect +++ /dev/null diff --git a/internal/compress/flate/testdata/huffman-shifts.sync.expect-noinput b/internal/compress/flate/testdata/huffman-shifts.sync.expect-noinput Binary files differdeleted file mode 100644 index 7812c1c6..00000000 --- a/internal/compress/flate/testdata/huffman-shifts.sync.expect-noinput +++ /dev/null diff --git a/internal/compress/flate/testdata/huffman-shifts.wb.expect b/internal/compress/flate/testdata/huffman-shifts.wb.expect Binary files differdeleted file mode 100644 index 7812c1c6..00000000 --- a/internal/compress/flate/testdata/huffman-shifts.wb.expect +++ /dev/null diff --git a/internal/compress/flate/testdata/huffman-shifts.wb.expect-noinput b/internal/compress/flate/testdata/huffman-shifts.wb.expect-noinput Binary files differdeleted file mode 100644 index 7812c1c6..00000000 --- a/internal/compress/flate/testdata/huffman-shifts.wb.expect-noinput +++ /dev/null diff --git a/internal/compress/flate/testdata/huffman-text-shift.dyn.expect b/internal/compress/flate/testdata/huffman-text-shift.dyn.expect Binary files differdeleted file mode 100644 index 71ce3aeb..00000000 --- a/internal/compress/flate/testdata/huffman-text-shift.dyn.expect +++ /dev/null diff --git a/internal/compress/flate/testdata/huffman-text-shift.dyn.expect-noinput b/internal/compress/flate/testdata/huffman-text-shift.dyn.expect-noinput Binary files differdeleted file mode 100644 index 71ce3aeb..00000000 --- a/internal/compress/flate/testdata/huffman-text-shift.dyn.expect-noinput +++ /dev/null diff --git a/internal/compress/flate/testdata/huffman-text-shift.golden b/internal/compress/flate/testdata/huffman-text-shift.golden Binary files differdeleted file mode 100644 index ff023114..00000000 --- a/internal/compress/flate/testdata/huffman-text-shift.golden +++ /dev/null diff --git a/internal/compress/flate/testdata/huffman-text-shift.in b/internal/compress/flate/testdata/huffman-text-shift.in deleted file mode 100644 index cc5c3ad6..00000000 --- a/internal/compress/flate/testdata/huffman-text-shift.in +++ /dev/null @@ -1,14 +0,0 @@ -//Copyright2009ThGoAuthor.Allrightrrvd.
-//UofthiourccodigovrndbyBSD-tyl
-//licnthtcnbfoundinthLICENSEfil.
-
-pckgmin
-
-import"o"
-
-funcmin(){
- vrb=mk([]byt,65535)
- f,_:=o.Crt("huffmn-null-mx.in")
- f.Writ(b)
-}
-ABCDEFGHIJKLMNOPQRSTUVXxyz!"#¤%&/?"
\ No newline at end of file diff --git a/internal/compress/flate/testdata/huffman-text-shift.sync.expect b/internal/compress/flate/testdata/huffman-text-shift.sync.expect Binary files differdeleted file mode 100644 index 71ce3aeb..00000000 --- a/internal/compress/flate/testdata/huffman-text-shift.sync.expect +++ /dev/null diff --git a/internal/compress/flate/testdata/huffman-text-shift.sync.expect-noinput b/internal/compress/flate/testdata/huffman-text-shift.sync.expect-noinput Binary files differdeleted file mode 100644 index 71ce3aeb..00000000 --- a/internal/compress/flate/testdata/huffman-text-shift.sync.expect-noinput +++ /dev/null diff --git a/internal/compress/flate/testdata/huffman-text-shift.wb.expect b/internal/compress/flate/testdata/huffman-text-shift.wb.expect Binary files differdeleted file mode 100644 index 71ce3aeb..00000000 --- a/internal/compress/flate/testdata/huffman-text-shift.wb.expect +++ /dev/null diff --git a/internal/compress/flate/testdata/huffman-text-shift.wb.expect-noinput b/internal/compress/flate/testdata/huffman-text-shift.wb.expect-noinput Binary files differdeleted file mode 100644 index 71ce3aeb..00000000 --- a/internal/compress/flate/testdata/huffman-text-shift.wb.expect-noinput +++ /dev/null diff --git a/internal/compress/flate/testdata/huffman-text.dyn.expect b/internal/compress/flate/testdata/huffman-text.dyn.expect deleted file mode 100644 index d448727c..00000000 --- a/internal/compress/flate/testdata/huffman-text.dyn.expect +++ /dev/null @@ -1 +0,0 @@ - Ë_Kó0 Åñëò ½ê`KÇó0AasÄ›)^ˆHšþ²„¥IÉŸbß»¬—_>ç4
a˜¢=›Œ›Í-^
á1`_² 1 ìÃÌ ‘Å‘:ÁYÓà-‚F66!…A…Ž`Îa¤è©C;Aâþô°Nyr4ßœUä!™¡¤GKСøÖ#ÂóÓáør:B[G‚3Ω.òLè¥õ׶ýbFRuM]¼š^⇳Å(#ZìÐË Õ Ÿí”i…›íöÿvÉÙB¯ð…» B‡H2S]™¢u/ýÚçÖ½üÖWóT¼G›©n—œýrö
\ No newline at end of file diff --git a/internal/compress/flate/testdata/huffman-text.dyn.expect-noinput b/internal/compress/flate/testdata/huffman-text.dyn.expect-noinput deleted file mode 100644 index d448727c..00000000 --- a/internal/compress/flate/testdata/huffman-text.dyn.expect-noinput +++ /dev/null @@ -1 +0,0 @@ - Ë_Kó0 Åñëò ½ê`KÇó0AasÄ›)^ˆHšþ²„¥IÉŸbß»¬—_>ç4
a˜¢=›Œ›Í-^
á1`_² 1 ìÃÌ ‘Å‘:ÁYÓà-‚F66!…A…Ž`Îa¤è©C;Aâþô°Nyr4ßœUä!™¡¤GKСøÖ#ÂóÓáør:B[G‚3Ω.òLè¥õ׶ýbFRuM]¼š^⇳Å(#ZìÐË Õ Ÿí”i…›íöÿvÉÙB¯ð…» B‡H2S]™¢u/ýÚçÖ½üÖWóT¼G›©n—œýrö
\ No newline at end of file diff --git a/internal/compress/flate/testdata/huffman-text.golden b/internal/compress/flate/testdata/huffman-text.golden deleted file mode 100644 index 6d34c61f..00000000 --- a/internal/compress/flate/testdata/huffman-text.golden +++ /dev/null @@ -1,3 +0,0 @@ -ÀAKó0ðó x¾ÃŸžZØÚñ¾LPØaÎ!‚x™âADÒöI–&#I‹EüîþšÇp]¢ LÆ¿íö¯Fðp˜² 1Õ88‡h“¢$‰³ô5SÓà- ‚F66!…)v‚.ô›0„Y¢— í…ûóÃ&åÅ SÓÀÙN|d£2:åÑ -t˜|ë‘àùéxz9Ÿ “š‰éªº‹£²ž‰ÉŽ×3Š -&&=ù£²¾¬ðôšUD‹=Fu‘òã³]²¬q³ÛýßUL+½Æîö©>FQYÊÂLZÊoüäÜfTßµõEÅ´Òõ{´Yʶbúeú
\ No newline at end of file diff --git a/internal/compress/flate/testdata/huffman-text.in b/internal/compress/flate/testdata/huffman-text.in deleted file mode 100644 index 73398b98..00000000 --- a/internal/compress/flate/testdata/huffman-text.in +++ /dev/null @@ -1,13 +0,0 @@ -// Copyright 2009 The Go Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style
-// license that can be found in the LICENSE file.
-
-package main
-
-import "os"
-
-func main() {
- var b = make([]byte, 65535)
- f, _ := os.Create("huffman-null-max.in")
- f.Write(b)
-}
diff --git a/internal/compress/flate/testdata/huffman-text.sync.expect b/internal/compress/flate/testdata/huffman-text.sync.expect deleted file mode 100644 index d448727c..00000000 --- a/internal/compress/flate/testdata/huffman-text.sync.expect +++ /dev/null @@ -1 +0,0 @@ - Ë_Kó0 Åñëò ½ê`KÇó0AasÄ›)^ˆHšþ²„¥IÉŸbß»¬—_>ç4
a˜¢=›Œ›Í-^
á1`_² 1 ìÃÌ ‘Å‘:ÁYÓà-‚F66!…A…Ž`Îa¤è©C;Aâþô°Nyr4ßœUä!™¡¤GKСøÖ#ÂóÓáør:B[G‚3Ω.òLè¥õ׶ýbFRuM]¼š^⇳Å(#ZìÐË Õ Ÿí”i…›íöÿvÉÙB¯ð…» B‡H2S]™¢u/ýÚçÖ½üÖWóT¼G›©n—œýrö
\ No newline at end of file diff --git a/internal/compress/flate/testdata/huffman-text.sync.expect-noinput b/internal/compress/flate/testdata/huffman-text.sync.expect-noinput deleted file mode 100644 index d448727c..00000000 --- a/internal/compress/flate/testdata/huffman-text.sync.expect-noinput +++ /dev/null @@ -1 +0,0 @@ - Ë_Kó0 Åñëò ½ê`KÇó0AasÄ›)^ˆHšþ²„¥IÉŸbß»¬—_>ç4
a˜¢=›Œ›Í-^
á1`_² 1 ìÃÌ ‘Å‘:ÁYÓà-‚F66!…A…Ž`Îa¤è©C;Aâþô°Nyr4ßœUä!™¡¤GKСøÖ#ÂóÓáør:B[G‚3Ω.òLè¥õ׶ýbFRuM]¼š^⇳Å(#ZìÐË Õ Ÿí”i…›íöÿvÉÙB¯ð…» B‡H2S]™¢u/ýÚçÖ½üÖWóT¼G›©n—œýrö
\ No newline at end of file diff --git a/internal/compress/flate/testdata/huffman-text.wb.expect b/internal/compress/flate/testdata/huffman-text.wb.expect deleted file mode 100644 index d448727c..00000000 --- a/internal/compress/flate/testdata/huffman-text.wb.expect +++ /dev/null @@ -1 +0,0 @@ - Ë_Kó0 Åñëò ½ê`KÇó0AasÄ›)^ˆHšþ²„¥IÉŸbß»¬—_>ç4
a˜¢=›Œ›Í-^
á1`_² 1 ìÃÌ ‘Å‘:ÁYÓà-‚F66!…A…Ž`Îa¤è©C;Aâþô°Nyr4ßœUä!™¡¤GKСøÖ#ÂóÓáør:B[G‚3Ω.òLè¥õ׶ýbFRuM]¼š^⇳Å(#ZìÐË Õ Ÿí”i…›íöÿvÉÙB¯ð…» B‡H2S]™¢u/ýÚçÖ½üÖWóT¼G›©n—œýrö
\ No newline at end of file diff --git a/internal/compress/flate/testdata/huffman-text.wb.expect-noinput b/internal/compress/flate/testdata/huffman-text.wb.expect-noinput deleted file mode 100644 index d448727c..00000000 --- a/internal/compress/flate/testdata/huffman-text.wb.expect-noinput +++ /dev/null @@ -1 +0,0 @@ - Ë_Kó0 Åñëò ½ê`KÇó0AasÄ›)^ˆHšþ²„¥IÉŸbß»¬—_>ç4
a˜¢=›Œ›Í-^
á1`_² 1 ìÃÌ ‘Å‘:ÁYÓà-‚F66!…A…Ž`Îa¤è©C;Aâþô°Nyr4ßœUä!™¡¤GKСøÖ#ÂóÓáør:B[G‚3Ω.òLè¥õ׶ýbFRuM]¼š^⇳Å(#ZìÐË Õ Ÿí”i…›íöÿvÉÙB¯ð…» B‡H2S]™¢u/ýÚçÖ½üÖWóT¼G›©n—œýrö
\ No newline at end of file diff --git a/internal/compress/flate/testdata/huffman-zero.dyn.expect b/internal/compress/flate/testdata/huffman-zero.dyn.expect Binary files differdeleted file mode 100644 index dbe401c5..00000000 --- a/internal/compress/flate/testdata/huffman-zero.dyn.expect +++ /dev/null diff --git a/internal/compress/flate/testdata/huffman-zero.dyn.expect-noinput b/internal/compress/flate/testdata/huffman-zero.dyn.expect-noinput Binary files differdeleted file mode 100644 index dbe401c5..00000000 --- a/internal/compress/flate/testdata/huffman-zero.dyn.expect-noinput +++ /dev/null diff --git a/internal/compress/flate/testdata/huffman-zero.golden b/internal/compress/flate/testdata/huffman-zero.golden Binary files differdeleted file mode 100644 index 5abdbaff..00000000 --- a/internal/compress/flate/testdata/huffman-zero.golden +++ /dev/null diff --git a/internal/compress/flate/testdata/huffman-zero.in b/internal/compress/flate/testdata/huffman-zero.in deleted file mode 100644 index 349be0e6..00000000 --- a/internal/compress/flate/testdata/huffman-zero.in +++ /dev/null @@ -1 +0,0 @@ -00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000
\ No newline at end of file diff --git a/internal/compress/flate/testdata/huffman-zero.sync.expect b/internal/compress/flate/testdata/huffman-zero.sync.expect Binary files differdeleted file mode 100644 index dbe401c5..00000000 --- a/internal/compress/flate/testdata/huffman-zero.sync.expect +++ /dev/null diff --git a/internal/compress/flate/testdata/huffman-zero.sync.expect-noinput b/internal/compress/flate/testdata/huffman-zero.sync.expect-noinput Binary files differdeleted file mode 100644 index dbe401c5..00000000 --- a/internal/compress/flate/testdata/huffman-zero.sync.expect-noinput +++ /dev/null diff --git a/internal/compress/flate/testdata/huffman-zero.wb.expect b/internal/compress/flate/testdata/huffman-zero.wb.expect Binary files differdeleted file mode 100644 index dbe401c5..00000000 --- a/internal/compress/flate/testdata/huffman-zero.wb.expect +++ /dev/null diff --git a/internal/compress/flate/testdata/huffman-zero.wb.expect-noinput b/internal/compress/flate/testdata/huffman-zero.wb.expect-noinput Binary files differdeleted file mode 100644 index dbe401c5..00000000 --- a/internal/compress/flate/testdata/huffman-zero.wb.expect-noinput +++ /dev/null diff --git a/internal/compress/flate/testdata/null-long-match.dyn.expect-noinput b/internal/compress/flate/testdata/null-long-match.dyn.expect-noinput Binary files differdeleted file mode 100644 index 8b92d9fc..00000000 --- a/internal/compress/flate/testdata/null-long-match.dyn.expect-noinput +++ /dev/null diff --git a/internal/compress/flate/testdata/null-long-match.sync.expect-noinput b/internal/compress/flate/testdata/null-long-match.sync.expect-noinput Binary files differdeleted file mode 100644 index 8b92d9fc..00000000 --- a/internal/compress/flate/testdata/null-long-match.sync.expect-noinput +++ /dev/null diff --git a/internal/compress/flate/testdata/null-long-match.wb.expect-noinput b/internal/compress/flate/testdata/null-long-match.wb.expect-noinput Binary files differdeleted file mode 100644 index 8b92d9fc..00000000 --- a/internal/compress/flate/testdata/null-long-match.wb.expect-noinput +++ /dev/null diff --git a/internal/compress/flate/testdata/partial-block b/internal/compress/flate/testdata/partial-block deleted file mode 100644 index b14e816a..00000000 --- a/internal/compress/flate/testdata/partial-block +++ /dev/null @@ -1 +0,0 @@ -ÊHÍÉÉ×Q(Ï/ÊI
\ No newline at end of file diff --git a/internal/compress/flate/testdata/regression.zip b/internal/compress/flate/testdata/regression.zip Binary files differdeleted file mode 100644 index 73cf8403..00000000 --- a/internal/compress/flate/testdata/regression.zip +++ /dev/null diff --git a/internal/compress/flate/testdata/tokens.bin b/internal/compress/flate/testdata/tokens.bin deleted file mode 100644 index b93c6968..00000000 --- a/internal/compress/flate/testdata/tokens.bin +++ /dev/null @@ -1,63 +0,0 @@ -<mediawiki xmlns="http://www.›€€Œ.org/xml/export-0.3/"°€€†:xsi´€€”w3€€„2001/XMLSchema-instance" xsi:s”€€„Locationó€€Î ¨€€È.xsd" version="0.3€„:lang="en"> - <siteinfo> - Ž€€ˆname>Wikipú€‚</“€€Œ¢€€†base>뀈en.¶‚€‚¨€€„Ÿ‚€„€€‚/Main_Page</¬€€„·€€†generator>MediaWiki 1.6alpha</€€Ž€€†c¸€€‚first-letter</’€€„€€„ <€‚spaces’€€ž key="-2"退†</¬€€Œ«€€¬1">Special€€À0" /É€€ª1">TalkÅ€€À2">User©€€À3©€€† tØ€€Æ4">Ü‚€‚”ƒ€„Ý€€À5®€€â€€Ê6">Image€À7ª€€ˆÚ€€Ê8µƒ€ˆ“€‚½€À9®€€â€€Ê10">Template À1Óƒ€‚®€€ˆâ€€Ì2">HelpÞ€€Â3ª€€†Ú€€Ì4">Category½€Â5®€€Žâ€€Ì00">PortŽ†€Ä101€€Šà€€¦Ã€€s·†€‚Òˆ€†õˆ€Š<pag˜‡€ˆ<title>AaA</Š€€†–€€†id>1</idŽ€€ˆreviÔ‰€‚€€†Ÿ€€†32899315¦€€ŽÎ€€„mestamp>2005-12-27T18:46:47Z　‚Ÿ€€ŠØ€€„ <contribuˆ‰€Œ€€€‚<user•Š€„Jsmethers</“€€Œ¦€€Žid>614213«€Žì€‚Ó€€ <text xml:É€„="preserve">#REDIRECT [[AAA]]</¬€€‚怀†</û€’</º‚€ŠÃ‚€ lgeriA。҂€‚¤€„ €‚ ‚€Ç‚€¤18063769€”§‚€ŽÇ‚€„07-03T11:13:13Z　‚Ç‚€ØDocu‚€ª802ý€€–À‚€¤minor†‹€comment>adding cur_id=5: {{R from CamelCase}}Ò€€‚®€€†—‚€„ƒ€ÎÔ‚€„a]]Ô€€¦§ƒ€ämericanSamoaÉ‚€‚ƒ€”6ˆ‚€Žƒ€°9Ôƒ€Ë€‚ƒ€Œƒ€˜4:1õ…€äƒ€¼to°ƒ€Š6 Û‚€¦°ƒ€èÝ‚€ˆ Þ‚€„·ƒ€”ppliedEthics·ƒ€œ8·ƒ€º5898943¦€€Ž·ƒ€š2-02-25T15:43:11·ƒ€Ðip>Conø“€ˆ script</i䉀ŽÐ†€ÔAutomated cÔ€€ŒÁ€€‚¿†€ä´‚€† ethics]] -³†€èccessibleComputingƒ€œ10“†€Žº†€¦‚ƒ€†º†€°3-04‚ƒ€‚22:18:38Z䀀‚艀ØAms80¬Œ€ª75„€˜˜ƒ€ÔFixing redirect¼€€‚“ƒ€äÌ‚€Œ_cÍ‚€ŠôŒ€ìd¸€ ¹€ºˆƒ€ˆê‰€‹†€ 9-22T16:02:5ˆƒ€ændre Engelsƒ€ª300Žƒ€ÞÀ…€‚ך€„É€¶da programming „›€‚uageé‚€ìnarchismï‚€ž2ÀŽø…€¤4213683–ƒ€ï‚€š6-û…€„T01:41:25ï‚€âCJames7߆€‚ì‚€¦83238«€Î‘€‚©’€¢è€ª/* „‚€Št Communism */ too many bracketsÙ€€‚›†€Ê{{Ö€€Šm}} -'''倀Šm''' originŽŠ€„as a term of abuse ¯€„ us€€‚gainst early [[work®‡€‚class]] [[radical]]s includž€€‚the [[Digger¦€€‚of’€€ˆEnglish Revolution]] and±€€ˆsans-culotte|''Ž€€’s''È€€’FrencÇ€€–.[Ο€ˆuk.encarta.msn.com/encyclop’›€‚_761568770/£‚€Œ.html] WhilstŒ€„’‚€„is stillŒ‚€†in a pejorative wayó€‚describe ''"any act thaÆ‚€ˆviolent means³€€ˆtroy退„organiz¿¢€„©‚€‚societyË€€†''<ref>à€Šwww.cas.sc.edu/socy/faculty/deflem/zhistorintpolý€‚င History of Intú—€‚µ£€‚al Police CoopeÞ€‚on],𔀆™ƒ€‚final protocols¥€‚–€€‚¢€†È€€–Conference¥€€‚Rome for®€€„Social Defense AÛ„€ˆ™…€Šts¾‚€†, 1898</÷€ˆ, it h©…€‚lso been taken up •€€‚ posi’ƒ€„label by self-definÅ…€‚惀ˆts. - -The word '''˜€€Š”†€„is [[etymology|derived„‚€„]]À€„[[Greekö‰€Œ|Greek]] ''[[Wik¶‚€‚ary:&#945;‰€€ˆ57;“€€œ61“€€Š6€€Ž3±€€Š45;|”€€ŠÆ€€ö]]'' (ªƒ€†without [[÷€‚ü†€‚s (ruler, chief, king)€€†).›ƒ€Œm§ˆ€†[[politð‡€‚ philosophy]], is¡‚€„beliefù…€†''Ù€€„s'' are unne𑀂aryñ‡€„should be abo”ˆ€‚ed, althoughÞ„€‚re·€€„differÕˆ€‚i±…€‚pretŸ…€„sß„€‚wha®‡€‚isÚ†€†°€”«„€‚referñ†€„rel„[[s†…€†move¼š€‚]]s)°€†advÆ©€‚eü€€‚ elimiž†€†è€€‚authoritarian institu‚s, particul£Š€„…‚€‚[[state]].Å…€‚¼‡€˜ì¨€ˆƒ€‚e쨀ŽDª…€‚iЀ€„_of_±…€Šm†€‚˜€€Š‡€„ƒ€Š] on WÈ€€Š, a€’€„ed 2006ö€€‚¼†€Š ð…€ŒÕƒ€†[[À€†¶ƒ€‚‘€€†Ûƒ€‚mostÜ€€Œts ‡Œ€‚it, does not imply [[chaos]], [[nihilism]], orÔ„€‚nomie]], but ra。 a harmoniouà†€‚anti-Í‚€”]]ò‰€Š. In pla»ˆ€†Ûƒ€„„€‚regardLJ€‚s‡ƒ€˜…€ŽstructuresÜ„€„coercive eco”€‚c²ƒ€˜Æ‚€Štsõƒ€Ž„€‚al¨„€†Øƒ€‚ õ«€‚d upon [[¯€‚nt¼…€„s©€€„£€€‚‘€†autonom߀‚individuals, [[mutual aid£‚€‚õ…€‚[[™‰€„govüŠ€‚nce]]. Ñ™€„ -üŒ€‚e™€ŒüŒ€„•ƒ€„easily ̉€Šby™‚€†it is €ŠÜ€€‚Ôƒ€‚Ñ€†ù…€‚offer˜Š€Žî¢€†Â„€„½€€„they‡€†ve to†‡€„ truly freŒ‚€†‘ƒ€„However, ideÿ‚€‚b¿ˆ€‚how an«€Œt»‚€„ety mightí„€‚k v臀‚considerably, esœ€†ly ‰‰€‚ r€€„tž‡€‚§ƒ€Šs; µ„€‚e‹€‚Ç€„disagreª‡€„ý€€’¦€”÷€€ˆbe brou‚€‚§€„. - -== Oí‘€„˜„€†predecessors == - -[[Peter Kropotkin|‰€€Œ³ƒ€Šo‘€‚s, arguœˆ€‚a退‚fo™…€„corded [[ú‚€‚ory³€€‚humanÁ‚€Š was ed oµ‚€’principles.倒‘€¶®€„.쀌[[Mä„€†Aid: A Factor¿†€‚E²…€‚¥…€†ª€€†'', 1902.žˆ€”Mù‡€†thropolog„€„follow”‚€Ž»…€„ö˜€††’€‚æ‹€„v“‹€‚‚€„hunter-ga¨‚€‚er bands wƃ€‚egalµŠ€Šÿ…€‚lÊ•€‚d di…€†§€‚l«ƒ€‚rÁ‰€„umuž€‚d we‰Œ€‚Öˆ€„decree¹€€‚wå…€‚d had eq׆€„󉀄¾„€‚resourcÈ‚€šFriedrich»€ˆ|†€€†¤ƒ€‚Freiš€€„Ê‚€’µ€’arx‰‚€‚¡‹€„Ù†€„ve/”€€‚/Ü…€‚s/1884/oØ„€„-family/indexæ’€‚ð„€Œ¦’€ˆFamiþ…€‚PrivÞŒ€‚Property,‘‚€„°Œ€‚S®Œ€„¥ƒ€884É€„Ä‹€Š -[[þ°€„:WilliamGodwin.jpg|thumb|right|150px|£€€ˆ ¤€€†]] - -Æ€Št½ƒ€‚¤—€–Œ€‚ž€€†y O¦…€„sÚ‰€Š€‚Þ„€‚rray Rothbard|ˆ€€Š]] find„ˆ€attitude¢„€„[[Taoisº’€‚Æ”€„[[H¡†€†•‚€‚China|AnciÕ‡€‚€€„¬Ž€–•€žŸ–€ˆ(Toronto)¶ƒ€‚က†§€„€ˆ.''𬀂pril 14]] [[2002ˆ€€‚܃€toxicpop.co.uk/library/t΀€„Áƒ€„T¡€€ˆ mirror¾€€–geocitiesÚ˜€„SoHo/5705¿€€‚anɘ€„ Vanity site€€ŠÎƒ€’‹€€‚–€Šà‚€°, ü‚€†ë€‚œ„€††€’lewrockwellˆ€„r˜ƒ€ˆ/aׂ€†-chinese‘€†Ñ‚€†˜€€„ð‚€„ese LiberÒ‘€ˆTÈ›€‚ˆ€„退†¬ˆ€‚an extrË™€‚À„…€¢mise¥†€†journals/jls/9_2ƒ€€‚_3.pdf”˜€‚cept‘†€ŒRol º˜€‚llecÀ‰€‚š…€‚¢˜€ŠChange Toward Laissez Fairež€Ò€‚Jô€€†Ì€€‚Í€’Studies, 9 (2) Fall 1990Ô†€” 銀´ found similarþ€ˆØ…€„stoicism|ˆ€€„Û„€‚[ZenòŽ€‚ CitiumÓ€‚Accord¢Š€„oÇŠ€Ž, Zeno‰“€ˆrepudi„„ˆ€‚omnipotºš€Š’€€‚s—ˆ€‚ý™€‚ù€€‚tÙ²€‚n¢ƒ€‚ñ…€„regi¿•€‚‡€„«ƒ€‚Ü€‚oclaimЀ€ˆs䀂eigntyŸ‚€‚’€€‚moral law€€Š°‘€Ž“€†. ‰€’ô…€b¹‹€‚crayonö…€„îµ€‚.jsp½†€Œbritt1910À†€‚l‡€ˆ”€‚, wš€€‚en by ð‚€˜ãœ€ˆEžŸ€ˆaŸŸ€‚ Britannica, 1910]´ƒ€’À›€†[[AnabaptistÄ™€‚of 16th century Eu㊀‚½”€„some𩀄Œ‘€Œž‚€‚ñ‘€„religi·“€‚Í€‚runn¬˜€‚of modernÚ‰€Œm. [[Bertrå‚€‚Russellž‡€‚in û˜€‚''Û‰€West¼€€‚PŸš€ŒÊ†€‚writes®Ž€†ˆƒ€‚¾€Žs„€ž…€‚sincñ€‚ey helÕƒ€‚¾€€ˆgoodœ¤€‚ will be gui‡‘€‚at “€‚y mo›„€‚¬”€‚[[the Holy Spirit]]...[f]•‹€‚ºš€„preŇ€‚—€‚y arrive a¤œ€‚‚¥€„²€€„.¡„€†Œƒ€‚»‰€Œ”‚€š|œ‚€›€€Š½‰€Žÿˆ€Šºœ€ŽÒ€€†'' in ''AŸ¡€’Ç‚€žä…€†Œ†€‚connecŠ†€„È”€„‡˜€Ž ƒ€‚ù’€‚al circum«Å€†sÀ‰€†º‚€‚Ê¥€‚iest …®€„‰„€‚•€€‚¹¹€„nt da®ƒ€„1945«ˆ€šÎ¥€ˆ (True Levº‰€‚rs)|‡‰€‚œ€€ˆ˜€‚rЃ€ˆ¦€€–逆”’€†×Š€‚¬œ€„à‚€Štic€Œ dur¶ˆ€„„¥€‚ime…€‚‹€€‚¼¦€ŽCivil War‚…€‚ð€‚¤š€‚é–€Š¡˜€†ô…€‚®š€‚×…€ºÌƒ€’쇀zpub净„notes/aan-ß•€‚Ù‡€Š½…€‚܇€„t Timeline],á‚€†É‡€²94„Ó‘€Œ -InÙ€ˆï†€ˆeraÒ€‚“€€‚ƒ©€†toˆ€„€€‚§€„to矀„Þ€„thing›—€†—†€‚n ™€„î–€„[[Louis-Armand de Lom d'Arce de Lahontan, Baron €€„’€€ˆ|¹€€’Ÿ€€ ]]ô‡€ŽNouveaux voyages dݧ€‚l'Amérique septentr㦀„eØ„€‚(1703), w©–€‚ he½¨€ŒÜ‡€‚䃀‚NÛ¨€†ò·€Šø‹€‚ÿ€„Uni¨Œ€‚©”€„s|indigenº‰€‚©€€ŠÆž€ŽÝ€€‚󕀂had noÄŒ€Šlaw«¡€‚ris´¡€†r„†€‚sâ–€„p˜•€ˆp´Š€‚rÀ€€‚as be󢀆…Š€ˆyºƒ€‚÷‡€Šª„€ˆeµ°€‚.lib.virginia©€„cgi-local/DHI/dhi.cgi?id=dv1-12 Dic©¦€ˆÛ…€Šö‡€I玀„- ANARCHISM]ø€€‚³„€Šˆ‹€‚∀ˆ Means¯„€‚—¥€‚lÊ‘€ŽÑ¬€ˆleader¢‹€‚Ø‚€†§‚€Š Ind。Mÿ£€Œ,Û¨€„repe‹€‚ly±‚€†ƒ€‚at hŠ€„½‡€†ò€’ù†€†soü†€„°‹€‚[his³–€‚cestorsŠ€ˆä…€„1793¬Œ€„‰€‚꺀‚kø€Š¢™€‚È€œÛ¢€‚×€š pub¤¦€† ''An EnquiryÙ’€†rning«€„š§€†Justice‹–€‚©ƒ€ˆweb.bilkent.edu.tr/On·‡€‚ˬ€„e•¯€†.upennƒ€„jlynch/Frank/ǘ€†/pjtpÛ€„]. A§§€Š›€€† did no퀄䀄઀ˆÀŽ€Œú…€‚˱€„Þ›€‚rÒ„€ˆ™€„havˆ€‚ᣀˆ€„book…€‚‚€‚‡ˆ€†majoµ€€ ÷„€‚€ƒ€†ÿ€€ˆ€€ˆ¦ƒ€†¸“€„eƒž€„錀Œ²©€„À€€Šmœƒ€ˆ Buµ™€‚õ€€†point no¥€€Œt’‹€Žyet exist£©€„®±€ˆ‘‰€„『tï‚€‚쟀‚known mainly˜€‚¯¨€†ult hurlŸ‹€†¬€‚[[bourgeoi¶Œ€‚[[GirondiÓ…€‚”€‚mà €„ݲ€† el«…€„ˆ€Œ·„€ŒrȲ€. - -==T´‚€Œ‚®€„®€„ô€€‚Õ€Œ==Ìœ€ŒPierre_Joseph_ProudhoÕœ€†110px|thumb|left|«€€† «€€†Â€‚«€€„]] -{{ma¥ˆ€‚rticles|[[Ö€€†-ª€€œ¹‚€„‚î—€‚¸›€‚(õ£€Š‚€‚ory)]]}} - -Itý£€‚›Ž€‚o‚€‚ù‘€Žit wasn't until¿—€‚•€„ꀀ ®†€’[[Wɇ€‚isùž€Œ?”®€„in 1840½Œ€‚ƒ„€‚у€ˆÃ„€†®‚€ŒŽ€€†Øƒ€„adop—ˆ€‚Óƒ€‚Ý‚€†¼‹€†pˆ‘€‚. I „ì±€†iψ€‚asoò‡€‚až¦€‚me š—€„¹‚€ŒÅ€€‚󀀂¬…€…Ž€ˆú€€ŒŸ€€‚or•ª€†Ã€¤Ì€€nsw»•€‚’€„Ö€€„a©€„Ó¢€‚ÏŸ€†×€ˆ[[ƒ‚€Š¬€‚thef”€‚䀈I®€‚is â¡€‚›€‚oppo‚ª€‚Ѐ€‚ª³€‚ò€ˆ¼‰€‚•£€Š¹€€†¸Œ€Š€€† (propriété뀌own¡€‚„¢¿€‚lete right„“€„º€€†¿ˆ€‚߆€‚ˆ¹€„’€€†ý€€‚i£€‚¶€ˆê‡€†Á¨€‚sh, such as exploiõ¿€‚ûˆ€‚k退‚‰ƒ€‚profitæ‘€Š éÍ€‚=Ö€€†pÁ…€ˆ-Ø€€‚’€€†â¶€„Ò„€¨|ñ…€Ž, õ„€”Þ•€ŽìŠ€ŒïÚ€‚¿¤€’…±€„×›€‚/subject/ø…€Šs/‡€Šˆ€€‚üŽ€„/ch03.htm…ž€‚pŠ€‚3. L£¦€„û€ˆ effi⟀„ ca±‚€‚¡”€ˆdo‰‡€„ö‰€‚Ë€€ˆ¬ž€Œœ“€†¿Ÿ€Œ†€¢ƒ‚€†¨‹€‚”†€‚±‚€‚·€‚“‚€‚—„€‚õ–€‚™¯€†ˆ‚€Š suppor†€‚Ѭ€„he caꈀ„'possesߧ€‚' -✀s can„€†limiº€€‚·¥€„€„€„È€‚Ƨ€Œ, capit㧀‚nd홀„ú‰€„aÔž€„㎀‚é…€†Àª€Žõ€‚”¨€„ö¢€‚³€€‚jƒŽ€†‚¬€‚±€Š's õ¨€Žå†€†«’€Ž½€Œµ¯€Šº€‚] (¯€‚ellisme), invol•¸€‚—€‚xc¦¡€†Ìƒ€†–¯€‚Ó“€‚ˆ€‚groups cíµ€„trad¡Ž€†produc’¢€Œir‡ª€„r usÃ…€‚''€€†ÿ–€„''Г€ˆre¦™€ˆœ‘€„e amount of±‡€„·€€‚°˜€„¤€Œin怀Š ‰€„TÛ‡€†‡€‚ensureœ‰€†¹¡€‚n²€‚˜€€‚°†€†®„€†â€€‚”€†ofè–€†s. W݆€ˆÏ€†«°€‚ly join toge¤€€‚ö€€‚co-Í…€‚Ë•€†œ€‚shopž·€„Å·€†est-»€€‚ bankú€€ˆb¯–€‚t upæ—€‚provideÍ€†ž€„°°€‚𫀎›€‚Š˜€‚¿‚€„䀒샀²£€†ñš€„influentialÉ€€„in ø’€ˆœ€‚¹‚€‚ÑÁ€„ñ€Œí¯€‚꬀‚iƈ€‚llowЈ€‚É€€„acŸ—€„Ì“€ˆ[[R€€Œ¥‚€‚1848¢˜€†º’€‚ce.ý„€”‘€Œø•€„¤‡€Šˆ‹€‚Š€†x:  deve®Á€„Ê€€‚a numbБ€†®€‚s overœ€„lifÜ¡€„Œ’€†ý‚€‚ô¡€†´º€´€„ofÀ€€„£‚€„‰€‚Forò€†detail£¯€„scuŇ€„ seꀂ쎀¦|õ…€‚]].'' - -==Max Stirner's EgѪ€‚==¤€ ¨€€™€Œ®€€†„€ˆåš€ŠÝž€‚Egoƒ€„Its Own''　Š“³€†ù€‚±’€„st¶€Ž«„€‚®€„Ž €ˆÀ€ß˜€‚‡®€–noÌ €„ofàš€†§š€‚¿…€‚†a¥‰€†, natur¢¸€‚Ž€€‚‰€„ï〆lþ‚€Œ ®…€‚Í€€û€€‚—¶€‚-¼„€†m„€€‚illuÕ‚€‚–´€‚ ''ghostsŽ‘€†Õ…€„iøœ€‚say•…€‚€€¬‡€†¥‹€†§€€‚爀”°¶€‚t󀄀Š€‚Ï€ŠHeݹ€Œd eÜ‚€„Æ‚€„a›Ž€‚mÜ€€‚amorꉀ„ž™€„刀†Ø€€’™‡€†uûœ€‚‰…€‚'û¹€ù†€„Ô€€„ts' ©“€„whe©Œ€‚¬’€„ †€†ir†•€„ô‡€ŒÚ‡€‚do so. For himë‚€sä¼€†come鸀Š–µ€‚À€‚ظ€„:þ€ˆWho奀‚é–€„sŒ¹€„to‘Ä€„,ˆ€€‚defe‚€‚¤‚€‚釀‚g•€€„him¶µ€‚ong𥀂܃€†§‚€ŠA®€€‚Ù‚€†WÈ„€‚I¥‘€†in my powžÂ€‚á„€„çÀ€‚yÉ‘€‚. So long’€‚I’¼€‚ert my쀄a“€‚lder, I aÇŠ€†ò€€‚rieš·€ˆš€Œ€ˆ - -³†€ˆ nØ€„ÈŽ€ˆhimЀ€†””€’-«€‚Ü…€Œ÷‚€„΀€‚è—€„ 'ƒ€†'. N¡‚€‚theless,ꇀŽ‡…€†ŽŠ€’o«™€‚nyöƒ€¥€„¥»€‚-¤†€‚ô¼€„Ž€†êƒ€‚£Ã€’€„À–瀀‚ÁÀ†º¿€†divers†Š€‚==ÅÙ€Œé€€”Ú€€Œ¦ˆ€‚˜™€ŠBenjaminTucker浀ീ†™€„ü¨€‚¥€€† ¦€€†]]™€¤îŸ€‚’¿€†Ò›€‚È€ŠÛ€‚³‡€‚— €–€¨Ž‰€Œ1825 [[Josiah Warreë €‚墀‚Ä€†ip膀„Œš€‚¨Â€‚ÌÏ€†Þ²€†]] experiÅ›€‚ hea쀂œ«€„Robº„€‚Owe¤€€‚ö‚€‚쀄New Harmon§Ã€‚™‡€ˆf˜‹€†à€€„few years amidst muc’ƒ€ˆÏË€‚conflict.¨€ˆ blam¨Š€„ý–€„Ö«€‚ty'sÒ€€„‰€‚o挀‚⯀‚•ˆ€‚[[Œ„€ŽÅ°€’¨‹€ˆ¨€€Ž¸¤€š. ŠŠ±€‚eedš‹€‚oþ½€ˆseƒ‚€ŽØ“€„耊©‚€Šies耈˜À€ˆˆ¼€‚‡€‚³€‚Ú”€†¤‡€†Ä€‚ܱ€ Àƒ€Ž¦€€†Åž€‚[[Utopia (žy)|œ€€†×€ˆ[[M÷¯€†T™¬€‚ij€‚õƒ€„33Ô€ŠwrotŸ™€†Þœ€’ €‚Peaceful Ž€ŽistÌ–€‚Ѐ†òŽ€„ø•€„Ñ“€‚”‚€„b‰€‚«Ÿ€Šœ€Žperiodÿ†€‚û‘€„退Ž. ï…€˜£Œ€‚톀‚atŸ€Šö€†ƒ¡€‚Ù€€Žman‚ƒ€‚íó€‚倂¼€‚forç¾€†¦€€„doctrin™€‚wÑŠ€„š€„Ϻ€†ª¢€(''඀†y'' XIV (Dece–‘€‚®€‚00):1)Í€‚”‡€œ becamƒ€‚÷“€†¡†€ˆýƒ€ˆŠ€‚á‹€ˆeetò€‚߆€”¯€„¦€ŽB. ò΀‚n݀‚ediÏ€€‚Ô€‚‘ƒ€’¨€Ž“€„AugustÔ¾€‚1Š‚€‚¼€ˆ908;¬€‚ide÷Œ€„¬Å€†Ÿƒ€˜n¬°€‚Û„€Žist-½€Š®ƒ€žiss‘€‚ý€†õÀ€‚ئ€‚°Ð€Œ. «‰€ˆ'ò“€‚“º€„ö€ˆß€€”‚€incorpora瀂Ճ€‚°“€„¤‡€‚aáÆ€‚Ü€„þ›€†orists: ‚€ˆ'Ú“€ˆØ‡€‚Ñ™€|Ι€‚alö–€„¡Þ€„;Ÿˆ€ˆ¯€€–cÉ’€‚퀀‚¦›€„à€€‚price|—€€„€Ÿ€†š€€˜ªš€‚õ‰€‚heterodox•š€ˆics|“€€Œ]]³€’[[l。½¯€‚æ¡€‚倀‚value]]);픀‚˜ž€ˆ]]'s marke—‚€»»€‚Á”€”'õ¯€‚‘€„mÔ€‚aÑ€‚[[HerÝŠ€„Spenc¦€€ˆ¤†€†‚º€ˆ²œ€„Å™€„dom™€€†£ƒ€ŒÓÌ€‚ong¬€‚逌“‚€‚ªƒ€Ž'sŠ”€†Õ„€‚·†€‚€€‚‹™€ˆÍ€‚ŒŽ€‚ã‘€‚eÛ›€†¿Ÿ€†ä€€†[[󮀚€Ÿ€ŠÊ”€†àÆ€†È„€†a’»€Š¾¡€’tÆ„€„-pa󢀈§Ÿ€‚ü†€œ|ì„€†²¡€‚–‡€Šê—€‚±€€†¡€’Ü»€¼î€€††€€ˆ37ž¡€„“¡€‚”«€‚–—€†Pay €Œâ®€„󎀆•¿€ˆy: Sel²·€†s F‹·€ŠWr²£€„Ï…€„¡€Š R.Ѐˆ,Æ€‚guard Press, •Ž€‚Yorkòˆ€‚26, Kraus Re‡ €‚¾Þ€‚., Millwood, NY¦€€‚73.Ë¡€’[[Ê„€ˆž…€†Ò€‚§¤€‚—Ÿ€‚üˆ€‚þ¥€„«“€‚û—€‚∀„à˜€’øŽ€„¤Î€Ê€€ˆsystem¡€„œÎ€‚…„€‚Á°€„´”€‚e abun²¡€†of 뀂pet耄ܖ€ˆèÔ€Šº„€„fÅ—€‚Ò‚€‚ Ÿ€’Ý°€‚÷›€‚ƒ€‚receiv‹ˆ€ˆfull³†€†Û€€‚î–€†é„€‚rõ¡€‚Oœ€‚ 19•¾€¹…€ŽÉ“€‚Ûˆ€‚ludLysa³®€„Spoon²„€†[[StepÒ—€‚P¹€‚¥–€‚rew¼Ê€„¡€‚[[ViêÌ€„Yarrosþ¬€Fïý€‚ÛÀ„rna适alÛ“€’akunin»€‚鬀„㬀†Ø“€†Í†€„Ù“€‚£€€†|Mikhail €€ˆ 1814-1876ë“€¬ó€€ª¡€„ingm݉€„AµÓ€’Ü€‚ဌ̀„M¯§€„ì“€ŽÀÀ€†™´€‚rsh reöŸ€‚onŒ €Šþ‚€†û®€ŽÀ…€„ûŸ€‚. Twentyš“€ˆí±€†¸€„64„€„•€‚£€Ê´€‚ÜŸ€‚¶¦€‚–”€„À€€‚'Ý‚€ ',¢›€†ë€‚mô–€ŒÃ€ˆan¨€ó¶€‚currenÿƒ€Œö…€‚׊€Œ. DâÑ€‚oÍœ€„genuiné‹€‚nks”€€‚„‚€‚÷ဂ‚¤€ˆ±½€Šî€†×€–ô€ˆsign«©€‚ant“…€‚ø‡€Œstarœ“€‚Karlû‚€„]] ·‘€‚a¶·€„’€‚fig—•€‚ÈŽ€ˆÜ€€”:”€‚€€‚ш€„Œƒ€‚o eøž€„sucõ”€‚¿€€‚Gene«Ÿ€‚Counci È€„¾‚€‚À€. ÂÈ€„²€†o˜«€„÷×€„toŽ„€„ ¶‘€„Á€„¹€€‚¸±€|±€Št«³€‚who©®€ŒÊ•€ˆÔ„€ˆ…ˆ€‚Â’€‚ ShortŽ´€‚fЫ€‚[[Ø…€˜€•€ˆ°Œ€‚á„€ˆrsÔ¦€„Œ€†1868̈€†üƒ€ ê‚€Špolari®¯€‚into two camps,˜‰€„ä‚€„é…€„ˆ€ŠÆ€†irì–€ŠÙƒ€‚ô‚€†ž™€‚s®¨€‚³±€‚earÿ‘€‚Ф€†Ã€‚ betw‚〄Ÿ€€‚က‚‹ƒ€„ëÈ€‚±Ž€‚Òÿ€‚y¼€€†Õ„€ŠäÚ€‚rý”€„û€€Šfavoured (inœÊ€Ž'ï €‚rds)¤Ž€ˆ‘ð€†©‹€ˆ‘–€„åÛ€‚ggle—逌¥¬€ˆÀ¡€„怂ÿŠ€‚蔀„²à€Œ¦€„ŸÄ€Žpar¿Ò€‚eÊÛ€ˆgi¿Ê€†‡Ÿ€ˆ A¿‹€‚Ž´€‚ù€‚…„€„뉀‚˜ƒ€–focÕ瀄on΀€šÅ‚€‚ity. - -逊cha¿Î€‚e™ƒ€†Ð€€‚é’€Œa²‘€‚×à€”˜‰€‚‚¤Ù€‚iÙ…€†‹€‚iì“€‚º€€‚𔀄a•¥€‚þ€‚ž€€‚ož¡€†®‡€„𽀆€‚È«€‚endʪ€‚È¿€‚a²µ€„®ƒ€‚[[rulØü€„¿ë€ˆÚÈ€†a“˜€‚¶Ú€†Ø€€‚st²Ž€„ì¿€Š¬…€š|Û€ˆ€‚ ‹€ˆÕ€¦lñŠ‚cyc¯Ì€†hp/adpage.phpÀ€‚1969 S¿Í€‚®†€ˆ‘‹€†´±€Ž„€‚ƒ±€’1872±€‚Ü€Œ climax¦œ€‚¬€„§é€ˆspli¢Û€‚¦…€Žtwo‡¯€ŠÛƒ€‚‡‹€„Hague CongÛ€‚ (1872)|•€€–ƒœ€‚µ®€„iá‹€‚ÂÍ€‚cµ±€„΂€ˆÿ×€†Úˆ€ŠÒŒ€Žt_͈€Ž_to_Œ´€†m|”Ÿ€ŠÂ†€Œ¡†€’nž„€†®†€‚ö€€‚ªŠ€ˆñ¤€‚ÊË€†µ€‚˹€‚Åꀈdemocracy|‘§€„€€Œt‘Ñ€‚ò怂n©À€‚[[´›€†ãÓ€†©€€†‰€‚®Â€”''“Œ€Žð¨€ˆ¥€€„˜„€„d¤€‚nЕ€‚ÄŸ€†®†€„»Œ€’Ç•€‚lòÆ€‚…€‚of œ£€‚-w»Ä€†é€s|'€€–'.]]–¤€„‰œ€ˆ¿“€‚—¡€„‰¤€„°¬€†|œ€€ŽÙŠ€Œ}}¦¤€ŒÌЀ„¾ˆ€ŒÍ€¨¤€€„䈀ŽŽÚ€‚Ú˜€Š¤‚€„Í…€ˆ bothÇ‹€Œ‚š€‚Œ€ˆæ…€‚µŒ€ŒÕ€‚tí„€†Ü‹€Œ¡â€Žìˆ€†1870±™€‚…ð€‚Û€Š¬Ž€†d awayÒŒ€†ö€€ˆ's‰•€ˆ„¾€„i•›€„ (ϖ€†co¢Ö€„±Þ€‚m‘€€†)¸€„embracÑý€„ª€†¤‚€‚Óœ€„s.Ë‚€ŒtÈÇ€‚å〄¾€†³²€ ‰ë€£»€‚Ï€€‚“×€„iveµÝ€‚È„€‚†‰€„…·€†b߀‚s݈†ëÌ€†need,£è€„û”€ˆÙ˜€Šnefac.net/node/157ŠÝ€„ûÍ€ˆ›‚€ŒÚƒ€Œ¸¬€†¶À€ˆDéjacqueø€†¤€†perÓ¾€„ä«€‚‘€‚be£ª€ŽÚš€ŽÜ…€ò…€ªÇ‚€†±ô€ŒÇ。ó€†onbooks²‰€„bleed/ÑÍ€ŠšÕ€‚/De”€†¤€†Î€‚×Û€„ªŠ€ŠíÆ€ˆjoseph.d¬€€ˆ.éµ€‚.fr/ecrits/lettreapjpÀ€€‚ De l'être-¹ä€‚õ¬€‚âle et femelle - L³€€„ à P.Jåµ€ŽŽŒ€‚±‚€œ] (§Ù€„áµ€ˆ¤û€Š|€€†]])ØŠ€”Unlike·¶€Œ,Ü’€‚ô™€’ª‚€†í耆䃀‚‡‰€‚䀮͟€‚⊀ˆ¦”€†ÛÊ€„Á²€ˆ tošì€†®¯€‚he ̈€‚sfaö’€„ºˆ€‚퉀‚°ž€ˆÔ„€‚º‘€‚hat܆€‚ ma®é€‚µ€€‚ir‡³€†e³€Šå±€‚nnounce©€†ø¡€†¦Ï€ˆUS pub…£€‚ed ¢Ý€ˆ Le—€ˆaireú‹€‚58-1861). - -PÁ¡€‚¯ˆ€Ž,ø‹€ˆsee³’€Š–뀆imp¬‚n‘‚€„ü¢€„, outl˜€„†€‚ׇ€Œ€Œ”’€‚ConqueªŠ€†Bre›€‚nd Fiel½ç€†ˆª€‚•€€‚À™€‚¸º€ˆHe fel«†€‚-œ€‚…‹€„•€‚¥È€„beneå–€„l©‡€‚ýº€‚䜀Œ,î´€†—“€„™”€‚õ€†×–€„Ä•€†·è€°˜‚€‚97). Subsequº€‚µ‡€ ê‚€‚Èœ€„ Emma Goldô¨€‚À€‚Alexל€†Berkman. M—Š€‚©Š€ˆÌ€€†o-syndß“€‚‚ˆ€‚Ô˜€Ž(õ¸€‚²³€‚w) saw€€ˆ£€€‚ð‰€Š•õ€„ö€†¾—€ˆve. Isaac Puent†¦€‚1932Ùþ€‚ÙŒ€„o÷ƒ€Šrio¶•€„ÓÇ€ŠbŲ€†SpanŸý€‚CNT샀‚Ÿ€‚manifesto¨ €„ëø€„t-Ñš€–׎€‚¬î€‚ - -Sÿš€‚¨€Œs½Ž€‚liked mergÚ‹€‚º€Ž²‘€† Œ€ˆm. Sç…€‚§¹€„»¢€Šê€‚ €€ŠÁŒ€‚aintaÞ„€„ㆀ„¶ö€„º†€Š‰¯€šÜ€„®‡€‚Ô©€„s…Õ€‚ñ€€†œŠ€†y. 𻀂exaˆÇ€‚,ü¢€Žù¢€Šw‰þ€†ŽÀ€‚ess¹€‚¦˜€ˆ…‚€„󎀌僀„߆€ˆ¤€€‚÷…€‚÷‡€‚‰Š€ŠŠÆ€‚zeteticÓŠ€†mac±à€‚debates/apx1pubsˆŠ€‚l],Ö€¨‚€Š‚‚€Žíƒ€„‰€†pseudo†«€Œ–Ž€ˆ.׉€‚§¦€¸/¨¦€‚ö€‚ŠÌ€‚agandaùˆ€‚ûƒ€‚deedœÎ€Johann؀„Ö‘€‚¥¡€ˆ±¡€†[[¡€€†À¾€Šn‚ˆ€‚spokº€‚úº€ˆß€€‚€€†¦ª€‚Ù‘€Šÿ€€¦þÍ€‚󑀌¬Ê€ˆÿˆ€†õý€„ïÆ€‚rayÈ€‚íØ€„ger‚Ø€‚ꆀ‚က†±÷€‚u˜¥€‚i¹Î€‚²Ÿ€‚»À€highîÉ€‚filô€‚—€‚t³™€‚sü»€‚Á‡€‚Áƒ€‚[[riot]]s£¢€‚assaýƒ€‚„“€€ˆËÑ€‚rre´‹€„’€€„ý€€‚[[te¨ê€‚¢€„̆€‚ˆ¡€„Ÿƒ€Š‘€‚™†€„[[¸†€Ž]]a艀„ý‚€ˆª¢€‚¥€–encouraged¤€„¨€€‚Ç›€Ž¾€†ce, ôŸ€‚‚Ì€„ÏÒ€‚mb]]Ш€„”€‚Ôƒ€‚ÀžÇ€€‚§€‚adì•€„î¿€‚|€€‚˜€€„Ž€€„íÿ€‚o furtº«€‚Ä€Š¢‡€‚退„ù€„›ƒ€ˆæ€‚ªœ€‚—’€‚en§£€Š'Þ«€‚Þ„€ŒÚˆ€ˆÞ„€‚]]'†¢€‚One…€‚Ÿ€‚¿‹€„³„€žŒ€„ý—€„º‹€†gЇ€†ó„€˜¦‡€‚o said—†€ˆÑ€€‚§Õ€„Ƀ€‚–©€Œ¡ä€†quickeφ€„dÁ€†ÜÔ€ˆly¼Ÿ€„‡³€‚w£ç€„Å€€‚anÝû€„øƒ€„«€‚‰Š€‚ô€‚n¹¤€‚”š€‚ereÜà€‚Ö×€‚ssacøú€‚Ò€ˆenem؃€people mus÷™€‚âÇ€„â‘€‚ð€„€ˆ{{fact}} 톀‚'€«€‚¶ÿ€‚‡³€‚æ߀‚o¨ƒ€„ß„€Œ, dynamite, earŽŠ€‚Û¿€‚´€‚åü€‚kerŒ‚€ˆD¦€€‚¾‡€‚Ž€€† - -ª—€Œ›ª€‚ØÛ€„noë—€„nsensuÍ£€‚Þ”€‚ݱ€‚egitimacy‘‘€‚uti×€‚ဂø„€ŠÄ€‚ýÀˆ.У€¬[[ErricϤ€‚ß…€‚sta´ƒ€‚ÇŒ€‚ùŠ€ŒÓ¸€†Ø€€’†¦€„«‚˜Ã逌desirableÅ€€‚…€†Ý§€–setýƒ€‚s.ÜÙ€Žè’€‚me §…€‚ù€„y dºà€‚¯’€„Ù†€ŠõŒ€Žé‚€Ž. (·€Œî€‚Ç‚€†On V»ˆ€„c Ó€ŠÐ„€‚Ι€ˆ‚Ä€†›ª€„fŸ˜€„NechaevØ’€‚€†Î†€Šõˆ€‚Ö€ŽidentifiÙ‰€†¥€‚÷›€‚o-pacif¬€€‚|‰€€ŠÀ€€Ë‚€‚䊀Š°Œ€„ÓÔ€†[[nonv¢€ˆÊž€‚[[Leo Tolsto;€ˆoseË€’iƒ‚€‚å“€‚vieÝ«€‚‚ƒ€„¦º€‚˜ƒ€‚[[ChÛ¶€‚´à€‚—€†Áž€‚“€€ ˆˆ„µ‘€þ»€„Ê€€„not³ƒ€„Á†€ŠÖ€€†¦€ŒtÜŽ€‚i©é€†Ç®€ˆ„Œ€Š¸€‚…€†«¶€†ž’€ŠÓÊ€„see£þ€‚|¬€€†Ç’€’¥ž€‚É€ŒFlaÉÈ€„¦€€ˆ î’€Žm.sv¯€Œ75pxË逄red-and-bÀ€„flag, comÓˆ€‚º€„Ñ„€‚É¿€†„„€‚Ù€€‚–‚€Š‹”€„Ÿ€€„®€˜ ˆ„º¥€‚§†ŒÒž€Š¼£€Œñƒ€ˆœ€’«Ÿ€‚Õ¢€Š™€€”Ž™±€‚y 20å±€ÿ€‚«€‚㥀„°‚€ŒÀ—€‚–€€„Á€‚Š¿¨€Å…€„Ù€‚‚Œ€„¥€€‚ˆ¦€‚ñÙ€‚™‚‚Ó€€‚—Ç€‚צ‚©œ€‚鮀‚󓀆ԗ€„•ƒ€Š pursuѲ€„indö–€‚“—€‚±Œ€†«Ž€„®Ù€Š[[gŸ€ˆ£€€‚k³†„©„€„prim™©€‚ö‹€Š°–€ˆ‚€žØŸ€’훀„¢€€†°’€‚º’€Šº¡€„“ª€‚ghŠ”€„Ç´€‚›€€Š¾€€˜õ‚€’ - -A¥€„Œ‚€‚[[PÔ¬€‚Î €ˆe|1871†Õ€ˆÂÍ€‚]] Ôœ€† €ˆ¯…€‚«ƒ„rgedÕ€‚õÒ€„cƒ€‚Í€€‚''Bourses„뀂TravailÝÕ€‚ù”‚ÆŽë‚€†s°§€Œî ¶Ö€„unioã‹‚š¦€’v— ݀€„[[Confédér·€†GénŒ€€‚le duð€€Š©½€‚œ°€ŽnfedΚ€Šofñš€„, CGT)¹‡€„‚ˆ€‚¶š€†1895¡ƒ€‚Æ€‚‹¡€†“倒¶…€’™€Ž.ÿ‹€‚mì’€‚Pataud寀ˆ”€€Œougetº½€„ô€‚¤‚€‚™Œ€‚က‚CGT saw¢¾€‚®™€ŠÖ›€†Å™€†]] Ô€Š´€€„Ô²€‚û¾€‚Ž€ˆ»„€‚Ò„€„. Ƀ€†191ý´€†×€€‚³‚€‚ˆ¥€’·„€†Á€’mÔ€„£Ÿ€ˆappeßú€†[[BolshevÙ„€„. úƒ€ˆ-styleÁ„€’º‚€„aû³€„ific¢ž€‚ô€€‚ô®€‚ª…€‚Ú¶€†é…€‚ €‚o 1921Å”€†remß™€†¼€€®Spainþ〈©‚€‚mid¬ó€‚0Ù“Šß¶€‚ø†€ŒÍƒ€‚É„€‚¾€‚he Worl¶‚(IWW),ù ½†€„1905š€‚¥€€‚US¨‰„o⯀„LJ€”ÿ„€„û‹€‚ˆ€€ŠÐƒ€ˆsᆀ‚t‚ƒ€¤ß€‚ush¡¯€‚È¿€‚Ûˆ€„‘Ï€‚²œ€ŒâÈ€„923 100,000 mÓÆ€„ì’€‚µ›€‚dÖ±€†˜€„…Á€ˆÍ€‚£Ú€†3²€€†‚‰€‚€‚匀‚—‹€‚licië´€„ÑŠ€„—¡€„毀„À«€†„‰‚by rank󂀄Œ˜€„̬€Œ,Õ¨€‚odyׄ€‚a s¯ø€„ð€€‚Ÿ€Žûœ€†hœž€‚nspi¤“€„ÒŸ€‚AnglophªÛ€‚Š„€Žð–„«€ŠCNT_tu_votar_y_ellos_decû€‚⚀„‡€’€Ÿ€‚րㅀ„¸Ç€†2004. Reads: DoŽè€‚let。Ÿ˜€Ši—Ü€‚Ö•‚ ô€‚lives/ You voÍ‹€‚Ј€‚…’€†cide/À€ˆaæÛ€‚ it/êò€„y, Aè—€„, Self-managÂ…€„•€„§ €Šã‚€Œ¢‰€’ Óˆ€ŽïÒ€ˆÇˆ€Ž¥€‚ ¬€‚'s,õ„€‚0€„äý€‚Ń€‚ÈŒ€‚ÌŽ‚Ź€‚ssful„‰€„²€€‚Û‰€†Ö€€„ción Nacؾ€„ delÊŠ€‚bajoÙ‰€‚¾ô€‚—€€„Ú‰€œ®Ã€‚ur:Ú¡€‚ü…€šý€€‚PŽ‡€Šî€€‚ц€„‰€€„‡ƒ€‚„€Šñ‰€†÷”€Œƒ‚€Š¯‹€‚¯„€‚†¢›€ˆsŠÐ€‚¦®€‚aÑ…€ŠhipÉ„€‚1.58 mill™¾€‚934 …€„ꀀ„play‰€‚뀀ˆrolè–€„“€‚[[œ£€Œÿù€Žø´€‚Seeª„:ïñ€‚Ö¿€Î„‚ˆ€‚Ñ’€„SŸ‹€ŽµÞ€‚kÏ€€‚Ricardo Flores MagónÚ€‚›˜€‚keyŽº€ˆß¥€‚€ˆMex层ö –ß”€‚atin‰÷€Š|€€”£Ž€‚¥„€Š™Š€†ùŸ‚ÖÈ€„žŽ€Šå‡€‚exteüŸ‘€„ZapŸª€‚ta Armû˜€„܃€Œ¼©€„´€„|¥€€Œ]] rebªå€‚ر€ˆÅ€€†Þª€‚ory occup²€€„Œ‹€Œî™€‚in ArgÖ„‚naŒ‰€„Berlin—€€‚192†ƒ„€Š½€ˆ‰€ŒëŠ€‚᳀Œê€˜½Ù€ŒãŒ€’å…€Œô„€Ž[[¸Ä€ ¼ƒ€„Contempor™’€‚Ì‚€†Ë€€’m—“€„inuµÜ€‚ùŒ€‚즆¨…€ŒŸ‰€„¯“€‚áÔ€‚;¨Ö€†smalle펀‚žó€„瀄ˆ†€‚s, 20s•‡€„30s.´œ€‚—‡€‚larg†¾€‚Ê€†Ê»€‚†€†È€‚Û€Œtoda˘€„ù„€Š’€‚Ù€€„ÿ‡€‚Àƒ€‚’ƒ€†š‡€ŒÊ‡€†Œ‘€ŠÉ‡€–€‚®€€ˆNT„‡€‚À€‚Çð€„怂paid-upà†€˜6ÇŒ€„ö‚€‚ÐŽ€‚È€ˆñ•€‚Õ†€‚÷†€ˆvÖÿ€‚¦€ˆï©€„[[s †€ˆ©Ù€‚î´€„ê’€„ ´›€ˆ´€„Ø‹€’õ€ŒÆ€€‚»¤€‚˜Œ€‚¼Ž€‚Ó€‚ŸÄ€ˆSolidarity A‘‘‚ÿ¥€„˜€†«€€„KÓ‡€‚£€€ŽFÞ‰€„ø€€‚ׇ€‚ڀ–Ò–€€ˆÂÄ€‚œˆ€‚–¢€†×쀂…ó€‚熀‚2»Ž€‚€„À‚€Š. é„€”cÍ€‚c¾€„타†ô„€–à‚€†Î¥€Šš…€‚ €žm†æ„€‚¤•€„߆‚é…€‚³Š€‚“À€‚¸¨€‚Ç€€‚ÙÛ€‚«€‚dë’€‚à†€„½°€Œâ瀂¡¤€„Žê€‚·€€‚. Pƒ€‚‘©€‚‚‚€‚¦€Šs×…€‚ÔÀ€„Bob BlackÑÜ€‚ÿ€€„¿€¤Ÿ©€ˆs½Å€‚ƒ¾€‚Ä€‚¦†€‚”€†Îžˆ³™€’Man¼Ô€‚Û›€†|‹€‚’€ˆÎƒ€‚Ô€†£ð€„´‡€„휀ˆ™–€’«„€‚´¢€Š up¤â€‚†”€‚ƒšˆ †€‚Ó€€‚ý‘€‚s–„€Œ†©€‚ˆÄ€ˆ……€‚œ„€‚„€†™â€‚–€’Ô«€‚¢†€‚ž€‚–€‚‰Û€Ž«ž€„main‰„›€€–ô€€‚1917øª€‚½€€‚Þ€†£€€¦šŒ€‚߈€„seis°»€‚e‚Ɔ€‚ÿ€‚‘—€ˆ¢€‚Ô€€‚ñ‚€†„€‚²€€„¤ˆ€Œíƒ€‚‘€€„þ €Žñš€‚«“€Š¥ž€ŠËà€ˆa°å€„ùƒ€‚ ‚Ùƒ€‚—€†k»‹€ˆ´½€„FebrušŠ€†d Octoberº†€s£§€‚Œ½€–Ѥ„ù§€„Ô”€ˆ†€‚က‚߀€ŒßpÛ½€Œ›€€–žƒ€‚¦¦€‚urn«€‚²Ä€†¢€€„ô€ŠñÇ€‚¸žŠ §…€‚À€†Ý¾€„i。´ƒ‚¢á€Šë°€‚ch cul𦄑—€†Œ€‚1918 [[KroñЂd¥¢€‚߀ˆ ˆ€‚²‚€inâÑ€„½²€‚ýƒ€†Â€†imí…†öž‚r dׂº˜€‚der‹Ä€‚¡Ø€‚orÔ€Šï€€‚vÏÑ€„§¥„‡‚€Œs¡—€„[[Ukra扂®‡€‚좀‚〄´Æ€ˆä‹€ˆº„€Ž ‘€Œ|civil waã퀄¨‚€ˆ³ƒ€„Whiù†€‚¥‚€‚뀀Žù‡€„Í€€ˆMakhnovshÅ—‚ö°‚asa®¯€‚Š€‚ÿ €‚¼ä€„Në„„«€€ˆ]]). - -Expðÿ€†ï…Œ°€úœ€‚¢·€Ž“€‚n㙀‚¦·€œÃ¯€‚ᡆleavœ„€‚ׂ€’amongñƒ€„Ʀ€‚ÃÊ€ˆ™Â€‚‹©€‚sponseˉ€‚×€Œ“”€„cž…€†ç‚€‚󄀂„þ†€†’€€‚ダŽu¡‰‚ing. Bothžª€ˆÚ”€„icÿ÷€„u󌀂ћ€†iǪ€‚°â€‚‡¥€‚†„€„¦€†, ã‹€ˆ˜Ð€‚xpo€‚×¥€„¼€€‚›€ŽÒ£€ˆ¶€†themðˆ€‚Í©€ˆ¯Ü€„ŒË€ˆŽ€‚ò¶€‚u÷€‚‚É€„’º€†ø€€‚ofºÇ€Šõ˜€†ÐÊ€‚Àõ€‚†€‚l׋€‚oµØ€‚…ꀂ∀‚æ„€††€„É€€‚Š€ŒÀ€„‘€€‚‡€ŠÁ‰€Ž§ƒ€„š€€‚resulì‚€„逆넀ƒ‡„s½ä€‚u›´€‚ ‚ÿ€‚ýƒ€Œ‚‰€Œç€€‚¶“€ly”»€ˆûŠ€Š¦ƒ€‚ꊀ‚vã…€„†¡€‚¢€ŒãŒ€‚š™€‚Á‰€‚謀ˆÞ“€‚Ì€ˆ;¦¤€ˆ”€‚맀„eÄ£€‚ew û€†Šƒ€‚ø³„ÏŽ€ŒŠ€„§ˆ€ŒúŒ€†¼€€‚¬Š€Šü†€†Éö€†¨€€„‚‚€‚US¦¢€„󀀈º˜€†™™€†÷œ€Â€€Ží€€‚¶€€‚[[CGT¾‡€„ñ…€‚IWWã…€„g怆™„€„gý‚€„mselves©Ç€Á‚€Š‰€€‚¹ç€‚“‚œš€†Ê“€‚mÌ‚€†|¡¦€†½€‚”–€”ŠŽ€„In Ʀ€„¬€†É”‚elo Truda]] Œ‰€‚p¦€‚Òƒ€Šð€€ŠÍŸ€‚iles Š€ˆç’€ˆ¾€‚÷‡€˜Ä…€‚ƒ“€„¡‚€‚a㌀‚¢‰€Œn„뀌Ÿ€ˆ new•€„™‚€„Ï•€ˆõ’€„¿‡€šµ‚€‚ðÒ€‚Ö®ˆ¯Á€‚ýŒ€ˆš°€‚Tþ†€„†ß€‚ß–€‚¡¾€†,¿è€Ž¼€€‚[[Platû•€‚´¢€‚O¿Ì€ð¸‚š€€ˆ×€€‚¬€€‚Ò™€„—¿€‚Ó…€‚Æ„€Šâ¯€„󎀂²ˆ€‚〆ܕŠÈ©€Œ®‚€Žß‚€„‰¢€„茀„É€†ú€„ˆ€„s. €‡€†''ü€€ŠùÍ€‚º˜€ŒõÙ€‚Ò¡€†˜±€„™€€„õ˜€˜“ƒ€‚®ƒ€„“ƒ€‚Â뀂ÙÊ€„¸‚€‚䙀Œ¢€€‚¿…€ŠÞ‚€Š‡ˆ€‚ñŒ€„ƶ€„¬á€‚õ«†û€‚'½Ä€„ ¾€‚“’€‚ù €‚', 'tacʆ‘€€Œ É€Šv„ˆ€„䊀‚ib”µ€„'Ò…€„'œ €†ÜÏ€‚'.Ü‚€ŒŒ€‚¯€ˆœ™€ˆ§„€†ñ‚€„¨—€¤•’ŽÕ€‚IŒ²‚¨Ž€‚¡ƒ€‚UK'ÁŒ€‚÷Ž€Œ°—€”““€†¦€€‚èµ€‚ [[North Eas†€‚ª€€ó€‚€€Žéƒ€’ð€€‚À€€‚n¹€€‚e¸€€ˆö•”逄bñ®‚¡š„Canadaø“€ŒfŠç€„厀Šfas££‚û“€Œ ’Aƒµ‚ €€ˆ„Ž€Ž´†€ˆ¸ €’ëЀCNT-ǵ‚¶Ù€‚-car-Øž€ˆ¹¥€„·á€†¿¥€†270px|¡€„ÁŒ€‚ÎŒ€‚1936²‘€‚M¨›€†‚‚€‚耂ڛ€ˆè„€‚熀†øµ€‚ ˆ carØ¡‚ˆ½€‚×€”¸€€†¼€†»¹‚µ‚€‚ô¦€‚‘Ž€†¸Ñ€„ÉÍ€†vÖ‡€ˆ|€€½€‚ ‘ €†¾©‚]] -I󂀆1920Þ‚€†193ú£€ˆÖ„iØŒ€‚Ⱥ€‚ˆš€”±¾‚Á퀄ãô€„¶‘€„˜€„ß©€„ü’€†– ‚sû‡€‚†‡€†ä€€‚ƒÚ€‚§€‚¾€Šû‚€‚¬€Œç’€„¬‡€„³â‚s,–€€ˆ×†€’õ€‚c˜„ffiíÁ‚ choiceÎÿ€‚ͼˆ×§€„Ò“€‚Ø€€‚½‚p–»‚¾¨€‚軄©€†¹½€„›†€„÷©€ˆ®•€ˆSoviet-Ý’€‚Ù‹€„쮀„ƒ‰€‚Ô„€Œ´•€‚•‘‚Ý„€ˆtë½€„y? Luigi Fabbri‰†€‚휀‚¢»€‚™¯€‚Itaû€‚²€€ˆmω€†Â’€˜rguÓ€‚¼‹€„Úƒ€†ª¤€ˆ÷»€‚ØÆ€„ó€‚ÿÞ€‚t: - -:÷º€†Fá…€†¼¹€‚ݳ€‚ð‰‚ anꉀ„à‹€„ê‚€‚縆“™€„ˆõ€„,û¥€†•‘€‚¨€€„s, u³€‚‡¹€†®Š‚ü’†®ƒ€‚‹©€„ŸÝ€”ÿ†€„š€€Œ³€€†È€„뀀šimaginò¹€‚Ѐ€„·‰ŒØ€€†utæ©€„glorŸ°€„Û’€‚¬€‚Ù€€‚Š€„ª”€†p¸Þ€‚”Å‚Ú…€ˆ·Š€Œ€€‚—€ŠyÝ¿€ŠÃÀ€ŠíŽ€„²€†žÂ€‚ʧ€‚×€€‚Õ‚€†tÊ怂ڋ€‚l¹•€‚ð‹€†ÌÅ€Œ²„€‚ª€€‚Κ€Œš©€‚ rioØŒ€‚ÒŽ€žË€„Ó˜€‚—Ñ€‚a '¢‹€‚eŠŠ‚ont'ë•€ˆ„Ö€Žž¢‚or.univ-montp3.fr/ra_forum/en/ €†/berry_david/€†m_or_á €Ž£Ê€†£§€‚Ö©€„”‹€†Ãˆ€ŠÑ›€Žù¾€‚ñŒ€‚toˆ†€†±Í€‚…†€Ý£€††¬„a”£€ˆÝ€‚ò’‚¥Ì€„”—€‚by󧀄ôŽ€Šrs “†€‚ÍÈ€„鉀„ ÞÛ€„¦¤€ŠÐ•€ŠñÀ€†¸¨€„36,Ê‚€„Å€€‚¤Ž†ú€ˆœ‚€„Æ„€„Ђ€ŒŒ¥€ˆhelbÙ‹€„»€€‚µ€–bŠ½€‚îည. Monthßꀈˆ‚€†Øဒ¦–€‚pon¤ƒ€‚Ë¢€„¹…€‚tË€‚§ƒ€‚coupñ€†å€€‚§‹€„‚¦€„´›€ŒÆ®€‚€‚-39)ü€„²œ€„w†³‚Ü„€„repü—€‚€‚Á€€‚‘›€„™€Œ Ž€ÈŒ€Ž|€€‚ÞŒ€ˆt-䀈‰µ€‚¨€ŽfᛀŠ…‹€†w™€†ü¢€‚ø‘€aÍŠ€„militiaí‘€‚¨ž‚À‘€‚rol€€‚Ž€‚Š–€†[[ciÓô€‚•€€‚Ι€‚rcelon£¬€‚ø€‚¥ß€‚Щ€‚·ˆ‚¢ø€‚f ru¼ž€‚ׄ€„ÂÀ‚š„eyšÆ€„¼Œ€Š€·†ÍŒ€z€„󀀂£€‚ýƒ€†‘¢€‚ºœ€Š™€€‚€€‚ªê€‚ÿ†€Š«„€Š¦„€ˆ9»ƒ€†›‚€Œ®²€ˆlos˃€‚¹‘€‚³’€‚Ã…€‚bÁ°‚à耄€è€†×ƒ€„»€€‚Áƒ€‚ኀ‚ú–€‚¸â€„⚀‚÷„€‚Žæ€ˆö°€†ÛÀ‚ɹ€„¯µ€‚ø €‚¢†€„‹‚€‚͇€ˆç¶€ˆðŠ€‚Ž¨€Š§€€‚tro¬Ü„¦š€‚ ÐÔ€‚ž…€ŒÙŠ€Ž.Ä倂€€†»Œ€„troops½†€„倂¹†€‚°œ€„È‚€ˆÛ€‚ø€€‚ËÝ€‚ecu’…€‚‘ €„[[POUM|dis‚nༀ‚rƒ¸€‚飀‚¯€€‚¤‚€ŽçÔ€‚˜°Šî’€†197¾€„ö—€Œê€„’Í€„Œ”’É€„Ù‚€‚€‚€Žneo’€Š|‹€€Žš©€‚ƒ€‚¶“‚I¥€‚̪‚»‡€†¾€€‚˜“€ˆKingdom¸‚€†€€À…€„þ†€†Á€„´…€„a„€‚ä‘€‚ÿ’€|Œ€€ò€€Žø¥€º€Šß…€ˆ…©€‚—‚€„]]Ù¤€„ðƒ€„yÌ©€ŒÐƒ€‚recÿ¹€‚comb—³€‚gü„€Šþ€ˆphysí–€„ °€†¦Éˆˆ°€„rely°€€‚Ý¥€‚¶Ê€‚œÐ€‚½ï€‚í‚€š9§°€‚¤½€‚›¸†tendé适–º€„皀ˆü€ŽUS˜‚€Œm¯µ€”«ˆ€„ti-RaË€„︀†ýˆ€‚US)¶”€‚œ€€„倆€€”K€€Œfa]]÷’†RʵŒë€€ŒËÕ€LeoþÉ€ˆ®•€„¨•€†òö€†œÊ€”|¨Ê€”ñ‰‚8-1910¶Õ€ŒÅ–€’Ê€¦ª…€‚œ€‚Š…€†³ž€†ç¶ˆÐ–€† -äÏ€‚¥€Œt cul‘Ý€„Ä‚€‚ï•€„b䃀‚š†€‚l‹‹€„ifÞ‘€„outûŒ€†Ë„€Ž athie„€Šä€€ˆu„ˆ€„”瀎ǃ€‚‰„€‚Ú¨€‚Û€‚쉀‚«‚€Œò €†è‡€‚s™¾‚ŠŽ€‚À›€‚orûš€„§€€ŒÁ±€‚±“€ŠoÕ‡€Œ¼€€Šlas°•€„ù€‚퀀ˆ’·€‚›€†ï€€‚Ôˆ€„¦ €„¬Ž–߀€Œ¬–€‚di⤀†Ø‰€„ÀŒÑ„€„ô£€†ÂŽ€‚¡…€„烀šâ¾€„Œƒ€ ¢Í€ ̈€„µ€Ší”€„”€‚ˆÒ€ŠŸØ€‚erÚ’€Ž¡€€‚Ѐ‚GodÇ™€‚߀‚Øž€†õÊ€‚thly…“€Ž˜¤€‚ü°€ˆ˜Š€ŽíŒ€ˆöÑ€‚»ŠchurcheÁÔ€†Ñ。Àž€†Š€„Jesus' teaò©€‚g‹Œ€ˆõ€„¹ö€‚Ñ„€ˆticÃ䀆€€†orruÖ€„û”„¡Ñ€†‹‚€Œi‘¥Œó–€‚declÿ‹€„¤ƒ€‚o˜€‚é±€‚„ƒ€ˆ˜„€„Ù‚. ÙÏ€ž€”€‚‚›÷€†¨€Šèˆ€†Ÿ€‚¾¥Œ’®€‚Þ€€†×‚€‚ cheek€†½Ì‚Ÿš€‚ricÂ…€‚ÍÑ€ˆmÖÑ€Œµ…€„º€‚õ–€„½§ŠÔ‰€ˆÄ„€‚Ô€Œ‰‚€Œ¢˜€†¿‡€”¦ƒ€„“ƒ€„²€€‚þҀ€‚£‹€Šof God¯˜€‚™¿€‚in Youý܀†»Ù€ˆÛ¦€‚aÛ…€„·™‚basé•€‚Ò䀆ŠÜ€‚º¯€‚·Ñ€“—€Œ÷ˆ€‚ƒ–€‚Ȇ„ȶ€‚Ä‚€ ⇀„¬‚€‚–œ€‚ [[»«€†œ™€‚ƒ‡€„„´¨€‚ªž€‚‹‚€‚y¹·€†occaø€€‚ƒ€„[[taxŽÅ€|Š€€† tax߇€„»€€„Ω€„€¤„ƒ€‚[[vege꤀†‚ƒ€‚€€Ž¯’€‚»’€‚v—¨€‚š€€ˆ•€€‚Û˜€Š. - -Õ€€š®€‚ñ‘€‚…Û€†ç˜‚¹˜‚rooÈ“€‚s olí³€„ñ„€‚è„€Š's birÜÍ‚˜€€ˆ[[á…€†›†€†ÿº€‚xhibðÚ€‚Žž€„ø…€À‚€„Ï΀‚‚î·€Š¸‚€†alŸï€ˆ†ƒ€‚½Î†. Byˆ®€obey utte뀀„릀‚ºõ€ŽBibl«æ€‚û†€Ž €€Š×Ѐ„©Á†çŠ€‚Š€€ˆÚ‡€‚›€†Ô„€‚sixteen‚Ó€»Ÿ€†ô–€ˆ¯€‚óµÓ€†ð€†'†‹€Š-ò¹€Œ•©€”Ѐ‚«¶€Ž›™€‚´†§€‚iŒ‡š„Ç€€†´©€ŽÞÓ€‚„€‚¤€‚t¿‚obeû·‚ÅÉ€‚oƇ€Šó€Œá‰€†¡‚€Šreje‚û€„¢ý€‚•€†)±š€‚»€‚û‰€„hier䀄û…€ˆ÷†€†ªŒ€‚(²€‚indµÑ‚non¦å€‚Á‚€†ú®€„€º€ŠÄó€†Äž€†Þ€‚瘀‚godŽ¯€‚çà€„ß‚€†Ü€”‡€‚Õ‚£€„itežª€„typÍ‘€‚þž‚µ”€‚ó €„œ€Ž½„€’beginnÚŠ€„««€‚¹‚€†iÓØ€ŽšÅ€‚abal§¤€‚,™á€†Ô€Š¬†€„ß‚€ŠmodelsöŒ€ˆž’‚ÎÅ„À„Ò£€‚늀‚ʼn€„ßÅ€ˆé¸€‚º€‚„†€‚礀„Ñ’€‚ž¤€‚ò €€Š©€Š½€‚Ì€ˆÃЂi-Xu‡³€„[[Buddž€‚¿¨€Œí€€„›®€„󀀂ˆ¡¤€‚by ï‘€ˆ¾„€„ù€€‚[[well-fiÇÆ‚šâ€†›¯€„Ì®€‚ªã€†Ï€€„‡˜€ˆý†€‚æÿ€„û‰€‚¾€€‚envž·‚¬‰€‚Ò€€„¸„¦Ì€„¢»€„¶€†µ€€‚똀†ãØ‚¹€„¥š€‚ó̈‚Éž€‚aŽ†€„˜ç€’n€ˆƒ‚€Œù±€‚¤ä€”Šë€†min⃀‚¨å„Starhawk²€„hoÂÔ€„e¶Ð€‚tenÉ¿€‚²Ž€‚µ€„ܘ€ˆóœ€‚iŠ›€‚—œ€†¢…€‚[[a¿€‚÷€„Ū€„µ“€–femÕš€‚Ç”€’Û-4’Ÿ„—€‚Ë”€ŒÞ¸€–¶”€Œ|Ù€€†a-FÓ€€ˆ‡”€‚Eõ‰€„¦õ€Œî€€Š¦õ€ˆ€€Š™Ž€‚뉀Œ[[Jenny d'HéØ ur倊[[Juliette AdamÖª€‚¾Ã€„Ì…€‚Áƒ€„[[mysogynó‚亀ˆó„€Ž§ž€„ƒö€ŠßÖ€‚u雀Œ185ÞÔ€‚ ¤¿€ˆa-fÄ€ˆˆƒ€‚a°æ‚Ñž€„ŠîŒ¹€Š…€‚Ø‘€„Œ¡€‚妀„È„€‚ò‘€„ªŸ€‚áÊ„patriõ†]΀€†f‚æ„€‚âæ‚roblem€‚Ú‰€‚âð€‚ ¹¼€‚l¹€Ž£ø€„¦ƒ€„ª”€„µÔ€ˆáŽ€‚À‡€†Ö™€„a¡Û‚dú€‚–„ÐІ±¶‚©Ô€‚«€€‚š‹€„„’€‚aÌÍ‚Û€€†ç€Ž'' d„£€ŒÇ€‚‹€†70sô×õĀ€ˆ¿Ú„sallydaœŠ€‚òîˆo‰„€ŒTwoòª„ò€‚†ð€„݇€ˆ¡€€Ž - Two䞀‚¥€€ˆ - Wñ…€‚¿¡€„:Ó°‚Þ„€†²€€Œt€‚’µ€†žû€„Ä„Š,Öƒ€’㘀„ond-wavÜö€‚Å‚€†|”€€«®€‚–€€†Ò³€ŽÓÚ€‚†„€˜,µä€„œ…€‚̓€”ÞŠ€†¬Ü€Œ€†å‚€†û®„½Œ€†‰¢€„öû€‚î„©Ï€‚ð¹‚uÜ£€‚¼€€Œ®’€„‚³‚ú˜€†§ƒ€‚ÍÏ€‚ ꀄȅ€ˆd›ˆ€‚Ø€„½Ñ‚ðÝ€‚©€„femaƒ«€‚³€š‹“€‚Ø“‚¯€‚‚”‚¦„€‚爄©’€†Ë²€Š×€ŽÏš€„yÌ뀆ÖÊ€„·ä‚×Á€ˆÔ¬€‚ª€‚ƒŠ€‚áညԊ€‚‚’–€‚ÔÀ€†‹ì€„àŽ€†•€€‚匀˜Ýû€†È€‚‚‚€‚³‚€‚î…€‚‘²€‚‘¸„Åž€„‘“€ŠæÈ€Ž¸â€‚ƒð‚Ü€„¹ø€‚¼€€‚creñ‚€Œg¤Ÿ€‚ÒÔ€†¥’€†Õ€Ž×¯€‚©€€¹€€‚ ‘†È€„ÃÀ„þ¥€ˆË€ž»ì€‚áå‚Š–€‚€€ˆ§Í„‚€ŒÕ‹€ schoo‹‘ˆðŠ†‡ˆ€„addÓœ€„sÿ‚€ŒÖ€€‚×Ç‚«Ž€‚[[Eco-™ƒ€ˆÔˆ€‚逌û€€Šß¸€‚º€€Žâž‚óÞ€‚of ¸æ‚ÿ€€’Ä€€ˆö€€‚þ€€„ßÑ€„Ï…€šß€€‚ñŒ€„Ê€†¨ç€šø‡€†20th-cܧ€‚¯€‚ò˜€„Õ‚€†ª’€„„ƒ€†’œ€Œ•Œ€–Œ„€„[[Voltairà”‚de CleyÆ´‚ÏŽ€‚œ€†»¨€„퀀ˆÄ†€„¨‡€”Äš€‚Mó΀‚WollstonecraËÁ‚Ȭ€‚»°€‚otoø’€‚•‘€„ö¨€‚ews†…€†Ùʖ쀂瀆̂€¦Â€€ŽprecursÜš‚Itï´€Šb®¹€‚šŸ€‚ǃ€„žŽ€ˆæ€€„Ó€Œ²†€‚ì‚€‚™‚€‚y€†Åž€†´€€‚è„€„©œ‚²Ä€‚–€€„Ö•€‚î•€Œè—€„ê÷€‚Ì€€Šµ‹€„’û‚Š²ŠMiss÷€€Œì‚˜€ˆist;˜±‚€£€‚‹•€‚ŽŸˆist. Sö‚‚‚‡”€‚Ñð€‚ŠùŒá¢€†Œ€‚¡²ˆÁ±‚ª€€‚¯–€‚æ¡€‚r„Ç‚. I make ó½‚ar¡î†€ý€‚ileÔ¬€‚Ì°€‚‘„€„Ÿ’€„ƒ³€„Õô€ˆß€€ ƒ‚€‚›—€„›€€†Ï‰€„aµ€ˆÐ€‚‰€†ÿ€€‚©€€‚¦ñ€ŽÑ쀄¥õ€Œžª‚SÿôŠ¾Ã€‚Ä€€„¬‚”–’Štirœ’€‚ž«€‚l¦†€‚¬‚±‚€‚”Ñ€„½€€ˆÅ‚Œ–€‚亀‚Þ‰€‚¢œ€‚涀ŠÛö€„alwˆ«‚úŽ€„â„€†¾€€„±††Ó¢€‚¼ï„¶ó€Ši»„€Š¸µ€ŠÜª€‚Ú€‚‰±€¤Ï€€‚ƒ…€ˆ€ŒÒ¿€ˆ±µŠFreÚ倂mené €Š¾š€†è。ރ€„fŠ”‚…”€„Ì…€Ž£ƒ€‚Œˆ€ŒŸ…„.Û±€„ˆ€‚Þ—€‚rn day°€€£˜€ŠËЀœ€€‚ ‚쌀Šªð€‚‘Š€ˆÍƀ뀀Šäš€ˆ×€€†Ñ€‚€„Ÿ€„groÝ´€‚ €†Ú€’of Quiet RumoÄ‘€‚™‚€¢Ð†‚‘´€‚é–€‚Ñ´€ˆto sprõ–€‚˜Š€‚œÊ€‚kiú˜€ˆú¬€„ª…€Šâ™€†˜‚€‚€†è‚€‚¶€ŠŸ€ˆ’ꀈbro倀‚“‚€Œ. Wendy McElÔÿ‚ü€€‚½µ€ˆ…ƒ€„‡€š—‰€Œm take☀‚耀ˆû™€‚¯¨„‚Ò‚«‚€‚°ò€‚lퟀ‚—€‚ü…€Ž›€”websiteýŠˆ“€šiÛ€€ˆts.net I-€€”Ÿ’€”Ȭ€‚Œ€†o-펀ŽÇ¬€ªé˜ Smile.JPGÒ¬€ˆ‹˜€”¬€€˜Ø€‚®È€‚-1995)š˜€šð€€’}} -…€ž‹ˆ€‚žò€‚¨«‚ê‘€‚ᯀ‚ƒ±€ˆ™”€„s-«¥€†ÿŒ€„ÎÆ€ˆ¶ð€‚Öª€„‘‡€†„‡€„ü€‚Ý€‚‡ë€–¿›€†ý…€‚“”€‚“„£œ€†ö„€‚[[fㆀ‚í§†À±€„Š‘€Žï¤€‚ÁŽˆó§€†braná©€‚¸€€‚È„€ŠÝ¾‚tÊúŽç‹€„ð‡€‚Ý†‚ソ†€‚Ò€€Ž¼“‚ÜЀˆt€“‚‹€„Ê€€ŠÓˆ‚oï¿€„³‘‚g‹á€‚›€€ˆÁ‚€’³Œ€‚Àù€Ž»€€Š©¬€†–ƒ€œ€£€‚ynthÓ¦€‚µ€‚[[Ò€„Ô €‚ˆ†ï„€‚÷€‚ƒ…€‚[[AÝn©‚€Œ µ€‚ÿž€‚ger™ƒ€‚„”€„Ê‚€„‰²€ˆÙ—€‚×€€‚¨¹€‚ÏÊ€˜³€“‘€‚†°‚ê„ëù†ß‹€‚À€‚¥€€ †€„øÖ‚©€„¯ƒ€‚Æú€„-agÅ›„š–€‚’©€Œ½Š€‚¿©€Œª€€‚í•€‚ep‹€„çބ󹀄LawÀ‚Cì‹€Šô›€†Â‘€ˆ“Í‚Ô°†³€‚…€†§ˆ€„‚€†(Ÿ€„‰×€„David ïô„m²¨€‚)†‹€‚Ê‚€‚ø€‚acµ€€¨Jan NarvesÚ”€‚)¥†ŒÇ‚€‚¹‘€„m|Š€€Œˆž€‚õ§€Ž[[Ayn RanÙü„ƼŒNozick’€€‚¬ƒ€†Ý¼ˆAÕ‚€‚inle‘É€†£©€„®¢€æ‚€žâ¸€‚Ñ·€Ž€ƒ€Á‹€„”Ý€‚±†€†Ú±€„¹€„áÛ€†®õ€’ÁŠ€‚oö€‚¾Ï€‚°€†¥‚€†HarÐ…€‚Ì„€‚Ralph Raico]]ºŒ€Žßú€‚Û¶€ˆ‹’€’㎀†¢ƒ€‚p퀊Rôñˆ›‹€‚‹™€‚¬‚€„¯€’²‚€Ž®Ù€†üƒ€„Gust§«€‚de Molá‚€‚i—•€ŒAuberon »´ˆ]] ဒº€€¢|À€Ž, Ú€€Žë·€‚‘ž€Špraxeology.net/MR-GM-PSÿ—„Pre×É€‚Á€‚Á€†³¯€‚P·ÏŒ±€‚SecuÍ’€‚Ÿ€€†²¯€„À¦€‚sÿÁ€‚€…€„J. Huston McCulloch, O¸®€Œ Paî¼€‚ Se÷Ë€‚ #2 (Riš¥‚d M. E“¡€‚¦Æ‚Editor)粎:†¯€‚ Ce¯Ø€‚ü‘€‚¥Ó€”ãòŒMay½€‚7€„란Š´‚€ˆ‹’ÛÅ€„-harÔÚˆ‹„§€€ˆµ€„Ž„€’|”„€Œ£„€†É‚€ŠÑ‚í‚óô†á„€‚y/1787–â‚uÔ€‚Ëꂬ½€†€†½‰€„Ä‚€‚‚ˆ€‚Ž²„Ï€‚ury''] Ecáð€‚Polytechn´ç‚,Ÿ€€„rœ„€„Rechercþ§€‚ Ep÷€‚mologie Appl€€‚e,…€„駚€ˆÃ©e au CNRS (ªõ€‚)•‚€” Opʈ‘€‚…€žÝÈ€‚spu€‚hת€‚½ç€„s€€„㜈McKa–—€‚ain; Elkin, Gary; Neal, D×…€‚''eŸ€‚ò€‚’ˆ¶‚€‚infoΘ‚“£€„faq/‘€‚nd11ÙÈ€„ Rep˜î‚‡…€‚û‡€„EЂƒ‹€‚d Dš¡€„Éž€‚檀‚ Bry¥×€‚aÀ–€‚ѺŒ²€†È‹€‚T¿Š€‚y FAQ烀†”΂š‹€„5.2]§ƒ€‚°£€Š¬€€‚FAQ VŸ€€ˆ11.2''ºö‚ž€‚d‰Ë€Ž20,÷÷€‚6ˆ‚€„á„€ŠÌ×€†ð耄̂€¤â–†³½€‚”ˆ€„†€„Ì€‚™¯€ˆwhe‘¡€‚™€‚·š€„Û—€†”ƒ€Š¾€€‚ÉÚ€‚l蘀†Æ—€ˆÍ‹€„Ù¢€‚‘Ä€‚²à€†¦®€„ငi⸀‚b¡€‚þ €‚ü¶€‚’‚€Š„›€‚劀¢ü¶€‚øª€ ¨…€‚ffÂÕ€„ꇚŽm|G埀”m|Eco-󀌀€‚ñ“€ˆØ’€‚¹Ä€¼ô€‚Êû€Ž·€‚̳€†ÿ€„untÀˆ€„ÀŒ€„ÞÄ€„tak‹€‚¯à€‚Æ•€‚”‹€‚he£€†ÁÅ‚µ®€Ž.Ç¡€ˆ˜€”€€ŒÜ€‚…’€‚[[Ì€šïă€Œ§€€„®‰€‚öÚ€ˆ[[deepŠ‘€‚ÛŠ€‚È€‚¯«Œ´‡„ldviewþ“€†¯¦ˆ»Ž€‚bâÆ‚ØÇ€„¤Ê€„倂[[s¹‹€‚µÑ€‚ßÊ€‚Ë€€„¡€–§ €ˆ“€‚’퀜„¦€Œ°¡‚‚‚€‚ˆ‚Ê„earth-䞀ˆª‚€‚ÖˆŠí¿€‚. Ofªâ€†íÁ€„±é€‚Ó „Ø‚°€‚退‚EÀ€€‚ø³†!]]ô˜€Œ¹ž€‚ꀀ‚akeæã‚΀€‚ñ¢€t°•€‚sit¬Ž€‚•‚€‚Anoì…€†à€€ˆö€†‰‰€„退‚[[e¥„€ŽÅ€‚þž€†see…€†Þ¨€†ã€€„oÒ‹€‚°Ã€„°›€‚ “‚aphý›€‚úƒ€ˆª€€–w—€‚´ÇˆÄƒ€Œmœ¦€†™É€ˆ©ƒ€‚ª¯€„qƽ†Äò€“–€Ž¹‡€„‹¶€†ò€„Ø¥€˜×ó€‚þ €‚¥§€ˆˆ³€‚elf.¥¨€ŽPœ§€í‡€„Ș€–ëøŠì€ŒûÁ€„˜§€‚vocú€„a reÝ¿€†º³€‚pre-§€¿„€‚usu倂š€€„agrЃ€‚à”€†¹¯€ŽI‚™€‚×Û„Ï€€‚ó€ªÇ€.œ €ˆis€€Žéš‚chnï…€ˆú€€„„†€‚Ö€€„é–€‚¦€‚‚Œ€‚[[alie´ƒ€†|Š€€ˆ°Ï€„îÔ€†§¸€†á„€‚̓€„–Þ‚†€‚ƒÁ€‚i¨Ý”Ú€€ˆ‡„€„療ÓÍ€‚„µ€„Ó–€†ÑÓ€‚©á€†ƒ…€ˆ¦Ë€ŒLudË‚˜€‚Ÿ‚€‚ꀀ‚£µ€‚þ¸€„„“€„Jean-J¡ª„s RoÕþ‚au]]. ¬ƒ€’þ€€ˆüÅ€†Ã€€‚‚‹€‚ext€€‚Ž€€‚€ß‚¤Ž€„€€„S÷…€‚£ˆ€‚, ¶†€’€€ŒÍ†€Šì䀄š‚€„ÌŒ‚´‚€„׆€Šs¨‰€‚John ZerzÊ—€‚ƒí€ˆî†€„¬€š &mdash;•›€„…Û€„˜€‚·€†ž€€’€¥€†n•¼€‚Õ߀‚®®€ˆ•†€ˆ¬Ÿ„À•€‚숂ved»…€Œº¯€ˆñ‹€’¤ô€‚¬ñ„¡€ˆÆ¡€ Ôƒ€„–Ž€„ÃÀ€„'®Œ€Še'઄uГ€‚-ga“ˆ€‚ÕÒ€‚íÀ„Ø‹€‚„œ‚ughå퀊“„€„'þ€‚ÿ€‚Œ°‚ƒ—€‚a薂ă€„Ì‹€Šáí‚Ÿµ€‚Ôà€‚Ä €‚µü€†Šž€Œÿƒ€‚offshoots== -—Ž€Ž²†à†€‚§€„ec鱄c±€€„syncÉŸ€„–‡€Ž¢€‚Ì€€‚¬ƒ€œÏ€Å‡€Š‹€‚†€†me¨…€Œ196¢Ò€„áဂ7‰€€‚˜¢Šê€‚´²€‚ʈ£€€†¨¯€†è€ˆ¦’€‚¸È€‚. »Í€„ú„€†sダ„ÿÕÄ€‚là‚€‚Ó«€‚‹Á„nà€Œºã†Ù‚€‚ê쀂𫀂ƒ°€„怀†ô€€ŽÒ£€‚Ư€‚bov§ð€‚ᵊHakim Bey.jpe¸Ï€Šïš€„|Á¹‚œ€€ˆéº€„*'''—ü€Œ™ƒ€ˆy''ë‚–€€œ (‹€„¶À€‚™àŠ•¥€)ø‹€‚ñÆ곂。ъ€ˆµˆ€ŽÚ¢€Œal •™€†û£€‚‰€€† -”Å€ˆª…€‚ñà€‚…—€„ÌÙ€‚ǂ肺¹„›€€‚etc. -Óƒ€„¸‚€‚scap醀‚懀„ùÒ„ì‚€‚[[id¼š€†¤º€‚À€Œ.×€Žè€€‚Å ¼€€‚·‹€‚à„€Œ‹§€‚¨‘€„weakenš€†§¿€‚À„attach¶Š€‚þ€€‚Úˆ€‚rÓ°€‚Æ€†Ð€‚€ôŽªƒ€Ž¬€‚¦¯€‚l·Î€‚sue„ï†s («Æ€‚ti-í÷€„“Ÿ€‚€€„nuÄ‘€€‚쀂)½°€„ø‚€‚àÀ€‚êÊ€„û¢€’¸€ŠÃˆ€‚ˆ‡€‚ȃ€‚Ž’„pecË’„⌀‚×€€„¶©€–íû€Ž¿€‚«€Š´Ò€‚œµ€‚Ä‚€‚Ô‚€‚Ñ€ˆÁÜ€„eu.€‚û±€†êÁ€„ì‚€‚㊀†è¨€ŽÖ€”speí“€†Ò†€‚ƒ£€ŠþÑ€„º“€‚þ¥€ŠoadÖ‡€ˆŽ€Š²‚€ˆhun¨€€‚gan¦€ˆë „ÐÉ€ŠÔ€€‚–Ä‚¹Í€„¬€‚þ®€‚£ô„absÕÀ‚Ä€‚þ±€„´½€„ݸ€Œ. IÓ‘€Œ²Æ€† ƒ€„€Ž‡Ë€‚ø„€„߆€‚ÝÒ€„Ž†€žñÄ€ˆ‘×€‚C‘Í€‚thInc]]ã’€‚÷§€‚gaz¦µ€‚З€Šy: A JÊ´ˆof Dܯ€‚e A߀‚’€ˆá„€‚œÙ‚ƒ¤€„Jˆø„McQuinÁ¸€„Çÿ€‚¡ƒŒŽ€€‚ɇ€½€„Ë‚€„sà˜€º›€‚rm“€„,‡€‚ûŠ„Í›€ŽÄ§€„''Aö…€„yõ—ˆLefڂdž½œ€ˆŽœ€¬a²€€‚Ÿ„€†mŽœ€†ß€€’ - ³‹€†Û€€˜½¬€” se†•€„«“€„ƒ€„퀀ŠÕ…€Šm.ws/post耀‚倀†‹ƒ€Ž½€€ˆ§„¸€€¤œ†€ŒÆ€€‚.Èœ€‚ù–Ê€€Žï€€†y¾ ‚•Š€Œá鑀‚“‚€„–Š€‚碀‚Ç…€‚ po剀‚»‚€‚œ•€‚Ë©€‚ÙƲ„€„Þ§€„SaulýÔ‚Û§€„,¹€†«€‚eivÓ—€‚í¯€ˆÏˆ€‚€¡€‚•€‚ÿ¥€„ǯ€„Šœ€„³„Öü€ˆëˆ€‚LacÛ€‚''þø€†fâµ€†a¶‚€‚¾€ŒÆ‡€‚ïø€Œ¨ˆ€˜ö†€„¢«€„¸€Š²ˆ€‚Àª€‚À„€„ú퀂stŇŠó€‚˜”€‚Óˆ€ˆàÌ€‚зŒto×€ˆ'î©€†Ü€†—‚€„,Úê‚•µ‚°Ä€‚ÉŠ€„혀‚‰¶€‚§Ï‚–‰„˜¬†·¬‚§‰€†ÉÞ‚ r•ö„›€€‚dÞ䀂܆€†¦‚€‚îÉ„Þ®€„嘀†¥€†È‰„Æ€ˆ¬À€„[[s©É€‚Ç‚€‚¨€€–囀‚n›Å†¨†€ˆý‘ˆmoìÑ€„õ†€‚‡Ð„þ‘€ˆƒ‚€‚À›€Œ™¡‚”Ç‚Š€‚ÆÞ†Ò“€†·€Œþ’€„ co“’€‚¯¤„Š€‚žâŒÖˆ€†úœ€„fÈõ‚Ô©€ˆå·€†ï€‚ñ—€„,®Ý€Šèš€‚ssib¶€‚µ€‚𰀂ýˆ€†Ä deg ›€‚ofçÒ€Š±€‚™§‚Õ¼€†or‰€€ˆÛ™‚b홀‚oupe‡ï‚ö´€‚¾€‚rubric. Noneä눱•„ï¡€‚¸è€‚¬»€‚쎀‚󉀌Š‚€˜“ƒ€†Æˆ€‚Ù…€˜[[ToddÒ§€‚ç…€‚[[Gilles Deleuz‘Æ€‚烀‚[[Félix Guat®€‚✀‚''Ex²÷€‚´Œ€‚Ï…€‚³‹€‚:߇€„¡…€Šm CŠŽ€‚inghouseàˆ˜‰€ «€‚³€€Œ¬¥€„Æ€€„¢„€ˆé…€‚”Ü€„Ñ„€‚È€€„…‰€––ˆ€Žþ¥€‚•ˆ€†Í€€Œ’ˆ€ŠI÷Ž€‚ €€Œ™ˆ€†ž€€¬¼œ€†‹‹€‚¼ƒ€‚õŽ€–À€ŒÚ›€†Áà€‚of«€€„¥‡€”½Ž‚r×û€‚˜Ž€Š¹û€†í…€‚ï„ñ€Ž˜€’ß‘€‚Œ€ˆŒŒ€ŠlÌŽ€”딂ËÇ€‚•‚€‚Ü•„ aff‡Ô€‚y–Ž€ˆÒþ‚rryŸ€€‚Þ—€‚ø¯Š¿Ú€†ã“€†n¢º€ŒŠ€‚èÒ‚ƒ¹€‚®´‚〆º€ŽÇò€‚»”€„ಀ‚€€„ÛŠ€‚®€€‚¹ €†”Ú€‚€†²Á€†o›–€„”€’î…€‚·–€‚´š€‚œ‚€Œsü›€‚o—¡€†¸ˆ€Š¤…€‚Wolfi L²Ô€‚treiche¥ï€‚™€‚¼‰‚f”¸€‚È€‚Bon‰Ä‚¹…€‚ž’€†€ƒ€‚‰é€‚Ë€€Šü€‚Ï“€†¬€„ JoÒ®€ˆË€€„Š€€†•Œ€‚쀆鄀‚TÍÒ€‚onâ•€†ž€ˆ´‘€ˆê—„Úõ€ŒÞ€„›‡€„USൄµ€ˆ§ˆ€ˆ—“„úÄ€‚fulΪ€‚‰Ù€„¢†€‚Ò†€ŒK›—‚ng߀„ AbacuˆÞ€„Î…€Žß† „€ªÚ…€ŒSø˜€‚ 'a—€‚Ò€ˆÔ…€†›€€®‘¢€„‰Ž€„е€„Ç€‚wû—€‚òø€ˆï ¨ƒ€‚unconn—Ú€†–€‚exts.ò¬€†ý¬€‚刀‚胀„¢‰€‚Ï€€„âÜ€‚Ò΂iÞ…€‚ˆŠ€‚big 'A'á‚€Œ¢€€„®€€‚„¦€„l³€Ž°€‚spunk.org²À‚™—€‚/introͲ€‚¥œ€‚e/sp001689Ë€†µƒ€Šm: I°˜€ˆÊŠ€‚MethodŽ€€„?].¿Ò€ˆ‡€ž´‰€„ù´€‚–ø€‚¾€€†Ã‡€„Õ€€Š®™€‚؆€†Û‚€”®€„a°€‚ü›€†ŽÅ†Â”€‚󀀆À€„¹ª€„erparts;ˆµ€ˆ€†€‚–É€‚õ›€‚Œ‚€ŒÅÍŒa wa¼è€„适›³€‚«˜€„’…€‚œé€ˆ„…€Œþ΀Š¨‹€‚³¹†¯Î€‚ÿÆ€ŠåÄ€ŠA¿€‚¹˜€†Ü€’šƒ€ŠÚ€¦sha¯¾€‚ñ¦€„·€ˆ 〄Ȁ†½€¤½¢€‚å·€†Graeb§‡€’Õ€‚j Grubacic]]’ €‚î耂nÑø‚ªä†veƒ‘€žÛ‚€‚¸ª€†¬‘€„‹Ž€„Ì€‚½ž€’š‰€†”›€‚™ú‚oœà€†è³‚r¿‚€ˆˆ…€‚Ô¤‚nãú‚帀„ìù€„È€†þ‚€ŠãË€‚š‡’ÄÛ€Ž Š†, volu¹ÜˆßŽ€ˆ—€„Ôîˆaid±˜€†netõˆ€‚ÉÆ€„lŽ–€†cru”¡€‚Ï¿€‚戀†ó…€‚‘€†®”€‚胀‚ƒû‚•‘€Œž€€‚endϤ€„if–³€„Š™€‚eanÑž€‚Šî‚lÉ€‚퀄™€€‚busine¥‹€‚¬Ü‚˜€–ʬ€‚o´™‚zª‹€‚ž„€‚Óý„󉀄±¶€‚˜à€†²‘€†¸‚€‚o¾ð„vis™€‚㬀‚ˆÇ€‚𤀄܀€„gunÕ„€ˆ«‡€’zmag«€„—ˆ€„nt/showÔ‡€ˆ.cfm?SÙ£€‚onID=41–¦€„ItemID=4796]´¤€ŽÒž€„s==¹—€„C…Ò€‚p¶€€‚Ø€†Á…€‚Ó‰€†ñ¤€Šá¡€ˆñ瀄‹©€Ž¿©€Œù¢€‚‚€†ñË€†¿®ˆÑ€‚‰‚€†Ä€ˆ¤½€‚™ð€„g˜ö‚„‚€‚˜ã€Š¥½‚æ‚€ŽÇ¯€„uÈ€„öŸ€†Ÿ˜€„¿×€†Öø€‚òú‚ï‚€‚ñÆ€‚müÇ€ˆÝŸ„æâ‚͆‘ù€‚¬¡€‚e˜î€’¡¯€‚Ø€ˆ›€€‚È¡€ŒÏ¸„‚¯†¸€†eµâ€‚ÉÄ€ŒÚ€ŠÏ€‚æΡ€„©ƒ€„ ;Ú¦€†Ô€€€€‚š€€„©€‚àô‚¯ñ€†l •€ˆž‰€†goºž€„É›€ˆ''cause''Ø€€‚chaos,¡‚€¥€€‚warÄ€ªï€€Ž’Ë€‚฀„îÖ€„غ‚óú‚nopo„Å‚ˆŒ€†Å‰€‹ˆ€ˆ°÷€–|m°€€ƒ€Šò€„ñá‚Å¡€‚”€€Šõ…€‚advÛ‘€„eŒá‚„ó€†e瀄 Much eff’„€‚¥¤€Œdø₵–€„Ž¥€‚ëÒ€†„€‚how¦ˆ€Ú–€‚ì©€„Ó«€ˆhandleö“€‚¸Ã€‚á .¶€šÞƒ€ŠŠ¸€†Ãß‚ety•€Šú€†Ö§€„ò‚€†¼°€vereignè…€Š[[¶ €„Œ€Œˆš€‚ᎀ„¡ˆ€ˆ…Ñ€„×€‚¨ƒ€‚„€ˆù“€‚ÌË€Š¬ƒ€†subjug½‰€„¶‚€„´³€–¡°€ˆéƒ€‚olor²€€†›£€„†€‚Ñ适ý¢€ŠÜ€„üŠ€†Ò£€†´€€‚‚€†™¾†ÇÆ€ˆŠ”€Š[[Ashanti A´×€„¶ƒ€‚[[Lorenzo Komboa Ervi™€€„ö€€‚ ™€‚m MbahÛŒ€†Ç‚€Št P¶†„ð€€‚C¥€‚§¯€‚a§²€‚e¾ƒ€„ׇ€„ↄ…€€‚´€„caucas¹É€‚Ƀ€Œ‹…€„exp»È€‚¦…€‚iÝš€‚÷¦€„뛊¤«‚¾é€‚觀‚÷€†’€‚³Ý€‚¯‡€Š¦Ÿ€„󆀂, p„·ù€‚뀂«€€Ž¡À€‚§Í€Œ¡Ž€‚ȯŒÅ€€ŠÆƒ€‚…“€„É€Ž›€‚Í–€ˆŽ«€Œêˆ„„€Žor ethn¡®€‚ߊ‚rvу€†ù€€‚Ù€€ŒûÙ€ˆö¨€‚íÕ‚Ü€†Æ³€‚o—žˆà‹€„Í€‚prÏ¥„î…€‚Õ€€ˆëŽ€ˆ(Å¥€„rac̳€‚)ƒ¬€†lø¡€†™å€‚epa¯€‚ó€‚ÛŸ€‚áÒ€‚¸€€„é³€„Ç€†Ä€‚蘀†ê‰€‚Ń€‚öý€ iÓ‰€†Ÿƒ€’È€€†ÿ”€† °€„¯‚€ŠË€‚·ù€‚nvol²€‚ܶ€„Å©€òÛ„cèñ€‚onfá³€‚¬€‚˃€‚Æœ€‚ôë„oߦ€‚ë—€‚è²Þº€×„€„hiap熂ö³„o €‚aü‚€Žš‡€¦€†÷‚€†¬’€„¦€Œ¾Ù€‚c«‚€‚š²€ˆÔ€‚Neo׆€‚È¡€ˆå…€„Glob®ª€Œè‡€†NçÞ€†¥´€‚Í€€Œ©«„ªƒ€‚³ƒ„Ä€€ä’€‚¹ˆ€‚€¥€„mpt½‰€‚¼Š€‚ÌÑ€ŒcoercŸ‡€„‡’€‚g„ scaž¡€‚–œ€‚¼•€‚•œ€‚ÿ³€ˆžŒ€ŠÆˆª€„ÃË€Šü…€‚‘›€‚rld Ban€í€„€€ŠT§Ã„O¨œ€Ç‡€‚[[G8|G„€‚Õ‚€‚EÅÿ€‚–€€‚Þ‡€‚Ì€€’EÏЀˆ Forum†˜‚€”ý¯€‚倆mbigu¸þ€„erm€†Â‹€‚±™€Œ¨’€„ÿ„€„Ú€‚Ç™€ŠÈ‚€Žƒ–„Õ€‚ÉŒ‚’„€‚̇€Šs¶‚€„•€‚Ù€€†oÉ€€„ 킀¹€‚/“†€„§‹€Œimpeëǂɇ€†(„ž€†ÂŠ„¸á‚ŠÀ€ˆ€„ed)ÓÇŠreÝ—€„ÉÁ€†ó€€‚²°€ˆžƒ€†„€ˆ“…€. O°·€‚s舀˜¼€†óÍ€’Õ“†è”€ˆÍ€€”·“€†×€Œô€€‚ã·€„·§€„expansÁˆ€‚„¨€ˆµà€‚€€Š¶¡€†³ƒ€„䃀‚蟀Šã€„߀ˆ†ásoಀˆs¨Š¸‘€‚Æœ€„‹‚„venéµ€‚‘†€‚ParÍ €‚lÈ €„†€†Ö“€„߀†ó‡€„try»€‚š§€‚Ûñ‚¶˜€Žèƒ€„Ó‘€„-Ø߀‚ª †¾…€”¹€‚ý€†outᤀ‚s,ÔŸ€† Ù…€Œ[[Food Not BombÝö€„€Ä‚简†¿Ë„edu¯•†¾†€‚öÿ„˜†€Žhome-Ý·€†±›€„neighborhÔ€€‚m¾è‚ò„€‚/arbit†·ˆ€†ã¢€ˆso ñ¿€„“‰€‚‡˜€„¶—€†Ð€†ç‚€‚‚€Ó€„aº¹€„Œ´€ €€ˆœ‹€‚àƒ€‚shelŠ¥€„Œ€€‚oldœ¢€ŠÊÕ€ˆî—€‚‡ƒ€ˆTÙÀ€Œþ‚€†Rec‰ª€‚ðÀ€Œî€„‘¿€ˆñ€‚Ÿº€ˆmad €†©ƒ€Œ›–€†ôà€†easier¶ƒ€‚‹“€ŠŠƒ€‚¶Ê€„×€„ivaú”€‚Û“‚Ñ€„†€ˆŒ€€†Ž‡€ŒÔ¦†æ¾€ˆon-l”³€‚鸀ˆßŠ€„•²€„tþ‚céõ€‚üŠ€‚perç’€‚Ì‚€‚½ª€„Âö€‚ŠË€‚Ô¶€‚gift-Ï‹€†”†¶„€†ð¯€„–Æ„ÿ€‚ing|‡€€ˆ·í€‚ic š€€‚›„€„ [[opeï‚€‚urÈ£€„programú¬‚Þ‰€’•‚‚ softwò‡€‚Õ¯€‚¼‚€‚à‰€‚TƼ€„cyber-È€¯’€Œ±ƒ€‚[[GNU䊀†LinuxŠ€€†IndyÓ„€„¤…€‚΀‚ݤ€‚kˆ«€„Õª€‚!-- ***NEEDS SOURCE THAT E-GOLD IS USED BY ANARCHISTS*** [[PºŽ€„ͬ€„cryptograp˜¯€‚⊀„냀„anony耄digÿú‚ ñ€„˜„͆€e-goðÊ„Ÿ€†Local Exch¸á‚«Œ€„•€‚SƆ€†¦€€„Ò‡€’郀‚ч€‚²€‚ƒ€‚aé’€‚nì½€‚. --¯Ô€‚¦Ñ€†ô„€Œö°‡€†«¶€ˆº…€Žƒ„€‚‘ó€ˆbÈ¿‚weaµé€‚žŒ€‚efe²¼€‚±¢€ˆa±Ù€‚ã´€‚Ý€€†À•‚Õ®€†Œƒ€„Œˆ€‚Û€€Šage†ê€„s¤€€„ð„€„‹€†ù†€‚eviõ‰‚•ã€¬modulaware.com/a/?m=sÑš„›ž€„id=0684832720¦„€‚ S‰˜€Š In—‹€‚dual -¢Š€‚s•••Ø€„Ñ£€„Ú€‚¾„€‚±€˜«®€®ü¹€†ypt•Ï€Ž×€„§Û€„CypherŽ§€‚ªü€„´‹€†Û„ܨ€ˆ±Ô€„þ€†©‹€„㢀„±€‚ÁÔ‚¨€€†]] (ª™€„Ÿ€Œ[[w™¾€‚)«€‚bª€‚•€†¸‰€ˆiüۀО‚û—€ˆ”ÜŒ…„€€’øÇ€‚¢µ€„ª¤€‚¾ãŽú€‚ÛË„ªŠ˜ˆ‡‚ÿŒ‚òÌ€‚°©„´ÜŒ|㙀‚Åœ€Šï„€‚õ˜€Œã„€†ô퀂ˆ¿€‚㮀‚·š„ô€ˆ¶…€„…Ô€„µ—€†Ÿö€‚Ç‚€„”‚€‚ؽ„‚£€Š,¢Š€†domeØ‚Ý¿€ˆ¹€‚á‹€‚÷›€‚Ûš„¡Œ€Šubôõˆ»‚€„ü߀‚olph BŠ½€‚eЀ„´˜€„’€„뀆©€‚ÇŒ€‚hûž€‚Ùò€‚Ú€ŠÙ†€‚¿ßˆ¯ˆ€‚…³€˜þ¯€Š¯»€‚σ€‚_é«€„/warÅ€€†½€€„ž»‚øª€†WºÑ€‚™”€†H倀”ñ™€„ò„€”. A loÞ–€„„€ŒÅ‚€ŽÄ“€‚À„‰’€ˆÊ€„ã°€„d¢€ˆParlia¿Š€‚¨‡€Šª€†I–Ä€Œ¹“€„¿ú€ˆâ€€Žƒ¬€‚¾–€Šsò ‚úˆ€‚in ò†€„‰”€‚, beÈÀ„œ€€Šam긆˜Ä€†don±€€‚È‚€‚ü€„‰Ù€„µ‚€˜©™ˆ.aolÔ‡€„vlntryst/hitler«‚€†Þ€‚Vƨ€ŠÛ‰€‚Ìý€‚y IÊ €ˆš€‚Ç‚Ö«€ŒH»€€„€˜½ö€„€€ŠØ†€‚—–€ˆ†€„Ô€€‚äÈ€†å‚€‚Ï€ˆ…€ˆemphasizeÉîŠÆ¬€‚Ø€‚y©Ò‚Ç»€‚gardÊð€Š´…€„Š€€†neiΆ€„b¦Ò‚t©›€‚r bulletsŒ¨€ˆÙ€„€ŽÅ«‚Ä€Œï‚€Š½Ì€‚Ó¢€„²€†''The EthܬˆV¦€€„°Á€¬ˆ¦‚÷ƒ€‚¶‚€‚Ìð„ˆ‘‚²€€„/¥ª‚cs_of_€€†.phpæ‚€šù‚€‚§€€†—‚€‚§€€†×‚€”ù€†George H.Ý퀂th]]. (Alsõ”€‚Ò–€Š§€†•ˆ€„„Oxymor½–€‚r What?—‚€ˆÏ€„Joe Peacotû™€ŒÒ€Œ €€„Fú°€‚Woodworõ€€‚±æ‚技‚Secê…€˜±™€–ꃀˆÓ€„냀Šaõø€„o‹€†ë¼€†³€‚ဂʀ€„Ò‰€‚Ž‚€‚Áž€‚¸€€‚¢„§š€Œ»¤€†ÛÜ‚ú—€‚''È¡€†'' eachˆ¨Í€‚ut–¦€„¿Á€„aÉÿ€‚o¥±€„Ü—€„€€‚†¡€‚,ÊÅ€†µ€€ˆö¦€‚¨Œ€‚“瀎³‚‚´…€„y|ˆ€€ˆÆ‹€†‡€‚äÛ€†Ì‰€„å¾€†Ñ¾€Š›Š€‚ aÎ󀄣•‚¼Ï€„d½É€‚û¤€†¶í€„f¤Š€„coe¢‚Ç…€„æ©€‚eΟ‚€‚¨…€Ž³Š€ˆ˜€†ng¹†€ˆÀ¾€‚¾¦€‚þ€‚Ý›€„Ì¿€„˜Ü€„Ì€€„Ì¡€†ë€Œ»ø€„逄 Œ€†Í€€‚rt–•€ˆ®€„coa¨Ö€‚on-builò†€‚¬‚€„at lea»Ã€‚™î€Ž•«ˆ·€€†øË€Š‰¬€‚úŸ€‚s‘¸€Š¡Š€‚ý€‚Ò¨€‚é‚€‚Žƒ€‚adj䮀‚vž•€‚.— †C±³€‚¦°‚烀‚õ€ŒÓó€‚:''M‚”‚³¥€„le:¾€„®€€¨¨©€‚'''÷ëŠ.󊀂øÒ€ˆ…€Ž¼‹€‚Ë„€†Â‹€„¡²€Š››‚ith ÏŽ€Š¿‚€„Ç®€à®€‚meï—€Š»Ù€„ëþ„Û“†beöŠ€„o«¶€‚¿€€‚„ž€‚“€‚îò€Šúª€‚„€€„à…€‚Æ€‚erick Engels„ŒŒ§¯€‚¨€ŠÆ´‚‘¨€„Ú€‚ñƒ€‚Ø€€ˆ “î‚gh:˜‚€‚¹‡€†ADz€¨†€‚«Ã€ˆlﮀ†–󀆵”€’§”€„£Œ€„؆€„;€‚–Ž€ˆ†€„—€€‚by ·³€‚ÀŸ€‚¯…€‚È€€‚¼È€†›„€‚Æ €‚É«€„ÈÅ€‚Ûü€„Úþ€„¤€€‚Ü„€„¶€€†by°±€iflõ¾€‚bay‰Ä€‚怆c„¾€‚n —Ÿ€˜¶€€„¢Å€„Ì£€„¯…€‚Ž–€‚……€‚allî‹€†i”€†¤Å†®ú€„«€‚y󰀎w½à€‚虊ʎvaiƒÁ€‚t¤š€‚¬§€‚«ý†’‚€‚Šßˆ¶€’က‚Ðó†—€ˆð€‚aÑЀ‚¿Šº€€„´¨€‚á’€‚‡È€‚›À€‚Woul³¤€†Î„´ó€‚mun´„€ˆlˆ–€‚Ö›€‚Ƈ€‚ñ炋ׂŸÉ€‚Û‚„‚»™€„ဂ‹ž€„²Ž€‚‚€ˆÃˆ€„Ö€€‚ஆŠ…€ˆØº€Š˜€€‚ºÞ‚geois?š„€†³’€ °½€‚²«Šˆ¶€„‡€„ve/”€€‚/¸€‚s/1872/10/ø€€Œ“€‚ ''On ¼í€‚“Ÿ€‚y''”€”ø†€„Utop–™€ˆú†€‚Ž•€ŠŽÅ€„ò†€†Ÿà€„쑆ɘ„föž€‚ýÈ€„or òŽ€„Àõ€„Æ€€„ÞŠ€‚×™€„’ƒ€‚›´€‚Š½€‚aÒ 튀Š'ü‰€‚nic‰¡€ŠÕö€”Ñ€†„Ü€ˆ, Carl LandauËꀂõÍ€”Á»Œî±€‚›¦€„€†‘€‚‘‡€Šö½€„‹‡€†un¤Ë„Œ–€Œ±€ˆâ€†šƒ‚唀‚at 䤎ᆀ‚a¤‘€ˆ†Ê€‚er evil¦ƒ€†Ï€‚𩀂Nj€‚ªŒ€‚ˆ‰€‚¨Š€‚€€†»Ã€†¬–‚‘ˆ€‚ce.–€€†€ù€‚ꀀþƒ€„Œ€‚¢„€‚a»€€ŠÔú‚适°¶„¬‡€ˆcease iÓÆ€‚¯€‚Ø€€ŽÆÚ€‚ÄÜ€†s½€€†´€„®¥ŠabsurdʹŽýƒ€‚Ù”€Š[ß‚€”|è‚€ŠÁƒ€‚™€€‚ŒŒ€‚å‚€žžÕ€‚H±ß€†ê…€‚IdeÌ€‚ndÜÃŒóî‚(1959) (retrñà€„×¯€†Ÿ€†à„€ŠËï€ ¬‘€„€ð€’È¿€‚Ü„€‚Jan„Õ„2§Ö‚[[2006]É…€”“«Ž¥ˆ€Š½Š–¢‡€‚Œ€†ž§€„Š„€Šâ™€†Ë€’‘…€˜ÍŒ€‚à€ŽBenjamin Tuck𚀂ÅÊ€Œ°‰‚…€‚y-f™¤€‚ôš€‚í€êƒ€„š €‚›€„𞀌ËÝ€„ƒ€ˆÂ²€‚Œ€€‚.Ø€Ž¥‹€†×…€Š‡«€‚¼€†s SiŠž€‚Óø€ˆ¼é„À‚proudly´Á€‚˜ý€‚ê„€‚•÷€‚acterѶ€Š‘ƒ€†ˆ†€ˆë–€Š•ˆ€„[[SŸ‘€†ŒÔ€„|Clasò€„Á€€Š«ˆ€„¦Ž„„’€‚ᢀ‚Ï‚€„Ÿ€€Œšš„¬€Š„‡€„Õÿ€‚¶€†ú“€‚¦Š€„ƒõ€„s˜‹€‚ºº€ˆ–€€Š[[petit›Š€iƒ¦€‚‚erha绀‚¿í€„lumpenprol妆ø¥€„œº€‚e.g©É‚ekhanovõ…€”G. V›€€Øú„‡‚€†òÀ€’ØŠ€‚å‚€‚튀”pЀ€Š/²ó‚úØ€ˆ/óó€‚x슀„Ú…€ŠÅ¯€„™‰€Ž]Þ€€†û˜€†áô€Š–ˆ€‚ a 𴈺‰€†¥í€ˆÜ€€‚— €”Ú‚€„ô“ˆenä‚€šs spoilÉ¥€‚middle-³ƒ€„©†€„dilettante⬀„×€„¸ã‚ýƒ‚ÑŠ€‚¶°€„ƒ€‚õŸ€„ï«€„Ï»€Ž¾¶†€€Œå×€‚û‘‚e”¤€‚‹ç€‚'''TacË’€‚ߌ€ˆ‚ƒ€‚ÈŒ€ŒIÝ€€‚退‚ëÅ€‚aÖ€‚Ž„€ŽÎ“€‚Ê€†±‹€†Ã’€‚by '›×€ts', 'Ì®€‚²¤€Œts'ꈀ„‰†€„«„€„ŠŠ€‚eÒ»€‚g 'tŠ€‚üŽ‚ק€†'Š€œÔœ€‚reaucraø瀂€‚Ö¨€ˆbehèž‚a dogmœ€€„facadˆŸ€¤Ì€ˆ.ôñ€‚ing—Æ€‚£„€ˆŽÅ€„s/SI/en/display/20å΀‚À逄°Š€„·…€‚Spectacle]뀂aƒª€„ 91쀀‚„€ŠÜ‚€„Hypocrisy솀„Ú˜€„¯‚€†®™€‚i¼®€‚†§€†ýš€‚¬…€‚’Š€†Þ‹€ŒÎ¡†P. d'H¹•€‚С†,î…€¤pinn½€„~suÅÄ€‚ne/whm2003/h»€€Š2ã…€‚üï„И€Žý÷€ˆ¿…€ˆc¿‡€‚行倀†õ…€˜¿ƒ€‚ﻀ„Ï„€†Ó¤€‚Ƀ€„ÙŠ€‚Ý„€‚Ͷ€’s’°‚tably [[PiÅ΀‚-Joseph耀Œ|ñ€€Œ¬€ŠMikhail“Þ€Š|›Þ€ˆü€‚ä쀘[[hä‚€ŠÈ¯€‚¤¨€ŠË€‚צ€„ç’€„†Œ€‚ÙÏ€†Ñ‘€„•€€Š›€‚h‹€Šdism®–‚Û‹€„䃀‚“‚‚ʳ€„prejudÊÌ„¢ñ‚9ÊÔÞ€„È€€‚ɪ€‚Íý„¥’€‚¶–€‚‰Ü€„¯€€‚á…€„nt-ñ”€‚ø€€’ƒ†€ˆ„€†iseÌ€„rŠ¨€‚²€ˆßˆ€„¯Ê€‚inuŸ†€‚†ª€†[[euro•‡€‚ric¯‚€ˆœÑ€„šŸ€„û€€‚impИ€‚„Ž€Ï€‚ùÛ€ˆÜñ„î‚€Šš€†Ü¿ˆª€‚ßø´€ˆ«€€†[[Cercl´€€Ž]].ç…€‚¤…€ŠÁ€steÓË€‚éµ€‚Ò’€ˆ×†€†iö„€‚€å€ŒŽ€‚‡³€‚gøà€„¤ä€˜ÿŠ€†¹€’ÓÅ€‚Ѐñ€‚€Ê€‚§ƒ€tÔÀ‚󧀂ò€‚Ì€†ê™€ˆsus¨‡€‚½‘€‚㿆 syÍÉ€‚hiz‹‚€‚by¥õŒ÷€€†¸€€„Û˜€ˆ™€€ŠýÆ€†Ã©€‚ÀÄ€‚µÄ€„ÝÛ€„ÐÒ€”À¾€Šî•‚ƒê€„‡ƒ€‚¿‚fª†Óƒ€Štئ€Š¦ß‚ŠÓ€‚ly¥Ë€„Ú〄irdµ‰€‚Ì‹€„ÜŽ€‚o·Ä‚‘‹€†¹€Œ×Ï€‚ñ玔´€ˆÖ€€‚•‡€‚䀈s΋‚쌀†Ü„€ˆÎ–€‚‘€‚´œ€‚ñ‰€ˆá€€„‹Æˆ‹ÌŠ€€‚(üŠ€ˆ) Re¦Â€†anÏ–€’ù©‹ù€„퀂ÿ–€†„…€Œ·£€‚Stanley G. Payþ¬€ˆï倆ü°€‚he„φÜ倂gimîÕ€‚盀‚ª÷€„ª€‚õÄ€„œ€†ÖÆ„¥®‚egot¤º€†Ð–€‚ÒÔˆò€ˆÆ¼€ï´‚È©† µé€„쀀„þÌ€‚´…€’Ò„€”|ª…†Ûˆ€‚é„€„¶–€‚¥ƒ€†Ü…€’gmu.edu/dÐÅ€‚î„€„s/e“Á€ˆs/bcî…„/spainî…€„Ϩ€‚Õ„€†o-SЀ†ô‚€„ÅÊ„ÙŠ€¤£€„︀„œ¹€‚henþ€‚aß–Noam_chomsk©ÃžŸÌ€„|©±€„am C¥€€†Ô²€‚1928–)]]ÞÝ„Þ™‚üƒ€‚’…€Ššú€‚ݧ‚Õ£ˆ˜œ€‚ÌÉ„”Ù€ˆù•€„„¡€ˆrÞÅ€ˆ™Ð€„Å™€†ƒ€‚ø€„celebrÑ¥€†°œ€‚»„€†¢‘€‚É‚àÒ€‚ï›ð˜€„剀†…€‚AÓ£ŠÂŒ€„™€€ŽÖ€ˆûÕ€„½Ç€„»„€‚²–€„¾ˆºÆ€‚½×€„—€ŠÚ€€„ƒŒ€Šly éli¢¸€‚øÖ€†è³€†±€€‚ºûŠñº€‚¬€ˆœƒ€„¾€Ž¿€Œª€‚-avo‰Û€’ts: - -*Ñ€‚À–€‚I¸ä‚쇀‚essÔà€†‹»€„gu¨€„Å“€†”ƒ€–»€€Œ×Ĉ fiö‡€„܉€‚ˆ“€„ [[Ursula K. Le Guû€‚¶€€ˆÛŠ€‚al ‰€‚À’„ó·‚öä„ ZinÀ€‚ -* —ƒ€‚r¯…‚È×€ˆÖ€€ŒHËÕ€‚” sÖ”„Ô€€ˆ[[Avant-ீ„‡€‚r·‚€‚΄€‚icoláƒÿ€‚sselló‚õí„Denõ™‚¸†€‚Š‚€†ì ‚t¦°€‚‡¾Œ½€‚¸Š€„¨Á€†Æš€‚ down¥€€„ÆÚ‚penhageÕŒ€‚²†€„²ç€‚¡ƒ€‚ω€‚employÙ‹€„Ä’€‚ºú†€–€„¡€„Ðû€Šâ€†ðË„˜Ó€„‹€‚‡»€Žª€€„¨Š€†e»‚”Ÿ€ˆ•½€‚Ÿ¤€„ity)|Š€€†˜ª€‚ø€€„squa軂¿€Œê„€‚kÝŽ€†Ÿ§€‚sti¸ë‚hrÿ„€Š¤€‚ÅÒŽì怂܀‚ataÈË€„Ю€„Mž¬€‚œ¦€‚¤—€†fa|¥¬€†ÿÔ€„to Ú΂NaziüŽ€ˆË‘€„¾†ü€€ˆµ‘†åþ€‚õÖ€ˆ‹îŠ¹…€„¼î€Œ÷¾€‚’™€†ª‡„ܰ€Œ[[AŽÏ€†“€€„ò€€ˆù€‚•‹€„ƒ‡€‚d™Ý„áÌ€‚so˜×€„Û‚€†gÉ€‚±Øˆ™¦€‚·Ã€‚Ï€‚ß„€†€ˆð“€‚ÐÒ€‚ÕÔ€ˆŸ¢ˆ. ¡Ö†™¯€‚¥€‚¯Â€„©Æ€‚ty¥©€„ø‡€Šm rÅÍ€‚þ€‚耀Œ‘×€‚ŠË€‚ø‡€ˆ¢«€‚Œƒ€‚lin„ë‘€”žÆ€‚Ž€†á€€„ª™€Ž ½€‚ rock,ÉÀ€‚òˆ€†í€ˆû’€Šg‡»‚ip hop€‚œ¼€„kÇ€€ˆ¬ˆ€‚Ž‚€„becå•€‚gé’€‚‹„ÑÌ€‚dium˜€ŽÓ¤†£€€‚É€‚Ô€–essage”Ñ€‚nš€€„[[U£Ü†Û瀂dom|UK‹“€„isò€†Ëà€ˆ©®€ŠÍ€€‚[[Ë€ŒÌ€Ð¥€‚e b¶€‚«¯€‚Áœ€†ˆ‚›‹€„Á€€„Ÿ€‚ͪ€„þŠ€Šä€„ç²€‚…¿€†ñÈŒÿ„€‚™¤€‚•‡€†[[Dutchû©€ˆ|DutŠÅ„Ђ€„ˆï€‚Ex]]Ú¾€Œexemplညܷ€†ÉÖ€‚ion. -''¼ù€‚…¿€Šdetai‚€„»ù€„œ€†o-ÐÀ€†©±€†’ €‚ü‚€‚== -È€‚ÁÅ€‚(PlÒ¦€„öô€‚¦Ñ€‚²„€„adõ‚€„ך€„i¹†€‚ü•€„à€‚Â¤€†À¨‚ ex”‚†¾†êÙ„ÛœŽÀ€€”à„€‚툀†’€†Œ€€‚•É€„p£Ö€‚age)ëÄ€ˆÙŽ€‚r ¬€‚à˜€ˆ¤€€Œrele׊€‚€†è‡€‚p㚀†Ð€€Šm»Š€‚Ù‰€‚Û”€‚brieý®€‚mmaê¡‚Š€‚Õ€‚sù€„¡å€‚–Ê€‚ÿŒ [[ž€®ˆÃ€†ïÅ€„œŽ€Š«€€‚ö€€Œ¥‰€‚é‚€ŽŠŠ€ˆ–€€š²òŽ¯€€šÛ‡€Š—€€ˆâ倆Ž€€Œþ‹€Œï€€¦Ù«Œ½€€ˆÛ¬€‚ª€€œnihi²€²»†ô€€bùÝ€šà€‚¥€€Šˆ€’ ¾€‚•¡€”†€ˆ“€€„ûü€‚¬€€’ÌŽ€‚΀‚˜§€‚œ¢€„”¤€»Ž€‚÷Ä€ŠÜ‚€‚symbolÏ€€„ú‚€„þ“€†ism/Links|LÌ„€˜§Ú€‚n«ÿ‚€€„€€šÃƒ€‚˜€€¦ÓÞ€”¥€€ŠMaj®œ€†å䀄¢–€†€ž€‚œ€‚Àµ€‚t®€€ˆPas瀂§©€‚Å“€„ó€Œá…€‚ŒŽ€‚Ê“€„ꀂ===®¬€†¬‹€„™°€‚¥‹‚= -*Ä€€‚粀딀„871)™€€‚Hayš€‚¯‚iotš€€†86š€€„¨ˆ€‚¾ýŠÊÓ‚×Ú€‚(1917 ÛÉ€„Žˆ192É€€†ûûŽÅâŒÎ€€„Ÿ€€Š§›€ŠRŒ·€Œž€€†36) (ψ€†‡ƒ€Œí‹€‚ÿ–€†ç€†½€€¦) -*³¡†68,Ž€Š(€€‚€„WTO Miôš€‚Ò×€„ Conúê„—¾€†1999|¢€€„eetˆ‰€‚in SeattlŸ‚€†999)Ò‚€„Books=¼›„main›„€œ™Ç‚oksý‚°‚€‚º•€Žˆ€‚ sß²€„Lj€‚¥€€„³¡€†ã¨€„“§€„Š›€Œï’€„߈€„‰€‚”݂ˆ€„ì½€‚¼†„à†€„ᛀ‚b ‚Îâ‚ùš€ˆÚˆ€¢ê€€„׈€† £€œí†€‚ò¬€‚ÁÔ‚Õ„€‚뉀‚ÞÇ€†Ä‚Çš€Šd¬‚mac.pitzerÏš€„œƒ€Št_AÈ©‚ves/b죀†/godÓ÷€„ate‹€€’_ch“¡†]ˆ€‚½¼–…€†Ó€€Šmá’†˜Ú€† Essays€âgÁº†/Á»ˆCW¬Ç€„þ€€„Peter K¯â€‚tk‡‚€ŽMutÏ€‚Aidî²€‚FŒô„®ƒ€‚E¥Å€‚ê—€‚|¡€€Ž‘€”§¸€‚gutenberg¾¢€„eÝÊ€‚/4341Á¦€¦÷€€ŠWꀄsᣀ‚ý°€„?Ü€€Ä360Ì€„Rudolf RockÅ´€†Òƒ€‚Ì‚€†o-Synd‚ˆ€‚Ö‚€‚(‹„€‚)|AÅ€„š€€”‚Ê€€‚Ç°œË€€„Ê…€‚ÐÇ€ŽÕŠ€†y²€€‚¢„€Œÿ€‚À§Žr¿«ˆÁÇ€ˆ†€€ˆ.asp¯€„Max Stirn󵀆€‚က„go AÁ•€„s Owdž†‚…€Šß€€‚df.€Ö‚se/~triad/sÀ€†/Ô€€„äá”Ô€€Žœ“€ˆÄƒ€‚ä…€‚¦Úœá€€””•€‚domnowƒ€„®¡€‚inyou·„€ˆÓŠ€‚Ö‚€†Ì‚€‚bÎÔ‚gion/Þž€ˆÝŠ€„ÙŒ€‚f󊀂Ž¦ŽñŒ€šÏ‰€’°€€†ƒ€„˜€€ˆ‚‡€‚EnglishøÓ€‚di€Ú†¨€€„ÕҜ苀†R™ƒŠáò€‚æ’€‚divž—€†=¢€†font-size: 85%º€†ëÖ€‚¬€€„õˆ€Œsº«„“€€„/div €€„ê°€„ÏÙ€„‘‚²€ˆnƒÇ€‚r°ý†¬–€„€€Œ·Š€‚À€‚†€‚cÖ¼€‚T´¢‚mœä€‚«‰€‚re-‹ý€‚Õ®€‚ -# {{note|bill}}‡ƒ€Œns52.super-hÐ߀‚“Ì€„~vaz1net/¬€€‚þ³€ˆ €‚“°€ˆ/†‚aw¦ƒ€ˆÙ€€ŒŸ¯Žß€€”€€Žö®€„±¯Œ·€€’èþˆµ€€flag.ª‘€„¹€€„Ž÷€„t/¬€€Šˆ€€„_pö¯†ç‰€ˆ‰€Œîù€‚etonÑ€€Ñ…€‚ŒÁ€ˆžì€ˆê€†ÿŽ€†_ñ“€Œ/»€€Š_boÝڀ€‚ Ag®Ä‚tエŸ‚¼Î‚A ˆ - Bo´°„Ö„€Šts‰€Yarros-NotþÀ€ˆ’€‚[Vi½ô‚ œ€€†£€€ˆ, •€€†ƒˆ€„󇀈'' VII¡“€‚˜¼€Œíµ€„1892⌀‚怀Œtotseõ‚€í€‚“€€„΃€„Ÿö‚¹€ˆÌƒ€161594߀‚l ¦€’¾¤€‚Ë€Šö†€„•£€”•‡€‚£‰Œ˜“€„== -Έ€‚overwhelmÏ€‚˜¨Œæ€„¾›†“ˆ€‚°€€„î—€‚Ÿø€†—ž€„ø‚€ˆ«ˆ€‚´—€Œô쀄«Û€‚d€‚â…€‚ù“€žm´¿‚ÄÌ€‚¹à€„s|link¹Þ€‚bÄ€‚¿•€‚ -{{wikiþ†€‚e|Deȇ„¥Õ€‚ˆ€‚ö€€Œ}} -*«…€Š”€€†oblogs.¸€ˆû„€„‚€ˆ™€€†] Blog—®€„¦‚€ŠÎð‚Å€€Œ¾€È¼€€ˆy ЀŠ]þ€”•‚€„“瀂ûÜ€’Ź€†¸›€†ãŸ€ˆ³€†‚€„‰š€‚£š€‚‰„esij€†Øž€†iˆŽƒ€„o³É€‚º§€ˆ”º€„s. -*HuÚ€‚ÒÊý€€„Š«€‚«›€‚þ‘€‚ÌÝ‚Ÿ‹€‚‹ñ€‚rt bioÂþ€‚ñ‚€„•€„ô÷€þ‚€‚s‰…€Œá¾‚´ó†on«“€„‘…€„bleed/g°é€‚ry‡€€Šß¼€ŽÞ’€ˆDaily B«€€‚'sì‚€Ž EncycÐ¥„ia]ø‚€Žô…€‚‚€‚ЗŠ/þ–”] (ñ。ž€€Ž|œ„€‚À€„²€„Ñ•€„‰„€ŠÊ€€„w–€†«à€‚®¿†l Work“²€‚Õ‚€†Ž€€‚ld] -Ë€„란‚Aº•Š!‹ƒ€‚„€†Ì„©‚€‚ …¢€‚‚½€‚¼€€„ •€‚ɺ€†°þ€ˆ¨ñ€‚öž€‚s far beyoƒ‚anageable sizÆò‚ÍŸ€ˆªã„˜ƒ€‚îÀ€‚¡Šå€€„®Ë€†Å‰€‚裂ᆀ–ßÛ€Žý”€‚i•…†í®„‚´Ö€„О€‚‡›€„º€€‚‹È€ˆknown •‰‚¥„‚ↀ‚ÿ怊Àš€‚ll bç‚utin¸‡€‚›€€‚ᨀŒ¢€„‘„€‚΀€‚kæ°‚ú€”Õ瀄£ú€‚lŽã„§ €Š»£‚Category:÷ƒ€Šm|*]]˜€€’For‘Ó€†ø²€Žë’€ˆ„€†ª€€Pþ‰€††Õ€‚•‰†ˆù„rî‚ints¶€€¾Á¹€†e©€€¬ù½€‚±€„ilosophy¨€€” -[[ar:لاسلطوية̀„ast:ª€€‚quismu’€€„bg:Ðнархизъ왈€†s«€€„hiza‹‘€‚ÃÆ‚€€„¼€€„e¢€€„c¢€€†ƒ‚€„Æó‚ -[[d¤€€†k£€€Œde€€„£€€’eo’€€„kiismoµ€€„eÈ€€†Ú€€„‘€€ˆt쀀„Ý‚€Œeu¡€€„×€€‚ €€†fa:دولت‌زدائی½€€„fi€€Œi€€†r΀€„…€„—€†gl¢€€„ò€€he:×× ×¨×›×™×–×Ê€€„hø€€†chiz《id’€€„‹€‚Ë€€†is:Stjór඀‚sisstefÄž€‚ -[[iÇ€†ø€€„¸€†ja:アナã‚ズム¨€€„ko:아나키즘“€€„l¼€€Œzmaû€€†n¶€†Ï€€„ü€€†nn‘€€„€ŽnÞ‚€Š¡€€Šp³€€ŒzɃ€†p×€€†û€r³€€†î‚€Œru–„€˜”„€ˆscoÈ‚€Ž€„simple”€€šk€€Š´‚€Œs„€†ç»€‚¸€€ˆr š°ä€€Šv½€€„À€‚€€„th:ลัทธิà¸à¸™à¸²Ž€€Š›à¹„ตย±€€†îƒ€†ÅŸñ€Šzh:æ— æ”¿åºœä¸»ä¹‰§€€„zh-min-nan:Hui-thóng-tÄ«-chú-gÄ«]]</text> - <™”€‚áö€„€€‚</page‰€€„õˆ€‚ˆ€€‚ <tÿ。>Afgha’¡€‚anª£€†y</™€€†È€€†id>13</id€€ˆ×€€’€€€‚ €€„5898948¦€€ŽÞ€€„m¸ˆ‚mp>2002-08-27T03:07:44Z</Ÿ€€ŽØ€€„ <¯Á€‚ributor“€€Š <username>Magnus¹÷€‚ske</—€€Œ¾€€ˆŠ€†4ƒ€”²Œ„Ò€€˜<miâ䀂/·€€Š<³¥€‚ent>whoops³€€‚€€†×€€ˆ<Þ€‚ xml:space="õ¥€„rve">#REDIRECT [[¼‚€ˆ·‹€‚Ò‚€•ƒ€€Geo£È€„—ƒ€ 쀗ƒ€²9“‚€”÷‚€Ž—ƒ€†2-25T15:43:11—ƒ€Ðip>Con꽄 ¼‚cript</ip·‚€Œü‚€ÔAutoÔÁ€‚Í Ԁ€ŠÁ€€‚Šƒ€â±‚€ŒŒƒ€œ -£†€üPű€„¢†€ž5ã‚€ŽŠƒ€°50Ɇ€ŽÙ„€‚¢†€ž1T10:42:35Šƒ€Ð¢†€Œ-‚›€‚ril†€ª166§€Ž—ˆ€‚ò†€ Ÿ†€¬fixÌ’€„µ€€‚–ƒ€âDemÉ…€†«œ€‚¦†€œãƒ†¤†€ 7ñ€Ž™ƒ€²1¦€€Ž™ƒ€ 5-17¤†€‚30:05Zþƒ€‚¼‰€ØAxelBoldtšƒ€ª2ÿ€€”˜ƒ€¤ö‚€Šˆƒ€‚redirect</•€€Šö†€Šà…€‚®‰€¾®‚€ˆ‡ƒ€œ¡†€¢
\ No newline at end of file diff --git a/internal/compress/flate/token.go b/internal/compress/flate/token.go deleted file mode 100644 index 40fa9454..00000000 --- a/internal/compress/flate/token.go +++ /dev/null @@ -1,379 +0,0 @@ -// Copyright 2009 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package flate - -import ( - "bytes" - "encoding/binary" - "fmt" - "io" - "math" -) - -const ( - // bits 0-16 xoffset = offset - MIN_OFFSET_SIZE, or literal - 16 bits - // bits 16-22 offsetcode - 5 bits - // bits 22-30 xlength = length - MIN_MATCH_LENGTH - 8 bits - // bits 30-32 type 0 = literal 1=EOF 2=Match 3=Unused - 2 bits - lengthShift = 22 - offsetMask = 1<<lengthShift - 1 - typeMask = 3 << 30 - literalType = 0 << 30 - matchType = 1 << 30 - matchOffsetOnlyMask = 0xffff -) - -// The length code for length X (MIN_MATCH_LENGTH <= X <= MAX_MATCH_LENGTH) -// is lengthCodes[length - MIN_MATCH_LENGTH] -var lengthCodes = [256]uint8{ - 0, 1, 2, 3, 4, 5, 6, 7, 8, 8, - 9, 9, 10, 10, 11, 11, 12, 12, 12, 12, - 13, 13, 13, 13, 14, 14, 14, 14, 15, 15, - 15, 15, 16, 16, 16, 16, 16, 16, 16, 16, - 17, 17, 17, 17, 17, 17, 17, 17, 18, 18, - 18, 18, 18, 18, 18, 18, 19, 19, 19, 19, - 19, 19, 19, 19, 20, 20, 20, 20, 20, 20, - 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, - 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, - 21, 21, 21, 21, 21, 21, 22, 22, 22, 22, - 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, - 22, 22, 23, 23, 23, 23, 23, 23, 23, 23, - 23, 23, 23, 23, 23, 23, 23, 23, 24, 24, - 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, - 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, - 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, - 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, - 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, - 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, - 25, 25, 26, 26, 26, 26, 26, 26, 26, 26, - 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, - 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, - 26, 26, 26, 26, 27, 27, 27, 27, 27, 27, - 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, - 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, - 27, 27, 27, 27, 27, 28, -} - -// lengthCodes1 is length codes, but starting at 1. -var lengthCodes1 = [256]uint8{ - 1, 2, 3, 4, 5, 6, 7, 8, 9, 9, - 10, 10, 11, 11, 12, 12, 13, 13, 13, 13, - 14, 14, 14, 14, 15, 15, 15, 15, 16, 16, - 16, 16, 17, 17, 17, 17, 17, 17, 17, 17, - 18, 18, 18, 18, 18, 18, 18, 18, 19, 19, - 19, 19, 19, 19, 19, 19, 20, 20, 20, 20, - 20, 20, 20, 20, 21, 21, 21, 21, 21, 21, - 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, - 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, - 22, 22, 22, 22, 22, 22, 23, 23, 23, 23, - 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, - 23, 23, 24, 24, 24, 24, 24, 24, 24, 24, - 24, 24, 24, 24, 24, 24, 24, 24, 25, 25, - 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, - 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, - 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, - 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, - 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, - 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, - 26, 26, 27, 27, 27, 27, 27, 27, 27, 27, - 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, - 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, - 27, 27, 27, 27, 28, 28, 28, 28, 28, 28, - 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, - 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, - 28, 28, 28, 28, 28, 29, -} - -var offsetCodes = [256]uint32{ - 0, 1, 2, 3, 4, 4, 5, 5, 6, 6, 6, 6, 7, 7, 7, 7, - 8, 8, 8, 8, 8, 8, 8, 8, 9, 9, 9, 9, 9, 9, 9, 9, - 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, - 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, - 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, - 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, - 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, - 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, - 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, - 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, - 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, - 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, - 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, - 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, - 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, - 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, -} - -// offsetCodes14 are offsetCodes, but with 14 added. -var offsetCodes14 = [256]uint32{ - 14, 15, 16, 17, 18, 18, 19, 19, 20, 20, 20, 20, 21, 21, 21, 21, - 22, 22, 22, 22, 22, 22, 22, 22, 23, 23, 23, 23, 23, 23, 23, 23, - 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, - 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, - 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, - 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, - 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, - 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, - 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, - 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, - 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, - 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, - 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, - 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, - 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, - 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, -} - -type token uint32 - -type tokens struct { - extraHist [32]uint16 // codes 256->maxnumlit - offHist [32]uint16 // offset codes - litHist [256]uint16 // codes 0->255 - nFilled int - n uint16 // Must be able to contain maxStoreBlockSize - tokens [maxStoreBlockSize + 1]token -} - -func (t *tokens) Reset() { - if t.n == 0 { - return - } - t.n = 0 - t.nFilled = 0 - for i := range t.litHist[:] { - t.litHist[i] = 0 - } - for i := range t.extraHist[:] { - t.extraHist[i] = 0 - } - for i := range t.offHist[:] { - t.offHist[i] = 0 - } -} - -func (t *tokens) Fill() { - if t.n == 0 { - return - } - for i, v := range t.litHist[:] { - if v == 0 { - t.litHist[i] = 1 - t.nFilled++ - } - } - for i, v := range t.extraHist[:literalCount-256] { - if v == 0 { - t.nFilled++ - t.extraHist[i] = 1 - } - } - for i, v := range t.offHist[:offsetCodeCount] { - if v == 0 { - t.offHist[i] = 1 - } - } -} - -func indexTokens(in []token) tokens { - var t tokens - t.indexTokens(in) - return t -} - -func (t *tokens) indexTokens(in []token) { - t.Reset() - for _, tok := range in { - if tok < matchType { - t.AddLiteral(tok.literal()) - continue - } - t.AddMatch(uint32(tok.length()), tok.offset()&matchOffsetOnlyMask) - } -} - -// emitLiteral writes a literal chunk and returns the number of bytes written. -func emitLiteral(dst *tokens, lit []byte) { - for _, v := range lit { - dst.tokens[dst.n] = token(v) - dst.litHist[v]++ - dst.n++ - } -} - -func (t *tokens) AddLiteral(lit byte) { - t.tokens[t.n] = token(lit) - t.litHist[lit]++ - t.n++ -} - -// from https://stackoverflow.com/a/28730362 -func mFastLog2(val float32) float32 { - ux := int32(math.Float32bits(val)) - log2 := (float32)(((ux >> 23) & 255) - 128) - ux &= -0x7f800001 - ux += 127 << 23 - uval := math.Float32frombits(uint32(ux)) - log2 += ((-0.34484843)*uval+2.02466578)*uval - 0.67487759 - return log2 -} - -// EstimatedBits will return an minimum size estimated by an *optimal* -// compression of the block. -// The size of the block -func (t *tokens) EstimatedBits() int { - shannon := float32(0) - bits := int(0) - nMatches := 0 - total := int(t.n) + t.nFilled - if total > 0 { - invTotal := 1.0 / float32(total) - for _, v := range t.litHist[:] { - if v > 0 { - n := float32(v) - shannon += atLeastOne(-mFastLog2(n*invTotal)) * n - } - } - // Just add 15 for EOB - shannon += 15 - for i, v := range t.extraHist[1 : literalCount-256] { - if v > 0 { - n := float32(v) - shannon += atLeastOne(-mFastLog2(n*invTotal)) * n - bits += int(lengthExtraBits[i&31]) * int(v) - nMatches += int(v) - } - } - } - if nMatches > 0 { - invTotal := 1.0 / float32(nMatches) - for i, v := range t.offHist[:offsetCodeCount] { - if v > 0 { - n := float32(v) - shannon += atLeastOne(-mFastLog2(n*invTotal)) * n - bits += int(offsetExtraBits[i&31]) * int(v) - } - } - } - return int(shannon) + bits -} - -// AddMatch adds a match to the tokens. -// This function is very sensitive to inlining and right on the border. -func (t *tokens) AddMatch(xlength uint32, xoffset uint32) { - if debugDeflate { - if xlength >= maxMatchLength+baseMatchLength { - panic(fmt.Errorf("invalid length: %v", xlength)) - } - if xoffset >= maxMatchOffset+baseMatchOffset { - panic(fmt.Errorf("invalid offset: %v", xoffset)) - } - } - oCode := offsetCode(xoffset) - xoffset |= oCode << 16 - - t.extraHist[lengthCodes1[uint8(xlength)]]++ - t.offHist[oCode&31]++ - t.tokens[t.n] = token(matchType | xlength<<lengthShift | xoffset) - t.n++ -} - -// AddMatchLong adds a match to the tokens, potentially longer than max match length. -// Length should NOT have the base subtracted, only offset should. -func (t *tokens) AddMatchLong(xlength int32, xoffset uint32) { - if debugDeflate { - if xoffset >= maxMatchOffset+baseMatchOffset { - panic(fmt.Errorf("invalid offset: %v", xoffset)) - } - } - oc := offsetCode(xoffset) - xoffset |= oc << 16 - for xlength > 0 { - xl := xlength - if xl > 258 { - // We need to have at least baseMatchLength left over for next loop. - if xl > 258+baseMatchLength { - xl = 258 - } else { - xl = 258 - baseMatchLength - } - } - xlength -= xl - xl -= baseMatchLength - t.extraHist[lengthCodes1[uint8(xl)]]++ - t.offHist[oc&31]++ - t.tokens[t.n] = token(matchType | uint32(xl)<<lengthShift | xoffset) - t.n++ - } -} - -func (t *tokens) AddEOB() { - t.tokens[t.n] = token(endBlockMarker) - t.extraHist[0]++ - t.n++ -} - -func (t *tokens) Slice() []token { - return t.tokens[:t.n] -} - -// VarInt returns the tokens as varint encoded bytes. -func (t *tokens) VarInt() []byte { - b := make([]byte, binary.MaxVarintLen32*int(t.n)) - var off int - for _, v := range t.tokens[:t.n] { - off += binary.PutUvarint(b[off:], uint64(v)) - } - return b[:off] -} - -// FromVarInt restores t to the varint encoded tokens provided. -// Any data in t is removed. -func (t *tokens) FromVarInt(b []byte) error { - buf := bytes.NewReader(b) - var toks []token - for { - r, err := binary.ReadUvarint(buf) - if err == io.EOF { - break - } - if err != nil { - return err - } - toks = append(toks, token(r)) - } - t.indexTokens(toks) - return nil -} - -// Returns the type of a token -func (t token) typ() uint32 { return uint32(t) & typeMask } - -// Returns the literal of a literal token -func (t token) literal() uint8 { return uint8(t) } - -// Returns the extra offset of a match token -func (t token) offset() uint32 { return uint32(t) & offsetMask } - -func (t token) length() uint8 { return uint8(t >> lengthShift) } - -// Convert length to code. -func lengthCode(len uint8) uint8 { return lengthCodes[len] } - -// Returns the offset code corresponding to a specific offset -func offsetCode(off uint32) uint32 { - if false { - if off < uint32(len(offsetCodes)) { - return offsetCodes[off&255] - } else if off>>7 < uint32(len(offsetCodes)) { - return offsetCodes[(off>>7)&255] + 14 - } else { - return offsetCodes[(off>>14)&255] + 28 - } - } - if off < uint32(len(offsetCodes)) { - return offsetCodes[uint8(off)] - } - return offsetCodes14[uint8(off>>7)] -} diff --git a/internal/compress/flate/token_test.go b/internal/compress/flate/token_test.go deleted file mode 100644 index 9070c341..00000000 --- a/internal/compress/flate/token_test.go +++ /dev/null @@ -1,54 +0,0 @@ -package flate - -import ( - "bytes" - "os" - "testing" -) - -type testFatal interface { - Fatal(args ...any) -} - -// loadTestTokens will load test tokens. -// First block from enwik9, varint encoded. -func loadTestTokens(t testFatal) *tokens { - b, err := os.ReadFile("testdata/tokens.bin") - if err != nil { - t.Fatal(err) - } - var tokens tokens - err = tokens.FromVarInt(b) - if err != nil { - t.Fatal(err) - } - return &tokens -} - -func Test_tokens_EstimatedBits(t *testing.T) { - tok := loadTestTokens(t) - // The estimated size, update if method changes. - const expect = 221057 - n := tok.EstimatedBits() - var buf bytes.Buffer - wr := newHuffmanBitWriter(&buf) - wr.writeBlockDynamic(tok, true, nil, true) - if wr.err != nil { - t.Fatal(wr.err) - } - wr.flush() - t.Log("got:", n, "actual:", buf.Len()*8, "(header not part of estimate)") - if n != expect { - t.Error("want:", expect, "bits, got:", n) - } -} - -func Benchmark_tokens_EstimatedBits(b *testing.B) { - tok := loadTestTokens(b) - b.ResetTimer() - // One "byte", one token iteration. - b.SetBytes(1) - for i := 0; i < b.N; i++ { - _ = tok.EstimatedBits() - } -} diff --git a/internal/compress/flate/writer_test.go b/internal/compress/flate/writer_test.go deleted file mode 100644 index 01893e50..00000000 --- a/internal/compress/flate/writer_test.go +++ /dev/null @@ -1,544 +0,0 @@ -// Copyright 2012 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package flate - -import ( - "archive/zip" - "bytes" - "compress/flate" - "fmt" - "io" - "math" - "math/rand" - "os" - "runtime" - "strconv" - "strings" - "testing" -) - -func TestWriterMemUsage(t *testing.T) { - testMem := func(t *testing.T, fn func()) { - var before, after runtime.MemStats - runtime.GC() - runtime.ReadMemStats(&before) - fn() - runtime.GC() - runtime.ReadMemStats(&after) - t.Logf("%s: Memory Used: %dKB, %d allocs", t.Name(), (after.HeapInuse-before.HeapInuse)/1024, after.HeapObjects-before.HeapObjects) - } - data := make([]byte, 100000) - t.Run("stateless", func(t *testing.T) { - testMem(t, func() { - StatelessDeflate(io.Discard, data, false, nil) - }) - }) - for level := HuffmanOnly; level <= BestCompression; level++ { - t.Run(fmt.Sprint("level-", level), func(t *testing.T) { - var zr *Writer - var err error - testMem(t, func() { - zr, err = NewWriter(io.Discard, level) - if err != nil { - t.Fatal(err) - } - zr.Write(data) - }) - zr.Close() - }) - } - for level := HuffmanOnly; level <= BestCompression; level++ { - t.Run(fmt.Sprint("stdlib-", level), func(t *testing.T) { - var zr *flate.Writer - var err error - testMem(t, func() { - zr, err = flate.NewWriter(io.Discard, level) - if err != nil { - t.Fatal(err) - } - zr.Write(data) - }) - zr.Close() - }) - } -} - -func TestWriterRegression(t *testing.T) { - data, err := os.ReadFile("testdata/regression.zip") - if err != nil { - t.Fatal(err) - } - for level := HuffmanOnly; level <= BestCompression; level++ { - t.Run(fmt.Sprint("level_", level), func(t *testing.T) { - zr, err := zip.NewReader(bytes.NewReader(data), int64(len(data))) - if err != nil { - t.Fatal(err) - } - - for _, tt := range zr.File { - if !strings.HasSuffix(t.Name(), "") { - continue - } - - t.Run(tt.Name, func(t *testing.T) { - if testing.Short() && tt.FileInfo().Size() > 10000 { - t.SkipNow() - } - r, err := tt.Open() - if err != nil { - t.Error(err) - return - } - in, err := io.ReadAll(r) - if err != nil { - t.Error(err) - } - msg := "level " + strconv.Itoa(level) + ":" - buf := new(bytes.Buffer) - fw, err := NewWriter(buf, level) - if err != nil { - t.Fatal(msg + err.Error()) - } - n, err := fw.Write(in) - if n != len(in) { - t.Fatal(msg + "short write") - } - if err != nil { - t.Fatal(msg + err.Error()) - } - err = fw.Close() - if err != nil { - t.Fatal(msg + err.Error()) - } - fr1 := NewReader(buf) - data2, err := io.ReadAll(fr1) - if err != nil { - t.Fatal(msg + err.Error()) - } - if !bytes.Equal(in, data2) { - t.Fatal(msg + "not equal") - } - // Do it again... - msg = "level " + strconv.Itoa(level) + " (reset):" - buf.Reset() - fw.Reset(buf) - n, err = fw.Write(in) - if n != len(in) { - t.Fatal(msg + "short write") - } - if err != nil { - t.Fatal(msg + err.Error()) - } - err = fw.Close() - if err != nil { - t.Fatal(msg + err.Error()) - } - fr1 = NewReader(buf) - data2, err = io.ReadAll(fr1) - if err != nil { - t.Fatal(msg + err.Error()) - } - if !bytes.Equal(in, data2) { - t.Fatal(msg + "not equal") - } - }) - } - }) - } -} - -func benchmarkEncoder(b *testing.B, testfile, level, n int) { - b.SetBytes(int64(n)) - buf0, err := os.ReadFile(testfiles[testfile]) - if err != nil { - b.Fatal(err) - } - if len(buf0) == 0 { - b.Fatalf("test file %q has no data", testfiles[testfile]) - } - buf1 := make([]byte, n) - for i := 0; i < n; i += len(buf0) { - if len(buf0) > n-i { - buf0 = buf0[:n-i] - } - copy(buf1[i:], buf0) - } - buf0 = nil - runtime.GC() - w, err := NewWriter(io.Discard, level) - if err != nil { - b.Fatal(err) - } - b.ResetTimer() - b.ReportAllocs() - for i := 0; i < b.N; i++ { - w.Reset(io.Discard) - _, err = w.Write(buf1) - if err != nil { - b.Fatal(err) - } - err = w.Close() - if err != nil { - b.Fatal(err) - } - } -} - -func BenchmarkEncodeDigitsConstant1e4(b *testing.B) { benchmarkEncoder(b, digits, constant, 1e4) } -func BenchmarkEncodeDigitsConstant1e5(b *testing.B) { benchmarkEncoder(b, digits, constant, 1e5) } -func BenchmarkEncodeDigitsConstant1e6(b *testing.B) { benchmarkEncoder(b, digits, constant, 1e6) } -func BenchmarkEncodeDigitsSpeed1e4(b *testing.B) { benchmarkEncoder(b, digits, speed, 1e4) } -func BenchmarkEncodeDigitsSpeed1e5(b *testing.B) { benchmarkEncoder(b, digits, speed, 1e5) } -func BenchmarkEncodeDigitsSpeed1e6(b *testing.B) { benchmarkEncoder(b, digits, speed, 1e6) } -func BenchmarkEncodeDigitsDefault1e4(b *testing.B) { benchmarkEncoder(b, digits, default_, 1e4) } -func BenchmarkEncodeDigitsDefault1e5(b *testing.B) { benchmarkEncoder(b, digits, default_, 1e5) } -func BenchmarkEncodeDigitsDefault1e6(b *testing.B) { benchmarkEncoder(b, digits, default_, 1e6) } -func BenchmarkEncodeDigitsCompress1e4(b *testing.B) { benchmarkEncoder(b, digits, compress, 1e4) } -func BenchmarkEncodeDigitsCompress1e5(b *testing.B) { benchmarkEncoder(b, digits, compress, 1e5) } -func BenchmarkEncodeDigitsCompress1e6(b *testing.B) { benchmarkEncoder(b, digits, compress, 1e6) } -func BenchmarkEncodeDigitsSL1e4(b *testing.B) { benchmarkStatelessEncoder(b, digits, 1e4) } -func BenchmarkEncodeDigitsSL1e5(b *testing.B) { benchmarkStatelessEncoder(b, digits, 1e5) } -func BenchmarkEncodeDigitsSL1e6(b *testing.B) { benchmarkStatelessEncoder(b, digits, 1e6) } -func BenchmarkEncodeTwainConstant1e4(b *testing.B) { benchmarkEncoder(b, twain, constant, 1e4) } -func BenchmarkEncodeTwainConstant1e5(b *testing.B) { benchmarkEncoder(b, twain, constant, 1e5) } -func BenchmarkEncodeTwainConstant1e6(b *testing.B) { benchmarkEncoder(b, twain, constant, 1e6) } -func BenchmarkEncodeTwainSpeed1e4(b *testing.B) { benchmarkEncoder(b, twain, speed, 1e4) } -func BenchmarkEncodeTwainSpeed1e5(b *testing.B) { benchmarkEncoder(b, twain, speed, 1e5) } -func BenchmarkEncodeTwainSpeed1e6(b *testing.B) { benchmarkEncoder(b, twain, speed, 1e6) } -func BenchmarkEncodeTwainDefault1e4(b *testing.B) { benchmarkEncoder(b, twain, default_, 1e4) } -func BenchmarkEncodeTwainDefault1e5(b *testing.B) { benchmarkEncoder(b, twain, default_, 1e5) } -func BenchmarkEncodeTwainDefault1e6(b *testing.B) { benchmarkEncoder(b, twain, default_, 1e6) } -func BenchmarkEncodeTwainCompress1e4(b *testing.B) { benchmarkEncoder(b, twain, compress, 1e4) } -func BenchmarkEncodeTwainCompress1e5(b *testing.B) { benchmarkEncoder(b, twain, compress, 1e5) } -func BenchmarkEncodeTwainCompress1e6(b *testing.B) { benchmarkEncoder(b, twain, compress, 1e6) } -func BenchmarkEncodeTwainSL1e4(b *testing.B) { benchmarkStatelessEncoder(b, twain, 1e4) } -func BenchmarkEncodeTwainSL1e5(b *testing.B) { benchmarkStatelessEncoder(b, twain, 1e5) } -func BenchmarkEncodeTwainSL1e6(b *testing.B) { benchmarkStatelessEncoder(b, twain, 1e6) } - -func BenchmarkEncodeTwain1024Win1e4(b *testing.B) { benchmarkEncoder(b, twain, oneK, 1e4) } -func BenchmarkEncodeTwain1024Win1e5(b *testing.B) { benchmarkEncoder(b, twain, oneK, 1e5) } -func BenchmarkEncodeTwain1024Win1e6(b *testing.B) { benchmarkEncoder(b, twain, oneK, 1e6) } - -func benchmarkStatelessEncoder(b *testing.B, testfile, n int) { - b.SetBytes(int64(n)) - buf0, err := os.ReadFile(testfiles[testfile]) - if err != nil { - b.Fatal(err) - } - if len(buf0) == 0 { - b.Fatalf("test file %q has no data", testfiles[testfile]) - } - buf1 := make([]byte, n) - for i := 0; i < n; i += len(buf0) { - if len(buf0) > n-i { - buf0 = buf0[:n-i] - } - copy(buf1[i:], buf0) - } - buf0 = nil - runtime.GC() - b.ResetTimer() - b.ReportAllocs() - for i := 0; i < b.N; i++ { - w := NewStatelessWriter(io.Discard) - _, err = w.Write(buf1) - if err != nil { - b.Fatal(err) - } - err = w.Close() - if err != nil { - b.Fatal(err) - } - } -} - -// A writer that fails after N writes. -type errorWriter struct { - N int -} - -func (e *errorWriter) Write(b []byte) (int, error) { - if e.N <= 0 { - return 0, io.ErrClosedPipe - } - e.N-- - return len(b), nil -} - -// Test if errors from the underlying writer is passed upwards. -func TestWriteError(t *testing.T) { - buf := new(bytes.Buffer) - n := 65536 - if !testing.Short() { - n *= 4 - } - for i := 0; i < n; i++ { - fmt.Fprintf(buf, "asdasfasf%d%dfghfgujyut%dyutyu\n", i, i, i) - } - in := buf.Bytes() - // We create our own buffer to control number of writes. - copyBuf := make([]byte, 128) - for l := range 10 { - for fail := 1; fail <= 256; fail *= 2 { - // Fail after 'fail' writes - ew := &errorWriter{N: fail} - w, err := NewWriter(ew, l) - if err != nil { - t.Fatalf("NewWriter: level %d: %v", l, err) - } - n, err := copyBuffer(w, bytes.NewBuffer(in), copyBuf) - if err == nil { - t.Fatalf("Level %d: Expected an error, writer was %#v", l, ew) - } - n2, err := w.Write([]byte{1, 2, 2, 3, 4, 5}) - if n2 != 0 { - t.Fatal("Level", l, "Expected 0 length write, got", n) - } - if err == nil { - t.Fatal("Level", l, "Expected an error") - } - err = w.Flush() - if err == nil { - t.Fatal("Level", l, "Expected an error on flush") - } - err = w.Close() - if err == nil { - t.Fatal("Level", l, "Expected an error on close") - } - - w.Reset(io.Discard) - n2, err = w.Write([]byte{1, 2, 3, 4, 5, 6}) - if err != nil { - t.Fatal("Level", l, "Got unexpected error after reset:", err) - } - if n2 == 0 { - t.Fatal("Level", l, "Got 0 length write, expected > 0") - } - if testing.Short() { - return - } - } - } -} - -// Test if errors from the underlying writer is passed upwards. -func TestWriter_Reset(t *testing.T) { - buf := new(bytes.Buffer) - n := 65536 - if !testing.Short() { - n *= 4 - } - for i := 0; i < n; i++ { - fmt.Fprintf(buf, "asdasfasf%d%dfghfgujyut%dyutyu\n", i, i, i) - } - in := buf.Bytes() - for l := range 10 { - if testing.Short() && l > 1 { - continue - } - t.Run(fmt.Sprintf("level-%d", l), func(t *testing.T) { - t.Parallel() - offset := 1 - if testing.Short() { - offset = 256 - } - for ; offset <= 256; offset *= 2 { - // Fail after 'fail' writes - w, err := NewWriter(io.Discard, l) - if err != nil { - t.Fatalf("NewWriter: level %d: %v", l, err) - } - if w.d.fast == nil { - t.Skip("Not Fast...") - return - } - for i := 0; i < (bufferReset-len(in)-offset-maxMatchOffset)/maxMatchOffset; i++ { - // skip ahead to where we are close to wrap around... - w.d.fast.Reset() - } - w.d.fast.Reset() - _, err = w.Write(in) - if err != nil { - t.Fatal(err) - } - for range 50 { - // skip ahead again... This should wrap around... - w.d.fast.Reset() - } - w.d.fast.Reset() - - _, err = w.Write(in) - if err != nil { - t.Fatal(err) - } - for range (math.MaxUint32 - bufferReset) / maxMatchOffset { - // skip ahead to where we are close to wrap around... - w.d.fast.Reset() - } - - _, err = w.Write(in) - if err != nil { - t.Fatal(err) - } - err = w.Close() - if err != nil { - t.Fatal(err) - } - } - }) - } -} - -func TestDeterministicL1(t *testing.T) { testDeterministic(1, t) } -func TestDeterministicL2(t *testing.T) { testDeterministic(2, t) } -func TestDeterministicL3(t *testing.T) { testDeterministic(3, t) } -func TestDeterministicL4(t *testing.T) { testDeterministic(4, t) } -func TestDeterministicL5(t *testing.T) { testDeterministic(5, t) } -func TestDeterministicL6(t *testing.T) { testDeterministic(6, t) } -func TestDeterministicL7(t *testing.T) { testDeterministic(7, t) } -func TestDeterministicL8(t *testing.T) { testDeterministic(8, t) } -func TestDeterministicL9(t *testing.T) { testDeterministic(9, t) } -func TestDeterministicL0(t *testing.T) { testDeterministic(0, t) } -func TestDeterministicLM2(t *testing.T) { testDeterministic(-2, t) } - -func testDeterministic(i int, t *testing.T) { - // Test so much we cross a good number of block boundaries. - length := maxStoreBlockSize*30 + 500 - if testing.Short() { - length /= 10 - } - - // Create a random, but compressible stream. - rng := rand.New(rand.NewSource(1)) - t1 := make([]byte, length) - for i := range t1 { - t1[i] = byte(rng.Int63() & 7) - } - - // Do our first encode. - var b1 bytes.Buffer - br := bytes.NewBuffer(t1) - w, err := NewWriter(&b1, i) - if err != nil { - t.Fatal(err) - } - // Use a very small prime sized buffer. - cbuf := make([]byte, 787) - _, err = copyBuffer(w, br, cbuf) - if err != nil { - t.Fatal(err) - } - w.Close() - - // We choose a different buffer size, - // bigger than a maximum block, and also a prime. - var b2 bytes.Buffer - cbuf = make([]byte, 81761) - br2 := bytes.NewBuffer(t1) - w2, err := NewWriter(&b2, i) - if err != nil { - t.Fatal(err) - } - _, err = copyBuffer(w2, br2, cbuf) - if err != nil { - t.Fatal(err) - } - w2.Close() - - b1b := b1.Bytes() - b2b := b2.Bytes() - - if !bytes.Equal(b1b, b2b) { - t.Errorf("level %d did not produce deterministic result, result mismatch, len(a) = %d, len(b) = %d", i, len(b1b), len(b2b)) - } - - // Test using io.WriterTo interface. - var b3 bytes.Buffer - br = bytes.NewBuffer(t1) - w, err = NewWriter(&b3, i) - if err != nil { - t.Fatal(err) - } - _, err = br.WriteTo(w) - if err != nil { - t.Fatal(err) - } - w.Close() - - b3b := b3.Bytes() - if !bytes.Equal(b1b, b3b) { - t.Errorf("level %d (io.WriterTo) did not produce deterministic result, result mismatch, len(a) = %d, len(b) = %d", i, len(b1b), len(b3b)) - } -} - -// copyBuffer is a copy of io.CopyBuffer, since we want to support older go versions. -// This is modified to never use io.WriterTo or io.ReaderFrom interfaces. -func copyBuffer(dst io.Writer, src io.Reader, buf []byte) (written int64, err error) { - if buf == nil { - buf = make([]byte, 32*1024) - } - for { - nr, er := src.Read(buf) - if nr > 0 { - nw, ew := dst.Write(buf[0:nr]) - if nw > 0 { - written += int64(nw) - } - if ew != nil { - err = ew - break - } - if nr != nw { - err = io.ErrShortWrite - break - } - } - if er == io.EOF { - break - } - if er != nil { - err = er - break - } - } - return written, err -} - -func BenchmarkCompressAllocations(b *testing.B) { - payload := []byte(strings.Repeat("Tiny payload", 20)) - for j := -2; j <= 9; j++ { - b.Run("level("+strconv.Itoa(j)+")", func(b *testing.B) { - b.Run("flate", func(b *testing.B) { - b.ReportAllocs() - - for i := 0; i < b.N; i++ { - w, err := NewWriter(io.Discard, j) - if err != nil { - b.Fatal(err) - } - w.Write(payload) - w.Close() - } - }) - }) - } -} - -func BenchmarkCompressAllocationsSingle(b *testing.B) { - payload := []byte(strings.Repeat("Tiny payload", 20)) - const level = 2 - b.Run("flate", func(b *testing.B) { - b.ReportAllocs() - - for i := 0; i < b.N; i++ { - w, err := NewWriter(io.Discard, level) - if err != nil { - b.Fatal(err) - } - w.Write(payload) - w.Close() - } - }) -} diff --git a/internal/compress/internal/doc.go b/internal/compress/internal/doc.go deleted file mode 100644 index b28bad09..00000000 --- a/internal/compress/internal/doc.go +++ /dev/null @@ -1,2 +0,0 @@ -// Package internal provides utilities internal to the compression library. -package internal diff --git a/internal/compress/internal/fuzz/helpers.go b/internal/compress/internal/fuzz/helpers.go deleted file mode 100644 index 71332ac6..00000000 --- a/internal/compress/internal/fuzz/helpers.go +++ /dev/null @@ -1,218 +0,0 @@ -// Copyright (c) 2024+ Klaus Post. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -// Package fuzz provides a way to add test cases to a testing.F instance from a zip file. -package fuzz - -import ( - "archive/zip" - "bytes" - "encoding/binary" - "fmt" - "go/ast" - "go/parser" - "go/token" - "io" - "os" - "strconv" - "testing" -) - -type InputType uint8 - -const ( - // TypeRaw indicates that files are raw bytes. - TypeRaw InputType = iota - // TypeGoFuzz indicates files are from Go Fuzzer. - TypeGoFuzz - // TypeOSSFuzz indicates that files are from OSS fuzzer with size before data. - TypeOSSFuzz -) - -// AddFromZip will read the supplied zip and add all as corpus for f. -// Byte slices only. -func AddFromZip(f *testing.F, filename string, t InputType, short bool) { - file, err := os.Open(filename) - if err != nil { - f.Fatal(err) - } - fi, err := file.Stat() - if fi == nil { - return - } - - if err != nil { - f.Fatal(err) - } - zr, err := zip.NewReader(file, fi.Size()) - if err != nil { - f.Fatal(err) - } - for i, file := range zr.File { - if short && i%10 != 0 { - continue - } - rc, err := file.Open() - if err != nil { - f.Fatal(err) - } - - b, err := io.ReadAll(rc) - if err != nil { - f.Fatal(err) - } - rc.Close() - t := t - if t == TypeOSSFuzz { - t = TypeRaw // Fallback - if len(b) >= 4 { - sz := binary.BigEndian.Uint32(b) - if sz <= uint32(len(b))-4 { - f.Add(b[4 : 4+sz]) - continue - } - } - } - - if bytes.HasPrefix(b, []byte("go test fuzz")) { - t = TypeGoFuzz - } else { - t = TypeRaw - } - - if t == TypeRaw { - f.Add(b) - continue - } - vals, err := unmarshalCorpusFile(b) - if err != nil { - f.Fatal(err) - } - for _, v := range vals { - f.Add(v) - } - } -} - -// ReturnFromZip will read the supplied zip and add all as corpus for f. -// Byte slices only. -func ReturnFromZip(tb testing.TB, filename string, t InputType, fn func([]byte)) { - file, err := os.Open(filename) - if err != nil { - tb.Fatal(err) - } - fi, err := file.Stat() - if fi == nil { - return - } - if err != nil { - tb.Fatal(err) - } - zr, err := zip.NewReader(file, fi.Size()) - if err != nil { - tb.Fatal(err) - } - for _, file := range zr.File { - rc, err := file.Open() - if err != nil { - tb.Fatal(err) - } - - b, err := io.ReadAll(rc) - if err != nil { - tb.Fatal(err) - } - rc.Close() - t := t - if t == TypeOSSFuzz { - t = TypeRaw // Fallback - if len(b) >= 4 { - sz := binary.BigEndian.Uint32(b) - if sz <= uint32(len(b))-4 { - fn(b[4 : 4+sz]) - continue - } - } - } - - if bytes.HasPrefix(b, []byte("go test fuzz")) { - t = TypeGoFuzz - } else { - t = TypeRaw - } - - if t == TypeRaw { - fn(b) - continue - } - vals, err := unmarshalCorpusFile(b) - if err != nil { - tb.Fatal(err) - } - for _, v := range vals { - fn(v) - } - } -} - -// unmarshalCorpusFile decodes corpus bytes into their respective values. -func unmarshalCorpusFile(b []byte) ([][]byte, error) { - if len(b) == 0 { - return nil, fmt.Errorf("cannot unmarshal empty string") - } - lines := bytes.Split(b, []byte("\n")) - if len(lines) < 2 { - return nil, fmt.Errorf("must include version and at least one value") - } - vals := make([][]byte, 0, len(lines)-1) - for _, line := range lines[1:] { - line = bytes.TrimSpace(line) - if len(line) == 0 { - continue - } - v, err := parseCorpusValue(line) - if err != nil { - return nil, fmt.Errorf("malformed line %q: %v", line, err) - } - vals = append(vals, v) - } - return vals, nil -} - -// parseCorpusValue -func parseCorpusValue(line []byte) ([]byte, error) { - fs := token.NewFileSet() - expr, err := parser.ParseExprFrom(fs, "(test)", line, 0) - if err != nil { - return nil, err - } - call, ok := expr.(*ast.CallExpr) - if !ok { - return nil, fmt.Errorf("expected call expression") - } - if len(call.Args) != 1 { - return nil, fmt.Errorf("expected call expression with 1 argument; got %d", len(call.Args)) - } - arg := call.Args[0] - - if arrayType, ok := call.Fun.(*ast.ArrayType); ok { - if arrayType.Len != nil { - return nil, fmt.Errorf("expected []byte or primitive type") - } - elt, ok := arrayType.Elt.(*ast.Ident) - if !ok || elt.Name != "byte" { - return nil, fmt.Errorf("expected []byte") - } - lit, ok := arg.(*ast.BasicLit) - if !ok || lit.Kind != token.STRING { - return nil, fmt.Errorf("string literal required for type []byte") - } - s, err := strconv.Unquote(lit.Value) - if err != nil { - return nil, err - } - return []byte(s), nil - } - return nil, fmt.Errorf("expected []byte") -} diff --git a/internal/compress/internal/le/le.go b/internal/compress/internal/le/le.go deleted file mode 100644 index 890ba873..00000000 --- a/internal/compress/internal/le/le.go +++ /dev/null @@ -1,6 +0,0 @@ -// Package le provides fast little endian integer routines. -package le - -type Indexer interface { - int | int8 | int16 | int32 | int64 | uint | uint8 | uint16 | uint32 | uint64 -} diff --git a/internal/compress/internal/le/unsafe_disabled.go b/internal/compress/internal/le/unsafe_disabled.go deleted file mode 100644 index 4f2a0d8c..00000000 --- a/internal/compress/internal/le/unsafe_disabled.go +++ /dev/null @@ -1,42 +0,0 @@ -//go:build !(amd64 || arm64 || ppc64le || riscv64) || nounsafe || purego || appengine - -package le - -import ( - "encoding/binary" -) - -// Load8 will load from b at index i. -func Load8[I Indexer](b []byte, i I) byte { - return b[i] -} - -// Load16 will load from b at index i. -func Load16[I Indexer](b []byte, i I) uint16 { - return binary.LittleEndian.Uint16(b[i:]) -} - -// Load32 will load from b at index i. -func Load32[I Indexer](b []byte, i I) uint32 { - return binary.LittleEndian.Uint32(b[i:]) -} - -// Load64 will load from b at index i. -func Load64[I Indexer](b []byte, i I) uint64 { - return binary.LittleEndian.Uint64(b[i:]) -} - -// Store16 will store v at b. -func Store16(b []byte, v uint16) { - binary.LittleEndian.PutUint16(b, v) -} - -// Store32 will store v at b. -func Store32(b []byte, v uint32) { - binary.LittleEndian.PutUint32(b, v) -} - -// Store64 will store v at b. -func Store64[I Indexer](b []byte, i I, v uint64) { - binary.LittleEndian.PutUint64(b[i:], v) -} diff --git a/internal/compress/internal/le/unsafe_enabled.go b/internal/compress/internal/le/unsafe_enabled.go deleted file mode 100644 index b47fd0db..00000000 --- a/internal/compress/internal/le/unsafe_enabled.go +++ /dev/null @@ -1,52 +0,0 @@ -// We enable 64 bit LE platforms: - -//go:build (amd64 || arm64 || ppc64le || riscv64) && !nounsafe && !purego && !appengine - -package le - -import ( - "unsafe" -) - -// Load8 will load from b at index i. -func Load8[I Indexer](b []byte, i I) byte { - // return binary.LittleEndian.Uint16(b[i:]) - // return *(*uint16)(unsafe.Pointer(&b[i])) - return *(*byte)(unsafe.Add(unsafe.Pointer(unsafe.SliceData(b)), i)) -} - -// Load16 will load from b at index i. -func Load16[I Indexer](b []byte, i I) uint16 { - // return binary.LittleEndian.Uint16(b[i:]) - // return *(*uint16)(unsafe.Pointer(&b[i])) - return *(*uint16)(unsafe.Add(unsafe.Pointer(unsafe.SliceData(b)), i)) -} - -// Load32 will load from b at index i. -func Load32[I Indexer](b []byte, i I) uint32 { - // return binary.LittleEndian.Uint32(b[i:]) - // return *(*uint32)(unsafe.Pointer(&b[i])) - return *(*uint32)(unsafe.Add(unsafe.Pointer(unsafe.SliceData(b)), i)) -} - -// Load64 will load from b at index i. -func Load64[I Indexer](b []byte, i I) uint64 { - // return binary.LittleEndian.Uint64(b[i:]) - // return *(*uint64)(unsafe.Pointer(&b[i])) - return *(*uint64)(unsafe.Add(unsafe.Pointer(unsafe.SliceData(b)), i)) -} - -// Store16 will store v at b. -func Store16(b []byte, v uint16) { - *(*uint16)(unsafe.Pointer(unsafe.SliceData(b))) = v -} - -// Store32 will store v at b. -func Store32(b []byte, v uint32) { - *(*uint32)(unsafe.Pointer(unsafe.SliceData(b))) = v -} - -// Store64 will store v at b[i:]. -func Store64[I Indexer](b []byte, i I, v uint64) { - *(*uint64)(unsafe.Add(unsafe.Pointer(unsafe.SliceData(b)), i)) = v -} diff --git a/internal/compress/zlib/reader.go b/internal/compress/zlib/reader.go index f58a904a..74357525 100644 --- a/internal/compress/zlib/reader.go +++ b/internal/compress/zlib/reader.go @@ -34,12 +34,14 @@ and to read that data back: package zlib import ( + "bytes" "encoding/binary" "errors" "hash" "io" - "lindenii.org/go/furgit/internal/compress/flate" + "github.com/klauspost/compress/flate" + "lindenii.org/go/lgo/intconv" "lindenii.org/go/lgo/sync" ) @@ -74,6 +76,7 @@ type Reader struct { trailerRead uint64 err error scratch [4]byte + br bytes.Reader } // NewReader creates a new ReadCloser. @@ -99,6 +102,23 @@ func NewReaderDict(r io.Reader, dict []byte) (*Reader, error) { return z, nil } +// NewReaderBytes is like [NewReader] but reads directly from payload, +// reusing a [bytes.Reader] pooled with the returned Reader +// instead of allocating a fresh one per call. +// It is the caller's responsibility to call Close on the ReadCloser when done. +func NewReaderBytes(payload []byte) (*Reader, error) { + z := readerPool.Get() + + z.br.Reset(payload) + + err := z.reset(&z.br, nil) + if err != nil { + return nil, err + } + + return z, nil +} + // Read decompresses bytes from receiver into p. func (z *Reader) Read(p []byte) (int, error) { if z.err != nil { @@ -167,6 +187,7 @@ func (z *Reader) Close() error { return z.err } + z.br.Reset(nil) readerPool.Put(z) return nil diff --git a/internal/compress/zlib/reader_reset.go b/internal/compress/zlib/reader_reset.go index b1b9afa6..6a9340c2 100644 --- a/internal/compress/zlib/reader_reset.go +++ b/internal/compress/zlib/reader_reset.go @@ -10,14 +10,15 @@ import ( "errors" "io" + "github.com/klauspost/compress/flate" + "lindenii.org/go/furgit/internal/adler32" - "lindenii.org/go/furgit/internal/compress/flate" "lindenii.org/go/lgo/intconv" ) // reset resets receiver to read a new zlib stream. func (z *Reader) reset(r io.Reader, dict []byte) error { - *z = Reader{decompressor: z.decompressor} + *z = Reader{decompressor: z.decompressor, digest: z.digest, br: z.br} var input flate.Reader if fr, ok := r.(flate.Reader); ok { @@ -95,7 +96,11 @@ func (z *Reader) reset(r io.Reader, dict []byte) error { return z.err } - z.digest = adler32.New() + if z.digest == nil { + z.digest = adler32.New() + } else { + z.digest.Reset() + } return nil } diff --git a/internal/compress/zlib/writer.go b/internal/compress/zlib/writer.go index ae327e6d..0fcb2ca8 100644 --- a/internal/compress/zlib/writer.go +++ b/internal/compress/zlib/writer.go @@ -10,7 +10,8 @@ import ( "hash" "io" - "lindenii.org/go/furgit/internal/compress/flate" + "github.com/klauspost/compress/flate" + "lindenii.org/go/lgo/sync" ) @@ -70,7 +71,7 @@ func NewWriterLevel(w io.Writer, level int) (*Writer, error) { // the Writer is closed. func NewWriterLevelDict(w io.Writer, level int, dict []byte) (*Writer, error) { if level < HuffmanOnly || level > BestCompression { - return nil, fmt.Errorf("zlib: invalid compression level: %d", level) + return nil, fmt.Errorf("zlib: invalid compression level: %d", level) //nolint:err113 } z := writerPool.Get() @@ -139,7 +140,7 @@ func (z *Writer) Write(p []byte) (n int, err error) { if err != nil { z.err = err - return n, err + return n, err //nolint:wrapcheck } _, err = z.digest.Write(p) diff --git a/internal/compress/zlib/writer_header.go b/internal/compress/zlib/writer_header.go index 6a5f09de..00407f49 100644 --- a/internal/compress/zlib/writer_header.go +++ b/internal/compress/zlib/writer_header.go @@ -7,8 +7,9 @@ package zlib import ( "encoding/binary" + "github.com/klauspost/compress/flate" + "lindenii.org/go/furgit/internal/adler32" - "lindenii.org/go/furgit/internal/compress/flate" ) // writeHeader writes the ZLIB header. @@ -61,7 +62,7 @@ func (z *Writer) writeHeader() (err error) { // after a Reset call. z.compressor, err = flate.NewWriterDict(z.w, z.level, z.dict) if err != nil { - return err + return err //nolint:wrapcheck } z.digest = adler32.New() diff --git a/internal/format/doc.go b/internal/format/doc.go new file mode 100644 index 00000000..f0a89b22 --- /dev/null +++ b/internal/format/doc.go @@ -0,0 +1,2 @@ +// Package format provides low-level routines for various file formats. +package format diff --git a/internal/format/packfile/delta/apply.go b/internal/format/packfile/delta/apply.go new file mode 100644 index 00000000..4210d1b3 --- /dev/null +++ b/internal/format/packfile/delta/apply.go @@ -0,0 +1,122 @@ +package delta + +import ( + "fmt" + + "lindenii.org/go/lgo/intconv" +) + +// MaxChainDepth is the maximum supported delta chain length. +// Resolvers reject chains deeper than this +// to bound recursion depth and reconstruction work. +const MaxChainDepth = 1 << 12 + +// Apply applies one inflated delta payload to base +// and returns the reconstructed result. +// +// delta must include the leading base/result size header. +func Apply(base, delta []byte) ([]byte, error) { + baseSize, resultSize, pos, err := ParseHeaderSizes(delta) + if err != nil { + return nil, err + } + + if baseSize != uint64(len(base)) { + return nil, fmt.Errorf("%w: base size mismatch", ErrMalformedDelta) + } + + outLen, err := intconv.Uint64ToInt(resultSize) + if err != nil { + return nil, fmt.Errorf("%w: result size overflows int", ErrMalformedDelta) + } + + out := make([]byte, outLen) + outPos := 0 + + for pos < len(delta) { + op := delta[pos] + pos++ + + switch { + case op&0x80 != 0: + outPos, err = applyCopy(out, outPos, base, delta, &pos, op) + case op != 0: + outPos, err = applyInsert(out, outPos, delta, &pos, int(op)) + default: + err = fmt.Errorf("%w: invalid opcode 0", ErrMalformedDelta) + } + + if err != nil { + return nil, err + } + } + + if outPos != len(out) { + return nil, fmt.Errorf("%w: result size mismatch", ErrMalformedDelta) + } + + return out, nil +} + +// applyCopy executes one copy instruction, +// copying a base range into out, +// and returns the new output position. +func applyCopy(out []byte, outPos int, base, delta []byte, pos *int, op byte) (int, error) { + off, err := parseCopyOperand(delta, pos, op, 0, 4) + if err != nil { + return 0, err + } + + n, err := parseCopyOperand(delta, pos, op, 4, 3) + if err != nil { + return 0, err + } + + if n == 0 { + n = 0x10000 + } + + if off+n > len(base) || outPos+n > len(out) { + return 0, fmt.Errorf("%w: copy out of bounds", ErrMalformedDelta) + } + + copy(out[outPos:outPos+n], base[off:off+n]) + + return outPos + n, nil +} + +// applyInsert executes one insert instruction, +// copying n literal delta bytes into out, +// and returns the new output position. +func applyInsert(out []byte, outPos int, delta []byte, pos *int, n int) (int, error) { + if *pos+n > len(delta) || outPos+n > len(out) { + return 0, fmt.Errorf("%w: insert out of bounds", ErrMalformedDelta) + } + + copy(out[outPos:outPos+n], delta[*pos:*pos+n]) + *pos += n + + return outPos + n, nil +} + +// parseCopyOperand assembles one little-endian copy instruction operand +// from the operand bytes selected by op's flag bits +// firstBit through firstBit+count-1. +func parseCopyOperand(delta []byte, pos *int, op byte, firstBit uint, count int) (int, error) { + value := 0 + + for i := range count { + if op&(1<<(firstBit+uint(i))) == 0 { + continue + } + + if *pos >= len(delta) { + return 0, fmt.Errorf("%w: truncated copy operand", ErrMalformedDelta) + } + + value |= int(delta[*pos]) << (8 * i) + *pos++ + } + + return value, nil +} diff --git a/internal/format/packfile/delta/apply_test.go b/internal/format/packfile/delta/apply_test.go new file mode 100644 index 00000000..bdf454d9 --- /dev/null +++ b/internal/format/packfile/delta/apply_test.go @@ -0,0 +1,174 @@ +package delta_test + +import ( + "bytes" + "errors" + "testing" + + "lindenii.org/go/furgit/internal/format/packfile/delta" +) + +func TestApplyInsert(t *testing.T) { + t.Parallel() + + data := delta.AppendHeaderSizes(nil, 0, 5) + data = append(data, 5, 'h', 'e', 'l', 'l', 'o') + + out, err := delta.Apply(nil, data) + if err != nil { + t.Fatalf("Apply: %v", err) + } + + if string(out) != "hello" { + t.Fatalf("Apply = %q, want %q", out, "hello") + } +} + +func TestApplyCopy(t *testing.T) { + t.Parallel() + + base := []byte("0123456789") + + // One offset byte and one size byte. + data := delta.AppendHeaderSizes(nil, 10, 4) + data = append(data, 0x91, 3, 4) + + out, err := delta.Apply(base, data) + if err != nil { + t.Fatalf("Apply: %v", err) + } + + if string(out) != "3456" { + t.Fatalf("Apply = %q, want %q", out, "3456") + } +} + +func TestApplyCopyImplicitSize(t *testing.T) { + t.Parallel() + + base := bytes.Repeat([]byte{0xab}, 0x10000+10) + + // Offset 0 and the implicit copy size 0x10000. + data := delta.AppendHeaderSizes(nil, uint64(len(base)), 0x10000) + data = append(data, 0x80) + + out, err := delta.Apply(base, data) + if err != nil { + t.Fatalf("Apply: %v", err) + } + + if !bytes.Equal(out, base[:0x10000]) { + t.Fatalf("Apply implicit-size copy mismatch") + } +} + +func TestApplyMixed(t *testing.T) { + t.Parallel() + + base := []byte("the quick brown fox") + + data := delta.AppendHeaderSizes(nil, uint64(len(base)), 9) + data = append(data, 0x91, 4, 5) // copy "quick" + data = append(data, 3, '!', '?', '!') // insert "!?!" + data = append(data, 0x91, 16, 1) // copy "f" + + out, err := delta.Apply(base, data) + if err != nil { + t.Fatalf("Apply: %v", err) + } + + if string(out) != "quick!?!f" { + t.Fatalf("Apply = %q, want %q", out, "quick!?!f") + } +} + +func TestApplyEmptyResult(t *testing.T) { + t.Parallel() + + data := delta.AppendHeaderSizes(nil, 3, 0) + + out, err := delta.Apply([]byte("abc"), data) + if err != nil { + t.Fatalf("Apply: %v", err) + } + + if len(out) != 0 { + t.Fatalf("Apply = %q, want empty", out) + } +} + +func TestApplyMalformed(t *testing.T) { + t.Parallel() + + base := []byte("0123456789") + + cases := []struct { + name string + ops []byte + baseSize uint64 + resultSize uint64 + }{ + { + name: "opcode zero", + ops: []byte{0x00}, + baseSize: 10, + resultSize: 1, + }, + { + name: "truncated copy operand", + ops: []byte{0x91, 3}, + baseSize: 10, + resultSize: 4, + }, + { + name: "copy past base", + ops: []byte{0x91, 8, 5}, + baseSize: 10, + resultSize: 5, + }, + { + name: "copy past result", + ops: []byte{0x91, 0, 5}, + baseSize: 10, + resultSize: 3, + }, + { + name: "insert past delta", + ops: []byte{5, 'a', 'b'}, + baseSize: 10, + resultSize: 5, + }, + { + name: "insert past result", + ops: []byte{5, 'a', 'b', 'c', 'd', 'e'}, + baseSize: 10, + resultSize: 3, + }, + { + name: "base size mismatch", + ops: []byte{0x91, 0, 1}, + baseSize: 9, + resultSize: 1, + }, + { + name: "result shorter than declared", + ops: []byte{0x91, 0, 1}, + baseSize: 10, + resultSize: 2, + }, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + + data := delta.AppendHeaderSizes(nil, tc.baseSize, tc.resultSize) + data = append(data, tc.ops...) + + _, err := delta.Apply(base, data) + if !errors.Is(err, delta.ErrMalformedDelta) { + t.Fatalf("Apply error = %v, want ErrMalformedDelta", err) + } + }) + } +} diff --git a/internal/format/packfile/delta/doc.go b/internal/format/packfile/delta/doc.go new file mode 100644 index 00000000..f63c96a8 --- /dev/null +++ b/internal/format/packfile/delta/doc.go @@ -0,0 +1,2 @@ +// Package delta provides various routines to handle Git delta compression. +package delta diff --git a/internal/format/packfile/delta/header.go b/internal/format/packfile/delta/header.go new file mode 100644 index 00000000..d5dd93fb --- /dev/null +++ b/internal/format/packfile/delta/header.go @@ -0,0 +1,98 @@ +package delta + +import ( + "errors" + "fmt" +) + +// ErrMalformedDelta reports that +// a delta payload is truncated, overlong, +// declares sizes that overflow uint64, +// contains invalid instructions, +// or does not match the supplied base or declared sizes. +var ErrMalformedDelta = errors.New("internal/format/packfile/delta: malformed delta") + +// MaxSizeVarintLen is the maximum encoded length +// of one delta header size varint. +// Every uint64 size is encodable within this bound, +// and parsing rejects longer encodings. +const MaxSizeVarintLen = 10 + +// MaxHeaderSizesLen is the maximum encoded length +// of the base/result size header of a delta payload. +// +// Callers reading a delta from a stream may inflate +// MaxHeaderSizesLen bytes +// (or fewer if the payload ends sooner) +// and parse with [ParseHeaderSizes]; +// no valid header is longer. +const MaxHeaderSizesLen = 2 * MaxSizeVarintLen + +// ParseHeaderSizes parses the base size and result size varints +// at the beginning of one inflated delta payload. +func ParseHeaderSizes(data []byte) (baseSize, resultSize uint64, consumed int, err error) { + baseSize, consumed, err = parseSizeVarint(data, 0) + if err != nil { + return 0, 0, 0, err + } + + resultSize, consumed, err = parseSizeVarint(data, consumed) + if err != nil { + return 0, 0, 0, err + } + + return baseSize, resultSize, consumed, nil +} + +// AppendHeaderSizes appends the base/result size header encoding +// of one delta payload to dst. +func AppendHeaderSizes(dst []byte, baseSize, resultSize uint64) []byte { + dst = appendSizeVarint(dst, baseSize) + + return appendSizeVarint(dst, resultSize) +} + +// appendSizeVarint appends the encoding of one delta header size varint to dst. +func appendSizeVarint(dst []byte, value uint64) []byte { + for value >= 0x80 { + dst = append(dst, byte(value)|0x80) + value >>= 7 + } + + return append(dst, byte(value)) +} + +// parseSizeVarint parses one delta header size varint +// beginning at pos, +// and returns the position past it. +func parseSizeVarint(data []byte, pos int) (value uint64, consumed int, err error) { + start := pos + + shift := uint(0) + + for { + if pos-start >= MaxSizeVarintLen { + return 0, 0, fmt.Errorf("%w: overlong size varint", ErrMalformedDelta) + } + + if pos >= len(data) { + return 0, 0, fmt.Errorf("%w: truncated size varint", ErrMalformedDelta) + } + + b := data[pos] + pos++ + + group := uint64(b & 0x7f) + if group<<shift>>shift != group { + return 0, 0, fmt.Errorf("%w: size overflows uint64", ErrMalformedDelta) + } + + value |= group << shift + + if b&0x80 == 0 { + return value, pos, nil + } + + shift += 7 + } +} diff --git a/internal/format/packfile/delta/header_test.go b/internal/format/packfile/delta/header_test.go new file mode 100644 index 00000000..8d97674f --- /dev/null +++ b/internal/format/packfile/delta/header_test.go @@ -0,0 +1,88 @@ +package delta_test + +import ( + "bytes" + "errors" + "math" + "testing" + + "lindenii.org/go/furgit/internal/format/packfile/delta" +) + +func TestParseHeaderSizesRoundTrip(t *testing.T) { + t.Parallel() + + cases := []struct { + name string + baseSize uint64 + resultSize uint64 + }{ + {name: "zero", baseSize: 0, resultSize: 0}, + {name: "small", baseSize: 5, resultSize: 130}, + {name: "boundaries", baseSize: 127, resultSize: 128}, + {name: "large", baseSize: 1 << 32, resultSize: 1 << 57}, + {name: "max", baseSize: math.MaxUint64, resultSize: math.MaxUint64}, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + + data := delta.AppendHeaderSizes(nil, tc.baseSize, tc.resultSize) + + wantConsumed := len(data) + + data = append(data, 0xde, 0xad) + + baseSize, resultSize, consumed, err := delta.ParseHeaderSizes(data) + if err != nil { + t.Fatalf("ParseHeaderSizes: %v", err) + } + + if baseSize != tc.baseSize { + t.Fatalf("ParseHeaderSizes base size = %d, want %d", baseSize, tc.baseSize) + } + + if resultSize != tc.resultSize { + t.Fatalf("ParseHeaderSizes result size = %d, want %d", resultSize, tc.resultSize) + } + + if consumed != wantConsumed { + t.Fatalf("ParseHeaderSizes consumed = %d, want %d", consumed, wantConsumed) + } + }) + } +} + +func TestParseHeaderSizesMalformed(t *testing.T) { + t.Parallel() + + cases := []struct { + name string + data []byte + }{ + {name: "empty", data: []byte{}}, + {name: "truncated first varint", data: []byte{0x80}}, + {name: "missing second varint", data: []byte{0x05}}, + {name: "truncated second varint", data: []byte{0x05, 0x80}}, + { + name: "overflow", + data: append(bytes.Repeat([]byte{0xff}, 9), 0x7f), + }, + { + name: "overlong", + data: append(bytes.Repeat([]byte{0x80}, 10), 0x00), + }, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + + _, _, _, err := delta.ParseHeaderSizes(tc.data) + if !errors.Is(err, delta.ErrMalformedDelta) { + t.Fatalf("ParseHeaderSizes error = %v, want ErrMalformedDelta", err) + } + }) + } +} diff --git a/internal/format/packfile/doc.go b/internal/format/packfile/doc.go new file mode 100644 index 00000000..d656e256 --- /dev/null +++ b/internal/format/packfile/doc.go @@ -0,0 +1,2 @@ +// Package packfile provides Git packfile format parsing primitives. +package packfile diff --git a/internal/format/packfile/entry_header.go b/internal/format/packfile/entry_header.go new file mode 100644 index 00000000..e5f2d62d --- /dev/null +++ b/internal/format/packfile/entry_header.go @@ -0,0 +1,165 @@ +package packfile + +import ( + "errors" + "fmt" + + "lindenii.org/go/furgit/object/id" +) + +// ErrMalformedEntryHeader reports that +// a packfile entry header is truncated, overlong, +// has an unsupported entry type, +// or declares a size that overflows uint64. +var ErrMalformedEntryHeader = errors.New("internal/format/packfile: malformed entry header") + +// MaxTypeSizeLen is the maximum encoded length +// of the type/size prefix of an entry header. +// Every uint64 size is encodable within this bound, +// and [ParseEntryHeader] rejects longer prefixes. +const MaxTypeSizeLen = 10 + +// MaxEntryHeaderLen returns the maximum encoded length +// of a full entry header +// for packs whose object IDs are hashSize bytes. +// +// Callers parsing from a stream may buffer +// MaxEntryHeaderLen bytes +// (or fewer if the pack data ends sooner) +// and parse with [ParseEntryHeader]; +// no valid entry header is longer. +func MaxEntryHeaderLen(hashSize int) int { + return MaxTypeSizeLen + max(hashSize, MaxOfsDeltaDistanceLen) +} + +// EntryHeader is one parsed packfile entry header: +// everything from the start of the entry +// up to its zlib payload. +type EntryHeader struct { + // Type is the packfile entry type. + Type EntryType + + // Size is the declared inflated size + // of the entry's payload. + // For delta entries this is the delta size, + // not the reconstructed object size. + Size uint64 + + // HeaderLen is the number of bytes + // the header occupies in the pack; + // the zlib payload begins HeaderLen bytes + // after the start of the entry. + HeaderLen int + + // RefBase holds the base object ID + // for ref-delta entries. + // Only the first hashSize bytes are meaningful. + RefBase [id.MaxObjectIDSize]byte + + // OfsDistance is the backward distance + // from the start of this entry + // to the start of the base entry, + // for ofs-delta entries. + OfsDistance uint64 +} + +// ParseEntryHeader parses one packfile entry header +// from the beginning of data. +// +// hashSize must be the object ID size +// of the pack's object format; +// ParseEntryHeader panics on implausible hash sizes. +// +// data need not contain the whole entry; +// [MaxEntryHeaderLen] bytes always suffice. +// Headers of types [EntryTypeInvalid] and [EntryTypeFuture] +// are rejected as malformed. +func ParseEntryHeader(data []byte, hashSize int) (EntryHeader, error) { + var zero EntryHeader + + if hashSize <= 0 || hashSize > id.MaxObjectIDSize { + panic("internal/format/packfile: invalid hash size") + } + + if len(data) == 0 { + return zero, fmt.Errorf("%w: truncated type/size prefix", ErrMalformedEntryHeader) + } + + first := data[0] + header := EntryHeader{ + Type: EntryType((first >> 4) & 0x07), + Size: uint64(first & 0x0f), + HeaderLen: 1, + } + + shift := uint(4) + + b := first + for b&0x80 != 0 { + if header.HeaderLen >= MaxTypeSizeLen { + return zero, fmt.Errorf("%w: overlong type/size prefix", ErrMalformedEntryHeader) + } + + if header.HeaderLen >= len(data) { + return zero, fmt.Errorf("%w: truncated type/size prefix", ErrMalformedEntryHeader) + } + + b = data[header.HeaderLen] + header.HeaderLen++ + + group := uint64(b & 0x7f) + if group<<shift>>shift != group { + return zero, fmt.Errorf("%w: size overflows uint64", ErrMalformedEntryHeader) + } + + header.Size |= group << shift + shift += 7 + } + + switch header.Type { + case EntryTypeCommit, EntryTypeTree, EntryTypeBlob, EntryTypeTag: + // Base entries have nothing between the type/size prefix and the payload. + case EntryTypeRefDelta: + end := header.HeaderLen + hashSize + if end > len(data) { + return zero, fmt.Errorf("%w: truncated ref-delta base ID", ErrMalformedEntryHeader) + } + + copy(header.RefBase[:], data[header.HeaderLen:end]) + header.HeaderLen = end + case EntryTypeOfsDelta: + dist, consumed, err := ParseOfsDeltaDistance(data[header.HeaderLen:]) + if err != nil { + return zero, fmt.Errorf("%w: %w", ErrMalformedEntryHeader, err) + } + + header.OfsDistance = dist + header.HeaderLen += consumed + case EntryTypeInvalid, EntryTypeFuture: + return zero, fmt.Errorf("%w: unsupported entry type", ErrMalformedEntryHeader) + default: + return zero, fmt.Errorf("%w: unsupported entry type", ErrMalformedEntryHeader) + } + + return header, nil +} + +// AppendTypeSize appends the type/size prefix encoding +// of an entry header to dst. +// +// entryType must be a valid on-disk entry type; +// [EntryTypeInvalid] and [EntryTypeFuture] and +// values that do not fit in three bits +// produce garbage encodings. +func AppendTypeSize(dst []byte, entryType EntryType, size uint64) []byte { + b := byte(entryType)<<4 | byte(size&0x0f) + size >>= 4 + + for size != 0 { + dst = append(dst, b|0x80) + b = byte(size & 0x7f) + size >>= 7 + } + + return append(dst, b) +} diff --git a/internal/format/packfile/entry_header_test.go b/internal/format/packfile/entry_header_test.go new file mode 100644 index 00000000..79dc2740 --- /dev/null +++ b/internal/format/packfile/entry_header_test.go @@ -0,0 +1,251 @@ +package packfile_test + +import ( + "bytes" + "errors" + "math" + "testing" + + "lindenii.org/go/furgit/internal/format/packfile" + "lindenii.org/go/furgit/object/id" +) + +func TestParseEntryHeaderBase(t *testing.T) { + t.Parallel() + + // Commit entry of size 85: + // 0x95 has the continuation bit set, type 1, and size low bits 0x5; + // 0x05 contributes 5<<4. + header, err := packfile.ParseEntryHeader([]byte{0x95, 0x05}, 20) + if err != nil { + t.Fatalf("ParseEntryHeader: %v", err) + } + + if header.Type != packfile.EntryTypeCommit { + t.Fatalf("ParseEntryHeader type = %d, want %d", header.Type, packfile.EntryTypeCommit) + } + + if header.Size != 85 { + t.Fatalf("ParseEntryHeader size = %d, want 85", header.Size) + } + + if header.HeaderLen != 2 { + t.Fatalf("ParseEntryHeader header len = %d, want 2", header.HeaderLen) + } +} + +func TestParseEntryHeaderRefDelta(t *testing.T) { + t.Parallel() + + for _, objectFormat := range id.SupportedObjectFormats() { + t.Run(objectFormat.String(), func(t *testing.T) { + t.Parallel() + + hashSize := objectFormat.Size() + + base := bytes.Repeat([]byte{0xab}, hashSize) + data := append([]byte{0x73}, base...) + + header, err := packfile.ParseEntryHeader(data, hashSize) + if err != nil { + t.Fatalf("ParseEntryHeader: %v", err) + } + + if header.Type != packfile.EntryTypeRefDelta { + t.Fatalf("ParseEntryHeader type = %d, want %d", header.Type, packfile.EntryTypeRefDelta) + } + + if header.Size != 3 { + t.Fatalf("ParseEntryHeader size = %d, want 3", header.Size) + } + + if header.HeaderLen != 1+hashSize { + t.Fatalf("ParseEntryHeader header len = %d, want %d", header.HeaderLen, 1+hashSize) + } + + if !bytes.Equal(header.RefBase[:hashSize], base) { + t.Fatalf("ParseEntryHeader ref base mismatch") + } + }) + } +} + +func TestParseEntryHeaderOfsDelta(t *testing.T) { + t.Parallel() + + header, err := packfile.ParseEntryHeader([]byte{0x65, 0x80, 0x00}, 20) + if err != nil { + t.Fatalf("ParseEntryHeader: %v", err) + } + + if header.Type != packfile.EntryTypeOfsDelta { + t.Fatalf("ParseEntryHeader type = %d, want %d", header.Type, packfile.EntryTypeOfsDelta) + } + + if header.OfsDistance != 128 { + t.Fatalf("ParseEntryHeader distance = %d, want 128", header.OfsDistance) + } + + if header.HeaderLen != 3 { + t.Fatalf("ParseEntryHeader header len = %d, want 3", header.HeaderLen) + } +} + +func TestParseEntryHeaderMalformed(t *testing.T) { + t.Parallel() + + cases := []struct { + name string + data []byte + }{ + {name: "empty", data: []byte{}}, + {name: "truncated type/size", data: []byte{0x95}}, + { + name: "size overflow", + data: append([]byte{0x9f}, bytes.Repeat([]byte{0xff}, 9)...), + }, + { + name: "overlong type/size", + data: append([]byte{0x95}, append(bytes.Repeat([]byte{0x80}, 9), 0x00)...), + }, + {name: "truncated ref-delta base", data: []byte{0x73, 0xab, 0xab}}, + {name: "truncated ofs distance", data: []byte{0x65, 0x80}}, + {name: "type invalid", data: []byte{0x05}}, + {name: "type future", data: []byte{0x55}}, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + + _, err := packfile.ParseEntryHeader(tc.data, 20) + if !errors.Is(err, packfile.ErrMalformedEntryHeader) { + t.Fatalf("ParseEntryHeader error = %v, want ErrMalformedEntryHeader", err) + } + }) + } +} + +func TestParseEntryHeaderBadHashSize(t *testing.T) { + t.Parallel() + + for _, hashSize := range []int{-1, 0, id.MaxObjectIDSize + 1} { + func() { + defer func() { + if recover() == nil { + t.Fatalf("ParseEntryHeader hash size %d: expected panic", hashSize) + } + }() + + _, _ = packfile.ParseEntryHeader([]byte{0x95, 0x05}, hashSize) + }() + } +} + +func TestTypeSizeRoundTrip(t *testing.T) { + t.Parallel() + + baseTypes := []packfile.EntryType{ + packfile.EntryTypeCommit, + packfile.EntryTypeTree, + packfile.EntryTypeBlob, + packfile.EntryTypeTag, + } + + sizes := []uint64{ + 0, 1, 15, 16, 127, 128, 1 << 20, 1 << 57, math.MaxUint64, + } + + for _, entryType := range baseTypes { + for _, size := range sizes { + data := packfile.AppendTypeSize(nil, entryType, size) + if len(data) > packfile.MaxTypeSizeLen { + t.Fatalf("AppendTypeSize(%d, %d) length = %d, want <= %d", + entryType, size, len(data), packfile.MaxTypeSizeLen) + } + + header, err := packfile.ParseEntryHeader(data, 20) + if err != nil { + t.Fatalf("ParseEntryHeader: %v", err) + } + + if header.Type != entryType { + t.Fatalf("round trip type = %d, want %d", header.Type, entryType) + } + + if header.Size != size { + t.Fatalf("round trip size = %d, want %d", header.Size, size) + } + + if header.HeaderLen != len(data) { + t.Fatalf("round trip header len = %d, want %d", header.HeaderLen, len(data)) + } + } + } +} + +func TestRefDeltaHeaderRoundTrip(t *testing.T) { + t.Parallel() + + for _, objectFormat := range id.SupportedObjectFormats() { + t.Run(objectFormat.String(), func(t *testing.T) { + t.Parallel() + + hashSize := objectFormat.Size() + + base := bytes.Repeat([]byte{0xcd}, hashSize) + data := packfile.AppendTypeSize(nil, packfile.EntryTypeRefDelta, 42) + data = append(data, base...) + + header, err := packfile.ParseEntryHeader(data, hashSize) + if err != nil { + t.Fatalf("ParseEntryHeader: %v", err) + } + + if !bytes.Equal(header.RefBase[:hashSize], base) { + t.Fatalf("round trip ref base mismatch") + } + + if header.HeaderLen != len(data) { + t.Fatalf("round trip header len = %d, want %d", header.HeaderLen, len(data)) + } + }) + } +} + +func TestOfsDeltaHeaderRoundTrip(t *testing.T) { + t.Parallel() + + data := packfile.AppendTypeSize(nil, packfile.EntryTypeOfsDelta, 7) + data = packfile.AppendOfsDeltaDistance(data, 123456) + + header, err := packfile.ParseEntryHeader(data, 20) + if err != nil { + t.Fatalf("ParseEntryHeader: %v", err) + } + + if header.OfsDistance != 123456 { + t.Fatalf("round trip distance = %d, want 123456", header.OfsDistance) + } + + if header.HeaderLen != len(data) { + t.Fatalf("round trip header len = %d, want %d", header.HeaderLen, len(data)) + } +} + +func TestMaxEntryHeaderLenBounds(t *testing.T) { + t.Parallel() + + for _, objectFormat := range id.SupportedObjectFormats() { + hashSize := objectFormat.Size() + + maxLen := packfile.MaxEntryHeaderLen(hashSize) + if maxLen < packfile.MaxTypeSizeLen+hashSize { + t.Fatalf("MaxEntryHeaderLen(%d) = %d cannot fit a ref-delta header", hashSize, maxLen) + } + + if maxLen < packfile.MaxTypeSizeLen+packfile.MaxOfsDeltaDistanceLen { + t.Fatalf("MaxEntryHeaderLen(%d) = %d cannot fit an ofs-delta header", hashSize, maxLen) + } + } +} diff --git a/internal/format/packfile/entry_type.go b/internal/format/packfile/entry_type.go new file mode 100644 index 00000000..b4baceb4 --- /dev/null +++ b/internal/format/packfile/entry_type.go @@ -0,0 +1,86 @@ +package packfile + +import ( + "errors" + + "lindenii.org/go/furgit/object/typ" +) + +var ( + // ErrInternalEntryType reports that + // a supplied packfile [EntryType] + // cannot be converted into an ordinary [typ.Type] + // since it is a packfile implementation detail. + ErrInternalEntryType = errors.New("internal/format/packfile: packfile-internal entry type cannot be converted to an object type") + + // ErrUnrepresentableObjectType reports that + // a supplied ordinary [typ.Type] + // is not currently representable + // as a packfile [EntryType]. + ErrUnrepresentableObjectType = errors.New("internal/format/packfile: object type not representable in packfiles") +) + +// EntryType represents the type of an entry in a git packfile. +type EntryType uint8 + +const ( + EntryTypeInvalid EntryType = 0 + EntryTypeCommit EntryType = 1 + EntryTypeTree EntryType = 2 + EntryTypeBlob EntryType = 3 + EntryTypeTag EntryType = 4 + EntryTypeFuture EntryType = 5 + EntryTypeOfsDelta EntryType = 6 + EntryTypeRefDelta EntryType = 7 +) + +// EntryTypeFromObjectType converts an ordinary [typ.Type] into a packfile [EntryType]. +func EntryTypeFromObjectType(ty typ.Type) (EntryType, error) { + switch ty { + case typ.Commit: + return EntryTypeCommit, nil + case typ.Tree: + return EntryTypeTree, nil + case typ.Blob: + return EntryTypeBlob, nil + case typ.Tag: + return EntryTypeTag, nil + case typ.Unknown: + } + + return EntryTypeInvalid, ErrUnrepresentableObjectType +} + +// ObjectType converts a packfile [EntryType] into an ordinary [typ.Type]. +func (entryType EntryType) ObjectType() (typ.Type, error) { + switch entryType { + case EntryTypeCommit: + return typ.Commit, nil + case EntryTypeTree: + return typ.Tree, nil + case EntryTypeBlob: + return typ.Blob, nil + case EntryTypeTag: + return typ.Tag, nil + case EntryTypeInvalid, EntryTypeFuture, EntryTypeOfsDelta, EntryTypeRefDelta: + } + + return typ.Unknown, ErrInternalEntryType +} + +// IsBase reports whether the entry type +// is a base (non-delta) object type. +func (entryType EntryType) IsBase() bool { + switch entryType { + case EntryTypeCommit, EntryTypeTree, EntryTypeBlob, EntryTypeTag: + return true + case EntryTypeInvalid, EntryTypeFuture, EntryTypeOfsDelta, EntryTypeRefDelta: + } + + return false +} + +// IsDelta reports whether the entry type is a delta type. +func (entryType EntryType) IsDelta() bool { + return entryType == EntryTypeOfsDelta || entryType == EntryTypeRefDelta +} diff --git a/internal/format/packfile/header.go b/internal/format/packfile/header.go new file mode 100644 index 00000000..64c7d140 --- /dev/null +++ b/internal/format/packfile/header.go @@ -0,0 +1,63 @@ +package packfile + +import ( + "encoding/binary" + "errors" + "fmt" +) + +const ( + // signature is the 4-byte "PACK" magic at the start of pack files. + signature = 0x5041434b + + // version is the only pack version this package reads or writes. + version = 2 +) + +// HeaderLen is the size of a pack header: +// signature, version, and object count. +const HeaderLen = 12 + +// ErrMalformedHeader reports that +// a pack header is truncated, +// or has a bad signature or unsupported version. +var ErrMalformedHeader = errors.New("internal/format/packfile: malformed pack header") + +// Header is one parsed pack header. +type Header struct { + // ObjectCount is the number of object entries + // declared to follow the header. + ObjectCount uint32 +} + +// ParseHeader parses and validates the pack header +// at the start of data. +func ParseHeader(data []byte) (Header, error) { + var zero Header + + if len(data) < HeaderLen { + return zero, fmt.Errorf("%w: truncated", ErrMalformedHeader) + } + + if binary.BigEndian.Uint32(data[0:4]) != signature { + return zero, fmt.Errorf("%w: bad signature", ErrMalformedHeader) + } + + parsedVersion := binary.BigEndian.Uint32(data[4:8]) + if parsedVersion != version { + return zero, fmt.Errorf("%w: unsupported version %d", ErrMalformedHeader, parsedVersion) + } + + return Header{ + ObjectCount: binary.BigEndian.Uint32(data[8:12]), + }, nil +} + +// AppendHeader appends an encoded pack header for objectCount to dst. +func AppendHeader(dst []byte, objectCount uint32) []byte { + dst = binary.BigEndian.AppendUint32(dst, signature) + dst = binary.BigEndian.AppendUint32(dst, version) + dst = binary.BigEndian.AppendUint32(dst, objectCount) + + return dst +} diff --git a/internal/format/packfile/ofs.go b/internal/format/packfile/ofs.go new file mode 100644 index 00000000..5fb8ecdd --- /dev/null +++ b/internal/format/packfile/ofs.go @@ -0,0 +1,68 @@ +package packfile + +import ( + "errors" + "fmt" + "math" +) + +// ErrMalformedOfsDeltaDistance reports that +// an ofs-delta backward distance encoding +// is truncated, overlong, or overflows uint64. +var ErrMalformedOfsDeltaDistance = errors.New("internal/format/packfile: malformed ofs-delta distance") + +// MaxOfsDeltaDistanceLen is the maximum encoded length +// of an ofs-delta backward distance. +// Every uint64 distance is encodable within this bound, +// and [ParseOfsDeltaDistance] rejects longer encodings. +const MaxOfsDeltaDistanceLen = 10 + +// ParseOfsDeltaDistance parses an ofs-delta backward distance. +func ParseOfsDeltaDistance(buf []byte) (dist uint64, consumed int, err error) { + if len(buf) == 0 { + return 0, 0, fmt.Errorf("%w: truncated", ErrMalformedOfsDeltaDistance) + } + + b := buf[0] + dist = uint64(b & 0x7f) + + consumed = 1 + for b&0x80 != 0 { + if consumed >= MaxOfsDeltaDistanceLen { + return 0, 0, fmt.Errorf("%w: overlong encoding", ErrMalformedOfsDeltaDistance) + } + + if consumed >= len(buf) { + return 0, 0, fmt.Errorf("%w: truncated", ErrMalformedOfsDeltaDistance) + } + + if dist >= math.MaxUint64>>7 { + return 0, 0, fmt.Errorf("%w: overflows uint64", ErrMalformedOfsDeltaDistance) + } + + b = buf[consumed] + consumed++ + dist = ((dist + 1) << 7) | uint64(b&0x7f) + } + + return dist, consumed, nil +} + +// AppendOfsDeltaDistance appends the encoding of +// an ofs-delta backward distance to dst. +func AppendOfsDeltaDistance(dst []byte, dist uint64) []byte { + var buf [MaxOfsDeltaDistanceLen]byte + + pos := len(buf) - 1 + buf[pos] = byte(dist & 0x7f) + + dist >>= 7 + for dist != 0 { + dist-- + pos-- + buf[pos] = 0x80 | byte(dist&0x7f) + dist >>= 7 + } + + return append(dst, buf[pos:]...) +} diff --git a/internal/format/packfile/ofs_test.go b/internal/format/packfile/ofs_test.go new file mode 100644 index 00000000..736f1ac9 --- /dev/null +++ b/internal/format/packfile/ofs_test.go @@ -0,0 +1,100 @@ +package packfile_test + +import ( + "bytes" + "errors" + "math" + "testing" + + "lindenii.org/go/furgit/internal/format/packfile" +) + +func TestParseOfsDeltaDistance(t *testing.T) { + t.Parallel() + + cases := []struct { + name string + data []byte + dist uint64 + consumed int + }{ + {name: "zero", data: []byte{0x00}, dist: 0, consumed: 1}, + {name: "one byte max", data: []byte{0x7f}, dist: 127, consumed: 1}, + {name: "two byte min", data: []byte{0x80, 0x00}, dist: 128, consumed: 2}, + {name: "two byte max", data: []byte{0xff, 0x7f}, dist: 16511, consumed: 2}, + {name: "trailing bytes ignored", data: []byte{0x7f, 0xde, 0xad}, dist: 127, consumed: 1}, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + + dist, consumed, err := packfile.ParseOfsDeltaDistance(tc.data) + if err != nil { + t.Fatalf("ParseOfsDeltaDistance: %v", err) + } + + if dist != tc.dist { + t.Fatalf("ParseOfsDeltaDistance = %d, want %d", dist, tc.dist) + } + + if consumed != tc.consumed { + t.Fatalf("ParseOfsDeltaDistance consumed = %d, want %d", consumed, tc.consumed) + } + }) + } +} + +func TestParseOfsDeltaDistanceMalformed(t *testing.T) { + t.Parallel() + + cases := []struct { + name string + data []byte + }{ + {name: "empty", data: []byte{}}, + {name: "truncated", data: []byte{0x80}}, + {name: "overflow", data: bytes.Repeat([]byte{0xff}, 10)}, + {name: "overlong", data: append(bytes.Repeat([]byte{0x80}, 10), 0x00)}, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + + _, _, err := packfile.ParseOfsDeltaDistance(tc.data) + if !errors.Is(err, packfile.ErrMalformedOfsDeltaDistance) { + t.Fatalf("ParseOfsDeltaDistance error = %v, want ErrMalformedOfsDeltaDistance", err) + } + }) + } +} + +func TestOfsDeltaDistanceRoundTrip(t *testing.T) { + t.Parallel() + + distances := []uint64{ + 0, 1, 127, 128, 16511, 16512, 1 << 31, 1 << 57, math.MaxUint64, + } + + for _, dist := range distances { + data := packfile.AppendOfsDeltaDistance(nil, dist) + if len(data) > packfile.MaxOfsDeltaDistanceLen { + t.Fatalf("AppendOfsDeltaDistance(%d) length = %d, want <= %d", + dist, len(data), packfile.MaxOfsDeltaDistanceLen) + } + + got, consumed, err := packfile.ParseOfsDeltaDistance(data) + if err != nil { + t.Fatalf("ParseOfsDeltaDistance: %v", err) + } + + if got != dist { + t.Fatalf("round trip = %d, want %d", got, dist) + } + + if consumed != len(data) { + t.Fatalf("round trip consumed = %d, want %d", consumed, len(data)) + } + } +} diff --git a/internal/format/packidx/bloom/bloom.go b/internal/format/packidx/bloom/bloom.go new file mode 100644 index 00000000..8b608221 --- /dev/null +++ b/internal/format/packidx/bloom/bloom.go @@ -0,0 +1,180 @@ +package bloom + +import ( + "bytes" + "encoding/binary" + "errors" + "fmt" + "math/bits" + + "lindenii.org/go/furgit/object/id" +) + +// ErrMalformedBloomFilter reports that +// a Bloom filter is truncated, +// has a bad signature, version, or hash function, +// or has inconsistent parameters. +var ErrMalformedBloomFilter = errors.New("internal/format/packidx/bloom: malformed bloom filter") + +const ( + signature = 0x4944424c // "IDBL" + version = 1 + + // HeaderLen is the fixed header length in octets, + // i.e., the signature, version, hash function identifier, + // B, K, and the trailing zero padding. + HeaderLen = 64 + + // BucketLen is the length of one bucket in octets, + // chosen to match the most common cache-line size. + BucketLen = 64 + + // wordBits is the bit width of one bucket word. + wordBits = 64 + + // fieldBits is the width of one in-bucket position field. + fieldBits = 9 +) + +// checkParams validates the filter parameters +// against one object hash size, +// returning log2(bucketCount) on success. +func checkParams(bucketCount uint32, k uint16, hashSize int) (uint, error) { + switch { + case bucketCount == 0 || bucketCount&(bucketCount-1) != 0: + return 0, errors.New("bucket count not a nonzero power of two") //nolint:err113 + case k == 0: + return 0, errors.New("zero probe count") //nolint:err113 + } + + log2B := uint(bits.TrailingZeros32(bucketCount)) + if log2B+fieldBits*uint(k) > uint(hashSize)*8 { + return 0, errors.New("parameters exceed hash length") //nolint:err113 + } + + return log2B, nil +} + +// hashFunctionID returns the on-disk hash function identifier +// for one object format. +func hashFunctionID(objectFormat id.ObjectFormat) (uint32, error) { + switch objectFormat { + case id.ObjectFormatSHA1: + return 1, nil + case id.ObjectFormatSHA256: + return 2, nil + case id.ObjectFormatUnknown: + } + + return 0, id.ErrInvalidObjectFormat +} + +// Bloom is a parsed blocked Bloom filter view over borrowed bytes. +// +// Labels: Deps-Borrowed, Life-Parent, MT-Safe. +type Bloom struct { + // data is the entire filter payload. + data []byte + + // buckets is the bucket region, between the header and the trailer. + buckets []byte + + // objectFormat is the filter's object format. + objectFormat id.ObjectFormat + + // log2B is the base-2 logarithm of the bucket count, + // i.e. the number of leading object ID bits that select a bucket. + log2B uint + + // k is the number of bits set and tested per object ID. + k int +} + +// Parse parses one Bloom filter from data. +// +// Labels: Deps-Borrowed, Life-Parent. +func Parse(data []byte, objectFormat id.ObjectFormat) (Bloom, error) { + var zero Bloom + + wantHashID, err := hashFunctionID(objectFormat) + if err != nil { + return zero, err + } + + hashSize := objectFormat.Size() + + if len(data) < HeaderLen { + return zero, fmt.Errorf("%w: truncated", ErrMalformedBloomFilter) + } + + if binary.BigEndian.Uint32(data) != signature { + return zero, fmt.Errorf("%w: bad signature", ErrMalformedBloomFilter) + } + + if binary.BigEndian.Uint32(data[4:]) != version { + return zero, fmt.Errorf("%w: unsupported version", ErrMalformedBloomFilter) + } + + if binary.BigEndian.Uint32(data[8:]) != wantHashID { + return zero, fmt.Errorf("%w: hash function mismatch", ErrMalformedBloomFilter) + } + + bucketCount := binary.BigEndian.Uint32(data[12:]) + k := binary.BigEndian.Uint16(data[16:]) + + for _, octet := range data[18:HeaderLen] { + if octet != 0 { + return zero, fmt.Errorf("%w: nonzero padding", ErrMalformedBloomFilter) + } + } + + log2B, err := checkParams(bucketCount, k, hashSize) + if err != nil { + return zero, fmt.Errorf("%w: %w", ErrMalformedBloomFilter, err) + } + + want := uint64(HeaderLen) + uint64(BucketLen)*uint64(bucketCount) + 2*uint64(hashSize) //#nosec G115 + if uint64(len(data)) != want { + return zero, fmt.Errorf("%w: file size disagrees with bucket count", ErrMalformedBloomFilter) + } + + return Bloom{ + data: data, + buckets: data[HeaderLen : len(data)-2*hashSize], + objectFormat: objectFormat, + log2B: log2B, + k: int(k), + }, nil +} + +// PackHash returns the pack hash recorded in the filter trailer. +// +// Labels: Life-Parent, Mut-No. +func (f *Bloom) PackHash() []byte { + hashSize := f.objectFormat.Size() + end := len(f.data) - hashSize + + return f.data[end-hashSize : end] +} + +// Verify recomputes the filter's trailing checksum and reports any mismatch. +// +// Verify reads the whole filter, +// so callers should treat it as a deliberate integrity check +// rather than part of the open path. +func (f *Bloom) Verify() error { + hashImpl, err := f.objectFormat.New() + if err != nil { + return fmt.Errorf("internal/format/packidx/bloom: %w", err) + } + + checksumOff := len(f.data) - f.objectFormat.Size() + + _, _ = hashImpl.Write(f.data[:checksumOff]) + + if !bytes.Equal(hashImpl.Sum(nil), f.data[checksumOff:]) { + return fmt.Errorf("%w: checksum mismatch", ErrMalformedBloomFilter) + } + + return nil +} diff --git a/internal/format/packidx/bloom/bloom_test.go b/internal/format/packidx/bloom/bloom_test.go new file mode 100644 index 00000000..bcfb4419 --- /dev/null +++ b/internal/format/packidx/bloom/bloom_test.go @@ -0,0 +1,159 @@ +package bloom_test + +import ( + "encoding/binary" + "errors" + "testing" + + "lindenii.org/go/furgit/internal/format/packidx/bloom" + "lindenii.org/go/furgit/object/id" +) + +func validFilter(t *testing.T, format id.ObjectFormat) []byte { + // TODO: maybe testgit should have something like this? + t.Helper() + + builder, err := bloom.NewBuilder(format, 4, 2, make([]byte, format.Size())) + if err != nil { + t.Fatal(err) + } + + return builder.Bytes() +} + +func otherFormat(t *testing.T, format id.ObjectFormat) id.ObjectFormat { + t.Helper() + + for _, candidate := range id.SupportedObjectFormats() { + if candidate != format { + return candidate + } + } + + t.Skip("only one supported object format") + + return id.ObjectFormatUnknown +} + +func TestParseValid(t *testing.T) { + t.Parallel() + + for _, format := range id.SupportedObjectFormats() { + t.Run(format.String(), func(t *testing.T) { + t.Parallel() + + _, err := bloom.Parse(validFilter(t, format), format) + if err != nil { + t.Fatalf("Parse rejected a valid filter: %v", err) + } + }) + } +} + +func TestParseMalformed(t *testing.T) { + t.Parallel() + + cases := []struct { + name string + mangle func(data []byte) []byte + }{ + {"truncated", func(data []byte) []byte { return data[:bloom.HeaderLen-1] }}, + {"bad signature", func(data []byte) []byte { + data[0] ^= 0xff + + return data + }}, + {"bad version", func(data []byte) []byte { + binary.BigEndian.PutUint32(data[4:], 99) + + return data + }}, + {"non power of two", func(data []byte) []byte { + binary.BigEndian.PutUint32(data[12:], 3) + + return data + }}, + {"zero probe count", func(data []byte) []byte { + binary.BigEndian.PutUint16(data[16:], 0) + + return data + }}, + {"parameters exceed hash", func(data []byte) []byte { + binary.BigEndian.PutUint32(data[12:], 1<<31) + binary.BigEndian.PutUint16(data[16:], 30) + + return data + }}, + {"nonzero padding", func(data []byte) []byte { + data[20] = 1 + + return data + }}, + {"size disagrees", func(data []byte) []byte { return data[:len(data)-1] }}, + } + + for _, format := range id.SupportedObjectFormats() { + t.Run(format.String(), func(t *testing.T) { + t.Parallel() + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + + data := tc.mangle(append([]byte(nil), validFilter(t, format)...)) + + _, err := bloom.Parse(data, format) + if !errors.Is(err, bloom.ErrMalformedBloomFilter) { + t.Fatalf("Parse error = %v, want ErrMalformedBloomFilter", err) + } + }) + } + }) + } +} + +// TestVerifyDetectsCorruption checks that Verify accepts a sound filter +// and rejects one whose bucket bytes have been altered. +func TestVerifyDetectsCorruption(t *testing.T) { + t.Parallel() + + for _, format := range id.SupportedObjectFormats() { + t.Run(format.String(), func(t *testing.T) { + t.Parallel() + + data := validFilter(t, format) + + filter, err := bloom.Parse(data, format) + if err != nil { + t.Fatal(err) + } + + err = filter.Verify() + if err != nil { + t.Fatalf("Verify on a sound filter: %v", err) + } + + data[bloom.HeaderLen] ^= 0xff + + err = filter.Verify() + if !errors.Is(err, bloom.ErrMalformedBloomFilter) { + t.Fatalf("Verify error = %v, want ErrMalformedBloomFilter", err) + } + }) + } +} + +func TestParseHashMismatch(t *testing.T) { + t.Parallel() + + for _, format := range id.SupportedObjectFormats() { + t.Run(format.String(), func(t *testing.T) { + t.Parallel() + + _, err := bloom.Parse(validFilter(t, format), otherFormat(t, format)) + if !errors.Is(err, bloom.ErrMalformedBloomFilter) { + t.Fatalf("Parse error = %v, want ErrMalformedBloomFilter", err) + } + }) + } +} diff --git a/internal/format/packidx/bloom/doc.go b/internal/format/packidx/bloom/doc.go new file mode 100644 index 00000000..06ca57cd --- /dev/null +++ b/internal/format/packidx/bloom/doc.go @@ -0,0 +1,138 @@ +// Package bloom provides a blocked Bloom filter +// for pack indexes. +// +// A filter answers, from a single cache-line-sized read, +// whether an object ID is definitely absent from the index it covers. +// A lookup that must consult many packs +// can then skip the full binary search +// in every pack whose filter rejects the object, +// decreasing the cost of misses. +// +// # Rationale +// +// Especially for server-side usage, +// repacking is expensive, +// and creating multi-pack-indexes is still rather expensive. +// Incremental multi-pack-indexes partially solve this, +// but having too many of them defeats the purpose, +// since the indexes must still be walked in order +// while performing expensive lookups. +// +// Instead, each multi-pack-index layer, +// and each ordinary pack index, +// may carry its own filter. +// The indexes are still traversed in their usual order, +// but the first step when traversing one +// is to check whether it could possibly hold the wanted object. +// +// The filter is split into 64-octet buckets, +// matching the most common cache-line size. +// Some bits of the object ID choose the bucket, +// and the rest choose several bit positions inside it, +// so a lookup reads one 64-octet bucket +// and checks whether all required bits are set. +// +// # Parameters +// +// A filter is parameterized by +// the number of buckets B +// and the number of bits set and tested per object ID, K. +// All integers in the format are big endian. +// The object ID is interpreted as a big-endian bitstring, +// where bit offset 0 is the most significant bit of octet 0. +// B must be a nonzero power of two, +// K must be nonzero, +// and log2(B) + 9*K must not exceed the hash length in bits. +// +// # File format +// +// A filter file is a 64-octet header, +// then B buckets of 64 octets each, +// then a two-hash trailer: +// +// - 4-octet signature: {'I', 'D', 'B', 'L'}. +// - 4-octet version identifier (= 1). +// - 4-octet object hash algorithm identifier +// (= 1 for SHA-1, 2 for SHA-256). +// - 4-octet B, the number of buckets. +// - 2-octet K, the number of bits set and tested per object ID. +// - 46-octet padding, which must be all zero. +// - B buckets of 64 octets each. +// - the pack trailer hash, which binds the filter to its pack. +// - the checksum of everything before it, over the filter's hash function. +// +// The hash length is that of the object format, +// so the trailer is 2 hashes wide +// and the file size is exactly 64 + 64*B + 2*hashlen octets. +// +// A reader must validate that +// the signature matches, +// the version is supported, +// the hash function identifier is recognized, +// B is nonzero and a power of two, +// K is nonzero, +// log2(B) + 9*K does not exceed the hash length in bits, +// the padding is all zero, +// and the file size is exactly 64 + 64*B + 2*hashlen octets. +// +// # Binding and integrity +// +// The pack hash binds a filter to one pack; +// a reader trusts a filter only when the recorded pack hash +// matches the pack it accompanies. +// +// The checksum guards against corruption of the filter itself. +// Recomputing it reads the whole file and rehashes it as fsck. +// +// # Lookup +// +// A lookup against one filter proceeds as follows: +// +// 1. Let b be the unsigned integer encoded +// by the most significant log2(B) bits of the object ID. +// B is a power of two, so 0 <= b < B. +// 2. Select and read bucket b. +// 3. For each 0 <= i < K, +// take the i-th 9-bit field +// from the 9*K bits that follow the bucket-selecting bits, +// and let pi be the unsigned integer it encodes, +// so 0 <= pi < 512. +// Compute wi = pi >> 6 and bi = pi & 63, +// so wi identifies one of the eight 64-bit words in bucket b +// and bi identifies one bit within that word. +// Within each 64-bit word, +// bit index 0 is the most significant bit +// and bit index 63 is the least significant bit. +// Test whether bit bi is set in word wi of bucket b. +// +// If any test fails, +// the object ID is definitely not in the covered index. +// If all tests succeed, +// the object ID may be in it. +// Two of the K 9-bit fields can decode to the same pi, +// so an insertion may set fewer than K distinct bits; +// this only raises the false positive rate +// and never causes a false negative. +// +// # Worked example +// +// Let B = 1 << 15 = 32768 and K = 8. +// Then log2(B) = 15, +// so each lookup uses 15 bits to choose the bucket +// and 8*9 = 72 bits to choose the in-bucket positions, +// for a total of 87 bits taken from the object ID. +// A SHA-1 has 160 bits and a SHA-256 has 256 bits, +// so both leave ample headroom. +// +// # Security considerations +// +// Object IDs are public unkeyed hashes, +// so an adversary can mine packs +// whose object IDs share a chosen prefix +// to crowd objects into one bucket +// and fill its bits. +// In the worst case this renders some buckets useless, +// making the filter degrade to "may contain" for those buckets, +// but it never produces a false negative +// and is not a significant denial-of-service vector. +package bloom diff --git a/internal/format/packidx/bloom/lookup.go b/internal/format/packidx/bloom/lookup.go new file mode 100644 index 00000000..4a8d7bda --- /dev/null +++ b/internal/format/packidx/bloom/lookup.go @@ -0,0 +1,42 @@ +package bloom + +import ( + "encoding/binary" +) + +// MayContain reports whether oid may be present +// in the index covered by the filter. +// +// oid must be exactly the filter's hash size; +// MayContain panics otherwise. +// +// Labels: Mut-No. +func (f *Bloom) MayContain(oid []byte) bool { + if len(oid) != f.objectFormat.Size() { + panic("internal/format/packidx/bloom: invalid object ID length") + } + + base := int(binary.BigEndian.Uint32(oid[:4])>>(32-f.log2B)) * BucketLen + + for i := range f.k { + word, mask := probe(oid, f.log2B, i) + if binary.BigEndian.Uint64(f.buckets[base+word*8:])&mask == 0 { + return false + } + } + + return true +} + +// probe returns the bucket word index and single-bit mask +// addressed by the i-th probe of oid. +func probe(oid []byte, log2B uint, i int) (word int, mask uint64) { + bitOff := log2B + fieldBits*uint(i) + byteOff := bitOff >> 3 + bitInByte := bitOff & 7 + + window := uint32(oid[byteOff])<<8 | uint32(oid[byteOff+1]) + pi := (window >> (16 - bitInByte - fieldBits)) & 0x1ff + + return int(pi >> 6), 1 << (wordBits - 1 - (pi & 63)) +} diff --git a/internal/format/packidx/bloom/lookup_test.go b/internal/format/packidx/bloom/lookup_test.go new file mode 100644 index 00000000..e6264f9a --- /dev/null +++ b/internal/format/packidx/bloom/lookup_test.go @@ -0,0 +1,32 @@ +package bloom_test + +import ( + "testing" + + "lindenii.org/go/furgit/internal/format/packidx/bloom" + "lindenii.org/go/furgit/object/id" +) + +func TestMayContainBadLength(t *testing.T) { + t.Parallel() + + format := id.ObjectFormatSHA256 + + builder, err := bloom.NewBuilder(format, 4, 2, make([]byte, format.Size())) + if err != nil { + t.Fatal(err) + } + + filter, err := bloom.Parse(builder.Bytes(), format) + if err != nil { + t.Fatal(err) + } + + defer func() { + if recover() == nil { + t.Fatal("MayContain did not panic on a short object ID") + } + }() + + filter.MayContain(make([]byte, format.Size()-1)) +} diff --git a/internal/format/packidx/bloom/roundtrip_test.go b/internal/format/packidx/bloom/roundtrip_test.go new file mode 100644 index 00000000..28db17a5 --- /dev/null +++ b/internal/format/packidx/bloom/roundtrip_test.go @@ -0,0 +1,92 @@ +package bloom_test + +import ( + "bytes" + "encoding/binary" + "testing" + + "lindenii.org/go/furgit/internal/format/packidx/bloom" + "lindenii.org/go/furgit/object/id" +) + +func makeOID(size int, seed uint64) []byte { + out := make([]byte, size) + state := seed + + for i := 0; i < size; i += 8 { + state = state*6364136223846793005 + 1442695040888963407 + + var word [8]byte + + binary.BigEndian.PutUint64(word[:], state) + copy(out[i:], word[:]) + } + + return out +} + +func TestRoundTrip(t *testing.T) { + t.Parallel() + + for _, format := range id.SupportedObjectFormats() { + t.Run(format.String(), func(t *testing.T) { + t.Parallel() + + const objects = 10000 + + bucketCount, k, err := bloom.RecommendParams(format, objects) + if err != nil { + t.Fatal(err) + } + + size := format.Size() + packHash := makeOID(size, 0xC0FFEE) + + builder, err := bloom.NewBuilder(format, bucketCount, k, packHash) + if err != nil { + t.Fatal(err) + } + + for i := range objects { + builder.Add(makeOID(size, uint64(i))) + } + + filter, err := bloom.Parse(builder.Bytes(), format) + if err != nil { + t.Fatal(err) + } + + if !bytes.Equal(filter.PackHash(), packHash) { + t.Fatalf("PackHash = %x, want %x", filter.PackHash(), packHash) + } + + err = filter.Verify() + if err != nil { + t.Fatalf("Verify on a freshly built filter: %v", err) + } + + for i := range objects { + if !filter.MayContain(makeOID(size, uint64(i))) { + t.Fatalf("false negative for added object %d", i) + } + } + + const probes = 10000 + + falsePositives := 0 + + for i := range probes { + if filter.MayContain(makeOID(size, uint64(1)<<40+uint64(i))) { + falsePositives++ + } + } + + rate := float64(falsePositives) / float64(probes) + if rate > 0.05 { + t.Errorf("false positive rate %.4f exceeds 0.05", rate) + } + + t.Logf("B=%d K=%d false positive rate %.4f", bucketCount, k, rate) + }) + } +} diff --git a/internal/format/packidx/bloom/write.go b/internal/format/packidx/bloom/write.go new file mode 100644 index 00000000..e6213a2c --- /dev/null +++ b/internal/format/packidx/bloom/write.go @@ -0,0 +1,164 @@ +package bloom + +import ( + "encoding/binary" + "errors" + "fmt" + "hash" + "math/bits" + + "lindenii.org/go/furgit/object/id" + "lindenii.org/go/lgo/intconv" +) + +// ErrInvalidParameters reports that +// the parameters supplied for a filter build +// are not representable in the format. +var ErrInvalidParameters = errors.New("internal/format/packidx/bloom: invalid parameters") + +// defaultK is the probe count used by [RecommendParams]. +// +// With 512-bit buckets it keeps the false positive rate near one percent +// at the target bucket load. +const defaultK = 8 + +// targetLoad is the object count per bucket that [RecommendParams] aims for. +const targetLoad = 48 + +// Builder accumulates object IDs into an in-memory Bloom filter +// and serializes it. +// +// Labels: MT-Unsafe. +type Builder struct { + // data is the full filter file, header and trailer included. + data []byte + + // buckets aliases the bucket region of data, between header and trailer. + buckets []byte + + // hashImpl computes the trailing checksum and gives the hash size. + hashImpl hash.Hash + + log2B uint + k int +} + +// NewBuilder creates a filter builder +// for bucketCount buckets and k probes per object ID, +// binding the filter to packHash. +// +// bucketCount must be a nonzero power of two, +// k must be nonzero, +// and log2(bucketCount) + 9*k must not exceed the hash length in bits. +// packHash must be the pack's trailer hash; +// NewBuilder panics when its length does not match the object format. +func NewBuilder(objectFormat id.ObjectFormat, bucketCount uint32, k uint16, packHash []byte) (*Builder, error) { + hashID, err := hashFunctionID(objectFormat) + if err != nil { + return nil, err + } + + hashImpl, err := objectFormat.New() + if err != nil { + return nil, fmt.Errorf("internal/format/packidx/bloom: %w", err) + } + + hashSize := objectFormat.Size() + + if len(packHash) != hashSize { + panic("internal/format/packidx/bloom: invalid pack hash length") + } + + log2B, err := checkParams(bucketCount, k, hashSize) + if err != nil { + return nil, fmt.Errorf("%w: %w", ErrInvalidParameters, err) + } + + total, err := intconv.Uint64ToInt(uint64(HeaderLen) + uint64(BucketLen)*uint64(bucketCount) + 2*uint64(hashSize)) //#nosec G115 + if err != nil { + return nil, fmt.Errorf("%w: %w", ErrInvalidParameters, err) + } + + data := make([]byte, total) + binary.BigEndian.PutUint32(data[0:], signature) + binary.BigEndian.PutUint32(data[4:], version) + binary.BigEndian.PutUint32(data[8:], hashID) + binary.BigEndian.PutUint32(data[12:], bucketCount) + binary.BigEndian.PutUint16(data[16:], k) + + bucketsEnd := total - 2*hashSize + copy(data[bucketsEnd:], packHash) + + return &Builder{ + data: data, + buckets: data[HeaderLen:bucketsEnd], + hashImpl: hashImpl, + log2B: log2B, + k: int(k), + }, nil +} + +// Add records oid in the filter. +// +// oid must be exactly the filter's hash size; +// Add panics otherwise. +func (b *Builder) Add(oid []byte) { + if len(oid) != b.hashImpl.Size() { + panic("internal/format/packidx/bloom: invalid object ID length") + } + + base := int(binary.BigEndian.Uint32(oid[:4])>>(32-b.log2B)) * BucketLen + + for i := range b.k { + word, mask := probe(oid, b.log2B, i) + + off := base + word*8 + set := binary.BigEndian.Uint64(b.buckets[off:]) | mask + binary.BigEndian.PutUint64(b.buckets[off:], set) + } +} + +// Bytes returns the serialized filter, including its trailing checksum. +// +// Labels: Life-Parent, Mut-No. +func (b *Builder) Bytes() []byte { + checksumOff := len(b.data) - b.hashImpl.Size() + + b.hashImpl.Reset() + _, _ = b.hashImpl.Write(b.data[:checksumOff]) + b.hashImpl.Sum(b.data[checksumOff:checksumOff]) + + return b.data +} + +// RecommendParams returns filter parameters for an index of n objects, +// targeting a false positive rate near one percent. +func RecommendParams(objectFormat id.ObjectFormat, n int) (bucketCount uint32, k uint16, err error) { + hashSize := objectFormat.Size() + if hashSize == 0 { + return 0, 0, id.ErrInvalidObjectFormat + } + + const maxPow2 = uint32(1) << 31 + + wanted := uint64(0) + if n > 0 { + wanted = (uint64(n) + targetLoad - 1) / targetLoad + } + + switch { + case wanted <= 1: + bucketCount = 1 + case wanted > uint64(maxPow2): + bucketCount = maxPow2 + default: + bucketCount = uint32(1) << bits.Len64(wanted-1) + } + + _, err = checkParams(bucketCount, defaultK, hashSize) + if err != nil { + return 0, 0, fmt.Errorf("%w: %w", ErrInvalidParameters, err) + } + + return bucketCount, defaultK, nil +} diff --git a/internal/format/packidx/bloom/write_test.go b/internal/format/packidx/bloom/write_test.go new file mode 100644 index 00000000..74173921 --- /dev/null +++ b/internal/format/packidx/bloom/write_test.go @@ -0,0 +1,95 @@ +package bloom_test + +import ( + "errors" + "testing" + + "lindenii.org/go/furgit/internal/format/packidx/bloom" + "lindenii.org/go/furgit/object/id" +) + +func TestRecommendParams(t *testing.T) { + t.Parallel() + + for _, format := range id.SupportedObjectFormats() { + t.Run(format.String(), func(t *testing.T) { + t.Parallel() + + for _, n := range []int{0, 1, 1000, 10000, 1000000} { + bucketCount, k, err := bloom.RecommendParams(format, n) + if err != nil { + t.Fatalf("n=%d: %v", n, err) + } + + if bucketCount == 0 || bucketCount&(bucketCount-1) != 0 { + t.Errorf("n=%d: bucket count %d not a power of two", n, bucketCount) + } + + _, err = bloom.NewBuilder(format, bucketCount, k, make([]byte, format.Size())) + if err != nil { + t.Errorf("n=%d: recommended parameters rejected: %v", n, err) + } + } + }) + } +} + +func TestNewBuilderRejects(t *testing.T) { + t.Parallel() + + cases := []struct { + name string + bucketCount uint32 + k uint16 + }{ + {"zero buckets", 0, 8}, + {"non power of two", 3, 8}, + {"zero probe count", 4, 0}, + } + + for _, format := range id.SupportedObjectFormats() { + t.Run(format.String(), func(t *testing.T) { + t.Parallel() + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + + _, err := bloom.NewBuilder(format, tc.bucketCount, tc.k, make([]byte, format.Size())) + if !errors.Is(err, bloom.ErrInvalidParameters) { + t.Fatalf("error = %v, want ErrInvalidParameters", err) + } + }) + } + }) + } +} + +func TestNewBuilderBadPackHash(t *testing.T) { + t.Parallel() + + defer func() { + if recover() == nil { + t.Fatal("NewBuilder did not panic on a short pack hash") + } + }() + + _, _ = bloom.NewBuilder(id.ObjectFormatSHA256, 4, 2, make([]byte, id.ObjectFormatSHA256.Size()-1)) +} + +func TestAddBadLength(t *testing.T) { + t.Parallel() + + builder, err := bloom.NewBuilder(id.ObjectFormatSHA256, 4, 2, make([]byte, id.ObjectFormatSHA256.Size())) + if err != nil { + t.Fatal(err) + } + + defer func() { + if recover() == nil { + t.Fatal("Add did not panic on a short object ID") + } + }() + + builder.Add(make([]byte, id.ObjectFormatSHA256.Size()-1)) +} diff --git a/internal/format/packidx/doc.go b/internal/format/packidx/doc.go new file mode 100644 index 00000000..5c6015b4 --- /dev/null +++ b/internal/format/packidx/doc.go @@ -0,0 +1,3 @@ +// Package packidx provides Git pack index (version 2) format +// parsing and writing primitives. +package packidx diff --git a/internal/format/packidx/helpers_test.go b/internal/format/packidx/helpers_test.go new file mode 100644 index 00000000..be9b47d1 --- /dev/null +++ b/internal/format/packidx/helpers_test.go @@ -0,0 +1,86 @@ +package packidx_test + +import ( + "os" + "strconv" + "strings" + "testing" + + "lindenii.org/go/furgit/internal/format/packidx" + "lindenii.org/go/furgit/internal/testgit" + "lindenii.org/go/furgit/object/id" +) + +// makeGitPack seeds a repository, +// packs the seeded objects with git pack-objects, +// and returns the repository, the artifact path prefix, +// and the packed object IDs. +func makeGitPack(t *testing.T, objectFormat id.ObjectFormat) (*testgit.Repo, string, []id.ObjectID) { + t.Helper() + + repo, err := testgit.NewRepo(t, testgit.RepoOptions{ObjectFormat: objectFormat}) + if err != nil { + t.Fatalf("NewRepo: %v", err) + } + + seeded, err := repo.SeedHistory(t) + if err != nil { + t.Fatalf("SeedHistory: %v", err) + } + + oids := seeded.All() + + prefix, err := repo.PackObjects(t, oids, testgit.PackObjectsOptions{}) + if err != nil { + t.Fatalf("PackObjects: %v", err) + } + + return repo, prefix, oids +} + +// parseGitIdxFile reads and parses one .idx file produced by git. +func parseGitIdxFile(t *testing.T, prefix string, objectFormat id.ObjectFormat) ([]byte, packidx.Packidx) { + t.Helper() + + data, err := os.ReadFile(prefix + ".idx") //nolint:gosec + if err != nil { + t.Fatalf("ReadFile: %v", err) + } + + idx, err := packidx.Parse(data, objectFormat.Size()) + if err != nil { + t.Fatalf("Parse: %v", err) + } + + return data, idx +} + +// gitPackOffsets extracts the object-to-offset mapping +// from git verify-pack -v output. +func gitPackOffsets(t *testing.T, repo *testgit.Repo, idxPath string, objectFormat id.ObjectFormat) map[string]uint64 { + t.Helper() + + out, err := repo.VerifyPack(t, idxPath) + if err != nil { + t.Fatalf("VerifyPack: %v", err) + } + + hexLen := objectFormat.HexLen() + offsets := make(map[string]uint64) + + for line := range strings.Lines(string(out)) { + fields := strings.Fields(line) + if len(fields) < 5 || len(fields[0]) != hexLen { + continue + } + + offset, err := strconv.ParseUint(fields[4], 10, 64) + if err != nil { + continue + } + + offsets[fields[0]] = offset + } + + return offsets +} diff --git a/internal/format/packidx/lookup.go b/internal/format/packidx/lookup.go new file mode 100644 index 00000000..e4b565b6 --- /dev/null +++ b/internal/format/packidx/lookup.go @@ -0,0 +1,121 @@ +package packidx + +import ( + "bytes" + "encoding/binary" + "fmt" + "math/bits" +) + +// Lookup searches the index for one object ID +// and returns its pack file offset. +// +// oid must be exactly the index's hash size; +// Lookup panics otherwise. +func (idx *Packidx) Lookup(oid []byte) (offset uint64, found bool, err error) { + if len(oid) != idx.hashSize { + panic("internal/format/packidx: invalid object ID length") + } + + lo, hi := idx.fanoutRange(oid[0]) + + // Object IDs are uniform for honest inputs, + // interp on next 8 octets converges in O(log log n). + // + // OIDs are public unkeyed hashes, + // so an attacker may sample/filter/mine prefix clusters, + // making interpolation mis-estimate every probe. + // See https://runxiyu.org/comp/ch4ht/ + // for why cryptographic hash algorithms are insufficient. + // + // Cap the interp at bisect's probe count and finish by bisect; + // adversarial at O(log n) plus small interpolation overhead, + // honest exit well under the cap. + target := binary.BigEndian.Uint64(oid[1:9]) + + for budget := bits.Len(uint(hi - lo)); hi-lo > 8 && budget > 0; budget-- { + loKey := binary.BigEndian.Uint64(idx.OIDAt(lo)[1:9]) + hiKey := binary.BigEndian.Uint64(idx.OIDAt(hi - 1)[1:9]) + + var mid int + + switch { + case target <= loKey: + mid = lo + case target >= hiKey: + mid = hi - 1 + default: + hi128, lo128 := bits.Mul64(target-loKey, uint64(hi-lo-1)) //#nosec G115 + q, _ := bits.Div64(hi128, lo128, hiKey-loKey) + mid = lo + int(q) //#nosec G115 + } + + switch cmp := bytes.Compare(oid, idx.OIDAt(mid)); { + case cmp == 0: + offset, err = idx.OffsetAt(mid) + if err != nil { + return 0, false, err + } + + return offset, true, nil + case cmp < 0: + hi = mid + default: + lo = mid + 1 + } + } + + // Interpolation narrowed or capped; bisect to finish. + for lo < hi { + mid := lo + (hi-lo)/2 + + cmp := bytes.Compare(oid, idx.OIDAt(mid)) + + switch { + case cmp == 0: + offset, err = idx.OffsetAt(mid) + if err != nil { + return 0, false, err + } + + return offset, true, nil + case cmp < 0: + hi = mid + default: + lo = mid + 1 + } + } + + return 0, false, nil +} + +// OffsetAt returns the pack file offset at one index position. +// +// OffsetAt panics when pos is out of range. +func (idx *Packidx) OffsetAt(pos int) (uint64, error) { + idx.checkPos(pos) + + raw := binary.BigEndian.Uint32(idx.data[idx.off32Off+4*pos:]) + if raw&largeOffsetFlag == 0 { + return uint64(raw), nil + } + + slot := raw &^ largeOffsetFlag + if uint64(slot) >= idx.off64Count { + return 0, fmt.Errorf("%w: 64-bit offset reference out of range", ErrMalformedPackIndex) + } + + return binary.BigEndian.Uint64(idx.data[idx.off64Off+8*int(slot):]), nil +} + +// fanoutRange returns the index position range [lo, hi) +// of object IDs beginning with first. +func (idx *Packidx) fanoutRange(first byte) (lo, hi int) { + hi = int(binary.BigEndian.Uint32(idx.data[headerLen+4*int(first):])) + + if first > 0 { + lo = int(binary.BigEndian.Uint32(idx.data[headerLen+4*(int(first)-1):])) + } + + return lo, hi +} diff --git a/internal/format/packidx/lookup_test.go b/internal/format/packidx/lookup_test.go new file mode 100644 index 00000000..514aad7a --- /dev/null +++ b/internal/format/packidx/lookup_test.go @@ -0,0 +1,99 @@ +package packidx_test + +import ( + "bytes" + "encoding/binary" + "errors" + "testing" + + "lindenii.org/go/furgit/internal/format/packidx" + "lindenii.org/go/furgit/object/id" +) + +func TestLookupGitIndex(t *testing.T) { + t.Parallel() + + for _, objectFormat := range id.SupportedObjectFormats() { + t.Run(objectFormat.String(), func(t *testing.T) { + t.Parallel() + + repo, prefix, oids := makeGitPack(t, objectFormat) + _, idx := parseGitIdxFile(t, prefix, objectFormat) + + wantOffsets := gitPackOffsets(t, repo, prefix+".idx", objectFormat) + if len(wantOffsets) != len(oids) { + t.Fatalf("verify-pack offsets = %d entries, want %d", len(wantOffsets), len(oids)) + } + + for _, oid := range oids { + offset, found, err := idx.Lookup(oid.RawBytes()) + if err != nil { + t.Fatalf("Lookup(%s): %v", oid, err) + } + + if !found { + t.Fatalf("Lookup(%s) not found", oid) + } + + if offset != wantOffsets[oid.String()] { + t.Fatalf("Lookup(%s) = %d, want %d", oid, offset, wantOffsets[oid.String()]) + } + } + }) + } +} + +func TestLookupMissing(t *testing.T) { + t.Parallel() + + for _, objectFormat := range id.SupportedObjectFormats() { + t.Run(objectFormat.String(), func(t *testing.T) { + t.Parallel() + + _, prefix, oids := makeGitPack(t, objectFormat) + _, idx := parseGitIdxFile(t, prefix, objectFormat) + + missing := bytes.Clone(oids[0].RawBytes()) + missing[len(missing)-1] ^= 0xff + + _, found, err := idx.Lookup(missing) + if err != nil { + t.Fatalf("Lookup: %v", err) + } + + if found { + t.Fatalf("Lookup of mutated oid unexpectedly found") + } + }) + } +} + +func TestOffsetAtMalformedLargeReference(t *testing.T) { + t.Parallel() + + for _, objectFormat := range id.SupportedObjectFormats() { + t.Run(objectFormat.String(), func(t *testing.T) { + t.Parallel() + + hashSize := objectFormat.Size() + + entries := syntheticEntries(3) + data := writeSyntheticIndex(t, objectFormat, entries) + + // Mark the first 32-bit offset entry as a 64-bit reference; + // the index has no 64-bit offset table. + off32Off := 8 + 256*4 + len(entries)*hashSize + len(entries)*4 + binary.BigEndian.PutUint32(data[off32Off:], 0x80000000) + + idx, err := packidx.Parse(data, hashSize) + if err != nil { + t.Fatalf("Parse: %v", err) + } + + _, err = idx.OffsetAt(0) + if !errors.Is(err, packidx.ErrMalformedPackIndex) { + t.Fatalf("OffsetAt error = %v, want ErrMalformedPackIndex", err) + } + }) + } +} diff --git a/internal/format/packidx/packidx.go b/internal/format/packidx/packidx.go new file mode 100644 index 00000000..ef2c93f4 --- /dev/null +++ b/internal/format/packidx/packidx.go @@ -0,0 +1,197 @@ +package packidx + +import ( + "encoding/binary" + "errors" + "fmt" + + "lindenii.org/go/furgit/object/id" + "lindenii.org/go/lgo/intconv" +) + +// ErrMalformedPackIndex reports that +// a pack index is truncated, +// has a bad signature or unsupported version, +// or has inconsistent tables. +var ErrMalformedPackIndex = errors.New("internal/format/packidx: malformed pack index") + +const ( + signature = 0xff744f63 + version = 2 + + headerLen = 8 + fanoutLen = 256 * 4 + + // largeOffsetFlag marks one 32-bit offset table entry + // as an index into the 64-bit offset table. + largeOffsetFlag = 0x80000000 +) + +// Packidx is one parsed pack index view over borrowed bytes. +// +// Labels: Deps-Borrowed, Life-Parent, MT-Safe. +type Packidx struct { + // data is the entire pack index payload. + data []byte + // hashSize is the object ID size of the index's object format. + hashSize int + + // numObjects is the object count from the last fanout entry. + numObjects int + + // namesOff, crcOff, off32Off, and off64Off are + // the byte offsets of the object ID, CRC32, + // 32-bit offset, and 64-bit offset tables. + namesOff int + crcOff int + off32Off int + off64Off int + // off64Count is the number of 64-bit offset table entries. + off64Count uint64 +} + +// Parse parses one pack index from data. +// +// hashSize must be the object ID size +// of the pack's object format; +// Parse panics on implausible hash sizes. +func Parse(data []byte, hashSize int) (Packidx, error) { + var zero Packidx + + if hashSize <= 0 || hashSize > id.MaxObjectIDSize { + panic("internal/format/packidx: invalid hash size") + } + + if len(data) < headerLen+fanoutLen+2*hashSize { + return zero, fmt.Errorf("%w: truncated", ErrMalformedPackIndex) + } + + if binary.BigEndian.Uint32(data) != signature { + return zero, fmt.Errorf("%w: bad signature", ErrMalformedPackIndex) + } + + if binary.BigEndian.Uint32(data[4:]) != version { + return zero, fmt.Errorf("%w: unsupported version", ErrMalformedPackIndex) + } + + prev := uint32(0) + + for i := range 256 { + count := binary.BigEndian.Uint32(data[headerLen+4*i:]) + if count < prev { + return zero, fmt.Errorf("%w: non-monotonic fanout", ErrMalformedPackIndex) + } + + prev = count + } + + numObjects := uint64(prev) + hashSize64 := uint64(hashSize) + + namesOff := uint64(headerLen + fanoutLen) + crcOff := namesOff + numObjects*hashSize64 + off32Off := crcOff + 4*numObjects + off64Off := off32Off + 4*numObjects + + minTotal := off64Off + 2*hashSize64 + + dataLen, err := intconv.IntToUint64(len(data)) + if err != nil { + return zero, fmt.Errorf("%w: %w", ErrMalformedPackIndex, err) + } + + if dataLen < minTotal { + return zero, fmt.Errorf("%w: tables exceed index size", ErrMalformedPackIndex) + } + + off64Bytes := dataLen - minTotal + if off64Bytes%8 != 0 { + return zero, fmt.Errorf("%w: trailing table size not a 64-bit offset multiple", ErrMalformedPackIndex) + } + + off64Count := off64Bytes / 8 + if off64Count > numObjects { + return zero, fmt.Errorf("%w: more 64-bit offsets than objects", ErrMalformedPackIndex) + } + + idx := Packidx{ + data: data, + hashSize: hashSize, + off64Count: off64Count, + } + + idx.numObjects, err = intconv.Uint64ToInt(numObjects) + if err != nil { + return zero, fmt.Errorf("%w: %w", ErrMalformedPackIndex, err) + } + + idx.namesOff, err = intconv.Uint64ToInt(namesOff) + if err != nil { + return zero, fmt.Errorf("%w: %w", ErrMalformedPackIndex, err) + } + + idx.crcOff, err = intconv.Uint64ToInt(crcOff) + if err != nil { + return zero, fmt.Errorf("%w: %w", ErrMalformedPackIndex, err) + } + + idx.off32Off, err = intconv.Uint64ToInt(off32Off) + if err != nil { + return zero, fmt.Errorf("%w: %w", ErrMalformedPackIndex, err) + } + + idx.off64Off, err = intconv.Uint64ToInt(off64Off) + if err != nil { + return zero, fmt.Errorf("%w: %w", ErrMalformedPackIndex, err) + } + + return idx, nil +} + +// NumObjects returns the number of objects in the index. +func (idx *Packidx) NumObjects() int { + return idx.numObjects +} + +// PackHash returns the pack hash recorded in the index trailer. +// +// Labels: Life-Parent, Mut-No. +func (idx *Packidx) PackHash() []byte { + return idx.data[len(idx.data)-2*idx.hashSize : len(idx.data)-idx.hashSize] +} + +// OIDAt returns the object ID bytes at one index position. +// Positions follow object ID sort order. +// +// OIDAt panics when pos is out of range. +// +// Labels: Life-Parent, Mut-No. +func (idx *Packidx) OIDAt(pos int) []byte { + idx.checkPos(pos) + + start := idx.namesOff + pos*idx.hashSize + + return idx.data[start : start+idx.hashSize] +} + +// CRCAt returns the CRC32 of the packed entry data +// at one index position. +// +// CRCAt panics when pos is out of range. +func (idx *Packidx) CRCAt(pos int) uint32 { + idx.checkPos(pos) + + return binary.BigEndian.Uint32(idx.data[idx.crcOff+4*pos:]) +} + +// checkPos panics when pos is not a valid index position. +// +// An out-of-range position is a caller bug +// that slice bounds checking would not catch, +// since the tables share one data slice; +// an unchecked access would silently read other tables' bytes. +func (idx *Packidx) checkPos(pos int) { + if pos < 0 || pos >= idx.numObjects { + panic("internal/format/packidx: index position out of range") + } +} diff --git a/internal/format/packidx/packidx_test.go b/internal/format/packidx/packidx_test.go new file mode 100644 index 00000000..d225a993 --- /dev/null +++ b/internal/format/packidx/packidx_test.go @@ -0,0 +1,124 @@ +package packidx_test + +import ( + "bytes" + "encoding/binary" + "errors" + "os" + "testing" + + "lindenii.org/go/furgit/internal/format/packidx" + "lindenii.org/go/furgit/object/id" +) + +func TestParseGitIndex(t *testing.T) { + t.Parallel() + + for _, objectFormat := range id.SupportedObjectFormats() { + t.Run(objectFormat.String(), func(t *testing.T) { + t.Parallel() + + _, prefix, oids := makeGitPack(t, objectFormat) + _, idx := parseGitIdxFile(t, prefix, objectFormat) + + if idx.NumObjects() != len(oids) { + t.Fatalf("NumObjects = %d, want %d", idx.NumObjects(), len(oids)) + } + + for pos := 1; pos < idx.NumObjects(); pos++ { + if bytes.Compare(idx.OIDAt(pos-1), idx.OIDAt(pos)) >= 0 { + t.Fatalf("OIDAt(%d) not sorted after predecessor", pos) + } + } + + packData, err := os.ReadFile(prefix + ".pack") //nolint:gosec + if err != nil { + t.Fatalf("ReadFile: %v", err) + } + + packTrailer := packData[len(packData)-objectFormat.Size():] + if !bytes.Equal(idx.PackHash(), packTrailer) { + t.Fatalf("PackHash does not match pack trailer") + } + }) + } +} + +func TestParseMalformed(t *testing.T) { + t.Parallel() + + for _, objectFormat := range id.SupportedObjectFormats() { + t.Run(objectFormat.String(), func(t *testing.T) { + t.Parallel() + + hashSize := objectFormat.Size() + + valid := writeSyntheticIndex(t, objectFormat, syntheticEntries(8)) + + corrupt := func(mutate func(data []byte) []byte) []byte { + return mutate(bytes.Clone(valid)) + } + + cases := []struct { + name string + data []byte + }{ + {name: "empty", data: []byte{}}, + {name: "truncated", data: corrupt(func(d []byte) []byte { return d[:20] })}, + { + name: "bad signature", + data: corrupt(func(d []byte) []byte { + d[0] ^= 0xff + + return d + }), + }, + { + name: "bad version", + data: corrupt(func(d []byte) []byte { + d[7] = 3 + + return d + }), + }, + { + name: "non-monotonic fanout", + data: corrupt(func(d []byte) []byte { + binary.BigEndian.PutUint32(d[8:], 0xffffffff) + + return d + }), + }, + { + name: "tables exceed index size", + data: corrupt(func(d []byte) []byte { + binary.BigEndian.PutUint32(d[8+255*4:], 0x00ffffff) + + return d + }), + }, + { + name: "trailing size not 64-bit multiple", + data: corrupt(func(d []byte) []byte { return append(d, 0xde, 0xad, 0xbe, 0xef) }), + }, + { + name: "more 64-bit offsets than objects", + data: corrupt(func(d []byte) []byte { + return append(d, bytes.Repeat([]byte{0x00}, 9*8)...) + }), + }, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + + _, err := packidx.Parse(tc.data, hashSize) + if !errors.Is(err, packidx.ErrMalformedPackIndex) { + t.Fatalf("Parse error = %v, want ErrMalformedPackIndex", err) + } + }) + } + }) + } +} diff --git a/internal/format/packidx/roundtrip_test.go b/internal/format/packidx/roundtrip_test.go new file mode 100644 index 00000000..f5bd82e7 --- /dev/null +++ b/internal/format/packidx/roundtrip_test.go @@ -0,0 +1,88 @@ +package packidx_test + +import ( + "bytes" + "math" + "testing" + + "lindenii.org/go/furgit/internal/format/packidx" + "lindenii.org/go/furgit/object/id" +) + +func TestWriteRoundTrip(t *testing.T) { + t.Parallel() + + for _, objectFormat := range id.SupportedObjectFormats() { + t.Run(objectFormat.String(), func(t *testing.T) { + t.Parallel() + + hashSize := objectFormat.Size() + + entries := syntheticEntries(20) + entries[3].Offset = 1 << 33 + entries[11].Offset = math.MaxUint64 + entries[12].Offset = 0x7fffffff + entries[13].Offset = 0x80000000 + + data := writeSyntheticIndex(t, objectFormat, entries) + + idx, err := packidx.Parse(data, hashSize) + if err != nil { + t.Fatalf("Parse: %v", err) + } + + if idx.NumObjects() != len(entries) { + t.Fatalf("NumObjects = %d, want %d", idx.NumObjects(), len(entries)) + } + + if !bytes.Equal(idx.PackHash(), bytes.Repeat([]byte{0x5a}, hashSize)) { + t.Fatalf("PackHash mismatch") + } + + for pos, entry := range entries { + if !bytes.Equal(idx.OIDAt(pos), entry.OID[:hashSize]) { + t.Fatalf("OIDAt(%d) mismatch", pos) + } + + if idx.CRCAt(pos) != entry.CRC32 { + t.Fatalf("CRCAt(%d) = %x, want %x", pos, idx.CRCAt(pos), entry.CRC32) + } + + offset, err := idx.OffsetAt(pos) + if err != nil { + t.Fatalf("OffsetAt(%d): %v", pos, err) + } + + if offset != entry.Offset { + t.Fatalf("OffsetAt(%d) = %d, want %d", pos, offset, entry.Offset) + } + + offset, found, err := idx.Lookup(entry.OID[:hashSize]) + if err != nil { + t.Fatalf("Lookup(%d): %v", pos, err) + } + + if !found || offset != entry.Offset { + t.Fatalf("Lookup(%d) = %d %v, want %d found", pos, offset, found, entry.Offset) + } + } + }) + } +} + +// writeSyntheticIndex writes one index over entries +// with a fixed fake pack hash. +func writeSyntheticIndex(t *testing.T, objectFormat id.ObjectFormat, entries []packidx.Entry) []byte { + t.Helper() + + packHash := bytes.Repeat([]byte{0x5a}, objectFormat.Size()) + + var buf bytes.Buffer + + err := packidx.Write(&buf, objectFormat, entries, packHash) + if err != nil { + t.Fatalf("Write: %v", err) + } + + return buf.Bytes() +} diff --git a/internal/format/packidx/write.go b/internal/format/packidx/write.go new file mode 100644 index 00000000..35b2805f --- /dev/null +++ b/internal/format/packidx/write.go @@ -0,0 +1,129 @@ +package packidx + +import ( + "bufio" + "bytes" + "errors" + "fmt" + "io" + "math" + + "lindenii.org/go/furgit/internal/stickyio" + "lindenii.org/go/furgit/object/id" +) + +// ErrInvalidEntries reports that +// entries supplied for an index write +// are unsorted, duplicated, or not representable +// in the pack index format. +var ErrInvalidEntries = errors.New("internal/format/packidx: invalid entries") + +// Entry is one object record for an index write. +type Entry struct { + // OID holds the object ID bytes; + // only the first hash-size bytes are meaningful. + OID [id.MaxObjectIDSize]byte + // Offset is the entry's pack file offset. + Offset uint64 + // CRC32 is the CRC32 of the entry's packed data. + CRC32 uint32 +} + +// Write writes one pack index over entries to w. +// +// entries must be sorted by object ID without duplicates. +// packHash must be the pack's trailer hash; +// Write panics when its length does not match the object format. +func Write(w io.Writer, objectFormat id.ObjectFormat, entries []Entry, packHash []byte) error { + hashSize := objectFormat.Size() + if hashSize == 0 { + return id.ErrInvalidObjectFormat + } + + if len(packHash) != hashSize { + panic("internal/format/packidx: invalid pack hash length") + } + + if len(entries) > math.MaxUint32 { + return fmt.Errorf("%w: too many entries", ErrInvalidEntries) + } + + for i := 1; i < len(entries); i++ { + if bytes.Compare(entries[i-1].OID[:hashSize], entries[i].OID[:hashSize]) >= 0 { + return fmt.Errorf("%w: not sorted by object ID", ErrInvalidEntries) + } + } + + hashImpl, err := objectFormat.New() + if err != nil { + return fmt.Errorf("internal/format/packidx: %w", err) + } + + bw := bufio.NewWriter(io.MultiWriter(w, hashImpl)) + sw := stickyio.New(bw) + + sw.PutUint32(signature) + sw.PutUint32(version) + + var counts [256]uint32 + for i := range entries { + counts[entries[i].OID[0]]++ + } + + cumulative := uint32(0) + for _, count := range counts { + cumulative += count + sw.PutUint32(cumulative) + } + + for i := range entries { + sw.Put(entries[i].OID[:hashSize]) + } + + for i := range entries { + sw.PutUint32(entries[i].CRC32) + } + + largeOffsets := make([]uint64, 0, len(entries)) + + for i := range entries { + offset := entries[i].Offset + if offset < largeOffsetFlag { + sw.PutUint32(uint32(offset)) + + continue + } + + slot := len(largeOffsets) + if slot >= largeOffsetFlag { + return fmt.Errorf("%w: too many large offsets", ErrInvalidEntries) + } + + sw.PutUint32(largeOffsetFlag | uint32(slot)) + + largeOffsets = append(largeOffsets, offset) + } + + for _, offset := range largeOffsets { + sw.PutUint64(offset) + } + + sw.Put(packHash) + + err = sw.Err() + if err != nil { + return fmt.Errorf("internal/format/packidx: %w", err) + } + + err = bw.Flush() + if err != nil { + return fmt.Errorf("internal/format/packidx: %w", err) + } + + _, err = w.Write(hashImpl.Sum(nil)) + if err != nil { + return fmt.Errorf("internal/format/packidx: %w", err) + } + + return nil +} diff --git a/internal/format/packidx/write_test.go b/internal/format/packidx/write_test.go new file mode 100644 index 00000000..68df3ece --- /dev/null +++ b/internal/format/packidx/write_test.go @@ -0,0 +1,112 @@ +package packidx_test + +import ( + "bytes" + "errors" + "testing" + + "lindenii.org/go/furgit/internal/format/packidx" + "lindenii.org/go/furgit/object/id" +) + +// syntheticEntries builds n distinct entries sorted by object ID, +// spread over the fanout table. +func syntheticEntries(n int) []packidx.Entry { + entries := make([]packidx.Entry, n) + for i := range entries { + entries[i].OID[0] = byte(i * 7) + entries[i].OID[1] = byte(i + 1) + entries[i].Offset = uint64(i+1) * 100 + entries[i].CRC32 = uint32(i+1) * 0x01010101 + } + + return entries +} + +func TestWriteMatchesGit(t *testing.T) { + t.Parallel() + + for _, objectFormat := range id.SupportedObjectFormats() { + t.Run(objectFormat.String(), func(t *testing.T) { + t.Parallel() + + _, prefix, _ := makeGitPack(t, objectFormat) + gitData, idx := parseGitIdxFile(t, prefix, objectFormat) + + entries := make([]packidx.Entry, idx.NumObjects()) + for pos := range entries { + copy(entries[pos].OID[:], idx.OIDAt(pos)) + entries[pos].CRC32 = idx.CRCAt(pos) + + offset, err := idx.OffsetAt(pos) + if err != nil { + t.Fatalf("OffsetAt(%d): %v", pos, err) + } + + entries[pos].Offset = offset + } + + var buf bytes.Buffer + + err := packidx.Write(&buf, objectFormat, entries, idx.PackHash()) + if err != nil { + t.Fatalf("Write: %v", err) + } + + if !bytes.Equal(buf.Bytes(), gitData) { + t.Fatalf("Write output differs from git's index (%d vs %d bytes)", buf.Len(), len(gitData)) + } + }) + } +} + +func TestWriteInvalidEntries(t *testing.T) { + t.Parallel() + + unsorted := syntheticEntries(3) + unsorted[0], unsorted[2] = unsorted[2], unsorted[0] + + duplicated := syntheticEntries(3) + duplicated[1] = duplicated[0] + + cases := []struct { + name string + entries []packidx.Entry + }{ + {name: "unsorted", entries: unsorted}, + {name: "duplicated", entries: duplicated}, + } + + for _, objectFormat := range id.SupportedObjectFormats() { + t.Run(objectFormat.String(), func(t *testing.T) { + t.Parallel() + + packHash := bytes.Repeat([]byte{0x5a}, objectFormat.Size()) + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + + err := packidx.Write(&bytes.Buffer{}, objectFormat, tc.entries, packHash) + if !errors.Is(err, packidx.ErrInvalidEntries) { + t.Fatalf("Write error = %v, want ErrInvalidEntries", err) + } + }) + } + }) + } +} + +func TestWriteBadPackHashPanics(t *testing.T) { + t.Parallel() + + objectFormat := id.ObjectFormatSHA256 + + defer func() { + if recover() == nil { + t.Fatalf("Write with short pack hash: expected panic") + } + }() + + _ = packidx.Write(&bytes.Buffer{}, objectFormat, nil, []byte{0x01}) +} diff --git a/internal/format/packrev/doc.go b/internal/format/packrev/doc.go new file mode 100644 index 00000000..6ce8113e --- /dev/null +++ b/internal/format/packrev/doc.go @@ -0,0 +1,3 @@ +// Package packrev provides Git pack reverse index (version 1) format +// parsing and writing primitives. +package packrev diff --git a/internal/format/packrev/helpers_test.go b/internal/format/packrev/helpers_test.go new file mode 100644 index 00000000..2d781669 --- /dev/null +++ b/internal/format/packrev/helpers_test.go @@ -0,0 +1,75 @@ +package packrev_test + +import ( + "cmp" + "slices" + "testing" + + "lindenii.org/go/furgit/internal/format/packidx" + "lindenii.org/go/furgit/internal/testgit" + "lindenii.org/go/furgit/object/id" + "lindenii.org/go/lgo/intconv" +) + +// makeGitPack seeds a repository, +// packs the seeded objects with git pack-objects +// including a reverse index, +// and returns the artifact path prefix. +func makeGitPack(t *testing.T, objectFormat id.ObjectFormat) string { + t.Helper() + + repo, err := testgit.NewRepo(t, testgit.RepoOptions{ObjectFormat: objectFormat}) + if err != nil { + t.Fatalf("NewRepo: %v", err) + } + + seeded, err := repo.SeedHistory(t) + if err != nil { + t.Fatalf("SeedHistory: %v", err) + } + + prefix, err := repo.PackObjects(t, seeded.All(), testgit.PackObjectsOptions{RevIndex: true}) + if err != nil { + t.Fatalf("PackObjects: %v", err) + } + + return prefix +} + +// packOrderPositions derives the pack-offset-order index positions +// from one parsed pack index. +func packOrderPositions(t *testing.T, idx *packidx.Packidx) []uint32 { + t.Helper() + + type pair struct { + offset uint64 + position uint32 + } + + pairs := make([]pair, 0, idx.NumObjects()) + + for pos := range idx.NumObjects() { + offset, err := idx.OffsetAt(pos) + if err != nil { + t.Fatalf("OffsetAt(%d): %v", pos, err) + } + + position, err := intconv.IntToUint32(pos) + if err != nil { + t.Fatalf("IntToUint32(%d): %v", pos, err) + } + + pairs = append(pairs, pair{offset: offset, position: position}) + } + + slices.SortFunc(pairs, func(a, b pair) int { + return cmp.Compare(a.offset, b.offset) + }) + + positions := make([]uint32, 0, len(pairs)) + for _, p := range pairs { + positions = append(positions, p.position) + } + + return positions +} diff --git a/internal/format/packrev/packrev.go b/internal/format/packrev/packrev.go new file mode 100644 index 00000000..3a6dc2de --- /dev/null +++ b/internal/format/packrev/packrev.go @@ -0,0 +1,124 @@ +package packrev + +import ( + "encoding/binary" + "errors" + "fmt" + + "lindenii.org/go/furgit/object/id" + "lindenii.org/go/lgo/intconv" +) + +// ErrMalformedReverseIndex reports that +// a pack reverse index is truncated, +// has a bad signature, version, or hash function, +// or contains invalid index positions. +var ErrMalformedReverseIndex = errors.New("internal/format/packrev: malformed pack reverse index") + +const ( + signature = 0x52494458 // "RIDX" + version = 1 + + headerLen = 12 +) + +// hashFunctionID returns the on-disk hash function identifier +// for one object format. +func hashFunctionID(objectFormat id.ObjectFormat) (uint32, error) { + switch objectFormat { + case id.ObjectFormatSHA1: + return 1, nil + case id.ObjectFormatSHA256: + return 2, nil + case id.ObjectFormatUnknown: + } + + return 0, id.ErrInvalidObjectFormat +} + +// Packrev is a parsed pack reverse index view over borrowed bytes. +// +// Labels: Deps-Borrowed, Life-Parent, MT-Safe. +type Packrev struct { + // data is the entire pack reverse index payload. + data []byte + // hashSize is the object ID size of the object format. + hashSize int + // numObjects is the number of index position entries. + numObjects int +} + +// Parse parses a pack reverse index from data. +func Parse(data []byte, objectFormat id.ObjectFormat) (Packrev, error) { + var zero Packrev + + wantHashID, err := hashFunctionID(objectFormat) + if err != nil { + return zero, err + } + + hashSize := objectFormat.Size() + + if len(data) < headerLen+2*hashSize { + return zero, fmt.Errorf("%w: truncated", ErrMalformedReverseIndex) + } + + if binary.BigEndian.Uint32(data) != signature { + return zero, fmt.Errorf("%w: bad signature", ErrMalformedReverseIndex) + } + + if binary.BigEndian.Uint32(data[4:]) != version { + return zero, fmt.Errorf("%w: unsupported version", ErrMalformedReverseIndex) + } + + if binary.BigEndian.Uint32(data[8:]) != wantHashID { + return zero, fmt.Errorf("%w: hash function mismatch", ErrMalformedReverseIndex) + } + + positionBytes := len(data) - headerLen - 2*hashSize + if positionBytes%4 != 0 { + return zero, fmt.Errorf("%w: position table size not a 32-bit multiple", ErrMalformedReverseIndex) + } + + return Packrev{ + data: data, + hashSize: hashSize, + numObjects: positionBytes / 4, + }, nil +} + +// NumObjects returns the number of index position entries. +func (rev *Packrev) NumObjects() int { + return rev.numObjects +} + +// PackHash returns the pack hash recorded in the trailer. +// +// Labels: Life-Parent, Mut-No. +func (rev *Packrev) PackHash() []byte { + return rev.data[len(rev.data)-2*rev.hashSize : len(rev.data)-rev.hashSize] +} + +// PositionAt returns the pack index position +// of the object at a pack offset order position. +// +// PositionAt panics when packOrder is out of range, +// and errors when the stored position is not a valid index position. +func (rev *Packrev) PositionAt(packOrder int) (int, error) { + if packOrder < 0 || packOrder >= rev.numObjects { + panic("internal/format/packrev: pack order position out of range") + } + + stored := binary.BigEndian.Uint32(rev.data[headerLen+4*packOrder:]) + + position, err := intconv.Uint32ToInt(stored) + if err != nil { + return 0, fmt.Errorf("%w: %w", ErrMalformedReverseIndex, err) + } + + if position >= rev.numObjects { + return 0, fmt.Errorf("%w: index position out of range", ErrMalformedReverseIndex) + } + + return position, nil +} diff --git a/internal/format/packrev/packrev_test.go b/internal/format/packrev/packrev_test.go new file mode 100644 index 00000000..b644e15e --- /dev/null +++ b/internal/format/packrev/packrev_test.go @@ -0,0 +1,160 @@ +package packrev_test + +import ( + "bytes" + "encoding/binary" + "errors" + "os" + "testing" + + "lindenii.org/go/furgit/internal/format/packidx" + "lindenii.org/go/furgit/internal/format/packrev" + "lindenii.org/go/furgit/object/id" +) + +func TestParseGitReverseIndex(t *testing.T) { + t.Parallel() + + for _, objectFormat := range id.SupportedObjectFormats() { + t.Run(objectFormat.String(), func(t *testing.T) { + t.Parallel() + + prefix := makeGitPack(t, objectFormat) + + revData, err := os.ReadFile(prefix + ".rev") //nolint:gosec + if err != nil { + t.Fatalf("ReadFile: %v", err) + } + + rev, err := packrev.Parse(revData, objectFormat) + if err != nil { + t.Fatalf("Parse: %v", err) + } + + idxData, err := os.ReadFile(prefix + ".idx") //nolint:gosec + if err != nil { + t.Fatalf("ReadFile: %v", err) + } + + idx, err := packidx.Parse(idxData, objectFormat.Size()) + if err != nil { + t.Fatalf("packidx.Parse: %v", err) + } + + if rev.NumObjects() != idx.NumObjects() { + t.Fatalf("NumObjects = %d, want %d", rev.NumObjects(), idx.NumObjects()) + } + + packData, err := os.ReadFile(prefix + ".pack") //nolint:gosec + if err != nil { + t.Fatalf("ReadFile: %v", err) + } + + packTrailer := packData[len(packData)-objectFormat.Size():] + if !bytes.Equal(rev.PackHash(), packTrailer) { + t.Fatalf("PackHash does not match pack trailer") + } + + want := packOrderPositions(t, &idx) + + for packOrder, wantPosition := range want { + position, err := rev.PositionAt(packOrder) + if err != nil { + t.Fatalf("PositionAt(%d): %v", packOrder, err) + } + + if position != int(wantPosition) { + t.Fatalf("PositionAt(%d) = %d, want %d", packOrder, position, wantPosition) + } + } + }) + } +} + +func TestParseMalformed(t *testing.T) { + t.Parallel() + + for _, objectFormat := range id.SupportedObjectFormats() { + t.Run(objectFormat.String(), func(t *testing.T) { + t.Parallel() + + valid := writeSyntheticRev(t, objectFormat, []uint32{2, 0, 1, 3}) + + corrupt := func(mutate func(data []byte) []byte) []byte { + return mutate(bytes.Clone(valid)) + } + + cases := []struct { + name string + data []byte + }{ + {name: "empty", data: []byte{}}, + {name: "truncated", data: corrupt(func(d []byte) []byte { return d[:10] })}, + { + name: "bad signature", + data: corrupt(func(d []byte) []byte { + d[0] ^= 0xff + + return d + }), + }, + { + name: "bad version", + data: corrupt(func(d []byte) []byte { + d[7] = 2 + + return d + }), + }, + { + name: "hash function mismatch", + data: corrupt(func(d []byte) []byte { + d[11] ^= 0xff + + return d + }), + }, + { + name: "position table size not 32-bit multiple", + data: corrupt(func(d []byte) []byte { return append(d, 0xde, 0xad) }), + }, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + + _, err := packrev.Parse(tc.data, objectFormat) + if !errors.Is(err, packrev.ErrMalformedReverseIndex) { + t.Fatalf("Parse error = %v, want ErrMalformedReverseIndex", err) + } + }) + } + }) + } +} + +func TestPositionAtMalformed(t *testing.T) { + t.Parallel() + + for _, objectFormat := range id.SupportedObjectFormats() { + t.Run(objectFormat.String(), func(t *testing.T) { + t.Parallel() + + data := writeSyntheticRev(t, objectFormat, []uint32{2, 0, 1, 3}) + + // Corrupt the first stored position to one past the object count. + binary.BigEndian.PutUint32(data[12:], 4) + + rev, err := packrev.Parse(data, objectFormat) + if err != nil { + t.Fatalf("Parse: %v", err) + } + + _, err = rev.PositionAt(0) + if !errors.Is(err, packrev.ErrMalformedReverseIndex) { + t.Fatalf("PositionAt error = %v, want ErrMalformedReverseIndex", err) + } + }) + } +} diff --git a/internal/format/packrev/write.go b/internal/format/packrev/write.go new file mode 100644 index 00000000..399c9157 --- /dev/null +++ b/internal/format/packrev/write.go @@ -0,0 +1,79 @@ +package packrev + +import ( + "bufio" + "errors" + "fmt" + "io" + "math" + + "lindenii.org/go/furgit/internal/stickyio" + "lindenii.org/go/furgit/object/id" +) + +// ErrInvalidPositions reports that +// positions supplied for a reverse index write +// are out of range or too numerous. +var ErrInvalidPositions = errors.New("internal/format/packrev: invalid positions") + +// Write writes one pack reverse index to w. +// +// positions holds, for each object in pack offset order, +// the object's pack index position. +// packHash must be the pack's trailer hash; +// Write panics when its length does not match the object format. +func Write(w io.Writer, objectFormat id.ObjectFormat, positions []uint32, packHash []byte) error { + hashID, err := hashFunctionID(objectFormat) + if err != nil { + return err + } + + if len(packHash) != objectFormat.Size() { + panic("internal/format/packrev: invalid pack hash length") + } + + if len(positions) > math.MaxUint32 { + return fmt.Errorf("%w: too many positions", ErrInvalidPositions) + } + + for _, position := range positions { + if uint64(position) >= uint64(len(positions)) { + return fmt.Errorf("%w: index position out of range", ErrInvalidPositions) + } + } + + hashImpl, err := objectFormat.New() + if err != nil { + return fmt.Errorf("internal/format/packrev: %w", err) + } + + bw := bufio.NewWriter(io.MultiWriter(w, hashImpl)) + sw := stickyio.New(bw) + + sw.PutUint32(signature) + sw.PutUint32(version) + sw.PutUint32(hashID) + + for _, position := range positions { + sw.PutUint32(position) + } + + sw.Put(packHash) + + err = sw.Err() + if err != nil { + return fmt.Errorf("internal/format/packrev: %w", err) + } + + err = bw.Flush() + if err != nil { + return fmt.Errorf("internal/format/packrev: %w", err) + } + + _, err = w.Write(hashImpl.Sum(nil)) + if err != nil { + return fmt.Errorf("internal/format/packrev: %w", err) + } + + return nil +} diff --git a/internal/format/packrev/write_test.go b/internal/format/packrev/write_test.go new file mode 100644 index 00000000..b5c1fcb9 --- /dev/null +++ b/internal/format/packrev/write_test.go @@ -0,0 +1,135 @@ +package packrev_test + +import ( + "bytes" + "errors" + "os" + "testing" + + "lindenii.org/go/furgit/internal/format/packidx" + "lindenii.org/go/furgit/internal/format/packrev" + "lindenii.org/go/furgit/object/id" +) + +// writeSyntheticRev writes one reverse index over positions +// with a fixed fake pack hash. +func writeSyntheticRev(t *testing.T, objectFormat id.ObjectFormat, positions []uint32) []byte { + t.Helper() + + packHash := bytes.Repeat([]byte{0x5a}, objectFormat.Size()) + + var buf bytes.Buffer + + err := packrev.Write(&buf, objectFormat, positions, packHash) + if err != nil { + t.Fatalf("Write: %v", err) + } + + return buf.Bytes() +} + +func TestWriteRoundTrip(t *testing.T) { + t.Parallel() + + for _, objectFormat := range id.SupportedObjectFormats() { + t.Run(objectFormat.String(), func(t *testing.T) { + t.Parallel() + + positions := []uint32{8, 6, 7, 5, 3, 0, 4, 1, 2} + data := writeSyntheticRev(t, objectFormat, positions) + + rev, err := packrev.Parse(data, objectFormat) + if err != nil { + t.Fatalf("Parse: %v", err) + } + + if rev.NumObjects() != len(positions) { + t.Fatalf("NumObjects = %d, want %d", rev.NumObjects(), len(positions)) + } + + if !bytes.Equal(rev.PackHash(), bytes.Repeat([]byte{0x5a}, objectFormat.Size())) { + t.Fatalf("PackHash mismatch") + } + + for packOrder, want := range positions { + position, err := rev.PositionAt(packOrder) + if err != nil { + t.Fatalf("PositionAt(%d): %v", packOrder, err) + } + + if position != int(want) { + t.Fatalf("PositionAt(%d) = %d, want %d", packOrder, position, want) + } + } + }) + } +} + +func TestWriteMatchesGit(t *testing.T) { + t.Parallel() + + for _, objectFormat := range id.SupportedObjectFormats() { + t.Run(objectFormat.String(), func(t *testing.T) { + t.Parallel() + + prefix := makeGitPack(t, objectFormat) + + gitData, err := os.ReadFile(prefix + ".rev") //nolint:gosec + if err != nil { + t.Fatalf("ReadFile: %v", err) + } + + idxData, err := os.ReadFile(prefix + ".idx") //nolint:gosec + if err != nil { + t.Fatalf("ReadFile: %v", err) + } + + idx, err := packidx.Parse(idxData, objectFormat.Size()) + if err != nil { + t.Fatalf("packidx.Parse: %v", err) + } + + positions := packOrderPositions(t, &idx) + + var buf bytes.Buffer + + err = packrev.Write(&buf, objectFormat, positions, idx.PackHash()) + if err != nil { + t.Fatalf("Write: %v", err) + } + + if !bytes.Equal(buf.Bytes(), gitData) { + t.Fatalf("Write output differs from git's reverse index (%d vs %d bytes)", buf.Len(), len(gitData)) + } + }) + } +} + +func TestWriteInvalidPositions(t *testing.T) { + t.Parallel() + + for _, objectFormat := range id.SupportedObjectFormats() { + t.Run(objectFormat.String(), func(t *testing.T) { + t.Parallel() + + packHash := bytes.Repeat([]byte{0x5a}, objectFormat.Size()) + + err := packrev.Write(&bytes.Buffer{}, objectFormat, []uint32{0, 5}, packHash) + if !errors.Is(err, packrev.ErrInvalidPositions) { + t.Fatalf("Write error = %v, want ErrInvalidPositions", err) + } + }) + } +} + +func TestWriteBadPackHashPanics(t *testing.T) { + t.Parallel() + + defer func() { + if recover() == nil { + t.Fatalf("Write with short pack hash: expected panic") + } + }() + + _ = packrev.Write(&bytes.Buffer{}, id.ObjectFormatSHA256, nil, []byte{0x01}) +} diff --git a/internal/iolimit/capped_capture_writer.go b/internal/iolimit/capped_capture_writer.go new file mode 100644 index 00000000..e4a64590 --- /dev/null +++ b/internal/iolimit/capped_capture_writer.go @@ -0,0 +1,70 @@ +package iolimit + +import ( + "bytes" + "fmt" + + "lindenii.org/go/lgo/intconv" +) + +// CappedCaptureWriter captures written bytes up to a fixed limit. +// +// Once the total written bytes would exceed the limit, +// capture is disabled and Bytes returns nil. +// Write still reports success for the full input length. +type CappedCaptureWriter struct { + limit uint64 + buf bytes.Buffer + full bool +} + +// NewCappedCaptureWriter constructs one capped capture writer. +func NewCappedCaptureWriter(limit uint64) *CappedCaptureWriter { + return &CappedCaptureWriter{limit: limit} +} + +// Write captures up to the configured limit +// and always reports len(src) bytes written. +func (writer *CappedCaptureWriter) Write(src []byte) (int, error) { + if writer.full { + return len(src), nil + } + + used, err := intconv.IntToUint64(writer.buf.Len()) + if err != nil { + return 0, fmt.Errorf("iolimit: %w", err) + } + + if used >= writer.limit { + writer.full = true + + return len(src), nil + } + + room := writer.limit - used + if uint64(len(src)) > room { + take, err := intconv.Uint64ToInt(room) + if err != nil { + return 0, fmt.Errorf("iolimit: %w", err) + } + + _, _ = writer.buf.Write(src[:take]) + writer.full = true + + return len(src), nil + } + + _, _ = writer.buf.Write(src) + + return len(src), nil +} + +// Bytes returns captured bytes, +// or nil when capture exceeded the limit. +func (writer *CappedCaptureWriter) Bytes() []byte { + if writer.full { + return nil + } + + return writer.buf.Bytes() +} diff --git a/internal/iolimit/capped_capture_writer_test.go b/internal/iolimit/capped_capture_writer_test.go new file mode 100644 index 00000000..2793f8cb --- /dev/null +++ b/internal/iolimit/capped_capture_writer_test.go @@ -0,0 +1,45 @@ +package iolimit_test + +import ( + "bytes" + "testing" + + "lindenii.org/go/furgit/internal/iolimit" +) + +func TestCappedCaptureWriterWithinLimit(t *testing.T) { + t.Parallel() + + writer := iolimit.NewCappedCaptureWriter(8) + + _, _ = writer.Write([]byte("hello")) + _, _ = writer.Write([]byte("!")) + + if got := writer.Bytes(); !bytes.Equal(got, []byte("hello!")) { + t.Fatalf("Bytes() = %q, want %q", got, "hello!") + } +} + +func TestCappedCaptureWriterExceededLimit(t *testing.T) { + t.Parallel() + + writer := iolimit.NewCappedCaptureWriter(4) + + _, _ = writer.Write([]byte("abcd")) + _, _ = writer.Write([]byte("x")) + + if got := writer.Bytes(); got != nil { + t.Fatalf("Bytes() = %q, want nil after overflow", got) + } +} + +func TestCappedCaptureWriterZeroLimit(t *testing.T) { + t.Parallel() + + writer := iolimit.NewCappedCaptureWriter(0) + + _, _ = writer.Write([]byte("x")) + if got := writer.Bytes(); got != nil { + t.Fatalf("Bytes() = %q, want nil at zero limit", got) + } +} diff --git a/internal/iolimit/doc.go b/internal/iolimit/doc.go new file mode 100644 index 00000000..5eb72b2b --- /dev/null +++ b/internal/iolimit/doc.go @@ -0,0 +1,6 @@ +// Package iolimit provides small internal I/O wrappers with bounded behavior. +// +// It includes helpers for both readers and writers +// that enforce configured limits +// (length checks, capped capture, etc.). +package iolimit diff --git a/internal/iolimit/expect_length_reader.go b/internal/iolimit/expect_length_reader.go new file mode 100644 index 00000000..4e5e5950 --- /dev/null +++ b/internal/iolimit/expect_length_reader.go @@ -0,0 +1,78 @@ +package iolimit + +import ( + "errors" + "io" +) + +// ErrExpectedLengthExceeded reports that a stream +// produced bytes beyond the expected length. +var ErrExpectedLengthExceeded = errors.New("iolimit: stream exceeded expected length") + +// ExpectLengthReader wraps src and enforces an expected byte length. +// +// It returns io.ErrUnexpectedEOF +// if src ends before expected bytes are read. +// It returns ErrExpectedLengthExceeded +// if reads continue beyond the expected boundary +// and src still produces bytes. +// +// This reader does not drain src on close or at the expected boundary. +// As a result, +// overlength streams are detected only +// when a caller reads at or past the boundary. +func ExpectLengthReader(src io.Reader, expected int) io.Reader { + return &expectLengthReader{ + src: src, + remaining: expected, + } +} + +type expectLengthReader struct { + src io.Reader + remaining int +} + +func (reader *expectLengthReader) Read(dst []byte) (int, error) { + if len(dst) == 0 { + return 0, nil + } + + if reader.remaining == 0 { + var probe [1]byte + + n, err := reader.src.Read(probe[:]) + if n > 0 { + return 0, ErrExpectedLengthExceeded + } + + if err == nil { + return 0, nil + } + + return 0, err + } + + if len(dst) > reader.remaining { + dst = dst[:reader.remaining] + } + + n, err := reader.src.Read(dst) + if n > 0 { + reader.remaining -= n + } + + if errors.Is(err, io.EOF) { + if reader.remaining > 0 { + return n, io.ErrUnexpectedEOF + } + + if n > 0 { + return n, nil + } + + return 0, io.EOF + } + + return n, err +} diff --git a/internal/iolimit/expect_length_reader_test.go b/internal/iolimit/expect_length_reader_test.go new file mode 100644 index 00000000..6508e5eb --- /dev/null +++ b/internal/iolimit/expect_length_reader_test.go @@ -0,0 +1,78 @@ +package iolimit_test + +import ( + "bytes" + "errors" + "io" + "testing" + + "lindenii.org/go/furgit/internal/iolimit" +) + +func TestExpectLengthReaderExact(t *testing.T) { + t.Parallel() + + r := iolimit.ExpectLengthReader(bytes.NewReader([]byte("hello")), 5) + + got, err := io.ReadAll(r) + if err != nil { + t.Fatalf("ReadAll error: %v", err) + } + + if !bytes.Equal(got, []byte("hello")) { + t.Fatalf("ReadAll = %q, want %q", got, "hello") + } + + buf := make([]byte, 1) + + n, err := r.Read(buf) + if n != 0 || !errors.Is(err, io.EOF) { + t.Fatalf("post-boundary Read = (%d,%v), want (0,EOF)", n, err) + } +} + +func TestExpectLengthReaderShort(t *testing.T) { + t.Parallel() + + r := iolimit.ExpectLengthReader(bytes.NewReader([]byte("hey")), 5) + + _, err := io.ReadAll(r) + if !errors.Is(err, io.ErrUnexpectedEOF) { + t.Fatalf("ReadAll error = %v, want ErrUnexpectedEOF", err) + } +} + +func TestExpectLengthReaderLongDetectedOnNextRead(t *testing.T) { + t.Parallel() + + r := iolimit.ExpectLengthReader(bytes.NewReader([]byte("hello!")), 5) + buf := make([]byte, 5) + + n, err := io.ReadFull(r, buf) + if err != nil { + t.Fatalf("ReadFull error: %v", err) + } + + if n != 5 || !bytes.Equal(buf, []byte("hello")) { + t.Fatalf("ReadFull = (%d,%q), want (5,hello)", n, buf) + } + + probe := make([]byte, 1) + + n, err = r.Read(probe) + if n != 0 || !errors.Is(err, iolimit.ErrExpectedLengthExceeded) { + t.Fatalf("overflow Read = (%d,%v), want (0,ErrExpectedLengthExceeded)", n, err) + } +} + +func TestExpectLengthReaderEmptyExpected(t *testing.T) { + t.Parallel() + + r := iolimit.ExpectLengthReader(bytes.NewReader(nil), 0) + buf := make([]byte, 1) + + n, err := r.Read(buf) + if n != 0 || !errors.Is(err, io.EOF) { + t.Fatalf("Read = (%d,%v), want (0,EOF)", n, err) + } +} diff --git a/internal/mmap/doc.go b/internal/mmap/doc.go new file mode 100644 index 00000000..c4f799fb --- /dev/null +++ b/internal/mmap/doc.go @@ -0,0 +1,2 @@ +// Package mmap provides read-only memory-mapped file views. +package mmap diff --git a/internal/mmap/mmap.go b/internal/mmap/mmap.go new file mode 100644 index 00000000..a8644a3b --- /dev/null +++ b/internal/mmap/mmap.go @@ -0,0 +1,92 @@ +//go:build unix + +package mmap + +import ( + "fmt" + "os" + "syscall" + + "lindenii.org/go/lgo/intconv" +) + +// Mmap is one read-only memory mapping of an entire file. +// +// The mapping remains valid +// after the originating file is closed. +// +// Truncating the mapped file can cause +// a fatal SIGBUS on later access; +// callers should only map files +// that are never truncated while in use. +// +// Labels: Close-Caller. +type Mmap struct { + // data is the mapped region, + // or nil for empty files and closed mappings. + data []byte +} + +// Open maps file read-only in its entirety. +// +// file is only used during Open +// and may be closed once Open returns. +func Open(file *os.File) (*Mmap, error) { + info, err := file.Stat() + if err != nil { + return nil, fmt.Errorf("internal/mmap: %w", err) + } + + size64, err := intconv.Int64ToUint64(info.Size()) + if err != nil { + return nil, fmt.Errorf("internal/mmap: %w", err) + } + + size, err := intconv.Uint64ToInt(size64) + if err != nil { + return nil, fmt.Errorf("internal/mmap: %w", err) + } + + if size == 0 { + return &Mmap{data: nil}, nil + } + + fd, err := intconv.UintptrToInt(file.Fd()) + if err != nil { + return nil, fmt.Errorf("internal/mmap: %w", err) + } + + data, err := syscall.Mmap(fd, 0, size, syscall.PROT_READ, syscall.MAP_PRIVATE) + if err != nil { + return nil, fmt.Errorf("internal/mmap: %w", err) + } + + return &Mmap{data: data}, nil +} + +// Data returns the mapped bytes. +// +// Labels: Life-Parent, Mut-No. +func (mmap *Mmap) Data() []byte { + return mmap.data +} + +// Close unmaps the mapping, +// invalidating previously returned data. +// +// Labels: Idem-Yes, MT-Unsafe. +func (mmap *Mmap) Close() error { + if mmap.data == nil { + return nil + } + + data := mmap.data + mmap.data = nil + + err := syscall.Munmap(data) + if err != nil { + return fmt.Errorf("internal/mmap: %w", err) + } + + return nil +} diff --git a/internal/mmap/mmap_test.go b/internal/mmap/mmap_test.go new file mode 100644 index 00000000..814c23a3 --- /dev/null +++ b/internal/mmap/mmap_test.go @@ -0,0 +1,111 @@ +//go:build unix + +package mmap_test + +import ( + "bytes" + "os" + "path/filepath" + "testing" + + "lindenii.org/go/furgit/internal/mmap" +) + +// makeTempFileWithContents creates one file with content and opens it for reading. +func makeTempFileWithContents(t *testing.T, content []byte) *os.File { + t.Helper() + + path := filepath.Join(t.TempDir(), "data") + + err := os.WriteFile(path, content, 0o600) + if err != nil { + t.Fatalf("WriteFile: %v", err) + } + + file, err := os.Open(path) //nolint:gosec + if err != nil { + t.Fatalf("Open: %v", err) + } + + t.Cleanup(func() { _ = file.Close() }) + + return file +} + +// makeLargeTempFileLongerThanPage creates and opens one file +// whose content repeats seed +// until it spans more than one OS page. +func makeLargeTempFileLongerThanPage(t *testing.T, seed []byte) (*os.File, []byte) { + t.Helper() + + content := bytes.Repeat(seed, os.Getpagesize()/len(seed)+2) + + return makeTempFileWithContents(t, content), content +} + +func TestOpenData(t *testing.T) { + t.Parallel() + + file, content := makeLargeTempFileLongerThanPage(t, []byte("mapped bytes\n")) + + mapping, err := mmap.Open(file) + if err != nil { + t.Fatalf("Open: %v", err) + } + + // The mapping must survive closing the originating file. + err = file.Close() + if err != nil { + t.Fatalf("file Close: %v", err) + } + + if !bytes.Equal(mapping.Data(), content) { + t.Fatalf("Data mismatch") + } + + err = mapping.Close() + if err != nil { + t.Fatalf("Close: %v", err) + } +} + +func TestOpenEmpty(t *testing.T) { + t.Parallel() + + file := makeTempFileWithContents(t, nil) + + mapping, err := mmap.Open(file) + if err != nil { + t.Fatalf("Open: %v", err) + } + + if len(mapping.Data()) != 0 { + t.Fatalf("Data = %d bytes, want empty", len(mapping.Data())) + } + + err = mapping.Close() + if err != nil { + t.Fatalf("Close: %v", err) + } +} + +func TestCloseIdempotent(t *testing.T) { + t.Parallel() + + file, _ := makeLargeTempFileLongerThanPage(t, []byte("once")) + + mapping, err := mmap.Open(file) + if err != nil { + t.Fatalf("Open: %v", err) + } + + err = mapping.Close() + if err != nil { + t.Fatalf("first Close: %v", err) + } + + err = mapping.Close() + if err != nil { + t.Fatalf("second Close: %v", err) + } +} diff --git a/internal/mru/keys.go b/internal/mru/keys.go deleted file mode 100644 index 0cd9a319..00000000 --- a/internal/mru/keys.go +++ /dev/null @@ -1,17 +0,0 @@ -package mru - -// Keys returns the keys in most-recently-used order, -// front first. -// -// The result is the immutable snapshot current at the call: -// a concurrent Touch or Sync does not affect it. -// -// Labels: Mut-No. -func (order *Order[K]) Keys() []K { - keys := order.snapshot.Load() - if keys == nil { - return nil - } - - return *keys -} diff --git a/internal/mru/len.go b/internal/mru/len.go deleted file mode 100644 index 38863c66..00000000 --- a/internal/mru/len.go +++ /dev/null @@ -1,6 +0,0 @@ -package mru - -// Len returns the number of keys in the order. -func (order *Order[K]) Len() int { - return len(order.Keys()) -} diff --git a/internal/mru/new.go b/internal/mru/new.go deleted file mode 100644 index 221b650b..00000000 --- a/internal/mru/new.go +++ /dev/null @@ -1,6 +0,0 @@ -package mru - -// New returns a new, empty order. -func New[K comparable]() *Order[K] { - return &Order[K]{} //nolint:exhaustruct -} diff --git a/internal/mru/order.go b/internal/mru/order.go index 9721d852..1ea1ac09 100644 --- a/internal/mru/order.go +++ b/internal/mru/order.go @@ -22,4 +22,52 @@ import ( type Order[K comparable] struct { snapshot atomic.Pointer[[]K] mu sync.Mutex + + interval uint64 + pending atomic.Uint64 +} + +// Options configures a new Order. +type Options struct { + // Interval applies a reorder at most once per Interval + // eligible (non-front, member) Touch calls. + // + // A larger Interval decreases recency precision + // but uses fewer allocations. + // Each applied reorder allocates one snapshot, + // so throttling decreases the snapshot-allocation rate + // by roughly Interval. + // + // An Interval of 1 reorders on every eligible Touch. + Interval uint64 +} + +// New returns a new, empty order configured by opts. +func New[K comparable](opts Options) *Order[K] { + if opts.Interval == 0 { + panic("internal/mru: Options.Interval must be at least 1") + } + + return &Order[K]{interval: opts.Interval} //nolint:exhaustruct +} + +// Len returns the number of keys in the order. +func (order *Order[K]) Len() int { + return len(order.Keys()) +} + +// Keys returns the keys in most-recently-used order, +// front first. +// +// The result is the immutable snapshot current at the call: +// a concurrent Touch or Sync does not affect it. +// +// Labels: Mut-No. +func (order *Order[K]) Keys() []K { + keys := order.snapshot.Load() + if keys == nil { + return nil + } + + return *keys } diff --git a/internal/mru/order_test.go b/internal/mru/order_test.go index 8b667e77..2f16eac3 100644 --- a/internal/mru/order_test.go +++ b/internal/mru/order_test.go @@ -20,7 +20,7 @@ func set(keys ...string) map[string]struct{} { func TestTouchMovesToFront(t *testing.T) { t.Parallel() - order := mru.New[string]() + order := mru.New[string](mru.Options{Interval: 1}) order.Sync(set("a", "b", "c")) order.Touch("a") @@ -36,10 +36,60 @@ func TestTouchMovesToFront(t *testing.T) { } } +func TestIntervalThrottlesReorder(t *testing.T) { + t.Parallel() + + const interval = 4 + + order := mru.New[string](mru.Options{Interval: interval}) + order.Sync(set("a", "b", "c")) + + front := order.Keys()[0] + + other := "a" + if other == front { + other = "b" + } + + for range interval - 1 { + order.Touch(other) + + if got := order.Keys()[0]; got != front { + t.Fatalf("reordered early: front = %q, want %q", got, front) + } + } + + order.Touch(other) + + if got := order.Keys()[0]; got != other { + t.Fatalf("after interval touches, front = %q, want %q", got, other) + } +} + +func TestIntervalKeepsMembershipUnderReorder(t *testing.T) { + t.Parallel() + + order := mru.New[string](mru.Options{Interval: 8}) + order.Sync(set("a", "b", "c", "d")) + + for range 100 { + for _, key := range []string{"a", "b", "c", "d"} { + order.Touch(key) + } + } + + got := slices.Clone(order.Keys()) + slices.Sort(got) + + if want := []string{"a", "b", "c", "d"}; !slices.Equal(got, want) { + t.Fatalf("membership corrupted: %v", got) + } +} + func TestSyncDropsAbsentAndKeepsSurvivorOrder(t *testing.T) { t.Parallel() - order := mru.New[string]() + order := mru.New[string](mru.Options{Interval: 1}) order.Sync(set("a", "b", "c")) // Establish a deterministic recency order: c, b, a. @@ -58,10 +108,26 @@ func TestSyncDropsAbsentAndKeepsSurvivorOrder(t *testing.T) { } } +func TestSyncPlacesNewKeysFirst(t *testing.T) { + t.Parallel() + + order := mru.New[string](mru.Options{Interval: 1}) + order.Sync(set("a", "b")) + + order.Touch("a") + order.Touch("b") + + order.Sync(set("a", "b", "z")) + + if got, want := order.Keys(), []string{"z", "b", "a"}; !slices.Equal(got, want) { + t.Fatalf("after Sync, order = %v, want %v", got, want) + } +} + func TestTouchAbsentIsNoOp(t *testing.T) { t.Parallel() - order := mru.New[string]() + order := mru.New[string](mru.Options{Interval: 1}) order.Sync(set("a", "b")) order.Touch("a") order.Touch("z") @@ -74,7 +140,7 @@ func TestTouchAbsentIsNoOp(t *testing.T) { func TestKeysIsConsistentSnapshot(t *testing.T) { t.Parallel() - order := mru.New[string]() + order := mru.New[string](mru.Options{Interval: 1}) order.Sync(set("a", "b")) snapshot := order.Keys() @@ -95,7 +161,7 @@ func TestKeysIsConsistentSnapshot(t *testing.T) { func TestConcurrentTouchAndKeys(t *testing.T) { t.Parallel() - order := mru.New[string]() + order := mru.New[string](mru.Options{Interval: 1}) order.Sync(set("a", "b", "c", "d")) var wg sync.WaitGroup diff --git a/internal/mru/sync.go b/internal/mru/sync.go index 69fc3446..f10da5f4 100644 --- a/internal/mru/sync.go +++ b/internal/mru/sync.go @@ -4,25 +4,32 @@ package mru // // Surviving keys keep their relative recency order, // absent keys are dropped, -// and newly present keys are appended after the survivors. +// and newly present keys are placed before the survivors, +// since new members are presumed recently used. func (order *Order[K]) Sync(present map[K]struct{}) { order.mu.Lock() defer order.mu.Unlock() keys := order.Keys() - next := make([]K, 0, len(present)) - seen := make(map[K]struct{}, len(present)) + survivors := make(map[K]struct{}, len(present)) for _, key := range keys { if _, ok := present[key]; ok { - next = append(next, key) - seen[key] = struct{}{} + survivors[key] = struct{}{} } } + next := make([]K, 0, len(present)) + for key := range present { - if _, ok := seen[key]; !ok { + if _, ok := survivors[key]; !ok { + next = append(next, key) + } + } + + for _, key := range keys { + if _, ok := survivors[key]; ok { next = append(next, key) } } diff --git a/internal/mru/touch.go b/internal/mru/touch.go index 67ac6706..420bbc4a 100644 --- a/internal/mru/touch.go +++ b/internal/mru/touch.go @@ -10,12 +10,21 @@ package mru // A contended attempt, // or a key that is not a member, // leaves the order unchanged. +// +// When the order has a reorder interval above 1, +// an eligible (non-front) Touch records its recency +// but applies the reorder only once per interval such calls; +// the recording itself is lock-free and allocation-free. func (order *Order[K]) Touch(key K) { keys := order.Keys() if len(keys) == 0 || keys[0] == key { return } + if order.interval > 1 && order.pending.Add(1)%order.interval != 0 { + return + } + if !order.mu.TryLock() { return } diff --git a/internal/priorityqueue/queue.go b/internal/priorityqueue/queue.go index 2d09add8..9e64c692 100644 --- a/internal/priorityqueue/queue.go +++ b/internal/priorityqueue/queue.go @@ -19,7 +19,7 @@ func (queue *Queue[T]) Len() int { } // Pop removes one highest-priority item. -func (queue *Queue[T]) Pop() (T, bool) { +func (queue *Queue[T]) Pop() (T, bool) { //nolint:ireturn if len(queue.items) == 0 { var zero T diff --git a/internal/progress/meter.go b/internal/progress/meter.go index 0e4138de..e5e64fb4 100644 --- a/internal/progress/meter.go +++ b/internal/progress/meter.go @@ -3,7 +3,6 @@ package progress import ( "time" - "lindenii.org/go/lgo/intconv" "lindenii.org/go/lgo/iowrap" ) @@ -17,7 +16,7 @@ type Meter struct { writer iowrap.WriteFlusher title string - total uint64 + total int delay time.Duration sparse bool throughput bool @@ -26,8 +25,8 @@ type Meter struct { nextUpdateAt time.Time nextThroughput time.Time - lastDone uint64 - lastBytes uint64 + lastDone int + lastBytes int lastPercent int lastCounterW int sawValue bool @@ -58,7 +57,7 @@ type Options struct { Writer iowrap.WriteFlusher Title string - Total uint64 + Total int // Delay suppresses progress output until Delay has elapsed since Start. Delay time.Duration @@ -70,7 +69,7 @@ type Options struct { // Set records current progress // and renders when percent changed or the 1s tick elapsed. -func (meter *Meter) Set(done uint64, bytes uint64) { +func (meter *Meter) Set(done int, bytes int) { meter.lastDone = done meter.lastBytes = bytes meter.sawValue = true @@ -85,11 +84,7 @@ func (meter *Meter) Set(done uint64, bytes uint64) { percentChanged := false if meter.total > 0 { - percent, err := intconv.Uint64ToInt(done * 100 / meter.total) - if err != nil { - return // TODO - } - + percent := int(int64(done) * 100 / int64(meter.total)) percentChanged = percent != meter.lastPercent } diff --git a/internal/progress/render.go b/internal/progress/render.go index f51852eb..814ced98 100644 --- a/internal/progress/render.go +++ b/internal/progress/render.go @@ -7,7 +7,6 @@ import ( "lindenii.org/go/furgit/internal/utils" "lindenii.org/go/lgo/fmt/humanize" - "lindenii.org/go/lgo/intconv" ) func (meter *Meter) render(now time.Time, eol string) { @@ -42,13 +41,7 @@ func (meter *Meter) render(now time.Time, eol string) { func (meter *Meter) renderCounters() string { if meter.total > 0 { - u, err := intconv.Uint64ToInt(meter.lastDone * 100 / meter.total) - if err != nil { - return "overflow" - // TODO - } - - meter.lastPercent = u + meter.lastPercent = int(int64(meter.lastDone) * 100 / int64(meter.total)) return fmt.Sprintf("%3d%% (%d/%d)%s", meter.lastPercent, meter.lastDone, meter.total, meter.throughputSuffix) } @@ -75,5 +68,5 @@ func (meter *Meter) refreshThroughput(now time.Time) { } rate := uint64(float64(meter.lastBytes) / elapsed.Seconds()) - meter.throughputSuffix = ", " + humanize.Bytes(meter.lastBytes) + " | " + humanize.Bytes(rate) + "/s" + meter.throughputSuffix = ", " + humanize.Bytes(uint64(meter.lastBytes)) + " | " + humanize.Bytes(rate) + "/s" //nolint:gosec } diff --git a/internal/stickyio/doc.go b/internal/stickyio/doc.go new file mode 100644 index 00000000..9c98e667 --- /dev/null +++ b/internal/stickyio/doc.go @@ -0,0 +1,4 @@ +// Package stickyio provides error-retaining I/O wrappers +// for serializers that emit many fields +// and handle failures once. +package stickyio diff --git a/internal/stickyio/stickyio.go b/internal/stickyio/stickyio.go new file mode 100644 index 00000000..45ae84b6 --- /dev/null +++ b/internal/stickyio/stickyio.go @@ -0,0 +1,80 @@ +package stickyio + +import ( + "encoding/binary" + "io" +) + +// Writer forwards writes to an underlying writer, +// retains the first write error, +// and turns subsequent writes into no-ops. +// +// Call sites that cannot act on individual write errors +// should use [Writer.Put] and its sibling methods +// and check [Writer.Err] once after emission; +// the retained error is deferred there, not discarded. +// +// Labels: MT-Unsafe. +type Writer struct { + w io.Writer + err error +} + +var _ io.Writer = (*Writer)(nil) + +// New creates a sticky writer over w. +// +// Labels: Deps-Borrowed, Life-Parent. +func New(w io.Writer) *Writer { + return &Writer{ + w: w, + err: nil, + } +} + +// Write implements [io.Writer], +// forwarding p to the underlying writer +// and reporting its result truthfully. +// After a write error, +// Write does nothing and returns the retained error. +func (writer *Writer) Write(p []byte) (int, error) { + if writer.err != nil { + return 0, writer.err + } + + n, err := writer.w.Write(p) + if err != nil { + writer.err = err + } + + return n, err +} + +// Put writes p without per-call error reporting; +// failures are retained for [Writer.Err]. +func (writer *Writer) Put(p []byte) { + _, _ = writer.Write(p) +} + +// PutUint32 writes one big-endian uint32 +// without per-call error reporting. +func (writer *Writer) PutUint32(v uint32) { + var buf [4]byte + + binary.BigEndian.PutUint32(buf[:], v) + writer.Put(buf[:]) +} + +// PutUint64 writes one big-endian uint64 +// without per-call error reporting. +func (writer *Writer) PutUint64(v uint64) { + var buf [8]byte + + binary.BigEndian.PutUint64(buf[:], v) + writer.Put(buf[:]) +} + +// Err returns the first write error, if any. +func (writer *Writer) Err() error { + return writer.err +} diff --git a/internal/testgit/command.go b/internal/testgit/command.go index db874bd1..4e696ece 100644 --- a/internal/testgit/command.go +++ b/internal/testgit/command.go @@ -15,7 +15,7 @@ func (repo *Repo) command( ) *exec.Cmd { tb.Helper() - cmd := exec.CommandContext(tb.Context(), command, args...) //nolint:gosec // Test helper runs caller-selected commands. + cmd := exec.CommandContext(tb.Context(), command, args...) //nolint:gosec cmd.Dir = repo.path cmd.Env = repo.env diff --git a/internal/testgit/packobjects.go b/internal/testgit/packobjects.go new file mode 100644 index 00000000..62f86963 --- /dev/null +++ b/internal/testgit/packobjects.go @@ -0,0 +1,145 @@ +package testgit + +import ( + "bytes" + "errors" + "fmt" + "path/filepath" + "strings" + "testing" + + "lindenii.org/go/furgit/object/id" +) + +// ErrInvalidPackObjectsOptions reports an inconsistent +// combination of pack-objects options. +var ErrInvalidPackObjectsOptions = errors.New("internal/testgit: invalid pack-objects options") + +// PackObjectsOptions controls one on-disk pack-objects invocation. +type PackObjectsOptions struct { + // RevIndex requests writing a .rev reverse index alongside the pack. + RevIndex bool + + // Revs changes how the input is interpreted. + // + // When false, each include is one object name, + // and exactly those objects are packed. + // + // When true, each include is a revision argument, + // and pack-objects packs the closure of objects + // reachable from the includes but not from the excludes, + // the same walk git rev-list --objects performs. + Revs bool + + // Exclude lists objects to omit by reachability, + // fed as "^<oid>" revision arguments. + // A non-nil Exclude requires Revs. + Exclude []id.ObjectID +} + +// PackObjects packs the include objects with git pack-objects +// into a temporary directory, +// and returns the artifact path prefix "<dir>/pack-<hash>", +// to which ".pack", ".idx", and ".rev" suffixes apply. +func (repo *Repo) PackObjects(tb testing.TB, include []id.ObjectID, opts PackObjectsOptions) (string, error) { + tb.Helper() + + if opts.Exclude != nil && !opts.Revs { + return "", fmt.Errorf("%w: Exclude requires Revs", ErrInvalidPackObjectsOptions) + } + + dir := tb.TempDir() + + revIndex := "false" + if opts.RevIndex { + revIndex = "true" + } + + args := []string{"-c", "pack.writeReverseIndex=" + revIndex, "pack-objects"} + if opts.Revs { + args = append(args, "--revs") + } + + args = append(args, "--end-of-options", filepath.Join(dir, "pack")) + + out, err := repo.run(tb, packObjectsStdin(include, opts.Exclude), "git", args...) + if err != nil { + return "", err + } + + return filepath.Join(dir, "pack-"+strings.TrimSpace(string(out))), nil +} + +// PackObjectsStdoutOptions controls one streamed pack-objects invocation. +type PackObjectsStdoutOptions struct { + // Revs changes how the input is interpreted. + // + // When false, each include is one object name, + // and exactly those objects are packed. + // + // When true, each include is a revision argument, + // and pack-objects packs the closure of objects + // reachable from the includes but not from the excludes, + // the same walk git rev-list --objects performs. + Revs bool + + // Thin omits excluded base objects from the pack, + // producing a thin pack that must be completed before use. + // Thin requires Revs. + Thin bool + + // Exclude lists objects to omit by reachability, + // fed as "^<oid>" revision arguments. + // A non-nil Exclude requires Revs. + Exclude []id.ObjectID +} + +// PackObjectsStdout packs the include objects with git pack-objects +// and returns the pack stream written to standard output. +func (repo *Repo) PackObjectsStdout(tb testing.TB, include []id.ObjectID, opts PackObjectsStdoutOptions) ([]byte, error) { + tb.Helper() + + if opts.Exclude != nil && !opts.Revs { + return nil, fmt.Errorf("%w: Exclude requires Revs", ErrInvalidPackObjectsOptions) + } + + if opts.Thin && !opts.Revs { + return nil, fmt.Errorf("%w: Thin requires Revs", ErrInvalidPackObjectsOptions) + } + + args := []string{"pack-objects", "--stdout"} + if opts.Revs { + args = append(args, "--revs") + } + + if opts.Thin { + args = append(args, "--thin") + } + + out, err := repo.run(tb, packObjectsStdin(include, opts.Exclude), "git", args...) + if err != nil { + return nil, err + } + + return out, nil +} + +// packObjectsStdin builds the pack-objects standard input: +// the include object IDs, +// followed by the exclude object IDs as "^<oid>" lines. +func packObjectsStdin(include, exclude []id.ObjectID) *bytes.Buffer { + var stdin bytes.Buffer + + for _, oid := range include { + stdin.WriteString(oid.String()) + stdin.WriteByte('\n') + } + + for _, oid := range exclude { + stdin.WriteByte('^') + stdin.WriteString(oid.String()) + stdin.WriteByte('\n') + } + + return &stdin +} diff --git a/internal/testgit/seed.go b/internal/testgit/seed.go new file mode 100644 index 00000000..9f60b23e --- /dev/null +++ b/internal/testgit/seed.go @@ -0,0 +1,206 @@ +package testgit + +import ( + "fmt" + "strings" + "testing" + + "lindenii.org/go/furgit/object/id" + "lindenii.org/go/furgit/object/typ" +) + +// Seeded lists the objects created by one [Repo.SeedHistory], by type. +// Each object ID appears exactly once. +type Seeded struct { + Blobs []id.ObjectID + Trees []id.ObjectID + Commits []id.ObjectID + Tags []id.ObjectID +} + +// All returns all seeded object IDs. +func (seeded *Seeded) All() []id.ObjectID { + all := make([]id.ObjectID, 0, len(seeded.Blobs)+len(seeded.Trees)+len(seeded.Commits)+len(seeded.Tags)) + + all = append(all, seeded.Blobs...) + all = append(all, seeded.Trees...) + all = append(all, seeded.Commits...) + all = append(all, seeded.Tags...) + + return all +} + +// SeedHistory creates one deterministic synthetic history +// of 64 commits on refs/heads/main, +// covering all four object types and various situations, +// such as stable, growing, unique, and empty blobs, +// nested subtrees shared across several commits, +// executable and symlink tree entries, +// and an annotated tag every sixteenth commit. +func (repo *Repo) SeedHistory(tb testing.TB) (Seeded, error) { + tb.Helper() + + seeder := &historySeeder{ + repo: repo, + tb: tb, + seen: make(map[id.ObjectID]struct{}), + seeded: Seeded{ + Blobs: nil, + Trees: nil, + Commits: nil, + Tags: nil, + }, + } + + err := seeder.run() + if err != nil { + return Seeded{}, err + } + + return seeder.seeded, nil +} + +type historySeeder struct { + repo *Repo + tb testing.TB + seen map[id.ObjectID]struct{} + seeded Seeded +} + +func (seeder *historySeeder) run() error { + readme, err := seeder.blob("stable readme\n") + if err != nil { + return err + } + + empty, err := seeder.blob("") + if err != nil { + return err + } + + link, err := seeder.blob("README") + if err != nil { + return err + } + + var parents []id.ObjectID + + for i := range 64 { + commitID, err := seeder.commit(i, readme, empty, link, parents) + if err != nil { + return fmt.Errorf("seed commit %d: %w", i, err) + } + + seeder.seeded.Commits = append(seeder.seeded.Commits, commitID) + parents = []id.ObjectID{commitID} + + if (i+1)%16 == 0 { + err = seeder.tag(fmt.Sprintf("v%d", (i+1)/16), commitID) + if err != nil { + return fmt.Errorf("seed tag at commit %d: %w", i, err) + } + } + } + + return seeder.repo.UpdateRef(seeder.tb, "refs/heads/main", parents[0]) +} + +func (seeder *historySeeder) commit(i int, readme, empty, link id.ObjectID, parents []id.ObjectID) (id.ObjectID, error) { + var growingContent strings.Builder + for j := range i + 1 { + fmt.Fprintf(&growingContent, "entry %d\n", j) + } + + growing, err := seeder.blob(growingContent.String()) + if err != nil { + return id.ObjectID{}, err + } + + tool, err := seeder.blob(fmt.Sprintf("tool %d\n", i)) + if err != nil { + return id.ObjectID{}, err + } + + inner, err := seeder.blob(fmt.Sprintf("inner %d\n", i/4)) + if err != nil { + return id.ObjectID{}, err + } + + subtree, err := seeder.tree([]TreeEntry{ + {Mode: "100644", Type: typ.Blob, OID: inner, Name: "inner"}, + {Mode: "120000", Type: typ.Blob, OID: link, Name: "link"}, + }) + if err != nil { + return id.ObjectID{}, err + } + + root, err := seeder.tree([]TreeEntry{ + {Mode: "100644", Type: typ.Blob, OID: readme, Name: "README"}, + {Mode: "100644", Type: typ.Blob, OID: empty, Name: "empty"}, + {Mode: "100644", Type: typ.Blob, OID: growing, Name: "log"}, + {Mode: "040000", Type: typ.Tree, OID: subtree, Name: "sub"}, + {Mode: "100755", Type: typ.Blob, OID: tool, Name: "tool"}, + }) + if err != nil { + return id.ObjectID{}, err + } + + return seeder.repo.CommitTree( + seeder.tb, + root, + CommitTreeOptions{Message: fmt.Sprintf("commit %d", i)}, //nolint:exhaustruct + parents..., + ) +} + +func (seeder *historySeeder) blob(content string) (id.ObjectID, error) { + oid, err := seeder.repo.HashObject(seeder.tb, typ.Blob, strings.NewReader(content)) + if err != nil { + return id.ObjectID{}, err + } + + if seeder.record(oid) { + seeder.seeded.Blobs = append(seeder.seeded.Blobs, oid) + } + + return oid, nil +} + +func (seeder *historySeeder) tree(entries []TreeEntry) (id.ObjectID, error) { + oid, err := seeder.repo.MkTree(seeder.tb, entries) + if err != nil { + return id.ObjectID{}, err + } + + if seeder.record(oid) { + seeder.seeded.Trees = append(seeder.seeded.Trees, oid) + } + + return oid, nil +} + +func (seeder *historySeeder) tag(name string, target id.ObjectID) error { + oid, err := seeder.repo.TagAnnotated( + seeder.tb, + name, + target, + TagAnnotatedOptions{Message: "tag " + name}, //nolint:exhaustruct + ) + if err != nil { + return err + } + + seeder.seeded.Tags = append(seeder.seeded.Tags, oid) + + return nil +} + +func (seeder *historySeeder) record(oid id.ObjectID) bool { + if _, ok := seeder.seen[oid]; ok { + return false + } + + seeder.seen[oid] = struct{}{} + + return true +} diff --git a/internal/testgit/tree.go b/internal/testgit/tree.go index 501c7949..ff9b1918 100644 --- a/internal/testgit/tree.go +++ b/internal/testgit/tree.go @@ -50,7 +50,7 @@ func (repo *Repo) LsTree(tb testing.TB, oid id.ObjectID) ([]TreeEntry, error) { return nil, fmt.Errorf("ls-tree: %w", err) } - var entries []TreeEntry + entries := make([]TreeEntry, 0, bytes.Count(stdout, []byte{0})) for record := range bytes.SplitSeq(stdout, []byte{0}) { if len(record) == 0 { diff --git a/internal/testgit/verifypack.go b/internal/testgit/verifypack.go new file mode 100644 index 00000000..ec59572a --- /dev/null +++ b/internal/testgit/verifypack.go @@ -0,0 +1,13 @@ +package testgit + +import ( + "testing" +) + +// VerifyPack runs git verify-pack -v on one pack index path +// and returns its raw verbose output. +func (repo *Repo) VerifyPack(tb testing.TB, idxPath string) ([]byte, error) { + tb.Helper() + + return repo.run(tb, nil, "git", "verify-pack", "-v", "--end-of-options", idxPath) +} |