diff options
| author | 2026-03-04 08:26:56 +0800 | |
|---|---|---|
| committer | 2026-03-04 08:59:53 +0800 | |
| commit | ab7501be34032fb9e5c48726a68ae90a917af9eb (patch) | |
| tree | 20d005647569befea8133e953c3270e8fd2a2a5b /objectstore/packed | |
| parent | *: gofumpt (diff) | |
| signature | No signature | |
*: Lint
Diffstat (limited to 'objectstore/packed')
| -rw-r--r-- | objectstore/packed/delta_apply.go | 13 | ||||
| -rw-r--r-- | objectstore/packed/delta_cache.go | 1 | ||||
| -rw-r--r-- | objectstore/packed/delta_plan.go | 7 | ||||
| -rw-r--r-- | objectstore/packed/entry_inflate.go | 8 | ||||
| -rw-r--r-- | objectstore/packed/entry_parse.go | 4 | ||||
| -rw-r--r-- | objectstore/packed/helpers_test.go | 17 | ||||
| -rw-r--r-- | objectstore/packed/idx_lookup_candidates.go | 24 | ||||
| -rw-r--r-- | objectstore/packed/idx_open.go | 40 | ||||
| -rw-r--r-- | objectstore/packed/idx_parse.go | 24 | ||||
| -rw-r--r-- | objectstore/packed/pack.go | 19 | ||||
| -rw-r--r-- | objectstore/packed/pack_idx_checksum.go | 4 | ||||
| -rw-r--r-- | objectstore/packed/read_bytes.go | 4 | ||||
| -rw-r--r-- | objectstore/packed/read_header.go | 1 | ||||
| -rw-r--r-- | objectstore/packed/read_header_resolve.go | 5 | ||||
| -rw-r--r-- | objectstore/packed/read_reader.go | 7 | ||||
| -rw-r--r-- | objectstore/packed/read_size.go | 3 | ||||
| -rw-r--r-- | objectstore/packed/read_test.go | 75 | ||||
| -rw-r--r-- | objectstore/packed/store.go | 64 |
18 files changed, 287 insertions, 33 deletions
diff --git a/objectstore/packed/delta_apply.go b/objectstore/packed/delta_apply.go index 5245e0ba..71f09ead 100644 --- a/objectstore/packed/delta_apply.go +++ b/objectstore/packed/delta_apply.go @@ -14,10 +14,12 @@ func (store *Store) deltaResolveContent(start location) (objecttype.Type, []byte if err != nil { return objecttype.TypeInvalid, nil, err } + pack, meta, err := store.entryMetaAt(start) if err != nil { return objecttype.TypeInvalid, nil, err } + declaredSize := meta.size if !packfmt.IsBaseObjectType(meta.ty) { declaredSize, err = deltaDeclaredSizeAt(pack, meta.dataOffset) @@ -25,6 +27,7 @@ func (store *Store) deltaResolveContent(start location) (objecttype.Type, []byte return objecttype.TypeInvalid, nil, err } } + return store.deltaResolveChain(chain, declaredSize) } @@ -37,18 +40,22 @@ func (store *Store) deltaResolveChain(chain deltaChain, declaredSize int64) (obj for i := nextDelta; i >= 0; i-- { node := chain.deltas[i] + pack, err := store.openPack(node.loc.packName) if err != nil { return objecttype.TypeInvalid, nil, err } + delta, err := inflateAt(pack, node.dataOffset, -1) if err != nil { return objecttype.TypeInvalid, nil, err } + out, err = deltaapply.Apply(out, delta) if err != nil { return objecttype.TypeInvalid, nil, err } + store.cacheMu.Lock() store.deltaCache.add( deltaBaseKey{packName: node.loc.packName, offset: node.loc.offset}, @@ -65,6 +72,7 @@ func (store *Store) deltaResolveChain(chain deltaChain, declaredSize int64) (obj declaredSize, ) } + if ty != chain.baseType { return objecttype.TypeInvalid, nil, fmt.Errorf( "objectstore/packed: resolved content type mismatch: got %d want %d", @@ -72,6 +80,7 @@ func (store *Store) deltaResolveChain(chain deltaChain, declaredSize int64) (obj chain.baseType, ) } + return ty, out, nil } @@ -85,6 +94,7 @@ func (store *Store) deltaResolveChainStart(chain deltaChain) (objecttype.Type, [ deltaBaseKey{packName: node.loc.packName, offset: node.loc.offset}, ) store.cacheMu.RUnlock() + if ok { return ty, out, i - 1, nil } @@ -95,6 +105,7 @@ func (store *Store) deltaResolveChainStart(chain deltaChain) (objecttype.Type, [ deltaBaseKey{packName: chain.baseLoc.packName, offset: chain.baseLoc.offset}, ) store.cacheMu.RUnlock() + if ok { return ty, out, len(chain.deltas) - 1, nil } @@ -103,9 +114,11 @@ func (store *Store) deltaResolveChainStart(chain deltaChain) (objecttype.Type, [ if err != nil { return objecttype.TypeInvalid, nil, 0, err } + if !packfmt.IsBaseObjectType(meta.ty) { return objecttype.TypeInvalid, nil, 0, fmt.Errorf("objectstore/packed: delta chain base is not a base object") } + base, err := inflateAt(pack, meta.dataOffset, meta.size) if err != nil { return objecttype.TypeInvalid, nil, 0, err diff --git a/objectstore/packed/delta_cache.go b/objectstore/packed/delta_cache.go index add21698..a911b254 100644 --- a/objectstore/packed/delta_cache.go +++ b/objectstore/packed/delta_cache.go @@ -41,6 +41,7 @@ func (cache *deltaCache) get(key deltaBaseKey) (objecttype.Type, []byte, bool) { if !ok { return objecttype.TypeInvalid, nil, false } + return value.ty, append([]byte(nil), value.content...), true } diff --git a/objectstore/packed/delta_plan.go b/objectstore/packed/delta_plan.go index 5f2ae959..b0b0324c 100644 --- a/objectstore/packed/delta_plan.go +++ b/objectstore/packed/delta_plan.go @@ -38,6 +38,7 @@ func (store *Store) deltaBuildChain(start location) (deltaChain, error) { if _, ok := visited[current]; ok { return deltaChain{}, fmt.Errorf("objectstore/packed: delta cycle while resolving object") } + visited[current] = struct{}{} _, meta, err := store.entryMetaAt(current) @@ -48,6 +49,7 @@ func (store *Store) deltaBuildChain(start location) (deltaChain, error) { if packfmt.IsBaseObjectType(meta.ty) { chain.baseLoc = current chain.baseType = meta.ty + return chain, nil } @@ -57,10 +59,12 @@ func (store *Store) deltaBuildChain(start location) (deltaChain, error) { loc: current, dataOffset: meta.dataOffset, }) + next, err := store.lookup(meta.baseRefID) if err != nil { return deltaChain{}, err } + current = next case objecttype.TypeOfsDelta: chain.deltas = append(chain.deltas, deltaNode{ @@ -88,12 +92,15 @@ func deltaDeclaredSizeAt(pack *packFile, dataOffset int) (int64, error) { if err != nil { return 0, err } + defer func() { _ = reader.Close() }() br := bufio.NewReaderSize(reader, 32) + _, size, err := deltaapply.ReadHeaderSizes(br) if err != nil { return 0, err } + return int64(size), nil } diff --git a/objectstore/packed/entry_inflate.go b/objectstore/packed/entry_inflate.go index 4f91710e..cbdb6a89 100644 --- a/objectstore/packed/entry_inflate.go +++ b/objectstore/packed/entry_inflate.go @@ -14,6 +14,7 @@ func zlibReaderAt(pack *packFile, offset int) (io.ReadCloser, error) { if offset < 0 || offset > len(pack.data) { return nil, fmt.Errorf("objectstore/packed: pack %q zlib offset out of bounds", pack.name) } + return zlib.NewReader(bytes.NewReader(pack.data[offset:])) } @@ -23,6 +24,7 @@ func inflateAt(pack *packFile, offset int, expectedSize int64) ([]byte, error) { if err != nil { return nil, err } + defer func() { _ = reader.Close() }() if expectedSize >= 0 { @@ -35,9 +37,12 @@ func inflateAt(pack *packFile, offset int, expectedSize int64) ([]byte, error) { } body := make([]byte, int(expectedSize)) - if _, err := io.ReadFull(reader, body); err != nil { + + _, err := io.ReadFull(reader, body) + if err != nil { return nil, err } + return body, nil } @@ -45,5 +50,6 @@ func inflateAt(pack *packFile, offset int, expectedSize int64) ([]byte, error) { if err != nil { return nil, err } + return body, nil } diff --git a/objectstore/packed/entry_parse.go b/objectstore/packed/entry_parse.go index 56287386..7af20af1 100644 --- a/objectstore/packed/entry_parse.go +++ b/objectstore/packed/entry_parse.go @@ -34,6 +34,7 @@ func parseEntryMeta(pack *packFile, algo objectid.Algorithm, offset uint64) (ent if err != nil { return zero, fmt.Errorf("objectstore/packed: pack %q offset conversion: %w", pack.name, err) } + entry, err := packfmt.ParseEntry(pack.data[pos:], algo.Size()) if err != nil { return zero, fmt.Errorf("objectstore/packed: pack %q: %w", pack.name, err) @@ -50,11 +51,13 @@ func parseEntryMeta(pack *packFile, algo objectid.Algorithm, offset uint64) (ent if err != nil { return zero, fmt.Errorf("objectstore/packed: pack %q invalid ref-delta base id: %w", pack.name, err) } + meta.baseRefID = baseID case objecttype.TypeOfsDelta: if offset <= entry.OfsBaseDistance { return zero, fmt.Errorf("objectstore/packed: pack %q has invalid ofs-delta base", pack.name) } + meta.baseOfs = offset - entry.OfsBaseDistance case objecttype.TypeCommit, objecttype.TypeTree, objecttype.TypeBlob, objecttype.TypeTag: // Base object types do not have delta base metadata. @@ -63,5 +66,6 @@ func parseEntryMeta(pack *packFile, algo objectid.Algorithm, offset uint64) (ent default: return zero, fmt.Errorf("objectstore/packed: pack %q has unsupported entry type %d", pack.name, meta.ty) } + return meta, nil } diff --git a/objectstore/packed/helpers_test.go b/objectstore/packed/helpers_test.go index f8cbd439..1b517294 100644 --- a/objectstore/packed/helpers_test.go +++ b/objectstore/packed/helpers_test.go @@ -18,30 +18,39 @@ import ( func openPackedStore(t *testing.T, repoPath string, algo objectid.Algorithm) *packed.Store { t.Helper() + packPath := filepath.Join(repoPath, "objects", "pack") + root, err := os.OpenRoot(packPath) if err != nil { t.Fatalf("OpenRoot(%q): %v", packPath, err) } + t.Cleanup(func() { _ = root.Close() }) store, err := packed.New(root, algo) if err != nil { t.Fatalf("packed.New: %v", err) } + return store } func mustReadAllAndClose(t *testing.T, reader io.ReadCloser) []byte { t.Helper() + data, err := io.ReadAll(reader) if err != nil { _ = reader.Close() + t.Fatalf("ReadAll: %v", err) } - if err := reader.Close(); err != nil { + + err = reader.Close() + if err != nil { t.Fatalf("Close: %v", err) } + return data } @@ -49,11 +58,14 @@ func expectedRawObject(t *testing.T, testRepo *testgit.TestRepo, id objectid.Obj t.Helper() typeName := testRepo.Run(t, "cat-file", "-t", id.String()) + ty, ok := objecttype.ParseName(typeName) if !ok { t.Fatalf("ParseName(%q) failed", typeName) } + body := testRepo.CatFile(t, typeName, id) + header, ok := objectheader.Encode(ty, int64(len(body))) if !ok { t.Fatalf("objectheader.Encode failed") @@ -62,6 +74,7 @@ func expectedRawObject(t *testing.T, testRepo *testgit.TestRepo, id objectid.Obj raw := make([]byte, len(header)+len(body)) copy(raw, header) copy(raw[len(header):], body) + return ty, body, raw } @@ -74,6 +87,7 @@ func createPackedFixtureRepo(t *testing.T, algo objectid.Algorithm) (*testgit.Te tagID := testRepo.TagAnnotated(t, "v1.0.0", commitID, "packed-store-tag") parent := commitID + for i := range 24 { content := "common-prefix\n" + strings.Repeat("line-"+strconv.Itoa(i%3)+"\n", 256) + fmt.Sprintf("tail-%d\n", i) nextBlob, nextTree := testRepo.MakeSingleFileTree(t, fmt.Sprintf("file-%02d.txt", i), []byte(content)) @@ -86,6 +100,7 @@ func createPackedFixtureRepo(t *testing.T, algo objectid.Algorithm) (*testgit.Te } testRepo.Repack(t, "-a", "-d", "-f", "--window=64", "--depth=64") + return testRepo, []objectid.ObjectID{ blobID, treeID, diff --git a/objectstore/packed/idx_lookup_candidates.go b/objectstore/packed/idx_lookup_candidates.go index 83055aac..72121b25 100644 --- a/objectstore/packed/idx_lookup_candidates.go +++ b/objectstore/packed/idx_lookup_candidates.go @@ -37,8 +37,11 @@ func (store *Store) ensureCandidates() error { candidateByPack := make(map[string]packCandidate, len(candidates)) nodeByPack := make(map[string]*packCandidateNode, len(candidates)) - var head *packCandidateNode - var tail *packCandidateNode + var ( + head *packCandidateNode + tail *packCandidateNode + ) + for _, candidate := range candidates { node := &packCandidateNode{ candidate: candidate, @@ -47,9 +50,11 @@ func (store *Store) ensureCandidates() error { if tail != nil { tail.next = node } + if head == nil { head = node } + tail = node candidateByPack[candidate.packName] = candidate nodeByPack[candidate.packName] = node @@ -67,6 +72,7 @@ func (store *Store) ensureCandidates() error { store.candidatesMu.RLock() err := store.discoverErr store.candidatesMu.RUnlock() + return err } @@ -78,8 +84,10 @@ func (store *Store) discoverCandidates() ([]packCandidate, error) { if os.IsNotExist(err) { return nil, nil } + return nil, err } + defer func() { _ = dir.Close() }() entries, err := dir.ReadDir(-1) @@ -95,11 +103,13 @@ func (store *Store) discoverCandidates() ([]packCandidate, error) { idxName := entry.Name() packName := strings.TrimSuffix(idxName, ".idx") + ".pack" + packInfo, err := store.root.Stat(packName) if err != nil { if os.IsNotExist(err) { return nil, fmt.Errorf("objectstore/packed: missing pack file for index %q", idxName) } + return nil, err } @@ -115,8 +125,10 @@ func (store *Store) discoverCandidates() ([]packCandidate, error) { if a.mtime > b.mtime { return -1 } + return 1 } + return strings.Compare(a.packName, b.packName) }) @@ -139,18 +151,22 @@ func (store *Store) touchCandidate(packName string) { if node.prev != nil { node.prev.next = node.next } + if node.next != nil { node.next.prev = node.prev } + if store.candidateTail == node { store.candidateTail = node.prev } node.prev = nil + node.next = store.candidateHead if store.candidateHead != nil { store.candidateHead.prev = node } + store.candidateHead = node if store.candidateTail == nil { store.candidateTail = node @@ -162,9 +178,11 @@ func (store *Store) touchCandidate(packName string) { func (store *Store) firstCandidatePackName() string { store.candidatesMu.RLock() defer store.candidatesMu.RUnlock() + if store.candidateHead == nil { return "" } + return store.candidateHead.candidate.packName } @@ -173,9 +191,11 @@ func (store *Store) firstCandidatePackName() string { func (store *Store) nextCandidatePackName(currentPack string) string { store.candidatesMu.RLock() defer store.candidatesMu.RUnlock() + node := store.candidateNodeByPack[currentPack] if node == nil || node.next == nil { return "" } + return node.next.candidate.packName } diff --git a/objectstore/packed/idx_open.go b/objectstore/packed/idx_open.go index c00a7bac..c3c97e4d 100644 --- a/objectstore/packed/idx_open.go +++ b/objectstore/packed/idx_open.go @@ -43,16 +43,21 @@ func (store *Store) candidateForPack(packName string) (packCandidate, bool) { store.candidatesMu.RLock() candidate, ok := store.candidateByPack[packName] store.candidatesMu.RUnlock() + return candidate, ok } // openIndex returns one opened and parsed index, caching it by pack basename. func (store *Store) openIndex(candidate packCandidate) (*idxFile, error) { store.idxMu.RLock() - if index, ok := store.idxByPack[candidate.packName]; ok { + + index, ok := store.idxByPack[candidate.packName] + if ok { store.idxMu.RUnlock() + return index, nil } + store.idxMu.RUnlock() index, err := openIdxFile(store.root, candidate.idxName, candidate.packName, store.algo) @@ -61,13 +66,19 @@ func (store *Store) openIndex(candidate packCandidate) (*idxFile, error) { } store.idxMu.Lock() - if existing, ok := store.idxByPack[candidate.packName]; ok { + + existing, ok := store.idxByPack[candidate.packName] + if ok { store.idxMu.Unlock() + _ = index.close() + return existing, nil } + store.idxByPack[candidate.packName] = index store.idxMu.Unlock() + return index, nil } @@ -77,24 +88,32 @@ func openIdxFile(root *os.Root, idxName, packName string, algo objectid.Algorith if err != nil { return nil, err } + info, err := file.Stat() if err != nil { _ = file.Close() + return nil, err } + size := info.Size() if size < 0 || size > int64(int(^uint(0)>>1)) { _ = file.Close() + return nil, fmt.Errorf("objectstore/packed: idx %q has unsupported size", idxName) } + fd, err := intconv.UintptrToInt(file.Fd()) if err != nil { _ = file.Close() + return nil, err } + data, err := syscall.Mmap(fd, 0, int(size), syscall.PROT_READ, syscall.MAP_PRIVATE) if err != nil { _ = file.Close() + return nil, err } @@ -105,27 +124,38 @@ func openIdxFile(root *os.Root, idxName, packName string, algo objectid.Algorith file: file, data: data, } - if err := index.parse(); err != nil { + + err = index.parse() + if err != nil { _ = index.close() + return nil, err } + return index, nil } // close unmaps and closes one idx handle. func (index *idxFile) close() error { var closeErr error + if index.data != nil { - if err := syscall.Munmap(index.data); err != nil && closeErr == nil { + err := syscall.Munmap(index.data) + if err != nil && closeErr == nil { closeErr = err } + index.data = nil } + if index.file != nil { - if err := index.file.Close(); err != nil && closeErr == nil { + err := index.file.Close() + if err != nil && closeErr == nil { closeErr = err } + index.file = nil } + return closeErr } diff --git a/objectstore/packed/idx_parse.go b/objectstore/packed/idx_parse.go index 0af72594..870ffdae 100644 --- a/objectstore/packed/idx_parse.go +++ b/objectstore/packed/idx_parse.go @@ -19,27 +19,34 @@ func (index *idxFile) parse() error { if hashSize <= 0 { return fmt.Errorf("objectstore/packed: idx %q has invalid hash algorithm", index.idxName) } + minLen := 8 + 256*4 + 2*hashSize if len(index.data) < minLen { return fmt.Errorf("objectstore/packed: idx %q too short", index.idxName) } + if binary.BigEndian.Uint32(index.data[:4]) != idxMagicV2 { return fmt.Errorf("objectstore/packed: idx %q invalid magic", index.idxName) } + if binary.BigEndian.Uint32(index.data[4:8]) != idxVersionV2 { return fmt.Errorf("objectstore/packed: idx %q unsupported version", index.idxName) } prev := uint32(0) + for i := range 256 { base := 8 + i*4 + cur := binary.BigEndian.Uint32(index.data[base : base+4]) if cur < prev { return fmt.Errorf("objectstore/packed: idx %q has non-monotonic fanout table", index.idxName) } + index.fanout[i] = cur prev = cur } + index.numObjects = int(index.fanout[255]) if index.numObjects < 0 { return fmt.Errorf("objectstore/packed: idx %q has invalid object count", index.idxName) @@ -48,6 +55,7 @@ func (index *idxFile) parse() error { namesBytes := index.numObjects * hashSize crcBytes := index.numObjects * 4 offset32Bytes := index.numObjects * 4 + minSize := 8 + 256*4 + namesBytes + crcBytes + offset32Bytes + 2*hashSize if minSize < 0 || len(index.data) < minSize { return fmt.Errorf("objectstore/packed: idx %q has truncated tables", index.idxName) @@ -61,11 +69,14 @@ func (index *idxFile) parse() error { if offset64Bytes < 0 || offset64Bytes%8 != 0 { return fmt.Errorf("objectstore/packed: idx %q has malformed 64-bit offset table", index.idxName) } + index.offset64Count = offset64Bytes / 8 + maxOffset64Count := max(index.numObjects-1, 0) if index.offset64Count > maxOffset64Count { return fmt.Errorf("objectstore/packed: idx %q has oversized 64-bit offset table", index.idxName) } + return nil } @@ -74,17 +85,21 @@ func (index *idxFile) lookup(id objectid.ObjectID) (uint64, bool, error) { if id.Algorithm() != index.algo { return 0, false, fmt.Errorf("objectstore/packed: object id algorithm mismatch") } + idBytes := (&id).RawBytes() + hashSize := len(idBytes) if hashSize != index.algo.Size() { return 0, false, fmt.Errorf("objectstore/packed: unexpected object id length") } first := int(idBytes[0]) + lo := 0 if first > 0 { lo = int(index.fanout[first-1]) } + hi := int(index.fanout[first]) if lo < 0 || hi < 0 || lo > hi || hi > index.numObjects { return 0, false, fmt.Errorf("objectstore/packed: idx %q has invalid fanout bounds", index.idxName) @@ -92,24 +107,29 @@ func (index *idxFile) lookup(id objectid.ObjectID) (uint64, bool, error) { for lo < hi { mid := lo + (hi-lo)/2 + nameOffset := index.namesOffset + mid*hashSize if nameOffset < 0 || nameOffset+hashSize > len(index.data) { return 0, false, fmt.Errorf("objectstore/packed: idx %q truncated name table", index.idxName) } + cmp := bytes.Compare(index.data[nameOffset:nameOffset+hashSize], idBytes) if cmp == 0 { offset, err := index.offsetAt(mid) if err != nil { return 0, false, err } + return offset, true, nil } + if cmp < 0 { lo = mid + 1 } else { hi = mid } } + return 0, false, nil } @@ -118,10 +138,12 @@ func (index *idxFile) offsetAt(objectIndex int) (uint64, error) { if objectIndex < 0 || objectIndex >= index.numObjects { return 0, fmt.Errorf("objectstore/packed: idx %q offset index out of bounds", index.idxName) } + wordOffset := index.offset32Offset + objectIndex*4 if wordOffset < 0 || wordOffset+4 > len(index.data) { return 0, fmt.Errorf("objectstore/packed: idx %q truncated 32-bit offset table", index.idxName) } + word := binary.BigEndian.Uint32(index.data[wordOffset : wordOffset+4]) if word&0x80000000 == 0 { return uint64(word), nil @@ -131,9 +153,11 @@ func (index *idxFile) offsetAt(objectIndex int) (uint64, error) { if pos < 0 || pos >= index.offset64Count { return 0, fmt.Errorf("objectstore/packed: idx %q invalid 64-bit offset position", index.idxName) } + offOffset := index.offset64Offset + pos*8 if offOffset < 0 || offOffset+8 > len(index.data)-2*index.algo.Size() { return 0, fmt.Errorf("objectstore/packed: idx %q truncated 64-bit offset table", index.idxName) } + return binary.BigEndian.Uint64(index.data[offOffset : offOffset+8]), nil } diff --git a/objectstore/packed/pack.go b/objectstore/packed/pack.go index 9af4c860..874b2b76 100644 --- a/objectstore/packed/pack.go +++ b/objectstore/packed/pack.go @@ -25,43 +25,58 @@ func openPackFile(name string, file *os.File, size int64) (*packFile, error) { if size < 12 { return nil, fmt.Errorf("objectstore/packed: pack %q too short", name) } + if size > int64(int(^uint(0)>>1)) { return nil, fmt.Errorf("objectstore/packed: pack %q has unsupported size", name) } + fd, err := intconv.UintptrToInt(file.Fd()) if err != nil { return nil, err } + data, err := syscall.Mmap(fd, 0, int(size), syscall.PROT_READ, syscall.MAP_PRIVATE) if err != nil { return nil, err } + if binary.BigEndian.Uint32(data[:4]) != packfmt.Signature { _ = syscall.Munmap(data) + return nil, fmt.Errorf("objectstore/packed: pack %q invalid signature", name) } + version := binary.BigEndian.Uint32(data[4:8]) if !packfmt.VersionSupported(version) { _ = syscall.Munmap(data) + return nil, fmt.Errorf("objectstore/packed: pack %q unsupported version %d", name, version) } + return &packFile{name: name, file: file, data: data}, nil } // close unmaps and closes one pack handle. func (pack *packFile) close() error { var closeErr error + if pack.data != nil { - if err := syscall.Munmap(pack.data); err != nil && closeErr == nil { + err := syscall.Munmap(pack.data) + if err != nil && closeErr == nil { closeErr = err } + pack.data = nil } + if pack.file != nil { - if err := pack.file.Close(); err != nil && closeErr == nil { + err := pack.file.Close() + if err != nil && closeErr == nil { closeErr = err } + pack.file = nil } + return closeErr } diff --git a/objectstore/packed/pack_idx_checksum.go b/objectstore/packed/pack_idx_checksum.go index 2f55a469..25556088 100644 --- a/objectstore/packed/pack_idx_checksum.go +++ b/objectstore/packed/pack_idx_checksum.go @@ -14,17 +14,21 @@ func verifyMappedPackMatchesMappedIdx(packData, idxData []byte, algo objectid.Al if hashSize <= 0 { return objectid.ErrInvalidAlgorithm } + if len(packData) < hashSize { return fmt.Errorf("objectstore/packed: pack too short for trailer hash") } + if len(idxData) < hashSize*2 { return fmt.Errorf("objectstore/packed: idx too short for trailer hashes") } packTrailerHash := packData[len(packData)-hashSize:] + idxPackHash := idxData[len(idxData)-hashSize*2 : len(idxData)-hashSize] if !bytes.Equal(packTrailerHash, idxPackHash) { return fmt.Errorf("objectstore/packed: pack hash does not match idx") } + return nil } diff --git a/objectstore/packed/read_bytes.go b/objectstore/packed/read_bytes.go index b6f42a0d..e272b626 100644 --- a/objectstore/packed/read_bytes.go +++ b/objectstore/packed/read_bytes.go @@ -14,6 +14,7 @@ func (store *Store) ReadBytesContent(id objectid.ObjectID) (objecttype.Type, []b if err != nil { return objecttype.TypeInvalid, nil, err } + return store.deltaResolveContent(loc) } @@ -23,12 +24,15 @@ func (store *Store) ReadBytesFull(id objectid.ObjectID) ([]byte, error) { if err != nil { return nil, err } + header, ok := objectheader.Encode(ty, int64(len(content))) if !ok { return nil, fmt.Errorf("objectstore/packed: failed to encode object header for type %d", ty) } + out := make([]byte, len(header)+len(content)) copy(out, header) copy(out[len(header):], content) + return out, nil } diff --git a/objectstore/packed/read_header.go b/objectstore/packed/read_header.go index 6822975c..5eb37c92 100644 --- a/objectstore/packed/read_header.go +++ b/objectstore/packed/read_header.go @@ -11,5 +11,6 @@ func (store *Store) ReadHeader(id objectid.ObjectID) (objecttype.Type, int64, er if err != nil { return objecttype.TypeInvalid, 0, err } + return store.resolveHeaderAt(loc) } diff --git a/objectstore/packed/read_header_resolve.go b/objectstore/packed/read_header_resolve.go index cf49fe2b..420d9363 100644 --- a/objectstore/packed/read_header_resolve.go +++ b/objectstore/packed/read_header_resolve.go @@ -17,12 +17,14 @@ func (store *Store) resolveHeaderAt(start location) (objecttype.Type, int64, err if _, ok := visited[current]; ok { return objecttype.TypeInvalid, 0, fmt.Errorf("objectstore/packed: delta cycle while resolving object header") } + visited[current] = struct{}{} pack, meta, err := store.entryMetaAt(current) if err != nil { return objecttype.TypeInvalid, 0, err } + if declaredSize < 0 { if packfmt.IsBaseObjectType(meta.ty) { declaredSize = meta.size @@ -31,9 +33,11 @@ func (store *Store) resolveHeaderAt(start location) (objecttype.Type, int64, err if err != nil { return objecttype.TypeInvalid, 0, err } + declaredSize = size } } + if packfmt.IsBaseObjectType(meta.ty) { return meta.ty, declaredSize, nil } @@ -44,6 +48,7 @@ func (store *Store) resolveHeaderAt(start location) (objecttype.Type, int64, err if err != nil { return objecttype.TypeInvalid, 0, err } + current = next case objecttype.TypeOfsDelta: current = location{ diff --git a/objectstore/packed/read_reader.go b/objectstore/packed/read_reader.go index a1f24799..d8dfdca9 100644 --- a/objectstore/packed/read_reader.go +++ b/objectstore/packed/read_reader.go @@ -41,11 +41,13 @@ func (store *Store) ReadReaderContent(id objectid.ObjectID) (objecttype.Type, in if err != nil { return objecttype.TypeInvalid, 0, nil, err } + if packfmt.IsBaseObjectType(meta.ty) { zr, err := zlibReaderAt(pack, meta.dataOffset) if err != nil { return objecttype.TypeInvalid, 0, nil, err } + return meta.ty, meta.size, &readCloser{ reader: iolimit.ExpectLengthReader(zr, meta.size), closer: zr, @@ -56,6 +58,7 @@ func (store *Store) ReadReaderContent(id objectid.ObjectID) (objecttype.Type, in if err != nil { return objecttype.TypeInvalid, 0, nil, err } + return ty, int64(len(content)), io.NopCloser(bytes.NewReader(content)), nil } @@ -72,15 +75,18 @@ func (store *Store) ReadReaderFull(id objectid.ObjectID) (io.ReadCloser, error) if err != nil { return nil, err } + if packfmt.IsBaseObjectType(meta.ty) { header, ok := objectheader.Encode(meta.ty, meta.size) if !ok { return nil, fmt.Errorf("objectstore/packed: failed to encode object header for type %d", meta.ty) } + zr, err := zlibReaderAt(pack, meta.dataOffset) if err != nil { return nil, err } + return &readCloser{ reader: io.MultiReader(bytes.NewReader(header), iolimit.ExpectLengthReader(zr, meta.size)), closer: zr, @@ -91,5 +97,6 @@ func (store *Store) ReadReaderFull(id objectid.ObjectID) (io.ReadCloser, error) if err != nil { return nil, err } + return io.NopCloser(bytes.NewReader(raw)), nil } diff --git a/objectstore/packed/read_size.go b/objectstore/packed/read_size.go index e162586a..a0a75db7 100644 --- a/objectstore/packed/read_size.go +++ b/objectstore/packed/read_size.go @@ -14,6 +14,7 @@ func (store *Store) ReadSize(id objectid.ObjectID) (int64, error) { if err != nil { return 0, err } + return store.resolveSizeAt(loc) } @@ -23,9 +24,11 @@ func (store *Store) resolveSizeAt(start location) (int64, error) { if err != nil { return 0, err } + if packfmt.IsBaseObjectType(meta.ty) { return meta.size, nil } + switch meta.ty { case objecttype.TypeRefDelta, objecttype.TypeOfsDelta: return deltaDeclaredSizeAt(pack, meta.dataOffset) diff --git a/objectstore/packed/read_test.go b/objectstore/packed/read_test.go index 9bfa6610..9ba89fdf 100644 --- a/objectstore/packed/read_test.go +++ b/objectstore/packed/read_test.go @@ -30,16 +30,20 @@ func TestPackedStoreReadAgainstGit(t *testing.T) { if err != nil { t.Fatalf("ReadHeader: %v", err) } + if gotHeaderType != wantType { t.Fatalf("ReadHeader type = %v, want %v", gotHeaderType, wantType) } + if gotHeaderSize != int64(len(wantBody)) { t.Fatalf("ReadHeader size = %d, want %d", gotHeaderSize, len(wantBody)) } + gotSize, err := store.ReadSize(id) if err != nil { t.Fatalf("ReadSize: %v", err) } + if gotSize != int64(len(wantBody)) { t.Fatalf("ReadSize = %d, want %d", gotSize, len(wantBody)) } @@ -48,6 +52,7 @@ func TestPackedStoreReadAgainstGit(t *testing.T) { if err != nil { t.Fatalf("ReadBytesFull: %v", err) } + if !bytes.Equal(gotRaw, wantRaw) { t.Fatalf("ReadBytesFull mismatch") } @@ -56,9 +61,11 @@ func TestPackedStoreReadAgainstGit(t *testing.T) { if err != nil { t.Fatalf("ReadBytesContent: %v", err) } + if gotType != wantType { t.Fatalf("ReadBytesContent type = %v, want %v", gotType, wantType) } + if !bytes.Equal(gotBody, wantBody) { t.Fatalf("ReadBytesContent mismatch") } @@ -67,7 +74,9 @@ func TestPackedStoreReadAgainstGit(t *testing.T) { if err != nil { t.Fatalf("ReadReaderFull: %v", err) } - if got := mustReadAllAndClose(t, fullReader); !bytes.Equal(got, wantRaw) { + + got := mustReadAllAndClose(t, fullReader) + if !bytes.Equal(got, wantRaw) { t.Fatalf("ReadReaderFull mismatch") } @@ -75,13 +84,17 @@ func TestPackedStoreReadAgainstGit(t *testing.T) { if err != nil { t.Fatalf("ReadReaderContent: %v", err) } + if contentType != wantType { t.Fatalf("ReadReaderContent type = %v, want %v", contentType, wantType) } + if contentSize != int64(len(wantBody)) { t.Fatalf("ReadReaderContent size = %d, want %d", contentSize, len(wantBody)) } - if got := mustReadAllAndClose(t, contentReader); !bytes.Equal(got, wantBody) { + + got = mustReadAllAndClose(t, contentReader) + if !bytes.Equal(got, wantBody) { t.Fatalf("ReadReaderContent mismatch") } }) @@ -100,38 +113,54 @@ func TestPackedStoreErrors(t *testing.T) { t.Fatalf("ParseHex(notFound): %v", err) } - if _, err := store.ReadBytesFull(notFoundID); !errors.Is(err, objectstore.ErrObjectNotFound) { + _, err = store.ReadBytesFull(notFoundID) + if !errors.Is(err, objectstore.ErrObjectNotFound) { t.Fatalf("ReadBytesFull not-found error = %v", err) } - if _, _, err := store.ReadBytesContent(notFoundID); !errors.Is(err, objectstore.ErrObjectNotFound) { + + _, _, err = store.ReadBytesContent(notFoundID) + if !errors.Is(err, objectstore.ErrObjectNotFound) { t.Fatalf("ReadBytesContent not-found error = %v", err) } - if _, err := store.ReadReaderFull(notFoundID); !errors.Is(err, objectstore.ErrObjectNotFound) { + + _, err = store.ReadReaderFull(notFoundID) + if !errors.Is(err, objectstore.ErrObjectNotFound) { t.Fatalf("ReadReaderFull not-found error = %v", err) } - if _, _, _, err := store.ReadReaderContent(notFoundID); !errors.Is(err, objectstore.ErrObjectNotFound) { + + _, _, _, err = store.ReadReaderContent(notFoundID) + if !errors.Is(err, objectstore.ErrObjectNotFound) { t.Fatalf("ReadReaderContent not-found error = %v", err) } - if _, _, err := store.ReadHeader(notFoundID); !errors.Is(err, objectstore.ErrObjectNotFound) { + + _, _, err = store.ReadHeader(notFoundID) + if !errors.Is(err, objectstore.ErrObjectNotFound) { t.Fatalf("ReadHeader not-found error = %v", err) } - if _, err := store.ReadSize(notFoundID); !errors.Is(err, objectstore.ErrObjectNotFound) { + + _, err = store.ReadSize(notFoundID) + if !errors.Is(err, objectstore.ErrObjectNotFound) { t.Fatalf("ReadSize not-found error = %v", err) } var otherAlgo objectid.Algorithm + for _, candidate := range objectid.SupportedAlgorithms() { if candidate != algo { otherAlgo = candidate + break } } + if otherAlgo != objectid.AlgorithmUnknown { mismatchID, err := objectid.ParseHex(otherAlgo, strings.Repeat("0", otherAlgo.HexLen())) if err != nil { t.Fatalf("ParseHex(mismatch): %v", err) } - if _, err := store.ReadBytesFull(mismatchID); err == nil || !strings.Contains(err.Error(), "algorithm mismatch") { + + _, err = store.ReadBytesFull(mismatchID) + if err == nil || !strings.Contains(err.Error(), "algorithm mismatch") { t.Fatalf("ReadBytesFull algorithm-mismatch error = %v", err) } } @@ -141,11 +170,16 @@ func TestPackedStoreErrors(t *testing.T) { func TestPackedStoreNewValidation(t *testing.T) { t.Parallel() testRepo, _ := createPackedFixtureRepo(t, objectid.AlgorithmSHA1) + store := openPackedStore(t, testRepo.Dir(), objectid.AlgorithmSHA1) - if err := store.Close(); err != nil { + + err := store.Close() + if err != nil { t.Fatalf("Close: %v", err) } - if err := store.Close(); err != nil { + + err = store.Close() + if err != nil { t.Fatalf("Close second: %v", err) } } @@ -153,13 +187,16 @@ func TestPackedStoreNewValidation(t *testing.T) { func TestPackedStoreInvalidAlgorithm(t *testing.T) { t.Parallel() testRepo := testgit.NewRepo(t, testgit.RepoOptions{ObjectFormat: objectid.AlgorithmSHA1, Bare: true}) + root, err := os.OpenRoot(testRepo.Dir()) if err != nil { t.Fatalf("OpenRoot(%q): %v", testRepo.Dir(), err) } + t.Cleanup(func() { _ = root.Close() }) - if _, err := packed.New(root, objectid.AlgorithmUnknown); !errors.Is(err, objectid.ErrInvalidAlgorithm) { + _, err = packed.New(root, objectid.AlgorithmUnknown) + if !errors.Is(err, objectid.ErrInvalidAlgorithm) { t.Fatalf("packed.New invalid algorithm error = %v", err) } } @@ -170,15 +207,20 @@ func TestPackedStoreReadHeaderUsesResolvedObjectSizeForDelta(t *testing.T) { testRepo := testgit.NewRepo(t, testgit.RepoOptions{ObjectFormat: algo, Bare: true}) var parent objectid.ObjectID + for i := range 96 { content := strings.Repeat("common-line-"+strconv.Itoa(i%7)+"\n", 384) + fmt.Sprintf("tail-%03d\n", i) + _, treeID := testRepo.MakeSingleFileTree(t, "file.txt", []byte(content)) if i == 0 { parent = testRepo.CommitTree(t, treeID, "delta-header-size-0") + continue } + parent = testRepo.CommitTree(t, treeID, fmt.Sprintf("delta-header-size-%03d", i), parent) } + testRepo.UpdateRef(t, "refs/heads/main", parent) testRepo.Repack(t, "-a", "-d", "-f", "--window=128", "--depth=128") @@ -189,13 +231,16 @@ func TestPackedStoreReadHeaderUsesResolvedObjectSizeForDelta(t *testing.T) { if err != nil { t.Fatalf("ReadHeader(%s): %v", deltaID, err) } + if gotSize != wantResolvedSize { t.Fatalf("ReadHeader(%s) size = %d, want resolved size %d", deltaID, gotSize, wantResolvedSize) } + gotReadSize, err := store.ReadSize(deltaID) if err != nil { t.Fatalf("ReadSize(%s): %v", deltaID, err) } + if gotReadSize != wantResolvedSize { t.Fatalf("ReadSize(%s) = %d, want resolved size %d", deltaID, gotReadSize, wantResolvedSize) } @@ -209,6 +254,7 @@ func findDeltaObjectWithResolvedSizeMismatch(t *testing.T, testRepo *testgit.Tes if err != nil { t.Fatalf("Glob idx: %v", err) } + if len(idxFiles) == 0 { t.Fatalf("no idx files found") } @@ -221,16 +267,19 @@ func findDeltaObjectWithResolvedSizeMismatch(t *testing.T, testRepo *testgit.Tes } idHex := fields[0] + deltaStreamSize, err := strconv.ParseInt(fields[2], 10, 64) if err != nil { continue } resolvedSizeStr := testRepo.Run(t, "cat-file", "-s", idHex) + resolvedSize, err := strconv.ParseInt(strings.TrimSpace(resolvedSizeStr), 10, 64) if err != nil { t.Fatalf("parse cat-file size for %s: %v", idHex, err) } + if deltaStreamSize == resolvedSize { continue } @@ -239,9 +288,11 @@ func findDeltaObjectWithResolvedSizeMismatch(t *testing.T, testRepo *testgit.Tes if err != nil { t.Fatalf("ParseHex(%s): %v", idHex, err) } + return id, resolvedSize } t.Fatalf("did not find a delta object with mismatched stream/resolved size") + return objectid.ObjectID{}, 0 } diff --git a/objectstore/packed/store.go b/objectstore/packed/store.go index abd7175f..d28113d1 100644 --- a/objectstore/packed/store.go +++ b/objectstore/packed/store.go @@ -60,6 +60,7 @@ func New(root *os.Root, algo objectid.Algorithm) (*Store, error) { if algo.Size() == 0 { return nil, objectid.ErrInvalidAlgorithm } + return &Store{ root: root, algo: algo, @@ -76,8 +77,10 @@ func (store *Store) Close() error { store.stateMu.Lock() if store.closed { store.stateMu.Unlock() + return nil } + store.closed = true root := store.root packs := store.packs @@ -87,23 +90,30 @@ func (store *Store) Close() error { store.idxMu.RUnlock() var closeErr error + for _, pack := range packs { - if err := pack.close(); err != nil && closeErr == nil { + err := pack.close() + if err != nil && closeErr == nil { closeErr = err } } + for _, index := range indexes { - if err := index.close(); err != nil && closeErr == nil { + err := index.close() + if err != nil && closeErr == nil { closeErr = err } } + store.cacheMu.Lock() store.deltaCache.clear() store.cacheMu.Unlock() - if err := root.Close(); err != nil && closeErr == nil { + err := root.Close() + if err != nil && closeErr == nil { closeErr = err } + return closeErr } @@ -113,7 +123,9 @@ func (store *Store) lookup(id objectid.ObjectID) (location, error) { if id.Algorithm() != store.algo { return zero, errors.New("objectstore/packed: object id algorithm mismatch") } - if err := store.ensureCandidates(); err != nil { + + err := store.ensureCandidates() + if err != nil { return zero, err } @@ -122,81 +134,111 @@ func (store *Store) lookup(id objectid.ObjectID) (location, error) { candidate, ok := store.candidateForPack(nextPackName) if !ok { nextPackName = store.firstCandidatePackName() + continue } + nextPackName = store.nextCandidatePackName(candidate.packName) + index, err := store.openIndex(candidate) if err != nil { return zero, err } + offset, ok, err := index.lookup(id) if err != nil { return zero, err } + if ok { store.touchCandidate(candidate.packName) + return location{packName: index.packName, offset: offset}, nil } } + return zero, objectstore.ErrObjectNotFound } // openPack returns one opened and validated pack handle. func (store *Store) openPack(name string) (*packFile, error) { store.stateMu.RLock() - if pack, ok := store.packs[name]; ok { + + pack, ok := store.packs[name] + if ok { store.stateMu.RUnlock() + return pack, nil } + store.stateMu.RUnlock() file, err := store.root.Open(name) if err != nil { return nil, err } + info, err := file.Stat() if err != nil { _ = file.Close() + return nil, err } - pack, err := openPackFile(name, file, info.Size()) + + pack, err = openPackFile(name, file, info.Size()) if err != nil { _ = file.Close() + return nil, err } - if err := store.verifyPackMatchesIndexes(pack); err != nil { + + err = store.verifyPackMatchesIndexes(pack) + if err != nil { _ = pack.close() + return nil, err } store.stateMu.Lock() - if existing, ok := store.packs[name]; ok { + + existing, ok := store.packs[name] + if ok { store.stateMu.Unlock() + _ = pack.close() + return existing, nil } + store.packs[name] = pack store.stateMu.Unlock() + return pack, nil } // verifyPackMatchesIndexes checks that one opened pack's trailer hash matches // every loaded index that references the same pack name. func (store *Store) verifyPackMatchesIndexes(pack *packFile) error { - if err := store.ensureCandidates(); err != nil { + err := store.ensureCandidates() + if err != nil { return err } + candidate, ok := store.candidateForPack(pack.name) if !ok { return fmt.Errorf("objectstore/packed: missing index for pack %q", pack.name) } + index, err := store.openIndex(candidate) if err != nil { return err } - if err := verifyMappedPackMatchesMappedIdx(pack.data, index.data, store.algo); err != nil { + + err = verifyMappedPackMatchesMappedIdx(pack.data, index.data, store.algo) + if err != nil { return fmt.Errorf("objectstore/packed: pack %q does not match idx %q: %w", pack.name, index.idxName, err) } + return nil } @@ -206,9 +248,11 @@ func (store *Store) entryMetaAt(loc location) (*packFile, entryMeta, error) { if err != nil { return nil, entryMeta{}, err } + meta, err := parseEntryMeta(pack, store.algo, loc.offset) if err != nil { return nil, entryMeta{}, err } + return pack, meta, nil } |
