aboutsummaryrefslogtreecommitdiff
path: root/objectstore/packed
diff options
context:
space:
mode:
authorGravatar Runxi Yu2026-03-04 08:26:56 +0800
committerGravatar Runxi Yu2026-03-04 08:59:53 +0800
commitab7501be34032fb9e5c48726a68ae90a917af9eb (patch)
tree20d005647569befea8133e953c3270e8fd2a2a5b /objectstore/packed
parent*: gofumpt (diff)
signatureNo signature
*: Lint
Diffstat (limited to 'objectstore/packed')
-rw-r--r--objectstore/packed/delta_apply.go13
-rw-r--r--objectstore/packed/delta_cache.go1
-rw-r--r--objectstore/packed/delta_plan.go7
-rw-r--r--objectstore/packed/entry_inflate.go8
-rw-r--r--objectstore/packed/entry_parse.go4
-rw-r--r--objectstore/packed/helpers_test.go17
-rw-r--r--objectstore/packed/idx_lookup_candidates.go24
-rw-r--r--objectstore/packed/idx_open.go40
-rw-r--r--objectstore/packed/idx_parse.go24
-rw-r--r--objectstore/packed/pack.go19
-rw-r--r--objectstore/packed/pack_idx_checksum.go4
-rw-r--r--objectstore/packed/read_bytes.go4
-rw-r--r--objectstore/packed/read_header.go1
-rw-r--r--objectstore/packed/read_header_resolve.go5
-rw-r--r--objectstore/packed/read_reader.go7
-rw-r--r--objectstore/packed/read_size.go3
-rw-r--r--objectstore/packed/read_test.go75
-rw-r--r--objectstore/packed/store.go64
18 files changed, 287 insertions, 33 deletions
diff --git a/objectstore/packed/delta_apply.go b/objectstore/packed/delta_apply.go
index 5245e0ba..71f09ead 100644
--- a/objectstore/packed/delta_apply.go
+++ b/objectstore/packed/delta_apply.go
@@ -14,10 +14,12 @@ func (store *Store) deltaResolveContent(start location) (objecttype.Type, []byte
if err != nil {
return objecttype.TypeInvalid, nil, err
}
+
pack, meta, err := store.entryMetaAt(start)
if err != nil {
return objecttype.TypeInvalid, nil, err
}
+
declaredSize := meta.size
if !packfmt.IsBaseObjectType(meta.ty) {
declaredSize, err = deltaDeclaredSizeAt(pack, meta.dataOffset)
@@ -25,6 +27,7 @@ func (store *Store) deltaResolveContent(start location) (objecttype.Type, []byte
return objecttype.TypeInvalid, nil, err
}
}
+
return store.deltaResolveChain(chain, declaredSize)
}
@@ -37,18 +40,22 @@ func (store *Store) deltaResolveChain(chain deltaChain, declaredSize int64) (obj
for i := nextDelta; i >= 0; i-- {
node := chain.deltas[i]
+
pack, err := store.openPack(node.loc.packName)
if err != nil {
return objecttype.TypeInvalid, nil, err
}
+
delta, err := inflateAt(pack, node.dataOffset, -1)
if err != nil {
return objecttype.TypeInvalid, nil, err
}
+
out, err = deltaapply.Apply(out, delta)
if err != nil {
return objecttype.TypeInvalid, nil, err
}
+
store.cacheMu.Lock()
store.deltaCache.add(
deltaBaseKey{packName: node.loc.packName, offset: node.loc.offset},
@@ -65,6 +72,7 @@ func (store *Store) deltaResolveChain(chain deltaChain, declaredSize int64) (obj
declaredSize,
)
}
+
if ty != chain.baseType {
return objecttype.TypeInvalid, nil, fmt.Errorf(
"objectstore/packed: resolved content type mismatch: got %d want %d",
@@ -72,6 +80,7 @@ func (store *Store) deltaResolveChain(chain deltaChain, declaredSize int64) (obj
chain.baseType,
)
}
+
return ty, out, nil
}
@@ -85,6 +94,7 @@ func (store *Store) deltaResolveChainStart(chain deltaChain) (objecttype.Type, [
deltaBaseKey{packName: node.loc.packName, offset: node.loc.offset},
)
store.cacheMu.RUnlock()
+
if ok {
return ty, out, i - 1, nil
}
@@ -95,6 +105,7 @@ func (store *Store) deltaResolveChainStart(chain deltaChain) (objecttype.Type, [
deltaBaseKey{packName: chain.baseLoc.packName, offset: chain.baseLoc.offset},
)
store.cacheMu.RUnlock()
+
if ok {
return ty, out, len(chain.deltas) - 1, nil
}
@@ -103,9 +114,11 @@ func (store *Store) deltaResolveChainStart(chain deltaChain) (objecttype.Type, [
if err != nil {
return objecttype.TypeInvalid, nil, 0, err
}
+
if !packfmt.IsBaseObjectType(meta.ty) {
return objecttype.TypeInvalid, nil, 0, fmt.Errorf("objectstore/packed: delta chain base is not a base object")
}
+
base, err := inflateAt(pack, meta.dataOffset, meta.size)
if err != nil {
return objecttype.TypeInvalid, nil, 0, err
diff --git a/objectstore/packed/delta_cache.go b/objectstore/packed/delta_cache.go
index add21698..a911b254 100644
--- a/objectstore/packed/delta_cache.go
+++ b/objectstore/packed/delta_cache.go
@@ -41,6 +41,7 @@ func (cache *deltaCache) get(key deltaBaseKey) (objecttype.Type, []byte, bool) {
if !ok {
return objecttype.TypeInvalid, nil, false
}
+
return value.ty, append([]byte(nil), value.content...), true
}
diff --git a/objectstore/packed/delta_plan.go b/objectstore/packed/delta_plan.go
index 5f2ae959..b0b0324c 100644
--- a/objectstore/packed/delta_plan.go
+++ b/objectstore/packed/delta_plan.go
@@ -38,6 +38,7 @@ func (store *Store) deltaBuildChain(start location) (deltaChain, error) {
if _, ok := visited[current]; ok {
return deltaChain{}, fmt.Errorf("objectstore/packed: delta cycle while resolving object")
}
+
visited[current] = struct{}{}
_, meta, err := store.entryMetaAt(current)
@@ -48,6 +49,7 @@ func (store *Store) deltaBuildChain(start location) (deltaChain, error) {
if packfmt.IsBaseObjectType(meta.ty) {
chain.baseLoc = current
chain.baseType = meta.ty
+
return chain, nil
}
@@ -57,10 +59,12 @@ func (store *Store) deltaBuildChain(start location) (deltaChain, error) {
loc: current,
dataOffset: meta.dataOffset,
})
+
next, err := store.lookup(meta.baseRefID)
if err != nil {
return deltaChain{}, err
}
+
current = next
case objecttype.TypeOfsDelta:
chain.deltas = append(chain.deltas, deltaNode{
@@ -88,12 +92,15 @@ func deltaDeclaredSizeAt(pack *packFile, dataOffset int) (int64, error) {
if err != nil {
return 0, err
}
+
defer func() { _ = reader.Close() }()
br := bufio.NewReaderSize(reader, 32)
+
_, size, err := deltaapply.ReadHeaderSizes(br)
if err != nil {
return 0, err
}
+
return int64(size), nil
}
diff --git a/objectstore/packed/entry_inflate.go b/objectstore/packed/entry_inflate.go
index 4f91710e..cbdb6a89 100644
--- a/objectstore/packed/entry_inflate.go
+++ b/objectstore/packed/entry_inflate.go
@@ -14,6 +14,7 @@ func zlibReaderAt(pack *packFile, offset int) (io.ReadCloser, error) {
if offset < 0 || offset > len(pack.data) {
return nil, fmt.Errorf("objectstore/packed: pack %q zlib offset out of bounds", pack.name)
}
+
return zlib.NewReader(bytes.NewReader(pack.data[offset:]))
}
@@ -23,6 +24,7 @@ func inflateAt(pack *packFile, offset int, expectedSize int64) ([]byte, error) {
if err != nil {
return nil, err
}
+
defer func() { _ = reader.Close() }()
if expectedSize >= 0 {
@@ -35,9 +37,12 @@ func inflateAt(pack *packFile, offset int, expectedSize int64) ([]byte, error) {
}
body := make([]byte, int(expectedSize))
- if _, err := io.ReadFull(reader, body); err != nil {
+
+ _, err := io.ReadFull(reader, body)
+ if err != nil {
return nil, err
}
+
return body, nil
}
@@ -45,5 +50,6 @@ func inflateAt(pack *packFile, offset int, expectedSize int64) ([]byte, error) {
if err != nil {
return nil, err
}
+
return body, nil
}
diff --git a/objectstore/packed/entry_parse.go b/objectstore/packed/entry_parse.go
index 56287386..7af20af1 100644
--- a/objectstore/packed/entry_parse.go
+++ b/objectstore/packed/entry_parse.go
@@ -34,6 +34,7 @@ func parseEntryMeta(pack *packFile, algo objectid.Algorithm, offset uint64) (ent
if err != nil {
return zero, fmt.Errorf("objectstore/packed: pack %q offset conversion: %w", pack.name, err)
}
+
entry, err := packfmt.ParseEntry(pack.data[pos:], algo.Size())
if err != nil {
return zero, fmt.Errorf("objectstore/packed: pack %q: %w", pack.name, err)
@@ -50,11 +51,13 @@ func parseEntryMeta(pack *packFile, algo objectid.Algorithm, offset uint64) (ent
if err != nil {
return zero, fmt.Errorf("objectstore/packed: pack %q invalid ref-delta base id: %w", pack.name, err)
}
+
meta.baseRefID = baseID
case objecttype.TypeOfsDelta:
if offset <= entry.OfsBaseDistance {
return zero, fmt.Errorf("objectstore/packed: pack %q has invalid ofs-delta base", pack.name)
}
+
meta.baseOfs = offset - entry.OfsBaseDistance
case objecttype.TypeCommit, objecttype.TypeTree, objecttype.TypeBlob, objecttype.TypeTag:
// Base object types do not have delta base metadata.
@@ -63,5 +66,6 @@ func parseEntryMeta(pack *packFile, algo objectid.Algorithm, offset uint64) (ent
default:
return zero, fmt.Errorf("objectstore/packed: pack %q has unsupported entry type %d", pack.name, meta.ty)
}
+
return meta, nil
}
diff --git a/objectstore/packed/helpers_test.go b/objectstore/packed/helpers_test.go
index f8cbd439..1b517294 100644
--- a/objectstore/packed/helpers_test.go
+++ b/objectstore/packed/helpers_test.go
@@ -18,30 +18,39 @@ import (
func openPackedStore(t *testing.T, repoPath string, algo objectid.Algorithm) *packed.Store {
t.Helper()
+
packPath := filepath.Join(repoPath, "objects", "pack")
+
root, err := os.OpenRoot(packPath)
if err != nil {
t.Fatalf("OpenRoot(%q): %v", packPath, err)
}
+
t.Cleanup(func() { _ = root.Close() })
store, err := packed.New(root, algo)
if err != nil {
t.Fatalf("packed.New: %v", err)
}
+
return store
}
func mustReadAllAndClose(t *testing.T, reader io.ReadCloser) []byte {
t.Helper()
+
data, err := io.ReadAll(reader)
if err != nil {
_ = reader.Close()
+
t.Fatalf("ReadAll: %v", err)
}
- if err := reader.Close(); err != nil {
+
+ err = reader.Close()
+ if err != nil {
t.Fatalf("Close: %v", err)
}
+
return data
}
@@ -49,11 +58,14 @@ func expectedRawObject(t *testing.T, testRepo *testgit.TestRepo, id objectid.Obj
t.Helper()
typeName := testRepo.Run(t, "cat-file", "-t", id.String())
+
ty, ok := objecttype.ParseName(typeName)
if !ok {
t.Fatalf("ParseName(%q) failed", typeName)
}
+
body := testRepo.CatFile(t, typeName, id)
+
header, ok := objectheader.Encode(ty, int64(len(body)))
if !ok {
t.Fatalf("objectheader.Encode failed")
@@ -62,6 +74,7 @@ func expectedRawObject(t *testing.T, testRepo *testgit.TestRepo, id objectid.Obj
raw := make([]byte, len(header)+len(body))
copy(raw, header)
copy(raw[len(header):], body)
+
return ty, body, raw
}
@@ -74,6 +87,7 @@ func createPackedFixtureRepo(t *testing.T, algo objectid.Algorithm) (*testgit.Te
tagID := testRepo.TagAnnotated(t, "v1.0.0", commitID, "packed-store-tag")
parent := commitID
+
for i := range 24 {
content := "common-prefix\n" + strings.Repeat("line-"+strconv.Itoa(i%3)+"\n", 256) + fmt.Sprintf("tail-%d\n", i)
nextBlob, nextTree := testRepo.MakeSingleFileTree(t, fmt.Sprintf("file-%02d.txt", i), []byte(content))
@@ -86,6 +100,7 @@ func createPackedFixtureRepo(t *testing.T, algo objectid.Algorithm) (*testgit.Te
}
testRepo.Repack(t, "-a", "-d", "-f", "--window=64", "--depth=64")
+
return testRepo, []objectid.ObjectID{
blobID,
treeID,
diff --git a/objectstore/packed/idx_lookup_candidates.go b/objectstore/packed/idx_lookup_candidates.go
index 83055aac..72121b25 100644
--- a/objectstore/packed/idx_lookup_candidates.go
+++ b/objectstore/packed/idx_lookup_candidates.go
@@ -37,8 +37,11 @@ func (store *Store) ensureCandidates() error {
candidateByPack := make(map[string]packCandidate, len(candidates))
nodeByPack := make(map[string]*packCandidateNode, len(candidates))
- var head *packCandidateNode
- var tail *packCandidateNode
+ var (
+ head *packCandidateNode
+ tail *packCandidateNode
+ )
+
for _, candidate := range candidates {
node := &packCandidateNode{
candidate: candidate,
@@ -47,9 +50,11 @@ func (store *Store) ensureCandidates() error {
if tail != nil {
tail.next = node
}
+
if head == nil {
head = node
}
+
tail = node
candidateByPack[candidate.packName] = candidate
nodeByPack[candidate.packName] = node
@@ -67,6 +72,7 @@ func (store *Store) ensureCandidates() error {
store.candidatesMu.RLock()
err := store.discoverErr
store.candidatesMu.RUnlock()
+
return err
}
@@ -78,8 +84,10 @@ func (store *Store) discoverCandidates() ([]packCandidate, error) {
if os.IsNotExist(err) {
return nil, nil
}
+
return nil, err
}
+
defer func() { _ = dir.Close() }()
entries, err := dir.ReadDir(-1)
@@ -95,11 +103,13 @@ func (store *Store) discoverCandidates() ([]packCandidate, error) {
idxName := entry.Name()
packName := strings.TrimSuffix(idxName, ".idx") + ".pack"
+
packInfo, err := store.root.Stat(packName)
if err != nil {
if os.IsNotExist(err) {
return nil, fmt.Errorf("objectstore/packed: missing pack file for index %q", idxName)
}
+
return nil, err
}
@@ -115,8 +125,10 @@ func (store *Store) discoverCandidates() ([]packCandidate, error) {
if a.mtime > b.mtime {
return -1
}
+
return 1
}
+
return strings.Compare(a.packName, b.packName)
})
@@ -139,18 +151,22 @@ func (store *Store) touchCandidate(packName string) {
if node.prev != nil {
node.prev.next = node.next
}
+
if node.next != nil {
node.next.prev = node.prev
}
+
if store.candidateTail == node {
store.candidateTail = node.prev
}
node.prev = nil
+
node.next = store.candidateHead
if store.candidateHead != nil {
store.candidateHead.prev = node
}
+
store.candidateHead = node
if store.candidateTail == nil {
store.candidateTail = node
@@ -162,9 +178,11 @@ func (store *Store) touchCandidate(packName string) {
func (store *Store) firstCandidatePackName() string {
store.candidatesMu.RLock()
defer store.candidatesMu.RUnlock()
+
if store.candidateHead == nil {
return ""
}
+
return store.candidateHead.candidate.packName
}
@@ -173,9 +191,11 @@ func (store *Store) firstCandidatePackName() string {
func (store *Store) nextCandidatePackName(currentPack string) string {
store.candidatesMu.RLock()
defer store.candidatesMu.RUnlock()
+
node := store.candidateNodeByPack[currentPack]
if node == nil || node.next == nil {
return ""
}
+
return node.next.candidate.packName
}
diff --git a/objectstore/packed/idx_open.go b/objectstore/packed/idx_open.go
index c00a7bac..c3c97e4d 100644
--- a/objectstore/packed/idx_open.go
+++ b/objectstore/packed/idx_open.go
@@ -43,16 +43,21 @@ func (store *Store) candidateForPack(packName string) (packCandidate, bool) {
store.candidatesMu.RLock()
candidate, ok := store.candidateByPack[packName]
store.candidatesMu.RUnlock()
+
return candidate, ok
}
// openIndex returns one opened and parsed index, caching it by pack basename.
func (store *Store) openIndex(candidate packCandidate) (*idxFile, error) {
store.idxMu.RLock()
- if index, ok := store.idxByPack[candidate.packName]; ok {
+
+ index, ok := store.idxByPack[candidate.packName]
+ if ok {
store.idxMu.RUnlock()
+
return index, nil
}
+
store.idxMu.RUnlock()
index, err := openIdxFile(store.root, candidate.idxName, candidate.packName, store.algo)
@@ -61,13 +66,19 @@ func (store *Store) openIndex(candidate packCandidate) (*idxFile, error) {
}
store.idxMu.Lock()
- if existing, ok := store.idxByPack[candidate.packName]; ok {
+
+ existing, ok := store.idxByPack[candidate.packName]
+ if ok {
store.idxMu.Unlock()
+
_ = index.close()
+
return existing, nil
}
+
store.idxByPack[candidate.packName] = index
store.idxMu.Unlock()
+
return index, nil
}
@@ -77,24 +88,32 @@ func openIdxFile(root *os.Root, idxName, packName string, algo objectid.Algorith
if err != nil {
return nil, err
}
+
info, err := file.Stat()
if err != nil {
_ = file.Close()
+
return nil, err
}
+
size := info.Size()
if size < 0 || size > int64(int(^uint(0)>>1)) {
_ = file.Close()
+
return nil, fmt.Errorf("objectstore/packed: idx %q has unsupported size", idxName)
}
+
fd, err := intconv.UintptrToInt(file.Fd())
if err != nil {
_ = file.Close()
+
return nil, err
}
+
data, err := syscall.Mmap(fd, 0, int(size), syscall.PROT_READ, syscall.MAP_PRIVATE)
if err != nil {
_ = file.Close()
+
return nil, err
}
@@ -105,27 +124,38 @@ func openIdxFile(root *os.Root, idxName, packName string, algo objectid.Algorith
file: file,
data: data,
}
- if err := index.parse(); err != nil {
+
+ err = index.parse()
+ if err != nil {
_ = index.close()
+
return nil, err
}
+
return index, nil
}
// close unmaps and closes one idx handle.
func (index *idxFile) close() error {
var closeErr error
+
if index.data != nil {
- if err := syscall.Munmap(index.data); err != nil && closeErr == nil {
+ err := syscall.Munmap(index.data)
+ if err != nil && closeErr == nil {
closeErr = err
}
+
index.data = nil
}
+
if index.file != nil {
- if err := index.file.Close(); err != nil && closeErr == nil {
+ err := index.file.Close()
+ if err != nil && closeErr == nil {
closeErr = err
}
+
index.file = nil
}
+
return closeErr
}
diff --git a/objectstore/packed/idx_parse.go b/objectstore/packed/idx_parse.go
index 0af72594..870ffdae 100644
--- a/objectstore/packed/idx_parse.go
+++ b/objectstore/packed/idx_parse.go
@@ -19,27 +19,34 @@ func (index *idxFile) parse() error {
if hashSize <= 0 {
return fmt.Errorf("objectstore/packed: idx %q has invalid hash algorithm", index.idxName)
}
+
minLen := 8 + 256*4 + 2*hashSize
if len(index.data) < minLen {
return fmt.Errorf("objectstore/packed: idx %q too short", index.idxName)
}
+
if binary.BigEndian.Uint32(index.data[:4]) != idxMagicV2 {
return fmt.Errorf("objectstore/packed: idx %q invalid magic", index.idxName)
}
+
if binary.BigEndian.Uint32(index.data[4:8]) != idxVersionV2 {
return fmt.Errorf("objectstore/packed: idx %q unsupported version", index.idxName)
}
prev := uint32(0)
+
for i := range 256 {
base := 8 + i*4
+
cur := binary.BigEndian.Uint32(index.data[base : base+4])
if cur < prev {
return fmt.Errorf("objectstore/packed: idx %q has non-monotonic fanout table", index.idxName)
}
+
index.fanout[i] = cur
prev = cur
}
+
index.numObjects = int(index.fanout[255])
if index.numObjects < 0 {
return fmt.Errorf("objectstore/packed: idx %q has invalid object count", index.idxName)
@@ -48,6 +55,7 @@ func (index *idxFile) parse() error {
namesBytes := index.numObjects * hashSize
crcBytes := index.numObjects * 4
offset32Bytes := index.numObjects * 4
+
minSize := 8 + 256*4 + namesBytes + crcBytes + offset32Bytes + 2*hashSize
if minSize < 0 || len(index.data) < minSize {
return fmt.Errorf("objectstore/packed: idx %q has truncated tables", index.idxName)
@@ -61,11 +69,14 @@ func (index *idxFile) parse() error {
if offset64Bytes < 0 || offset64Bytes%8 != 0 {
return fmt.Errorf("objectstore/packed: idx %q has malformed 64-bit offset table", index.idxName)
}
+
index.offset64Count = offset64Bytes / 8
+
maxOffset64Count := max(index.numObjects-1, 0)
if index.offset64Count > maxOffset64Count {
return fmt.Errorf("objectstore/packed: idx %q has oversized 64-bit offset table", index.idxName)
}
+
return nil
}
@@ -74,17 +85,21 @@ func (index *idxFile) lookup(id objectid.ObjectID) (uint64, bool, error) {
if id.Algorithm() != index.algo {
return 0, false, fmt.Errorf("objectstore/packed: object id algorithm mismatch")
}
+
idBytes := (&id).RawBytes()
+
hashSize := len(idBytes)
if hashSize != index.algo.Size() {
return 0, false, fmt.Errorf("objectstore/packed: unexpected object id length")
}
first := int(idBytes[0])
+
lo := 0
if first > 0 {
lo = int(index.fanout[first-1])
}
+
hi := int(index.fanout[first])
if lo < 0 || hi < 0 || lo > hi || hi > index.numObjects {
return 0, false, fmt.Errorf("objectstore/packed: idx %q has invalid fanout bounds", index.idxName)
@@ -92,24 +107,29 @@ func (index *idxFile) lookup(id objectid.ObjectID) (uint64, bool, error) {
for lo < hi {
mid := lo + (hi-lo)/2
+
nameOffset := index.namesOffset + mid*hashSize
if nameOffset < 0 || nameOffset+hashSize > len(index.data) {
return 0, false, fmt.Errorf("objectstore/packed: idx %q truncated name table", index.idxName)
}
+
cmp := bytes.Compare(index.data[nameOffset:nameOffset+hashSize], idBytes)
if cmp == 0 {
offset, err := index.offsetAt(mid)
if err != nil {
return 0, false, err
}
+
return offset, true, nil
}
+
if cmp < 0 {
lo = mid + 1
} else {
hi = mid
}
}
+
return 0, false, nil
}
@@ -118,10 +138,12 @@ func (index *idxFile) offsetAt(objectIndex int) (uint64, error) {
if objectIndex < 0 || objectIndex >= index.numObjects {
return 0, fmt.Errorf("objectstore/packed: idx %q offset index out of bounds", index.idxName)
}
+
wordOffset := index.offset32Offset + objectIndex*4
if wordOffset < 0 || wordOffset+4 > len(index.data) {
return 0, fmt.Errorf("objectstore/packed: idx %q truncated 32-bit offset table", index.idxName)
}
+
word := binary.BigEndian.Uint32(index.data[wordOffset : wordOffset+4])
if word&0x80000000 == 0 {
return uint64(word), nil
@@ -131,9 +153,11 @@ func (index *idxFile) offsetAt(objectIndex int) (uint64, error) {
if pos < 0 || pos >= index.offset64Count {
return 0, fmt.Errorf("objectstore/packed: idx %q invalid 64-bit offset position", index.idxName)
}
+
offOffset := index.offset64Offset + pos*8
if offOffset < 0 || offOffset+8 > len(index.data)-2*index.algo.Size() {
return 0, fmt.Errorf("objectstore/packed: idx %q truncated 64-bit offset table", index.idxName)
}
+
return binary.BigEndian.Uint64(index.data[offOffset : offOffset+8]), nil
}
diff --git a/objectstore/packed/pack.go b/objectstore/packed/pack.go
index 9af4c860..874b2b76 100644
--- a/objectstore/packed/pack.go
+++ b/objectstore/packed/pack.go
@@ -25,43 +25,58 @@ func openPackFile(name string, file *os.File, size int64) (*packFile, error) {
if size < 12 {
return nil, fmt.Errorf("objectstore/packed: pack %q too short", name)
}
+
if size > int64(int(^uint(0)>>1)) {
return nil, fmt.Errorf("objectstore/packed: pack %q has unsupported size", name)
}
+
fd, err := intconv.UintptrToInt(file.Fd())
if err != nil {
return nil, err
}
+
data, err := syscall.Mmap(fd, 0, int(size), syscall.PROT_READ, syscall.MAP_PRIVATE)
if err != nil {
return nil, err
}
+
if binary.BigEndian.Uint32(data[:4]) != packfmt.Signature {
_ = syscall.Munmap(data)
+
return nil, fmt.Errorf("objectstore/packed: pack %q invalid signature", name)
}
+
version := binary.BigEndian.Uint32(data[4:8])
if !packfmt.VersionSupported(version) {
_ = syscall.Munmap(data)
+
return nil, fmt.Errorf("objectstore/packed: pack %q unsupported version %d", name, version)
}
+
return &packFile{name: name, file: file, data: data}, nil
}
// close unmaps and closes one pack handle.
func (pack *packFile) close() error {
var closeErr error
+
if pack.data != nil {
- if err := syscall.Munmap(pack.data); err != nil && closeErr == nil {
+ err := syscall.Munmap(pack.data)
+ if err != nil && closeErr == nil {
closeErr = err
}
+
pack.data = nil
}
+
if pack.file != nil {
- if err := pack.file.Close(); err != nil && closeErr == nil {
+ err := pack.file.Close()
+ if err != nil && closeErr == nil {
closeErr = err
}
+
pack.file = nil
}
+
return closeErr
}
diff --git a/objectstore/packed/pack_idx_checksum.go b/objectstore/packed/pack_idx_checksum.go
index 2f55a469..25556088 100644
--- a/objectstore/packed/pack_idx_checksum.go
+++ b/objectstore/packed/pack_idx_checksum.go
@@ -14,17 +14,21 @@ func verifyMappedPackMatchesMappedIdx(packData, idxData []byte, algo objectid.Al
if hashSize <= 0 {
return objectid.ErrInvalidAlgorithm
}
+
if len(packData) < hashSize {
return fmt.Errorf("objectstore/packed: pack too short for trailer hash")
}
+
if len(idxData) < hashSize*2 {
return fmt.Errorf("objectstore/packed: idx too short for trailer hashes")
}
packTrailerHash := packData[len(packData)-hashSize:]
+
idxPackHash := idxData[len(idxData)-hashSize*2 : len(idxData)-hashSize]
if !bytes.Equal(packTrailerHash, idxPackHash) {
return fmt.Errorf("objectstore/packed: pack hash does not match idx")
}
+
return nil
}
diff --git a/objectstore/packed/read_bytes.go b/objectstore/packed/read_bytes.go
index b6f42a0d..e272b626 100644
--- a/objectstore/packed/read_bytes.go
+++ b/objectstore/packed/read_bytes.go
@@ -14,6 +14,7 @@ func (store *Store) ReadBytesContent(id objectid.ObjectID) (objecttype.Type, []b
if err != nil {
return objecttype.TypeInvalid, nil, err
}
+
return store.deltaResolveContent(loc)
}
@@ -23,12 +24,15 @@ func (store *Store) ReadBytesFull(id objectid.ObjectID) ([]byte, error) {
if err != nil {
return nil, err
}
+
header, ok := objectheader.Encode(ty, int64(len(content)))
if !ok {
return nil, fmt.Errorf("objectstore/packed: failed to encode object header for type %d", ty)
}
+
out := make([]byte, len(header)+len(content))
copy(out, header)
copy(out[len(header):], content)
+
return out, nil
}
diff --git a/objectstore/packed/read_header.go b/objectstore/packed/read_header.go
index 6822975c..5eb37c92 100644
--- a/objectstore/packed/read_header.go
+++ b/objectstore/packed/read_header.go
@@ -11,5 +11,6 @@ func (store *Store) ReadHeader(id objectid.ObjectID) (objecttype.Type, int64, er
if err != nil {
return objecttype.TypeInvalid, 0, err
}
+
return store.resolveHeaderAt(loc)
}
diff --git a/objectstore/packed/read_header_resolve.go b/objectstore/packed/read_header_resolve.go
index cf49fe2b..420d9363 100644
--- a/objectstore/packed/read_header_resolve.go
+++ b/objectstore/packed/read_header_resolve.go
@@ -17,12 +17,14 @@ func (store *Store) resolveHeaderAt(start location) (objecttype.Type, int64, err
if _, ok := visited[current]; ok {
return objecttype.TypeInvalid, 0, fmt.Errorf("objectstore/packed: delta cycle while resolving object header")
}
+
visited[current] = struct{}{}
pack, meta, err := store.entryMetaAt(current)
if err != nil {
return objecttype.TypeInvalid, 0, err
}
+
if declaredSize < 0 {
if packfmt.IsBaseObjectType(meta.ty) {
declaredSize = meta.size
@@ -31,9 +33,11 @@ func (store *Store) resolveHeaderAt(start location) (objecttype.Type, int64, err
if err != nil {
return objecttype.TypeInvalid, 0, err
}
+
declaredSize = size
}
}
+
if packfmt.IsBaseObjectType(meta.ty) {
return meta.ty, declaredSize, nil
}
@@ -44,6 +48,7 @@ func (store *Store) resolveHeaderAt(start location) (objecttype.Type, int64, err
if err != nil {
return objecttype.TypeInvalid, 0, err
}
+
current = next
case objecttype.TypeOfsDelta:
current = location{
diff --git a/objectstore/packed/read_reader.go b/objectstore/packed/read_reader.go
index a1f24799..d8dfdca9 100644
--- a/objectstore/packed/read_reader.go
+++ b/objectstore/packed/read_reader.go
@@ -41,11 +41,13 @@ func (store *Store) ReadReaderContent(id objectid.ObjectID) (objecttype.Type, in
if err != nil {
return objecttype.TypeInvalid, 0, nil, err
}
+
if packfmt.IsBaseObjectType(meta.ty) {
zr, err := zlibReaderAt(pack, meta.dataOffset)
if err != nil {
return objecttype.TypeInvalid, 0, nil, err
}
+
return meta.ty, meta.size, &readCloser{
reader: iolimit.ExpectLengthReader(zr, meta.size),
closer: zr,
@@ -56,6 +58,7 @@ func (store *Store) ReadReaderContent(id objectid.ObjectID) (objecttype.Type, in
if err != nil {
return objecttype.TypeInvalid, 0, nil, err
}
+
return ty, int64(len(content)), io.NopCloser(bytes.NewReader(content)), nil
}
@@ -72,15 +75,18 @@ func (store *Store) ReadReaderFull(id objectid.ObjectID) (io.ReadCloser, error)
if err != nil {
return nil, err
}
+
if packfmt.IsBaseObjectType(meta.ty) {
header, ok := objectheader.Encode(meta.ty, meta.size)
if !ok {
return nil, fmt.Errorf("objectstore/packed: failed to encode object header for type %d", meta.ty)
}
+
zr, err := zlibReaderAt(pack, meta.dataOffset)
if err != nil {
return nil, err
}
+
return &readCloser{
reader: io.MultiReader(bytes.NewReader(header), iolimit.ExpectLengthReader(zr, meta.size)),
closer: zr,
@@ -91,5 +97,6 @@ func (store *Store) ReadReaderFull(id objectid.ObjectID) (io.ReadCloser, error)
if err != nil {
return nil, err
}
+
return io.NopCloser(bytes.NewReader(raw)), nil
}
diff --git a/objectstore/packed/read_size.go b/objectstore/packed/read_size.go
index e162586a..a0a75db7 100644
--- a/objectstore/packed/read_size.go
+++ b/objectstore/packed/read_size.go
@@ -14,6 +14,7 @@ func (store *Store) ReadSize(id objectid.ObjectID) (int64, error) {
if err != nil {
return 0, err
}
+
return store.resolveSizeAt(loc)
}
@@ -23,9 +24,11 @@ func (store *Store) resolveSizeAt(start location) (int64, error) {
if err != nil {
return 0, err
}
+
if packfmt.IsBaseObjectType(meta.ty) {
return meta.size, nil
}
+
switch meta.ty {
case objecttype.TypeRefDelta, objecttype.TypeOfsDelta:
return deltaDeclaredSizeAt(pack, meta.dataOffset)
diff --git a/objectstore/packed/read_test.go b/objectstore/packed/read_test.go
index 9bfa6610..9ba89fdf 100644
--- a/objectstore/packed/read_test.go
+++ b/objectstore/packed/read_test.go
@@ -30,16 +30,20 @@ func TestPackedStoreReadAgainstGit(t *testing.T) {
if err != nil {
t.Fatalf("ReadHeader: %v", err)
}
+
if gotHeaderType != wantType {
t.Fatalf("ReadHeader type = %v, want %v", gotHeaderType, wantType)
}
+
if gotHeaderSize != int64(len(wantBody)) {
t.Fatalf("ReadHeader size = %d, want %d", gotHeaderSize, len(wantBody))
}
+
gotSize, err := store.ReadSize(id)
if err != nil {
t.Fatalf("ReadSize: %v", err)
}
+
if gotSize != int64(len(wantBody)) {
t.Fatalf("ReadSize = %d, want %d", gotSize, len(wantBody))
}
@@ -48,6 +52,7 @@ func TestPackedStoreReadAgainstGit(t *testing.T) {
if err != nil {
t.Fatalf("ReadBytesFull: %v", err)
}
+
if !bytes.Equal(gotRaw, wantRaw) {
t.Fatalf("ReadBytesFull mismatch")
}
@@ -56,9 +61,11 @@ func TestPackedStoreReadAgainstGit(t *testing.T) {
if err != nil {
t.Fatalf("ReadBytesContent: %v", err)
}
+
if gotType != wantType {
t.Fatalf("ReadBytesContent type = %v, want %v", gotType, wantType)
}
+
if !bytes.Equal(gotBody, wantBody) {
t.Fatalf("ReadBytesContent mismatch")
}
@@ -67,7 +74,9 @@ func TestPackedStoreReadAgainstGit(t *testing.T) {
if err != nil {
t.Fatalf("ReadReaderFull: %v", err)
}
- if got := mustReadAllAndClose(t, fullReader); !bytes.Equal(got, wantRaw) {
+
+ got := mustReadAllAndClose(t, fullReader)
+ if !bytes.Equal(got, wantRaw) {
t.Fatalf("ReadReaderFull mismatch")
}
@@ -75,13 +84,17 @@ func TestPackedStoreReadAgainstGit(t *testing.T) {
if err != nil {
t.Fatalf("ReadReaderContent: %v", err)
}
+
if contentType != wantType {
t.Fatalf("ReadReaderContent type = %v, want %v", contentType, wantType)
}
+
if contentSize != int64(len(wantBody)) {
t.Fatalf("ReadReaderContent size = %d, want %d", contentSize, len(wantBody))
}
- if got := mustReadAllAndClose(t, contentReader); !bytes.Equal(got, wantBody) {
+
+ got = mustReadAllAndClose(t, contentReader)
+ if !bytes.Equal(got, wantBody) {
t.Fatalf("ReadReaderContent mismatch")
}
})
@@ -100,38 +113,54 @@ func TestPackedStoreErrors(t *testing.T) {
t.Fatalf("ParseHex(notFound): %v", err)
}
- if _, err := store.ReadBytesFull(notFoundID); !errors.Is(err, objectstore.ErrObjectNotFound) {
+ _, err = store.ReadBytesFull(notFoundID)
+ if !errors.Is(err, objectstore.ErrObjectNotFound) {
t.Fatalf("ReadBytesFull not-found error = %v", err)
}
- if _, _, err := store.ReadBytesContent(notFoundID); !errors.Is(err, objectstore.ErrObjectNotFound) {
+
+ _, _, err = store.ReadBytesContent(notFoundID)
+ if !errors.Is(err, objectstore.ErrObjectNotFound) {
t.Fatalf("ReadBytesContent not-found error = %v", err)
}
- if _, err := store.ReadReaderFull(notFoundID); !errors.Is(err, objectstore.ErrObjectNotFound) {
+
+ _, err = store.ReadReaderFull(notFoundID)
+ if !errors.Is(err, objectstore.ErrObjectNotFound) {
t.Fatalf("ReadReaderFull not-found error = %v", err)
}
- if _, _, _, err := store.ReadReaderContent(notFoundID); !errors.Is(err, objectstore.ErrObjectNotFound) {
+
+ _, _, _, err = store.ReadReaderContent(notFoundID)
+ if !errors.Is(err, objectstore.ErrObjectNotFound) {
t.Fatalf("ReadReaderContent not-found error = %v", err)
}
- if _, _, err := store.ReadHeader(notFoundID); !errors.Is(err, objectstore.ErrObjectNotFound) {
+
+ _, _, err = store.ReadHeader(notFoundID)
+ if !errors.Is(err, objectstore.ErrObjectNotFound) {
t.Fatalf("ReadHeader not-found error = %v", err)
}
- if _, err := store.ReadSize(notFoundID); !errors.Is(err, objectstore.ErrObjectNotFound) {
+
+ _, err = store.ReadSize(notFoundID)
+ if !errors.Is(err, objectstore.ErrObjectNotFound) {
t.Fatalf("ReadSize not-found error = %v", err)
}
var otherAlgo objectid.Algorithm
+
for _, candidate := range objectid.SupportedAlgorithms() {
if candidate != algo {
otherAlgo = candidate
+
break
}
}
+
if otherAlgo != objectid.AlgorithmUnknown {
mismatchID, err := objectid.ParseHex(otherAlgo, strings.Repeat("0", otherAlgo.HexLen()))
if err != nil {
t.Fatalf("ParseHex(mismatch): %v", err)
}
- if _, err := store.ReadBytesFull(mismatchID); err == nil || !strings.Contains(err.Error(), "algorithm mismatch") {
+
+ _, err = store.ReadBytesFull(mismatchID)
+ if err == nil || !strings.Contains(err.Error(), "algorithm mismatch") {
t.Fatalf("ReadBytesFull algorithm-mismatch error = %v", err)
}
}
@@ -141,11 +170,16 @@ func TestPackedStoreErrors(t *testing.T) {
func TestPackedStoreNewValidation(t *testing.T) {
t.Parallel()
testRepo, _ := createPackedFixtureRepo(t, objectid.AlgorithmSHA1)
+
store := openPackedStore(t, testRepo.Dir(), objectid.AlgorithmSHA1)
- if err := store.Close(); err != nil {
+
+ err := store.Close()
+ if err != nil {
t.Fatalf("Close: %v", err)
}
- if err := store.Close(); err != nil {
+
+ err = store.Close()
+ if err != nil {
t.Fatalf("Close second: %v", err)
}
}
@@ -153,13 +187,16 @@ func TestPackedStoreNewValidation(t *testing.T) {
func TestPackedStoreInvalidAlgorithm(t *testing.T) {
t.Parallel()
testRepo := testgit.NewRepo(t, testgit.RepoOptions{ObjectFormat: objectid.AlgorithmSHA1, Bare: true})
+
root, err := os.OpenRoot(testRepo.Dir())
if err != nil {
t.Fatalf("OpenRoot(%q): %v", testRepo.Dir(), err)
}
+
t.Cleanup(func() { _ = root.Close() })
- if _, err := packed.New(root, objectid.AlgorithmUnknown); !errors.Is(err, objectid.ErrInvalidAlgorithm) {
+ _, err = packed.New(root, objectid.AlgorithmUnknown)
+ if !errors.Is(err, objectid.ErrInvalidAlgorithm) {
t.Fatalf("packed.New invalid algorithm error = %v", err)
}
}
@@ -170,15 +207,20 @@ func TestPackedStoreReadHeaderUsesResolvedObjectSizeForDelta(t *testing.T) {
testRepo := testgit.NewRepo(t, testgit.RepoOptions{ObjectFormat: algo, Bare: true})
var parent objectid.ObjectID
+
for i := range 96 {
content := strings.Repeat("common-line-"+strconv.Itoa(i%7)+"\n", 384) + fmt.Sprintf("tail-%03d\n", i)
+
_, treeID := testRepo.MakeSingleFileTree(t, "file.txt", []byte(content))
if i == 0 {
parent = testRepo.CommitTree(t, treeID, "delta-header-size-0")
+
continue
}
+
parent = testRepo.CommitTree(t, treeID, fmt.Sprintf("delta-header-size-%03d", i), parent)
}
+
testRepo.UpdateRef(t, "refs/heads/main", parent)
testRepo.Repack(t, "-a", "-d", "-f", "--window=128", "--depth=128")
@@ -189,13 +231,16 @@ func TestPackedStoreReadHeaderUsesResolvedObjectSizeForDelta(t *testing.T) {
if err != nil {
t.Fatalf("ReadHeader(%s): %v", deltaID, err)
}
+
if gotSize != wantResolvedSize {
t.Fatalf("ReadHeader(%s) size = %d, want resolved size %d", deltaID, gotSize, wantResolvedSize)
}
+
gotReadSize, err := store.ReadSize(deltaID)
if err != nil {
t.Fatalf("ReadSize(%s): %v", deltaID, err)
}
+
if gotReadSize != wantResolvedSize {
t.Fatalf("ReadSize(%s) = %d, want resolved size %d", deltaID, gotReadSize, wantResolvedSize)
}
@@ -209,6 +254,7 @@ func findDeltaObjectWithResolvedSizeMismatch(t *testing.T, testRepo *testgit.Tes
if err != nil {
t.Fatalf("Glob idx: %v", err)
}
+
if len(idxFiles) == 0 {
t.Fatalf("no idx files found")
}
@@ -221,16 +267,19 @@ func findDeltaObjectWithResolvedSizeMismatch(t *testing.T, testRepo *testgit.Tes
}
idHex := fields[0]
+
deltaStreamSize, err := strconv.ParseInt(fields[2], 10, 64)
if err != nil {
continue
}
resolvedSizeStr := testRepo.Run(t, "cat-file", "-s", idHex)
+
resolvedSize, err := strconv.ParseInt(strings.TrimSpace(resolvedSizeStr), 10, 64)
if err != nil {
t.Fatalf("parse cat-file size for %s: %v", idHex, err)
}
+
if deltaStreamSize == resolvedSize {
continue
}
@@ -239,9 +288,11 @@ func findDeltaObjectWithResolvedSizeMismatch(t *testing.T, testRepo *testgit.Tes
if err != nil {
t.Fatalf("ParseHex(%s): %v", idHex, err)
}
+
return id, resolvedSize
}
t.Fatalf("did not find a delta object with mismatched stream/resolved size")
+
return objectid.ObjectID{}, 0
}
diff --git a/objectstore/packed/store.go b/objectstore/packed/store.go
index abd7175f..d28113d1 100644
--- a/objectstore/packed/store.go
+++ b/objectstore/packed/store.go
@@ -60,6 +60,7 @@ func New(root *os.Root, algo objectid.Algorithm) (*Store, error) {
if algo.Size() == 0 {
return nil, objectid.ErrInvalidAlgorithm
}
+
return &Store{
root: root,
algo: algo,
@@ -76,8 +77,10 @@ func (store *Store) Close() error {
store.stateMu.Lock()
if store.closed {
store.stateMu.Unlock()
+
return nil
}
+
store.closed = true
root := store.root
packs := store.packs
@@ -87,23 +90,30 @@ func (store *Store) Close() error {
store.idxMu.RUnlock()
var closeErr error
+
for _, pack := range packs {
- if err := pack.close(); err != nil && closeErr == nil {
+ err := pack.close()
+ if err != nil && closeErr == nil {
closeErr = err
}
}
+
for _, index := range indexes {
- if err := index.close(); err != nil && closeErr == nil {
+ err := index.close()
+ if err != nil && closeErr == nil {
closeErr = err
}
}
+
store.cacheMu.Lock()
store.deltaCache.clear()
store.cacheMu.Unlock()
- if err := root.Close(); err != nil && closeErr == nil {
+ err := root.Close()
+ if err != nil && closeErr == nil {
closeErr = err
}
+
return closeErr
}
@@ -113,7 +123,9 @@ func (store *Store) lookup(id objectid.ObjectID) (location, error) {
if id.Algorithm() != store.algo {
return zero, errors.New("objectstore/packed: object id algorithm mismatch")
}
- if err := store.ensureCandidates(); err != nil {
+
+ err := store.ensureCandidates()
+ if err != nil {
return zero, err
}
@@ -122,81 +134,111 @@ func (store *Store) lookup(id objectid.ObjectID) (location, error) {
candidate, ok := store.candidateForPack(nextPackName)
if !ok {
nextPackName = store.firstCandidatePackName()
+
continue
}
+
nextPackName = store.nextCandidatePackName(candidate.packName)
+
index, err := store.openIndex(candidate)
if err != nil {
return zero, err
}
+
offset, ok, err := index.lookup(id)
if err != nil {
return zero, err
}
+
if ok {
store.touchCandidate(candidate.packName)
+
return location{packName: index.packName, offset: offset}, nil
}
}
+
return zero, objectstore.ErrObjectNotFound
}
// openPack returns one opened and validated pack handle.
func (store *Store) openPack(name string) (*packFile, error) {
store.stateMu.RLock()
- if pack, ok := store.packs[name]; ok {
+
+ pack, ok := store.packs[name]
+ if ok {
store.stateMu.RUnlock()
+
return pack, nil
}
+
store.stateMu.RUnlock()
file, err := store.root.Open(name)
if err != nil {
return nil, err
}
+
info, err := file.Stat()
if err != nil {
_ = file.Close()
+
return nil, err
}
- pack, err := openPackFile(name, file, info.Size())
+
+ pack, err = openPackFile(name, file, info.Size())
if err != nil {
_ = file.Close()
+
return nil, err
}
- if err := store.verifyPackMatchesIndexes(pack); err != nil {
+
+ err = store.verifyPackMatchesIndexes(pack)
+ if err != nil {
_ = pack.close()
+
return nil, err
}
store.stateMu.Lock()
- if existing, ok := store.packs[name]; ok {
+
+ existing, ok := store.packs[name]
+ if ok {
store.stateMu.Unlock()
+
_ = pack.close()
+
return existing, nil
}
+
store.packs[name] = pack
store.stateMu.Unlock()
+
return pack, nil
}
// verifyPackMatchesIndexes checks that one opened pack's trailer hash matches
// every loaded index that references the same pack name.
func (store *Store) verifyPackMatchesIndexes(pack *packFile) error {
- if err := store.ensureCandidates(); err != nil {
+ err := store.ensureCandidates()
+ if err != nil {
return err
}
+
candidate, ok := store.candidateForPack(pack.name)
if !ok {
return fmt.Errorf("objectstore/packed: missing index for pack %q", pack.name)
}
+
index, err := store.openIndex(candidate)
if err != nil {
return err
}
- if err := verifyMappedPackMatchesMappedIdx(pack.data, index.data, store.algo); err != nil {
+
+ err = verifyMappedPackMatchesMappedIdx(pack.data, index.data, store.algo)
+ if err != nil {
return fmt.Errorf("objectstore/packed: pack %q does not match idx %q: %w", pack.name, index.idxName, err)
}
+
return nil
}
@@ -206,9 +248,11 @@ func (store *Store) entryMetaAt(loc location) (*packFile, entryMeta, error) {
if err != nil {
return nil, entryMeta{}, err
}
+
meta, err := parseEntryMeta(pack, store.algo, loc.offset)
if err != nil {
return nil, entryMeta{}, err
}
+
return pack, meta, nil
}