aboutsummaryrefslogtreecommitdiff
path: root/objectstore
diff options
context:
space:
mode:
authorGravatar Runxi Yu2026-03-04 08:26:56 +0800
committerGravatar Runxi Yu2026-03-04 08:59:53 +0800
commitab7501be34032fb9e5c48726a68ae90a917af9eb (patch)
tree20d005647569befea8133e953c3270e8fd2a2a5b /objectstore
parent*: gofumpt (diff)
signatureNo signature
*: Lint
Diffstat (limited to 'objectstore')
-rw-r--r--objectstore/chain/chain.go51
-rw-r--r--objectstore/loose/helpers_test.go15
-rw-r--r--objectstore/loose/parse.go6
-rw-r--r--objectstore/loose/paths.go5
-rw-r--r--objectstore/loose/read_bytes.go5
-rw-r--r--objectstore/loose/read_header.go3
-rw-r--r--objectstore/loose/read_reader.go8
-rw-r--r--objectstore/loose/read_size.go1
-rw-r--r--objectstore/loose/read_test.go43
-rw-r--r--objectstore/loose/store.go1
-rw-r--r--objectstore/loose/write_reader.go17
-rw-r--r--objectstore/loose/write_test.go20
-rw-r--r--objectstore/loose/write_writer.go57
-rw-r--r--objectstore/packed/delta_apply.go13
-rw-r--r--objectstore/packed/delta_cache.go1
-rw-r--r--objectstore/packed/delta_plan.go7
-rw-r--r--objectstore/packed/entry_inflate.go8
-rw-r--r--objectstore/packed/entry_parse.go4
-rw-r--r--objectstore/packed/helpers_test.go17
-rw-r--r--objectstore/packed/idx_lookup_candidates.go24
-rw-r--r--objectstore/packed/idx_open.go40
-rw-r--r--objectstore/packed/idx_parse.go24
-rw-r--r--objectstore/packed/pack.go19
-rw-r--r--objectstore/packed/pack_idx_checksum.go4
-rw-r--r--objectstore/packed/read_bytes.go4
-rw-r--r--objectstore/packed/read_header.go1
-rw-r--r--objectstore/packed/read_header_resolve.go5
-rw-r--r--objectstore/packed/read_reader.go7
-rw-r--r--objectstore/packed/read_size.go3
-rw-r--r--objectstore/packed/read_test.go75
-rw-r--r--objectstore/packed/store.go64
31 files changed, 490 insertions, 62 deletions
diff --git a/objectstore/chain/chain.go b/objectstore/chain/chain.go
index f2992b34..8e10feb6 100644
--- a/objectstore/chain/chain.go
+++ b/objectstore/chain/chain.go
@@ -25,13 +25,17 @@ type Chain struct {
// New creates a Chain from backends.
func New(backends ...objectstore.Store) *Chain {
nodeByStore := make(map[objectstore.Store]*backendNode, len(backends))
- var head *backendNode
- var tail *backendNode
+
+ var (
+ head *backendNode
+ tail *backendNode
+ )
for _, backend := range backends {
if backend == nil {
continue
}
+
node := &backendNode{
backend: backend,
prev: tail,
@@ -39,9 +43,11 @@ func New(backends ...objectstore.Store) *Chain {
if tail != nil {
tail.next = node
}
+
if head == nil {
head = node
}
+
tail = node
nodeByStore[backend] = node
}
@@ -59,13 +65,17 @@ func (chain *Chain) ReadBytesFull(id objectid.ObjectID) ([]byte, error) {
full, err := backend.ReadBytesFull(id)
if err == nil {
chain.touchBackend(backend)
+
return full, nil
}
+
if errors.Is(err, objectstore.ErrObjectNotFound) {
continue
}
+
return nil, fmt.Errorf("objectstore: backend %d read bytes full: %w", i, err)
}
+
return nil, objectstore.ErrObjectNotFound
}
@@ -76,13 +86,17 @@ func (chain *Chain) ReadBytesContent(id objectid.ObjectID) (objecttype.Type, []b
ty, content, err := backend.ReadBytesContent(id)
if err == nil {
chain.touchBackend(backend)
+
return ty, content, nil
}
+
if errors.Is(err, objectstore.ErrObjectNotFound) {
continue
}
+
return objecttype.TypeInvalid, nil, fmt.Errorf("objectstore: backend %d read bytes content: %w", i, err)
}
+
return objecttype.TypeInvalid, nil, objectstore.ErrObjectNotFound
}
@@ -93,13 +107,17 @@ func (chain *Chain) ReadReaderFull(id objectid.ObjectID) (io.ReadCloser, error)
reader, err := backend.ReadReaderFull(id)
if err == nil {
chain.touchBackend(backend)
+
return reader, nil
}
+
if errors.Is(err, objectstore.ErrObjectNotFound) {
continue
}
+
return nil, fmt.Errorf("objectstore: backend %d read reader full: %w", i, err)
}
+
return nil, objectstore.ErrObjectNotFound
}
@@ -110,13 +128,17 @@ func (chain *Chain) ReadReaderContent(id objectid.ObjectID) (objecttype.Type, in
ty, size, reader, err := backend.ReadReaderContent(id)
if err == nil {
chain.touchBackend(backend)
+
return ty, size, reader, nil
}
+
if errors.Is(err, objectstore.ErrObjectNotFound) {
continue
}
+
return objecttype.TypeInvalid, 0, nil, fmt.Errorf("objectstore: backend %d read reader content: %w", i, err)
}
+
return objecttype.TypeInvalid, 0, nil, objectstore.ErrObjectNotFound
}
@@ -126,13 +148,17 @@ func (chain *Chain) ReadSize(id objectid.ObjectID) (int64, error) {
size, err := backend.ReadSize(id)
if err == nil {
chain.touchBackend(backend)
+
return size, nil
}
+
if errors.Is(err, objectstore.ErrObjectNotFound) {
continue
}
+
return 0, fmt.Errorf("objectstore: backend %d read size: %w", i, err)
}
+
return 0, objectstore.ErrObjectNotFound
}
@@ -142,31 +168,40 @@ func (chain *Chain) ReadHeader(id objectid.ObjectID) (objecttype.Type, int64, er
ty, size, err := backend.ReadHeader(id)
if err == nil {
chain.touchBackend(backend)
+
return ty, size, nil
}
+
if errors.Is(err, objectstore.ErrObjectNotFound) {
continue
}
+
return objecttype.TypeInvalid, 0, fmt.Errorf("objectstore: backend %d read header: %w", i, err)
}
+
return objecttype.TypeInvalid, 0, objectstore.ErrObjectNotFound
}
// Close closes all backends and joins close errors.
func (chain *Chain) Close() error {
chain.mu.RLock()
+
backends := make([]objectstore.Store, 0, len(chain.backendNodeByStore))
for node := chain.backendHead; node != nil; node = node.next {
backends = append(backends, node.backend)
}
+
chain.mu.RUnlock()
var errs []error
+
for _, backend := range backends {
- if err := backend.Close(); err != nil {
+ err := backend.Close()
+ if err != nil {
errs = append(errs, err)
}
}
+
return errors.Join(errs...)
}
@@ -179,19 +214,23 @@ type backendNode struct {
func (chain *Chain) firstBackend() objectstore.Store {
chain.mu.RLock()
defer chain.mu.RUnlock()
+
if chain.backendHead == nil {
return nil
}
+
return chain.backendHead.backend
}
func (chain *Chain) nextBackend(current objectstore.Store) objectstore.Store {
chain.mu.RLock()
defer chain.mu.RUnlock()
+
node := chain.backendNodeByStore[current]
if node == nil || node.next == nil {
return nil
}
+
return node.next.backend
}
@@ -199,6 +238,7 @@ func (chain *Chain) touchBackend(backend objectstore.Store) {
if backend == nil {
return
}
+
if !chain.mu.TryLock() {
return
}
@@ -208,21 +248,26 @@ func (chain *Chain) touchBackend(backend objectstore.Store) {
if node == nil || node == chain.backendHead {
return
}
+
if node.prev != nil {
node.prev.next = node.next
}
+
if node.next != nil {
node.next.prev = node.prev
}
+
if chain.backendTail == node {
chain.backendTail = node.prev
}
node.prev = nil
+
node.next = chain.backendHead
if chain.backendHead != nil {
chain.backendHead.prev = node
}
+
chain.backendHead = node
if chain.backendTail == nil {
chain.backendTail = node
diff --git a/objectstore/loose/helpers_test.go b/objectstore/loose/helpers_test.go
index 972059e0..4b0bb60e 100644
--- a/objectstore/loose/helpers_test.go
+++ b/objectstore/loose/helpers_test.go
@@ -15,30 +15,39 @@ import (
func openLooseStore(t *testing.T, repoPath string, algo objectid.Algorithm) *loose.Store {
t.Helper()
+
objectsPath := filepath.Join(repoPath, "objects")
+
root, err := os.OpenRoot(objectsPath)
if err != nil {
t.Fatalf("OpenRoot(%q): %v", objectsPath, err)
}
+
t.Cleanup(func() { _ = root.Close() })
store, err := loose.New(root, algo)
if err != nil {
t.Fatalf("loose.New: %v", err)
}
+
return store
}
func mustReadAllAndClose(t *testing.T, reader io.ReadCloser) []byte {
t.Helper()
+
data, err := io.ReadAll(reader)
if err != nil {
_ = reader.Close()
+
t.Fatalf("ReadAll: %v", err)
}
- if err := reader.Close(); err != nil {
+
+ err = reader.Close()
+ if err != nil {
t.Fatalf("Close: %v", err)
}
+
return data
}
@@ -46,11 +55,14 @@ func expectedRawObject(t *testing.T, testRepo *testgit.TestRepo, id objectid.Obj
t.Helper()
typeName := testRepo.Run(t, "cat-file", "-t", id.String())
+
ty, ok := objecttype.ParseName(typeName)
if !ok {
t.Fatalf("ParseName(%q) failed", typeName)
}
+
body := testRepo.CatFile(t, typeName, id)
+
header, ok := objectheader.Encode(ty, int64(len(body)))
if !ok {
t.Fatalf("objectheader.Encode failed")
@@ -59,5 +71,6 @@ func expectedRawObject(t *testing.T, testRepo *testgit.TestRepo, id objectid.Obj
raw := make([]byte, len(header)+len(body))
copy(raw, header)
copy(raw[len(header):], body)
+
return ty, body, raw
}
diff --git a/objectstore/loose/parse.go b/objectstore/loose/parse.go
index 54bb2375..e88d7c6c 100644
--- a/objectstore/loose/parse.go
+++ b/objectstore/loose/parse.go
@@ -17,7 +17,9 @@ func decodeAll(file *os.File) ([]byte, error) {
if err != nil {
return nil, err
}
+
defer func() { _ = zr.Close() }()
+
return io.ReadAll(zr)
}
@@ -27,10 +29,12 @@ func parseRaw(raw []byte) (objecttype.Type, []byte, error) {
if !ok {
return objecttype.TypeInvalid, nil, errors.New("objectstore/loose: malformed object header")
}
+
content := raw[headerLen:]
if int64(len(content)) != size {
return objecttype.TypeInvalid, nil, errors.New("objectstore/loose: object header size/content mismatch")
}
+
return ty, content, nil
}
@@ -41,9 +45,11 @@ func readHeader(br *bufio.Reader) ([]byte, objecttype.Type, int64, error) {
if err != nil {
return nil, objecttype.TypeInvalid, 0, err
}
+
ty, size, _, ok := objectheader.Parse(header)
if !ok {
return nil, objecttype.TypeInvalid, 0, errors.New("objectstore/loose: malformed object header")
}
+
return header, ty, size, nil
}
diff --git a/objectstore/loose/paths.go b/objectstore/loose/paths.go
index 04730bd3..e8020d72 100644
--- a/objectstore/loose/paths.go
+++ b/objectstore/loose/paths.go
@@ -16,7 +16,9 @@ func (store *Store) objectPath(id objectid.ObjectID) (string, error) {
if id.Algorithm() != store.algo {
return "", fmt.Errorf("objectstore/loose: object id algorithm mismatch: got %s want %s", id.Algorithm(), store.algo)
}
+
hex := id.String()
+
return filepath.Join(hex[:2], hex[2:]), nil
}
@@ -27,12 +29,15 @@ func (store *Store) openObject(id objectid.ObjectID) (*os.File, error) {
if err != nil {
return nil, err
}
+
file, err := store.root.Open(relPath)
if err != nil {
if errors.Is(err, fs.ErrNotExist) {
return nil, objectstore.ErrObjectNotFound
}
+
return nil, err
}
+
return file, nil
}
diff --git a/objectstore/loose/read_bytes.go b/objectstore/loose/read_bytes.go
index 2f7c24bc..78e1009e 100644
--- a/objectstore/loose/read_bytes.go
+++ b/objectstore/loose/read_bytes.go
@@ -12,16 +12,19 @@ func (store *Store) readBytesParsed(id objectid.ObjectID) ([]byte, objecttype.Ty
if err != nil {
return nil, objecttype.TypeInvalid, nil, err
}
+
defer func() { _ = file.Close() }()
raw, err := decodeAll(file)
if err != nil {
return nil, objecttype.TypeInvalid, nil, err
}
+
ty, content, err := parseRaw(raw)
if err != nil {
return nil, objecttype.TypeInvalid, nil, err
}
+
return raw, ty, content, nil
}
@@ -31,6 +34,7 @@ func (store *Store) ReadBytesFull(id objectid.ObjectID) ([]byte, error) {
if err != nil {
return nil, err
}
+
return raw, nil
}
@@ -40,5 +44,6 @@ func (store *Store) ReadBytesContent(id objectid.ObjectID) (objecttype.Type, []b
if err != nil {
return objecttype.TypeInvalid, nil, err
}
+
return ty, content, nil
}
diff --git a/objectstore/loose/read_header.go b/objectstore/loose/read_header.go
index ce76600e..abfb1a02 100644
--- a/objectstore/loose/read_header.go
+++ b/objectstore/loose/read_header.go
@@ -14,17 +14,20 @@ func (store *Store) ReadHeader(id objectid.ObjectID) (objecttype.Type, int64, er
if err != nil {
return objecttype.TypeInvalid, 0, err
}
+
defer func() { _ = file.Close() }()
zr, err := zlib.NewReader(file)
if err != nil {
return objecttype.TypeInvalid, 0, err
}
+
defer func() { _ = zr.Close() }()
_, ty, size, err := readHeader(bufio.NewReader(zr))
if err != nil {
return objecttype.TypeInvalid, 0, err
}
+
return ty, size, nil
}
diff --git a/objectstore/loose/read_reader.go b/objectstore/loose/read_reader.go
index 6a377ba3..a0a51cc1 100644
--- a/objectstore/loose/read_reader.go
+++ b/objectstore/loose/read_reader.go
@@ -29,6 +29,7 @@ func (reader *objectReader) Read(dst []byte) (int, error) {
func (reader *objectReader) Close() error {
errZlib := reader.zr.Close()
errFile := reader.file.Close()
+
return errors.Join(errZlib, errFile)
}
@@ -39,11 +40,14 @@ func (store *Store) openInflated(id objectid.ObjectID) (*os.File, io.ReadCloser,
if err != nil {
return nil, nil, err
}
+
zr, err := zlib.NewReader(file)
if err != nil {
_ = file.Close()
+
return nil, nil, err
}
+
return file, zr, nil
}
@@ -56,10 +60,12 @@ func (store *Store) ReadReaderFull(id objectid.ObjectID) (io.ReadCloser, error)
}
br := bufio.NewReader(zr)
+
header, _, size, err := readHeader(br)
if err != nil {
_ = zr.Close()
_ = file.Close()
+
return nil, err
}
@@ -82,10 +88,12 @@ func (store *Store) ReadReaderContent(id objectid.ObjectID) (objecttype.Type, in
}
br := bufio.NewReader(zr)
+
_, ty, size, err := readHeader(br)
if err != nil {
_ = zr.Close()
_ = file.Close()
+
return objecttype.TypeInvalid, 0, nil, err
}
diff --git a/objectstore/loose/read_size.go b/objectstore/loose/read_size.go
index 45f1f0fe..2a1eaec9 100644
--- a/objectstore/loose/read_size.go
+++ b/objectstore/loose/read_size.go
@@ -5,5 +5,6 @@ import "codeberg.org/lindenii/furgit/objectid"
// ReadSize reads an object's declared content length.
func (store *Store) ReadSize(id objectid.ObjectID) (int64, error) {
_, size, err := store.ReadHeader(id)
+
return size, err
}
diff --git a/objectstore/loose/read_test.go b/objectstore/loose/read_test.go
index d8166c9e..1efc1682 100644
--- a/objectstore/loose/read_test.go
+++ b/objectstore/loose/read_test.go
@@ -41,6 +41,7 @@ func TestLooseStoreReadAgainstGit(t *testing.T) {
if err != nil {
t.Fatalf("ReadBytesFull: %v", err)
}
+
if !bytes.Equal(gotRaw, wantRaw) {
t.Fatalf("ReadBytesFull mismatch")
}
@@ -49,9 +50,11 @@ func TestLooseStoreReadAgainstGit(t *testing.T) {
if err != nil {
t.Fatalf("ReadBytesContent: %v", err)
}
+
if gotType != wantType {
t.Fatalf("ReadBytesContent type = %v, want %v", gotType, wantType)
}
+
if !bytes.Equal(gotBody, wantBody) {
t.Fatalf("ReadBytesContent body mismatch")
}
@@ -60,9 +63,11 @@ func TestLooseStoreReadAgainstGit(t *testing.T) {
if err != nil {
t.Fatalf("ReadHeader: %v", err)
}
+
if headType != wantType {
t.Fatalf("ReadHeader type = %v, want %v", headType, wantType)
}
+
if headSize != int64(len(wantBody)) {
t.Fatalf("ReadHeader size = %d, want %d", headSize, len(wantBody))
}
@@ -71,7 +76,9 @@ func TestLooseStoreReadAgainstGit(t *testing.T) {
if err != nil {
t.Fatalf("ReadReaderFull: %v", err)
}
- if got := mustReadAllAndClose(t, fullReader); !bytes.Equal(got, wantRaw) {
+
+ got := mustReadAllAndClose(t, fullReader)
+ if !bytes.Equal(got, wantRaw) {
t.Fatalf("ReadReaderFull stream mismatch")
}
@@ -79,13 +86,17 @@ func TestLooseStoreReadAgainstGit(t *testing.T) {
if err != nil {
t.Fatalf("ReadReaderContent: %v", err)
}
+
if contentType != wantType {
t.Fatalf("ReadReaderContent type = %v, want %v", contentType, wantType)
}
+
if contentSize != int64(len(wantBody)) {
t.Fatalf("ReadReaderContent size = %d, want %d", contentSize, len(wantBody))
}
- if got := mustReadAllAndClose(t, contentReader); !bytes.Equal(got, wantBody) {
+
+ got = mustReadAllAndClose(t, contentReader)
+ if !bytes.Equal(got, wantBody) {
t.Fatalf("ReadReaderContent stream mismatch")
}
})
@@ -104,19 +115,28 @@ func TestLooseStoreErrors(t *testing.T) {
t.Fatalf("ParseHex(notFoundID): %v", err)
}
- if _, err := store.ReadBytesFull(notFoundID); !errors.Is(err, objectstore.ErrObjectNotFound) {
+ _, err = store.ReadBytesFull(notFoundID)
+ if !errors.Is(err, objectstore.ErrObjectNotFound) {
t.Fatalf("ReadBytesFull not-found error = %v", err)
}
- if _, _, err := store.ReadBytesContent(notFoundID); !errors.Is(err, objectstore.ErrObjectNotFound) {
+
+ _, _, err = store.ReadBytesContent(notFoundID)
+ if !errors.Is(err, objectstore.ErrObjectNotFound) {
t.Fatalf("ReadBytesContent not-found error = %v", err)
}
- if _, err := store.ReadReaderFull(notFoundID); !errors.Is(err, objectstore.ErrObjectNotFound) {
+
+ _, err = store.ReadReaderFull(notFoundID)
+ if !errors.Is(err, objectstore.ErrObjectNotFound) {
t.Fatalf("ReadReaderFull not-found error = %v", err)
}
- if _, _, _, err := store.ReadReaderContent(notFoundID); !errors.Is(err, objectstore.ErrObjectNotFound) {
+
+ _, _, _, err = store.ReadReaderContent(notFoundID)
+ if !errors.Is(err, objectstore.ErrObjectNotFound) {
t.Fatalf("ReadReaderContent not-found error = %v", err)
}
- if _, _, err := store.ReadHeader(notFoundID); !errors.Is(err, objectstore.ErrObjectNotFound) {
+
+ _, _, err = store.ReadHeader(notFoundID)
+ if !errors.Is(err, objectstore.ErrObjectNotFound) {
t.Fatalf("ReadHeader not-found error = %v", err)
}
@@ -126,12 +146,14 @@ func TestLooseStoreErrors(t *testing.T) {
} else {
otherAlgo = objectid.AlgorithmSHA1
}
+
otherID, err := objectid.ParseHex(otherAlgo, strings.Repeat("1", otherAlgo.HexLen()))
if err != nil {
t.Fatalf("ParseHex(otherID): %v", err)
}
- if _, err := store.ReadBytesFull(otherID); err == nil || !strings.Contains(err.Error(), "algorithm mismatch") {
+ _, err = store.ReadBytesFull(otherID)
+ if err == nil || !strings.Contains(err.Error(), "algorithm mismatch") {
t.Fatalf("ReadBytesFull algorithm-mismatch error = %v", err)
}
})
@@ -139,13 +161,16 @@ func TestLooseStoreErrors(t *testing.T) {
func TestLooseStoreNewValidation(t *testing.T) {
t.Parallel()
+
root, err := os.OpenRoot(t.TempDir())
if err != nil {
t.Fatalf("OpenRoot: %v", err)
}
+
defer func() { _ = root.Close() }()
- if _, err := loose.New(root, objectid.AlgorithmUnknown); err == nil {
+ _, err = loose.New(root, objectid.AlgorithmUnknown)
+ if err == nil {
t.Fatalf("loose.New(root, unknown) expected error")
}
}
diff --git a/objectstore/loose/store.go b/objectstore/loose/store.go
index 05459a6c..c3ae989c 100644
--- a/objectstore/loose/store.go
+++ b/objectstore/loose/store.go
@@ -24,6 +24,7 @@ func New(root *os.Root, algo objectid.Algorithm) (*Store, error) {
if algo.Size() == 0 {
return nil, objectid.ErrInvalidAlgorithm
}
+
return &Store{
root: root,
algo: algo,
diff --git a/objectstore/loose/write_reader.go b/objectstore/loose/write_reader.go
index b2329f02..9dbf3818 100644
--- a/objectstore/loose/write_reader.go
+++ b/objectstore/loose/write_reader.go
@@ -27,12 +27,15 @@ func (store *Store) WriteReaderContent(ty objecttype.Type, size int64, src io.Re
if err != nil {
return objectid.ObjectID{}, err
}
+
writer.headerDone = true
writer.expectedContentLeft = size
- if err := writer.writeRawChunk(header); err != nil {
+ err = writer.writeRawChunk(header)
+ if err != nil {
_ = writer.Close()
_ = store.root.Remove(writer.tmpRelPath)
+
return objectid.ObjectID{}, err
}
@@ -46,25 +49,33 @@ func (store *Store) WriteReaderFull(src io.Reader) (objectid.ObjectID, error) {
if err != nil {
return objectid.ObjectID{}, err
}
+
return writeReaderIntoStreamWriter(writer, src)
}
// writeReaderIntoStreamWriter copies src into writer and publishes the object.
func writeReaderIntoStreamWriter(writer *streamWriter, src io.Reader) (objectid.ObjectID, error) {
- if _, err := io.Copy(writer, src); err != nil {
+ _, err := io.Copy(writer, src)
+ if err != nil {
_ = writer.Close()
_ = writer.store.root.Remove(writer.tmpRelPath)
+
return objectid.ObjectID{}, err
}
- if err := writer.Close(); err != nil {
+
+ err = writer.Close()
+ if err != nil {
_ = writer.store.root.Remove(writer.tmpRelPath)
+
return objectid.ObjectID{}, err
}
id, err := writer.finalize()
if err != nil {
_ = writer.store.root.Remove(writer.tmpRelPath)
+
return objectid.ObjectID{}, err
}
+
return id, nil
}
diff --git a/objectstore/loose/write_test.go b/objectstore/loose/write_test.go
index cceabe5a..5604c5b0 100644
--- a/objectstore/loose/write_test.go
+++ b/objectstore/loose/write_test.go
@@ -18,6 +18,7 @@ func TestLooseStoreWriteReaderContentAgainstGit(t *testing.T) {
content := []byte("written-by-content-reader\n")
expectedHex := testRepo.RunInput(t, content, "hash-object", "-t", "blob", "--stdin")
+
expectedID, err := objectid.ParseHex(algo, expectedHex)
if err != nil {
t.Fatalf("ParseHex(expected): %v", err)
@@ -27,6 +28,7 @@ func TestLooseStoreWriteReaderContentAgainstGit(t *testing.T) {
if err != nil {
t.Fatalf("WriteReaderContent: %v", err)
}
+
if writtenID != expectedID {
t.Fatalf("WriteReaderContent id = %s, want %s", writtenID, expectedID)
}
@@ -41,6 +43,7 @@ func TestLooseStoreWriteReaderContentAgainstGit(t *testing.T) {
if err != nil {
t.Fatalf("WriteReaderContent second: %v", err)
}
+
if writtenID2 != expectedID {
t.Fatalf("WriteReaderContent second id = %s, want %s", writtenID2, expectedID)
}
@@ -54,19 +57,23 @@ func TestLooseStoreWriteReaderFullAgainstGit(t *testing.T) {
store := openLooseStore(t, testRepo.Dir(), algo)
body := []byte("full-reader-body\n")
+
header, ok := objectheader.Encode(objecttype.TypeBlob, int64(len(body)))
if !ok {
t.Fatalf("objectheader.Encode failed")
}
+
raw := make([]byte, len(header)+len(body))
copy(raw, header)
copy(raw[len(header):], body)
wantID := algo.Sum(raw)
+
gotID, err := store.WriteReaderFull(bytes.NewReader(raw))
if err != nil {
t.Fatalf("WriteReaderFull: %v", err)
}
+
if gotID != wantID {
t.Fatalf("WriteReaderFull id = %s, want %s", gotID, wantID)
}
@@ -86,7 +93,8 @@ func TestLooseStoreReaderValidationErrors(t *testing.T) {
testRepo := testgit.NewRepo(t, testgit.RepoOptions{ObjectFormat: algo, Bare: true})
store := openLooseStore(t, testRepo.Dir(), algo)
- if _, err := store.WriteReaderContent(objecttype.TypeBlob, 1, bytes.NewReader([]byte("hello"))); err == nil {
+ _, err := store.WriteReaderContent(objecttype.TypeBlob, 1, bytes.NewReader([]byte("hello")))
+ if err == nil {
t.Fatalf("expected error after overflow")
}
})
@@ -96,7 +104,8 @@ func TestLooseStoreReaderValidationErrors(t *testing.T) {
testRepo := testgit.NewRepo(t, testgit.RepoOptions{ObjectFormat: algo, Bare: true})
store := openLooseStore(t, testRepo.Dir(), algo)
- if _, err := store.WriteReaderContent(objecttype.TypeBlob, 5, bytes.NewReader([]byte("x"))); err == nil {
+ _, err := store.WriteReaderContent(objecttype.TypeBlob, 5, bytes.NewReader([]byte("x")))
+ if err == nil {
t.Fatalf("expected error for short content")
}
})
@@ -106,7 +115,8 @@ func TestLooseStoreReaderValidationErrors(t *testing.T) {
testRepo := testgit.NewRepo(t, testgit.RepoOptions{ObjectFormat: algo, Bare: true})
store := openLooseStore(t, testRepo.Dir(), algo)
- if _, err := store.WriteReaderFull(bytes.NewReader([]byte("not-a-header"))); err == nil {
+ _, err := store.WriteReaderFull(bytes.NewReader([]byte("not-a-header")))
+ if err == nil {
t.Fatalf("expected error for malformed header")
}
})
@@ -117,7 +127,9 @@ func TestLooseStoreReaderValidationErrors(t *testing.T) {
store := openLooseStore(t, testRepo.Dir(), algo)
raw := []byte("blob 1\x00hello")
- if _, err := store.WriteReaderFull(bytes.NewReader(raw)); err == nil {
+
+ _, err := store.WriteReaderFull(bytes.NewReader(raw))
+ if err == nil {
t.Fatalf("expected error after mismatch")
}
})
diff --git a/objectstore/loose/write_writer.go b/objectstore/loose/write_writer.go
index c075f2ba..a0f24f2b 100644
--- a/objectstore/loose/write_writer.go
+++ b/objectstore/loose/write_writer.go
@@ -76,23 +76,28 @@ func (writer *streamWriter) Write(src []byte) (int, error) {
if writer.finalized {
return 0, errors.New("objectstore/loose: write after finalize")
}
+
if writer.closed {
return 0, errors.New("objectstore/loose: write after close")
}
if writer.fullMode {
- if err := writer.acceptFull(src); err != nil {
+ err := writer.acceptFull(src)
+ if err != nil {
return 0, err
}
} else {
- if err := writer.acceptContent(int64(len(src))); err != nil {
+ err := writer.acceptContent(int64(len(src)))
+ if err != nil {
return 0, err
}
}
- if err := writer.writeRawChunk(src); err != nil {
+ err := writer.writeRawChunk(src)
+ if err != nil {
return 0, err
}
+
return len(src), nil
}
@@ -102,12 +107,14 @@ func (writer *streamWriter) Close() error {
if writer.closed {
return nil
}
+
writer.closed = true
errZlib := writer.zw.Close()
errSync := writer.file.Sync()
errFile := writer.file.Close()
writer.file = nil
+
return errors.Join(errZlib, errSync, errFile)
}
@@ -118,84 +125,107 @@ func (writer *streamWriter) finalize() (objectid.ObjectID, error) {
if writer.finalized {
return writer.finalID, writer.finalErr
}
+
writer.finalized = true
var zero objectid.ObjectID
if !writer.closed {
- if err := writer.Close(); err != nil {
+ err := writer.Close()
+ if err != nil {
writer.finalErr = err
+
return zero, err
}
}
if writer.fullMode && !writer.headerDone {
writer.finalErr = errors.New("objectstore/loose: missing full object header")
+
return zero, writer.finalErr
}
+
if writer.expectedContentLeft != 0 {
writer.finalErr = errors.New("objectstore/loose: object content shorter than declared size")
+
return zero, writer.finalErr
}
idBytes := writer.hash.Sum(nil)
+
id, err := objectid.FromBytes(writer.store.algo, idBytes)
if err != nil {
writer.finalErr = err
+
return zero, err
}
relPath, err := writer.store.objectPath(id)
if err != nil {
writer.finalErr = err
+
return zero, err
}
dir := filepath.Dir(relPath)
- if err := writer.store.root.MkdirAll(dir, 0o755); err != nil {
+
+ err = writer.store.root.MkdirAll(dir, 0o755)
+ if err != nil {
writer.finalErr = err
+
return zero, err
}
cleanup := true
+
defer func() {
if cleanup {
_ = writer.store.root.Remove(writer.tmpRelPath)
}
}()
- if err := writer.store.root.Link(writer.tmpRelPath, relPath); err != nil {
+ err = writer.store.root.Link(writer.tmpRelPath, relPath)
+ if err != nil {
if errors.Is(err, fs.ErrExist) {
writer.finalID = id
cleanup = false
_ = writer.store.root.Remove(writer.tmpRelPath)
+
return id, nil
}
+
writer.finalErr = err
+
return zero, err
}
writer.finalID = id
cleanup = false
+
return id, nil
}
// acceptFull validates and accounts raw full-object input.
func (writer *streamWriter) acceptFull(src []byte) error {
if !writer.headerDone {
- if nul := bytes.IndexByte(src, 0); nul >= 0 {
+ nul := bytes.IndexByte(src, 0)
+ if nul >= 0 {
headerChunkLen := nul + 1
writer.headerBuf = append(writer.headerBuf, src[:headerChunkLen]...)
+
_, size, _, ok := objectheader.Parse(writer.headerBuf)
if !ok {
return errors.New("objectstore/loose: malformed object header")
}
+
writer.headerDone = true
writer.expectedContentLeft = size
+
return writer.acceptContent(int64(len(src) - headerChunkLen))
}
writer.headerBuf = append(writer.headerBuf, src...)
+
return nil
}
@@ -207,18 +237,24 @@ func (writer *streamWriter) acceptContent(n int64) error {
if n > writer.expectedContentLeft {
return errors.New("objectstore/loose: object content exceeds declared size")
}
+
writer.expectedContentLeft -= n
+
return nil
}
// writeRawChunk forwards raw bytes to the hash and deflate pipeline.
func (writer *streamWriter) writeRawChunk(src []byte) error {
- if _, err := writer.hash.Write(src); err != nil {
+ _, err := writer.hash.Write(src)
+ if err != nil {
return err
}
- if _, err := writer.zw.Write(src); err != nil {
+
+ _, err = writer.zw.Write(src)
+ if err != nil {
return err
}
+
return nil
}
@@ -227,13 +263,16 @@ func (writer *streamWriter) writeRawChunk(src []byte) error {
func (store *Store) createTempObjectFile(dir string) (string, *os.File, error) {
for range 16 {
relPath := filepath.Join(dir, tempObjectFilePrefix+rand.Text())
+
file, err := store.root.OpenFile(relPath, os.O_CREATE|os.O_EXCL|os.O_WRONLY, 0o644)
if err == nil {
return relPath, file, nil
}
+
if errors.Is(err, fs.ErrExist) {
continue
}
+
return "", nil, err
}
diff --git a/objectstore/packed/delta_apply.go b/objectstore/packed/delta_apply.go
index 5245e0ba..71f09ead 100644
--- a/objectstore/packed/delta_apply.go
+++ b/objectstore/packed/delta_apply.go
@@ -14,10 +14,12 @@ func (store *Store) deltaResolveContent(start location) (objecttype.Type, []byte
if err != nil {
return objecttype.TypeInvalid, nil, err
}
+
pack, meta, err := store.entryMetaAt(start)
if err != nil {
return objecttype.TypeInvalid, nil, err
}
+
declaredSize := meta.size
if !packfmt.IsBaseObjectType(meta.ty) {
declaredSize, err = deltaDeclaredSizeAt(pack, meta.dataOffset)
@@ -25,6 +27,7 @@ func (store *Store) deltaResolveContent(start location) (objecttype.Type, []byte
return objecttype.TypeInvalid, nil, err
}
}
+
return store.deltaResolveChain(chain, declaredSize)
}
@@ -37,18 +40,22 @@ func (store *Store) deltaResolveChain(chain deltaChain, declaredSize int64) (obj
for i := nextDelta; i >= 0; i-- {
node := chain.deltas[i]
+
pack, err := store.openPack(node.loc.packName)
if err != nil {
return objecttype.TypeInvalid, nil, err
}
+
delta, err := inflateAt(pack, node.dataOffset, -1)
if err != nil {
return objecttype.TypeInvalid, nil, err
}
+
out, err = deltaapply.Apply(out, delta)
if err != nil {
return objecttype.TypeInvalid, nil, err
}
+
store.cacheMu.Lock()
store.deltaCache.add(
deltaBaseKey{packName: node.loc.packName, offset: node.loc.offset},
@@ -65,6 +72,7 @@ func (store *Store) deltaResolveChain(chain deltaChain, declaredSize int64) (obj
declaredSize,
)
}
+
if ty != chain.baseType {
return objecttype.TypeInvalid, nil, fmt.Errorf(
"objectstore/packed: resolved content type mismatch: got %d want %d",
@@ -72,6 +80,7 @@ func (store *Store) deltaResolveChain(chain deltaChain, declaredSize int64) (obj
chain.baseType,
)
}
+
return ty, out, nil
}
@@ -85,6 +94,7 @@ func (store *Store) deltaResolveChainStart(chain deltaChain) (objecttype.Type, [
deltaBaseKey{packName: node.loc.packName, offset: node.loc.offset},
)
store.cacheMu.RUnlock()
+
if ok {
return ty, out, i - 1, nil
}
@@ -95,6 +105,7 @@ func (store *Store) deltaResolveChainStart(chain deltaChain) (objecttype.Type, [
deltaBaseKey{packName: chain.baseLoc.packName, offset: chain.baseLoc.offset},
)
store.cacheMu.RUnlock()
+
if ok {
return ty, out, len(chain.deltas) - 1, nil
}
@@ -103,9 +114,11 @@ func (store *Store) deltaResolveChainStart(chain deltaChain) (objecttype.Type, [
if err != nil {
return objecttype.TypeInvalid, nil, 0, err
}
+
if !packfmt.IsBaseObjectType(meta.ty) {
return objecttype.TypeInvalid, nil, 0, fmt.Errorf("objectstore/packed: delta chain base is not a base object")
}
+
base, err := inflateAt(pack, meta.dataOffset, meta.size)
if err != nil {
return objecttype.TypeInvalid, nil, 0, err
diff --git a/objectstore/packed/delta_cache.go b/objectstore/packed/delta_cache.go
index add21698..a911b254 100644
--- a/objectstore/packed/delta_cache.go
+++ b/objectstore/packed/delta_cache.go
@@ -41,6 +41,7 @@ func (cache *deltaCache) get(key deltaBaseKey) (objecttype.Type, []byte, bool) {
if !ok {
return objecttype.TypeInvalid, nil, false
}
+
return value.ty, append([]byte(nil), value.content...), true
}
diff --git a/objectstore/packed/delta_plan.go b/objectstore/packed/delta_plan.go
index 5f2ae959..b0b0324c 100644
--- a/objectstore/packed/delta_plan.go
+++ b/objectstore/packed/delta_plan.go
@@ -38,6 +38,7 @@ func (store *Store) deltaBuildChain(start location) (deltaChain, error) {
if _, ok := visited[current]; ok {
return deltaChain{}, fmt.Errorf("objectstore/packed: delta cycle while resolving object")
}
+
visited[current] = struct{}{}
_, meta, err := store.entryMetaAt(current)
@@ -48,6 +49,7 @@ func (store *Store) deltaBuildChain(start location) (deltaChain, error) {
if packfmt.IsBaseObjectType(meta.ty) {
chain.baseLoc = current
chain.baseType = meta.ty
+
return chain, nil
}
@@ -57,10 +59,12 @@ func (store *Store) deltaBuildChain(start location) (deltaChain, error) {
loc: current,
dataOffset: meta.dataOffset,
})
+
next, err := store.lookup(meta.baseRefID)
if err != nil {
return deltaChain{}, err
}
+
current = next
case objecttype.TypeOfsDelta:
chain.deltas = append(chain.deltas, deltaNode{
@@ -88,12 +92,15 @@ func deltaDeclaredSizeAt(pack *packFile, dataOffset int) (int64, error) {
if err != nil {
return 0, err
}
+
defer func() { _ = reader.Close() }()
br := bufio.NewReaderSize(reader, 32)
+
_, size, err := deltaapply.ReadHeaderSizes(br)
if err != nil {
return 0, err
}
+
return int64(size), nil
}
diff --git a/objectstore/packed/entry_inflate.go b/objectstore/packed/entry_inflate.go
index 4f91710e..cbdb6a89 100644
--- a/objectstore/packed/entry_inflate.go
+++ b/objectstore/packed/entry_inflate.go
@@ -14,6 +14,7 @@ func zlibReaderAt(pack *packFile, offset int) (io.ReadCloser, error) {
if offset < 0 || offset > len(pack.data) {
return nil, fmt.Errorf("objectstore/packed: pack %q zlib offset out of bounds", pack.name)
}
+
return zlib.NewReader(bytes.NewReader(pack.data[offset:]))
}
@@ -23,6 +24,7 @@ func inflateAt(pack *packFile, offset int, expectedSize int64) ([]byte, error) {
if err != nil {
return nil, err
}
+
defer func() { _ = reader.Close() }()
if expectedSize >= 0 {
@@ -35,9 +37,12 @@ func inflateAt(pack *packFile, offset int, expectedSize int64) ([]byte, error) {
}
body := make([]byte, int(expectedSize))
- if _, err := io.ReadFull(reader, body); err != nil {
+
+ _, err := io.ReadFull(reader, body)
+ if err != nil {
return nil, err
}
+
return body, nil
}
@@ -45,5 +50,6 @@ func inflateAt(pack *packFile, offset int, expectedSize int64) ([]byte, error) {
if err != nil {
return nil, err
}
+
return body, nil
}
diff --git a/objectstore/packed/entry_parse.go b/objectstore/packed/entry_parse.go
index 56287386..7af20af1 100644
--- a/objectstore/packed/entry_parse.go
+++ b/objectstore/packed/entry_parse.go
@@ -34,6 +34,7 @@ func parseEntryMeta(pack *packFile, algo objectid.Algorithm, offset uint64) (ent
if err != nil {
return zero, fmt.Errorf("objectstore/packed: pack %q offset conversion: %w", pack.name, err)
}
+
entry, err := packfmt.ParseEntry(pack.data[pos:], algo.Size())
if err != nil {
return zero, fmt.Errorf("objectstore/packed: pack %q: %w", pack.name, err)
@@ -50,11 +51,13 @@ func parseEntryMeta(pack *packFile, algo objectid.Algorithm, offset uint64) (ent
if err != nil {
return zero, fmt.Errorf("objectstore/packed: pack %q invalid ref-delta base id: %w", pack.name, err)
}
+
meta.baseRefID = baseID
case objecttype.TypeOfsDelta:
if offset <= entry.OfsBaseDistance {
return zero, fmt.Errorf("objectstore/packed: pack %q has invalid ofs-delta base", pack.name)
}
+
meta.baseOfs = offset - entry.OfsBaseDistance
case objecttype.TypeCommit, objecttype.TypeTree, objecttype.TypeBlob, objecttype.TypeTag:
// Base object types do not have delta base metadata.
@@ -63,5 +66,6 @@ func parseEntryMeta(pack *packFile, algo objectid.Algorithm, offset uint64) (ent
default:
return zero, fmt.Errorf("objectstore/packed: pack %q has unsupported entry type %d", pack.name, meta.ty)
}
+
return meta, nil
}
diff --git a/objectstore/packed/helpers_test.go b/objectstore/packed/helpers_test.go
index f8cbd439..1b517294 100644
--- a/objectstore/packed/helpers_test.go
+++ b/objectstore/packed/helpers_test.go
@@ -18,30 +18,39 @@ import (
func openPackedStore(t *testing.T, repoPath string, algo objectid.Algorithm) *packed.Store {
t.Helper()
+
packPath := filepath.Join(repoPath, "objects", "pack")
+
root, err := os.OpenRoot(packPath)
if err != nil {
t.Fatalf("OpenRoot(%q): %v", packPath, err)
}
+
t.Cleanup(func() { _ = root.Close() })
store, err := packed.New(root, algo)
if err != nil {
t.Fatalf("packed.New: %v", err)
}
+
return store
}
func mustReadAllAndClose(t *testing.T, reader io.ReadCloser) []byte {
t.Helper()
+
data, err := io.ReadAll(reader)
if err != nil {
_ = reader.Close()
+
t.Fatalf("ReadAll: %v", err)
}
- if err := reader.Close(); err != nil {
+
+ err = reader.Close()
+ if err != nil {
t.Fatalf("Close: %v", err)
}
+
return data
}
@@ -49,11 +58,14 @@ func expectedRawObject(t *testing.T, testRepo *testgit.TestRepo, id objectid.Obj
t.Helper()
typeName := testRepo.Run(t, "cat-file", "-t", id.String())
+
ty, ok := objecttype.ParseName(typeName)
if !ok {
t.Fatalf("ParseName(%q) failed", typeName)
}
+
body := testRepo.CatFile(t, typeName, id)
+
header, ok := objectheader.Encode(ty, int64(len(body)))
if !ok {
t.Fatalf("objectheader.Encode failed")
@@ -62,6 +74,7 @@ func expectedRawObject(t *testing.T, testRepo *testgit.TestRepo, id objectid.Obj
raw := make([]byte, len(header)+len(body))
copy(raw, header)
copy(raw[len(header):], body)
+
return ty, body, raw
}
@@ -74,6 +87,7 @@ func createPackedFixtureRepo(t *testing.T, algo objectid.Algorithm) (*testgit.Te
tagID := testRepo.TagAnnotated(t, "v1.0.0", commitID, "packed-store-tag")
parent := commitID
+
for i := range 24 {
content := "common-prefix\n" + strings.Repeat("line-"+strconv.Itoa(i%3)+"\n", 256) + fmt.Sprintf("tail-%d\n", i)
nextBlob, nextTree := testRepo.MakeSingleFileTree(t, fmt.Sprintf("file-%02d.txt", i), []byte(content))
@@ -86,6 +100,7 @@ func createPackedFixtureRepo(t *testing.T, algo objectid.Algorithm) (*testgit.Te
}
testRepo.Repack(t, "-a", "-d", "-f", "--window=64", "--depth=64")
+
return testRepo, []objectid.ObjectID{
blobID,
treeID,
diff --git a/objectstore/packed/idx_lookup_candidates.go b/objectstore/packed/idx_lookup_candidates.go
index 83055aac..72121b25 100644
--- a/objectstore/packed/idx_lookup_candidates.go
+++ b/objectstore/packed/idx_lookup_candidates.go
@@ -37,8 +37,11 @@ func (store *Store) ensureCandidates() error {
candidateByPack := make(map[string]packCandidate, len(candidates))
nodeByPack := make(map[string]*packCandidateNode, len(candidates))
- var head *packCandidateNode
- var tail *packCandidateNode
+ var (
+ head *packCandidateNode
+ tail *packCandidateNode
+ )
+
for _, candidate := range candidates {
node := &packCandidateNode{
candidate: candidate,
@@ -47,9 +50,11 @@ func (store *Store) ensureCandidates() error {
if tail != nil {
tail.next = node
}
+
if head == nil {
head = node
}
+
tail = node
candidateByPack[candidate.packName] = candidate
nodeByPack[candidate.packName] = node
@@ -67,6 +72,7 @@ func (store *Store) ensureCandidates() error {
store.candidatesMu.RLock()
err := store.discoverErr
store.candidatesMu.RUnlock()
+
return err
}
@@ -78,8 +84,10 @@ func (store *Store) discoverCandidates() ([]packCandidate, error) {
if os.IsNotExist(err) {
return nil, nil
}
+
return nil, err
}
+
defer func() { _ = dir.Close() }()
entries, err := dir.ReadDir(-1)
@@ -95,11 +103,13 @@ func (store *Store) discoverCandidates() ([]packCandidate, error) {
idxName := entry.Name()
packName := strings.TrimSuffix(idxName, ".idx") + ".pack"
+
packInfo, err := store.root.Stat(packName)
if err != nil {
if os.IsNotExist(err) {
return nil, fmt.Errorf("objectstore/packed: missing pack file for index %q", idxName)
}
+
return nil, err
}
@@ -115,8 +125,10 @@ func (store *Store) discoverCandidates() ([]packCandidate, error) {
if a.mtime > b.mtime {
return -1
}
+
return 1
}
+
return strings.Compare(a.packName, b.packName)
})
@@ -139,18 +151,22 @@ func (store *Store) touchCandidate(packName string) {
if node.prev != nil {
node.prev.next = node.next
}
+
if node.next != nil {
node.next.prev = node.prev
}
+
if store.candidateTail == node {
store.candidateTail = node.prev
}
node.prev = nil
+
node.next = store.candidateHead
if store.candidateHead != nil {
store.candidateHead.prev = node
}
+
store.candidateHead = node
if store.candidateTail == nil {
store.candidateTail = node
@@ -162,9 +178,11 @@ func (store *Store) touchCandidate(packName string) {
func (store *Store) firstCandidatePackName() string {
store.candidatesMu.RLock()
defer store.candidatesMu.RUnlock()
+
if store.candidateHead == nil {
return ""
}
+
return store.candidateHead.candidate.packName
}
@@ -173,9 +191,11 @@ func (store *Store) firstCandidatePackName() string {
func (store *Store) nextCandidatePackName(currentPack string) string {
store.candidatesMu.RLock()
defer store.candidatesMu.RUnlock()
+
node := store.candidateNodeByPack[currentPack]
if node == nil || node.next == nil {
return ""
}
+
return node.next.candidate.packName
}
diff --git a/objectstore/packed/idx_open.go b/objectstore/packed/idx_open.go
index c00a7bac..c3c97e4d 100644
--- a/objectstore/packed/idx_open.go
+++ b/objectstore/packed/idx_open.go
@@ -43,16 +43,21 @@ func (store *Store) candidateForPack(packName string) (packCandidate, bool) {
store.candidatesMu.RLock()
candidate, ok := store.candidateByPack[packName]
store.candidatesMu.RUnlock()
+
return candidate, ok
}
// openIndex returns one opened and parsed index, caching it by pack basename.
func (store *Store) openIndex(candidate packCandidate) (*idxFile, error) {
store.idxMu.RLock()
- if index, ok := store.idxByPack[candidate.packName]; ok {
+
+ index, ok := store.idxByPack[candidate.packName]
+ if ok {
store.idxMu.RUnlock()
+
return index, nil
}
+
store.idxMu.RUnlock()
index, err := openIdxFile(store.root, candidate.idxName, candidate.packName, store.algo)
@@ -61,13 +66,19 @@ func (store *Store) openIndex(candidate packCandidate) (*idxFile, error) {
}
store.idxMu.Lock()
- if existing, ok := store.idxByPack[candidate.packName]; ok {
+
+ existing, ok := store.idxByPack[candidate.packName]
+ if ok {
store.idxMu.Unlock()
+
_ = index.close()
+
return existing, nil
}
+
store.idxByPack[candidate.packName] = index
store.idxMu.Unlock()
+
return index, nil
}
@@ -77,24 +88,32 @@ func openIdxFile(root *os.Root, idxName, packName string, algo objectid.Algorith
if err != nil {
return nil, err
}
+
info, err := file.Stat()
if err != nil {
_ = file.Close()
+
return nil, err
}
+
size := info.Size()
if size < 0 || size > int64(int(^uint(0)>>1)) {
_ = file.Close()
+
return nil, fmt.Errorf("objectstore/packed: idx %q has unsupported size", idxName)
}
+
fd, err := intconv.UintptrToInt(file.Fd())
if err != nil {
_ = file.Close()
+
return nil, err
}
+
data, err := syscall.Mmap(fd, 0, int(size), syscall.PROT_READ, syscall.MAP_PRIVATE)
if err != nil {
_ = file.Close()
+
return nil, err
}
@@ -105,27 +124,38 @@ func openIdxFile(root *os.Root, idxName, packName string, algo objectid.Algorith
file: file,
data: data,
}
- if err := index.parse(); err != nil {
+
+ err = index.parse()
+ if err != nil {
_ = index.close()
+
return nil, err
}
+
return index, nil
}
// close unmaps and closes one idx handle.
func (index *idxFile) close() error {
var closeErr error
+
if index.data != nil {
- if err := syscall.Munmap(index.data); err != nil && closeErr == nil {
+ err := syscall.Munmap(index.data)
+ if err != nil && closeErr == nil {
closeErr = err
}
+
index.data = nil
}
+
if index.file != nil {
- if err := index.file.Close(); err != nil && closeErr == nil {
+ err := index.file.Close()
+ if err != nil && closeErr == nil {
closeErr = err
}
+
index.file = nil
}
+
return closeErr
}
diff --git a/objectstore/packed/idx_parse.go b/objectstore/packed/idx_parse.go
index 0af72594..870ffdae 100644
--- a/objectstore/packed/idx_parse.go
+++ b/objectstore/packed/idx_parse.go
@@ -19,27 +19,34 @@ func (index *idxFile) parse() error {
if hashSize <= 0 {
return fmt.Errorf("objectstore/packed: idx %q has invalid hash algorithm", index.idxName)
}
+
minLen := 8 + 256*4 + 2*hashSize
if len(index.data) < minLen {
return fmt.Errorf("objectstore/packed: idx %q too short", index.idxName)
}
+
if binary.BigEndian.Uint32(index.data[:4]) != idxMagicV2 {
return fmt.Errorf("objectstore/packed: idx %q invalid magic", index.idxName)
}
+
if binary.BigEndian.Uint32(index.data[4:8]) != idxVersionV2 {
return fmt.Errorf("objectstore/packed: idx %q unsupported version", index.idxName)
}
prev := uint32(0)
+
for i := range 256 {
base := 8 + i*4
+
cur := binary.BigEndian.Uint32(index.data[base : base+4])
if cur < prev {
return fmt.Errorf("objectstore/packed: idx %q has non-monotonic fanout table", index.idxName)
}
+
index.fanout[i] = cur
prev = cur
}
+
index.numObjects = int(index.fanout[255])
if index.numObjects < 0 {
return fmt.Errorf("objectstore/packed: idx %q has invalid object count", index.idxName)
@@ -48,6 +55,7 @@ func (index *idxFile) parse() error {
namesBytes := index.numObjects * hashSize
crcBytes := index.numObjects * 4
offset32Bytes := index.numObjects * 4
+
minSize := 8 + 256*4 + namesBytes + crcBytes + offset32Bytes + 2*hashSize
if minSize < 0 || len(index.data) < minSize {
return fmt.Errorf("objectstore/packed: idx %q has truncated tables", index.idxName)
@@ -61,11 +69,14 @@ func (index *idxFile) parse() error {
if offset64Bytes < 0 || offset64Bytes%8 != 0 {
return fmt.Errorf("objectstore/packed: idx %q has malformed 64-bit offset table", index.idxName)
}
+
index.offset64Count = offset64Bytes / 8
+
maxOffset64Count := max(index.numObjects-1, 0)
if index.offset64Count > maxOffset64Count {
return fmt.Errorf("objectstore/packed: idx %q has oversized 64-bit offset table", index.idxName)
}
+
return nil
}
@@ -74,17 +85,21 @@ func (index *idxFile) lookup(id objectid.ObjectID) (uint64, bool, error) {
if id.Algorithm() != index.algo {
return 0, false, fmt.Errorf("objectstore/packed: object id algorithm mismatch")
}
+
idBytes := (&id).RawBytes()
+
hashSize := len(idBytes)
if hashSize != index.algo.Size() {
return 0, false, fmt.Errorf("objectstore/packed: unexpected object id length")
}
first := int(idBytes[0])
+
lo := 0
if first > 0 {
lo = int(index.fanout[first-1])
}
+
hi := int(index.fanout[first])
if lo < 0 || hi < 0 || lo > hi || hi > index.numObjects {
return 0, false, fmt.Errorf("objectstore/packed: idx %q has invalid fanout bounds", index.idxName)
@@ -92,24 +107,29 @@ func (index *idxFile) lookup(id objectid.ObjectID) (uint64, bool, error) {
for lo < hi {
mid := lo + (hi-lo)/2
+
nameOffset := index.namesOffset + mid*hashSize
if nameOffset < 0 || nameOffset+hashSize > len(index.data) {
return 0, false, fmt.Errorf("objectstore/packed: idx %q truncated name table", index.idxName)
}
+
cmp := bytes.Compare(index.data[nameOffset:nameOffset+hashSize], idBytes)
if cmp == 0 {
offset, err := index.offsetAt(mid)
if err != nil {
return 0, false, err
}
+
return offset, true, nil
}
+
if cmp < 0 {
lo = mid + 1
} else {
hi = mid
}
}
+
return 0, false, nil
}
@@ -118,10 +138,12 @@ func (index *idxFile) offsetAt(objectIndex int) (uint64, error) {
if objectIndex < 0 || objectIndex >= index.numObjects {
return 0, fmt.Errorf("objectstore/packed: idx %q offset index out of bounds", index.idxName)
}
+
wordOffset := index.offset32Offset + objectIndex*4
if wordOffset < 0 || wordOffset+4 > len(index.data) {
return 0, fmt.Errorf("objectstore/packed: idx %q truncated 32-bit offset table", index.idxName)
}
+
word := binary.BigEndian.Uint32(index.data[wordOffset : wordOffset+4])
if word&0x80000000 == 0 {
return uint64(word), nil
@@ -131,9 +153,11 @@ func (index *idxFile) offsetAt(objectIndex int) (uint64, error) {
if pos < 0 || pos >= index.offset64Count {
return 0, fmt.Errorf("objectstore/packed: idx %q invalid 64-bit offset position", index.idxName)
}
+
offOffset := index.offset64Offset + pos*8
if offOffset < 0 || offOffset+8 > len(index.data)-2*index.algo.Size() {
return 0, fmt.Errorf("objectstore/packed: idx %q truncated 64-bit offset table", index.idxName)
}
+
return binary.BigEndian.Uint64(index.data[offOffset : offOffset+8]), nil
}
diff --git a/objectstore/packed/pack.go b/objectstore/packed/pack.go
index 9af4c860..874b2b76 100644
--- a/objectstore/packed/pack.go
+++ b/objectstore/packed/pack.go
@@ -25,43 +25,58 @@ func openPackFile(name string, file *os.File, size int64) (*packFile, error) {
if size < 12 {
return nil, fmt.Errorf("objectstore/packed: pack %q too short", name)
}
+
if size > int64(int(^uint(0)>>1)) {
return nil, fmt.Errorf("objectstore/packed: pack %q has unsupported size", name)
}
+
fd, err := intconv.UintptrToInt(file.Fd())
if err != nil {
return nil, err
}
+
data, err := syscall.Mmap(fd, 0, int(size), syscall.PROT_READ, syscall.MAP_PRIVATE)
if err != nil {
return nil, err
}
+
if binary.BigEndian.Uint32(data[:4]) != packfmt.Signature {
_ = syscall.Munmap(data)
+
return nil, fmt.Errorf("objectstore/packed: pack %q invalid signature", name)
}
+
version := binary.BigEndian.Uint32(data[4:8])
if !packfmt.VersionSupported(version) {
_ = syscall.Munmap(data)
+
return nil, fmt.Errorf("objectstore/packed: pack %q unsupported version %d", name, version)
}
+
return &packFile{name: name, file: file, data: data}, nil
}
// close unmaps and closes one pack handle.
func (pack *packFile) close() error {
var closeErr error
+
if pack.data != nil {
- if err := syscall.Munmap(pack.data); err != nil && closeErr == nil {
+ err := syscall.Munmap(pack.data)
+ if err != nil && closeErr == nil {
closeErr = err
}
+
pack.data = nil
}
+
if pack.file != nil {
- if err := pack.file.Close(); err != nil && closeErr == nil {
+ err := pack.file.Close()
+ if err != nil && closeErr == nil {
closeErr = err
}
+
pack.file = nil
}
+
return closeErr
}
diff --git a/objectstore/packed/pack_idx_checksum.go b/objectstore/packed/pack_idx_checksum.go
index 2f55a469..25556088 100644
--- a/objectstore/packed/pack_idx_checksum.go
+++ b/objectstore/packed/pack_idx_checksum.go
@@ -14,17 +14,21 @@ func verifyMappedPackMatchesMappedIdx(packData, idxData []byte, algo objectid.Al
if hashSize <= 0 {
return objectid.ErrInvalidAlgorithm
}
+
if len(packData) < hashSize {
return fmt.Errorf("objectstore/packed: pack too short for trailer hash")
}
+
if len(idxData) < hashSize*2 {
return fmt.Errorf("objectstore/packed: idx too short for trailer hashes")
}
packTrailerHash := packData[len(packData)-hashSize:]
+
idxPackHash := idxData[len(idxData)-hashSize*2 : len(idxData)-hashSize]
if !bytes.Equal(packTrailerHash, idxPackHash) {
return fmt.Errorf("objectstore/packed: pack hash does not match idx")
}
+
return nil
}
diff --git a/objectstore/packed/read_bytes.go b/objectstore/packed/read_bytes.go
index b6f42a0d..e272b626 100644
--- a/objectstore/packed/read_bytes.go
+++ b/objectstore/packed/read_bytes.go
@@ -14,6 +14,7 @@ func (store *Store) ReadBytesContent(id objectid.ObjectID) (objecttype.Type, []b
if err != nil {
return objecttype.TypeInvalid, nil, err
}
+
return store.deltaResolveContent(loc)
}
@@ -23,12 +24,15 @@ func (store *Store) ReadBytesFull(id objectid.ObjectID) ([]byte, error) {
if err != nil {
return nil, err
}
+
header, ok := objectheader.Encode(ty, int64(len(content)))
if !ok {
return nil, fmt.Errorf("objectstore/packed: failed to encode object header for type %d", ty)
}
+
out := make([]byte, len(header)+len(content))
copy(out, header)
copy(out[len(header):], content)
+
return out, nil
}
diff --git a/objectstore/packed/read_header.go b/objectstore/packed/read_header.go
index 6822975c..5eb37c92 100644
--- a/objectstore/packed/read_header.go
+++ b/objectstore/packed/read_header.go
@@ -11,5 +11,6 @@ func (store *Store) ReadHeader(id objectid.ObjectID) (objecttype.Type, int64, er
if err != nil {
return objecttype.TypeInvalid, 0, err
}
+
return store.resolveHeaderAt(loc)
}
diff --git a/objectstore/packed/read_header_resolve.go b/objectstore/packed/read_header_resolve.go
index cf49fe2b..420d9363 100644
--- a/objectstore/packed/read_header_resolve.go
+++ b/objectstore/packed/read_header_resolve.go
@@ -17,12 +17,14 @@ func (store *Store) resolveHeaderAt(start location) (objecttype.Type, int64, err
if _, ok := visited[current]; ok {
return objecttype.TypeInvalid, 0, fmt.Errorf("objectstore/packed: delta cycle while resolving object header")
}
+
visited[current] = struct{}{}
pack, meta, err := store.entryMetaAt(current)
if err != nil {
return objecttype.TypeInvalid, 0, err
}
+
if declaredSize < 0 {
if packfmt.IsBaseObjectType(meta.ty) {
declaredSize = meta.size
@@ -31,9 +33,11 @@ func (store *Store) resolveHeaderAt(start location) (objecttype.Type, int64, err
if err != nil {
return objecttype.TypeInvalid, 0, err
}
+
declaredSize = size
}
}
+
if packfmt.IsBaseObjectType(meta.ty) {
return meta.ty, declaredSize, nil
}
@@ -44,6 +48,7 @@ func (store *Store) resolveHeaderAt(start location) (objecttype.Type, int64, err
if err != nil {
return objecttype.TypeInvalid, 0, err
}
+
current = next
case objecttype.TypeOfsDelta:
current = location{
diff --git a/objectstore/packed/read_reader.go b/objectstore/packed/read_reader.go
index a1f24799..d8dfdca9 100644
--- a/objectstore/packed/read_reader.go
+++ b/objectstore/packed/read_reader.go
@@ -41,11 +41,13 @@ func (store *Store) ReadReaderContent(id objectid.ObjectID) (objecttype.Type, in
if err != nil {
return objecttype.TypeInvalid, 0, nil, err
}
+
if packfmt.IsBaseObjectType(meta.ty) {
zr, err := zlibReaderAt(pack, meta.dataOffset)
if err != nil {
return objecttype.TypeInvalid, 0, nil, err
}
+
return meta.ty, meta.size, &readCloser{
reader: iolimit.ExpectLengthReader(zr, meta.size),
closer: zr,
@@ -56,6 +58,7 @@ func (store *Store) ReadReaderContent(id objectid.ObjectID) (objecttype.Type, in
if err != nil {
return objecttype.TypeInvalid, 0, nil, err
}
+
return ty, int64(len(content)), io.NopCloser(bytes.NewReader(content)), nil
}
@@ -72,15 +75,18 @@ func (store *Store) ReadReaderFull(id objectid.ObjectID) (io.ReadCloser, error)
if err != nil {
return nil, err
}
+
if packfmt.IsBaseObjectType(meta.ty) {
header, ok := objectheader.Encode(meta.ty, meta.size)
if !ok {
return nil, fmt.Errorf("objectstore/packed: failed to encode object header for type %d", meta.ty)
}
+
zr, err := zlibReaderAt(pack, meta.dataOffset)
if err != nil {
return nil, err
}
+
return &readCloser{
reader: io.MultiReader(bytes.NewReader(header), iolimit.ExpectLengthReader(zr, meta.size)),
closer: zr,
@@ -91,5 +97,6 @@ func (store *Store) ReadReaderFull(id objectid.ObjectID) (io.ReadCloser, error)
if err != nil {
return nil, err
}
+
return io.NopCloser(bytes.NewReader(raw)), nil
}
diff --git a/objectstore/packed/read_size.go b/objectstore/packed/read_size.go
index e162586a..a0a75db7 100644
--- a/objectstore/packed/read_size.go
+++ b/objectstore/packed/read_size.go
@@ -14,6 +14,7 @@ func (store *Store) ReadSize(id objectid.ObjectID) (int64, error) {
if err != nil {
return 0, err
}
+
return store.resolveSizeAt(loc)
}
@@ -23,9 +24,11 @@ func (store *Store) resolveSizeAt(start location) (int64, error) {
if err != nil {
return 0, err
}
+
if packfmt.IsBaseObjectType(meta.ty) {
return meta.size, nil
}
+
switch meta.ty {
case objecttype.TypeRefDelta, objecttype.TypeOfsDelta:
return deltaDeclaredSizeAt(pack, meta.dataOffset)
diff --git a/objectstore/packed/read_test.go b/objectstore/packed/read_test.go
index 9bfa6610..9ba89fdf 100644
--- a/objectstore/packed/read_test.go
+++ b/objectstore/packed/read_test.go
@@ -30,16 +30,20 @@ func TestPackedStoreReadAgainstGit(t *testing.T) {
if err != nil {
t.Fatalf("ReadHeader: %v", err)
}
+
if gotHeaderType != wantType {
t.Fatalf("ReadHeader type = %v, want %v", gotHeaderType, wantType)
}
+
if gotHeaderSize != int64(len(wantBody)) {
t.Fatalf("ReadHeader size = %d, want %d", gotHeaderSize, len(wantBody))
}
+
gotSize, err := store.ReadSize(id)
if err != nil {
t.Fatalf("ReadSize: %v", err)
}
+
if gotSize != int64(len(wantBody)) {
t.Fatalf("ReadSize = %d, want %d", gotSize, len(wantBody))
}
@@ -48,6 +52,7 @@ func TestPackedStoreReadAgainstGit(t *testing.T) {
if err != nil {
t.Fatalf("ReadBytesFull: %v", err)
}
+
if !bytes.Equal(gotRaw, wantRaw) {
t.Fatalf("ReadBytesFull mismatch")
}
@@ -56,9 +61,11 @@ func TestPackedStoreReadAgainstGit(t *testing.T) {
if err != nil {
t.Fatalf("ReadBytesContent: %v", err)
}
+
if gotType != wantType {
t.Fatalf("ReadBytesContent type = %v, want %v", gotType, wantType)
}
+
if !bytes.Equal(gotBody, wantBody) {
t.Fatalf("ReadBytesContent mismatch")
}
@@ -67,7 +74,9 @@ func TestPackedStoreReadAgainstGit(t *testing.T) {
if err != nil {
t.Fatalf("ReadReaderFull: %v", err)
}
- if got := mustReadAllAndClose(t, fullReader); !bytes.Equal(got, wantRaw) {
+
+ got := mustReadAllAndClose(t, fullReader)
+ if !bytes.Equal(got, wantRaw) {
t.Fatalf("ReadReaderFull mismatch")
}
@@ -75,13 +84,17 @@ func TestPackedStoreReadAgainstGit(t *testing.T) {
if err != nil {
t.Fatalf("ReadReaderContent: %v", err)
}
+
if contentType != wantType {
t.Fatalf("ReadReaderContent type = %v, want %v", contentType, wantType)
}
+
if contentSize != int64(len(wantBody)) {
t.Fatalf("ReadReaderContent size = %d, want %d", contentSize, len(wantBody))
}
- if got := mustReadAllAndClose(t, contentReader); !bytes.Equal(got, wantBody) {
+
+ got = mustReadAllAndClose(t, contentReader)
+ if !bytes.Equal(got, wantBody) {
t.Fatalf("ReadReaderContent mismatch")
}
})
@@ -100,38 +113,54 @@ func TestPackedStoreErrors(t *testing.T) {
t.Fatalf("ParseHex(notFound): %v", err)
}
- if _, err := store.ReadBytesFull(notFoundID); !errors.Is(err, objectstore.ErrObjectNotFound) {
+ _, err = store.ReadBytesFull(notFoundID)
+ if !errors.Is(err, objectstore.ErrObjectNotFound) {
t.Fatalf("ReadBytesFull not-found error = %v", err)
}
- if _, _, err := store.ReadBytesContent(notFoundID); !errors.Is(err, objectstore.ErrObjectNotFound) {
+
+ _, _, err = store.ReadBytesContent(notFoundID)
+ if !errors.Is(err, objectstore.ErrObjectNotFound) {
t.Fatalf("ReadBytesContent not-found error = %v", err)
}
- if _, err := store.ReadReaderFull(notFoundID); !errors.Is(err, objectstore.ErrObjectNotFound) {
+
+ _, err = store.ReadReaderFull(notFoundID)
+ if !errors.Is(err, objectstore.ErrObjectNotFound) {
t.Fatalf("ReadReaderFull not-found error = %v", err)
}
- if _, _, _, err := store.ReadReaderContent(notFoundID); !errors.Is(err, objectstore.ErrObjectNotFound) {
+
+ _, _, _, err = store.ReadReaderContent(notFoundID)
+ if !errors.Is(err, objectstore.ErrObjectNotFound) {
t.Fatalf("ReadReaderContent not-found error = %v", err)
}
- if _, _, err := store.ReadHeader(notFoundID); !errors.Is(err, objectstore.ErrObjectNotFound) {
+
+ _, _, err = store.ReadHeader(notFoundID)
+ if !errors.Is(err, objectstore.ErrObjectNotFound) {
t.Fatalf("ReadHeader not-found error = %v", err)
}
- if _, err := store.ReadSize(notFoundID); !errors.Is(err, objectstore.ErrObjectNotFound) {
+
+ _, err = store.ReadSize(notFoundID)
+ if !errors.Is(err, objectstore.ErrObjectNotFound) {
t.Fatalf("ReadSize not-found error = %v", err)
}
var otherAlgo objectid.Algorithm
+
for _, candidate := range objectid.SupportedAlgorithms() {
if candidate != algo {
otherAlgo = candidate
+
break
}
}
+
if otherAlgo != objectid.AlgorithmUnknown {
mismatchID, err := objectid.ParseHex(otherAlgo, strings.Repeat("0", otherAlgo.HexLen()))
if err != nil {
t.Fatalf("ParseHex(mismatch): %v", err)
}
- if _, err := store.ReadBytesFull(mismatchID); err == nil || !strings.Contains(err.Error(), "algorithm mismatch") {
+
+ _, err = store.ReadBytesFull(mismatchID)
+ if err == nil || !strings.Contains(err.Error(), "algorithm mismatch") {
t.Fatalf("ReadBytesFull algorithm-mismatch error = %v", err)
}
}
@@ -141,11 +170,16 @@ func TestPackedStoreErrors(t *testing.T) {
func TestPackedStoreNewValidation(t *testing.T) {
t.Parallel()
testRepo, _ := createPackedFixtureRepo(t, objectid.AlgorithmSHA1)
+
store := openPackedStore(t, testRepo.Dir(), objectid.AlgorithmSHA1)
- if err := store.Close(); err != nil {
+
+ err := store.Close()
+ if err != nil {
t.Fatalf("Close: %v", err)
}
- if err := store.Close(); err != nil {
+
+ err = store.Close()
+ if err != nil {
t.Fatalf("Close second: %v", err)
}
}
@@ -153,13 +187,16 @@ func TestPackedStoreNewValidation(t *testing.T) {
func TestPackedStoreInvalidAlgorithm(t *testing.T) {
t.Parallel()
testRepo := testgit.NewRepo(t, testgit.RepoOptions{ObjectFormat: objectid.AlgorithmSHA1, Bare: true})
+
root, err := os.OpenRoot(testRepo.Dir())
if err != nil {
t.Fatalf("OpenRoot(%q): %v", testRepo.Dir(), err)
}
+
t.Cleanup(func() { _ = root.Close() })
- if _, err := packed.New(root, objectid.AlgorithmUnknown); !errors.Is(err, objectid.ErrInvalidAlgorithm) {
+ _, err = packed.New(root, objectid.AlgorithmUnknown)
+ if !errors.Is(err, objectid.ErrInvalidAlgorithm) {
t.Fatalf("packed.New invalid algorithm error = %v", err)
}
}
@@ -170,15 +207,20 @@ func TestPackedStoreReadHeaderUsesResolvedObjectSizeForDelta(t *testing.T) {
testRepo := testgit.NewRepo(t, testgit.RepoOptions{ObjectFormat: algo, Bare: true})
var parent objectid.ObjectID
+
for i := range 96 {
content := strings.Repeat("common-line-"+strconv.Itoa(i%7)+"\n", 384) + fmt.Sprintf("tail-%03d\n", i)
+
_, treeID := testRepo.MakeSingleFileTree(t, "file.txt", []byte(content))
if i == 0 {
parent = testRepo.CommitTree(t, treeID, "delta-header-size-0")
+
continue
}
+
parent = testRepo.CommitTree(t, treeID, fmt.Sprintf("delta-header-size-%03d", i), parent)
}
+
testRepo.UpdateRef(t, "refs/heads/main", parent)
testRepo.Repack(t, "-a", "-d", "-f", "--window=128", "--depth=128")
@@ -189,13 +231,16 @@ func TestPackedStoreReadHeaderUsesResolvedObjectSizeForDelta(t *testing.T) {
if err != nil {
t.Fatalf("ReadHeader(%s): %v", deltaID, err)
}
+
if gotSize != wantResolvedSize {
t.Fatalf("ReadHeader(%s) size = %d, want resolved size %d", deltaID, gotSize, wantResolvedSize)
}
+
gotReadSize, err := store.ReadSize(deltaID)
if err != nil {
t.Fatalf("ReadSize(%s): %v", deltaID, err)
}
+
if gotReadSize != wantResolvedSize {
t.Fatalf("ReadSize(%s) = %d, want resolved size %d", deltaID, gotReadSize, wantResolvedSize)
}
@@ -209,6 +254,7 @@ func findDeltaObjectWithResolvedSizeMismatch(t *testing.T, testRepo *testgit.Tes
if err != nil {
t.Fatalf("Glob idx: %v", err)
}
+
if len(idxFiles) == 0 {
t.Fatalf("no idx files found")
}
@@ -221,16 +267,19 @@ func findDeltaObjectWithResolvedSizeMismatch(t *testing.T, testRepo *testgit.Tes
}
idHex := fields[0]
+
deltaStreamSize, err := strconv.ParseInt(fields[2], 10, 64)
if err != nil {
continue
}
resolvedSizeStr := testRepo.Run(t, "cat-file", "-s", idHex)
+
resolvedSize, err := strconv.ParseInt(strings.TrimSpace(resolvedSizeStr), 10, 64)
if err != nil {
t.Fatalf("parse cat-file size for %s: %v", idHex, err)
}
+
if deltaStreamSize == resolvedSize {
continue
}
@@ -239,9 +288,11 @@ func findDeltaObjectWithResolvedSizeMismatch(t *testing.T, testRepo *testgit.Tes
if err != nil {
t.Fatalf("ParseHex(%s): %v", idHex, err)
}
+
return id, resolvedSize
}
t.Fatalf("did not find a delta object with mismatched stream/resolved size")
+
return objectid.ObjectID{}, 0
}
diff --git a/objectstore/packed/store.go b/objectstore/packed/store.go
index abd7175f..d28113d1 100644
--- a/objectstore/packed/store.go
+++ b/objectstore/packed/store.go
@@ -60,6 +60,7 @@ func New(root *os.Root, algo objectid.Algorithm) (*Store, error) {
if algo.Size() == 0 {
return nil, objectid.ErrInvalidAlgorithm
}
+
return &Store{
root: root,
algo: algo,
@@ -76,8 +77,10 @@ func (store *Store) Close() error {
store.stateMu.Lock()
if store.closed {
store.stateMu.Unlock()
+
return nil
}
+
store.closed = true
root := store.root
packs := store.packs
@@ -87,23 +90,30 @@ func (store *Store) Close() error {
store.idxMu.RUnlock()
var closeErr error
+
for _, pack := range packs {
- if err := pack.close(); err != nil && closeErr == nil {
+ err := pack.close()
+ if err != nil && closeErr == nil {
closeErr = err
}
}
+
for _, index := range indexes {
- if err := index.close(); err != nil && closeErr == nil {
+ err := index.close()
+ if err != nil && closeErr == nil {
closeErr = err
}
}
+
store.cacheMu.Lock()
store.deltaCache.clear()
store.cacheMu.Unlock()
- if err := root.Close(); err != nil && closeErr == nil {
+ err := root.Close()
+ if err != nil && closeErr == nil {
closeErr = err
}
+
return closeErr
}
@@ -113,7 +123,9 @@ func (store *Store) lookup(id objectid.ObjectID) (location, error) {
if id.Algorithm() != store.algo {
return zero, errors.New("objectstore/packed: object id algorithm mismatch")
}
- if err := store.ensureCandidates(); err != nil {
+
+ err := store.ensureCandidates()
+ if err != nil {
return zero, err
}
@@ -122,81 +134,111 @@ func (store *Store) lookup(id objectid.ObjectID) (location, error) {
candidate, ok := store.candidateForPack(nextPackName)
if !ok {
nextPackName = store.firstCandidatePackName()
+
continue
}
+
nextPackName = store.nextCandidatePackName(candidate.packName)
+
index, err := store.openIndex(candidate)
if err != nil {
return zero, err
}
+
offset, ok, err := index.lookup(id)
if err != nil {
return zero, err
}
+
if ok {
store.touchCandidate(candidate.packName)
+
return location{packName: index.packName, offset: offset}, nil
}
}
+
return zero, objectstore.ErrObjectNotFound
}
// openPack returns one opened and validated pack handle.
func (store *Store) openPack(name string) (*packFile, error) {
store.stateMu.RLock()
- if pack, ok := store.packs[name]; ok {
+
+ pack, ok := store.packs[name]
+ if ok {
store.stateMu.RUnlock()
+
return pack, nil
}
+
store.stateMu.RUnlock()
file, err := store.root.Open(name)
if err != nil {
return nil, err
}
+
info, err := file.Stat()
if err != nil {
_ = file.Close()
+
return nil, err
}
- pack, err := openPackFile(name, file, info.Size())
+
+ pack, err = openPackFile(name, file, info.Size())
if err != nil {
_ = file.Close()
+
return nil, err
}
- if err := store.verifyPackMatchesIndexes(pack); err != nil {
+
+ err = store.verifyPackMatchesIndexes(pack)
+ if err != nil {
_ = pack.close()
+
return nil, err
}
store.stateMu.Lock()
- if existing, ok := store.packs[name]; ok {
+
+ existing, ok := store.packs[name]
+ if ok {
store.stateMu.Unlock()
+
_ = pack.close()
+
return existing, nil
}
+
store.packs[name] = pack
store.stateMu.Unlock()
+
return pack, nil
}
// verifyPackMatchesIndexes checks that one opened pack's trailer hash matches
// every loaded index that references the same pack name.
func (store *Store) verifyPackMatchesIndexes(pack *packFile) error {
- if err := store.ensureCandidates(); err != nil {
+ err := store.ensureCandidates()
+ if err != nil {
return err
}
+
candidate, ok := store.candidateForPack(pack.name)
if !ok {
return fmt.Errorf("objectstore/packed: missing index for pack %q", pack.name)
}
+
index, err := store.openIndex(candidate)
if err != nil {
return err
}
- if err := verifyMappedPackMatchesMappedIdx(pack.data, index.data, store.algo); err != nil {
+
+ err = verifyMappedPackMatchesMappedIdx(pack.data, index.data, store.algo)
+ if err != nil {
return fmt.Errorf("objectstore/packed: pack %q does not match idx %q: %w", pack.name, index.idxName, err)
}
+
return nil
}
@@ -206,9 +248,11 @@ func (store *Store) entryMetaAt(loc location) (*packFile, entryMeta, error) {
if err != nil {
return nil, entryMeta{}, err
}
+
meta, err := parseEntryMeta(pack, store.algo, loc.offset)
if err != nil {
return nil, entryMeta{}, err
}
+
return pack, meta, nil
}