objectstore/packed: Lazily parse idx metadata

1 files changed, 106 insertions, 16 deletions

diff --git a/objectstore/packed/idx_load.go b/objectstore/packed/idx_load.go
index 293e005f..35e8b925 100644
--- a/objectstore/packed/idx_load.go
+++ b/objectstore/packed/idx_load.go
@@ -17,6 +17,16 @@ type location struct {
 	offset   uint64
 }
 
+// packCandidate describes one discovered pack/index pair.
+type packCandidate struct {
+	// packName is the .pack basename.
+	packName string
+	// idxName is the .idx basename.
+	idxName string
+	// mtime is the pack file modification time for initial ordering.
+	mtime int64
+}
+
 // idxFile stores one mapped and validated idx v2 file.
 type idxFile struct {
 	// idxName is the basename of this .idx file.
@@ -46,8 +56,30 @@ type idxFile struct {
 	offset64Count int
 }
 
-// loadIndexes loads and validates all .idx files under objects/pack.
-func (store *Store) loadIndexes() ([]*idxFile, error) {
+// ensureCandidates discovers pack/index pairs once.
+func (store *Store) ensureCandidates() error {
+	store.discoverOnce.Do(func() {
+		candidates, err := store.discoverCandidates()
+		candidateByPack := make(map[string]packCandidate, len(candidates))
+		for _, candidate := range candidates {
+			candidateByPack[candidate.packName] = candidate
+		}
+		store.stateMu.Lock()
+		store.candidates = candidates
+		store.candidateByPack = candidateByPack
+		store.discoverErr = err
+		store.stateMu.Unlock()
+	})
+
+	store.stateMu.RLock()
+	err := store.discoverErr
+	store.stateMu.RUnlock()
+	return err
+}
+
+// discoverCandidates scans the objects/pack root and returns sorted pack/index
+// pairs.
+func (store *Store) discoverCandidates() ([]packCandidate, error) {
 	dir, err := store.root.Open(".")
 	if err != nil {
 		if os.IsNotExist(err) {
@@ -56,39 +88,97 @@ func (store *Store) loadIndexes() ([]*idxFile, error) {
 		return nil, err
 	}
 	defer func() { _ = dir.Close() }()
+
 	entries, err := dir.ReadDir(-1)
 	if err != nil {
 		return nil, err
 	}
 
-	idxNames := make([]string, 0, len(entries))
+	candidates := make([]packCandidate, 0, len(entries))
 	for _, entry := range entries {
 		if entry.IsDir() || !strings.HasSuffix(entry.Name(), ".idx") {
 			continue
 		}
-		idxNames = append(idxNames, entry.Name())
-	}
-	slices.Sort(idxNames)
 
-	out := make([]*idxFile, 0, len(idxNames))
-	for _, idxName := range idxNames {
+		idxName := entry.Name()
 		packName := strings.TrimSuffix(idxName, ".idx") + ".pack"
-		if _, err := store.root.Stat(packName); err != nil {
+		packInfo, err := store.root.Stat(packName)
+		if err != nil {
 			if os.IsNotExist(err) {
 				return nil, fmt.Errorf("objectstore/packed: missing pack file for index %q", idxName)
 			}
 			return nil, err
 		}
-		index, err := openIdxFile(store.root, idxName, packName, store.algo)
-		if err != nil {
-			for _, loaded := range out {
-				_ = loaded.close()
+
+		candidates = append(candidates, packCandidate{
+			packName: packName,
+			idxName:  idxName,
+			mtime:    packInfo.ModTime().UnixNano(),
+		})
+	}
+
+	slices.SortFunc(candidates, func(a, b packCandidate) int {
+		if a.mtime != b.mtime {
+			if a.mtime > b.mtime {
+				return -1
 			}
-			return nil, err
+			return 1
+		}
+		return strings.Compare(a.packName, b.packName)
+	})
+
+	return candidates, nil
+}
+
+// candidateForPack returns one discovered candidate for a pack basename.
+func (store *Store) candidateForPack(packName string) (packCandidate, bool) {
+	store.stateMu.RLock()
+	candidate, ok := store.candidateByPack[packName]
+	store.stateMu.RUnlock()
+	return candidate, ok
+}
+
+// openIndex returns one opened and parsed index, caching it by pack basename.
+func (store *Store) openIndex(candidate packCandidate) (*idxFile, error) {
+	store.stateMu.RLock()
+	if index, ok := store.idxByPack[candidate.packName]; ok {
+		store.stateMu.RUnlock()
+		return index, nil
+	}
+	store.stateMu.RUnlock()
+
+	index, err := openIdxFile(store.root, candidate.idxName, candidate.packName, store.algo)
+	if err != nil {
+		return nil, err
+	}
+
+	store.stateMu.Lock()
+	if existing, ok := store.idxByPack[candidate.packName]; ok {
+		store.stateMu.Unlock()
+		_ = index.close()
+		return existing, nil
+	}
+	store.idxByPack[candidate.packName] = index
+	store.stateMu.Unlock()
+	return index, nil
+}
+
+// touchCandidate moves one candidate to the front of the lookup order.
+func (store *Store) touchCandidate(packName string) {
+	store.stateMu.Lock()
+	defer store.stateMu.Unlock()
+	for i := range store.candidates {
+		if store.candidates[i].packName != packName {
+			continue
+		}
+		if i == 0 {
+			return
 		}
-		out = append(out, index)
+		candidate := store.candidates[i]
+		copy(store.candidates[1:i+1], store.candidates[0:i])
+		store.candidates[0] = candidate
+		return
 	}
-	return out, nil
 }
 
 // openIdxFile maps and validates one idx v2 file.