objectstore: Refresh v0.1.80

* Add manual Refresh for various objectstore's
* RefreshPolicy option
* Refreshable MRU and atomic snapshotting

10 files changed, 238 insertions, 109 deletions

diff --git a/objectstore/packed/helpers_test.go b/objectstore/packed/helpers_test.go
index 581c0dd7..f07ff4e3 100644
--- a/objectstore/packed/helpers_test.go
+++ b/objectstore/packed/helpers_test.go
@@ -19,7 +19,7 @@ func openPackedStore(t *testing.T, testRepo *testgit.TestRepo, algo objectid.Alg
 
 	root := testRepo.OpenPackRoot(t)
 
-	store, err := packed.New(root, algo)
+	store, err := packed.New(root, algo, packed.Options{})
 	if err != nil {
 		t.Fatalf("packed.New: %v", err)
 	}
diff --git a/objectstore/packed/idx_candidate.go b/objectstore/packed/idx_candidate.go
deleted file mode 100644
index 2f2ad7a9..00000000
--- a/objectstore/packed/idx_candidate.go
+++ /dev/null
@@ -1,10 +0,0 @@
-package packed
-
-// candidateForPack returns one discovered candidate for a pack basename.
-func (store *Store) candidateForPack(packName string) (packCandidate, bool) {
-	store.candidatesMu.RLock()
-	candidate, ok := store.candidateByPack[packName]
-	store.candidatesMu.RUnlock()
-
-	return candidate, ok
-}
diff --git a/objectstore/packed/idx_candidates_mru.go b/objectstore/packed/idx_candidates_mru.go
index 593e84f4..b0960df5 100644
--- a/objectstore/packed/idx_candidates_mru.go
+++ b/objectstore/packed/idx_candidates_mru.go
@@ -2,21 +2,76 @@ package packed
 
 // packCandidateNode is one node in the candidate MRU order list.
 type packCandidateNode struct {
-	candidate packCandidate
-	prev      *packCandidateNode
-	next      *packCandidateNode
+	packName string
+	prev     *packCandidateNode
+	next     *packCandidateNode
+}
+
+func (store *Store) reconcileMRU(candidates []packCandidate) {
+	store.mruMu.Lock()
+	defer store.mruMu.Unlock()
+
+	if store.mruNodeByPack == nil {
+		store.mruNodeByPack = make(map[string]*packCandidateNode, len(candidates))
+	}
+
+	present := make(map[string]struct{}, len(candidates))
+	for _, candidate := range candidates {
+		present[candidate.packName] = struct{}{}
+	}
+
+	ordered := make([]string, 0, len(candidates))
+
+	for node := store.mruHead; node != nil; node = node.next {
+		if _, ok := present[node.packName]; !ok {
+			continue
+		}
+
+		ordered = append(ordered, node.packName)
+		delete(present, node.packName)
+	}
+
+	for _, candidate := range candidates {
+		if _, ok := present[candidate.packName]; !ok {
+			continue
+		}
+
+		ordered = append(ordered, candidate.packName)
+		delete(present, candidate.packName)
+	}
+
+	store.mruHead = nil
+	store.mruTail = nil
+	store.mruNodeByPack = make(map[string]*packCandidateNode, len(ordered))
+
+	for _, packName := range ordered {
+		node := &packCandidateNode{
+			packName: packName,
+			prev:     store.mruTail,
+		}
+		if store.mruTail != nil {
+			store.mruTail.next = node
+		}
+
+		if store.mruHead == nil {
+			store.mruHead = node
+		}
+
+		store.mruTail = node
+		store.mruNodeByPack[packName] = node
+	}
 }
 
 // touchCandidate moves one candidate to the front of the lookup order.
 // This is done on a best-effort basis.
 func (store *Store) touchCandidate(packName string) {
-	if !store.candidatesMu.TryLock() {
+	if !store.mruMu.TryLock() {
 		return
 	}
-	defer store.candidatesMu.Unlock()
+	defer store.mruMu.Unlock()
 
-	node := store.candidateNodeByPack[packName]
-	if node == nil || node == store.candidateHead {
+	node := store.mruNodeByPack[packName]
+	if node == nil || node == store.mruHead {
 		return
 	}
 
@@ -28,46 +83,53 @@ func (store *Store) touchCandidate(packName string) {
 		node.next.prev = node.prev
 	}
 
-	if store.candidateTail == node {
-		store.candidateTail = node.prev
+	if store.mruTail == node {
+		store.mruTail = node.prev
 	}
 
 	node.prev = nil
-
-	node.next = store.candidateHead
-	if store.candidateHead != nil {
-		store.candidateHead.prev = node
+	node.next = store.mruHead
+	if store.mruHead != nil {
+		store.mruHead.prev = node
 	}
 
-	store.candidateHead = node
-	if store.candidateTail == nil {
-		store.candidateTail = node
+	store.mruHead = node
+	if store.mruTail == nil {
+		store.mruTail = node
 	}
 }
 
 // firstCandidatePackName returns the current head pack name, or "" when none
 // are available.
-func (store *Store) firstCandidatePackName() string {
-	store.candidatesMu.RLock()
-	defer store.candidatesMu.RUnlock()
+func (store *Store) firstCandidatePackName(snapshot *candidateSnapshot) string {
+	store.mruMu.RLock()
+	defer store.mruMu.RUnlock()
 
-	if store.candidateHead == nil {
-		return ""
+	for node := store.mruHead; node != nil; node = node.next {
+		if _, ok := snapshot.candidateByPack[node.packName]; ok {
+			return node.packName
+		}
 	}
 
-	return store.candidateHead.candidate.packName
+	return ""
 }
 
 // nextCandidatePackName returns the pack name after currentPack in current MRU
 // order, or "" at end / when currentPack is not present.
-func (store *Store) nextCandidatePackName(currentPack string) string {
-	store.candidatesMu.RLock()
-	defer store.candidatesMu.RUnlock()
+func (store *Store) nextCandidatePackName(currentPack string, snapshot *candidateSnapshot) string {
+	store.mruMu.RLock()
+	defer store.mruMu.RUnlock()
 
-	node := store.candidateNodeByPack[currentPack]
-	if node == nil || node.next == nil {
+	node := store.mruNodeByPack[currentPack]
+	if node == nil {
 		return ""
 	}
 
-	return node.next.candidate.packName
+	for node = node.next; node != nil; node = node.next {
+		if _, ok := snapshot.candidateByPack[node.packName]; ok {
+			return node.packName
+		}
+	}
+
+	return ""
 }
diff --git a/objectstore/packed/idx_lookup_candidates.go b/objectstore/packed/idx_lookup_candidates.go
index 9534476a..e69d07d7 100644
--- a/objectstore/packed/idx_lookup_candidates.go
+++ b/objectstore/packed/idx_lookup_candidates.go
@@ -17,50 +17,48 @@ type packCandidate struct {
 	mtime int64
 }
 
-// ensureCandidates discovers pack/index pairs once.
-func (store *Store) ensureCandidates() error {
-	store.discoverOnce.Do(func() {
-		candidates, err := store.discoverCandidates()
-		candidateByPack := make(map[string]packCandidate, len(candidates))
-		nodeByPack := make(map[string]*packCandidateNode, len(candidates))
+type candidateSnapshot struct {
+	candidates      []packCandidate
+	candidateByPack map[string]packCandidate
+}
 
-		var (
-			head *packCandidateNode
-			tail *packCandidateNode
-		)
+// Refresh rescans objects/pack and atomically installs a fresh candidate list.
+func (store *Store) Refresh() error {
+	store.refreshMu.Lock()
+	defer store.refreshMu.Unlock()
 
-		for _, candidate := range candidates {
-			node := &packCandidateNode{
-				candidate: candidate,
-				prev:      tail,
-			}
-			if tail != nil {
-				tail.next = node
-			}
+	candidates, err := store.discoverCandidates()
+	if err != nil {
+		return err
+	}
 
-			if head == nil {
-				head = node
-			}
+	candidateByPack := make(map[string]packCandidate, len(candidates))
+	for _, candidate := range candidates {
+		candidateByPack[candidate.packName] = candidate
+	}
 
-			tail = node
-			candidateByPack[candidate.packName] = candidate
-			nodeByPack[candidate.packName] = node
-		}
+	store.reconcileMRU(candidates)
 
-		store.candidatesMu.Lock()
-		store.candidateHead = head
-		store.candidateTail = tail
-		store.candidateByPack = candidateByPack
-		store.candidateNodeByPack = nodeByPack
-		store.discoverErr = err
-		store.candidatesMu.Unlock()
+	store.candidates.Store(&candidateSnapshot{
+		candidates:      candidates,
+		candidateByPack: candidateByPack,
 	})
 
-	store.candidatesMu.RLock()
-	err := store.discoverErr
-	store.candidatesMu.RUnlock()
+	return nil
+}
+
+func (store *Store) ensureCandidates() (*candidateSnapshot, error) {
+	snapshot := store.candidates.Load()
+	if snapshot != nil {
+		return snapshot, nil
+	}
+
+	err := store.Refresh()
+	if err != nil {
+		return nil, err
+	}
 
-	return err
+	return store.candidates.Load(), nil
 }
 
 // discoverCandidates scans the objects/pack root and returns sorted pack/index
diff --git a/objectstore/packed/new.go b/objectstore/packed/new.go
index 407bc1d0..c8e7338e 100644
--- a/objectstore/packed/new.go
+++ b/objectstore/packed/new.go
@@ -1,24 +1,31 @@
 package packed
 
 import (
+	"fmt"
 	"os"
 
 	"codeberg.org/lindenii/furgit/objectid"
 )
 
 // New creates a packed-object store rooted at an objects/pack directory.
-func New(root *os.Root, algo objectid.Algorithm) (*Store, error) {
+func New(root *os.Root, algo objectid.Algorithm, opts Options) (*Store, error) {
 	if algo.Size() == 0 {
 		return nil, objectid.ErrInvalidAlgorithm
 	}
 
+	switch opts.RefreshPolicy {
+	case RefreshPolicyOnMissing, RefreshPolicyNever:
+	default:
+		return nil, fmt.Errorf("objectstore/packed: invalid refresh policy %d", opts.RefreshPolicy)
+	}
+
 	return &Store{
-		root:                root,
-		algo:                algo,
-		candidateByPack:     make(map[string]packCandidate),
-		candidateNodeByPack: make(map[string]*packCandidateNode),
-		idxByPack:           make(map[string]*idxFile),
-		packs:               make(map[string]*packFile),
-		deltaCache:          newDeltaCache(defaultDeltaCacheMaxBytes),
+		root:          root,
+		algo:          algo,
+		refreshPolicy: opts.RefreshPolicy,
+		mruNodeByPack: make(map[string]*packCandidateNode),
+		idxByPack:     make(map[string]*idxFile),
+		packs:         make(map[string]*packFile),
+		deltaCache:    newDeltaCache(defaultDeltaCacheMaxBytes),
 	}, nil
 }
diff --git a/objectstore/packed/options.go b/objectstore/packed/options.go
new file mode 100644
index 00000000..05cbee30
--- /dev/null
+++ b/objectstore/packed/options.go
@@ -0,0 +1,16 @@
+package packed
+
+// RefreshPolicy configures when candidate pack/index discovery refreshes.
+type RefreshPolicy uint8
+
+const (
+	// RefreshPolicyOnMissing refreshes candidates once after a lookup miss.
+	RefreshPolicyOnMissing RefreshPolicy = iota
+	// RefreshPolicyNever disables automatic refresh after lookup misses.
+	RefreshPolicyNever
+)
+
+// Options configures a packed object store.
+type Options struct {
+	RefreshPolicy RefreshPolicy
+}
diff --git a/objectstore/packed/read_test.go b/objectstore/packed/read_test.go
index 435bc350..0e5da5d8 100644
--- a/objectstore/packed/read_test.go
+++ b/objectstore/packed/read_test.go
@@ -191,7 +191,7 @@ func TestPackedStoreInvalidAlgorithm(t *testing.T) {
 
 	root := testRepo.OpenPackRoot(t)
 
-	_, err := packed.New(root, objectid.AlgorithmUnknown)
+	_, err := packed.New(root, objectid.AlgorithmUnknown, packed.Options{})
 	if !errors.Is(err, objectid.ErrInvalidAlgorithm) {
 		t.Fatalf("packed.New invalid algorithm error = %v", err)
 	}
diff --git a/objectstore/packed/store.go b/objectstore/packed/store.go
index 000e04f2..1c6082f6 100644
--- a/objectstore/packed/store.go
+++ b/objectstore/packed/store.go
@@ -4,6 +4,7 @@ package packed
 import (
 	"os"
 	"sync"
+	"sync/atomic"
 
 	"codeberg.org/lindenii/furgit/objectid"
 	"codeberg.org/lindenii/furgit/objectstore"
@@ -17,26 +18,26 @@ type Store struct {
 	root *os.Root
 	// algo is the expected object ID algorithm for lookups.
 	algo objectid.Algorithm
+	// refreshPolicy controls automatic candidate refresh on lookup misses.
+	refreshPolicy RefreshPolicy
 
-	// discoverOnce guards one-time pack candidate discovery.
-	discoverOnce sync.Once
-	// discoverErr stores candidate discovery failures.
-	discoverErr error
-	// candidateHead is the first candidate in lookup priority order.
-	candidateHead *packCandidateNode
-	// candidateTail is the last candidate in lookup priority order.
-	candidateTail *packCandidateNode
-	// candidateByPack maps pack basename to discovered candidate.
-	candidateByPack map[string]packCandidate
-	// candidateNodeByPack maps pack basename to linked-list node.
-	candidateNodeByPack map[string]*packCandidateNode
+	// candidates stores the latest immutable candidate snapshot.
+	candidates atomic.Pointer[candidateSnapshot]
+	// refreshMu serializes candidate refresh.
+	refreshMu sync.Mutex
+	// mruMu guards candidate MRU linked-list state.
+	mruMu sync.RWMutex
+	// mruHead is the first pack in MRU order.
+	mruHead *packCandidateNode
+	// mruTail is the last pack in MRU order.
+	mruTail *packCandidateNode
+	// mruNodeByPack maps pack basename to MRU node.
+	mruNodeByPack map[string]*packCandidateNode
 	// idxByPack caches opened and parsed indexes by pack basename.
 	idxByPack map[string]*idxFile
 
 	// stateMu guards pack cache and close state.
 	stateMu sync.RWMutex
-	// candidatesMu guards discovered candidates and MRU order.
-	candidatesMu sync.RWMutex
 	// idxMu guards parsed index cache.
 	idxMu sync.RWMutex
 	// cacheMu guards delta cache operations.
diff --git a/objectstore/packed/store_lookup.go b/objectstore/packed/store_lookup.go
index cf5a580d..a1bd9b85 100644
--- a/objectstore/packed/store_lookup.go
+++ b/objectstore/packed/store_lookup.go
@@ -14,38 +14,93 @@ func (store *Store) lookup(id objectid.ObjectID) (location, error) {
 		return zero, errors.New("objectstore/packed: object id algorithm mismatch")
 	}
 
-	err := store.ensureCandidates()
+	snapshot, err := store.ensureCandidates()
 	if err != nil {
 		return zero, err
 	}
 
-	nextPackName := store.firstCandidatePackName()
+	loc, ok, err := store.lookupInCandidates(id, snapshot)
+	if err != nil {
+		return zero, err
+	}
+
+	if ok {
+		return loc, nil
+	}
+
+	if store.refreshPolicy == RefreshPolicyOnMissing {
+		err = store.Refresh()
+		if err != nil {
+			return zero, err
+		}
+
+		refreshed := store.candidates.Load()
+		if refreshed != nil && refreshed != snapshot {
+			loc, ok, err = store.lookupInCandidates(id, refreshed)
+			if err != nil {
+				return zero, err
+			}
+
+			if ok {
+				return loc, nil
+			}
+		}
+	}
+
+	return zero, objectstore.ErrObjectNotFound
+}
+
+func (store *Store) lookupInCandidates(
+	id objectid.ObjectID,
+	snapshot *candidateSnapshot,
+) (location, bool, error) {
+	var zero location
+
+	nextPackName := store.firstCandidatePackName(snapshot)
 	for nextPackName != "" {
-		candidate, ok := store.candidateForPack(nextPackName)
+		candidate, ok := snapshot.candidateByPack[nextPackName]
 		if !ok {
-			nextPackName = store.firstCandidatePackName()
+			nextPackName = store.firstCandidatePackName(snapshot)
 
 			continue
 		}
 
-		nextPackName = store.nextCandidatePackName(candidate.packName)
+		nextPackName = store.nextCandidatePackName(candidate.packName, snapshot)
 
 		index, err := store.openIndex(candidate)
 		if err != nil {
-			return zero, err
+			return zero, false, err
 		}
 
 		offset, ok, err := index.lookup(id)
 		if err != nil {
-			return zero, err
+			return zero, false, err
 		}
 
 		if ok {
 			store.touchCandidate(candidate.packName)
 
-			return location{packName: index.packName, offset: offset}, nil
+			return location{packName: index.packName, offset: offset}, true, nil
 		}
 	}
 
-	return zero, objectstore.ErrObjectNotFound
+	for _, candidate := range snapshot.candidates {
+		index, err := store.openIndex(candidate)
+		if err != nil {
+			return zero, false, err
+		}
+
+		offset, ok, err := index.lookup(id)
+		if err != nil {
+			return zero, false, err
+		}
+
+		if ok {
+			store.touchCandidate(candidate.packName)
+
+			return location{packName: index.packName, offset: offset}, true, nil
+		}
+	}
+
+	return zero, false, nil
 }
diff --git a/objectstore/packed/trailer_match.go b/objectstore/packed/trailer_match.go
index 25337cd7..dc43e37d 100644
--- a/objectstore/packed/trailer_match.go
+++ b/objectstore/packed/trailer_match.go
@@ -5,12 +5,12 @@ import "fmt"
 // verifyPackMatchesIndexes checks that one opened pack's trailer hash matches
 // every loaded index that references the same pack name.
 func (store *Store) verifyPackMatchesIndexes(pack *packFile) error {
-	err := store.ensureCandidates()
+	snapshot, err := store.ensureCandidates()
 	if err != nil {
 		return err
 	}
 
-	candidate, ok := store.candidateForPack(pack.name)
+	candidate, ok := snapshot.candidateByPack[pack.name]
 	if !ok {
 		return fmt.Errorf("objectstore/packed: missing index for pack %q", pack.name)
 	}