diff options
| -rw-r--r-- | objectstore/packed/idx_load.go | 122 | ||||
| -rw-r--r-- | objectstore/packed/store.go | 91 |
2 files changed, 144 insertions, 69 deletions
diff --git a/objectstore/packed/idx_load.go b/objectstore/packed/idx_load.go index 293e005f..35e8b925 100644 --- a/objectstore/packed/idx_load.go +++ b/objectstore/packed/idx_load.go @@ -17,6 +17,16 @@ type location struct { offset uint64 } +// packCandidate describes one discovered pack/index pair. +type packCandidate struct { + // packName is the .pack basename. + packName string + // idxName is the .idx basename. + idxName string + // mtime is the pack file modification time for initial ordering. + mtime int64 +} + // idxFile stores one mapped and validated idx v2 file. type idxFile struct { // idxName is the basename of this .idx file. @@ -46,8 +56,30 @@ type idxFile struct { offset64Count int } -// loadIndexes loads and validates all .idx files under objects/pack. -func (store *Store) loadIndexes() ([]*idxFile, error) { +// ensureCandidates discovers pack/index pairs once. +func (store *Store) ensureCandidates() error { + store.discoverOnce.Do(func() { + candidates, err := store.discoverCandidates() + candidateByPack := make(map[string]packCandidate, len(candidates)) + for _, candidate := range candidates { + candidateByPack[candidate.packName] = candidate + } + store.stateMu.Lock() + store.candidates = candidates + store.candidateByPack = candidateByPack + store.discoverErr = err + store.stateMu.Unlock() + }) + + store.stateMu.RLock() + err := store.discoverErr + store.stateMu.RUnlock() + return err +} + +// discoverCandidates scans the objects/pack root and returns sorted pack/index +// pairs. +func (store *Store) discoverCandidates() ([]packCandidate, error) { dir, err := store.root.Open(".") if err != nil { if os.IsNotExist(err) { @@ -56,39 +88,97 @@ func (store *Store) loadIndexes() ([]*idxFile, error) { return nil, err } defer func() { _ = dir.Close() }() + entries, err := dir.ReadDir(-1) if err != nil { return nil, err } - idxNames := make([]string, 0, len(entries)) + candidates := make([]packCandidate, 0, len(entries)) for _, entry := range entries { if entry.IsDir() || !strings.HasSuffix(entry.Name(), ".idx") { continue } - idxNames = append(idxNames, entry.Name()) - } - slices.Sort(idxNames) - out := make([]*idxFile, 0, len(idxNames)) - for _, idxName := range idxNames { + idxName := entry.Name() packName := strings.TrimSuffix(idxName, ".idx") + ".pack" - if _, err := store.root.Stat(packName); err != nil { + packInfo, err := store.root.Stat(packName) + if err != nil { if os.IsNotExist(err) { return nil, fmt.Errorf("objectstore/packed: missing pack file for index %q", idxName) } return nil, err } - index, err := openIdxFile(store.root, idxName, packName, store.algo) - if err != nil { - for _, loaded := range out { - _ = loaded.close() + + candidates = append(candidates, packCandidate{ + packName: packName, + idxName: idxName, + mtime: packInfo.ModTime().UnixNano(), + }) + } + + slices.SortFunc(candidates, func(a, b packCandidate) int { + if a.mtime != b.mtime { + if a.mtime > b.mtime { + return -1 } - return nil, err + return 1 + } + return strings.Compare(a.packName, b.packName) + }) + + return candidates, nil +} + +// candidateForPack returns one discovered candidate for a pack basename. +func (store *Store) candidateForPack(packName string) (packCandidate, bool) { + store.stateMu.RLock() + candidate, ok := store.candidateByPack[packName] + store.stateMu.RUnlock() + return candidate, ok +} + +// openIndex returns one opened and parsed index, caching it by pack basename. +func (store *Store) openIndex(candidate packCandidate) (*idxFile, error) { + store.stateMu.RLock() + if index, ok := store.idxByPack[candidate.packName]; ok { + store.stateMu.RUnlock() + return index, nil + } + store.stateMu.RUnlock() + + index, err := openIdxFile(store.root, candidate.idxName, candidate.packName, store.algo) + if err != nil { + return nil, err + } + + store.stateMu.Lock() + if existing, ok := store.idxByPack[candidate.packName]; ok { + store.stateMu.Unlock() + _ = index.close() + return existing, nil + } + store.idxByPack[candidate.packName] = index + store.stateMu.Unlock() + return index, nil +} + +// touchCandidate moves one candidate to the front of the lookup order. +func (store *Store) touchCandidate(packName string) { + store.stateMu.Lock() + defer store.stateMu.Unlock() + for i := range store.candidates { + if store.candidates[i].packName != packName { + continue + } + if i == 0 { + return } - out = append(out, index) + candidate := store.candidates[i] + copy(store.candidates[1:i+1], store.candidates[0:i]) + store.candidates[0] = candidate + return } - return out, nil } // openIdxFile maps and validates one idx v2 file. diff --git a/objectstore/packed/store.go b/objectstore/packed/store.go index 4b2b011e..00b8c962 100644 --- a/objectstore/packed/store.go +++ b/objectstore/packed/store.go @@ -21,16 +21,16 @@ type Store struct { // algo is the expected object ID algorithm for lookups. algo objectid.Algorithm - // loadOnce guards one-time index loading. - loadOnce sync.Once - // loadErr stores index loading failures. - loadErr error - // indexesLoaded reports whether indexes/loadErr have been initialized. - indexesLoaded bool - // indexes stores parsed .idx handles. - indexes []*idxFile - // indexByPack maps one pack basename to its parsed index. - indexByPack map[string]*idxFile + // discoverOnce guards one-time pack candidate discovery. + discoverOnce sync.Once + // discoverErr stores candidate discovery failures. + discoverErr error + // candidates stores known pack/index pairs in lookup priority order. + candidates []packCandidate + // candidateByPack maps pack basename to discovered candidate. + candidateByPack map[string]packCandidate + // idxByPack caches opened and parsed indexes by pack basename. + idxByPack map[string]*idxFile // stateMu guards index publication, pack cache, and close state. stateMu sync.RWMutex @@ -54,10 +54,12 @@ func New(root *os.Root, algo objectid.Algorithm) (*Store, error) { return nil, objectid.ErrInvalidAlgorithm } return &Store{ - root: root, - algo: algo, - packs: make(map[string]*packFile), - deltaCache: newDeltaCache(defaultDeltaCacheMaxBytes), + root: root, + algo: algo, + candidateByPack: make(map[string]packCandidate), + idxByPack: make(map[string]*idxFile), + packs: make(map[string]*packFile), + deltaCache: newDeltaCache(defaultDeltaCacheMaxBytes), }, nil } @@ -71,7 +73,7 @@ func (store *Store) Close() error { store.closed = true root := store.root packs := store.packs - indexes := store.indexes + indexes := store.idxByPack store.stateMu.Unlock() var closeErr error @@ -81,9 +83,6 @@ func (store *Store) Close() error { } } for _, index := range indexes { - if index == nil { - continue - } if err := index.close(); err != nil && closeErr == nil { closeErr = err } @@ -98,45 +97,31 @@ func (store *Store) Close() error { return closeErr } -// ensureIndexes loads and validates all pack indexes once. -func (store *Store) ensureIndexes() error { - store.loadOnce.Do(func() { - indexes, err := store.loadIndexes() - indexByPack := make(map[string]*idxFile, len(indexes)) - for _, index := range indexes { - indexByPack[index.packName] = index - } - store.stateMu.Lock() - store.indexes = indexes - store.indexByPack = indexByPack - store.loadErr = err - store.indexesLoaded = true - store.stateMu.Unlock() - }) - - store.stateMu.RLock() - defer store.stateMu.RUnlock() - if store.indexesLoaded { - return store.loadErr - } - return errors.New("objectstore/packed: indexes were not initialized") -} - // lookup resolves one object ID to its pack location. func (store *Store) lookup(id objectid.ObjectID) (location, error) { var zero location if id.Algorithm() != store.algo { return zero, errors.New("objectstore/packed: object id algorithm mismatch") } - if err := store.ensureIndexes(); err != nil { + if err := store.ensureCandidates(); err != nil { return zero, err } - for _, index := range store.indexes { + + store.stateMu.RLock() + candidates := append([]packCandidate(nil), store.candidates...) + store.stateMu.RUnlock() + + for _, candidate := range candidates { + index, err := store.openIndex(candidate) + if err != nil { + return zero, err + } offset, ok, err := index.lookup(id) if err != nil { return zero, err } if ok { + store.touchCandidate(candidate.packName) return location{packName: index.packName, offset: offset}, nil } } @@ -185,16 +170,16 @@ func (store *Store) openPack(name string) (*packFile, error) { // verifyPackMatchesIndexes checks that one opened pack's trailer hash matches // every loaded index that references the same pack name. func (store *Store) verifyPackMatchesIndexes(pack *packFile) error { - store.stateMu.RLock() - index := store.indexByPack[pack.name] - indexesLoaded := store.indexesLoaded - store.stateMu.RUnlock() - - if !indexesLoaded { - return nil + if err := store.ensureCandidates(); err != nil { + return err } - if index == nil { - return nil + candidate, ok := store.candidateForPack(pack.name) + if !ok { + return fmt.Errorf("objectstore/packed: missing index for pack %q", pack.name) + } + index, err := store.openIndex(candidate) + if err != nil { + return err } if err := packchecksum.VerifyPackMatchesIdx(pack.data, index.data, store.algo); err != nil { return fmt.Errorf("objectstore/packed: pack %q does not match idx %q: %w", pack.name, index.idxName, err) |