diff options
Diffstat (limited to 'object/store/dual')
| -rw-r--r-- | object/store/dual/doc.go | 8 | ||||
| -rw-r--r-- | object/store/dual/dual.go | 35 | ||||
| -rw-r--r-- | object/store/dual/dual_test.go | 263 | ||||
| -rw-r--r-- | object/store/dual/new.go | 29 | ||||
| -rw-r--r-- | object/store/dual/quarantine.go | 113 | ||||
| -rw-r--r-- | object/store/dual/quarantine_begin.go | 47 | ||||
| -rw-r--r-- | object/store/dual/quarantine_discard.go | 11 | ||||
| -rw-r--r-- | object/store/dual/quarantine_promote.go | 13 | ||||
| -rw-r--r-- | object/store/dual/reader.go | 57 | ||||
| -rw-r--r-- | object/store/dual/writer_object.go | 32 | ||||
| -rw-r--r-- | object/store/dual/writer_pack.go | 12 |
11 files changed, 620 insertions, 0 deletions
diff --git a/object/store/dual/doc.go b/object/store/dual/doc.go new file mode 100644 index 00000000..104120ec --- /dev/null +++ b/object/store/dual/doc.go @@ -0,0 +1,8 @@ +// Package dual provides one logical object store backed by separate object-wise +// and pack-wise stores. +// +// Dual composes a store that handles individual object writes with a store that +// handles pack-wise writes, while exposing one mixed reader over both. +// Coordinated quarantine operations span both stores, but quarantine promotion +// is non-atomic. +package dual diff --git a/object/store/dual/dual.go b/object/store/dual/dual.go new file mode 100644 index 00000000..627cc121 --- /dev/null +++ b/object/store/dual/dual.go @@ -0,0 +1,35 @@ +package dual + +import objectstore "codeberg.org/lindenii/furgit/object/store" + +type objectSide interface { + objectstore.Reader + objectstore.ObjectWriter + objectstore.ObjectQuarantiner +} + +type packSide interface { + objectstore.Reader + objectstore.PackWriter + objectstore.PackQuarantiner +} + +// Dual composes one object-wise store and one pack-wise store into one logical +// object store. +// +// Reads are served from the combined object reader of both stores. Individual +// object writes are routed to the object-wise store, and pack writes are routed +// to the pack-wise store. Coordinated quarantines go across both stores. +type Dual struct { + object objectSide + pack packSide + reader objectstore.Reader +} + +var ( + _ objectstore.Reader = (*Dual)(nil) + _ objectstore.ObjectWriter = (*Dual)(nil) + _ objectstore.PackWriter = (*Dual)(nil) + _ objectstore.ObjectQuarantiner = (*Dual)(nil) + _ objectstore.PackQuarantiner = (*Dual)(nil) +) diff --git a/object/store/dual/dual_test.go b/object/store/dual/dual_test.go new file mode 100644 index 00000000..bccd378f --- /dev/null +++ b/object/store/dual/dual_test.go @@ -0,0 +1,263 @@ +package dual_test + +import ( + "bytes" + "os" + "path/filepath" + "strings" + "testing" + + "codeberg.org/lindenii/furgit/internal/testgit" + objectid "codeberg.org/lindenii/furgit/object/id" + objectstore "codeberg.org/lindenii/furgit/object/store" + "codeberg.org/lindenii/furgit/object/store/dual" + "codeberg.org/lindenii/furgit/object/store/loose" + "codeberg.org/lindenii/furgit/object/store/packed" + objecttype "codeberg.org/lindenii/furgit/object/type" +) + +func fixturePath(t *testing.T, algo objectid.Algorithm, name string) string { + t.Helper() + + return filepath.Join("..", "packed", "internal", "ingest", "testdata", "fixtures", algo.String(), name) +} + +func fixtureBytes(t *testing.T, algo objectid.Algorithm, name string) []byte { + t.Helper() + + path := fixturePath(t, algo, name) + dir := filepath.Dir(path) + base := filepath.Base(path) + + root, err := os.OpenRoot(dir) + if err != nil { + t.Fatalf("open fixture root %q: %v", dir, err) + } + + defer func() { + err := root.Close() + if err != nil { + t.Fatalf("close fixture root %q: %v", dir, err) + } + }() + + data, err := root.ReadFile(base) + if err != nil { + t.Fatalf("read fixture %q: %v", base, err) + } + + return data +} + +func fixtureMetadata(t *testing.T, algo objectid.Algorithm) map[string]string { + t.Helper() + + data := fixtureBytes(t, algo, "METADATA.txt") + out := make(map[string]string) + + for line := range strings.SplitSeq(strings.TrimSpace(string(data)), "\n") { + line = strings.TrimSpace(line) + if line == "" { + continue + } + + key, value, ok := strings.Cut(line, "=") + if !ok { + t.Fatalf("invalid fixture metadata line %q", line) + } + + out[strings.TrimSpace(key)] = strings.TrimSpace(value) + } + + return out +} + +func fixtureOID(t *testing.T, algo objectid.Algorithm, key string) objectid.ObjectID { + t.Helper() + + meta := fixtureMetadata(t, algo) + hex, ok := meta[key] + if !ok { + t.Fatalf("missing fixture metadata key %q", key) + } + + id, err := objectid.ParseHex(algo, hex) + if err != nil { + t.Fatalf("parse fixture metadata oid %q: %v", hex, err) + } + + return id +} + +func newDualStore(t *testing.T, repo *testgit.TestRepo, algo objectid.Algorithm) *dual.Dual { + t.Helper() + + objectsRoot := repo.OpenObjectsRoot(t) + looseStore, err := loose.New(objectsRoot, algo) + if err != nil { + t.Fatalf("loose.New: %v", err) + } + + packRoot := repo.OpenPackRoot(t) + packedStore, err := packed.New(packRoot, algo, packed.Options{WriteRev: true}) + if err != nil { + t.Fatalf("packed.New: %v", err) + } + + return dual.New(looseStore, packedStore) +} + +func TestDualReadsWritesAndQuarantine(t *testing.T) { + t.Parallel() + + testgit.ForEachAlgorithm(t, func(t *testing.T, algo objectid.Algorithm) { + head := fixtureOID(t, algo, "head") + packBytes := fixtureBytes(t, algo, "nonthin.pack") + + repo := testgit.NewRepo(t, testgit.RepoOptions{ObjectFormat: algo, Bare: true}) + store := newDualStore(t, repo, algo) + + quarantiner, ok := any(store).(objectstore.PackQuarantiner) + if !ok { + t.Fatal("dual does not implement PackQuarantiner") + } + + quarantine, err := quarantiner.BeginPackQuarantine(objectstore.PackQuarantineOptions{}) + if err != nil { + t.Fatalf("BeginPackQuarantine: %v", err) + } + + err = quarantine.WritePack(bytes.NewReader(packBytes), objectstore.PackWriteOptions{RequireTrailingEOF: true}) + if err != nil { + t.Fatalf("quarantine.WritePack: %v", err) + } + + objectQ, ok := any(quarantine).(objectstore.ObjectQuarantine) + if !ok { + t.Fatal("pack quarantine does not also implement ObjectQuarantine") + } + + looseContent := []byte("dual quarantine loose object\n") + looseID, err := objectQ.WriteBytesContent(objecttype.TypeBlob, looseContent) + if err != nil { + t.Fatalf("quarantine.WriteBytesContent: %v", err) + } + + ty, _, err := quarantine.ReadHeader(head) + if err != nil { + t.Fatalf("quarantine.ReadHeader(pack): %v", err) + } + + if ty != objecttype.TypeCommit { + t.Fatalf("quarantine.ReadHeader(pack) type = %v, want commit", ty) + } + + ty, got, err := quarantine.ReadBytesContent(looseID) + if err != nil { + t.Fatalf("quarantine.ReadBytesContent(loose): %v", err) + } + + if ty != objecttype.TypeBlob { + t.Fatalf("quarantine.ReadBytesContent(loose) type = %v, want blob", ty) + } + + if !bytes.Equal(got, looseContent) { + t.Fatal("quarantine.ReadBytesContent(loose) mismatch") + } + + _, _, err = store.ReadHeader(head) + if err == nil { + t.Fatal("store.ReadHeader unexpectedly saw quarantined pack object before promote") + } + + _, _, err = store.ReadBytesContent(looseID) + if err == nil { + t.Fatal("store.ReadBytesContent unexpectedly saw quarantined loose object before promote") + } + + err = quarantine.Promote() + if err != nil { + t.Fatalf("quarantine.Promote: %v", err) + } + + err = store.Refresh() + if err != nil { + t.Fatalf("store.Refresh: %v", err) + } + + ty, _, err = store.ReadHeader(head) + if err != nil { + t.Fatalf("store.ReadHeader(pack): %v", err) + } + + if ty != objecttype.TypeCommit { + t.Fatalf("store.ReadHeader(pack) type = %v, want commit", ty) + } + + ty, got, err = store.ReadBytesContent(looseID) + if err != nil { + t.Fatalf("store.ReadBytesContent(loose): %v", err) + } + + if ty != objecttype.TypeBlob { + t.Fatalf("store.ReadBytesContent(loose) type = %v, want blob", ty) + } + + if !bytes.Equal(got, looseContent) { + t.Fatal("store.ReadBytesContent(loose) mismatch") + } + }) +} + +func TestDualQuarantineDiscardDropsBothHalves(t *testing.T) { + t.Parallel() + + testgit.ForEachAlgorithm(t, func(t *testing.T, algo objectid.Algorithm) { + head := fixtureOID(t, algo, "head") + packBytes := fixtureBytes(t, algo, "nonthin.pack") + + repo := testgit.NewRepo(t, testgit.RepoOptions{ObjectFormat: algo, Bare: true}) + store := newDualStore(t, repo, algo) + + quarantiner := any(store).(objectstore.ObjectQuarantiner) + quarantine, err := quarantiner.BeginObjectQuarantine(objectstore.ObjectQuarantineOptions{}) + if err != nil { + t.Fatalf("BeginObjectQuarantine: %v", err) + } + + packQ, ok := any(quarantine).(objectstore.PackQuarantine) + if !ok { + t.Fatal("object quarantine does not also implement PackQuarantine") + } + + err = packQ.WritePack(bytes.NewReader(packBytes), objectstore.PackWriteOptions{RequireTrailingEOF: true}) + if err != nil { + t.Fatalf("quarantine.WritePack: %v", err) + } + + looseID, err := quarantine.WriteBytesContent(objecttype.TypeBlob, []byte("discarded dual object\n")) + if err != nil { + t.Fatalf("quarantine.WriteBytesContent: %v", err) + } + + err = quarantine.Discard() + if err != nil { + t.Fatalf("quarantine.Discard: %v", err) + } + + err = store.Refresh() + if err != nil { + t.Fatalf("store.Refresh: %v", err) + } + + _, _, err = store.ReadHeader(head) + if err == nil { + t.Fatal("store.ReadHeader unexpectedly saw discarded pack object") + } + + _, _, err = store.ReadBytesContent(looseID) + if err == nil { + t.Fatal("store.ReadBytesContent unexpectedly saw discarded loose object") + } + }) +} diff --git a/object/store/dual/new.go b/object/store/dual/new.go new file mode 100644 index 00000000..ef38bc7a --- /dev/null +++ b/object/store/dual/new.go @@ -0,0 +1,29 @@ +package dual + +import ( + objectstore "codeberg.org/lindenii/furgit/object/store" + objectmix "codeberg.org/lindenii/furgit/object/store/mix" +) + +// New creates one dual object store from borrowed object-wise and pack-wise +// stores. +// +// Labels: Deps-Borrowed, Life-Parent. +func New( + object interface { + objectstore.Reader + objectstore.ObjectWriter + objectstore.ObjectQuarantiner + }, + pack interface { + objectstore.Reader + objectstore.PackWriter + objectstore.PackQuarantiner + }, +) *Dual { + return &Dual{ + object: object, + pack: pack, + reader: objectmix.New(object, pack), + } +} diff --git a/object/store/dual/quarantine.go b/object/store/dual/quarantine.go new file mode 100644 index 00000000..ff561c7c --- /dev/null +++ b/object/store/dual/quarantine.go @@ -0,0 +1,113 @@ +package dual + +import ( + "io" + + objectid "codeberg.org/lindenii/furgit/object/id" + objectstore "codeberg.org/lindenii/furgit/object/store" + objectmix "codeberg.org/lindenii/furgit/object/store/mix" + objecttype "codeberg.org/lindenii/furgit/object/type" +) + +// quarantine is one coordinated dual quarantine over both stores. +type quarantine struct { + objectQ objectstore.ObjectQuarantine + packQ objectstore.PackQuarantine + reader objectstore.Reader +} + +var ( + _ objectstore.ObjectQuarantine = (*quarantine)(nil) + _ objectstore.PackQuarantine = (*quarantine)(nil) +) + +func newQuarantine( + objectQ objectstore.ObjectQuarantine, + packQ objectstore.PackQuarantine, +) *quarantine { + return &quarantine{ + objectQ: objectQ, + packQ: packQ, + reader: objectmix.New(objectQ, packQ), + } +} + +// ReadBytesFull reads a full serialized object as "type size\0content" from +// either quarantined store. +func (quarantine *quarantine) ReadBytesFull(id objectid.ObjectID) ([]byte, error) { + return quarantine.reader.ReadBytesFull(id) +} + +// ReadBytesContent reads an object's type and content bytes from either +// quarantined store. +func (quarantine *quarantine) ReadBytesContent(id objectid.ObjectID) (objecttype.Type, []byte, error) { + return quarantine.reader.ReadBytesContent(id) +} + +// ReadReaderFull reads a full serialized object stream as +// "type size\0content" from either quarantined store. +func (quarantine *quarantine) ReadReaderFull(id objectid.ObjectID) (io.ReadCloser, error) { + return quarantine.reader.ReadReaderFull(id) +} + +// ReadReaderContent reads an object's type, declared content length, and +// content stream from either quarantined store. +func (quarantine *quarantine) ReadReaderContent(id objectid.ObjectID) (objecttype.Type, int64, io.ReadCloser, error) { + return quarantine.reader.ReadReaderContent(id) +} + +// ReadSize reads an object's declared content length from either quarantined +// store. +func (quarantine *quarantine) ReadSize(id objectid.ObjectID) (int64, error) { + return quarantine.reader.ReadSize(id) +} + +// ReadHeader reads an object's type and declared content length from either +// quarantined store. +func (quarantine *quarantine) ReadHeader(id objectid.ObjectID) (objecttype.Type, int64, error) { + return quarantine.reader.ReadHeader(id) +} + +// Refresh refreshes both quarantined stores and the combined quarantined reader. +func (quarantine *quarantine) Refresh() error { + err := quarantine.objectQ.Refresh() + if err != nil { + return err + } + + err = quarantine.packQ.Refresh() + if err != nil { + return err + } + + return quarantine.reader.Refresh() +} + +// WriteReaderContent writes one typed object content stream to the quarantined +// object-wise store. +func (quarantine *quarantine) WriteReaderContent(ty objecttype.Type, size int64, src io.Reader) (objectid.ObjectID, error) { + return quarantine.objectQ.WriteReaderContent(ty, size, src) +} + +// WriteReaderFull writes one full serialized object stream as +// "type size\0content" to the quarantined object-wise store. +func (quarantine *quarantine) WriteReaderFull(src io.Reader) (objectid.ObjectID, error) { + return quarantine.objectQ.WriteReaderFull(src) +} + +// WriteBytesContent writes one typed object content byte slice to the +// quarantined object-wise store. +func (quarantine *quarantine) WriteBytesContent(ty objecttype.Type, content []byte) (objectid.ObjectID, error) { + return quarantine.objectQ.WriteBytesContent(ty, content) +} + +// WriteBytesFull writes one full serialized object byte slice as +// "type size\0content" to the quarantined object-wise store. +func (quarantine *quarantine) WriteBytesFull(raw []byte) (objectid.ObjectID, error) { + return quarantine.objectQ.WriteBytesFull(raw) +} + +// WritePack ingests one pack stream into the quarantined pack-wise store. +func (quarantine *quarantine) WritePack(src io.Reader, opts objectstore.PackWriteOptions) error { + return quarantine.packQ.WritePack(src, opts) +} diff --git a/object/store/dual/quarantine_begin.go b/object/store/dual/quarantine_begin.go new file mode 100644 index 00000000..bca688b7 --- /dev/null +++ b/object/store/dual/quarantine_begin.go @@ -0,0 +1,47 @@ +package dual + +import objectstore "codeberg.org/lindenii/furgit/object/store" + +// TODO: This doesn't actually make sense. We need a combined quarantine. + +// BeginObjectQuarantine creates one coordinated dual quarantine spanning both +// stores and returns it as an object-wise quarantine. +// +// Labels: Deps-Borrowed, Life-Parent, Close-No. +func (dual *Dual) BeginObjectQuarantine(_ objectstore.ObjectQuarantineOptions) (objectstore.ObjectQuarantine, error) { + quarantine, err := dual.beginQuarantine() + if err != nil { + return nil, err + } + + return quarantine, nil +} + +// BeginPackQuarantine creates one coordinated dual quarantine spanning both +// stores and returns it as a pack-wise quarantine. +// +// Labels: Deps-Borrowed, Life-Parent, Close-No. +func (dual *Dual) BeginPackQuarantine(_ objectstore.PackQuarantineOptions) (objectstore.PackQuarantine, error) { + quarantine, err := dual.beginQuarantine() + if err != nil { + return nil, err + } + + return quarantine, nil +} + +func (dual *Dual) beginQuarantine() (*quarantine, error) { + objectQ, err := dual.object.BeginObjectQuarantine(objectstore.ObjectQuarantineOptions{}) + if err != nil { + return nil, err + } + + packQ, err := dual.pack.BeginPackQuarantine(objectstore.PackQuarantineOptions{}) + if err != nil { + _ = objectQ.Discard() + + return nil, err + } + + return newQuarantine(objectQ, packQ), nil +} diff --git a/object/store/dual/quarantine_discard.go b/object/store/dual/quarantine_discard.go new file mode 100644 index 00000000..67f15d6c --- /dev/null +++ b/object/store/dual/quarantine_discard.go @@ -0,0 +1,11 @@ +package dual + +// Discard abandons both quarantine halves and invalidates the receiver. +func (quarantine *quarantine) Discard() error { + err := quarantine.packQ.Discard() + if err != nil { + return err + } + + return quarantine.objectQ.Discard() +} diff --git a/object/store/dual/quarantine_promote.go b/object/store/dual/quarantine_promote.go new file mode 100644 index 00000000..4d0a45b8 --- /dev/null +++ b/object/store/dual/quarantine_promote.go @@ -0,0 +1,13 @@ +package dual + +// Promote publishes both quarantine halves and invalidates the receiver. +// +// Promotion is coordinated and ordered, but not atomic. +func (quarantine *quarantine) Promote() error { + err := quarantine.packQ.Promote() + if err != nil { + return err + } + + return quarantine.objectQ.Promote() +} diff --git a/object/store/dual/reader.go b/object/store/dual/reader.go new file mode 100644 index 00000000..7b499d5d --- /dev/null +++ b/object/store/dual/reader.go @@ -0,0 +1,57 @@ +package dual + +import ( + "io" + + objectid "codeberg.org/lindenii/furgit/object/id" + objecttype "codeberg.org/lindenii/furgit/object/type" +) + +// ReadBytesFull reads a full serialized object as "type size\0content" from +// either store. +func (dual *Dual) ReadBytesFull(id objectid.ObjectID) ([]byte, error) { + return dual.reader.ReadBytesFull(id) +} + +// ReadBytesContent reads an object's type and content bytes from either store. +func (dual *Dual) ReadBytesContent(id objectid.ObjectID) (objecttype.Type, []byte, error) { + return dual.reader.ReadBytesContent(id) +} + +// ReadReaderFull reads a full serialized object stream as "type size\0content" +// from either store. +func (dual *Dual) ReadReaderFull(id objectid.ObjectID) (io.ReadCloser, error) { + return dual.reader.ReadReaderFull(id) +} + +// ReadReaderContent reads an object's type, declared content length, and +// content stream from either store. +func (dual *Dual) ReadReaderContent(id objectid.ObjectID) (objecttype.Type, int64, io.ReadCloser, error) { + return dual.reader.ReadReaderContent(id) +} + +// ReadSize reads an object's declared content length from either store. +func (dual *Dual) ReadSize(id objectid.ObjectID) (int64, error) { + return dual.reader.ReadSize(id) +} + +// ReadHeader reads an object's type and declared content length from either +// store. +func (dual *Dual) ReadHeader(id objectid.ObjectID) (objecttype.Type, int64, error) { + return dual.reader.ReadHeader(id) +} + +// Refresh refreshes both underlying stores and the combined read view. +func (dual *Dual) Refresh() error { + err := dual.object.Refresh() + if err != nil { + return err + } + + err = dual.pack.Refresh() + if err != nil { + return err + } + + return dual.reader.Refresh() +} diff --git a/object/store/dual/writer_object.go b/object/store/dual/writer_object.go new file mode 100644 index 00000000..7aefe9ea --- /dev/null +++ b/object/store/dual/writer_object.go @@ -0,0 +1,32 @@ +package dual + +import ( + "io" + + objectid "codeberg.org/lindenii/furgit/object/id" + objecttype "codeberg.org/lindenii/furgit/object/type" +) + +// WriteReaderContent writes one typed object content stream to the object-wise +// store. +func (dual *Dual) WriteReaderContent(ty objecttype.Type, size int64, src io.Reader) (objectid.ObjectID, error) { + return dual.object.WriteReaderContent(ty, size, src) +} + +// WriteReaderFull writes one full serialized object stream as +// "type size\0content" to the object-wise store. +func (dual *Dual) WriteReaderFull(src io.Reader) (objectid.ObjectID, error) { + return dual.object.WriteReaderFull(src) +} + +// WriteBytesContent writes one typed object content byte slice to the +// object-wise store. +func (dual *Dual) WriteBytesContent(ty objecttype.Type, content []byte) (objectid.ObjectID, error) { + return dual.object.WriteBytesContent(ty, content) +} + +// WriteBytesFull writes one full serialized object byte slice as +// "type size\0content" to the object-wise store. +func (dual *Dual) WriteBytesFull(raw []byte) (objectid.ObjectID, error) { + return dual.object.WriteBytesFull(raw) +} diff --git a/object/store/dual/writer_pack.go b/object/store/dual/writer_pack.go new file mode 100644 index 00000000..5ac8648b --- /dev/null +++ b/object/store/dual/writer_pack.go @@ -0,0 +1,12 @@ +package dual + +import ( + "io" + + objectstore "codeberg.org/lindenii/furgit/object/store" +) + +// WritePack ingests one pack stream into the pack-wise store. +func (dual *Dual) WritePack(src io.Reader, opts objectstore.PackWriteOptions) error { + return dual.pack.WritePack(src, opts) +} |
