diff options
| -rw-r--r-- | format/packfile/ingest/api.go | 196 | ||||
| -rw-r--r-- | object/store/packed/doc.go | 4 | ||||
| -rw-r--r-- | object/store/packed/internal/ingest/byteslice_reader.go (renamed from format/packfile/ingest/byteslice_reader.go) | 0 | ||||
| -rw-r--r-- | object/store/packed/internal/ingest/cache.go (renamed from format/packfile/ingest/cache.go) | 0 | ||||
| -rw-r--r-- | object/store/packed/internal/ingest/counting_writer.go (renamed from format/packfile/ingest/counting_writer.go) | 0 | ||||
| -rw-r--r-- | object/store/packed/internal/ingest/crc.go (renamed from format/packfile/ingest/crc.go) | 0 | ||||
| -rw-r--r-- | object/store/packed/internal/ingest/delta_header.go (renamed from format/packfile/ingest/delta_header.go) | 0 | ||||
| -rw-r--r-- | object/store/packed/internal/ingest/distance.go (renamed from format/packfile/ingest/distance.go) | 0 | ||||
| -rw-r--r-- | object/store/packed/internal/ingest/doc.go (renamed from format/packfile/ingest/doc.go) | 2 | ||||
| -rw-r--r-- | object/store/packed/internal/ingest/drain.go (renamed from format/packfile/ingest/drain.go) | 0 | ||||
| -rw-r--r-- | object/store/packed/internal/ingest/entry.go (renamed from format/packfile/ingest/entry.go) | 0 | ||||
| -rw-r--r-- | object/store/packed/internal/ingest/entry_header.go (renamed from format/packfile/ingest/entry_header.go) | 0 | ||||
| -rw-r--r-- | object/store/packed/internal/ingest/entry_prefix.go (renamed from format/packfile/ingest/entry_prefix.go) | 0 | ||||
| -rw-r--r-- | object/store/packed/internal/ingest/errors.go (renamed from format/packfile/ingest/errors.go) | 7 | ||||
| -rw-r--r-- | object/store/packed/internal/ingest/file_section_writer.go (renamed from format/packfile/ingest/file_section_writer.go) | 0 | ||||
| -rw-r--r-- | object/store/packed/internal/ingest/fill.go (renamed from format/packfile/ingest/fill.go) | 0 | ||||
| -rw-r--r-- | object/store/packed/internal/ingest/finalize.go (renamed from format/packfile/ingest/finalize.go) | 0 | ||||
| -rw-r--r-- | object/store/packed/internal/ingest/flush.go (renamed from format/packfile/ingest/flush.go) | 0 | ||||
| -rw-r--r-- | object/store/packed/internal/ingest/hash.go (renamed from format/packfile/ingest/hash.go) | 0 | ||||
| -rw-r--r-- | object/store/packed/internal/ingest/header.go (renamed from format/packfile/ingest/header.go) | 19 | ||||
| -rw-r--r-- | object/store/packed/internal/ingest/idx_write.go (renamed from format/packfile/ingest/idx_write.go) | 0 | ||||
| -rw-r--r-- | object/store/packed/internal/ingest/ingest.go (renamed from format/packfile/ingest/ingest.go) | 0 | ||||
| -rw-r--r-- | object/store/packed/internal/ingest/ingest_test.go (renamed from format/packfile/ingest/ingest_test.go) | 67 | ||||
| -rw-r--r-- | object/store/packed/internal/ingest/options.go | 26 | ||||
| -rw-r--r-- | object/store/packed/internal/ingest/progress_write.go (renamed from format/packfile/ingest/progress_write.go) | 0 | ||||
| -rw-r--r-- | object/store/packed/internal/ingest/record_content.go (renamed from format/packfile/ingest/record_content.go) | 0 | ||||
| -rw-r--r-- | object/store/packed/internal/ingest/record_delta.go (renamed from format/packfile/ingest/record_delta.go) | 0 | ||||
| -rw-r--r-- | object/store/packed/internal/ingest/record_inflate.go (renamed from format/packfile/ingest/record_inflate.go) | 0 | ||||
| -rw-r--r-- | object/store/packed/internal/ingest/record_resolve.go (renamed from format/packfile/ingest/record_resolve.go) | 0 | ||||
| -rw-r--r-- | object/store/packed/internal/ingest/records.go (renamed from format/packfile/ingest/records.go) | 0 | ||||
| -rw-r--r-- | object/store/packed/internal/ingest/resolve_all.go (renamed from format/packfile/ingest/resolve_all.go) | 0 | ||||
| -rw-r--r-- | object/store/packed/internal/ingest/result.go | 23 | ||||
| -rw-r--r-- | object/store/packed/internal/ingest/rev_write.go (renamed from format/packfile/ingest/rev_write.go) | 0 | ||||
| -rw-r--r-- | object/store/packed/internal/ingest/rewrite_header_trailer.go (renamed from format/packfile/ingest/rewrite_header_trailer.go) | 0 | ||||
| -rw-r--r-- | object/store/packed/internal/ingest/scan.go (renamed from format/packfile/ingest/scan.go) | 0 | ||||
| -rw-r--r-- | object/store/packed/internal/ingest/state.go (renamed from format/packfile/ingest/state.go) | 2 | ||||
| -rw-r--r-- | object/store/packed/internal/ingest/stream.go (renamed from format/packfile/ingest/stream.go) | 0 | ||||
| -rw-r--r-- | object/store/packed/internal/ingest/temp.go (renamed from format/packfile/ingest/temp.go) | 0 | ||||
| -rw-r--r-- | object/store/packed/internal/ingest/testdata/fixtures/sha1/METADATA.txt (renamed from format/packfile/ingest/testdata/fixtures/sha1/METADATA.txt) | 0 | ||||
| -rw-r--r-- | object/store/packed/internal/ingest/testdata/fixtures/sha1/base.pack (renamed from format/packfile/ingest/testdata/fixtures/sha1/base.pack) | bin | 81007 -> 81007 bytes | |||
| -rw-r--r-- | object/store/packed/internal/ingest/testdata/fixtures/sha1/nonthin.pack (renamed from format/packfile/ingest/testdata/fixtures/sha1/nonthin.pack) | bin | 117458 -> 117458 bytes | |||
| -rw-r--r-- | object/store/packed/internal/ingest/testdata/fixtures/sha1/thin.pack (renamed from format/packfile/ingest/testdata/fixtures/sha1/thin.pack) | bin | 38581 -> 38581 bytes | |||
| -rw-r--r-- | object/store/packed/internal/ingest/testdata/fixtures/sha256/METADATA.txt (renamed from format/packfile/ingest/testdata/fixtures/sha256/METADATA.txt) | 0 | ||||
| -rw-r--r-- | object/store/packed/internal/ingest/testdata/fixtures/sha256/base.pack (renamed from format/packfile/ingest/testdata/fixtures/sha256/base.pack) | bin | 105138 -> 105138 bytes | |||
| -rw-r--r-- | object/store/packed/internal/ingest/testdata/fixtures/sha256/nonthin.pack (renamed from format/packfile/ingest/testdata/fixtures/sha256/nonthin.pack) | bin | 152284 -> 152284 bytes | |||
| -rw-r--r-- | object/store/packed/internal/ingest/testdata/fixtures/sha256/thin.pack (renamed from format/packfile/ingest/testdata/fixtures/sha256/thin.pack) | bin | 49412 -> 49412 bytes | |||
| -rw-r--r-- | object/store/packed/internal/ingest/thin_append.go (renamed from format/packfile/ingest/thin_append.go) | 0 | ||||
| -rw-r--r-- | object/store/packed/internal/ingest/thin_fix.go (renamed from format/packfile/ingest/thin_fix.go) | 0 | ||||
| -rw-r--r-- | object/store/packed/internal/ingest/thin_unresolved.go (renamed from format/packfile/ingest/thin_unresolved.go) | 0 | ||||
| -rw-r--r-- | object/store/packed/internal/ingest/trailer.go (renamed from format/packfile/ingest/trailer.go) | 0 | ||||
| -rw-r--r-- | object/store/packed/internal/ingest/use.go (renamed from format/packfile/ingest/use.go) | 0 | ||||
| -rw-r--r-- | object/store/packed/internal/ingest/write.go | 50 | ||||
| -rw-r--r-- | object/store/packed/internal/ingest/write_empty.go | 58 | ||||
| -rw-r--r-- | object/store/packed/writer.go | 17 |
54 files changed, 217 insertions, 254 deletions
diff --git a/format/packfile/ingest/api.go b/format/packfile/ingest/api.go deleted file mode 100644 index 03774d6f..00000000 --- a/format/packfile/ingest/api.go +++ /dev/null @@ -1,196 +0,0 @@ -package ingest - -import ( - "bufio" - "bytes" - "errors" - "io" - "os" - - "codeberg.org/lindenii/furgit/common/iowrap" - objectid "codeberg.org/lindenii/furgit/object/id" - objectstore "codeberg.org/lindenii/furgit/object/store" -) - -// Options controls one pack ingest operation. -type Options struct { - // FixThin appends missing local bases for thin packs. - FixThin bool - // WriteRev writes a .rev alongside the .pack and .idx. - WriteRev bool - // Base supplies existing objects for thin-pack fixup. - Base objectstore.Reader - // Progress receives human-readable progress messages. - // - // When nil, no progress output is emitted. - Progress iowrap.WriteFlusher - // RequireTrailingEOF requires the source to hit EOF after the pack trailer. - // - // This is suitable for exact pack-file readers, but should be disabled for - // full-duplex transport streams like receive-pack where the peer keeps the - // connection open to read the server response. - RequireTrailingEOF bool -} - -// Result describes one successful ingest transaction. -type Result struct { - // PackName is the destination-relative filename of the written .pack. - PackName string - // IdxName is the destination-relative filename of the written .idx. - IdxName string - // RevName is the destination-relative filename of the written .rev. - // - // RevName is empty when writeRev is false. - RevName string - // PackHash is the final pack hash (same hash embedded in .idx/.rev trailers). - PackHash objectid.ObjectID - // ObjectCount is the final object count in the resulting pack. - // - // If thin fixup appends objects, this includes appended base objects. - ObjectCount uint32 - // ThinFixed reports whether thin fixup appended local bases. - ThinFixed bool -} - -// HeaderInfo describes the parsed PACK header. -type HeaderInfo struct { - Version uint32 - ObjectCount uint32 -} - -// DiscardResult describes one successful Discard call. -type DiscardResult struct { - PackHash objectid.ObjectID - ObjectCount uint32 -} - -// Pending is one started ingest operation awaiting Continue or Discard. -// -// Exactly one of Continue or Discard may be called. -// -// Labels: MT-Unsafe. -type Pending struct { - reader *bufio.Reader - algo objectid.Algorithm - opts Options - header HeaderInfo - headerRaw [packHeaderSize]byte - - finalized bool -} - -// Ingest reads and validates one PACK header, returning one pending operation. -// -// Labels: Deps-Borrowed, Life-Parent. -func Ingest( - src io.Reader, - algo objectid.Algorithm, - opts Options, -) (*Pending, error) { - if algo.Size() == 0 { - return nil, objectid.ErrInvalidAlgorithm - } - - reader := bufio.NewReader(src) - - header, headerRaw, err := readAndValidatePackHeader(reader) - if err != nil { - return nil, err - } - - return &Pending{ - reader: reader, - algo: algo, - opts: opts, - header: header, - headerRaw: headerRaw, - }, nil -} - -// Header returns parsed PACK header info. -func (pending *Pending) Header() HeaderInfo { - return pending.header -} - -// Continue ingests the pack stream into destination and writes pack artifacts. -// -// Continue invalidates the receiver. -// -// Artifacts are published under content-addressed final names derived from the -// resulting pack hash. If those final names already exist, Continue treats that -// as success and removes its temporary files. -func (pending *Pending) Continue(destination *os.Root) (Result, error) { - pending.finalized = true - - if pending.header.ObjectCount == 0 { - return Result{}, ErrZeroObjectContinue - } - - state, err := newIngestState( - pending.reader, - destination, - pending.algo, - pending.opts, - pending.header, - pending.headerRaw, - ) - if err != nil { - return Result{}, err - } - - return ingest(state) -} - -// Discard consumes and verifies one zero-object pack stream without writing -// files. -// -// Discard invalidates the receiver. -func (pending *Pending) Discard() (DiscardResult, error) { - pending.finalized = true - - if pending.header.ObjectCount != 0 { - return DiscardResult{}, ErrNonZeroDiscard - } - - hashImpl, err := pending.algo.New() - if err != nil { - return DiscardResult{}, err - } - - _, _ = hashImpl.Write(pending.headerRaw[:]) - - trailer := make([]byte, pending.algo.Size()) - - _, err = io.ReadFull(pending.reader, trailer) - if err != nil { - return DiscardResult{}, &PackTrailerMismatchError{} - } - - computed := hashImpl.Sum(nil) - if !bytes.Equal(computed, trailer) { - return DiscardResult{}, &PackTrailerMismatchError{} - } - - if pending.opts.RequireTrailingEOF { - var probe [1]byte - - n, err := pending.reader.Read(probe[:]) - if n > 0 || err == nil { - return DiscardResult{}, errors.New("packfile/ingest: pack has trailing garbage") - } - - if err != io.EOF { - return DiscardResult{}, err - } - } - - packHash, err := objectid.FromBytes(pending.algo, trailer) - if err != nil { - return DiscardResult{}, err - } - - return DiscardResult{ - PackHash: packHash, - ObjectCount: 0, - }, nil -} diff --git a/object/store/packed/doc.go b/object/store/packed/doc.go index 252a2baf..55189aa1 100644 --- a/object/store/packed/doc.go +++ b/object/store/packed/doc.go @@ -1,3 +1,3 @@ -// Package packed provides Git object reading from pack/index files under an -// objects/pack directory. +// Package packed provides Git object reading from, and pack writing to, +// an objects/pack directory. package packed diff --git a/format/packfile/ingest/byteslice_reader.go b/object/store/packed/internal/ingest/byteslice_reader.go index a1570ef3..a1570ef3 100644 --- a/format/packfile/ingest/byteslice_reader.go +++ b/object/store/packed/internal/ingest/byteslice_reader.go diff --git a/format/packfile/ingest/cache.go b/object/store/packed/internal/ingest/cache.go index 9a15f55f..9a15f55f 100644 --- a/format/packfile/ingest/cache.go +++ b/object/store/packed/internal/ingest/cache.go diff --git a/format/packfile/ingest/counting_writer.go b/object/store/packed/internal/ingest/counting_writer.go index 051ad9d1..051ad9d1 100644 --- a/format/packfile/ingest/counting_writer.go +++ b/object/store/packed/internal/ingest/counting_writer.go diff --git a/format/packfile/ingest/crc.go b/object/store/packed/internal/ingest/crc.go index f55af4ff..f55af4ff 100644 --- a/format/packfile/ingest/crc.go +++ b/object/store/packed/internal/ingest/crc.go diff --git a/format/packfile/ingest/delta_header.go b/object/store/packed/internal/ingest/delta_header.go index 110cf83b..110cf83b 100644 --- a/format/packfile/ingest/delta_header.go +++ b/object/store/packed/internal/ingest/delta_header.go diff --git a/format/packfile/ingest/distance.go b/object/store/packed/internal/ingest/distance.go index 9bc4d886..9bc4d886 100644 --- a/format/packfile/ingest/distance.go +++ b/object/store/packed/internal/ingest/distance.go diff --git a/format/packfile/ingest/doc.go b/object/store/packed/internal/ingest/doc.go index 2095068a..074012de 100644 --- a/format/packfile/ingest/doc.go +++ b/object/store/packed/internal/ingest/doc.go @@ -1,3 +1,3 @@ // Package ingest implements streaming ingestion of one Git pack stream into a -// destination root, producing .pack/.idx and optionally .rev. +// packed destination root, producing .pack/.idx and optionally .rev. package ingest diff --git a/format/packfile/ingest/drain.go b/object/store/packed/internal/ingest/drain.go index 7179a823..7179a823 100644 --- a/format/packfile/ingest/drain.go +++ b/object/store/packed/internal/ingest/drain.go diff --git a/format/packfile/ingest/entry.go b/object/store/packed/internal/ingest/entry.go index 363e213c..363e213c 100644 --- a/format/packfile/ingest/entry.go +++ b/object/store/packed/internal/ingest/entry.go diff --git a/format/packfile/ingest/entry_header.go b/object/store/packed/internal/ingest/entry_header.go index c74fdc16..c74fdc16 100644 --- a/format/packfile/ingest/entry_header.go +++ b/object/store/packed/internal/ingest/entry_header.go diff --git a/format/packfile/ingest/entry_prefix.go b/object/store/packed/internal/ingest/entry_prefix.go index a107b4e8..a107b4e8 100644 --- a/format/packfile/ingest/entry_prefix.go +++ b/object/store/packed/internal/ingest/entry_prefix.go diff --git a/format/packfile/ingest/errors.go b/object/store/packed/internal/ingest/errors.go index f6ee9757..cbad1e77 100644 --- a/format/packfile/ingest/errors.go +++ b/object/store/packed/internal/ingest/errors.go @@ -66,10 +66,3 @@ func (err *DestinationWriteError) Error() string { } var errExternalThinBase = errors.New("packfile/ingest: external thin base required") - -var ( - // ErrZeroObjectContinue indicates Continue was called for a zero-object pack. - ErrZeroObjectContinue = errors.New("packfile/ingest: cannot continue zero-object pack") - // ErrNonZeroDiscard indicates Discard was called for a non-zero-object pack. - ErrNonZeroDiscard = errors.New("packfile/ingest: cannot discard non-zero pack") -) diff --git a/format/packfile/ingest/file_section_writer.go b/object/store/packed/internal/ingest/file_section_writer.go index fa28c1a9..fa28c1a9 100644 --- a/format/packfile/ingest/file_section_writer.go +++ b/object/store/packed/internal/ingest/file_section_writer.go diff --git a/format/packfile/ingest/fill.go b/object/store/packed/internal/ingest/fill.go index eca4e4d6..eca4e4d6 100644 --- a/format/packfile/ingest/fill.go +++ b/object/store/packed/internal/ingest/fill.go diff --git a/format/packfile/ingest/finalize.go b/object/store/packed/internal/ingest/finalize.go index 6fe4edb2..6fe4edb2 100644 --- a/format/packfile/ingest/finalize.go +++ b/object/store/packed/internal/ingest/finalize.go diff --git a/format/packfile/ingest/flush.go b/object/store/packed/internal/ingest/flush.go index 96753170..96753170 100644 --- a/format/packfile/ingest/flush.go +++ b/object/store/packed/internal/ingest/flush.go diff --git a/format/packfile/ingest/hash.go b/object/store/packed/internal/ingest/hash.go index 4b739c20..4b739c20 100644 --- a/format/packfile/ingest/hash.go +++ b/object/store/packed/internal/ingest/hash.go diff --git a/format/packfile/ingest/header.go b/object/store/packed/internal/ingest/header.go index 5fae4c41..6b90becc 100644 --- a/format/packfile/ingest/header.go +++ b/object/store/packed/internal/ingest/header.go @@ -10,39 +10,44 @@ import ( const packHeaderSize = 12 +type packHeader struct { + Version uint32 + ObjectCount uint32 +} + // readAndValidatePackHeader reads one PACK header from src and validates it. -func readAndValidatePackHeader(src io.Reader) (HeaderInfo, [packHeaderSize]byte, error) { +func readAndValidatePackHeader(src io.Reader) (packHeader, [packHeaderSize]byte, error) { var hdr [packHeaderSize]byte _, err := io.ReadFull(src, hdr[:]) if err != nil { - return HeaderInfo{}, [packHeaderSize]byte{}, &InvalidPackHeaderError{ + return packHeader{}, [packHeaderSize]byte{}, &InvalidPackHeaderError{ Reason: fmt.Sprintf("read header: %v", err), } } header, err := parseAndValidatePackHeader(hdr) if err != nil { - return HeaderInfo{}, [packHeaderSize]byte{}, err + return packHeader{}, [packHeaderSize]byte{}, err } return header, hdr, nil } // parseAndValidatePackHeader validates one already-read PACK header. -func parseAndValidatePackHeader(hdr [packHeaderSize]byte) (HeaderInfo, error) { +func parseAndValidatePackHeader(hdr [packHeaderSize]byte) (packHeader, error) { if binary.BigEndian.Uint32(hdr[:4]) != packfile.Signature { - return HeaderInfo{}, &InvalidPackHeaderError{Reason: "signature mismatch"} + return packHeader{}, &InvalidPackHeaderError{Reason: "signature mismatch"} } version := binary.BigEndian.Uint32(hdr[4:8]) if !packfile.SupportedVersion(version) { - return HeaderInfo{}, &InvalidPackHeaderError{ + return packHeader{}, &InvalidPackHeaderError{ Reason: fmt.Sprintf("unsupported version %d", version), } } - return HeaderInfo{ + return packHeader{ Version: version, ObjectCount: binary.BigEndian.Uint32(hdr[8:12]), }, nil diff --git a/format/packfile/ingest/idx_write.go b/object/store/packed/internal/ingest/idx_write.go index fa139264..fa139264 100644 --- a/format/packfile/ingest/idx_write.go +++ b/object/store/packed/internal/ingest/idx_write.go diff --git a/format/packfile/ingest/ingest.go b/object/store/packed/internal/ingest/ingest.go index be65ff5f..be65ff5f 100644 --- a/format/packfile/ingest/ingest.go +++ b/object/store/packed/internal/ingest/ingest.go diff --git a/format/packfile/ingest/ingest_test.go b/object/store/packed/internal/ingest/ingest_test.go index c9661762..86a3e8e1 100644 --- a/format/packfile/ingest/ingest_test.go +++ b/object/store/packed/internal/ingest/ingest_test.go @@ -11,9 +11,9 @@ import ( "strings" "testing" - "codeberg.org/lindenii/furgit/format/packfile/ingest" "codeberg.org/lindenii/furgit/internal/testgit" objectid "codeberg.org/lindenii/furgit/object/id" + "codeberg.org/lindenii/furgit/object/store/packed/internal/ingest" ) type noExtraReadReader struct { @@ -28,18 +28,13 @@ func (r *noExtraReadReader) Read(p []byte) (int, error) { return r.reader.Read(p) } -func beginAndContinue( +func writePack( src io.Reader, packRoot *os.Root, algo objectid.Algorithm, opts ingest.Options, ) (ingest.Result, error) { - pending, err := ingest.Ingest(src, algo, opts) - if err != nil { - return ingest.Result{}, err - } - - return pending.Continue(packRoot) + return ingest.WritePack(packRoot, algo, src, opts) } // fixturePath returns one fixture file path for the selected algorithm. @@ -189,7 +184,7 @@ func TestIngestNonThinPackWritesPackIdxRev(t *testing.T) { packRoot := receiver.OpenPackRoot(t) - result, err := beginAndContinue(bytes.NewReader(packBytes), packRoot, algo, ingest.Options{ + result, err := writePack(bytes.NewReader(packBytes), packRoot, algo, ingest.Options{ WriteRev: true, RequireTrailingEOF: true, }) @@ -237,7 +232,7 @@ func TestIngestThinPackWithoutFixReturnsUnresolved(t *testing.T) { receiver := testgit.NewRepo(t, testgit.RepoOptions{ObjectFormat: algo, Bare: true}) packRoot := receiver.OpenPackRoot(t) - _, err := beginAndContinue(bytes.NewReader(thinPack), packRoot, algo, ingest.Options{ + _, err := writePack(bytes.NewReader(thinPack), packRoot, algo, ingest.Options{ WriteRev: true, RequireTrailingEOF: true, }) @@ -273,7 +268,7 @@ func TestIngestThinPackWithFixThin(t *testing.T) { packRoot := receiver.OpenPackRoot(t) - _, err := beginAndContinue(bytes.NewReader(basePack), packRoot, algo, ingest.Options{ + _, err := writePack(bytes.NewReader(basePack), packRoot, algo, ingest.Options{ RequireTrailingEOF: true, }) if err != nil { @@ -282,7 +277,7 @@ func TestIngestThinPackWithFixThin(t *testing.T) { receiverRepo := receiver.OpenRepository(t) - result, err := beginAndContinue(bytes.NewReader(thinPack), packRoot, algo, ingest.Options{ + result, err := writePack(bytes.NewReader(thinPack), packRoot, algo, ingest.Options{ FixThin: true, WriteRev: true, Base: receiverRepo.Objects(), @@ -317,7 +312,7 @@ func TestIngestPackTrailerMismatch(t *testing.T) { receiver := testgit.NewRepo(t, testgit.RepoOptions{ObjectFormat: algo, Bare: true}) packRoot := receiver.OpenPackRoot(t) - _, err := beginAndContinue(bytes.NewReader(packBytes), packRoot, algo, ingest.Options{ + _, err := writePack(bytes.NewReader(packBytes), packRoot, algo, ingest.Options{ WriteRev: true, RequireTrailingEOF: true, }) @@ -360,52 +355,44 @@ func zeroObjectPackBytes(t *testing.T, algo objectid.Algorithm) []byte { return append(header[:], hashImpl.Sum(nil)...) } -func TestIngestDiscardZeroObjectPack(t *testing.T) { +func TestIngestZeroObjectPackIsDiscardedInternally(t *testing.T) { t.Parallel() testgit.ForEachAlgorithm(t, func(t *testing.T, algo objectid.Algorithm) { //nolint:thelper packBytes := zeroObjectPackBytes(t, algo) + receiver := testgit.NewRepo(t, testgit.RepoOptions{ObjectFormat: algo, Bare: true}) + packRoot := receiver.OpenPackRoot(t) - pending, err := ingest.Ingest(bytes.NewReader(packBytes), algo, ingest.Options{ + result, err := writePack(bytes.NewReader(packBytes), packRoot, algo, ingest.Options{ RequireTrailingEOF: true, }) if err != nil { - t.Fatalf("Ingest: %v", err) + t.Fatalf("WritePack: %v", err) } - if pending.Header().ObjectCount != 0 { - t.Fatalf("ObjectCount = %d, want 0", pending.Header().ObjectCount) + if result.ObjectCount != 0 { + t.Fatalf("ObjectCount = %d, want 0", result.ObjectCount) } - discarded, err := pending.Discard() - if err != nil { - t.Fatalf("Discard: %v", err) + if result.PackName != "" { + t.Fatalf("PackName = %q, want empty", result.PackName) } - if discarded.ObjectCount != 0 { - t.Fatalf("Discard.ObjectCount = %d, want 0", discarded.ObjectCount) + if result.IdxName != "" { + t.Fatalf("IdxName = %q, want empty", result.IdxName) } - }) -} - -func TestIngestContinueRejectsZeroObjectPack(t *testing.T) { - t.Parallel() - testgit.ForEachAlgorithm(t, func(t *testing.T, algo objectid.Algorithm) { //nolint:thelper - packBytes := zeroObjectPackBytes(t, algo) - receiver := testgit.NewRepo(t, testgit.RepoOptions{ObjectFormat: algo, Bare: true}) - packRoot := receiver.OpenPackRoot(t) + if result.RevName != "" { + t.Fatalf("RevName = %q, want empty", result.RevName) + } - pending, err := ingest.Ingest(bytes.NewReader(packBytes), algo, ingest.Options{ - RequireTrailingEOF: true, - }) + entries, err := fs.ReadDir(packRoot.FS(), ".") if err != nil { - t.Fatalf("Ingest: %v", err) + t.Fatalf("ReadDir(pack): %v", err) } - _, err = pending.Continue(packRoot) - if !errors.Is(err, ingest.ErrZeroObjectContinue) { - t.Fatalf("Continue error = %v, want ErrZeroObjectContinue", err) + if len(entries) != 0 { + t.Fatalf("unexpected files after zero-object pack: %d", len(entries)) } }) } @@ -420,7 +407,7 @@ func TestIngestCanFinishWithoutTrailingEOF(t *testing.T) { receiver := testgit.NewRepo(t, testgit.RepoOptions{ObjectFormat: algo, Bare: true}) packRoot := receiver.OpenPackRoot(t) - result, err := beginAndContinue(&noExtraReadReader{reader: bytes.NewReader(packBytes)}, packRoot, algo, ingest.Options{ + result, err := writePack(&noExtraReadReader{reader: bytes.NewReader(packBytes)}, packRoot, algo, ingest.Options{ WriteRev: true, }) if err != nil { diff --git a/object/store/packed/internal/ingest/options.go b/object/store/packed/internal/ingest/options.go new file mode 100644 index 00000000..1ed4a123 --- /dev/null +++ b/object/store/packed/internal/ingest/options.go @@ -0,0 +1,26 @@ +package ingest + +import ( + "codeberg.org/lindenii/furgit/common/iowrap" + objectstore "codeberg.org/lindenii/furgit/object/store" +) + +// Options controls one pack ingest operation. +type Options struct { + // FixThin appends missing local bases for thin packs. + FixThin bool + // WriteRev writes a .rev alongside the .pack and .idx. + WriteRev bool + // Base supplies existing objects for thin-pack fixup. + Base objectstore.Reader + // Progress receives human-readable progress messages. + // + // When nil, no progress output is emitted. + Progress iowrap.WriteFlusher + // RequireTrailingEOF requires the source to hit EOF after the pack trailer. + // + // This is suitable for exact pack-file readers, but should be disabled for + // full-duplex transport streams like receive-pack where the peer keeps the + // connection open to read the server response. + RequireTrailingEOF bool +} diff --git a/format/packfile/ingest/progress_write.go b/object/store/packed/internal/ingest/progress_write.go index afb39305..afb39305 100644 --- a/format/packfile/ingest/progress_write.go +++ b/object/store/packed/internal/ingest/progress_write.go diff --git a/format/packfile/ingest/record_content.go b/object/store/packed/internal/ingest/record_content.go index c66a1234..c66a1234 100644 --- a/format/packfile/ingest/record_content.go +++ b/object/store/packed/internal/ingest/record_content.go diff --git a/format/packfile/ingest/record_delta.go b/object/store/packed/internal/ingest/record_delta.go index bc40367f..bc40367f 100644 --- a/format/packfile/ingest/record_delta.go +++ b/object/store/packed/internal/ingest/record_delta.go diff --git a/format/packfile/ingest/record_inflate.go b/object/store/packed/internal/ingest/record_inflate.go index b8eca25b..b8eca25b 100644 --- a/format/packfile/ingest/record_inflate.go +++ b/object/store/packed/internal/ingest/record_inflate.go diff --git a/format/packfile/ingest/record_resolve.go b/object/store/packed/internal/ingest/record_resolve.go index 7a9471dc..7a9471dc 100644 --- a/format/packfile/ingest/record_resolve.go +++ b/object/store/packed/internal/ingest/record_resolve.go diff --git a/format/packfile/ingest/records.go b/object/store/packed/internal/ingest/records.go index 75f157fa..75f157fa 100644 --- a/format/packfile/ingest/records.go +++ b/object/store/packed/internal/ingest/records.go diff --git a/format/packfile/ingest/resolve_all.go b/object/store/packed/internal/ingest/resolve_all.go index 90464015..90464015 100644 --- a/format/packfile/ingest/resolve_all.go +++ b/object/store/packed/internal/ingest/resolve_all.go diff --git a/object/store/packed/internal/ingest/result.go b/object/store/packed/internal/ingest/result.go new file mode 100644 index 00000000..9a285f09 --- /dev/null +++ b/object/store/packed/internal/ingest/result.go @@ -0,0 +1,23 @@ +package ingest + +import objectid "codeberg.org/lindenii/furgit/object/id" + +// Result describes one successful ingest transaction. +type Result struct { + // PackName is the destination-relative filename of the written .pack. + PackName string + // IdxName is the destination-relative filename of the written .idx. + IdxName string + // RevName is the destination-relative filename of the written .rev. + // + // RevName is empty when writeRev is false. + RevName string + // PackHash is the final pack hash (same hash embedded in .idx/.rev trailers). + PackHash objectid.ObjectID + // ObjectCount is the final object count in the resulting pack. + // + // If thin fixup appends objects, this includes appended base objects. + ObjectCount uint32 + // ThinFixed reports whether thin fixup appended local bases. + ThinFixed bool +} diff --git a/format/packfile/ingest/rev_write.go b/object/store/packed/internal/ingest/rev_write.go index 16d27085..16d27085 100644 --- a/format/packfile/ingest/rev_write.go +++ b/object/store/packed/internal/ingest/rev_write.go diff --git a/format/packfile/ingest/rewrite_header_trailer.go b/object/store/packed/internal/ingest/rewrite_header_trailer.go index f1f18a39..f1f18a39 100644 --- a/format/packfile/ingest/rewrite_header_trailer.go +++ b/object/store/packed/internal/ingest/rewrite_header_trailer.go diff --git a/format/packfile/ingest/scan.go b/object/store/packed/internal/ingest/scan.go index ddd1eaf3..ddd1eaf3 100644 --- a/format/packfile/ingest/scan.go +++ b/object/store/packed/internal/ingest/scan.go diff --git a/format/packfile/ingest/state.go b/object/store/packed/internal/ingest/state.go index 797323b2..0412eb32 100644 --- a/format/packfile/ingest/state.go +++ b/object/store/packed/internal/ingest/state.go @@ -49,7 +49,7 @@ func newIngestState( destination *os.Root, algo objectid.Algorithm, opts Options, - header HeaderInfo, + header packHeader, headerRaw [packHeaderSize]byte, ) (*ingestState, error) { if algo.Size() == 0 { diff --git a/format/packfile/ingest/stream.go b/object/store/packed/internal/ingest/stream.go index a403087a..a403087a 100644 --- a/format/packfile/ingest/stream.go +++ b/object/store/packed/internal/ingest/stream.go diff --git a/format/packfile/ingest/temp.go b/object/store/packed/internal/ingest/temp.go index d0b7862c..d0b7862c 100644 --- a/format/packfile/ingest/temp.go +++ b/object/store/packed/internal/ingest/temp.go diff --git a/format/packfile/ingest/testdata/fixtures/sha1/METADATA.txt b/object/store/packed/internal/ingest/testdata/fixtures/sha1/METADATA.txt index 5fcbfe26..5fcbfe26 100644 --- a/format/packfile/ingest/testdata/fixtures/sha1/METADATA.txt +++ b/object/store/packed/internal/ingest/testdata/fixtures/sha1/METADATA.txt diff --git a/format/packfile/ingest/testdata/fixtures/sha1/base.pack b/object/store/packed/internal/ingest/testdata/fixtures/sha1/base.pack Binary files differindex 3d7a4903..3d7a4903 100644 --- a/format/packfile/ingest/testdata/fixtures/sha1/base.pack +++ b/object/store/packed/internal/ingest/testdata/fixtures/sha1/base.pack diff --git a/format/packfile/ingest/testdata/fixtures/sha1/nonthin.pack b/object/store/packed/internal/ingest/testdata/fixtures/sha1/nonthin.pack Binary files differindex ea07c9a0..ea07c9a0 100644 --- a/format/packfile/ingest/testdata/fixtures/sha1/nonthin.pack +++ b/object/store/packed/internal/ingest/testdata/fixtures/sha1/nonthin.pack diff --git a/format/packfile/ingest/testdata/fixtures/sha1/thin.pack b/object/store/packed/internal/ingest/testdata/fixtures/sha1/thin.pack Binary files differindex 95084feb..95084feb 100644 --- a/format/packfile/ingest/testdata/fixtures/sha1/thin.pack +++ b/object/store/packed/internal/ingest/testdata/fixtures/sha1/thin.pack diff --git a/format/packfile/ingest/testdata/fixtures/sha256/METADATA.txt b/object/store/packed/internal/ingest/testdata/fixtures/sha256/METADATA.txt index 8a5ea0a2..8a5ea0a2 100644 --- a/format/packfile/ingest/testdata/fixtures/sha256/METADATA.txt +++ b/object/store/packed/internal/ingest/testdata/fixtures/sha256/METADATA.txt diff --git a/format/packfile/ingest/testdata/fixtures/sha256/base.pack b/object/store/packed/internal/ingest/testdata/fixtures/sha256/base.pack Binary files differindex 52ceef74..52ceef74 100644 --- a/format/packfile/ingest/testdata/fixtures/sha256/base.pack +++ b/object/store/packed/internal/ingest/testdata/fixtures/sha256/base.pack diff --git a/format/packfile/ingest/testdata/fixtures/sha256/nonthin.pack b/object/store/packed/internal/ingest/testdata/fixtures/sha256/nonthin.pack Binary files differindex 50db05d0..50db05d0 100644 --- a/format/packfile/ingest/testdata/fixtures/sha256/nonthin.pack +++ b/object/store/packed/internal/ingest/testdata/fixtures/sha256/nonthin.pack diff --git a/format/packfile/ingest/testdata/fixtures/sha256/thin.pack b/object/store/packed/internal/ingest/testdata/fixtures/sha256/thin.pack Binary files differindex b331b915..b331b915 100644 --- a/format/packfile/ingest/testdata/fixtures/sha256/thin.pack +++ b/object/store/packed/internal/ingest/testdata/fixtures/sha256/thin.pack diff --git a/format/packfile/ingest/thin_append.go b/object/store/packed/internal/ingest/thin_append.go index 779d477f..779d477f 100644 --- a/format/packfile/ingest/thin_append.go +++ b/object/store/packed/internal/ingest/thin_append.go diff --git a/format/packfile/ingest/thin_fix.go b/object/store/packed/internal/ingest/thin_fix.go index f66ed279..f66ed279 100644 --- a/format/packfile/ingest/thin_fix.go +++ b/object/store/packed/internal/ingest/thin_fix.go diff --git a/format/packfile/ingest/thin_unresolved.go b/object/store/packed/internal/ingest/thin_unresolved.go index 757cc0e2..757cc0e2 100644 --- a/format/packfile/ingest/thin_unresolved.go +++ b/object/store/packed/internal/ingest/thin_unresolved.go diff --git a/format/packfile/ingest/trailer.go b/object/store/packed/internal/ingest/trailer.go index 7a26a8f2..7a26a8f2 100644 --- a/format/packfile/ingest/trailer.go +++ b/object/store/packed/internal/ingest/trailer.go diff --git a/format/packfile/ingest/use.go b/object/store/packed/internal/ingest/use.go index 97f8757a..97f8757a 100644 --- a/format/packfile/ingest/use.go +++ b/object/store/packed/internal/ingest/use.go diff --git a/object/store/packed/internal/ingest/write.go b/object/store/packed/internal/ingest/write.go new file mode 100644 index 00000000..efd27323 --- /dev/null +++ b/object/store/packed/internal/ingest/write.go @@ -0,0 +1,50 @@ +package ingest + +import ( + "bufio" + "io" + "os" + + objectid "codeberg.org/lindenii/furgit/object/id" +) + +// WritePack ingests one pack stream into destination and writes pack artifacts. +// +// Artifacts are published under content-addressed final names derived from the +// resulting pack hash. If those final names already exist, WritePack treats +// that as success and removes its temporary files. +func WritePack( + destination *os.Root, + algo objectid.Algorithm, + src io.Reader, + opts Options, +) (Result, error) { + if algo.Size() == 0 { + return Result{}, objectid.ErrInvalidAlgorithm + } + + reader := bufio.NewReader(src) + + header, headerRaw, err := readAndValidatePackHeader(reader) + if err != nil { + return Result{}, err + } + + if header.ObjectCount == 0 { + return discardZeroObjectPack(reader, algo, opts, headerRaw) + } + + state, err := newIngestState( + reader, + destination, + algo, + opts, + header, + headerRaw, + ) + if err != nil { + return Result{}, err + } + + return ingest(state) +} diff --git a/object/store/packed/internal/ingest/write_empty.go b/object/store/packed/internal/ingest/write_empty.go new file mode 100644 index 00000000..0d3401f0 --- /dev/null +++ b/object/store/packed/internal/ingest/write_empty.go @@ -0,0 +1,58 @@ +package ingest + +import ( + "bytes" + "errors" + "io" + + objectid "codeberg.org/lindenii/furgit/object/id" +) + +func discardZeroObjectPack( + src io.Reader, + algo objectid.Algorithm, + opts Options, + headerRaw [packHeaderSize]byte, +) (Result, error) { + hashImpl, err := algo.New() + if err != nil { + return Result{}, err + } + + _, _ = hashImpl.Write(headerRaw[:]) + + trailer := make([]byte, algo.Size()) + + _, err = io.ReadFull(src, trailer) + if err != nil { + return Result{}, &PackTrailerMismatchError{} + } + + computed := hashImpl.Sum(nil) + if !bytes.Equal(computed, trailer) { + return Result{}, &PackTrailerMismatchError{} + } + + if opts.RequireTrailingEOF { + var probe [1]byte + + n, err := src.Read(probe[:]) + if n > 0 || err == nil { + return Result{}, errors.New("packfile/ingest: pack has trailing garbage") + } + + if err != io.EOF { + return Result{}, err + } + } + + packHash, err := objectid.FromBytes(algo, trailer) + if err != nil { + return Result{}, err + } + + return Result{ + PackHash: packHash, + ObjectCount: 0, + }, nil +} diff --git a/object/store/packed/writer.go b/object/store/packed/writer.go new file mode 100644 index 00000000..28867291 --- /dev/null +++ b/object/store/packed/writer.go @@ -0,0 +1,17 @@ +package packed + +import ( + "io" + + objectstore "codeberg.org/lindenii/furgit/object/store" + "codeberg.org/lindenii/furgit/object/store/packed/internal/ingest" +) + +var _ objectstore.PackWriter = (*Store)(nil) + +// WritePack ingests one pack stream into the packed store. +func (store *Store) WritePack(src io.Reader, _ objectstore.PackWriteOptions) error { + _, err := ingest.WritePack(store.root, store.algo, src, ingest.Options{}) + + return err +} |
