object,store/packed{,/internal/ingest}: Move from format/packfile/ingest

1 files changed, 262 insertions, 0 deletions

diff --git a/object/store/packed/internal/ingest/idx_write.go b/object/store/packed/internal/ingest/idx_write.go
new file mode 100644
index 00000000..fa139264
--- /dev/null
+++ b/object/store/packed/internal/ingest/idx_write.go
@@ -0,0 +1,262 @@
+package ingest
+
+import (
+	"bytes"
+	"encoding/binary"
+	"fmt"
+	"hash"
+	"io"
+	"slices"
+
+	"codeberg.org/lindenii/furgit/internal/intconv"
+	"codeberg.org/lindenii/furgit/internal/progress"
+)
+
+const (
+	idxMagicV2   = 0xff744f63
+	idxVersionV2 = 2
+)
+
+// writeIdx writes idx v2 for resolved records.
+func writeIdx(state *ingestState) error {
+	order := buildIdxOrder(state)
+
+	hashImpl, err := state.algo.New()
+	if err != nil {
+		return err
+	}
+
+	write := func(src []byte) error {
+		_, writeErr := state.idxFile.Write(src)
+		if writeErr != nil {
+			return writeErr
+		}
+
+		_, writeErr = hashImpl.Write(src)
+		if writeErr != nil {
+			return writeErr
+		}
+
+		return nil
+	}
+
+	var (
+		scratch [8]byte
+		fanout  [256]uint32
+	)
+
+	writeProgressf(state, "writing index fanout...\r")
+
+	for _, recordIdx := range order {
+		idRaw := state.records[recordIdx].objectID.Bytes()
+		fanout[idRaw[0]]++
+	}
+
+	binary.BigEndian.PutUint32(scratch[:4], idxMagicV2)
+	binary.BigEndian.PutUint32(scratch[4:8], idxVersionV2)
+
+	err = write(scratch[:8])
+	if err != nil {
+		return err
+	}
+
+	var cumulative uint32
+	for i := range fanout {
+		cumulative += fanout[i]
+		binary.BigEndian.PutUint32(scratch[:4], cumulative)
+
+		err := write(scratch[:4])
+		if err != nil {
+			return err
+		}
+	}
+
+	writeProgressf(state, "writing index fanout: done.\n")
+
+	largeOffsetCount := 0
+
+	for idx := range state.records {
+		if state.records[idx].offset >= 0x80000000 {
+			largeOffsetCount++
+		}
+	}
+
+	oidMeter := progress.New(progress.Options{
+		Writer: state.opts.Progress,
+		Title:  "writing index object ids",
+		Total:  uint64(len(order)),
+	})
+
+	var oidDone uint64
+
+	for _, recordIdx := range order {
+		idRaw := state.records[recordIdx].objectID.Bytes()
+
+		err := write(idRaw)
+		if err != nil {
+			return err
+		}
+
+		oidDone++
+		oidMeter.Set(oidDone, 0)
+	}
+
+	if oidDone > 0 {
+		oidMeter.Stop("done")
+	}
+
+	crcMeter := progress.New(progress.Options{
+		Writer: state.opts.Progress,
+		Title:  "writing index crc32",
+		Total:  uint64(len(order)),
+	})
+
+	var crcDone uint64
+
+	for _, recordIdx := range order {
+		binary.BigEndian.PutUint32(scratch[:4], state.records[recordIdx].crc32)
+
+		err := write(scratch[:4])
+		if err != nil {
+			return err
+		}
+
+		crcDone++
+		crcMeter.Set(crcDone, 0)
+	}
+
+	if crcDone > 0 {
+		crcMeter.Stop("done")
+	}
+
+	largeOffsets := make([]uint64, 0)
+	offsetMeter := progress.New(progress.Options{
+		Writer: state.opts.Progress,
+		Title:  "writing index offsets",
+		Total:  uint64(len(order)),
+	})
+
+	var offsetDone uint64
+
+	for _, recordIdx := range order {
+		offset := state.records[recordIdx].offset
+		if offset >= 0x80000000 {
+			largeOffsetIdx, err := intconv.IntToUint32(len(largeOffsets))
+			if err != nil {
+				return err
+			}
+
+			word := 0x80000000 | largeOffsetIdx
+
+			largeOffsets = append(largeOffsets, offset)
+
+			binary.BigEndian.PutUint32(scratch[:4], word)
+		} else {
+			binary.BigEndian.PutUint32(scratch[:4], uint32(offset))
+		}
+
+		err := write(scratch[:4])
+		if err != nil {
+			return err
+		}
+
+		offsetDone++
+		offsetMeter.Set(offsetDone, 0)
+	}
+
+	if offsetDone > 0 {
+		offsetMeter.Stop("done")
+	}
+
+	total, err := intconv.IntToUint64(largeOffsetCount)
+	if err != nil {
+		return err
+	}
+
+	largeOffsetMeter := progress.New(progress.Options{
+		Writer: state.opts.Progress,
+		Title:  "writing index large offsets",
+		Total:  total,
+	})
+
+	var largeOffsetDone uint64
+
+	for _, off := range largeOffsets {
+		binary.BigEndian.PutUint64(scratch[:8], off)
+
+		err := write(scratch[:8])
+		if err != nil {
+			return err
+		}
+
+		largeOffsetDone++
+		largeOffsetMeter.Set(largeOffsetDone, 0)
+	}
+
+	if largeOffsetDone > 0 {
+		largeOffsetMeter.Stop("done")
+	}
+
+	writeProgressf(state, "writing index trailer...\r")
+
+	err = write(state.packHash.Bytes())
+	if err != nil {
+		return err
+	}
+
+	idxHash := hashImpl.Sum(nil)
+
+	_, err = state.idxFile.Write(idxHash)
+	if err != nil {
+		return err
+	}
+
+	err = state.idxFile.Sync()
+	if err != nil {
+		return err
+	}
+
+	writeProgressf(state, "writing index trailer: done.\n")
+
+	return nil
+}
+
+// buildIdxOrder returns record indexes sorted by ObjectID.
+func buildIdxOrder(state *ingestState) []int {
+	out := make([]int, 0, len(state.records))
+	for idx := range state.records {
+		out = append(out, idx)
+	}
+
+	slices.SortFunc(out, func(a, b int) int {
+		return bytes.Compare(state.records[a].objectID.Bytes(), state.records[b].objectID.Bytes())
+	})
+
+	return out
+}
+
+// verifyResolvedRecords checks that all records are fully resolved before index writing.
+func verifyResolvedRecords(state *ingestState) error {
+	for idx, record := range state.records {
+		if !record.resolved {
+			return fmt.Errorf("packfile/ingest: unresolved record %d at offset %d", idx, record.offset)
+		}
+	}
+
+	return nil
+}
+
+// writeAndHash writes src to dst and updates hash.
+func writeAndHash(dst io.Writer, hashImpl hash.Hash, src []byte) error {
+	_, err := dst.Write(src)
+	if err != nil {
+		return err
+	}
+
+	_, err = hashImpl.Write(src)
+	if err != nil {
+		return err
+	}
+
+	return nil
+}