format/pack/ingest: Init

1 files changed, 138 insertions, 0 deletions

diff --git a/format/pack/ingest/idx_write.go b/format/pack/ingest/idx_write.go
new file mode 100644
index 00000000..1e5f20c4
--- /dev/null
+++ b/format/pack/ingest/idx_write.go
@@ -0,0 +1,138 @@
+package ingest
+
+import (
+	"bytes"
+	"encoding/binary"
+	"fmt"
+	"hash"
+	"io"
+	"slices"
+)
+
+const (
+	idxMagicV2   = 0xff744f63
+	idxVersionV2 = 2
+)
+
+// writeIdx writes idx v2 for resolved records.
+func writeIdx(state *ingestState) error {
+	order := buildIdxOrder(state)
+	hashImpl, err := state.algo.New()
+	if err != nil {
+		return err
+	}
+
+	write := func(src []byte) error {
+		if _, err := state.idxFile.Write(src); err != nil {
+			return err
+		}
+		if _, err := hashImpl.Write(src); err != nil {
+			return err
+		}
+
+		return nil
+	}
+
+	var scratch [8]byte
+	binary.BigEndian.PutUint32(scratch[:4], idxMagicV2)
+	binary.BigEndian.PutUint32(scratch[4:8], idxVersionV2)
+	if err := write(scratch[:8]); err != nil {
+		return err
+	}
+
+	var fanout [256]uint32
+	for _, recordIdx := range order {
+		idRaw := state.records[recordIdx].objectID.Bytes()
+		fanout[idRaw[0]]++
+	}
+	var cumulative uint32
+	for i := range fanout {
+		cumulative += fanout[i]
+		binary.BigEndian.PutUint32(scratch[:4], cumulative)
+		if err := write(scratch[:4]); err != nil {
+			return err
+		}
+	}
+
+	for _, recordIdx := range order {
+		idRaw := state.records[recordIdx].objectID.Bytes()
+		if err := write(idRaw); err != nil {
+			return err
+		}
+	}
+
+	for _, recordIdx := range order {
+		binary.BigEndian.PutUint32(scratch[:4], state.records[recordIdx].crc32)
+		if err := write(scratch[:4]); err != nil {
+			return err
+		}
+	}
+
+	largeOffsets := make([]uint64, 0)
+	for _, recordIdx := range order {
+		offset := state.records[recordIdx].offset
+		if offset >= 0x80000000 {
+			word := 0x80000000 | uint32(len(largeOffsets))
+			largeOffsets = append(largeOffsets, offset)
+			binary.BigEndian.PutUint32(scratch[:4], word)
+		} else {
+			binary.BigEndian.PutUint32(scratch[:4], uint32(offset))
+		}
+		if err := write(scratch[:4]); err != nil {
+			return err
+		}
+	}
+	for _, off := range largeOffsets {
+		binary.BigEndian.PutUint64(scratch[:8], off)
+		if err := write(scratch[:8]); err != nil {
+			return err
+		}
+	}
+
+	if err := write(state.packHash.Bytes()); err != nil {
+		return err
+	}
+
+	idxHash := hashImpl.Sum(nil)
+	if _, err := state.idxFile.Write(idxHash); err != nil {
+		return err
+	}
+
+	return state.idxFile.Sync()
+}
+
+// buildIdxOrder returns record indexes sorted by ObjectID.
+func buildIdxOrder(state *ingestState) []int {
+	out := make([]int, 0, len(state.records))
+	for idx := range state.records {
+		out = append(out, idx)
+	}
+	slices.SortFunc(out, func(a, b int) int {
+		return bytes.Compare(state.records[a].objectID.Bytes(), state.records[b].objectID.Bytes())
+	})
+
+	return out
+}
+
+// verifyResolvedRecords checks that all records are fully resolved before index writing.
+func verifyResolvedRecords(state *ingestState) error {
+	for idx, record := range state.records {
+		if !record.resolved {
+			return fmt.Errorf("format/pack/ingest: unresolved record %d at offset %d", idx, record.offset)
+		}
+	}
+
+	return nil
+}
+
+// writeAndHash writes src to dst and updates hash.
+func writeAndHash(dst io.Writer, hashImpl hash.Hash, src []byte) error {
+	if _, err := dst.Write(src); err != nil {
+		return err
+	}
+	if _, err := hashImpl.Write(src); err != nil {
+		return err
+	}
+
+	return nil
+}