format/pack/ingest: Write index progress too v0.1.79

3 files changed, 112 insertions, 29 deletions

diff --git a/format/pack/ingest/idx_write.go b/format/pack/ingest/idx_write.go
index 730548aa..0523ac1d 100644
--- a/format/pack/ingest/idx_write.go
+++ b/format/pack/ingest/idx_write.go
@@ -9,6 +9,7 @@ import (
 	"slices"
 
 	"codeberg.org/lindenii/furgit/internal/intconv"
+	"codeberg.org/lindenii/furgit/internal/progress"
 )
 
 const (
@@ -26,35 +27,35 @@ func writeIdx(state *ingestState) error {
 	}
 
 	write := func(src []byte) error {
-		_, err := state.idxFile.Write(src)
-		if err != nil {
-			return err
+		_, writeErr := state.idxFile.Write(src)
+		if writeErr != nil {
+			return writeErr
 		}
 
-		_, err = hashImpl.Write(src)
-		if err != nil {
-			return err
+		_, writeErr = hashImpl.Write(src)
+		if writeErr != nil {
+			return writeErr
 		}
 
 		return nil
 	}
 
 	var scratch [8]byte
-	binary.BigEndian.PutUint32(scratch[:4], idxMagicV2)
-	binary.BigEndian.PutUint32(scratch[4:8], idxVersionV2)
-
-	err = write(scratch[:8])
-	if err != nil {
-		return err
-	}
-
 	var fanout [256]uint32
+	writeProgress(state, "writing index fanout...\r")
 
 	for _, recordIdx := range order {
 		idRaw := state.records[recordIdx].objectID.Bytes()
 		fanout[idRaw[0]]++
 	}
 
+	binary.BigEndian.PutUint32(scratch[:4], idxMagicV2)
+	binary.BigEndian.PutUint32(scratch[4:8], idxVersionV2)
+	err = write(scratch[:8])
+	if err != nil {
+		return err
+	}
+
 	var cumulative uint32
 	for i := range fanout {
 		cumulative += fanout[i]
@@ -65,6 +66,22 @@ func writeIdx(state *ingestState) error {
 			return err
 		}
 	}
+	writeProgress(state, "writing index fanout: done.\n")
+
+	largeOffsetCount := 0
+	for idx := range state.records {
+		if state.records[idx].offset >= 0x80000000 {
+			largeOffsetCount++
+		}
+	}
+
+	oidMeter := progress.New(progress.Options{
+		Writer: state.opts.Progress,
+		Flush:  state.opts.ProgressFlush,
+		Title:  "writing index object ids",
+		Total:  uint64(len(order)),
+	})
+	var oidDone uint64
 
 	for _, recordIdx := range order {
 		idRaw := state.records[recordIdx].objectID.Bytes()
@@ -73,8 +90,22 @@ func writeIdx(state *ingestState) error {
 		if err != nil {
 			return err
 		}
+
+		oidDone++
+		oidMeter.Set(oidDone, 0)
+	}
+	if oidDone > 0 {
+		oidMeter.Stop("done")
 	}
 
+	crcMeter := progress.New(progress.Options{
+		Writer: state.opts.Progress,
+		Flush:  state.opts.ProgressFlush,
+		Title:  "writing index crc32",
+		Total:  uint64(len(order)),
+	})
+	var crcDone uint64
+
 	for _, recordIdx := range order {
 		binary.BigEndian.PutUint32(scratch[:4], state.records[recordIdx].crc32)
 
@@ -82,9 +113,22 @@ func writeIdx(state *ingestState) error {
 		if err != nil {
 			return err
 		}
+
+		crcDone++
+		crcMeter.Set(crcDone, 0)
+	}
+	if crcDone > 0 {
+		crcMeter.Stop("done")
 	}
 
 	largeOffsets := make([]uint64, 0)
+	offsetMeter := progress.New(progress.Options{
+		Writer: state.opts.Progress,
+		Flush:  state.opts.ProgressFlush,
+		Title:  "writing index offsets",
+		Total:  uint64(len(order)),
+	})
+	var offsetDone uint64
 
 	for _, recordIdx := range order {
 		offset := state.records[recordIdx].offset
@@ -107,8 +151,21 @@ func writeIdx(state *ingestState) error {
 		if err != nil {
 			return err
 		}
+
+		offsetDone++
+		offsetMeter.Set(offsetDone, 0)
+	}
+	if offsetDone > 0 {
+		offsetMeter.Stop("done")
 	}
 
+	largeOffsetMeter := progress.New(progress.Options{
+		Writer: state.opts.Progress,
+		Flush:  state.opts.ProgressFlush,
+		Title:  "writing index large offsets",
+		Total:  uint64(largeOffsetCount),
+	})
+	var largeOffsetDone uint64
 	for _, off := range largeOffsets {
 		binary.BigEndian.PutUint64(scratch[:8], off)
 
@@ -116,7 +173,15 @@ func writeIdx(state *ingestState) error {
 		if err != nil {
 			return err
 		}
+
+		largeOffsetDone++
+		largeOffsetMeter.Set(largeOffsetDone, 0)
 	}
+	if largeOffsetDone > 0 {
+		largeOffsetMeter.Stop("done")
+	}
+
+	writeProgress(state, "writing index trailer...\r")
 
 	err = write(state.packHash.Bytes())
 	if err != nil {
@@ -130,7 +195,13 @@ func writeIdx(state *ingestState) error {
 		return err
 	}
 
-	return state.idxFile.Sync()
+	err = state.idxFile.Sync()
+	if err != nil {
+		return err
+	}
+	writeProgress(state, "writing index trailer: done.\n")
+
+	return nil
 }
 
 // buildIdxOrder returns record indexes sorted by ObjectID.
diff --git a/format/pack/ingest/ingest.go b/format/pack/ingest/ingest.go
index 493942a0..be65ff5f 100644
--- a/format/pack/ingest/ingest.go
+++ b/format/pack/ingest/ingest.go
@@ -34,7 +34,7 @@ func ingest(state *ingestState) (out Result, err error) {
 	}
 
 	if state.thinFixed {
-		err := resolveAll(state)
+		err = resolveAll(state)
 		if err != nil {
 			return Result{}, err
 		}
@@ -49,8 +49,6 @@ func ingest(state *ingestState) (out Result, err error) {
 		return Result{}, err
 	}
 
-	writeProgress(state, "writing index...\r")
-
 	err = state.packFile.Sync()
 	if err != nil {
 		return Result{}, &DestinationWriteError{Op: fmt.Sprintf("sync pack: %v", err)}
@@ -61,20 +59,10 @@ func ingest(state *ingestState) (out Result, err error) {
 		return Result{}, err
 	}
 
-	writeProgress(state, "writing index: done.\n")
-
-	if state.opts.WriteRev {
-		writeProgress(state, "writing reverse index...\r")
-	}
-
 	err = writeRev(state)
 	if err != nil {
 		return Result{}, err
 	}
 
-	if state.opts.WriteRev {
-		writeProgress(state, "writing reverse index: done.\n")
-	}
-
 	return finalizeArtifacts(state)
 }
diff --git a/format/pack/ingest/rev_write.go b/format/pack/ingest/rev_write.go
index 5a2f5375..9ed8dc6c 100644
--- a/format/pack/ingest/rev_write.go
+++ b/format/pack/ingest/rev_write.go
@@ -5,6 +5,7 @@ import (
 	"slices"
 
 	"codeberg.org/lindenii/furgit/internal/intconv"
+	"codeberg.org/lindenii/furgit/internal/progress"
 )
 
 const (
@@ -33,6 +34,7 @@ func writeRev(state *ingestState) error {
 	}
 
 	var scratch [8]byte
+	writeProgress(state, "writing reverse index header...\r")
 	binary.BigEndian.PutUint32(scratch[:4], revMagic)
 
 	err = writeAndHash(state.revFile, hashImpl, scratch[:4])
@@ -53,6 +55,15 @@ func writeRev(state *ingestState) error {
 	if err != nil {
 		return err
 	}
+	writeProgress(state, "writing reverse index header: done.\n")
+
+	entriesMeter := progress.New(progress.Options{
+		Writer: state.opts.Progress,
+		Flush:  state.opts.ProgressFlush,
+		Title:  "writing reverse index entries",
+		Total:  uint64(len(packOrder)),
+	})
+	var entriesDone uint64
 
 	for _, recordIdx := range packOrder {
 		recordPos, err := intconv.IntToUint32(recordToIdxPos[recordIdx])
@@ -66,7 +77,14 @@ func writeRev(state *ingestState) error {
 		if err != nil {
 			return err
 		}
+		entriesDone++
+		entriesMeter.Set(entriesDone, 0)
 	}
+	if entriesDone > 0 {
+		entriesMeter.Stop("done")
+	}
+
+	writeProgress(state, "writing reverse index trailer...\r")
 
 	err = writeAndHash(state.revFile, hashImpl, state.packHash.Bytes())
 	if err != nil {
@@ -80,7 +98,13 @@ func writeRev(state *ingestState) error {
 		return err
 	}
 
-	return state.revFile.Sync()
+	err = state.revFile.Sync()
+	if err != nil {
+		return err
+	}
+	writeProgress(state, "writing reverse index trailer: done.\n")
+
+	return nil
 }
 
 // buildPackOrder returns record indexes sorted by pack offset.