From b90caf8f6606cdecd8529dc365b53584ee402d8c Mon Sep 17 00:00:00 2001 From: Runxi Yu Date: Sun, 8 Mar 2026 14:25:33 +0800 Subject: format/pack/ingest: Write index progress too --- format/pack/ingest/idx_write.go | 101 ++++++++++++++++++++++++++++++++++------ format/pack/ingest/ingest.go | 14 +----- format/pack/ingest/rev_write.go | 26 ++++++++++- 3 files changed, 112 insertions(+), 29 deletions(-) diff --git a/format/pack/ingest/idx_write.go b/format/pack/ingest/idx_write.go index 730548aa..0523ac1d 100644 --- a/format/pack/ingest/idx_write.go +++ b/format/pack/ingest/idx_write.go @@ -9,6 +9,7 @@ import ( "slices" "codeberg.org/lindenii/furgit/internal/intconv" + "codeberg.org/lindenii/furgit/internal/progress" ) const ( @@ -26,35 +27,35 @@ func writeIdx(state *ingestState) error { } write := func(src []byte) error { - _, err := state.idxFile.Write(src) - if err != nil { - return err + _, writeErr := state.idxFile.Write(src) + if writeErr != nil { + return writeErr } - _, err = hashImpl.Write(src) - if err != nil { - return err + _, writeErr = hashImpl.Write(src) + if writeErr != nil { + return writeErr } return nil } var scratch [8]byte - binary.BigEndian.PutUint32(scratch[:4], idxMagicV2) - binary.BigEndian.PutUint32(scratch[4:8], idxVersionV2) - - err = write(scratch[:8]) - if err != nil { - return err - } - var fanout [256]uint32 + writeProgress(state, "writing index fanout...\r") for _, recordIdx := range order { idRaw := state.records[recordIdx].objectID.Bytes() fanout[idRaw[0]]++ } + binary.BigEndian.PutUint32(scratch[:4], idxMagicV2) + binary.BigEndian.PutUint32(scratch[4:8], idxVersionV2) + err = write(scratch[:8]) + if err != nil { + return err + } + var cumulative uint32 for i := range fanout { cumulative += fanout[i] @@ -65,6 +66,22 @@ func writeIdx(state *ingestState) error { return err } } + writeProgress(state, "writing index fanout: done.\n") + + largeOffsetCount := 0 + for idx := range state.records { + if state.records[idx].offset >= 0x80000000 { + largeOffsetCount++ + } + } + + oidMeter := progress.New(progress.Options{ + Writer: state.opts.Progress, + Flush: state.opts.ProgressFlush, + Title: "writing index object ids", + Total: uint64(len(order)), + }) + var oidDone uint64 for _, recordIdx := range order { idRaw := state.records[recordIdx].objectID.Bytes() @@ -73,8 +90,22 @@ func writeIdx(state *ingestState) error { if err != nil { return err } + + oidDone++ + oidMeter.Set(oidDone, 0) + } + if oidDone > 0 { + oidMeter.Stop("done") } + crcMeter := progress.New(progress.Options{ + Writer: state.opts.Progress, + Flush: state.opts.ProgressFlush, + Title: "writing index crc32", + Total: uint64(len(order)), + }) + var crcDone uint64 + for _, recordIdx := range order { binary.BigEndian.PutUint32(scratch[:4], state.records[recordIdx].crc32) @@ -82,9 +113,22 @@ func writeIdx(state *ingestState) error { if err != nil { return err } + + crcDone++ + crcMeter.Set(crcDone, 0) + } + if crcDone > 0 { + crcMeter.Stop("done") } largeOffsets := make([]uint64, 0) + offsetMeter := progress.New(progress.Options{ + Writer: state.opts.Progress, + Flush: state.opts.ProgressFlush, + Title: "writing index offsets", + Total: uint64(len(order)), + }) + var offsetDone uint64 for _, recordIdx := range order { offset := state.records[recordIdx].offset @@ -107,8 +151,21 @@ func writeIdx(state *ingestState) error { if err != nil { return err } + + offsetDone++ + offsetMeter.Set(offsetDone, 0) + } + if offsetDone > 0 { + offsetMeter.Stop("done") } + largeOffsetMeter := progress.New(progress.Options{ + Writer: state.opts.Progress, + Flush: state.opts.ProgressFlush, + Title: "writing index large offsets", + Total: uint64(largeOffsetCount), + }) + var largeOffsetDone uint64 for _, off := range largeOffsets { binary.BigEndian.PutUint64(scratch[:8], off) @@ -116,7 +173,15 @@ func writeIdx(state *ingestState) error { if err != nil { return err } + + largeOffsetDone++ + largeOffsetMeter.Set(largeOffsetDone, 0) } + if largeOffsetDone > 0 { + largeOffsetMeter.Stop("done") + } + + writeProgress(state, "writing index trailer...\r") err = write(state.packHash.Bytes()) if err != nil { @@ -130,7 +195,13 @@ func writeIdx(state *ingestState) error { return err } - return state.idxFile.Sync() + err = state.idxFile.Sync() + if err != nil { + return err + } + writeProgress(state, "writing index trailer: done.\n") + + return nil } // buildIdxOrder returns record indexes sorted by ObjectID. diff --git a/format/pack/ingest/ingest.go b/format/pack/ingest/ingest.go index 493942a0..be65ff5f 100644 --- a/format/pack/ingest/ingest.go +++ b/format/pack/ingest/ingest.go @@ -34,7 +34,7 @@ func ingest(state *ingestState) (out Result, err error) { } if state.thinFixed { - err := resolveAll(state) + err = resolveAll(state) if err != nil { return Result{}, err } @@ -49,8 +49,6 @@ func ingest(state *ingestState) (out Result, err error) { return Result{}, err } - writeProgress(state, "writing index...\r") - err = state.packFile.Sync() if err != nil { return Result{}, &DestinationWriteError{Op: fmt.Sprintf("sync pack: %v", err)} @@ -61,20 +59,10 @@ func ingest(state *ingestState) (out Result, err error) { return Result{}, err } - writeProgress(state, "writing index: done.\n") - - if state.opts.WriteRev { - writeProgress(state, "writing reverse index...\r") - } - err = writeRev(state) if err != nil { return Result{}, err } - if state.opts.WriteRev { - writeProgress(state, "writing reverse index: done.\n") - } - return finalizeArtifacts(state) } diff --git a/format/pack/ingest/rev_write.go b/format/pack/ingest/rev_write.go index 5a2f5375..9ed8dc6c 100644 --- a/format/pack/ingest/rev_write.go +++ b/format/pack/ingest/rev_write.go @@ -5,6 +5,7 @@ import ( "slices" "codeberg.org/lindenii/furgit/internal/intconv" + "codeberg.org/lindenii/furgit/internal/progress" ) const ( @@ -33,6 +34,7 @@ func writeRev(state *ingestState) error { } var scratch [8]byte + writeProgress(state, "writing reverse index header...\r") binary.BigEndian.PutUint32(scratch[:4], revMagic) err = writeAndHash(state.revFile, hashImpl, scratch[:4]) @@ -53,6 +55,15 @@ func writeRev(state *ingestState) error { if err != nil { return err } + writeProgress(state, "writing reverse index header: done.\n") + + entriesMeter := progress.New(progress.Options{ + Writer: state.opts.Progress, + Flush: state.opts.ProgressFlush, + Title: "writing reverse index entries", + Total: uint64(len(packOrder)), + }) + var entriesDone uint64 for _, recordIdx := range packOrder { recordPos, err := intconv.IntToUint32(recordToIdxPos[recordIdx]) @@ -66,7 +77,14 @@ func writeRev(state *ingestState) error { if err != nil { return err } + entriesDone++ + entriesMeter.Set(entriesDone, 0) } + if entriesDone > 0 { + entriesMeter.Stop("done") + } + + writeProgress(state, "writing reverse index trailer...\r") err = writeAndHash(state.revFile, hashImpl, state.packHash.Bytes()) if err != nil { @@ -80,7 +98,13 @@ func writeRev(state *ingestState) error { return err } - return state.revFile.Sync() + err = state.revFile.Sync() + if err != nil { + return err + } + writeProgress(state, "writing reverse index trailer: done.\n") + + return nil } // buildPackOrder returns record indexes sorted by pack offset. -- cgit v1.3.1-10-gc9f91