aboutsummaryrefslogtreecommitdiff
path: root/object/store/packed/internal
diff options
context:
space:
mode:
authorGravatar Runxi Yu2026-06-13 03:32:21 +0000
committerGravatar Runxi Yu2026-06-13 04:59:51 +0000
commitbe63ecd9711b46135bbff1769c2e4c3642255ef1 (patch)
treed5581c8b2e438af2b8ae82dd8f200393b2dbc5d3 /object/store/packed/internal
parentTODO: Update (diff)
Unify lengths
Diffstat (limited to 'object/store/packed/internal')
-rw-r--r--object/store/packed/internal/ingest/finalize.go7
-rw-r--r--object/store/packed/internal/ingest/ingest.go10
-rw-r--r--object/store/packed/internal/ingest/record.go12
-rw-r--r--object/store/packed/internal/ingest/resolve.go28
-rw-r--r--object/store/packed/internal/ingest/scan.go52
-rw-r--r--object/store/packed/internal/ingest/thin.go39
6 files changed, 62 insertions, 86 deletions
diff --git a/object/store/packed/internal/ingest/finalize.go b/object/store/packed/internal/ingest/finalize.go
index 7dca131a..f0ab6622 100644
--- a/object/store/packed/internal/ingest/finalize.go
+++ b/object/store/packed/internal/ingest/finalize.go
@@ -96,9 +96,14 @@ func (ingestion *ingestion) indexEntries() ([]packidx.Entry, []uint32, error) {
var oidBytes [id.MaxObjectIDSize]byte
copy(oidBytes[:], rec.oid.RawBytes())
+ offset, err := intconv.IntToUint64(rec.offset)
+ if err != nil {
+ return nil, nil, fmt.Errorf("object/store/packed/internal/ingest: %w", err)
+ }
+
entries[indexPosition] = packidx.Entry{
OID: oidBytes,
- Offset: rec.offset,
+ Offset: offset,
CRC32: rec.crc32,
}
diff --git a/object/store/packed/internal/ingest/ingest.go b/object/store/packed/internal/ingest/ingest.go
index 324ed8ce..5422b4af 100644
--- a/object/store/packed/internal/ingest/ingest.go
+++ b/object/store/packed/internal/ingest/ingest.go
@@ -46,17 +46,17 @@ type ingestion struct {
// byOffset maps an entry offset to its record index,
// and byOID maps a resolved object ID to its record index.
- byOffset map[uint64]int
+ byOffset map[int]int
byOID map[id.ObjectID]int
// headerCount is the object count declared by the pack header.
- headerCount uint32
+ headerCount int
// deltaCount counts delta records, accumulated during scanning.
- deltaCount uint64
+ deltaCount int
// deltasResolved counts resolved delta records, for progress.
- deltasResolved uint64
+ deltasResolved int
// packHash is the final pack trailer hash.
packHash id.ObjectID
@@ -101,7 +101,7 @@ func WritePack(root *os.Root, objectFormat id.ObjectFormat, src io.Reader, opts
temps: nil,
scanner: nil,
records: nil,
- byOffset: make(map[uint64]int),
+ byOffset: make(map[int]int),
byOID: make(map[id.ObjectID]int),
headerCount: count,
deltaCount: 0,
diff --git a/object/store/packed/internal/ingest/record.go b/object/store/packed/internal/ingest/record.go
index 460365fd..69101293 100644
--- a/object/store/packed/internal/ingest/record.go
+++ b/object/store/packed/internal/ingest/record.go
@@ -12,15 +12,15 @@ import (
// so a record's index in the slice is also its pack order.
type record struct {
// offset is the entry's start offset in the pack.
- offset uint64
+ offset int
// headerLen is the entry header length in bytes,
// so the zlib payload begins at offset+headerLen.
- headerLen uint64
+ headerLen int
// packedLen is the total on-disk entry length in bytes,
// covering the header and the compressed payload.
- packedLen uint64
+ packedLen int
// crc32 is the CRC32 of the entry's packed bytes.
crc32 uint32
@@ -29,10 +29,10 @@ type record struct {
packedType packfile.EntryType
// declaredSize is the declared inflated payload size.
- declaredSize uint64
+ declaredSize int
// baseOffset is the base entry offset for an ofs-delta.
- baseOffset uint64
+ baseOffset int
// baseOID is the base object ID for a ref-delta.
baseOID id.ObjectID
@@ -50,6 +50,6 @@ type record struct {
}
// dataOffset returns the entry's compressed payload start offset.
-func (record *record) dataOffset() uint64 {
+func (record *record) dataOffset() int {
return record.offset + record.headerLen
}
diff --git a/object/store/packed/internal/ingest/resolve.go b/object/store/packed/internal/ingest/resolve.go
index 7e163f2d..8595d366 100644
--- a/object/store/packed/internal/ingest/resolve.go
+++ b/object/store/packed/internal/ingest/resolve.go
@@ -10,13 +10,12 @@ import (
"lindenii.org/go/furgit/internal/progress"
"lindenii.org/go/furgit/object/header"
"lindenii.org/go/furgit/object/id"
- "lindenii.org/go/lgo/intconv"
)
// adjacency maps each resolvable base to its delta children:
// ofs-deltas keyed by base offset, ref-deltas keyed by base object ID.
type adjacency struct {
- byOffset map[uint64][]int
+ byOffset map[int][]int
byOID map[id.ObjectID][]int
}
@@ -60,7 +59,7 @@ func (ingestion *ingestion) resolveDeltas() error {
// so a resolved base can find the children that delta against it.
func (ingestion *ingestion) buildAdjacency() adjacency {
out := adjacency{
- byOffset: make(map[uint64][]int),
+ byOffset: make(map[int][]int),
byOID: make(map[id.ObjectID][]int),
}
@@ -192,20 +191,9 @@ func (ingestion *ingestion) resolveChild(
func (ingestion *ingestion) inflateRecord(index int) ([]byte, error) {
rec := &ingestion.records[index]
- offset, err := intconv.Uint64ToInt64(rec.dataOffset())
- if err != nil {
- return nil, fmt.Errorf("object/store/packed/internal/ingest: %w", err)
- }
-
- compressedLen, err := intconv.Uint64ToInt64(rec.packedLen - rec.headerLen)
- if err != nil {
- return nil, fmt.Errorf("object/store/packed/internal/ingest: %w", err)
- }
-
- size, err := intconv.Uint64ToInt(rec.declaredSize)
- if err != nil {
- return nil, fmt.Errorf("object/store/packed/internal/ingest: %w", err)
- }
+ offset := int64(rec.dataOffset())
+ compressedLen := int64(rec.packedLen - rec.headerLen)
+ size := rec.declaredSize
zr, err := zlib.NewReader(io.NewSectionReader(ingestion.packFile, offset, compressedLen))
if err != nil {
@@ -238,7 +226,7 @@ func (ingestion *ingestion) hashObject(objectType packfile.EntryType, content []
return zero, fmt.Errorf("object/store/packed/internal/ingest: %w", err)
}
- _, _ = hashImpl.Write(header.Append(nil, ty, uint64(len(content))))
+ _, _ = hashImpl.Write(header.Append(nil, ty, len(content)))
_, _ = hashImpl.Write(content)
oid, err := ingestion.objectFormat.FromBytes(hashImpl.Sum(nil))
@@ -263,7 +251,7 @@ func (ingestion *ingestion) resolvedRoots() []int {
}
// countDeltas returns the number of delta records.
-func (ingestion *ingestion) countDeltas() uint64 {
+func (ingestion *ingestion) countDeltas() int {
return ingestion.deltaCount
}
@@ -272,7 +260,7 @@ func (ingestion *ingestion) countDeltas() uint64 {
// Every base is resolved during scanning or thin completion,
// so the unresolved records are exactly the unresolved deltas:
// the delta records minus those already resolved.
-func (ingestion *ingestion) countUnresolved() uint64 {
+func (ingestion *ingestion) countUnresolved() int {
return ingestion.deltaCount - ingestion.deltasResolved
}
diff --git a/object/store/packed/internal/ingest/scan.go b/object/store/packed/internal/ingest/scan.go
index 56e42cea..6b3b73b7 100644
--- a/object/store/packed/internal/ingest/scan.go
+++ b/object/store/packed/internal/ingest/scan.go
@@ -36,7 +36,7 @@ type scanner struct {
n int
// consumed counts stream bytes consumed so far.
- consumed uint64
+ consumed int
// hash accumulates the pack hash over consumed bytes
// while hashing is true.
@@ -66,7 +66,7 @@ func newScanner(src io.Reader, dst io.Writer, packHash hash.Hash) *scanner {
// readPackHeader reads and validates the pack header from src,
// returning the raw header and its declared object count.
-func readPackHeader(src io.Reader) ([packfile.HeaderLen]byte, uint32, error) {
+func readPackHeader(src io.Reader) ([packfile.HeaderLen]byte, int, error) {
var raw [packfile.HeaderLen]byte
_, err := io.ReadFull(src, raw[:])
@@ -79,7 +79,12 @@ func readPackHeader(src io.Reader) ([packfile.HeaderLen]byte, uint32, error) {
return raw, 0, fmt.Errorf("%w: %w", ErrMalformedPack, err)
}
- return raw, packHeader.ObjectCount, nil
+ count, err := intconv.Uint32ToInt(packHeader.ObjectCount)
+ if err != nil {
+ return raw, 0, fmt.Errorf("%w: object count: %w", ErrMalformedPack, err)
+ }
+
+ return raw, count, nil
}
// Read implements [io.Reader].
@@ -205,7 +210,7 @@ func (scanner *scanner) use(n int) error {
}
scanner.off += n
- scanner.consumed += uint64(n) //nolint:gosec
+ scanner.consumed += n
return nil
}
@@ -283,7 +288,7 @@ func (ingestion *ingestion) streamAndScan() error {
meter := progress.New(progress.Options{
Writer: ingestion.opts.Progress,
Title: "receiving objects",
- Total: uint64(ingestion.headerCount),
+ Total: ingestion.headerCount,
Delay: 0,
Sparse: false,
Throughput: true,
@@ -295,7 +300,7 @@ func (ingestion *ingestion) streamAndScan() error {
return err
}
- meter.Set(uint64(done)+1, ingestion.scanner.consumed)
+ meter.Set(done+1, ingestion.scanner.consumed)
}
meter.Stop("done")
@@ -316,7 +321,7 @@ func (ingestion *ingestion) streamAndScan() error {
}
// scanEntry scans the entry beginning at start into one record.
-func (ingestion *ingestion) scanEntry(start uint64) error {
+func (ingestion *ingestion) scanEntry(start int) error {
ingestion.scanner.beginCRC()
rec, err := ingestion.scanHeader(start)
@@ -329,7 +334,7 @@ func (ingestion *ingestion) scanEntry(start uint64) error {
return err
}
- if inflated != rec.declaredSize {
+ if inflated != int64(rec.declaredSize) {
return fmt.Errorf(
"%w: entry at %d: inflated size %d differs from declared %d",
ErrMalformedPack, start, inflated, rec.declaredSize,
@@ -359,7 +364,7 @@ func (ingestion *ingestion) scanEntry(start uint64) error {
}
// scanHeader parses and consumes the entry header at start.
-func (ingestion *ingestion) scanHeader(start uint64) (record, error) {
+func (ingestion *ingestion) scanHeader(start int) (record, error) {
var rec record
rec.offset = start
@@ -374,23 +379,24 @@ func (ingestion *ingestion) scanHeader(start uint64) (record, error) {
return rec, fmt.Errorf("%w: entry at %d: %w", ErrMalformedPack, start, err)
}
- headerLen, err := intconv.IntToUint64(entryHeader.HeaderLen)
+ declaredSize, err := intconv.Uint64ToInt(entryHeader.Size)
if err != nil {
- return rec, fmt.Errorf("object/store/packed/internal/ingest: %w", err)
+ return rec, fmt.Errorf("%w: entry at %d: declared size overflows int: %w", ErrMalformedPack, start, err)
}
rec.packedType = entryHeader.Type
- rec.declaredSize = entryHeader.Size
- rec.headerLen = headerLen
+ rec.declaredSize = declaredSize
+ rec.headerLen = entryHeader.HeaderLen
switch entryHeader.Type {
case packfile.EntryTypeCommit, packfile.EntryTypeTree, packfile.EntryTypeBlob, packfile.EntryTypeTag:
case packfile.EntryTypeOfsDelta:
- if entryHeader.OfsDistance == 0 || entryHeader.OfsDistance > start {
+ dist, err := intconv.Uint64ToInt(entryHeader.OfsDistance)
+ if err != nil || dist == 0 || dist > start {
return rec, fmt.Errorf("%w: entry at %d: ofs-delta base out of bounds", ErrMalformedPack, start)
}
- rec.baseOffset = start - entryHeader.OfsDistance
+ rec.baseOffset = start - dist
case packfile.EntryTypeRefDelta:
baseID, err := ingestion.objectFormat.FromBytes(entryHeader.RefBase[:ingestion.objectFormat.Size()])
if err != nil {
@@ -412,7 +418,7 @@ func (ingestion *ingestion) scanHeader(start uint64) (record, error) {
// drainPayload consumes one entry's compressed payload from the stream,
// returning its inflated length and, for base entries, its object ID.
-func (ingestion *ingestion) drainPayload(rec *record) (uint64, id.ObjectID, error) {
+func (ingestion *ingestion) drainPayload(rec *record) (int64, id.ObjectID, error) {
var zero id.ObjectID
zr, err := zlib.NewReader(ingestion.scanner)
@@ -428,12 +434,7 @@ func (ingestion *ingestion) drainPayload(rec *record) (uint64, id.ObjectID, erro
return 0, zero, fmt.Errorf("%w: entry at %d: %w", ErrMalformedPack, rec.offset, err)
}
- inflated, err := intconv.Int64ToUint64(read)
- if err != nil {
- return 0, zero, fmt.Errorf("object/store/packed/internal/ingest: %w", err)
- }
-
- return inflated, zero, nil
+ return read, zero, nil
}
objectType, err := rec.packedType.ObjectType()
@@ -458,10 +459,5 @@ func (ingestion *ingestion) drainPayload(rec *record) (uint64, id.ObjectID, erro
return 0, zero, fmt.Errorf("object/store/packed/internal/ingest: %w", err)
}
- inflated, err := intconv.Int64ToUint64(read)
- if err != nil {
- return 0, zero, fmt.Errorf("object/store/packed/internal/ingest: %w", err)
- }
-
- return inflated, oid, nil
+ return read, oid, nil
}
diff --git a/object/store/packed/internal/ingest/thin.go b/object/store/packed/internal/ingest/thin.go
index e96846cb..8d1566e0 100644
--- a/object/store/packed/internal/ingest/thin.go
+++ b/object/store/packed/internal/ingest/thin.go
@@ -25,8 +25,8 @@ func (ingestion *ingestion) fixThin(external []id.ObjectID, adjacency adjacency,
return ErrThinPackNotPermitted
}
- hashSize := uint64(ingestion.objectFormat.Size()) //nolint:gosec
- if ingestion.scanner.consumed < uint64(packfile.HeaderLen)+hashSize {
+ hashSize := ingestion.objectFormat.Size()
+ if ingestion.scanner.consumed < packfile.HeaderLen+hashSize {
return fmt.Errorf("%w: pack shorter than trailer", ErrMalformedPack)
}
@@ -96,11 +96,7 @@ func (ingestion *ingestion) appendBaseObject(objectID id.ObjectID, objectType ty
}
start := ingestion.scanner.consumed
-
- startOffset, err := intconv.Uint64ToInt64(start)
- if err != nil {
- return 0, fmt.Errorf("object/store/packed/internal/ingest: %w", err)
- }
+ startOffset := int64(start)
headerBytes := packfile.AppendTypeSize(nil, entryType, uint64(len(content)))
@@ -129,17 +125,8 @@ func (ingestion *ingestion) appendBaseObject(objectID id.ObjectID, objectType ty
return 0, fmt.Errorf("object/store/packed/internal/ingest: compressing thin base: %w", err)
}
- compressedLen, err := intconv.Int64ToUint64(writer.offset - dataOffset)
- if err != nil {
- return 0, fmt.Errorf("object/store/packed/internal/ingest: %w", err)
- }
-
- headerLen, err := intconv.IntToUint64(len(headerBytes))
- if err != nil {
- return 0, fmt.Errorf("object/store/packed/internal/ingest: %w", err)
- }
-
- packedLen := headerLen + compressedLen
+ headerLen := len(headerBytes)
+ packedLen := headerLen + writer.written
ingestion.scanner.consumed = start + packedLen
rec := record{
@@ -148,7 +135,7 @@ func (ingestion *ingestion) appendBaseObject(objectID id.ObjectID, objectType ty
packedLen: packedLen,
crc32: crc.Sum32(),
packedType: entryType,
- declaredSize: uint64(len(content)),
+ declaredSize: len(content),
baseOffset: 0,
baseOID: id.ObjectID{},
objectType: entryType,
@@ -178,10 +165,7 @@ func (ingestion *ingestion) rewriteHeaderTrailer() error {
return fmt.Errorf("object/store/packed/internal/ingest: rewriting header: %w", err)
}
- bodyEnd, err := intconv.Uint64ToInt64(ingestion.scanner.consumed)
- if err != nil {
- return fmt.Errorf("object/store/packed/internal/ingest: %w", err)
- }
+ bodyEnd := int64(ingestion.scanner.consumed)
hashImpl, err := ingestion.objectFormat.New()
if err != nil {
@@ -211,16 +195,19 @@ func (ingestion *ingestion) rewriteHeaderTrailer() error {
}
// offsetWriter writes to a file via WriteAt,
-// advancing sequentially from a base offset.
+// advancing sequentially from a base offset
+// and counting the bytes written.
type offsetWriter struct {
- file *os.File
- offset int64
+ file *os.File
+ offset int64
+ written int
}
// Write implements [io.Writer].
func (writer *offsetWriter) Write(p []byte) (int, error) {
n, err := writer.file.WriteAt(p, writer.offset)
writer.offset += int64(n)
+ writer.written += n
return n, err //nolint:wrapcheck
}