From be63ecd9711b46135bbff1769c2e4c3642255ef1 Mon Sep 17 00:00:00 2001 From: Runxi Yu Date: Sat, 13 Jun 2026 03:32:21 +0000 Subject: Unify lengths --- object/blob/append.go | 2 +- object/commit/append.go | 2 +- object/fetch/blob.go | 4 +- object/fetch/header.go | 4 +- object/fetch/reader.go | 2 +- object/fetch/treefs.go | 8 +--- object/header/append.go | 4 +- object/header/parse.go | 10 ++++- object/parse.go | 2 +- object/store/chain/header.go | 4 +- object/store/chain/reader.go | 2 +- object/store/dual/quarantine.go | 8 ++-- object/store/dual/reader.go | 6 +-- object/store/dual/writer.go | 2 +- object/store/loose/helpers_test.go | 2 +- object/store/loose/parse.go | 4 +- object/store/loose/read_test.go | 6 +-- object/store/loose/reader.go | 6 +-- object/store/loose/roundtrip_test.go | 6 +-- object/store/loose/streamwriter.go | 26 +++---------- object/store/loose/write_test.go | 6 +-- object/store/loose/writer.go | 4 +- object/store/memory/reader.go | 12 +++--- object/store/memory/writer.go | 18 +++------ object/store/memory/writer_test.go | 8 ++-- object/store/mix/header.go | 4 +- object/store/mix/reader.go | 2 +- object/store/packed/basecache.go | 2 +- object/store/packed/delta.go | 30 +++++++++----- object/store/packed/entry.go | 6 +-- object/store/packed/internal/ingest/finalize.go | 7 +++- object/store/packed/internal/ingest/ingest.go | 10 ++--- object/store/packed/internal/ingest/record.go | 12 +++--- object/store/packed/internal/ingest/resolve.go | 28 ++++--------- object/store/packed/internal/ingest/scan.go | 52 ++++++++++++------------- object/store/packed/internal/ingest/thin.go | 39 +++++++------------ object/store/packed/lookup.go | 10 ++++- object/store/packed/read_test.go | 6 +-- object/store/packed/reader.go | 34 +++++++++++----- object/store/reader.go | 6 +-- object/store/writer.go | 2 +- object/tag/append.go | 2 +- object/tree/append.go | 2 +- 43 files changed, 201 insertions(+), 211 deletions(-) (limited to 'object') diff --git a/object/blob/append.go b/object/blob/append.go index 2376d65f..8106f1c5 100644 --- a/object/blob/append.go +++ b/object/blob/append.go @@ -12,7 +12,7 @@ func (blob *Blob) AppendWithoutHeader(dst []byte) ([]byte, error) { // AppendWithHeader renders the raw object (header + body). func (blob *Blob) AppendWithHeader(dst []byte) ([]byte, error) { - dst = header.Append(dst, typ.Blob, uint64(len(blob.Data))) + dst = header.Append(dst, typ.Blob, len(blob.Data)) return blob.AppendWithoutHeader(dst) } diff --git a/object/commit/append.go b/object/commit/append.go index b637620e..d5258b97 100644 --- a/object/commit/append.go +++ b/object/commit/append.go @@ -61,7 +61,7 @@ func (commit *Commit) AppendWithHeader(dst []byte) ([]byte, error) { return dst, err } - dst = header.Append(dst, typ.Commit, uint64(len(body))) + dst = header.Append(dst, typ.Commit, len(body)) return append(dst, body...), nil } diff --git a/object/fetch/blob.go b/object/fetch/blob.go index 9af34922..d40ec875 100644 --- a/object/fetch/blob.go +++ b/object/fetch/blob.go @@ -32,7 +32,7 @@ func (fetcher *Fetcher) ExactBlob(id oid.ObjectID) (*stored.Stored[*blob.Blob], // together with its content size in bytes. // // Labels: Life-Parent, Close-Caller. -func (fetcher *Fetcher) ExactBlobReader(id oid.ObjectID) (io.ReadCloser, uint64, error) { +func (fetcher *Fetcher) ExactBlobReader(id oid.ObjectID) (io.ReadCloser, int, error) { return fetcher.exactReader(id, typ.Blob) } @@ -87,7 +87,7 @@ func (fetcher *Fetcher) PeelToBlobID(id oid.ObjectID) (oid.ObjectID, error) { // together with its content size in bytes. // // Labels: Life-Parent, Close-Caller. -func (fetcher *Fetcher) PeelToBlobReader(id oid.ObjectID) (io.ReadCloser, uint64, error) { +func (fetcher *Fetcher) PeelToBlobReader(id oid.ObjectID) (io.ReadCloser, int, error) { blobID, err := fetcher.PeelToBlobID(id) if err != nil { return nil, 0, err diff --git a/object/fetch/header.go b/object/fetch/header.go index 7a8df483..ee02ef69 100644 --- a/object/fetch/header.go +++ b/object/fetch/header.go @@ -8,7 +8,7 @@ import ( // Header returns the object type and content size at id. // // Labels: Life-Parent. -func (fetcher *Fetcher) Header(id oid.ObjectID) (typ.Type, uint64, error) { +func (fetcher *Fetcher) Header(id oid.ObjectID) (typ.Type, int, error) { ty, size, err := fetcher.store.ReadHeader(id) if err != nil { return typ.Unknown, 0, wrapObjectReadError(id, err) @@ -20,7 +20,7 @@ func (fetcher *Fetcher) Header(id oid.ObjectID) (typ.Type, uint64, error) { // Size returns the object content size at id. // // Labels: Life-Parent. -func (fetcher *Fetcher) Size(id oid.ObjectID) (uint64, error) { +func (fetcher *Fetcher) Size(id oid.ObjectID) (int, error) { size, err := fetcher.store.ReadSize(id) if err != nil { return 0, wrapObjectReadError(id, err) diff --git a/object/fetch/reader.go b/object/fetch/reader.go index 8baf1119..b1b4f7c2 100644 --- a/object/fetch/reader.go +++ b/object/fetch/reader.go @@ -10,7 +10,7 @@ import ( // exactReader reads one object's content stream // and verifies that its header type matches wantType. -func (fetcher *Fetcher) exactReader(id oid.ObjectID, wantType typ.Type) (io.ReadCloser, uint64, error) { +func (fetcher *Fetcher) exactReader(id oid.ObjectID, wantType typ.Type) (io.ReadCloser, int, error) { gotType, size, rc, err := fetcher.store.ReadReaderContent(id) if err != nil { return nil, 0, wrapObjectReadError(id, err) diff --git a/object/fetch/treefs.go b/object/fetch/treefs.go index da92af51..9d88abb2 100644 --- a/object/fetch/treefs.go +++ b/object/fetch/treefs.go @@ -11,7 +11,6 @@ import ( oid "lindenii.org/go/furgit/object/id" "lindenii.org/go/furgit/object/tree" "lindenii.org/go/furgit/object/tree/mode" - "lindenii.org/go/lgo/intconv" ) // TreeFS exposes one Git tree as an fs.FS view backed by a Fetcher. @@ -69,7 +68,7 @@ func (entry treeEntryValue) isDir() bool { return entry.mode == mode.Directory } -func (entry treeEntryValue) blobSize(fetcher *Fetcher) (uint64, error) { +func (entry treeEntryValue) blobSize(fetcher *Fetcher) (int, error) { return fetcher.Size(entry.objectID) } @@ -434,10 +433,7 @@ func (treeFS *TreeFS) statEntry(entry treeEntryValue) (*treeFSInfo, error) { return nil, err } - size, err = intconv.Uint64ToInt64(sz) - if err != nil { - return nil, fmt.Errorf("object/fetch: blob size overflows int64: %w", err) - } + size = int64(sz) } var sys any diff --git a/object/header/append.go b/object/header/append.go index b204002d..b8d6669d 100644 --- a/object/header/append.go +++ b/object/header/append.go @@ -8,13 +8,13 @@ import ( ) // Append appends a canonical loose-object header ("type size\x00") to dst. -func Append(dst []byte, ty typ.Type, size uint64) []byte { +func Append(dst []byte, ty typ.Type, size int) []byte { tyName := ty.Name() dst = slices.Grow(dst, len(tyName)+1+19+1) dst = append(dst, tyName...) dst = append(dst, ' ') - dst = strconv.AppendUint(dst, size, 10) + dst = strconv.AppendInt(dst, int64(size), 10) dst = append(dst, 0) return dst diff --git a/object/header/parse.go b/object/header/parse.go index 5829a755..2b0c78e1 100644 --- a/object/header/parse.go +++ b/object/header/parse.go @@ -7,13 +7,14 @@ import ( "strconv" "lindenii.org/go/furgit/object/typ" + "lindenii.org/go/lgo/intconv" ) // ErrInvalidHeader indicates a malformed loose-object header. var ErrInvalidHeader = errors.New("object/header: invalid header") // Parse parses a canonical loose-object header ("type size\x00"). -func Parse(data []byte) (ty typ.Type, size uint64, consumed int, err error) { +func Parse(data []byte) (ty typ.Type, size int, consumed int, err error) { space := bytes.IndexByte(data, ' ') if space <= 0 { return 0, 0, 0, fmt.Errorf("%w: missing ' ' type/size separator", ErrInvalidHeader) @@ -36,7 +37,12 @@ func Parse(data []byte) (ty typ.Type, size uint64, consumed int, err error) { return 0, 0, 0, fmt.Errorf("%w: empty size field", ErrInvalidHeader) } - size, err = strconv.ParseUint(string(sizeBytes), 10, 64) + sizeU, err := strconv.ParseUint(string(sizeBytes), 10, 64) + if err != nil { + return 0, 0, 0, fmt.Errorf("%w: size %q: %w", ErrInvalidHeader, sizeBytes, err) + } + + size, err = intconv.Uint64ToInt(sizeU) if err != nil { return 0, 0, 0, fmt.Errorf("%w: size %q: %w", ErrInvalidHeader, sizeBytes, err) } diff --git a/object/parse.go b/object/parse.go index afcdfe28..f9779171 100644 --- a/object/parse.go +++ b/object/parse.go @@ -29,7 +29,7 @@ func ParseWithHeader(raw []byte, objectFormat id.ObjectFormat) (Object, error) { } body := raw[headerLen:] - if uint64(len(body)) != size { + if len(body) != size { return nil, fmt.Errorf("%w: header declares %d bytes, body has %d", ErrSizeMismatch, size, len(body)) } diff --git a/object/store/chain/header.go b/object/store/chain/header.go index c12fc27f..3a5ad815 100644 --- a/object/store/chain/header.go +++ b/object/store/chain/header.go @@ -11,7 +11,7 @@ import ( // ReadHeader reads object header data // from the first backend that has it. -func (chain *Chain) ReadHeader(id id.ObjectID) (typ.Type, uint64, error) { +func (chain *Chain) ReadHeader(id id.ObjectID) (typ.Type, int, error) { for _, backend := range chain.backends { ty, size, err := backend.ReadHeader(id) if err == nil { @@ -30,7 +30,7 @@ func (chain *Chain) ReadHeader(id id.ObjectID) (typ.Type, uint64, error) { // ReadSize reads object content length // from the first backend that has it. -func (chain *Chain) ReadSize(id id.ObjectID) (uint64, error) { +func (chain *Chain) ReadSize(id id.ObjectID) (int, error) { for _, backend := range chain.backends { size, err := backend.ReadSize(id) if err == nil { diff --git a/object/store/chain/reader.go b/object/store/chain/reader.go index 744838dd..e7f07c33 100644 --- a/object/store/chain/reader.go +++ b/object/store/chain/reader.go @@ -31,7 +31,7 @@ func (chain *Chain) ReadReaderFull(id id.ObjectID) (io.ReadCloser, error) { // ReadReaderContent reads an object's type, declared content length, // and content stream from the first backend that has it. -func (chain *Chain) ReadReaderContent(id id.ObjectID) (typ.Type, uint64, io.ReadCloser, error) { +func (chain *Chain) ReadReaderContent(id id.ObjectID) (typ.Type, int, io.ReadCloser, error) { for _, backend := range chain.backends { ty, size, reader, err := backend.ReadReaderContent(id) if err == nil { diff --git a/object/store/dual/quarantine.go b/object/store/dual/quarantine.go index eb1fca21..b73e48fe 100644 --- a/object/store/dual/quarantine.go +++ b/object/store/dual/quarantine.go @@ -79,15 +79,15 @@ func (quarantine *coordinatedQuarantine) ReadReaderFull(id id.ObjectID) (io.Read return quarantine.reader.ReadReaderFull(id) //nolint:wrapcheck } -func (quarantine *coordinatedQuarantine) ReadReaderContent(id id.ObjectID) (typ.Type, uint64, io.ReadCloser, error) { +func (quarantine *coordinatedQuarantine) ReadReaderContent(id id.ObjectID) (typ.Type, int, io.ReadCloser, error) { return quarantine.reader.ReadReaderContent(id) //nolint:wrapcheck } -func (quarantine *coordinatedQuarantine) ReadSize(id id.ObjectID) (uint64, error) { +func (quarantine *coordinatedQuarantine) ReadSize(id id.ObjectID) (int, error) { return quarantine.reader.ReadSize(id) //nolint:wrapcheck } -func (quarantine *coordinatedQuarantine) ReadHeader(id id.ObjectID) (typ.Type, uint64, error) { +func (quarantine *coordinatedQuarantine) ReadHeader(id id.ObjectID) (typ.Type, int, error) { return quarantine.reader.ReadHeader(id) //nolint:wrapcheck } @@ -107,7 +107,7 @@ func (quarantine *coordinatedQuarantine) WriteReaderFull(src io.Reader) (id.Obje return quarantine.objectQ.WriteReaderFull(src) //nolint:wrapcheck } -func (quarantine *coordinatedQuarantine) WriteReaderContent(ty typ.Type, size uint64, src io.Reader) (id.ObjectID, error) { +func (quarantine *coordinatedQuarantine) WriteReaderContent(ty typ.Type, size int, src io.Reader) (id.ObjectID, error) { return quarantine.objectQ.WriteReaderContent(ty, size, src) //nolint:wrapcheck } diff --git a/object/store/dual/reader.go b/object/store/dual/reader.go index a51cfbd0..7e5c8d6b 100644 --- a/object/store/dual/reader.go +++ b/object/store/dual/reader.go @@ -24,17 +24,17 @@ func (dual *Dual) ReadReaderFull(id id.ObjectID) (io.ReadCloser, error) { // ReadReaderContent reads an object's type, declared content length, // and content stream from the combined view. -func (dual *Dual) ReadReaderContent(id id.ObjectID) (typ.Type, uint64, io.ReadCloser, error) { +func (dual *Dual) ReadReaderContent(id id.ObjectID) (typ.Type, int, io.ReadCloser, error) { return dual.reader.ReadReaderContent(id) //nolint:wrapcheck } // ReadSize reads an object's declared content length from the combined view. -func (dual *Dual) ReadSize(id id.ObjectID) (uint64, error) { +func (dual *Dual) ReadSize(id id.ObjectID) (int, error) { return dual.reader.ReadSize(id) //nolint:wrapcheck } // ReadHeader reads an object's type and declared content length from the combined view. -func (dual *Dual) ReadHeader(id id.ObjectID) (typ.Type, uint64, error) { +func (dual *Dual) ReadHeader(id id.ObjectID) (typ.Type, int, error) { return dual.reader.ReadHeader(id) //nolint:wrapcheck } diff --git a/object/store/dual/writer.go b/object/store/dual/writer.go index 5961e7c7..f75f49e1 100644 --- a/object/store/dual/writer.go +++ b/object/store/dual/writer.go @@ -24,7 +24,7 @@ func (dual *Dual) WriteReaderFull(src io.Reader) (id.ObjectID, error) { } // WriteReaderContent writes one typed object content stream to the object side. -func (dual *Dual) WriteReaderContent(ty typ.Type, size uint64, src io.Reader) (id.ObjectID, error) { +func (dual *Dual) WriteReaderContent(ty typ.Type, size int, src io.Reader) (id.ObjectID, error) { return dual.object.WriteReaderContent(ty, size, src) //nolint:wrapcheck } diff --git a/object/store/loose/helpers_test.go b/object/store/loose/helpers_test.go index db2c5bd2..d1e9a50d 100644 --- a/object/store/loose/helpers_test.go +++ b/object/store/loose/helpers_test.go @@ -77,7 +77,7 @@ func gitOracleObjects(t *testing.T, repo *testgit.Repo) []gitOracleObject { t.Fatalf("CatFile(%s %s): %v", group.name, oid, err) } - raw := header.Append(nil, group.ty, uint64(len(body))) + raw := header.Append(nil, group.ty, len(body)) raw = append(raw, body...) objects = append(objects, gitOracleObject{ diff --git a/object/store/loose/parse.go b/object/store/loose/parse.go index abfd527b..c3af6159 100644 --- a/object/store/loose/parse.go +++ b/object/store/loose/parse.go @@ -37,7 +37,7 @@ func parseRaw(raw []byte) (typ.Type, []byte, error) { } content := raw[consumed:] - if uint64(len(content)) != size { + if len(content) != size { return typ.Unknown, nil, fmt.Errorf("%w: header size/content mismatch", store.ErrInvalidObject) } @@ -46,7 +46,7 @@ func parseRaw(raw []byte) (typ.Type, []byte, error) { // readHeader reads and parses a loose object header from br, // and returns the raw header bytes including the trailing NUL. -func readHeader(br *bufio.Reader) ([]byte, typ.Type, uint64, error) { +func readHeader(br *bufio.Reader) ([]byte, typ.Type, int, error) { headerBytes, err := br.ReadSlice(0) if err != nil { return nil, typ.Unknown, 0, fmt.Errorf("object/store/loose: %w", err) diff --git a/object/store/loose/read_test.go b/object/store/loose/read_test.go index a0cea970..d8fdad1a 100644 --- a/object/store/loose/read_test.go +++ b/object/store/loose/read_test.go @@ -75,7 +75,7 @@ func TestRead(t *testing.T) { t.Fatalf("%s: ReadHeader type = %v, want %v", o.name, gotType, o.ty) } - if gotSize != uint64(len(o.body)) { + if gotSize != len(o.body) { t.Fatalf("%s: ReadHeader size = %d, want %d", o.name, gotSize, len(o.body)) } } @@ -133,7 +133,7 @@ func TestRead(t *testing.T) { t.Fatalf("%s: ReadReaderContent type = %v, want %v", o.name, gotType, o.ty) } - if gotSize != uint64(len(o.body)) { + if gotSize != len(o.body) { t.Fatalf("%s: ReadReaderContent size = %d, want %d", o.name, gotSize, len(o.body)) } @@ -250,7 +250,7 @@ func TestReadCorruptTrailer(t *testing.T) { t.Fatalf("ReadHeader type = %v, want %v", ty, typ.Blob) } - if size != uint64(len(content)) { + if size != len(content) { t.Fatalf("ReadHeader size = %d, want %d", size, len(content)) } diff --git a/object/store/loose/reader.go b/object/store/loose/reader.go index 25dc29e3..2f26efe5 100644 --- a/object/store/loose/reader.go +++ b/object/store/loose/reader.go @@ -48,7 +48,7 @@ func (loose *Loose) ReadBytesContent(objectID id.ObjectID) (typ.Type, []byte, er // It parses only enough of the zlib-decoded object to recover the object header. // It does not verify that the remaining object content is readable // and does not verify the zlib Adler-32 trailer. -func (loose *Loose) ReadHeader(objectID id.ObjectID) (typ.Type, uint64, error) { +func (loose *Loose) ReadHeader(objectID id.ObjectID) (typ.Type, int, error) { file, err := loose.openObject(objectID) if err != nil { return typ.Unknown, 0, err @@ -76,7 +76,7 @@ func (loose *Loose) ReadHeader(objectID id.ObjectID) (typ.Type, uint64, error) { // Like ReadHeader, // it parses only enough of the zlib-decoded object to recover the header // and does not verify the zlib Adler-32 trailer. -func (loose *Loose) ReadSize(objectID id.ObjectID) (uint64, error) { +func (loose *Loose) ReadSize(objectID id.ObjectID) (int, error) { _, size, err := loose.ReadHeader(objectID) return size, err @@ -127,7 +127,7 @@ func (loose *Loose) ReadReaderFull(objectID id.ObjectID) (io.ReadCloser, error) // trailing bytes past the declared object size, // and the zlib Adler-32 trailer // may go unverified unless the caller reads to io.EOF. -func (loose *Loose) ReadReaderContent(objectID id.ObjectID) (typ.Type, uint64, io.ReadCloser, error) { +func (loose *Loose) ReadReaderContent(objectID id.ObjectID) (typ.Type, int, io.ReadCloser, error) { file, zr, err := loose.openInflated(objectID) if err != nil { return typ.Unknown, 0, nil, err diff --git a/object/store/loose/roundtrip_test.go b/object/store/loose/roundtrip_test.go index 1f65667c..cc989b5b 100644 --- a/object/store/loose/roundtrip_test.go +++ b/object/store/loose/roundtrip_test.go @@ -39,7 +39,7 @@ func TestRoundTrip(t *testing.T) { for _, tc := range cases { t.Run(tc.name, func(t *testing.T) { - wantRaw := header.Append(nil, tc.ty, uint64(len(tc.content))) + wantRaw := header.Append(nil, tc.ty, len(tc.content)) wantRaw = append(wantRaw, tc.content...) objectID, err := looseStore.WriteBytesContent(tc.ty, tc.content) @@ -78,7 +78,7 @@ func TestRoundTrip(t *testing.T) { t.Fatalf("ReadHeader type = %v, want %v", headType, tc.ty) } - if headSize != uint64(len(tc.content)) { + if headSize != len(tc.content) { t.Fatalf("ReadHeader size = %d, want %d", headSize, len(tc.content)) } @@ -124,7 +124,7 @@ func TestRoundTrip(t *testing.T) { t.Fatalf("ReadReaderContent type = %v, want %v", contentType, tc.ty) } - if contentSize != uint64(len(tc.content)) { + if contentSize != len(tc.content) { t.Fatalf("ReadReaderContent size = %d, want %d", contentSize, len(tc.content)) } diff --git a/object/store/loose/streamwriter.go b/object/store/loose/streamwriter.go index c32d66b6..bdf78155 100644 --- a/object/store/loose/streamwriter.go +++ b/object/store/loose/streamwriter.go @@ -14,7 +14,6 @@ import ( "lindenii.org/go/furgit/object/header" "lindenii.org/go/furgit/object/id" "lindenii.org/go/furgit/object/store" - "lindenii.org/go/lgo/intconv" ) const tempObjectFilePrefix = "tmp_obj_" @@ -43,7 +42,7 @@ type streamWriter struct { // headerDone reports whether the full-object header has been parsed. headerDone bool // expectedContentLeft tracks remaining declared content bytes. - expectedContentLeft uint64 + expectedContentLeft int closed bool finalized bool @@ -66,12 +65,7 @@ func (writer *streamWriter) Write(src []byte) (int, error) { return 0, err } } else { - n, err := intconv.IntToUint64(len(src)) - if err != nil { - return 0, fmt.Errorf("object/store/loose: %w", err) - } - - err = writer.acceptContent(n) + err := writer.acceptContent(len(src)) if err != nil { return 0, err } @@ -100,12 +94,7 @@ func (writer *streamWriter) Close() error { // acceptFull validates and accounts raw full-object input. func (writer *streamWriter) acceptFull(src []byte) error { if writer.headerDone { - n, err := intconv.IntToUint64(len(src)) - if err != nil { - return fmt.Errorf("object/store/loose: %w", err) - } - - return writer.acceptContent(n) + return writer.acceptContent(len(src)) } nul := bytes.IndexByte(src, 0) @@ -126,16 +115,11 @@ func (writer *streamWriter) acceptFull(src []byte) error { writer.headerDone = true writer.expectedContentLeft = size - rest, err := intconv.IntToUint64(len(src) - headerChunkLen) - if err != nil { - return fmt.Errorf("object/store/loose: %w", err) - } - - return writer.acceptContent(rest) + return writer.acceptContent(len(src) - headerChunkLen) } // acceptContent validates and accounts content byte counts. -func (writer *streamWriter) acceptContent(n uint64) error { +func (writer *streamWriter) acceptContent(n int) error { if n > writer.expectedContentLeft { return fmt.Errorf("%w: object content exceeds declared size", store.ErrInvalidObject) } diff --git a/object/store/loose/write_test.go b/object/store/loose/write_test.go index 4e9eb9d0..e3ea2d6c 100644 --- a/object/store/loose/write_test.go +++ b/object/store/loose/write_test.go @@ -29,13 +29,13 @@ func TestWrite(t *testing.T) { { name: "ReaderContent", write: func(looseStore *loose.Loose, content []byte) (id.ObjectID, error) { - return looseStore.WriteReaderContent(typ.Blob, uint64(len(content)), bytes.NewReader(content)) + return looseStore.WriteReaderContent(typ.Blob, len(content), bytes.NewReader(content)) }, }, { name: "BytesFull", write: func(looseStore *loose.Loose, content []byte) (id.ObjectID, error) { - raw := header.Append(nil, typ.Blob, uint64(len(content))) + raw := header.Append(nil, typ.Blob, len(content)) raw = append(raw, content...) return looseStore.WriteBytesFull(raw) @@ -44,7 +44,7 @@ func TestWrite(t *testing.T) { { name: "ReaderFull", write: func(looseStore *loose.Loose, content []byte) (id.ObjectID, error) { - raw := header.Append(nil, typ.Blob, uint64(len(content))) + raw := header.Append(nil, typ.Blob, len(content)) raw = append(raw, content...) return looseStore.WriteReaderFull(bytes.NewReader(raw)) diff --git a/object/store/loose/writer.go b/object/store/loose/writer.go index b02fe2d9..1133e8c8 100644 --- a/object/store/loose/writer.go +++ b/object/store/loose/writer.go @@ -17,14 +17,14 @@ func (loose *Loose) WriteBytesFull(raw []byte) (id.ObjectID, error) { // WriteBytesContent writes typed content bytes as a loose object. func (loose *Loose) WriteBytesContent(ty typ.Type, content []byte) (id.ObjectID, error) { - return loose.WriteReaderContent(ty, uint64(len(content)), bytes.NewReader(content)) + return loose.WriteReaderContent(ty, len(content), bytes.NewReader(content)) } // WriteReaderContent writes one loose object from typed content bytes read from src. // src must provide exactly size bytes. // size is required because loose object headers are "type size\x00content", // so the header must be emitted before streaming content without buffering. -func (loose *Loose) WriteReaderContent(ty typ.Type, size uint64, src io.Reader) (id.ObjectID, error) { +func (loose *Loose) WriteReaderContent(ty typ.Type, size int, src io.Reader) (id.ObjectID, error) { headerBytes := header.Append(nil, ty, size) writer, err := loose.newStreamWriter(false) diff --git a/object/store/memory/reader.go b/object/store/memory/reader.go index bcde0a78..6b8fae55 100644 --- a/object/store/memory/reader.go +++ b/object/store/memory/reader.go @@ -17,7 +17,7 @@ func (memory *Memory) ReadBytesFull(id id.ObjectID) ([]byte, error) { return nil, store.ErrObjectNotFound } - raw := header.Append(nil, obj.ty, uint64(len(obj.content))) + raw := header.Append(nil, obj.ty, len(obj.content)) raw = append(raw, obj.content...) return raw, nil @@ -34,17 +34,17 @@ func (memory *Memory) ReadBytesContent(id id.ObjectID) (typ.Type, []byte, error) } // ReadHeader reads one object header. -func (memory *Memory) ReadHeader(id id.ObjectID) (typ.Type, uint64, error) { +func (memory *Memory) ReadHeader(id id.ObjectID) (typ.Type, int, error) { obj, ok := memory.objects.Load(id) if !ok { return typ.Unknown, 0, store.ErrObjectNotFound } - return obj.ty, uint64(len(obj.content)), nil + return obj.ty, len(obj.content), nil } // ReadSize reads one object size. -func (memory *Memory) ReadSize(id id.ObjectID) (uint64, error) { +func (memory *Memory) ReadSize(id id.ObjectID) (int, error) { _, size, err := memory.ReadHeader(id) if err != nil { return 0, err @@ -64,13 +64,13 @@ func (memory *Memory) ReadReaderFull(id id.ObjectID) (io.ReadCloser, error) { } // ReadReaderContent reads one object body through a reader. -func (memory *Memory) ReadReaderContent(id id.ObjectID) (typ.Type, uint64, io.ReadCloser, error) { +func (memory *Memory) ReadReaderContent(id id.ObjectID) (typ.Type, int, io.ReadCloser, error) { ty, content, err := memory.ReadBytesContent(id) if err != nil { return typ.Unknown, 0, nil, err } - return ty, uint64(len(content)), io.NopCloser(bytes.NewReader(content)), nil + return ty, len(content), io.NopCloser(bytes.NewReader(content)), nil } // Refresh is a no-op for in-memory object stores. diff --git a/object/store/memory/writer.go b/object/store/memory/writer.go index 185b082b..d76a1f41 100644 --- a/object/store/memory/writer.go +++ b/object/store/memory/writer.go @@ -8,12 +8,11 @@ import ( "lindenii.org/go/furgit/object/id" "lindenii.org/go/furgit/object/store" "lindenii.org/go/furgit/object/typ" - "lindenii.org/go/lgo/intconv" ) // WriteBytesContent writes one typed object content byte slice. func (memory *Memory) WriteBytesContent(ty typ.Type, content []byte) (id.ObjectID, error) { - raw := header.Append(nil, ty, uint64(len(content))) + raw := header.Append(nil, ty, len(content)) raw = append(raw, content...) objectID := memory.objectFormat.Sum(raw) @@ -30,7 +29,7 @@ func (memory *Memory) WriteBytesFull(raw []byte) (id.ObjectID, error) { } content := raw[consumed:] - if uint64(len(content)) != size { + if len(content) != size { return id.ObjectID{}, fmt.Errorf("%w: header size/content mismatch", store.ErrInvalidObject) } @@ -38,21 +37,16 @@ func (memory *Memory) WriteBytesFull(raw []byte) (id.ObjectID, error) { } // WriteReaderContent writes one typed object content stream. -func (memory *Memory) WriteReaderContent(ty typ.Type, size uint64, src io.Reader) (id.ObjectID, error) { - limit, err := intconv.Uint64ToInt64(size) - if err != nil { - return id.ObjectID{}, fmt.Errorf("object/store/memory: content size: %w", err) - } - - content, err := io.ReadAll(io.LimitReader(src, limit+1)) +func (memory *Memory) WriteReaderContent(ty typ.Type, size int, src io.Reader) (id.ObjectID, error) { + content, err := io.ReadAll(io.LimitReader(src, int64(size)+1)) if err != nil { return id.ObjectID{}, fmt.Errorf("object/store/memory: read content: %w", err) } switch { - case uint64(len(content)) > size: + case len(content) > size: return id.ObjectID{}, fmt.Errorf("%w: content longer than declared size", store.ErrInvalidObject) - case uint64(len(content)) < size: + case len(content) < size: return id.ObjectID{}, fmt.Errorf("%w: content shorter than declared size", store.ErrInvalidObject) } diff --git a/object/store/memory/writer_test.go b/object/store/memory/writer_test.go index 18223642..ad0d8722 100644 --- a/object/store/memory/writer_test.go +++ b/object/store/memory/writer_test.go @@ -19,9 +19,9 @@ func TestWriteReaderContent(t *testing.T) { store := memory.New(objectFormat) content := []byte("memory-content\n") - raw := append(header.Append(nil, typ.Blob, uint64(len(content))), content...) + raw := append(header.Append(nil, typ.Blob, len(content)), content...) - gotID, err := store.WriteReaderContent(typ.Blob, uint64(len(content)), bytes.NewReader(content)) + gotID, err := store.WriteReaderContent(typ.Blob, len(content), bytes.NewReader(content)) if err != nil { t.Fatalf("WriteReaderContent: %v", err) } @@ -56,7 +56,7 @@ func TestWriteReaderFull(t *testing.T) { store := memory.New(objectFormat) content := []byte("memory-full\n") - raw := append(header.Append(nil, typ.Blob, uint64(len(content))), content...) + raw := append(header.Append(nil, typ.Blob, len(content)), content...) gotID, err := store.WriteReaderFull(bytes.NewReader(raw)) if err != nil { @@ -89,7 +89,7 @@ func TestWriteBytes(t *testing.T) { store := memory.New(objectFormat) content := []byte("memory-bytes\n") - raw := append(header.Append(nil, typ.Blob, uint64(len(content))), content...) + raw := append(header.Append(nil, typ.Blob, len(content)), content...) gotID, err := store.WriteBytesContent(typ.Blob, content) if err != nil { diff --git a/object/store/mix/header.go b/object/store/mix/header.go index 7f3c0b1b..aefa2907 100644 --- a/object/store/mix/header.go +++ b/object/store/mix/header.go @@ -11,7 +11,7 @@ import ( // ReadHeader reads object header data // from the most-recently-used backend that has it. -func (mix *Mix) ReadHeader(id id.ObjectID) (typ.Type, uint64, error) { +func (mix *Mix) ReadHeader(id id.ObjectID) (typ.Type, int, error) { for _, backend := range mix.order.Keys() { ty, size, err := backend.ReadHeader(id) if err == nil { @@ -32,7 +32,7 @@ func (mix *Mix) ReadHeader(id id.ObjectID) (typ.Type, uint64, error) { // ReadSize reads object content length // from the most-recently-used backend that has it. -func (mix *Mix) ReadSize(id id.ObjectID) (uint64, error) { +func (mix *Mix) ReadSize(id id.ObjectID) (int, error) { for _, backend := range mix.order.Keys() { size, err := backend.ReadSize(id) if err == nil { diff --git a/object/store/mix/reader.go b/object/store/mix/reader.go index 81b0474b..46a3aedf 100644 --- a/object/store/mix/reader.go +++ b/object/store/mix/reader.go @@ -33,7 +33,7 @@ func (mix *Mix) ReadReaderFull(id id.ObjectID) (io.ReadCloser, error) { // ReadReaderContent reads an object's type, declared content length, // and content stream from the most-recently-used backend that has it. -func (mix *Mix) ReadReaderContent(id id.ObjectID) (typ.Type, uint64, io.ReadCloser, error) { +func (mix *Mix) ReadReaderContent(id id.ObjectID) (typ.Type, int, io.ReadCloser, error) { for _, backend := range mix.order.Keys() { ty, size, reader, err := backend.ReadReaderContent(id) if err == nil { diff --git a/object/store/packed/basecache.go b/object/store/packed/basecache.go index 7de4ec7b..88597404 100644 --- a/object/store/packed/basecache.go +++ b/object/store/packed/basecache.go @@ -12,7 +12,7 @@ const baseCacheMaxWeight = 96 << 20 // as a delta base cache key. type baseKey struct { pack *pack - offset uint64 + offset int } // cachedBase is a cached delta base, i.e., diff --git a/object/store/packed/delta.go b/object/store/packed/delta.go index c20a3eb5..567fd679 100644 --- a/object/store/packed/delta.go +++ b/object/store/packed/delta.go @@ -10,6 +10,7 @@ import ( "lindenii.org/go/furgit/internal/compress/zlib" "lindenii.org/go/furgit/internal/format/packfile" "lindenii.org/go/furgit/internal/format/packfile/delta" + "lindenii.org/go/lgo/intconv" ) // deltaNode is a delta entry on a resolution chain. @@ -21,14 +22,14 @@ type deltaNode struct { size uint64 // baseOffset is the entry's base entry offset. - baseOffset uint64 + baseOffset int } // unpackEntry reconstructs the object stored at offset in p, // following ref- and ofs-delta chains within the pack. // // Labels: Life-Independent. -func (packed *Packed) unpackEntry(p *pack, offset uint64) (packfile.EntryType, []byte, error) { +func (packed *Packed) unpackEntry(p *pack, offset int) (packfile.EntryType, []byte, error) { var zero packfile.EntryType var ( @@ -118,18 +119,19 @@ func (packed *Packed) unpackEntry(p *pack, offset uint64) (packfile.EntryType, [ // deltaBaseOffset resolves a delta entry's base entry offset // within the same pack. -func (packed *Packed) deltaBaseOffset(p *pack, offset uint64, header packfile.EntryHeader) (uint64, error) { +func (packed *Packed) deltaBaseOffset(p *pack, offset int, header packfile.EntryHeader) (int, error) { switch header.Type { case packfile.EntryTypeOfsDelta: - if header.OfsDistance == 0 || header.OfsDistance > offset { + dist, err := intconv.Uint64ToInt(header.OfsDistance) + if err != nil || dist == 0 || dist > offset { return 0, fmt.Errorf("%w: pack %q: invalid ofs-delta distance", ErrMalformedPackedStore, p.name) } - return offset - header.OfsDistance, nil + return offset - dist, nil case packfile.EntryTypeRefDelta: refBase := header.RefBase[:packed.objectFormat.Size()] - baseOffset, found, err := p.idx.Lookup(refBase) + baseOffsetU, found, err := p.idx.Lookup(refBase) if err != nil { return 0, fmt.Errorf("%w: pack %q: %w", ErrMalformedPackedStore, p.name, err) } @@ -146,6 +148,11 @@ func (packed *Packed) deltaBaseOffset(p *pack, offset uint64, header packfile.En ) } + baseOffset, err := intconv.Uint64ToInt(baseOffsetU) + if err != nil { + return 0, fmt.Errorf("%w: pack %q: ref-delta base offset overflows int: %w", ErrMalformedPackedStore, p.name, err) + } + return baseOffset, nil case packfile.EntryTypeInvalid, packfile.EntryTypeCommit, @@ -161,7 +168,7 @@ func (packed *Packed) deltaBaseOffset(p *pack, offset uint64, header packfile.En // resolveType walks one delta chain // to find the chained base object entry type, // without inflating any content. -func (packed *Packed) resolveType(p *pack, offset uint64, entryHeader packfile.EntryHeader) (packfile.EntryType, error) { +func (packed *Packed) resolveType(p *pack, offset int, entryHeader packfile.EntryHeader) (packfile.EntryType, error) { var zero packfile.EntryType depth := 0 @@ -194,7 +201,7 @@ func (packed *Packed) resolveType(p *pack, offset uint64, entryHeader packfile.E // deltaResultSize reads the declared result size // from one compressed delta payload prefix. -func deltaResultSize(payload []byte, deltaSize uint64) (uint64, error) { +func deltaResultSize(payload []byte, deltaSize uint64) (int, error) { zr, err := zlib.NewReader(bytes.NewReader(payload)) if err != nil { return 0, fmt.Errorf("reading delta header: %w", err) @@ -216,5 +223,10 @@ func deltaResultSize(payload []byte, deltaSize uint64) (uint64, error) { return 0, fmt.Errorf("reading delta header: %w", err) } - return resultSize, nil + size, err := intconv.Uint64ToInt(resultSize) + if err != nil { + return 0, fmt.Errorf("reading delta header: result size overflows int: %w", err) + } + + return size, nil } diff --git a/object/store/packed/entry.go b/object/store/packed/entry.go index 23f389a3..e9d45bb4 100644 --- a/object/store/packed/entry.go +++ b/object/store/packed/entry.go @@ -23,11 +23,11 @@ var errPayloadOverlong = errors.New("entry payload longer than declared") // not the slice length. // // Labels: Life-Parent, Mut-No. -func (pack *pack) entryHeaderAt(offset uint64, objectFormat id.ObjectFormat) (packfile.EntryHeader, []byte, error) { +func (pack *pack) entryHeaderAt(offset int, objectFormat id.ObjectFormat) (packfile.EntryHeader, []byte, error) { var zero packfile.EntryHeader - pos, err := intconv.Uint64ToInt(offset) - if err != nil || pos >= len(pack.data) { + pos := offset + if pos < 0 || pos >= len(pack.data) { return zero, nil, fmt.Errorf("%w: pack %q: entry offset out of bounds", ErrMalformedPackedStore, pack.name) } diff --git a/object/store/packed/internal/ingest/finalize.go b/object/store/packed/internal/ingest/finalize.go index 7dca131a..f0ab6622 100644 --- a/object/store/packed/internal/ingest/finalize.go +++ b/object/store/packed/internal/ingest/finalize.go @@ -96,9 +96,14 @@ func (ingestion *ingestion) indexEntries() ([]packidx.Entry, []uint32, error) { var oidBytes [id.MaxObjectIDSize]byte copy(oidBytes[:], rec.oid.RawBytes()) + offset, err := intconv.IntToUint64(rec.offset) + if err != nil { + return nil, nil, fmt.Errorf("object/store/packed/internal/ingest: %w", err) + } + entries[indexPosition] = packidx.Entry{ OID: oidBytes, - Offset: rec.offset, + Offset: offset, CRC32: rec.crc32, } diff --git a/object/store/packed/internal/ingest/ingest.go b/object/store/packed/internal/ingest/ingest.go index 324ed8ce..5422b4af 100644 --- a/object/store/packed/internal/ingest/ingest.go +++ b/object/store/packed/internal/ingest/ingest.go @@ -46,17 +46,17 @@ type ingestion struct { // byOffset maps an entry offset to its record index, // and byOID maps a resolved object ID to its record index. - byOffset map[uint64]int + byOffset map[int]int byOID map[id.ObjectID]int // headerCount is the object count declared by the pack header. - headerCount uint32 + headerCount int // deltaCount counts delta records, accumulated during scanning. - deltaCount uint64 + deltaCount int // deltasResolved counts resolved delta records, for progress. - deltasResolved uint64 + deltasResolved int // packHash is the final pack trailer hash. packHash id.ObjectID @@ -101,7 +101,7 @@ func WritePack(root *os.Root, objectFormat id.ObjectFormat, src io.Reader, opts temps: nil, scanner: nil, records: nil, - byOffset: make(map[uint64]int), + byOffset: make(map[int]int), byOID: make(map[id.ObjectID]int), headerCount: count, deltaCount: 0, diff --git a/object/store/packed/internal/ingest/record.go b/object/store/packed/internal/ingest/record.go index 460365fd..69101293 100644 --- a/object/store/packed/internal/ingest/record.go +++ b/object/store/packed/internal/ingest/record.go @@ -12,15 +12,15 @@ import ( // so a record's index in the slice is also its pack order. type record struct { // offset is the entry's start offset in the pack. - offset uint64 + offset int // headerLen is the entry header length in bytes, // so the zlib payload begins at offset+headerLen. - headerLen uint64 + headerLen int // packedLen is the total on-disk entry length in bytes, // covering the header and the compressed payload. - packedLen uint64 + packedLen int // crc32 is the CRC32 of the entry's packed bytes. crc32 uint32 @@ -29,10 +29,10 @@ type record struct { packedType packfile.EntryType // declaredSize is the declared inflated payload size. - declaredSize uint64 + declaredSize int // baseOffset is the base entry offset for an ofs-delta. - baseOffset uint64 + baseOffset int // baseOID is the base object ID for a ref-delta. baseOID id.ObjectID @@ -50,6 +50,6 @@ type record struct { } // dataOffset returns the entry's compressed payload start offset. -func (record *record) dataOffset() uint64 { +func (record *record) dataOffset() int { return record.offset + record.headerLen } diff --git a/object/store/packed/internal/ingest/resolve.go b/object/store/packed/internal/ingest/resolve.go index 7e163f2d..8595d366 100644 --- a/object/store/packed/internal/ingest/resolve.go +++ b/object/store/packed/internal/ingest/resolve.go @@ -10,13 +10,12 @@ import ( "lindenii.org/go/furgit/internal/progress" "lindenii.org/go/furgit/object/header" "lindenii.org/go/furgit/object/id" - "lindenii.org/go/lgo/intconv" ) // adjacency maps each resolvable base to its delta children: // ofs-deltas keyed by base offset, ref-deltas keyed by base object ID. type adjacency struct { - byOffset map[uint64][]int + byOffset map[int][]int byOID map[id.ObjectID][]int } @@ -60,7 +59,7 @@ func (ingestion *ingestion) resolveDeltas() error { // so a resolved base can find the children that delta against it. func (ingestion *ingestion) buildAdjacency() adjacency { out := adjacency{ - byOffset: make(map[uint64][]int), + byOffset: make(map[int][]int), byOID: make(map[id.ObjectID][]int), } @@ -192,20 +191,9 @@ func (ingestion *ingestion) resolveChild( func (ingestion *ingestion) inflateRecord(index int) ([]byte, error) { rec := &ingestion.records[index] - offset, err := intconv.Uint64ToInt64(rec.dataOffset()) - if err != nil { - return nil, fmt.Errorf("object/store/packed/internal/ingest: %w", err) - } - - compressedLen, err := intconv.Uint64ToInt64(rec.packedLen - rec.headerLen) - if err != nil { - return nil, fmt.Errorf("object/store/packed/internal/ingest: %w", err) - } - - size, err := intconv.Uint64ToInt(rec.declaredSize) - if err != nil { - return nil, fmt.Errorf("object/store/packed/internal/ingest: %w", err) - } + offset := int64(rec.dataOffset()) + compressedLen := int64(rec.packedLen - rec.headerLen) + size := rec.declaredSize zr, err := zlib.NewReader(io.NewSectionReader(ingestion.packFile, offset, compressedLen)) if err != nil { @@ -238,7 +226,7 @@ func (ingestion *ingestion) hashObject(objectType packfile.EntryType, content [] return zero, fmt.Errorf("object/store/packed/internal/ingest: %w", err) } - _, _ = hashImpl.Write(header.Append(nil, ty, uint64(len(content)))) + _, _ = hashImpl.Write(header.Append(nil, ty, len(content))) _, _ = hashImpl.Write(content) oid, err := ingestion.objectFormat.FromBytes(hashImpl.Sum(nil)) @@ -263,7 +251,7 @@ func (ingestion *ingestion) resolvedRoots() []int { } // countDeltas returns the number of delta records. -func (ingestion *ingestion) countDeltas() uint64 { +func (ingestion *ingestion) countDeltas() int { return ingestion.deltaCount } @@ -272,7 +260,7 @@ func (ingestion *ingestion) countDeltas() uint64 { // Every base is resolved during scanning or thin completion, // so the unresolved records are exactly the unresolved deltas: // the delta records minus those already resolved. -func (ingestion *ingestion) countUnresolved() uint64 { +func (ingestion *ingestion) countUnresolved() int { return ingestion.deltaCount - ingestion.deltasResolved } diff --git a/object/store/packed/internal/ingest/scan.go b/object/store/packed/internal/ingest/scan.go index 56e42cea..6b3b73b7 100644 --- a/object/store/packed/internal/ingest/scan.go +++ b/object/store/packed/internal/ingest/scan.go @@ -36,7 +36,7 @@ type scanner struct { n int // consumed counts stream bytes consumed so far. - consumed uint64 + consumed int // hash accumulates the pack hash over consumed bytes // while hashing is true. @@ -66,7 +66,7 @@ func newScanner(src io.Reader, dst io.Writer, packHash hash.Hash) *scanner { // readPackHeader reads and validates the pack header from src, // returning the raw header and its declared object count. -func readPackHeader(src io.Reader) ([packfile.HeaderLen]byte, uint32, error) { +func readPackHeader(src io.Reader) ([packfile.HeaderLen]byte, int, error) { var raw [packfile.HeaderLen]byte _, err := io.ReadFull(src, raw[:]) @@ -79,7 +79,12 @@ func readPackHeader(src io.Reader) ([packfile.HeaderLen]byte, uint32, error) { return raw, 0, fmt.Errorf("%w: %w", ErrMalformedPack, err) } - return raw, packHeader.ObjectCount, nil + count, err := intconv.Uint32ToInt(packHeader.ObjectCount) + if err != nil { + return raw, 0, fmt.Errorf("%w: object count: %w", ErrMalformedPack, err) + } + + return raw, count, nil } // Read implements [io.Reader]. @@ -205,7 +210,7 @@ func (scanner *scanner) use(n int) error { } scanner.off += n - scanner.consumed += uint64(n) //nolint:gosec + scanner.consumed += n return nil } @@ -283,7 +288,7 @@ func (ingestion *ingestion) streamAndScan() error { meter := progress.New(progress.Options{ Writer: ingestion.opts.Progress, Title: "receiving objects", - Total: uint64(ingestion.headerCount), + Total: ingestion.headerCount, Delay: 0, Sparse: false, Throughput: true, @@ -295,7 +300,7 @@ func (ingestion *ingestion) streamAndScan() error { return err } - meter.Set(uint64(done)+1, ingestion.scanner.consumed) + meter.Set(done+1, ingestion.scanner.consumed) } meter.Stop("done") @@ -316,7 +321,7 @@ func (ingestion *ingestion) streamAndScan() error { } // scanEntry scans the entry beginning at start into one record. -func (ingestion *ingestion) scanEntry(start uint64) error { +func (ingestion *ingestion) scanEntry(start int) error { ingestion.scanner.beginCRC() rec, err := ingestion.scanHeader(start) @@ -329,7 +334,7 @@ func (ingestion *ingestion) scanEntry(start uint64) error { return err } - if inflated != rec.declaredSize { + if inflated != int64(rec.declaredSize) { return fmt.Errorf( "%w: entry at %d: inflated size %d differs from declared %d", ErrMalformedPack, start, inflated, rec.declaredSize, @@ -359,7 +364,7 @@ func (ingestion *ingestion) scanEntry(start uint64) error { } // scanHeader parses and consumes the entry header at start. -func (ingestion *ingestion) scanHeader(start uint64) (record, error) { +func (ingestion *ingestion) scanHeader(start int) (record, error) { var rec record rec.offset = start @@ -374,23 +379,24 @@ func (ingestion *ingestion) scanHeader(start uint64) (record, error) { return rec, fmt.Errorf("%w: entry at %d: %w", ErrMalformedPack, start, err) } - headerLen, err := intconv.IntToUint64(entryHeader.HeaderLen) + declaredSize, err := intconv.Uint64ToInt(entryHeader.Size) if err != nil { - return rec, fmt.Errorf("object/store/packed/internal/ingest: %w", err) + return rec, fmt.Errorf("%w: entry at %d: declared size overflows int: %w", ErrMalformedPack, start, err) } rec.packedType = entryHeader.Type - rec.declaredSize = entryHeader.Size - rec.headerLen = headerLen + rec.declaredSize = declaredSize + rec.headerLen = entryHeader.HeaderLen switch entryHeader.Type { case packfile.EntryTypeCommit, packfile.EntryTypeTree, packfile.EntryTypeBlob, packfile.EntryTypeTag: case packfile.EntryTypeOfsDelta: - if entryHeader.OfsDistance == 0 || entryHeader.OfsDistance > start { + dist, err := intconv.Uint64ToInt(entryHeader.OfsDistance) + if err != nil || dist == 0 || dist > start { return rec, fmt.Errorf("%w: entry at %d: ofs-delta base out of bounds", ErrMalformedPack, start) } - rec.baseOffset = start - entryHeader.OfsDistance + rec.baseOffset = start - dist case packfile.EntryTypeRefDelta: baseID, err := ingestion.objectFormat.FromBytes(entryHeader.RefBase[:ingestion.objectFormat.Size()]) if err != nil { @@ -412,7 +418,7 @@ func (ingestion *ingestion) scanHeader(start uint64) (record, error) { // drainPayload consumes one entry's compressed payload from the stream, // returning its inflated length and, for base entries, its object ID. -func (ingestion *ingestion) drainPayload(rec *record) (uint64, id.ObjectID, error) { +func (ingestion *ingestion) drainPayload(rec *record) (int64, id.ObjectID, error) { var zero id.ObjectID zr, err := zlib.NewReader(ingestion.scanner) @@ -428,12 +434,7 @@ func (ingestion *ingestion) drainPayload(rec *record) (uint64, id.ObjectID, erro return 0, zero, fmt.Errorf("%w: entry at %d: %w", ErrMalformedPack, rec.offset, err) } - inflated, err := intconv.Int64ToUint64(read) - if err != nil { - return 0, zero, fmt.Errorf("object/store/packed/internal/ingest: %w", err) - } - - return inflated, zero, nil + return read, zero, nil } objectType, err := rec.packedType.ObjectType() @@ -458,10 +459,5 @@ func (ingestion *ingestion) drainPayload(rec *record) (uint64, id.ObjectID, erro return 0, zero, fmt.Errorf("object/store/packed/internal/ingest: %w", err) } - inflated, err := intconv.Int64ToUint64(read) - if err != nil { - return 0, zero, fmt.Errorf("object/store/packed/internal/ingest: %w", err) - } - - return inflated, oid, nil + return read, oid, nil } diff --git a/object/store/packed/internal/ingest/thin.go b/object/store/packed/internal/ingest/thin.go index e96846cb..8d1566e0 100644 --- a/object/store/packed/internal/ingest/thin.go +++ b/object/store/packed/internal/ingest/thin.go @@ -25,8 +25,8 @@ func (ingestion *ingestion) fixThin(external []id.ObjectID, adjacency adjacency, return ErrThinPackNotPermitted } - hashSize := uint64(ingestion.objectFormat.Size()) //nolint:gosec - if ingestion.scanner.consumed < uint64(packfile.HeaderLen)+hashSize { + hashSize := ingestion.objectFormat.Size() + if ingestion.scanner.consumed < packfile.HeaderLen+hashSize { return fmt.Errorf("%w: pack shorter than trailer", ErrMalformedPack) } @@ -96,11 +96,7 @@ func (ingestion *ingestion) appendBaseObject(objectID id.ObjectID, objectType ty } start := ingestion.scanner.consumed - - startOffset, err := intconv.Uint64ToInt64(start) - if err != nil { - return 0, fmt.Errorf("object/store/packed/internal/ingest: %w", err) - } + startOffset := int64(start) headerBytes := packfile.AppendTypeSize(nil, entryType, uint64(len(content))) @@ -129,17 +125,8 @@ func (ingestion *ingestion) appendBaseObject(objectID id.ObjectID, objectType ty return 0, fmt.Errorf("object/store/packed/internal/ingest: compressing thin base: %w", err) } - compressedLen, err := intconv.Int64ToUint64(writer.offset - dataOffset) - if err != nil { - return 0, fmt.Errorf("object/store/packed/internal/ingest: %w", err) - } - - headerLen, err := intconv.IntToUint64(len(headerBytes)) - if err != nil { - return 0, fmt.Errorf("object/store/packed/internal/ingest: %w", err) - } - - packedLen := headerLen + compressedLen + headerLen := len(headerBytes) + packedLen := headerLen + writer.written ingestion.scanner.consumed = start + packedLen rec := record{ @@ -148,7 +135,7 @@ func (ingestion *ingestion) appendBaseObject(objectID id.ObjectID, objectType ty packedLen: packedLen, crc32: crc.Sum32(), packedType: entryType, - declaredSize: uint64(len(content)), + declaredSize: len(content), baseOffset: 0, baseOID: id.ObjectID{}, objectType: entryType, @@ -178,10 +165,7 @@ func (ingestion *ingestion) rewriteHeaderTrailer() error { return fmt.Errorf("object/store/packed/internal/ingest: rewriting header: %w", err) } - bodyEnd, err := intconv.Uint64ToInt64(ingestion.scanner.consumed) - if err != nil { - return fmt.Errorf("object/store/packed/internal/ingest: %w", err) - } + bodyEnd := int64(ingestion.scanner.consumed) hashImpl, err := ingestion.objectFormat.New() if err != nil { @@ -211,16 +195,19 @@ func (ingestion *ingestion) rewriteHeaderTrailer() error { } // offsetWriter writes to a file via WriteAt, -// advancing sequentially from a base offset. +// advancing sequentially from a base offset +// and counting the bytes written. type offsetWriter struct { - file *os.File - offset int64 + file *os.File + offset int64 + written int } // Write implements [io.Writer]. func (writer *offsetWriter) Write(p []byte) (int, error) { n, err := writer.file.WriteAt(p, writer.offset) writer.offset += int64(n) + writer.written += n return n, err //nolint:wrapcheck } diff --git a/object/store/packed/lookup.go b/object/store/packed/lookup.go index 74087072..e54d34b2 100644 --- a/object/store/packed/lookup.go +++ b/object/store/packed/lookup.go @@ -5,6 +5,7 @@ import ( "lindenii.org/go/furgit/object/id" "lindenii.org/go/furgit/object/store" + "lindenii.org/go/lgo/intconv" ) // lookup finds the pack containing objectID @@ -12,7 +13,7 @@ import ( // probing packs in most-recently-used-ish order. // // Labels: Life-Parent. -func (packed *Packed) lookup(objectID id.ObjectID) (*pack, uint64, error) { +func (packed *Packed) lookup(objectID id.ObjectID) (*pack, int, error) { if objectID.ObjectFormat() != packed.objectFormat { return nil, 0, fmt.Errorf( "%w: got %s want %s", @@ -23,7 +24,7 @@ func (packed *Packed) lookup(objectID id.ObjectID) (*pack, uint64, error) { oid := objectID.RawBytes() for _, p := range packed.order.Keys() { - offset, found, err := p.idx.Lookup(oid) + offsetU, found, err := p.idx.Lookup(oid) if err != nil { return nil, 0, fmt.Errorf("%w: pack %q: %w", ErrMalformedPackedStore, p.name, err) } @@ -32,6 +33,11 @@ func (packed *Packed) lookup(objectID id.ObjectID) (*pack, uint64, error) { continue } + offset, err := intconv.Uint64ToInt(offsetU) + if err != nil { + return nil, 0, fmt.Errorf("%w: pack %q: entry offset overflows int: %w", ErrMalformedPackedStore, p.name, err) + } + packed.order.Touch(p) return p, offset, nil diff --git a/object/store/packed/read_test.go b/object/store/packed/read_test.go index 02a3a5b0..64faaf5b 100644 --- a/object/store/packed/read_test.go +++ b/object/store/packed/read_test.go @@ -70,7 +70,7 @@ func TestReadGitPack(t *testing.T) { t.Fatalf("ReadHeader(%s) type = %v, want %v", oid, ty, group.ty) } - if size != uint64(len(wantContent)) { + if size != len(wantContent) { t.Fatalf("ReadHeader(%s) size = %d, want %d", oid, size, len(wantContent)) } @@ -79,7 +79,7 @@ func TestReadGitPack(t *testing.T) { t.Fatalf("ReadSize(%s): %v", oid, err) } - if size != uint64(len(wantContent)) { + if size != len(wantContent) { t.Fatalf("ReadSize(%s) = %d, want %d", oid, size, len(wantContent)) } @@ -105,7 +105,7 @@ func checkReaderContent(t *testing.T, packedStore *packed.Packed, oid id.ObjectI t.Fatalf("ReadReaderContent(%s) type = %v, want %v", oid, ty, wantType) } - if size != uint64(len(wantContent)) { + if size != len(wantContent) { t.Fatalf("ReadReaderContent(%s) size = %d, want %d", oid, size, len(wantContent)) } diff --git a/object/store/packed/reader.go b/object/store/packed/reader.go index 9c183575..bfc82eff 100644 --- a/object/store/packed/reader.go +++ b/object/store/packed/reader.go @@ -11,6 +11,7 @@ import ( "lindenii.org/go/furgit/object/header" "lindenii.org/go/furgit/object/id" "lindenii.org/go/furgit/object/typ" + "lindenii.org/go/lgo/intconv" ) // ReadBytesContent reads an object's type and content bytes, @@ -42,7 +43,7 @@ func (packed *Packed) ReadBytesFull(objectID id.ObjectID) ([]byte, error) { return nil, err } - raw := header.Append(make([]byte, 0, len(content)+32), ty, uint64(len(content))) + raw := header.Append(make([]byte, 0, len(content)+32), ty, len(content)) return append(raw, content...), nil } @@ -52,7 +53,7 @@ func (packed *Packed) ReadBytesFull(objectID id.ObjectID) ([]byte, error) { // For delta entries this resolves the chained base type // and the declared delta result size, // without reconstructing content. -func (packed *Packed) ReadHeader(objectID id.ObjectID) (typ.Type, uint64, error) { +func (packed *Packed) ReadHeader(objectID id.ObjectID) (typ.Type, int, error) { p, offset, err := packed.lookup(objectID) if err != nil { return typ.Unknown, 0, err @@ -63,13 +64,18 @@ func (packed *Packed) ReadHeader(objectID id.ObjectID) (typ.Type, uint64, error) return typ.Unknown, 0, err } - size := entryHeader.Size + var size int if entryHeader.Type.IsDelta() { size, err = deltaResultSize(payload, entryHeader.Size) if err != nil { return typ.Unknown, 0, fmt.Errorf("%w: pack %q: %w", ErrMalformedPackedStore, p.name, err) } + } else { + size, err = intconv.Uint64ToInt(entryHeader.Size) + if err != nil { + return typ.Unknown, 0, fmt.Errorf("%w: pack %q: object size overflows int: %w", ErrMalformedPackedStore, p.name, err) + } } entryType, err := packed.resolveType(p, offset, entryHeader) @@ -89,7 +95,7 @@ func (packed *Packed) ReadHeader(objectID id.ObjectID) (typ.Type, uint64, error) // // Unlike ReadHeader, // this never walks delta chains. -func (packed *Packed) ReadSize(objectID id.ObjectID) (uint64, error) { +func (packed *Packed) ReadSize(objectID id.ObjectID) (int, error) { p, offset, err := packed.lookup(objectID) if err != nil { return 0, err @@ -101,7 +107,12 @@ func (packed *Packed) ReadSize(objectID id.ObjectID) (uint64, error) { } if !entryHeader.Type.IsDelta() { - return entryHeader.Size, nil + size, err := intconv.Uint64ToInt(entryHeader.Size) + if err != nil { + return 0, fmt.Errorf("%w: pack %q: object size overflows int: %w", ErrMalformedPackedStore, p.name, err) + } + + return size, nil } size, err := deltaResultSize(payload, entryHeader.Size) @@ -119,7 +130,7 @@ func (packed *Packed) ReadSize(objectID id.ObjectID) (uint64, error) { // delta entries are fully resolved in memory first. // Close releases resources only // and does not drain unread data for additional validation. -func (packed *Packed) ReadReaderContent(objectID id.ObjectID) (typ.Type, uint64, io.ReadCloser, error) { +func (packed *Packed) ReadReaderContent(objectID id.ObjectID) (typ.Type, int, io.ReadCloser, error) { p, offset, err := packed.lookup(objectID) if err != nil { return typ.Unknown, 0, nil, err @@ -141,7 +152,7 @@ func (packed *Packed) ReadReaderContent(objectID id.ObjectID) (typ.Type, uint64, return typ.Unknown, 0, nil, err } - return ty, uint64(len(content)), io.NopCloser(bytes.NewReader(content)), nil + return ty, len(content), io.NopCloser(bytes.NewReader(content)), nil } ty, err := objectTypeOf(entryHeader.Type) @@ -149,13 +160,18 @@ func (packed *Packed) ReadReaderContent(objectID id.ObjectID) (typ.Type, uint64, return typ.Unknown, 0, nil, err } + size, err := intconv.Uint64ToInt(entryHeader.Size) + if err != nil { + return typ.Unknown, 0, nil, fmt.Errorf("%w: pack %q: object size overflows int: %w", ErrMalformedPackedStore, p.name, err) + } + zr, err := zlib.NewReader(bytes.NewReader(payload)) if err != nil { return typ.Unknown, 0, nil, fmt.Errorf("%w: pack %q: %w", ErrMalformedPackedStore, p.name, err) } - return ty, entryHeader.Size, &objectReader{ - reader: iolimit.ExpectLengthReader(zr, entryHeader.Size), + return ty, size, &objectReader{ + reader: iolimit.ExpectLengthReader(zr, size), zr: zr, }, nil } diff --git a/object/store/reader.go b/object/store/reader.go index e8829b87..7979fb6c 100644 --- a/object/store/reader.go +++ b/object/store/reader.go @@ -41,17 +41,17 @@ type ObjectReader interface { // declared content length, and content stream. // // Labels: Life-Parent, Close-Caller. - ReadReaderContent(id id.ObjectID) (typ.Type, uint64, io.ReadCloser, error) + ReadReaderContent(id id.ObjectID) (typ.Type, int, io.ReadCloser, error) // ReadSize reads an object's declared content length. // // This returns the same size as the second result of [ObjectReader.ReadHeader]; // for some implementations, this may be cheaper than ReadHeader // when callers do not need the object type. - ReadSize(id id.ObjectID) (uint64, error) + ReadSize(id id.ObjectID) (int, error) // ReadHeader reads an object's type and declared content length. - ReadHeader(id id.ObjectID) (typ.Type, uint64, error) + ReadHeader(id id.ObjectID) (typ.Type, int, error) // Refresh updates any backend-local discovery/cache view of on-disk objects. // diff --git a/object/store/writer.go b/object/store/writer.go index ce3284d2..d83eec6a 100644 --- a/object/store/writer.go +++ b/object/store/writer.go @@ -24,7 +24,7 @@ type ObjectWriter interface { WriteReaderFull(src io.Reader) (id.ObjectID, error) // WriteReaderContent writes one typed object content stream. - WriteReaderContent(ty typ.Type, size uint64, src io.Reader) (id.ObjectID, error) + WriteReaderContent(ty typ.Type, size int, src io.Reader) (id.ObjectID, error) } // PackWriter writes Git pack streams. diff --git a/object/tag/append.go b/object/tag/append.go index bf519c55..15a6fde9 100644 --- a/object/tag/append.go +++ b/object/tag/append.go @@ -47,7 +47,7 @@ func (tag *Tag) AppendWithHeader(dst []byte) ([]byte, error) { return dst, err } - dst = header.Append(dst, typ.Tag, uint64(len(body))) + dst = header.Append(dst, typ.Tag, len(body)) return append(dst, body...), nil } diff --git a/object/tree/append.go b/object/tree/append.go index 5fe040f1..edabf714 100644 --- a/object/tree/append.go +++ b/object/tree/append.go @@ -39,7 +39,7 @@ func (tree *Tree) AppendWithHeader(dst []byte) ([]byte, error) { return dst, err } - dst = header.Append(dst, typ.Tree, uint64(len(body))) + dst = header.Append(dst, typ.Tree, len(body)) return append(dst, body...), nil } -- cgit v1.3.1-10-gc9f91