From 3e884f5f3d42cbc4874a04da31dde10314b0cfad Mon Sep 17 00:00:00 2001 From: Runxi Yu Date: Thu, 26 Mar 2026 09:17:14 +0000 Subject: format: Move commitgraph and packfile here --- cmd/index-pack/main.go | 2 +- commitgraph/TODO | 6 - commitgraph/bloom/bloom.go | 3 - commitgraph/bloom/constants.go | 8 - commitgraph/bloom/contain.go | 25 -- commitgraph/bloom/errors.go | 5 - commitgraph/bloom/filter.go | 26 -- commitgraph/bloom/key.go | 117 ------ commitgraph/bloom/murmur.go | 127 ------ commitgraph/bloom/settings.go | 50 --- commitgraph/constants.go | 32 -- commitgraph/doc.go | 2 - commitgraph/read/bloom.go | 117 ------ commitgraph/read/close.go | 20 - commitgraph/read/commitat.go | 85 ---- commitgraph/read/commits.go | 20 - commitgraph/read/doc.go | 2 - commitgraph/read/edges.go | 48 --- commitgraph/read/errors.go | 58 --- commitgraph/read/generation.go | 43 -- commitgraph/read/hash.go | 79 ---- commitgraph/read/iterators.go | 45 --- commitgraph/read/layer.go | 28 -- commitgraph/read/layer_close.go | 33 -- commitgraph/read/layer_lookup.go | 53 --- commitgraph/read/layer_open.go | 81 ---- commitgraph/read/layer_parse.go | 276 ------------- commitgraph/read/layer_pos.go | 21 - commitgraph/read/layerinfo.go | 23 -- commitgraph/read/lookup.go | 29 -- commitgraph/read/mode.go | 11 - commitgraph/read/oidat.go | 36 -- commitgraph/read/open.go | 26 -- commitgraph/read/open_chain.go | 133 ------- commitgraph/read/open_single.go | 32 -- commitgraph/read/parents.go | 67 ---- commitgraph/read/position.go | 38 -- commitgraph/read/read_test.go | 322 --------------- commitgraph/read/reader.go | 16 - .../fixtures/sha1/chain_changed/repo.git/HEAD | 1 - .../fixtures/sha1/chain_changed/repo.git/config | 4 - .../objects/info/commit-graphs/commit-graph-chain | 2 - ...-bf985c21612a52070d8b008e6ef51edf8b609401.graph | Bin 4810 -> 0 bytes ...-dd7578d5216ca76c25b19631ba90f7498aeabbe7.graph | Bin 7088 -> 0 bytes .../sha1/chain_changed/repo.git/objects/info/packs | 2 - ...15b064d6a8ef8cff520565f6db8c006b2e6f7f2f.bitmap | Bin 8234 -> 0 bytes ...ck-15b064d6a8ef8cff520565f6db8c006b2e6f7f2f.idx | Bin 13252 -> 0 bytes ...k-15b064d6a8ef8cff520565f6db8c006b2e6f7f2f.pack | Bin 34730 -> 0 bytes ...ck-15b064d6a8ef8cff520565f6db8c006b2e6f7f2f.rev | Bin 1792 -> 0 bytes .../sha1/chain_changed/repo.git/refs/heads/master | 1 - .../fixtures/sha1/single_changed/repo.git/HEAD | 1 - .../fixtures/sha1/single_changed/repo.git/config | 4 - .../repo.git/objects/info/commit-graph | Bin 9068 -> 0 bytes .../single_changed/repo.git/objects/info/packs | 2 - ...34e9e132566989e2abfe8821731236c77f9bcbe9.bitmap | Bin 7780 -> 0 bytes ...ck-34e9e132566989e2abfe8821731236c77f9bcbe9.idx | Bin 11152 -> 0 bytes ...k-34e9e132566989e2abfe8821731236c77f9bcbe9.pack | Bin 28664 -> 0 bytes ...ck-34e9e132566989e2abfe8821731236c77f9bcbe9.rev | Bin 1492 -> 0 bytes .../sha1/single_changed/repo.git/refs/heads/main | 1 - .../fixtures/sha1/single_nochanged/repo.git/HEAD | 1 - .../fixtures/sha1/single_nochanged/repo.git/config | 4 - .../repo.git/objects/info/commit-graph | Bin 5912 -> 0 bytes .../single_nochanged/repo.git/objects/info/packs | 2 - ...a3da595034c94bb16b6829d757a66b7d259b9ffc.bitmap | Bin 5452 -> 0 bytes ...ck-a3da595034c94bb16b6829d757a66b7d259b9ffc.idx | Bin 7792 -> 0 bytes ...k-a3da595034c94bb16b6829d757a66b7d259b9ffc.pack | Bin 18969 -> 0 bytes ...ck-a3da595034c94bb16b6829d757a66b7d259b9ffc.rev | Bin 1012 -> 0 bytes .../single_nochanged/repo.git/refs/heads/master | 1 - .../fixtures/sha256/chain_changed/repo.git/HEAD | 1 - .../fixtures/sha256/chain_changed/repo.git/config | 6 - .../objects/info/commit-graphs/commit-graph-chain | 2 - ...97943112739236c6bcd19ed4d1f7c6b830cab4f62.graph | Bin 9260 -> 0 bytes ...717a5823c0cb4b5ee336a14959678e060d674ffb6.graph | Bin 6154 -> 0 bytes .../chain_changed/repo.git/objects/info/packs | 2 - ...f045957e44ccee06d812b5e531ae666014a26ed1.bitmap | Bin 8234 -> 0 bytes ...fbcf045957e44ccee06d812b5e531ae666014a26ed1.idx | Bin 18496 -> 0 bytes ...bcf045957e44ccee06d812b5e531ae666014a26ed1.pack | Bin 41482 -> 0 bytes ...fbcf045957e44ccee06d812b5e531ae666014a26ed1.rev | Bin 1816 -> 0 bytes .../chain_changed/repo.git/refs/heads/master | 1 - .../fixtures/sha256/single_changed/repo.git/HEAD | 1 - .../fixtures/sha256/single_changed/repo.git/config | 6 - .../repo.git/objects/info/commit-graph | Bin 11960 -> 0 bytes .../single_changed/repo.git/objects/info/packs | 2 - ...a0c55b76387cbf1fd2a117ab3d7ca0d854a031c9.bitmap | Bin 7804 -> 0 bytes ...40da0c55b76387cbf1fd2a117ab3d7ca0d854a031c9.idx | Bin 15496 -> 0 bytes ...0da0c55b76387cbf1fd2a117ab3d7ca0d854a031c9.pack | Bin 34252 -> 0 bytes ...40da0c55b76387cbf1fd2a117ab3d7ca0d854a031c9.rev | Bin 1516 -> 0 bytes .../sha256/single_changed/repo.git/refs/heads/main | 1 - .../fixtures/sha256/single_nochanged/repo.git/HEAD | 1 - .../sha256/single_nochanged/repo.git/config | 6 - .../repo.git/objects/info/commit-graph | Bin 7844 -> 0 bytes .../single_nochanged/repo.git/objects/info/packs | 2 - ...0ec9bf0e5dd596c0ee1ac6310136067c4f13438b.bitmap | Bin 5476 -> 0 bytes ...d780ec9bf0e5dd596c0ee1ac6310136067c4f13438b.idx | Bin 10696 -> 0 bytes ...780ec9bf0e5dd596c0ee1ac6310136067c4f13438b.pack | Bin 22569 -> 0 bytes ...d780ec9bf0e5dd596c0ee1ac6310136067c4f13438b.rev | Bin 1036 -> 0 bytes .../single_nochanged/repo.git/refs/heads/master | 1 - commitquery/commit.go | 2 +- commitquery/context.go | 2 +- commitquery/graph_pos.go | 2 +- commitquery/node.go | 2 +- commitquery/oid.go | 2 +- commitquery/parent.go | 2 +- format/commitgraph/TODO | 6 + format/commitgraph/bloom/bloom.go | 3 + format/commitgraph/bloom/constants.go | 8 + format/commitgraph/bloom/contain.go | 25 ++ format/commitgraph/bloom/errors.go | 5 + format/commitgraph/bloom/filter.go | 26 ++ format/commitgraph/bloom/key.go | 117 ++++++ format/commitgraph/bloom/murmur.go | 127 ++++++ format/commitgraph/bloom/settings.go | 50 +++ format/commitgraph/constants.go | 32 ++ format/commitgraph/doc.go | 2 + format/commitgraph/read/bloom.go | 117 ++++++ format/commitgraph/read/close.go | 20 + format/commitgraph/read/commitat.go | 85 ++++ format/commitgraph/read/commits.go | 20 + format/commitgraph/read/doc.go | 2 + format/commitgraph/read/edges.go | 48 +++ format/commitgraph/read/errors.go | 58 +++ format/commitgraph/read/generation.go | 43 ++ format/commitgraph/read/hash.go | 79 ++++ format/commitgraph/read/iterators.go | 45 +++ format/commitgraph/read/layer.go | 28 ++ format/commitgraph/read/layer_close.go | 33 ++ format/commitgraph/read/layer_lookup.go | 53 +++ format/commitgraph/read/layer_open.go | 81 ++++ format/commitgraph/read/layer_parse.go | 276 +++++++++++++ format/commitgraph/read/layer_pos.go | 21 + format/commitgraph/read/layerinfo.go | 23 ++ format/commitgraph/read/lookup.go | 29 ++ format/commitgraph/read/mode.go | 11 + format/commitgraph/read/oidat.go | 36 ++ format/commitgraph/read/open.go | 26 ++ format/commitgraph/read/open_chain.go | 133 +++++++ format/commitgraph/read/open_single.go | 32 ++ format/commitgraph/read/parents.go | 67 ++++ format/commitgraph/read/position.go | 38 ++ format/commitgraph/read/read_test.go | 322 +++++++++++++++ format/commitgraph/read/reader.go | 16 + .../fixtures/sha1/chain_changed/repo.git/HEAD | 1 + .../fixtures/sha1/chain_changed/repo.git/config | 4 + .../objects/info/commit-graphs/commit-graph-chain | 2 + ...-bf985c21612a52070d8b008e6ef51edf8b609401.graph | Bin 0 -> 4810 bytes ...-dd7578d5216ca76c25b19631ba90f7498aeabbe7.graph | Bin 0 -> 7088 bytes .../sha1/chain_changed/repo.git/objects/info/packs | 2 + ...15b064d6a8ef8cff520565f6db8c006b2e6f7f2f.bitmap | Bin 0 -> 8234 bytes ...ck-15b064d6a8ef8cff520565f6db8c006b2e6f7f2f.idx | Bin 0 -> 13252 bytes ...k-15b064d6a8ef8cff520565f6db8c006b2e6f7f2f.pack | Bin 0 -> 34730 bytes ...ck-15b064d6a8ef8cff520565f6db8c006b2e6f7f2f.rev | Bin 0 -> 1792 bytes .../sha1/chain_changed/repo.git/refs/heads/master | 1 + .../fixtures/sha1/single_changed/repo.git/HEAD | 1 + .../fixtures/sha1/single_changed/repo.git/config | 4 + .../repo.git/objects/info/commit-graph | Bin 0 -> 9068 bytes .../single_changed/repo.git/objects/info/packs | 2 + ...34e9e132566989e2abfe8821731236c77f9bcbe9.bitmap | Bin 0 -> 7780 bytes ...ck-34e9e132566989e2abfe8821731236c77f9bcbe9.idx | Bin 0 -> 11152 bytes ...k-34e9e132566989e2abfe8821731236c77f9bcbe9.pack | Bin 0 -> 28664 bytes ...ck-34e9e132566989e2abfe8821731236c77f9bcbe9.rev | Bin 0 -> 1492 bytes .../sha1/single_changed/repo.git/refs/heads/main | 1 + .../fixtures/sha1/single_nochanged/repo.git/HEAD | 1 + .../fixtures/sha1/single_nochanged/repo.git/config | 4 + .../repo.git/objects/info/commit-graph | Bin 0 -> 5912 bytes .../single_nochanged/repo.git/objects/info/packs | 2 + ...a3da595034c94bb16b6829d757a66b7d259b9ffc.bitmap | Bin 0 -> 5452 bytes ...ck-a3da595034c94bb16b6829d757a66b7d259b9ffc.idx | Bin 0 -> 7792 bytes ...k-a3da595034c94bb16b6829d757a66b7d259b9ffc.pack | Bin 0 -> 18969 bytes ...ck-a3da595034c94bb16b6829d757a66b7d259b9ffc.rev | Bin 0 -> 1012 bytes .../single_nochanged/repo.git/refs/heads/master | 1 + .../fixtures/sha256/chain_changed/repo.git/HEAD | 1 + .../fixtures/sha256/chain_changed/repo.git/config | 6 + .../objects/info/commit-graphs/commit-graph-chain | 2 + ...97943112739236c6bcd19ed4d1f7c6b830cab4f62.graph | Bin 0 -> 9260 bytes ...717a5823c0cb4b5ee336a14959678e060d674ffb6.graph | Bin 0 -> 6154 bytes .../chain_changed/repo.git/objects/info/packs | 2 + ...f045957e44ccee06d812b5e531ae666014a26ed1.bitmap | Bin 0 -> 8234 bytes ...fbcf045957e44ccee06d812b5e531ae666014a26ed1.idx | Bin 0 -> 18496 bytes ...bcf045957e44ccee06d812b5e531ae666014a26ed1.pack | Bin 0 -> 41482 bytes ...fbcf045957e44ccee06d812b5e531ae666014a26ed1.rev | Bin 0 -> 1816 bytes .../chain_changed/repo.git/refs/heads/master | 1 + .../fixtures/sha256/single_changed/repo.git/HEAD | 1 + .../fixtures/sha256/single_changed/repo.git/config | 6 + .../repo.git/objects/info/commit-graph | Bin 0 -> 11960 bytes .../single_changed/repo.git/objects/info/packs | 2 + ...a0c55b76387cbf1fd2a117ab3d7ca0d854a031c9.bitmap | Bin 0 -> 7804 bytes ...40da0c55b76387cbf1fd2a117ab3d7ca0d854a031c9.idx | Bin 0 -> 15496 bytes ...0da0c55b76387cbf1fd2a117ab3d7ca0d854a031c9.pack | Bin 0 -> 34252 bytes ...40da0c55b76387cbf1fd2a117ab3d7ca0d854a031c9.rev | Bin 0 -> 1516 bytes .../sha256/single_changed/repo.git/refs/heads/main | 1 + .../fixtures/sha256/single_nochanged/repo.git/HEAD | 1 + .../sha256/single_nochanged/repo.git/config | 6 + .../repo.git/objects/info/commit-graph | Bin 0 -> 7844 bytes .../single_nochanged/repo.git/objects/info/packs | 2 + ...0ec9bf0e5dd596c0ee1ac6310136067c4f13438b.bitmap | Bin 0 -> 5476 bytes ...d780ec9bf0e5dd596c0ee1ac6310136067c4f13438b.idx | Bin 0 -> 10696 bytes ...780ec9bf0e5dd596c0ee1ac6310136067c4f13438b.pack | Bin 0 -> 22569 bytes ...d780ec9bf0e5dd596c0ee1ac6310136067c4f13438b.rev | Bin 0 -> 1036 bytes .../single_nochanged/repo.git/refs/heads/master | 1 + format/doc.go | 5 + format/packfile/delta/apply/apply.go | 160 ++++++++ format/packfile/delta/apply/header.go | 47 +++ format/packfile/delta/doc.go | 2 + format/packfile/doc.go | 5 + format/packfile/entry.go | 76 ++++ format/packfile/entry_header.go | 52 +++ format/packfile/header.go | 9 + format/packfile/ingest/api.go | 195 +++++++++ format/packfile/ingest/byteslice_reader.go | 21 + format/packfile/ingest/cache.go | 53 +++ format/packfile/ingest/counting_writer.go | 17 + format/packfile/ingest/crc.go | 22 ++ format/packfile/ingest/delta_header.go | 11 + format/packfile/ingest/distance.go | 30 ++ format/packfile/ingest/doc.go | 3 + format/packfile/ingest/drain.go | 68 ++++ format/packfile/ingest/entry.go | 92 +++++ format/packfile/ingest/entry_header.go | 33 ++ format/packfile/ingest/entry_prefix.go | 95 +++++ format/packfile/ingest/errors.go | 75 ++++ format/packfile/ingest/file_section_writer.go | 22 ++ format/packfile/ingest/fill.go | 44 +++ format/packfile/ingest/finalize.go | 94 +++++ format/packfile/ingest/flush.go | 37 ++ format/packfile/ingest/hash.go | 27 ++ format/packfile/ingest/header.go | 49 +++ format/packfile/ingest/idx_write.go | 266 +++++++++++++ format/packfile/ingest/ingest.go | 68 ++++ format/packfile/ingest/ingest_test.go | 434 +++++++++++++++++++++ format/packfile/ingest/progress_write.go | 11 + format/packfile/ingest/record_content.go | 30 ++ format/packfile/ingest/record_delta.go | 60 +++ format/packfile/ingest/record_inflate.go | 46 +++ format/packfile/ingest/record_resolve.go | 117 ++++++ format/packfile/ingest/records.go | 46 +++ format/packfile/ingest/resolve_all.go | 71 ++++ format/packfile/ingest/rev_write.go | 138 +++++++ format/packfile/ingest/rewrite_header_trailer.go | 89 +++++ format/packfile/ingest/scan.go | 106 +++++ format/packfile/ingest/state.go | 70 ++++ format/packfile/ingest/stream.go | 111 ++++++ format/packfile/ingest/temp.go | 103 +++++ .../ingest/testdata/fixtures/sha1/METADATA.txt | 3 + .../ingest/testdata/fixtures/sha1/base.pack | Bin 0 -> 81007 bytes .../ingest/testdata/fixtures/sha1/nonthin.pack | Bin 0 -> 117458 bytes .../ingest/testdata/fixtures/sha1/thin.pack | Bin 0 -> 38581 bytes .../ingest/testdata/fixtures/sha256/METADATA.txt | 3 + .../ingest/testdata/fixtures/sha256/base.pack | Bin 0 -> 105138 bytes .../ingest/testdata/fixtures/sha256/nonthin.pack | Bin 0 -> 152284 bytes .../ingest/testdata/fixtures/sha256/thin.pack | Bin 0 -> 49412 bytes format/packfile/ingest/thin_append.go | 91 +++++ format/packfile/ingest/thin_fix.go | 100 +++++ format/packfile/ingest/thin_unresolved.go | 34 ++ format/packfile/ingest/trailer.go | 58 +++ format/packfile/ingest/use.go | 34 ++ format/packfile/object_type.go | 16 + format/packfile/ofs.go | 26 ++ internal/testgit/repo_open_commit_graph.go | 2 +- network/receivepack/service/ingest_quarantine.go | 2 +- network/receivepack/service/result.go | 2 +- object/storer/packed/delta_build_chain.go | 2 +- object/storer/packed/delta_resolve_chain.go | 2 +- object/storer/packed/delta_resolve_chain_start.go | 2 +- object/storer/packed/delta_resolve_content.go | 2 +- object/storer/packed/delta_size.go | 2 +- object/storer/packed/entry_parse.go | 2 +- object/storer/packed/pack.go | 2 +- object/storer/packed/read_header_resolve.go | 2 +- object/storer/packed/read_reader.go | 2 +- object/storer/packed/read_size.go | 2 +- packfile/delta/apply/apply.go | 160 -------- packfile/delta/apply/header.go | 47 --- packfile/delta/doc.go | 2 - packfile/doc.go | 2 - packfile/entry.go | 76 ---- packfile/entry_header.go | 52 --- packfile/header.go | 9 - packfile/ingest/api.go | 195 --------- packfile/ingest/byteslice_reader.go | 21 - packfile/ingest/cache.go | 53 --- packfile/ingest/counting_writer.go | 17 - packfile/ingest/crc.go | 22 -- packfile/ingest/delta_header.go | 11 - packfile/ingest/distance.go | 30 -- packfile/ingest/doc.go | 3 - packfile/ingest/drain.go | 68 ---- packfile/ingest/entry.go | 92 ----- packfile/ingest/entry_header.go | 33 -- packfile/ingest/entry_prefix.go | 95 ----- packfile/ingest/errors.go | 75 ---- packfile/ingest/file_section_writer.go | 22 -- packfile/ingest/fill.go | 44 --- packfile/ingest/finalize.go | 94 ----- packfile/ingest/flush.go | 37 -- packfile/ingest/hash.go | 27 -- packfile/ingest/header.go | 49 --- packfile/ingest/idx_write.go | 266 ------------- packfile/ingest/ingest.go | 68 ---- packfile/ingest/ingest_test.go | 434 --------------------- packfile/ingest/progress_write.go | 11 - packfile/ingest/record_content.go | 30 -- packfile/ingest/record_delta.go | 60 --- packfile/ingest/record_inflate.go | 46 --- packfile/ingest/record_resolve.go | 117 ------ packfile/ingest/records.go | 46 --- packfile/ingest/resolve_all.go | 71 ---- packfile/ingest/rev_write.go | 138 ------- packfile/ingest/rewrite_header_trailer.go | 89 ----- packfile/ingest/scan.go | 106 ----- packfile/ingest/state.go | 70 ---- packfile/ingest/stream.go | 111 ------ packfile/ingest/temp.go | 103 ----- .../ingest/testdata/fixtures/sha1/METADATA.txt | 3 - packfile/ingest/testdata/fixtures/sha1/base.pack | Bin 81007 -> 0 bytes .../ingest/testdata/fixtures/sha1/nonthin.pack | Bin 117458 -> 0 bytes packfile/ingest/testdata/fixtures/sha1/thin.pack | Bin 38581 -> 0 bytes .../ingest/testdata/fixtures/sha256/METADATA.txt | 3 - packfile/ingest/testdata/fixtures/sha256/base.pack | Bin 105138 -> 0 bytes .../ingest/testdata/fixtures/sha256/nonthin.pack | Bin 152284 -> 0 bytes packfile/ingest/testdata/fixtures/sha256/thin.pack | Bin 49412 -> 0 bytes packfile/ingest/thin_append.go | 91 ----- packfile/ingest/thin_fix.go | 100 ----- packfile/ingest/thin_unresolved.go | 34 -- packfile/ingest/trailer.go | 58 --- packfile/ingest/use.go | 34 -- packfile/object_type.go | 16 - packfile/ofs.go | 26 -- reachability/reachability.go | 2 +- reachability/walk_expand_commits_graph.go | 2 +- 329 files changed, 5698 insertions(+), 5690 deletions(-) delete mode 100644 commitgraph/TODO delete mode 100644 commitgraph/bloom/bloom.go delete mode 100644 commitgraph/bloom/constants.go delete mode 100644 commitgraph/bloom/contain.go delete mode 100644 commitgraph/bloom/errors.go delete mode 100644 commitgraph/bloom/filter.go delete mode 100644 commitgraph/bloom/key.go delete mode 100644 commitgraph/bloom/murmur.go delete mode 100644 commitgraph/bloom/settings.go delete mode 100644 commitgraph/constants.go delete mode 100644 commitgraph/doc.go delete mode 100644 commitgraph/read/bloom.go delete mode 100644 commitgraph/read/close.go delete mode 100644 commitgraph/read/commitat.go delete mode 100644 commitgraph/read/commits.go delete mode 100644 commitgraph/read/doc.go delete mode 100644 commitgraph/read/edges.go delete mode 100644 commitgraph/read/errors.go delete mode 100644 commitgraph/read/generation.go delete mode 100644 commitgraph/read/hash.go delete mode 100644 commitgraph/read/iterators.go delete mode 100644 commitgraph/read/layer.go delete mode 100644 commitgraph/read/layer_close.go delete mode 100644 commitgraph/read/layer_lookup.go delete mode 100644 commitgraph/read/layer_open.go delete mode 100644 commitgraph/read/layer_parse.go delete mode 100644 commitgraph/read/layer_pos.go delete mode 100644 commitgraph/read/layerinfo.go delete mode 100644 commitgraph/read/lookup.go delete mode 100644 commitgraph/read/mode.go delete mode 100644 commitgraph/read/oidat.go delete mode 100644 commitgraph/read/open.go delete mode 100644 commitgraph/read/open_chain.go delete mode 100644 commitgraph/read/open_single.go delete mode 100644 commitgraph/read/parents.go delete mode 100644 commitgraph/read/position.go delete mode 100644 commitgraph/read/read_test.go delete mode 100644 commitgraph/read/reader.go delete mode 100644 commitgraph/read/testdata/fixtures/sha1/chain_changed/repo.git/HEAD delete mode 100644 commitgraph/read/testdata/fixtures/sha1/chain_changed/repo.git/config delete mode 100644 commitgraph/read/testdata/fixtures/sha1/chain_changed/repo.git/objects/info/commit-graphs/commit-graph-chain delete mode 100644 commitgraph/read/testdata/fixtures/sha1/chain_changed/repo.git/objects/info/commit-graphs/graph-bf985c21612a52070d8b008e6ef51edf8b609401.graph delete mode 100644 commitgraph/read/testdata/fixtures/sha1/chain_changed/repo.git/objects/info/commit-graphs/graph-dd7578d5216ca76c25b19631ba90f7498aeabbe7.graph delete mode 100644 commitgraph/read/testdata/fixtures/sha1/chain_changed/repo.git/objects/info/packs delete mode 100644 commitgraph/read/testdata/fixtures/sha1/chain_changed/repo.git/objects/pack/pack-15b064d6a8ef8cff520565f6db8c006b2e6f7f2f.bitmap delete mode 100644 commitgraph/read/testdata/fixtures/sha1/chain_changed/repo.git/objects/pack/pack-15b064d6a8ef8cff520565f6db8c006b2e6f7f2f.idx delete mode 100644 commitgraph/read/testdata/fixtures/sha1/chain_changed/repo.git/objects/pack/pack-15b064d6a8ef8cff520565f6db8c006b2e6f7f2f.pack delete mode 100644 commitgraph/read/testdata/fixtures/sha1/chain_changed/repo.git/objects/pack/pack-15b064d6a8ef8cff520565f6db8c006b2e6f7f2f.rev delete mode 100644 commitgraph/read/testdata/fixtures/sha1/chain_changed/repo.git/refs/heads/master delete mode 100644 commitgraph/read/testdata/fixtures/sha1/single_changed/repo.git/HEAD delete mode 100644 commitgraph/read/testdata/fixtures/sha1/single_changed/repo.git/config delete mode 100644 commitgraph/read/testdata/fixtures/sha1/single_changed/repo.git/objects/info/commit-graph delete mode 100644 commitgraph/read/testdata/fixtures/sha1/single_changed/repo.git/objects/info/packs delete mode 100644 commitgraph/read/testdata/fixtures/sha1/single_changed/repo.git/objects/pack/pack-34e9e132566989e2abfe8821731236c77f9bcbe9.bitmap delete mode 100644 commitgraph/read/testdata/fixtures/sha1/single_changed/repo.git/objects/pack/pack-34e9e132566989e2abfe8821731236c77f9bcbe9.idx delete mode 100644 commitgraph/read/testdata/fixtures/sha1/single_changed/repo.git/objects/pack/pack-34e9e132566989e2abfe8821731236c77f9bcbe9.pack delete mode 100644 commitgraph/read/testdata/fixtures/sha1/single_changed/repo.git/objects/pack/pack-34e9e132566989e2abfe8821731236c77f9bcbe9.rev delete mode 100644 commitgraph/read/testdata/fixtures/sha1/single_changed/repo.git/refs/heads/main delete mode 100644 commitgraph/read/testdata/fixtures/sha1/single_nochanged/repo.git/HEAD delete mode 100644 commitgraph/read/testdata/fixtures/sha1/single_nochanged/repo.git/config delete mode 100644 commitgraph/read/testdata/fixtures/sha1/single_nochanged/repo.git/objects/info/commit-graph delete mode 100644 commitgraph/read/testdata/fixtures/sha1/single_nochanged/repo.git/objects/info/packs delete mode 100644 commitgraph/read/testdata/fixtures/sha1/single_nochanged/repo.git/objects/pack/pack-a3da595034c94bb16b6829d757a66b7d259b9ffc.bitmap delete mode 100644 commitgraph/read/testdata/fixtures/sha1/single_nochanged/repo.git/objects/pack/pack-a3da595034c94bb16b6829d757a66b7d259b9ffc.idx delete mode 100644 commitgraph/read/testdata/fixtures/sha1/single_nochanged/repo.git/objects/pack/pack-a3da595034c94bb16b6829d757a66b7d259b9ffc.pack delete mode 100644 commitgraph/read/testdata/fixtures/sha1/single_nochanged/repo.git/objects/pack/pack-a3da595034c94bb16b6829d757a66b7d259b9ffc.rev delete mode 100644 commitgraph/read/testdata/fixtures/sha1/single_nochanged/repo.git/refs/heads/master delete mode 100644 commitgraph/read/testdata/fixtures/sha256/chain_changed/repo.git/HEAD delete mode 100644 commitgraph/read/testdata/fixtures/sha256/chain_changed/repo.git/config delete mode 100644 commitgraph/read/testdata/fixtures/sha256/chain_changed/repo.git/objects/info/commit-graphs/commit-graph-chain delete mode 100644 commitgraph/read/testdata/fixtures/sha256/chain_changed/repo.git/objects/info/commit-graphs/graph-505cab61f8ddfa614301e8f97943112739236c6bcd19ed4d1f7c6b830cab4f62.graph delete mode 100644 commitgraph/read/testdata/fixtures/sha256/chain_changed/repo.git/objects/info/commit-graphs/graph-77c47bd6ca2ce17208c9361717a5823c0cb4b5ee336a14959678e060d674ffb6.graph delete mode 100644 commitgraph/read/testdata/fixtures/sha256/chain_changed/repo.git/objects/info/packs delete mode 100644 commitgraph/read/testdata/fixtures/sha256/chain_changed/repo.git/objects/pack/pack-04168d0884c910f505cb9fbcf045957e44ccee06d812b5e531ae666014a26ed1.bitmap delete mode 100644 commitgraph/read/testdata/fixtures/sha256/chain_changed/repo.git/objects/pack/pack-04168d0884c910f505cb9fbcf045957e44ccee06d812b5e531ae666014a26ed1.idx delete mode 100644 commitgraph/read/testdata/fixtures/sha256/chain_changed/repo.git/objects/pack/pack-04168d0884c910f505cb9fbcf045957e44ccee06d812b5e531ae666014a26ed1.pack delete mode 100644 commitgraph/read/testdata/fixtures/sha256/chain_changed/repo.git/objects/pack/pack-04168d0884c910f505cb9fbcf045957e44ccee06d812b5e531ae666014a26ed1.rev delete mode 100644 commitgraph/read/testdata/fixtures/sha256/chain_changed/repo.git/refs/heads/master delete mode 100644 commitgraph/read/testdata/fixtures/sha256/single_changed/repo.git/HEAD delete mode 100644 commitgraph/read/testdata/fixtures/sha256/single_changed/repo.git/config delete mode 100644 commitgraph/read/testdata/fixtures/sha256/single_changed/repo.git/objects/info/commit-graph delete mode 100644 commitgraph/read/testdata/fixtures/sha256/single_changed/repo.git/objects/info/packs delete mode 100644 commitgraph/read/testdata/fixtures/sha256/single_changed/repo.git/objects/pack/pack-316dbc67dac12d131591640da0c55b76387cbf1fd2a117ab3d7ca0d854a031c9.bitmap delete mode 100644 commitgraph/read/testdata/fixtures/sha256/single_changed/repo.git/objects/pack/pack-316dbc67dac12d131591640da0c55b76387cbf1fd2a117ab3d7ca0d854a031c9.idx delete mode 100644 commitgraph/read/testdata/fixtures/sha256/single_changed/repo.git/objects/pack/pack-316dbc67dac12d131591640da0c55b76387cbf1fd2a117ab3d7ca0d854a031c9.pack delete mode 100644 commitgraph/read/testdata/fixtures/sha256/single_changed/repo.git/objects/pack/pack-316dbc67dac12d131591640da0c55b76387cbf1fd2a117ab3d7ca0d854a031c9.rev delete mode 100644 commitgraph/read/testdata/fixtures/sha256/single_changed/repo.git/refs/heads/main delete mode 100644 commitgraph/read/testdata/fixtures/sha256/single_nochanged/repo.git/HEAD delete mode 100644 commitgraph/read/testdata/fixtures/sha256/single_nochanged/repo.git/config delete mode 100644 commitgraph/read/testdata/fixtures/sha256/single_nochanged/repo.git/objects/info/commit-graph delete mode 100644 commitgraph/read/testdata/fixtures/sha256/single_nochanged/repo.git/objects/info/packs delete mode 100644 commitgraph/read/testdata/fixtures/sha256/single_nochanged/repo.git/objects/pack/pack-d335453f760b064e36459d780ec9bf0e5dd596c0ee1ac6310136067c4f13438b.bitmap delete mode 100644 commitgraph/read/testdata/fixtures/sha256/single_nochanged/repo.git/objects/pack/pack-d335453f760b064e36459d780ec9bf0e5dd596c0ee1ac6310136067c4f13438b.idx delete mode 100644 commitgraph/read/testdata/fixtures/sha256/single_nochanged/repo.git/objects/pack/pack-d335453f760b064e36459d780ec9bf0e5dd596c0ee1ac6310136067c4f13438b.pack delete mode 100644 commitgraph/read/testdata/fixtures/sha256/single_nochanged/repo.git/objects/pack/pack-d335453f760b064e36459d780ec9bf0e5dd596c0ee1ac6310136067c4f13438b.rev delete mode 100644 commitgraph/read/testdata/fixtures/sha256/single_nochanged/repo.git/refs/heads/master create mode 100644 format/commitgraph/TODO create mode 100644 format/commitgraph/bloom/bloom.go create mode 100644 format/commitgraph/bloom/constants.go create mode 100644 format/commitgraph/bloom/contain.go create mode 100644 format/commitgraph/bloom/errors.go create mode 100644 format/commitgraph/bloom/filter.go create mode 100644 format/commitgraph/bloom/key.go create mode 100644 format/commitgraph/bloom/murmur.go create mode 100644 format/commitgraph/bloom/settings.go create mode 100644 format/commitgraph/constants.go create mode 100644 format/commitgraph/doc.go create mode 100644 format/commitgraph/read/bloom.go create mode 100644 format/commitgraph/read/close.go create mode 100644 format/commitgraph/read/commitat.go create mode 100644 format/commitgraph/read/commits.go create mode 100644 format/commitgraph/read/doc.go create mode 100644 format/commitgraph/read/edges.go create mode 100644 format/commitgraph/read/errors.go create mode 100644 format/commitgraph/read/generation.go create mode 100644 format/commitgraph/read/hash.go create mode 100644 format/commitgraph/read/iterators.go create mode 100644 format/commitgraph/read/layer.go create mode 100644 format/commitgraph/read/layer_close.go create mode 100644 format/commitgraph/read/layer_lookup.go create mode 100644 format/commitgraph/read/layer_open.go create mode 100644 format/commitgraph/read/layer_parse.go create mode 100644 format/commitgraph/read/layer_pos.go create mode 100644 format/commitgraph/read/layerinfo.go create mode 100644 format/commitgraph/read/lookup.go create mode 100644 format/commitgraph/read/mode.go create mode 100644 format/commitgraph/read/oidat.go create mode 100644 format/commitgraph/read/open.go create mode 100644 format/commitgraph/read/open_chain.go create mode 100644 format/commitgraph/read/open_single.go create mode 100644 format/commitgraph/read/parents.go create mode 100644 format/commitgraph/read/position.go create mode 100644 format/commitgraph/read/read_test.go create mode 100644 format/commitgraph/read/reader.go create mode 100644 format/commitgraph/read/testdata/fixtures/sha1/chain_changed/repo.git/HEAD create mode 100644 format/commitgraph/read/testdata/fixtures/sha1/chain_changed/repo.git/config create mode 100644 format/commitgraph/read/testdata/fixtures/sha1/chain_changed/repo.git/objects/info/commit-graphs/commit-graph-chain create mode 100644 format/commitgraph/read/testdata/fixtures/sha1/chain_changed/repo.git/objects/info/commit-graphs/graph-bf985c21612a52070d8b008e6ef51edf8b609401.graph create mode 100644 format/commitgraph/read/testdata/fixtures/sha1/chain_changed/repo.git/objects/info/commit-graphs/graph-dd7578d5216ca76c25b19631ba90f7498aeabbe7.graph create mode 100644 format/commitgraph/read/testdata/fixtures/sha1/chain_changed/repo.git/objects/info/packs create mode 100644 format/commitgraph/read/testdata/fixtures/sha1/chain_changed/repo.git/objects/pack/pack-15b064d6a8ef8cff520565f6db8c006b2e6f7f2f.bitmap create mode 100644 format/commitgraph/read/testdata/fixtures/sha1/chain_changed/repo.git/objects/pack/pack-15b064d6a8ef8cff520565f6db8c006b2e6f7f2f.idx create mode 100644 format/commitgraph/read/testdata/fixtures/sha1/chain_changed/repo.git/objects/pack/pack-15b064d6a8ef8cff520565f6db8c006b2e6f7f2f.pack create mode 100644 format/commitgraph/read/testdata/fixtures/sha1/chain_changed/repo.git/objects/pack/pack-15b064d6a8ef8cff520565f6db8c006b2e6f7f2f.rev create mode 100644 format/commitgraph/read/testdata/fixtures/sha1/chain_changed/repo.git/refs/heads/master create mode 100644 format/commitgraph/read/testdata/fixtures/sha1/single_changed/repo.git/HEAD create mode 100644 format/commitgraph/read/testdata/fixtures/sha1/single_changed/repo.git/config create mode 100644 format/commitgraph/read/testdata/fixtures/sha1/single_changed/repo.git/objects/info/commit-graph create mode 100644 format/commitgraph/read/testdata/fixtures/sha1/single_changed/repo.git/objects/info/packs create mode 100644 format/commitgraph/read/testdata/fixtures/sha1/single_changed/repo.git/objects/pack/pack-34e9e132566989e2abfe8821731236c77f9bcbe9.bitmap create mode 100644 format/commitgraph/read/testdata/fixtures/sha1/single_changed/repo.git/objects/pack/pack-34e9e132566989e2abfe8821731236c77f9bcbe9.idx create mode 100644 format/commitgraph/read/testdata/fixtures/sha1/single_changed/repo.git/objects/pack/pack-34e9e132566989e2abfe8821731236c77f9bcbe9.pack create mode 100644 format/commitgraph/read/testdata/fixtures/sha1/single_changed/repo.git/objects/pack/pack-34e9e132566989e2abfe8821731236c77f9bcbe9.rev create mode 100644 format/commitgraph/read/testdata/fixtures/sha1/single_changed/repo.git/refs/heads/main create mode 100644 format/commitgraph/read/testdata/fixtures/sha1/single_nochanged/repo.git/HEAD create mode 100644 format/commitgraph/read/testdata/fixtures/sha1/single_nochanged/repo.git/config create mode 100644 format/commitgraph/read/testdata/fixtures/sha1/single_nochanged/repo.git/objects/info/commit-graph create mode 100644 format/commitgraph/read/testdata/fixtures/sha1/single_nochanged/repo.git/objects/info/packs create mode 100644 format/commitgraph/read/testdata/fixtures/sha1/single_nochanged/repo.git/objects/pack/pack-a3da595034c94bb16b6829d757a66b7d259b9ffc.bitmap create mode 100644 format/commitgraph/read/testdata/fixtures/sha1/single_nochanged/repo.git/objects/pack/pack-a3da595034c94bb16b6829d757a66b7d259b9ffc.idx create mode 100644 format/commitgraph/read/testdata/fixtures/sha1/single_nochanged/repo.git/objects/pack/pack-a3da595034c94bb16b6829d757a66b7d259b9ffc.pack create mode 100644 format/commitgraph/read/testdata/fixtures/sha1/single_nochanged/repo.git/objects/pack/pack-a3da595034c94bb16b6829d757a66b7d259b9ffc.rev create mode 100644 format/commitgraph/read/testdata/fixtures/sha1/single_nochanged/repo.git/refs/heads/master create mode 100644 format/commitgraph/read/testdata/fixtures/sha256/chain_changed/repo.git/HEAD create mode 100644 format/commitgraph/read/testdata/fixtures/sha256/chain_changed/repo.git/config create mode 100644 format/commitgraph/read/testdata/fixtures/sha256/chain_changed/repo.git/objects/info/commit-graphs/commit-graph-chain create mode 100644 format/commitgraph/read/testdata/fixtures/sha256/chain_changed/repo.git/objects/info/commit-graphs/graph-505cab61f8ddfa614301e8f97943112739236c6bcd19ed4d1f7c6b830cab4f62.graph create mode 100644 format/commitgraph/read/testdata/fixtures/sha256/chain_changed/repo.git/objects/info/commit-graphs/graph-77c47bd6ca2ce17208c9361717a5823c0cb4b5ee336a14959678e060d674ffb6.graph create mode 100644 format/commitgraph/read/testdata/fixtures/sha256/chain_changed/repo.git/objects/info/packs create mode 100644 format/commitgraph/read/testdata/fixtures/sha256/chain_changed/repo.git/objects/pack/pack-04168d0884c910f505cb9fbcf045957e44ccee06d812b5e531ae666014a26ed1.bitmap create mode 100644 format/commitgraph/read/testdata/fixtures/sha256/chain_changed/repo.git/objects/pack/pack-04168d0884c910f505cb9fbcf045957e44ccee06d812b5e531ae666014a26ed1.idx create mode 100644 format/commitgraph/read/testdata/fixtures/sha256/chain_changed/repo.git/objects/pack/pack-04168d0884c910f505cb9fbcf045957e44ccee06d812b5e531ae666014a26ed1.pack create mode 100644 format/commitgraph/read/testdata/fixtures/sha256/chain_changed/repo.git/objects/pack/pack-04168d0884c910f505cb9fbcf045957e44ccee06d812b5e531ae666014a26ed1.rev create mode 100644 format/commitgraph/read/testdata/fixtures/sha256/chain_changed/repo.git/refs/heads/master create mode 100644 format/commitgraph/read/testdata/fixtures/sha256/single_changed/repo.git/HEAD create mode 100644 format/commitgraph/read/testdata/fixtures/sha256/single_changed/repo.git/config create mode 100644 format/commitgraph/read/testdata/fixtures/sha256/single_changed/repo.git/objects/info/commit-graph create mode 100644 format/commitgraph/read/testdata/fixtures/sha256/single_changed/repo.git/objects/info/packs create mode 100644 format/commitgraph/read/testdata/fixtures/sha256/single_changed/repo.git/objects/pack/pack-316dbc67dac12d131591640da0c55b76387cbf1fd2a117ab3d7ca0d854a031c9.bitmap create mode 100644 format/commitgraph/read/testdata/fixtures/sha256/single_changed/repo.git/objects/pack/pack-316dbc67dac12d131591640da0c55b76387cbf1fd2a117ab3d7ca0d854a031c9.idx create mode 100644 format/commitgraph/read/testdata/fixtures/sha256/single_changed/repo.git/objects/pack/pack-316dbc67dac12d131591640da0c55b76387cbf1fd2a117ab3d7ca0d854a031c9.pack create mode 100644 format/commitgraph/read/testdata/fixtures/sha256/single_changed/repo.git/objects/pack/pack-316dbc67dac12d131591640da0c55b76387cbf1fd2a117ab3d7ca0d854a031c9.rev create mode 100644 format/commitgraph/read/testdata/fixtures/sha256/single_changed/repo.git/refs/heads/main create mode 100644 format/commitgraph/read/testdata/fixtures/sha256/single_nochanged/repo.git/HEAD create mode 100644 format/commitgraph/read/testdata/fixtures/sha256/single_nochanged/repo.git/config create mode 100644 format/commitgraph/read/testdata/fixtures/sha256/single_nochanged/repo.git/objects/info/commit-graph create mode 100644 format/commitgraph/read/testdata/fixtures/sha256/single_nochanged/repo.git/objects/info/packs create mode 100644 format/commitgraph/read/testdata/fixtures/sha256/single_nochanged/repo.git/objects/pack/pack-d335453f760b064e36459d780ec9bf0e5dd596c0ee1ac6310136067c4f13438b.bitmap create mode 100644 format/commitgraph/read/testdata/fixtures/sha256/single_nochanged/repo.git/objects/pack/pack-d335453f760b064e36459d780ec9bf0e5dd596c0ee1ac6310136067c4f13438b.idx create mode 100644 format/commitgraph/read/testdata/fixtures/sha256/single_nochanged/repo.git/objects/pack/pack-d335453f760b064e36459d780ec9bf0e5dd596c0ee1ac6310136067c4f13438b.pack create mode 100644 format/commitgraph/read/testdata/fixtures/sha256/single_nochanged/repo.git/objects/pack/pack-d335453f760b064e36459d780ec9bf0e5dd596c0ee1ac6310136067c4f13438b.rev create mode 100644 format/commitgraph/read/testdata/fixtures/sha256/single_nochanged/repo.git/refs/heads/master create mode 100644 format/doc.go create mode 100644 format/packfile/delta/apply/apply.go create mode 100644 format/packfile/delta/apply/header.go create mode 100644 format/packfile/delta/doc.go create mode 100644 format/packfile/doc.go create mode 100644 format/packfile/entry.go create mode 100644 format/packfile/entry_header.go create mode 100644 format/packfile/header.go create mode 100644 format/packfile/ingest/api.go create mode 100644 format/packfile/ingest/byteslice_reader.go create mode 100644 format/packfile/ingest/cache.go create mode 100644 format/packfile/ingest/counting_writer.go create mode 100644 format/packfile/ingest/crc.go create mode 100644 format/packfile/ingest/delta_header.go create mode 100644 format/packfile/ingest/distance.go create mode 100644 format/packfile/ingest/doc.go create mode 100644 format/packfile/ingest/drain.go create mode 100644 format/packfile/ingest/entry.go create mode 100644 format/packfile/ingest/entry_header.go create mode 100644 format/packfile/ingest/entry_prefix.go create mode 100644 format/packfile/ingest/errors.go create mode 100644 format/packfile/ingest/file_section_writer.go create mode 100644 format/packfile/ingest/fill.go create mode 100644 format/packfile/ingest/finalize.go create mode 100644 format/packfile/ingest/flush.go create mode 100644 format/packfile/ingest/hash.go create mode 100644 format/packfile/ingest/header.go create mode 100644 format/packfile/ingest/idx_write.go create mode 100644 format/packfile/ingest/ingest.go create mode 100644 format/packfile/ingest/ingest_test.go create mode 100644 format/packfile/ingest/progress_write.go create mode 100644 format/packfile/ingest/record_content.go create mode 100644 format/packfile/ingest/record_delta.go create mode 100644 format/packfile/ingest/record_inflate.go create mode 100644 format/packfile/ingest/record_resolve.go create mode 100644 format/packfile/ingest/records.go create mode 100644 format/packfile/ingest/resolve_all.go create mode 100644 format/packfile/ingest/rev_write.go create mode 100644 format/packfile/ingest/rewrite_header_trailer.go create mode 100644 format/packfile/ingest/scan.go create mode 100644 format/packfile/ingest/state.go create mode 100644 format/packfile/ingest/stream.go create mode 100644 format/packfile/ingest/temp.go create mode 100644 format/packfile/ingest/testdata/fixtures/sha1/METADATA.txt create mode 100644 format/packfile/ingest/testdata/fixtures/sha1/base.pack create mode 100644 format/packfile/ingest/testdata/fixtures/sha1/nonthin.pack create mode 100644 format/packfile/ingest/testdata/fixtures/sha1/thin.pack create mode 100644 format/packfile/ingest/testdata/fixtures/sha256/METADATA.txt create mode 100644 format/packfile/ingest/testdata/fixtures/sha256/base.pack create mode 100644 format/packfile/ingest/testdata/fixtures/sha256/nonthin.pack create mode 100644 format/packfile/ingest/testdata/fixtures/sha256/thin.pack create mode 100644 format/packfile/ingest/thin_append.go create mode 100644 format/packfile/ingest/thin_fix.go create mode 100644 format/packfile/ingest/thin_unresolved.go create mode 100644 format/packfile/ingest/trailer.go create mode 100644 format/packfile/ingest/use.go create mode 100644 format/packfile/object_type.go create mode 100644 format/packfile/ofs.go delete mode 100644 packfile/delta/apply/apply.go delete mode 100644 packfile/delta/apply/header.go delete mode 100644 packfile/delta/doc.go delete mode 100644 packfile/doc.go delete mode 100644 packfile/entry.go delete mode 100644 packfile/entry_header.go delete mode 100644 packfile/header.go delete mode 100644 packfile/ingest/api.go delete mode 100644 packfile/ingest/byteslice_reader.go delete mode 100644 packfile/ingest/cache.go delete mode 100644 packfile/ingest/counting_writer.go delete mode 100644 packfile/ingest/crc.go delete mode 100644 packfile/ingest/delta_header.go delete mode 100644 packfile/ingest/distance.go delete mode 100644 packfile/ingest/doc.go delete mode 100644 packfile/ingest/drain.go delete mode 100644 packfile/ingest/entry.go delete mode 100644 packfile/ingest/entry_header.go delete mode 100644 packfile/ingest/entry_prefix.go delete mode 100644 packfile/ingest/errors.go delete mode 100644 packfile/ingest/file_section_writer.go delete mode 100644 packfile/ingest/fill.go delete mode 100644 packfile/ingest/finalize.go delete mode 100644 packfile/ingest/flush.go delete mode 100644 packfile/ingest/hash.go delete mode 100644 packfile/ingest/header.go delete mode 100644 packfile/ingest/idx_write.go delete mode 100644 packfile/ingest/ingest.go delete mode 100644 packfile/ingest/ingest_test.go delete mode 100644 packfile/ingest/progress_write.go delete mode 100644 packfile/ingest/record_content.go delete mode 100644 packfile/ingest/record_delta.go delete mode 100644 packfile/ingest/record_inflate.go delete mode 100644 packfile/ingest/record_resolve.go delete mode 100644 packfile/ingest/records.go delete mode 100644 packfile/ingest/resolve_all.go delete mode 100644 packfile/ingest/rev_write.go delete mode 100644 packfile/ingest/rewrite_header_trailer.go delete mode 100644 packfile/ingest/scan.go delete mode 100644 packfile/ingest/state.go delete mode 100644 packfile/ingest/stream.go delete mode 100644 packfile/ingest/temp.go delete mode 100644 packfile/ingest/testdata/fixtures/sha1/METADATA.txt delete mode 100644 packfile/ingest/testdata/fixtures/sha1/base.pack delete mode 100644 packfile/ingest/testdata/fixtures/sha1/nonthin.pack delete mode 100644 packfile/ingest/testdata/fixtures/sha1/thin.pack delete mode 100644 packfile/ingest/testdata/fixtures/sha256/METADATA.txt delete mode 100644 packfile/ingest/testdata/fixtures/sha256/base.pack delete mode 100644 packfile/ingest/testdata/fixtures/sha256/nonthin.pack delete mode 100644 packfile/ingest/testdata/fixtures/sha256/thin.pack delete mode 100644 packfile/ingest/thin_append.go delete mode 100644 packfile/ingest/thin_fix.go delete mode 100644 packfile/ingest/thin_unresolved.go delete mode 100644 packfile/ingest/trailer.go delete mode 100644 packfile/ingest/use.go delete mode 100644 packfile/object_type.go delete mode 100644 packfile/ofs.go diff --git a/cmd/index-pack/main.go b/cmd/index-pack/main.go index 69cf648a..30de3917 100644 --- a/cmd/index-pack/main.go +++ b/cmd/index-pack/main.go @@ -10,7 +10,7 @@ import ( objectid "codeberg.org/lindenii/furgit/object/id" objectstorer "codeberg.org/lindenii/furgit/object/storer" - "codeberg.org/lindenii/furgit/packfile/ingest" + "codeberg.org/lindenii/furgit/format/packfile/ingest" "codeberg.org/lindenii/furgit/repository" ) diff --git a/commitgraph/TODO b/commitgraph/TODO deleted file mode 100644 index 87e0888d..00000000 --- a/commitgraph/TODO +++ /dev/null @@ -1,6 +0,0 @@ -Paranoia mode -Split commit-graph chain with mixed generation and bloom setting -Separate chunk parsing layer -Config stuff - -Writing diff --git a/commitgraph/bloom/bloom.go b/commitgraph/bloom/bloom.go deleted file mode 100644 index 9653d595..00000000 --- a/commitgraph/bloom/bloom.go +++ /dev/null @@ -1,3 +0,0 @@ -// Package bloom provides a bloom filter implementation used for changed-path -// filters in Git commit graphs. -package bloom diff --git a/commitgraph/bloom/constants.go b/commitgraph/bloom/constants.go deleted file mode 100644 index 958e551e..00000000 --- a/commitgraph/bloom/constants.go +++ /dev/null @@ -1,8 +0,0 @@ -package bloom - -const ( - // DataHeaderSize is the size of the BDAT header in commit-graph files. - DataHeaderSize = 3 * 4 - // DefaultMaxChange matches Git's default max-changed-paths behavior. - DefaultMaxChange = 512 -) diff --git a/commitgraph/bloom/contain.go b/commitgraph/bloom/contain.go deleted file mode 100644 index 331b7687..00000000 --- a/commitgraph/bloom/contain.go +++ /dev/null @@ -1,25 +0,0 @@ -package bloom - -// MightContain reports whether the Bloom filter may contain the given path. -// -// Evaluated against the full path and each of its directory prefixes. A true -// result indicates a possible match; false means the path definitely did not -// change. -func (f *Filter) MightContain(path []byte) (bool, error) { - if len(f.Data) == 0 { - return false, nil - } - - keys, err := keyvec(path, f) - if err != nil { - return false, err - } - - for i := range keys { - if filterContainsKey(f, keys[i]) { - return true, nil - } - } - - return false, nil -} diff --git a/commitgraph/bloom/errors.go b/commitgraph/bloom/errors.go deleted file mode 100644 index fe38d1bc..00000000 --- a/commitgraph/bloom/errors.go +++ /dev/null @@ -1,5 +0,0 @@ -package bloom - -import "errors" - -var ErrInvalid = errors.New("bloom: invalid data") diff --git a/commitgraph/bloom/filter.go b/commitgraph/bloom/filter.go deleted file mode 100644 index 395dd5ce..00000000 --- a/commitgraph/bloom/filter.go +++ /dev/null @@ -1,26 +0,0 @@ -package bloom - -// Filter represents a changed-paths Bloom filter associated with a commit. -// -// The filter encodes which paths changed between a commit and its first -// parent. Paths are expected to be in Git's slash-separated form and -// are queried using a path and its prefixes (e.g. "a/b/c", "a/b", "a"). -type Filter struct { - Data []byte - - HashVersion uint32 - NumHashes uint32 - BitsPerEntry uint32 - MaxChangePaths uint32 -} - -// NewFilter constructs one query-ready bloom filter from raw data/settings. -func NewFilter(data []byte, settings Settings) Filter { - return Filter{ - Data: data, - HashVersion: settings.HashVersion, - NumHashes: settings.NumHashes, - BitsPerEntry: settings.BitsPerEntry, - MaxChangePaths: settings.MaxChangePaths, - } -} diff --git a/commitgraph/bloom/key.go b/commitgraph/bloom/key.go deleted file mode 100644 index a15df904..00000000 --- a/commitgraph/bloom/key.go +++ /dev/null @@ -1,117 +0,0 @@ -package bloom - -import "codeberg.org/lindenii/furgit/internal/intconv" - -type key struct { - hashes []uint32 -} - -func keyvec(path []byte, filter *Filter) ([]key, error) { - if len(path) == 0 { - return nil, nil - } - - count := 1 - - for _, b := range path { - if b == '/' { - count++ - } - } - - keys := make([]key, 0, count) - - full, err := keyFill(path, filter) - if err != nil { - return nil, err - } - - keys = append(keys, full) - - for i := len(path) - 1; i >= 0; i-- { - if path[i] == '/' { - k, err := keyFill(path[:i], filter) - if err != nil { - return nil, err - } - - keys = append(keys, k) - } - } - - return keys, nil -} - -func keyFill(path []byte, filter *Filter) (key, error) { - const ( - seed0 = 0x293ae76f - seed1 = 0x7e646e2c - ) - - var ( - h0 uint32 - h1 uint32 - err error - ) - - switch filter.HashVersion { - case 2: - h0, err = murmur3SeededV2(seed0, path) - if err != nil { - return key{}, err - } - - h1, err = murmur3SeededV2(seed1, path) - if err != nil { - return key{}, err - } - case 1: - h0, err = murmur3SeededV1(seed0, path) - if err != nil { - return key{}, err - } - - h1, err = murmur3SeededV1(seed1, path) - if err != nil { - return key{}, err - } - default: - return key{}, ErrInvalid - } - - hashCount, err := intconv.Uint32ToInt(filter.NumHashes) - if err != nil { - return key{}, ErrInvalid - } - - hashes := make([]uint32, hashCount) - for i := range hashCount { - iU32, err := intconv.IntToUint32(i) - if err != nil { - return key{}, ErrInvalid - } - - hashes[i] = h0 + iU32*h1 - } - - return key{hashes: hashes}, nil -} - -func filterContainsKey(filter *Filter, key key) bool { - if len(filter.Data) == 0 { - return false - } - - mod := uint64(len(filter.Data)) * 8 - for _, h := range key.hashes { - idx := uint64(h) % mod - bytePos := idx / 8 - - bit := byte(1 << (idx & 7)) - if filter.Data[bytePos]&bit == 0 { - return false - } - } - - return true -} diff --git a/commitgraph/bloom/murmur.go b/commitgraph/bloom/murmur.go deleted file mode 100644 index 363b63ae..00000000 --- a/commitgraph/bloom/murmur.go +++ /dev/null @@ -1,127 +0,0 @@ -package bloom - -import "codeberg.org/lindenii/furgit/internal/intconv" - -func murmur3SeededV2(seed uint32, data []byte) (uint32, error) { - const ( - c1 = 0xcc9e2d51 - c2 = 0x1b873593 - r1 = 15 - r2 = 13 - m = 5 - n = 0xe6546b64 - ) - - h := seed - - nblocks := len(data) / 4 - for i := range nblocks { - k := uint32(data[4*i]) | - (uint32(data[4*i+1]) << 8) | - (uint32(data[4*i+2]) << 16) | - (uint32(data[4*i+3]) << 24) - k *= c1 - k = (k << r1) | (k >> (32 - r1)) - k *= c2 - - h ^= k - h = (h << r2) | (h >> (32 - r2)) - h = h*m + n - } - - var k1 uint32 - - tail := data[nblocks*4:] - switch len(tail) & 3 { - case 3: - k1 ^= uint32(tail[2]) << 16 - - fallthrough - case 2: - k1 ^= uint32(tail[1]) << 8 - - fallthrough - case 1: - k1 ^= uint32(tail[0]) - k1 *= c1 - k1 = (k1 << r1) | (k1 >> (32 - r1)) - k1 *= c2 - h ^= k1 - } - - dataLen, err := intconv.IntToUint32(len(data)) - if err != nil { - return 0, err - } - - h ^= dataLen - h ^= h >> 16 - h *= 0x85ebca6b - h ^= h >> 13 - h *= 0xc2b2ae35 - h ^= h >> 16 - - return h, nil -} - -func murmur3SeededV1(seed uint32, data []byte) (uint32, error) { - const ( - c1 = 0xcc9e2d51 - c2 = 0x1b873593 - r1 = 15 - r2 = 13 - m = 5 - n = 0xe6546b64 - ) - - h := seed - - nblocks := len(data) / 4 - for i := range nblocks { - k := intconv.SignExtendByteToUint32(data[4*i]) | - (intconv.SignExtendByteToUint32(data[4*i+1]) << 8) | - (intconv.SignExtendByteToUint32(data[4*i+2]) << 16) | - (intconv.SignExtendByteToUint32(data[4*i+3]) << 24) - k *= c1 - k = (k << r1) | (k >> (32 - r1)) - k *= c2 - - h ^= k - h = (h << r2) | (h >> (32 - r2)) - h = h*m + n - } - - var k1 uint32 - - tail := data[nblocks*4:] - switch len(tail) & 3 { - case 3: - k1 ^= intconv.SignExtendByteToUint32(tail[2]) << 16 - - fallthrough - case 2: - k1 ^= intconv.SignExtendByteToUint32(tail[1]) << 8 - - fallthrough - case 1: - k1 ^= intconv.SignExtendByteToUint32(tail[0]) - k1 *= c1 - k1 = (k1 << r1) | (k1 >> (32 - r1)) - k1 *= c2 - h ^= k1 - } - - dataLen, err := intconv.IntToUint32(len(data)) - if err != nil { - return 0, err - } - - h ^= dataLen - h ^= h >> 16 - h *= 0x85ebca6b - h ^= h >> 13 - h *= 0xc2b2ae35 - h ^= h >> 16 - - return h, nil -} diff --git a/commitgraph/bloom/settings.go b/commitgraph/bloom/settings.go deleted file mode 100644 index 764653bd..00000000 --- a/commitgraph/bloom/settings.go +++ /dev/null @@ -1,50 +0,0 @@ -package bloom - -import ( - "encoding/binary" - - "codeberg.org/lindenii/furgit/internal/intconv" -) - -// Settings describe the changed-paths Bloom filter parameters stored in -// commit-graph BDAT chunks. -// -// Obviously, they must match the repository's commit-graph settings to -// interpret filters correctly. -type Settings struct { - HashVersion uint32 - NumHashes uint32 - BitsPerEntry uint32 - MaxChangePaths uint32 -} - -// ParseSettings reads Bloom filter settings from a BDAT chunk header. -func ParseSettings(bdat []byte) (*Settings, error) { - if len(bdat) < DataHeaderSize { - return nil, ErrInvalid - } - - settings := &Settings{ - HashVersion: binary.BigEndian.Uint32(bdat[0:4]), - NumHashes: binary.BigEndian.Uint32(bdat[4:8]), - BitsPerEntry: binary.BigEndian.Uint32(bdat[8:12]), - MaxChangePaths: DefaultMaxChange, - } - - switch settings.HashVersion { - case 1, 2: - default: - return nil, ErrInvalid - } - - if settings.NumHashes == 0 { - return nil, ErrInvalid - } - - _, err := intconv.Uint32ToInt(settings.NumHashes) - if err != nil { - return nil, ErrInvalid - } - - return settings, nil -} diff --git a/commitgraph/constants.go b/commitgraph/constants.go deleted file mode 100644 index 3a06a290..00000000 --- a/commitgraph/constants.go +++ /dev/null @@ -1,32 +0,0 @@ -package commitgraph - -const ( - FileSignature = 0x43475048 // "CGPH" - FileVersion = 1 -) - -const ( - ChunkOIDF = 0x4f494446 // "OIDF" - ChunkOIDL = 0x4f49444c // "OIDL" - ChunkCDAT = 0x43444154 // "CDAT" - ChunkGDA2 = 0x47444132 // "GDA2" - ChunkGDO2 = 0x47444f32 // "GDO2" - ChunkEDGE = 0x45444745 // "EDGE" - ChunkBIDX = 0x42494458 // "BIDX" - ChunkBDAT = 0x42444154 // "BDAT" - ChunkBASE = 0x42415345 // "BASE" -) - -const ( - HeaderSize = 8 - ChunkEntrySize = 12 - FanoutSize = 256 * 4 -) - -const ( - ParentNone = 0x70000000 - ParentExtraMask = 0x80000000 - ParentLastMask = 0x7fffffff - - GenerationOverflow = 0x80000000 -) diff --git a/commitgraph/doc.go b/commitgraph/doc.go deleted file mode 100644 index abf5f3d3..00000000 --- a/commitgraph/doc.go +++ /dev/null @@ -1,2 +0,0 @@ -// Package commitgraph provides constants and common utilities for handling commit graphs. -package commitgraph diff --git a/commitgraph/read/bloom.go b/commitgraph/read/bloom.go deleted file mode 100644 index eca88c36..00000000 --- a/commitgraph/read/bloom.go +++ /dev/null @@ -1,117 +0,0 @@ -package read - -import ( - "encoding/binary" - - "codeberg.org/lindenii/furgit/commitgraph/bloom" - "codeberg.org/lindenii/furgit/internal/intconv" -) - -// HasBloom reports whether any layer has changed-path Bloom data. -func (reader *Reader) HasBloom() bool { - for i := range reader.layers { - layer := &reader.layers[i] - if layer.chunkBloomIndex != nil && layer.chunkBloomData != nil && layer.bloomSettings != nil { - return true - } - } - - return false -} - -// BloomVersion returns the changed-path Bloom hash version, or 0 if absent. -func (reader *Reader) BloomVersion() uint8 { - for i := len(reader.layers) - 1; i >= 0; i-- { - layer := &reader.layers[i] - if layer.bloomSettings != nil { - version, err := intconv.Uint32ToUint8(layer.bloomSettings.HashVersion) - if err != nil { - return 0 - } - - return version - } - } - - return 0 -} - -// BloomFilterAt returns one commit's changed-path Bloom filter. -// -// The returned filter borrows reader-owned mapped commit-graph data and is -// only valid until the reader is closed. -// -// Returns BloomUnavailableError when this commit graph has no Bloom data. -func (reader *Reader) BloomFilterAt(pos Position) (bloom.Filter, error) { - layer, err := reader.layerByPosition(pos) - if err != nil { - return bloom.Filter{}, err - } - - if layer.chunkBloomIndex == nil || layer.chunkBloomData == nil || layer.bloomSettings == nil { - return bloom.Filter{}, &BloomUnavailableError{Pos: pos} - } - - start, end, err := bloomRange(layer, pos.Index) - if err != nil { - return bloom.Filter{}, err - } - - filter := bloom.NewFilter( - layer.chunkBloomData[bloom.DataHeaderSize+start:bloom.DataHeaderSize+end], - *layer.bloomSettings, - ) - - return filter, nil -} - -func bloomRange(layer *layer, commitIndex uint32) (int, int, error) { - off64 := uint64(commitIndex) * 4 - - off, err := intconv.Uint64ToInt(off64) - if err != nil { - return 0, 0, err - } - - end := binary.BigEndian.Uint32(layer.chunkBloomIndex[off : off+4]) - - var start uint32 - - if commitIndex > 0 { - prevOff64 := uint64(commitIndex-1) * 4 - - prevOff, err := intconv.Uint64ToInt(prevOff64) - if err != nil { - return 0, 0, err - } - - start = binary.BigEndian.Uint32(layer.chunkBloomIndex[prevOff : prevOff+4]) - } - - if end < start { - return 0, 0, &MalformedError{Path: layer.path, Reason: "invalid BIDX range"} - } - - bdatLen := len(layer.chunkBloomData) - bloom.DataHeaderSize - - bdatLenU32, err := intconv.IntToUint32(bdatLen) - if err != nil { - return 0, 0, err - } - - if end > bdatLenU32 { - return 0, 0, &MalformedError{Path: layer.path, Reason: "BIDX range out of BDAT bounds"} - } - - startInt, err := intconv.Uint64ToInt(uint64(start)) - if err != nil { - return 0, 0, err - } - - endInt, err := intconv.Uint64ToInt(uint64(end)) - if err != nil { - return 0, 0, err - } - - return startInt, endInt, nil -} diff --git a/commitgraph/read/close.go b/commitgraph/read/close.go deleted file mode 100644 index f8b6141a..00000000 --- a/commitgraph/read/close.go +++ /dev/null @@ -1,20 +0,0 @@ -package read - -// Close releases all mapped commit-graph files. -// -// Repeated calls to Close are undefined behavior. -func (reader *Reader) Close() error { - var closeErr error - - for i := len(reader.layers) - 1; i >= 0; i-- { - err := reader.layers[i].close() - if err != nil && closeErr == nil { - closeErr = err - } - } - - reader.layers = nil - reader.total = 0 - - return closeErr -} diff --git a/commitgraph/read/commitat.go b/commitgraph/read/commitat.go deleted file mode 100644 index a39c5ccd..00000000 --- a/commitgraph/read/commitat.go +++ /dev/null @@ -1,85 +0,0 @@ -package read - -import ( - "encoding/binary" - - "codeberg.org/lindenii/furgit/internal/intconv" - objectid "codeberg.org/lindenii/furgit/object/id" -) - -// CommitAt returns decoded commit-graph metadata at one position. -func (reader *Reader) CommitAt(pos Position) (Commit, error) { - layer, err := reader.layerByPosition(pos) - if err != nil { - return Commit{}, err - } - - hashSize := reader.algo.Size() - stride := hashSize + 16 - - strideU64, err := intconv.IntToUint64(stride) - if err != nil { - return Commit{}, err - } - - start64 := uint64(pos.Index) * strideU64 - end64 := start64 + strideU64 - - start, err := intconv.Uint64ToInt(start64) - if err != nil { - return Commit{}, err - } - - end, err := intconv.Uint64ToInt(end64) - if err != nil { - return Commit{}, err - } - - record := layer.chunkCommit[start:end] - - treeOID, err := objectid.FromBytes(reader.algo, record[:hashSize]) - if err != nil { - return Commit{}, err - } - - oid, err := reader.OIDAt(pos) - if err != nil { - return Commit{}, err - } - - p1 := binary.BigEndian.Uint32(record[hashSize : hashSize+4]) - p2 := binary.BigEndian.Uint32(record[hashSize+4 : hashSize+8]) - genAndTimeHi := binary.BigEndian.Uint32(record[hashSize+8 : hashSize+12]) - timeLow := binary.BigEndian.Uint32(record[hashSize+12 : hashSize+16]) - - timeHigh := uint64(genAndTimeHi & 0x3) - commitTimeU64 := (timeHigh << 32) | uint64(timeLow) - - commitTime, err := intconv.Uint64ToInt64(commitTimeU64) - if err != nil { - return Commit{}, err - } - - generationV1 := genAndTimeHi >> 2 - - generationV2, err := reader.readGenerationV2(layer, pos.Index, commitTimeU64) - if err != nil { - return Commit{}, err - } - - parent1, parent2, extra, err := reader.decodeParents(layer, p1, p2) - if err != nil { - return Commit{}, err - } - - return Commit{ - OID: oid, - TreeOID: treeOID, - Parent1: parent1, - Parent2: parent2, - ExtraParents: extra, - CommitTimeUnix: commitTime, - GenerationV1: generationV1, - GenerationV2: generationV2, - }, nil -} diff --git a/commitgraph/read/commits.go b/commitgraph/read/commits.go deleted file mode 100644 index 48984ecb..00000000 --- a/commitgraph/read/commits.go +++ /dev/null @@ -1,20 +0,0 @@ -package read - -import objectid "codeberg.org/lindenii/furgit/object/id" - -// Commit stores decoded commit-graph record data. -type Commit struct { - OID objectid.ObjectID - TreeOID objectid.ObjectID - Parent1 ParentRef - Parent2 ParentRef - ExtraParents []Position - CommitTimeUnix int64 - GenerationV1 uint32 - GenerationV2 uint64 -} - -// NumCommits returns total commits across loaded layers. -func (reader *Reader) NumCommits() uint32 { - return reader.total -} diff --git a/commitgraph/read/doc.go b/commitgraph/read/doc.go deleted file mode 100644 index 573ddc19..00000000 --- a/commitgraph/read/doc.go +++ /dev/null @@ -1,2 +0,0 @@ -// Package read provides routines for reading commit graphs. -package read diff --git a/commitgraph/read/edges.go b/commitgraph/read/edges.go deleted file mode 100644 index 33a6c9fc..00000000 --- a/commitgraph/read/edges.go +++ /dev/null @@ -1,48 +0,0 @@ -package read - -import ( - "encoding/binary" - - "codeberg.org/lindenii/furgit/commitgraph" - "codeberg.org/lindenii/furgit/internal/intconv" -) - -func (reader *Reader) decodeExtraEdgeList(layer *layer, edgeStart uint32) ([]Position, error) { - if len(layer.chunkExtraEdges) == 0 { - return nil, &MalformedError{Path: layer.path, Reason: "missing EDGE chunk"} - } - - out := make([]Position, 0) - - cur := edgeStart - for { - off64 := uint64(cur) * 4 - - off, err := intconv.Uint64ToInt(off64) - if err != nil { - return nil, err - } - - if off+4 > len(layer.chunkExtraEdges) { - return nil, &MalformedError{Path: layer.path, Reason: "EDGE index out of range"} - } - - word := binary.BigEndian.Uint32(layer.chunkExtraEdges[off : off+4]) - parentGlobal := word & commitgraph.ParentLastMask - - parentPos, err := reader.globalToPosition(parentGlobal) - if err != nil { - return nil, err - } - - out = append(out, parentPos) - - if word&commitgraph.ParentExtraMask != 0 { - break - } - - cur++ - } - - return out, nil -} diff --git a/commitgraph/read/errors.go b/commitgraph/read/errors.go deleted file mode 100644 index 0a32a368..00000000 --- a/commitgraph/read/errors.go +++ /dev/null @@ -1,58 +0,0 @@ -package read - -import ( - "fmt" - - objectid "codeberg.org/lindenii/furgit/object/id" -) - -// NotFoundError reports a missing commit graph entry by object ID. -type NotFoundError struct { - OID objectid.ObjectID -} - -// Error implements error. -func (err *NotFoundError) Error() string { - return fmt.Sprintf("commitgraph: object not found: %s", err.OID) -} - -// PositionOutOfRangeError reports an invalid graph position. -type PositionOutOfRangeError struct { - Pos Position -} - -// Error implements error. -func (err *PositionOutOfRangeError) Error() string { - return fmt.Sprintf("commitgraph: position out of range: graph=%d index=%d", err.Pos.Graph, err.Pos.Index) -} - -// MalformedError reports malformed commit-graph data. -type MalformedError struct { - Path string - Reason string -} - -// Error implements error. -func (err *MalformedError) Error() string { - return fmt.Sprintf("commitgraph: malformed %q: %s", err.Path, err.Reason) -} - -// UnsupportedVersionError reports unsupported commit-graph version. -type UnsupportedVersionError struct { - Version uint8 -} - -// Error implements error. -func (err *UnsupportedVersionError) Error() string { - return fmt.Sprintf("commitgraph: unsupported version %d", err.Version) -} - -// BloomUnavailableError reports missing changed-path bloom data at one position. -type BloomUnavailableError struct { - Pos Position -} - -// Error implements error. -func (err *BloomUnavailableError) Error() string { - return fmt.Sprintf("commitgraph: bloom unavailable at position graph=%d index=%d", err.Pos.Graph, err.Pos.Index) -} diff --git a/commitgraph/read/generation.go b/commitgraph/read/generation.go deleted file mode 100644 index 53dabe2d..00000000 --- a/commitgraph/read/generation.go +++ /dev/null @@ -1,43 +0,0 @@ -package read - -import ( - "encoding/binary" - - "codeberg.org/lindenii/furgit/commitgraph" - "codeberg.org/lindenii/furgit/internal/intconv" -) - -func (reader *Reader) readGenerationV2(layer *layer, index uint32, commitTime uint64) (uint64, error) { - if len(layer.chunkGeneration) == 0 { - return 0, nil - } - - off64 := uint64(index) * 4 - - off, err := intconv.Uint64ToInt(off64) - if err != nil { - return 0, err - } - - value := binary.BigEndian.Uint32(layer.chunkGeneration[off : off+4]) - - if value&commitgraph.GenerationOverflow == 0 { - return commitTime + uint64(value), nil - } - - gdo2Index := value ^ commitgraph.GenerationOverflow - gdo2Off64 := uint64(gdo2Index) * 8 - - gdo2Off, err := intconv.Uint64ToInt(gdo2Off64) - if err != nil { - return 0, err - } - - if gdo2Off+8 > len(layer.chunkGenerationOv) { - return 0, &MalformedError{Path: layer.path, Reason: "GDO2 index out of range"} - } - - overflow := binary.BigEndian.Uint64(layer.chunkGenerationOv[gdo2Off : gdo2Off+8]) - - return commitTime + overflow, nil -} diff --git a/commitgraph/read/hash.go b/commitgraph/read/hash.go deleted file mode 100644 index 3a525afe..00000000 --- a/commitgraph/read/hash.go +++ /dev/null @@ -1,79 +0,0 @@ -package read - -import ( - "bytes" - "fmt" - "io" - - objectid "codeberg.org/lindenii/furgit/object/id" -) - -// HashVersion returns the commit-graph hash version. -func (reader *Reader) HashVersion() uint8 { - return reader.hashVersion -} - -func validateChainBaseHashes(algo objectid.Algorithm, chain []string, idx int, graph *layer) error { - if idx == 0 { - if len(graph.chunkBaseGraphs) != 0 { - return &MalformedError{Path: graph.path, Reason: "unexpected BASE chunk in first graph"} - } - - return nil - } - - hashSize := algo.Size() - - expectedLen := idx * hashSize - if len(graph.chunkBaseGraphs) != expectedLen { - return &MalformedError{ - Path: graph.path, - Reason: fmt.Sprintf("BASE chunk length %d does not match expected %d", len(graph.chunkBaseGraphs), expectedLen), - } - } - - for i := range idx { - start := i * hashSize - end := start + hashSize - - baseHash, err := objectid.FromBytes(algo, graph.chunkBaseGraphs[start:end]) - if err != nil { - return err - } - - if baseHash.String() != chain[i] { - return &MalformedError{ - Path: graph.path, - Reason: fmt.Sprintf("BASE chunk mismatch at index %d", i), - } - } - } - - return nil -} - -func verifyTrailerHash(data []byte, algo objectid.Algorithm, path string) error { - hashSize := algo.Size() - if len(data) < hashSize { - return &MalformedError{Path: path, Reason: "file too short for trailer"} - } - - hashImpl, err := algo.New() - if err != nil { - return err - } - - _, err = io.Copy(hashImpl, bytes.NewReader(data[:len(data)-hashSize])) - if err != nil { - return err - } - - got := hashImpl.Sum(nil) - - want := data[len(data)-hashSize:] - if !bytes.Equal(got, want) { - return &MalformedError{Path: path, Reason: "trailer hash mismatch"} - } - - return nil -} diff --git a/commitgraph/read/iterators.go b/commitgraph/read/iterators.go deleted file mode 100644 index 85c56ff1..00000000 --- a/commitgraph/read/iterators.go +++ /dev/null @@ -1,45 +0,0 @@ -package read - -import ( - "iter" - - "codeberg.org/lindenii/furgit/internal/intconv" - objectid "codeberg.org/lindenii/furgit/object/id" -) - -// AllPositions iterates all commit positions in native layer order. -func (reader *Reader) AllPositions() iter.Seq[Position] { - return func(yield func(Position) bool) { - for layerIdx := range reader.layers { - layer := &reader.layers[layerIdx] - - graph, err := intconv.IntToUint32(layerIdx) - if err != nil { - return - } - - for idx := range layer.numCommits { - if !yield(Position{Graph: graph, Index: idx}) { - return - } - } - } - } -} - -// AllOIDs iterates all commit object IDs in native layer order. -func (reader *Reader) AllOIDs() iter.Seq[objectid.ObjectID] { - return func(yield func(objectid.ObjectID) bool) { - positions := reader.AllPositions() - for pos := range positions { - oid, err := reader.OIDAt(pos) - if err != nil { - return - } - - if !yield(oid) { - return - } - } - } -} diff --git a/commitgraph/read/layer.go b/commitgraph/read/layer.go deleted file mode 100644 index 3b8abcd2..00000000 --- a/commitgraph/read/layer.go +++ /dev/null @@ -1,28 +0,0 @@ -package read - -import ( - "os" - - "codeberg.org/lindenii/furgit/commitgraph/bloom" -) - -type layer struct { - path string - file *os.File - data []byte - numCommits uint32 - baseCount uint32 - globalFrom uint32 - - chunkOIDFanout []byte - chunkOIDLookup []byte - chunkCommit []byte - chunkGeneration []byte - chunkGenerationOv []byte - chunkExtraEdges []byte - chunkBloomIndex []byte - chunkBloomData []byte - chunkBaseGraphs []byte - - bloomSettings *bloom.Settings -} diff --git a/commitgraph/read/layer_close.go b/commitgraph/read/layer_close.go deleted file mode 100644 index 03dc91d5..00000000 --- a/commitgraph/read/layer_close.go +++ /dev/null @@ -1,33 +0,0 @@ -package read - -import "syscall" - -func closeLayers(layers []layer) { - for i := len(layers) - 1; i >= 0; i-- { - _ = layers[i].close() - } -} - -func (layer *layer) close() error { - var closeErr error - - if layer.data != nil { - err := syscall.Munmap(layer.data) - if err != nil { - closeErr = err - } - - layer.data = nil - } - - if layer.file != nil { - err := layer.file.Close() - if err != nil && closeErr == nil { - closeErr = err - } - - layer.file = nil - } - - return closeErr -} diff --git a/commitgraph/read/layer_lookup.go b/commitgraph/read/layer_lookup.go deleted file mode 100644 index 84095788..00000000 --- a/commitgraph/read/layer_lookup.go +++ /dev/null @@ -1,53 +0,0 @@ -package read - -import ( - "bytes" - "encoding/binary" - - "codeberg.org/lindenii/furgit/internal/intconv" - objectid "codeberg.org/lindenii/furgit/object/id" -) - -func layerLookup(layer *layer, oid objectid.ObjectID) (uint32, bool) { - hashSize := oid.Size() - first := int(oid.RawBytes()[0]) - - var lo uint32 - if first > 0 { - lo = binary.BigEndian.Uint32(layer.chunkOIDFanout[(first-1)*4 : first*4]) - } - - hi := binary.BigEndian.Uint32(layer.chunkOIDFanout[first*4 : (first+1)*4]) - if hi == 0 || lo >= hi { - return 0, false - } - - target := oid.RawBytes() - left := int(lo) - - right := int(hi) - 1 - for left <= right { - mid := left + (right-left)/2 - start := mid * hashSize - end := start + hashSize - - current := layer.chunkOIDLookup[start:end] - - cmp := bytes.Compare(current, target) - switch { - case cmp == 0: - pos, err := intconv.IntToUint32(mid) - if err != nil { - return 0, false - } - - return pos, true - case cmp < 0: - left = mid + 1 - default: - right = mid - 1 - } - } - - return 0, false -} diff --git a/commitgraph/read/layer_open.go b/commitgraph/read/layer_open.go deleted file mode 100644 index 3ecd4672..00000000 --- a/commitgraph/read/layer_open.go +++ /dev/null @@ -1,81 +0,0 @@ -package read - -import ( - "os" - "syscall" - - "codeberg.org/lindenii/furgit/commitgraph" - "codeberg.org/lindenii/furgit/internal/intconv" - objectid "codeberg.org/lindenii/furgit/object/id" -) - -func openLayer(root *os.Root, relPath string, algo objectid.Algorithm) (*layer, error) { - file, err := root.Open(relPath) - if err != nil { - return nil, err - } - - info, err := file.Stat() - if err != nil { - _ = file.Close() - - return nil, err - } - - size := info.Size() - if size < int64(commitgraph.HeaderSize+commitgraph.FanoutSize+algo.Size()) { - _ = file.Close() - - return nil, &MalformedError{Path: relPath, Reason: "file too short"} - } - - mapLen, err := intconv.Int64ToUint64(size) - if err != nil { - _ = file.Close() - - return nil, err - } - - mapLenInt, err := intconv.Uint64ToInt(mapLen) - if err != nil { - _ = file.Close() - - return nil, err - } - - fd, err := intconv.UintptrToInt(file.Fd()) - if err != nil { - _ = file.Close() - - return nil, err - } - - data, err := syscall.Mmap(fd, 0, mapLenInt, syscall.PROT_READ, syscall.MAP_PRIVATE) - if err != nil { - _ = file.Close() - - return nil, err - } - - out := &layer{ - path: relPath, - file: file, - data: data, - } - - parseErr := parseLayer(out, algo) - if parseErr != nil { - _ = out.close() - - return nil, parseErr - } - - verifyErr := verifyTrailerHash(out.data, algo, relPath) - if verifyErr != nil { - _ = out.close() - - return nil, verifyErr - } - - return out, nil -} diff --git a/commitgraph/read/layer_parse.go b/commitgraph/read/layer_parse.go deleted file mode 100644 index 2aa71428..00000000 --- a/commitgraph/read/layer_parse.go +++ /dev/null @@ -1,276 +0,0 @@ -package read - -import ( - "encoding/binary" - - "codeberg.org/lindenii/furgit/commitgraph" - "codeberg.org/lindenii/furgit/commitgraph/bloom" - "codeberg.org/lindenii/furgit/internal/intconv" - objectid "codeberg.org/lindenii/furgit/object/id" -) - -func parseLayer(layer *layer, algo objectid.Algorithm) error { //nolint:maintidx - if len(layer.data) < commitgraph.HeaderSize { - return &MalformedError{Path: layer.path, Reason: "file too short"} - } - - header := layer.data[:commitgraph.HeaderSize] - - signature := binary.BigEndian.Uint32(header[:4]) - if signature != commitgraph.FileSignature { - return &MalformedError{Path: layer.path, Reason: "invalid signature"} - } - - version := header[4] - if version != commitgraph.FileVersion { - return &UnsupportedVersionError{Version: version} - } - - expectedHashVersion, err := intconv.Uint32ToUint8(algo.PackHashID()) - if err != nil { - return err - } - - hashVersion := header[5] - if hashVersion != expectedHashVersion { - return &MalformedError{Path: layer.path, Reason: "hash version does not match object format"} - } - - numChunks := int(header[6]) - baseCount := uint32(header[7]) - - tocLen := (numChunks + 1) * commitgraph.ChunkEntrySize - tocStart := commitgraph.HeaderSize - - tocEnd := tocStart + tocLen - if tocEnd > len(layer.data) { - return &MalformedError{Path: layer.path, Reason: "truncated chunk table"} - } - - type tocEntry struct { - id uint32 - offset uint64 - } - - entries := make([]tocEntry, 0, numChunks+1) - for i := range numChunks + 1 { - entryOff := tocStart + i*commitgraph.ChunkEntrySize - entryData := layer.data[entryOff : entryOff+commitgraph.ChunkEntrySize] - - entry := tocEntry{ - id: binary.BigEndian.Uint32(entryData[:4]), - offset: binary.BigEndian.Uint64(entryData[4:]), - } - entries = append(entries, entry) - } - - if entries[len(entries)-1].id != 0 { - return &MalformedError{Path: layer.path, Reason: "missing chunk table terminator"} - } - - trailerStart := len(layer.data) - algo.Size() - - chunks := make(map[uint32][]byte, numChunks) - for i := range numChunks { - entry := entries[i] - if entry.id == 0 { - return &MalformedError{Path: layer.path, Reason: "early chunk table terminator"} - } - - next := entries[i+1] - - start, err := intconv.Uint64ToInt(entry.offset) - if err != nil { - return err - } - - end, err := intconv.Uint64ToInt(next.offset) - if err != nil { - return err - } - - if start < tocEnd || end < start || end > trailerStart { - return &MalformedError{Path: layer.path, Reason: "invalid chunk offsets"} - } - - if _, exists := chunks[entry.id]; exists { - return &MalformedError{Path: layer.path, Reason: "duplicate chunk id"} - } - - chunks[entry.id] = layer.data[start:end] - } - - oidf := chunks[commitgraph.ChunkOIDF] - if len(oidf) != commitgraph.FanoutSize { - return &MalformedError{Path: layer.path, Reason: "invalid OIDF length"} - } - - layer.chunkOIDFanout = oidf - layer.numCommits = binary.BigEndian.Uint32(oidf[commitgraph.FanoutSize-4:]) - - for i := range 255 { - cur := binary.BigEndian.Uint32(oidf[i*4 : (i+1)*4]) - - next := binary.BigEndian.Uint32(oidf[(i+1)*4 : (i+2)*4]) - if cur > next { - return &MalformedError{Path: layer.path, Reason: "non-monotonic OIDF fanout"} - } - } - - hashSize := algo.Size() - - hashSizeU64, err := intconv.IntToUint64(hashSize) - if err != nil { - return err - } - - oidl := chunks[commitgraph.ChunkOIDL] - oidlWantLen64 := uint64(layer.numCommits) * hashSizeU64 - - oidlWantLen, err := intconv.Uint64ToInt(oidlWantLen64) - if err != nil { - return err - } - - if len(oidl) != oidlWantLen { - return &MalformedError{Path: layer.path, Reason: "invalid OIDL length"} - } - - layer.chunkOIDLookup = oidl - - stride := hashSize + 16 - - strideU64, err := intconv.IntToUint64(stride) - if err != nil { - return err - } - - cdat := chunks[commitgraph.ChunkCDAT] - cdatWantLen64 := uint64(layer.numCommits) * strideU64 - - cdatWantLen, err := intconv.Uint64ToInt(cdatWantLen64) - if err != nil { - return err - } - - if len(cdat) != cdatWantLen { - return &MalformedError{Path: layer.path, Reason: "invalid CDAT length"} - } - - layer.chunkCommit = cdat - - gda2 := chunks[commitgraph.ChunkGDA2] - if len(gda2) != 0 { - wantLen64 := uint64(layer.numCommits) * 4 - - wantLen, err := intconv.Uint64ToInt(wantLen64) - if err != nil { - return err - } - - if len(gda2) != wantLen { - return &MalformedError{Path: layer.path, Reason: "invalid GDA2 length"} - } - - layer.chunkGeneration = gda2 - } - - gdo2 := chunks[commitgraph.ChunkGDO2] - if len(gdo2) != 0 { - if len(gdo2)%8 != 0 { - return &MalformedError{Path: layer.path, Reason: "invalid GDO2 length"} - } - - layer.chunkGenerationOv = gdo2 - } - - edge := chunks[commitgraph.ChunkEDGE] - if len(edge) != 0 { - if len(edge)%4 != 0 { - return &MalformedError{Path: layer.path, Reason: "invalid EDGE length"} - } - - layer.chunkExtraEdges = edge - } - - base := chunks[commitgraph.ChunkBASE] - if baseCount == 0 { - if len(base) != 0 { - return &MalformedError{Path: layer.path, Reason: "unexpected BASE chunk"} - } - } else { - wantLen64 := uint64(baseCount) * hashSizeU64 - - wantLen, err := intconv.Uint64ToInt(wantLen64) - if err != nil { - return err - } - - if len(base) != wantLen { - return &MalformedError{Path: layer.path, Reason: "invalid BASE length"} - } - - layer.chunkBaseGraphs = base - } - - layer.baseCount = baseCount - - bidx := chunks[commitgraph.ChunkBIDX] - - bdat := chunks[commitgraph.ChunkBDAT] - if len(bidx) != 0 || len(bdat) != 0 { //nolint:nestif - if len(bidx) == 0 || len(bdat) == 0 { - return &MalformedError{Path: layer.path, Reason: "BIDX/BDAT must both be present"} - } - - bidxWantLen64 := uint64(layer.numCommits) * 4 - - bidxWantLen, err := intconv.Uint64ToInt(bidxWantLen64) - if err != nil { - return err - } - - if len(bidx) != bidxWantLen { - return &MalformedError{Path: layer.path, Reason: "invalid BIDX length"} - } - - if len(bdat) < bloom.DataHeaderSize { - return &MalformedError{Path: layer.path, Reason: "invalid BDAT length"} - } - - settings, err := bloom.ParseSettings(bdat) - if err != nil { - return err - } - - prev := uint32(0) - - for i := range layer.numCommits { - off := int(i) * 4 - - cur := binary.BigEndian.Uint32(bidx[off : off+4]) - if i > 0 && cur < prev { - return &MalformedError{Path: layer.path, Reason: "non-monotonic BIDX"} - } - - bdatDataLen := len(bdat) - bloom.DataHeaderSize - - bdatDataLenU32, err := intconv.IntToUint32(bdatDataLen) - if err != nil { - return err - } - - if cur > bdatDataLenU32 { - return &MalformedError{Path: layer.path, Reason: "BIDX offset out of range"} - } - - prev = cur - } - - layer.chunkBloomIndex = bidx - layer.chunkBloomData = bdat - layer.bloomSettings = settings - } - - return nil -} diff --git a/commitgraph/read/layer_pos.go b/commitgraph/read/layer_pos.go deleted file mode 100644 index 7b87b381..00000000 --- a/commitgraph/read/layer_pos.go +++ /dev/null @@ -1,21 +0,0 @@ -package read - -import "codeberg.org/lindenii/furgit/internal/intconv" - -func (reader *Reader) layerByPosition(pos Position) (*layer, error) { - graphIdx, err := intconv.Uint64ToInt(uint64(pos.Graph)) - if err != nil { - return nil, err - } - - if graphIdx < 0 || graphIdx >= len(reader.layers) { - return nil, &PositionOutOfRangeError{Pos: pos} - } - - layer := &reader.layers[graphIdx] - if pos.Index >= layer.numCommits { - return nil, &PositionOutOfRangeError{Pos: pos} - } - - return layer, nil -} diff --git a/commitgraph/read/layerinfo.go b/commitgraph/read/layerinfo.go deleted file mode 100644 index 83c4407d..00000000 --- a/commitgraph/read/layerinfo.go +++ /dev/null @@ -1,23 +0,0 @@ -package read - -// LayerInfo describes one loaded commit-graph layer. -type LayerInfo struct { - Path string - BaseCount uint32 - Commits uint32 -} - -// Layers returns loaded layer metadata in native chain order. -func (reader *Reader) Layers() []LayerInfo { - out := make([]LayerInfo, 0, len(reader.layers)) - for i := range reader.layers { - layer := reader.layers[i] - out = append(out, LayerInfo{ - Path: layer.path, - BaseCount: layer.baseCount, - Commits: layer.numCommits, - }) - } - - return out -} diff --git a/commitgraph/read/lookup.go b/commitgraph/read/lookup.go deleted file mode 100644 index 5f1b08f6..00000000 --- a/commitgraph/read/lookup.go +++ /dev/null @@ -1,29 +0,0 @@ -package read - -import ( - "codeberg.org/lindenii/furgit/internal/intconv" - objectid "codeberg.org/lindenii/furgit/object/id" -) - -// Lookup resolves one object ID to one graph position. -func (reader *Reader) Lookup(oid objectid.ObjectID) (Position, error) { - if oid.Algorithm() != reader.algo { - return Position{}, &NotFoundError{OID: oid} - } - - for layerIdx := len(reader.layers) - 1; layerIdx >= 0; layerIdx-- { - layer := &reader.layers[layerIdx] - - found, ok := layerLookup(layer, oid) - if ok { - idxU32, err := intconv.IntToUint32(layerIdx) - if err != nil { - return Position{}, err - } - - return Position{Graph: idxU32, Index: found}, nil - } - } - - return Position{}, &NotFoundError{OID: oid} -} diff --git a/commitgraph/read/mode.go b/commitgraph/read/mode.go deleted file mode 100644 index 76afa21f..00000000 --- a/commitgraph/read/mode.go +++ /dev/null @@ -1,11 +0,0 @@ -package read - -// OpenMode controls which commit-graph layout Open loads. -type OpenMode uint8 - -const ( - // OpenSingle opens one commit-graph file at info/commit-graph. - OpenSingle OpenMode = iota - // OpenChain opens chained commit-graphs from info/commit-graphs. - OpenChain -) diff --git a/commitgraph/read/oidat.go b/commitgraph/read/oidat.go deleted file mode 100644 index 908cbc1c..00000000 --- a/commitgraph/read/oidat.go +++ /dev/null @@ -1,36 +0,0 @@ -package read - -import ( - "codeberg.org/lindenii/furgit/internal/intconv" - objectid "codeberg.org/lindenii/furgit/object/id" -) - -// OIDAt returns object ID at one position. -func (reader *Reader) OIDAt(pos Position) (objectid.ObjectID, error) { - layer, err := reader.layerByPosition(pos) - if err != nil { - return objectid.ObjectID{}, err - } - - hashSize := reader.algo.Size() - - hashSizeU64, err := intconv.IntToUint64(hashSize) - if err != nil { - return objectid.ObjectID{}, err - } - - start64 := uint64(pos.Index) * hashSizeU64 - end64 := start64 + hashSizeU64 - - start, err := intconv.Uint64ToInt(start64) - if err != nil { - return objectid.ObjectID{}, err - } - - end, err := intconv.Uint64ToInt(end64) - if err != nil { - return objectid.ObjectID{}, err - } - - return objectid.FromBytes(reader.algo, layer.chunkOIDLookup[start:end]) -} diff --git a/commitgraph/read/open.go b/commitgraph/read/open.go deleted file mode 100644 index 9c708b49..00000000 --- a/commitgraph/read/open.go +++ /dev/null @@ -1,26 +0,0 @@ -package read - -import ( - "fmt" - "os" - - objectid "codeberg.org/lindenii/furgit/object/id" -) - -// Open opens commit-graph data from one objects root. -// -// Open borrows root during construction and does not close it. -func Open(root *os.Root, algo objectid.Algorithm, mode OpenMode) (*Reader, error) { - if algo.Size() == 0 { - return nil, objectid.ErrInvalidAlgorithm - } - - switch mode { - case OpenSingle: - return openSingle(root, algo) - case OpenChain: - return openChain(root, algo) - default: - return nil, fmt.Errorf("commitgraph: invalid open mode %d", mode) - } -} diff --git a/commitgraph/read/open_chain.go b/commitgraph/read/open_chain.go deleted file mode 100644 index b55f3e57..00000000 --- a/commitgraph/read/open_chain.go +++ /dev/null @@ -1,133 +0,0 @@ -package read - -import ( - "bufio" - "errors" - "fmt" - "os" - "strings" - - "codeberg.org/lindenii/furgit/internal/intconv" - objectid "codeberg.org/lindenii/furgit/object/id" -) - -func openChain(root *os.Root, algo objectid.Algorithm) (*Reader, error) { - chainPath := "info/commit-graphs/commit-graph-chain" - - file, err := root.Open(chainPath) - if err != nil { - if errors.Is(err, os.ErrNotExist) { - return nil, &MalformedError{Path: chainPath, Reason: "missing commit-graph-chain"} - } - - return nil, err - } - - scanner := bufio.NewScanner(file) - hashes := make([]string, 0) - - for scanner.Scan() { - line := strings.TrimSpace(scanner.Text()) - if line == "" { - continue - } - - hashes = append(hashes, line) - } - - scanErr := scanner.Err() - closeErr := file.Close() - - if scanErr != nil { - return nil, scanErr - } - - if closeErr != nil { - return nil, closeErr - } - - if len(hashes) == 0 { - return nil, &MalformedError{Path: chainPath, Reason: "empty chain"} - } - - layers := make([]layer, 0, len(hashes)) - - var total uint32 - - hashVersion, err := intconv.Uint32ToUint8(algo.PackHashID()) - if err != nil { - return nil, err - } - - for i, hashHex := range hashes { - expectedBaseCount, err := intconv.IntToUint32(i) - if err != nil { - closeLayers(layers) - - return nil, err - } - - if len(hashHex) != algo.HexLen() { - closeLayers(layers) - - return nil, &MalformedError{ - Path: chainPath, - Reason: fmt.Sprintf("invalid graph hash length at line %d", i+1), - } - } - - relPath := fmt.Sprintf("info/commit-graphs/graph-%s.graph", hashHex) - - loaded, loadErr := openLayer(root, relPath, algo) - if loadErr != nil { - closeLayers(layers) - - return nil, loadErr - } - - if loaded.baseCount != expectedBaseCount { - _ = loaded.close() - - closeLayers(layers) - - return nil, &MalformedError{ - Path: relPath, - Reason: fmt.Sprintf("BASE count %d does not match chain depth %d", loaded.baseCount, i), - } - } - - validateErr := validateChainBaseHashes(algo, hashes, i, loaded) - if validateErr != nil { - _ = loaded.close() - - closeLayers(layers) - - return nil, validateErr - } - - loaded.globalFrom = total - loaded.baseCount = expectedBaseCount - - totalNext := total + loaded.numCommits - if totalNext < total { - _ = loaded.close() - - closeLayers(layers) - - return nil, &MalformedError{Path: relPath, Reason: "total commit count overflow"} - } - - total = totalNext - - layers = append(layers, *loaded) - } - - out := &Reader{ - algo: algo, - hashVersion: hashVersion, - layers: layers, - total: total, - } - - return out, nil -} diff --git a/commitgraph/read/open_single.go b/commitgraph/read/open_single.go deleted file mode 100644 index 9ad6607f..00000000 --- a/commitgraph/read/open_single.go +++ /dev/null @@ -1,32 +0,0 @@ -package read - -import ( - "os" - - "codeberg.org/lindenii/furgit/internal/intconv" - objectid "codeberg.org/lindenii/furgit/object/id" -) - -func openSingle(root *os.Root, algo objectid.Algorithm) (*Reader, error) { - graph, err := openLayer(root, "info/commit-graph", algo) - if err != nil { - return nil, err - } - - graph.baseCount = 0 - graph.globalFrom = 0 - - hashVersion, err := intconv.Uint32ToUint8(algo.PackHashID()) - if err != nil { - return nil, err - } - - out := &Reader{ - algo: algo, - hashVersion: hashVersion, - layers: []layer{*graph}, - total: graph.numCommits, - } - - return out, nil -} diff --git a/commitgraph/read/parents.go b/commitgraph/read/parents.go deleted file mode 100644 index deb5ea98..00000000 --- a/commitgraph/read/parents.go +++ /dev/null @@ -1,67 +0,0 @@ -package read - -import "codeberg.org/lindenii/furgit/commitgraph" - -// ParentRef references one parent position. -type ParentRef struct { - Valid bool - Pos Position -} - -func (reader *Reader) decodeParents(layer *layer, p1, p2 uint32) (ParentRef, ParentRef, []Position, error) { - parent1, err := reader.decodeSingleParent(p1) - if err != nil { - return ParentRef{}, ParentRef{}, nil, err - } - - if p2 == commitgraph.ParentNone { - return parent1, ParentRef{}, nil, nil - } - - if p2&commitgraph.ParentExtraMask == 0 { - parent2, err := reader.decodeSingleParent(p2) - if err != nil { - return ParentRef{}, ParentRef{}, nil, err - } - - return parent1, parent2, nil, nil - } - - edgeStart := p2 & commitgraph.ParentLastMask - - parents, err := reader.decodeExtraEdgeList(layer, edgeStart) - if err != nil { - return ParentRef{}, ParentRef{}, nil, err - } - - if len(parents) == 0 { - return ParentRef{}, ParentRef{}, nil, &MalformedError{Path: layer.path, Reason: "empty EDGE list"} - } - - parent2 := ParentRef{Valid: true, Pos: parents[0]} - if len(parents) == 1 { - return parent1, parent2, nil, nil - } - - return parent1, parent2, parents[1:], nil -} - -func (reader *Reader) decodeSingleParent(raw uint32) (ParentRef, error) { - if raw == commitgraph.ParentNone { - return ParentRef{}, nil - } - - if raw&commitgraph.ParentExtraMask != 0 { - return ParentRef{}, &MalformedError{ - Path: "commit-graph", - Reason: "unexpected EDGE marker in single-parent slot", - } - } - - pos, err := reader.globalToPosition(raw) - if err != nil { - return ParentRef{}, err - } - - return ParentRef{Valid: true, Pos: pos}, nil -} diff --git a/commitgraph/read/position.go b/commitgraph/read/position.go deleted file mode 100644 index b2e1138b..00000000 --- a/commitgraph/read/position.go +++ /dev/null @@ -1,38 +0,0 @@ -package read - -import ( - "fmt" - - "codeberg.org/lindenii/furgit/internal/intconv" -) - -// Position identifies one commit record by layer and row index. -type Position struct { - Graph uint32 - Index uint32 -} - -func (reader *Reader) globalToPosition(global uint32) (Position, error) { - for i := range reader.layers { - layer := &reader.layers[i] - from := layer.globalFrom - - to := from + layer.numCommits - if global >= from && global < to { - graph, err := intconv.IntToUint32(i) - if err != nil { - return Position{}, err - } - - return Position{ - Graph: graph, - Index: global - from, - }, nil - } - } - - return Position{}, &MalformedError{ - Path: "commit-graph", - Reason: fmt.Sprintf("parent global position out of range: %d", global), - } -} diff --git a/commitgraph/read/read_test.go b/commitgraph/read/read_test.go deleted file mode 100644 index bf87ec5d..00000000 --- a/commitgraph/read/read_test.go +++ /dev/null @@ -1,322 +0,0 @@ -package read_test - -import ( - "errors" - "path/filepath" - "strconv" - "strings" - "testing" - - "codeberg.org/lindenii/furgit/commitgraph/bloom" - "codeberg.org/lindenii/furgit/commitgraph/read" - "codeberg.org/lindenii/furgit/internal/intconv" - "codeberg.org/lindenii/furgit/internal/testgit" - objectid "codeberg.org/lindenii/furgit/object/id" -) - -func fixtureRepoPath(t *testing.T, algo objectid.Algorithm, name string) string { - t.Helper() - - return filepath.Join("testdata", "fixtures", algo.String(), name, "repo.git") -} - -func fixtureRepo(t *testing.T, algo objectid.Algorithm, name string) *testgit.TestRepo { - t.Helper() - - return testgit.NewRepoFromFixture(t, algo, fixtureRepoPath(t, algo, name)) -} - -func TestReadSingleMatchesGit(t *testing.T) { - t.Parallel() - - testgit.ForEachAlgorithm(t, func(t *testing.T, algo objectid.Algorithm) { //nolint:thelper - testRepo := fixtureRepo(t, algo, "single_changed") - - reader := openReader(t, testRepo, read.OpenSingle) - - defer func() { _ = reader.Close() }() - - allIDs := testRepo.RevList(t, "--all") - if len(allIDs) == 0 { - t.Fatal("git rev-list --all returned no commits") - } - - wantCommitCount, err := intconv.IntToUint32(len(allIDs)) - if err != nil { - t.Fatalf("len(allIDs) convert: %v", err) - } - - if got := reader.NumCommits(); got != wantCommitCount { - t.Fatalf("NumCommits() = %d, want %d", got, len(allIDs)) - } - - if !reader.HasBloom() { - t.Fatal("HasBloom() = false, want true") - } - - bloomVersion := reader.BloomVersion() - if bloomVersion == 0 { - t.Fatal("BloomVersion() = 0, want non-zero when HasBloom() is true") - } - - for _, id := range allIDs { - pos, err := reader.Lookup(id) - if err != nil { - t.Fatalf("Lookup(%s): %v", id, err) - } - - gotID, err := reader.OIDAt(pos) - if err != nil { - t.Fatalf("OIDAt(%+v): %v", pos, err) - } - - if gotID != id { - t.Fatalf("OIDAt(Lookup(%s)) = %s, want %s", id, gotID, id) - } - } - - step := max(len(allIDs)/24, 1) - - for i, id := range allIDs { - if i%step != 0 && i != len(allIDs)-1 { - continue - } - - verifyCommitAgainstGit(t, testRepo, reader, id) - } - }) -} - -func TestReadChainMatchesGit(t *testing.T) { - t.Parallel() - - testgit.ForEachAlgorithm(t, func(t *testing.T, algo objectid.Algorithm) { //nolint:thelper - testRepo := fixtureRepo(t, algo, "chain_changed") - - reader := openReader(t, testRepo, read.OpenChain) - - defer func() { _ = reader.Close() }() - - layers := reader.Layers() - if len(layers) < 2 { - t.Fatalf("Layers len = %d, want >= 2", len(layers)) - } - - allIDs := testRepo.RevList(t, "--all") - - wantCommitCount, err := intconv.IntToUint32(len(allIDs)) - if err != nil { - t.Fatalf("len(allIDs) convert: %v", err) - } - - if got := reader.NumCommits(); got != wantCommitCount { - t.Fatalf("NumCommits() = %d, want %d", got, len(allIDs)) - } - - step := max(len(allIDs)/20, 1) - - for i, id := range allIDs { - pos, err := reader.Lookup(id) - if err != nil { - t.Fatalf("Lookup(%s): %v", id, err) - } - - if i%step != 0 && i != len(allIDs)-1 { - continue - } - - gotID, err := reader.OIDAt(pos) - if err != nil { - t.Fatalf("OIDAt(%+v): %v", pos, err) - } - - if gotID != id { - t.Fatalf("OIDAt(Lookup(%s)) = %s, want %s", id, gotID, id) - } - } - }) -} - -func TestBloomUnavailableWithoutChangedPaths(t *testing.T) { - t.Parallel() - - testgit.ForEachAlgorithm(t, func(t *testing.T, algo objectid.Algorithm) { //nolint:thelper - testRepo := fixtureRepo(t, algo, "single_nochanged") - - reader := openReader(t, testRepo, read.OpenSingle) - - defer func() { _ = reader.Close() }() - - head := testRepo.RevParse(t, "HEAD") - - pos, err := reader.Lookup(head) - if err != nil { - t.Fatalf("Lookup(%s): %v", head, err) - } - - _, err = reader.BloomFilterAt(pos) - if err == nil { - t.Fatal("BloomFilterAt() error = nil, want BloomUnavailableError") - } - - unavailable, ok := errors.AsType[*read.BloomUnavailableError](err) - if !ok { - t.Fatalf("BloomFilterAt() error type = %T, want *BloomUnavailableError", err) - } - - if unavailable.Pos != pos { - t.Fatalf("BloomUnavailableError.Pos = %+v, want %+v", unavailable.Pos, pos) - } - }) -} - -func openReader(tb testing.TB, testRepo *testgit.TestRepo, mode read.OpenMode) *read.Reader { - tb.Helper() - - root := testRepo.OpenObjectsRoot(tb) - - reader, err := read.Open(root, testRepo.Algorithm(), mode) - if err != nil { - tb.Fatalf("read.Open(objects): %v", err) - } - - return reader -} - -func verifyCommitAgainstGit(tb testing.TB, testRepo *testgit.TestRepo, reader *read.Reader, id objectid.ObjectID) { - tb.Helper() - - pos, err := reader.Lookup(id) - if err != nil { - tb.Fatalf("Lookup(%s): %v", id, err) - } - - commit, err := reader.CommitAt(pos) - if err != nil { - tb.Fatalf("CommitAt(%+v): %v", pos, err) - } - - if commit.OID != id { - tb.Fatalf("CommitAt(%+v).OID = %s, want %s", pos, commit.OID, id) - } - - treeHex := testRepo.Run(tb, "show", "-s", "--format=%T", id.String()) - - wantTree, err := objectid.ParseHex(testRepo.Algorithm(), treeHex) - if err != nil { - tb.Fatalf("parse tree id %q: %v", treeHex, err) - } - - if commit.TreeOID != wantTree { - tb.Fatalf("CommitAt(%+v).TreeOID = %s, want %s", pos, commit.TreeOID, wantTree) - } - - wantParents := parseOIDLine(tb, testRepo.Algorithm(), testRepo.Run(tb, "show", "-s", "--format=%P", id.String())) - - gotParents := commitParents(tb, reader, commit) - if len(gotParents) != len(wantParents) { - tb.Fatalf("parent count for %s = %d, want %d", id, len(gotParents), len(wantParents)) - } - - for i := range gotParents { - if gotParents[i] != wantParents[i] { - tb.Fatalf("parent %d for %s = %s, want %s", i, id, gotParents[i], wantParents[i]) - } - } - - commitTimeRaw := testRepo.Run(tb, "show", "-s", "--format=%ct", id.String()) - - wantCommitTime, err := strconv.ParseInt(strings.TrimSpace(commitTimeRaw), 10, 64) - if err != nil { - tb.Fatalf("parse commit time %q: %v", commitTimeRaw, err) - } - - if commit.CommitTimeUnix != wantCommitTime { - tb.Fatalf("CommitAt(%+v).CommitTimeUnix = %d, want %d", pos, commit.CommitTimeUnix, wantCommitTime) - } - - filter, err := reader.BloomFilterAt(pos) - if err != nil { - tb.Fatalf("BloomFilterAt(%+v): %v", pos, err) - } - - if filter.HashVersion != uint32(reader.BloomVersion()) { - tb.Fatalf("filter.HashVersion = %d, want %d", filter.HashVersion, reader.BloomVersion()) - } - - assertChangedPathsBloomPositive(tb, testRepo, filter, id) -} - -func commitParents(tb testing.TB, reader *read.Reader, commit read.Commit) []objectid.ObjectID { - tb.Helper() - - out := make([]objectid.ObjectID, 0, 2+len(commit.ExtraParents)) - - if commit.Parent1.Valid { - id, err := reader.OIDAt(commit.Parent1.Pos) - if err != nil { - tb.Fatalf("OIDAt(parent1 %+v): %v", commit.Parent1.Pos, err) - } - - out = append(out, id) - } - - if commit.Parent2.Valid { - id, err := reader.OIDAt(commit.Parent2.Pos) - if err != nil { - tb.Fatalf("OIDAt(parent2 %+v): %v", commit.Parent2.Pos, err) - } - - out = append(out, id) - } - - for _, parentPos := range commit.ExtraParents { - id, err := reader.OIDAt(parentPos) - if err != nil { - tb.Fatalf("OIDAt(extra parent %+v): %v", parentPos, err) - } - - out = append(out, id) - } - - return out -} - -func assertChangedPathsBloomPositive(tb testing.TB, testRepo *testgit.TestRepo, filter bloom.Filter, commitID objectid.ObjectID) { - tb.Helper() - - changedPaths := testRepo.Run(tb, "diff-tree", "--no-commit-id", "--name-only", "-r", "--root", commitID.String()) - for line := range strings.SplitSeq(strings.TrimSpace(changedPaths), "\n") { - path := strings.TrimSpace(line) - if path == "" { - continue - } - - mightContain, err := filter.MightContain([]byte(path)) - if err != nil { - tb.Fatalf("MightContain(%q): %v", path, err) - } - - if !mightContain { - tb.Fatalf("Bloom filter false negative for commit %s path %q", commitID, path) - } - } -} - -func parseOIDLine(tb testing.TB, algo objectid.Algorithm, line string) []objectid.ObjectID { - tb.Helper() - - toks := strings.Fields(line) - - out := make([]objectid.ObjectID, 0, len(toks)) - for _, tok := range toks { - id, err := objectid.ParseHex(algo, tok) - if err != nil { - tb.Fatalf("parse object id %q: %v", tok, err) - } - - out = append(out, id) - } - - return out -} diff --git a/commitgraph/read/reader.go b/commitgraph/read/reader.go deleted file mode 100644 index d5c84a70..00000000 --- a/commitgraph/read/reader.go +++ /dev/null @@ -1,16 +0,0 @@ -package read - -import objectid "codeberg.org/lindenii/furgit/object/id" - -// Reader provides read-only access to one mmap-backed commit-graph snapshot. -// -// It is safe for concurrent read-only queries. -// Values returned by Reader methods are only valid until the reader is closed -// when explicitly documented on that method. -type Reader struct { - algo objectid.Algorithm - hashVersion uint8 - - layers []layer - total uint32 -} diff --git a/commitgraph/read/testdata/fixtures/sha1/chain_changed/repo.git/HEAD b/commitgraph/read/testdata/fixtures/sha1/chain_changed/repo.git/HEAD deleted file mode 100644 index cb089cd8..00000000 --- a/commitgraph/read/testdata/fixtures/sha1/chain_changed/repo.git/HEAD +++ /dev/null @@ -1 +0,0 @@ -ref: refs/heads/master diff --git a/commitgraph/read/testdata/fixtures/sha1/chain_changed/repo.git/config b/commitgraph/read/testdata/fixtures/sha1/chain_changed/repo.git/config deleted file mode 100644 index 07d359d0..00000000 --- a/commitgraph/read/testdata/fixtures/sha1/chain_changed/repo.git/config +++ /dev/null @@ -1,4 +0,0 @@ -[core] - repositoryformatversion = 0 - filemode = true - bare = true diff --git a/commitgraph/read/testdata/fixtures/sha1/chain_changed/repo.git/objects/info/commit-graphs/commit-graph-chain b/commitgraph/read/testdata/fixtures/sha1/chain_changed/repo.git/objects/info/commit-graphs/commit-graph-chain deleted file mode 100644 index 74c46b64..00000000 --- a/commitgraph/read/testdata/fixtures/sha1/chain_changed/repo.git/objects/info/commit-graphs/commit-graph-chain +++ /dev/null @@ -1,2 +0,0 @@ -dd7578d5216ca76c25b19631ba90f7498aeabbe7 -bf985c21612a52070d8b008e6ef51edf8b609401 diff --git a/commitgraph/read/testdata/fixtures/sha1/chain_changed/repo.git/objects/info/commit-graphs/graph-bf985c21612a52070d8b008e6ef51edf8b609401.graph b/commitgraph/read/testdata/fixtures/sha1/chain_changed/repo.git/objects/info/commit-graphs/graph-bf985c21612a52070d8b008e6ef51edf8b609401.graph deleted file mode 100644 index c31869c1..00000000 Binary files a/commitgraph/read/testdata/fixtures/sha1/chain_changed/repo.git/objects/info/commit-graphs/graph-bf985c21612a52070d8b008e6ef51edf8b609401.graph and /dev/null differ diff --git a/commitgraph/read/testdata/fixtures/sha1/chain_changed/repo.git/objects/info/commit-graphs/graph-dd7578d5216ca76c25b19631ba90f7498aeabbe7.graph b/commitgraph/read/testdata/fixtures/sha1/chain_changed/repo.git/objects/info/commit-graphs/graph-dd7578d5216ca76c25b19631ba90f7498aeabbe7.graph deleted file mode 100644 index 241eb3cc..00000000 Binary files a/commitgraph/read/testdata/fixtures/sha1/chain_changed/repo.git/objects/info/commit-graphs/graph-dd7578d5216ca76c25b19631ba90f7498aeabbe7.graph and /dev/null differ diff --git a/commitgraph/read/testdata/fixtures/sha1/chain_changed/repo.git/objects/info/packs b/commitgraph/read/testdata/fixtures/sha1/chain_changed/repo.git/objects/info/packs deleted file mode 100644 index 61decf9b..00000000 --- a/commitgraph/read/testdata/fixtures/sha1/chain_changed/repo.git/objects/info/packs +++ /dev/null @@ -1,2 +0,0 @@ -P pack-15b064d6a8ef8cff520565f6db8c006b2e6f7f2f.pack - diff --git a/commitgraph/read/testdata/fixtures/sha1/chain_changed/repo.git/objects/pack/pack-15b064d6a8ef8cff520565f6db8c006b2e6f7f2f.bitmap b/commitgraph/read/testdata/fixtures/sha1/chain_changed/repo.git/objects/pack/pack-15b064d6a8ef8cff520565f6db8c006b2e6f7f2f.bitmap deleted file mode 100644 index 1508cf18..00000000 Binary files a/commitgraph/read/testdata/fixtures/sha1/chain_changed/repo.git/objects/pack/pack-15b064d6a8ef8cff520565f6db8c006b2e6f7f2f.bitmap and /dev/null differ diff --git a/commitgraph/read/testdata/fixtures/sha1/chain_changed/repo.git/objects/pack/pack-15b064d6a8ef8cff520565f6db8c006b2e6f7f2f.idx b/commitgraph/read/testdata/fixtures/sha1/chain_changed/repo.git/objects/pack/pack-15b064d6a8ef8cff520565f6db8c006b2e6f7f2f.idx deleted file mode 100644 index 00ee2646..00000000 Binary files a/commitgraph/read/testdata/fixtures/sha1/chain_changed/repo.git/objects/pack/pack-15b064d6a8ef8cff520565f6db8c006b2e6f7f2f.idx and /dev/null differ diff --git a/commitgraph/read/testdata/fixtures/sha1/chain_changed/repo.git/objects/pack/pack-15b064d6a8ef8cff520565f6db8c006b2e6f7f2f.pack b/commitgraph/read/testdata/fixtures/sha1/chain_changed/repo.git/objects/pack/pack-15b064d6a8ef8cff520565f6db8c006b2e6f7f2f.pack deleted file mode 100644 index c65ae27f..00000000 Binary files a/commitgraph/read/testdata/fixtures/sha1/chain_changed/repo.git/objects/pack/pack-15b064d6a8ef8cff520565f6db8c006b2e6f7f2f.pack and /dev/null differ diff --git a/commitgraph/read/testdata/fixtures/sha1/chain_changed/repo.git/objects/pack/pack-15b064d6a8ef8cff520565f6db8c006b2e6f7f2f.rev b/commitgraph/read/testdata/fixtures/sha1/chain_changed/repo.git/objects/pack/pack-15b064d6a8ef8cff520565f6db8c006b2e6f7f2f.rev deleted file mode 100644 index d0689f72..00000000 Binary files a/commitgraph/read/testdata/fixtures/sha1/chain_changed/repo.git/objects/pack/pack-15b064d6a8ef8cff520565f6db8c006b2e6f7f2f.rev and /dev/null differ diff --git a/commitgraph/read/testdata/fixtures/sha1/chain_changed/repo.git/refs/heads/master b/commitgraph/read/testdata/fixtures/sha1/chain_changed/repo.git/refs/heads/master deleted file mode 100644 index 8942d437..00000000 --- a/commitgraph/read/testdata/fixtures/sha1/chain_changed/repo.git/refs/heads/master +++ /dev/null @@ -1 +0,0 @@ -46ca641fd65e566b8ecfa567a1f01766289192f8 diff --git a/commitgraph/read/testdata/fixtures/sha1/single_changed/repo.git/HEAD b/commitgraph/read/testdata/fixtures/sha1/single_changed/repo.git/HEAD deleted file mode 100644 index b870d826..00000000 --- a/commitgraph/read/testdata/fixtures/sha1/single_changed/repo.git/HEAD +++ /dev/null @@ -1 +0,0 @@ -ref: refs/heads/main diff --git a/commitgraph/read/testdata/fixtures/sha1/single_changed/repo.git/config b/commitgraph/read/testdata/fixtures/sha1/single_changed/repo.git/config deleted file mode 100644 index 07d359d0..00000000 --- a/commitgraph/read/testdata/fixtures/sha1/single_changed/repo.git/config +++ /dev/null @@ -1,4 +0,0 @@ -[core] - repositoryformatversion = 0 - filemode = true - bare = true diff --git a/commitgraph/read/testdata/fixtures/sha1/single_changed/repo.git/objects/info/commit-graph b/commitgraph/read/testdata/fixtures/sha1/single_changed/repo.git/objects/info/commit-graph deleted file mode 100644 index 56b59a54..00000000 Binary files a/commitgraph/read/testdata/fixtures/sha1/single_changed/repo.git/objects/info/commit-graph and /dev/null differ diff --git a/commitgraph/read/testdata/fixtures/sha1/single_changed/repo.git/objects/info/packs b/commitgraph/read/testdata/fixtures/sha1/single_changed/repo.git/objects/info/packs deleted file mode 100644 index ecf5d272..00000000 --- a/commitgraph/read/testdata/fixtures/sha1/single_changed/repo.git/objects/info/packs +++ /dev/null @@ -1,2 +0,0 @@ -P pack-34e9e132566989e2abfe8821731236c77f9bcbe9.pack - diff --git a/commitgraph/read/testdata/fixtures/sha1/single_changed/repo.git/objects/pack/pack-34e9e132566989e2abfe8821731236c77f9bcbe9.bitmap b/commitgraph/read/testdata/fixtures/sha1/single_changed/repo.git/objects/pack/pack-34e9e132566989e2abfe8821731236c77f9bcbe9.bitmap deleted file mode 100644 index 9fec7b16..00000000 Binary files a/commitgraph/read/testdata/fixtures/sha1/single_changed/repo.git/objects/pack/pack-34e9e132566989e2abfe8821731236c77f9bcbe9.bitmap and /dev/null differ diff --git a/commitgraph/read/testdata/fixtures/sha1/single_changed/repo.git/objects/pack/pack-34e9e132566989e2abfe8821731236c77f9bcbe9.idx b/commitgraph/read/testdata/fixtures/sha1/single_changed/repo.git/objects/pack/pack-34e9e132566989e2abfe8821731236c77f9bcbe9.idx deleted file mode 100644 index e30cbb5a..00000000 Binary files a/commitgraph/read/testdata/fixtures/sha1/single_changed/repo.git/objects/pack/pack-34e9e132566989e2abfe8821731236c77f9bcbe9.idx and /dev/null differ diff --git a/commitgraph/read/testdata/fixtures/sha1/single_changed/repo.git/objects/pack/pack-34e9e132566989e2abfe8821731236c77f9bcbe9.pack b/commitgraph/read/testdata/fixtures/sha1/single_changed/repo.git/objects/pack/pack-34e9e132566989e2abfe8821731236c77f9bcbe9.pack deleted file mode 100644 index 8da45eab..00000000 Binary files a/commitgraph/read/testdata/fixtures/sha1/single_changed/repo.git/objects/pack/pack-34e9e132566989e2abfe8821731236c77f9bcbe9.pack and /dev/null differ diff --git a/commitgraph/read/testdata/fixtures/sha1/single_changed/repo.git/objects/pack/pack-34e9e132566989e2abfe8821731236c77f9bcbe9.rev b/commitgraph/read/testdata/fixtures/sha1/single_changed/repo.git/objects/pack/pack-34e9e132566989e2abfe8821731236c77f9bcbe9.rev deleted file mode 100644 index 3bcd2e2c..00000000 Binary files a/commitgraph/read/testdata/fixtures/sha1/single_changed/repo.git/objects/pack/pack-34e9e132566989e2abfe8821731236c77f9bcbe9.rev and /dev/null differ diff --git a/commitgraph/read/testdata/fixtures/sha1/single_changed/repo.git/refs/heads/main b/commitgraph/read/testdata/fixtures/sha1/single_changed/repo.git/refs/heads/main deleted file mode 100644 index 090ca933..00000000 --- a/commitgraph/read/testdata/fixtures/sha1/single_changed/repo.git/refs/heads/main +++ /dev/null @@ -1 +0,0 @@ -d02a8dbd1a8fbaac8ab7f7f1533cc312ab2c9eec diff --git a/commitgraph/read/testdata/fixtures/sha1/single_nochanged/repo.git/HEAD b/commitgraph/read/testdata/fixtures/sha1/single_nochanged/repo.git/HEAD deleted file mode 100644 index cb089cd8..00000000 --- a/commitgraph/read/testdata/fixtures/sha1/single_nochanged/repo.git/HEAD +++ /dev/null @@ -1 +0,0 @@ -ref: refs/heads/master diff --git a/commitgraph/read/testdata/fixtures/sha1/single_nochanged/repo.git/config b/commitgraph/read/testdata/fixtures/sha1/single_nochanged/repo.git/config deleted file mode 100644 index 07d359d0..00000000 --- a/commitgraph/read/testdata/fixtures/sha1/single_nochanged/repo.git/config +++ /dev/null @@ -1,4 +0,0 @@ -[core] - repositoryformatversion = 0 - filemode = true - bare = true diff --git a/commitgraph/read/testdata/fixtures/sha1/single_nochanged/repo.git/objects/info/commit-graph b/commitgraph/read/testdata/fixtures/sha1/single_nochanged/repo.git/objects/info/commit-graph deleted file mode 100644 index 28f7d06a..00000000 Binary files a/commitgraph/read/testdata/fixtures/sha1/single_nochanged/repo.git/objects/info/commit-graph and /dev/null differ diff --git a/commitgraph/read/testdata/fixtures/sha1/single_nochanged/repo.git/objects/info/packs b/commitgraph/read/testdata/fixtures/sha1/single_nochanged/repo.git/objects/info/packs deleted file mode 100644 index 8434a002..00000000 --- a/commitgraph/read/testdata/fixtures/sha1/single_nochanged/repo.git/objects/info/packs +++ /dev/null @@ -1,2 +0,0 @@ -P pack-a3da595034c94bb16b6829d757a66b7d259b9ffc.pack - diff --git a/commitgraph/read/testdata/fixtures/sha1/single_nochanged/repo.git/objects/pack/pack-a3da595034c94bb16b6829d757a66b7d259b9ffc.bitmap b/commitgraph/read/testdata/fixtures/sha1/single_nochanged/repo.git/objects/pack/pack-a3da595034c94bb16b6829d757a66b7d259b9ffc.bitmap deleted file mode 100644 index 64a36c71..00000000 Binary files a/commitgraph/read/testdata/fixtures/sha1/single_nochanged/repo.git/objects/pack/pack-a3da595034c94bb16b6829d757a66b7d259b9ffc.bitmap and /dev/null differ diff --git a/commitgraph/read/testdata/fixtures/sha1/single_nochanged/repo.git/objects/pack/pack-a3da595034c94bb16b6829d757a66b7d259b9ffc.idx b/commitgraph/read/testdata/fixtures/sha1/single_nochanged/repo.git/objects/pack/pack-a3da595034c94bb16b6829d757a66b7d259b9ffc.idx deleted file mode 100644 index f5e16674..00000000 Binary files a/commitgraph/read/testdata/fixtures/sha1/single_nochanged/repo.git/objects/pack/pack-a3da595034c94bb16b6829d757a66b7d259b9ffc.idx and /dev/null differ diff --git a/commitgraph/read/testdata/fixtures/sha1/single_nochanged/repo.git/objects/pack/pack-a3da595034c94bb16b6829d757a66b7d259b9ffc.pack b/commitgraph/read/testdata/fixtures/sha1/single_nochanged/repo.git/objects/pack/pack-a3da595034c94bb16b6829d757a66b7d259b9ffc.pack deleted file mode 100644 index 8f82b451..00000000 Binary files a/commitgraph/read/testdata/fixtures/sha1/single_nochanged/repo.git/objects/pack/pack-a3da595034c94bb16b6829d757a66b7d259b9ffc.pack and /dev/null differ diff --git a/commitgraph/read/testdata/fixtures/sha1/single_nochanged/repo.git/objects/pack/pack-a3da595034c94bb16b6829d757a66b7d259b9ffc.rev b/commitgraph/read/testdata/fixtures/sha1/single_nochanged/repo.git/objects/pack/pack-a3da595034c94bb16b6829d757a66b7d259b9ffc.rev deleted file mode 100644 index 64771f70..00000000 Binary files a/commitgraph/read/testdata/fixtures/sha1/single_nochanged/repo.git/objects/pack/pack-a3da595034c94bb16b6829d757a66b7d259b9ffc.rev and /dev/null differ diff --git a/commitgraph/read/testdata/fixtures/sha1/single_nochanged/repo.git/refs/heads/master b/commitgraph/read/testdata/fixtures/sha1/single_nochanged/repo.git/refs/heads/master deleted file mode 100644 index 475cb2c1..00000000 --- a/commitgraph/read/testdata/fixtures/sha1/single_nochanged/repo.git/refs/heads/master +++ /dev/null @@ -1 +0,0 @@ -dda8217252bdf3e01fdf31309d0e5c3051b00945 diff --git a/commitgraph/read/testdata/fixtures/sha256/chain_changed/repo.git/HEAD b/commitgraph/read/testdata/fixtures/sha256/chain_changed/repo.git/HEAD deleted file mode 100644 index cb089cd8..00000000 --- a/commitgraph/read/testdata/fixtures/sha256/chain_changed/repo.git/HEAD +++ /dev/null @@ -1 +0,0 @@ -ref: refs/heads/master diff --git a/commitgraph/read/testdata/fixtures/sha256/chain_changed/repo.git/config b/commitgraph/read/testdata/fixtures/sha256/chain_changed/repo.git/config deleted file mode 100644 index 7d1c0006..00000000 --- a/commitgraph/read/testdata/fixtures/sha256/chain_changed/repo.git/config +++ /dev/null @@ -1,6 +0,0 @@ -[extensions] - objectformat = sha256 -[core] - repositoryformatversion = 1 - filemode = true - bare = true diff --git a/commitgraph/read/testdata/fixtures/sha256/chain_changed/repo.git/objects/info/commit-graphs/commit-graph-chain b/commitgraph/read/testdata/fixtures/sha256/chain_changed/repo.git/objects/info/commit-graphs/commit-graph-chain deleted file mode 100644 index 4e7d76fe..00000000 --- a/commitgraph/read/testdata/fixtures/sha256/chain_changed/repo.git/objects/info/commit-graphs/commit-graph-chain +++ /dev/null @@ -1,2 +0,0 @@ -505cab61f8ddfa614301e8f97943112739236c6bcd19ed4d1f7c6b830cab4f62 -77c47bd6ca2ce17208c9361717a5823c0cb4b5ee336a14959678e060d674ffb6 diff --git a/commitgraph/read/testdata/fixtures/sha256/chain_changed/repo.git/objects/info/commit-graphs/graph-505cab61f8ddfa614301e8f97943112739236c6bcd19ed4d1f7c6b830cab4f62.graph b/commitgraph/read/testdata/fixtures/sha256/chain_changed/repo.git/objects/info/commit-graphs/graph-505cab61f8ddfa614301e8f97943112739236c6bcd19ed4d1f7c6b830cab4f62.graph deleted file mode 100644 index 4a93de94..00000000 Binary files a/commitgraph/read/testdata/fixtures/sha256/chain_changed/repo.git/objects/info/commit-graphs/graph-505cab61f8ddfa614301e8f97943112739236c6bcd19ed4d1f7c6b830cab4f62.graph and /dev/null differ diff --git a/commitgraph/read/testdata/fixtures/sha256/chain_changed/repo.git/objects/info/commit-graphs/graph-77c47bd6ca2ce17208c9361717a5823c0cb4b5ee336a14959678e060d674ffb6.graph b/commitgraph/read/testdata/fixtures/sha256/chain_changed/repo.git/objects/info/commit-graphs/graph-77c47bd6ca2ce17208c9361717a5823c0cb4b5ee336a14959678e060d674ffb6.graph deleted file mode 100644 index 7807351d..00000000 Binary files a/commitgraph/read/testdata/fixtures/sha256/chain_changed/repo.git/objects/info/commit-graphs/graph-77c47bd6ca2ce17208c9361717a5823c0cb4b5ee336a14959678e060d674ffb6.graph and /dev/null differ diff --git a/commitgraph/read/testdata/fixtures/sha256/chain_changed/repo.git/objects/info/packs b/commitgraph/read/testdata/fixtures/sha256/chain_changed/repo.git/objects/info/packs deleted file mode 100644 index 3b1241c4..00000000 --- a/commitgraph/read/testdata/fixtures/sha256/chain_changed/repo.git/objects/info/packs +++ /dev/null @@ -1,2 +0,0 @@ -P pack-04168d0884c910f505cb9fbcf045957e44ccee06d812b5e531ae666014a26ed1.pack - diff --git a/commitgraph/read/testdata/fixtures/sha256/chain_changed/repo.git/objects/pack/pack-04168d0884c910f505cb9fbcf045957e44ccee06d812b5e531ae666014a26ed1.bitmap b/commitgraph/read/testdata/fixtures/sha256/chain_changed/repo.git/objects/pack/pack-04168d0884c910f505cb9fbcf045957e44ccee06d812b5e531ae666014a26ed1.bitmap deleted file mode 100644 index 007fcd0e..00000000 Binary files a/commitgraph/read/testdata/fixtures/sha256/chain_changed/repo.git/objects/pack/pack-04168d0884c910f505cb9fbcf045957e44ccee06d812b5e531ae666014a26ed1.bitmap and /dev/null differ diff --git a/commitgraph/read/testdata/fixtures/sha256/chain_changed/repo.git/objects/pack/pack-04168d0884c910f505cb9fbcf045957e44ccee06d812b5e531ae666014a26ed1.idx b/commitgraph/read/testdata/fixtures/sha256/chain_changed/repo.git/objects/pack/pack-04168d0884c910f505cb9fbcf045957e44ccee06d812b5e531ae666014a26ed1.idx deleted file mode 100644 index 248cf8fc..00000000 Binary files a/commitgraph/read/testdata/fixtures/sha256/chain_changed/repo.git/objects/pack/pack-04168d0884c910f505cb9fbcf045957e44ccee06d812b5e531ae666014a26ed1.idx and /dev/null differ diff --git a/commitgraph/read/testdata/fixtures/sha256/chain_changed/repo.git/objects/pack/pack-04168d0884c910f505cb9fbcf045957e44ccee06d812b5e531ae666014a26ed1.pack b/commitgraph/read/testdata/fixtures/sha256/chain_changed/repo.git/objects/pack/pack-04168d0884c910f505cb9fbcf045957e44ccee06d812b5e531ae666014a26ed1.pack deleted file mode 100644 index 92cea7fb..00000000 Binary files a/commitgraph/read/testdata/fixtures/sha256/chain_changed/repo.git/objects/pack/pack-04168d0884c910f505cb9fbcf045957e44ccee06d812b5e531ae666014a26ed1.pack and /dev/null differ diff --git a/commitgraph/read/testdata/fixtures/sha256/chain_changed/repo.git/objects/pack/pack-04168d0884c910f505cb9fbcf045957e44ccee06d812b5e531ae666014a26ed1.rev b/commitgraph/read/testdata/fixtures/sha256/chain_changed/repo.git/objects/pack/pack-04168d0884c910f505cb9fbcf045957e44ccee06d812b5e531ae666014a26ed1.rev deleted file mode 100644 index 569862ce..00000000 Binary files a/commitgraph/read/testdata/fixtures/sha256/chain_changed/repo.git/objects/pack/pack-04168d0884c910f505cb9fbcf045957e44ccee06d812b5e531ae666014a26ed1.rev and /dev/null differ diff --git a/commitgraph/read/testdata/fixtures/sha256/chain_changed/repo.git/refs/heads/master b/commitgraph/read/testdata/fixtures/sha256/chain_changed/repo.git/refs/heads/master deleted file mode 100644 index 29d83be8..00000000 --- a/commitgraph/read/testdata/fixtures/sha256/chain_changed/repo.git/refs/heads/master +++ /dev/null @@ -1 +0,0 @@ -10d2943dc7ad88011cae3b776d9565d6451a350ce1d16949bc8546a5fe6c0a53 diff --git a/commitgraph/read/testdata/fixtures/sha256/single_changed/repo.git/HEAD b/commitgraph/read/testdata/fixtures/sha256/single_changed/repo.git/HEAD deleted file mode 100644 index b870d826..00000000 --- a/commitgraph/read/testdata/fixtures/sha256/single_changed/repo.git/HEAD +++ /dev/null @@ -1 +0,0 @@ -ref: refs/heads/main diff --git a/commitgraph/read/testdata/fixtures/sha256/single_changed/repo.git/config b/commitgraph/read/testdata/fixtures/sha256/single_changed/repo.git/config deleted file mode 100644 index 7d1c0006..00000000 --- a/commitgraph/read/testdata/fixtures/sha256/single_changed/repo.git/config +++ /dev/null @@ -1,6 +0,0 @@ -[extensions] - objectformat = sha256 -[core] - repositoryformatversion = 1 - filemode = true - bare = true diff --git a/commitgraph/read/testdata/fixtures/sha256/single_changed/repo.git/objects/info/commit-graph b/commitgraph/read/testdata/fixtures/sha256/single_changed/repo.git/objects/info/commit-graph deleted file mode 100644 index f4dd0e0c..00000000 Binary files a/commitgraph/read/testdata/fixtures/sha256/single_changed/repo.git/objects/info/commit-graph and /dev/null differ diff --git a/commitgraph/read/testdata/fixtures/sha256/single_changed/repo.git/objects/info/packs b/commitgraph/read/testdata/fixtures/sha256/single_changed/repo.git/objects/info/packs deleted file mode 100644 index 0f39ed89..00000000 --- a/commitgraph/read/testdata/fixtures/sha256/single_changed/repo.git/objects/info/packs +++ /dev/null @@ -1,2 +0,0 @@ -P pack-316dbc67dac12d131591640da0c55b76387cbf1fd2a117ab3d7ca0d854a031c9.pack - diff --git a/commitgraph/read/testdata/fixtures/sha256/single_changed/repo.git/objects/pack/pack-316dbc67dac12d131591640da0c55b76387cbf1fd2a117ab3d7ca0d854a031c9.bitmap b/commitgraph/read/testdata/fixtures/sha256/single_changed/repo.git/objects/pack/pack-316dbc67dac12d131591640da0c55b76387cbf1fd2a117ab3d7ca0d854a031c9.bitmap deleted file mode 100644 index b5c5055c..00000000 Binary files a/commitgraph/read/testdata/fixtures/sha256/single_changed/repo.git/objects/pack/pack-316dbc67dac12d131591640da0c55b76387cbf1fd2a117ab3d7ca0d854a031c9.bitmap and /dev/null differ diff --git a/commitgraph/read/testdata/fixtures/sha256/single_changed/repo.git/objects/pack/pack-316dbc67dac12d131591640da0c55b76387cbf1fd2a117ab3d7ca0d854a031c9.idx b/commitgraph/read/testdata/fixtures/sha256/single_changed/repo.git/objects/pack/pack-316dbc67dac12d131591640da0c55b76387cbf1fd2a117ab3d7ca0d854a031c9.idx deleted file mode 100644 index 144778cd..00000000 Binary files a/commitgraph/read/testdata/fixtures/sha256/single_changed/repo.git/objects/pack/pack-316dbc67dac12d131591640da0c55b76387cbf1fd2a117ab3d7ca0d854a031c9.idx and /dev/null differ diff --git a/commitgraph/read/testdata/fixtures/sha256/single_changed/repo.git/objects/pack/pack-316dbc67dac12d131591640da0c55b76387cbf1fd2a117ab3d7ca0d854a031c9.pack b/commitgraph/read/testdata/fixtures/sha256/single_changed/repo.git/objects/pack/pack-316dbc67dac12d131591640da0c55b76387cbf1fd2a117ab3d7ca0d854a031c9.pack deleted file mode 100644 index 599ccae0..00000000 Binary files a/commitgraph/read/testdata/fixtures/sha256/single_changed/repo.git/objects/pack/pack-316dbc67dac12d131591640da0c55b76387cbf1fd2a117ab3d7ca0d854a031c9.pack and /dev/null differ diff --git a/commitgraph/read/testdata/fixtures/sha256/single_changed/repo.git/objects/pack/pack-316dbc67dac12d131591640da0c55b76387cbf1fd2a117ab3d7ca0d854a031c9.rev b/commitgraph/read/testdata/fixtures/sha256/single_changed/repo.git/objects/pack/pack-316dbc67dac12d131591640da0c55b76387cbf1fd2a117ab3d7ca0d854a031c9.rev deleted file mode 100644 index 3c093f31..00000000 Binary files a/commitgraph/read/testdata/fixtures/sha256/single_changed/repo.git/objects/pack/pack-316dbc67dac12d131591640da0c55b76387cbf1fd2a117ab3d7ca0d854a031c9.rev and /dev/null differ diff --git a/commitgraph/read/testdata/fixtures/sha256/single_changed/repo.git/refs/heads/main b/commitgraph/read/testdata/fixtures/sha256/single_changed/repo.git/refs/heads/main deleted file mode 100644 index 4ba32358..00000000 --- a/commitgraph/read/testdata/fixtures/sha256/single_changed/repo.git/refs/heads/main +++ /dev/null @@ -1 +0,0 @@ -a9ff114900e6be139ec66a2a61c930973d8c4bc6fd3b899405ee7ab8740bdbd3 diff --git a/commitgraph/read/testdata/fixtures/sha256/single_nochanged/repo.git/HEAD b/commitgraph/read/testdata/fixtures/sha256/single_nochanged/repo.git/HEAD deleted file mode 100644 index cb089cd8..00000000 --- a/commitgraph/read/testdata/fixtures/sha256/single_nochanged/repo.git/HEAD +++ /dev/null @@ -1 +0,0 @@ -ref: refs/heads/master diff --git a/commitgraph/read/testdata/fixtures/sha256/single_nochanged/repo.git/config b/commitgraph/read/testdata/fixtures/sha256/single_nochanged/repo.git/config deleted file mode 100644 index 7d1c0006..00000000 --- a/commitgraph/read/testdata/fixtures/sha256/single_nochanged/repo.git/config +++ /dev/null @@ -1,6 +0,0 @@ -[extensions] - objectformat = sha256 -[core] - repositoryformatversion = 1 - filemode = true - bare = true diff --git a/commitgraph/read/testdata/fixtures/sha256/single_nochanged/repo.git/objects/info/commit-graph b/commitgraph/read/testdata/fixtures/sha256/single_nochanged/repo.git/objects/info/commit-graph deleted file mode 100644 index f98ca4a1..00000000 Binary files a/commitgraph/read/testdata/fixtures/sha256/single_nochanged/repo.git/objects/info/commit-graph and /dev/null differ diff --git a/commitgraph/read/testdata/fixtures/sha256/single_nochanged/repo.git/objects/info/packs b/commitgraph/read/testdata/fixtures/sha256/single_nochanged/repo.git/objects/info/packs deleted file mode 100644 index 65184c9a..00000000 --- a/commitgraph/read/testdata/fixtures/sha256/single_nochanged/repo.git/objects/info/packs +++ /dev/null @@ -1,2 +0,0 @@ -P pack-d335453f760b064e36459d780ec9bf0e5dd596c0ee1ac6310136067c4f13438b.pack - diff --git a/commitgraph/read/testdata/fixtures/sha256/single_nochanged/repo.git/objects/pack/pack-d335453f760b064e36459d780ec9bf0e5dd596c0ee1ac6310136067c4f13438b.bitmap b/commitgraph/read/testdata/fixtures/sha256/single_nochanged/repo.git/objects/pack/pack-d335453f760b064e36459d780ec9bf0e5dd596c0ee1ac6310136067c4f13438b.bitmap deleted file mode 100644 index 53530f4c..00000000 Binary files a/commitgraph/read/testdata/fixtures/sha256/single_nochanged/repo.git/objects/pack/pack-d335453f760b064e36459d780ec9bf0e5dd596c0ee1ac6310136067c4f13438b.bitmap and /dev/null differ diff --git a/commitgraph/read/testdata/fixtures/sha256/single_nochanged/repo.git/objects/pack/pack-d335453f760b064e36459d780ec9bf0e5dd596c0ee1ac6310136067c4f13438b.idx b/commitgraph/read/testdata/fixtures/sha256/single_nochanged/repo.git/objects/pack/pack-d335453f760b064e36459d780ec9bf0e5dd596c0ee1ac6310136067c4f13438b.idx deleted file mode 100644 index b3a417a8..00000000 Binary files a/commitgraph/read/testdata/fixtures/sha256/single_nochanged/repo.git/objects/pack/pack-d335453f760b064e36459d780ec9bf0e5dd596c0ee1ac6310136067c4f13438b.idx and /dev/null differ diff --git a/commitgraph/read/testdata/fixtures/sha256/single_nochanged/repo.git/objects/pack/pack-d335453f760b064e36459d780ec9bf0e5dd596c0ee1ac6310136067c4f13438b.pack b/commitgraph/read/testdata/fixtures/sha256/single_nochanged/repo.git/objects/pack/pack-d335453f760b064e36459d780ec9bf0e5dd596c0ee1ac6310136067c4f13438b.pack deleted file mode 100644 index d8dcedbf..00000000 Binary files a/commitgraph/read/testdata/fixtures/sha256/single_nochanged/repo.git/objects/pack/pack-d335453f760b064e36459d780ec9bf0e5dd596c0ee1ac6310136067c4f13438b.pack and /dev/null differ diff --git a/commitgraph/read/testdata/fixtures/sha256/single_nochanged/repo.git/objects/pack/pack-d335453f760b064e36459d780ec9bf0e5dd596c0ee1ac6310136067c4f13438b.rev b/commitgraph/read/testdata/fixtures/sha256/single_nochanged/repo.git/objects/pack/pack-d335453f760b064e36459d780ec9bf0e5dd596c0ee1ac6310136067c4f13438b.rev deleted file mode 100644 index e50d1a81..00000000 Binary files a/commitgraph/read/testdata/fixtures/sha256/single_nochanged/repo.git/objects/pack/pack-d335453f760b064e36459d780ec9bf0e5dd596c0ee1ac6310136067c4f13438b.rev and /dev/null differ diff --git a/commitgraph/read/testdata/fixtures/sha256/single_nochanged/repo.git/refs/heads/master b/commitgraph/read/testdata/fixtures/sha256/single_nochanged/repo.git/refs/heads/master deleted file mode 100644 index a4e184b4..00000000 --- a/commitgraph/read/testdata/fixtures/sha256/single_nochanged/repo.git/refs/heads/master +++ /dev/null @@ -1 +0,0 @@ -7e396bf648e3b045c293d9fbdc533d4377d4e801d5d1fb57b84d22dd054a5860 diff --git a/commitquery/commit.go b/commitquery/commit.go index 8e60e3fd..dff6a91c 100644 --- a/commitquery/commit.go +++ b/commitquery/commit.go @@ -1,7 +1,7 @@ package commitquery import ( - commitgraphread "codeberg.org/lindenii/furgit/commitgraph/read" + commitgraphread "codeberg.org/lindenii/furgit/format/commitgraph/read" objectid "codeberg.org/lindenii/furgit/object/id" ) diff --git a/commitquery/context.go b/commitquery/context.go index df8ddd97..74f094b1 100644 --- a/commitquery/context.go +++ b/commitquery/context.go @@ -2,7 +2,7 @@ package commitquery import ( - commitgraphread "codeberg.org/lindenii/furgit/commitgraph/read" + commitgraphread "codeberg.org/lindenii/furgit/format/commitgraph/read" objectid "codeberg.org/lindenii/furgit/object/id" objectstorer "codeberg.org/lindenii/furgit/object/storer" ) diff --git a/commitquery/graph_pos.go b/commitquery/graph_pos.go index b1d27129..b5bc02d8 100644 --- a/commitquery/graph_pos.go +++ b/commitquery/graph_pos.go @@ -1,6 +1,6 @@ package commitquery -import commitgraphread "codeberg.org/lindenii/furgit/commitgraph/read" +import commitgraphread "codeberg.org/lindenii/furgit/format/commitgraph/read" // resolveGraphPos resolves one commit-graph position to one internal query node. func (query *Query) resolveGraphPos(pos commitgraphread.Position) (nodeIndex, error) { diff --git a/commitquery/node.go b/commitquery/node.go index 5b1c7142..c0f25c55 100644 --- a/commitquery/node.go +++ b/commitquery/node.go @@ -1,7 +1,7 @@ package commitquery import ( - commitgraphread "codeberg.org/lindenii/furgit/commitgraph/read" + commitgraphread "codeberg.org/lindenii/furgit/format/commitgraph/read" objectid "codeberg.org/lindenii/furgit/object/id" ) diff --git a/commitquery/oid.go b/commitquery/oid.go index 0308c85e..7f1e9db1 100644 --- a/commitquery/oid.go +++ b/commitquery/oid.go @@ -3,7 +3,7 @@ package commitquery import ( stderrors "errors" - commitgraphread "codeberg.org/lindenii/furgit/commitgraph/read" + commitgraphread "codeberg.org/lindenii/furgit/format/commitgraph/read" giterrors "codeberg.org/lindenii/furgit/errors" "codeberg.org/lindenii/furgit/internal/peel" objectcommit "codeberg.org/lindenii/furgit/object/commit" diff --git a/commitquery/parent.go b/commitquery/parent.go index 41877975..e7703f77 100644 --- a/commitquery/parent.go +++ b/commitquery/parent.go @@ -1,7 +1,7 @@ package commitquery import ( - commitgraphread "codeberg.org/lindenii/furgit/commitgraph/read" + commitgraphread "codeberg.org/lindenii/furgit/format/commitgraph/read" objectid "codeberg.org/lindenii/furgit/object/id" ) diff --git a/format/commitgraph/TODO b/format/commitgraph/TODO new file mode 100644 index 00000000..87e0888d --- /dev/null +++ b/format/commitgraph/TODO @@ -0,0 +1,6 @@ +Paranoia mode +Split commit-graph chain with mixed generation and bloom setting +Separate chunk parsing layer +Config stuff + +Writing diff --git a/format/commitgraph/bloom/bloom.go b/format/commitgraph/bloom/bloom.go new file mode 100644 index 00000000..9653d595 --- /dev/null +++ b/format/commitgraph/bloom/bloom.go @@ -0,0 +1,3 @@ +// Package bloom provides a bloom filter implementation used for changed-path +// filters in Git commit graphs. +package bloom diff --git a/format/commitgraph/bloom/constants.go b/format/commitgraph/bloom/constants.go new file mode 100644 index 00000000..958e551e --- /dev/null +++ b/format/commitgraph/bloom/constants.go @@ -0,0 +1,8 @@ +package bloom + +const ( + // DataHeaderSize is the size of the BDAT header in commit-graph files. + DataHeaderSize = 3 * 4 + // DefaultMaxChange matches Git's default max-changed-paths behavior. + DefaultMaxChange = 512 +) diff --git a/format/commitgraph/bloom/contain.go b/format/commitgraph/bloom/contain.go new file mode 100644 index 00000000..331b7687 --- /dev/null +++ b/format/commitgraph/bloom/contain.go @@ -0,0 +1,25 @@ +package bloom + +// MightContain reports whether the Bloom filter may contain the given path. +// +// Evaluated against the full path and each of its directory prefixes. A true +// result indicates a possible match; false means the path definitely did not +// change. +func (f *Filter) MightContain(path []byte) (bool, error) { + if len(f.Data) == 0 { + return false, nil + } + + keys, err := keyvec(path, f) + if err != nil { + return false, err + } + + for i := range keys { + if filterContainsKey(f, keys[i]) { + return true, nil + } + } + + return false, nil +} diff --git a/format/commitgraph/bloom/errors.go b/format/commitgraph/bloom/errors.go new file mode 100644 index 00000000..fe38d1bc --- /dev/null +++ b/format/commitgraph/bloom/errors.go @@ -0,0 +1,5 @@ +package bloom + +import "errors" + +var ErrInvalid = errors.New("bloom: invalid data") diff --git a/format/commitgraph/bloom/filter.go b/format/commitgraph/bloom/filter.go new file mode 100644 index 00000000..395dd5ce --- /dev/null +++ b/format/commitgraph/bloom/filter.go @@ -0,0 +1,26 @@ +package bloom + +// Filter represents a changed-paths Bloom filter associated with a commit. +// +// The filter encodes which paths changed between a commit and its first +// parent. Paths are expected to be in Git's slash-separated form and +// are queried using a path and its prefixes (e.g. "a/b/c", "a/b", "a"). +type Filter struct { + Data []byte + + HashVersion uint32 + NumHashes uint32 + BitsPerEntry uint32 + MaxChangePaths uint32 +} + +// NewFilter constructs one query-ready bloom filter from raw data/settings. +func NewFilter(data []byte, settings Settings) Filter { + return Filter{ + Data: data, + HashVersion: settings.HashVersion, + NumHashes: settings.NumHashes, + BitsPerEntry: settings.BitsPerEntry, + MaxChangePaths: settings.MaxChangePaths, + } +} diff --git a/format/commitgraph/bloom/key.go b/format/commitgraph/bloom/key.go new file mode 100644 index 00000000..a15df904 --- /dev/null +++ b/format/commitgraph/bloom/key.go @@ -0,0 +1,117 @@ +package bloom + +import "codeberg.org/lindenii/furgit/internal/intconv" + +type key struct { + hashes []uint32 +} + +func keyvec(path []byte, filter *Filter) ([]key, error) { + if len(path) == 0 { + return nil, nil + } + + count := 1 + + for _, b := range path { + if b == '/' { + count++ + } + } + + keys := make([]key, 0, count) + + full, err := keyFill(path, filter) + if err != nil { + return nil, err + } + + keys = append(keys, full) + + for i := len(path) - 1; i >= 0; i-- { + if path[i] == '/' { + k, err := keyFill(path[:i], filter) + if err != nil { + return nil, err + } + + keys = append(keys, k) + } + } + + return keys, nil +} + +func keyFill(path []byte, filter *Filter) (key, error) { + const ( + seed0 = 0x293ae76f + seed1 = 0x7e646e2c + ) + + var ( + h0 uint32 + h1 uint32 + err error + ) + + switch filter.HashVersion { + case 2: + h0, err = murmur3SeededV2(seed0, path) + if err != nil { + return key{}, err + } + + h1, err = murmur3SeededV2(seed1, path) + if err != nil { + return key{}, err + } + case 1: + h0, err = murmur3SeededV1(seed0, path) + if err != nil { + return key{}, err + } + + h1, err = murmur3SeededV1(seed1, path) + if err != nil { + return key{}, err + } + default: + return key{}, ErrInvalid + } + + hashCount, err := intconv.Uint32ToInt(filter.NumHashes) + if err != nil { + return key{}, ErrInvalid + } + + hashes := make([]uint32, hashCount) + for i := range hashCount { + iU32, err := intconv.IntToUint32(i) + if err != nil { + return key{}, ErrInvalid + } + + hashes[i] = h0 + iU32*h1 + } + + return key{hashes: hashes}, nil +} + +func filterContainsKey(filter *Filter, key key) bool { + if len(filter.Data) == 0 { + return false + } + + mod := uint64(len(filter.Data)) * 8 + for _, h := range key.hashes { + idx := uint64(h) % mod + bytePos := idx / 8 + + bit := byte(1 << (idx & 7)) + if filter.Data[bytePos]&bit == 0 { + return false + } + } + + return true +} diff --git a/format/commitgraph/bloom/murmur.go b/format/commitgraph/bloom/murmur.go new file mode 100644 index 00000000..363b63ae --- /dev/null +++ b/format/commitgraph/bloom/murmur.go @@ -0,0 +1,127 @@ +package bloom + +import "codeberg.org/lindenii/furgit/internal/intconv" + +func murmur3SeededV2(seed uint32, data []byte) (uint32, error) { + const ( + c1 = 0xcc9e2d51 + c2 = 0x1b873593 + r1 = 15 + r2 = 13 + m = 5 + n = 0xe6546b64 + ) + + h := seed + + nblocks := len(data) / 4 + for i := range nblocks { + k := uint32(data[4*i]) | + (uint32(data[4*i+1]) << 8) | + (uint32(data[4*i+2]) << 16) | + (uint32(data[4*i+3]) << 24) + k *= c1 + k = (k << r1) | (k >> (32 - r1)) + k *= c2 + + h ^= k + h = (h << r2) | (h >> (32 - r2)) + h = h*m + n + } + + var k1 uint32 + + tail := data[nblocks*4:] + switch len(tail) & 3 { + case 3: + k1 ^= uint32(tail[2]) << 16 + + fallthrough + case 2: + k1 ^= uint32(tail[1]) << 8 + + fallthrough + case 1: + k1 ^= uint32(tail[0]) + k1 *= c1 + k1 = (k1 << r1) | (k1 >> (32 - r1)) + k1 *= c2 + h ^= k1 + } + + dataLen, err := intconv.IntToUint32(len(data)) + if err != nil { + return 0, err + } + + h ^= dataLen + h ^= h >> 16 + h *= 0x85ebca6b + h ^= h >> 13 + h *= 0xc2b2ae35 + h ^= h >> 16 + + return h, nil +} + +func murmur3SeededV1(seed uint32, data []byte) (uint32, error) { + const ( + c1 = 0xcc9e2d51 + c2 = 0x1b873593 + r1 = 15 + r2 = 13 + m = 5 + n = 0xe6546b64 + ) + + h := seed + + nblocks := len(data) / 4 + for i := range nblocks { + k := intconv.SignExtendByteToUint32(data[4*i]) | + (intconv.SignExtendByteToUint32(data[4*i+1]) << 8) | + (intconv.SignExtendByteToUint32(data[4*i+2]) << 16) | + (intconv.SignExtendByteToUint32(data[4*i+3]) << 24) + k *= c1 + k = (k << r1) | (k >> (32 - r1)) + k *= c2 + + h ^= k + h = (h << r2) | (h >> (32 - r2)) + h = h*m + n + } + + var k1 uint32 + + tail := data[nblocks*4:] + switch len(tail) & 3 { + case 3: + k1 ^= intconv.SignExtendByteToUint32(tail[2]) << 16 + + fallthrough + case 2: + k1 ^= intconv.SignExtendByteToUint32(tail[1]) << 8 + + fallthrough + case 1: + k1 ^= intconv.SignExtendByteToUint32(tail[0]) + k1 *= c1 + k1 = (k1 << r1) | (k1 >> (32 - r1)) + k1 *= c2 + h ^= k1 + } + + dataLen, err := intconv.IntToUint32(len(data)) + if err != nil { + return 0, err + } + + h ^= dataLen + h ^= h >> 16 + h *= 0x85ebca6b + h ^= h >> 13 + h *= 0xc2b2ae35 + h ^= h >> 16 + + return h, nil +} diff --git a/format/commitgraph/bloom/settings.go b/format/commitgraph/bloom/settings.go new file mode 100644 index 00000000..764653bd --- /dev/null +++ b/format/commitgraph/bloom/settings.go @@ -0,0 +1,50 @@ +package bloom + +import ( + "encoding/binary" + + "codeberg.org/lindenii/furgit/internal/intconv" +) + +// Settings describe the changed-paths Bloom filter parameters stored in +// commit-graph BDAT chunks. +// +// Obviously, they must match the repository's commit-graph settings to +// interpret filters correctly. +type Settings struct { + HashVersion uint32 + NumHashes uint32 + BitsPerEntry uint32 + MaxChangePaths uint32 +} + +// ParseSettings reads Bloom filter settings from a BDAT chunk header. +func ParseSettings(bdat []byte) (*Settings, error) { + if len(bdat) < DataHeaderSize { + return nil, ErrInvalid + } + + settings := &Settings{ + HashVersion: binary.BigEndian.Uint32(bdat[0:4]), + NumHashes: binary.BigEndian.Uint32(bdat[4:8]), + BitsPerEntry: binary.BigEndian.Uint32(bdat[8:12]), + MaxChangePaths: DefaultMaxChange, + } + + switch settings.HashVersion { + case 1, 2: + default: + return nil, ErrInvalid + } + + if settings.NumHashes == 0 { + return nil, ErrInvalid + } + + _, err := intconv.Uint32ToInt(settings.NumHashes) + if err != nil { + return nil, ErrInvalid + } + + return settings, nil +} diff --git a/format/commitgraph/constants.go b/format/commitgraph/constants.go new file mode 100644 index 00000000..3a06a290 --- /dev/null +++ b/format/commitgraph/constants.go @@ -0,0 +1,32 @@ +package commitgraph + +const ( + FileSignature = 0x43475048 // "CGPH" + FileVersion = 1 +) + +const ( + ChunkOIDF = 0x4f494446 // "OIDF" + ChunkOIDL = 0x4f49444c // "OIDL" + ChunkCDAT = 0x43444154 // "CDAT" + ChunkGDA2 = 0x47444132 // "GDA2" + ChunkGDO2 = 0x47444f32 // "GDO2" + ChunkEDGE = 0x45444745 // "EDGE" + ChunkBIDX = 0x42494458 // "BIDX" + ChunkBDAT = 0x42444154 // "BDAT" + ChunkBASE = 0x42415345 // "BASE" +) + +const ( + HeaderSize = 8 + ChunkEntrySize = 12 + FanoutSize = 256 * 4 +) + +const ( + ParentNone = 0x70000000 + ParentExtraMask = 0x80000000 + ParentLastMask = 0x7fffffff + + GenerationOverflow = 0x80000000 +) diff --git a/format/commitgraph/doc.go b/format/commitgraph/doc.go new file mode 100644 index 00000000..abf5f3d3 --- /dev/null +++ b/format/commitgraph/doc.go @@ -0,0 +1,2 @@ +// Package commitgraph provides constants and common utilities for handling commit graphs. +package commitgraph diff --git a/format/commitgraph/read/bloom.go b/format/commitgraph/read/bloom.go new file mode 100644 index 00000000..12dd6db3 --- /dev/null +++ b/format/commitgraph/read/bloom.go @@ -0,0 +1,117 @@ +package read + +import ( + "encoding/binary" + + "codeberg.org/lindenii/furgit/format/commitgraph/bloom" + "codeberg.org/lindenii/furgit/internal/intconv" +) + +// HasBloom reports whether any layer has changed-path Bloom data. +func (reader *Reader) HasBloom() bool { + for i := range reader.layers { + layer := &reader.layers[i] + if layer.chunkBloomIndex != nil && layer.chunkBloomData != nil && layer.bloomSettings != nil { + return true + } + } + + return false +} + +// BloomVersion returns the changed-path Bloom hash version, or 0 if absent. +func (reader *Reader) BloomVersion() uint8 { + for i := len(reader.layers) - 1; i >= 0; i-- { + layer := &reader.layers[i] + if layer.bloomSettings != nil { + version, err := intconv.Uint32ToUint8(layer.bloomSettings.HashVersion) + if err != nil { + return 0 + } + + return version + } + } + + return 0 +} + +// BloomFilterAt returns one commit's changed-path Bloom filter. +// +// The returned filter borrows reader-owned mapped commit-graph data and is +// only valid until the reader is closed. +// +// Returns BloomUnavailableError when this commit graph has no Bloom data. +func (reader *Reader) BloomFilterAt(pos Position) (bloom.Filter, error) { + layer, err := reader.layerByPosition(pos) + if err != nil { + return bloom.Filter{}, err + } + + if layer.chunkBloomIndex == nil || layer.chunkBloomData == nil || layer.bloomSettings == nil { + return bloom.Filter{}, &BloomUnavailableError{Pos: pos} + } + + start, end, err := bloomRange(layer, pos.Index) + if err != nil { + return bloom.Filter{}, err + } + + filter := bloom.NewFilter( + layer.chunkBloomData[bloom.DataHeaderSize+start:bloom.DataHeaderSize+end], + *layer.bloomSettings, + ) + + return filter, nil +} + +func bloomRange(layer *layer, commitIndex uint32) (int, int, error) { + off64 := uint64(commitIndex) * 4 + + off, err := intconv.Uint64ToInt(off64) + if err != nil { + return 0, 0, err + } + + end := binary.BigEndian.Uint32(layer.chunkBloomIndex[off : off+4]) + + var start uint32 + + if commitIndex > 0 { + prevOff64 := uint64(commitIndex-1) * 4 + + prevOff, err := intconv.Uint64ToInt(prevOff64) + if err != nil { + return 0, 0, err + } + + start = binary.BigEndian.Uint32(layer.chunkBloomIndex[prevOff : prevOff+4]) + } + + if end < start { + return 0, 0, &MalformedError{Path: layer.path, Reason: "invalid BIDX range"} + } + + bdatLen := len(layer.chunkBloomData) - bloom.DataHeaderSize + + bdatLenU32, err := intconv.IntToUint32(bdatLen) + if err != nil { + return 0, 0, err + } + + if end > bdatLenU32 { + return 0, 0, &MalformedError{Path: layer.path, Reason: "BIDX range out of BDAT bounds"} + } + + startInt, err := intconv.Uint64ToInt(uint64(start)) + if err != nil { + return 0, 0, err + } + + endInt, err := intconv.Uint64ToInt(uint64(end)) + if err != nil { + return 0, 0, err + } + + return startInt, endInt, nil +} diff --git a/format/commitgraph/read/close.go b/format/commitgraph/read/close.go new file mode 100644 index 00000000..f8b6141a --- /dev/null +++ b/format/commitgraph/read/close.go @@ -0,0 +1,20 @@ +package read + +// Close releases all mapped commit-graph files. +// +// Repeated calls to Close are undefined behavior. +func (reader *Reader) Close() error { + var closeErr error + + for i := len(reader.layers) - 1; i >= 0; i-- { + err := reader.layers[i].close() + if err != nil && closeErr == nil { + closeErr = err + } + } + + reader.layers = nil + reader.total = 0 + + return closeErr +} diff --git a/format/commitgraph/read/commitat.go b/format/commitgraph/read/commitat.go new file mode 100644 index 00000000..a39c5ccd --- /dev/null +++ b/format/commitgraph/read/commitat.go @@ -0,0 +1,85 @@ +package read + +import ( + "encoding/binary" + + "codeberg.org/lindenii/furgit/internal/intconv" + objectid "codeberg.org/lindenii/furgit/object/id" +) + +// CommitAt returns decoded commit-graph metadata at one position. +func (reader *Reader) CommitAt(pos Position) (Commit, error) { + layer, err := reader.layerByPosition(pos) + if err != nil { + return Commit{}, err + } + + hashSize := reader.algo.Size() + stride := hashSize + 16 + + strideU64, err := intconv.IntToUint64(stride) + if err != nil { + return Commit{}, err + } + + start64 := uint64(pos.Index) * strideU64 + end64 := start64 + strideU64 + + start, err := intconv.Uint64ToInt(start64) + if err != nil { + return Commit{}, err + } + + end, err := intconv.Uint64ToInt(end64) + if err != nil { + return Commit{}, err + } + + record := layer.chunkCommit[start:end] + + treeOID, err := objectid.FromBytes(reader.algo, record[:hashSize]) + if err != nil { + return Commit{}, err + } + + oid, err := reader.OIDAt(pos) + if err != nil { + return Commit{}, err + } + + p1 := binary.BigEndian.Uint32(record[hashSize : hashSize+4]) + p2 := binary.BigEndian.Uint32(record[hashSize+4 : hashSize+8]) + genAndTimeHi := binary.BigEndian.Uint32(record[hashSize+8 : hashSize+12]) + timeLow := binary.BigEndian.Uint32(record[hashSize+12 : hashSize+16]) + + timeHigh := uint64(genAndTimeHi & 0x3) + commitTimeU64 := (timeHigh << 32) | uint64(timeLow) + + commitTime, err := intconv.Uint64ToInt64(commitTimeU64) + if err != nil { + return Commit{}, err + } + + generationV1 := genAndTimeHi >> 2 + + generationV2, err := reader.readGenerationV2(layer, pos.Index, commitTimeU64) + if err != nil { + return Commit{}, err + } + + parent1, parent2, extra, err := reader.decodeParents(layer, p1, p2) + if err != nil { + return Commit{}, err + } + + return Commit{ + OID: oid, + TreeOID: treeOID, + Parent1: parent1, + Parent2: parent2, + ExtraParents: extra, + CommitTimeUnix: commitTime, + GenerationV1: generationV1, + GenerationV2: generationV2, + }, nil +} diff --git a/format/commitgraph/read/commits.go b/format/commitgraph/read/commits.go new file mode 100644 index 00000000..48984ecb --- /dev/null +++ b/format/commitgraph/read/commits.go @@ -0,0 +1,20 @@ +package read + +import objectid "codeberg.org/lindenii/furgit/object/id" + +// Commit stores decoded commit-graph record data. +type Commit struct { + OID objectid.ObjectID + TreeOID objectid.ObjectID + Parent1 ParentRef + Parent2 ParentRef + ExtraParents []Position + CommitTimeUnix int64 + GenerationV1 uint32 + GenerationV2 uint64 +} + +// NumCommits returns total commits across loaded layers. +func (reader *Reader) NumCommits() uint32 { + return reader.total +} diff --git a/format/commitgraph/read/doc.go b/format/commitgraph/read/doc.go new file mode 100644 index 00000000..573ddc19 --- /dev/null +++ b/format/commitgraph/read/doc.go @@ -0,0 +1,2 @@ +// Package read provides routines for reading commit graphs. +package read diff --git a/format/commitgraph/read/edges.go b/format/commitgraph/read/edges.go new file mode 100644 index 00000000..96ffeb6d --- /dev/null +++ b/format/commitgraph/read/edges.go @@ -0,0 +1,48 @@ +package read + +import ( + "encoding/binary" + + "codeberg.org/lindenii/furgit/format/commitgraph" + "codeberg.org/lindenii/furgit/internal/intconv" +) + +func (reader *Reader) decodeExtraEdgeList(layer *layer, edgeStart uint32) ([]Position, error) { + if len(layer.chunkExtraEdges) == 0 { + return nil, &MalformedError{Path: layer.path, Reason: "missing EDGE chunk"} + } + + out := make([]Position, 0) + + cur := edgeStart + for { + off64 := uint64(cur) * 4 + + off, err := intconv.Uint64ToInt(off64) + if err != nil { + return nil, err + } + + if off+4 > len(layer.chunkExtraEdges) { + return nil, &MalformedError{Path: layer.path, Reason: "EDGE index out of range"} + } + + word := binary.BigEndian.Uint32(layer.chunkExtraEdges[off : off+4]) + parentGlobal := word & commitgraph.ParentLastMask + + parentPos, err := reader.globalToPosition(parentGlobal) + if err != nil { + return nil, err + } + + out = append(out, parentPos) + + if word&commitgraph.ParentExtraMask != 0 { + break + } + + cur++ + } + + return out, nil +} diff --git a/format/commitgraph/read/errors.go b/format/commitgraph/read/errors.go new file mode 100644 index 00000000..0a32a368 --- /dev/null +++ b/format/commitgraph/read/errors.go @@ -0,0 +1,58 @@ +package read + +import ( + "fmt" + + objectid "codeberg.org/lindenii/furgit/object/id" +) + +// NotFoundError reports a missing commit graph entry by object ID. +type NotFoundError struct { + OID objectid.ObjectID +} + +// Error implements error. +func (err *NotFoundError) Error() string { + return fmt.Sprintf("commitgraph: object not found: %s", err.OID) +} + +// PositionOutOfRangeError reports an invalid graph position. +type PositionOutOfRangeError struct { + Pos Position +} + +// Error implements error. +func (err *PositionOutOfRangeError) Error() string { + return fmt.Sprintf("commitgraph: position out of range: graph=%d index=%d", err.Pos.Graph, err.Pos.Index) +} + +// MalformedError reports malformed commit-graph data. +type MalformedError struct { + Path string + Reason string +} + +// Error implements error. +func (err *MalformedError) Error() string { + return fmt.Sprintf("commitgraph: malformed %q: %s", err.Path, err.Reason) +} + +// UnsupportedVersionError reports unsupported commit-graph version. +type UnsupportedVersionError struct { + Version uint8 +} + +// Error implements error. +func (err *UnsupportedVersionError) Error() string { + return fmt.Sprintf("commitgraph: unsupported version %d", err.Version) +} + +// BloomUnavailableError reports missing changed-path bloom data at one position. +type BloomUnavailableError struct { + Pos Position +} + +// Error implements error. +func (err *BloomUnavailableError) Error() string { + return fmt.Sprintf("commitgraph: bloom unavailable at position graph=%d index=%d", err.Pos.Graph, err.Pos.Index) +} diff --git a/format/commitgraph/read/generation.go b/format/commitgraph/read/generation.go new file mode 100644 index 00000000..62e47996 --- /dev/null +++ b/format/commitgraph/read/generation.go @@ -0,0 +1,43 @@ +package read + +import ( + "encoding/binary" + + "codeberg.org/lindenii/furgit/format/commitgraph" + "codeberg.org/lindenii/furgit/internal/intconv" +) + +func (reader *Reader) readGenerationV2(layer *layer, index uint32, commitTime uint64) (uint64, error) { + if len(layer.chunkGeneration) == 0 { + return 0, nil + } + + off64 := uint64(index) * 4 + + off, err := intconv.Uint64ToInt(off64) + if err != nil { + return 0, err + } + + value := binary.BigEndian.Uint32(layer.chunkGeneration[off : off+4]) + + if value&commitgraph.GenerationOverflow == 0 { + return commitTime + uint64(value), nil + } + + gdo2Index := value ^ commitgraph.GenerationOverflow + gdo2Off64 := uint64(gdo2Index) * 8 + + gdo2Off, err := intconv.Uint64ToInt(gdo2Off64) + if err != nil { + return 0, err + } + + if gdo2Off+8 > len(layer.chunkGenerationOv) { + return 0, &MalformedError{Path: layer.path, Reason: "GDO2 index out of range"} + } + + overflow := binary.BigEndian.Uint64(layer.chunkGenerationOv[gdo2Off : gdo2Off+8]) + + return commitTime + overflow, nil +} diff --git a/format/commitgraph/read/hash.go b/format/commitgraph/read/hash.go new file mode 100644 index 00000000..3a525afe --- /dev/null +++ b/format/commitgraph/read/hash.go @@ -0,0 +1,79 @@ +package read + +import ( + "bytes" + "fmt" + "io" + + objectid "codeberg.org/lindenii/furgit/object/id" +) + +// HashVersion returns the commit-graph hash version. +func (reader *Reader) HashVersion() uint8 { + return reader.hashVersion +} + +func validateChainBaseHashes(algo objectid.Algorithm, chain []string, idx int, graph *layer) error { + if idx == 0 { + if len(graph.chunkBaseGraphs) != 0 { + return &MalformedError{Path: graph.path, Reason: "unexpected BASE chunk in first graph"} + } + + return nil + } + + hashSize := algo.Size() + + expectedLen := idx * hashSize + if len(graph.chunkBaseGraphs) != expectedLen { + return &MalformedError{ + Path: graph.path, + Reason: fmt.Sprintf("BASE chunk length %d does not match expected %d", len(graph.chunkBaseGraphs), expectedLen), + } + } + + for i := range idx { + start := i * hashSize + end := start + hashSize + + baseHash, err := objectid.FromBytes(algo, graph.chunkBaseGraphs[start:end]) + if err != nil { + return err + } + + if baseHash.String() != chain[i] { + return &MalformedError{ + Path: graph.path, + Reason: fmt.Sprintf("BASE chunk mismatch at index %d", i), + } + } + } + + return nil +} + +func verifyTrailerHash(data []byte, algo objectid.Algorithm, path string) error { + hashSize := algo.Size() + if len(data) < hashSize { + return &MalformedError{Path: path, Reason: "file too short for trailer"} + } + + hashImpl, err := algo.New() + if err != nil { + return err + } + + _, err = io.Copy(hashImpl, bytes.NewReader(data[:len(data)-hashSize])) + if err != nil { + return err + } + + got := hashImpl.Sum(nil) + + want := data[len(data)-hashSize:] + if !bytes.Equal(got, want) { + return &MalformedError{Path: path, Reason: "trailer hash mismatch"} + } + + return nil +} diff --git a/format/commitgraph/read/iterators.go b/format/commitgraph/read/iterators.go new file mode 100644 index 00000000..85c56ff1 --- /dev/null +++ b/format/commitgraph/read/iterators.go @@ -0,0 +1,45 @@ +package read + +import ( + "iter" + + "codeberg.org/lindenii/furgit/internal/intconv" + objectid "codeberg.org/lindenii/furgit/object/id" +) + +// AllPositions iterates all commit positions in native layer order. +func (reader *Reader) AllPositions() iter.Seq[Position] { + return func(yield func(Position) bool) { + for layerIdx := range reader.layers { + layer := &reader.layers[layerIdx] + + graph, err := intconv.IntToUint32(layerIdx) + if err != nil { + return + } + + for idx := range layer.numCommits { + if !yield(Position{Graph: graph, Index: idx}) { + return + } + } + } + } +} + +// AllOIDs iterates all commit object IDs in native layer order. +func (reader *Reader) AllOIDs() iter.Seq[objectid.ObjectID] { + return func(yield func(objectid.ObjectID) bool) { + positions := reader.AllPositions() + for pos := range positions { + oid, err := reader.OIDAt(pos) + if err != nil { + return + } + + if !yield(oid) { + return + } + } + } +} diff --git a/format/commitgraph/read/layer.go b/format/commitgraph/read/layer.go new file mode 100644 index 00000000..53ab1663 --- /dev/null +++ b/format/commitgraph/read/layer.go @@ -0,0 +1,28 @@ +package read + +import ( + "os" + + "codeberg.org/lindenii/furgit/format/commitgraph/bloom" +) + +type layer struct { + path string + file *os.File + data []byte + numCommits uint32 + baseCount uint32 + globalFrom uint32 + + chunkOIDFanout []byte + chunkOIDLookup []byte + chunkCommit []byte + chunkGeneration []byte + chunkGenerationOv []byte + chunkExtraEdges []byte + chunkBloomIndex []byte + chunkBloomData []byte + chunkBaseGraphs []byte + + bloomSettings *bloom.Settings +} diff --git a/format/commitgraph/read/layer_close.go b/format/commitgraph/read/layer_close.go new file mode 100644 index 00000000..03dc91d5 --- /dev/null +++ b/format/commitgraph/read/layer_close.go @@ -0,0 +1,33 @@ +package read + +import "syscall" + +func closeLayers(layers []layer) { + for i := len(layers) - 1; i >= 0; i-- { + _ = layers[i].close() + } +} + +func (layer *layer) close() error { + var closeErr error + + if layer.data != nil { + err := syscall.Munmap(layer.data) + if err != nil { + closeErr = err + } + + layer.data = nil + } + + if layer.file != nil { + err := layer.file.Close() + if err != nil && closeErr == nil { + closeErr = err + } + + layer.file = nil + } + + return closeErr +} diff --git a/format/commitgraph/read/layer_lookup.go b/format/commitgraph/read/layer_lookup.go new file mode 100644 index 00000000..84095788 --- /dev/null +++ b/format/commitgraph/read/layer_lookup.go @@ -0,0 +1,53 @@ +package read + +import ( + "bytes" + "encoding/binary" + + "codeberg.org/lindenii/furgit/internal/intconv" + objectid "codeberg.org/lindenii/furgit/object/id" +) + +func layerLookup(layer *layer, oid objectid.ObjectID) (uint32, bool) { + hashSize := oid.Size() + first := int(oid.RawBytes()[0]) + + var lo uint32 + if first > 0 { + lo = binary.BigEndian.Uint32(layer.chunkOIDFanout[(first-1)*4 : first*4]) + } + + hi := binary.BigEndian.Uint32(layer.chunkOIDFanout[first*4 : (first+1)*4]) + if hi == 0 || lo >= hi { + return 0, false + } + + target := oid.RawBytes() + left := int(lo) + + right := int(hi) - 1 + for left <= right { + mid := left + (right-left)/2 + start := mid * hashSize + end := start + hashSize + + current := layer.chunkOIDLookup[start:end] + + cmp := bytes.Compare(current, target) + switch { + case cmp == 0: + pos, err := intconv.IntToUint32(mid) + if err != nil { + return 0, false + } + + return pos, true + case cmp < 0: + left = mid + 1 + default: + right = mid - 1 + } + } + + return 0, false +} diff --git a/format/commitgraph/read/layer_open.go b/format/commitgraph/read/layer_open.go new file mode 100644 index 00000000..21a97644 --- /dev/null +++ b/format/commitgraph/read/layer_open.go @@ -0,0 +1,81 @@ +package read + +import ( + "os" + "syscall" + + "codeberg.org/lindenii/furgit/format/commitgraph" + "codeberg.org/lindenii/furgit/internal/intconv" + objectid "codeberg.org/lindenii/furgit/object/id" +) + +func openLayer(root *os.Root, relPath string, algo objectid.Algorithm) (*layer, error) { + file, err := root.Open(relPath) + if err != nil { + return nil, err + } + + info, err := file.Stat() + if err != nil { + _ = file.Close() + + return nil, err + } + + size := info.Size() + if size < int64(commitgraph.HeaderSize+commitgraph.FanoutSize+algo.Size()) { + _ = file.Close() + + return nil, &MalformedError{Path: relPath, Reason: "file too short"} + } + + mapLen, err := intconv.Int64ToUint64(size) + if err != nil { + _ = file.Close() + + return nil, err + } + + mapLenInt, err := intconv.Uint64ToInt(mapLen) + if err != nil { + _ = file.Close() + + return nil, err + } + + fd, err := intconv.UintptrToInt(file.Fd()) + if err != nil { + _ = file.Close() + + return nil, err + } + + data, err := syscall.Mmap(fd, 0, mapLenInt, syscall.PROT_READ, syscall.MAP_PRIVATE) + if err != nil { + _ = file.Close() + + return nil, err + } + + out := &layer{ + path: relPath, + file: file, + data: data, + } + + parseErr := parseLayer(out, algo) + if parseErr != nil { + _ = out.close() + + return nil, parseErr + } + + verifyErr := verifyTrailerHash(out.data, algo, relPath) + if verifyErr != nil { + _ = out.close() + + return nil, verifyErr + } + + return out, nil +} diff --git a/format/commitgraph/read/layer_parse.go b/format/commitgraph/read/layer_parse.go new file mode 100644 index 00000000..13e36c0a --- /dev/null +++ b/format/commitgraph/read/layer_parse.go @@ -0,0 +1,276 @@ +package read + +import ( + "encoding/binary" + + "codeberg.org/lindenii/furgit/format/commitgraph" + "codeberg.org/lindenii/furgit/format/commitgraph/bloom" + "codeberg.org/lindenii/furgit/internal/intconv" + objectid "codeberg.org/lindenii/furgit/object/id" +) + +func parseLayer(layer *layer, algo objectid.Algorithm) error { //nolint:maintidx + if len(layer.data) < commitgraph.HeaderSize { + return &MalformedError{Path: layer.path, Reason: "file too short"} + } + + header := layer.data[:commitgraph.HeaderSize] + + signature := binary.BigEndian.Uint32(header[:4]) + if signature != commitgraph.FileSignature { + return &MalformedError{Path: layer.path, Reason: "invalid signature"} + } + + version := header[4] + if version != commitgraph.FileVersion { + return &UnsupportedVersionError{Version: version} + } + + expectedHashVersion, err := intconv.Uint32ToUint8(algo.PackHashID()) + if err != nil { + return err + } + + hashVersion := header[5] + if hashVersion != expectedHashVersion { + return &MalformedError{Path: layer.path, Reason: "hash version does not match object format"} + } + + numChunks := int(header[6]) + baseCount := uint32(header[7]) + + tocLen := (numChunks + 1) * commitgraph.ChunkEntrySize + tocStart := commitgraph.HeaderSize + + tocEnd := tocStart + tocLen + if tocEnd > len(layer.data) { + return &MalformedError{Path: layer.path, Reason: "truncated chunk table"} + } + + type tocEntry struct { + id uint32 + offset uint64 + } + + entries := make([]tocEntry, 0, numChunks+1) + for i := range numChunks + 1 { + entryOff := tocStart + i*commitgraph.ChunkEntrySize + entryData := layer.data[entryOff : entryOff+commitgraph.ChunkEntrySize] + + entry := tocEntry{ + id: binary.BigEndian.Uint32(entryData[:4]), + offset: binary.BigEndian.Uint64(entryData[4:]), + } + entries = append(entries, entry) + } + + if entries[len(entries)-1].id != 0 { + return &MalformedError{Path: layer.path, Reason: "missing chunk table terminator"} + } + + trailerStart := len(layer.data) - algo.Size() + + chunks := make(map[uint32][]byte, numChunks) + for i := range numChunks { + entry := entries[i] + if entry.id == 0 { + return &MalformedError{Path: layer.path, Reason: "early chunk table terminator"} + } + + next := entries[i+1] + + start, err := intconv.Uint64ToInt(entry.offset) + if err != nil { + return err + } + + end, err := intconv.Uint64ToInt(next.offset) + if err != nil { + return err + } + + if start < tocEnd || end < start || end > trailerStart { + return &MalformedError{Path: layer.path, Reason: "invalid chunk offsets"} + } + + if _, exists := chunks[entry.id]; exists { + return &MalformedError{Path: layer.path, Reason: "duplicate chunk id"} + } + + chunks[entry.id] = layer.data[start:end] + } + + oidf := chunks[commitgraph.ChunkOIDF] + if len(oidf) != commitgraph.FanoutSize { + return &MalformedError{Path: layer.path, Reason: "invalid OIDF length"} + } + + layer.chunkOIDFanout = oidf + layer.numCommits = binary.BigEndian.Uint32(oidf[commitgraph.FanoutSize-4:]) + + for i := range 255 { + cur := binary.BigEndian.Uint32(oidf[i*4 : (i+1)*4]) + + next := binary.BigEndian.Uint32(oidf[(i+1)*4 : (i+2)*4]) + if cur > next { + return &MalformedError{Path: layer.path, Reason: "non-monotonic OIDF fanout"} + } + } + + hashSize := algo.Size() + + hashSizeU64, err := intconv.IntToUint64(hashSize) + if err != nil { + return err + } + + oidl := chunks[commitgraph.ChunkOIDL] + oidlWantLen64 := uint64(layer.numCommits) * hashSizeU64 + + oidlWantLen, err := intconv.Uint64ToInt(oidlWantLen64) + if err != nil { + return err + } + + if len(oidl) != oidlWantLen { + return &MalformedError{Path: layer.path, Reason: "invalid OIDL length"} + } + + layer.chunkOIDLookup = oidl + + stride := hashSize + 16 + + strideU64, err := intconv.IntToUint64(stride) + if err != nil { + return err + } + + cdat := chunks[commitgraph.ChunkCDAT] + cdatWantLen64 := uint64(layer.numCommits) * strideU64 + + cdatWantLen, err := intconv.Uint64ToInt(cdatWantLen64) + if err != nil { + return err + } + + if len(cdat) != cdatWantLen { + return &MalformedError{Path: layer.path, Reason: "invalid CDAT length"} + } + + layer.chunkCommit = cdat + + gda2 := chunks[commitgraph.ChunkGDA2] + if len(gda2) != 0 { + wantLen64 := uint64(layer.numCommits) * 4 + + wantLen, err := intconv.Uint64ToInt(wantLen64) + if err != nil { + return err + } + + if len(gda2) != wantLen { + return &MalformedError{Path: layer.path, Reason: "invalid GDA2 length"} + } + + layer.chunkGeneration = gda2 + } + + gdo2 := chunks[commitgraph.ChunkGDO2] + if len(gdo2) != 0 { + if len(gdo2)%8 != 0 { + return &MalformedError{Path: layer.path, Reason: "invalid GDO2 length"} + } + + layer.chunkGenerationOv = gdo2 + } + + edge := chunks[commitgraph.ChunkEDGE] + if len(edge) != 0 { + if len(edge)%4 != 0 { + return &MalformedError{Path: layer.path, Reason: "invalid EDGE length"} + } + + layer.chunkExtraEdges = edge + } + + base := chunks[commitgraph.ChunkBASE] + if baseCount == 0 { + if len(base) != 0 { + return &MalformedError{Path: layer.path, Reason: "unexpected BASE chunk"} + } + } else { + wantLen64 := uint64(baseCount) * hashSizeU64 + + wantLen, err := intconv.Uint64ToInt(wantLen64) + if err != nil { + return err + } + + if len(base) != wantLen { + return &MalformedError{Path: layer.path, Reason: "invalid BASE length"} + } + + layer.chunkBaseGraphs = base + } + + layer.baseCount = baseCount + + bidx := chunks[commitgraph.ChunkBIDX] + + bdat := chunks[commitgraph.ChunkBDAT] + if len(bidx) != 0 || len(bdat) != 0 { //nolint:nestif + if len(bidx) == 0 || len(bdat) == 0 { + return &MalformedError{Path: layer.path, Reason: "BIDX/BDAT must both be present"} + } + + bidxWantLen64 := uint64(layer.numCommits) * 4 + + bidxWantLen, err := intconv.Uint64ToInt(bidxWantLen64) + if err != nil { + return err + } + + if len(bidx) != bidxWantLen { + return &MalformedError{Path: layer.path, Reason: "invalid BIDX length"} + } + + if len(bdat) < bloom.DataHeaderSize { + return &MalformedError{Path: layer.path, Reason: "invalid BDAT length"} + } + + settings, err := bloom.ParseSettings(bdat) + if err != nil { + return err + } + + prev := uint32(0) + + for i := range layer.numCommits { + off := int(i) * 4 + + cur := binary.BigEndian.Uint32(bidx[off : off+4]) + if i > 0 && cur < prev { + return &MalformedError{Path: layer.path, Reason: "non-monotonic BIDX"} + } + + bdatDataLen := len(bdat) - bloom.DataHeaderSize + + bdatDataLenU32, err := intconv.IntToUint32(bdatDataLen) + if err != nil { + return err + } + + if cur > bdatDataLenU32 { + return &MalformedError{Path: layer.path, Reason: "BIDX offset out of range"} + } + + prev = cur + } + + layer.chunkBloomIndex = bidx + layer.chunkBloomData = bdat + layer.bloomSettings = settings + } + + return nil +} diff --git a/format/commitgraph/read/layer_pos.go b/format/commitgraph/read/layer_pos.go new file mode 100644 index 00000000..7b87b381 --- /dev/null +++ b/format/commitgraph/read/layer_pos.go @@ -0,0 +1,21 @@ +package read + +import "codeberg.org/lindenii/furgit/internal/intconv" + +func (reader *Reader) layerByPosition(pos Position) (*layer, error) { + graphIdx, err := intconv.Uint64ToInt(uint64(pos.Graph)) + if err != nil { + return nil, err + } + + if graphIdx < 0 || graphIdx >= len(reader.layers) { + return nil, &PositionOutOfRangeError{Pos: pos} + } + + layer := &reader.layers[graphIdx] + if pos.Index >= layer.numCommits { + return nil, &PositionOutOfRangeError{Pos: pos} + } + + return layer, nil +} diff --git a/format/commitgraph/read/layerinfo.go b/format/commitgraph/read/layerinfo.go new file mode 100644 index 00000000..83c4407d --- /dev/null +++ b/format/commitgraph/read/layerinfo.go @@ -0,0 +1,23 @@ +package read + +// LayerInfo describes one loaded commit-graph layer. +type LayerInfo struct { + Path string + BaseCount uint32 + Commits uint32 +} + +// Layers returns loaded layer metadata in native chain order. +func (reader *Reader) Layers() []LayerInfo { + out := make([]LayerInfo, 0, len(reader.layers)) + for i := range reader.layers { + layer := reader.layers[i] + out = append(out, LayerInfo{ + Path: layer.path, + BaseCount: layer.baseCount, + Commits: layer.numCommits, + }) + } + + return out +} diff --git a/format/commitgraph/read/lookup.go b/format/commitgraph/read/lookup.go new file mode 100644 index 00000000..5f1b08f6 --- /dev/null +++ b/format/commitgraph/read/lookup.go @@ -0,0 +1,29 @@ +package read + +import ( + "codeberg.org/lindenii/furgit/internal/intconv" + objectid "codeberg.org/lindenii/furgit/object/id" +) + +// Lookup resolves one object ID to one graph position. +func (reader *Reader) Lookup(oid objectid.ObjectID) (Position, error) { + if oid.Algorithm() != reader.algo { + return Position{}, &NotFoundError{OID: oid} + } + + for layerIdx := len(reader.layers) - 1; layerIdx >= 0; layerIdx-- { + layer := &reader.layers[layerIdx] + + found, ok := layerLookup(layer, oid) + if ok { + idxU32, err := intconv.IntToUint32(layerIdx) + if err != nil { + return Position{}, err + } + + return Position{Graph: idxU32, Index: found}, nil + } + } + + return Position{}, &NotFoundError{OID: oid} +} diff --git a/format/commitgraph/read/mode.go b/format/commitgraph/read/mode.go new file mode 100644 index 00000000..76afa21f --- /dev/null +++ b/format/commitgraph/read/mode.go @@ -0,0 +1,11 @@ +package read + +// OpenMode controls which commit-graph layout Open loads. +type OpenMode uint8 + +const ( + // OpenSingle opens one commit-graph file at info/commit-graph. + OpenSingle OpenMode = iota + // OpenChain opens chained commit-graphs from info/commit-graphs. + OpenChain +) diff --git a/format/commitgraph/read/oidat.go b/format/commitgraph/read/oidat.go new file mode 100644 index 00000000..908cbc1c --- /dev/null +++ b/format/commitgraph/read/oidat.go @@ -0,0 +1,36 @@ +package read + +import ( + "codeberg.org/lindenii/furgit/internal/intconv" + objectid "codeberg.org/lindenii/furgit/object/id" +) + +// OIDAt returns object ID at one position. +func (reader *Reader) OIDAt(pos Position) (objectid.ObjectID, error) { + layer, err := reader.layerByPosition(pos) + if err != nil { + return objectid.ObjectID{}, err + } + + hashSize := reader.algo.Size() + + hashSizeU64, err := intconv.IntToUint64(hashSize) + if err != nil { + return objectid.ObjectID{}, err + } + + start64 := uint64(pos.Index) * hashSizeU64 + end64 := start64 + hashSizeU64 + + start, err := intconv.Uint64ToInt(start64) + if err != nil { + return objectid.ObjectID{}, err + } + + end, err := intconv.Uint64ToInt(end64) + if err != nil { + return objectid.ObjectID{}, err + } + + return objectid.FromBytes(reader.algo, layer.chunkOIDLookup[start:end]) +} diff --git a/format/commitgraph/read/open.go b/format/commitgraph/read/open.go new file mode 100644 index 00000000..9c708b49 --- /dev/null +++ b/format/commitgraph/read/open.go @@ -0,0 +1,26 @@ +package read + +import ( + "fmt" + "os" + + objectid "codeberg.org/lindenii/furgit/object/id" +) + +// Open opens commit-graph data from one objects root. +// +// Open borrows root during construction and does not close it. +func Open(root *os.Root, algo objectid.Algorithm, mode OpenMode) (*Reader, error) { + if algo.Size() == 0 { + return nil, objectid.ErrInvalidAlgorithm + } + + switch mode { + case OpenSingle: + return openSingle(root, algo) + case OpenChain: + return openChain(root, algo) + default: + return nil, fmt.Errorf("commitgraph: invalid open mode %d", mode) + } +} diff --git a/format/commitgraph/read/open_chain.go b/format/commitgraph/read/open_chain.go new file mode 100644 index 00000000..b55f3e57 --- /dev/null +++ b/format/commitgraph/read/open_chain.go @@ -0,0 +1,133 @@ +package read + +import ( + "bufio" + "errors" + "fmt" + "os" + "strings" + + "codeberg.org/lindenii/furgit/internal/intconv" + objectid "codeberg.org/lindenii/furgit/object/id" +) + +func openChain(root *os.Root, algo objectid.Algorithm) (*Reader, error) { + chainPath := "info/commit-graphs/commit-graph-chain" + + file, err := root.Open(chainPath) + if err != nil { + if errors.Is(err, os.ErrNotExist) { + return nil, &MalformedError{Path: chainPath, Reason: "missing commit-graph-chain"} + } + + return nil, err + } + + scanner := bufio.NewScanner(file) + hashes := make([]string, 0) + + for scanner.Scan() { + line := strings.TrimSpace(scanner.Text()) + if line == "" { + continue + } + + hashes = append(hashes, line) + } + + scanErr := scanner.Err() + closeErr := file.Close() + + if scanErr != nil { + return nil, scanErr + } + + if closeErr != nil { + return nil, closeErr + } + + if len(hashes) == 0 { + return nil, &MalformedError{Path: chainPath, Reason: "empty chain"} + } + + layers := make([]layer, 0, len(hashes)) + + var total uint32 + + hashVersion, err := intconv.Uint32ToUint8(algo.PackHashID()) + if err != nil { + return nil, err + } + + for i, hashHex := range hashes { + expectedBaseCount, err := intconv.IntToUint32(i) + if err != nil { + closeLayers(layers) + + return nil, err + } + + if len(hashHex) != algo.HexLen() { + closeLayers(layers) + + return nil, &MalformedError{ + Path: chainPath, + Reason: fmt.Sprintf("invalid graph hash length at line %d", i+1), + } + } + + relPath := fmt.Sprintf("info/commit-graphs/graph-%s.graph", hashHex) + + loaded, loadErr := openLayer(root, relPath, algo) + if loadErr != nil { + closeLayers(layers) + + return nil, loadErr + } + + if loaded.baseCount != expectedBaseCount { + _ = loaded.close() + + closeLayers(layers) + + return nil, &MalformedError{ + Path: relPath, + Reason: fmt.Sprintf("BASE count %d does not match chain depth %d", loaded.baseCount, i), + } + } + + validateErr := validateChainBaseHashes(algo, hashes, i, loaded) + if validateErr != nil { + _ = loaded.close() + + closeLayers(layers) + + return nil, validateErr + } + + loaded.globalFrom = total + loaded.baseCount = expectedBaseCount + + totalNext := total + loaded.numCommits + if totalNext < total { + _ = loaded.close() + + closeLayers(layers) + + return nil, &MalformedError{Path: relPath, Reason: "total commit count overflow"} + } + + total = totalNext + + layers = append(layers, *loaded) + } + + out := &Reader{ + algo: algo, + hashVersion: hashVersion, + layers: layers, + total: total, + } + + return out, nil +} diff --git a/format/commitgraph/read/open_single.go b/format/commitgraph/read/open_single.go new file mode 100644 index 00000000..9ad6607f --- /dev/null +++ b/format/commitgraph/read/open_single.go @@ -0,0 +1,32 @@ +package read + +import ( + "os" + + "codeberg.org/lindenii/furgit/internal/intconv" + objectid "codeberg.org/lindenii/furgit/object/id" +) + +func openSingle(root *os.Root, algo objectid.Algorithm) (*Reader, error) { + graph, err := openLayer(root, "info/commit-graph", algo) + if err != nil { + return nil, err + } + + graph.baseCount = 0 + graph.globalFrom = 0 + + hashVersion, err := intconv.Uint32ToUint8(algo.PackHashID()) + if err != nil { + return nil, err + } + + out := &Reader{ + algo: algo, + hashVersion: hashVersion, + layers: []layer{*graph}, + total: graph.numCommits, + } + + return out, nil +} diff --git a/format/commitgraph/read/parents.go b/format/commitgraph/read/parents.go new file mode 100644 index 00000000..fcaad8b6 --- /dev/null +++ b/format/commitgraph/read/parents.go @@ -0,0 +1,67 @@ +package read + +import "codeberg.org/lindenii/furgit/format/commitgraph" + +// ParentRef references one parent position. +type ParentRef struct { + Valid bool + Pos Position +} + +func (reader *Reader) decodeParents(layer *layer, p1, p2 uint32) (ParentRef, ParentRef, []Position, error) { + parent1, err := reader.decodeSingleParent(p1) + if err != nil { + return ParentRef{}, ParentRef{}, nil, err + } + + if p2 == commitgraph.ParentNone { + return parent1, ParentRef{}, nil, nil + } + + if p2&commitgraph.ParentExtraMask == 0 { + parent2, err := reader.decodeSingleParent(p2) + if err != nil { + return ParentRef{}, ParentRef{}, nil, err + } + + return parent1, parent2, nil, nil + } + + edgeStart := p2 & commitgraph.ParentLastMask + + parents, err := reader.decodeExtraEdgeList(layer, edgeStart) + if err != nil { + return ParentRef{}, ParentRef{}, nil, err + } + + if len(parents) == 0 { + return ParentRef{}, ParentRef{}, nil, &MalformedError{Path: layer.path, Reason: "empty EDGE list"} + } + + parent2 := ParentRef{Valid: true, Pos: parents[0]} + if len(parents) == 1 { + return parent1, parent2, nil, nil + } + + return parent1, parent2, parents[1:], nil +} + +func (reader *Reader) decodeSingleParent(raw uint32) (ParentRef, error) { + if raw == commitgraph.ParentNone { + return ParentRef{}, nil + } + + if raw&commitgraph.ParentExtraMask != 0 { + return ParentRef{}, &MalformedError{ + Path: "commit-graph", + Reason: "unexpected EDGE marker in single-parent slot", + } + } + + pos, err := reader.globalToPosition(raw) + if err != nil { + return ParentRef{}, err + } + + return ParentRef{Valid: true, Pos: pos}, nil +} diff --git a/format/commitgraph/read/position.go b/format/commitgraph/read/position.go new file mode 100644 index 00000000..b2e1138b --- /dev/null +++ b/format/commitgraph/read/position.go @@ -0,0 +1,38 @@ +package read + +import ( + "fmt" + + "codeberg.org/lindenii/furgit/internal/intconv" +) + +// Position identifies one commit record by layer and row index. +type Position struct { + Graph uint32 + Index uint32 +} + +func (reader *Reader) globalToPosition(global uint32) (Position, error) { + for i := range reader.layers { + layer := &reader.layers[i] + from := layer.globalFrom + + to := from + layer.numCommits + if global >= from && global < to { + graph, err := intconv.IntToUint32(i) + if err != nil { + return Position{}, err + } + + return Position{ + Graph: graph, + Index: global - from, + }, nil + } + } + + return Position{}, &MalformedError{ + Path: "commit-graph", + Reason: fmt.Sprintf("parent global position out of range: %d", global), + } +} diff --git a/format/commitgraph/read/read_test.go b/format/commitgraph/read/read_test.go new file mode 100644 index 00000000..c65b183e --- /dev/null +++ b/format/commitgraph/read/read_test.go @@ -0,0 +1,322 @@ +package read_test + +import ( + "errors" + "path/filepath" + "strconv" + "strings" + "testing" + + "codeberg.org/lindenii/furgit/format/commitgraph/bloom" + "codeberg.org/lindenii/furgit/format/commitgraph/read" + "codeberg.org/lindenii/furgit/internal/intconv" + "codeberg.org/lindenii/furgit/internal/testgit" + objectid "codeberg.org/lindenii/furgit/object/id" +) + +func fixtureRepoPath(t *testing.T, algo objectid.Algorithm, name string) string { + t.Helper() + + return filepath.Join("testdata", "fixtures", algo.String(), name, "repo.git") +} + +func fixtureRepo(t *testing.T, algo objectid.Algorithm, name string) *testgit.TestRepo { + t.Helper() + + return testgit.NewRepoFromFixture(t, algo, fixtureRepoPath(t, algo, name)) +} + +func TestReadSingleMatchesGit(t *testing.T) { + t.Parallel() + + testgit.ForEachAlgorithm(t, func(t *testing.T, algo objectid.Algorithm) { //nolint:thelper + testRepo := fixtureRepo(t, algo, "single_changed") + + reader := openReader(t, testRepo, read.OpenSingle) + + defer func() { _ = reader.Close() }() + + allIDs := testRepo.RevList(t, "--all") + if len(allIDs) == 0 { + t.Fatal("git rev-list --all returned no commits") + } + + wantCommitCount, err := intconv.IntToUint32(len(allIDs)) + if err != nil { + t.Fatalf("len(allIDs) convert: %v", err) + } + + if got := reader.NumCommits(); got != wantCommitCount { + t.Fatalf("NumCommits() = %d, want %d", got, len(allIDs)) + } + + if !reader.HasBloom() { + t.Fatal("HasBloom() = false, want true") + } + + bloomVersion := reader.BloomVersion() + if bloomVersion == 0 { + t.Fatal("BloomVersion() = 0, want non-zero when HasBloom() is true") + } + + for _, id := range allIDs { + pos, err := reader.Lookup(id) + if err != nil { + t.Fatalf("Lookup(%s): %v", id, err) + } + + gotID, err := reader.OIDAt(pos) + if err != nil { + t.Fatalf("OIDAt(%+v): %v", pos, err) + } + + if gotID != id { + t.Fatalf("OIDAt(Lookup(%s)) = %s, want %s", id, gotID, id) + } + } + + step := max(len(allIDs)/24, 1) + + for i, id := range allIDs { + if i%step != 0 && i != len(allIDs)-1 { + continue + } + + verifyCommitAgainstGit(t, testRepo, reader, id) + } + }) +} + +func TestReadChainMatchesGit(t *testing.T) { + t.Parallel() + + testgit.ForEachAlgorithm(t, func(t *testing.T, algo objectid.Algorithm) { //nolint:thelper + testRepo := fixtureRepo(t, algo, "chain_changed") + + reader := openReader(t, testRepo, read.OpenChain) + + defer func() { _ = reader.Close() }() + + layers := reader.Layers() + if len(layers) < 2 { + t.Fatalf("Layers len = %d, want >= 2", len(layers)) + } + + allIDs := testRepo.RevList(t, "--all") + + wantCommitCount, err := intconv.IntToUint32(len(allIDs)) + if err != nil { + t.Fatalf("len(allIDs) convert: %v", err) + } + + if got := reader.NumCommits(); got != wantCommitCount { + t.Fatalf("NumCommits() = %d, want %d", got, len(allIDs)) + } + + step := max(len(allIDs)/20, 1) + + for i, id := range allIDs { + pos, err := reader.Lookup(id) + if err != nil { + t.Fatalf("Lookup(%s): %v", id, err) + } + + if i%step != 0 && i != len(allIDs)-1 { + continue + } + + gotID, err := reader.OIDAt(pos) + if err != nil { + t.Fatalf("OIDAt(%+v): %v", pos, err) + } + + if gotID != id { + t.Fatalf("OIDAt(Lookup(%s)) = %s, want %s", id, gotID, id) + } + } + }) +} + +func TestBloomUnavailableWithoutChangedPaths(t *testing.T) { + t.Parallel() + + testgit.ForEachAlgorithm(t, func(t *testing.T, algo objectid.Algorithm) { //nolint:thelper + testRepo := fixtureRepo(t, algo, "single_nochanged") + + reader := openReader(t, testRepo, read.OpenSingle) + + defer func() { _ = reader.Close() }() + + head := testRepo.RevParse(t, "HEAD") + + pos, err := reader.Lookup(head) + if err != nil { + t.Fatalf("Lookup(%s): %v", head, err) + } + + _, err = reader.BloomFilterAt(pos) + if err == nil { + t.Fatal("BloomFilterAt() error = nil, want BloomUnavailableError") + } + + unavailable, ok := errors.AsType[*read.BloomUnavailableError](err) + if !ok { + t.Fatalf("BloomFilterAt() error type = %T, want *BloomUnavailableError", err) + } + + if unavailable.Pos != pos { + t.Fatalf("BloomUnavailableError.Pos = %+v, want %+v", unavailable.Pos, pos) + } + }) +} + +func openReader(tb testing.TB, testRepo *testgit.TestRepo, mode read.OpenMode) *read.Reader { + tb.Helper() + + root := testRepo.OpenObjectsRoot(tb) + + reader, err := read.Open(root, testRepo.Algorithm(), mode) + if err != nil { + tb.Fatalf("read.Open(objects): %v", err) + } + + return reader +} + +func verifyCommitAgainstGit(tb testing.TB, testRepo *testgit.TestRepo, reader *read.Reader, id objectid.ObjectID) { + tb.Helper() + + pos, err := reader.Lookup(id) + if err != nil { + tb.Fatalf("Lookup(%s): %v", id, err) + } + + commit, err := reader.CommitAt(pos) + if err != nil { + tb.Fatalf("CommitAt(%+v): %v", pos, err) + } + + if commit.OID != id { + tb.Fatalf("CommitAt(%+v).OID = %s, want %s", pos, commit.OID, id) + } + + treeHex := testRepo.Run(tb, "show", "-s", "--format=%T", id.String()) + + wantTree, err := objectid.ParseHex(testRepo.Algorithm(), treeHex) + if err != nil { + tb.Fatalf("parse tree id %q: %v", treeHex, err) + } + + if commit.TreeOID != wantTree { + tb.Fatalf("CommitAt(%+v).TreeOID = %s, want %s", pos, commit.TreeOID, wantTree) + } + + wantParents := parseOIDLine(tb, testRepo.Algorithm(), testRepo.Run(tb, "show", "-s", "--format=%P", id.String())) + + gotParents := commitParents(tb, reader, commit) + if len(gotParents) != len(wantParents) { + tb.Fatalf("parent count for %s = %d, want %d", id, len(gotParents), len(wantParents)) + } + + for i := range gotParents { + if gotParents[i] != wantParents[i] { + tb.Fatalf("parent %d for %s = %s, want %s", i, id, gotParents[i], wantParents[i]) + } + } + + commitTimeRaw := testRepo.Run(tb, "show", "-s", "--format=%ct", id.String()) + + wantCommitTime, err := strconv.ParseInt(strings.TrimSpace(commitTimeRaw), 10, 64) + if err != nil { + tb.Fatalf("parse commit time %q: %v", commitTimeRaw, err) + } + + if commit.CommitTimeUnix != wantCommitTime { + tb.Fatalf("CommitAt(%+v).CommitTimeUnix = %d, want %d", pos, commit.CommitTimeUnix, wantCommitTime) + } + + filter, err := reader.BloomFilterAt(pos) + if err != nil { + tb.Fatalf("BloomFilterAt(%+v): %v", pos, err) + } + + if filter.HashVersion != uint32(reader.BloomVersion()) { + tb.Fatalf("filter.HashVersion = %d, want %d", filter.HashVersion, reader.BloomVersion()) + } + + assertChangedPathsBloomPositive(tb, testRepo, filter, id) +} + +func commitParents(tb testing.TB, reader *read.Reader, commit read.Commit) []objectid.ObjectID { + tb.Helper() + + out := make([]objectid.ObjectID, 0, 2+len(commit.ExtraParents)) + + if commit.Parent1.Valid { + id, err := reader.OIDAt(commit.Parent1.Pos) + if err != nil { + tb.Fatalf("OIDAt(parent1 %+v): %v", commit.Parent1.Pos, err) + } + + out = append(out, id) + } + + if commit.Parent2.Valid { + id, err := reader.OIDAt(commit.Parent2.Pos) + if err != nil { + tb.Fatalf("OIDAt(parent2 %+v): %v", commit.Parent2.Pos, err) + } + + out = append(out, id) + } + + for _, parentPos := range commit.ExtraParents { + id, err := reader.OIDAt(parentPos) + if err != nil { + tb.Fatalf("OIDAt(extra parent %+v): %v", parentPos, err) + } + + out = append(out, id) + } + + return out +} + +func assertChangedPathsBloomPositive(tb testing.TB, testRepo *testgit.TestRepo, filter bloom.Filter, commitID objectid.ObjectID) { + tb.Helper() + + changedPaths := testRepo.Run(tb, "diff-tree", "--no-commit-id", "--name-only", "-r", "--root", commitID.String()) + for line := range strings.SplitSeq(strings.TrimSpace(changedPaths), "\n") { + path := strings.TrimSpace(line) + if path == "" { + continue + } + + mightContain, err := filter.MightContain([]byte(path)) + if err != nil { + tb.Fatalf("MightContain(%q): %v", path, err) + } + + if !mightContain { + tb.Fatalf("Bloom filter false negative for commit %s path %q", commitID, path) + } + } +} + +func parseOIDLine(tb testing.TB, algo objectid.Algorithm, line string) []objectid.ObjectID { + tb.Helper() + + toks := strings.Fields(line) + + out := make([]objectid.ObjectID, 0, len(toks)) + for _, tok := range toks { + id, err := objectid.ParseHex(algo, tok) + if err != nil { + tb.Fatalf("parse object id %q: %v", tok, err) + } + + out = append(out, id) + } + + return out +} diff --git a/format/commitgraph/read/reader.go b/format/commitgraph/read/reader.go new file mode 100644 index 00000000..d5c84a70 --- /dev/null +++ b/format/commitgraph/read/reader.go @@ -0,0 +1,16 @@ +package read + +import objectid "codeberg.org/lindenii/furgit/object/id" + +// Reader provides read-only access to one mmap-backed commit-graph snapshot. +// +// It is safe for concurrent read-only queries. +// Values returned by Reader methods are only valid until the reader is closed +// when explicitly documented on that method. +type Reader struct { + algo objectid.Algorithm + hashVersion uint8 + + layers []layer + total uint32 +} diff --git a/format/commitgraph/read/testdata/fixtures/sha1/chain_changed/repo.git/HEAD b/format/commitgraph/read/testdata/fixtures/sha1/chain_changed/repo.git/HEAD new file mode 100644 index 00000000..cb089cd8 --- /dev/null +++ b/format/commitgraph/read/testdata/fixtures/sha1/chain_changed/repo.git/HEAD @@ -0,0 +1 @@ +ref: refs/heads/master diff --git a/format/commitgraph/read/testdata/fixtures/sha1/chain_changed/repo.git/config b/format/commitgraph/read/testdata/fixtures/sha1/chain_changed/repo.git/config new file mode 100644 index 00000000..07d359d0 --- /dev/null +++ b/format/commitgraph/read/testdata/fixtures/sha1/chain_changed/repo.git/config @@ -0,0 +1,4 @@ +[core] + repositoryformatversion = 0 + filemode = true + bare = true diff --git a/format/commitgraph/read/testdata/fixtures/sha1/chain_changed/repo.git/objects/info/commit-graphs/commit-graph-chain b/format/commitgraph/read/testdata/fixtures/sha1/chain_changed/repo.git/objects/info/commit-graphs/commit-graph-chain new file mode 100644 index 00000000..74c46b64 --- /dev/null +++ b/format/commitgraph/read/testdata/fixtures/sha1/chain_changed/repo.git/objects/info/commit-graphs/commit-graph-chain @@ -0,0 +1,2 @@ +dd7578d5216ca76c25b19631ba90f7498aeabbe7 +bf985c21612a52070d8b008e6ef51edf8b609401 diff --git a/format/commitgraph/read/testdata/fixtures/sha1/chain_changed/repo.git/objects/info/commit-graphs/graph-bf985c21612a52070d8b008e6ef51edf8b609401.graph b/format/commitgraph/read/testdata/fixtures/sha1/chain_changed/repo.git/objects/info/commit-graphs/graph-bf985c21612a52070d8b008e6ef51edf8b609401.graph new file mode 100644 index 00000000..c31869c1 Binary files /dev/null and b/format/commitgraph/read/testdata/fixtures/sha1/chain_changed/repo.git/objects/info/commit-graphs/graph-bf985c21612a52070d8b008e6ef51edf8b609401.graph differ diff --git a/format/commitgraph/read/testdata/fixtures/sha1/chain_changed/repo.git/objects/info/commit-graphs/graph-dd7578d5216ca76c25b19631ba90f7498aeabbe7.graph b/format/commitgraph/read/testdata/fixtures/sha1/chain_changed/repo.git/objects/info/commit-graphs/graph-dd7578d5216ca76c25b19631ba90f7498aeabbe7.graph new file mode 100644 index 00000000..241eb3cc Binary files /dev/null and b/format/commitgraph/read/testdata/fixtures/sha1/chain_changed/repo.git/objects/info/commit-graphs/graph-dd7578d5216ca76c25b19631ba90f7498aeabbe7.graph differ diff --git a/format/commitgraph/read/testdata/fixtures/sha1/chain_changed/repo.git/objects/info/packs b/format/commitgraph/read/testdata/fixtures/sha1/chain_changed/repo.git/objects/info/packs new file mode 100644 index 00000000..61decf9b --- /dev/null +++ b/format/commitgraph/read/testdata/fixtures/sha1/chain_changed/repo.git/objects/info/packs @@ -0,0 +1,2 @@ +P pack-15b064d6a8ef8cff520565f6db8c006b2e6f7f2f.pack + diff --git a/format/commitgraph/read/testdata/fixtures/sha1/chain_changed/repo.git/objects/pack/pack-15b064d6a8ef8cff520565f6db8c006b2e6f7f2f.bitmap b/format/commitgraph/read/testdata/fixtures/sha1/chain_changed/repo.git/objects/pack/pack-15b064d6a8ef8cff520565f6db8c006b2e6f7f2f.bitmap new file mode 100644 index 00000000..1508cf18 Binary files /dev/null and b/format/commitgraph/read/testdata/fixtures/sha1/chain_changed/repo.git/objects/pack/pack-15b064d6a8ef8cff520565f6db8c006b2e6f7f2f.bitmap differ diff --git a/format/commitgraph/read/testdata/fixtures/sha1/chain_changed/repo.git/objects/pack/pack-15b064d6a8ef8cff520565f6db8c006b2e6f7f2f.idx b/format/commitgraph/read/testdata/fixtures/sha1/chain_changed/repo.git/objects/pack/pack-15b064d6a8ef8cff520565f6db8c006b2e6f7f2f.idx new file mode 100644 index 00000000..00ee2646 Binary files /dev/null and b/format/commitgraph/read/testdata/fixtures/sha1/chain_changed/repo.git/objects/pack/pack-15b064d6a8ef8cff520565f6db8c006b2e6f7f2f.idx differ diff --git a/format/commitgraph/read/testdata/fixtures/sha1/chain_changed/repo.git/objects/pack/pack-15b064d6a8ef8cff520565f6db8c006b2e6f7f2f.pack b/format/commitgraph/read/testdata/fixtures/sha1/chain_changed/repo.git/objects/pack/pack-15b064d6a8ef8cff520565f6db8c006b2e6f7f2f.pack new file mode 100644 index 00000000..c65ae27f Binary files /dev/null and b/format/commitgraph/read/testdata/fixtures/sha1/chain_changed/repo.git/objects/pack/pack-15b064d6a8ef8cff520565f6db8c006b2e6f7f2f.pack differ diff --git a/format/commitgraph/read/testdata/fixtures/sha1/chain_changed/repo.git/objects/pack/pack-15b064d6a8ef8cff520565f6db8c006b2e6f7f2f.rev b/format/commitgraph/read/testdata/fixtures/sha1/chain_changed/repo.git/objects/pack/pack-15b064d6a8ef8cff520565f6db8c006b2e6f7f2f.rev new file mode 100644 index 00000000..d0689f72 Binary files /dev/null and b/format/commitgraph/read/testdata/fixtures/sha1/chain_changed/repo.git/objects/pack/pack-15b064d6a8ef8cff520565f6db8c006b2e6f7f2f.rev differ diff --git a/format/commitgraph/read/testdata/fixtures/sha1/chain_changed/repo.git/refs/heads/master b/format/commitgraph/read/testdata/fixtures/sha1/chain_changed/repo.git/refs/heads/master new file mode 100644 index 00000000..8942d437 --- /dev/null +++ b/format/commitgraph/read/testdata/fixtures/sha1/chain_changed/repo.git/refs/heads/master @@ -0,0 +1 @@ +46ca641fd65e566b8ecfa567a1f01766289192f8 diff --git a/format/commitgraph/read/testdata/fixtures/sha1/single_changed/repo.git/HEAD b/format/commitgraph/read/testdata/fixtures/sha1/single_changed/repo.git/HEAD new file mode 100644 index 00000000..b870d826 --- /dev/null +++ b/format/commitgraph/read/testdata/fixtures/sha1/single_changed/repo.git/HEAD @@ -0,0 +1 @@ +ref: refs/heads/main diff --git a/format/commitgraph/read/testdata/fixtures/sha1/single_changed/repo.git/config b/format/commitgraph/read/testdata/fixtures/sha1/single_changed/repo.git/config new file mode 100644 index 00000000..07d359d0 --- /dev/null +++ b/format/commitgraph/read/testdata/fixtures/sha1/single_changed/repo.git/config @@ -0,0 +1,4 @@ +[core] + repositoryformatversion = 0 + filemode = true + bare = true diff --git a/format/commitgraph/read/testdata/fixtures/sha1/single_changed/repo.git/objects/info/commit-graph b/format/commitgraph/read/testdata/fixtures/sha1/single_changed/repo.git/objects/info/commit-graph new file mode 100644 index 00000000..56b59a54 Binary files /dev/null and b/format/commitgraph/read/testdata/fixtures/sha1/single_changed/repo.git/objects/info/commit-graph differ diff --git a/format/commitgraph/read/testdata/fixtures/sha1/single_changed/repo.git/objects/info/packs b/format/commitgraph/read/testdata/fixtures/sha1/single_changed/repo.git/objects/info/packs new file mode 100644 index 00000000..ecf5d272 --- /dev/null +++ b/format/commitgraph/read/testdata/fixtures/sha1/single_changed/repo.git/objects/info/packs @@ -0,0 +1,2 @@ +P pack-34e9e132566989e2abfe8821731236c77f9bcbe9.pack + diff --git a/format/commitgraph/read/testdata/fixtures/sha1/single_changed/repo.git/objects/pack/pack-34e9e132566989e2abfe8821731236c77f9bcbe9.bitmap b/format/commitgraph/read/testdata/fixtures/sha1/single_changed/repo.git/objects/pack/pack-34e9e132566989e2abfe8821731236c77f9bcbe9.bitmap new file mode 100644 index 00000000..9fec7b16 Binary files /dev/null and b/format/commitgraph/read/testdata/fixtures/sha1/single_changed/repo.git/objects/pack/pack-34e9e132566989e2abfe8821731236c77f9bcbe9.bitmap differ diff --git a/format/commitgraph/read/testdata/fixtures/sha1/single_changed/repo.git/objects/pack/pack-34e9e132566989e2abfe8821731236c77f9bcbe9.idx b/format/commitgraph/read/testdata/fixtures/sha1/single_changed/repo.git/objects/pack/pack-34e9e132566989e2abfe8821731236c77f9bcbe9.idx new file mode 100644 index 00000000..e30cbb5a Binary files /dev/null and b/format/commitgraph/read/testdata/fixtures/sha1/single_changed/repo.git/objects/pack/pack-34e9e132566989e2abfe8821731236c77f9bcbe9.idx differ diff --git a/format/commitgraph/read/testdata/fixtures/sha1/single_changed/repo.git/objects/pack/pack-34e9e132566989e2abfe8821731236c77f9bcbe9.pack b/format/commitgraph/read/testdata/fixtures/sha1/single_changed/repo.git/objects/pack/pack-34e9e132566989e2abfe8821731236c77f9bcbe9.pack new file mode 100644 index 00000000..8da45eab Binary files /dev/null and b/format/commitgraph/read/testdata/fixtures/sha1/single_changed/repo.git/objects/pack/pack-34e9e132566989e2abfe8821731236c77f9bcbe9.pack differ diff --git a/format/commitgraph/read/testdata/fixtures/sha1/single_changed/repo.git/objects/pack/pack-34e9e132566989e2abfe8821731236c77f9bcbe9.rev b/format/commitgraph/read/testdata/fixtures/sha1/single_changed/repo.git/objects/pack/pack-34e9e132566989e2abfe8821731236c77f9bcbe9.rev new file mode 100644 index 00000000..3bcd2e2c Binary files /dev/null and b/format/commitgraph/read/testdata/fixtures/sha1/single_changed/repo.git/objects/pack/pack-34e9e132566989e2abfe8821731236c77f9bcbe9.rev differ diff --git a/format/commitgraph/read/testdata/fixtures/sha1/single_changed/repo.git/refs/heads/main b/format/commitgraph/read/testdata/fixtures/sha1/single_changed/repo.git/refs/heads/main new file mode 100644 index 00000000..090ca933 --- /dev/null +++ b/format/commitgraph/read/testdata/fixtures/sha1/single_changed/repo.git/refs/heads/main @@ -0,0 +1 @@ +d02a8dbd1a8fbaac8ab7f7f1533cc312ab2c9eec diff --git a/format/commitgraph/read/testdata/fixtures/sha1/single_nochanged/repo.git/HEAD b/format/commitgraph/read/testdata/fixtures/sha1/single_nochanged/repo.git/HEAD new file mode 100644 index 00000000..cb089cd8 --- /dev/null +++ b/format/commitgraph/read/testdata/fixtures/sha1/single_nochanged/repo.git/HEAD @@ -0,0 +1 @@ +ref: refs/heads/master diff --git a/format/commitgraph/read/testdata/fixtures/sha1/single_nochanged/repo.git/config b/format/commitgraph/read/testdata/fixtures/sha1/single_nochanged/repo.git/config new file mode 100644 index 00000000..07d359d0 --- /dev/null +++ b/format/commitgraph/read/testdata/fixtures/sha1/single_nochanged/repo.git/config @@ -0,0 +1,4 @@ +[core] + repositoryformatversion = 0 + filemode = true + bare = true diff --git a/format/commitgraph/read/testdata/fixtures/sha1/single_nochanged/repo.git/objects/info/commit-graph b/format/commitgraph/read/testdata/fixtures/sha1/single_nochanged/repo.git/objects/info/commit-graph new file mode 100644 index 00000000..28f7d06a Binary files /dev/null and b/format/commitgraph/read/testdata/fixtures/sha1/single_nochanged/repo.git/objects/info/commit-graph differ diff --git a/format/commitgraph/read/testdata/fixtures/sha1/single_nochanged/repo.git/objects/info/packs b/format/commitgraph/read/testdata/fixtures/sha1/single_nochanged/repo.git/objects/info/packs new file mode 100644 index 00000000..8434a002 --- /dev/null +++ b/format/commitgraph/read/testdata/fixtures/sha1/single_nochanged/repo.git/objects/info/packs @@ -0,0 +1,2 @@ +P pack-a3da595034c94bb16b6829d757a66b7d259b9ffc.pack + diff --git a/format/commitgraph/read/testdata/fixtures/sha1/single_nochanged/repo.git/objects/pack/pack-a3da595034c94bb16b6829d757a66b7d259b9ffc.bitmap b/format/commitgraph/read/testdata/fixtures/sha1/single_nochanged/repo.git/objects/pack/pack-a3da595034c94bb16b6829d757a66b7d259b9ffc.bitmap new file mode 100644 index 00000000..64a36c71 Binary files /dev/null and b/format/commitgraph/read/testdata/fixtures/sha1/single_nochanged/repo.git/objects/pack/pack-a3da595034c94bb16b6829d757a66b7d259b9ffc.bitmap differ diff --git a/format/commitgraph/read/testdata/fixtures/sha1/single_nochanged/repo.git/objects/pack/pack-a3da595034c94bb16b6829d757a66b7d259b9ffc.idx b/format/commitgraph/read/testdata/fixtures/sha1/single_nochanged/repo.git/objects/pack/pack-a3da595034c94bb16b6829d757a66b7d259b9ffc.idx new file mode 100644 index 00000000..f5e16674 Binary files /dev/null and b/format/commitgraph/read/testdata/fixtures/sha1/single_nochanged/repo.git/objects/pack/pack-a3da595034c94bb16b6829d757a66b7d259b9ffc.idx differ diff --git a/format/commitgraph/read/testdata/fixtures/sha1/single_nochanged/repo.git/objects/pack/pack-a3da595034c94bb16b6829d757a66b7d259b9ffc.pack b/format/commitgraph/read/testdata/fixtures/sha1/single_nochanged/repo.git/objects/pack/pack-a3da595034c94bb16b6829d757a66b7d259b9ffc.pack new file mode 100644 index 00000000..8f82b451 Binary files /dev/null and b/format/commitgraph/read/testdata/fixtures/sha1/single_nochanged/repo.git/objects/pack/pack-a3da595034c94bb16b6829d757a66b7d259b9ffc.pack differ diff --git a/format/commitgraph/read/testdata/fixtures/sha1/single_nochanged/repo.git/objects/pack/pack-a3da595034c94bb16b6829d757a66b7d259b9ffc.rev b/format/commitgraph/read/testdata/fixtures/sha1/single_nochanged/repo.git/objects/pack/pack-a3da595034c94bb16b6829d757a66b7d259b9ffc.rev new file mode 100644 index 00000000..64771f70 Binary files /dev/null and b/format/commitgraph/read/testdata/fixtures/sha1/single_nochanged/repo.git/objects/pack/pack-a3da595034c94bb16b6829d757a66b7d259b9ffc.rev differ diff --git a/format/commitgraph/read/testdata/fixtures/sha1/single_nochanged/repo.git/refs/heads/master b/format/commitgraph/read/testdata/fixtures/sha1/single_nochanged/repo.git/refs/heads/master new file mode 100644 index 00000000..475cb2c1 --- /dev/null +++ b/format/commitgraph/read/testdata/fixtures/sha1/single_nochanged/repo.git/refs/heads/master @@ -0,0 +1 @@ +dda8217252bdf3e01fdf31309d0e5c3051b00945 diff --git a/format/commitgraph/read/testdata/fixtures/sha256/chain_changed/repo.git/HEAD b/format/commitgraph/read/testdata/fixtures/sha256/chain_changed/repo.git/HEAD new file mode 100644 index 00000000..cb089cd8 --- /dev/null +++ b/format/commitgraph/read/testdata/fixtures/sha256/chain_changed/repo.git/HEAD @@ -0,0 +1 @@ +ref: refs/heads/master diff --git a/format/commitgraph/read/testdata/fixtures/sha256/chain_changed/repo.git/config b/format/commitgraph/read/testdata/fixtures/sha256/chain_changed/repo.git/config new file mode 100644 index 00000000..7d1c0006 --- /dev/null +++ b/format/commitgraph/read/testdata/fixtures/sha256/chain_changed/repo.git/config @@ -0,0 +1,6 @@ +[extensions] + objectformat = sha256 +[core] + repositoryformatversion = 1 + filemode = true + bare = true diff --git a/format/commitgraph/read/testdata/fixtures/sha256/chain_changed/repo.git/objects/info/commit-graphs/commit-graph-chain b/format/commitgraph/read/testdata/fixtures/sha256/chain_changed/repo.git/objects/info/commit-graphs/commit-graph-chain new file mode 100644 index 00000000..4e7d76fe --- /dev/null +++ b/format/commitgraph/read/testdata/fixtures/sha256/chain_changed/repo.git/objects/info/commit-graphs/commit-graph-chain @@ -0,0 +1,2 @@ +505cab61f8ddfa614301e8f97943112739236c6bcd19ed4d1f7c6b830cab4f62 +77c47bd6ca2ce17208c9361717a5823c0cb4b5ee336a14959678e060d674ffb6 diff --git a/format/commitgraph/read/testdata/fixtures/sha256/chain_changed/repo.git/objects/info/commit-graphs/graph-505cab61f8ddfa614301e8f97943112739236c6bcd19ed4d1f7c6b830cab4f62.graph b/format/commitgraph/read/testdata/fixtures/sha256/chain_changed/repo.git/objects/info/commit-graphs/graph-505cab61f8ddfa614301e8f97943112739236c6bcd19ed4d1f7c6b830cab4f62.graph new file mode 100644 index 00000000..4a93de94 Binary files /dev/null and b/format/commitgraph/read/testdata/fixtures/sha256/chain_changed/repo.git/objects/info/commit-graphs/graph-505cab61f8ddfa614301e8f97943112739236c6bcd19ed4d1f7c6b830cab4f62.graph differ diff --git a/format/commitgraph/read/testdata/fixtures/sha256/chain_changed/repo.git/objects/info/commit-graphs/graph-77c47bd6ca2ce17208c9361717a5823c0cb4b5ee336a14959678e060d674ffb6.graph b/format/commitgraph/read/testdata/fixtures/sha256/chain_changed/repo.git/objects/info/commit-graphs/graph-77c47bd6ca2ce17208c9361717a5823c0cb4b5ee336a14959678e060d674ffb6.graph new file mode 100644 index 00000000..7807351d Binary files /dev/null and b/format/commitgraph/read/testdata/fixtures/sha256/chain_changed/repo.git/objects/info/commit-graphs/graph-77c47bd6ca2ce17208c9361717a5823c0cb4b5ee336a14959678e060d674ffb6.graph differ diff --git a/format/commitgraph/read/testdata/fixtures/sha256/chain_changed/repo.git/objects/info/packs b/format/commitgraph/read/testdata/fixtures/sha256/chain_changed/repo.git/objects/info/packs new file mode 100644 index 00000000..3b1241c4 --- /dev/null +++ b/format/commitgraph/read/testdata/fixtures/sha256/chain_changed/repo.git/objects/info/packs @@ -0,0 +1,2 @@ +P pack-04168d0884c910f505cb9fbcf045957e44ccee06d812b5e531ae666014a26ed1.pack + diff --git a/format/commitgraph/read/testdata/fixtures/sha256/chain_changed/repo.git/objects/pack/pack-04168d0884c910f505cb9fbcf045957e44ccee06d812b5e531ae666014a26ed1.bitmap b/format/commitgraph/read/testdata/fixtures/sha256/chain_changed/repo.git/objects/pack/pack-04168d0884c910f505cb9fbcf045957e44ccee06d812b5e531ae666014a26ed1.bitmap new file mode 100644 index 00000000..007fcd0e Binary files /dev/null and b/format/commitgraph/read/testdata/fixtures/sha256/chain_changed/repo.git/objects/pack/pack-04168d0884c910f505cb9fbcf045957e44ccee06d812b5e531ae666014a26ed1.bitmap differ diff --git a/format/commitgraph/read/testdata/fixtures/sha256/chain_changed/repo.git/objects/pack/pack-04168d0884c910f505cb9fbcf045957e44ccee06d812b5e531ae666014a26ed1.idx b/format/commitgraph/read/testdata/fixtures/sha256/chain_changed/repo.git/objects/pack/pack-04168d0884c910f505cb9fbcf045957e44ccee06d812b5e531ae666014a26ed1.idx new file mode 100644 index 00000000..248cf8fc Binary files /dev/null and b/format/commitgraph/read/testdata/fixtures/sha256/chain_changed/repo.git/objects/pack/pack-04168d0884c910f505cb9fbcf045957e44ccee06d812b5e531ae666014a26ed1.idx differ diff --git a/format/commitgraph/read/testdata/fixtures/sha256/chain_changed/repo.git/objects/pack/pack-04168d0884c910f505cb9fbcf045957e44ccee06d812b5e531ae666014a26ed1.pack b/format/commitgraph/read/testdata/fixtures/sha256/chain_changed/repo.git/objects/pack/pack-04168d0884c910f505cb9fbcf045957e44ccee06d812b5e531ae666014a26ed1.pack new file mode 100644 index 00000000..92cea7fb Binary files /dev/null and b/format/commitgraph/read/testdata/fixtures/sha256/chain_changed/repo.git/objects/pack/pack-04168d0884c910f505cb9fbcf045957e44ccee06d812b5e531ae666014a26ed1.pack differ diff --git a/format/commitgraph/read/testdata/fixtures/sha256/chain_changed/repo.git/objects/pack/pack-04168d0884c910f505cb9fbcf045957e44ccee06d812b5e531ae666014a26ed1.rev b/format/commitgraph/read/testdata/fixtures/sha256/chain_changed/repo.git/objects/pack/pack-04168d0884c910f505cb9fbcf045957e44ccee06d812b5e531ae666014a26ed1.rev new file mode 100644 index 00000000..569862ce Binary files /dev/null and b/format/commitgraph/read/testdata/fixtures/sha256/chain_changed/repo.git/objects/pack/pack-04168d0884c910f505cb9fbcf045957e44ccee06d812b5e531ae666014a26ed1.rev differ diff --git a/format/commitgraph/read/testdata/fixtures/sha256/chain_changed/repo.git/refs/heads/master b/format/commitgraph/read/testdata/fixtures/sha256/chain_changed/repo.git/refs/heads/master new file mode 100644 index 00000000..29d83be8 --- /dev/null +++ b/format/commitgraph/read/testdata/fixtures/sha256/chain_changed/repo.git/refs/heads/master @@ -0,0 +1 @@ +10d2943dc7ad88011cae3b776d9565d6451a350ce1d16949bc8546a5fe6c0a53 diff --git a/format/commitgraph/read/testdata/fixtures/sha256/single_changed/repo.git/HEAD b/format/commitgraph/read/testdata/fixtures/sha256/single_changed/repo.git/HEAD new file mode 100644 index 00000000..b870d826 --- /dev/null +++ b/format/commitgraph/read/testdata/fixtures/sha256/single_changed/repo.git/HEAD @@ -0,0 +1 @@ +ref: refs/heads/main diff --git a/format/commitgraph/read/testdata/fixtures/sha256/single_changed/repo.git/config b/format/commitgraph/read/testdata/fixtures/sha256/single_changed/repo.git/config new file mode 100644 index 00000000..7d1c0006 --- /dev/null +++ b/format/commitgraph/read/testdata/fixtures/sha256/single_changed/repo.git/config @@ -0,0 +1,6 @@ +[extensions] + objectformat = sha256 +[core] + repositoryformatversion = 1 + filemode = true + bare = true diff --git a/format/commitgraph/read/testdata/fixtures/sha256/single_changed/repo.git/objects/info/commit-graph b/format/commitgraph/read/testdata/fixtures/sha256/single_changed/repo.git/objects/info/commit-graph new file mode 100644 index 00000000..f4dd0e0c Binary files /dev/null and b/format/commitgraph/read/testdata/fixtures/sha256/single_changed/repo.git/objects/info/commit-graph differ diff --git a/format/commitgraph/read/testdata/fixtures/sha256/single_changed/repo.git/objects/info/packs b/format/commitgraph/read/testdata/fixtures/sha256/single_changed/repo.git/objects/info/packs new file mode 100644 index 00000000..0f39ed89 --- /dev/null +++ b/format/commitgraph/read/testdata/fixtures/sha256/single_changed/repo.git/objects/info/packs @@ -0,0 +1,2 @@ +P pack-316dbc67dac12d131591640da0c55b76387cbf1fd2a117ab3d7ca0d854a031c9.pack + diff --git a/format/commitgraph/read/testdata/fixtures/sha256/single_changed/repo.git/objects/pack/pack-316dbc67dac12d131591640da0c55b76387cbf1fd2a117ab3d7ca0d854a031c9.bitmap b/format/commitgraph/read/testdata/fixtures/sha256/single_changed/repo.git/objects/pack/pack-316dbc67dac12d131591640da0c55b76387cbf1fd2a117ab3d7ca0d854a031c9.bitmap new file mode 100644 index 00000000..b5c5055c Binary files /dev/null and b/format/commitgraph/read/testdata/fixtures/sha256/single_changed/repo.git/objects/pack/pack-316dbc67dac12d131591640da0c55b76387cbf1fd2a117ab3d7ca0d854a031c9.bitmap differ diff --git a/format/commitgraph/read/testdata/fixtures/sha256/single_changed/repo.git/objects/pack/pack-316dbc67dac12d131591640da0c55b76387cbf1fd2a117ab3d7ca0d854a031c9.idx b/format/commitgraph/read/testdata/fixtures/sha256/single_changed/repo.git/objects/pack/pack-316dbc67dac12d131591640da0c55b76387cbf1fd2a117ab3d7ca0d854a031c9.idx new file mode 100644 index 00000000..144778cd Binary files /dev/null and b/format/commitgraph/read/testdata/fixtures/sha256/single_changed/repo.git/objects/pack/pack-316dbc67dac12d131591640da0c55b76387cbf1fd2a117ab3d7ca0d854a031c9.idx differ diff --git a/format/commitgraph/read/testdata/fixtures/sha256/single_changed/repo.git/objects/pack/pack-316dbc67dac12d131591640da0c55b76387cbf1fd2a117ab3d7ca0d854a031c9.pack b/format/commitgraph/read/testdata/fixtures/sha256/single_changed/repo.git/objects/pack/pack-316dbc67dac12d131591640da0c55b76387cbf1fd2a117ab3d7ca0d854a031c9.pack new file mode 100644 index 00000000..599ccae0 Binary files /dev/null and b/format/commitgraph/read/testdata/fixtures/sha256/single_changed/repo.git/objects/pack/pack-316dbc67dac12d131591640da0c55b76387cbf1fd2a117ab3d7ca0d854a031c9.pack differ diff --git a/format/commitgraph/read/testdata/fixtures/sha256/single_changed/repo.git/objects/pack/pack-316dbc67dac12d131591640da0c55b76387cbf1fd2a117ab3d7ca0d854a031c9.rev b/format/commitgraph/read/testdata/fixtures/sha256/single_changed/repo.git/objects/pack/pack-316dbc67dac12d131591640da0c55b76387cbf1fd2a117ab3d7ca0d854a031c9.rev new file mode 100644 index 00000000..3c093f31 Binary files /dev/null and b/format/commitgraph/read/testdata/fixtures/sha256/single_changed/repo.git/objects/pack/pack-316dbc67dac12d131591640da0c55b76387cbf1fd2a117ab3d7ca0d854a031c9.rev differ diff --git a/format/commitgraph/read/testdata/fixtures/sha256/single_changed/repo.git/refs/heads/main b/format/commitgraph/read/testdata/fixtures/sha256/single_changed/repo.git/refs/heads/main new file mode 100644 index 00000000..4ba32358 --- /dev/null +++ b/format/commitgraph/read/testdata/fixtures/sha256/single_changed/repo.git/refs/heads/main @@ -0,0 +1 @@ +a9ff114900e6be139ec66a2a61c930973d8c4bc6fd3b899405ee7ab8740bdbd3 diff --git a/format/commitgraph/read/testdata/fixtures/sha256/single_nochanged/repo.git/HEAD b/format/commitgraph/read/testdata/fixtures/sha256/single_nochanged/repo.git/HEAD new file mode 100644 index 00000000..cb089cd8 --- /dev/null +++ b/format/commitgraph/read/testdata/fixtures/sha256/single_nochanged/repo.git/HEAD @@ -0,0 +1 @@ +ref: refs/heads/master diff --git a/format/commitgraph/read/testdata/fixtures/sha256/single_nochanged/repo.git/config b/format/commitgraph/read/testdata/fixtures/sha256/single_nochanged/repo.git/config new file mode 100644 index 00000000..7d1c0006 --- /dev/null +++ b/format/commitgraph/read/testdata/fixtures/sha256/single_nochanged/repo.git/config @@ -0,0 +1,6 @@ +[extensions] + objectformat = sha256 +[core] + repositoryformatversion = 1 + filemode = true + bare = true diff --git a/format/commitgraph/read/testdata/fixtures/sha256/single_nochanged/repo.git/objects/info/commit-graph b/format/commitgraph/read/testdata/fixtures/sha256/single_nochanged/repo.git/objects/info/commit-graph new file mode 100644 index 00000000..f98ca4a1 Binary files /dev/null and b/format/commitgraph/read/testdata/fixtures/sha256/single_nochanged/repo.git/objects/info/commit-graph differ diff --git a/format/commitgraph/read/testdata/fixtures/sha256/single_nochanged/repo.git/objects/info/packs b/format/commitgraph/read/testdata/fixtures/sha256/single_nochanged/repo.git/objects/info/packs new file mode 100644 index 00000000..65184c9a --- /dev/null +++ b/format/commitgraph/read/testdata/fixtures/sha256/single_nochanged/repo.git/objects/info/packs @@ -0,0 +1,2 @@ +P pack-d335453f760b064e36459d780ec9bf0e5dd596c0ee1ac6310136067c4f13438b.pack + diff --git a/format/commitgraph/read/testdata/fixtures/sha256/single_nochanged/repo.git/objects/pack/pack-d335453f760b064e36459d780ec9bf0e5dd596c0ee1ac6310136067c4f13438b.bitmap b/format/commitgraph/read/testdata/fixtures/sha256/single_nochanged/repo.git/objects/pack/pack-d335453f760b064e36459d780ec9bf0e5dd596c0ee1ac6310136067c4f13438b.bitmap new file mode 100644 index 00000000..53530f4c Binary files /dev/null and b/format/commitgraph/read/testdata/fixtures/sha256/single_nochanged/repo.git/objects/pack/pack-d335453f760b064e36459d780ec9bf0e5dd596c0ee1ac6310136067c4f13438b.bitmap differ diff --git a/format/commitgraph/read/testdata/fixtures/sha256/single_nochanged/repo.git/objects/pack/pack-d335453f760b064e36459d780ec9bf0e5dd596c0ee1ac6310136067c4f13438b.idx b/format/commitgraph/read/testdata/fixtures/sha256/single_nochanged/repo.git/objects/pack/pack-d335453f760b064e36459d780ec9bf0e5dd596c0ee1ac6310136067c4f13438b.idx new file mode 100644 index 00000000..b3a417a8 Binary files /dev/null and b/format/commitgraph/read/testdata/fixtures/sha256/single_nochanged/repo.git/objects/pack/pack-d335453f760b064e36459d780ec9bf0e5dd596c0ee1ac6310136067c4f13438b.idx differ diff --git a/format/commitgraph/read/testdata/fixtures/sha256/single_nochanged/repo.git/objects/pack/pack-d335453f760b064e36459d780ec9bf0e5dd596c0ee1ac6310136067c4f13438b.pack b/format/commitgraph/read/testdata/fixtures/sha256/single_nochanged/repo.git/objects/pack/pack-d335453f760b064e36459d780ec9bf0e5dd596c0ee1ac6310136067c4f13438b.pack new file mode 100644 index 00000000..d8dcedbf Binary files /dev/null and b/format/commitgraph/read/testdata/fixtures/sha256/single_nochanged/repo.git/objects/pack/pack-d335453f760b064e36459d780ec9bf0e5dd596c0ee1ac6310136067c4f13438b.pack differ diff --git a/format/commitgraph/read/testdata/fixtures/sha256/single_nochanged/repo.git/objects/pack/pack-d335453f760b064e36459d780ec9bf0e5dd596c0ee1ac6310136067c4f13438b.rev b/format/commitgraph/read/testdata/fixtures/sha256/single_nochanged/repo.git/objects/pack/pack-d335453f760b064e36459d780ec9bf0e5dd596c0ee1ac6310136067c4f13438b.rev new file mode 100644 index 00000000..e50d1a81 Binary files /dev/null and b/format/commitgraph/read/testdata/fixtures/sha256/single_nochanged/repo.git/objects/pack/pack-d335453f760b064e36459d780ec9bf0e5dd596c0ee1ac6310136067c4f13438b.rev differ diff --git a/format/commitgraph/read/testdata/fixtures/sha256/single_nochanged/repo.git/refs/heads/master b/format/commitgraph/read/testdata/fixtures/sha256/single_nochanged/repo.git/refs/heads/master new file mode 100644 index 00000000..a4e184b4 --- /dev/null +++ b/format/commitgraph/read/testdata/fixtures/sha256/single_nochanged/repo.git/refs/heads/master @@ -0,0 +1 @@ +7e396bf648e3b045c293d9fbdc533d4377d4e801d5d1fb57b84d22dd054a5860 diff --git a/format/doc.go b/format/doc.go new file mode 100644 index 00000000..0d2ec813 --- /dev/null +++ b/format/doc.go @@ -0,0 +1,5 @@ +// Package format encapsulates various git-related file formats. +// +// These are particularly the ones that aren't necessarily associated with +// a very clear domain that they obviously belong to. +package format diff --git a/format/packfile/delta/apply/apply.go b/format/packfile/delta/apply/apply.go new file mode 100644 index 00000000..f5006e3c --- /dev/null +++ b/format/packfile/delta/apply/apply.go @@ -0,0 +1,160 @@ +// Package apply applies Git delta instruction streams. +package apply + +import "fmt" + +// Apply applies one Git delta instruction stream to base. +func Apply(base, delta []byte) ([]byte, error) { + pos := 0 + + srcSize, err := readVarint(delta, &pos) + if err != nil { + return nil, err + } + + dstSize, err := readVarint(delta, &pos) + if err != nil { + return nil, err + } + + if srcSize != len(base) { + return nil, fmt.Errorf("delta/apply: delta source size mismatch: got %d want %d", srcSize, len(base)) + } + + out := make([]byte, dstSize) + outPos := 0 + + for pos < len(delta) { + op := delta[pos] + pos++ + + //nolint:nestif + if op&0x80 != 0 { + off := 0 + + if op&0x01 != 0 { + if pos >= len(delta) { + return nil, fmt.Errorf("delta/apply: malformed delta copy offset") + } + + off |= int(delta[pos]) + pos++ + } + + if op&0x02 != 0 { + if pos >= len(delta) { + return nil, fmt.Errorf("delta/apply: malformed delta copy offset") + } + + off |= int(delta[pos]) << 8 + pos++ + } + + if op&0x04 != 0 { + if pos >= len(delta) { + return nil, fmt.Errorf("delta/apply: malformed delta copy offset") + } + + off |= int(delta[pos]) << 16 + pos++ + } + + if op&0x08 != 0 { + if pos >= len(delta) { + return nil, fmt.Errorf("delta/apply: malformed delta copy offset") + } + + off |= int(delta[pos]) << 24 + pos++ + } + + n := 0 + + if op&0x10 != 0 { + if pos >= len(delta) { + return nil, fmt.Errorf("delta/apply: malformed delta copy size") + } + + n |= int(delta[pos]) + pos++ + } + + if op&0x20 != 0 { + if pos >= len(delta) { + return nil, fmt.Errorf("delta/apply: malformed delta copy size") + } + + n |= int(delta[pos]) << 8 + pos++ + } + + if op&0x40 != 0 { + if pos >= len(delta) { + return nil, fmt.Errorf("delta/apply: malformed delta copy size") + } + + n |= int(delta[pos]) << 16 + pos++ + } + + if n == 0 { + n = 0x10000 + } + + if off < 0 || n < 0 || off+n > len(base) || outPos+n > len(out) { + return nil, fmt.Errorf("delta/apply: delta copy out of bounds") + } + + copy(out[outPos:outPos+n], base[off:off+n]) + outPos += n + + continue + } + + if op == 0 { + return nil, fmt.Errorf("delta/apply: invalid delta opcode 0") + } + + n := int(op) + if pos+n > len(delta) || outPos+n > len(out) { + return nil, fmt.Errorf("delta/apply: delta insert out of bounds") + } + + copy(out[outPos:outPos+n], delta[pos:pos+n]) + outPos += n + pos += n + } + + if outPos != len(out) { + return nil, fmt.Errorf("delta/apply: delta output size mismatch: got %d want %d", outPos, len(out)) + } + + return out, nil +} + +// readVarint parses one Git delta varint and advances pos. +func readVarint(buf []byte, pos *int) (int, error) { + value := 0 + shift := uint(0) + + for { + if *pos >= len(buf) { + return 0, fmt.Errorf("delta/apply: malformed delta varint") + } + + b := buf[*pos] + *pos++ + + value |= int(b&0x7f) << shift + if b&0x80 == 0 { + break + } + + shift += 7 + if shift > 63 { + return 0, fmt.Errorf("delta/apply: delta varint overflow") + } + } + + return value, nil +} diff --git a/format/packfile/delta/apply/header.go b/format/packfile/delta/apply/header.go new file mode 100644 index 00000000..69c9659a --- /dev/null +++ b/format/packfile/delta/apply/header.go @@ -0,0 +1,47 @@ +package apply + +import ( + "fmt" + "io" +) + +// ReadHeaderSizes reads the first two varints in one inflated delta stream. +// +// Callers that continue reading the same stream should pass their own buffered +// byte reader and keep using that same reader afterwards. +func ReadHeaderSizes(reader io.ByteReader) (int, int, error) { + srcSize, err := readVarintFromByteReader(reader) + if err != nil { + return 0, 0, err + } + + dstSize, err := readVarintFromByteReader(reader) + if err != nil { + return 0, 0, err + } + + return srcSize, dstSize, nil +} + +// readVarintFromByteReader parses one Git delta varint from reader. +func readVarintFromByteReader(reader io.ByteReader) (int, error) { + value := 0 + shift := uint(0) + + for { + b, err := reader.ReadByte() + if err != nil { + return 0, fmt.Errorf("delta/apply: malformed delta varint: %w", err) + } + + value |= int(b&0x7f) << shift + if b&0x80 == 0 { + return value, nil + } + + shift += 7 + if shift > 63 { + return 0, fmt.Errorf("delta/apply: delta varint overflow") + } + } +} diff --git a/format/packfile/delta/doc.go b/format/packfile/delta/doc.go new file mode 100644 index 00000000..f63c96a8 --- /dev/null +++ b/format/packfile/delta/doc.go @@ -0,0 +1,2 @@ +// Package delta provides various routines to handle Git delta compression. +package delta diff --git a/format/packfile/doc.go b/format/packfile/doc.go new file mode 100644 index 00000000..cd4aacfc --- /dev/null +++ b/format/packfile/doc.go @@ -0,0 +1,5 @@ +// Package packfile provides Git packfile format parsing primitives. +package packfile + +// TODO: This could probably be moved into object/store/packed when we get the pack ingestion semantics right? +// Oh, wait, the other stores might still want pack constants like we provide here. diff --git a/format/packfile/entry.go b/format/packfile/entry.go new file mode 100644 index 00000000..0f9c7c8d --- /dev/null +++ b/format/packfile/entry.go @@ -0,0 +1,76 @@ +package packfile + +import ( + "fmt" + + objecttype "codeberg.org/lindenii/furgit/object/type" +) + +// Entry is one parsed pack entry prefix, including any delta base reference +// data that appears before the compressed payload. +type Entry struct { + // Type is the pack entry type. + Type objecttype.Type + // Size is the declared resulting object size. + Size int64 + // DataOffset is the byte offset from the start of the entry to the zlib + // payload bytes. + DataOffset int + // RefBaseID is the referenced base object ID bytes for ref-delta entries. + RefBaseID []byte + // OfsBaseDistance is the backward distance for ofs-delta entries. + OfsBaseDistance uint64 +} + +// ParseEntry parses one full pack entry prefix from data. +// +// hashSize must match the hash algorithm size used by the pack/index. +func ParseEntry(data []byte, hashSize int) (Entry, error) { + var zero Entry + + header, err := ParseEntryHeader(data) + if err != nil { + return zero, err + } + + entry := Entry{ + Type: header.Type, + Size: header.Size, + DataOffset: header.HeaderSize, + } + + switch entry.Type { + case objecttype.TypeCommit, objecttype.TypeTree, objecttype.TypeBlob, objecttype.TypeTag: + // Base object entries have no extra prefix fields. + case objecttype.TypeRefDelta: + if hashSize <= 0 { + return zero, fmt.Errorf("packfile: invalid hash size %d", hashSize) + } + + end := entry.DataOffset + hashSize + if end > len(data) { + return zero, fmt.Errorf("packfile: truncated ref-delta base id") + } + + entry.RefBaseID = data[entry.DataOffset:end] + entry.DataOffset = end + case objecttype.TypeOfsDelta: + dist, consumed, err := ParseOfsDeltaDistance(data[entry.DataOffset:]) + if err != nil { + return zero, err + } + + entry.OfsBaseDistance = dist + entry.DataOffset += consumed + case objecttype.TypeInvalid, objecttype.TypeFuture: + return zero, fmt.Errorf("packfile: unsupported object type %d", entry.Type) + default: + return zero, fmt.Errorf("packfile: unsupported object type %d", entry.Type) + } + + if entry.DataOffset > len(data) { + return zero, fmt.Errorf("packfile: entry data offset out of bounds") + } + + return entry, nil +} diff --git a/format/packfile/entry_header.go b/format/packfile/entry_header.go new file mode 100644 index 00000000..05664268 --- /dev/null +++ b/format/packfile/entry_header.go @@ -0,0 +1,52 @@ +package packfile + +import ( + "fmt" + + objecttype "codeberg.org/lindenii/furgit/object/type" +) + +// EntryHeader is one parsed pack entry header prefix. +type EntryHeader struct { + // Type is the entry type tag from the first header byte. + Type objecttype.Type + // Size is the declared resulting object size. + Size int64 + // HeaderSize is the number of bytes consumed by the type/size header. + HeaderSize int +} + +// ParseEntryHeader parses one pack entry type/size header from data. +func ParseEntryHeader(data []byte) (EntryHeader, error) { + var zero EntryHeader + if len(data) == 0 { + return zero, fmt.Errorf("packfile: truncated entry header") + } + + first := data[0] + header := EntryHeader{ + Type: objecttype.Type((first >> 4) & 0x07), + Size: int64(first & 0x0f), + HeaderSize: 1, + } + + shift := uint(4) + + b := first + for b&0x80 != 0 { + if header.HeaderSize >= len(data) { + return zero, fmt.Errorf("packfile: truncated entry header") + } + + b = data[header.HeaderSize] + header.HeaderSize++ + header.Size |= int64(b&0x7f) << shift + shift += 7 + } + + if header.Size < 0 { + return zero, fmt.Errorf("packfile: negative entry size") + } + + return header, nil +} diff --git a/format/packfile/header.go b/format/packfile/header.go new file mode 100644 index 00000000..bc859a55 --- /dev/null +++ b/format/packfile/header.go @@ -0,0 +1,9 @@ +package packfile + +// Signature is the 4-byte "PACK" magic at the start of pack files. +const Signature = 0x5041434b + +// VersionSupported reports whether one pack version is supported. +func VersionSupported(version uint32) bool { + return version == 2 || version == 3 +} diff --git a/format/packfile/ingest/api.go b/format/packfile/ingest/api.go new file mode 100644 index 00000000..ce366a4f --- /dev/null +++ b/format/packfile/ingest/api.go @@ -0,0 +1,195 @@ +package ingest + +import ( + "bufio" + "bytes" + "errors" + "io" + "os" + + objectid "codeberg.org/lindenii/furgit/object/id" + objectstorer "codeberg.org/lindenii/furgit/object/storer" +) + +// Options controls one pack ingest operation. +type Options struct { + // FixThin appends missing local bases for thin packs. + FixThin bool + // WriteRev writes a .rev alongside the .pack and .idx. + WriteRev bool + // Base supplies existing objects for thin-pack fixup. + Base objectstorer.Store + // Progress receives human-readable progress messages. + // + // When nil, no progress output is emitted. + Progress io.Writer + // ProgressFlush flushes transport output after progress writes. + // + // When nil, no explicit flush is attempted. + ProgressFlush func() error + // RequireTrailingEOF requires the source to hit EOF after the pack trailer. + // + // This is suitable for exact pack-file readers, but should be disabled for + // full-duplex transport streams like receive-pack where the peer keeps the + // connection open to read the server response. + RequireTrailingEOF bool +} + +// Result describes one successful ingest transaction. +type Result struct { + // PackName is the destination-relative filename of the written .pack. + PackName string + // IdxName is the destination-relative filename of the written .idx. + IdxName string + // RevName is the destination-relative filename of the written .rev. + // + // RevName is empty when writeRev is false. + RevName string + // PackHash is the final pack hash (same hash embedded in .idx/.rev trailers). + PackHash objectid.ObjectID + // ObjectCount is the final object count in the resulting pack. + // + // If thin fixup appends objects, this includes appended base objects. + ObjectCount uint32 + // ThinFixed reports whether thin fixup appended local bases. + ThinFixed bool +} + +// HeaderInfo describes the parsed PACK header. +type HeaderInfo struct { + Version uint32 + ObjectCount uint32 +} + +// DiscardResult describes one successful Discard call. +type DiscardResult struct { + PackHash objectid.ObjectID + ObjectCount uint32 +} + +// Pending is one started ingest operation awaiting Continue or Discard. +// +// Exactly one of Continue or Discard may be called. +type Pending struct { + reader *bufio.Reader + algo objectid.Algorithm + opts Options + header HeaderInfo + headerRaw [packHeaderSize]byte + + finalized bool +} + +// Ingest reads and validates one PACK header, returning one pending operation. +func Ingest( + src io.Reader, + algo objectid.Algorithm, + opts Options, +) (*Pending, error) { + if algo.Size() == 0 { + return nil, objectid.ErrInvalidAlgorithm + } + + reader := bufio.NewReader(src) + + header, headerRaw, err := readAndValidatePackHeader(reader) + if err != nil { + return nil, err + } + + return &Pending{ + reader: reader, + algo: algo, + opts: opts, + header: header, + headerRaw: headerRaw, + }, nil +} + +// Header returns parsed PACK header info. +func (pending *Pending) Header() HeaderInfo { + return pending.header +} + +// Continue ingests the pack stream into destination and writes pack artifacts. +// +// Continue is terminal. Further use of pending is undefined behavior. +// +// Artifacts are published under content-addressed final names derived from the +// resulting pack hash. If those final names already exist, Continue treats that +// as success and removes its temporary files. +func (pending *Pending) Continue(destination *os.Root) (Result, error) { + pending.finalized = true + + if pending.header.ObjectCount == 0 { + return Result{}, ErrZeroObjectContinue + } + + state, err := newIngestState( + pending.reader, + destination, + pending.algo, + pending.opts, + pending.header, + pending.headerRaw, + ) + if err != nil { + return Result{}, err + } + + return ingest(state) +} + +// Discard consumes and verifies one zero-object pack stream without writing +// files. +// +// Discard is terminal. Further use of pending is undefined behavior. +func (pending *Pending) Discard() (DiscardResult, error) { + pending.finalized = true + + if pending.header.ObjectCount != 0 { + return DiscardResult{}, ErrNonZeroDiscard + } + + hashImpl, err := pending.algo.New() + if err != nil { + return DiscardResult{}, err + } + + _, _ = hashImpl.Write(pending.headerRaw[:]) + + trailer := make([]byte, pending.algo.Size()) + + _, err = io.ReadFull(pending.reader, trailer) + if err != nil { + return DiscardResult{}, &PackTrailerMismatchError{} + } + + computed := hashImpl.Sum(nil) + if !bytes.Equal(computed, trailer) { + return DiscardResult{}, &PackTrailerMismatchError{} + } + + if pending.opts.RequireTrailingEOF { + var probe [1]byte + + n, err := pending.reader.Read(probe[:]) + if n > 0 || err == nil { + return DiscardResult{}, errors.New("packfile/ingest: pack has trailing garbage") + } + + if err != io.EOF { + return DiscardResult{}, err + } + } + + packHash, err := objectid.FromBytes(pending.algo, trailer) + if err != nil { + return DiscardResult{}, err + } + + return DiscardResult{ + PackHash: packHash, + ObjectCount: 0, + }, nil +} diff --git a/format/packfile/ingest/byteslice_reader.go b/format/packfile/ingest/byteslice_reader.go new file mode 100644 index 00000000..a1570ef3 --- /dev/null +++ b/format/packfile/ingest/byteslice_reader.go @@ -0,0 +1,21 @@ +package ingest + +import "io" + +// byteSliceReader implements io.ByteReader on []byte. +type byteSliceReader struct { + data []byte + pos int +} + +// ReadByte reads one byte from receiver. +func (reader *byteSliceReader) ReadByte() (byte, error) { + if reader.pos >= len(reader.data) { + return 0, io.EOF + } + + b := reader.data[reader.pos] + reader.pos++ + + return b, nil +} diff --git a/format/packfile/ingest/cache.go b/format/packfile/ingest/cache.go new file mode 100644 index 00000000..9a15f55f --- /dev/null +++ b/format/packfile/ingest/cache.go @@ -0,0 +1,53 @@ +package ingest + +import ( + "codeberg.org/lindenii/furgit/internal/lru" + objecttype "codeberg.org/lindenii/furgit/object/type" +) + +// deltaBaseCacheKey identifies one resolved base by record index. +type deltaBaseCacheKey struct { + recordIdx int +} + +// deltaBaseCacheValue stores one resolved base object payload. +type deltaBaseCacheValue struct { + realType objecttype.Type + content []byte +} + +// deltaBaseCache is a bounded LRU for resolved base payloads. +type deltaBaseCache struct { + lru *lru.Cache[deltaBaseCacheKey, deltaBaseCacheValue] +} + +// newDeltaBaseCache creates one bounded base cache. +func newDeltaBaseCache(maxBytes int64) *deltaBaseCache { + return &deltaBaseCache{ + lru: lru.New( + maxBytes, + func(_ deltaBaseCacheKey, value deltaBaseCacheValue) int64 { + return int64(len(value.content)) + }, + nil, + ), + } +} + +// get returns one cache entry for recordIdx. +func (cache *deltaBaseCache) get(recordIdx int) (objecttype.Type, []byte, bool) { + value, ok := cache.lru.Get(deltaBaseCacheKey{recordIdx: recordIdx}) + if !ok { + return objecttype.TypeInvalid, nil, false + } + + return value.realType, value.content, true +} + +// add stores one cache entry for recordIdx. +func (cache *deltaBaseCache) add(recordIdx int, realType objecttype.Type, content []byte) { + cache.lru.Add(deltaBaseCacheKey{recordIdx: recordIdx}, deltaBaseCacheValue{ + realType: realType, + content: content, + }) +} diff --git a/format/packfile/ingest/counting_writer.go b/format/packfile/ingest/counting_writer.go new file mode 100644 index 00000000..051ad9d1 --- /dev/null +++ b/format/packfile/ingest/counting_writer.go @@ -0,0 +1,17 @@ +package ingest + +import "io" + +// countingWriter counts bytes written to dst. +type countingWriter struct { + dst io.Writer + n int +} + +// Write writes src to dst and tracks output byte count. +func (writer *countingWriter) Write(src []byte) (int, error) { + n, err := writer.dst.Write(src) + writer.n += n + + return n, err +} diff --git a/format/packfile/ingest/crc.go b/format/packfile/ingest/crc.go new file mode 100644 index 00000000..f55af4ff --- /dev/null +++ b/format/packfile/ingest/crc.go @@ -0,0 +1,22 @@ +package ingest + +import "fmt" + +// beginEntryCRC starts inline CRC accumulation for one packed entry. +func (scanner *streamScanner) beginEntryCRC() { + scanner.entryCRC = 0 + scanner.inEntryCRC = true +} + +// endEntryCRC finishes inline CRC accumulation for one packed entry. +func (scanner *streamScanner) endEntryCRC() (uint32, error) { + if !scanner.inEntryCRC { + return 0, fmt.Errorf("packfile/ingest: entry CRC not started") + } + + crc := scanner.entryCRC + scanner.entryCRC = 0 + scanner.inEntryCRC = false + + return crc, nil +} diff --git a/format/packfile/ingest/delta_header.go b/format/packfile/ingest/delta_header.go new file mode 100644 index 00000000..110cf83b --- /dev/null +++ b/format/packfile/ingest/delta_header.go @@ -0,0 +1,11 @@ +package ingest + +import deltaapply "codeberg.org/lindenii/furgit/format/packfile/delta/apply" + +// finalizeStreamPackHash consumes trailer bytes and verifies stream integrity. +// readDeltaHeaderSizes reads source and destination sizes from one delta payload. +func readDeltaHeaderSizes(payload []byte) (int, int, error) { + reader := &byteSliceReader{data: payload} + + return deltaapply.ReadHeaderSizes(reader) +} diff --git a/format/packfile/ingest/distance.go b/format/packfile/ingest/distance.go new file mode 100644 index 00000000..9bc4d886 --- /dev/null +++ b/format/packfile/ingest/distance.go @@ -0,0 +1,30 @@ +package ingest + +import ( + "fmt" + "io" +) + +// readOfsDistanceFromStream reads one ofs-delta encoded distance. +func readOfsDistanceFromStream(reader io.ByteReader) (uint64, int, error) { + first, err := reader.ReadByte() + if err != nil { + return 0, 0, fmt.Errorf("read ofs distance first byte: %w", err) + } + + dist := uint64(first & 0x7f) + consumed := 1 + + b := first + for b&0x80 != 0 { + b, err = reader.ReadByte() + if err != nil { + return 0, 0, fmt.Errorf("read ofs distance continuation: %w", err) + } + + consumed++ + dist = ((dist + 1) << 7) + uint64(b&0x7f) + } + + return dist, consumed, nil +} diff --git a/format/packfile/ingest/doc.go b/format/packfile/ingest/doc.go new file mode 100644 index 00000000..2095068a --- /dev/null +++ b/format/packfile/ingest/doc.go @@ -0,0 +1,3 @@ +// Package ingest implements streaming ingestion of one Git pack stream into a +// destination root, producing .pack/.idx and optionally .rev. +package ingest diff --git a/format/packfile/ingest/drain.go b/format/packfile/ingest/drain.go new file mode 100644 index 00000000..ed6ec821 --- /dev/null +++ b/format/packfile/ingest/drain.go @@ -0,0 +1,68 @@ +package ingest + +import ( + "fmt" + "io" + + "codeberg.org/lindenii/furgit/internal/compress/zlib" + objectheader "codeberg.org/lindenii/furgit/object/header" + objectid "codeberg.org/lindenii/furgit/object/id" + objecttype "codeberg.org/lindenii/furgit/object/type" + packfmt "codeberg.org/lindenii/furgit/format/packfile" +) + +// drainEntryPayload inflates one entry payload from stream and returns +// (inflatedLength, oidForBaseEntry). +func drainEntryPayload(state *ingestState, record objectRecord) (int64, objectid.ObjectID, error) { + var zero objectid.ObjectID + + reader, err := zlib.NewReader(state.stream) + if err != nil { + return 0, zero, &MalformedPackEntryError{Offset: record.offset, Reason: fmt.Sprintf("open zlib stream: %v", err)} + } + + defer func() { _ = reader.Close() }() + + var total int64 + + if packfmt.IsBaseObjectType(record.packedType) { + header, ok := objectheader.Encode(record.packedType, record.declaredSize) + if !ok { + return 0, zero, &MalformedPackEntryError{Offset: record.offset, Reason: "encode object header"} + } + + hashImpl, err := state.algo.New() + if err != nil { + return 0, zero, err + } + + _, _ = hashImpl.Write(header) + + n, err := io.Copy(hashImpl, reader) + if err != nil { + return 0, zero, &MalformedPackEntryError{Offset: record.offset, Reason: fmt.Sprintf("inflate base object: %v", err)} + } + + total = n + + oid, err := objectid.FromBytes(state.algo, hashImpl.Sum(nil)) + if err != nil { + return 0, zero, err + } + + return total, oid, nil + } + + if record.packedType == objecttype.TypeOfsDelta || record.packedType == objecttype.TypeRefDelta { + n, err := io.Copy(io.Discard, reader) + if err != nil { + return 0, zero, &MalformedPackEntryError{Offset: record.offset, Reason: fmt.Sprintf("inflate delta payload: %v", err)} + } + + total = n + + return total, zero, nil + } + + return 0, zero, &MalformedPackEntryError{Offset: record.offset, Reason: "unsupported payload type"} +} diff --git a/format/packfile/ingest/entry.go b/format/packfile/ingest/entry.go new file mode 100644 index 00000000..4e2cab55 --- /dev/null +++ b/format/packfile/ingest/entry.go @@ -0,0 +1,92 @@ +package ingest + +import ( + "fmt" + + objecttype "codeberg.org/lindenii/furgit/object/type" + packfmt "codeberg.org/lindenii/furgit/format/packfile" +) + +// scanOneEntry scans one pack entry from stream and appends one record. +func scanOneEntry(state *ingestState, startOffset uint64) (uint64, error) { + state.stream.beginEntryCRC() + + record, err := parseEntryPrefix(state, startOffset) + if err != nil { + return 0, err + } + + payloadStartConsumed := state.stream.consumed + + contentLen, oid, err := drainEntryPayload(state, record) + if err != nil { + return 0, err + } + + consumedInput := state.stream.consumed - payloadStartConsumed + + if contentLen != record.declaredSize { + return 0, &MalformedPackEntryError{ + Offset: startOffset, + Reason: fmt.Sprintf("inflated size mismatch got %d want %d", contentLen, record.declaredSize), + } + } + + endOffset := startOffset + uint64(record.headerLen) + consumedInput + if endOffset > state.stream.consumed { + return 0, &MalformedPackEntryError{ + Offset: startOffset, + Reason: fmt.Sprintf("entry end offset overflow got %d > stream %d", endOffset, state.stream.consumed), + } + } + + record.packedLen = endOffset - startOffset + + record.dataOffset = startOffset + uint64(record.headerLen) + if record.packedLen < uint64(record.headerLen) { + return 0, &MalformedPackEntryError{Offset: startOffset, Reason: "negative payload span"} + } + + crc, err := state.stream.endEntryCRC() + if err != nil { + return 0, err + } + + record.crc32 = crc + + if packfmt.IsBaseObjectType(record.packedType) { + record.objectID = oid + record.realType = record.packedType + record.resolved = true + } + + recordIdx := len(state.records) + state.records = append(state.records, record) + + state.offsetToRecord[record.offset] = recordIdx + if record.resolved { + state.objectToRecord[record.objectID] = recordIdx + } + + switch record.packedType { + case objecttype.TypeOfsDelta: + state.ofsDeltas = append(state.ofsDeltas, ofsDeltaRef{ + baseOffset: record.baseOffset, + recordIdx: recordIdx, + }) + case objecttype.TypeRefDelta: + state.refDeltas = append(state.refDeltas, refDeltaRef{ + baseObject: record.baseObject, + recordIdx: recordIdx, + }) + case objecttype.TypeInvalid, + objecttype.TypeCommit, + objecttype.TypeTree, + objecttype.TypeBlob, + objecttype.TypeTag, + objecttype.TypeFuture: + default: + } + + return endOffset, nil +} diff --git a/format/packfile/ingest/entry_header.go b/format/packfile/ingest/entry_header.go new file mode 100644 index 00000000..c74fdc16 --- /dev/null +++ b/format/packfile/ingest/entry_header.go @@ -0,0 +1,33 @@ +package ingest + +import ( + "codeberg.org/lindenii/furgit/internal/intconv" + objecttype "codeberg.org/lindenii/furgit/object/type" +) + +// encodePackEntryHeader encodes one non-delta packed entry header. +func encodePackEntryHeader(ty objecttype.Type, size int64) []byte { + var out [16]byte + + n := 0 + + s, err := intconv.Int64ToUint64(size) + if err != nil { + panic(err) + } + + c := (uint8(ty) << 4) | byte(s&0x0f) + + s >>= 4 + for s != 0 { + out[n] = c | 0x80 + n++ + c = byte(s & 0x7f) + s >>= 7 + } + + out[n] = c + n++ + + return append([]byte(nil), out[:n]...) +} diff --git a/format/packfile/ingest/entry_prefix.go b/format/packfile/ingest/entry_prefix.go new file mode 100644 index 00000000..a107b4e8 --- /dev/null +++ b/format/packfile/ingest/entry_prefix.go @@ -0,0 +1,95 @@ +package ingest + +import ( + "fmt" + + "codeberg.org/lindenii/furgit/internal/intconv" + objectid "codeberg.org/lindenii/furgit/object/id" + objecttype "codeberg.org/lindenii/furgit/object/type" +) + +// parseEntryPrefix parses one entry prefix from stream. +func parseEntryPrefix(state *ingestState, startOffset uint64) (objectRecord, error) { + var record objectRecord + + record.offset = startOffset + + first, err := state.stream.ReadByte() + if err != nil { + return record, &MalformedPackEntryError{Offset: startOffset, Reason: fmt.Sprintf("read first header byte: %v", err)} + } + + record.packedType = objecttype.Type((first >> 4) & 0x07) + size := int64(first & 0x0f) + headerLen := uint32(1) + shift := uint(4) + b := first + + for b&0x80 != 0 { + b, err = state.stream.ReadByte() + if err != nil { + return record, &MalformedPackEntryError{Offset: startOffset, Reason: fmt.Sprintf("read size continuation: %v", err)} + } + + headerLen++ + size |= int64(b&0x7f) << shift + shift += 7 + } + + if size < 0 { + return record, &MalformedPackEntryError{Offset: startOffset, Reason: "negative declared size"} + } + + record.declaredSize = size + + switch record.packedType { + case objecttype.TypeCommit, objecttype.TypeTree, objecttype.TypeBlob, objecttype.TypeTag: + case objecttype.TypeRefDelta: + baseRaw := make([]byte, state.algo.Size()) + + err := state.stream.readFull(baseRaw) + if err != nil { + return record, &MalformedPackEntryError{Offset: startOffset, Reason: fmt.Sprintf("read ref base: %v", err)} + } + + baseID, err := objectid.FromBytes(state.algo, baseRaw) + if err != nil { + return record, &MalformedPackEntryError{Offset: startOffset, Reason: fmt.Sprintf("parse ref base: %v", err)} + } + + record.baseObject = baseID + + baseRawLen, err := intconv.IntToUint32(len(baseRaw)) + if err != nil { + return record, err + } + + headerLen += baseRawLen + case objecttype.TypeOfsDelta: + dist, consumed, err := readOfsDistanceFromStream(state.stream) + if err != nil { + return record, &MalformedPackEntryError{Offset: startOffset, Reason: err.Error()} + } + + if startOffset <= dist { + return record, &MalformedPackEntryError{Offset: startOffset, Reason: "ofs base offset out of bounds"} + } + + record.baseOffset = startOffset - dist + + consumedUint32, err := intconv.IntToUint32(consumed) + if err != nil { + return record, err + } + + headerLen += consumedUint32 + case objecttype.TypeInvalid, objecttype.TypeFuture: + return record, &MalformedPackEntryError{Offset: startOffset, Reason: fmt.Sprintf("unsupported object type %d", record.packedType)} + default: + return record, &MalformedPackEntryError{Offset: startOffset, Reason: fmt.Sprintf("unsupported object type %d", record.packedType)} + } + + record.headerLen = headerLen + + return record, nil +} diff --git a/format/packfile/ingest/errors.go b/format/packfile/ingest/errors.go new file mode 100644 index 00000000..f6ee9757 --- /dev/null +++ b/format/packfile/ingest/errors.go @@ -0,0 +1,75 @@ +package ingest + +import ( + "errors" + "fmt" +) + +// InvalidPackHeaderError reports an invalid or unsupported pack header. +type InvalidPackHeaderError struct { + Reason string +} + +// Error implements error. +func (err *InvalidPackHeaderError) Error() string { + return "packfile/ingest: invalid pack header: " + err.Reason +} + +// PackTrailerMismatchError reports a mismatch between computed and trailer pack hash. +type PackTrailerMismatchError struct{} + +// Error implements error. +func (err *PackTrailerMismatchError) Error() string { + return "packfile/ingest: pack trailer hash mismatch" +} + +// ThinPackUnresolvedError reports unresolved REF deltas when fixThin is disabled +// or when required bases cannot be found in base. +type ThinPackUnresolvedError struct { + Count int +} + +// Error implements error. +func (err *ThinPackUnresolvedError) Error() string { + return fmt.Sprintf("packfile/ingest: unresolved thin deltas: %d", err.Count) +} + +// MalformedPackEntryError reports malformed entry encoding at one pack offset. +type MalformedPackEntryError struct { + Offset uint64 + Reason string +} + +// Error implements error. +func (err *MalformedPackEntryError) Error() string { + return fmt.Sprintf("packfile/ingest: malformed pack entry at offset %d: %s", err.Offset, err.Reason) +} + +// DeltaCycleError reports a detected cycle in delta dependency resolution. +type DeltaCycleError struct { + Offset uint64 +} + +// Error implements error. +func (err *DeltaCycleError) Error() string { + return fmt.Sprintf("packfile/ingest: delta cycle detected at offset %d", err.Offset) +} + +// DestinationWriteError reports destination I/O failures. +type DestinationWriteError struct { + Op string +} + +// Error implements error. +func (err *DestinationWriteError) Error() string { + return "packfile/ingest: destination write failure: " + err.Op +} + +var errExternalThinBase = errors.New("packfile/ingest: external thin base required") + +var ( + // ErrZeroObjectContinue indicates Continue was called for a zero-object pack. + ErrZeroObjectContinue = errors.New("packfile/ingest: cannot continue zero-object pack") + // ErrNonZeroDiscard indicates Discard was called for a non-zero-object pack. + ErrNonZeroDiscard = errors.New("packfile/ingest: cannot discard non-zero pack") +) diff --git a/format/packfile/ingest/file_section_writer.go b/format/packfile/ingest/file_section_writer.go new file mode 100644 index 00000000..fa28c1a9 --- /dev/null +++ b/format/packfile/ingest/file_section_writer.go @@ -0,0 +1,22 @@ +package ingest + +import "os" + +// fileSectionWriter writes sequentially to file via WriteAt at one base offset. +type fileSectionWriter struct { + file *os.File + off int64 + pos int64 +} + +// Write writes src at current section position. +func (writer *fileSectionWriter) Write(src []byte) (int, error) { + if len(src) == 0 { + return 0, nil + } + + n, err := writer.file.WriteAt(src, writer.off+writer.pos) + writer.pos += int64(n) + + return n, err +} diff --git a/format/packfile/ingest/fill.go b/format/packfile/ingest/fill.go new file mode 100644 index 00000000..eca4e4d6 --- /dev/null +++ b/format/packfile/ingest/fill.go @@ -0,0 +1,44 @@ +package ingest + +import ( + "errors" + "fmt" + "io" +) + +// fill ensures at least min unread bytes are available in receiver's buffer. +func (scanner *streamScanner) fill(minLen int) error { + if minLen <= 0 { + return nil + } + + if minLen > len(scanner.buf) { + return fmt.Errorf("packfile/ingest: fill(%d) exceeds scanner buffer", minLen) + } + + for scanner.n-scanner.off < minLen { + err := scanner.flushConsumedPrefix() + if err != nil { + return err + } + + readN, err := scanner.src.Read(scanner.buf[scanner.n:]) + if readN > 0 { + scanner.n += readN + } + + if err != nil { + if errors.Is(err, io.EOF) && scanner.n-scanner.off >= minLen { + return nil + } + + return err + } + + if readN == 0 { + return io.ErrNoProgress + } + } + + return nil +} diff --git a/format/packfile/ingest/finalize.go b/format/packfile/ingest/finalize.go new file mode 100644 index 00000000..6fe4edb2 --- /dev/null +++ b/format/packfile/ingest/finalize.go @@ -0,0 +1,94 @@ +package ingest + +import ( + "errors" + "fmt" + "io/fs" + "strings" + + "codeberg.org/lindenii/furgit/internal/intconv" +) + +// finalizeArtifacts links temporary files to final names and returns Result. +func finalizeArtifacts(state *ingestState) (Result, error) { + base := "pack-" + state.packHash.String() + packFinal := base + ".pack" + idxFinal := base + ".idx" + + revFinal := "" + if state.opts.WriteRev { + revFinal = base + ".rev" + } + + err := linkTempToFinal(state, state.packTmpName, packFinal) + if err != nil { + return Result{}, err + } + + err = linkTempToFinal(state, state.idxTmpName, idxFinal) + if err != nil { + return Result{}, err + } + + if state.opts.WriteRev { + err := linkTempToFinal(state, state.revTmpName, revFinal) + if err != nil { + return Result{}, err + } + } + + objectCount, err := intconv.IntToUint32(len(state.records)) + if err != nil { + return Result{}, err + } + + return Result{ + PackName: packFinal, + IdxName: idxFinal, + RevName: revFinal, + PackHash: state.packHash, + ObjectCount: objectCount, + ThinFixed: state.thinFixed, + }, nil +} + +// rollbackTemporaryArtifacts removes temporary files after failure. +func rollbackTemporaryArtifacts(state *ingestState) { + if state.packTmpName != "" { + _ = state.destination.Remove(state.packTmpName) + } + + if state.idxTmpName != "" { + _ = state.destination.Remove(state.idxTmpName) + } + + if state.revTmpName != "" { + _ = state.destination.Remove(state.revTmpName) + } +} + +// linkTempToFinal hard-links tmp to final, tolerating existing final paths. +func linkTempToFinal(state *ingestState, tmp, final string) error { + if tmp == "" || final == "" { + return fmt.Errorf("packfile/ingest: invalid finalize names tmp=%q final=%q", tmp, final) + } + + if strings.Contains(final, "/") { + return fmt.Errorf("packfile/ingest: final name must be leaf: %q", final) + } + + err := state.destination.Link(tmp, final) + if err == nil { + _ = state.destination.Remove(tmp) + + return nil + } + + if errors.Is(err, fs.ErrExist) { + _ = state.destination.Remove(tmp) + + return nil + } + + return err +} diff --git a/format/packfile/ingest/flush.go b/format/packfile/ingest/flush.go new file mode 100644 index 00000000..96753170 --- /dev/null +++ b/format/packfile/ingest/flush.go @@ -0,0 +1,37 @@ +package ingest + +import "fmt" + +// flush writes all consumed-but-unflushed bytes to destination pack file. +func (scanner *streamScanner) flush() error { + return scanner.flushConsumedPrefix() +} + +// flushConsumedPrefix writes scanner.buf[:scanner.off] and compacts unread +// bytes to the start of buffer. +func (scanner *streamScanner) flushConsumedPrefix() error { + if scanner.off == 0 { + return nil + } + + written := 0 + for written < scanner.off { + n, err := scanner.dstFile.Write(scanner.buf[written:scanner.off]) + if err != nil { + return &DestinationWriteError{Op: fmt.Sprintf("write pack: %v", err)} + } + + if n == 0 { + return &DestinationWriteError{Op: "write pack: short write"} + } + + written += n + } + + unread := scanner.n - scanner.off + copy(scanner.buf[:unread], scanner.buf[scanner.off:scanner.n]) + scanner.off = 0 + scanner.n = unread + + return nil +} diff --git a/format/packfile/ingest/hash.go b/format/packfile/ingest/hash.go new file mode 100644 index 00000000..4b739c20 --- /dev/null +++ b/format/packfile/ingest/hash.go @@ -0,0 +1,27 @@ +package ingest + +import ( + "fmt" + + objectheader "codeberg.org/lindenii/furgit/object/header" + objectid "codeberg.org/lindenii/furgit/object/id" + objecttype "codeberg.org/lindenii/furgit/object/type" +) + +// hashCanonicalObject hashes canonical object bytes (header+content). +func hashCanonicalObject(algo objectid.Algorithm, ty objecttype.Type, content []byte) (objectid.ObjectID, error) { + header, ok := objectheader.Encode(ty, int64(len(content))) + if !ok { + return objectid.ObjectID{}, fmt.Errorf("packfile/ingest: encode object header for type %d", ty) + } + + hashImpl, err := algo.New() + if err != nil { + return objectid.ObjectID{}, err + } + + _, _ = hashImpl.Write(header) + _, _ = hashImpl.Write(content) + + return objectid.FromBytes(algo, hashImpl.Sum(nil)) +} diff --git a/format/packfile/ingest/header.go b/format/packfile/ingest/header.go new file mode 100644 index 00000000..6a214828 --- /dev/null +++ b/format/packfile/ingest/header.go @@ -0,0 +1,49 @@ +package ingest + +import ( + "encoding/binary" + "fmt" + "io" + + "codeberg.org/lindenii/furgit/format/packfile" +) + +const packHeaderSize = 12 + +// readAndValidatePackHeader reads one PACK header from src and validates it. +func readAndValidatePackHeader(src io.Reader) (HeaderInfo, [packHeaderSize]byte, error) { + var hdr [packHeaderSize]byte + + _, err := io.ReadFull(src, hdr[:]) + if err != nil { + return HeaderInfo{}, [packHeaderSize]byte{}, &InvalidPackHeaderError{ + Reason: fmt.Sprintf("read header: %v", err), + } + } + + header, err := parseAndValidatePackHeader(hdr) + if err != nil { + return HeaderInfo{}, [packHeaderSize]byte{}, err + } + + return header, hdr, nil +} + +// parseAndValidatePackHeader validates one already-read PACK header. +func parseAndValidatePackHeader(hdr [packHeaderSize]byte) (HeaderInfo, error) { + if binary.BigEndian.Uint32(hdr[:4]) != packfile.Signature { + return HeaderInfo{}, &InvalidPackHeaderError{Reason: "signature mismatch"} + } + + version := binary.BigEndian.Uint32(hdr[4:8]) + if !packfile.VersionSupported(version) { + return HeaderInfo{}, &InvalidPackHeaderError{ + Reason: fmt.Sprintf("unsupported version %d", version), + } + } + + return HeaderInfo{ + Version: version, + ObjectCount: binary.BigEndian.Uint32(hdr[8:12]), + }, nil +} diff --git a/format/packfile/ingest/idx_write.go b/format/packfile/ingest/idx_write.go new file mode 100644 index 00000000..506788b9 --- /dev/null +++ b/format/packfile/ingest/idx_write.go @@ -0,0 +1,266 @@ +package ingest + +import ( + "bytes" + "encoding/binary" + "fmt" + "hash" + "io" + "slices" + + "codeberg.org/lindenii/furgit/internal/intconv" + "codeberg.org/lindenii/furgit/internal/progress" +) + +const ( + idxMagicV2 = 0xff744f63 + idxVersionV2 = 2 +) + +// writeIdx writes idx v2 for resolved records. +func writeIdx(state *ingestState) error { + order := buildIdxOrder(state) + + hashImpl, err := state.algo.New() + if err != nil { + return err + } + + write := func(src []byte) error { + _, writeErr := state.idxFile.Write(src) + if writeErr != nil { + return writeErr + } + + _, writeErr = hashImpl.Write(src) + if writeErr != nil { + return writeErr + } + + return nil + } + + var ( + scratch [8]byte + fanout [256]uint32 + ) + + writeProgressf(state, "writing index fanout...\r") + + for _, recordIdx := range order { + idRaw := state.records[recordIdx].objectID.Bytes() + fanout[idRaw[0]]++ + } + + binary.BigEndian.PutUint32(scratch[:4], idxMagicV2) + binary.BigEndian.PutUint32(scratch[4:8], idxVersionV2) + + err = write(scratch[:8]) + if err != nil { + return err + } + + var cumulative uint32 + for i := range fanout { + cumulative += fanout[i] + binary.BigEndian.PutUint32(scratch[:4], cumulative) + + err := write(scratch[:4]) + if err != nil { + return err + } + } + + writeProgressf(state, "writing index fanout: done.\n") + + largeOffsetCount := 0 + + for idx := range state.records { + if state.records[idx].offset >= 0x80000000 { + largeOffsetCount++ + } + } + + oidMeter := progress.New(progress.Options{ + Writer: state.opts.Progress, + Flush: state.opts.ProgressFlush, + Title: "writing index object ids", + Total: uint64(len(order)), + }) + + var oidDone uint64 + + for _, recordIdx := range order { + idRaw := state.records[recordIdx].objectID.Bytes() + + err := write(idRaw) + if err != nil { + return err + } + + oidDone++ + oidMeter.Set(oidDone, 0) + } + + if oidDone > 0 { + oidMeter.Stop("done") + } + + crcMeter := progress.New(progress.Options{ + Writer: state.opts.Progress, + Flush: state.opts.ProgressFlush, + Title: "writing index crc32", + Total: uint64(len(order)), + }) + + var crcDone uint64 + + for _, recordIdx := range order { + binary.BigEndian.PutUint32(scratch[:4], state.records[recordIdx].crc32) + + err := write(scratch[:4]) + if err != nil { + return err + } + + crcDone++ + crcMeter.Set(crcDone, 0) + } + + if crcDone > 0 { + crcMeter.Stop("done") + } + + largeOffsets := make([]uint64, 0) + offsetMeter := progress.New(progress.Options{ + Writer: state.opts.Progress, + Flush: state.opts.ProgressFlush, + Title: "writing index offsets", + Total: uint64(len(order)), + }) + + var offsetDone uint64 + + for _, recordIdx := range order { + offset := state.records[recordIdx].offset + if offset >= 0x80000000 { + largeOffsetIdx, err := intconv.IntToUint32(len(largeOffsets)) + if err != nil { + return err + } + + word := 0x80000000 | largeOffsetIdx + + largeOffsets = append(largeOffsets, offset) + + binary.BigEndian.PutUint32(scratch[:4], word) + } else { + binary.BigEndian.PutUint32(scratch[:4], uint32(offset)) + } + + err := write(scratch[:4]) + if err != nil { + return err + } + + offsetDone++ + offsetMeter.Set(offsetDone, 0) + } + + if offsetDone > 0 { + offsetMeter.Stop("done") + } + + total, err := intconv.IntToUint64(largeOffsetCount) + if err != nil { + return err + } + + largeOffsetMeter := progress.New(progress.Options{ + Writer: state.opts.Progress, + Flush: state.opts.ProgressFlush, + Title: "writing index large offsets", + Total: total, + }) + + var largeOffsetDone uint64 + + for _, off := range largeOffsets { + binary.BigEndian.PutUint64(scratch[:8], off) + + err := write(scratch[:8]) + if err != nil { + return err + } + + largeOffsetDone++ + largeOffsetMeter.Set(largeOffsetDone, 0) + } + + if largeOffsetDone > 0 { + largeOffsetMeter.Stop("done") + } + + writeProgressf(state, "writing index trailer...\r") + + err = write(state.packHash.Bytes()) + if err != nil { + return err + } + + idxHash := hashImpl.Sum(nil) + + _, err = state.idxFile.Write(idxHash) + if err != nil { + return err + } + + err = state.idxFile.Sync() + if err != nil { + return err + } + + writeProgressf(state, "writing index trailer: done.\n") + + return nil +} + +// buildIdxOrder returns record indexes sorted by ObjectID. +func buildIdxOrder(state *ingestState) []int { + out := make([]int, 0, len(state.records)) + for idx := range state.records { + out = append(out, idx) + } + + slices.SortFunc(out, func(a, b int) int { + return bytes.Compare(state.records[a].objectID.Bytes(), state.records[b].objectID.Bytes()) + }) + + return out +} + +// verifyResolvedRecords checks that all records are fully resolved before index writing. +func verifyResolvedRecords(state *ingestState) error { + for idx, record := range state.records { + if !record.resolved { + return fmt.Errorf("packfile/ingest: unresolved record %d at offset %d", idx, record.offset) + } + } + + return nil +} + +// writeAndHash writes src to dst and updates hash. +func writeAndHash(dst io.Writer, hashImpl hash.Hash, src []byte) error { + _, err := dst.Write(src) + if err != nil { + return err + } + + _, err = hashImpl.Write(src) + if err != nil { + return err + } + + return nil +} diff --git a/format/packfile/ingest/ingest.go b/format/packfile/ingest/ingest.go new file mode 100644 index 00000000..be65ff5f --- /dev/null +++ b/format/packfile/ingest/ingest.go @@ -0,0 +1,68 @@ +package ingest + +import ( + "fmt" +) + +// ingest initializes transaction state and executes the ingest pipeline. +func ingest(state *ingestState) (out Result, err error) { + err = openTemporaryArtifacts(state) + if err != nil { + return Result{}, err + } + + defer func() { + _ = closeTemporaryArtifacts(state) + if err != nil { + rollbackTemporaryArtifacts(state) + } + }() + + err = streamPackAndScan(state) + if err != nil { + return Result{}, err + } + + err = resolveAll(state) + if err != nil { + return Result{}, err + } + + err = maybeFixThin(state) + if err != nil { + return Result{}, err + } + + if state.thinFixed { + err = resolveAll(state) + if err != nil { + return Result{}, err + } + } + + if len(state.unresolvedRefDeltas) > 0 { + return Result{}, &ThinPackUnresolvedError{Count: len(state.unresolvedRefDeltas)} + } + + err = verifyResolvedRecords(state) + if err != nil { + return Result{}, err + } + + err = state.packFile.Sync() + if err != nil { + return Result{}, &DestinationWriteError{Op: fmt.Sprintf("sync pack: %v", err)} + } + + err = writeIdx(state) + if err != nil { + return Result{}, err + } + + err = writeRev(state) + if err != nil { + return Result{}, err + } + + return finalizeArtifacts(state) +} diff --git a/format/packfile/ingest/ingest_test.go b/format/packfile/ingest/ingest_test.go new file mode 100644 index 00000000..fb50d241 --- /dev/null +++ b/format/packfile/ingest/ingest_test.go @@ -0,0 +1,434 @@ +package ingest_test + +import ( + "bytes" + "encoding/binary" + "errors" + "io" + "io/fs" + "os" + "path/filepath" + "strings" + "testing" + + "codeberg.org/lindenii/furgit/internal/testgit" + objectid "codeberg.org/lindenii/furgit/object/id" + "codeberg.org/lindenii/furgit/format/packfile/ingest" +) + +type noExtraReadReader struct { + reader *bytes.Reader +} + +func (r *noExtraReadReader) Read(p []byte) (int, error) { + if r.reader.Len() == 0 { + return 0, errors.New("unexpected extra read after pack trailer") + } + + return r.reader.Read(p) +} + +func beginAndContinue( + src io.Reader, + packRoot *os.Root, + algo objectid.Algorithm, + opts ingest.Options, +) (ingest.Result, error) { + pending, err := ingest.Ingest(src, algo, opts) + if err != nil { + return ingest.Result{}, err + } + + return pending.Continue(packRoot) +} + +// fixturePath returns one fixture file path for the selected algorithm. +func fixturePath(t *testing.T, algo objectid.Algorithm, name string) string { + t.Helper() + + dir := algo.String() + if dir == "" { + t.Fatalf("unsupported fixture algorithm: %v", algo) + } + + return filepath.Join("testdata", "fixtures", dir, name) +} + +// fixtureBytes reads one fixture file fully. +func fixtureBytes(t *testing.T, algo objectid.Algorithm, name string) []byte { + t.Helper() + + path := fixturePath(t, algo, name) + dir := filepath.Dir(path) + base := filepath.Base(path) + + root, err := os.OpenRoot(dir) + if err != nil { + t.Fatalf("open fixture root %q: %v", dir, err) + } + + defer func() { + err := root.Close() + if err != nil { + t.Fatalf("close fixture root %q: %v", dir, err) + } + }() + + data, err := root.ReadFile(base) + if err != nil { + t.Fatalf("read fixture %q: %v", base, err) + } + + return data +} + +// fixtureMetadata parses key=value metadata for one algorithm fixture set. +func fixtureMetadata(t *testing.T, algo objectid.Algorithm) map[string]string { + t.Helper() + + data := fixtureBytes(t, algo, "METADATA.txt") + + out := make(map[string]string) + for line := range strings.SplitSeq(strings.TrimSpace(string(data)), "\n") { + line = strings.TrimSpace(line) + if line == "" { + continue + } + + key, value, ok := strings.Cut(line, "=") + if !ok { + t.Fatalf("invalid fixture metadata line %q", line) + } + + out[strings.TrimSpace(key)] = strings.TrimSpace(value) + } + + return out +} + +// fixtureOID returns one fixture metadata object ID value. +func fixtureOID(t *testing.T, algo objectid.Algorithm, key string) objectid.ObjectID { + t.Helper() + + meta := fixtureMetadata(t, algo) + + hex, ok := meta[key] + if !ok { + t.Fatalf("missing fixture metadata key %q", key) + } + + id, err := objectid.ParseHex(algo, hex) + if err != nil { + t.Fatalf("parse fixture metadata oid %q: %v", hex, err) + } + + return id +} + +// verifyReindexOracle regenerates idx/rev with upstream git index-pack and +// compares bytes with files produced by ingest. +func verifyReindexOracle(t *testing.T, repo *testgit.TestRepo, packName, idxName, revName string) { + t.Helper() + + oracleDir := t.TempDir() + oracleIdxPath := filepath.Join(oracleDir, "oracle.idx") + _ = repo.Run(t, "index-pack", "--rev-index", "-o", oracleIdxPath, filepath.Join("objects", "pack", packName)) + oracleRevPath := strings.TrimSuffix(oracleIdxPath, ".idx") + ".rev" + + packRoot := repo.OpenPackRoot(t) + + gotIdx, err := packRoot.ReadFile(idxName) + if err != nil { + t.Fatalf("read idx: %v", err) + } + + oracleRoot, err := os.OpenRoot(oracleDir) + if err != nil { + t.Fatalf("open oracle root: %v", err) + } + + defer func() { + err := oracleRoot.Close() + if err != nil { + t.Fatalf("close oracle root: %v", err) + } + }() + + wantIdx, err := oracleRoot.ReadFile(filepath.Base(oracleIdxPath)) + if err != nil { + t.Fatalf("read oracle idx: %v", err) + } + + if !bytes.Equal(gotIdx, wantIdx) { + t.Fatal("idx bytes differ from git index-pack output") + } + + gotRev, err := packRoot.ReadFile(revName) + if err != nil { + t.Fatalf("read rev: %v", err) + } + + wantRev, err := oracleRoot.ReadFile(filepath.Base(oracleRevPath)) + if err != nil { + t.Fatalf("read oracle rev: %v", err) + } + + if !bytes.Equal(gotRev, wantRev) { + t.Fatal("rev bytes differ from git index-pack output") + } +} + +func TestIngestNonThinPackWritesPackIdxRev(t *testing.T) { + t.Parallel() + + testgit.ForEachAlgorithm(t, func(t *testing.T, algo objectid.Algorithm) { //nolint:thelper + head := fixtureOID(t, algo, "head") + packBytes := fixtureBytes(t, algo, "nonthin.pack") + + receiver := testgit.NewRepo(t, testgit.RepoOptions{ObjectFormat: algo, Bare: true}) + + packRoot := receiver.OpenPackRoot(t) + + result, err := beginAndContinue(bytes.NewReader(packBytes), packRoot, algo, ingest.Options{ + WriteRev: true, + RequireTrailingEOF: true, + }) + if err != nil { + t.Fatalf("Ingest: %v", err) + } + + if result.ThinFixed { + t.Fatalf("ThinFixed = true, want false") + } + + if result.RevName == "" { + t.Fatal("RevName is empty") + } + + _, err = packRoot.Stat(result.PackName) + if err != nil { + t.Fatalf("stat pack: %v", err) + } + + _, err = packRoot.Stat(result.IdxName) + if err != nil { + t.Fatalf("stat idx: %v", err) + } + + _, err = packRoot.Stat(result.RevName) + if err != nil { + t.Fatalf("stat rev: %v", err) + } + + _ = receiver.Run(t, "verify-pack", "-v", filepath.Join("objects", "pack", result.IdxName)) + verifyReindexOracle(t, receiver, result.PackName, result.IdxName, result.RevName) + + receiver.UpdateRef(t, "refs/heads/main", head) + _ = receiver.Run(t, "fsck", "--full", "--strict", "--no-progress", "--no-dangling") + }) +} + +func TestIngestThinPackWithoutFixReturnsUnresolved(t *testing.T) { + t.Parallel() + + testgit.ForEachAlgorithm(t, func(t *testing.T, algo objectid.Algorithm) { //nolint:thelper + thinPack := fixtureBytes(t, algo, "thin.pack") + + receiver := testgit.NewRepo(t, testgit.RepoOptions{ObjectFormat: algo, Bare: true}) + packRoot := receiver.OpenPackRoot(t) + + _, err := beginAndContinue(bytes.NewReader(thinPack), packRoot, algo, ingest.Options{ + WriteRev: true, + RequireTrailingEOF: true, + }) + if err == nil { + t.Fatal("Ingest error = nil, want error") + } + + if _, ok := errors.AsType[*ingest.ThinPackUnresolvedError](err); !ok { + t.Fatalf("Ingest error type = %T (%v), want *ThinPackUnresolvedError", err, err) + } + + entries, err := fs.ReadDir(packRoot.FS(), ".") + if err != nil { + t.Fatalf("ReadDir(pack): %v", err) + } + + for _, entry := range entries { + if strings.HasSuffix(entry.Name(), ".pack") { + t.Fatalf("found finalized pack file after failure: %v", entry.Name()) + } + } + }) +} + +func TestIngestThinPackWithFixThin(t *testing.T) { + t.Parallel() + + testgit.ForEachAlgorithm(t, func(t *testing.T, algo objectid.Algorithm) { //nolint:thelper + head := fixtureOID(t, algo, "head") + basePack := fixtureBytes(t, algo, "base.pack") + thinPack := fixtureBytes(t, algo, "thin.pack") + receiver := testgit.NewRepo(t, testgit.RepoOptions{ObjectFormat: algo, Bare: true}) + + packRoot := receiver.OpenPackRoot(t) + + _, err := beginAndContinue(bytes.NewReader(basePack), packRoot, algo, ingest.Options{ + RequireTrailingEOF: true, + }) + if err != nil { + t.Fatalf("ingest base pack: %v", err) + } + + receiverRepo := receiver.OpenRepository(t) + + result, err := beginAndContinue(bytes.NewReader(thinPack), packRoot, algo, ingest.Options{ + FixThin: true, + WriteRev: true, + Base: receiverRepo.Objects(), + RequireTrailingEOF: true, + }) + if err != nil { + t.Fatalf("Ingest(thin): %v", err) + } + + if !result.ThinFixed { + t.Fatal("ThinFixed = false, want true") + } + + _ = receiver.Run(t, "verify-pack", "-v", filepath.Join("objects", "pack", result.IdxName)) + verifyReindexOracle(t, receiver, result.PackName, result.IdxName, result.RevName) + receiver.UpdateRef(t, "refs/heads/main", head) + _ = receiver.Run(t, "fsck", "--full", "--strict", "--no-progress", "--no-dangling") + }) +} + +func TestIngestPackTrailerMismatch(t *testing.T) { + t.Parallel() + + testgit.ForEachAlgorithm(t, func(t *testing.T, algo objectid.Algorithm) { //nolint:thelper + packBytes := fixtureBytes(t, algo, "nonthin.pack") + if len(packBytes) == 0 { + t.Fatal("empty pack stream") + } + + packBytes[len(packBytes)-1] ^= 0xff + + receiver := testgit.NewRepo(t, testgit.RepoOptions{ObjectFormat: algo, Bare: true}) + packRoot := receiver.OpenPackRoot(t) + + _, err := beginAndContinue(bytes.NewReader(packBytes), packRoot, algo, ingest.Options{ + WriteRev: true, + RequireTrailingEOF: true, + }) + if err == nil { + t.Fatal("Ingest error = nil, want error") + } + + if _, ok := errors.AsType[*ingest.PackTrailerMismatchError](err); !ok { + t.Fatalf("Ingest error type = %T (%v), want *PackTrailerMismatchError", err, err) + } + + entries, err := fs.ReadDir(packRoot.FS(), ".") + if err != nil { + t.Fatalf("ReadDir(pack): %v", err) + } + + for _, entry := range entries { + if strings.HasSuffix(entry.Name(), ".pack") { + t.Fatalf("found finalized pack file after failure: %v", entry.Name()) + } + } + }) +} + +func zeroObjectPackBytes(t *testing.T, algo objectid.Algorithm) []byte { + t.Helper() + + hashImpl, err := algo.New() + if err != nil { + t.Fatalf("algo.New: %v", err) + } + + var header [12]byte + copy(header[:4], []byte{'P', 'A', 'C', 'K'}) + binary.BigEndian.PutUint32(header[4:8], 2) + binary.BigEndian.PutUint32(header[8:12], 0) + + _, _ = hashImpl.Write(header[:]) + + return append(header[:], hashImpl.Sum(nil)...) +} + +func TestIngestDiscardZeroObjectPack(t *testing.T) { + t.Parallel() + + testgit.ForEachAlgorithm(t, func(t *testing.T, algo objectid.Algorithm) { //nolint:thelper + packBytes := zeroObjectPackBytes(t, algo) + + pending, err := ingest.Ingest(bytes.NewReader(packBytes), algo, ingest.Options{ + RequireTrailingEOF: true, + }) + if err != nil { + t.Fatalf("Ingest: %v", err) + } + + if pending.Header().ObjectCount != 0 { + t.Fatalf("ObjectCount = %d, want 0", pending.Header().ObjectCount) + } + + discarded, err := pending.Discard() + if err != nil { + t.Fatalf("Discard: %v", err) + } + + if discarded.ObjectCount != 0 { + t.Fatalf("Discard.ObjectCount = %d, want 0", discarded.ObjectCount) + } + }) +} + +func TestIngestContinueRejectsZeroObjectPack(t *testing.T) { + t.Parallel() + + testgit.ForEachAlgorithm(t, func(t *testing.T, algo objectid.Algorithm) { //nolint:thelper + packBytes := zeroObjectPackBytes(t, algo) + receiver := testgit.NewRepo(t, testgit.RepoOptions{ObjectFormat: algo, Bare: true}) + packRoot := receiver.OpenPackRoot(t) + + pending, err := ingest.Ingest(bytes.NewReader(packBytes), algo, ingest.Options{ + RequireTrailingEOF: true, + }) + if err != nil { + t.Fatalf("Ingest: %v", err) + } + + _, err = pending.Continue(packRoot) + if !errors.Is(err, ingest.ErrZeroObjectContinue) { + t.Fatalf("Continue error = %v, want ErrZeroObjectContinue", err) + } + }) +} + +func TestIngestCanFinishWithoutTrailingEOF(t *testing.T) { + t.Parallel() + + testgit.ForEachAlgorithm(t, func(t *testing.T, algo objectid.Algorithm) { //nolint:thelper + head := fixtureOID(t, algo, "head") + packBytes := fixtureBytes(t, algo, "nonthin.pack") + + receiver := testgit.NewRepo(t, testgit.RepoOptions{ObjectFormat: algo, Bare: true}) + packRoot := receiver.OpenPackRoot(t) + + result, err := beginAndContinue(&noExtraReadReader{reader: bytes.NewReader(packBytes)}, packRoot, algo, ingest.Options{ + WriteRev: true, + }) + if err != nil { + t.Fatalf("Ingest without trailing EOF: %v", err) + } + + receiver.UpdateRef(t, "refs/heads/main", head) + _ = receiver.Run(t, "verify-pack", "-v", filepath.Join("objects", "pack", result.IdxName)) + _ = receiver.Run(t, "fsck", "--full", "--strict", "--no-progress", "--no-dangling") + }) +} diff --git a/format/packfile/ingest/progress_write.go b/format/packfile/ingest/progress_write.go new file mode 100644 index 00000000..5b9f184b --- /dev/null +++ b/format/packfile/ingest/progress_write.go @@ -0,0 +1,11 @@ +package ingest + +import "codeberg.org/lindenii/furgit/internal/utils" + +func writeProgressf(state *ingestState, format string, args ...any) { + utils.BestEffortFprintf(state.opts.Progress, format, args...) + + if state.opts.ProgressFlush != nil { + _ = state.opts.ProgressFlush() + } +} diff --git a/format/packfile/ingest/record_content.go b/format/packfile/ingest/record_content.go new file mode 100644 index 00000000..47f5321f --- /dev/null +++ b/format/packfile/ingest/record_content.go @@ -0,0 +1,30 @@ +package ingest + +import ( + "fmt" + + objecttype "codeberg.org/lindenii/furgit/object/type" + packfmt "codeberg.org/lindenii/furgit/format/packfile" +) + +// readBaseRecordContent reads canonical base content for one non-delta record. +func readBaseRecordContent(state *ingestState, idx int) (objecttype.Type, []byte, error) { + record := state.records[idx] + if !packfmt.IsBaseObjectType(record.packedType) { + return objecttype.TypeInvalid, nil, fmt.Errorf("packfile/ingest: record %d is not a base object", idx) + } + + content, err := inflateRecordPayload(state, idx) + if err != nil { + return objecttype.TypeInvalid, nil, err + } + + if int64(len(content)) != record.declaredSize { + return objecttype.TypeInvalid, nil, &MalformedPackEntryError{ + Offset: record.offset, + Reason: fmt.Sprintf("base content size mismatch got %d want %d", len(content), record.declaredSize), + } + } + + return record.packedType, content, nil +} diff --git a/format/packfile/ingest/record_delta.go b/format/packfile/ingest/record_delta.go new file mode 100644 index 00000000..31fb4b62 --- /dev/null +++ b/format/packfile/ingest/record_delta.go @@ -0,0 +1,60 @@ +package ingest + +import ( + "fmt" + + objecttype "codeberg.org/lindenii/furgit/object/type" + deltaapply "codeberg.org/lindenii/furgit/format/packfile/delta/apply" +) + +// applyDeltaRecord applies one delta record onto base content. +func applyDeltaRecord(state *ingestState, idx int, baseType objecttype.Type, baseContent []byte) (objecttype.Type, []byte, error) { + record := state.records[idx] + if record.packedType != objecttype.TypeOfsDelta && record.packedType != objecttype.TypeRefDelta { + return objecttype.TypeInvalid, nil, fmt.Errorf("packfile/ingest: record %d is not a delta record", idx) + } + + deltaPayload, err := inflateRecordPayload(state, idx) + if err != nil { + return objecttype.TypeInvalid, nil, err + } + + if int64(len(deltaPayload)) != record.declaredSize { + return objecttype.TypeInvalid, nil, &MalformedPackEntryError{ + Offset: record.offset, + Reason: fmt.Sprintf("delta payload size mismatch got %d want %d", len(deltaPayload), record.declaredSize), + } + } + + srcSize, dstSize, err := readDeltaHeaderSizes(deltaPayload) + if err != nil { + return objecttype.TypeInvalid, nil, &MalformedPackEntryError{ + Offset: record.offset, + Reason: fmt.Sprintf("read delta header: %v", err), + } + } + + if srcSize != len(baseContent) { + return objecttype.TypeInvalid, nil, &MalformedPackEntryError{ + Offset: record.offset, + Reason: fmt.Sprintf("delta source size mismatch got %d want %d", srcSize, len(baseContent)), + } + } + + content, err := deltaapply.Apply(baseContent, deltaPayload) + if err != nil { + return objecttype.TypeInvalid, nil, &MalformedPackEntryError{ + Offset: record.offset, + Reason: fmt.Sprintf("apply delta: %v", err), + } + } + + if len(content) != dstSize { + return objecttype.TypeInvalid, nil, &MalformedPackEntryError{ + Offset: record.offset, + Reason: fmt.Sprintf("delta result size mismatch got %d want %d", len(content), dstSize), + } + } + + return baseType, content, nil +} diff --git a/format/packfile/ingest/record_inflate.go b/format/packfile/ingest/record_inflate.go new file mode 100644 index 00000000..b8eca25b --- /dev/null +++ b/format/packfile/ingest/record_inflate.go @@ -0,0 +1,46 @@ +package ingest + +import ( + "compress/zlib" + "fmt" + "io" + + "codeberg.org/lindenii/furgit/internal/intconv" +) + +// inflateRecordPayload inflates one record's zlib payload from pack file. +func inflateRecordPayload(state *ingestState, idx int) ([]byte, error) { + record := state.records[idx] + if record.packedLen < uint64(record.headerLen) { + return nil, &MalformedPackEntryError{Offset: record.offset, Reason: "entry packed span underflow"} + } + + compressedOffset := record.offset + uint64(record.headerLen) + compressedLen := record.packedLen - uint64(record.headerLen) + + compressedOffsetInt64, err := intconv.Uint64ToInt64(compressedOffset) + if err != nil { + return nil, err + } + + compressedLenInt64, err := intconv.Uint64ToInt64(compressedLen) + if err != nil { + return nil, err + } + + section := io.NewSectionReader(state.packFile, compressedOffsetInt64, compressedLenInt64) + + reader, err := zlib.NewReader(section) + if err != nil { + return nil, &MalformedPackEntryError{Offset: record.offset, Reason: fmt.Sprintf("open payload zlib: %v", err)} + } + + defer func() { _ = reader.Close() }() + + out, err := io.ReadAll(reader) + if err != nil { + return nil, &MalformedPackEntryError{Offset: record.offset, Reason: fmt.Sprintf("inflate payload: %v", err)} + } + + return out, nil +} diff --git a/format/packfile/ingest/record_resolve.go b/format/packfile/ingest/record_resolve.go new file mode 100644 index 00000000..1ccc427b --- /dev/null +++ b/format/packfile/ingest/record_resolve.go @@ -0,0 +1,117 @@ +package ingest + +import ( + "fmt" + + objecttype "codeberg.org/lindenii/furgit/object/type" + packfmt "codeberg.org/lindenii/furgit/format/packfile" +) + +// resolveRecord resolves one record and returns canonical type/content. +func resolveRecord(state *ingestState, idx int, visiting map[int]struct{}) (objecttype.Type, []byte, error) { + if idx < 0 || idx >= len(state.records) { + return objecttype.TypeInvalid, nil, fmt.Errorf("packfile/ingest: record index out of bounds") + } + + if _, ok := visiting[idx]; ok { + return objecttype.TypeInvalid, nil, &DeltaCycleError{Offset: state.records[idx].offset} + } + + visiting[idx] = struct{}{} + defer delete(visiting, idx) + + record := &state.records[idx] + if ty, content, ok := state.baseCache.get(idx); ok { + return ty, content, nil + } + + if packfmt.IsBaseObjectType(record.packedType) { + ty, content, err := readBaseRecordContent(state, idx) + if err != nil { + return objecttype.TypeInvalid, nil, err + } + + if record.resolved { + state.baseCache.add(idx, record.realType, content) + + return record.realType, content, nil + } + + id, err := hashCanonicalObject(state.algo, ty, content) + if err != nil { + return objecttype.TypeInvalid, nil, err + } + + record.objectID = id + record.realType = ty + record.resolved = true + state.objectToRecord[id] = idx + state.baseCache.add(idx, ty, content) + + return ty, content, nil + } + + var ( + baseType objecttype.Type + baseContent []byte + err error + ) + switch record.packedType { + case objecttype.TypeOfsDelta: + baseIdx, ok := state.offsetToRecord[record.baseOffset] + if !ok { + return objecttype.TypeInvalid, nil, &MalformedPackEntryError{ + Offset: record.offset, + Reason: "missing ofs-delta base entry", + } + } + + baseType, baseContent, err = resolveRecord(state, baseIdx, visiting) + if err != nil { + return objecttype.TypeInvalid, nil, err + } + case objecttype.TypeRefDelta: + baseIdx, ok := state.objectToRecord[record.baseObject] + if ok { + baseType, baseContent, err = resolveRecord(state, baseIdx, visiting) + if err != nil { + return objecttype.TypeInvalid, nil, err + } + } else { + return objecttype.TypeInvalid, nil, errExternalThinBase + } + case objecttype.TypeInvalid, + objecttype.TypeCommit, + objecttype.TypeTree, + objecttype.TypeBlob, + objecttype.TypeTag, + objecttype.TypeFuture: + return objecttype.TypeInvalid, nil, &MalformedPackEntryError{ + Offset: record.offset, + Reason: "unsupported delta type", + } + default: + return objecttype.TypeInvalid, nil, &MalformedPackEntryError{ + Offset: record.offset, + Reason: "unsupported delta type", + } + } + + ty, content, err := applyDeltaRecord(state, idx, baseType, baseContent) + if err != nil { + return objecttype.TypeInvalid, nil, err + } + + id, err := hashCanonicalObject(state.algo, ty, content) + if err != nil { + return objecttype.TypeInvalid, nil, err + } + + record.objectID = id + record.realType = ty + record.resolved = true + state.objectToRecord[id] = idx + state.baseCache.add(idx, ty, content) + + return ty, content, nil +} diff --git a/format/packfile/ingest/records.go b/format/packfile/ingest/records.go new file mode 100644 index 00000000..75f157fa --- /dev/null +++ b/format/packfile/ingest/records.go @@ -0,0 +1,46 @@ +package ingest + +import ( + objectid "codeberg.org/lindenii/furgit/object/id" + objecttype "codeberg.org/lindenii/furgit/object/type" +) + +// objectRecord stores metadata for one packed object entry. +type objectRecord struct { + // offset is the entry start offset in the pack file. + offset uint64 + // headerLen is packed entry header length in bytes. + headerLen uint32 + // packedLen is total packed entry length in bytes. + packedLen uint64 + // crc32 is the CRC over the full packed entry. + crc32 uint32 + // packedType is the entry type tag from the pack stream. + packedType objecttype.Type + // realType is canonical object type after delta resolution. + realType objecttype.Type + // declaredSize is the declared output object size for this entry. + declaredSize int64 + // dataOffset is compressed payload start offset for this entry. + dataOffset uint64 + // baseOffset is OFS base offset when packedType is OFS delta. + baseOffset uint64 + // baseObject is REF base object ID when packedType is REF delta. + baseObject objectid.ObjectID + // objectID is final resolved object ID. + objectID objectid.ObjectID + // resolved reports whether objectID/realType are finalized. + resolved bool +} + +// ofsDeltaRef maps one OFS delta record to its base offset. +type ofsDeltaRef struct { + baseOffset uint64 + recordIdx int +} + +// refDeltaRef maps one REF delta record to its base object ID. +type refDeltaRef struct { + baseObject objectid.ObjectID + recordIdx int +} diff --git a/format/packfile/ingest/resolve_all.go b/format/packfile/ingest/resolve_all.go new file mode 100644 index 00000000..e0ad2281 --- /dev/null +++ b/format/packfile/ingest/resolve_all.go @@ -0,0 +1,71 @@ +package ingest + +import ( + "errors" + + "codeberg.org/lindenii/furgit/internal/progress" +) + +// resolveAll resolves all delta records and finalizes ObjectID/RealType for every record. +func resolveAll(state *ingestState) error { + state.unresolvedRefDeltas = state.unresolvedRefDeltas[:0] + + var pending uint32 + + for idx := range state.records { + if !state.records[idx].resolved { + pending++ + } + } + + if pending == 0 { + return nil + } + + var done uint32 + + meter := progress.New(progress.Options{ + Writer: state.opts.Progress, + Flush: state.opts.ProgressFlush, + Title: "resolving deltas", + Total: uint64(pending), + }) + + for idx := range state.records { + if state.records[idx].resolved { + continue + } + + done++ + meter.Set(uint64(done), 0) + + visiting := make(map[int]struct{}) + + ty, content, err := resolveRecord(state, idx, visiting) + if err != nil { + if errors.Is(err, errExternalThinBase) { + state.unresolvedRefDeltas = append(state.unresolvedRefDeltas, idx) + + continue + } + + return err + } + + id, err := hashCanonicalObject(state.algo, ty, content) + if err != nil { + return err + } + + record := &state.records[idx] + record.realType = ty + record.objectID = id + record.resolved = true + state.objectToRecord[id] = idx + state.baseCache.add(idx, ty, content) + } + + meter.Stop("done") + + return nil +} diff --git a/format/packfile/ingest/rev_write.go b/format/packfile/ingest/rev_write.go new file mode 100644 index 00000000..f8c30c1b --- /dev/null +++ b/format/packfile/ingest/rev_write.go @@ -0,0 +1,138 @@ +package ingest + +import ( + "encoding/binary" + "slices" + + "codeberg.org/lindenii/furgit/internal/intconv" + "codeberg.org/lindenii/furgit/internal/progress" +) + +const ( + revMagic = 0x52494458 + revVersion = 1 +) + +// writeRev writes rev index for resolved records. +func writeRev(state *ingestState) error { + if !state.opts.WriteRev { + return nil + } + + idxOrder := buildIdxOrder(state) + + recordToIdxPos := make([]int, len(state.records)) + for pos, recordIdx := range idxOrder { + recordToIdxPos[recordIdx] = pos + } + + packOrder := buildPackOrder(state) + + hashImpl, err := state.algo.New() + if err != nil { + return err + } + + var scratch [8]byte + + writeProgressf(state, "writing reverse index header...\r") + binary.BigEndian.PutUint32(scratch[:4], revMagic) + + err = writeAndHash(state.revFile, hashImpl, scratch[:4]) + if err != nil { + return err + } + + binary.BigEndian.PutUint32(scratch[:4], revVersion) + + err = writeAndHash(state.revFile, hashImpl, scratch[:4]) + if err != nil { + return err + } + + binary.BigEndian.PutUint32(scratch[:4], state.algo.PackHashID()) + + err = writeAndHash(state.revFile, hashImpl, scratch[:4]) + if err != nil { + return err + } + + writeProgressf(state, "writing reverse index header: done.\n") + + entriesMeter := progress.New(progress.Options{ + Writer: state.opts.Progress, + Flush: state.opts.ProgressFlush, + Title: "writing reverse index entries", + Total: uint64(len(packOrder)), + }) + + var entriesDone uint64 + + for _, recordIdx := range packOrder { + recordPos, err := intconv.IntToUint32(recordToIdxPos[recordIdx]) + if err != nil { + return err + } + + binary.BigEndian.PutUint32(scratch[:4], recordPos) + + err = writeAndHash(state.revFile, hashImpl, scratch[:4]) + if err != nil { + return err + } + + entriesDone++ + entriesMeter.Set(entriesDone, 0) + } + + if entriesDone > 0 { + entriesMeter.Stop("done") + } + + writeProgressf(state, "writing reverse index trailer...\r") + + err = writeAndHash(state.revFile, hashImpl, state.packHash.Bytes()) + if err != nil { + return err + } + + revHash := hashImpl.Sum(nil) + + _, err = state.revFile.Write(revHash) + if err != nil { + return err + } + + err = state.revFile.Sync() + if err != nil { + return err + } + + writeProgressf(state, "writing reverse index trailer: done.\n") + + return nil +} + +// buildPackOrder returns record indexes sorted by pack offset. +func buildPackOrder(state *ingestState) []int { + out := make([]int, 0, len(state.records)) + for idx := range state.records { + out = append(out, idx) + } + + slices.SortFunc(out, func(a, b int) int { + offA := state.records[a].offset + + offB := state.records[b].offset + switch { + case offA < offB: + return -1 + case offA > offB: + return 1 + default: + return 0 + } + }) + + return out +} diff --git a/format/packfile/ingest/rewrite_header_trailer.go b/format/packfile/ingest/rewrite_header_trailer.go new file mode 100644 index 00000000..f1f18a39 --- /dev/null +++ b/format/packfile/ingest/rewrite_header_trailer.go @@ -0,0 +1,89 @@ +package ingest + +import ( + "encoding/binary" + "io" + + "codeberg.org/lindenii/furgit/internal/intconv" + objectid "codeberg.org/lindenii/furgit/object/id" +) + +// rewritePackHeaderAndTrailer rewrites object count and trailer hash using ReadAt/WriteAt. +func rewritePackHeaderAndTrailer(state *ingestState) error { + var countRaw [4]byte + + recordCountUint32, err := intconv.IntToUint32(len(state.records)) + if err != nil { + return err + } + + binary.BigEndian.PutUint32(countRaw[:], recordCountUint32) + + _, err = state.packFile.WriteAt(countRaw[:], 8) + if err != nil { + return err + } + + info, err := state.packFile.Stat() + if err != nil { + return err + } + + endWithoutTrailer := info.Size() + + hashImpl, err := state.algo.New() + if err != nil { + return err + } + + var ( + buf [128 << 10]byte + pos int64 + ) + for pos < endWithoutTrailer { + want := int64(len(buf)) + + remaining := endWithoutTrailer - pos + if remaining < want { + want = remaining + } + + n, err := state.packFile.ReadAt(buf[:want], pos) + if err != nil && err != io.EOF { + return err + } + + if n == 0 { + return io.ErrUnexpectedEOF + } + + _, _ = hashImpl.Write(buf[:n]) + pos += int64(n) + } + + sum := hashImpl.Sum(nil) + + _, err = state.packFile.WriteAt(sum, endWithoutTrailer) + if err != nil { + return err + } + + packHash, err := objectid.FromBytes(state.algo, sum) + if err != nil { + return err + } + + state.packHash = packHash + state.objectCountHeader = recordCountUint32 + + sumLenInt64 := int64(len(sum)) + + newConsumed, err := intconv.Int64ToUint64(endWithoutTrailer + sumLenInt64) + if err != nil { + return err + } + + state.stream.consumed = newConsumed + + return nil +} diff --git a/format/packfile/ingest/scan.go b/format/packfile/ingest/scan.go new file mode 100644 index 00000000..de4e993c --- /dev/null +++ b/format/packfile/ingest/scan.go @@ -0,0 +1,106 @@ +package ingest + +import ( + "fmt" + + "codeberg.org/lindenii/furgit/internal/progress" + objectid "codeberg.org/lindenii/furgit/object/id" +) + +// streamPackAndScan copies src into temp .pack while scanning packed entries. +func streamPackAndScan(state *ingestState) error { + hashImpl, err := state.algo.New() + if err != nil { + return err + } + + state.stream = newStreamScanner( + state.src, + state.packFile, + hashImpl, + state.algo.Size(), + ) + + writeProgressf(state, "validating pack header...\r") + + err = seedStreamWithPackHeader(state) + if err != nil { + return err + } + + writeProgressf(state, "validating pack header: done.\n") + + state.records = make([]objectRecord, 0, state.objectCountHeader) + state.ofsDeltas = make([]ofsDeltaRef, 0, state.objectCountHeader) + state.refDeltas = make([]refDeltaRef, 0, state.objectCountHeader) + + total := state.objectCountHeader + meter := progress.New(progress.Options{ + Writer: state.opts.Progress, + Flush: state.opts.ProgressFlush, + Title: "receiving objects", + Total: uint64(total), + Throughput: true, + }) + + for i := range total { + nextOffset, err := scanOneEntry(state, state.stream.consumed) + if err != nil { + return err + } + + if nextOffset != state.stream.consumed { + return fmt.Errorf("packfile/ingest: internal stream offset mismatch") + } + + done := i + 1 + meter.Set(uint64(done), state.stream.consumed) + } + + meter.Stop("done") + + err = state.stream.finishAndFlushTrailer(state.opts.RequireTrailingEOF) + if err != nil { + return err + } + + if len(state.stream.packTrailer) != state.algo.Size() { + return fmt.Errorf("packfile/ingest: invalid trailer size") + } + + packHash, err := objectid.FromBytes(state.algo, state.stream.packTrailer) + if err != nil { + return err + } + + state.packHash = packHash + + return state.stream.flush() +} + +// seedStreamWithPackHeader writes the already-validated PACK header to output, +// seeds the running pack hash, and advances stream offset accounting. +func seedStreamWithPackHeader(state *ingestState) error { + written := 0 + for written < len(state.packHeaderRaw) { + n, err := state.packFile.Write(state.packHeaderRaw[written:]) + if err != nil { + return &DestinationWriteError{Op: fmt.Sprintf("write pack header: %v", err)} + } + + if n == 0 { + return &DestinationWriteError{Op: "write pack header: short write"} + } + + written += n + } + + _, err := state.stream.hash.Write(state.packHeaderRaw[:]) + if err != nil { + return err + } + + state.stream.consumed = packHeaderSize + + return nil +} diff --git a/format/packfile/ingest/state.go b/format/packfile/ingest/state.go new file mode 100644 index 00000000..797323b2 --- /dev/null +++ b/format/packfile/ingest/state.go @@ -0,0 +1,70 @@ +package ingest + +import ( + "io" + "os" + + objectid "codeberg.org/lindenii/furgit/object/id" +) + +const ( + defaultDeltaBaseCacheMaxBytes = 32 << 20 +) + +// ingestState holds mutable state for one Ingest call. +type ingestState struct { + src io.Reader + destination *os.Root + algo objectid.Algorithm + opts Options + + packHeaderRaw [packHeaderSize]byte + + packFile *os.File + packTmpName string + idxFile *os.File + idxTmpName string + revFile *os.File + revTmpName string + + stream *streamScanner + + records []objectRecord + ofsDeltas []ofsDeltaRef + refDeltas []refDeltaRef + unresolvedRefDeltas []int + offsetToRecord map[uint64]int + objectToRecord map[objectid.ObjectID]int + + baseCache *deltaBaseCache + packHash objectid.ObjectID + + objectCountHeader uint32 + thinFixed bool +} + +// newIngestState constructs one call-local ingest state. +func newIngestState( + src io.Reader, + destination *os.Root, + algo objectid.Algorithm, + opts Options, + header HeaderInfo, + headerRaw [packHeaderSize]byte, +) (*ingestState, error) { + if algo.Size() == 0 { + return nil, objectid.ErrInvalidAlgorithm + } + + return &ingestState{ + src: src, + destination: destination, + algo: algo, + opts: opts, + packHeaderRaw: headerRaw, + objectCountHeader: header.ObjectCount, + offsetToRecord: make(map[uint64]int), + objectToRecord: make(map[objectid.ObjectID]int), + baseCache: newDeltaBaseCache(defaultDeltaBaseCacheMaxBytes), + }, nil +} diff --git a/format/packfile/ingest/stream.go b/format/packfile/ingest/stream.go new file mode 100644 index 00000000..a403087a --- /dev/null +++ b/format/packfile/ingest/stream.go @@ -0,0 +1,111 @@ +package ingest + +import ( + "errors" + "hash" + "io" + "os" +) + +const streamScannerBufferSize = 64 << 10 + +// streamScanner incrementally reads/consumes one pack stream while mirroring +// consumed bytes into one destination pack file. +type streamScanner struct { + src io.Reader + dstFile *os.File + + // Input buffer window: buf[off:n] is unread. + buf []byte + off int + n int + + // Absolute consumed stream bytes. + consumed uint64 + + // Running pack hash over consumed bytes while hashEnabled is true. + hash hash.Hash + hashSize int + hashEnabled bool + + // Entry CRC state while one entry is being consumed. + entryCRC uint32 + inEntryCRC bool + + packTrailer []byte +} + +// newStreamScanner constructs one scanner with fixed input buffering. +func newStreamScanner(src io.Reader, dstFile *os.File, hash hash.Hash, hashSize int) *streamScanner { + return &streamScanner{ + src: src, + dstFile: dstFile, + buf: make([]byte, streamScannerBufferSize), + hash: hash, + hashSize: hashSize, + hashEnabled: true, + } +} + +// Read implements io.Reader. +func (scanner *streamScanner) Read(dst []byte) (int, error) { + if len(dst) == 0 { + return 0, nil + } + + if scanner.n-scanner.off == 0 { + err := scanner.fill(1) + if err != nil { + if errors.Is(err, io.EOF) { + return 0, io.EOF + } + + return 0, err + } + } + + unread := scanner.n - scanner.off + if unread == 0 { + return 0, io.EOF + } + + n := min(len(dst), unread) + + copy(dst, scanner.buf[scanner.off:scanner.off+n]) + + err := scanner.use(n) + if err != nil { + return 0, err + } + + return n, nil +} + +// ReadByte implements io.ByteReader without allocation. +func (scanner *streamScanner) ReadByte() (byte, error) { + if scanner.n-scanner.off == 0 { + err := scanner.fill(1) + if err != nil { + return 0, err + } + } + + b := scanner.buf[scanner.off] + + err := scanner.use(1) + if err != nil { + return 0, err + } + + return b, nil +} + +// readFull reads exactly len(dst) bytes through receiver. +func (scanner *streamScanner) readFull(dst []byte) error { + _, err := io.ReadFull(scanner, dst) + if err != nil { + return err + } + + return nil +} diff --git a/format/packfile/ingest/temp.go b/format/packfile/ingest/temp.go new file mode 100644 index 00000000..d0b7862c --- /dev/null +++ b/format/packfile/ingest/temp.go @@ -0,0 +1,103 @@ +package ingest + +import ( + "crypto/rand" + "errors" + "fmt" + "io/fs" + "os" +) + +// openTemporaryArtifacts creates/open temp pack/idx/(rev) files under destination. +func openTemporaryArtifacts(state *ingestState) error { + packName, packFile, err := createTempFile(state.destination, "tmp_pack_") + if err != nil { + return err + } + + idxName, idxFile, err := createTempFile(state.destination, "tmp_idx_") + if err != nil { + _ = packFile.Close() + _ = state.destination.Remove(packName) + + return err + } + + revName := "" + + var revFile *os.File + if state.opts.WriteRev { + revName, revFile, err = createTempFile(state.destination, "tmp_rev_") + if err != nil { + _ = idxFile.Close() + _ = state.destination.Remove(idxName) + _ = packFile.Close() + _ = state.destination.Remove(packName) + + return err + } + } + + state.packTmpName = packName + state.packFile = packFile + state.idxTmpName = idxName + state.idxFile = idxFile + state.revTmpName = revName + state.revFile = revFile + + return nil +} + +// closeTemporaryArtifacts closes all temporary artifact file descriptors. +func closeTemporaryArtifacts(state *ingestState) error { + var out error + + if state.packFile != nil { + err := state.packFile.Close() + if err != nil && out == nil { + out = err + } + + state.packFile = nil + } + + if state.idxFile != nil { + err := state.idxFile.Close() + if err != nil && out == nil { + out = err + } + + state.idxFile = nil + } + + if state.revFile != nil { + err := state.revFile.Close() + if err != nil && out == nil { + out = err + } + + state.revFile = nil + } + + return out +} + +// createTempFile creates one temporary file under root using prefix. +func createTempFile(root *os.Root, prefix string) (string, *os.File, error) { + for range 32 { + name := prefix + rand.Text() + + file, err := root.OpenFile(name, os.O_CREATE|os.O_EXCL|os.O_RDWR, 0o644) + if err == nil { + return name, file, nil + } + + if errors.Is(err, fs.ErrExist) { + continue + } + + return "", nil, fmt.Errorf("packfile/ingest: create temp file %q: %w", name, err) + } + + return "", nil, fmt.Errorf("packfile/ingest: unable to create temporary file for prefix %q", prefix) +} diff --git a/format/packfile/ingest/testdata/fixtures/sha1/METADATA.txt b/format/packfile/ingest/testdata/fixtures/sha1/METADATA.txt new file mode 100644 index 00000000..5fcbfe26 --- /dev/null +++ b/format/packfile/ingest/testdata/fixtures/sha1/METADATA.txt @@ -0,0 +1,3 @@ +format=sha1 +head=200c960359dad025b4170284c518919eb4a24305 +base=4bc507fc631ea78474d83c47548743c9f1dda0dc diff --git a/format/packfile/ingest/testdata/fixtures/sha1/base.pack b/format/packfile/ingest/testdata/fixtures/sha1/base.pack new file mode 100644 index 00000000..3d7a4903 Binary files /dev/null and b/format/packfile/ingest/testdata/fixtures/sha1/base.pack differ diff --git a/format/packfile/ingest/testdata/fixtures/sha1/nonthin.pack b/format/packfile/ingest/testdata/fixtures/sha1/nonthin.pack new file mode 100644 index 00000000..ea07c9a0 Binary files /dev/null and b/format/packfile/ingest/testdata/fixtures/sha1/nonthin.pack differ diff --git a/format/packfile/ingest/testdata/fixtures/sha1/thin.pack b/format/packfile/ingest/testdata/fixtures/sha1/thin.pack new file mode 100644 index 00000000..95084feb Binary files /dev/null and b/format/packfile/ingest/testdata/fixtures/sha1/thin.pack differ diff --git a/format/packfile/ingest/testdata/fixtures/sha256/METADATA.txt b/format/packfile/ingest/testdata/fixtures/sha256/METADATA.txt new file mode 100644 index 00000000..8a5ea0a2 --- /dev/null +++ b/format/packfile/ingest/testdata/fixtures/sha256/METADATA.txt @@ -0,0 +1,3 @@ +format=sha256 +head=35cc0f4cd1c73524187540494058d233a2ecbd071c85d496a2250d8e0c805ef8 +base=b4abe46895f0bb5aa22fd42d28d428413f265359734c288752e3c2d270eec276 diff --git a/format/packfile/ingest/testdata/fixtures/sha256/base.pack b/format/packfile/ingest/testdata/fixtures/sha256/base.pack new file mode 100644 index 00000000..52ceef74 Binary files /dev/null and b/format/packfile/ingest/testdata/fixtures/sha256/base.pack differ diff --git a/format/packfile/ingest/testdata/fixtures/sha256/nonthin.pack b/format/packfile/ingest/testdata/fixtures/sha256/nonthin.pack new file mode 100644 index 00000000..50db05d0 Binary files /dev/null and b/format/packfile/ingest/testdata/fixtures/sha256/nonthin.pack differ diff --git a/format/packfile/ingest/testdata/fixtures/sha256/thin.pack b/format/packfile/ingest/testdata/fixtures/sha256/thin.pack new file mode 100644 index 00000000..b331b915 Binary files /dev/null and b/format/packfile/ingest/testdata/fixtures/sha256/thin.pack differ diff --git a/format/packfile/ingest/thin_append.go b/format/packfile/ingest/thin_append.go new file mode 100644 index 00000000..779d477f --- /dev/null +++ b/format/packfile/ingest/thin_append.go @@ -0,0 +1,91 @@ +package ingest + +import ( + "compress/zlib" + "hash/crc32" + "io" + + "codeberg.org/lindenii/furgit/internal/intconv" + objectid "codeberg.org/lindenii/furgit/object/id" + objecttype "codeberg.org/lindenii/furgit/object/type" +) + +// appendBaseObject appends one base object as a new packed non-delta entry. +func appendBaseObject(state *ingestState, id objectid.ObjectID, realType objecttype.Type, content []byte) (int, error) { + start := state.stream.consumed + + header := encodePackEntryHeader(realType, int64(len(content))) + + startInt64, err := intconv.Uint64ToInt64(start) + if err != nil { + return 0, err + } + + _, err = state.packFile.WriteAt(header, startInt64) + if err != nil { + return 0, err + } + + headerLenInt64 := int64(len(header)) + section := &fileSectionWriter{file: state.packFile, off: startInt64 + headerLenInt64} + crc := crc32.NewIEEE() + + _, err = crc.Write(header) + if err != nil { + return 0, err + } + + counting := &countingWriter{dst: section} + + zw := zlib.NewWriter(io.MultiWriter(counting, crc)) + + _, err = zw.Write(content) + if err != nil { + return 0, err + } + + err = zw.Close() + if err != nil { + return 0, err + } + + headerLenUint64, err := intconv.IntToUint64(len(header)) + if err != nil { + return 0, err + } + + countingNUint64, err := intconv.IntToUint64(counting.n) + if err != nil { + return 0, err + } + + packedLen := headerLenUint64 + countingNUint64 + end := start + packedLen + state.stream.consumed = end + + headerLenUint32, err := intconv.IntToUint32(len(header)) + if err != nil { + return 0, err + } + + record := objectRecord{ + offset: start, + headerLen: headerLenUint32, + packedLen: packedLen, + crc32: crc.Sum32(), + packedType: realType, + realType: realType, + declaredSize: int64(len(content)), + dataOffset: start + headerLenUint64, + objectID: id, + resolved: true, + } + + recordIdx := len(state.records) + state.records = append(state.records, record) + state.offsetToRecord[start] = recordIdx + state.objectToRecord[id] = recordIdx + state.baseCache.add(recordIdx, realType, content) + + return recordIdx, nil +} diff --git a/format/packfile/ingest/thin_fix.go b/format/packfile/ingest/thin_fix.go new file mode 100644 index 00000000..83e5572a --- /dev/null +++ b/format/packfile/ingest/thin_fix.go @@ -0,0 +1,100 @@ +package ingest + +import ( + "errors" + "fmt" + + "codeberg.org/lindenii/furgit/internal/intconv" + "codeberg.org/lindenii/furgit/internal/progress" + objectstorer "codeberg.org/lindenii/furgit/object/storer" +) + +// maybeFixThin appends missing bases and rewrites pack header/trailer when needed. +func maybeFixThin(state *ingestState) error { + if len(state.unresolvedRefDeltas) == 0 { + return nil + } + + writeProgressf( + state, + "fixing thin pack: %d unresolved bases\r", + len(state.unresolvedRefDeltas), + ) + + if !state.opts.FixThin { + return &ThinPackUnresolvedError{Count: len(state.unresolvedRefDeltas)} + } + + if state.opts.Base == nil { + return &ThinPackUnresolvedError{Count: len(state.unresolvedRefDeltas)} + } + + hashSize := int64(state.algo.Size()) + + info, err := state.packFile.Stat() + if err != nil { + return err + } + + size := info.Size() + if size < hashSize { + return fmt.Errorf("packfile/ingest: pack too short to trim trailer") + } + + newEnd := size - hashSize + + err = state.packFile.Truncate(newEnd) + if err != nil { + return err + } + + consumed, err := intconv.Int64ToUint64(newEnd) + if err != nil { + return err + } + + state.stream.consumed = consumed + + baseIDs := unresolvedThinBaseIDs(state) + + total := len(baseIDs) + meter := progress.New(progress.Options{ + Writer: state.opts.Progress, + Flush: state.opts.ProgressFlush, + Title: "fixing thin pack", + Total: uint64(total), + }) + meter.Set(0, 0) + + var appended uint64 + + for _, id := range baseIDs { + ty, content, err := state.opts.Base.ReadBytesContent(id) + if err != nil { + if errors.Is(err, objectstorer.ErrObjectNotFound) { + continue + } + + return fmt.Errorf("packfile/ingest: read thin base %s: %w", id, err) + } + + _, err = appendBaseObject(state, id, ty, content) + if err != nil { + return err + } + + state.thinFixed = true + + appended++ + meter.Set(appended, 0) + } + + err = rewritePackHeaderAndTrailer(state) + if err != nil { + return err + } + + meter.Stop(fmt.Sprintf("appended %d/%d, done", appended, total)) + + return nil +} diff --git a/format/packfile/ingest/thin_unresolved.go b/format/packfile/ingest/thin_unresolved.go new file mode 100644 index 00000000..757cc0e2 --- /dev/null +++ b/format/packfile/ingest/thin_unresolved.go @@ -0,0 +1,34 @@ +package ingest + +import ( + "bytes" + "slices" + + objectid "codeberg.org/lindenii/furgit/object/id" + objecttype "codeberg.org/lindenii/furgit/object/type" +) + +// unresolvedThinBaseIDs returns sorted unique unresolved ref base IDs. +func unresolvedThinBaseIDs(state *ingestState) []objectid.ObjectID { + seen := make(map[objectid.ObjectID]struct{}) + + for _, idx := range state.unresolvedRefDeltas { + record := state.records[idx] + if record.packedType != objecttype.TypeRefDelta { + continue + } + + seen[record.baseObject] = struct{}{} + } + + out := make([]objectid.ObjectID, 0, len(seen)) + for id := range seen { + out = append(out, id) + } + + slices.SortFunc(out, func(a, b objectid.ObjectID) int { + return bytes.Compare(a.RawBytes(), b.RawBytes()) + }) + + return out +} diff --git a/format/packfile/ingest/trailer.go b/format/packfile/ingest/trailer.go new file mode 100644 index 00000000..7a26a8f2 --- /dev/null +++ b/format/packfile/ingest/trailer.go @@ -0,0 +1,58 @@ +package ingest + +import ( + "bytes" + "errors" + "fmt" + "io" +) + +// finishAndFlushTrailer reads trailer hash bytes, verifies trailer checksum, +// and optionally requires the source stream to hit EOF afterward. +func (scanner *streamScanner) finishAndFlushTrailer(requireTrailingEOF bool) error { + if scanner.hashSize <= 0 { + return fmt.Errorf("packfile/ingest: invalid hash size") + } + + trailer := make([]byte, scanner.hashSize) + + scanner.hashEnabled = false + + err := scanner.readFull(trailer) + if err != nil { + return &PackTrailerMismatchError{} + } + + scanner.packTrailer = append(scanner.packTrailer[:0], trailer...) + + if scanner.n-scanner.off > 0 { + return fmt.Errorf("packfile/ingest: pack has trailing garbage") + } + + if !requireTrailingEOF { + computed := scanner.hash.Sum(nil) + if !bytes.Equal(computed, trailer) { + return &PackTrailerMismatchError{} + } + + return nil + } + + var probe [1]byte + + n, err := scanner.Read(probe[:]) + if n > 0 || err == nil { + return fmt.Errorf("packfile/ingest: pack has trailing garbage") + } + + if !errors.Is(err, io.EOF) { + return err + } + + computed := scanner.hash.Sum(nil) + if !bytes.Equal(computed, trailer) { + return &PackTrailerMismatchError{} + } + + return nil +} diff --git a/format/packfile/ingest/use.go b/format/packfile/ingest/use.go new file mode 100644 index 00000000..97f8757a --- /dev/null +++ b/format/packfile/ingest/use.go @@ -0,0 +1,34 @@ +package ingest + +import ( + "fmt" + "hash/crc32" +) + +// use consumes n unread bytes and updates accounting/checksum state. +func (scanner *streamScanner) use(n int) error { + if n < 0 || n > scanner.n-scanner.off { + return fmt.Errorf("packfile/ingest: invalid consume length %d", n) + } + + if n == 0 { + return nil + } + + chunk := scanner.buf[scanner.off : scanner.off+n] + if scanner.hashEnabled { + _, err := scanner.hash.Write(chunk) + if err != nil { + return err + } + } + + if scanner.inEntryCRC { + scanner.entryCRC = crc32.Update(scanner.entryCRC, crc32.IEEETable, chunk) + } + + scanner.off += n + scanner.consumed += uint64(n) + + return nil +} diff --git a/format/packfile/object_type.go b/format/packfile/object_type.go new file mode 100644 index 00000000..8382baa9 --- /dev/null +++ b/format/packfile/object_type.go @@ -0,0 +1,16 @@ +package packfile + +import objecttype "codeberg.org/lindenii/furgit/object/type" + +// IsBaseObjectType reports whether ty is one of the four canonical object +// types encoded directly in pack entries. +func IsBaseObjectType(ty objecttype.Type) bool { + switch ty { + case objecttype.TypeCommit, objecttype.TypeTree, objecttype.TypeBlob, objecttype.TypeTag: + return true + case objecttype.TypeInvalid, objecttype.TypeFuture, objecttype.TypeOfsDelta, objecttype.TypeRefDelta: + return false + default: + return false + } +} diff --git a/format/packfile/ofs.go b/format/packfile/ofs.go new file mode 100644 index 00000000..4992a506 --- /dev/null +++ b/format/packfile/ofs.go @@ -0,0 +1,26 @@ +package packfile + +import "fmt" + +// ParseOfsDeltaDistance parses one ofs-delta backward distance. +func ParseOfsDeltaDistance(buf []byte) (uint64, int, error) { + if len(buf) == 0 { + return 0, 0, fmt.Errorf("packfile: malformed ofs-delta distance") + } + + b := buf[0] + dist := uint64(b & 0x7f) + + consumed := 1 + for b&0x80 != 0 { + if consumed >= len(buf) { + return 0, 0, fmt.Errorf("packfile: malformed ofs-delta distance") + } + + b = buf[consumed] + consumed++ + dist = ((dist + 1) << 7) + uint64(b&0x7f) + } + + return dist, consumed, nil +} diff --git a/internal/testgit/repo_open_commit_graph.go b/internal/testgit/repo_open_commit_graph.go index 6c0fee4a..4db7261b 100644 --- a/internal/testgit/repo_open_commit_graph.go +++ b/internal/testgit/repo_open_commit_graph.go @@ -3,7 +3,7 @@ package testgit import ( "testing" - commitgraphread "codeberg.org/lindenii/furgit/commitgraph/read" + commitgraphread "codeberg.org/lindenii/furgit/format/commitgraph/read" ) // OpenCommitGraph opens the repository commit-graph and registers cleanup on diff --git a/network/receivepack/service/ingest_quarantine.go b/network/receivepack/service/ingest_quarantine.go index 8e3e2455..6121ad6d 100644 --- a/network/receivepack/service/ingest_quarantine.go +++ b/network/receivepack/service/ingest_quarantine.go @@ -4,7 +4,7 @@ import ( "os" "codeberg.org/lindenii/furgit/internal/utils" - "codeberg.org/lindenii/furgit/packfile/ingest" + "codeberg.org/lindenii/furgit/format/packfile/ingest" ) func (service *Service) ingestQuarantine( diff --git a/network/receivepack/service/result.go b/network/receivepack/service/result.go index 17fc0b6b..c5ff3812 100644 --- a/network/receivepack/service/result.go +++ b/network/receivepack/service/result.go @@ -1,7 +1,7 @@ package service import ( - "codeberg.org/lindenii/furgit/packfile/ingest" + "codeberg.org/lindenii/furgit/format/packfile/ingest" ) // Result is one receive-pack execution result. diff --git a/object/storer/packed/delta_build_chain.go b/object/storer/packed/delta_build_chain.go index c348e4d5..553f96f6 100644 --- a/object/storer/packed/delta_build_chain.go +++ b/object/storer/packed/delta_build_chain.go @@ -4,7 +4,7 @@ import ( "fmt" objecttype "codeberg.org/lindenii/furgit/object/type" - packfmt "codeberg.org/lindenii/furgit/packfile" + packfmt "codeberg.org/lindenii/furgit/format/packfile" ) // deltaBuildChain walks one object's chain and builds a reconstruction chain. diff --git a/object/storer/packed/delta_resolve_chain.go b/object/storer/packed/delta_resolve_chain.go index 0c71b628..92df694e 100644 --- a/object/storer/packed/delta_resolve_chain.go +++ b/object/storer/packed/delta_resolve_chain.go @@ -4,7 +4,7 @@ import ( "fmt" objecttype "codeberg.org/lindenii/furgit/object/type" - deltaapply "codeberg.org/lindenii/furgit/packfile/delta/apply" + deltaapply "codeberg.org/lindenii/furgit/format/packfile/delta/apply" ) // deltaResolveChain resolves one object chain into content bytes. diff --git a/object/storer/packed/delta_resolve_chain_start.go b/object/storer/packed/delta_resolve_chain_start.go index 53050134..fa3bce98 100644 --- a/object/storer/packed/delta_resolve_chain_start.go +++ b/object/storer/packed/delta_resolve_chain_start.go @@ -4,7 +4,7 @@ import ( "fmt" objecttype "codeberg.org/lindenii/furgit/object/type" - packfmt "codeberg.org/lindenii/furgit/packfile" + packfmt "codeberg.org/lindenii/furgit/format/packfile" ) // deltaResolveChainStart finds the nearest cached chain node or inflates the diff --git a/object/storer/packed/delta_resolve_content.go b/object/storer/packed/delta_resolve_content.go index 06fc4226..9907f4c8 100644 --- a/object/storer/packed/delta_resolve_content.go +++ b/object/storer/packed/delta_resolve_content.go @@ -2,7 +2,7 @@ package packed import ( objecttype "codeberg.org/lindenii/furgit/object/type" - packfmt "codeberg.org/lindenii/furgit/packfile" + packfmt "codeberg.org/lindenii/furgit/format/packfile" ) // deltaResolveContent resolves one object's content bytes from its pack location. diff --git a/object/storer/packed/delta_size.go b/object/storer/packed/delta_size.go index 6896c939..e5ba3bb7 100644 --- a/object/storer/packed/delta_size.go +++ b/object/storer/packed/delta_size.go @@ -3,7 +3,7 @@ package packed import ( "bufio" - deltaapply "codeberg.org/lindenii/furgit/packfile/delta/apply" + deltaapply "codeberg.org/lindenii/furgit/format/packfile/delta/apply" ) // deltaDeclaredSizeAt returns the resolved object size declared by one delta diff --git a/object/storer/packed/entry_parse.go b/object/storer/packed/entry_parse.go index bbbbc469..95759d0e 100644 --- a/object/storer/packed/entry_parse.go +++ b/object/storer/packed/entry_parse.go @@ -6,7 +6,7 @@ import ( "codeberg.org/lindenii/furgit/internal/intconv" objectid "codeberg.org/lindenii/furgit/object/id" objecttype "codeberg.org/lindenii/furgit/object/type" - packfmt "codeberg.org/lindenii/furgit/packfile" + packfmt "codeberg.org/lindenii/furgit/format/packfile" ) // entryMeta describes one parsed pack entry header. diff --git a/object/storer/packed/pack.go b/object/storer/packed/pack.go index c8135d52..363fdc13 100644 --- a/object/storer/packed/pack.go +++ b/object/storer/packed/pack.go @@ -7,7 +7,7 @@ import ( "syscall" "codeberg.org/lindenii/furgit/internal/intconv" - packfmt "codeberg.org/lindenii/furgit/packfile" + packfmt "codeberg.org/lindenii/furgit/format/packfile" ) // packFile stores one mapped and validated .pack file. diff --git a/object/storer/packed/read_header_resolve.go b/object/storer/packed/read_header_resolve.go index 285387fa..614eacf8 100644 --- a/object/storer/packed/read_header_resolve.go +++ b/object/storer/packed/read_header_resolve.go @@ -4,7 +4,7 @@ import ( "fmt" objecttype "codeberg.org/lindenii/furgit/object/type" - packfmt "codeberg.org/lindenii/furgit/packfile" + packfmt "codeberg.org/lindenii/furgit/format/packfile" ) // resolveHeaderAt resolves one object's canonical type and declared content size. diff --git a/object/storer/packed/read_reader.go b/object/storer/packed/read_reader.go index 324ee033..01f38fcf 100644 --- a/object/storer/packed/read_reader.go +++ b/object/storer/packed/read_reader.go @@ -9,7 +9,7 @@ import ( objectheader "codeberg.org/lindenii/furgit/object/header" objectid "codeberg.org/lindenii/furgit/object/id" objecttype "codeberg.org/lindenii/furgit/object/type" - packfmt "codeberg.org/lindenii/furgit/packfile" + packfmt "codeberg.org/lindenii/furgit/format/packfile" ) // ReadReaderContent reads an object's type, declared content size, and content diff --git a/object/storer/packed/read_size.go b/object/storer/packed/read_size.go index 9d6c8e7d..18eec288 100644 --- a/object/storer/packed/read_size.go +++ b/object/storer/packed/read_size.go @@ -5,7 +5,7 @@ import ( objectid "codeberg.org/lindenii/furgit/object/id" objecttype "codeberg.org/lindenii/furgit/object/type" - packfmt "codeberg.org/lindenii/furgit/packfile" + packfmt "codeberg.org/lindenii/furgit/format/packfile" ) // ReadSize reads an object's declared content size. diff --git a/packfile/delta/apply/apply.go b/packfile/delta/apply/apply.go deleted file mode 100644 index f5006e3c..00000000 --- a/packfile/delta/apply/apply.go +++ /dev/null @@ -1,160 +0,0 @@ -// Package apply applies Git delta instruction streams. -package apply - -import "fmt" - -// Apply applies one Git delta instruction stream to base. -func Apply(base, delta []byte) ([]byte, error) { - pos := 0 - - srcSize, err := readVarint(delta, &pos) - if err != nil { - return nil, err - } - - dstSize, err := readVarint(delta, &pos) - if err != nil { - return nil, err - } - - if srcSize != len(base) { - return nil, fmt.Errorf("delta/apply: delta source size mismatch: got %d want %d", srcSize, len(base)) - } - - out := make([]byte, dstSize) - outPos := 0 - - for pos < len(delta) { - op := delta[pos] - pos++ - - //nolint:nestif - if op&0x80 != 0 { - off := 0 - - if op&0x01 != 0 { - if pos >= len(delta) { - return nil, fmt.Errorf("delta/apply: malformed delta copy offset") - } - - off |= int(delta[pos]) - pos++ - } - - if op&0x02 != 0 { - if pos >= len(delta) { - return nil, fmt.Errorf("delta/apply: malformed delta copy offset") - } - - off |= int(delta[pos]) << 8 - pos++ - } - - if op&0x04 != 0 { - if pos >= len(delta) { - return nil, fmt.Errorf("delta/apply: malformed delta copy offset") - } - - off |= int(delta[pos]) << 16 - pos++ - } - - if op&0x08 != 0 { - if pos >= len(delta) { - return nil, fmt.Errorf("delta/apply: malformed delta copy offset") - } - - off |= int(delta[pos]) << 24 - pos++ - } - - n := 0 - - if op&0x10 != 0 { - if pos >= len(delta) { - return nil, fmt.Errorf("delta/apply: malformed delta copy size") - } - - n |= int(delta[pos]) - pos++ - } - - if op&0x20 != 0 { - if pos >= len(delta) { - return nil, fmt.Errorf("delta/apply: malformed delta copy size") - } - - n |= int(delta[pos]) << 8 - pos++ - } - - if op&0x40 != 0 { - if pos >= len(delta) { - return nil, fmt.Errorf("delta/apply: malformed delta copy size") - } - - n |= int(delta[pos]) << 16 - pos++ - } - - if n == 0 { - n = 0x10000 - } - - if off < 0 || n < 0 || off+n > len(base) || outPos+n > len(out) { - return nil, fmt.Errorf("delta/apply: delta copy out of bounds") - } - - copy(out[outPos:outPos+n], base[off:off+n]) - outPos += n - - continue - } - - if op == 0 { - return nil, fmt.Errorf("delta/apply: invalid delta opcode 0") - } - - n := int(op) - if pos+n > len(delta) || outPos+n > len(out) { - return nil, fmt.Errorf("delta/apply: delta insert out of bounds") - } - - copy(out[outPos:outPos+n], delta[pos:pos+n]) - outPos += n - pos += n - } - - if outPos != len(out) { - return nil, fmt.Errorf("delta/apply: delta output size mismatch: got %d want %d", outPos, len(out)) - } - - return out, nil -} - -// readVarint parses one Git delta varint and advances pos. -func readVarint(buf []byte, pos *int) (int, error) { - value := 0 - shift := uint(0) - - for { - if *pos >= len(buf) { - return 0, fmt.Errorf("delta/apply: malformed delta varint") - } - - b := buf[*pos] - *pos++ - - value |= int(b&0x7f) << shift - if b&0x80 == 0 { - break - } - - shift += 7 - if shift > 63 { - return 0, fmt.Errorf("delta/apply: delta varint overflow") - } - } - - return value, nil -} diff --git a/packfile/delta/apply/header.go b/packfile/delta/apply/header.go deleted file mode 100644 index 69c9659a..00000000 --- a/packfile/delta/apply/header.go +++ /dev/null @@ -1,47 +0,0 @@ -package apply - -import ( - "fmt" - "io" -) - -// ReadHeaderSizes reads the first two varints in one inflated delta stream. -// -// Callers that continue reading the same stream should pass their own buffered -// byte reader and keep using that same reader afterwards. -func ReadHeaderSizes(reader io.ByteReader) (int, int, error) { - srcSize, err := readVarintFromByteReader(reader) - if err != nil { - return 0, 0, err - } - - dstSize, err := readVarintFromByteReader(reader) - if err != nil { - return 0, 0, err - } - - return srcSize, dstSize, nil -} - -// readVarintFromByteReader parses one Git delta varint from reader. -func readVarintFromByteReader(reader io.ByteReader) (int, error) { - value := 0 - shift := uint(0) - - for { - b, err := reader.ReadByte() - if err != nil { - return 0, fmt.Errorf("delta/apply: malformed delta varint: %w", err) - } - - value |= int(b&0x7f) << shift - if b&0x80 == 0 { - return value, nil - } - - shift += 7 - if shift > 63 { - return 0, fmt.Errorf("delta/apply: delta varint overflow") - } - } -} diff --git a/packfile/delta/doc.go b/packfile/delta/doc.go deleted file mode 100644 index f63c96a8..00000000 --- a/packfile/delta/doc.go +++ /dev/null @@ -1,2 +0,0 @@ -// Package delta provides various routines to handle Git delta compression. -package delta diff --git a/packfile/doc.go b/packfile/doc.go deleted file mode 100644 index d656e256..00000000 --- a/packfile/doc.go +++ /dev/null @@ -1,2 +0,0 @@ -// Package packfile provides Git packfile format parsing primitives. -package packfile diff --git a/packfile/entry.go b/packfile/entry.go deleted file mode 100644 index 0f9c7c8d..00000000 --- a/packfile/entry.go +++ /dev/null @@ -1,76 +0,0 @@ -package packfile - -import ( - "fmt" - - objecttype "codeberg.org/lindenii/furgit/object/type" -) - -// Entry is one parsed pack entry prefix, including any delta base reference -// data that appears before the compressed payload. -type Entry struct { - // Type is the pack entry type. - Type objecttype.Type - // Size is the declared resulting object size. - Size int64 - // DataOffset is the byte offset from the start of the entry to the zlib - // payload bytes. - DataOffset int - // RefBaseID is the referenced base object ID bytes for ref-delta entries. - RefBaseID []byte - // OfsBaseDistance is the backward distance for ofs-delta entries. - OfsBaseDistance uint64 -} - -// ParseEntry parses one full pack entry prefix from data. -// -// hashSize must match the hash algorithm size used by the pack/index. -func ParseEntry(data []byte, hashSize int) (Entry, error) { - var zero Entry - - header, err := ParseEntryHeader(data) - if err != nil { - return zero, err - } - - entry := Entry{ - Type: header.Type, - Size: header.Size, - DataOffset: header.HeaderSize, - } - - switch entry.Type { - case objecttype.TypeCommit, objecttype.TypeTree, objecttype.TypeBlob, objecttype.TypeTag: - // Base object entries have no extra prefix fields. - case objecttype.TypeRefDelta: - if hashSize <= 0 { - return zero, fmt.Errorf("packfile: invalid hash size %d", hashSize) - } - - end := entry.DataOffset + hashSize - if end > len(data) { - return zero, fmt.Errorf("packfile: truncated ref-delta base id") - } - - entry.RefBaseID = data[entry.DataOffset:end] - entry.DataOffset = end - case objecttype.TypeOfsDelta: - dist, consumed, err := ParseOfsDeltaDistance(data[entry.DataOffset:]) - if err != nil { - return zero, err - } - - entry.OfsBaseDistance = dist - entry.DataOffset += consumed - case objecttype.TypeInvalid, objecttype.TypeFuture: - return zero, fmt.Errorf("packfile: unsupported object type %d", entry.Type) - default: - return zero, fmt.Errorf("packfile: unsupported object type %d", entry.Type) - } - - if entry.DataOffset > len(data) { - return zero, fmt.Errorf("packfile: entry data offset out of bounds") - } - - return entry, nil -} diff --git a/packfile/entry_header.go b/packfile/entry_header.go deleted file mode 100644 index 05664268..00000000 --- a/packfile/entry_header.go +++ /dev/null @@ -1,52 +0,0 @@ -package packfile - -import ( - "fmt" - - objecttype "codeberg.org/lindenii/furgit/object/type" -) - -// EntryHeader is one parsed pack entry header prefix. -type EntryHeader struct { - // Type is the entry type tag from the first header byte. - Type objecttype.Type - // Size is the declared resulting object size. - Size int64 - // HeaderSize is the number of bytes consumed by the type/size header. - HeaderSize int -} - -// ParseEntryHeader parses one pack entry type/size header from data. -func ParseEntryHeader(data []byte) (EntryHeader, error) { - var zero EntryHeader - if len(data) == 0 { - return zero, fmt.Errorf("packfile: truncated entry header") - } - - first := data[0] - header := EntryHeader{ - Type: objecttype.Type((first >> 4) & 0x07), - Size: int64(first & 0x0f), - HeaderSize: 1, - } - - shift := uint(4) - - b := first - for b&0x80 != 0 { - if header.HeaderSize >= len(data) { - return zero, fmt.Errorf("packfile: truncated entry header") - } - - b = data[header.HeaderSize] - header.HeaderSize++ - header.Size |= int64(b&0x7f) << shift - shift += 7 - } - - if header.Size < 0 { - return zero, fmt.Errorf("packfile: negative entry size") - } - - return header, nil -} diff --git a/packfile/header.go b/packfile/header.go deleted file mode 100644 index bc859a55..00000000 --- a/packfile/header.go +++ /dev/null @@ -1,9 +0,0 @@ -package packfile - -// Signature is the 4-byte "PACK" magic at the start of pack files. -const Signature = 0x5041434b - -// VersionSupported reports whether one pack version is supported. -func VersionSupported(version uint32) bool { - return version == 2 || version == 3 -} diff --git a/packfile/ingest/api.go b/packfile/ingest/api.go deleted file mode 100644 index ce366a4f..00000000 --- a/packfile/ingest/api.go +++ /dev/null @@ -1,195 +0,0 @@ -package ingest - -import ( - "bufio" - "bytes" - "errors" - "io" - "os" - - objectid "codeberg.org/lindenii/furgit/object/id" - objectstorer "codeberg.org/lindenii/furgit/object/storer" -) - -// Options controls one pack ingest operation. -type Options struct { - // FixThin appends missing local bases for thin packs. - FixThin bool - // WriteRev writes a .rev alongside the .pack and .idx. - WriteRev bool - // Base supplies existing objects for thin-pack fixup. - Base objectstorer.Store - // Progress receives human-readable progress messages. - // - // When nil, no progress output is emitted. - Progress io.Writer - // ProgressFlush flushes transport output after progress writes. - // - // When nil, no explicit flush is attempted. - ProgressFlush func() error - // RequireTrailingEOF requires the source to hit EOF after the pack trailer. - // - // This is suitable for exact pack-file readers, but should be disabled for - // full-duplex transport streams like receive-pack where the peer keeps the - // connection open to read the server response. - RequireTrailingEOF bool -} - -// Result describes one successful ingest transaction. -type Result struct { - // PackName is the destination-relative filename of the written .pack. - PackName string - // IdxName is the destination-relative filename of the written .idx. - IdxName string - // RevName is the destination-relative filename of the written .rev. - // - // RevName is empty when writeRev is false. - RevName string - // PackHash is the final pack hash (same hash embedded in .idx/.rev trailers). - PackHash objectid.ObjectID - // ObjectCount is the final object count in the resulting pack. - // - // If thin fixup appends objects, this includes appended base objects. - ObjectCount uint32 - // ThinFixed reports whether thin fixup appended local bases. - ThinFixed bool -} - -// HeaderInfo describes the parsed PACK header. -type HeaderInfo struct { - Version uint32 - ObjectCount uint32 -} - -// DiscardResult describes one successful Discard call. -type DiscardResult struct { - PackHash objectid.ObjectID - ObjectCount uint32 -} - -// Pending is one started ingest operation awaiting Continue or Discard. -// -// Exactly one of Continue or Discard may be called. -type Pending struct { - reader *bufio.Reader - algo objectid.Algorithm - opts Options - header HeaderInfo - headerRaw [packHeaderSize]byte - - finalized bool -} - -// Ingest reads and validates one PACK header, returning one pending operation. -func Ingest( - src io.Reader, - algo objectid.Algorithm, - opts Options, -) (*Pending, error) { - if algo.Size() == 0 { - return nil, objectid.ErrInvalidAlgorithm - } - - reader := bufio.NewReader(src) - - header, headerRaw, err := readAndValidatePackHeader(reader) - if err != nil { - return nil, err - } - - return &Pending{ - reader: reader, - algo: algo, - opts: opts, - header: header, - headerRaw: headerRaw, - }, nil -} - -// Header returns parsed PACK header info. -func (pending *Pending) Header() HeaderInfo { - return pending.header -} - -// Continue ingests the pack stream into destination and writes pack artifacts. -// -// Continue is terminal. Further use of pending is undefined behavior. -// -// Artifacts are published under content-addressed final names derived from the -// resulting pack hash. If those final names already exist, Continue treats that -// as success and removes its temporary files. -func (pending *Pending) Continue(destination *os.Root) (Result, error) { - pending.finalized = true - - if pending.header.ObjectCount == 0 { - return Result{}, ErrZeroObjectContinue - } - - state, err := newIngestState( - pending.reader, - destination, - pending.algo, - pending.opts, - pending.header, - pending.headerRaw, - ) - if err != nil { - return Result{}, err - } - - return ingest(state) -} - -// Discard consumes and verifies one zero-object pack stream without writing -// files. -// -// Discard is terminal. Further use of pending is undefined behavior. -func (pending *Pending) Discard() (DiscardResult, error) { - pending.finalized = true - - if pending.header.ObjectCount != 0 { - return DiscardResult{}, ErrNonZeroDiscard - } - - hashImpl, err := pending.algo.New() - if err != nil { - return DiscardResult{}, err - } - - _, _ = hashImpl.Write(pending.headerRaw[:]) - - trailer := make([]byte, pending.algo.Size()) - - _, err = io.ReadFull(pending.reader, trailer) - if err != nil { - return DiscardResult{}, &PackTrailerMismatchError{} - } - - computed := hashImpl.Sum(nil) - if !bytes.Equal(computed, trailer) { - return DiscardResult{}, &PackTrailerMismatchError{} - } - - if pending.opts.RequireTrailingEOF { - var probe [1]byte - - n, err := pending.reader.Read(probe[:]) - if n > 0 || err == nil { - return DiscardResult{}, errors.New("packfile/ingest: pack has trailing garbage") - } - - if err != io.EOF { - return DiscardResult{}, err - } - } - - packHash, err := objectid.FromBytes(pending.algo, trailer) - if err != nil { - return DiscardResult{}, err - } - - return DiscardResult{ - PackHash: packHash, - ObjectCount: 0, - }, nil -} diff --git a/packfile/ingest/byteslice_reader.go b/packfile/ingest/byteslice_reader.go deleted file mode 100644 index a1570ef3..00000000 --- a/packfile/ingest/byteslice_reader.go +++ /dev/null @@ -1,21 +0,0 @@ -package ingest - -import "io" - -// byteSliceReader implements io.ByteReader on []byte. -type byteSliceReader struct { - data []byte - pos int -} - -// ReadByte reads one byte from receiver. -func (reader *byteSliceReader) ReadByte() (byte, error) { - if reader.pos >= len(reader.data) { - return 0, io.EOF - } - - b := reader.data[reader.pos] - reader.pos++ - - return b, nil -} diff --git a/packfile/ingest/cache.go b/packfile/ingest/cache.go deleted file mode 100644 index 9a15f55f..00000000 --- a/packfile/ingest/cache.go +++ /dev/null @@ -1,53 +0,0 @@ -package ingest - -import ( - "codeberg.org/lindenii/furgit/internal/lru" - objecttype "codeberg.org/lindenii/furgit/object/type" -) - -// deltaBaseCacheKey identifies one resolved base by record index. -type deltaBaseCacheKey struct { - recordIdx int -} - -// deltaBaseCacheValue stores one resolved base object payload. -type deltaBaseCacheValue struct { - realType objecttype.Type - content []byte -} - -// deltaBaseCache is a bounded LRU for resolved base payloads. -type deltaBaseCache struct { - lru *lru.Cache[deltaBaseCacheKey, deltaBaseCacheValue] -} - -// newDeltaBaseCache creates one bounded base cache. -func newDeltaBaseCache(maxBytes int64) *deltaBaseCache { - return &deltaBaseCache{ - lru: lru.New( - maxBytes, - func(_ deltaBaseCacheKey, value deltaBaseCacheValue) int64 { - return int64(len(value.content)) - }, - nil, - ), - } -} - -// get returns one cache entry for recordIdx. -func (cache *deltaBaseCache) get(recordIdx int) (objecttype.Type, []byte, bool) { - value, ok := cache.lru.Get(deltaBaseCacheKey{recordIdx: recordIdx}) - if !ok { - return objecttype.TypeInvalid, nil, false - } - - return value.realType, value.content, true -} - -// add stores one cache entry for recordIdx. -func (cache *deltaBaseCache) add(recordIdx int, realType objecttype.Type, content []byte) { - cache.lru.Add(deltaBaseCacheKey{recordIdx: recordIdx}, deltaBaseCacheValue{ - realType: realType, - content: content, - }) -} diff --git a/packfile/ingest/counting_writer.go b/packfile/ingest/counting_writer.go deleted file mode 100644 index 051ad9d1..00000000 --- a/packfile/ingest/counting_writer.go +++ /dev/null @@ -1,17 +0,0 @@ -package ingest - -import "io" - -// countingWriter counts bytes written to dst. -type countingWriter struct { - dst io.Writer - n int -} - -// Write writes src to dst and tracks output byte count. -func (writer *countingWriter) Write(src []byte) (int, error) { - n, err := writer.dst.Write(src) - writer.n += n - - return n, err -} diff --git a/packfile/ingest/crc.go b/packfile/ingest/crc.go deleted file mode 100644 index f55af4ff..00000000 --- a/packfile/ingest/crc.go +++ /dev/null @@ -1,22 +0,0 @@ -package ingest - -import "fmt" - -// beginEntryCRC starts inline CRC accumulation for one packed entry. -func (scanner *streamScanner) beginEntryCRC() { - scanner.entryCRC = 0 - scanner.inEntryCRC = true -} - -// endEntryCRC finishes inline CRC accumulation for one packed entry. -func (scanner *streamScanner) endEntryCRC() (uint32, error) { - if !scanner.inEntryCRC { - return 0, fmt.Errorf("packfile/ingest: entry CRC not started") - } - - crc := scanner.entryCRC - scanner.entryCRC = 0 - scanner.inEntryCRC = false - - return crc, nil -} diff --git a/packfile/ingest/delta_header.go b/packfile/ingest/delta_header.go deleted file mode 100644 index 63fda066..00000000 --- a/packfile/ingest/delta_header.go +++ /dev/null @@ -1,11 +0,0 @@ -package ingest - -import deltaapply "codeberg.org/lindenii/furgit/packfile/delta/apply" - -// finalizeStreamPackHash consumes trailer bytes and verifies stream integrity. -// readDeltaHeaderSizes reads source and destination sizes from one delta payload. -func readDeltaHeaderSizes(payload []byte) (int, int, error) { - reader := &byteSliceReader{data: payload} - - return deltaapply.ReadHeaderSizes(reader) -} diff --git a/packfile/ingest/distance.go b/packfile/ingest/distance.go deleted file mode 100644 index 9bc4d886..00000000 --- a/packfile/ingest/distance.go +++ /dev/null @@ -1,30 +0,0 @@ -package ingest - -import ( - "fmt" - "io" -) - -// readOfsDistanceFromStream reads one ofs-delta encoded distance. -func readOfsDistanceFromStream(reader io.ByteReader) (uint64, int, error) { - first, err := reader.ReadByte() - if err != nil { - return 0, 0, fmt.Errorf("read ofs distance first byte: %w", err) - } - - dist := uint64(first & 0x7f) - consumed := 1 - - b := first - for b&0x80 != 0 { - b, err = reader.ReadByte() - if err != nil { - return 0, 0, fmt.Errorf("read ofs distance continuation: %w", err) - } - - consumed++ - dist = ((dist + 1) << 7) + uint64(b&0x7f) - } - - return dist, consumed, nil -} diff --git a/packfile/ingest/doc.go b/packfile/ingest/doc.go deleted file mode 100644 index 2095068a..00000000 --- a/packfile/ingest/doc.go +++ /dev/null @@ -1,3 +0,0 @@ -// Package ingest implements streaming ingestion of one Git pack stream into a -// destination root, producing .pack/.idx and optionally .rev. -package ingest diff --git a/packfile/ingest/drain.go b/packfile/ingest/drain.go deleted file mode 100644 index 48fb91d9..00000000 --- a/packfile/ingest/drain.go +++ /dev/null @@ -1,68 +0,0 @@ -package ingest - -import ( - "fmt" - "io" - - "codeberg.org/lindenii/furgit/internal/compress/zlib" - objectheader "codeberg.org/lindenii/furgit/object/header" - objectid "codeberg.org/lindenii/furgit/object/id" - objecttype "codeberg.org/lindenii/furgit/object/type" - packfmt "codeberg.org/lindenii/furgit/packfile" -) - -// drainEntryPayload inflates one entry payload from stream and returns -// (inflatedLength, oidForBaseEntry). -func drainEntryPayload(state *ingestState, record objectRecord) (int64, objectid.ObjectID, error) { - var zero objectid.ObjectID - - reader, err := zlib.NewReader(state.stream) - if err != nil { - return 0, zero, &MalformedPackEntryError{Offset: record.offset, Reason: fmt.Sprintf("open zlib stream: %v", err)} - } - - defer func() { _ = reader.Close() }() - - var total int64 - - if packfmt.IsBaseObjectType(record.packedType) { - header, ok := objectheader.Encode(record.packedType, record.declaredSize) - if !ok { - return 0, zero, &MalformedPackEntryError{Offset: record.offset, Reason: "encode object header"} - } - - hashImpl, err := state.algo.New() - if err != nil { - return 0, zero, err - } - - _, _ = hashImpl.Write(header) - - n, err := io.Copy(hashImpl, reader) - if err != nil { - return 0, zero, &MalformedPackEntryError{Offset: record.offset, Reason: fmt.Sprintf("inflate base object: %v", err)} - } - - total = n - - oid, err := objectid.FromBytes(state.algo, hashImpl.Sum(nil)) - if err != nil { - return 0, zero, err - } - - return total, oid, nil - } - - if record.packedType == objecttype.TypeOfsDelta || record.packedType == objecttype.TypeRefDelta { - n, err := io.Copy(io.Discard, reader) - if err != nil { - return 0, zero, &MalformedPackEntryError{Offset: record.offset, Reason: fmt.Sprintf("inflate delta payload: %v", err)} - } - - total = n - - return total, zero, nil - } - - return 0, zero, &MalformedPackEntryError{Offset: record.offset, Reason: "unsupported payload type"} -} diff --git a/packfile/ingest/entry.go b/packfile/ingest/entry.go deleted file mode 100644 index da063e68..00000000 --- a/packfile/ingest/entry.go +++ /dev/null @@ -1,92 +0,0 @@ -package ingest - -import ( - "fmt" - - objecttype "codeberg.org/lindenii/furgit/object/type" - packfmt "codeberg.org/lindenii/furgit/packfile" -) - -// scanOneEntry scans one pack entry from stream and appends one record. -func scanOneEntry(state *ingestState, startOffset uint64) (uint64, error) { - state.stream.beginEntryCRC() - - record, err := parseEntryPrefix(state, startOffset) - if err != nil { - return 0, err - } - - payloadStartConsumed := state.stream.consumed - - contentLen, oid, err := drainEntryPayload(state, record) - if err != nil { - return 0, err - } - - consumedInput := state.stream.consumed - payloadStartConsumed - - if contentLen != record.declaredSize { - return 0, &MalformedPackEntryError{ - Offset: startOffset, - Reason: fmt.Sprintf("inflated size mismatch got %d want %d", contentLen, record.declaredSize), - } - } - - endOffset := startOffset + uint64(record.headerLen) + consumedInput - if endOffset > state.stream.consumed { - return 0, &MalformedPackEntryError{ - Offset: startOffset, - Reason: fmt.Sprintf("entry end offset overflow got %d > stream %d", endOffset, state.stream.consumed), - } - } - - record.packedLen = endOffset - startOffset - - record.dataOffset = startOffset + uint64(record.headerLen) - if record.packedLen < uint64(record.headerLen) { - return 0, &MalformedPackEntryError{Offset: startOffset, Reason: "negative payload span"} - } - - crc, err := state.stream.endEntryCRC() - if err != nil { - return 0, err - } - - record.crc32 = crc - - if packfmt.IsBaseObjectType(record.packedType) { - record.objectID = oid - record.realType = record.packedType - record.resolved = true - } - - recordIdx := len(state.records) - state.records = append(state.records, record) - - state.offsetToRecord[record.offset] = recordIdx - if record.resolved { - state.objectToRecord[record.objectID] = recordIdx - } - - switch record.packedType { - case objecttype.TypeOfsDelta: - state.ofsDeltas = append(state.ofsDeltas, ofsDeltaRef{ - baseOffset: record.baseOffset, - recordIdx: recordIdx, - }) - case objecttype.TypeRefDelta: - state.refDeltas = append(state.refDeltas, refDeltaRef{ - baseObject: record.baseObject, - recordIdx: recordIdx, - }) - case objecttype.TypeInvalid, - objecttype.TypeCommit, - objecttype.TypeTree, - objecttype.TypeBlob, - objecttype.TypeTag, - objecttype.TypeFuture: - default: - } - - return endOffset, nil -} diff --git a/packfile/ingest/entry_header.go b/packfile/ingest/entry_header.go deleted file mode 100644 index c74fdc16..00000000 --- a/packfile/ingest/entry_header.go +++ /dev/null @@ -1,33 +0,0 @@ -package ingest - -import ( - "codeberg.org/lindenii/furgit/internal/intconv" - objecttype "codeberg.org/lindenii/furgit/object/type" -) - -// encodePackEntryHeader encodes one non-delta packed entry header. -func encodePackEntryHeader(ty objecttype.Type, size int64) []byte { - var out [16]byte - - n := 0 - - s, err := intconv.Int64ToUint64(size) - if err != nil { - panic(err) - } - - c := (uint8(ty) << 4) | byte(s&0x0f) - - s >>= 4 - for s != 0 { - out[n] = c | 0x80 - n++ - c = byte(s & 0x7f) - s >>= 7 - } - - out[n] = c - n++ - - return append([]byte(nil), out[:n]...) -} diff --git a/packfile/ingest/entry_prefix.go b/packfile/ingest/entry_prefix.go deleted file mode 100644 index a107b4e8..00000000 --- a/packfile/ingest/entry_prefix.go +++ /dev/null @@ -1,95 +0,0 @@ -package ingest - -import ( - "fmt" - - "codeberg.org/lindenii/furgit/internal/intconv" - objectid "codeberg.org/lindenii/furgit/object/id" - objecttype "codeberg.org/lindenii/furgit/object/type" -) - -// parseEntryPrefix parses one entry prefix from stream. -func parseEntryPrefix(state *ingestState, startOffset uint64) (objectRecord, error) { - var record objectRecord - - record.offset = startOffset - - first, err := state.stream.ReadByte() - if err != nil { - return record, &MalformedPackEntryError{Offset: startOffset, Reason: fmt.Sprintf("read first header byte: %v", err)} - } - - record.packedType = objecttype.Type((first >> 4) & 0x07) - size := int64(first & 0x0f) - headerLen := uint32(1) - shift := uint(4) - b := first - - for b&0x80 != 0 { - b, err = state.stream.ReadByte() - if err != nil { - return record, &MalformedPackEntryError{Offset: startOffset, Reason: fmt.Sprintf("read size continuation: %v", err)} - } - - headerLen++ - size |= int64(b&0x7f) << shift - shift += 7 - } - - if size < 0 { - return record, &MalformedPackEntryError{Offset: startOffset, Reason: "negative declared size"} - } - - record.declaredSize = size - - switch record.packedType { - case objecttype.TypeCommit, objecttype.TypeTree, objecttype.TypeBlob, objecttype.TypeTag: - case objecttype.TypeRefDelta: - baseRaw := make([]byte, state.algo.Size()) - - err := state.stream.readFull(baseRaw) - if err != nil { - return record, &MalformedPackEntryError{Offset: startOffset, Reason: fmt.Sprintf("read ref base: %v", err)} - } - - baseID, err := objectid.FromBytes(state.algo, baseRaw) - if err != nil { - return record, &MalformedPackEntryError{Offset: startOffset, Reason: fmt.Sprintf("parse ref base: %v", err)} - } - - record.baseObject = baseID - - baseRawLen, err := intconv.IntToUint32(len(baseRaw)) - if err != nil { - return record, err - } - - headerLen += baseRawLen - case objecttype.TypeOfsDelta: - dist, consumed, err := readOfsDistanceFromStream(state.stream) - if err != nil { - return record, &MalformedPackEntryError{Offset: startOffset, Reason: err.Error()} - } - - if startOffset <= dist { - return record, &MalformedPackEntryError{Offset: startOffset, Reason: "ofs base offset out of bounds"} - } - - record.baseOffset = startOffset - dist - - consumedUint32, err := intconv.IntToUint32(consumed) - if err != nil { - return record, err - } - - headerLen += consumedUint32 - case objecttype.TypeInvalid, objecttype.TypeFuture: - return record, &MalformedPackEntryError{Offset: startOffset, Reason: fmt.Sprintf("unsupported object type %d", record.packedType)} - default: - return record, &MalformedPackEntryError{Offset: startOffset, Reason: fmt.Sprintf("unsupported object type %d", record.packedType)} - } - - record.headerLen = headerLen - - return record, nil -} diff --git a/packfile/ingest/errors.go b/packfile/ingest/errors.go deleted file mode 100644 index f6ee9757..00000000 --- a/packfile/ingest/errors.go +++ /dev/null @@ -1,75 +0,0 @@ -package ingest - -import ( - "errors" - "fmt" -) - -// InvalidPackHeaderError reports an invalid or unsupported pack header. -type InvalidPackHeaderError struct { - Reason string -} - -// Error implements error. -func (err *InvalidPackHeaderError) Error() string { - return "packfile/ingest: invalid pack header: " + err.Reason -} - -// PackTrailerMismatchError reports a mismatch between computed and trailer pack hash. -type PackTrailerMismatchError struct{} - -// Error implements error. -func (err *PackTrailerMismatchError) Error() string { - return "packfile/ingest: pack trailer hash mismatch" -} - -// ThinPackUnresolvedError reports unresolved REF deltas when fixThin is disabled -// or when required bases cannot be found in base. -type ThinPackUnresolvedError struct { - Count int -} - -// Error implements error. -func (err *ThinPackUnresolvedError) Error() string { - return fmt.Sprintf("packfile/ingest: unresolved thin deltas: %d", err.Count) -} - -// MalformedPackEntryError reports malformed entry encoding at one pack offset. -type MalformedPackEntryError struct { - Offset uint64 - Reason string -} - -// Error implements error. -func (err *MalformedPackEntryError) Error() string { - return fmt.Sprintf("packfile/ingest: malformed pack entry at offset %d: %s", err.Offset, err.Reason) -} - -// DeltaCycleError reports a detected cycle in delta dependency resolution. -type DeltaCycleError struct { - Offset uint64 -} - -// Error implements error. -func (err *DeltaCycleError) Error() string { - return fmt.Sprintf("packfile/ingest: delta cycle detected at offset %d", err.Offset) -} - -// DestinationWriteError reports destination I/O failures. -type DestinationWriteError struct { - Op string -} - -// Error implements error. -func (err *DestinationWriteError) Error() string { - return "packfile/ingest: destination write failure: " + err.Op -} - -var errExternalThinBase = errors.New("packfile/ingest: external thin base required") - -var ( - // ErrZeroObjectContinue indicates Continue was called for a zero-object pack. - ErrZeroObjectContinue = errors.New("packfile/ingest: cannot continue zero-object pack") - // ErrNonZeroDiscard indicates Discard was called for a non-zero-object pack. - ErrNonZeroDiscard = errors.New("packfile/ingest: cannot discard non-zero pack") -) diff --git a/packfile/ingest/file_section_writer.go b/packfile/ingest/file_section_writer.go deleted file mode 100644 index fa28c1a9..00000000 --- a/packfile/ingest/file_section_writer.go +++ /dev/null @@ -1,22 +0,0 @@ -package ingest - -import "os" - -// fileSectionWriter writes sequentially to file via WriteAt at one base offset. -type fileSectionWriter struct { - file *os.File - off int64 - pos int64 -} - -// Write writes src at current section position. -func (writer *fileSectionWriter) Write(src []byte) (int, error) { - if len(src) == 0 { - return 0, nil - } - - n, err := writer.file.WriteAt(src, writer.off+writer.pos) - writer.pos += int64(n) - - return n, err -} diff --git a/packfile/ingest/fill.go b/packfile/ingest/fill.go deleted file mode 100644 index eca4e4d6..00000000 --- a/packfile/ingest/fill.go +++ /dev/null @@ -1,44 +0,0 @@ -package ingest - -import ( - "errors" - "fmt" - "io" -) - -// fill ensures at least min unread bytes are available in receiver's buffer. -func (scanner *streamScanner) fill(minLen int) error { - if minLen <= 0 { - return nil - } - - if minLen > len(scanner.buf) { - return fmt.Errorf("packfile/ingest: fill(%d) exceeds scanner buffer", minLen) - } - - for scanner.n-scanner.off < minLen { - err := scanner.flushConsumedPrefix() - if err != nil { - return err - } - - readN, err := scanner.src.Read(scanner.buf[scanner.n:]) - if readN > 0 { - scanner.n += readN - } - - if err != nil { - if errors.Is(err, io.EOF) && scanner.n-scanner.off >= minLen { - return nil - } - - return err - } - - if readN == 0 { - return io.ErrNoProgress - } - } - - return nil -} diff --git a/packfile/ingest/finalize.go b/packfile/ingest/finalize.go deleted file mode 100644 index 6fe4edb2..00000000 --- a/packfile/ingest/finalize.go +++ /dev/null @@ -1,94 +0,0 @@ -package ingest - -import ( - "errors" - "fmt" - "io/fs" - "strings" - - "codeberg.org/lindenii/furgit/internal/intconv" -) - -// finalizeArtifacts links temporary files to final names and returns Result. -func finalizeArtifacts(state *ingestState) (Result, error) { - base := "pack-" + state.packHash.String() - packFinal := base + ".pack" - idxFinal := base + ".idx" - - revFinal := "" - if state.opts.WriteRev { - revFinal = base + ".rev" - } - - err := linkTempToFinal(state, state.packTmpName, packFinal) - if err != nil { - return Result{}, err - } - - err = linkTempToFinal(state, state.idxTmpName, idxFinal) - if err != nil { - return Result{}, err - } - - if state.opts.WriteRev { - err := linkTempToFinal(state, state.revTmpName, revFinal) - if err != nil { - return Result{}, err - } - } - - objectCount, err := intconv.IntToUint32(len(state.records)) - if err != nil { - return Result{}, err - } - - return Result{ - PackName: packFinal, - IdxName: idxFinal, - RevName: revFinal, - PackHash: state.packHash, - ObjectCount: objectCount, - ThinFixed: state.thinFixed, - }, nil -} - -// rollbackTemporaryArtifacts removes temporary files after failure. -func rollbackTemporaryArtifacts(state *ingestState) { - if state.packTmpName != "" { - _ = state.destination.Remove(state.packTmpName) - } - - if state.idxTmpName != "" { - _ = state.destination.Remove(state.idxTmpName) - } - - if state.revTmpName != "" { - _ = state.destination.Remove(state.revTmpName) - } -} - -// linkTempToFinal hard-links tmp to final, tolerating existing final paths. -func linkTempToFinal(state *ingestState, tmp, final string) error { - if tmp == "" || final == "" { - return fmt.Errorf("packfile/ingest: invalid finalize names tmp=%q final=%q", tmp, final) - } - - if strings.Contains(final, "/") { - return fmt.Errorf("packfile/ingest: final name must be leaf: %q", final) - } - - err := state.destination.Link(tmp, final) - if err == nil { - _ = state.destination.Remove(tmp) - - return nil - } - - if errors.Is(err, fs.ErrExist) { - _ = state.destination.Remove(tmp) - - return nil - } - - return err -} diff --git a/packfile/ingest/flush.go b/packfile/ingest/flush.go deleted file mode 100644 index 96753170..00000000 --- a/packfile/ingest/flush.go +++ /dev/null @@ -1,37 +0,0 @@ -package ingest - -import "fmt" - -// flush writes all consumed-but-unflushed bytes to destination pack file. -func (scanner *streamScanner) flush() error { - return scanner.flushConsumedPrefix() -} - -// flushConsumedPrefix writes scanner.buf[:scanner.off] and compacts unread -// bytes to the start of buffer. -func (scanner *streamScanner) flushConsumedPrefix() error { - if scanner.off == 0 { - return nil - } - - written := 0 - for written < scanner.off { - n, err := scanner.dstFile.Write(scanner.buf[written:scanner.off]) - if err != nil { - return &DestinationWriteError{Op: fmt.Sprintf("write pack: %v", err)} - } - - if n == 0 { - return &DestinationWriteError{Op: "write pack: short write"} - } - - written += n - } - - unread := scanner.n - scanner.off - copy(scanner.buf[:unread], scanner.buf[scanner.off:scanner.n]) - scanner.off = 0 - scanner.n = unread - - return nil -} diff --git a/packfile/ingest/hash.go b/packfile/ingest/hash.go deleted file mode 100644 index 4b739c20..00000000 --- a/packfile/ingest/hash.go +++ /dev/null @@ -1,27 +0,0 @@ -package ingest - -import ( - "fmt" - - objectheader "codeberg.org/lindenii/furgit/object/header" - objectid "codeberg.org/lindenii/furgit/object/id" - objecttype "codeberg.org/lindenii/furgit/object/type" -) - -// hashCanonicalObject hashes canonical object bytes (header+content). -func hashCanonicalObject(algo objectid.Algorithm, ty objecttype.Type, content []byte) (objectid.ObjectID, error) { - header, ok := objectheader.Encode(ty, int64(len(content))) - if !ok { - return objectid.ObjectID{}, fmt.Errorf("packfile/ingest: encode object header for type %d", ty) - } - - hashImpl, err := algo.New() - if err != nil { - return objectid.ObjectID{}, err - } - - _, _ = hashImpl.Write(header) - _, _ = hashImpl.Write(content) - - return objectid.FromBytes(algo, hashImpl.Sum(nil)) -} diff --git a/packfile/ingest/header.go b/packfile/ingest/header.go deleted file mode 100644 index c9db0f1b..00000000 --- a/packfile/ingest/header.go +++ /dev/null @@ -1,49 +0,0 @@ -package ingest - -import ( - "encoding/binary" - "fmt" - "io" - - "codeberg.org/lindenii/furgit/packfile" -) - -const packHeaderSize = 12 - -// readAndValidatePackHeader reads one PACK header from src and validates it. -func readAndValidatePackHeader(src io.Reader) (HeaderInfo, [packHeaderSize]byte, error) { - var hdr [packHeaderSize]byte - - _, err := io.ReadFull(src, hdr[:]) - if err != nil { - return HeaderInfo{}, [packHeaderSize]byte{}, &InvalidPackHeaderError{ - Reason: fmt.Sprintf("read header: %v", err), - } - } - - header, err := parseAndValidatePackHeader(hdr) - if err != nil { - return HeaderInfo{}, [packHeaderSize]byte{}, err - } - - return header, hdr, nil -} - -// parseAndValidatePackHeader validates one already-read PACK header. -func parseAndValidatePackHeader(hdr [packHeaderSize]byte) (HeaderInfo, error) { - if binary.BigEndian.Uint32(hdr[:4]) != packfile.Signature { - return HeaderInfo{}, &InvalidPackHeaderError{Reason: "signature mismatch"} - } - - version := binary.BigEndian.Uint32(hdr[4:8]) - if !packfile.VersionSupported(version) { - return HeaderInfo{}, &InvalidPackHeaderError{ - Reason: fmt.Sprintf("unsupported version %d", version), - } - } - - return HeaderInfo{ - Version: version, - ObjectCount: binary.BigEndian.Uint32(hdr[8:12]), - }, nil -} diff --git a/packfile/ingest/idx_write.go b/packfile/ingest/idx_write.go deleted file mode 100644 index 506788b9..00000000 --- a/packfile/ingest/idx_write.go +++ /dev/null @@ -1,266 +0,0 @@ -package ingest - -import ( - "bytes" - "encoding/binary" - "fmt" - "hash" - "io" - "slices" - - "codeberg.org/lindenii/furgit/internal/intconv" - "codeberg.org/lindenii/furgit/internal/progress" -) - -const ( - idxMagicV2 = 0xff744f63 - idxVersionV2 = 2 -) - -// writeIdx writes idx v2 for resolved records. -func writeIdx(state *ingestState) error { - order := buildIdxOrder(state) - - hashImpl, err := state.algo.New() - if err != nil { - return err - } - - write := func(src []byte) error { - _, writeErr := state.idxFile.Write(src) - if writeErr != nil { - return writeErr - } - - _, writeErr = hashImpl.Write(src) - if writeErr != nil { - return writeErr - } - - return nil - } - - var ( - scratch [8]byte - fanout [256]uint32 - ) - - writeProgressf(state, "writing index fanout...\r") - - for _, recordIdx := range order { - idRaw := state.records[recordIdx].objectID.Bytes() - fanout[idRaw[0]]++ - } - - binary.BigEndian.PutUint32(scratch[:4], idxMagicV2) - binary.BigEndian.PutUint32(scratch[4:8], idxVersionV2) - - err = write(scratch[:8]) - if err != nil { - return err - } - - var cumulative uint32 - for i := range fanout { - cumulative += fanout[i] - binary.BigEndian.PutUint32(scratch[:4], cumulative) - - err := write(scratch[:4]) - if err != nil { - return err - } - } - - writeProgressf(state, "writing index fanout: done.\n") - - largeOffsetCount := 0 - - for idx := range state.records { - if state.records[idx].offset >= 0x80000000 { - largeOffsetCount++ - } - } - - oidMeter := progress.New(progress.Options{ - Writer: state.opts.Progress, - Flush: state.opts.ProgressFlush, - Title: "writing index object ids", - Total: uint64(len(order)), - }) - - var oidDone uint64 - - for _, recordIdx := range order { - idRaw := state.records[recordIdx].objectID.Bytes() - - err := write(idRaw) - if err != nil { - return err - } - - oidDone++ - oidMeter.Set(oidDone, 0) - } - - if oidDone > 0 { - oidMeter.Stop("done") - } - - crcMeter := progress.New(progress.Options{ - Writer: state.opts.Progress, - Flush: state.opts.ProgressFlush, - Title: "writing index crc32", - Total: uint64(len(order)), - }) - - var crcDone uint64 - - for _, recordIdx := range order { - binary.BigEndian.PutUint32(scratch[:4], state.records[recordIdx].crc32) - - err := write(scratch[:4]) - if err != nil { - return err - } - - crcDone++ - crcMeter.Set(crcDone, 0) - } - - if crcDone > 0 { - crcMeter.Stop("done") - } - - largeOffsets := make([]uint64, 0) - offsetMeter := progress.New(progress.Options{ - Writer: state.opts.Progress, - Flush: state.opts.ProgressFlush, - Title: "writing index offsets", - Total: uint64(len(order)), - }) - - var offsetDone uint64 - - for _, recordIdx := range order { - offset := state.records[recordIdx].offset - if offset >= 0x80000000 { - largeOffsetIdx, err := intconv.IntToUint32(len(largeOffsets)) - if err != nil { - return err - } - - word := 0x80000000 | largeOffsetIdx - - largeOffsets = append(largeOffsets, offset) - - binary.BigEndian.PutUint32(scratch[:4], word) - } else { - binary.BigEndian.PutUint32(scratch[:4], uint32(offset)) - } - - err := write(scratch[:4]) - if err != nil { - return err - } - - offsetDone++ - offsetMeter.Set(offsetDone, 0) - } - - if offsetDone > 0 { - offsetMeter.Stop("done") - } - - total, err := intconv.IntToUint64(largeOffsetCount) - if err != nil { - return err - } - - largeOffsetMeter := progress.New(progress.Options{ - Writer: state.opts.Progress, - Flush: state.opts.ProgressFlush, - Title: "writing index large offsets", - Total: total, - }) - - var largeOffsetDone uint64 - - for _, off := range largeOffsets { - binary.BigEndian.PutUint64(scratch[:8], off) - - err := write(scratch[:8]) - if err != nil { - return err - } - - largeOffsetDone++ - largeOffsetMeter.Set(largeOffsetDone, 0) - } - - if largeOffsetDone > 0 { - largeOffsetMeter.Stop("done") - } - - writeProgressf(state, "writing index trailer...\r") - - err = write(state.packHash.Bytes()) - if err != nil { - return err - } - - idxHash := hashImpl.Sum(nil) - - _, err = state.idxFile.Write(idxHash) - if err != nil { - return err - } - - err = state.idxFile.Sync() - if err != nil { - return err - } - - writeProgressf(state, "writing index trailer: done.\n") - - return nil -} - -// buildIdxOrder returns record indexes sorted by ObjectID. -func buildIdxOrder(state *ingestState) []int { - out := make([]int, 0, len(state.records)) - for idx := range state.records { - out = append(out, idx) - } - - slices.SortFunc(out, func(a, b int) int { - return bytes.Compare(state.records[a].objectID.Bytes(), state.records[b].objectID.Bytes()) - }) - - return out -} - -// verifyResolvedRecords checks that all records are fully resolved before index writing. -func verifyResolvedRecords(state *ingestState) error { - for idx, record := range state.records { - if !record.resolved { - return fmt.Errorf("packfile/ingest: unresolved record %d at offset %d", idx, record.offset) - } - } - - return nil -} - -// writeAndHash writes src to dst and updates hash. -func writeAndHash(dst io.Writer, hashImpl hash.Hash, src []byte) error { - _, err := dst.Write(src) - if err != nil { - return err - } - - _, err = hashImpl.Write(src) - if err != nil { - return err - } - - return nil -} diff --git a/packfile/ingest/ingest.go b/packfile/ingest/ingest.go deleted file mode 100644 index be65ff5f..00000000 --- a/packfile/ingest/ingest.go +++ /dev/null @@ -1,68 +0,0 @@ -package ingest - -import ( - "fmt" -) - -// ingest initializes transaction state and executes the ingest pipeline. -func ingest(state *ingestState) (out Result, err error) { - err = openTemporaryArtifacts(state) - if err != nil { - return Result{}, err - } - - defer func() { - _ = closeTemporaryArtifacts(state) - if err != nil { - rollbackTemporaryArtifacts(state) - } - }() - - err = streamPackAndScan(state) - if err != nil { - return Result{}, err - } - - err = resolveAll(state) - if err != nil { - return Result{}, err - } - - err = maybeFixThin(state) - if err != nil { - return Result{}, err - } - - if state.thinFixed { - err = resolveAll(state) - if err != nil { - return Result{}, err - } - } - - if len(state.unresolvedRefDeltas) > 0 { - return Result{}, &ThinPackUnresolvedError{Count: len(state.unresolvedRefDeltas)} - } - - err = verifyResolvedRecords(state) - if err != nil { - return Result{}, err - } - - err = state.packFile.Sync() - if err != nil { - return Result{}, &DestinationWriteError{Op: fmt.Sprintf("sync pack: %v", err)} - } - - err = writeIdx(state) - if err != nil { - return Result{}, err - } - - err = writeRev(state) - if err != nil { - return Result{}, err - } - - return finalizeArtifacts(state) -} diff --git a/packfile/ingest/ingest_test.go b/packfile/ingest/ingest_test.go deleted file mode 100644 index 3eb821d3..00000000 --- a/packfile/ingest/ingest_test.go +++ /dev/null @@ -1,434 +0,0 @@ -package ingest_test - -import ( - "bytes" - "encoding/binary" - "errors" - "io" - "io/fs" - "os" - "path/filepath" - "strings" - "testing" - - "codeberg.org/lindenii/furgit/internal/testgit" - objectid "codeberg.org/lindenii/furgit/object/id" - "codeberg.org/lindenii/furgit/packfile/ingest" -) - -type noExtraReadReader struct { - reader *bytes.Reader -} - -func (r *noExtraReadReader) Read(p []byte) (int, error) { - if r.reader.Len() == 0 { - return 0, errors.New("unexpected extra read after pack trailer") - } - - return r.reader.Read(p) -} - -func beginAndContinue( - src io.Reader, - packRoot *os.Root, - algo objectid.Algorithm, - opts ingest.Options, -) (ingest.Result, error) { - pending, err := ingest.Ingest(src, algo, opts) - if err != nil { - return ingest.Result{}, err - } - - return pending.Continue(packRoot) -} - -// fixturePath returns one fixture file path for the selected algorithm. -func fixturePath(t *testing.T, algo objectid.Algorithm, name string) string { - t.Helper() - - dir := algo.String() - if dir == "" { - t.Fatalf("unsupported fixture algorithm: %v", algo) - } - - return filepath.Join("testdata", "fixtures", dir, name) -} - -// fixtureBytes reads one fixture file fully. -func fixtureBytes(t *testing.T, algo objectid.Algorithm, name string) []byte { - t.Helper() - - path := fixturePath(t, algo, name) - dir := filepath.Dir(path) - base := filepath.Base(path) - - root, err := os.OpenRoot(dir) - if err != nil { - t.Fatalf("open fixture root %q: %v", dir, err) - } - - defer func() { - err := root.Close() - if err != nil { - t.Fatalf("close fixture root %q: %v", dir, err) - } - }() - - data, err := root.ReadFile(base) - if err != nil { - t.Fatalf("read fixture %q: %v", base, err) - } - - return data -} - -// fixtureMetadata parses key=value metadata for one algorithm fixture set. -func fixtureMetadata(t *testing.T, algo objectid.Algorithm) map[string]string { - t.Helper() - - data := fixtureBytes(t, algo, "METADATA.txt") - - out := make(map[string]string) - for line := range strings.SplitSeq(strings.TrimSpace(string(data)), "\n") { - line = strings.TrimSpace(line) - if line == "" { - continue - } - - key, value, ok := strings.Cut(line, "=") - if !ok { - t.Fatalf("invalid fixture metadata line %q", line) - } - - out[strings.TrimSpace(key)] = strings.TrimSpace(value) - } - - return out -} - -// fixtureOID returns one fixture metadata object ID value. -func fixtureOID(t *testing.T, algo objectid.Algorithm, key string) objectid.ObjectID { - t.Helper() - - meta := fixtureMetadata(t, algo) - - hex, ok := meta[key] - if !ok { - t.Fatalf("missing fixture metadata key %q", key) - } - - id, err := objectid.ParseHex(algo, hex) - if err != nil { - t.Fatalf("parse fixture metadata oid %q: %v", hex, err) - } - - return id -} - -// verifyReindexOracle regenerates idx/rev with upstream git index-pack and -// compares bytes with files produced by ingest. -func verifyReindexOracle(t *testing.T, repo *testgit.TestRepo, packName, idxName, revName string) { - t.Helper() - - oracleDir := t.TempDir() - oracleIdxPath := filepath.Join(oracleDir, "oracle.idx") - _ = repo.Run(t, "index-pack", "--rev-index", "-o", oracleIdxPath, filepath.Join("objects", "pack", packName)) - oracleRevPath := strings.TrimSuffix(oracleIdxPath, ".idx") + ".rev" - - packRoot := repo.OpenPackRoot(t) - - gotIdx, err := packRoot.ReadFile(idxName) - if err != nil { - t.Fatalf("read idx: %v", err) - } - - oracleRoot, err := os.OpenRoot(oracleDir) - if err != nil { - t.Fatalf("open oracle root: %v", err) - } - - defer func() { - err := oracleRoot.Close() - if err != nil { - t.Fatalf("close oracle root: %v", err) - } - }() - - wantIdx, err := oracleRoot.ReadFile(filepath.Base(oracleIdxPath)) - if err != nil { - t.Fatalf("read oracle idx: %v", err) - } - - if !bytes.Equal(gotIdx, wantIdx) { - t.Fatal("idx bytes differ from git index-pack output") - } - - gotRev, err := packRoot.ReadFile(revName) - if err != nil { - t.Fatalf("read rev: %v", err) - } - - wantRev, err := oracleRoot.ReadFile(filepath.Base(oracleRevPath)) - if err != nil { - t.Fatalf("read oracle rev: %v", err) - } - - if !bytes.Equal(gotRev, wantRev) { - t.Fatal("rev bytes differ from git index-pack output") - } -} - -func TestIngestNonThinPackWritesPackIdxRev(t *testing.T) { - t.Parallel() - - testgit.ForEachAlgorithm(t, func(t *testing.T, algo objectid.Algorithm) { //nolint:thelper - head := fixtureOID(t, algo, "head") - packBytes := fixtureBytes(t, algo, "nonthin.pack") - - receiver := testgit.NewRepo(t, testgit.RepoOptions{ObjectFormat: algo, Bare: true}) - - packRoot := receiver.OpenPackRoot(t) - - result, err := beginAndContinue(bytes.NewReader(packBytes), packRoot, algo, ingest.Options{ - WriteRev: true, - RequireTrailingEOF: true, - }) - if err != nil { - t.Fatalf("Ingest: %v", err) - } - - if result.ThinFixed { - t.Fatalf("ThinFixed = true, want false") - } - - if result.RevName == "" { - t.Fatal("RevName is empty") - } - - _, err = packRoot.Stat(result.PackName) - if err != nil { - t.Fatalf("stat pack: %v", err) - } - - _, err = packRoot.Stat(result.IdxName) - if err != nil { - t.Fatalf("stat idx: %v", err) - } - - _, err = packRoot.Stat(result.RevName) - if err != nil { - t.Fatalf("stat rev: %v", err) - } - - _ = receiver.Run(t, "verify-pack", "-v", filepath.Join("objects", "pack", result.IdxName)) - verifyReindexOracle(t, receiver, result.PackName, result.IdxName, result.RevName) - - receiver.UpdateRef(t, "refs/heads/main", head) - _ = receiver.Run(t, "fsck", "--full", "--strict", "--no-progress", "--no-dangling") - }) -} - -func TestIngestThinPackWithoutFixReturnsUnresolved(t *testing.T) { - t.Parallel() - - testgit.ForEachAlgorithm(t, func(t *testing.T, algo objectid.Algorithm) { //nolint:thelper - thinPack := fixtureBytes(t, algo, "thin.pack") - - receiver := testgit.NewRepo(t, testgit.RepoOptions{ObjectFormat: algo, Bare: true}) - packRoot := receiver.OpenPackRoot(t) - - _, err := beginAndContinue(bytes.NewReader(thinPack), packRoot, algo, ingest.Options{ - WriteRev: true, - RequireTrailingEOF: true, - }) - if err == nil { - t.Fatal("Ingest error = nil, want error") - } - - if _, ok := errors.AsType[*ingest.ThinPackUnresolvedError](err); !ok { - t.Fatalf("Ingest error type = %T (%v), want *ThinPackUnresolvedError", err, err) - } - - entries, err := fs.ReadDir(packRoot.FS(), ".") - if err != nil { - t.Fatalf("ReadDir(pack): %v", err) - } - - for _, entry := range entries { - if strings.HasSuffix(entry.Name(), ".pack") { - t.Fatalf("found finalized pack file after failure: %v", entry.Name()) - } - } - }) -} - -func TestIngestThinPackWithFixThin(t *testing.T) { - t.Parallel() - - testgit.ForEachAlgorithm(t, func(t *testing.T, algo objectid.Algorithm) { //nolint:thelper - head := fixtureOID(t, algo, "head") - basePack := fixtureBytes(t, algo, "base.pack") - thinPack := fixtureBytes(t, algo, "thin.pack") - receiver := testgit.NewRepo(t, testgit.RepoOptions{ObjectFormat: algo, Bare: true}) - - packRoot := receiver.OpenPackRoot(t) - - _, err := beginAndContinue(bytes.NewReader(basePack), packRoot, algo, ingest.Options{ - RequireTrailingEOF: true, - }) - if err != nil { - t.Fatalf("ingest base pack: %v", err) - } - - receiverRepo := receiver.OpenRepository(t) - - result, err := beginAndContinue(bytes.NewReader(thinPack), packRoot, algo, ingest.Options{ - FixThin: true, - WriteRev: true, - Base: receiverRepo.Objects(), - RequireTrailingEOF: true, - }) - if err != nil { - t.Fatalf("Ingest(thin): %v", err) - } - - if !result.ThinFixed { - t.Fatal("ThinFixed = false, want true") - } - - _ = receiver.Run(t, "verify-pack", "-v", filepath.Join("objects", "pack", result.IdxName)) - verifyReindexOracle(t, receiver, result.PackName, result.IdxName, result.RevName) - receiver.UpdateRef(t, "refs/heads/main", head) - _ = receiver.Run(t, "fsck", "--full", "--strict", "--no-progress", "--no-dangling") - }) -} - -func TestIngestPackTrailerMismatch(t *testing.T) { - t.Parallel() - - testgit.ForEachAlgorithm(t, func(t *testing.T, algo objectid.Algorithm) { //nolint:thelper - packBytes := fixtureBytes(t, algo, "nonthin.pack") - if len(packBytes) == 0 { - t.Fatal("empty pack stream") - } - - packBytes[len(packBytes)-1] ^= 0xff - - receiver := testgit.NewRepo(t, testgit.RepoOptions{ObjectFormat: algo, Bare: true}) - packRoot := receiver.OpenPackRoot(t) - - _, err := beginAndContinue(bytes.NewReader(packBytes), packRoot, algo, ingest.Options{ - WriteRev: true, - RequireTrailingEOF: true, - }) - if err == nil { - t.Fatal("Ingest error = nil, want error") - } - - if _, ok := errors.AsType[*ingest.PackTrailerMismatchError](err); !ok { - t.Fatalf("Ingest error type = %T (%v), want *PackTrailerMismatchError", err, err) - } - - entries, err := fs.ReadDir(packRoot.FS(), ".") - if err != nil { - t.Fatalf("ReadDir(pack): %v", err) - } - - for _, entry := range entries { - if strings.HasSuffix(entry.Name(), ".pack") { - t.Fatalf("found finalized pack file after failure: %v", entry.Name()) - } - } - }) -} - -func zeroObjectPackBytes(t *testing.T, algo objectid.Algorithm) []byte { - t.Helper() - - hashImpl, err := algo.New() - if err != nil { - t.Fatalf("algo.New: %v", err) - } - - var header [12]byte - copy(header[:4], []byte{'P', 'A', 'C', 'K'}) - binary.BigEndian.PutUint32(header[4:8], 2) - binary.BigEndian.PutUint32(header[8:12], 0) - - _, _ = hashImpl.Write(header[:]) - - return append(header[:], hashImpl.Sum(nil)...) -} - -func TestIngestDiscardZeroObjectPack(t *testing.T) { - t.Parallel() - - testgit.ForEachAlgorithm(t, func(t *testing.T, algo objectid.Algorithm) { //nolint:thelper - packBytes := zeroObjectPackBytes(t, algo) - - pending, err := ingest.Ingest(bytes.NewReader(packBytes), algo, ingest.Options{ - RequireTrailingEOF: true, - }) - if err != nil { - t.Fatalf("Ingest: %v", err) - } - - if pending.Header().ObjectCount != 0 { - t.Fatalf("ObjectCount = %d, want 0", pending.Header().ObjectCount) - } - - discarded, err := pending.Discard() - if err != nil { - t.Fatalf("Discard: %v", err) - } - - if discarded.ObjectCount != 0 { - t.Fatalf("Discard.ObjectCount = %d, want 0", discarded.ObjectCount) - } - }) -} - -func TestIngestContinueRejectsZeroObjectPack(t *testing.T) { - t.Parallel() - - testgit.ForEachAlgorithm(t, func(t *testing.T, algo objectid.Algorithm) { //nolint:thelper - packBytes := zeroObjectPackBytes(t, algo) - receiver := testgit.NewRepo(t, testgit.RepoOptions{ObjectFormat: algo, Bare: true}) - packRoot := receiver.OpenPackRoot(t) - - pending, err := ingest.Ingest(bytes.NewReader(packBytes), algo, ingest.Options{ - RequireTrailingEOF: true, - }) - if err != nil { - t.Fatalf("Ingest: %v", err) - } - - _, err = pending.Continue(packRoot) - if !errors.Is(err, ingest.ErrZeroObjectContinue) { - t.Fatalf("Continue error = %v, want ErrZeroObjectContinue", err) - } - }) -} - -func TestIngestCanFinishWithoutTrailingEOF(t *testing.T) { - t.Parallel() - - testgit.ForEachAlgorithm(t, func(t *testing.T, algo objectid.Algorithm) { //nolint:thelper - head := fixtureOID(t, algo, "head") - packBytes := fixtureBytes(t, algo, "nonthin.pack") - - receiver := testgit.NewRepo(t, testgit.RepoOptions{ObjectFormat: algo, Bare: true}) - packRoot := receiver.OpenPackRoot(t) - - result, err := beginAndContinue(&noExtraReadReader{reader: bytes.NewReader(packBytes)}, packRoot, algo, ingest.Options{ - WriteRev: true, - }) - if err != nil { - t.Fatalf("Ingest without trailing EOF: %v", err) - } - - receiver.UpdateRef(t, "refs/heads/main", head) - _ = receiver.Run(t, "verify-pack", "-v", filepath.Join("objects", "pack", result.IdxName)) - _ = receiver.Run(t, "fsck", "--full", "--strict", "--no-progress", "--no-dangling") - }) -} diff --git a/packfile/ingest/progress_write.go b/packfile/ingest/progress_write.go deleted file mode 100644 index 5b9f184b..00000000 --- a/packfile/ingest/progress_write.go +++ /dev/null @@ -1,11 +0,0 @@ -package ingest - -import "codeberg.org/lindenii/furgit/internal/utils" - -func writeProgressf(state *ingestState, format string, args ...any) { - utils.BestEffortFprintf(state.opts.Progress, format, args...) - - if state.opts.ProgressFlush != nil { - _ = state.opts.ProgressFlush() - } -} diff --git a/packfile/ingest/record_content.go b/packfile/ingest/record_content.go deleted file mode 100644 index 4109bc01..00000000 --- a/packfile/ingest/record_content.go +++ /dev/null @@ -1,30 +0,0 @@ -package ingest - -import ( - "fmt" - - objecttype "codeberg.org/lindenii/furgit/object/type" - packfmt "codeberg.org/lindenii/furgit/packfile" -) - -// readBaseRecordContent reads canonical base content for one non-delta record. -func readBaseRecordContent(state *ingestState, idx int) (objecttype.Type, []byte, error) { - record := state.records[idx] - if !packfmt.IsBaseObjectType(record.packedType) { - return objecttype.TypeInvalid, nil, fmt.Errorf("packfile/ingest: record %d is not a base object", idx) - } - - content, err := inflateRecordPayload(state, idx) - if err != nil { - return objecttype.TypeInvalid, nil, err - } - - if int64(len(content)) != record.declaredSize { - return objecttype.TypeInvalid, nil, &MalformedPackEntryError{ - Offset: record.offset, - Reason: fmt.Sprintf("base content size mismatch got %d want %d", len(content), record.declaredSize), - } - } - - return record.packedType, content, nil -} diff --git a/packfile/ingest/record_delta.go b/packfile/ingest/record_delta.go deleted file mode 100644 index 69cb8524..00000000 --- a/packfile/ingest/record_delta.go +++ /dev/null @@ -1,60 +0,0 @@ -package ingest - -import ( - "fmt" - - objecttype "codeberg.org/lindenii/furgit/object/type" - deltaapply "codeberg.org/lindenii/furgit/packfile/delta/apply" -) - -// applyDeltaRecord applies one delta record onto base content. -func applyDeltaRecord(state *ingestState, idx int, baseType objecttype.Type, baseContent []byte) (objecttype.Type, []byte, error) { - record := state.records[idx] - if record.packedType != objecttype.TypeOfsDelta && record.packedType != objecttype.TypeRefDelta { - return objecttype.TypeInvalid, nil, fmt.Errorf("packfile/ingest: record %d is not a delta record", idx) - } - - deltaPayload, err := inflateRecordPayload(state, idx) - if err != nil { - return objecttype.TypeInvalid, nil, err - } - - if int64(len(deltaPayload)) != record.declaredSize { - return objecttype.TypeInvalid, nil, &MalformedPackEntryError{ - Offset: record.offset, - Reason: fmt.Sprintf("delta payload size mismatch got %d want %d", len(deltaPayload), record.declaredSize), - } - } - - srcSize, dstSize, err := readDeltaHeaderSizes(deltaPayload) - if err != nil { - return objecttype.TypeInvalid, nil, &MalformedPackEntryError{ - Offset: record.offset, - Reason: fmt.Sprintf("read delta header: %v", err), - } - } - - if srcSize != len(baseContent) { - return objecttype.TypeInvalid, nil, &MalformedPackEntryError{ - Offset: record.offset, - Reason: fmt.Sprintf("delta source size mismatch got %d want %d", srcSize, len(baseContent)), - } - } - - content, err := deltaapply.Apply(baseContent, deltaPayload) - if err != nil { - return objecttype.TypeInvalid, nil, &MalformedPackEntryError{ - Offset: record.offset, - Reason: fmt.Sprintf("apply delta: %v", err), - } - } - - if len(content) != dstSize { - return objecttype.TypeInvalid, nil, &MalformedPackEntryError{ - Offset: record.offset, - Reason: fmt.Sprintf("delta result size mismatch got %d want %d", len(content), dstSize), - } - } - - return baseType, content, nil -} diff --git a/packfile/ingest/record_inflate.go b/packfile/ingest/record_inflate.go deleted file mode 100644 index b8eca25b..00000000 --- a/packfile/ingest/record_inflate.go +++ /dev/null @@ -1,46 +0,0 @@ -package ingest - -import ( - "compress/zlib" - "fmt" - "io" - - "codeberg.org/lindenii/furgit/internal/intconv" -) - -// inflateRecordPayload inflates one record's zlib payload from pack file. -func inflateRecordPayload(state *ingestState, idx int) ([]byte, error) { - record := state.records[idx] - if record.packedLen < uint64(record.headerLen) { - return nil, &MalformedPackEntryError{Offset: record.offset, Reason: "entry packed span underflow"} - } - - compressedOffset := record.offset + uint64(record.headerLen) - compressedLen := record.packedLen - uint64(record.headerLen) - - compressedOffsetInt64, err := intconv.Uint64ToInt64(compressedOffset) - if err != nil { - return nil, err - } - - compressedLenInt64, err := intconv.Uint64ToInt64(compressedLen) - if err != nil { - return nil, err - } - - section := io.NewSectionReader(state.packFile, compressedOffsetInt64, compressedLenInt64) - - reader, err := zlib.NewReader(section) - if err != nil { - return nil, &MalformedPackEntryError{Offset: record.offset, Reason: fmt.Sprintf("open payload zlib: %v", err)} - } - - defer func() { _ = reader.Close() }() - - out, err := io.ReadAll(reader) - if err != nil { - return nil, &MalformedPackEntryError{Offset: record.offset, Reason: fmt.Sprintf("inflate payload: %v", err)} - } - - return out, nil -} diff --git a/packfile/ingest/record_resolve.go b/packfile/ingest/record_resolve.go deleted file mode 100644 index 5bff18ab..00000000 --- a/packfile/ingest/record_resolve.go +++ /dev/null @@ -1,117 +0,0 @@ -package ingest - -import ( - "fmt" - - objecttype "codeberg.org/lindenii/furgit/object/type" - packfmt "codeberg.org/lindenii/furgit/packfile" -) - -// resolveRecord resolves one record and returns canonical type/content. -func resolveRecord(state *ingestState, idx int, visiting map[int]struct{}) (objecttype.Type, []byte, error) { - if idx < 0 || idx >= len(state.records) { - return objecttype.TypeInvalid, nil, fmt.Errorf("packfile/ingest: record index out of bounds") - } - - if _, ok := visiting[idx]; ok { - return objecttype.TypeInvalid, nil, &DeltaCycleError{Offset: state.records[idx].offset} - } - - visiting[idx] = struct{}{} - defer delete(visiting, idx) - - record := &state.records[idx] - if ty, content, ok := state.baseCache.get(idx); ok { - return ty, content, nil - } - - if packfmt.IsBaseObjectType(record.packedType) { - ty, content, err := readBaseRecordContent(state, idx) - if err != nil { - return objecttype.TypeInvalid, nil, err - } - - if record.resolved { - state.baseCache.add(idx, record.realType, content) - - return record.realType, content, nil - } - - id, err := hashCanonicalObject(state.algo, ty, content) - if err != nil { - return objecttype.TypeInvalid, nil, err - } - - record.objectID = id - record.realType = ty - record.resolved = true - state.objectToRecord[id] = idx - state.baseCache.add(idx, ty, content) - - return ty, content, nil - } - - var ( - baseType objecttype.Type - baseContent []byte - err error - ) - switch record.packedType { - case objecttype.TypeOfsDelta: - baseIdx, ok := state.offsetToRecord[record.baseOffset] - if !ok { - return objecttype.TypeInvalid, nil, &MalformedPackEntryError{ - Offset: record.offset, - Reason: "missing ofs-delta base entry", - } - } - - baseType, baseContent, err = resolveRecord(state, baseIdx, visiting) - if err != nil { - return objecttype.TypeInvalid, nil, err - } - case objecttype.TypeRefDelta: - baseIdx, ok := state.objectToRecord[record.baseObject] - if ok { - baseType, baseContent, err = resolveRecord(state, baseIdx, visiting) - if err != nil { - return objecttype.TypeInvalid, nil, err - } - } else { - return objecttype.TypeInvalid, nil, errExternalThinBase - } - case objecttype.TypeInvalid, - objecttype.TypeCommit, - objecttype.TypeTree, - objecttype.TypeBlob, - objecttype.TypeTag, - objecttype.TypeFuture: - return objecttype.TypeInvalid, nil, &MalformedPackEntryError{ - Offset: record.offset, - Reason: "unsupported delta type", - } - default: - return objecttype.TypeInvalid, nil, &MalformedPackEntryError{ - Offset: record.offset, - Reason: "unsupported delta type", - } - } - - ty, content, err := applyDeltaRecord(state, idx, baseType, baseContent) - if err != nil { - return objecttype.TypeInvalid, nil, err - } - - id, err := hashCanonicalObject(state.algo, ty, content) - if err != nil { - return objecttype.TypeInvalid, nil, err - } - - record.objectID = id - record.realType = ty - record.resolved = true - state.objectToRecord[id] = idx - state.baseCache.add(idx, ty, content) - - return ty, content, nil -} diff --git a/packfile/ingest/records.go b/packfile/ingest/records.go deleted file mode 100644 index 75f157fa..00000000 --- a/packfile/ingest/records.go +++ /dev/null @@ -1,46 +0,0 @@ -package ingest - -import ( - objectid "codeberg.org/lindenii/furgit/object/id" - objecttype "codeberg.org/lindenii/furgit/object/type" -) - -// objectRecord stores metadata for one packed object entry. -type objectRecord struct { - // offset is the entry start offset in the pack file. - offset uint64 - // headerLen is packed entry header length in bytes. - headerLen uint32 - // packedLen is total packed entry length in bytes. - packedLen uint64 - // crc32 is the CRC over the full packed entry. - crc32 uint32 - // packedType is the entry type tag from the pack stream. - packedType objecttype.Type - // realType is canonical object type after delta resolution. - realType objecttype.Type - // declaredSize is the declared output object size for this entry. - declaredSize int64 - // dataOffset is compressed payload start offset for this entry. - dataOffset uint64 - // baseOffset is OFS base offset when packedType is OFS delta. - baseOffset uint64 - // baseObject is REF base object ID when packedType is REF delta. - baseObject objectid.ObjectID - // objectID is final resolved object ID. - objectID objectid.ObjectID - // resolved reports whether objectID/realType are finalized. - resolved bool -} - -// ofsDeltaRef maps one OFS delta record to its base offset. -type ofsDeltaRef struct { - baseOffset uint64 - recordIdx int -} - -// refDeltaRef maps one REF delta record to its base object ID. -type refDeltaRef struct { - baseObject objectid.ObjectID - recordIdx int -} diff --git a/packfile/ingest/resolve_all.go b/packfile/ingest/resolve_all.go deleted file mode 100644 index e0ad2281..00000000 --- a/packfile/ingest/resolve_all.go +++ /dev/null @@ -1,71 +0,0 @@ -package ingest - -import ( - "errors" - - "codeberg.org/lindenii/furgit/internal/progress" -) - -// resolveAll resolves all delta records and finalizes ObjectID/RealType for every record. -func resolveAll(state *ingestState) error { - state.unresolvedRefDeltas = state.unresolvedRefDeltas[:0] - - var pending uint32 - - for idx := range state.records { - if !state.records[idx].resolved { - pending++ - } - } - - if pending == 0 { - return nil - } - - var done uint32 - - meter := progress.New(progress.Options{ - Writer: state.opts.Progress, - Flush: state.opts.ProgressFlush, - Title: "resolving deltas", - Total: uint64(pending), - }) - - for idx := range state.records { - if state.records[idx].resolved { - continue - } - - done++ - meter.Set(uint64(done), 0) - - visiting := make(map[int]struct{}) - - ty, content, err := resolveRecord(state, idx, visiting) - if err != nil { - if errors.Is(err, errExternalThinBase) { - state.unresolvedRefDeltas = append(state.unresolvedRefDeltas, idx) - - continue - } - - return err - } - - id, err := hashCanonicalObject(state.algo, ty, content) - if err != nil { - return err - } - - record := &state.records[idx] - record.realType = ty - record.objectID = id - record.resolved = true - state.objectToRecord[id] = idx - state.baseCache.add(idx, ty, content) - } - - meter.Stop("done") - - return nil -} diff --git a/packfile/ingest/rev_write.go b/packfile/ingest/rev_write.go deleted file mode 100644 index f8c30c1b..00000000 --- a/packfile/ingest/rev_write.go +++ /dev/null @@ -1,138 +0,0 @@ -package ingest - -import ( - "encoding/binary" - "slices" - - "codeberg.org/lindenii/furgit/internal/intconv" - "codeberg.org/lindenii/furgit/internal/progress" -) - -const ( - revMagic = 0x52494458 - revVersion = 1 -) - -// writeRev writes rev index for resolved records. -func writeRev(state *ingestState) error { - if !state.opts.WriteRev { - return nil - } - - idxOrder := buildIdxOrder(state) - - recordToIdxPos := make([]int, len(state.records)) - for pos, recordIdx := range idxOrder { - recordToIdxPos[recordIdx] = pos - } - - packOrder := buildPackOrder(state) - - hashImpl, err := state.algo.New() - if err != nil { - return err - } - - var scratch [8]byte - - writeProgressf(state, "writing reverse index header...\r") - binary.BigEndian.PutUint32(scratch[:4], revMagic) - - err = writeAndHash(state.revFile, hashImpl, scratch[:4]) - if err != nil { - return err - } - - binary.BigEndian.PutUint32(scratch[:4], revVersion) - - err = writeAndHash(state.revFile, hashImpl, scratch[:4]) - if err != nil { - return err - } - - binary.BigEndian.PutUint32(scratch[:4], state.algo.PackHashID()) - - err = writeAndHash(state.revFile, hashImpl, scratch[:4]) - if err != nil { - return err - } - - writeProgressf(state, "writing reverse index header: done.\n") - - entriesMeter := progress.New(progress.Options{ - Writer: state.opts.Progress, - Flush: state.opts.ProgressFlush, - Title: "writing reverse index entries", - Total: uint64(len(packOrder)), - }) - - var entriesDone uint64 - - for _, recordIdx := range packOrder { - recordPos, err := intconv.IntToUint32(recordToIdxPos[recordIdx]) - if err != nil { - return err - } - - binary.BigEndian.PutUint32(scratch[:4], recordPos) - - err = writeAndHash(state.revFile, hashImpl, scratch[:4]) - if err != nil { - return err - } - - entriesDone++ - entriesMeter.Set(entriesDone, 0) - } - - if entriesDone > 0 { - entriesMeter.Stop("done") - } - - writeProgressf(state, "writing reverse index trailer...\r") - - err = writeAndHash(state.revFile, hashImpl, state.packHash.Bytes()) - if err != nil { - return err - } - - revHash := hashImpl.Sum(nil) - - _, err = state.revFile.Write(revHash) - if err != nil { - return err - } - - err = state.revFile.Sync() - if err != nil { - return err - } - - writeProgressf(state, "writing reverse index trailer: done.\n") - - return nil -} - -// buildPackOrder returns record indexes sorted by pack offset. -func buildPackOrder(state *ingestState) []int { - out := make([]int, 0, len(state.records)) - for idx := range state.records { - out = append(out, idx) - } - - slices.SortFunc(out, func(a, b int) int { - offA := state.records[a].offset - - offB := state.records[b].offset - switch { - case offA < offB: - return -1 - case offA > offB: - return 1 - default: - return 0 - } - }) - - return out -} diff --git a/packfile/ingest/rewrite_header_trailer.go b/packfile/ingest/rewrite_header_trailer.go deleted file mode 100644 index f1f18a39..00000000 --- a/packfile/ingest/rewrite_header_trailer.go +++ /dev/null @@ -1,89 +0,0 @@ -package ingest - -import ( - "encoding/binary" - "io" - - "codeberg.org/lindenii/furgit/internal/intconv" - objectid "codeberg.org/lindenii/furgit/object/id" -) - -// rewritePackHeaderAndTrailer rewrites object count and trailer hash using ReadAt/WriteAt. -func rewritePackHeaderAndTrailer(state *ingestState) error { - var countRaw [4]byte - - recordCountUint32, err := intconv.IntToUint32(len(state.records)) - if err != nil { - return err - } - - binary.BigEndian.PutUint32(countRaw[:], recordCountUint32) - - _, err = state.packFile.WriteAt(countRaw[:], 8) - if err != nil { - return err - } - - info, err := state.packFile.Stat() - if err != nil { - return err - } - - endWithoutTrailer := info.Size() - - hashImpl, err := state.algo.New() - if err != nil { - return err - } - - var ( - buf [128 << 10]byte - pos int64 - ) - for pos < endWithoutTrailer { - want := int64(len(buf)) - - remaining := endWithoutTrailer - pos - if remaining < want { - want = remaining - } - - n, err := state.packFile.ReadAt(buf[:want], pos) - if err != nil && err != io.EOF { - return err - } - - if n == 0 { - return io.ErrUnexpectedEOF - } - - _, _ = hashImpl.Write(buf[:n]) - pos += int64(n) - } - - sum := hashImpl.Sum(nil) - - _, err = state.packFile.WriteAt(sum, endWithoutTrailer) - if err != nil { - return err - } - - packHash, err := objectid.FromBytes(state.algo, sum) - if err != nil { - return err - } - - state.packHash = packHash - state.objectCountHeader = recordCountUint32 - - sumLenInt64 := int64(len(sum)) - - newConsumed, err := intconv.Int64ToUint64(endWithoutTrailer + sumLenInt64) - if err != nil { - return err - } - - state.stream.consumed = newConsumed - - return nil -} diff --git a/packfile/ingest/scan.go b/packfile/ingest/scan.go deleted file mode 100644 index de4e993c..00000000 --- a/packfile/ingest/scan.go +++ /dev/null @@ -1,106 +0,0 @@ -package ingest - -import ( - "fmt" - - "codeberg.org/lindenii/furgit/internal/progress" - objectid "codeberg.org/lindenii/furgit/object/id" -) - -// streamPackAndScan copies src into temp .pack while scanning packed entries. -func streamPackAndScan(state *ingestState) error { - hashImpl, err := state.algo.New() - if err != nil { - return err - } - - state.stream = newStreamScanner( - state.src, - state.packFile, - hashImpl, - state.algo.Size(), - ) - - writeProgressf(state, "validating pack header...\r") - - err = seedStreamWithPackHeader(state) - if err != nil { - return err - } - - writeProgressf(state, "validating pack header: done.\n") - - state.records = make([]objectRecord, 0, state.objectCountHeader) - state.ofsDeltas = make([]ofsDeltaRef, 0, state.objectCountHeader) - state.refDeltas = make([]refDeltaRef, 0, state.objectCountHeader) - - total := state.objectCountHeader - meter := progress.New(progress.Options{ - Writer: state.opts.Progress, - Flush: state.opts.ProgressFlush, - Title: "receiving objects", - Total: uint64(total), - Throughput: true, - }) - - for i := range total { - nextOffset, err := scanOneEntry(state, state.stream.consumed) - if err != nil { - return err - } - - if nextOffset != state.stream.consumed { - return fmt.Errorf("packfile/ingest: internal stream offset mismatch") - } - - done := i + 1 - meter.Set(uint64(done), state.stream.consumed) - } - - meter.Stop("done") - - err = state.stream.finishAndFlushTrailer(state.opts.RequireTrailingEOF) - if err != nil { - return err - } - - if len(state.stream.packTrailer) != state.algo.Size() { - return fmt.Errorf("packfile/ingest: invalid trailer size") - } - - packHash, err := objectid.FromBytes(state.algo, state.stream.packTrailer) - if err != nil { - return err - } - - state.packHash = packHash - - return state.stream.flush() -} - -// seedStreamWithPackHeader writes the already-validated PACK header to output, -// seeds the running pack hash, and advances stream offset accounting. -func seedStreamWithPackHeader(state *ingestState) error { - written := 0 - for written < len(state.packHeaderRaw) { - n, err := state.packFile.Write(state.packHeaderRaw[written:]) - if err != nil { - return &DestinationWriteError{Op: fmt.Sprintf("write pack header: %v", err)} - } - - if n == 0 { - return &DestinationWriteError{Op: "write pack header: short write"} - } - - written += n - } - - _, err := state.stream.hash.Write(state.packHeaderRaw[:]) - if err != nil { - return err - } - - state.stream.consumed = packHeaderSize - - return nil -} diff --git a/packfile/ingest/state.go b/packfile/ingest/state.go deleted file mode 100644 index 797323b2..00000000 --- a/packfile/ingest/state.go +++ /dev/null @@ -1,70 +0,0 @@ -package ingest - -import ( - "io" - "os" - - objectid "codeberg.org/lindenii/furgit/object/id" -) - -const ( - defaultDeltaBaseCacheMaxBytes = 32 << 20 -) - -// ingestState holds mutable state for one Ingest call. -type ingestState struct { - src io.Reader - destination *os.Root - algo objectid.Algorithm - opts Options - - packHeaderRaw [packHeaderSize]byte - - packFile *os.File - packTmpName string - idxFile *os.File - idxTmpName string - revFile *os.File - revTmpName string - - stream *streamScanner - - records []objectRecord - ofsDeltas []ofsDeltaRef - refDeltas []refDeltaRef - unresolvedRefDeltas []int - offsetToRecord map[uint64]int - objectToRecord map[objectid.ObjectID]int - - baseCache *deltaBaseCache - packHash objectid.ObjectID - - objectCountHeader uint32 - thinFixed bool -} - -// newIngestState constructs one call-local ingest state. -func newIngestState( - src io.Reader, - destination *os.Root, - algo objectid.Algorithm, - opts Options, - header HeaderInfo, - headerRaw [packHeaderSize]byte, -) (*ingestState, error) { - if algo.Size() == 0 { - return nil, objectid.ErrInvalidAlgorithm - } - - return &ingestState{ - src: src, - destination: destination, - algo: algo, - opts: opts, - packHeaderRaw: headerRaw, - objectCountHeader: header.ObjectCount, - offsetToRecord: make(map[uint64]int), - objectToRecord: make(map[objectid.ObjectID]int), - baseCache: newDeltaBaseCache(defaultDeltaBaseCacheMaxBytes), - }, nil -} diff --git a/packfile/ingest/stream.go b/packfile/ingest/stream.go deleted file mode 100644 index a403087a..00000000 --- a/packfile/ingest/stream.go +++ /dev/null @@ -1,111 +0,0 @@ -package ingest - -import ( - "errors" - "hash" - "io" - "os" -) - -const streamScannerBufferSize = 64 << 10 - -// streamScanner incrementally reads/consumes one pack stream while mirroring -// consumed bytes into one destination pack file. -type streamScanner struct { - src io.Reader - dstFile *os.File - - // Input buffer window: buf[off:n] is unread. - buf []byte - off int - n int - - // Absolute consumed stream bytes. - consumed uint64 - - // Running pack hash over consumed bytes while hashEnabled is true. - hash hash.Hash - hashSize int - hashEnabled bool - - // Entry CRC state while one entry is being consumed. - entryCRC uint32 - inEntryCRC bool - - packTrailer []byte -} - -// newStreamScanner constructs one scanner with fixed input buffering. -func newStreamScanner(src io.Reader, dstFile *os.File, hash hash.Hash, hashSize int) *streamScanner { - return &streamScanner{ - src: src, - dstFile: dstFile, - buf: make([]byte, streamScannerBufferSize), - hash: hash, - hashSize: hashSize, - hashEnabled: true, - } -} - -// Read implements io.Reader. -func (scanner *streamScanner) Read(dst []byte) (int, error) { - if len(dst) == 0 { - return 0, nil - } - - if scanner.n-scanner.off == 0 { - err := scanner.fill(1) - if err != nil { - if errors.Is(err, io.EOF) { - return 0, io.EOF - } - - return 0, err - } - } - - unread := scanner.n - scanner.off - if unread == 0 { - return 0, io.EOF - } - - n := min(len(dst), unread) - - copy(dst, scanner.buf[scanner.off:scanner.off+n]) - - err := scanner.use(n) - if err != nil { - return 0, err - } - - return n, nil -} - -// ReadByte implements io.ByteReader without allocation. -func (scanner *streamScanner) ReadByte() (byte, error) { - if scanner.n-scanner.off == 0 { - err := scanner.fill(1) - if err != nil { - return 0, err - } - } - - b := scanner.buf[scanner.off] - - err := scanner.use(1) - if err != nil { - return 0, err - } - - return b, nil -} - -// readFull reads exactly len(dst) bytes through receiver. -func (scanner *streamScanner) readFull(dst []byte) error { - _, err := io.ReadFull(scanner, dst) - if err != nil { - return err - } - - return nil -} diff --git a/packfile/ingest/temp.go b/packfile/ingest/temp.go deleted file mode 100644 index d0b7862c..00000000 --- a/packfile/ingest/temp.go +++ /dev/null @@ -1,103 +0,0 @@ -package ingest - -import ( - "crypto/rand" - "errors" - "fmt" - "io/fs" - "os" -) - -// openTemporaryArtifacts creates/open temp pack/idx/(rev) files under destination. -func openTemporaryArtifacts(state *ingestState) error { - packName, packFile, err := createTempFile(state.destination, "tmp_pack_") - if err != nil { - return err - } - - idxName, idxFile, err := createTempFile(state.destination, "tmp_idx_") - if err != nil { - _ = packFile.Close() - _ = state.destination.Remove(packName) - - return err - } - - revName := "" - - var revFile *os.File - if state.opts.WriteRev { - revName, revFile, err = createTempFile(state.destination, "tmp_rev_") - if err != nil { - _ = idxFile.Close() - _ = state.destination.Remove(idxName) - _ = packFile.Close() - _ = state.destination.Remove(packName) - - return err - } - } - - state.packTmpName = packName - state.packFile = packFile - state.idxTmpName = idxName - state.idxFile = idxFile - state.revTmpName = revName - state.revFile = revFile - - return nil -} - -// closeTemporaryArtifacts closes all temporary artifact file descriptors. -func closeTemporaryArtifacts(state *ingestState) error { - var out error - - if state.packFile != nil { - err := state.packFile.Close() - if err != nil && out == nil { - out = err - } - - state.packFile = nil - } - - if state.idxFile != nil { - err := state.idxFile.Close() - if err != nil && out == nil { - out = err - } - - state.idxFile = nil - } - - if state.revFile != nil { - err := state.revFile.Close() - if err != nil && out == nil { - out = err - } - - state.revFile = nil - } - - return out -} - -// createTempFile creates one temporary file under root using prefix. -func createTempFile(root *os.Root, prefix string) (string, *os.File, error) { - for range 32 { - name := prefix + rand.Text() - - file, err := root.OpenFile(name, os.O_CREATE|os.O_EXCL|os.O_RDWR, 0o644) - if err == nil { - return name, file, nil - } - - if errors.Is(err, fs.ErrExist) { - continue - } - - return "", nil, fmt.Errorf("packfile/ingest: create temp file %q: %w", name, err) - } - - return "", nil, fmt.Errorf("packfile/ingest: unable to create temporary file for prefix %q", prefix) -} diff --git a/packfile/ingest/testdata/fixtures/sha1/METADATA.txt b/packfile/ingest/testdata/fixtures/sha1/METADATA.txt deleted file mode 100644 index 5fcbfe26..00000000 --- a/packfile/ingest/testdata/fixtures/sha1/METADATA.txt +++ /dev/null @@ -1,3 +0,0 @@ -format=sha1 -head=200c960359dad025b4170284c518919eb4a24305 -base=4bc507fc631ea78474d83c47548743c9f1dda0dc diff --git a/packfile/ingest/testdata/fixtures/sha1/base.pack b/packfile/ingest/testdata/fixtures/sha1/base.pack deleted file mode 100644 index 3d7a4903..00000000 Binary files a/packfile/ingest/testdata/fixtures/sha1/base.pack and /dev/null differ diff --git a/packfile/ingest/testdata/fixtures/sha1/nonthin.pack b/packfile/ingest/testdata/fixtures/sha1/nonthin.pack deleted file mode 100644 index ea07c9a0..00000000 Binary files a/packfile/ingest/testdata/fixtures/sha1/nonthin.pack and /dev/null differ diff --git a/packfile/ingest/testdata/fixtures/sha1/thin.pack b/packfile/ingest/testdata/fixtures/sha1/thin.pack deleted file mode 100644 index 95084feb..00000000 Binary files a/packfile/ingest/testdata/fixtures/sha1/thin.pack and /dev/null differ diff --git a/packfile/ingest/testdata/fixtures/sha256/METADATA.txt b/packfile/ingest/testdata/fixtures/sha256/METADATA.txt deleted file mode 100644 index 8a5ea0a2..00000000 --- a/packfile/ingest/testdata/fixtures/sha256/METADATA.txt +++ /dev/null @@ -1,3 +0,0 @@ -format=sha256 -head=35cc0f4cd1c73524187540494058d233a2ecbd071c85d496a2250d8e0c805ef8 -base=b4abe46895f0bb5aa22fd42d28d428413f265359734c288752e3c2d270eec276 diff --git a/packfile/ingest/testdata/fixtures/sha256/base.pack b/packfile/ingest/testdata/fixtures/sha256/base.pack deleted file mode 100644 index 52ceef74..00000000 Binary files a/packfile/ingest/testdata/fixtures/sha256/base.pack and /dev/null differ diff --git a/packfile/ingest/testdata/fixtures/sha256/nonthin.pack b/packfile/ingest/testdata/fixtures/sha256/nonthin.pack deleted file mode 100644 index 50db05d0..00000000 Binary files a/packfile/ingest/testdata/fixtures/sha256/nonthin.pack and /dev/null differ diff --git a/packfile/ingest/testdata/fixtures/sha256/thin.pack b/packfile/ingest/testdata/fixtures/sha256/thin.pack deleted file mode 100644 index b331b915..00000000 Binary files a/packfile/ingest/testdata/fixtures/sha256/thin.pack and /dev/null differ diff --git a/packfile/ingest/thin_append.go b/packfile/ingest/thin_append.go deleted file mode 100644 index 779d477f..00000000 --- a/packfile/ingest/thin_append.go +++ /dev/null @@ -1,91 +0,0 @@ -package ingest - -import ( - "compress/zlib" - "hash/crc32" - "io" - - "codeberg.org/lindenii/furgit/internal/intconv" - objectid "codeberg.org/lindenii/furgit/object/id" - objecttype "codeberg.org/lindenii/furgit/object/type" -) - -// appendBaseObject appends one base object as a new packed non-delta entry. -func appendBaseObject(state *ingestState, id objectid.ObjectID, realType objecttype.Type, content []byte) (int, error) { - start := state.stream.consumed - - header := encodePackEntryHeader(realType, int64(len(content))) - - startInt64, err := intconv.Uint64ToInt64(start) - if err != nil { - return 0, err - } - - _, err = state.packFile.WriteAt(header, startInt64) - if err != nil { - return 0, err - } - - headerLenInt64 := int64(len(header)) - section := &fileSectionWriter{file: state.packFile, off: startInt64 + headerLenInt64} - crc := crc32.NewIEEE() - - _, err = crc.Write(header) - if err != nil { - return 0, err - } - - counting := &countingWriter{dst: section} - - zw := zlib.NewWriter(io.MultiWriter(counting, crc)) - - _, err = zw.Write(content) - if err != nil { - return 0, err - } - - err = zw.Close() - if err != nil { - return 0, err - } - - headerLenUint64, err := intconv.IntToUint64(len(header)) - if err != nil { - return 0, err - } - - countingNUint64, err := intconv.IntToUint64(counting.n) - if err != nil { - return 0, err - } - - packedLen := headerLenUint64 + countingNUint64 - end := start + packedLen - state.stream.consumed = end - - headerLenUint32, err := intconv.IntToUint32(len(header)) - if err != nil { - return 0, err - } - - record := objectRecord{ - offset: start, - headerLen: headerLenUint32, - packedLen: packedLen, - crc32: crc.Sum32(), - packedType: realType, - realType: realType, - declaredSize: int64(len(content)), - dataOffset: start + headerLenUint64, - objectID: id, - resolved: true, - } - - recordIdx := len(state.records) - state.records = append(state.records, record) - state.offsetToRecord[start] = recordIdx - state.objectToRecord[id] = recordIdx - state.baseCache.add(recordIdx, realType, content) - - return recordIdx, nil -} diff --git a/packfile/ingest/thin_fix.go b/packfile/ingest/thin_fix.go deleted file mode 100644 index 83e5572a..00000000 --- a/packfile/ingest/thin_fix.go +++ /dev/null @@ -1,100 +0,0 @@ -package ingest - -import ( - "errors" - "fmt" - - "codeberg.org/lindenii/furgit/internal/intconv" - "codeberg.org/lindenii/furgit/internal/progress" - objectstorer "codeberg.org/lindenii/furgit/object/storer" -) - -// maybeFixThin appends missing bases and rewrites pack header/trailer when needed. -func maybeFixThin(state *ingestState) error { - if len(state.unresolvedRefDeltas) == 0 { - return nil - } - - writeProgressf( - state, - "fixing thin pack: %d unresolved bases\r", - len(state.unresolvedRefDeltas), - ) - - if !state.opts.FixThin { - return &ThinPackUnresolvedError{Count: len(state.unresolvedRefDeltas)} - } - - if state.opts.Base == nil { - return &ThinPackUnresolvedError{Count: len(state.unresolvedRefDeltas)} - } - - hashSize := int64(state.algo.Size()) - - info, err := state.packFile.Stat() - if err != nil { - return err - } - - size := info.Size() - if size < hashSize { - return fmt.Errorf("packfile/ingest: pack too short to trim trailer") - } - - newEnd := size - hashSize - - err = state.packFile.Truncate(newEnd) - if err != nil { - return err - } - - consumed, err := intconv.Int64ToUint64(newEnd) - if err != nil { - return err - } - - state.stream.consumed = consumed - - baseIDs := unresolvedThinBaseIDs(state) - - total := len(baseIDs) - meter := progress.New(progress.Options{ - Writer: state.opts.Progress, - Flush: state.opts.ProgressFlush, - Title: "fixing thin pack", - Total: uint64(total), - }) - meter.Set(0, 0) - - var appended uint64 - - for _, id := range baseIDs { - ty, content, err := state.opts.Base.ReadBytesContent(id) - if err != nil { - if errors.Is(err, objectstorer.ErrObjectNotFound) { - continue - } - - return fmt.Errorf("packfile/ingest: read thin base %s: %w", id, err) - } - - _, err = appendBaseObject(state, id, ty, content) - if err != nil { - return err - } - - state.thinFixed = true - - appended++ - meter.Set(appended, 0) - } - - err = rewritePackHeaderAndTrailer(state) - if err != nil { - return err - } - - meter.Stop(fmt.Sprintf("appended %d/%d, done", appended, total)) - - return nil -} diff --git a/packfile/ingest/thin_unresolved.go b/packfile/ingest/thin_unresolved.go deleted file mode 100644 index 757cc0e2..00000000 --- a/packfile/ingest/thin_unresolved.go +++ /dev/null @@ -1,34 +0,0 @@ -package ingest - -import ( - "bytes" - "slices" - - objectid "codeberg.org/lindenii/furgit/object/id" - objecttype "codeberg.org/lindenii/furgit/object/type" -) - -// unresolvedThinBaseIDs returns sorted unique unresolved ref base IDs. -func unresolvedThinBaseIDs(state *ingestState) []objectid.ObjectID { - seen := make(map[objectid.ObjectID]struct{}) - - for _, idx := range state.unresolvedRefDeltas { - record := state.records[idx] - if record.packedType != objecttype.TypeRefDelta { - continue - } - - seen[record.baseObject] = struct{}{} - } - - out := make([]objectid.ObjectID, 0, len(seen)) - for id := range seen { - out = append(out, id) - } - - slices.SortFunc(out, func(a, b objectid.ObjectID) int { - return bytes.Compare(a.RawBytes(), b.RawBytes()) - }) - - return out -} diff --git a/packfile/ingest/trailer.go b/packfile/ingest/trailer.go deleted file mode 100644 index 7a26a8f2..00000000 --- a/packfile/ingest/trailer.go +++ /dev/null @@ -1,58 +0,0 @@ -package ingest - -import ( - "bytes" - "errors" - "fmt" - "io" -) - -// finishAndFlushTrailer reads trailer hash bytes, verifies trailer checksum, -// and optionally requires the source stream to hit EOF afterward. -func (scanner *streamScanner) finishAndFlushTrailer(requireTrailingEOF bool) error { - if scanner.hashSize <= 0 { - return fmt.Errorf("packfile/ingest: invalid hash size") - } - - trailer := make([]byte, scanner.hashSize) - - scanner.hashEnabled = false - - err := scanner.readFull(trailer) - if err != nil { - return &PackTrailerMismatchError{} - } - - scanner.packTrailer = append(scanner.packTrailer[:0], trailer...) - - if scanner.n-scanner.off > 0 { - return fmt.Errorf("packfile/ingest: pack has trailing garbage") - } - - if !requireTrailingEOF { - computed := scanner.hash.Sum(nil) - if !bytes.Equal(computed, trailer) { - return &PackTrailerMismatchError{} - } - - return nil - } - - var probe [1]byte - - n, err := scanner.Read(probe[:]) - if n > 0 || err == nil { - return fmt.Errorf("packfile/ingest: pack has trailing garbage") - } - - if !errors.Is(err, io.EOF) { - return err - } - - computed := scanner.hash.Sum(nil) - if !bytes.Equal(computed, trailer) { - return &PackTrailerMismatchError{} - } - - return nil -} diff --git a/packfile/ingest/use.go b/packfile/ingest/use.go deleted file mode 100644 index 97f8757a..00000000 --- a/packfile/ingest/use.go +++ /dev/null @@ -1,34 +0,0 @@ -package ingest - -import ( - "fmt" - "hash/crc32" -) - -// use consumes n unread bytes and updates accounting/checksum state. -func (scanner *streamScanner) use(n int) error { - if n < 0 || n > scanner.n-scanner.off { - return fmt.Errorf("packfile/ingest: invalid consume length %d", n) - } - - if n == 0 { - return nil - } - - chunk := scanner.buf[scanner.off : scanner.off+n] - if scanner.hashEnabled { - _, err := scanner.hash.Write(chunk) - if err != nil { - return err - } - } - - if scanner.inEntryCRC { - scanner.entryCRC = crc32.Update(scanner.entryCRC, crc32.IEEETable, chunk) - } - - scanner.off += n - scanner.consumed += uint64(n) - - return nil -} diff --git a/packfile/object_type.go b/packfile/object_type.go deleted file mode 100644 index 8382baa9..00000000 --- a/packfile/object_type.go +++ /dev/null @@ -1,16 +0,0 @@ -package packfile - -import objecttype "codeberg.org/lindenii/furgit/object/type" - -// IsBaseObjectType reports whether ty is one of the four canonical object -// types encoded directly in pack entries. -func IsBaseObjectType(ty objecttype.Type) bool { - switch ty { - case objecttype.TypeCommit, objecttype.TypeTree, objecttype.TypeBlob, objecttype.TypeTag: - return true - case objecttype.TypeInvalid, objecttype.TypeFuture, objecttype.TypeOfsDelta, objecttype.TypeRefDelta: - return false - default: - return false - } -} diff --git a/packfile/ofs.go b/packfile/ofs.go deleted file mode 100644 index 4992a506..00000000 --- a/packfile/ofs.go +++ /dev/null @@ -1,26 +0,0 @@ -package packfile - -import "fmt" - -// ParseOfsDeltaDistance parses one ofs-delta backward distance. -func ParseOfsDeltaDistance(buf []byte) (uint64, int, error) { - if len(buf) == 0 { - return 0, 0, fmt.Errorf("packfile: malformed ofs-delta distance") - } - - b := buf[0] - dist := uint64(b & 0x7f) - - consumed := 1 - for b&0x80 != 0 { - if consumed >= len(buf) { - return 0, 0, fmt.Errorf("packfile: malformed ofs-delta distance") - } - - b = buf[consumed] - consumed++ - dist = ((dist + 1) << 7) + uint64(b&0x7f) - } - - return dist, consumed, nil -} diff --git a/reachability/reachability.go b/reachability/reachability.go index 77a844a7..31947525 100644 --- a/reachability/reachability.go +++ b/reachability/reachability.go @@ -2,7 +2,7 @@ package reachability import ( - commitgraphread "codeberg.org/lindenii/furgit/commitgraph/read" + commitgraphread "codeberg.org/lindenii/furgit/format/commitgraph/read" objectstorer "codeberg.org/lindenii/furgit/object/storer" ) diff --git a/reachability/walk_expand_commits_graph.go b/reachability/walk_expand_commits_graph.go index 447879d8..863906f9 100644 --- a/reachability/walk_expand_commits_graph.go +++ b/reachability/walk_expand_commits_graph.go @@ -3,7 +3,7 @@ package reachability import ( "errors" - commitgraphread "codeberg.org/lindenii/furgit/commitgraph/read" + commitgraphread "codeberg.org/lindenii/furgit/format/commitgraph/read" objectid "codeberg.org/lindenii/furgit/object/id" objecttype "codeberg.org/lindenii/furgit/object/type" ) -- cgit v1.3.1-10-gc9f91