From: Johannes Schindelin via GitGitGadget Date: Tue, 16 Jun 2026 14:49:51 GMT Subject: [PATCH v2 0/6] Support hashing objects larger than 4GB on Windows Message-ID: In-Reply-To: Philip Oakley has contributed these patches ~4.5 years ago, and they have been carried in Git for Windows ever since. Now that there are already other patch series flying around that try to address various aspects about >4GB objects (which aren't handled well by Git until it stops forcing unsigned long to do size_t's job), it seems a good time to upstream these patches, too, at long last. Changes since v1: * Rebased to current master to resolve the conflicts with ps/odb-source-loose * Dropped the !LONG_IS_64BIT prereq from the added/touched tests, as it is now no longer needed Philip Oakley (6): hash-object: demonstrate a >4GB/LLP64 problem object-file.c: use size_t for header lengths hash algorithms: use size_t for section lengths hash-object --stdin: verify that it works with >4GB/LLP64 hash-object: add another >4GB/LLP64 test case hash-object: add a >4GB/LLP64 test case using filtered input object-file.c | 14 +++++++------- object-file.h | 6 +++--- odb/source-files.c | 2 +- odb/source-inmemory.c | 2 +- odb/source-loose.c | 4 ++-- odb/source.h | 2 +- sha1dc_git.c | 3 +-- sha1dc_git.h | 2 +- t/t1007-hash-object.sh | 39 +++++++++++++++++++++++++++++++++++++++ 9 files changed, 56 insertions(+), 18 deletions(-) base-commit: 700432b2ba22603a0bcb71475c9c333d17c9b0d1 Published-As: https://github.com/gitgitgadget/git/releases/tag/pr-2138%2Fdscho%2FPhilipOakley%2Fhashliteral_t-v2 Fetch-It-Via: git fetch https://github.com/gitgitgadget/git pr-2138/dscho/PhilipOakley/hashliteral_t-v2 Pull-Request: https://github.com/gitgitgadget/git/pull/2138 Range-diff vs v1: 1: 84e1cd0aa0 = 1: 9c01bac407 hash-object: demonstrate a >4GB/LLP64 problem 2: 809d83e46f ! 2: aa5859c14f object-file.c: use size_t for header lengths @@ Commit message Signed-off-by: Johannes Schindelin ## object-file.c ## -@@ object-file.c: int odb_source_loose_read_object_info(struct odb_source *source, +@@ object-file.c: int parse_loose_header(const char *hdr, struct object_info *oi) static void hash_object_body(const struct git_hash_algo *algo, struct git_hash_ctx *c, const void *buf, unsigned long len, struct object_id *oid, @@ object-file.c: int odb_source_loose_read_object_info(struct odb_source *source, @@ object-file.c: static void hash_object_body(const struct git_hash_algo *algo, struct git_hash_c } - static void write_object_file_prepare(const struct git_hash_algo *algo, -- const void *buf, unsigned long len, -+ const void *buf, size_t len, - enum object_type type, struct object_id *oid, -- char *hdr, int *hdrlen) -+ char *hdr, size_t *hdrlen) + void write_object_file_prepare(const struct git_hash_algo *algo, +- const void *buf, unsigned long len, ++ const void *buf, size_t len, + enum object_type type, struct object_id *oid, +- char *hdr, int *hdrlen) ++ char *hdr, size_t *hdrlen) { struct git_hash_ctx c; @@ object-file.c: out: write_object_file_prepare(algo, buf, len, type, oid, hdr, &hdrlen); } -@@ object-file.c: cleanup: + + ## object-file.h ## +@@ object-file.h: int finalize_object_file_flags(struct repository *repo, + enum finalize_object_file_flags flags); + + void hash_object_file(const struct git_hash_algo *algo, const void *buf, +- unsigned long len, enum object_type type, ++ size_t len, enum object_type type, + struct object_id *oid); + void write_object_file_prepare(const struct git_hash_algo *algo, +- const void *buf, unsigned long len, ++ const void *buf, size_t len, + enum object_type type, struct object_id *oid, +- char *hdr, int *hdrlen); ++ char *hdr, size_t *hdrlen); + int write_loose_object(struct odb_source_loose *loose, + const struct object_id *oid, char *hdr, + int hdrlen, const void *buf, unsigned long len, + + ## odb/source-files.c ## +@@ odb/source-files.c: static int odb_source_files_freshen_object(struct odb_source *source, + } + + static int odb_source_files_write_object(struct odb_source *source, +- const void *buf, unsigned long len, ++ const void *buf, size_t len, + enum object_type type, + struct object_id *oid, + struct object_id *compat_oid, + + ## odb/source-inmemory.c ## +@@ odb/source-inmemory.c: static int odb_source_inmemory_count_objects(struct odb_source *source, } - int odb_source_loose_write_object(struct odb_source *source, -- const void *buf, unsigned long len, -+ const void *buf, size_t len, - enum object_type type, struct object_id *oid, - struct object_id *compat_oid_in, - enum odb_write_object_flags flags) -@@ object-file.c: int odb_source_loose_write_object(struct odb_source *source, + static int odb_source_inmemory_write_object(struct odb_source *source, +- const void *buf, unsigned long len, ++ const void *buf, size_t len, + enum object_type type, + struct object_id *oid, + struct object_id *compat_oid UNUSED, + + ## odb/source-loose.c ## +@@ odb/source-loose.c: static int odb_source_loose_freshen_object(struct odb_source *source, + } + + static int odb_source_loose_write_object(struct odb_source *source, +- const void *buf, unsigned long len, ++ const void *buf, size_t len, + enum object_type type, struct object_id *oid, + struct object_id *compat_oid_in, + enum odb_write_object_flags flags) +@@ odb/source-loose.c: static int odb_source_loose_write_object(struct odb_source *source, const struct git_hash_algo *compat = source->odb->repo->compat_hash_algo; struct object_id compat_oid; char hdr[MAX_HEADER_LEN]; @@ object-file.c: int odb_source_loose_write_object(struct odb_source *source, /* Generate compat_oid */ if (compat) { - ## object-file.h ## -@@ object-file.h: int odb_source_loose_freshen_object(struct odb_source *source, - const struct object_id *oid); - - int odb_source_loose_write_object(struct odb_source *source, -- const void *buf, unsigned long len, -+ const void *buf, size_t len, - enum object_type type, struct object_id *oid, - struct object_id *compat_oid_in, - enum odb_write_object_flags flags); -@@ object-file.h: int finalize_object_file_flags(struct repository *repo, - enum finalize_object_file_flags flags); - - void hash_object_file(const struct git_hash_algo *algo, const void *buf, -- unsigned long len, enum object_type type, -+ size_t len, enum object_type type, - struct object_id *oid); - - /* Helper to check and "touch" a file */ + ## odb/source.h ## +@@ odb/source.h: struct odb_source { + * return 0 on success, a negative error code otherwise. + */ + int (*write_object)(struct odb_source *source, +- const void *buf, unsigned long len, ++ const void *buf, size_t len, + enum object_type type, + struct object_id *oid, + struct object_id *compat_oid, 3: 253d6f8004 ! 3: b401eb490f hash algorithms: use size_t for section lengths @@ Commit message Signed-off-by: Johannes Schindelin ## object-file.c ## -@@ object-file.c: int odb_source_loose_read_object_info(struct odb_source *source, +@@ object-file.c: int parse_loose_header(const char *hdr, struct object_info *oi) } static void hash_object_body(const struct git_hash_algo *algo, struct git_hash_ctx *c, @@ object-file.c: int odb_source_loose_read_object_info(struct odb_source *source, struct object_id *oid, char *hdr, size_t *hdrlen) { -@@ object-file.c: static void write_object_file_prepare(const struct git_hash_algo *algo, +@@ object-file.c: void write_object_file_prepare(const struct git_hash_algo *algo, /* Generate the header */ *hdrlen = format_object_header(hdr, *hdrlen, type, len); @@ t/t1007-hash-object.sh: test_expect_success '--stdin outside of repository (uses ' -test_expect_failure EXPENSIVE,SIZE_T_IS_64BIT,!LONG_IS_64BIT \ -+test_expect_success EXPENSIVE,SIZE_T_IS_64BIT,!LONG_IS_64BIT \ ++test_expect_success EXPENSIVE,SIZE_T_IS_64BIT \ 'files over 4GB hash literally' ' test-tool genzeros $((5*1024*1024*1024)) >big && test_oid large5GB >expect && 4: ba629a3f03 ! 4: 411727336a hash-object --stdin: verify that it works with >4GB/LLP64 @@ Commit message Signed-off-by: Johannes Schindelin ## t/t1007-hash-object.sh ## -@@ t/t1007-hash-object.sh: test_expect_success EXPENSIVE,SIZE_T_IS_64BIT,!LONG_IS_64BIT \ +@@ t/t1007-hash-object.sh: test_expect_success EXPENSIVE,SIZE_T_IS_64BIT \ test_cmp expect actual ' -+test_expect_success EXPENSIVE,SIZE_T_IS_64BIT,!LONG_IS_64BIT \ ++test_expect_success EXPENSIVE,SIZE_T_IS_64BIT \ + 'files over 4GB hash correctly via --stdin' ' + { test -f big || test-tool genzeros $((5*1024*1024*1024)) >big; } && + test_oid large5GB >expect && 5: f48d570bba ! 5: e6bb4e6228 hash-object: add another >4GB/LLP64 test case @@ Commit message Signed-off-by: Johannes Schindelin ## t/t1007-hash-object.sh ## -@@ t/t1007-hash-object.sh: test_expect_success EXPENSIVE,SIZE_T_IS_64BIT,!LONG_IS_64BIT \ +@@ t/t1007-hash-object.sh: test_expect_success EXPENSIVE,SIZE_T_IS_64BIT \ test_cmp expect actual ' -+test_expect_success EXPENSIVE,SIZE_T_IS_64BIT,!LONG_IS_64BIT \ ++test_expect_success EXPENSIVE,SIZE_T_IS_64BIT \ + 'files over 4GB hash correctly' ' + { test -f big || test-tool genzeros $((5*1024*1024*1024)) >big; } && + test_oid large5GB >expect && 6: 8a6beeb16d ! 6: 568807ac34 hash-object: add a >4GB/LLP64 test case using filtered input @@ Commit message Signed-off-by: Johannes Schindelin ## t/t1007-hash-object.sh ## -@@ t/t1007-hash-object.sh: test_expect_success EXPENSIVE,SIZE_T_IS_64BIT,!LONG_IS_64BIT \ +@@ t/t1007-hash-object.sh: test_expect_success EXPENSIVE,SIZE_T_IS_64BIT \ test_cmp expect actual ' +# This clean filter does nothing, other than excercising the interface. +# We ensure that cleaning doesn't mangle large files on 64-bit Windows. -+test_expect_success EXPENSIVE,SIZE_T_IS_64BIT,!LONG_IS_64BIT \ ++test_expect_success EXPENSIVE,SIZE_T_IS_64BIT \ + 'hash filtered files over 4GB correctly' ' + { test -f big || test-tool genzeros $((5*1024*1024*1024)) >big; } && + test_oid large5GB >expect && -- gitgitgadget