From: Han Xin <hanxin.hx@alibaba-inc.com>
Changes since v3:
* Add "size" to "struct input_stream" which used by following commits.
* Increase the buffer size of "struct input_zstream_data" from 4096 to
8192, which is consistent with the "fixed_buf" in the "index-pack.c".
* Refactor "read stream in a loop in write_loose_object()" which
introduced a performance problem reported by Derrick Stolee[1].
* Rewrite benchmarks in "unpack-objects: unpack_non_delta_entry() read
data in a stream" with sugguestions by Derrick Stolee[1] and
Ævar Arnfjörð Bjarmason[2].
Now use "scalar.git" to benchmark, which contains more than 28000
objects and 96 objects larger than 16kB.
1. https://lore.kernel.org/git/8ff89e50-1b80-7932-f0e2-af401ee04bb1@gmail.com/
2. https://lore.kernel.org/git/211201.86r1aw9gbd.gmgdl@evledraar.gmail.com/
Han Xin (5):
object-file: refactor write_loose_object() to read buffer from stream
object-file.c: handle undetermined oid in write_loose_object()
object-file.c: read stream in a loop in write_loose_object()
unpack-objects.c: add dry_run mode for get_data()
unpack-objects: unpack_non_delta_entry() read data in a stream
builtin/unpack-objects.c | 93 +++++++++++++++++++++++--
object-file.c | 102 ++++++++++++++++++++++++----
object-store.h | 10 +++
t/t5590-unpack-non-delta-objects.sh | 76 +++++++++++++++++++++
4 files changed, 262 insertions(+), 19 deletions(-)
create mode 100755 t/t5590-unpack-non-delta-objects.sh
Range-diff against v3:
1: 8640b04f6d ! 1: af707ef304 object-file: refactor write_loose_object() to read buffer from stream
@@ object-file.c: int write_object_file_flags(const void *buf, unsigned long len,
+ .buf = buf,
+ .len = len,
+ },
++ .size = len,
+ };
/* Normally if we have it in the pack then we do not bother writing
@@ object-file.c: int hash_object_file_literally(const void *buf, unsigned long len
+ .buf = buf,
+ .len = len,
+ },
++ .size = len,
+ };
/* type string, SP, %lu of the length plus NUL must fit this */
@@ object-file.c: int force_object_loose(const struct object_id *oid, time_t mtime)
if (has_loose_object(oid))
return 0;
buf = read_object(the_repository, oid, &type, &len);
++ in_stream.size = len;
if (!buf)
return error(_("cannot read object for %s"), oid_to_hex(oid));
+ data.buf = buf;
@@ object-store.h: struct object_directory {
+struct input_stream {
+ const void *(*read)(struct input_stream *, unsigned long *len);
+ void *data;
++ size_t size;
+};
+
KHASH_INIT(odb_path_map, const char * /* key: odb_path */,
2: d4a2caf2bd = 2: 321ad90d8e object-file.c: handle undetermined oid in write_loose_object()
3: 2575900449 ! 3: 1992ac39af object-file.c: read stream in a loop in write_loose_object()
@@ object-file.c: static int write_loose_object(const struct object_id *oid, char *
- ret = git_deflate(&stream, Z_FINISH);
+ if (!stream.avail_in) {
+ buf = in_stream->read(in_stream, &stream.avail_in);
-+ if (buf) {
-+ stream.next_in = (void *)buf;
-+ in0 = (unsigned char *)buf;
-+ } else {
++ stream.next_in = (void *)buf;
++ in0 = (unsigned char *)buf;
++ /* All data has been read. */
++ if (in_stream->size + hdrlen == stream.total_in + stream.avail_in)
+ flush = Z_FINISH;
-+ }
+ }
+ ret = git_deflate(&stream, flush);
the_hash_algo->update_fn(&c, in0, stream.next_in - in0);
if (write_buffer(fd, compressed, stream.next_out - compressed) < 0)
die(_("unable to write loose object file"));
+ stream.next_out = compressed;
+ stream.avail_out = sizeof(compressed);
+- } while (ret == Z_OK);
++ } while (ret == Z_OK || ret == Z_BUF_ERROR);
+
+ if (ret != Z_STREAM_END)
+ die(_("unable to deflate new object %s (%d)"), oid_to_hex(oid),
4: ca93ecc780 = 4: c41eb06533 unpack-objects.c: add dry_run mode for get_data()
5: 39a072ee2a ! 5: 9427775bdc unpack-objects: unpack_non_delta_entry() read data in a stream
@@ Commit message
larger than the "big_file_threshold" in zstream. See the following
benchmarks:
- $ hyperfine \
- --prepare 'rm -rf dest.git && git init --bare dest.git' \
- 'git -C dest.git unpack-objects <binary_320M.pack'
- Benchmark 1: git -C dest.git unpack-objects <binary_320M.pack
- Time (mean ± σ): 10.029 s ± 0.270 s [User: 8.265 s, System: 1.522 s]
- Range (min … max): 9.786 s … 10.603 s 10 runs
+ hyperfine \
+ --setup \
+ 'if ! test -d scalar.git; then git clone --bare https://github.com/microsoft/scalar.git; cp scalar.git/objects/pack/*.pack small.pack; fi' \
+ --prepare 'rm -rf dest.git && git init --bare dest.git' \
+ -n 'old' 'git -C dest.git unpack-objects <small.pack' \
+ -n 'new' 'new/git -C dest.git unpack-objects <small.pack' \
+ -n 'new (small threshold)' \
+ 'new/git -c core.bigfilethreshold=16k -C dest.git unpack-objects <small.pack'
+ Benchmark 1: old
+ Time (mean ± σ): 6.075 s ± 0.069 s [User: 5.047 s, System: 0.991 s]
+ Range (min … max): 6.018 s … 6.189 s 10 runs
- $ hyperfine \
- --prepare 'rm -rf dest.git && git init --bare dest.git' \
- 'git -c core.bigFileThreshold=2m -C dest.git unpack-objects <binary_320M.pack'
- Benchmark 1: git -c core.bigFileThreshold=2m -C dest.git unpack-objects <binary_320M.pack
- Time (mean ± σ): 10.859 s ± 0.774 s [User: 8.813 s, System: 1.898 s]
- Range (min … max): 9.884 s … 12.192 s 10 runs
+ Benchmark 2: new
+ Time (mean ± σ): 6.090 s ± 0.033 s [User: 5.075 s, System: 0.976 s]
+ Range (min … max): 6.030 s … 6.142 s 10 runs
- $ hyperfine \
- --prepare 'rm -rf dest.git && git init --bare dest.git' \
- 'git -C dest.git unpack-objects <binary_96M.pack'
- Benchmark 1: git -C dest.git unpack-objects <binary_96M.pack
- Time (mean ± σ): 2.678 s ± 0.037 s [User: 2.205 s, System: 0.450 s]
- Range (min … max): 2.639 s … 2.743 s 10 runs
+ Benchmark 3: new (small threshold)
+ Time (mean ± σ): 6.755 s ± 0.029 s [User: 5.150 s, System: 1.560 s]
+ Range (min … max): 6.711 s … 6.809 s 10 runs
- $ hyperfine \
- --prepare 'rm -rf dest.git && git init --bare dest.git' \
- 'git -c core.bigFileThreshold=2m -C dest.git unpack-objects <binary_96M.pack'
- Benchmark 1: git -c core.bigFileThreshold=2m -C dest.git unpack-objects <binary_96M.pack
- Time (mean ± σ): 2.819 s ± 0.124 s [User: 2.216 s, System: 0.564 s]
- Range (min … max): 2.679 s … 3.125 s 10 runs
+ Summary
+ 'old' ran
+ 1.00 ± 0.01 times faster than 'new'
+ 1.11 ± 0.01 times faster than 'new (small threshold)'
+ Helped-by: Derrick Stolee <stolee@gmail.com>
Helped-by: Jiang Xin <zhiyou.jx@alibaba-inc.com>
Signed-off-by: Han Xin <hanxin.hx@alibaba-inc.com>
@@ builtin/unpack-objects.c: static void added_object(unsigned nr, enum object_type
+struct input_zstream_data {
+ git_zstream *zstream;
-+ unsigned char buf[4096];
++ unsigned char buf[8192];
+ int status;
+};
+
@@ builtin/unpack-objects.c: static void added_object(unsigned nr, enum object_type
+ struct input_stream in_stream = {
+ .read = feed_input_zstream,
+ .data = &data,
++ .size = size,
+ };
+ struct object_id *oid = &obj_list[nr].oid;
+ int ret;