git/list[1] front-page[2] threads[3] people[4] search[5] about
 

[PATCH v1 3/4] fastindex: add test tools and a test script

From
Ben Peart <benpeart@microsoft.com>
Date
Nov 9, 2017, 14:17 UTC
Message-ID
<20171109141737.47976-4-benpeart@microsoft.com>
In-Reply-To
<20171109141737.47976-1-benpeart@microsoft.com>

Add a test utility (test-dump-fast-index) that dumps the contents of the IEOT to stdout. This enables checking the existance of the extension and the ability to parse and output its contents.

Add a test utility (test-fast-index) that loads the index using the fastindex logic then loads it using the regular logic and compares the results of the two. Any differences are reported and an error is returned.

Add a test script (t1800-fast-index.sh) to:

Test the ability to add/remove the fastindex index extension via update-index.

Verify that loading the index with and without the fastindex extension return the same results with both V2 and V4 indexes.

Signed-off-by: Ben Peart <benpeart@microsoft.com>
---
 Makefile                        |  2 +
 t/helper/test-dump-fast-index.c | 68 +++++++++++++++++++++++++++++++++
 t/helper/test-fast-index.c      | 84 +++++++++++++++++++++++++++++++++++++++++
 t/t1800-fast-index.sh           | 55 +++++++++++++++++++++++++++
 4 files changed, 209 insertions(+)
 create mode 100644 t/helper/test-dump-fast-index.c
 create mode 100644 t/helper/test-fast-index.c
 create mode 100644 t/t1800-fast-index.sh
diff --git a/Makefile b/Makefile
index cd75985991..a7df82721e 100644
--- a/Makefile
+++ b/Makefile
@@ -647,9 +647,11 @@ TEST_PROGRAMS_NEED_X += test-config
 TEST_PROGRAMS_NEED_X += test-date
 TEST_PROGRAMS_NEED_X += test-delta
 TEST_PROGRAMS_NEED_X += test-dump-cache-tree
+TEST_PROGRAMS_NEED_X += test-dump-fast-index
 TEST_PROGRAMS_NEED_X += test-dump-split-index
 TEST_PROGRAMS_NEED_X += test-dump-untracked-cache
 TEST_PROGRAMS_NEED_X += test-fake-ssh
+TEST_PROGRAMS_NEED_X += test-fast-index
 TEST_PROGRAMS_NEED_X += test-genrandom
 TEST_PROGRAMS_NEED_X += test-hashmap
 TEST_PROGRAMS_NEED_X += test-index-version
diff --git a/t/helper/test-dump-fast-index.c b/t/helper/test-dump-fast-index.c
new file mode 100644
index 0000000000..4e6d28d660
--- /dev/null
+++ b/t/helper/test-dump-fast-index.c
@@ -0,0 +1,68 @@
+#include "cache.h"
+
+int cmd_main(int ac, const char **av)
+{
+#ifndef NO_PTHREADS
+	const char *path;
+	int fd, i;
+	struct stat st;
+	void *mmap;
+	size_t mmap_size;
+	struct cache_header *hdr;
+	struct index_entry_offset_table *ieot;
+	struct strbuf previous_name_buf = STRBUF_INIT, *previous_name;
+	int err = 0;
+
+	setup_git_directory();
+	path = get_index_file();
+	fd = open(path, O_RDONLY);
+	if (fd < 0) {
+		die("%s: index file open failed", path);
+	}
+
+	if (fstat(fd, &st))
+		die("cannot stat the open index");
+
+	mmap_size = xsize_t(st.st_size);
+	if (mmap_size < sizeof(struct cache_header) + GIT_SHA1_RAWSZ)
+		die("index file smaller than expected");
+
+	mmap = xmmap(NULL, mmap_size, PROT_READ, MAP_PRIVATE, fd, 0);
+	if (mmap == MAP_FAILED)
+		die("unable to map index file");
+	close(fd);
+
+	hdr = mmap;
+	if (ntohl(hdr->hdr_version) == 4)
+		previous_name = &previous_name_buf;
+	else
+		previous_name = NULL;
+
+	ieot = read_ieot_extension(mmap, mmap_size);
+	if (ieot) {
+		printf("IEOT with %d entries\n", ieot->nr);
+		printf("  Offset    Count Name\n");
+		printf("-------- -------- ------------------------\n");
+		for (i = 0; i < ieot->nr; i++) {
+			struct ondisk_cache_entry *disk_ce;
+			struct cache_entry *ce;
+			unsigned long consumed;
+
+			disk_ce = (struct ondisk_cache_entry *)((char *)mmap + ieot->entries[i].offset);
+			ce = create_from_disk(disk_ce, &consumed, previous_name, 0);
+			printf("%8d %8d %.*s\n", ieot->entries[i].offset, ieot->entries[i].nr, ce->ce_namelen, ce->name);
+			free(ce);
+		}
+	} else {
+		printf("missing or invalid extension");
+		err = 1;
+	}
+
+	free(ieot);
+	munmap(mmap, mmap_size);
+	return err;
+#else
+	die("ieot only supported with PTHREADS");
+	return -1;
+#endif
+}
diff --git a/t/helper/test-fast-index.c b/t/helper/test-fast-index.c
new file mode 100644
index 0000000000..fe63130ba0
--- /dev/null
+++ b/t/helper/test-fast-index.c
@@ -0,0 +1,84 @@
+#include "cache.h"
+
+int compare_ce(const struct cache_entry *ce1, const struct cache_entry *ce2)
+{
+	/*	struct hashmap_entry ent; */
+	/*	struct stat_data ce_stat_data; */
+
+	if (ce1->ce_mode != ce2->ce_mode) {
+		printf("ce_mode: %d:%d\n", ce1->ce_mode, ce2->ce_mode);
+		return 1;
+	}
+
+	if (ce1->ce_flags != ce2->ce_flags) {
+		printf("ce_flags: %d:%d\n", ce1->ce_flags, ce2->ce_flags);
+		return 1;
+	}
+
+	if (ce1->ce_namelen != ce2->ce_namelen) {
+		printf("namelen: %d:%d\n", ce1->ce_namelen, ce2->ce_namelen);
+		return 1;
+	}
+
+	if (ce1->index != ce2->index) {
+		printf("index: %d:%d\n", ce1->index, ce2->index);
+		return 1;
+	}
+
+	if (oidcmp(&ce1->oid, &ce2->oid)) {
+		printf("oid: %s:%s\n", oid_to_hex(&ce1->oid), oid_to_hex(&ce2->oid));
+		return 1;
+	}
+
+	if (strcmp(ce1->name, ce2->name)) {
+		printf("name: %s:%s\n", ce1->name, ce2->name);
+		return 1;
+	}
+
+
+	return 0;
+}
+
+extern int ignore_fast_index_config;
+
+int cmd_main(int ac, const char **av)
+{
+#ifndef NO_PTHREADS
+	static struct index_state index;
+	static struct index_state ieot;
+	int i, err = 0;
+
+	setup_git_directory();
+	ignore_fast_index_config = 1;
+	core_fast_index = 0;
+	read_index(&index);
+	core_fast_index = 1;
+	read_index(&ieot);
+
+	for (i = 0; i < index.cache_nr; i++) {
+		if (compare_ce(index.cache[i], ieot.cache[i])) {
+			struct cache_entry *ce;
+
+			ce = index.cache[i];
+			printf("%06o %s %d\t%s\n", ce->ce_mode,
+				oid_to_hex(&ce->oid), ce_stage(ce), ce->name);
+			ce = ieot.cache[i];
+			printf("%06o %s %d\t%s\n", ce->ce_mode,
+				oid_to_hex(&ce->oid), ce_stage(ce), ce->name);
+
+			printf("cache entry %d does not match", i);
+			err = 1;
+			break;
+		}
+	}
+
+	discard_index(&ieot);
+	discard_index(&index);
+	if (!err)
+		printf("Cache entires are the same\n");
+	return err;
+#else
+	die("ieot only supported with PTHREADS");
+	return -1;
+#endif
+}
diff --git a/t/t1800-fast-index.sh b/t/t1800-fast-index.sh
new file mode 100644
index 0000000000..b3460f1698
--- /dev/null
+++ b/t/t1800-fast-index.sh
@@ -0,0 +1,55 @@
+#!/bin/sh
+
+test_description='git fast index tests'
+
+. ./test-lib.sh
+
+GIT_FASTINDEX_TEST=1; export GIT_FASTINDEX_TEST
+
+test_expect_success 'setup' '
+	: >tracked &&
+	: >modified &&
+	mkdir dir1 &&
+	: >dir1/tracked &&
+	: >dir1/modified &&
+	mkdir dir2 &&
+	: >dir2/tracked &&
+	: >dir2/modified &&
+	git add . &&
+	git commit -m initial &&
+	cat >.gitignore <<-\EOF
+	.gitignore
+	expect*
+	actual*
+	EOF
+'
+
+test_expect_success 'fastindex extension is off by default' '
+	test_must_fail test-dump-fast-index >actual 2>&1 &&
+	grep "^missing or invalid extension" actual
+'
+
+test_expect_success 'update-index --fastindex" adds the fsmonitor extension' '
+	git update-index --fastindex &&
+	test-dump-fast-index >actual &&
+	grep "^IEOT with" actual
+'
+
+test_expect_success 'update-index --no-fastindex" removes the fastindex extension' '
+	git update-index --no-fastindex &&
+	test_must_fail test-dump-fast-index >actual &&
+	grep "^missing or invalid extension" actual
+'
+
+test_expect_success 'verify with and without fastindex returns same result' '
+	git update-index --fastindex &&
+	test-fast-index
+'
+
+test_expect_success 'test with V4 index' '
+	git config core.fastindex 1 &&
+	git update-index --index-version 4 &&
+	test-fast-index
+'
+
+test_done

-- 
2.15.0.windows.1
Previous: Ben PeartNext: Ben Peart
Message 18 of 19 in “Speed up index load through parallelization”
  1. 0/4 Speed up index load through parallelizationBen Peart, Nov 9, 2017
  2. 1/4 fastindex: speed up index load through parallelizationBen Peart, Nov 9, 2017
  3. Junio C HamanoNov 10, 2017
  4. Ben PeartNov 13, 2017
  5. Junio C HamanoNov 14, 2017
  6. Ben PeartNov 14, 2017
  7. Junio C HamanoNov 14, 2017
  8. Ben PeartNov 14, 2017
  9. Junio C HamanoNov 15, 2017
  10. Ben PeartNov 15, 2017
  11. Junio C HamanoNov 15, 2017
  12. Ben PeartNov 20, 2017
  13. Jeff KingNov 20, 2017
  14. Jeff KingNov 20, 2017
  15. Ramsay JonesNov 20, 2017
  16. Ben PeartNov 21, 2017
  17. 2/4 update-index: add fastindex support to update-indexBen Peart, Nov 9, 2017
  18. 3/4 fastindex: add test tools and a test scriptBen Peart, Nov 9, 2017
  19. 4/4 fastindex: add documentation for the fastindex extensionBen Peart, Nov 9, 2017

Read the whole thread, see it on lore, or plain text.

$ cat FOOTERMessages come from the public archive at lore.kernel.org/git, fetched every hour. The front page is chosen and written each morning by an AI editor and can be wrong; the threads themselves are the record. About and API. For agents: an MCP server at https://gitlist.dev/mcp, and any thread, story or person page as Markdown by adding .md to its URL (or sending Accept: text/markdown). Details in /llms.txt.