git/list[1] front-page[2] threads[3] people[4] search[5] about
 

Re: [PATCH 0/19] pack bitmaps

From
Junio C Hamano <gitster@pobox.com>
Date
Oct 25, 2013, 03:07 UTC
Message-ID
<xmqq38nqkpzx.fsf@gitster.dls.corp.google.com>
In-Reply-To
<20131024175915.GA23398@sigill.intra.peff.net>

This is only to tentatively work-around the compilation breakages; the fixes need to be split into the respective patches that introduce breakages when the series is rerolled (the one I sent for pack-bitmap.c separately is also included in this message).

Thanks.
 ewah/ewah_bitmap.c  | 22 ++++++++++++++++------
 ewah/ewah_io.c      | 44 ++++++++++++++++++++++++++------------------
 pack-bitmap-write.c |  2 --
 pack-bitmap.c       | 13 ++++++-------
 4 files changed, 48 insertions(+), 33 deletions(-)
diff --git a/ewah/ewah_bitmap.c b/ewah/ewah_bitmap.c
index b74a1eb..7986720 100644
--- a/ewah/ewah_bitmap.c
+++ b/ewah/ewah_bitmap.c
@@ -65,6 +65,8 @@ static void buffer_push_rlw(struct ewah_bitmap *self, eword_t value)
 
 static size_t add_empty_words(struct ewah_bitmap *self, int v, size_t number)
 {
+	eword_t runlen;
+	eword_t can_add;
 	size_t added = 0;
 
 	if (rlw_get_run_bit(self->rlw) != v && rlw_size(self->rlw) == 0) {
@@ -76,8 +78,8 @@ static size_t add_empty_words(struct ewah_bitmap *self, int v, size_t number)
 		added++;
 	}
 
-	eword_t runlen = rlw_get_running_len(self->rlw);
-	eword_t can_add = min_size(number, RLW_LARGEST_RUNNING_COUNT - runlen);
+	runlen = rlw_get_running_len(self->rlw);
+	can_add = min_size(number, RLW_LARGEST_RUNNING_COUNT - runlen);
 
 	rlw_set_running_len(self->rlw, runlen + can_add);
 	number -= can_add;
@@ -426,6 +428,8 @@ void ewah_xor(
 	rlwit_init(&rlw_j, ewah_j);
 
 	while (rlwit_word_size(&rlw_i) > 0 && rlwit_word_size(&rlw_j) > 0) {
+		size_t literals;
+
 		while (rlw_i.rlw.running_len > 0 || rlw_j.rlw.running_len > 0) {
 			struct rlw_iterator *prey, *predator;
 			size_t index;
@@ -446,7 +450,7 @@ void ewah_xor(
 			rlwit_discard_first_words(predator, predator->rlw.running_len);
 		}
 
-		size_t literals = min_size(rlw_i.rlw.literal_words, rlw_j.rlw.literal_words);
+		literals = min_size(rlw_i.rlw.literal_words, rlw_j.rlw.literal_words);
 
 		if (literals) {
 			size_t k;
@@ -484,6 +488,8 @@ void ewah_and(
 	rlwit_init(&rlw_j, ewah_j);
 
 	while (rlwit_word_size(&rlw_i) > 0 && rlwit_word_size(&rlw_j) > 0) {
+		size_t literals;
+
 		while (rlw_i.rlw.running_len > 0 || rlw_j.rlw.running_len > 0) {
 			struct rlw_iterator *prey, *predator;
 
@@ -507,7 +513,7 @@ void ewah_and(
 			}
 		}
 
-		size_t literals = min_size(rlw_i.rlw.literal_words, rlw_j.rlw.literal_words);
+		literals = min_size(rlw_i.rlw.literal_words, rlw_j.rlw.literal_words);
 
 		if (literals) {
 			size_t k;
@@ -545,6 +551,8 @@ void ewah_and_not(
 	rlwit_init(&rlw_j, ewah_j);
 
 	while (rlwit_word_size(&rlw_i) > 0 && rlwit_word_size(&rlw_j) > 0) {
+		size_t literals;
+
 		while (rlw_i.rlw.running_len > 0 || rlw_j.rlw.running_len > 0) {
 			struct rlw_iterator *prey, *predator;
 
@@ -572,7 +580,7 @@ void ewah_and_not(
 			}
 		}
 
-		size_t literals = min_size(rlw_i.rlw.literal_words, rlw_j.rlw.literal_words);
+		literals = min_size(rlw_i.rlw.literal_words, rlw_j.rlw.literal_words);
 
 		if (literals) {
 			size_t k;
@@ -610,6 +618,8 @@ void ewah_or(
 	rlwit_init(&rlw_j, ewah_j);
 
 	while (rlwit_word_size(&rlw_i) > 0 && rlwit_word_size(&rlw_j) > 0) {
+		size_t literals;
+
 		while (rlw_i.rlw.running_len > 0 || rlw_j.rlw.running_len > 0) {
 			struct rlw_iterator *prey, *predator;
 
@@ -634,7 +644,7 @@ void ewah_or(
 			}
 		}
 
-		size_t literals = min_size(rlw_i.rlw.literal_words, rlw_j.rlw.literal_words);
+		literals = min_size(rlw_i.rlw.literal_words, rlw_j.rlw.literal_words);
 
 		if (literals) {
 			size_t k;
diff --git a/ewah/ewah_io.c b/ewah/ewah_io.c
index db6c062..05c51d9 100644
--- a/ewah/ewah_io.c
+++ b/ewah/ewah_io.c
@@ -58,19 +58,26 @@ int ewah_serialize_to(struct ewah_bitmap *self,
 	eword_t dump[2048];
 	const size_t words_per_dump = sizeof(dump) / sizeof(eword_t);
 
-	/* 32 bit -- bit size fr the map */
-	uint32_t bitsize =  htonl((uint32_t)self->bit_size);
+	/* 32 bit -- bit size for the map */
+	uint32_t bitsize;
+	/* 32 bit -- number of compressed 64-bit words */
+	uint32_t word_count;
+	/* 64 bit x N -- compressed words */
+	const eword_t *buffer = self->buffer;
+	size_t words_left;
+
+	/* 32 bit -- position for the RLW */
+	uint32_t rlw_pos;
+
+	bitsize =  htonl((uint32_t)self->bit_size);
 	if (write_fun(data, &bitsize, 4) != 4)
 		return -1;
 
-	/** 32 bit -- number of compressed 64-bit words */
-	uint32_t word_count =  htonl((uint32_t)self->buffer_size);
+	word_count =  htonl((uint32_t)self->buffer_size);
 	if (write_fun(data, &word_count, 4) != 4)
 		return -1;
 
-	/** 64 bit x N -- compressed words */
-	const eword_t *buffer = self->buffer;
-	size_t words_left = self->buffer_size;
+	words_left = self->buffer_size;
 
 	while (words_left >= words_per_dump) {
 		for (i = 0; i < words_per_dump; ++i, ++buffer)
@@ -90,8 +97,7 @@ int ewah_serialize_to(struct ewah_bitmap *self,
 			return -1;
 	}
 
-	/** 32 bit -- position for the RLW */
-	uint32_t rlw_pos = (uint8_t*)self->rlw - (uint8_t *)self->buffer;
+	rlw_pos = (uint8_t*)self->rlw - (uint8_t *)self->buffer;
 	rlw_pos = htonl(rlw_pos / sizeof(eword_t));
 
 	if (write_fun(data, &rlw_pos, 4) != 4)
@@ -138,18 +144,23 @@ int ewah_deserialize(struct ewah_bitmap *self, int fd)
 	size_t i;
 	eword_t dump[2048];
 	const size_t words_per_dump = sizeof(dump) / sizeof(eword_t);
+	/* 32 bit -- bit size for the map */
+	uint32_t bitsize;
+	/* 32 bit -- number of compressed 64-bit words */
+	uint32_t word_count;
+	/* 64 bit x N -- compressed words */
+	eword_t *buffer;
+	size_t words_left;
+	/** 32 bit -- position for the RLW */
+	uint32_t rlw_pos;
 
 	ewah_clear(self);
 
-	/* 32 bit -- bit size fr the map */
-	uint32_t bitsize;
 	if (read(fd, &bitsize, 4) != 4)
 		return -1;
 
 	self->bit_size = (size_t)ntohl(bitsize);
 
-	/** 32 bit -- number of compressed 64-bit words */
-	uint32_t word_count;
 	if (read(fd, &word_count, 4) != 4)
 		return -1;
 
@@ -159,9 +170,8 @@ int ewah_deserialize(struct ewah_bitmap *self, int fd)
 	if (!self->buffer)
 		return -1;
 
-	/** 64 bit x N -- compressed words */
-	eword_t *buffer = self->buffer;
-	size_t words_left = self->buffer_size;
+	buffer = self->buffer;
+	words_left = self->buffer_size;
 
 	while (words_left >= words_per_dump) {
 		if (read(fd, dump, sizeof(dump)) != sizeof(dump))
@@ -181,8 +191,6 @@ int ewah_deserialize(struct ewah_bitmap *self, int fd)
 			*buffer = ntohll(dump[i]);
 	}
 
-	/** 32 bit -- position for the RLW */
-	uint32_t rlw_pos;
 	if (read(fd, &rlw_pos, 4) != 4)
 		return -1;
 
diff --git a/pack-bitmap-write.c b/pack-bitmap-write.c
index c44874a..48087f1 100644
--- a/pack-bitmap-write.c
+++ b/pack-bitmap-write.c
@@ -1,5 +1,3 @@
-#include <stdlib.h>
-
 #include "cache.h"
 #include "commit.h"
 #include "tag.h"
diff --git a/pack-bitmap.c b/pack-bitmap.c
index a7c553d..3f1d369 100644
--- a/pack-bitmap.c
+++ b/pack-bitmap.c
@@ -1,5 +1,3 @@
-#include <stdlib.h>
-
 #include "cache.h"
 #include "commit.h"
 #include "tag.h"
@@ -123,8 +121,8 @@ static struct ewah_bitmap *read_bitmap_1(struct bitmap_index *index)
 	struct ewah_bitmap *b = ewah_pool_new();
 
 	int bitmap_size = ewah_read_mmap(b,
-		index->map + index->map_pos,
-		index->map_size - index->map_pos);
+					 (unsigned char *)index->map + index->map_pos,
+					 index->map_size - index->map_pos);
 
 	if (bitmap_size < 0) {
 		error("Failed to load bitmap index (corrupted?)");
@@ -159,8 +157,9 @@ static int load_bitmap_header(struct bitmap_index *index)
 				"(Git requires BITMAP_OPT_FULL_DAG)");
 
 		if (flags & BITMAP_OPT_HASH_CACHE) {
-			index->hashes = index->map + index->map_size - 20 -
-				(sizeof(uint32_t) * index->pack->num_objects);
+			index->hashes = (uint32_t *)((unsigned char *)index->map +
+						     index->map_size - 20 -
+						     (sizeof(uint32_t) * index->pack->num_objects));
 		}
 	}
 
@@ -216,7 +215,7 @@ static int load_bitmap_entries_v1(struct bitmap_index *index)
 		uint32_t commit_idx_pos;
 		const unsigned char *sha1;
 
-		entry = index->map + index->map_pos;
+		entry = (struct bitmap_disk_entry *)((unsigned char *)index->map + index->map_pos);
 		index->map_pos += sizeof(struct bitmap_disk_entry);
 
 		commit_idx_pos = ntohl(entry->object_pos);
Previous: Junio C HamanoNext: Jeff King
Message 50 of 87 in “pack bitmaps”
  1. 0/19 pack bitmapsJeff King, Oct 24, 2013
  2. 01/19 sha1write: make buffer const-correctJeff King, Oct 24, 2013
  3. 02/19 revindex: Export new APIsJeff King, Oct 24, 2013
  4. 03/19 pack-objects: Refactor the packing listJeff King, Oct 24, 2013
  5. 04/19 pack-objects: factor out name_hashJeff King, Oct 24, 2013
  6. 05/19 revision: allow setting custom limiter functionJeff King, Oct 24, 2013
  7. 06/19 sha1_file: export `git_open_noatime`Jeff King, Oct 24, 2013
  8. 07/19 compat: add endianness helpersJeff King, Oct 24, 2013
  9. Thomas RastOct 26, 2013
  10. Jeff KingOct 30, 2013
  11. Vicent MartíOct 30, 2013
  12. 08/19 ewah: compressed bitmap implementationJeff King, Oct 24, 2013
  13. Junio C HamanoOct 24, 2013
  14. Jeff KingOct 25, 2013
  15. Thomas RastOct 26, 2013
  16. 09/19 documentation: add documentation for the bitmap formatJeff King, Oct 24, 2013
  17. Duy NguyenOct 25, 2013
  18. Jeff KingOct 25, 2013
  19. Duy NguyenOct 25, 2013
  20. Shawn PearceOct 25, 2013
  21. Jeff KingOct 30, 2013
  22. Shawn PearceOct 30, 2013
  23. Vicent MartiOct 30, 2013
  24. Vicent MartiOct 30, 2013
  25. 10/19 pack-bitmap: add support for bitmap indexesJeff King, Oct 24, 2013
  26. Shawn PearceOct 25, 2013
  27. Jeff KingOct 30, 2013
  28. Shawn PearceOct 30, 2013
  29. Vicent MartiOct 30, 2013
  30. Shawn PearceOct 30, 2013
  31. Jeff KingOct 30, 2013
  32. 11/19 pack-objects: use bitmaps when packing objectsJeff King, Oct 24, 2013
  33. Shawn PearceOct 25, 2013
  34. Jeff KingOct 30, 2013
  35. Shawn PearceOct 30, 2013
  36. Vicent MartiOct 30, 2013
  37. 12/19 rev-list: add bitmap mode to speed up object listsJeff King, Oct 24, 2013
  38. Shawn PearceOct 25, 2013
  39. Jeff KingOct 30, 2013
  40. 13/19 pack-objects: implement bitmap writingJeff King, Oct 24, 2013
  41. Duy NguyenOct 25, 2013
  42. Jeff KingOct 25, 2013
  43. 14/19 repack: stop using magic number for ARRAY_SIZE(exts)Jeff King, Oct 24, 2013
  44. 15/19 repack: turn exts array into array-of-structJeff King, Oct 24, 2013
  45. 16/19 repack: handle optional files created by pack-objectsJeff King, Oct 24, 2013
  46. 17/19 repack: consider bitmaps when performing repacksJeff King, Oct 24, 2013
  47. 18/19 t: add basic bitmap functionality testsJeff King, Oct 24, 2013
  48. 19/19 pack-bitmap: implement optional name_hash cacheJeff King, Oct 24, 2013
  49. Junio C HamanoOct 24, 2013
  50. Junio C HamanoOct 25, 2013
  51. 0/19 pack bitmapsJeff King, Oct 25, 2013
  52. 01/19 sha1write: make buffer const-correctJeff King, Oct 25, 2013
  53. 02/19 revindex: Export new APIsJeff King, Oct 25, 2013
  54. 03/19 pack-objects: Refactor the packing listJeff King, Oct 25, 2013
  55. 04/19 pack-objects: factor out name_hashJeff King, Oct 25, 2013
  56. 05/19 revision: allow setting custom limiter functionJeff King, Oct 25, 2013
  57. 06/19 sha1_file: export `git_open_noatime`Jeff King, Oct 25, 2013
  58. 07/19 compat: add endianness helpersJeff King, Oct 25, 2013
  59. 08/19 ewah: compressed bitmap implementationJeff King, Oct 25, 2013
  60. 09/19 documentation: add documentation for the bitmap formatJeff King, Oct 25, 2013
  61. 10/19 pack-bitmap: add support for bitmap indexesJeff King, Oct 25, 2013
  62. Junio C HamanoOct 25, 2013
  63. Jeff KingOct 26, 2013
  64. Jeff KingOct 26, 2013
  65. Junio C HamanoOct 28, 2013
  66. Jeff KingOct 30, 2013
  67. Duy NguyenOct 26, 2013
  68. Jeff KingOct 30, 2013
  69. 11/19 pack-objects: use bitmaps when packing objectsJeff King, Oct 25, 2013
  70. Duy NguyenOct 26, 2013
  71. Jeff KingOct 30, 2013
  72. Duy NguyenOct 30, 2013
  73. Jeff KingOct 30, 2013
  74. Duy NguyenOct 31, 2013
  75. 12/19 rev-list: add bitmap mode to speed up object listsJeff King, Oct 25, 2013
  76. 13/19 pack-objects: implement bitmap writingJeff King, Oct 25, 2013
  77. 14/19 repack: stop using magic number for ARRAY_SIZE(exts)Jeff King, Oct 25, 2013
  78. 15/19 repack: turn exts array into array-of-structJeff King, Oct 25, 2013
  79. 16/19 repack: handle optional files created by pack-objectsJeff King, Oct 25, 2013
  80. 17/19 repack: consider bitmaps when performing repacksJeff King, Oct 25, 2013
  81. 18/19 t: add basic bitmap functionality testsJeff King, Oct 25, 2013
  82. SZEDER GáborOct 28, 2013
  83. Jeff KingOct 30, 2013
  84. 19/19 pack-bitmap: implement optional name_hash cacheJeff King, Oct 25, 2013
  85. 20/19 count-objects: consider .bitmap without .pack/.idx pair garbageNguyễn Thái Ngọc Duy, Oct 26, 2013
  86. Jeff KingOct 30, 2013
  87. Junio C HamanoOct 30, 2013

Read the whole thread, see it on lore, or plain text.

$ cat FOOTERMessages come from the public archive at lore.kernel.org/git, fetched every hour. The front page is chosen and written each morning by an AI editor and can be wrong; the threads themselves are the record. About and API. For agents: an MCP server at https://gitlist.dev/mcp, and any thread, story or person page as Markdown by adding .md to its URL (or sending Accept: text/markdown). Details in /llms.txt.