threads / patch / 1043

patchassorted delta code cleanup

Subject: [PATCH] assorted delta code cleanup

## tl;dr

3 messages between Jun 29, 2005 and Jun 29, 2005. Diffs are folded; open one to read it.

replies: 2people: 3as markdown or json

Nicolas Pitre· Jun 29, 2005, 06:49 UTC · lore

This is a wrap-up patch including all the cleanups I've done to the delta code and its usage. The most important change is the factorization of the delta header handling code.

Signed-off-by: Nicolas Pitre <nico@cam.org>
Show changes to 6 files +35 −58

count-delta.c, delta.h, diff-delta.c, pack-objects.c, patch-delta.c, sha1_file.c

diff --git a/count-delta.c b/count-delta.c
--- a/count-delta.c
+++ b/count-delta.c
@@ -6,23 +6,9 @@
 #include <stdlib.h>
 #include <string.h>
 #include <limits.h>
+#include "delta.h"
 #include "count-delta.h"
 
-static unsigned long get_hdr_size(const unsigned char **datap)
-{
-	const unsigned char *data = *datap;
-	unsigned char cmd = *data++;
-	unsigned long size = cmd & ~0x80;
-	int i = 7;
-	while (cmd & 0x80) {
-		cmd = *data++;
-		size |= (cmd & ~0x80) << i;
-		i += 7;
-	}
-	*datap = data;
-	return size;
-}
-
 /*
  * NOTE.  We do not _interpret_ delta fully.  As an approximation, we
  * just count the number of bytes that are copied from the source, and
@@ -44,15 +30,14 @@ int count_delta(void *delta_buf, unsigne
 	unsigned char cmd;
 	unsigned long src_size, dst_size, out;
 
-	/* the smallest delta size possible is 4 bytes */
-	if (delta_size < 4)
+	if (delta_size < DELTA_SIZE_MIN)
 		return -1;
 
 	data = delta_buf;
 	top = delta_buf + delta_size;
 
-	src_size = get_hdr_size(&data);
-	dst_size = get_hdr_size(&data);
+	src_size = get_delta_hdr_size(&data);
+	dst_size = get_delta_hdr_size(&data);
 
 	added_literal = copied_from_source = out = 0;
 	while (data < top) {
diff --git a/delta.h b/delta.h
--- a/delta.h
+++ b/delta.h
@@ -9,4 +9,26 @@ extern void *patch_delta(void *src_buf, 
 			 void *delta_buf, unsigned long delta_size,
 			 unsigned long *dst_size);
 
+/* the smallest possible delta size is 4 bytes */
+#define DELTA_SIZE_MIN	4
+
+/*
+ * This must be called twice on the delta data buffer, first to get the
+ * expected reference buffer size, and again to get the result buffer size.
+ */
+static inline unsigned long get_delta_hdr_size(const unsigned char **datap)
+{
+	const unsigned char *data = *datap;
+	unsigned char cmd = *data++;
+	unsigned long size = cmd & ~0x80;
+	int i = 7;
+	while (cmd & 0x80) {
+		cmd = *data++;
+		size |= (cmd & ~0x80) << i;
+		i += 7;
+	}
+	*datap = data;
+	return size;
+}
+
 #endif
diff --git a/diff-delta.c b/diff-delta.c
--- a/diff-delta.c
+++ b/diff-delta.c
@@ -306,12 +306,13 @@ void *diff_delta(void *from_buf, unsigne
 			*orig = i;
 		}
 
-		/* next time around the largest possible output is 1 + 4 + 3 */
 		if (max_size && outpos > max_size) {
 			free(out);
 			delta_cleanup(&bdf);
 			return NULL;
 		}
+
+		/* next time around the largest possible output is 1 + 4 + 3 */
 		if (outpos > outsize - 8) {
 			void *tmp = out;
 			outsize = outsize * 3 / 2;
diff --git a/pack-objects.c b/pack-objects.c
--- a/pack-objects.c
+++ b/pack-objects.c
@@ -34,7 +34,7 @@ static void *delta_against(void *buf, un
 	if (!otherbuf)
 		die("unable to read %s", sha1_to_hex(entry->delta->sha1));
         delta_buf = diff_delta(otherbuf, othersize,
-			       buf, size, &delta_size, ~0UL);
+			       buf, size, &delta_size, 0);
         if (!delta_buf || delta_size != entry->delta_size)
         	die("delta size changed");
         free(buf);
diff --git a/patch-delta.c b/patch-delta.c
--- a/patch-delta.c
+++ b/patch-delta.c
@@ -20,36 +20,20 @@ void *patch_delta(void *src_buf, unsigne
 	const unsigned char *data, *top;
 	unsigned char *dst_buf, *out, cmd;
 	unsigned long size;
-	int i;
 
-	/* the smallest delta size possible is 4 bytes */
-	if (delta_size < 4)
+	if (delta_size < DELTA_SIZE_MIN)
 		return NULL;
 
 	data = delta_buf;
 	top = delta_buf + delta_size;
 
 	/* make sure the orig file size matches what we expect */
-	cmd = *data++;
-	size = cmd & ~0x80;
-	i = 7;
-	while (cmd & 0x80) {
-		cmd = *data++;
-		size |= (cmd & ~0x80) << i;
-		i += 7;
-	}
+	size = get_delta_hdr_size(&data);
 	if (size != src_size)
 		return NULL;
 
 	/* now the result size */
-	cmd = *data++;
-	size = cmd & ~0x80;
-	i = 7;
-	while (cmd & 0x80) {
-		cmd = *data++;
-		size |= (cmd & ~0x80) << i;
-		i += 7;
-	}
+	size = get_delta_hdr_size(&data);
 	dst_buf = malloc(size);
 	if (!dst_buf)
 		return NULL;
diff --git a/sha1_file.c b/sha1_file.c
--- a/sha1_file.c
+++ b/sha1_file.c
@@ -592,22 +592,6 @@ void * unpack_sha1_file(void *map, unsig
 	return unpack_sha1_rest(&stream, hdr, *size);
 }
 
-static unsigned long parse_delta_size(unsigned char **p)
-{
-	unsigned char c;
-	unsigned long size = 0;
-	unsigned shift = 0;
-	unsigned char *data = *p;
-
-	do {
-		c = *data++;
-		size += (c & 0x7f) << shift;
-		shift += 7;
-	} while (c & 0x80);
-	*p = data;
-	return size;
-}
-
 static int packed_delta_info(unsigned char *base_sha1,
 			     unsigned long delta_size,
 			     unsigned long left,
@@ -645,11 +629,12 @@ static int packed_delta_info(unsigned ch
 	 * the result size.  Verify the base size while we are at it.
 	 */
 	data = delta_head;
-	verify_base_size = parse_delta_size(&data);
-	result_size = parse_delta_size(&data);
+	verify_base_size = get_delta_hdr_size(&data);
 	if (verify_base_size != base_size)
 		die("delta base size mismatch");
 
+	/* Read the result size */
+	result_size = get_delta_hdr_size(&data);
 	*sizep = result_size;
 	return 0;
 }
Matthias Urlichs· Jun 29, 2005, 07:10 UTC · re: Nicolas Pitre · lore

Re: [PATCH] assorted delta code cleanup

Hi, Nicolas Pitre wrote:
> This is a wrap-up patch including all the cleanups I've done to the
> delta code and its usage. 

FWIW, I think Linus said he'd prefer a do/while loop. In fact this old code ...

Show 16 quoted lines
> -static unsigned long parse_delta_size(unsigned char **p)
> -{
> -	unsigned char c;
> -	unsigned long size = 0;
> -	unsigned shift = 0;
> -	unsigned char *data = *p;
> -
> -	do {
> -		c = *data++;
> -		size += (c & 0x7f) << shift;
> -		shift += 7;
> -	} while (c & 0x80);
> -	*p = data;
> -	return size;
> -}
> -
... is IMHO much more readable than this ...
Show 15 quoted lines
> +static inline unsigned long get_delta_hdr_size(const unsigned char **datap)
> +{
> +	const unsigned char *data = *datap;
> +	unsigned char cmd = *data++;
> +	unsigned long size = cmd & ~0x80;
> +	int i = 7;
> +	while (cmd & 0x80) {
> +		cmd = *data++;
> +		size |= (cmd & ~0x80) << i;
> +		i += 7;
> +	}
> +	*datap = data;
> +	return size;
> +}
> +
(except that the first += should be |= ).

Yeah, I know, it's nitpicking. Regard it as a testimony to the code's quality that I didn't see worse. ;-)

NB, it might be a good idea to exit with an error if the shift exceeds 31.
-- 
Matthias Urlichs   |   {M:U} IT Design @ m-u-it.de   |  smurf@smurf.noris.de
Disclaimer: The quote was selected randomly. Really. | http://smurf.noris.de
 - -
When it's fall in New York, the air smells as if someone's been frying
goats in it, and if you are keen to breathe the best plan is to open a
window and stick your head in a building.
		-- The Hitch Hiker's Guide to the Galaxy
Junio C Hamano· Jun 29, 2005, 07:32 UTC · re: Nicolas Pitre · lore

[PATCH] Fixlets on top of Nico's clean-up.

Do we care about a corner case where delta_size is not smaller than 4 bytes, *datap passed to get_delta_hdr_size() points at some bytes before a mmaped page boundary, and all the bytes til the end of that page have high-bit set? The said inline function would step over the page boundary and would presumably segfault/sigbus.

Other than that, with the attached patch on top of it, I find your clean-up very pleasant to read. Please consider it acked.

------------ If we prefer 0 as maxsize for diff_delta() to say "unlimited", let's be consistent about it.

This patch also fixes type mismatch in a call to get_delta_hdr_size() from packed_delta_info().

Signed-off-by: Junio C Hamano <junkio@cox.net>
---
 diffcore-break.c |    2 +-
 sha1_file.c      |   11 ++++-------
 test-delta.c     |    2 +-
 3 files changed, 6 insertions(+), 9 deletions(-)
Show changes to 3 files +6 −9

diffcore-break.c, sha1_file.c, test-delta.c

diff --git a/diffcore-break.c b/diffcore-break.c
--- a/diffcore-break.c
+++ b/diffcore-break.c
@@ -65,7 +65,7 @@ static int should_break(struct diff_file
 
 	delta = diff_delta(src->data, src->size,
 			   dst->data, dst->size,
-			   &delta_size, ~0UL);
+			   &delta_size, 0);
 
 	/* Estimate the edit size by interpreting delta. */
 	if (count_delta(delta, delta_size,
diff --git a/sha1_file.c b/sha1_file.c
--- a/sha1_file.c
+++ b/sha1_file.c
@@ -598,9 +598,9 @@ static int packed_delta_info(unsigned ch
 			     char *type,
 			     unsigned long *sizep)
 {
-	unsigned char *data;
+	const unsigned char *data;
 	unsigned char delta_head[64];
-	unsigned long data_size, result_size, base_size, verify_base_size;
+	unsigned long result_size, base_size, verify_base_size;
 	z_stream stream;
 	int st;
 
@@ -609,13 +609,10 @@ static int packed_delta_info(unsigned ch
 	if (sha1_object_info(base_sha1, type, &base_size))
 		die("cannot get info for delta-pack base");
 
-	data = base_sha1 + 20;
-	data_size = left - 20;
-
 	memset(&stream, 0, sizeof(stream));
 
-	stream.next_in = data;
-	stream.avail_in = data_size;
+	data = stream.next_in = base_sha1 + 20;
+	stream.avail_in = left - 20;
 	stream.next_out = delta_head;
 	stream.avail_out = sizeof(delta_head);
 
diff --git a/test-delta.c b/test-delta.c
--- a/test-delta.c
+++ b/test-delta.c
@@ -61,7 +61,7 @@ int main(int argc, char *argv[])
 	if (argv[1][1] == 'd')
 		out_buf = diff_delta(from_buf, from_size,
 				     data_buf, data_size,
-				     &out_size, ~0UL);
+				     &out_size, 0);
 	else
 		out_buf = patch_delta(from_buf, from_size,
 				      data_buf, data_size,
------------------------------------------------

← back to recent threads