git/list[1] front-page[2] threads[3] people[4] search[5] about
 

Re: [PATCH 1/4] index-pack: Refactor base arguments of resolve_delta into a struct

From
Nicolas Pitre <nico@cam.org>
Date
Jul 15, 2008, 02:40 UTC
Message-ID
<alpine.LFD.1.10.0807142239390.12484@xanadu.home>
In-Reply-To
<1216001267-33235-2-git-send-email-spearce@spearce.org>
On Sun, 13 Jul 2008, Shawn O. Pearce wrote:
Show 9 quoted lines
> We need to discard base objects which are not recently used if our
> memory gets low, such as when we are unpacking a long delta chain
> of a very large object.
> 
> To support tracking the available base objects we combine the
> pointer and size into a struct.  Future changes would allow the
> data pointer to be free'd and marked NULL if memory gets low.
> 
> Signed-off-by: Shawn O. Pearce <spearce@spearce.org>
ACK
Show 172 quoted lines
> ---
>  index-pack.c |   60 +++++++++++++++++++++++++++++++--------------------------
>  1 files changed, 33 insertions(+), 27 deletions(-)
> 
> diff --git a/index-pack.c b/index-pack.c
> index 25db5db..db03478 100644
> --- a/index-pack.c
> +++ b/index-pack.c
> @@ -26,6 +26,11 @@ union delta_base {
>  	off_t offset;
>  };
>  
> +struct base_data {
> +	void *data;
> +	unsigned long size;
> +};
> +
>  /*
>   * Even if sizeof(union delta_base) == 24 on 64-bit archs, we really want
>   * to memcmp() only the first 20 bytes.
> @@ -426,25 +431,25 @@ static void sha1_object(const void *data, unsigned long size,
>  	}
>  }
>  
> -static void resolve_delta(struct object_entry *delta_obj, void *base_data,
> -			  unsigned long base_size, enum object_type type)
> +static void resolve_delta(struct object_entry *delta_obj,
> +			  struct base_data *base_obj, enum object_type type)
>  {
>  	void *delta_data;
>  	unsigned long delta_size;
> -	void *result;
> -	unsigned long result_size;
>  	union delta_base delta_base;
>  	int j, first, last;
> +	struct base_data result;
>  
>  	delta_obj->real_type = type;
>  	delta_data = get_data_from_pack(delta_obj);
>  	delta_size = delta_obj->size;
> -	result = patch_delta(base_data, base_size, delta_data, delta_size,
> -			     &result_size);
> +	result.data = patch_delta(base_obj->data, base_obj->size,
> +			     delta_data, delta_size,
> +			     &result.size);
>  	free(delta_data);
> -	if (!result)
> +	if (!result.data)
>  		bad_object(delta_obj->idx.offset, "failed to apply delta");
> -	sha1_object(result, result_size, type, delta_obj->idx.sha1);
> +	sha1_object(result.data, result.size, type, delta_obj->idx.sha1);
>  	nr_resolved_deltas++;
>  
>  	hashcpy(delta_base.sha1, delta_obj->idx.sha1);
> @@ -452,7 +457,7 @@ static void resolve_delta(struct object_entry *delta_obj, void *base_data,
>  		for (j = first; j <= last; j++) {
>  			struct object_entry *child = objects + deltas[j].obj_no;
>  			if (child->real_type == OBJ_REF_DELTA)
> -				resolve_delta(child, result, result_size, type);
> +				resolve_delta(child, &result, type);
>  		}
>  	}
>  
> @@ -462,11 +467,11 @@ static void resolve_delta(struct object_entry *delta_obj, void *base_data,
>  		for (j = first; j <= last; j++) {
>  			struct object_entry *child = objects + deltas[j].obj_no;
>  			if (child->real_type == OBJ_OFS_DELTA)
> -				resolve_delta(child, result, result_size, type);
> +				resolve_delta(child, &result, type);
>  		}
>  	}
>  
> -	free(result);
> +	free(result.data);
>  }
>  
>  static int compare_delta_entry(const void *a, const void *b)
> @@ -481,7 +486,6 @@ static void parse_pack_objects(unsigned char *sha1)
>  {
>  	int i;
>  	struct delta_entry *delta = deltas;
> -	void *data;
>  	struct stat st;
>  
>  	/*
> @@ -496,7 +500,7 @@ static void parse_pack_objects(unsigned char *sha1)
>  				nr_objects);
>  	for (i = 0; i < nr_objects; i++) {
>  		struct object_entry *obj = &objects[i];
> -		data = unpack_raw_entry(obj, &delta->base);
> +		void *data = unpack_raw_entry(obj, &delta->base);
>  		obj->real_type = obj->type;
>  		if (obj->type == OBJ_REF_DELTA || obj->type == OBJ_OFS_DELTA) {
>  			nr_deltas++;
> @@ -545,6 +549,7 @@ static void parse_pack_objects(unsigned char *sha1)
>  		struct object_entry *obj = &objects[i];
>  		union delta_base base;
>  		int j, ref, ref_first, ref_last, ofs, ofs_first, ofs_last;
> +		struct base_data base_obj;
>  
>  		if (obj->type == OBJ_REF_DELTA || obj->type == OBJ_OFS_DELTA)
>  			continue;
> @@ -555,22 +560,22 @@ static void parse_pack_objects(unsigned char *sha1)
>  		ofs = !find_delta_children(&base, &ofs_first, &ofs_last);
>  		if (!ref && !ofs)
>  			continue;
> -		data = get_data_from_pack(obj);
> +		base_obj.data = get_data_from_pack(obj);
> +		base_obj.size = obj->size;
> +
>  		if (ref)
>  			for (j = ref_first; j <= ref_last; j++) {
>  				struct object_entry *child = objects + deltas[j].obj_no;
>  				if (child->real_type == OBJ_REF_DELTA)
> -					resolve_delta(child, data,
> -						      obj->size, obj->type);
> +					resolve_delta(child, &base_obj, obj->type);
>  			}
>  		if (ofs)
>  			for (j = ofs_first; j <= ofs_last; j++) {
>  				struct object_entry *child = objects + deltas[j].obj_no;
>  				if (child->real_type == OBJ_OFS_DELTA)
> -					resolve_delta(child, data,
> -						      obj->size, obj->type);
> +					resolve_delta(child, &base_obj, obj->type);
>  			}
> -		free(data);
> +		free(base_obj.data);
>  		display_progress(progress, nr_resolved_deltas);
>  	}
>  }
> @@ -656,28 +661,29 @@ static void fix_unresolved_deltas(int nr_unresolved)
>  
>  	for (i = 0; i < n; i++) {
>  		struct delta_entry *d = sorted_by_pos[i];
> -		void *data;
> -		unsigned long size;
>  		enum object_type type;
>  		int j, first, last;
> +		struct base_data base_obj;
>  
>  		if (objects[d->obj_no].real_type != OBJ_REF_DELTA)
>  			continue;
> -		data = read_sha1_file(d->base.sha1, &type, &size);
> -		if (!data)
> +		base_obj.data = read_sha1_file(d->base.sha1, &type, &base_obj.size);
> +		if (!base_obj.data)
>  			continue;
>  
>  		find_delta_children(&d->base, &first, &last);
>  		for (j = first; j <= last; j++) {
>  			struct object_entry *child = objects + deltas[j].obj_no;
>  			if (child->real_type == OBJ_REF_DELTA)
> -				resolve_delta(child, data, size, type);
> +				resolve_delta(child, &base_obj, type);
>  		}
>  
> -		if (check_sha1_signature(d->base.sha1, data, size, typename(type)))
> +		if (check_sha1_signature(d->base.sha1, base_obj.data,
> +				base_obj.size, typename(type)))
>  			die("local object %s is corrupt", sha1_to_hex(d->base.sha1));
> -		append_obj_to_pack(d->base.sha1, data, size, type);
> -		free(data);
> +		append_obj_to_pack(d->base.sha1, base_obj.data,
> +			base_obj.size, type);
> +		free(base_obj.data);
>  		display_progress(progress, nr_resolved_deltas);
>  	}
>  	free(sorted_by_pos);
> -- 
> 1.5.6.2.393.g45096
> 
Nicolas
Previous: Nicolas PitreNext: Nicolas Pitre
Message 31 of 51 in “git pull is slow”
  1. Stephan HennigJul 10, 2008
  2. Martin LanghoffJul 10, 2008
  3. Petr BaudisJul 10, 2008
  4. Johannes SixtJul 10, 2008
  5. Stephan HennigJul 10, 2008
  6. Petr BaudisJul 10, 2008
  7. Stephan HennigJul 10, 2008
  8. Stephan HennigJul 11, 2008
  9. Andreas EricssonJul 11, 2008
  10. Johannes SchindelinJul 11, 2008
  11. Stephan HennigJul 12, 2008
  12. Johannes SchindelinJul 12, 2008
  13. Shawn O. PearceJul 13, 2008
  14. Johannes SchindelinJul 13, 2008
  15. Shawn O. PearceJul 13, 2008
  16. 0/4 Honor core.deltaBaseCacheLimit during index-packShawn O. Pearce, Jul 14, 2008
  17. 1/4 index-pack: Refactor base arguments of resolve_delta into a structShawn O. Pearce, Jul 14, 2008
  18. 2/4 index-pack: Chain the struct base_data on the stack for traversalShawn O. Pearce, Jul 14, 2008
  19. 3/4 index-pack: Track the object_entry that creates each base_dataShawn O. Pearce, Jul 14, 2008
  20. 4/4 index-pack: Honor core.deltaBaseCacheLimit when resolving deltasShawn O. Pearce, Jul 14, 2008
  21. Nicolas PitreJul 15, 2008
  22. Shawn O. PearceJul 15, 2008
  23. index-pack: Honor core.deltaBaseCacheLimit when resolving deltasShawn O. Pearce, Jul 15, 2008
  24. Nicolas PitreJul 15, 2008
  25. Junio C HamanoJul 15, 2008
  26. Johannes SchindelinJul 14, 2008
  27. Nicolas PitreJul 15, 2008
  28. Shawn O. PearceJul 15, 2008
  29. Nicolas PitreJul 15, 2008
  30. Nicolas PitreJul 15, 2008
  31. Nicolas PitreJul 15, 2008
  32. Nicolas PitreJul 14, 2008
  33. Shawn O. PearceJul 14, 2008
  34. Johannes SchindelinJul 14, 2008
  35. Jakub NarebskiJul 14, 2008
  36. Johannes SchindelinJul 14, 2008
  37. Andreas EricssonJul 14, 2008
  38. Johannes SchindelinJul 14, 2008
  39. Andreas EricssonJul 14, 2008
  40. Johannes SchindelinJul 14, 2008
  41. Nicolas PitreJul 15, 2008
  42. Shawn O. PearceJul 15, 2008
  43. Nicolas PitreJul 15, 2008
  44. Stephan HennigJul 17, 2008
  45. Nicolas PitreJul 17, 2008
  46. Shawn O. PearceJul 17, 2008
  47. index-pack: Issue a warning if deltaBaseCacheLimit is too smallShawn O. Pearce, Jul 17, 2008
  48. Nicolas PitreJul 17, 2008
  49. Shawn O. PearceJul 15, 2008
  50. Stephan HennigJul 13, 2008
  51. Stephan HennigJul 11, 2008

Read the whole thread, see it on lore, or plain text.

$ cat FOOTERMessages come from the public archive at lore.kernel.org/git, fetched every hour. The front page is chosen and written each morning by an AI editor and can be wrong; the threads themselves are the record. About and API. For agents: an MCP server at https://gitlist.dev/mcp, and any thread, story or person page as Markdown by adding .md to its URL (or sending Accept: text/markdown). Details in /llms.txt.