git/list[1] front-page[2] threads[3] people[4] search[5] about
 

Re: [PATCH 3/5] archive: delegate blob reading to backend

From
René Scharfe <rene.scharfe@lsrfire.ath.cx>
Date
Apr 30, 2012, 21:07 UTC
Message-ID
<4F9EFF22.4060105@lsrfire.ath.cx>
In-Reply-To
<1335761837-12482-4-git-send-email-pclouds@gmail.com>
Am 30.04.2012 06:57, schrieb Nguyễn Thái Ngọc Duy:
Show 44 quoted lines
> archive-tar.c and archive-zip.c now perform conversion check, with
> help of sha1_file_to_archive() from archive.c
> 
> This gives backends more freedom in dealing with (streaming) large
> blobs.
> 
> Signed-off-by: Nguyễn Thái Ngọc Duy<pclouds@gmail.com>
> ---
>   archive-tar.c |   20 +++++++++++++++++---
>   archive-zip.c |   14 ++++++++++++--
>   archive.c     |   28 +++++++++++-----------------
>   archive.h     |   10 +++++++++-
>   4 files changed, 49 insertions(+), 23 deletions(-)
> 
> diff --git a/archive-tar.c b/archive-tar.c
> index 6c8a0bd..61821f4 100644
> --- a/archive-tar.c
> +++ b/archive-tar.c
> @@ -161,11 +161,15 @@ static int write_extended_header(struct archiver_args *args,
>   }
> 
>   static int write_tar_entry(struct archiver_args *args,
> -		const unsigned char *sha1, const char *path, size_t pathlen,
> -		unsigned int mode, void *buffer, unsigned long size)
> +			   const unsigned char *sha1,
> +			   const char *path, size_t pathlen,
> +			   unsigned int mode)
>   {
>   	struct ustar_header header;
>   	struct strbuf ext_header = STRBUF_INIT;
> +	unsigned int old_mode = mode;
> +	unsigned long size;
> +	void *buffer;
>   	int err = 0;
> 
>   	memset(&header, 0, sizeof(header));
> @@ -199,7 +203,17 @@ static int write_tar_entry(struct archiver_args *args,
>   	} else
>   		memcpy(header.name, path, pathlen);
> 
> -	if (S_ISLNK(mode)&&  buffer) {
> +	if (S_ISLNK(mode) || S_ISREG(mode)) {
> +		enum object_type type;
> +		buffer = sha1_file_to_archive(args, path, sha1, old_mode,&type,&size);
This buffer is never freed.
Show 45 quoted lines
> +		if (!buffer)
> +			return error("cannot read %s", sha1_to_hex(sha1));
> +	} else {
> +		buffer = NULL;
> +		size = 0;
> +	}
> +
> +	if (S_ISLNK(mode)) {
>   		if (size>  sizeof(header.linkname)) {
>   			sprintf(header.linkname, "see %s.paxheader",
>   			        sha1_to_hex(sha1));
> diff --git a/archive-zip.c b/archive-zip.c
> index 02d1f37..f8039ba 100644
> --- a/archive-zip.c
> +++ b/archive-zip.c
> @@ -121,8 +121,9 @@ static void *zlib_deflate(void *data, unsigned long size,
>   }
> 
>   static int write_zip_entry(struct archiver_args *args,
> -		const unsigned char *sha1, const char *path, size_t pathlen,
> -		unsigned int mode, void *buffer, unsigned long size)
> +			   const unsigned char *sha1,
> +			   const char *path, size_t pathlen,
> +			   unsigned int mode)
>   {
>   	struct zip_local_header header;
>   	struct zip_dir_header dirent;
> @@ -134,6 +135,8 @@ static int write_zip_entry(struct archiver_args *args,
>   	int method;
>   	unsigned char *out;
>   	void *deflated = NULL;
> +	void *buffer;
> +	unsigned long size;
> 
>   	crc = crc32(0, NULL, 0);
> 
> @@ -148,7 +151,14 @@ static int write_zip_entry(struct archiver_args *args,
>   		out = NULL;
>   		uncompressed_size = 0;
>   		compressed_size = 0;
> +		buffer = NULL;
> +		size = 0;
>   	} else if (S_ISREG(mode) || S_ISLNK(mode)) {
> +		enum object_type type;
> +		buffer = sha1_file_to_archive(args, path, sha1, mode,&type,&size);
This one is leaked as well.
Show 6 quoted lines
> +		if (!buffer)
> +			return error("cannot read %s", sha1_to_hex(sha1));
> +
>   		method = 0;
>   		attr2 = S_ISLNK(mode) ? ((mode | 0777)<<  16) :
>   			(mode&  0111) ? ((mode)<<  16) : 0;
You could squash this in:
diff --git a/archive-tar.c b/archive-tar.c
index 61821f4..bb320ad 100644
--- a/archive-tar.c
+++ b/archive-tar.c
@@ -235,6 +235,7 @@ static int write_tar_entry(struct archiver_args *args,
 	write_blocked(&header, sizeof(header));
 	if (S_ISREG(mode) && buffer && size > 0)
 		write_blocked(buffer, size);
+	free(buffer);
 	return err;
 }
 
diff --git a/archive-zip.c b/archive-zip.c
index f8039ba..716cc42 100644
--- a/archive-zip.c
+++ b/archive-zip.c
@@ -239,6 +239,7 @@ static int write_zip_entry(struct archiver_args *args,
 	}
 
 	free(deflated);
+	free(buffer);
 
 	return 0;
 }
Previous: Nguyễn Thái Ngọc DuyNext: Nguyễn Thái Ngọc Duy
Message 7 of 23 in “Large file support for git-archive”
  1. 0/5 Large file support for git-archiveNguyễn Thái Ngọc Duy, Apr 30, 2012
  2. 1/5 archive-tar: turn write_tar_entry into blob-writing onlyNguyễn Thái Ngọc Duy, Apr 30, 2012
  3. Junio C HamanoApr 30, 2012
  4. René ScharfeApr 30, 2012
  5. 2/5 archive-tar: unindent write_tar_entry by one levelNguyễn Thái Ngọc Duy, Apr 30, 2012
  6. 3/5 archive: delegate blob reading to backendNguyễn Thái Ngọc Duy, Apr 30, 2012
  7. René ScharfeApr 30, 2012
  8. 4/5 archive-tar: stream large blobs to tar fileNguyễn Thái Ngọc Duy, Apr 30, 2012
  9. Junio C HamanoApr 30, 2012
  10. René ScharfeApr 30, 2012
  11. Junio C HamanoApr 30, 2012
  12. René ScharfeApr 30, 2012
  13. 5/5 archive-zip: stream large blobs into zip fileNguyễn Thái Ngọc Duy, Apr 30, 2012
  14. Junio C HamanoApr 30, 2012
  15. René ScharfeApr 30, 2012
  16. streaming: void pointer instead of char pointerRené Scharfe, Apr 30, 2012
  17. archive-zip: remove uncompressed_sizeRené Scharfe, Apr 30, 2012
  18. archive-zip: factor out helpers for writing sizes and CRCRené Scharfe, Apr 30, 2012
  19. archive-zip: streaming for stored filesRené Scharfe, Apr 30, 2012
  20. archive-zip: streaming for deflated filesRené Scharfe, Apr 30, 2012
  21. Junio C HamanoApr 30, 2012
  22. René ScharfeApr 30, 2012
  23. Nguyen Thai Ngoc DuyMay 1, 2012

Read the whole thread, see it on lore, or plain text.

$ cat FOOTERMessages come from the public archive at lore.kernel.org/git, fetched every hour. The front page is chosen and written each morning by an AI editor and can be wrong; the threads themselves are the record. About and API. For agents: an MCP server at https://gitlist.dev/mcp, and any thread, story or person page as Markdown by adding .md to its URL (or sending Accept: text/markdown). Details in /llms.txt.