git/list[1] front-page[2] threads[3] people[4] search[5] about
 

[PATCHv2 7/9] archive: implement configurable tar filters

From
Jeff King <peff@peff.net>
Date
Jun 22, 2011, 01:26 UTC
Message-ID
<20110622012631.GG30604@sigill.intra.peff.net>
In-Reply-To
<20110622011923.GA30370@sigill.intra.peff.net>

It's common to pipe the tar output produce by "git archive" through gzip or some other compressor. Locally, this can easily be done by using a shell pipe. When requesting a remote archive, though, it cannot be done through the upload-archive interface.

This patch allows configurable tar filters, so that one could define a "tar.gz" format that automatically pipes tar output through gzip.

Signed-off-by: Jeff King <peff@peff.net>
---
This was split across several commits in the previous version of the
series, but due to the cleanups it fits nicely into a single commit.
 Documentation/git-archive.txt |   16 ++++++
 archive-tar.c                 |  107 ++++++++++++++++++++++++++++++++++++++++-
 t/t5000-tar-tree.sh           |   43 ++++++++++++++++
 3 files changed, 165 insertions(+), 1 deletions(-)
diff --git a/Documentation/git-archive.txt b/Documentation/git-archive.txt
index 9c750e2..726bf63 100644
--- a/Documentation/git-archive.txt
+++ b/Documentation/git-archive.txt
@@ -101,6 +101,16 @@ tar.umask::
 	details.  If `--remote` is used then only the configuration of
 	the remote repository takes effect.
 
+tar.<format>.command::
+	This variable specifies a shell command through which the tar
+	output generated by `git archive` should be piped. The command
+	is executed using the shell with the generated tar file on its
+	standard input, and should produce the final output on its
+	standard output. Any compression-level options will be passed
+	to the command (e.g., "-9"). An output file with the same
+	extension as `<format>` will be use this format if no other
+	format is given.
+
 ATTRIBUTES
 ----------
 
@@ -149,6 +159,12 @@ git archive -o latest.zip HEAD::
 	commit on the current branch. Note that the output format is
 	inferred by the extension of the output file.
 
+git config tar.tar.xz.command "xz -c"::
+
+	Configure a "tar.xz" format for making LZMA-compressed tarfiles.
+	You can use it specifying `--format=tar.xz`, or by creating an
+	output file like `-o foo.tar.xz`.
+
 
 SEE ALSO
 --------
diff --git a/archive-tar.c b/archive-tar.c
index bed9a9b..5c30747 100644
--- a/archive-tar.c
+++ b/archive-tar.c
@@ -4,6 +4,7 @@
 #include "cache.h"
 #include "tar.h"
 #include "archive.h"
+#include "run-command.h"
 
 #define RECORDSIZE	(512)
 #define BLOCKSIZE	(RECORDSIZE * 20)
@@ -13,6 +14,9 @@ static unsigned long offset;
 
 static int tar_umask = 002;
 
+static int write_tar_filter_archive(const struct archiver *ar,
+				    struct archiver_args *args);
+
 /* writes out the whole block, but only if it is full */
 static void write_if_needed(void)
 {
@@ -220,6 +224,60 @@ static int write_global_extended_header(struct archiver_args *args)
 	return err;
 }
 
+static struct archiver **tar_filters;
+static int nr_tar_filters;
+static int alloc_tar_filters;
+
+static struct archiver *find_tar_filter(const char *name, int len)
+{
+	int i;
+	for (i = 0; i < nr_tar_filters; i++) {
+		struct archiver *ar = tar_filters[i];
+		if (!strncmp(ar->name, name, len) && !ar->name[len])
+			return ar;
+	}
+	return NULL;
+}
+
+static int tar_filter_config(const char *var, const char *value, void *data)
+{
+	struct archiver *ar;
+	const char *dot;
+	const char *name;
+	const char *type;
+	int namelen;
+
+	if (prefixcmp(var, "tar."))
+		return 0;
+	dot = strrchr(var, '.');
+	if (dot == var + 9)
+		return 0;
+
+	name = var + 4;
+	namelen = dot - name;
+	type = dot + 1;
+
+	ar = find_tar_filter(name, namelen);
+	if (!ar) {
+		ar = xcalloc(1, sizeof(*ar));
+		ar->name = xmemdupz(name, namelen);
+		ar->write_archive = write_tar_filter_archive;
+		ar->flags = ARCHIVER_WANT_COMPRESSION_LEVELS;
+		ALLOC_GROW(tar_filters, nr_tar_filters + 1, alloc_tar_filters);
+		tar_filters[nr_tar_filters++] = ar;
+	}
+
+	if (!strcmp(type, "command")) {
+		if (!value)
+			return config_error_nonbool(var);
+		free(ar->data);
+		ar->data = xstrdup(value);
+		return 0;
+	}
+
+	return 0;
+}
+
 static int git_tar_config(const char *var, const char *value, void *cb)
 {
 	if (!strcmp(var, "tar.umask")) {
@@ -231,7 +289,8 @@ static int git_tar_config(const char *var, const char *value, void *cb)
 		}
 		return 0;
 	}
-	return 0;
+
+	return tar_filter_config(var, value, cb);
 }
 
 static int write_tar_archive(const struct archiver *ar,
@@ -248,6 +307,45 @@ static int write_tar_archive(const struct archiver *ar,
 	return err;
 }
 
+static int write_tar_filter_archive(const struct archiver *ar,
+				    struct archiver_args *args)
+{
+	struct strbuf cmd = STRBUF_INIT;
+	struct child_process filter;
+	const char *argv[2];
+	int r;
+
+	if (!ar->data)
+		die("BUG: tar-filter archiver called with no filter defined");
+
+	strbuf_addstr(&cmd, ar->data);
+	if (args->compression_level >= 0)
+		strbuf_addf(&cmd, " -%d", args->compression_level);
+
+	memset(&filter, 0, sizeof(filter));
+	argv[0] = cmd.buf;
+	argv[1] = NULL;
+	filter.argv = argv;
+	filter.use_shell = 1;
+	filter.in = -1;
+
+	if (start_command(&filter) < 0)
+		die_errno("unable to start '%s' filter", argv[0]);
+	close(1);
+	if (dup2(filter.in, 1) < 0)
+		die_errno("unable to redirect descriptor");
+	close(filter.in);
+
+	r = write_tar_archive(ar, args);
+
+	close(1);
+	if (finish_command(&filter) != 0)
+		die("'%s' filter reported error", argv[0]);
+
+	strbuf_release(&cmd);
+	return r;
+}
+
 static struct archiver tar_archiver = {
 	"tar",
 	write_tar_archive,
@@ -256,6 +354,13 @@ static struct archiver tar_archiver = {
 
 void init_tar_archiver(void)
 {
+	int i;
 	register_archiver(&tar_archiver);
+
 	git_config(git_tar_config, NULL);
+	for (i = 0; i < nr_tar_filters; i++) {
+		/* omit any filters that never had a command configured */
+		if (tar_filters[i]->data)
+			register_archiver(tar_filters[i]);
+	}
 }
diff --git a/t/t5000-tar-tree.sh b/t/t5000-tar-tree.sh
index cff1b3e..1f90692 100755
--- a/t/t5000-tar-tree.sh
+++ b/t/t5000-tar-tree.sh
@@ -252,4 +252,47 @@ test_expect_success 'git-archive --prefix=olde-' '
 	test -f h/olde-a/bin/sh
 '
 
+test_expect_success 'setup tar filters' '
+	git config tar.tar.foo.command "tr ab ba" &&
+	git config tar.bar.command "tr ab ba"
+'
+
+test_expect_success 'archive --list mentions user filter' '
+	git archive --list >output &&
+	grep "^tar\.foo\$" output &&
+	grep "^bar\$" output
+'
+
+test_expect_success 'archive --list shows remote user filters' '
+	git archive --list --remote=. >output &&
+	grep "^tar\.foo\$" output &&
+	grep "^bar\$" output
+'
+
+test_expect_success 'invoke tar filter by format' '
+	git archive --format=tar.foo HEAD >config.tar.foo &&
+	tr ab ba <config.tar.foo >config.tar &&
+	test_cmp b.tar config.tar &&
+	git archive --format=bar HEAD >config.bar &&
+	tr ab ba <config.bar >config.tar &&
+	test_cmp b.tar config.tar
+'
+
+test_expect_success 'invoke tar filter by extension' '
+	git archive -o config-implicit.tar.foo HEAD &&
+	test_cmp config.tar.foo config-implicit.tar.foo &&
+	git archive -o config-implicit.bar HEAD &&
+	test_cmp config.tar.foo config-implicit.bar
+'
+
+test_expect_success 'default output format remains tar' '
+	git archive -o config-implicit.baz HEAD &&
+	test_cmp b.tar config-implicit.baz
+'
+
+test_expect_success 'extension matching requires dot' '
+	git archive -o config-implicittar.foo HEAD &&
+	test_cmp b.tar config-implicittar.foo
+'
+
 test_done

-- 
1.7.5.4.44.g4b107
Previous: Jeff KingNext: Jeff King
Message 44 of 56 in “archive: factor out write phase of tar format”
  1. 1/2 archive: factor out write phase of tar formatJeff King, Jun 14, 2011
  2. 2/2 archive: support gzipped tar filesJeff King, Jun 14, 2011
  3. J.H.Jun 14, 2011
  4. Jeff KingJun 14, 2011
  5. René ScharfeJun 14, 2011
  6. Jeff KingJun 14, 2011
  7. Jeff KingJun 14, 2011
  8. 0/7 user-configurable git-archive output formatsJeff King, Jun 15, 2011
  9. 1/7 archive: reorder option parsing and config readingJeff King, Jun 15, 2011
  10. 2/7 archive: add user-configurable tar-filter infrastructureJeff King, Jun 15, 2011
  11. Junio C HamanoJun 15, 2011
  12. Jeff KingJun 16, 2011
  13. 3/7 archive: support user tar-filters via --formatJeff King, Jun 15, 2011
  14. 4/7 archive: advertise user tar-filters in --listJeff King, Jun 15, 2011
  15. 5/7 archive: refactor format-guessing from filenameJeff King, Jun 15, 2011
  16. Junio C HamanoJun 15, 2011
  17. Jeff KingJun 16, 2011
  18. 6/7 archive: match extensions from user-configured formatsJeff King, Jun 15, 2011
  19. 7/7 archive: provide builtin .tar.gz filterJeff King, Jun 15, 2011
  20. Junio C HamanoJun 15, 2011
  21. Junio C HamanoJun 15, 2011
  22. Jeff KingJun 16, 2011
  23. Junio C HamanoJun 16, 2011
  24. Jeff KingJun 16, 2011
  25. Chris WebbJun 16, 2011
  26. Jeff KingJun 16, 2011
  27. Junio C HamanoJun 16, 2011
  28. Jeff KingJun 16, 2011
  29. John SzakmeisterJun 16, 2011
  30. Junio C HamanoJun 16, 2011
  31. Jeff KingJun 16, 2011
  32. René ScharfeJun 18, 2011
  33. Jakub NarebskiJun 18, 2011
  34. Junio C HamanoJun 20, 2011
  35. 0/9 configurable tar compressorsJeff King, Jun 22, 2011
  36. 1/9 archive: reorder option parsing and config readingJeff King, Jun 22, 2011
  37. 2/9 archive-tar: don't reload default config optionsJeff King, Jun 22, 2011
  38. 3/9 archive: refactor list of archive formatsJeff King, Jun 22, 2011
  39. Thiago FarinaJun 23, 2011
  40. Jeff KingJun 23, 2011
  41. 4/9 archive: pass archiver struct to write_archive callbackJeff King, Jun 22, 2011
  42. 5/9 archive: move file extension format-guessing lowerJeff King, Jun 22, 2011
  43. 6/9 archive: refactor file extension format-guessingJeff King, Jun 22, 2011
  44. 7/9 archive: implement configurable tar filtersJeff King, Jun 22, 2011
  45. Jeff KingJun 22, 2011
  46. René ScharfeJun 22, 2011
  47. Jeff KingJun 22, 2011
  48. 8/9 archive: provide builtin .tar.gz filterJeff King, Jun 22, 2011
  49. 9/9 upload-archive: allow user to turn off filtersJeff King, Jun 22, 2011
  50. Jeff KingJun 22, 2011
  51. Jeff KingJun 21, 2011
  52. René ScharfeJun 18, 2011
  53. Junio C HamanoJun 14, 2011
  54. Jeff KingJun 14, 2011
  55. Miles BaderJun 14, 2011
  56. Jeff KingJun 15, 2011

Read the whole thread, see it on lore, or plain text.

$ cat FOOTERMessages come from the public archive at lore.kernel.org/git, fetched every hour. The front page is chosen and written each morning by an AI editor and can be wrong; the threads themselves are the record. About and API. For agents: an MCP server at https://gitlist.dev/mcp, and any thread, story or person page as Markdown by adding .md to its URL (or sending Accept: text/markdown). Details in /llms.txt.