git/list[1] front-page[2] threads[3] people[4] search[5] about
 

[PATCH 2/7] archive: add user-configurable tar-filter infrastructure

From
Jeff King <peff@peff.net>
Date
Jun 15, 2011, 22:33 UTC
Message-ID
<20110615223301.GB16807@sigill.intra.peff.net>
In-Reply-To
<20110615223030.GA16110@sigill.intra.peff.net>

Archive supports two output formats: tar and zip. The tar files are totally uncompressed, and it is up to the user to pipe them through gzip or similar. This is no more than a minor inconvenience when running archive locally. However, for remote calls to upload-archive, having the server do the compression can save a lot of bandwidth.

This patch lays the foundation for tar filters; it parses user configuration into an internal representation, but doesn't yet do anything with the result.

It also introduces some tests to document the intended usage.

Signed-off-by: Jeff King <peff@peff.net>
---
 Makefile             |    1 +
 archive-tar-filter.c |  112 ++++++++++++++++++++++++++++++++++++++++++++++++++
 archive.c            |    1 +
 archive.h            |   15 +++++++
 t/t5000-tar-tree.sh  |   54 ++++++++++++++++++++++++
 5 files changed, 183 insertions(+), 0 deletions(-)
 create mode 100644 archive-tar-filter.c
diff --git a/Makefile b/Makefile
index e40ac0c..bd3002f 100644
--- a/Makefile
+++ b/Makefile
@@ -573,6 +573,7 @@ LIB_OBJS += advice.o
 LIB_OBJS += alias.o
 LIB_OBJS += alloc.o
 LIB_OBJS += archive.o
+LIB_OBJS += archive-tar-filter.o
 LIB_OBJS += archive-tar.o
 LIB_OBJS += archive-zip.o
 LIB_OBJS += attr.o
diff --git a/archive-tar-filter.c b/archive-tar-filter.c
new file mode 100644
index 0000000..211f1df
--- /dev/null
+++ b/archive-tar-filter.c
@@ -0,0 +1,112 @@
+#include "cache.h"
+#include "archive.h"
+
+struct tar_filter *tar_filters;
+static struct tar_filter **tar_filters_tail = &tar_filters;
+
+static struct tar_filter *tar_filter_new(const char *name, int namelen)
+{
+	struct tar_filter *tf;
+	tf = xcalloc(1, sizeof(*tf));
+	tf->name = xmemdupz(name, namelen);
+	tf->extensions.strdup_strings = 1;
+	*tar_filters_tail = tf;
+	tar_filters_tail = &tf->next;
+	return tf;
+}
+
+static void tar_filter_free(struct tar_filter *tf)
+{
+	string_list_clear(&tf->extensions, 0);
+	free(tf->name);
+	free(tf->command);
+	free(tf);
+}
+
+static struct tar_filter *tar_filter_by_namelen(const char *name,
+						int len)
+{
+	struct tar_filter *p;
+	for (p = tar_filters; p; p = p->next)
+		if (!strncmp(p->name, name, len) && !p->name[len])
+			return p;
+	return NULL;
+}
+
+struct tar_filter *tar_filter_by_name(const char *name)
+{
+	return tar_filter_by_namelen(name, strlen(name));
+}
+
+static int tar_filter_config(const char *var, const char *value, void *data)
+{
+	struct tar_filter *tf;
+	const char *dot;
+	const char *name;
+	const char *type;
+	int namelen;
+
+	if (prefixcmp(var, "tarfilter."))
+		return 0;
+	dot = strrchr(var, '.');
+	if (dot == var + 9)
+		return 0;
+
+	name = var + 10;
+	namelen = dot - name;
+	type = dot + 1;
+
+	tf = tar_filter_by_namelen(name, namelen);
+	if (!tf)
+		tf = tar_filter_new(name, namelen);
+
+	if (!strcmp(type, "command")) {
+		if (!value)
+			return config_error_nonbool(var);
+		tf->command = xstrdup(value);
+		return 0;
+	}
+	else if (!strcmp(type, "extension")) {
+		if (!value)
+			return config_error_nonbool(var);
+		string_list_append(&tf->extensions, value);
+		return 0;
+	}
+	else if (!strcmp(type, "compressionlevels")) {
+		tf->use_compression = git_config_bool(var, value);
+		return 0;
+	}
+
+	return 0;
+}
+
+static void remove_filters_without_command(void)
+{
+	struct tar_filter *p = tar_filters;
+	struct tar_filter **last = &tar_filters;
+
+	while (p) {
+		if (p->command && *p->command)
+			last = &p->next;
+		else {
+			*last = p->next;
+			tar_filter_free(p);
+		}
+		p = *last;
+	}
+}
+
+/*
+ * We don't want to load twice, since some of our
+ * values actually append rather than overwrite.
+ */
+static int tar_filter_config_loaded;
+extern void tar_filter_load_config(void)
+{
+	if (tar_filter_config_loaded)
+		return;
+	tar_filter_config_loaded = 1;
+
+	git_config(tar_filter_config, NULL);
+	remove_filters_without_command();
+}
diff --git a/archive.c b/archive.c
index 2616676..2ed9259 100644
--- a/archive.c
+++ b/archive.c
@@ -395,6 +395,7 @@ int write_archive(int argc, const char **argv, const char *prefix,
 		prefix = setup_git_directory_gently(&nongit);
 
 	git_config(git_default_config, NULL);
+	tar_filter_load_config();
 
 	argc = parse_archive_args(argc, argv, &ar, &args);
 	if (nongit) {
diff --git a/archive.h b/archive.h
index 038ac35..8386c46 100644
--- a/archive.h
+++ b/archive.h
@@ -1,6 +1,8 @@
 #ifndef ARCHIVE_H
 #define ARCHIVE_H
 
+#include "string-list.h"
+
 struct archiver_args {
 	const char *base;
 	size_t baselen;
@@ -27,4 +29,17 @@ extern int write_zip_archive(struct archiver_args *);
 extern int write_archive_entries(struct archiver_args *args, write_archive_entry_fn_t write_entry);
 extern int write_archive(int argc, const char **argv, const char *prefix, int setup_prefix);
 
+struct tar_filter {
+	char *name;
+	char *command;
+	struct string_list extensions;
+	unsigned use_compression:1;
+	struct tar_filter *next;
+};
+
+extern struct tar_filter *tar_filters;
+extern struct tar_filter *tar_filter_by_name(const char *name);
+
+extern void tar_filter_load_config(void);
+
 #endif	/* ARCHIVE_H */
diff --git a/t/t5000-tar-tree.sh b/t/t5000-tar-tree.sh
index cff1b3e..c3e1a4e 100755
--- a/t/t5000-tar-tree.sh
+++ b/t/t5000-tar-tree.sh
@@ -252,4 +252,58 @@ test_expect_success 'git-archive --prefix=olde-' '
 	test -f h/olde-a/bin/sh
 '
 
+test_expect_success 'setup fake tar filter' '
+	git config tarfilter.fake.command "cat >/dev/null; echo args: "
+'
+
+test_expect_failure 'filter does not allow compression levels by default' '
+	test_must_fail git archive --format=fake -9 HEAD >output
+'
+
+test_expect_failure 'filters can allow compression levels' '
+	git config tarfilter.fake.compressionlevels true &&
+	echo "args: -9" >expect &&
+	git archive --format=fake -9 HEAD >output &&
+	test_cmp expect output
+'
+
+test_expect_failure 'archive --list mentions user filter' '
+	git archive --list >output &&
+	grep "^fake\$" output
+'
+
+test_expect_failure 'archive --list shows remote user filters' '
+	git archive --list --remote=. >output &&
+	grep "^fake\$" output
+'
+
+test_expect_success 'setup slightly more useful tar filter' '
+	git config tarfilter.foo.command "tr ab ba" &&
+	git config --add tarfilter.foo.extension tar.foo &&
+	git config --add tarfilter.foo.extension bar
+'
+
+test_expect_failure 'archive outputs in configurable format' '
+	git archive --format=foo HEAD >config.tar.foo &&
+	tr ab ba <config.tar.foo >config.tar &&
+	test_cmp b.tar config.tar
+'
+
+test_expect_failure 'archive selects implicit format by configured extension' '
+	git archive -o config-implicit.tar.foo HEAD &&
+	test_cmp config.tar.foo config-implicit.tar.foo &&
+	git archive -o config-implicit.bar HEAD &&
+	test_cmp config.tar.foo config-implicit.bar
+'
+
+test_expect_success 'default output format remains tar' '
+	git archive -o config-implicit.baz HEAD &&
+	test_cmp b.tar config-implicit.baz
+'
+
+test_expect_failure 'extension matching requires dot' '
+	git archive -o config-implicittar.foo HEAD &&
+	test_cmp b.tar config-implicittar.foo
+'
+
 test_done

-- 
1.7.6.rc1.4.g49204
Previous: Jeff KingNext: Junio C Hamano
Message 10 of 56 in “archive: factor out write phase of tar format”
  1. 1/2 archive: factor out write phase of tar formatJeff King, Jun 14, 2011
  2. 2/2 archive: support gzipped tar filesJeff King, Jun 14, 2011
  3. J.H.Jun 14, 2011
  4. Jeff KingJun 14, 2011
  5. René ScharfeJun 14, 2011
  6. Jeff KingJun 14, 2011
  7. Jeff KingJun 14, 2011
  8. 0/7 user-configurable git-archive output formatsJeff King, Jun 15, 2011
  9. 1/7 archive: reorder option parsing and config readingJeff King, Jun 15, 2011
  10. 2/7 archive: add user-configurable tar-filter infrastructureJeff King, Jun 15, 2011
  11. Junio C HamanoJun 15, 2011
  12. Jeff KingJun 16, 2011
  13. 3/7 archive: support user tar-filters via --formatJeff King, Jun 15, 2011
  14. 4/7 archive: advertise user tar-filters in --listJeff King, Jun 15, 2011
  15. 5/7 archive: refactor format-guessing from filenameJeff King, Jun 15, 2011
  16. Junio C HamanoJun 15, 2011
  17. Jeff KingJun 16, 2011
  18. 6/7 archive: match extensions from user-configured formatsJeff King, Jun 15, 2011
  19. 7/7 archive: provide builtin .tar.gz filterJeff King, Jun 15, 2011
  20. Junio C HamanoJun 15, 2011
  21. Junio C HamanoJun 15, 2011
  22. Jeff KingJun 16, 2011
  23. Junio C HamanoJun 16, 2011
  24. Jeff KingJun 16, 2011
  25. Chris WebbJun 16, 2011
  26. Jeff KingJun 16, 2011
  27. Junio C HamanoJun 16, 2011
  28. Jeff KingJun 16, 2011
  29. John SzakmeisterJun 16, 2011
  30. Junio C HamanoJun 16, 2011
  31. Jeff KingJun 16, 2011
  32. René ScharfeJun 18, 2011
  33. Jakub NarebskiJun 18, 2011
  34. Junio C HamanoJun 20, 2011
  35. 0/9 configurable tar compressorsJeff King, Jun 22, 2011
  36. 1/9 archive: reorder option parsing and config readingJeff King, Jun 22, 2011
  37. 2/9 archive-tar: don't reload default config optionsJeff King, Jun 22, 2011
  38. 3/9 archive: refactor list of archive formatsJeff King, Jun 22, 2011
  39. Thiago FarinaJun 23, 2011
  40. Jeff KingJun 23, 2011
  41. 4/9 archive: pass archiver struct to write_archive callbackJeff King, Jun 22, 2011
  42. 5/9 archive: move file extension format-guessing lowerJeff King, Jun 22, 2011
  43. 6/9 archive: refactor file extension format-guessingJeff King, Jun 22, 2011
  44. 7/9 archive: implement configurable tar filtersJeff King, Jun 22, 2011
  45. Jeff KingJun 22, 2011
  46. René ScharfeJun 22, 2011
  47. Jeff KingJun 22, 2011
  48. 8/9 archive: provide builtin .tar.gz filterJeff King, Jun 22, 2011
  49. 9/9 upload-archive: allow user to turn off filtersJeff King, Jun 22, 2011
  50. Jeff KingJun 22, 2011
  51. Jeff KingJun 21, 2011
  52. René ScharfeJun 18, 2011
  53. Junio C HamanoJun 14, 2011
  54. Jeff KingJun 14, 2011
  55. Miles BaderJun 14, 2011
  56. Jeff KingJun 15, 2011

Read the whole thread, see it on lore, or plain text.

$ cat FOOTERMessages come from the public archive at lore.kernel.org/git, fetched every hour. The front page is chosen and written each morning by an AI editor and can be wrong; the threads themselves are the record. About and API. For agents: an MCP server at https://gitlist.dev/mcp, and any thread, story or person page as Markdown by adding .md to its URL (or sending Accept: text/markdown). Details in /llms.txt.