git/list[1] front-page[2] threads[3] people[4] search[5] about
 

[RFC PATCH v2 12/16] Smart fetch and push over HTTP: server side

From
Shawn O. Pearce <spearce@spearce.org>
Date
Oct 13, 2009, 02:25 UTC
Message-ID
<1255400715-10508-13-git-send-email-spearce@spearce.org>
In-Reply-To
<1255400715-10508-1-git-send-email-spearce@spearce.org>

Requests for $GIT_URL/git-receive-pack and $GIT_URL/git-upload-pack are forwarded to the corresponding backend process by directly executing it and leaving stdin and stdout connected to the invoking web server. Prior to starting the backend process the HTTP response headers are sent, thereby freeing the backend from needing to know about the HTTP protocol.

Requests that are encoded with Content-Encoding: gzip are automatically inflated before being streamed into the backend. This is primarily useful for the git-upload-pack backend, which receives highly repetitive text data from clients that easily compresses to 50% of its original size.

Signed-off-by: Shawn O. Pearce <spearce@spearce.org>
---
 http-backend.c |  192 +++++++++++++++++++++++++++++++++++++++++++++++++++++++-
 1 files changed, 191 insertions(+), 1 deletions(-)
diff --git a/http-backend.c b/http-backend.c
index 39cfd25..adb3256 100644
--- a/http-backend.c
+++ b/http-backend.c
@@ -77,6 +77,152 @@ static NORETURN void not_found(const char *err, ...)
 	exit(0);
 }
 
+static NORETURN void forbidden(const char *err, ...)
+{
+	va_list params;
+
+	write_status(403, "Forbidden");
+	write_nocache();
+	end_headers();
+
+	va_start(params, err);
+	if (err && *err) {
+		vsnprintf(buffer, sizeof(buffer), err, params);
+		fprintf(stderr, "%s\n", buffer);
+	}
+	va_end(params);
+	exit(0);
+}
+
+struct http_service {
+	const char *name;
+	const char *config_name;
+	int enabled;
+};
+static struct http_service *service;
+
+static struct http_service http_service[] = {
+	{ "upload-pack", "uploadpack", 1 },
+	{ "receive-pack", "receivepack", 0 },
+};
+
+static int http_config(const char *var, const char *value, void *cb)
+{
+	if (!prefixcmp(var, "http.") &&
+	    !strcmp(var + 5, service->config_name)) {
+		service->enabled = git_config_bool(var, value);
+		return 0;
+	}
+
+	/* we are not interested in parsing any other configuration here */
+	return 0;
+}
+
+static void select_service(const char *name)
+{
+	int i;
+
+	if (prefixcmp(name, "git-"))
+		forbidden("Unsupported service: '%s'", name);
+
+	for (i = 0; i < ARRAY_SIZE(http_service); i++) {
+		service = &http_service[i];
+		if (!strcmp(service->name, name + 4)) {
+			git_config(http_config, NULL);
+			if (!service->enabled)
+				forbidden("Service not enabled: '%s'", name);
+			return;
+		}
+	}
+	forbidden("Unsupported service: '%s'", name);
+}
+
+static void inflate_request(const char *prog_name, int out)
+{
+	z_stream stream;
+	unsigned char in_buf[8192];
+	unsigned char out_buf[8192];
+	unsigned long cnt = 0;
+	int ret;
+
+	memset(&stream, 0, sizeof(stream));
+	ret = inflateInit2(&stream, (15 + 16));
+	if (ret != Z_OK)
+		die("cannot start zlib inflater, zlib err %d", ret);
+
+	while (1) {
+		ssize_t n = xread(0, in_buf, sizeof(in_buf));
+		if (n <= 0)
+			die("request ended in the middle of the gzip stream");
+
+		stream.next_in = in_buf;
+		stream.avail_in = n;
+
+		while (0 < stream.avail_in) {
+			int ret;
+
+			stream.next_out = out_buf;
+			stream.avail_out = sizeof(out_buf);
+
+			ret = inflate(&stream, Z_NO_FLUSH);
+			if (ret != Z_OK && ret != Z_STREAM_END)
+				die("zlib error inflating request, result %d", ret);
+
+			n = stream.total_out - cnt;
+			if (write_in_full(out, out_buf, n) != n)
+				die("%s aborted reading request", prog_name);
+			cnt += n;
+
+			if (ret == Z_STREAM_END)
+				goto done;
+		}
+	}
+
+done:
+	inflateEnd(&stream);
+	close(out);
+}
+
+static void run_service(const char **argv)
+{
+	const char *encoding = getenv("HTTP_CONTENT_ENCODING");
+	int use_gzip = 0;
+	struct child_process cld;
+
+	if (encoding && !strcmp(encoding, "gzip"))
+		use_gzip = 1;
+	else if (encoding && !strcmp(encoding, "x-gzip"))
+		use_gzip = 1;
+
+	memset(&cld, 0, sizeof(cld));
+	cld.argv = argv;
+	if (use_gzip)
+		cld.in = -1;
+	cld.git_cmd = 1;
+	if (start_command(&cld))
+		die_errno("Cannot start %s", argv[0]);
+
+	close(1);
+	if (use_gzip)
+		inflate_request(argv[0], cld.in);
+	else
+		close(0);
+
+	if (finish_command(&cld))
+		die("%s terminated with error", argv[0]);
+}
+
+static void require_content_type(const char *need_type)
+{
+	const char *input_type = getenv("CONTENT_TYPE");
+	if (!input_type || strcmp(input_type, need_type)) {
+		write_status(415, "Unsupported Media Type");
+		write_nocache();
+		end_headers();
+		exit(0);
+	}
+}
+
 static void write_file(const char *the_type, const char *name)
 {
 	const char *p = git_path("%s", name);
@@ -151,6 +297,25 @@ static int show_text_ref(const char *name, const unsigned char *sha1,
 
 static void get_info_refs(char *arg)
 {
+	char *query = getenv("QUERY_STRING");
+
+	if (query && !prefixcmp(query, "service=")) {
+		const char *argv[] = {NULL /* service name */,
+			"--one-shot-rpc", "--advertise-refs",
+			".", NULL};
+
+		select_service(query + 8);
+
+		write_nocache();
+		format_write("%s: application/x-git-%s-advertisement\r\n",
+			content_type, service->name);
+		end_headers();
+		packet_write(1, "# service=git-%s\n", service->name);
+
+		argv[0] = service->name;
+		run_service(argv);
+	}
+
 	write_nocache();
 	write_header(content_type, "text/plain; charset=utf-8");
 	end_headers();
@@ -176,6 +341,28 @@ static void get_info_packs(char *arg)
 	safe_write(1, "\n", 1);
 }
 
+static void post_to_service(char *service_name)
+{
+	const char *argv[] = {NULL, "--one-shot-rpc", ".", NULL};
+	unsigned n;
+
+	select_service(service_name);
+
+	n = snprintf(buffer, sizeof(buffer),
+		"application/x-git-%s-request", service->name);
+	if (n >= sizeof(buffer))
+		die("impossibly long service name");
+	require_content_type(buffer);
+
+	write_nocache();
+	format_write("%s: application/x-git-%s-result\r\n",
+		content_type, service->name);
+	end_headers();
+
+	argv[0] = service->name;
+	run_service(argv);
+}
+
 static NORETURN void die_webcgi(const char *err, va_list params)
 {
 	write_status(500, "Internal Server Error");
@@ -198,7 +385,10 @@ static struct service_cmd {
 	{"GET", "/objects/info/[^/]*$", get_text_file},
 	{"GET", "/objects/[0-9a-f]{2}/[0-9a-f]{38}$", get_loose_object},
 	{"GET", "/objects/pack/pack-[0-9a-f]{40}\\.pack$", get_pack_file},
-	{"GET", "/objects/pack/pack-[0-9a-f]{40}\\.idx$", get_idx_file}
+	{"GET", "/objects/pack/pack-[0-9a-f]{40}\\.idx$", get_idx_file},
+
+	{"POST", "/git-upload-pack$", post_to_service},
+	{"POST", "/git-receive-pack$", post_to_service}
 };
 
 int main(int argc, char **argv)
-- 
1.6.5.52.g0ff2e
Previous: Shawn O. PearceNext: Johannes Sixt
Message 28 of 38 in “Return of smart HTTP”
  1. 00/16 Return of smart HTTPShawn O. Pearce, Oct 13, 2009
  2. 01/16 pkt-line: Add strbuf based functionsShawn O. Pearce, Oct 13, 2009
  3. Johannes SixtOct 13, 2009
  4. Shawn O. PearceOct 13, 2009
  5. 02/16 pkt-line: Make packet_read_line easier to debugShawn O. Pearce, Oct 13, 2009
  6. 03/16 fetch-pack: Use a strbuf to compose the want listShawn O. Pearce, Oct 13, 2009
  7. 04/16 Move "get_ack()" back to fetch-packShawn O. Pearce, Oct 13, 2009
  8. 05/16 Add multi_ack_2 capability to fetch-pack/upload-packShawn O. Pearce, Oct 13, 2009
  9. Jakub NarebskiOct 13, 2009
  10. Shawn O. PearceOct 13, 2009
  11. 06/16 remote-curl: Refactor walker initializationShawn O. Pearce, Oct 13, 2009
  12. 07/16 remote-helpers: Fetch more than one ref in a batchShawn O. Pearce, Oct 13, 2009
  13. Daniel BarkalowOct 13, 2009
  14. Shawn O. PearceOct 13, 2009
  15. 08/16 remote-helpers: Support custom transport optionsShawn O. Pearce, Oct 13, 2009
  16. Daniel BarkalowOct 13, 2009
  17. Shawn O. PearceOct 13, 2009
  18. Daniel BarkalowOct 13, 2009
  19. Shawn O. PearceOct 13, 2009
  20. Daniel BarkalowOct 13, 2009
  21. Shawn O. PearceOct 13, 2009
  22. 09/16 Move WebDAV HTTP push under remote-curlShawn O. Pearce, Oct 13, 2009
  23. Mike HommeyOct 13, 2009
  24. Johannes SchindelinOct 13, 2009
  25. 10/16 Git-aware CGI to provide dumb HTTP transportShawn O. Pearce, Oct 13, 2009
  26. Johannes SixtOct 13, 2009
  27. 11/16 Add one shot RPC options to upload-pack, receive-packShawn O. Pearce, Oct 13, 2009
  28. 12/16 Smart fetch and push over HTTP: server sideShawn O. Pearce, Oct 13, 2009
  29. Johannes SixtOct 13, 2009
  30. Shawn O. PearceOct 13, 2009
  31. 13/16 Discover refs via smart HTTP server when availableShawn O. Pearce, Oct 13, 2009
  32. 14/16 Smart push over HTTP: client sideShawn O. Pearce, Oct 13, 2009
  33. Felipe ContrerasOct 13, 2009
  34. 15/16 Smart fetch over HTTP: client sideShawn O. Pearce, Oct 13, 2009
  35. 16/16 Smart HTTP fetch: gzip requestsShawn O. Pearce, Oct 13, 2009
  36. Junio C HamanoOct 13, 2009
  37. eduard stefanOct 13, 2009
  38. Junio C HamanoOct 13, 2009

Read the whole thread, see it on lore, or plain text.

$ cat FOOTERMessages come from the public archive at lore.kernel.org/git, fetched every hour. The front page is chosen and written each morning by an AI editor and can be wrong; the threads themselves are the record. About and API. For agents: an MCP server at https://gitlist.dev/mcp, and any thread, story or person page as Markdown by adding .md to its URL (or sending Accept: text/markdown). Details in /llms.txt.