git/list[1] front-page[2] threads[3] people[4] search[5] about
 

[PATCH 13/15] unpack-trees.c: optionally clone submodules for later checkout

From
Sskimo@liacs.nl <skimo@liacs.nl>
Date
May 20, 2007, 18:04 UTC
Message-ID
<11796842904194-git-send-email-skimo@liacs.nl>
In-Reply-To
<11796842882917-git-send-email-skimo@liacs.nl>
From: Sven Verdoolaege <skimo@kotnet.org>

When the --submodules option is specified and a submodule to be checked out is not available locally, git-checkout will search for submodule.<submodule>.url options in the remote configuration and clone each submodule using the first url that it can use from the local site.

Signed-off-by: Sven Verdoolaege <skimo@kotnet.org>
---
 Documentation/config.txt |    3 +
 Makefile                 |    5 +-
 submodules.c             |  209 ++++++++++++++++++++++++++++++++++++++++++++++
 submodules.h             |    6 ++
 unpack-trees.c           |   51 +++++++++++
 5 files changed, 272 insertions(+), 2 deletions(-)
 create mode 100644 submodules.c
 create mode 100644 submodules.h
diff --git a/Documentation/config.txt b/Documentation/config.txt
index 5d891ac..cee9e40 100644
--- a/Documentation/config.txt
+++ b/Documentation/config.txt
@@ -610,6 +610,9 @@ showbranch.default::
 	The default set of branches for gitlink:git-show-branch[1].
 	See gitlink:git-show-branch[1].
 
+submodule.<submodule>.url
+	The URL of a submodule.  See gitlink:git-clone[1].
+
 tar.umask::
 	By default, gitlink:git-tar-tree[1] sets file and directories modes
 	to 0666 or 0777. While this is both useful and acceptable for projects
diff --git a/Makefile b/Makefile
index bce8514..c79e636 100644
--- a/Makefile
+++ b/Makefile
@@ -297,7 +297,8 @@ LIB_H = \
 	diff.h object.h pack.h pkt-line.h quote.h refs.h list-objects.h sideband.h \
 	run-command.h strbuf.h tag.h tree.h git-compat-util.h revision.h \
 	tree-walk.h log-tree.h dir.h path-list.h unpack-trees.h builtin.h \
-	utf8.h reflog-walk.h patch-ids.h attr.h decorate.h progress.h mailmap.h
+	utf8.h reflog-walk.h patch-ids.h attr.h decorate.h progress.h mailmap.h \
+	submodules.h
 
 DIFF_OBJS = \
 	diff.o diff-lib.o diffcore-break.o diffcore-order.o \
@@ -320,7 +321,7 @@ LIB_OBJS = \
 	alloc.o merge-file.o path-list.o help.o unpack-trees.o $(DIFF_OBJS) \
 	color.o wt-status.o archive-zip.o archive-tar.o shallow.o utf8.o \
 	convert.o attr.o decorate.o progress.o mailmap.o symlinks.o \
-	$(HTTP_CONFIG_OBJ)
+	$(HTTP_CONFIG_OBJ) submodules.o
 
 BUILTIN_OBJS = \
 	builtin-add.o \
diff --git a/submodules.c b/submodules.c
new file mode 100644
index 0000000..5035a05
--- /dev/null
+++ b/submodules.c
@@ -0,0 +1,209 @@
+#include "cache.h"
+#include "submodules.h"
+#include "run-command.h"
+
+struct key_val_list {
+	struct key_val_list *next;
+	char *key;
+	char *val;
+};
+
+static void free_key_val_list(struct key_val_list *list)
+{
+	struct key_val_list *next;
+	for (; list; list = next) {
+		next = list->next;
+		free(list->key);
+		free(list->val);
+		free(list);
+	}
+}
+
+static struct key_val_list *find_key_val_list(struct key_val_list *list,
+						const char *key)
+{
+	while (list && strcmp(list->key, key))
+		list = list->next;
+	return list;
+}
+
+struct collect_urls_data {
+	struct key_val_list **next;
+
+	const char *type;
+};
+
+static int collect_urls(const char *var, const char *value, void *cb_data)
+{
+	struct collect_urls_data *cb = (struct collect_urls_data*)cb_data;
+	int typelen = strlen(cb->type);
+	int len;
+	char *doturl;
+	struct key_val_list *item;
+
+	if (prefixcmp(var, cb->type))
+		return 0;
+
+	if (var[typelen] != '.')
+		return 0;
+
+	var += typelen+1;
+
+	doturl = strrchr(var, '.');
+	if (!doturl || strcmp(doturl, ".url"))
+		return 0;
+
+	len = doturl-var;
+	if (len <= 0)
+		return 0;
+
+	item = xmalloc(sizeof(struct key_val_list));
+	item->key = xmalloc(len+1);
+	memcpy(item->key, var, len);
+	item->key[len] = 0;
+	item->val = xstrdup(value);
+	item->next = NULL;
+	*cb->next = item;
+	cb->next = &item->next;
+
+	return 0;
+}
+
+static const char *local_URL(const char *remote, const char *url)
+{
+	static char local_url[PATH_MAX];
+
+	if (!prefixcmp(url, "https://"))
+		return url;
+
+	if (!prefixcmp(url, "http://"))
+		return url;
+
+	if (!prefixcmp(url, "ftp://"))
+		return url;
+
+	if (!prefixcmp(remote, "/"))
+		return url;
+
+	if (!prefixcmp(remote, "ssh://") && !prefixcmp(url, "/")) {
+		char *slash;
+		int len = strlen(url);
+
+		slash = strchr(remote+6, '/');
+		if (!slash || (slash-remote)+len+1 > sizeof(local_url))
+			return NULL;
+		memcpy(local_url, remote, slash-remote);
+		memcpy(local_url+(slash-remote), url, len+1);
+		return local_url;
+	}
+
+	return NULL;
+}
+
+static int fetch_submodule_urls(struct key_val_list **next_url)
+{
+	struct key_val_list *remotes = NULL;
+	struct collect_urls_data remotes_data = { &remotes, "remote" };
+	struct key_val_list *remote;
+	static char key[1024];
+
+	git_config(collect_urls, &remotes_data);
+	for (remote = remotes; remote; remote = remote->next) {
+		struct key_val_list *submodules = NULL;
+		struct collect_urls_data submodules_data =
+			{ &submodules, "submodule" };
+		struct key_val_list *submodule;
+		char *dest;
+
+		dest = xstrdup(remote->val);
+		git_config_from_remote(collect_urls, dest, &submodules_data);
+		free(dest);
+		for (submodule = submodules; submodule; submodule = submodule->next) {
+			const char *local_url;
+			struct key_val_list *item;
+
+			local_url = local_URL(remote->val, submodule->val);
+			if (!local_url)
+				continue;
+
+			if (snprintf(key, sizeof(key),
+			    "submodule.%s.url", submodule->key) > sizeof(key))
+				return error("submodule name too long");
+
+			git_config_set(key, local_url);
+
+			item = xmalloc(sizeof(struct key_val_list));
+			item->key = xstrdup(submodule->key);
+			item->val = xstrdup(local_url);
+			item->next = NULL;
+			*next_url = item;
+			next_url = &item->next;
+		}
+
+		free_key_val_list(submodules);
+	}
+
+	free_key_val_list(remotes);
+
+	return 0;
+}
+
+int clone_submodule(const char *submodule)
+{
+	struct key_val_list *submodules = NULL;
+	struct collect_urls_data submodules_data = { &submodules, "submodule" };
+	struct key_val_list *item;
+	char *path;
+	int err;
+	const char *gitdirenv;
+	const char *args[10];
+	int argc;
+
+	git_config(collect_urls, &submodules_data);
+	item = find_key_val_list(submodules, submodule);
+	if (!item) {
+		err = fetch_submodule_urls(submodules_data.next);
+		if (err)
+			return err;
+		item = find_key_val_list(*submodules_data.next, submodule);
+		if (!item)
+			return error("don't know where to get submodule '%s'",
+					submodule);
+	}
+
+	path = git_path("submodules/%s", submodule);
+
+	argc = 0;
+	args[argc++] = "clone";
+	args[argc++] = "--submodules";
+	args[argc++] = "-n";
+	args[argc++] = item->val;
+	args[argc++] = path;
+	args[argc] = NULL;
+
+	gitdirenv = getenv(GIT_DIR_ENVIRONMENT);
+	unsetenv(GIT_DIR_ENVIRONMENT);
+	err = run_command_v_opt(args, RUN_GIT_CMD);
+
+	setenv(GIT_DIR_ENVIRONMENT,
+		git_path("submodules/%s/.git", submodule), 1);
+
+	argc = 0;
+	args[argc++] = "update-ref";
+	args[argc++] = "--no-deref";
+	args[argc++] = "HEAD";
+	args[argc++] = "0000000000000000000000000000000000000000";
+	args[argc] = NULL;
+
+	if (!err)
+		err = run_command_v_opt(args, RUN_GIT_CMD);
+
+	setenv(GIT_DIR_ENVIRONMENT, gitdirenv, 1);
+
+	if (err)
+		return error("failed to clone submodule '%s'", submodule);
+
+	free_key_val_list(submodules);
+
+	return 0;
+}
diff --git a/submodules.h b/submodules.h
new file mode 100644
index 0000000..56590e7
--- /dev/null
+++ b/submodules.h
@@ -0,0 +1,6 @@
+#ifndef SUBMODULES_H
+#define SUBMODULES_H
+
+int clone_submodule(const char *submodule);
+
+#endif
diff --git a/unpack-trees.c b/unpack-trees.c
index 3dadebb..f9865b9 100644
--- a/unpack-trees.c
+++ b/unpack-trees.c
@@ -6,6 +6,7 @@
 #include "unpack-trees.h"
 #include "progress.h"
 #include "refs.h"
+#include "submodules.h"
 
 #define DBRT_DEBUG 1
 
@@ -808,6 +809,36 @@ int threeway_merge(struct cache_entry **stages,
 	return count;
 }
 
+static int ensure_submodule(struct cache_entry *ce,
+			    struct unpack_trees_options *o)
+{
+	struct stat st;
+	char *path;
+
+	if (!ce)
+		return 0;
+
+	if (!S_ISDIRLNK(ntohl(ce->ce_mode)))
+		return 0;
+
+	path = mkpath("%s/.git", ce->name);
+	fprintf(stderr, "path: %s\n", path);
+	if (lstat(path, &st)) {
+		path = git_path("submodules/%s/.git", ce->name);
+		fprintf(stderr, "path: %s\n", path);
+		if (lstat(path, &st)) {
+			int err;
+			err = clone_submodule(ce->name);
+			if (err)
+				return err;
+		}
+	}
+
+	/* Now check that the commit is available and fetch if needed */
+
+	return 0;
+}
+
 /*
  * Two-way merge.
  *
@@ -833,6 +864,17 @@ int twoway_merge(struct cache_entry **src,
 	if (newtree == o->df_conflict_entry)
 		newtree = NULL;
 
+	if (o->submodules && o->update) {
+		int err;
+		err = ensure_submodule(current, o);
+		if (!err)
+			err = ensure_submodule(oldtree, o);
+		if (!err)
+			err = ensure_submodule(newtree, o);
+		if (err)
+			return err;
+	}
+
 	if (current) {
 		if ((!oldtree && !newtree) || /* 4 and 5 */
 		    (!oldtree && newtree &&
@@ -909,6 +951,15 @@ int oneway_merge(struct cache_entry **src,
 		return error("Cannot do a oneway merge of %d trees",
 			     o->merge_size);
 
+	if (o->submodules && o->update) {
+		int err;
+		err = ensure_submodule(old, o);
+		if (!err)
+			err = ensure_submodule(a, o);
+		if (err)
+			return err;
+	}
+
 	if (!a)
 		return deleted_entry(old, old, o);
 	if (old && same(old, a)) {

-- 
1.5.2.rc3.815.g8fc2
Previous: skimo@liacs.nlNext: skimo@liacs.nl
Message 22 of 53 in “[RFC] Third round of support for cloning submodules”
  1. skimo@liacs.nlMay 20, 2007
  2. 01/15 Add dump-configskimo@liacs.nl, May 20, 2007
  3. 02/15 git-config: add --remote option for reading config from remote reposkimo@liacs.nl, May 20, 2007
  4. Frank LichtenheldMay 20, 2007
  5. Sven VerdoolaegeMay 20, 2007
  6. Frank LichtenheldMay 20, 2007
  7. 03/15 http.h: make fill_active_slots a function pointerskimo@liacs.nl, May 20, 2007
  8. 04/15 git-config: read remote config files over HTTPskimo@liacs.nl, May 20, 2007
  9. 05/15 unpack-trees.c: pass cache_entry * to verify_absent rather than just the nameskimo@liacs.nl, May 20, 2007
  10. 06/15 git-read-tree: take --submodules optionskimo@liacs.nl, May 20, 2007
  11. Martin WaitzMay 20, 2007
  12. Sven VerdoolaegeMay 20, 2007
  13. 07/15 unpack-trees.c: assume submodules are cleanskimo@liacs.nl, May 20, 2007
  14. 08/15 Add run_command_v_opt_cd: chdir into a directory before execskimo@liacs.nl, May 20, 2007
  15. 09/15 entry.c: optionally checkout submodulesskimo@liacs.nl, May 20, 2007
  16. Martin WaitzMay 20, 2007
  17. Sven VerdoolaegeMay 20, 2007
  18. Sven VerdoolaegeMay 24, 2007
  19. 10/15 git-checkout: pass --submodules option to git-read-treeskimo@liacs.nl, May 20, 2007
  20. 11/15 git-read-tree: treat null commit as empty treeskimo@liacs.nl, May 20, 2007
  21. 12/15 git_config: add void * for callback dataskimo@liacs.nl, May 20, 2007
  22. 13/15 unpack-trees.c: optionally clone submodules for later checkoutskimo@liacs.nl, May 20, 2007
  23. 14/15 entry.c: optionall checkout newly cloned submodulesskimo@liacs.nl, May 20, 2007
  24. 15/15 git-clone: add --submodules for cloning submodulesskimo@liacs.nl, May 20, 2007
  25. Junio C HamanoMay 20, 2007
  26. Sven VerdoolaegeMay 20, 2007
  27. Alex RiesenMay 20, 2007
  28. Sven VerdoolaegeMay 20, 2007
  29. Alex RiesenMay 20, 2007
  30. Sven VerdoolaegeMay 20, 2007
  31. Alex RiesenMay 20, 2007
  32. Sven VerdoolaegeMay 21, 2007
  33. Josef WeidendorferMay 21, 2007
  34. Martin WaitzMay 21, 2007
  35. Martin WaitzMay 20, 2007
  36. Alex RiesenMay 20, 2007
  37. Martin WaitzMay 20, 2007
  38. Alex RiesenMay 20, 2007
  39. Martin WaitzMay 20, 2007
  40. Alex RiesenMay 22, 2007
  41. Martin WaitzMay 24, 2007
  42. Steven GrimmMay 21, 2007
  43. Sven VerdoolaegeMay 21, 2007
  44. Alex RiesenMay 22, 2007
  45. Martin WaitzMay 20, 2007
  46. Alex RiesenMay 20, 2007
  47. Martin WaitzMay 20, 2007
  48. Martin WaitzMay 20, 2007
  49. Sven VerdoolaegeMay 21, 2007
  50. Martin WaitzMay 21, 2007
  51. Sven VerdoolaegeMay 21, 2007
  52. Martin WaitzMay 21, 2007
  53. Sven VerdoolaegeMay 21, 2007

Read the whole thread, see it on lore, or plain text.

$ cat FOOTERMessages come from the public archive at lore.kernel.org/git, fetched every hour. The front page is chosen and written each morning by an AI editor and can be wrong; the threads themselves are the record. About and API. For agents: an MCP server at https://gitlist.dev/mcp, and any thread, story or person page as Markdown by adding .md to its URL (or sending Accept: text/markdown). Details in /llms.txt.