Multi-head pulling series

Junio C Hamano· Aug 18, 2005, 07:39 UTC · re: Junio C Hamano · lore

[PATCH 1/3] Start adding the $GIT_DIR/remotes/ support.

Signed-off-by: Junio C Hamano <junkio@cox.net>
---

 Makefile                |    2 -
 git-parse-remote-script |  122 +++++++++++++++++++++++++++++++++++++++++++++++
 git-push-script         |   28 ++---------
 3 files changed, 129 insertions(+), 23 deletions(-)
 create mode 100755 git-parse-remote-script

f8892bf17675056cd18a252d3bc4e4ba381fb3bc

diff --git a/Makefile b/Makefile
--- a/Makefile
+++ b/Makefile
@@ -64,7 +64,7 @@ SCRIPTS=git git-apply-patch-script git-m
 	git-reset-script git-add-script git-checkout-script git-clone-script \
 	gitk git-cherry git-rebase-script git-relink-script git-repack-script \
 	git-format-patch-script git-sh-setup-script git-push-script \
-	git-branch-script git-parse-remote git-verify-tag-script \
+	git-branch-script git-parse-remote git-parse-remote-script git-verify-tag-script \
 	git-ls-remote-script git-clone-dumb-http git-rename-script \
 	git-request-pull-script git-bisect-script
 
diff --git a/git-parse-remote-script b/git-parse-remote-script
new file mode 100755
--- /dev/null
+++ b/git-parse-remote-script
@@ -0,0 +1,122 @@
+#!/bin/sh
+
+. git-sh-setup-script || die "Not a git archive"
+
+get_data_source () {
+	case "$1" in
+	*/*)
+		# Not so fast.  This could be the partial URL shorthand...
+		token=$(expr "$1" : '\([^/]*\)/')
+		remainder=$(expr "$1" : '[^/]*/\(.*\)')
+		if test -f "$GIT_DIR/branches/$token"
+		then
+			echo branches-partial
+		else
+			echo ''
+		fi
+		;;
+	*)
+		if test -f "$GIT_DIR/remotes/$1"
+		then
+			echo remotes
+		elif test -f "$GIT_DIR/branches/$1"
+		then
+			echo branches
+		else
+			echo ''
+		fi ;;
+	esac
+}
+
+get_remote_url () {
+	data_source=$(get_data_source "$1")
+	case "$data_source" in
+	'')
+		echo "$1" ;;
+	remotes)
+		sed -ne '/^URL: */{
+			s///p
+			q
+		}' "$GIT_DIR/remotes/$1" ;;
+	branches)
+		sed -e 's/#.*//' "$GIT_DIR/branches/$1" ;;
+	branches-partial)
+		token=$(expr "$1" : '\([^/]*\)/')
+		remainder=$(expr "$1" : '[^/]*/\(.*\)')
+		url=$(sed -e 's/#.*//' "$GIT_DIR/branches/$token")
+		echo "$url/$remainder"
+		;;
+	*)
+		die "internal error: get-remote-url $1" ;;
+	esac
+}
+
+get_remote_default_refs_for_push () {
+	data_source=$(get_data_source "$1")
+	case "$data_source" in
+	'' | branches | branches-partial)
+		;; # no default push mapping, just send matching refs.
+	remotes)
+		sed -ne '/^Push: */{
+			s///p
+		}' "$GIT_DIR/remotes/$1" ;;
+	*)
+		die "internal error: get-remote-default-ref-for-push $1" ;;
+	esac
+}
+
+# Subroutine to caninicalize remote:local notation
+canon_refs_list_for_fetch () {
+	for ref
+	do
+		expr "$ref" : '.*:' >/dev/null || ref="${ref}:"
+		remote=$(expr "$ref" : '\([^:]*\):')
+		local=$(expr "$ref" : '[^:]*:\(.*\)')
+		case "$remote" in
+		'') remote=HEAD ;;
+		*) remote="refs/heads/$remote" ;;
+		esac
+		case "$local" in
+		'') local= ;;
+		*) local="refs/heads/$local" ;;
+		esac
+		echo "${remote}:${local}"
+	done
+}
+
+# Returns list of src: (no store), or src:dst (store)
+get_remote_default_refs_for_fetch () {
+	data_source=$(get_data_source "$1")
+	case "$data_source" in
+	'' | branches-partial)
+		echo "HEAD:" ;;
+	branches)
+		remote_branch=$(sed -ne '/#/s/.*#//p' "$GIT_DIR/branches/$1")
+		case "$remote_branch" in '') remote_branch=master ;; esac
+		echo "refs/heads/${remote_branch}:refs/heads/$1"
+		;;
+	remotes)
+		canon_refs_list_for_fetch $(sed -ne '/^Pull: */{
+						s///p
+					}' "$GIT_DIR/remotes/$1")
+		;;
+	*)
+		die "internal error: get-remote-default-ref-for-push $1" ;;
+	esac
+}
+
+get_remote_refs_for_push () {
+	case "$#" in
+	0) die "internal error: get-remote-refs-for-push." ;;
+	1) get_remote_default_refs_for_push "$@" ;;
+	*) shift; echo "$@" ;;
+	esac
+}
+
+get_remote_refs_for_fetch () {
+	case "$#" in
+	0) die "internal error: get-remote-refs-for-fetch." ;;
+	1) get_remote_default_refs_for_fetch "$@" ;;
+	*) shift; canon_refs_list_for_fetch "$@" ;;
+	esac
+}
diff --git a/git-push-script b/git-push-script
--- a/git-push-script
+++ b/git-push-script
@@ -20,8 +20,6 @@ do
 	-*)
 		die "Unknown parameter $1" ;;
         *)
-		remote="$1"
-		shift
 		set x "$@"
 		shift
 		break ;;
@@ -29,27 +27,13 @@ do
 	shift
 done
 
-case "$remote" in
-*:* | /* | ../* | ./* )
-	# An URL, host:/path/to/git, absolute and relative paths.
-	;;
-* )
-	# Shorthand
-	if expr "$remote" : '..*/..*' >/dev/null
-	then
-		# a short-hand followed by a trailing path
-		shorthand=$(expr "$remote" : '\([^/]*\)')
-		remainder=$(expr "$remote" : '[^/]*\(/.*\)$')
-	else
-		shorthand="$remote"
-		remainder=
-	fi
-	remote=$(sed -e 's/#.*//' "$GIT_DIR/branches/$remote") &&
-	expr "$remote" : '..*:' >/dev/null &&
-	remote="$remote$remainder" ||
-	die "Cannot parse remote $remote"
-	;;
+. git-parse-remote-script
+remote=$(get_remote_url "$@")
+case "$has_all" in
+--all) set x ;;
+'')    set x $(get_remote_refs_for_push "$@") ;;
 esac
+shift
 
 case "$remote" in
 http://* | https://* | git://* | rsync://* )

Junio C Hamano· Aug 18, 2005, 07:39 UTC · re: Junio C Hamano · lore

[PATCH 2/3] Multi-head fetch.

Traditionally, fetch takes these forms:

    $ git fetch <remote>
    $ git fetch <remote> <head>
    $ git fetch <remote> tag <tag>

This patch updates it to take

    $ git fetch <remote> <refspec>...

where:

    - A <refspec> of form "<src>:<dst>" is to fetch the objects
      needed for the remote ref that matches <src>, and if <dst>
      is not empty, store it as a local <dst>.

    - "tag" followed by <next> is just an old way of saying
      "refs/tags/<next>:refs/tags/<next>"; this mimics the
      current behaviour of the third form above and means "fetch
      that tag and store it under the same name".

    - A single token <refspec> without colon is a shorthand for
      "<refspec>:"  That is, "fetch that ref but do not store
      anywhere".

    - when there is no <refspec> specified

      - if <remote> is the name of a file under $GIT_DIR/remotes/
	(i.e. a new-style shorthand), then it is the same as giving
	the <refspec>s listed on Pull: line in that file.

      - if <remote> is the name of a file under $GIT_DIR/branches/
	(i.e. an old-style shorthand, without trailing path), then it
	is the same as giving a single <refspec>
	"<remote-name>:refs/heads/<remote>" on the command line, where
	<remote-name> is the remote branch name (defaults to HEAD, but
	can be overridden by .git/branches/<remote> file having the
	URL fragment notation).  That is, "fetch that branch head and
	store it in refs/heads/<remote>".

      - otherwise, it is the same as giving a single <refspec>
        that is "HEAD:".

Signed-off-by: Junio C Hamano <junkio@cox.net>
---

 git-fetch-script |  154 ++++++++++++++++++++++++++++++++++++++----------------
 1 files changed, 108 insertions(+), 46 deletions(-)

7dbfeb149dfc049015f27f15dd9135fd15d5f99f

diff --git a/git-fetch-script b/git-fetch-script
--- a/git-fetch-script
+++ b/git-fetch-script
@@ -1,54 +1,116 @@
 #!/bin/sh
 #
 . git-sh-setup-script || die "Not a git archive"
-. git-parse-remote "$@"
-merge_repo="$_remote_repo"
-merge_head="$_remote_head"
-merge_store="$_remote_store"
-
-TMP_HEAD="$GIT_DIR/TMP_HEAD"
-
-case "$merge_repo" in
-http://* | https://*)
-        if [ -n "$GIT_SSL_NO_VERIFY" ]; then
-            curl_extra_args="-k"
-        fi
-	_x40='[0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f]' &&
-	_x40="$_x40$_x40$_x40$_x40$_x40$_x40$_x40$_x40" &&
-	head=$(curl -nsf $curl_extra_args "$merge_repo/$merge_head") &&
-	expr "$head" : "$_x40\$" >/dev/null || {
-		echo >&2 "Failed to fetch $merge_head from $merge_repo"
-	        exit 1
-	}
-	echo Fetching "$merge_head" using http
-	git-http-pull -v -a "$head" "$merge_repo/" || exit
-	;;
-rsync://*)
-	rsync -L "$merge_repo/$merge_head" "$TMP_HEAD" || exit 1
-	head=$(git-rev-parse TMP_HEAD)
-	rm -f "$TMP_HEAD"
-	rsync -avz --ignore-existing "$merge_repo/objects/" "$GIT_OBJECT_DIRECTORY/"
-	;;
-*)
-	head=$(git-fetch-pack "$merge_repo" "$merge_head")
-	if h=`expr "$head" : '\([^ ][^ ]*\) '`
+. git-parse-remote-script
+_x40='[0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f]'
+_x40="$_x40$_x40$_x40$_x40$_x40$_x40$_x40$_x40"
+
+remote_nick="$1"
+remote=$(get_remote_url "$@")
+refs=
+rref=
+prev_was_literal_tag=
+rsync_slurped_objects=
+: >$GIT_DIR/FETCH_HEAD
+
+fast_forward_local () {
+
+	# NEEDSWORK: probably use the same cmpxchg protocol here...
+	echo "$2" >"$GIT_DIR/$1.lock"
+	if test -f "$GIT_DIR/$1"
 	then
-	    head=$h
+		local=$(git-rev-parse --verify "$1^0") &&
+		mb=$(git-merge-base "$local" "$2") &&
+		test "$mb" = "$local" || {
+			rm -f "$GIT_DIR/$1.lock"
+			die "$1 does not fast forward to $4 from $3.";
+		}
 	fi
-	;;
-esac || exit 1
-
-git-rev-parse --verify "$head" > /dev/null || exit 1
+	mv "$GIT_DIR/$1.lock" "$GIT_DIR/$1"
+}
 
-case "$merge_store" in
-'')
-	;;
-*)
-	echo "$head" > "$GIT_DIR/$merge_store"
-esac &&
+for ref in $(get_remote_refs_for_fetch "$@")
+do
+	if test "$prev_was_literal_tag"
+	then
+		ref="refs/tags/${ref}:refs/tags/${ref}"
+		prev_was_literal_tag=
+	else
+		case "$ref" in
+		tag)
+			prev_was_literal_tag=yes
+			continue
+			;;
+		esac
+	fi
 
-# FETCH_HEAD is fed to git-resolve-script which will eventually be
-# passed to git-commit-tree as one of the parents.  Make sure we do
-# not give a tag object ID.
+	refs="$refs $ref"
 
-git-rev-parse "$head^0" >"$GIT_DIR/FETCH_HEAD"
+	# These are relative path from $GIT_DIR, typically starting at refs/
+	# but may be HEAD
+	remote_name=$(expr "$ref" : '\([^:]*\):')
+	local_name=$(expr "$ref" : '[^:]*:\(.*\)')
+
+	rref="$rref $remote_name"
+
+	# There are transports that can fetch only one head at a time...
+	case "$remote" in
+	http://* | https://*)
+		if [ -n "$GIT_SSL_NO_VERIFY" ]; then
+		    curl_extra_args="-k"
+		fi
+		head=$(curl -nsf $curl_extra_args "$remote/$remote_name") &&
+		expr "$head" : "$_x40\$" >/dev/null ||
+			die "Failed to fetch $remote_name from $remote"
+		echo Fetching "$remote_name from $remote" using http
+		git-http-pull -v -a "$head" "$remote/" || exit
+		;;
+	rsync://*)
+		TMP_HEAD="$GIT_DIR/TMP_HEAD"
+		rsync -L "$remote/$remote_name" "$TMP_HEAD" || exit 1
+		head=$(git-rev-parse TMP_HEAD)
+		rm -f "$TMP_HEAD"
+		test "$rsync_slurped_objects" || {
+			rsync -avz --ignore-existing "$remote/objects/" \
+				"$GIT_OBJECT_DIRECTORY/" || exit
+			rsync_slurped_objects=t
+		}
+		;;
+	*)
+		# We will do git native transport at one go.
+		continue ;;
+	esac
+	head=$(git-rev-parse --verify "$head^0") || exit
+	echo "$head	$remote_name from $remote_nick" >>$GIT_DIR/FETCH_HEAD
+	case "$local_name" in '') continue ;; esac
+
+	# We are storing the head locally.  Make sure that it is
+	# a fast forward (aka "reverse push").
+	fast_forward_local "$local_name" "$head" "$remote" "$remote_name"
+
+done
+
+case "$remote" in
+http://* | https://* | rsync://* )
+	;; # we are already done.
+*)
+	git-fetch-pack "$remote" $rref |
+	while read sha1 remote_name
+	do
+		found=
+		for ref in $refs
+		do
+			case "$ref" in
+			$remote_name:*)
+				found="$ref"
+				break ;;
+			esac
+		done
+		head=$(git-rev-parse --verify "$sha1^0") || exit
+		echo "$head	$remote_name from $remote_nick" >>$GIT_DIR/FETCH_HEAD
+		case "$found" in '' | *:) continue ;; esac
+		local=$(expr "$found" : '[^:]*:\(.*\)')
+		fast_forward_local "$local" "$sha1" "$remote" "$remote_name"
+	done
+	;;
+esac

Junio C Hamano· Aug 18, 2005, 07:39 UTC · re: Junio C Hamano · lore

[PATCH 3/3] Update git-pull to match updated git-fetch.

Signed-off-by: Junio C Hamano <junkio@cox.net>
---

 Makefile         |    2 +
 git-parse-remote |   79 ------------------------------------------------------
 git-pull-script  |   14 ++++++----
 3 files changed, 10 insertions(+), 85 deletions(-)
 delete mode 100755 git-parse-remote

3a071a02828c71bbfdc2749d25814906cd9c8b18

diff --git a/Makefile b/Makefile
--- a/Makefile
+++ b/Makefile
@@ -64,7 +64,7 @@ SCRIPTS=git git-apply-patch-script git-m
 	git-reset-script git-add-script git-checkout-script git-clone-script \
 	gitk git-cherry git-rebase-script git-relink-script git-repack-script \
 	git-format-patch-script git-sh-setup-script git-push-script \
-	git-branch-script git-parse-remote git-parse-remote-script git-verify-tag-script \
+	git-branch-script git-parse-remote-script git-verify-tag-script \
 	git-ls-remote-script git-clone-dumb-http git-rename-script \
 	git-request-pull-script git-bisect-script
 
diff --git a/git-parse-remote b/git-parse-remote
deleted file mode 100755
--- a/git-parse-remote
+++ /dev/null
@@ -1,79 +0,0 @@
-: To be included in git-pull and git-fetch scripts.
-
-# A remote repository can be specified on the command line
-# in one of the following formats:
-#
-#	<repo>
-#	<repo> <head>
-#	<repo> tag <tag>
-#
-# where <repo> could be one of:
-#
-#	a URL (including absolute or local pathname)
-#	a short-hand
-#	a short-hand followed by a trailing path
-#
-# A short-hand <name> has a corresponding file $GIT_DIR/branches/<name>,
-# whose contents is a URL, possibly followed by a URL fragment #<head>
-# to name the default branch on the remote side to fetch from.
-
-_remote_repo= _remote_store= _remote_head= _remote_name=
-
-case "$1" in
-*:* | /* | ../* | ./* )
-	_remote_repo="$1"
-	;;
-* )
-	# otherwise, it is a short hand.
-	case "$1" in
-	*/*)
-		# a short-hand followed by a trailing path
-		_token=$(expr "$1" : '\([^/]*\)/')
-		_rest=$(expr "$1" : '[^/]*\(/.*\)$')
-		;;
-	*)
-		_token="$1"
-		_rest=
-		_remote_store="refs/heads/$_token"
-		;;
-	esac
-	test -f "$GIT_DIR/branches/$_token" ||
-	die "No such remote branch: $_token"
-
-	_remote_repo=$(cat "$GIT_DIR/branches/$_token")"$_rest"
-	;;
-esac
-
-case "$_remote_repo" in
-*"#"*)
-	_remote_head=`expr "$_remote_repo" : '.*#\(.*\)$'`
-	_remote_repo=`expr "$_remote_repo" : '\(.*\)#'`
-	;;
-esac
-
-_remote_name=$(echo "$_remote_repo" | sed 's|\.git/*$||')
-
-case "$2" in
-tag)
-	_remote_name="tag '$3' of $_remote_name"
-	_remote_head="refs/tags/$3"
-	_remote_store="$_remote_head"
-	;;
-?*)
-	# command line specified a head explicitly; do not
-	# store the fetched head as a branch head.
-	_remote_name="head '$2' of $_remote_name"
-	_remote_head="refs/heads/$2"
-	_remote_store=''
-	;;
-'')
-	case "$_remote_head" in
-	'')
-		_remote_head=HEAD ;;
-	*)
-		_remote_name="head '$_remote_head' of $_remote_name"
-		_remote_head="refs/heads/$_remote_head"
-		;;
-	esac
-	;;
-esac
diff --git a/git-pull-script b/git-pull-script
--- a/git-pull-script
+++ b/git-pull-script
@@ -1,12 +1,16 @@
 #!/bin/sh
 #
 . git-sh-setup-script || die "Not a git archive"
-. git-parse-remote "$@"
-merge_name="$_remote_name"
-
 git-fetch-script "$@" || exit 1
+merge_head=$(sed -e 's/	.*//' "$GIT_DIR"/FETCH_HEAD | tr '\012' ' ')
+merge_name=$(sed -e 's/^[0-9a-f]*	//' "$GIT_DIR"/FETCH_HEAD |
+	 tr '\012' ' ')
+
+case "$merge_head" in
+'' | *' '?*) die "Cannot resolve multiple heads at the same time (yet)." ;;
+esac
+
 
 git-resolve-script \
 	"$(cat "$GIT_DIR"/HEAD)" \
-	"$(cat "$GIT_DIR"/FETCH_HEAD)" \
-	"Merge $merge_name"
+	$merge_head "Merge $merge_name"

Re: Multi-head pulling series

On Thursday 18 August 2005 09:24, Junio C Hamano wrote:

> 	$ cat $GIT_DIR/remotes/www
> 	URL: http://www.kernel.org/pub/scm/git/git.git/
> 	Pull: master:ko-master pu:ko-pu
>       Push: master:master pu:pu foo:bar

Isn't this mixing two kinds of information:
1) Some default/persistent mapping of local to remote heads
2) The default heads which are pulled/pushed when only giving the remote, like 
in:

> 	$ git push mko ;# push our master and pu to the same
> 			# name, foo to .git/refs/heads/bar.

Can we put the default pull/push actions in separate lines, like

 	$ cat $GIT_DIR/remotes/mko
 	URL: master.kernel.org:/pub/scm/git/git.git/
 	Pull: master:ko-master pu:ko-pu mylocal:myremote
	Push: master:master pu:pu foo:bar
	Default-Pull: master pu
	Default-Push: master pu foo
	

> 	$ git push mko pu:refs/heads/testing
> 			# instead of pushing to the usual ref,
>                         # push our pu to refs/heads/testing,
>                         # this time only.

	git push mko pu

And finally, it would be nice to specify a default mapping for arbitrary heads

	$ cat $GIT_DIR/remotes/www
 	URL: http://www.kernel.org/pub/scm/git/git.git/
 	Pull: www#*:*

Such that

	git fetch www#pu

will fetch head pu into local head .refs/heads/www#pu

Josef

PS: I know I should provide patches for my proposals. But let discuss them 
first.

Junio C Hamano· Aug 20, 2005, 18:54 UTC · re: Josef Weidendorfer · lore

Re: Multi-head pulling series

Josef Weidendorfer <Josef.Weidendorfer@gmx.de> writes:

>> 	$ cat $GIT_DIR/remotes/www
>> 	URL: http://www.kernel.org/pub/scm/git/git.git/
>>	Pull: master:ko-master pu:ko-pu
>>	Push: master:master pu:pu foo:bar
>
> Isn't this mixing two kinds of information:
> 1) Some default/persistent mapping of local to remote heads
> 2) The default heads which are pulled/pushed when only giving the remote

Re: Multi-head pulling series

On Saturday 20 August 2005 20:54, Junio C Hamano wrote:

> Josef Weidendorfer <Josef.Weidendorfer@gmx.de> writes:
> >> 	$ cat $GIT_DIR/remotes/www
> >> 	URL: http://www.kernel.org/pub/scm/git/git.git/
> >>	Pull: master:ko-master pu:ko-pu
> >>	Push: master:master pu:pu foo:bar
> >
> > Isn't this mixing two kinds of information:
> > 1) Some default/persistent mapping of local to remote heads
> > 2) The default heads which are pulled/pushed when only giving the remote
>
> I am not convinced myself that default/persistent mapping makes
> much sense,

> and updating the stored mappings dynamically when 
> the user uses non-default mappings in push/pull as a one-shot
> operation does not make any sense at all at least to me.

Yes, you are right.

Josef

Junio C Hamano· Aug 23, 2005, 01:07 UTC · re: Josef Weidendorfer · lore

Re: Multi-head pulling series

Josef Weidendorfer <Josef.Weidendorfer@gmx.de> writes:

> I think that both, default and non-default persistant
> mappings, are only a handy convenience issue for less
> typing. If GIT core is not doing this, some (most?) porcelains
> will invent their own way for storing these mappings (like
> Cogito is doing with branches/). I think it is better if this
> is "standardized".

> Currently missing here is automatic detection of the remote
> repository for a given head to be pulled/pushed. But this is
> no format issue.

"git ls-remote <repo>" perhaps?

Junio C Hamano· Aug 23, 2005, 05:37 UTC · re: Josef Weidendorfer · lore

Re: Multi-head pulling series

Josef Weidendorfer <Josef.Weidendorfer@gmx.de> writes:

> I think that both, default and non-default persistant
> mappings, are only a handy convenience issue for less
> typing. If GIT core is not doing this, some (most?) porcelains
> will invent their own way for storing these mappings.

Suppose I have something like this in $GIT_DIR/remotes/ko:

    URL: master.kernel.org:/pub/scm/git/git.git/
    Pull: master:ko-master rc:ko-rc pu:ko-pu
    Push: master:master pu:pu rc:rc
    Default-Pull: master rc

Am I reading you right so far?

    $ git push ko  ;# my master, rc, and pu goes there, under
                    # the same name.
    $ git fetch ko ;# their master comes to my ko-master, rc to my ko-rc
                    # pu is not touched.

But now, what should this do?

    $ git fetch ko master

(1) does it use "Pull" mappings and use their "master" to fast
    forward my "ko-master"?

or

(2) does it obey "the command line overrides the configuration
    file" principle and fetch their "master" but not touch my
    local refs (remember, refspec "master" without colon is
    equivalent to "master:" in fetch, which means "get the
    objects to complete it, but do not store the ref locally")?