threads / discuss / 5561

Enable the packed refs file format

Subject: Enable the packed refs file format

## tl;dr

8 messages between Sep 14, 2006 and Sep 23, 2006.

replies: 7people: 4as markdown or json

Linus Torvalds· Sep 14, 2006, 17:14 UTC · lore

This actually "turns on" the packed ref file format, now that the infrastructure to do so sanely exists (ie notably the change to make the reference reading logic take refnames rather than pathnames to the loose objects that no longer necessarily even exist).

In particular, when the ref lookup hits a refname that has no loose file associated with it, it falls back on the packed-ref information. Also, the ref-locking code, while still using a loose file for the locking itself (and _creating_ a loose file for the new ref) no longer requires that the old ref be in such an unpacked state.

Finally, this does a minimal hack to git-checkout.sh to rather than check the ref-file directly, do a "git-rev-parse" on the "heads/$refname". That's not really wonderful - we should rather really have a special routine to verify the names as proper branch head names, but it is a workable solution for now.

With this, I can literally do something like
	git pack-refs
	find .git/refs -type f -print0 | xargs -0 rm -f --

and the end result is a largely working repository (ie I've done two commits - which creates _one_ unpacked ref file - done things like run "gitk" and "git log" etc, and it all looks ok).

There are probably things missing, but I'm hoping that the missing things are now of the "small and obvious" kind, and that somebody else might want to start looking at this too. Hint hint ;)

Signed-off-by: Linus Torvalds <torvalds@osdl.org>
---

This obviously depends on the lt/refs branch in Junio's tree, that is currently only in -pu.

diff --git a/git-checkout.sh b/git-checkout.sh
index 580a9e8..c60e029 100755
--- a/git-checkout.sh
+++ b/git-checkout.sh
@@ -22,7 +22,7 @@ while [ "$#" != "0" ]; do
 		shift
 		[ -z "$newbranch" ] &&
 			die "git checkout: -b needs a branch name"
-		[ -e "$GIT_DIR/refs/heads/$newbranch" ] &&
+		git-rev-parse --symbolic "heads/$newbranch" >&/dev/null &&
 			die "git checkout: branch $newbranch already exists"
 		git-check-ref-format "heads/$newbranch" ||
 			die "git checkout: we do not like '$newbranch' as a branch name."
@@ -51,7 +51,7 @@ while [ "$#" != "0" ]; do
 			fi
 			new="$rev"
 			new_name="$arg^0"
-			if [ -f "$GIT_DIR/refs/heads/$arg" ]; then
+			if git-rev-parse "heads/$arg^0" >&/dev/null; then
 				branch="$arg"
 			fi
 		elif rev=$(git-rev-parse --verify "$arg^{tree}" 2>/dev/null)
diff --git a/refs.c b/refs.c
index 50c25d3..134c0fc 100644
--- a/refs.c
+++ b/refs.c
@@ -28,6 +28,8 @@ static const char *parse_ref_line(char *
 	if (!isspace(line[40]))
 		return NULL;
 	line += 41;
+	if (isspace(*line))
+		return NULL;
 	if (line[len] != '\n')
 		return NULL;
 	line[len] = 0;
@@ -168,6 +170,14 @@ const char *resolve_ref(const char *ref,
 		 * reading.
 		 */
 		if (lstat(path, &st) < 0) {
+			struct ref_list *list = get_packed_refs();
+			while (list) {
+				if (!strcmp(ref, list->name)) {
+					hashcpy(sha1, list->sha1);
+					return ref;
+				}
+				list = list->next;
+			}
 			if (reading || errno != ENOENT)
 				return NULL;
 			hashclr(sha1);
@@ -400,22 +410,13 @@ int check_ref_format(const char *ref)
 static struct ref_lock *verify_lock(struct ref_lock *lock,
 	const unsigned char *old_sha1, int mustexist)
 {
-	char buf[40];
-	int nr, fd = open(lock->ref_file, O_RDONLY);
-	if (fd < 0 && (mustexist || errno != ENOENT)) {
-		error("Can't verify ref %s", lock->ref_file);
-		unlock_ref(lock);
-		return NULL;
-	}
-	nr = read(fd, buf, 40);
-	close(fd);
-	if (nr != 40 || get_sha1_hex(buf, lock->old_sha1) < 0) {
-		error("Can't verify ref %s", lock->ref_file);
+	if (!resolve_ref(lock->ref_name, lock->old_sha1, mustexist)) {
+		error("Can't verify ref %s", lock->ref_name);
 		unlock_ref(lock);
 		return NULL;
 	}
 	if (hashcmp(lock->old_sha1, old_sha1)) {
-		error("Ref %s is at %s but expected %s", lock->ref_file,
+		error("Ref %s is at %s but expected %s", lock->ref_name,
 			sha1_to_hex(lock->old_sha1), sha1_to_hex(old_sha1));
 		unlock_ref(lock);
 		return NULL;
@@ -427,6 +428,7 @@ static struct ref_lock *lock_ref_sha1_ba
 	int plen,
 	const unsigned char *old_sha1, int mustexist)
 {
+	char *ref_file;
 	const char *orig_ref = ref;
 	struct ref_lock *lock;
 	struct stat st;
@@ -445,13 +447,14 @@ static struct ref_lock *lock_ref_sha1_ba
 	}
 	lock->lk = xcalloc(1, sizeof(struct lock_file));
 
-	lock->ref_file = xstrdup(git_path("%s", ref));
+	lock->ref_name = xstrdup(ref);
 	lock->log_file = xstrdup(git_path("logs/%s", ref));
-	lock->force_write = lstat(lock->ref_file, &st) && errno == ENOENT;
+	ref_file = git_path(ref);
+	lock->force_write = lstat(ref_file, &st) && errno == ENOENT;
 
-	if (safe_create_leading_directories(lock->ref_file))
-		die("unable to create directory for %s", lock->ref_file);
-	lock->lock_fd = hold_lock_file_for_update(lock->lk, lock->ref_file, 1);
+	if (safe_create_leading_directories(ref_file))
+		die("unable to create directory for %s", ref_file);
+	lock->lock_fd = hold_lock_file_for_update(lock->lk, ref_file, 1);
 
 	return old_sha1 ? verify_lock(lock, old_sha1, mustexist) : lock;
 }
@@ -479,7 +482,7 @@ void unlock_ref(struct ref_lock *lock)
 		if (lock->lk)
 			rollback_lock_file(lock->lk);
 	}
-	free(lock->ref_file);
+	free(lock->ref_name);
 	free(lock->log_file);
 	free(lock);
 }
@@ -556,7 +559,7 @@ int write_ref_sha1(struct ref_lock *lock
 		return -1;
 	}
 	if (commit_lock_file(lock->lk)) {
-		error("Couldn't set %s", lock->ref_file);
+		error("Couldn't set %s", lock->ref_name);
 		unlock_ref(lock);
 		return -1;
 	}
diff --git a/refs.h b/refs.h
index 553155c..af347e6 100644
--- a/refs.h
+++ b/refs.h
@@ -2,7 +2,7 @@ #ifndef REFS_H
 #define REFS_H
 
 struct ref_lock {
-	char *ref_file;
+	char *ref_name;
 	char *log_file;
 	struct lock_file *lk;
 	unsigned char old_sha1[20];
Petr Baudis· Sep 19, 2006, 20:55 UTC · re: Linus Torvalds · lore

Re: Enable the packed refs file format

Dear diary, on Thu, Sep 14, 2006 at 07:14:47PM CEST, I got a letter where Linus Torvalds <torvalds@osdl.org> said that...

> +	ref_file = git_path(ref);

You slip... You fall... *BLAMMMM!!!*

Cloning a repository with '%s' tag over HTTP now dumps core nicely, and I guess this kind of bugs tends to be exploitable.

-- 
				Petr "Pasky Yay for Obscure ADOM
					References" Baudis
Stuff: http://pasky.or.cz/
Snow falling on Perl. White noise covering line noise.
Hides all the bugs too. -- J. Putnam
Linus Torvalds· Sep 19, 2006, 21:09 UTC · re: Petr Baudis · lore

Re: Enable the packed refs file format

On Tue, 19 Sep 2006, Petr Baudis wrote:
Show 7 quoted lines
> Dear diary, on Thu, Sep 14, 2006 at 07:14:47PM CEST, I got a letter
> where Linus Torvalds <torvalds@osdl.org> said that...
> > +	ref_file = git_path(ref);
> 
> You slip...
> You fall...
> *BLAMMMM!!!*
Gaah. Yes. I fixed one such mistake already.

Too bad that we can't get gcc to warn on these things. We do mark it as "format(printf)", but I don't know of any way to tell gcc that it _has_ to have that initial constant string.

		Linus
Phil Richards· Sep 20, 2006, 20:19 UTC · re: Linus Torvalds · lore

Re: Enable the packed refs file format

On 2006-09-19, Linus Torvalds <torvalds@osdl.org> wrote:
>  Too bad that we can't get gcc to warn on these things. We do mark it as 
>  "format(printf)", but I don't know of any way to tell gcc that it _has_ to 
>  have that initial constant string.

Not sure if it just a gcc 4.x-ism, but -Wformat-nonliteral or -Wformat-security might be what you are looking for.

`-Wformat-nonliteral'
     If `-Wformat' is specified, also warn if the format string is not a
     string literal and so cannot be checked, unless the format function
     takes its format arguments as a `va_list'.
`-Wformat-security'
     If `-Wformat' is specified, also warn about uses of format
     functions that represent possible security problems.  At present,
     this warns about calls to `printf' and `scanf' functions where the
     format string is not a string literal and there are no format
     arguments, as in `printf (foo);'.  This may be a security hole if
     the format string came from untrusted input and contains `%n'.
     (This is currently a subset of what `-Wformat-nonliteral' warns
     about, but in future warnings may be added to `-Wformat-security'
     that are not included in `-Wformat-nonliteral'.)
phil
-- 
change name before "@" to "phil" for email
Petr Baudis· Sep 22, 2006, 23:08 UTC · re: Petr Baudis · lore

[PATCH] Fix buggy ref recording

Dear diary, on Tue, Sep 19, 2006 at 10:55:54PM CEST, I got a letter where Petr Baudis <pasky@suse.cz> said that...

Show 10 quoted lines
> Dear diary, on Thu, Sep 14, 2006 at 07:14:47PM CEST, I got a letter
> where Linus Torvalds <torvalds@osdl.org> said that...
> > +	ref_file = git_path(ref);
> 
> You slip...
> You fall...
> *BLAMMMM!!!*
> 
> Cloning a repository with '%s' tag over HTTP now dumps core nicely, and
> I guess this kind of bugs tends to be exploitable.
And since just reporting it did not magically result in a fix... ;-)
-8<-

There is a format string vulnerability introduced with the packed refs file format.

Signed-off-by: Petr Baudis <pasky@suse.cz>
---
 refs.c |    2 +-
 1 files changed, 1 insertions(+), 1 deletions(-)
diff --git a/refs.c b/refs.c
index 40f16af..5fdf9c4 100644
--- a/refs.c
+++ b/refs.c
@@ -472,7 +472,7 @@ static struct ref_lock *lock_ref_sha1_ba

 	lock->ref_name = xstrdup(ref);
 	lock->log_file = xstrdup(git_path("logs/%s", ref));
-	ref_file = git_path(ref);
+	ref_file = git_path("%s", ref);
 	lock->force_write = lstat(ref_file, &st) && errno == ENOENT;

 	if (safe_create_leading_directories(ref_file))
-- 
				Petr "Pasky" Baudis
Stuff: http://pasky.or.cz/
#!/bin/perl -sp0777i<X+d*lMLa^*lN%0]dsXx++lMlN/dsM0<j]dsj
$/=unpack('H*',$_);$_=`echo 16dio\U$k"SK$/SM$n\EsN0p[lN*1
lK[d2%Sa2/d0$^Ixp"|dc`;s/\W//g;$_=pack('H*',/((..)*)$/)
Junio C Hamano· Sep 23, 2006, 00:44 UTC · re: Petr Baudis · lore

Re: [PATCH] Fix buggy ref recording

Petr Baudis <pasky@suse.cz> writes:
> And since just reporting it did not magically result in a fix... ;-)

Yes, please always send in a patch to be applied to get the attribution right.

I've never seen you send out a corrupt patch over e-mail. What's different this time?

Show 8 quoted lines
> diff --git a/refs.c b/refs.c
> index 40f16af..5fdf9c4 100644
> --- a/refs.c
> +++ b/refs.c
> @@ -472,7 +472,7 @@ static struct ref_lock *lock_ref_sha1_ba
>
>  	lock->ref_name = xstrdup(ref);
>  	lock->log_file = xstrdup(git_path("logs/%s", ref));

The empty line at the beginning of the hunk is totally empty, not even with a SP to show it is a context line.

Will hand-apply, no need to resend.
Petr Baudis· Sep 23, 2006, 01:16 UTC · re: Junio C Hamano · lore

Re: [PATCH] Fix buggy ref recording

Dear diary, on Sat, Sep 23, 2006 at 02:44:31AM CEST, I got a letter where Junio C Hamano <junkio@cox.net> said that...

Show 14 quoted lines
> I've never seen you send out a corrupt patch over e-mail.
> What's different this time?
> 
> > diff --git a/refs.c b/refs.c
> > index 40f16af..5fdf9c4 100644
> > --- a/refs.c
> > +++ b/refs.c
> > @@ -472,7 +472,7 @@ static struct ref_lock *lock_ref_sha1_ba
> >
> >  	lock->ref_name = xstrdup(ref);
> >  	lock->log_file = xstrdup(git_path("logs/%s", ref));
> 
> The empty line at the beginning of the hunk is totally empty,
> not even with a SP to show it is a context line.

Sorry, I've cut'n'pasted from an ssh session on repo.or.cz and *thought* that I fixed the whitespaces...

-- 
				Petr "Pasky" Baudis
Stuff: http://pasky.or.cz/
#!/bin/perl -sp0777i<X+d*lMLa^*lN%0]dsXx++lMlN/dsM0<j]dsj
$/=unpack('H*',$_);$_=`echo 16dio\U$k"SK$/SM$n\EsN0p[lN*1
lK[d2%Sa2/d0$^Ixp"|dc`;s/\W//g;$_=pack('H*',/((..)*)$/)
Junio C Hamano· Sep 23, 2006, 04:34 UTC · re: Petr Baudis · lore

[PATCH] pack-refs: fix git_path() usage.

Signed-off-by: Junio C Hamano <junkio@cox.net>
---
 * A valid ref name can contain %.
 builtin-pack-refs.c |    2 +-
 1 files changed, 1 insertions(+), 1 deletions(-)
diff --git a/builtin-pack-refs.c b/builtin-pack-refs.c
index 246dd63..db57fee 100644
--- a/builtin-pack-refs.c
+++ b/builtin-pack-refs.c
@@ -56,7 +56,7 @@ static void prune_ref(struct ref_to_prun
 	struct ref_lock *lock = lock_ref_sha1(r->name + 5, r->sha1, 1);
 
 	if (lock) {
-		unlink(git_path(r->name));
+		unlink(git_path("%s", r->name));
 		unlock_ref(lock);
 	}
 }
-- 
1.4.2.1.gf2bba

← back to recent threads