# Enable the packed refs file format 8 messages from 2006-09-14 to 2006-09-23. Participants: Linus Torvalds, Petr Baudis, Phil Richards, Junio C Hamano. Thread: https://gitlist.dev/t/5561 ## Linus Torvalds, 2006-09-14 17:14 Subject: Enable the packed refs file format Message-ID: URL: https://gitlist.dev/e/Pine.LNX.4.64.0609141005440.4388%40g5.osdl.org ``` This actually "turns on" the packed ref file format, now that the infrastructure to do so sanely exists (ie notably the change to make the reference reading logic take refnames rather than pathnames to the loose objects that no longer necessarily even exist). In particular, when the ref lookup hits a refname that has no loose file associated with it, it falls back on the packed-ref information. Also, the ref-locking code, while still using a loose file for the locking itself (and _creating_ a loose file for the new ref) no longer requires that the old ref be in such an unpacked state. Finally, this does a minimal hack to git-checkout.sh to rather than check the ref-file directly, do a "git-rev-parse" on the "heads/$refname". That's not really wonderful - we should rather really have a special routine to verify the names as proper branch head names, but it is a workable solution for now. With this, I can literally do something like git pack-refs find .git/refs -type f -print0 | xargs -0 rm -f -- and the end result is a largely working repository (ie I've done two commits - which creates _one_ unpacked ref file - done things like run "gitk" and "git log" etc, and it all looks ok). There are probably things missing, but I'm hoping that the missing things are now of the "small and obvious" kind, and that somebody else might want to start looking at this too. Hint hint ;) Signed-off-by: Linus Torvalds --- This obviously depends on the lt/refs branch in Junio's tree, that is currently only in -pu. diff --git a/git-checkout.sh b/git-checkout.sh index 580a9e8..c60e029 100755 --- a/git-checkout.sh +++ b/git-checkout.sh @@ -22,7 +22,7 @@ while [ "$#" != "0" ]; do shift [ -z "$newbranch" ] && die "git checkout: -b needs a branch name" - [ -e "$GIT_DIR/refs/heads/$newbranch" ] && + git-rev-parse --symbolic "heads/$newbranch" >&/dev/null && die "git checkout: branch $newbranch already exists" git-check-ref-format "heads/$newbranch" || die "git checkout: we do not like '$newbranch' as a branch name." @@ -51,7 +51,7 @@ while [ "$#" != "0" ]; do fi new="$rev" new_name="$arg^0" - if [ -f "$GIT_DIR/refs/heads/$arg" ]; then + if git-rev-parse "heads/$arg^0" >&/dev/null; then branch="$arg" fi elif rev=$(git-rev-parse --verify "$arg^{tree}" 2>/dev/null) diff --git a/refs.c b/refs.c index 50c25d3..134c0fc 100644 --- a/refs.c +++ b/refs.c @@ -28,6 +28,8 @@ static const char *parse_ref_line(char * if (!isspace(line[40])) return NULL; line += 41; + if (isspace(*line)) + return NULL; if (line[len] != '\n') return NULL; line[len] = 0; @@ -168,6 +170,14 @@ const char *resolve_ref(const char *ref, * reading. */ if (lstat(path, &st) < 0) { + struct ref_list *list = get_packed_refs(); + while (list) { + if (!strcmp(ref, list->name)) { + hashcpy(sha1, list->sha1); + return ref; + } + list = list->next; + } if (reading || errno != ENOENT) return NULL; hashclr(sha1); @@ -400,22 +410,13 @@ int check_ref_format(const char *ref) static struct ref_lock *verify_lock(struct ref_lock *lock, const unsigned char *old_sha1, int mustexist) { - char buf[40]; - int nr, fd = open(lock->ref_file, O_RDONLY); - if (fd < 0 && (mustexist || errno != ENOENT)) { - error("Can't verify ref %s", lock->ref_file); - unlock_ref(lock); - return NULL; - } - nr = read(fd, buf, 40); - close(fd); - if (nr != 40 || get_sha1_hex(buf, lock->old_sha1) < 0) { - error("Can't verify ref %s", lock->ref_file); + if (!resolve_ref(lock->ref_name, lock->old_sha1, mustexist)) { + error("Can't verify ref %s", lock->ref_name); unlock_ref(lock); return NULL; } if (hashcmp(lock->old_sha1, old_sha1)) { - error("Ref %s is at %s but expected %s", lock->ref_file, + error("Ref %s is at %s but expected %s", lock->ref_name, sha1_to_hex(lock->old_sha1), sha1_to_hex(old_sha1)); unlock_ref(lock); return NULL; @@ -427,6 +428,7 @@ static struct ref_lock *lock_ref_sha1_ba int plen, const unsigned char *old_sha1, int mustexist) { + char *ref_file; const char *orig_ref = ref; struct ref_lock *lock; struct stat st; @@ -445,13 +447,14 @@ static struct ref_lock *lock_ref_sha1_ba } lock->lk = xcalloc(1, sizeof(struct lock_file)); - lock->ref_file = xstrdup(git_path("%s", ref)); + lock->ref_name = xstrdup(ref); lock->log_file = xstrdup(git_path("logs/%s", ref)); - lock->force_write = lstat(lock->ref_file, &st) && errno == ENOENT; + ref_file = git_path(ref); + lock->force_write = lstat(ref_file, &st) && errno == ENOENT; - if (safe_create_leading_directories(lock->ref_file)) - die("unable to create directory for %s", lock->ref_file); - lock->lock_fd = hold_lock_file_for_update(lock->lk, lock->ref_file, 1); + if (safe_create_leading_directories(ref_file)) + die("unable to create directory for %s", ref_file); + lock->lock_fd = hold_lock_file_for_update(lock->lk, ref_file, 1); return old_sha1 ? verify_lock(lock, old_sha1, mustexist) : lock; } @@ -479,7 +482,7 @@ void unlock_ref(struct ref_lock *lock) if (lock->lk) rollback_lock_file(lock->lk); } - free(lock->ref_file); + free(lock->ref_name); free(lock->log_file); free(lock); } @@ -556,7 +559,7 @@ int write_ref_sha1(struct ref_lock *lock return -1; } if (commit_lock_file(lock->lk)) { - error("Couldn't set %s", lock->ref_file); + error("Couldn't set %s", lock->ref_name); unlock_ref(lock); return -1; } diff --git a/refs.h b/refs.h index 553155c..af347e6 100644 --- a/refs.h +++ b/refs.h @@ -2,7 +2,7 @@ #ifndef REFS_H #define REFS_H struct ref_lock { - char *ref_file; + char *ref_name; char *log_file; struct lock_file *lk; unsigned char old_sha1[20]; ``` ## Petr Baudis, 2006-09-19 20:55 Subject: Re: Enable the packed refs file format Message-ID: <20060919205554.GA8259@pasky.or.cz> URL: https://gitlist.dev/e/20060919205554.GA8259%40pasky.or.cz In-Reply-To: ``` Dear diary, on Thu, Sep 14, 2006 at 07:14:47PM CEST, I got a letter where Linus Torvalds said that... > + ref_file = git_path(ref); You slip... You fall... *BLAMMMM!!!* Cloning a repository with '%s' tag over HTTP now dumps core nicely, and I guess this kind of bugs tends to be exploitable. -- Petr "Pasky Yay for Obscure ADOM References" Baudis Stuff: http://pasky.or.cz/ Snow falling on Perl. White noise covering line noise. Hides all the bugs too. -- J. Putnam ``` ## Linus Torvalds, 2006-09-19 21:09 Subject: Re: Enable the packed refs file format Message-ID: URL: https://gitlist.dev/e/Pine.LNX.4.64.0609191407340.4388%40g5.osdl.org In-Reply-To: <20060919205554.GA8259@pasky.or.cz> ``` On Tue, 19 Sep 2006, Petr Baudis wrote: > Dear diary, on Thu, Sep 14, 2006 at 07:14:47PM CEST, I got a letter > where Linus Torvalds said that... > > + ref_file = git_path(ref); > > You slip... > You fall... > *BLAMMMM!!!* Gaah. Yes. I fixed one such mistake already. Too bad that we can't get gcc to warn on these things. We do mark it as "format(printf)", but I don't know of any way to tell gcc that it _has_ to have that initial constant string. Linus ``` ## Phil Richards, 2006-09-20 20:19 Subject: Re: Enable the packed refs file format Message-ID: <20060920201930.9E48F487D@derisoft.derived-software.demon.co.uk> URL: https://gitlist.dev/e/20060920201930.9E48F487D%40derisoft.derived-software.demon.co.uk In-Reply-To: ``` On 2006-09-19, Linus Torvalds wrote: > Too bad that we can't get gcc to warn on these things. We do mark it as > "format(printf)", but I don't know of any way to tell gcc that it _has_ to > have that initial constant string. Not sure if it just a gcc 4.x-ism, but -Wformat-nonliteral or -Wformat-security might be what you are looking for. `-Wformat-nonliteral' If `-Wformat' is specified, also warn if the format string is not a string literal and so cannot be checked, unless the format function takes its format arguments as a `va_list'. `-Wformat-security' If `-Wformat' is specified, also warn about uses of format functions that represent possible security problems. At present, this warns about calls to `printf' and `scanf' functions where the format string is not a string literal and there are no format arguments, as in `printf (foo);'. This may be a security hole if the format string came from untrusted input and contains `%n'. (This is currently a subset of what `-Wformat-nonliteral' warns about, but in future warnings may be added to `-Wformat-security' that are not included in `-Wformat-nonliteral'.) phil -- change name before "@" to "phil" for email ``` ## Petr Baudis, 2006-09-22 23:08 Subject: [PATCH] Fix buggy ref recording Message-ID: <20060922230845.GB8259@pasky.or.cz> URL: https://gitlist.dev/e/20060922230845.GB8259%40pasky.or.cz In-Reply-To: <20060919205554.GA8259@pasky.or.cz> ``` Dear diary, on Tue, Sep 19, 2006 at 10:55:54PM CEST, I got a letter where Petr Baudis said that... > Dear diary, on Thu, Sep 14, 2006 at 07:14:47PM CEST, I got a letter > where Linus Torvalds said that... > > + ref_file = git_path(ref); > > You slip... > You fall... > *BLAMMMM!!!* > > Cloning a repository with '%s' tag over HTTP now dumps core nicely, and > I guess this kind of bugs tends to be exploitable. And since just reporting it did not magically result in a fix... ;-) -8<- There is a format string vulnerability introduced with the packed refs file format. Signed-off-by: Petr Baudis --- refs.c | 2 +- 1 files changed, 1 insertions(+), 1 deletions(-) diff --git a/refs.c b/refs.c index 40f16af..5fdf9c4 100644 --- a/refs.c +++ b/refs.c @@ -472,7 +472,7 @@ static struct ref_lock *lock_ref_sha1_ba lock->ref_name = xstrdup(ref); lock->log_file = xstrdup(git_path("logs/%s", ref)); - ref_file = git_path(ref); + ref_file = git_path("%s", ref); lock->force_write = lstat(ref_file, &st) && errno == ENOENT; if (safe_create_leading_directories(ref_file)) -- Petr "Pasky" Baudis Stuff: http://pasky.or.cz/ #!/bin/perl -sp0777i URL: https://gitlist.dev/e/7vzmcrxvgw.fsf%40assigned-by-dhcp.cox.net In-Reply-To: <20060922230845.GB8259@pasky.or.cz> ``` Petr Baudis writes: > And since just reporting it did not magically result in a fix... ;-) Yes, please always send in a patch to be applied to get the attribution right. I've never seen you send out a corrupt patch over e-mail. What's different this time? > diff --git a/refs.c b/refs.c > index 40f16af..5fdf9c4 100644 > --- a/refs.c > +++ b/refs.c > @@ -472,7 +472,7 @@ static struct ref_lock *lock_ref_sha1_ba > > lock->ref_name = xstrdup(ref); > lock->log_file = xstrdup(git_path("logs/%s", ref)); The empty line at the beginning of the hunk is totally empty, not even with a SP to show it is a context line. Will hand-apply, no need to resend. ``` ## Petr Baudis, 2006-09-23 01:16 Subject: Re: [PATCH] Fix buggy ref recording Message-ID: <20060923011634.GL13132@pasky.or.cz> URL: https://gitlist.dev/e/20060923011634.GL13132%40pasky.or.cz In-Reply-To: <7vzmcrxvgw.fsf@assigned-by-dhcp.cox.net> ``` Dear diary, on Sat, Sep 23, 2006 at 02:44:31AM CEST, I got a letter where Junio C Hamano said that... > I've never seen you send out a corrupt patch over e-mail. > What's different this time? > > > diff --git a/refs.c b/refs.c > > index 40f16af..5fdf9c4 100644 > > --- a/refs.c > > +++ b/refs.c > > @@ -472,7 +472,7 @@ static struct ref_lock *lock_ref_sha1_ba > > > > lock->ref_name = xstrdup(ref); > > lock->log_file = xstrdup(git_path("logs/%s", ref)); > > The empty line at the beginning of the hunk is totally empty, > not even with a SP to show it is a context line. Sorry, I've cut'n'pasted from an ssh session on repo.or.cz and *thought* that I fixed the whitespaces... -- Petr "Pasky" Baudis Stuff: http://pasky.or.cz/ #!/bin/perl -sp0777i URL: https://gitlist.dev/e/7vmz8rw68n.fsf_-_%40assigned-by-dhcp.cox.net In-Reply-To: <20060923011634.GL13132@pasky.or.cz> ``` Signed-off-by: Junio C Hamano --- * A valid ref name can contain %. builtin-pack-refs.c | 2 +- 1 files changed, 1 insertions(+), 1 deletions(-) diff --git a/builtin-pack-refs.c b/builtin-pack-refs.c index 246dd63..db57fee 100644 --- a/builtin-pack-refs.c +++ b/builtin-pack-refs.c @@ -56,7 +56,7 @@ static void prune_ref(struct ref_to_prun struct ref_lock *lock = lock_ref_sha1(r->name + 5, r->sha1, 1); if (lock) { - unlink(git_path(r->name)); + unlink(git_path("%s", r->name)); unlock_ref(lock); } } -- 1.4.2.1.gf2bba ```