Enable the packed refs file format

Enable the packed refs file format

[Linus Torvalds]

This actually "turns on" the packed ref file format, now that the 
infrastructure to do so sanely exists (ie notably the change to make the 
reference reading logic take refnames rather than pathnames to the loose 
objects that no longer necessarily even exist).

In particular, when the ref lookup hits a refname that has no loose file 
associated with it, it falls back on the packed-ref information. Also, the 
ref-locking code, while still using a loose file for the locking itself 
(and _creating_ a loose file for the new ref) no longer requires that the 
old ref be in such an unpacked state.

Finally, this does a minimal hack to git-checkout.sh to rather than check 
the ref-file directly, do a "git-rev-parse" on the "heads/$refname". 
That's not really wonderful - we should rather really have a special 
routine to verify the names as proper branch head names, but it is a 
workable solution for now.

With this, I can literally do something like

	git pack-refs
	find .git/refs -type f -print0 | xargs -0 rm -f --

and the end result is a largely working repository (ie I've done two 
commits - which creates _one_ unpacked ref file - done things like run 
"gitk" and "git log" etc, and it all looks ok).

There are probably things missing, but I'm hoping that the missing things 
are now of the "small and obvious" kind, and that somebody else might want 
to start looking at this too. Hint hint ;)

Signed-off-by: Linus Torvalds <torvalds@osdl.org>
---

This obviously depends on the lt/refs branch in Junio's tree, that is 
currently only in -pu.

diff --git a/git-checkout.sh b/git-checkout.sh
index 580a9e8..c60e029 100755
--- a/git-checkout.sh
+++ b/git-checkout.sh
@@ -22,7 +22,7 @@ while [ "$#" != "0" ]; do
 		shift
 		[ -z "$newbranch" ] &&
 			die "git checkout: -b needs a branch name"
-		[ -e "$GIT_DIR/refs/heads/$newbranch" ] &&
+		git-rev-parse --symbolic "heads/$newbranch" >&/dev/null &&
 			die "git checkout: branch $newbranch already exists"
 		git-check-ref-format "heads/$newbranch" ||
 			die "git checkout: we do not like '$newbranch' as a branch name."
@@ -51,7 +51,7 @@ while [ "$#" != "0" ]; do
 			fi
 			new="$rev"
 			new_name="$arg^0"
-			if [ -f "$GIT_DIR/refs/heads/$arg" ]; then
+			if git-rev-parse "heads/$arg^0" >&/dev/null; then
 				branch="$arg"
 			fi
 		elif rev=$(git-rev-parse --verify "$arg^{tree}" 2>/dev/null)
diff --git a/refs.c b/refs.c
index 50c25d3..134c0fc 100644
--- a/refs.c
+++ b/refs.c
@@ -28,6 +28,8 @@ static const char *parse_ref_line(char *
 	if (!isspace(line[40]))
 		return NULL;
 	line += 41;
+	if (isspace(*line))
+		return NULL;
 	if (line[len] != '\n')
 		return NULL;
 	line[len] = 0;
@@ -168,6 +170,14 @@ const char *resolve_ref(const char *ref,
 		 * reading.
 		 */
 		if (lstat(path, &st) < 0) {
+			struct ref_list *list = get_packed_refs();
+			while (list) {
+				if (!strcmp(ref, list->name)) {
+					hashcpy(sha1, list->sha1);
+					return ref;
+				}
+				list = list->next;
+			}
 			if (reading || errno != ENOENT)
 				return NULL;
 			hashclr(sha1);
@@ -400,22 +410,13 @@ int check_ref_format(const char *ref)
 static struct ref_lock *verify_lock(struct ref_lock *lock,
 	const unsigned char *old_sha1, int mustexist)
 {
-	char buf[40];
-	int nr, fd = open(lock->ref_file, O_RDONLY);
-	if (fd < 0 && (mustexist || errno != ENOENT)) {
-		error("Can't verify ref %s", lock->ref_file);
-		unlock_ref(lock);
-		return NULL;
-	}
-	nr = read(fd, buf, 40);
-	close(fd);
-	if (nr != 40 || get_sha1_hex(buf, lock->old_sha1) < 0) {
-		error("Can't verify ref %s", lock->ref_file);
+	if (!resolve_ref(lock->ref_name, lock->old_sha1, mustexist)) {
+		error("Can't verify ref %s", lock->ref_name);
 		unlock_ref(lock);
 		return NULL;
 	}
 	if (hashcmp(lock->old_sha1, old_sha1)) {
-		error("Ref %s is at %s but expected %s", lock->ref_file,
+		error("Ref %s is at %s but expected %s", lock->ref_name,
 			sha1_to_hex(lock->old_sha1), sha1_to_hex(old_sha1));
 		unlock_ref(lock);
 		return NULL;
@@ -427,6 +428,7 @@ static struct ref_lock *lock_ref_sha1_ba
 	int plen,
 	const unsigned char *old_sha1, int mustexist)
 {
+	char *ref_file;
 	const char *orig_ref = ref;
 	struct ref_lock *lock;
 	struct stat st;
@@ -445,13 +447,14 @@ static struct ref_lock *lock_ref_sha1_ba
 	}
 	lock->lk = xcalloc(1, sizeof(struct lock_file));
 
-	lock->ref_file = xstrdup(git_path("%s", ref));
+	lock->ref_name = xstrdup(ref);
 	lock->log_file = xstrdup(git_path("logs/%s", ref));
-	lock->force_write = lstat(lock->ref_file, &st) && errno == ENOENT;
+	ref_file = git_path(ref);
+	lock->force_write = lstat(ref_file, &st) && errno == ENOENT;
 
-	if (safe_create_leading_directories(lock->ref_file))
-		die("unable to create directory for %s", lock->ref_file);
-	lock->lock_fd = hold_lock_file_for_update(lock->lk, lock->ref_file, 1);
+	if (safe_create_leading_directories(ref_file))
+		die("unable to create directory for %s", ref_file);
+	lock->lock_fd = hold_lock_file_for_update(lock->lk, ref_file, 1);
 
 	return old_sha1 ? verify_lock(lock, old_sha1, mustexist) : lock;
 }
@@ -479,7 +482,7 @@ void unlock_ref(struct ref_lock *lock)
 		if (lock->lk)
 			rollback_lock_file(lock->lk);
 	}
-	free(lock->ref_file);
+	free(lock->ref_name);
 	free(lock->log_file);
 	free(lock);
 }
@@ -556,7 +559,7 @@ int write_ref_sha1(struct ref_lock *lock
 		return -1;
 	}
 	if (commit_lock_file(lock->lk)) {
-		error("Couldn't set %s", lock->ref_file);
+		error("Couldn't set %s", lock->ref_name);
 		unlock_ref(lock);
 		return -1;
 	}
diff --git a/refs.h b/refs.h
index 553155c..af347e6 100644
--- a/refs.h
+++ b/refs.h
@@ -2,7 +2,7 @@ #ifndef REFS_H
 #define REFS_H
 
 struct ref_lock {
-	char *ref_file;
+	char *ref_name;
 	char *log_file;
 	struct lock_file *lk;
 	unsigned char old_sha1[20];

Re: Enable the packed refs file format

[Petr Baudis]

Dear diary, on Thu, Sep 14, 2006 at 07:14:47PM CEST, I got a letter
where Linus Torvalds <torvalds@osdl.org> said that...
> +	ref_file = git_path(ref);

You slip...
You fall...
*BLAMMMM!!!*

Cloning a repository with '%s' tag over HTTP now dumps core nicely, and
I guess this kind of bugs tends to be exploitable.

-- 
				Petr "Pasky Yay for Obscure ADOM
					References" Baudis
Stuff: http://pasky.or.cz/
Snow falling on Perl. White noise covering line noise.
Hides all the bugs too. -- J. Putnam

Re: Enable the packed refs file format

[Linus Torvalds]

On Tue, 19 Sep 2006, Petr Baudis wrote:

> Dear diary, on Thu, Sep 14, 2006 at 07:14:47PM CEST, I got a letter
> where Linus Torvalds <torvalds@osdl.org> said that...
> > +	ref_file = git_path(ref);
> 
> You slip...
> You fall...
> *BLAMMMM!!!*

Gaah. Yes. I fixed one such mistake already.

Too bad that we can't get gcc to warn on these things. We do mark it as 
"format(printf)", but I don't know of any way to tell gcc that it _has_ to 
have that initial constant string.

		Linus

Re: Enable the packed refs file format

[Phil Richards]

On 2006-09-19, Linus Torvalds <torvalds@osdl.org> wrote:
>  Too bad that we can't get gcc to warn on these things. We do mark it as 
>  "format(printf)", but I don't know of any way to tell gcc that it _has_ to 
>  have that initial constant string.

Not sure if it just a gcc 4.x-ism, but -Wformat-nonliteral or -Wformat-security
might be what you are looking for.

`-Wformat-nonliteral'
     If `-Wformat' is specified, also warn if the format string is not a
     string literal and so cannot be checked, unless the format function
     takes its format arguments as a `va_list'.

`-Wformat-security'
     If `-Wformat' is specified, also warn about uses of format
     functions that represent possible security problems.  At present,
     this warns about calls to `printf' and `scanf' functions where the
     format string is not a string literal and there are no format
     arguments, as in `printf (foo);'.  This may be a security hole if
     the format string came from untrusted input and contains `%n'.
     (This is currently a subset of what `-Wformat-nonliteral' warns
     about, but in future warnings may be added to `-Wformat-security'
     that are not included in `-Wformat-nonliteral'.)


phil
-- 
change name before "@" to "phil" for email

[PATCH] Fix buggy ref recording

[Petr Baudis]

Dear diary, on Tue, Sep 19, 2006 at 10:55:54PM CEST, I got a letter
where Petr Baudis <pasky@suse.cz> said that...
> Dear diary, on Thu, Sep 14, 2006 at 07:14:47PM CEST, I got a letter
> where Linus Torvalds <torvalds@osdl.org> said that...
> > +	ref_file = git_path(ref);
> 
> You slip...
> You fall...
> *BLAMMMM!!!*
> 
> Cloning a repository with '%s' tag over HTTP now dumps core nicely, and
> I guess this kind of bugs tends to be exploitable.

And since just reporting it did not magically result in a fix... ;-)

-8<-

There is a format string vulnerability introduced with the packed refs
file format.

Signed-off-by: Petr Baudis <pasky@suse.cz>
---

 refs.c |    2 +-
 1 files changed, 1 insertions(+), 1 deletions(-)

diff --git a/refs.c b/refs.c
index 40f16af..5fdf9c4 100644
--- a/refs.c
+++ b/refs.c
@@ -472,7 +472,7 @@ static struct ref_lock *lock_ref_sha1_ba

 	lock->ref_name = xstrdup(ref);
 	lock->log_file = xstrdup(git_path("logs/%s", ref));
-	ref_file = git_path(ref);
+	ref_file = git_path("%s", ref);
 	lock->force_write = lstat(ref_file, &st) && errno == ENOENT;

 	if (safe_create_leading_directories(ref_file))


-- 
				Petr "Pasky" Baudis
Stuff: http://pasky.or.cz/
#!/bin/perl -sp0777i<X+d*lMLa^*lN%0]dsXx++lMlN/dsM0<j]dsj
$/=unpack('H*',$_);$_=`echo 16dio\U$k"SK$/SM$n\EsN0p[lN*1
lK[d2%Sa2/d0$^Ixp"|dc`;s/\W//g;$_=pack('H*',/((..)*)$/)

Re: [PATCH] Fix buggy ref recording

[Junio C Hamano]

Petr Baudis <pasky@suse.cz> writes:

> And since just reporting it did not magically result in a fix... ;-)

Yes, please always send in a patch to be applied to get the
attribution right.

I've never seen you send out a corrupt patch over e-mail.
What's different this time?

> diff --git a/refs.c b/refs.c
> index 40f16af..5fdf9c4 100644
> --- a/refs.c
> +++ b/refs.c
> @@ -472,7 +472,7 @@ static struct ref_lock *lock_ref_sha1_ba
>
>  	lock->ref_name = xstrdup(ref);
>  	lock->log_file = xstrdup(git_path("logs/%s", ref));

The empty line at the beginning of the hunk is totally empty,
not even with a SP to show it is a context line.

Will hand-apply, no need to resend.

Re: [PATCH] Fix buggy ref recording

[Petr Baudis]

Dear diary, on Sat, Sep 23, 2006 at 02:44:31AM CEST, I got a letter
where Junio C Hamano <junkio@cox.net> said that...
> I've never seen you send out a corrupt patch over e-mail.
> What's different this time?
> 
> > diff --git a/refs.c b/refs.c
> > index 40f16af..5fdf9c4 100644
> > --- a/refs.c
> > +++ b/refs.c
> > @@ -472,7 +472,7 @@ static struct ref_lock *lock_ref_sha1_ba
> >
> >  	lock->ref_name = xstrdup(ref);
> >  	lock->log_file = xstrdup(git_path("logs/%s", ref));
> 
> The empty line at the beginning of the hunk is totally empty,
> not even with a SP to show it is a context line.

Sorry, I've cut'n'pasted from an ssh session on repo.or.cz and *thought*
that I fixed the whitespaces...

-- 
				Petr "Pasky" Baudis
Stuff: http://pasky.or.cz/
#!/bin/perl -sp0777i<X+d*lMLa^*lN%0]dsXx++lMlN/dsM0<j]dsj
$/=unpack('H*',$_);$_=`echo 16dio\U$k"SK$/SM$n\EsN0p[lN*1
lK[d2%Sa2/d0$^Ixp"|dc`;s/\W//g;$_=pack('H*',/((..)*)$/)

[PATCH] pack-refs: fix git_path() usage.

[Junio C Hamano]

Signed-off-by: Junio C Hamano <junkio@cox.net>
---

 * A valid ref name can contain %.

 builtin-pack-refs.c |    2 +-
 1 files changed, 1 insertions(+), 1 deletions(-)

diff --git a/builtin-pack-refs.c b/builtin-pack-refs.c
index 246dd63..db57fee 100644
--- a/builtin-pack-refs.c
+++ b/builtin-pack-refs.c
@@ -56,7 +56,7 @@ static void prune_ref(struct ref_to_prun
 	struct ref_lock *lock = lock_ref_sha1(r->name + 5, r->sha1, 1);
 
 	if (lock) {
-		unlink(git_path(r->name));
+		unlink(git_path("%s", r->name));
 		unlock_ref(lock);
 	}
 }
-- 
1.4.2.1.gf2bba