[PATCH] add a diff-files command (revised and cleaned up)

[PATCH] add a diff-files command (revised and cleaned up)

[Nicolas Pitre]

[ sorry for the resent
  still experiencing glitches with cogito
  please ignore previous patch ]

In the same spirit as diff-tree and diff-cache, here is a diff-files 
command that processes differences between the index cache and the 
working directory content.  It produces lists of files that are either 
changed, deleted and/or unknown with regards to the current cache, 
content. The -p option can also be used to generate a patch describing 
the differences in patch form.

It also has the ability to accept exclude patterns for files and the 
ability to read those exclude patterns from a file.

Typical usage looks like:

    diff-files --others --exclude=\*.o arch/arm/ include/asm-arm/

which lists all files the git cache doesn't know about in arch/arm/ and 
include/asm-arm/ but ignoring any object files.  Or:

    diff-files --all -p --exclude-from=dontdiff.list

which produces a patch of all changes currently in the work tree while 
excluding all files matching any of the patterns listed in 
dontdiff.list (useful when one doesn't want to run 'make distclean').

(revised after comments from Junio C Hamano)
Signed-off-by: Nicolas Pitre <nico@cam.org>

--- k/Makefile
+++ l/Makefile
@@ -18,7 +18,7 @@ PROG=   update-cache show-diff init-db w
 	cat-file fsck-cache checkout-cache diff-tree rev-tree show-files \
 	check-files ls-tree merge-base merge-cache unpack-file git-export \
 	diff-cache convert-cache http-pull rpush rpull rev-list git-mktag \
-	diff-tree-helper
+	diff-tree-helper diff-files
 
 all: $(PROG)
 
--- k/diff-files.c
+++ l/diff-files.c
@@ -0,0 +1,347 @@
+/*
+ * GIT - The information manager from hell
+ *
+ * Copyright (C) Linus Torvalds, 2005
+ */
+
+#include <dirent.h>
+#include <fnmatch.h>
+#include "cache.h"
+#include "diff.h"
+
+static const char *diff_files_usage =
+	"diff-files [--all] [--changed] [--deleted] [--others] [-p | -z] "
+	"[--exclude=<pattern>] [--exclude-from=<file>] [paths...]";
+
+/* What paths are we interested in? */
+static int nr_paths = 0;
+static char **paths = NULL;
+static int *pathlens = NULL;
+
+static int nr_excludes;
+static const char **excludes;
+static int excludes_alloc;
+
+static void add_exclude(const char *string)
+{
+	if (nr_excludes == excludes_alloc) {
+		excludes_alloc = alloc_nr(excludes_alloc);
+		excludes = realloc(excludes, excludes_alloc*sizeof(char *));
+	}
+	excludes[nr_excludes++] = string;
+}
+
+static void add_excludes_from_file(const char *fname)
+{
+	int fd, i;
+	long size;
+	char *buf, *entry;
+
+	fd = open(fname, O_RDONLY);
+	if (fd < 0)
+		goto err;
+	size = lseek(fd, 0, SEEK_END);
+	if (size < 0)
+		goto err;
+	lseek(fd, 0, SEEK_SET);
+	if (size == 0) {
+		close(fd);
+		return;
+	}
+	buf = malloc(size);
+	if (!buf) {
+		errno = ENOMEM;
+		goto err;
+	}
+	if (read(fd, buf, size) != size)
+		goto err;
+	close(fd);
+
+	entry = buf;
+	for (i = 0; i < size; i++) {
+		if (buf[i] == '\n') {
+			if (entry != buf + i) {
+				buf[i] = 0;
+				add_exclude(entry);
+			}
+			entry = buf + i + 1;
+		}
+	}
+	return;
+
+err:	perror(fname);
+	exit(1);
+}
+
+/*
+ * See if name matches our specified paths and is not excluded.
+ * return value:
+ *	-1 if no match
+ *	0 if partial match (name is a directory component)
+ *	1 = exact match
+ *	2 = name is under a specified directory path with no excludes
+ */
+static int path_match(const char *name, int namelen)
+{
+	int i, ret;
+
+	/* fast case: no path list and no exclude list */
+	if (!nr_paths && !nr_excludes)
+		return 2;
+
+	ret = (nr_paths) ? -1 : 1;
+	for (i = 0; i < nr_paths; i++) {
+		int pathlen = pathlens[i];
+		if (pathlen == namelen &&
+		    strncmp(paths[i], name, pathlen) == 0) {
+			ret = 1;
+			break;
+		} else if (pathlen > namelen && 
+			   strncmp(paths[i], name, namelen) == 0 &&
+			   paths[i][namelen] == '/') {
+			ret = 0;
+			break;
+		} else if (pathlen < namelen &&
+			   strncmp(paths[i], name, pathlen) == 0 &&
+			   name[pathlen] == '/') {
+			ret = (nr_excludes) ? 1 : 2;
+			break;
+		}
+	}
+
+	if (ret >= 0 && nr_excludes) {
+		const char *basename = strrchr(name, '/');
+		basename = (basename) ? basename+1 : name;
+		for (i = 0; i < nr_excludes; i++) {
+			if (fnmatch(excludes[i], basename, 0) == 0) {
+				ret = -1;
+				break;
+			}
+		}
+	}
+
+	return ret;
+}
+
+static const char **others;
+static int nr_others;
+static int others_alloc;
+
+static void add_name(const char *pathname, int len)
+{
+	char *name;
+
+	if (cache_name_pos(pathname, len) >= 0)
+		return;
+
+	if (nr_others == others_alloc) {
+		others_alloc = alloc_nr(others_alloc);
+		others = realloc(others, others_alloc*sizeof(char *));
+	}
+	name = malloc(len + 1);
+	memcpy(name, pathname, len + 1);
+	others[nr_others++] = name;
+}
+
+/*
+ * Read a directory tree. We currently ignore anything but
+ * directories and regular files. That's because git doesn't
+ * handle them at all yet. Maybe that will change some day.
+ *
+ * Also, we currently ignore all names starting with a dot.
+ * That likely will not change.
+ */
+static void read_directory(const char *path, const char *base, int baselen, int match)
+{
+	DIR *dir = opendir(path);
+
+	if (dir) {
+		struct dirent *de;
+		char fullname[MAXPATHLEN + 1];
+		memcpy(fullname, base, baselen);
+
+		while ((de = readdir(dir)) != NULL) {
+			int len;
+
+			if (de->d_name[0] == '.')
+				continue;
+			len = strlen(de->d_name);
+			memcpy(fullname + baselen, de->d_name, len+1);
+			if (match < 2)
+				match = path_match(fullname, baselen+len);
+			if (match < 0)
+				continue;
+
+			switch (de->d_type) {
+			struct stat st;
+			default:
+				continue;
+			case DT_UNKNOWN:
+				if (lstat(fullname, &st))
+					continue;
+				if (S_ISREG(st.st_mode))
+					break;
+				if (!S_ISDIR(st.st_mode))
+					continue;
+				/* fallthrough */
+			case DT_DIR:
+				memcpy(fullname + baselen + len, "/", 2);
+				read_directory(fullname, fullname,
+					       baselen + len + 1, match);
+				continue;
+			case DT_REG:
+				break;
+			}
+			if (match > 0)
+				add_name(fullname, baselen + len);
+		}
+		closedir(dir);
+	}
+}
+
+static int cmp_name(const void *p1, const void *p2)
+{
+	const char *n1 = *(const char **)p1;
+	const char *n2 = *(const char **)p2;
+	int l1 = strlen(n1), l2 = strlen(n2);
+
+	return cache_name_compare(n1, l1, n2, l2);
+}
+
+static int show_changed = 0;
+static int show_deleted = 0;
+static int show_others = 0;
+static int generate_patch = 0;
+static int line_terminator = '\n';
+
+static const char null_sha1[20];
+static const char null_sha1_hex[] = "0000000000000000000000000000000000000000";
+
+static void show_file(int prefix, unsigned int mode,
+		      const char *sha1, const char *name)
+{
+	if (generate_patch)
+		diff_addremove(prefix, mode, sha1, name, NULL);
+	else
+		printf("%c%o\t%s\t%s\t%s%c", prefix, mode, "blob",
+		       sha1_to_hex(sha1), name, line_terminator);
+}
+
+int main(int argc, char **argv)
+{
+	int i, entries;
+
+	for (i = 1; i < argc; i++) {
+		char *arg = argv[i];
+
+		if (*arg != '-') {
+			break;
+		} else if (!strcmp(arg, "-z")) {
+			line_terminator = 0;
+		} else if (!strcmp(arg, "-a") || !strcmp(arg, "--all")) {
+			show_changed = show_deleted = show_others = 1;
+		} else if (!strcmp(arg, "-c") || !strcmp(arg, "--changed")) {
+			show_changed = 1;
+		} else if (!strcmp(arg, "-d") || !strcmp(arg, "--deleted")) {
+			show_deleted = 1;
+		} else if (!strcmp(arg, "-o") || !strcmp(arg, "--others")) {
+			show_others = 1;
+		} else if (!strcmp(arg, "-p")) {
+			generate_patch = 1;
+		} else if (!strcmp(arg, "-x") && i+1 < argc) {
+			add_exclude(argv[++i]);
+		} else if (!strncmp(arg, "--exclude=", 10)) {
+			add_exclude(arg+10);
+		} else if (!strcmp(arg, "-X") && i+1 < argc) {
+			add_excludes_from_file(argv[++i]);
+		} else if (!strncmp(arg, "--exclude-from=", 15)) {
+			add_excludes_from_file(arg+15);
+		} else if (!strcmp(arg, "--")) {
+			i++;
+			break;
+		} else
+			usage(diff_files_usage);
+	}
+
+	/* default to -c if none of -c, -d nor -o have been specified */
+	if (!show_changed && !show_deleted && !show_others)
+		show_changed = 1;
+
+	if (i < argc) {
+		paths = &argv[i];
+		nr_paths = argc - i;
+		pathlens = malloc(nr_paths * sizeof(int));
+		for (i=0; i<nr_paths; i++) {
+			pathlens[i] = strlen(paths[i]);
+			if (paths[i][pathlens[i] - 1] == '/')
+				pathlens[i]--;
+		}
+	}
+
+	entries = read_cache();
+	if (entries < 0) {
+		perror("read_cache");
+		exit(1);
+	}
+
+	if (show_others) {
+		read_directory(".", "", 0, 0);
+		qsort(others, nr_others, sizeof(char *), cmp_name);
+		for (i = 0; i < nr_others; i++) {
+			struct stat st;
+			unsigned int mode;
+			if (stat(others[i], &st) < 0) {
+				perror(others[i]);
+			} else {
+				mode = S_IFREG | ce_permissions(st.st_mode);
+				show_file('+', mode, null_sha1, others[i]);
+			}
+		}
+	}
+
+	for (i = 0; i < entries; i++) {
+		struct stat st;
+		unsigned int ce_mode, mode;
+		struct cache_entry *ce = active_cache[i];
+
+		if (path_match(ce->name, ce_namelen(ce)) < 1)
+			continue;
+
+		if (show_changed && ce_stage(ce)) {
+			if (generate_patch)
+				diff_unmerge(ce->name);
+			else
+				printf("U %s%c", ce->name, line_terminator);
+			do {
+				i++;
+			} while (i < entries &&
+				 !strcmp(ce->name, active_cache[i]->name));
+			continue;
+		}
+
+		ce_mode = ntohl(ce->ce_mode);
+		if (stat(ce->name, &st) < 0) {
+			if (errno != ENOENT) {
+				perror(ce->name);
+			} else if (show_deleted) {
+				show_file('-', ce_mode, ce->sha1, ce->name);
+			}
+			continue;
+		}
+
+		if (!show_changed || !cache_match_stat(ce, &st))
+			continue;
+
+		mode = S_IFREG | ce_permissions(st.st_mode);
+		if (generate_patch)
+			diff_change(ce_mode, mode, ce->sha1,
+				    null_sha1, ce->name, NULL);
+		else
+			printf("*%o->%o\t%s\t%s->%s\t%s%c",
+			       ce_mode, mode, "blob",
+			       sha1_to_hex(ce->sha1), null_sha1_hex,
+			       ce->name, line_terminator);
+	}
+
+	return 0;
+}

Re: [PATCH] add a diff-files command (revised and cleaned up)

[Linus Torvalds]

On Wed, 27 Apr 2005, Nicolas Pitre wrote:
>
> In the same spirit as diff-tree and diff-cache, here is a diff-files 
> command that processes differences between the index cache and the 
> working directory content.

I really think the current "show-diff" does that very well, and what 
you're doing is really different.

I think this thing is really a replacement for "show-files", which is a 
piece of crap (hey, I wrote it, but I don't have to be proud of it), and 
which really was meant to be more of what your diff-files is.

The thing is, I really don't want the "core" diff-xxx programs to worry 
about exclude patters, and current directory contents. They do one thing, 
and one thing only: compare the files they were explicitly told to 
compare. 

HOWEVER, there clearly is a separate problem, which is what "show-files"  
currently does very badly (and not at all in some cases), which is the 
"ok, what about the _other_ files?"

And once you start talking about files that are _not_ mentioned in the
index, now you really do have something totally different, and now it does
need to be able to have exclude patterns to know to avoid object files and
other crud that we know we're not interested in).

But for the crud we don't know about, we're not really interested in the
diff against something we _do_ know about. So I think that the whole
"--others" and "--all" thing is wrong (yeah, yeah, it was me that started
it with show-files), and that this thing should always _only_ look at
files that aren't mentioned in the index file (ie "others" is always
enabled, and "all" is pointless).

Because those are special files: they are files we don't know what to do
with (conversely, files that _are_ mentioned in the index but don't 
actually seem to show up are interesting for the exact same reason).

That set of files is interesting for several reasons:

 - maybe we're about to check something in. We want to know whether maybe 
   we've forgotten to "add" a file or "remove" a file.

 - is it a file we've lost track of, and if so, does it look anything like 
   some _other_ file in the index. This happens if you do a "mv", and 
   don't tell the SCM about it: git doesn't care (it looks like a remove 
   and add), but it would be good to have a tool that warns about it and 
   then it would be nice if it could actually say "files xxx and yyy seem 
   to have gone away, but I see new files aaa and bbb, and it looks like 
   bbb might be similar to yyy, and aaa looks a lot like the old xxx"

This was all stuff that "show-files" was kind of supposed to work up to, 
but I just couldn't find it in myself to be interested enough.

		Linus

Re: [PATCH] add a diff-files command (revised and cleaned up)

[Nicolas Pitre]

On Thu, 28 Apr 2005, Linus Torvalds wrote:

> HOWEVER, there clearly is a separate problem, which is what "show-files"  
> currently does very badly (and not at all in some cases), which is the 
> "ok, what about the _other_ files?"

Right.  And that's the problem I'm trying to solve.

What about this patch then?

=====

Give show-files the ability to process exclusion pattern.

This can be used with the famous dontdiff file as follows to find out 
about uncommitted files just like dontdiff is used with the diff 
command:

    show-files --others --exclude-from=dontdiff

and the exclude list can be reversed with the --ignore switch.

Signed-off-by: Nicolas Pitre <nico@cam.org>

--- a/show-files.c
+++ b/show-files.c
@@ -6,6 +6,7 @@
  * Copyright (C) Linus Torvalds, 2005
  */
 #include <dirent.h>
+#include <fnmatch.h>
 
 #include "cache.h"
 
@@ -17,6 +18,70 @@ static int show_stage = 0;
 static int show_unmerged = 0;
 static int line_terminator = '\n';
 
+static int nr_excludes;
+static const char **excludes;
+static int excludes_alloc;
+
+static void add_exclude(const char *string)
+{
+	if (nr_excludes == excludes_alloc) {
+		excludes_alloc = alloc_nr(excludes_alloc);
+		excludes = realloc(excludes, excludes_alloc*sizeof(char *));
+	}
+	excludes[nr_excludes++] = string;
+}
+
+static void add_excludes_from_file(const char *fname)
+{
+	int fd, i;
+	long size;
+	char *buf, *entry;
+
+	fd = open(fname, O_RDONLY);
+	if (fd < 0)
+		goto err;
+	size = lseek(fd, 0, SEEK_END);
+	if (size < 0)
+		goto err;
+	lseek(fd, 0, SEEK_SET);
+	if (size == 0) {
+		close(fd);
+		return;
+	}
+	buf = xmalloc(size);
+	if (read(fd, buf, size) != size)
+		goto err;
+	close(fd);
+
+	entry = buf;
+	for (i = 0; i < size; i++) {
+		if (buf[i] == '\n') {
+			if (entry != buf + i) {
+				buf[i] = 0;
+				add_exclude(entry);
+			}
+			entry = buf + i + 1;
+		}
+	}
+	return;
+
+err:	perror(fname);
+	exit(1);
+}
+
+static int excluded(const char *pathname)
+{
+	int i;
+	if (nr_excludes) {
+		const char *basename = strrchr(pathname, '/');
+		basename = (basename) ? basename+1 : pathname;
+		for (i = 0; i < nr_excludes; i++)
+			if (fnmatch(excludes[i], basename, 0) == 0)
+				return 1;
+	}
+	return 0;
+}
+
 static const char **dir;
 static int nr_dir;
 static int dir_alloc;
@@ -59,6 +124,8 @@ static void read_directory(const char *p
 
 			if (de->d_name[0] == '.')
 				continue;
+			if (excluded(de->d_name) != show_ignored)
+				continue;
 			len = strlen(de->d_name);
 			memcpy(fullname + baselen, de->d_name, len+1);
 
@@ -101,17 +168,17 @@ static void show_files(void)
 	int i;
 
 	/* For cached/deleted files we don't need to even do the readdir */
-	if (show_others | show_ignored) {
+	if (show_others) {
 		read_directory(".", "", 0);
 		qsort(dir, nr_dir, sizeof(char *), cmp_name);
-	}
-	if (show_others) {
 		for (i = 0; i < nr_dir; i++)
 			printf("%s%c", dir[i], line_terminator);
 	}
 	if (show_cached | show_stage) {
 		for (i = 0; i < active_nr; i++) {
 			struct cache_entry *ce = active_cache[i];
+			if (excluded(ce->name) != show_ignored)
+				continue;
 			if (show_unmerged && !ce_stage(ce))
 				continue;
 			if (!show_stage)
@@ -130,14 +197,13 @@ static void show_files(void)
 		for (i = 0; i < active_nr; i++) {
 			struct cache_entry *ce = active_cache[i];
 			struct stat st;
+			if (excluded(ce->name) != show_ignored)
+				continue;
 			if (!stat(ce->name, &st))
 				continue;
 			printf("%s%c", ce->name, line_terminator);
 		}
 	}
-	if (show_ignored) {
-		/* We don't have any "ignore" list yet */
-	}
 }
 
 int main(int argc, char **argv)
@@ -179,11 +245,34 @@ int main(int argc, char **argv)
 			continue;
 		}
 
-		usage("show-files [-z] (--[cached|deleted|others|ignored|stage])*");
+		if (!strcmp(arg, "-x") && i+1 < argc) {
+			add_exclude(argv[++i]);
+			continue;
+		}
+		if (!strncmp(arg, "--exclude=", 10)) {
+			add_exclude(arg+10);
+			continue;
+		}
+		if (!strcmp(arg, "-X") && i+1 < argc) {
+			add_excludes_from_file(argv[++i]);
+			continue;
+		}
+		if (!strncmp(arg, "--exclude-from=", 15)) {
+			add_excludes_from_file(arg+15);
+			continue;
+		}
+
+		usage("show-files [-z] (--[cached|deleted|others|stage])* "
+		      "[ --ignored [--exclude=<pattern>] [--exclude-from=<file>) ]");
+	}
+
+	if (show_ignored && !nr_excludes) {
+		fprintf(stderr, "%s: --ignored needs some exclude pattern\n", argv[0]);
+		exit(1);
 	}
 
 	/* With no flags, we default to showing the cached files */
-	if (!(show_stage | show_deleted | show_others | show_ignored | show_unmerged))
+	if (!(show_stage | show_deleted | show_others | show_unmerged))
 		show_cached = 1;
 
 	read_cache();

Re: [PATCH] add a diff-files command (revised and cleaned up)

[Junio C Hamano]

>>>>> "NP" == Nicolas Pitre <nico@cam.org> writes:

NP> What about this patch then?
NP> =====
NP> Give show-files the ability to process exclusion pattern.

I just tried this, and I like it.  Thanks!