backups with git and inotify

backups with git and inotify

[Luciano Rocha]

[-- Attachment #1.1: Type: text/plain, Size: 1152 bytes --]


Hello,

The following is a work in progress. There are some problems in how I'm
using git and recording the history:

1. I use an opened fd for each monitored directory (and subdirectories),
   (inotify_add_watch_at would be nice).
   I fchdir(fd) when a change happens to register and commit it.

2. git-rm dir/file also removes <dir> if file was the only entry of
   <dir>. So, when committing the removal, git complains that it can't
   find cwd. So I record the parent directory, do the git command, check
   if getcwd() works, and if not do the commit in the parent directory.

3. git-rm (empty) directory fails

4. Changes aren't atomic, but I can live with that and I doubt I would
   be able to make it atomic without implementing a filesystem (FUSE or
   not).

I can work around most of the problems, and rewrite to use recorded path
names instead of directories fd, but before I do that, and while I'm
at the beginning, I'd like to probe for opinions and suggestions.

So, please, suggest.

Regards,
Luciano Rocha

-- 
Luciano Rocha <luciano@eurotux.com>
Eurotux Informática, S.A. <http://www.eurotux.com/>

[-- Attachment #1.2: ino.c --]
[-- Type: text/plain, Size: 9197 bytes --]

/*
    monitor with inotify, record with git

    Copyright (C) 2007, Luciano Rocha <luciano@nsk.pt>
    Released under the GPL v2 or later. See LICENSE.GPL.
 
    This program is free software; you can redistribute it and/or modify
    it under the terms of the GNU General Public License as published by
    the Free Software Foundation; either version 2 of the License, or
    (at your option) any later version.

    This program is distributed in the hope that it will be useful,
    but WITHOUT ANY WARRANTY; without even the implied warranty of
    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
    GNU General Public License for more details.

    You should have received a copy of the GNU General Public License
    along with this program; if not, write to the Free Software
    Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
*/

#define _ATFILE_SOURCE 1
#define _GNU_SOURCE 1
#include <sys/inotify.h>
#include <sys/types.h>
#include <sys/stat.h>
#include <sys/wait.h>
#include <errno.h>
#include <fcntl.h>
#include <unistd.h>
#include <dirent.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>

struct ilist {
	int fd;
	uint32_t wd;
	struct ilist *prev, *next;
};

typedef struct ilist *ilist;

/* inotify_add_watch macro, with desired mask */
#define INOTIFY_ADD(ifd, dir) (inotify_add_watch((ifd), (dir), \
			IN_CLOSE_WRITE | IN_CREATE \
			| IN_DELETE | IN_DELETE_SELF \
			| IN_MOVED_FROM | IN_MOVED_TO))

/* add directory to inotify watch list
 * inotify_add_watchat would be nice, but it doesn't exist, so
 * read symlink in /proc/self/fd/<dirfd> instead
 */
int add_inotify(int fd, int ifd)
{
	char p[32];
	char *dir;
	int l;
	int wd;
	ssize_t ll;

	snprintf(p, sizeof p, "/proc/self/fd/%d", fd);
	dir = NULL;
	l = 0;
	do {
		l += 256;
		if (dir)
			free(dir);
		dir = malloc(l);
		ll = readlink(p, dir, l);
		if (ll < 0) {
			perror(p);
			free(dir);
			return -1;
		}
	} while (ll >= l);
	dir[strlen(dir) - 1] = '\0';
	wd = INOTIFY_ADD(ifd, dir);
	if (wd < 0)
		perror(dir);
	free(dir);
	return wd;
}

/* add watch for directory and each sub-directory, unless
 * there's a .git inside
 */
void add_watch(int root, const char *name, ilist *head, int ifd,
		const char **except, const char **skip)
{
	ilist new;
	DIR *dir;
	struct dirent *d;
	int dirfd, DIRfd;
	int wd;
	int i;

	new = malloc(sizeof *new);
	if (!new) {
		perror(name);
		return;
	}

	dirfd = openat(root, name, O_RDONLY | O_DIRECTORY | O_NOATIME
			| O_NOFOLLOW);

	if (dirfd < 0) {
		perror(name);
		free(new);
		return;
	}

	if (except) {
		for (i = 0; except[i] && faccessat(dirfd, except[i],
					R_OK | X_OK,
					AT_EACCESS | AT_SYMLINK_NOFOLLOW); i++);
		if (except[i]) {
			printf("skipping %s (%s exists)\n", name, except[i]);
			free(new);
			close(dirfd);
			return;
		}
	}

	wd = add_inotify(dirfd, ifd);
	if (wd < 0) {
		free(new);
		close(dirfd);
		return;
	}

	DIRfd = dup(dirfd);
	dir = fdopendir(DIRfd);
	if (!dir) {
		perror(name);
		free(new);
		close(dirfd);
		close(DIRfd);
		inotify_rm_watch(ifd, wd);
		return;
	}

	while ((d = readdir(dir))) {
		if (!S_ISDIR(d->d_type << 12))
			continue;
		if (d->d_name[0] == '.' && (d->d_name[1] == '\0'
					|| (d->d_name[1] == '.'
						&& d->d_name[2] == '\0')))
			continue;

		if (skip) {
			for (i = 0; skip[i] && strcmp(skip[i], d->d_name); i++);
			if (skip[i])
				continue;
		}

		add_watch(dirfd, d->d_name, head, ifd, except, skip);
	}
	closedir(dir);

	/* add to list */
	new->fd = dirfd;
	new->wd = wd;
	new->next = *head;
	if (*head) (*head)->prev = new;
	new->prev = NULL;
	*head = new;
}

static const char *default_except[] = {
	".git",
	NULL,
};

/* add watch to a directory and its sub-directories, complain and do
 * nothing if no .git exists
 */
void git_watch(const char *name, ilist *head, int ifd, const char **skip)
{
	ilist new;
	DIR *dir;
	struct dirent *d;
	int dirfd, DIRfd;
	int wd;
	int i;

	new = malloc(sizeof *new);
	if (!new) {
		perror(name);
		return;
	}

	dirfd = open(name, O_RDONLY | O_DIRECTORY | O_NOATIME | O_NOFOLLOW);

	if (dirfd < 0) {
		perror(name);
		free(new);
		return;
	}

	if (faccessat(dirfd, ".git", R_OK | X_OK,
				AT_EACCESS | AT_SYMLINK_NOFOLLOW)) {
		fprintf(stderr, "couldn't access .git subdir of %s: %s\n",
				name, strerror(errno));
		free(new);
		close(dirfd);
		return;
	}

	wd = INOTIFY_ADD(ifd, name);
	if (wd < 0) {
		perror(name);
		free(new);
		close(dirfd);
		return;
	}

	DIRfd = dup(dirfd);
	dir = fdopendir(DIRfd);
	if (!dir) {
		perror(name);
		free(new);
		close(dirfd);
		close(DIRfd);
		inotify_rm_watch(ifd, wd);
		return;
	}

	while ((d = readdir(dir))) {
		if (!S_ISDIR(d->d_type << 12))
			continue;
		if (d->d_name[0] == '.' && (d->d_name[1] == '\0'
					|| (d->d_name[1] == '.'
						&& d->d_name[2] == '\0')))
			continue;
		if (!strcmp(".git", d->d_name))
			continue;
		if (skip) {
			for (i = 0; skip[i] && strcmp(skip[i], d->d_name); i++);
			if (skip[i])
				continue;
		}

		add_watch(dirfd, d->d_name, head, ifd, default_except, skip);
	}
	closedir(dir);

	/* add to list */
	new->fd = dirfd;
	new->wd = wd;
	new->next = *head;
	if (*head) (*head)->prev = new;
	new->prev = NULL;
	*head = new;
}

/* run a shell command, abort on error
 */
void run_command(char *argv[])
{
	pid_t pid;
	int status;

	pid = fork();
	if (pid < 0) {
		perror("fork(2)");
		exit(1);
	}

	if (pid == 0) {
		execvp(argv[0], argv);
		perror(argv[0]);
		exit(1);
	}

	if (waitpid(pid, &status, 0) < 0) {
		perror("couldn't wait for child");
		exit(1);
	}

	if (WIFEXITED(status) && WEXITSTATUS(status) == 0)
		return;

	fprintf(stderr, "sub-command %s returned invalid exit code: %d\n",
			argv[0], status);
	exit(1);
}

/* run git command, and follow it with git-commit
 */
void run_git(int fd, char *cmd, char *what)
{
	int parent;
	int cl = strlen(cmd);
	int wl = strlen(what);
	char commit[cl + wl + 5];
	char *argv[] = {
		"git",
		cmd,
		what,
		NULL,
	};

	if (fchdir(fd))
		return;

	/* save parent:
	 * git-rm of last file in subdir removes the directory, so the
	 * following git-commit fails
	 */
	parent = open("..", O_RDONLY | O_DIRECTORY | O_NOATIME | O_NOFOLLOW);

	/* run git sub-command */
	run_command(argv);

	if (getcwd(commit, cl + wl) == NULL && errno != ERANGE && parent >= 0) {
		printf("errno: %d, %s\n", errno, strerror(errno));
		fchdir(parent);
	}

	/* create commit message */
	commit[0] = '-';
	commit[1] = 'm';
	memcpy(commit + 2, cmd, cl);
	commit[cl + 2] = ':';
	commit[cl + 3] = ' ';
	memcpy(commit + cl + 4, what, wl + 1);

	/* commit change(s) */
	argv[1] = "commit";
	argv[2] = commit;
	run_command(argv);
}

/* get inotify events, run git as appropriate
 */
void check_event(void *buffer, int len, ilist *head, int ifd)
{
	while (len >= sizeof(struct inotify_event)) {
		struct inotify_event *p = buffer;
		ilist l;

		/* advance buffer position */
		len -= sizeof(struct inotify_event) + p->len;
		buffer += sizeof(struct inotify_event) + p->len;

		for (l = *head; l && l->wd != p->wd; l = l->next);

		if (!l) {
			/* not found in list? */
			continue;
		}

		if (p->mask & (IN_IGNORED | IN_UNMOUNT | IN_DELETE_SELF)) {
			/* remove it */
			inotify_rm_watch(ifd, p->wd);
			close(l->fd);

			if (l->prev)
				l->prev->next = l->next;
			else
				*head = l->next;

			if (l->next)
				l->next->prev = l->prev;
		}

		/* the following events require a file name specification,
		 * changes to the directory itself aren't of our interest
		 */
		if (p->len == 0)
			continue;

		if (p->mask & IN_CREATE) {
			/* add new watch if directory, otherwise ignore,
			 * IN_CLOSE_WRITE should follow
			 */
			struct stat st;
			if (!fstatat(l->fd, p->name, &st, AT_SYMLINK_NOFOLLOW)
					&& S_ISDIR(st.st_mode))
				add_watch(l->fd, p->name, head, ifd,
						default_except, NULL);
		}

		if (p->mask & (IN_CLOSE_WRITE | IN_MOVED_TO)) {
			/* add/commit */
			run_git(l->fd, "add", p->name);
		}


		if (p->mask & (IN_DELETE | IN_MOVED_FROM)) {
			/* rm/commit */
			run_git(l->fd, "rm", p->name);
		}
	}
}

int main(int argc, char *argv[])
{
	int fd;
	int i;
	ilist head;
	void *buffer;

	fd = inotify_init();
	if (fd < 0) {
		perror("init inotify");
		return 1;
	}


	buffer = malloc(1<<20);
	if (!buffer) {
		perror("buffer allocation");
		return 1;
	}

	head = NULL;

	while (*++argv) {
		git_watch(*argv, &head, fd, NULL);
	}

	if (!head) {
		printf("nothing to do\n");
		return 0;
	}

	/* loop until there's an error or all watched elements are
	 * removed or made inaccessible
	 */
	while (head) {
		i = read(fd, buffer, 1<<20);
		if (i == 0 || (i < 0 && errno != EINTR && errno != EAGAIN))
			break;
		if (i < 0)
			continue;
		check_event(buffer, i, &head, fd);
	}

	if (i < 0) {
		perror("reading event");
		return 1;
	}

	return 0;
}

[-- Attachment #2: Type: application/pgp-signature, Size: 189 bytes --]

Re: backups with git and inotify

[David Tweed]

Hi, looks interesting project.

I've been doing something similar-ish (but periodically rather than
upon changes). Here are some rough off-the-cuff observations (probably
telling you things you already know).

Firstly, are you doing backups (be able to restore to n previous
states upon catastrophe) or archives (being able to lookup arbitrary
points in history to compare with current stuff, eg, for regressions)?
(Archiving is useful if you aren't in a disciplined enough project to
do rewritten proper commits but still want to be able to look around
and try to figure out what's caused regressions, etc.)

The scripts I use are at

http://www.personal.rdg.ac.uk/~sis05dst/chronoversion.tgz

but they're designed around archiving rather than backups.

On Dec 10, 2007 8:29 PM, Luciano Rocha <luciano@eurotux.com> wrote:
> The following is a work in progress. There are some problems in how I'm
> using git and recording the history:
>
> 1. I use an opened fd for each monitored directory (and subdirectories),
>    (inotify_add_watch_at would be nice).
>    I fchdir(fd) when a change happens to register and commit it.

I thought about trying to have a daemon using inotify to record the
git-add's/git-rm's but keeping the cron driven actual commits, and
looked at python support module. I didn't because firstly I wasn't
sure how far inotify scaled (the fact the Linux VFS maintainer insists
on calling it "idiotify" doesn't inspire confidence). If it was me,
I'd pull the git-commit outside your loop that does the git-add/git-rm
(see later comment about emacs, etc). Obviously if your buffer isn't
completely emptied you'll get a misleading granularity of commits, but
then I guess that'll happen anyway. I think inotify drops events if an
internal queue fills: personally I'd try to check for that and
initiate manually scanning if I detected that happening.

> 2. git-rm dir/file also removes <dir> if file was the only entry of
>    <dir>. So, when committing the removal, git complains that it can't
>    find cwd. So I record the parent directory, do the git command, check
>    if getcwd() works, and if not do the commit in the parent directory.
>
> 3. git-rm (empty) directory fails
>
> 4. Changes aren't atomic, but I can live with that and I doubt I would
>    be able to make it atomic without implementing a filesystem (FUSE or
>    not).

With things like emacs that do update writes by writing a new file
with a temporary name and then copying it over the top of the old
file, you'll get presumably 3 commits. Is that acceptable?

> I can work around most of the problems, and rewrite to use recorded path
> names instead of directories fd, but before I do that, and while I'm
> at the beginning, I'd like to probe for opinions and suggestions.

The only other thing that occurs to me is whether you need any greater
support for stopping the automatic monitoring than just stopping the
daemon. Eg, what happens if you decide you need to recover a previous
version of a file. Git checks it out, presumably updates the index
itself and then inotify fires off a git-add that will want to write to
the index. Basically, I'm trying to think if there's any situation
where you can have a delete event that git causes, followed by a
creating some new content where delay in your program processing the
delete will cause the new content to be `lost'? (I know, I sould read
the code.) In chronoversion, the first thing it does is check for a
"suppress" file which stops it doing anything automatically and I put
one in there whenever I'm doing anything more than looking at the data
(eg, switch branch, checkout old version, etc). But I might be being
hyper-cautious.

-- 
cheers, dave tweed__________________________
david.tweed@gmail.com
Rm 124, School of Systems Engineering, University of Reading.
"we had no idea that when we added templates we were adding a Turing-
complete compile-time language." -- C++ standardisation committee

Re: backups with git and inotify

[Luciano Rocha]

[-- Attachment #1: Type: text/plain, Size: 5048 bytes --]

On Mon, Dec 10, 2007 at 09:18:18PM +0000, David Tweed wrote:
> Hi, looks interesting project.
> 
> I've been doing something similar-ish (but periodically rather than
> upon changes). Here are some rough off-the-cuff observations (probably
> telling you things you already know).

Thanks.

> 
> Firstly, are you doing backups (be able to restore to n previous
> states upon catastrophe) or archives (being able to lookup arbitrary
> points in history to compare with current stuff, eg, for regressions)?
> (Archiving is useful if you aren't in a disciplined enough project to
> do rewritten proper commits but still want to be able to look around
> and try to figure out what's caused regressions, etc.)

Archives, only. If the project requires a coherent state, then I don't
think that automatic commits are the way to go.

> 
> The scripts I use are at
> 
> http://www.personal.rdg.ac.uk/~sis05dst/chronoversion.tgz
> 
> but they're designed around archiving rather than backups.

Thanks, I'll take a look, and maybe borrow some ideas. :)

> 
> On Dec 10, 2007 8:29 PM, Luciano Rocha <luciano@eurotux.com> wrote:
> > The following is a work in progress. There are some problems in how I'm
> > using git and recording the history:
> >
> > 1. I use an opened fd for each monitored directory (and subdirectories),
> >    (inotify_add_watch_at would be nice).
> >    I fchdir(fd) when a change happens to register and commit it.
> 
> I thought about trying to have a daemon using inotify to record the
> git-add's/git-rm's but keeping the cron driven actual commits, and
> looked at python support module. I didn't because firstly I wasn't
> sure how far inotify scaled (the fact the Linux VFS maintainer insists
> on calling it "idiotify" doesn't inspire confidence). If it was me,
> I'd pull the git-commit outside your loop that does the git-add/git-rm
> (see later comment about emacs, etc).

I'd like to have the changes committed as soon as an application closes
its files. As I monitor a small subset of possible inotify events, I
think I shouldn't have much problems with scale. I'll have to test it
with my Maildir, though, to have a definitive answer.

> Obviously if your buffer isn't
> completely emptied you'll get a misleading granularity of commits, but
> then I guess that'll happen anyway. I think inotify drops events if an
> internal queue fills: personally I'd try to check for that and
> initiate manually scanning if I detected that happening.

Hm, that could be a problem. Maybe a periodic git-status followed by
git-add/rm, etc.. Hourly, perhaps.

> 
> > 2. git-rm dir/file also removes <dir> if file was the only entry of
> >    <dir>. So, when committing the removal, git complains that it can't
> >    find cwd. So I record the parent directory, do the git command, check
> >    if getcwd() works, and if not do the commit in the parent directory.
> >
> > 3. git-rm (empty) directory fails
> >
> > 4. Changes aren't atomic, but I can live with that and I doubt I would
> >    be able to make it atomic without implementing a filesystem (FUSE or
> >    not).
> 
> With things like emacs that do update writes by writing a new file
> with a temporary name and then copying it over the top of the old
> file, you'll get presumably 3 commits. Is that acceptable?

No. Vim also has that behaviour. I plan on accepting ignore patterns,
and maybe also parse .gitignore, and add those temporary files (*~,
.*.sw[po]), etc.) implicitly.

> 
> > I can work around most of the problems, and rewrite to use recorded path
> > names instead of directories fd, but before I do that, and while I'm
> > at the beginning, I'd like to probe for opinions and suggestions.
> 
> The only other thing that occurs to me is whether you need any greater
> support for stopping the automatic monitoring than just stopping the
> daemon. Eg, what happens if you decide you need to recover a previous
> version of a file. Git checks it out, presumably updates the index
> itself and then inotify fires off a git-add that will want to write to
> the index. Basically, I'm trying to think if there's any situation
> where you can have a delete event that git causes, followed by a
> creating some new content where delay in your program processing the
> delete will cause the new content to be `lost'? (I know, I sould read
> the code.) In chronoversion, the first thing it does is check for a
> "suppress" file which stops it doing anything automatically and I put
> one in there whenever I'm doing anything more than looking at the data
> (eg, switch branch, checkout old version, etc). But I might be being
> hyper-cautious.

I'll have to think about that. A stop/pause button is a good idea, and
checking if the tree is at HEAD. I don't think a commit of changes to a
file checked-out to a previous version will lose any information,
but I'll check.

-- 
Luciano Rocha <luciano@eurotux.com>
Eurotux Informática, S.A. <http://www.eurotux.com/>

[-- Attachment #2: Type: application/pgp-signature, Size: 189 bytes --]

Re: backups with git and inotify

[Björn Steinbrink]

On 2007.12.10 20:29:11 +0000, Luciano Rocha wrote:
> 
> Hello,
> 
> The following is a work in progress. There are some problems in how I'm
> using git and recording the history:
> 
> 1. I use an opened fd for each monitored directory (and subdirectories),
>    (inotify_add_watch_at would be nice).
>    I fchdir(fd) when a change happens to register and commit it.
> 
> 2. git-rm dir/file also removes <dir> if file was the only entry of
>    <dir>. So, when committing the removal, git complains that it can't
>    find cwd. So I record the parent directory, do the git command, check
>    if getcwd() works, and if not do the commit in the parent directory.
> 
> 3. git-rm (empty) directory fails
> 
> 4. Changes aren't atomic, but I can live with that and I doubt I would
>    be able to make it atomic without implementing a filesystem (FUSE or
>    not).
> 
> I can work around most of the problems, and rewrite to use recorded path
> names instead of directories fd, but before I do that, and while I'm
> at the beginning, I'd like to probe for opinions and suggestions.
> 
> So, please, suggest.

I posted an extremely simple bash script here:
http://lkml.org/lkml/2007/12/7/279

It just employs inotifywait to do all watching and just needs to
translate the events to the different git command. Did just glance over
your code, but it seems to do basically the same thing, just that it's a
lot shorter. The overhead of being a shell script is probably neglible,
as the amount of git calls are likely dominating anyway.

Feel free to ignore my comments on why I think that that is crap anyway
and do whatever you want with the script.

HTH
Björn

Re: backups with git and inotify

[Luciano Rocha]

[-- Attachment #1: Type: text/plain, Size: 1738 bytes --]

On Mon, Dec 10, 2007 at 10:57:46PM +0100, Björn Steinbrink wrote:
> On 2007.12.10 20:29:11 +0000, Luciano Rocha wrote:
<snip>
> > So, please, suggest.
> 
> I posted an extremely simple bash script here:
> http://lkml.org/lkml/2007/12/7/279

That thread was what motivated me to write the program, but I must've
missed your post.

> 
> It just employs inotifywait to do all watching and just needs to
> translate the events to the different git command. Did just glance over
> your code, but it seems to do basically the same thing, just that it's a
> lot shorter. The overhead of being a shell script is probably neglible,
> as the amount of git calls are likely dominating anyway.

Yes, being a shell script isn't a problem. I wasn't aware of
inotify-tools, so I wrote my own. Still, the inotify-tools programs miss
a check for ignoring directories with predetermined contents (I want to
ignore sub-directories with their own git repository).

I think I'll contribute to inotify-tools a switch for that, and switch
to perl. That will simplify the development and allow for coalescing
events, so that updates with temporary files would be a simple update.

> Feel free to ignore my comments on why I think that that is crap anyway
> and do whatever you want with the script.

FWIW, I also think that trying to keep a coherent stat with automatic
commits isn't possible. As for the temporary, unneeded files, a
exclusion pattern will suffice, and using .git directly, instead of a
(FUSE) filesystem, will allow permanent storage of those temporary
files, until explicitly removed.

Thanks,
Luciano Rocha

-- 
Luciano Rocha <luciano@eurotux.com>
Eurotux Informática, S.A. <http://www.eurotux.com/>

[-- Attachment #2: Type: application/pgp-signature, Size: 189 bytes --]

Re: backups with git and inotify

[David Tweed]

On Dec 11, 2007 10:25 AM, Luciano Rocha <luciano@eurotux.com> wrote:
> On Mon, Dec 10, 2007 at 10:57:46PM +0100, Björn Steinbrink wrote:
> > On 2007.12.10 20:29:11 +0000, Luciano Rocha wrote:
> FWIW, I also think that trying to keep a coherent stat with automatic
> commits isn't possible. As for the temporary, unneeded files, a
> exclusion pattern will suffice, and using .git directly, instead of a
> (FUSE) filesystem, will allow permanent storage of those temporary
> files, until explicitly removed.

As a data point, I find it easier for my work to have rules that
specify the files you DO want to keep. (I actually have a set of
common suffixes to ignore that's checked first but purely for
efficiency: it's quicker to throw out the usual suspects like *~, *.o,
*.pyc, etc, immediately rather than fail to match all the more
complicated "keep this" rules every single time.) One advantage of
this is that if I do something stupid like drop a 2G video file into
the tracked tree it's doesn't get sucked in to the git repo.

-- 
cheers, dave tweed__________________________
david.tweed@gmail.com
Rm 124, School of Systems Engineering, University of Reading.
"we had no idea that when we added templates we were adding a Turing-
complete compile-time language." -- C++ standardisation committee