[PATCH v3 02/15] libxl: functions to lock / unlock libxl userdata store

[Wei Liu <wei.liu2@citrix.com>]

This lock is used to protect all userdata files related to a particular
domain, which include but are not limited to domain configuration.  A
new "domain-userdata-lock" entry is introduced in libxl registry.

This lock works among different processes and different threads within
the same process.

Locking protocol inspired by Ian Jackson's chiark-utils with-lock-ex. A
file lock is taken with flock(2). If that succeeds that thread fstat the
fd and stat the lock file path. If the device and inode match then the
lock has been successfully acquired. This lock remains acquired until
the lock file gets deleted or released by flock(2). If device and inode
don't match then another thread acquired the lock and deleted the file
in the meantime; lock procedure should restart.

Portability note: this lock utilises flock(2) so a proper implementation
of flock(2) is required -- that is, it should not be implemented with
fcntl(2).

Signed-off-by: Wei Liu <wei.liu2@citrix.com>
Acked-by: Ian Campbell <ian.campbell@citrix.com>

---
change in v3:
rename functions to libxl__{un,}lock_domain_userdata
rename registry entry to "domain-userdata-lock"
---
 tools/libxl/libxl.h          |    3 ++
 tools/libxl/libxl_internal.c |   69 ++++++++++++++++++++++++++++++++++++++++++
 tools/libxl/libxl_internal.h |    5 +++
 3 files changed, 77 insertions(+)

diff --git a/tools/libxl/libxl.h b/tools/libxl/libxl.h
index 460207b..1ca25ae 100644
--- a/tools/libxl/libxl.h
+++ b/tools/libxl/libxl.h
@@ -1220,6 +1220,9 @@ void libxl_cpuid_set(libxl_ctx *ctx, uint32_t domid,
  *  "xl"          domain config file in xl format, Unix line endings
  *  "libvirt-xml" domain config file in libvirt XML format.  See
  *                http://libvirt.org/formatdomain.html
+ *  "domain-userdata-lock"  lock file to protect domain userdata in libxl.
+ *                          It's a per-domain lock. Applications should
+ *                          not touch this file.
  *
  * libxl does not enforce the registration of userdata userids or the
  * semantics of the data.  For specifications of the data formats
diff --git a/tools/libxl/libxl_internal.c b/tools/libxl/libxl_internal.c
index b880c89..edf864b 100644
--- a/tools/libxl/libxl_internal.c
+++ b/tools/libxl/libxl_internal.c
@@ -381,6 +381,75 @@ out:
     return rc;
 }
 
+/* Portability note: this lock utilises flock(2) so a proper implementation of
+ * flock(2) is required.
+ */
+libxl__carefd *libxl__lock_domain_userdata(libxl__gc *gc, uint32_t domid)
+{
+    libxl__carefd *carefd = NULL;
+    const char *lockfile;
+    int fd;
+    struct stat stab, fstab;
+
+    lockfile = libxl__userdata_path(gc, domid, "domain-userdata-lock", "l");
+    if (!lockfile) goto out;
+
+    while (true) {
+        libxl__carefd_begin();
+        fd = open(lockfile, O_RDWR|O_CREAT, 0666);
+        if (fd < 0)
+            LOGE(ERROR, "cannot open lockfile %s, errno=%d", lockfile, errno);
+        carefd = libxl__carefd_opened(CTX, fd);
+        if (fd < 0) goto out;
+
+        /* Lock the file in exclusive mode, wait indefinitely to
+         * acquire the lock
+         */
+        while (flock(fd, LOCK_EX)) {
+            switch (errno) {
+            case EINTR:
+                /* Signal received, retry */
+                continue;
+            default:
+                /* All other errno: EBADF, EINVAL, ENOLCK, EWOULDBLOCK */
+                LOGE(ERROR,
+                     "unexpected error while trying to lock %s, fd=%d, errno=%d",
+                     lockfile, fd, errno);
+                goto out;
+            }
+        }
+
+        if (fstat(fd, &fstab)) {
+            LOGE(ERROR, "cannot fstat %s, fd=%d, errno=%d",
+                 lockfile, fd, errno);
+            goto out;
+        }
+        if (stat(lockfile, &stab)) {
+            if (errno != ENOENT) {
+                LOGE(ERROR, "cannot stat %s, errno=%d", lockfile, errno);
+                goto out;
+            }
+        } else {
+            if (stab.st_dev == fstab.st_dev && stab.st_ino == fstab.st_ino)
+                break;
+        }
+
+        libxl__carefd_close(carefd);
+    }
+
+    return carefd;
+
+out:
+    if (carefd) libxl__carefd_close(carefd);
+    return NULL;
+}
+
+void libxl__unlock_domain_userdata(libxl__carefd *lock_carefd)
+{
+    /* Simply closing the file descriptor releases the lock */
+    libxl__carefd_close(lock_carefd);
+}
+
 /*
  * Local variables:
  * mode: C
diff --git a/tools/libxl/libxl_internal.h b/tools/libxl/libxl_internal.h
index 7244473..0cedd30 100644
--- a/tools/libxl/libxl_internal.h
+++ b/tools/libxl/libxl_internal.h
@@ -43,6 +43,7 @@
 #include <sys/types.h>
 #include <sys/wait.h>
 #include <sys/socket.h>
+#include <sys/file.h>
 
 #include <xenstore.h>
 #include <xenctrl.h>
@@ -3223,6 +3224,10 @@ static inline int libxl__key_value_list_is_empty(libxl_key_value_list *pkvl)
 
 int libxl__cpuid_policy_is_empty(libxl_cpuid_policy_list *pl);
 
+/* Portability note: a proper flock(2) implementation is required */
+libxl__carefd *libxl__lock_domain_userdata(libxl__gc *gc, uint32_t domid);
+void libxl__unlock_domain_userdata(libxl__carefd *lock_carefd);
+
 #endif
 
 /*
-- 
1.7.10.4