[RFC v5][PATCH 1/8] Create syscalls: sys_checkpoint, sys_restart

[RFC v5][PATCH 1/8] Create syscalls: sys_checkpoint, sys_restart

[Oren Laadan]

Create trivial sys_checkpoint and sys_restore system calls. They will
enable to checkpoint and restart an entire container, to and from a
checkpoint image file descriptor.

The syscalls take a file descriptor (for the image file) and flags as
arguments. For sys_checkpoint the first argument identifies the target
container; for sys_restart it will identify the checkpoint image.

Signed-off-by: Oren Laadan <orenl-eQaUEPhvms7ENvBUuze7eA@public.gmane.org>
---
 arch/x86/kernel/syscall_table_32.S |    2 +
 checkpoint/Kconfig                 |   11 +++++++++
 checkpoint/Makefile                |    5 ++++
 checkpoint/sys.c                   |   41 ++++++++++++++++++++++++++++++++++++
 include/asm-x86/unistd_32.h        |    2 +
 include/linux/syscalls.h           |    2 +
 init/Kconfig                       |    2 +
 kernel/sys_ni.c                    |    4 +++
 8 files changed, 69 insertions(+), 0 deletions(-)
 create mode 100644 checkpoint/Kconfig
 create mode 100644 checkpoint/Makefile
 create mode 100644 checkpoint/sys.c

diff --git a/arch/x86/kernel/syscall_table_32.S b/arch/x86/kernel/syscall_table_32.S
index d44395f..5543136 100644
--- a/arch/x86/kernel/syscall_table_32.S
+++ b/arch/x86/kernel/syscall_table_32.S
@@ -332,3 +332,5 @@ ENTRY(sys_call_table)
 	.long sys_dup3			/* 330 */
 	.long sys_pipe2
 	.long sys_inotify_init1
+	.long sys_checkpoint
+	.long sys_restart
diff --git a/checkpoint/Kconfig b/checkpoint/Kconfig
new file mode 100644
index 0000000..ffaa635
--- /dev/null
+++ b/checkpoint/Kconfig
@@ -0,0 +1,11 @@
+config CHECKPOINT_RESTART
+	prompt "Enable checkpoint/restart (EXPERIMENTAL)"
+	def_bool n
+	depends on X86_32 && EXPERIMENTAL
+	help
+	  Application checkpoint/restart is the ability to save the
+	  state of a running application so that it can later resume
+	  its execution from the time at which it was checkpointed.
+
+	  Turning this option on will enable checkpoint and restart
+	  functionality in the kernel.
diff --git a/checkpoint/Makefile b/checkpoint/Makefile
new file mode 100644
index 0000000..07d018b
--- /dev/null
+++ b/checkpoint/Makefile
@@ -0,0 +1,5 @@
+#
+# Makefile for linux checkpoint/restart.
+#
+
+obj-$(CONFIG_CHECKPOINT_RESTART) += sys.o
diff --git a/checkpoint/sys.c b/checkpoint/sys.c
new file mode 100644
index 0000000..375129c
--- /dev/null
+++ b/checkpoint/sys.c
@@ -0,0 +1,41 @@
+/*
+ *  Generic container checkpoint-restart
+ *
+ *  Copyright (C) 2008 Oren Laadan
+ *
+ *  This file is subject to the terms and conditions of the GNU General Public
+ *  License.  See the file COPYING in the main directory of the Linux
+ *  distribution for more details.
+ */
+
+#include <linux/sched.h>
+#include <linux/kernel.h>
+
+/**
+ * sys_checkpoint - checkpoint a container
+ * @pid: pid of the container init(1) process
+ * @fd: file to which dump the checkpoint image
+ * @flags: checkpoint operation flags
+ *
+ * Returns positive identifier on success, 0 when returning from restart
+ * or negative value on error
+ */
+asmlinkage long sys_checkpoint(pid_t pid, int fd, unsigned long flags)
+{
+	pr_debug("sys_checkpoint not implemented yet\n");
+	return -ENOSYS;
+}
+/**
+ * sys_restart - restart a container
+ * @crid: checkpoint image identifier
+ * @fd: file from which read the checkpoint image
+ * @flags: restart operation flags
+ *
+ * Returns negative value on error, or otherwise returns in the realm
+ * of the original checkpoint
+ */
+asmlinkage long sys_restart(int crid, int fd, unsigned long flags)
+{
+	pr_debug("sys_restart not implemented yet\n");
+	return -ENOSYS;
+}
diff --git a/include/asm-x86/unistd_32.h b/include/asm-x86/unistd_32.h
index d739467..88bdec4 100644
--- a/include/asm-x86/unistd_32.h
+++ b/include/asm-x86/unistd_32.h
@@ -338,6 +338,8 @@
 #define __NR_dup3		330
 #define __NR_pipe2		331
 #define __NR_inotify_init1	332
+#define __NR_checkpoint		333
+#define __NR_restart		334
 
 #ifdef __KERNEL__
 
diff --git a/include/linux/syscalls.h b/include/linux/syscalls.h
index d6ff145..edc218b 100644
--- a/include/linux/syscalls.h
+++ b/include/linux/syscalls.h
@@ -622,6 +622,8 @@ asmlinkage long sys_timerfd_gettime(int ufd, struct itimerspec __user *otmr);
 asmlinkage long sys_eventfd(unsigned int count);
 asmlinkage long sys_eventfd2(unsigned int count, int flags);
 asmlinkage long sys_fallocate(int fd, int mode, loff_t offset, loff_t len);
+asmlinkage long sys_checkpoint(pid_t pid, int fd, unsigned long flags);
+asmlinkage long sys_restart(int crid, int fd, unsigned long flags);
 
 int kernel_execve(const char *filename, char *const argv[], char *const envp[]);
 
diff --git a/init/Kconfig b/init/Kconfig
index c11da38..fd5f7bf 100644
--- a/init/Kconfig
+++ b/init/Kconfig
@@ -779,6 +779,8 @@ config MARKERS
 
 source "arch/Kconfig"
 
+source "checkpoint/Kconfig"
+
 config PROC_PAGE_MONITOR
  	default y
 	depends on PROC_FS && MMU
diff --git a/kernel/sys_ni.c b/kernel/sys_ni.c
index 08d6e1b..ca95c25 100644
--- a/kernel/sys_ni.c
+++ b/kernel/sys_ni.c
@@ -168,3 +168,7 @@ cond_syscall(compat_sys_timerfd_settime);
 cond_syscall(compat_sys_timerfd_gettime);
 cond_syscall(sys_eventfd);
 cond_syscall(sys_eventfd2);
+
+/* checkpoint/restart */
+cond_syscall(sys_checkpoint);
+cond_syscall(sys_restart);
-- 
1.5.4.3

Re: [RFC v5][PATCH 1/8] Create syscalls: sys_checkpoint, sys_restart

[Serge E. Hallyn]

Quoting Oren Laadan (orenl-eQaUEPhvms7ENvBUuze7eA@public.gmane.org):
> Create trivial sys_checkpoint and sys_restore system calls. They will
> enable to checkpoint and restart an entire container, to and from a
> checkpoint image file descriptor.
> 
> The syscalls take a file descriptor (for the image file) and flags as
> arguments. For sys_checkpoint the first argument identifies the target
> container; for sys_restart it will identify the checkpoint image.
> 
> Signed-off-by: Oren Laadan <orenl-eQaUEPhvms7ENvBUuze7eA@public.gmane.org>

Acked-by: Serge Hallyn <serue-r/Jw6+rmf7HQT0dZR+AlfA@public.gmane.org>

thanks,
-serge

> ---
>  arch/x86/kernel/syscall_table_32.S |    2 +
>  checkpoint/Kconfig                 |   11 +++++++++
>  checkpoint/Makefile                |    5 ++++
>  checkpoint/sys.c                   |   41 ++++++++++++++++++++++++++++++++++++
>  include/asm-x86/unistd_32.h        |    2 +
>  include/linux/syscalls.h           |    2 +
>  init/Kconfig                       |    2 +
>  kernel/sys_ni.c                    |    4 +++
>  8 files changed, 69 insertions(+), 0 deletions(-)
>  create mode 100644 checkpoint/Kconfig
>  create mode 100644 checkpoint/Makefile
>  create mode 100644 checkpoint/sys.c
> 
> diff --git a/arch/x86/kernel/syscall_table_32.S b/arch/x86/kernel/syscall_table_32.S
> index d44395f..5543136 100644
> --- a/arch/x86/kernel/syscall_table_32.S
> +++ b/arch/x86/kernel/syscall_table_32.S
> @@ -332,3 +332,5 @@ ENTRY(sys_call_table)
>  	.long sys_dup3			/* 330 */
>  	.long sys_pipe2
>  	.long sys_inotify_init1
> +	.long sys_checkpoint
> +	.long sys_restart
> diff --git a/checkpoint/Kconfig b/checkpoint/Kconfig
> new file mode 100644
> index 0000000..ffaa635
> --- /dev/null
> +++ b/checkpoint/Kconfig
> @@ -0,0 +1,11 @@
> +config CHECKPOINT_RESTART
> +	prompt "Enable checkpoint/restart (EXPERIMENTAL)"
> +	def_bool n
> +	depends on X86_32 && EXPERIMENTAL
> +	help
> +	  Application checkpoint/restart is the ability to save the
> +	  state of a running application so that it can later resume
> +	  its execution from the time at which it was checkpointed.
> +
> +	  Turning this option on will enable checkpoint and restart
> +	  functionality in the kernel.
> diff --git a/checkpoint/Makefile b/checkpoint/Makefile
> new file mode 100644
> index 0000000..07d018b
> --- /dev/null
> +++ b/checkpoint/Makefile
> @@ -0,0 +1,5 @@
> +#
> +# Makefile for linux checkpoint/restart.
> +#
> +
> +obj-$(CONFIG_CHECKPOINT_RESTART) += sys.o
> diff --git a/checkpoint/sys.c b/checkpoint/sys.c
> new file mode 100644
> index 0000000..375129c
> --- /dev/null
> +++ b/checkpoint/sys.c
> @@ -0,0 +1,41 @@
> +/*
> + *  Generic container checkpoint-restart
> + *
> + *  Copyright (C) 2008 Oren Laadan
> + *
> + *  This file is subject to the terms and conditions of the GNU General Public
> + *  License.  See the file COPYING in the main directory of the Linux
> + *  distribution for more details.
> + */
> +
> +#include <linux/sched.h>
> +#include <linux/kernel.h>
> +
> +/**
> + * sys_checkpoint - checkpoint a container
> + * @pid: pid of the container init(1) process
> + * @fd: file to which dump the checkpoint image
> + * @flags: checkpoint operation flags
> + *
> + * Returns positive identifier on success, 0 when returning from restart
> + * or negative value on error
> + */
> +asmlinkage long sys_checkpoint(pid_t pid, int fd, unsigned long flags)
> +{
> +	pr_debug("sys_checkpoint not implemented yet\n");
> +	return -ENOSYS;
> +}
> +/**
> + * sys_restart - restart a container
> + * @crid: checkpoint image identifier
> + * @fd: file from which read the checkpoint image
> + * @flags: restart operation flags
> + *
> + * Returns negative value on error, or otherwise returns in the realm
> + * of the original checkpoint
> + */
> +asmlinkage long sys_restart(int crid, int fd, unsigned long flags)
> +{
> +	pr_debug("sys_restart not implemented yet\n");
> +	return -ENOSYS;
> +}
> diff --git a/include/asm-x86/unistd_32.h b/include/asm-x86/unistd_32.h
> index d739467..88bdec4 100644
> --- a/include/asm-x86/unistd_32.h
> +++ b/include/asm-x86/unistd_32.h
> @@ -338,6 +338,8 @@
>  #define __NR_dup3		330
>  #define __NR_pipe2		331
>  #define __NR_inotify_init1	332
> +#define __NR_checkpoint		333
> +#define __NR_restart		334
> 
>  #ifdef __KERNEL__
> 
> diff --git a/include/linux/syscalls.h b/include/linux/syscalls.h
> index d6ff145..edc218b 100644
> --- a/include/linux/syscalls.h
> +++ b/include/linux/syscalls.h
> @@ -622,6 +622,8 @@ asmlinkage long sys_timerfd_gettime(int ufd, struct itimerspec __user *otmr);
>  asmlinkage long sys_eventfd(unsigned int count);
>  asmlinkage long sys_eventfd2(unsigned int count, int flags);
>  asmlinkage long sys_fallocate(int fd, int mode, loff_t offset, loff_t len);
> +asmlinkage long sys_checkpoint(pid_t pid, int fd, unsigned long flags);
> +asmlinkage long sys_restart(int crid, int fd, unsigned long flags);
> 
>  int kernel_execve(const char *filename, char *const argv[], char *const envp[]);
> 
> diff --git a/init/Kconfig b/init/Kconfig
> index c11da38..fd5f7bf 100644
> --- a/init/Kconfig
> +++ b/init/Kconfig
> @@ -779,6 +779,8 @@ config MARKERS
> 
>  source "arch/Kconfig"
> 
> +source "checkpoint/Kconfig"
> +
>  config PROC_PAGE_MONITOR
>   	default y
>  	depends on PROC_FS && MMU
> diff --git a/kernel/sys_ni.c b/kernel/sys_ni.c
> index 08d6e1b..ca95c25 100644
> --- a/kernel/sys_ni.c
> +++ b/kernel/sys_ni.c
> @@ -168,3 +168,7 @@ cond_syscall(compat_sys_timerfd_settime);
>  cond_syscall(compat_sys_timerfd_gettime);
>  cond_syscall(sys_eventfd);
>  cond_syscall(sys_eventfd2);
> +
> +/* checkpoint/restart */
> +cond_syscall(sys_checkpoint);
> +cond_syscall(sys_restart);
> -- 
> 1.5.4.3
> 
> _______________________________________________
> Containers mailing list
> Containers-cunTk1MwBs9QetFLy7KEm3xJsTq8ys+cHZ5vskTnxNA@public.gmane.org
> https://lists.linux-foundation.org/mailman/listinfo/containers

[RFC v5][PATCH 2/8] General infrastructure for checkpoint restart

[Oren Laadan]

Add those interfaces, as well as helpers needed to easily manage the
file format. The code is roughly broken out as follows:

checkpoint/sys.c - user/kernel data transfer, as well as setup of the
checkpoint/restart context (a per-checkpoint data structure for
housekeeping)

checkpoint/checkpoint.c - output wrappers and basic checkpoint handling

checkpoint/restart.c - input wrappers and basic restart handling

Patches to add the per-architecture support as well as the actual
work to do the memory checkpoint follow in subsequent patches.

Signed-off-by: Oren Laadan <orenl-eQaUEPhvms7ENvBUuze7eA@public.gmane.org>
---
 Makefile                       |    2 +-
 checkpoint/Makefile            |    2 +-
 checkpoint/checkpoint.c        |  174 ++++++++++++++++++++++++++++++++
 checkpoint/restart.c           |  189 ++++++++++++++++++++++++++++++++++
 checkpoint/sys.c               |  218 +++++++++++++++++++++++++++++++++++++++-
 include/linux/checkpoint.h     |   60 +++++++++++
 include/linux/checkpoint_hdr.h |   75 ++++++++++++++
 include/linux/magic.h          |    3 +
 8 files changed, 717 insertions(+), 6 deletions(-)
 create mode 100644 checkpoint/checkpoint.c
 create mode 100644 checkpoint/restart.c
 create mode 100644 include/linux/checkpoint.h
 create mode 100644 include/linux/checkpoint_hdr.h

diff --git a/Makefile b/Makefile
index f448e00..a558ad2 100644
--- a/Makefile
+++ b/Makefile
@@ -619,7 +619,7 @@ export mod_strip_cmd
 
 
 ifeq ($(KBUILD_EXTMOD),)
-core-y		+= kernel/ mm/ fs/ ipc/ security/ crypto/ block/
+core-y		+= kernel/ mm/ fs/ ipc/ security/ crypto/ block/ checkpoint/
 
 vmlinux-dirs	:= $(patsubst %/,%,$(filter %/, $(init-y) $(init-m) \
 		     $(core-y) $(core-m) $(drivers-y) $(drivers-m) \
diff --git a/checkpoint/Makefile b/checkpoint/Makefile
index 07d018b..d2df68c 100644
--- a/checkpoint/Makefile
+++ b/checkpoint/Makefile
@@ -2,4 +2,4 @@
 # Makefile for linux checkpoint/restart.
 #
 
-obj-$(CONFIG_CHECKPOINT_RESTART) += sys.o
+obj-$(CONFIG_CHECKPOINT_RESTART) += sys.o checkpoint.o restart.o
diff --git a/checkpoint/checkpoint.c b/checkpoint/checkpoint.c
new file mode 100644
index 0000000..e5e188f
--- /dev/null
+++ b/checkpoint/checkpoint.c
@@ -0,0 +1,174 @@
+/*
+ *  Checkpoint logic and helpers
+ *
+ *  Copyright (C) 2008 Oren Laadan
+ *
+ *  This file is subject to the terms and conditions of the GNU General Public
+ *  License.  See the file COPYING in the main directory of the Linux
+ *  distribution for more details.
+ */
+
+#include <linux/version.h>
+#include <linux/sched.h>
+#include <linux/time.h>
+#include <linux/fs.h>
+#include <linux/file.h>
+#include <linux/dcache.h>
+#include <linux/mount.h>
+#include <linux/utsname.h>
+#include <linux/magic.h>
+#include <linux/checkpoint.h>
+#include <linux/checkpoint_hdr.h>
+
+/**
+ * cr_write_obj - write a record described by a cr_hdr
+ * @ctx: checkpoint context
+ * @h: record descriptor
+ * @buf: record buffer
+ */
+int cr_write_obj(struct cr_ctx *ctx, struct cr_hdr *h, void *buf)
+{
+	int ret;
+
+	ret = cr_kwrite(ctx, h, sizeof(*h));
+	if (ret < 0)
+		return ret;
+	return cr_kwrite(ctx, buf, h->len);
+}
+
+/**
+ * cr_write_string - write a string
+ * @ctx: checkpoint context
+ * @str: string pointer
+ * @len: string length
+ */
+int cr_write_string(struct cr_ctx *ctx, char *str, int len)
+{
+	struct cr_hdr h;
+
+	h.type = CR_HDR_STRING;
+	h.len = len;
+	h.parent = 0;
+
+	return cr_write_obj(ctx, &h, str);
+}
+
+/* write the checkpoint header */
+static int cr_write_head(struct cr_ctx *ctx)
+{
+	struct cr_hdr h;
+	struct cr_hdr_head *hh = cr_hbuf_get(ctx, sizeof(*hh));
+	struct new_utsname *uts;
+	struct timeval ktv;
+	int ret;
+
+	h.type = CR_HDR_HEAD;
+	h.len = sizeof(*hh);
+	h.parent = 0;
+
+	do_gettimeofday(&ktv);
+
+	hh->magic = CHECKPOINT_MAGIC_HEAD;
+	hh->major = (LINUX_VERSION_CODE >> 16) & 0xff;
+	hh->minor = (LINUX_VERSION_CODE >> 8) & 0xff;
+	hh->patch = (LINUX_VERSION_CODE) & 0xff;
+
+	hh->rev = CR_VERSION;
+
+	hh->flags = ctx->flags;
+	hh->time = ktv.tv_sec;
+
+	uts = utsname();
+	memcpy(hh->release, uts->release, __NEW_UTS_LEN);
+	memcpy(hh->version, uts->version, __NEW_UTS_LEN);
+	memcpy(hh->machine, uts->machine, __NEW_UTS_LEN);
+
+	ret = cr_write_obj(ctx, &h, hh);
+	cr_hbuf_put(ctx, sizeof(*hh));
+	return ret;
+}
+
+/* write the checkpoint trailer */
+static int cr_write_tail(struct cr_ctx *ctx)
+{
+	struct cr_hdr h;
+	struct cr_hdr_tail *hh = cr_hbuf_get(ctx, sizeof(*hh));
+	int ret;
+
+	h.type = CR_HDR_TAIL;
+	h.len = sizeof(*hh);
+	h.parent = 0;
+
+	hh->magic = CHECKPOINT_MAGIC_TAIL;
+
+	ret = cr_write_obj(ctx, &h, hh);
+	cr_hbuf_put(ctx, sizeof(*hh));
+	return ret;
+}
+
+/* dump the task_struct of a given task */
+static int cr_write_task_struct(struct cr_ctx *ctx, struct task_struct *t)
+{
+	struct cr_hdr h;
+	struct cr_hdr_task *hh = cr_hbuf_get(ctx, sizeof(*hh));
+	int ret;
+
+	h.type = CR_HDR_TASK;
+	h.len = sizeof(*hh);
+	h.parent = 0;
+
+	hh->state = t->state;
+	hh->exit_state = t->exit_state;
+	hh->exit_code = t->exit_code;
+	hh->exit_signal = t->exit_signal;
+
+	hh->task_comm_len = TASK_COMM_LEN;
+
+	/* FIXME: save remaining relevant task_struct fields */
+
+	ret = cr_write_obj(ctx, &h, hh);
+	cr_hbuf_put(ctx, sizeof(*hh));
+	if (ret < 0)
+		return ret;
+
+	return cr_write_string(ctx, t->comm, TASK_COMM_LEN);
+}
+
+/* dump the entire state of a given task */
+static int cr_write_task(struct cr_ctx *ctx, struct task_struct *t)
+{
+	int ret ;
+
+	if (t->state == TASK_DEAD) {
+		pr_warning("CR: task may not be in state TASK_DEAD\n");
+		return -EAGAIN;
+	}
+
+	ret = cr_write_task_struct(ctx, t);
+	cr_debug("ret %d\n", ret);
+
+	return ret;
+}
+
+int do_checkpoint(struct cr_ctx *ctx)
+{
+	int ret;
+
+	/* FIX: need to test whether container is checkpointable */
+
+	ret = cr_write_head(ctx);
+	if (ret < 0)
+		goto out;
+	ret = cr_write_task(ctx, current);
+	if (ret < 0)
+		goto out;
+	ret = cr_write_tail(ctx);
+	if (ret < 0)
+		goto out;
+
+	/* on success, return (unique) checkpoint identifier */
+	ret = ctx->crid;
+
+ out:
+	return ret;
+}
diff --git a/checkpoint/restart.c b/checkpoint/restart.c
new file mode 100644
index 0000000..ef6bc37
--- /dev/null
+++ b/checkpoint/restart.c
@@ -0,0 +1,189 @@
+/*
+ *  Restart logic and helpers
+ *
+ *  Copyright (C) 2008 Oren Laadan
+ *
+ *  This file is subject to the terms and conditions of the GNU General Public
+ *  License.  See the file COPYING in the main directory of the Linux
+ *  distribution for more details.
+ */
+
+#include <linux/version.h>
+#include <linux/sched.h>
+#include <linux/file.h>
+#include <linux/magic.h>
+#include <linux/checkpoint.h>
+#include <linux/checkpoint_hdr.h>
+
+/**
+ * cr_read_obj - read a whole record (cr_hdr followed by payload)
+ * @ctx: checkpoint context
+ * @h: record descriptor
+ * @buf: record buffer
+ * @n: available buffer size
+ *
+ * Returns size of payload
+ */
+int cr_read_obj(struct cr_ctx *ctx, struct cr_hdr *h, void *buf, int n)
+{
+	int ret;
+
+	ret = cr_kread(ctx, h, sizeof(*h));
+	if (ret < 0)
+		return ret;
+
+	cr_debug("type %d len %d parent %d\n", h->type, h->len, h->parent);
+
+	if (h->len < 0 || h->len > n)
+		return -EINVAL;
+
+	return cr_kread(ctx, buf, h->len);
+}
+
+/**
+ * cr_read_obj_type - read a whole record of expected type
+ * @ctx: checkpoint context
+ * @buf: record buffer
+ * @n: available buffer size
+ * @type: expected record type
+ *
+ * Returns object reference of the parent object
+ */
+int cr_read_obj_type(struct cr_ctx *ctx, void *buf, int n, int type)
+{
+	struct cr_hdr h;
+	int ret;
+
+	ret = cr_read_obj(ctx, &h, buf, n);
+	if (!ret) {
+		if (h.type == type)
+			ret = h.parent;
+		else
+			ret = -EINVAL;
+	}
+	return ret;
+}
+
+/**
+ * cr_read_string - read a string
+ * @ctx: checkpoint context
+ * @str: string buffer
+ * @len: buffer buffer length
+ */
+int cr_read_string(struct cr_ctx *ctx, void *str, int len)
+{
+	return cr_read_obj_type(ctx, str, len, CR_HDR_STRING);
+}
+
+/* read the checkpoint header */
+static int cr_read_head(struct cr_ctx *ctx)
+{
+	struct cr_hdr_head *hh = cr_hbuf_get(ctx, sizeof(*hh));
+	int parent;
+
+	parent = cr_read_obj_type(ctx, hh, sizeof(*hh), CR_HDR_HEAD);
+	if (parent < 0)
+		return parent;
+	else if (parent != 0)
+		return -EINVAL;
+
+	if (hh->magic != CHECKPOINT_MAGIC_HEAD || hh->rev != CR_VERSION ||
+	    hh->major != ((LINUX_VERSION_CODE >> 16) & 0xff) ||
+	    hh->minor != ((LINUX_VERSION_CODE >> 8) & 0xff) ||
+	    hh->patch != ((LINUX_VERSION_CODE) & 0xff))
+		return -EINVAL;
+
+	if (hh->flags & ~CR_CTX_CKPT)
+		return -EINVAL;
+
+	ctx->oflags = hh->flags;
+
+	/* FIX: verify compatibility of release, version and machine */
+
+	cr_hbuf_put(ctx, sizeof(*hh));
+	return 0;
+}
+
+/* read the checkpoint trailer */
+static int cr_read_tail(struct cr_ctx *ctx)
+{
+	struct cr_hdr_tail *hh = cr_hbuf_get(ctx, sizeof(*hh));
+	int parent;
+
+	parent = cr_read_obj_type(ctx, hh, sizeof(*hh), CR_HDR_TAIL);
+	if (parent < 0)
+		return parent;
+	else if (parent != 0)
+		return -EINVAL;
+
+	if (hh->magic != CHECKPOINT_MAGIC_TAIL)
+		return -EINVAL;
+
+	cr_hbuf_put(ctx, sizeof(*hh));
+	return 0;
+}
+
+/* read the task_struct into the current task */
+static int cr_read_task_struct(struct cr_ctx *ctx)
+{
+	struct cr_hdr_task *hh = cr_hbuf_get(ctx, sizeof(*hh));
+	struct task_struct *t = current;
+	char *buf;
+	int parent, ret;
+
+	parent = cr_read_obj_type(ctx, hh, sizeof(*hh), CR_HDR_TASK);
+	if (parent < 0)
+		return parent;
+	else if (parent != 0)
+		return -EINVAL;
+
+	/* upper limit for task_comm_len to prevent DoS */
+	if (hh->task_comm_len < 0 || hh->task_comm_len > PAGE_SIZE)
+		return -EINVAL;
+
+	buf = kmalloc(hh->task_comm_len, GFP_KERNEL);
+	if (!buf)
+		return -ENOMEM;
+	ret = cr_read_string(ctx, buf, hh->task_comm_len);
+	if (!ret) {
+		/* if t->comm is too long, silently truncate */
+		memset(t->comm, 0, TASK_COMM_LEN);
+		memcpy(t->comm, buf, min(hh->task_comm_len, TASK_COMM_LEN));
+	}
+	kfree(buf);
+
+	/* FIXME: restore remaining relevant task_struct fields */
+
+	cr_hbuf_put(ctx, sizeof(*hh));
+	return ret;
+}
+
+/* read the entire state of the current task */
+static int cr_read_task(struct cr_ctx *ctx)
+{
+	int ret;
+
+	ret = cr_read_task_struct(ctx);
+	cr_debug("ret %d\n", ret);
+
+	return ret;
+}
+
+int do_restart(struct cr_ctx *ctx)
+{
+	int ret;
+
+	ret = cr_read_head(ctx);
+	if (ret < 0)
+		goto out;
+	ret = cr_read_task(ctx);
+	if (ret < 0)
+		goto out;
+	ret = cr_read_tail(ctx);
+	if (ret < 0)
+		goto out;
+
+	/* on success, adjust the return value if needed [TODO] */
+ out:
+	return ret;
+}
diff --git a/checkpoint/sys.c b/checkpoint/sys.c
index 375129c..30863c6 100644
--- a/checkpoint/sys.c
+++ b/checkpoint/sys.c
@@ -10,6 +10,189 @@
 
 #include <linux/sched.h>
 #include <linux/kernel.h>
+#include <linux/fs.h>
+#include <linux/file.h>
+#include <linux/uaccess.h>
+#include <linux/capability.h>
+#include <linux/checkpoint.h>
+
+/*
+ * helpers to write/read to/from the image file descriptor
+ *
+ *   cr_uwrite() - write a user-space buffer to the checkpoint image
+ *   cr_kwrite() - write a kernel-space buffer to the checkpoint image
+ *   cr_uread() - read from the checkpoint image to a user-space buffer
+ *   cr_kread() - read from the checkpoint image to a kernel-space buffer
+ */
+
+/*
+ * FIXME: (temporarily added file_pos_read() and file_pos_write() because
+ * they are static in fs/read_write.c... should cleanup and remove later)
+ */
+static inline loff_t file_pos_read(struct file *file)
+{
+	return file->f_pos;
+}
+
+static inline void file_pos_write(struct file *file, loff_t pos)
+{
+	file->f_pos = pos;
+}
+
+int cr_uwrite(struct cr_ctx *ctx, void *buf, int count)
+{
+	struct file *file = ctx->file;
+	ssize_t nwrite;
+	int nleft;
+
+	for (nleft = count; nleft; nleft -= nwrite) {
+		loff_t pos = file_pos_read(file);
+		nwrite = vfs_write(file, (char __user *) buf, nleft, &pos);
+		file_pos_write(file, pos);
+		if (nwrite <= 0) {
+			if (nwrite == -EAGAIN)
+				nwrite = 0;
+			else
+				return nwrite;
+		}
+		buf += nwrite;
+	}
+
+	ctx->total += count;
+	return 0;
+}
+
+int cr_kwrite(struct cr_ctx *ctx, void *buf, int count)
+{
+	mm_segment_t oldfs;
+	int ret;
+
+	oldfs = get_fs();
+	set_fs(KERNEL_DS);
+	ret = cr_uwrite(ctx, buf, count);
+	set_fs(oldfs);
+
+	return ret;
+}
+
+int cr_uread(struct cr_ctx *ctx, void *buf, int count)
+{
+	struct file *file = ctx->file;
+	ssize_t nread;
+	int nleft;
+
+	for (nleft = count; nleft; nleft -= nread) {
+		loff_t pos = file_pos_read(file);
+		nread = vfs_read(file, (char __user *) buf, nleft, &pos);
+		file_pos_write(file, pos);
+		if (nread <= 0) {
+			if (nread == -EAGAIN)
+				nread = 0;
+			else
+				return nread;
+		}
+		buf += nread;
+	}
+
+	ctx->total += count;
+	return 0;
+}
+
+int cr_kread(struct cr_ctx *ctx, void *buf, int count)
+{
+	mm_segment_t oldfs;
+	int ret;
+
+	oldfs = get_fs();
+	set_fs(KERNEL_DS);
+	ret = cr_uread(ctx, buf, count);
+	set_fs(oldfs);
+
+	return ret;
+}
+
+
+/*
+ * helpers to manage CR contexts: allocated for each checkpoint and/or
+ * restart operation, and persists until the operation is completed.
+ */
+
+/* unique checkpoint identifier (FIXME: should be per-container) */
+static atomic_t cr_ctx_count;
+
+void cr_ctx_free(struct cr_ctx *ctx)
+{
+	if (ctx->file)
+		fput(ctx->file);
+
+	free_pages((unsigned long) ctx->hbuf, CR_HBUF_ORDER);
+
+	kfree(ctx);
+}
+
+struct cr_ctx *cr_ctx_alloc(pid_t pid, int fd, unsigned long flags)
+{
+	struct cr_ctx *ctx;
+
+	ctx = kzalloc(sizeof(*ctx), GFP_KERNEL);
+	if (!ctx)
+		return ERR_PTR(-ENOMEM);
+
+	ctx->file = fget(fd);
+	if (!ctx->file) {
+		cr_ctx_free(ctx);
+		return ERR_PTR(-EBADF);
+	}
+
+	ctx->hbuf = (void *) __get_free_pages(GFP_KERNEL, CR_HBUF_ORDER);
+	if (!ctx->hbuf) {
+		cr_ctx_free(ctx);
+		return ERR_PTR(-ENOMEM);
+	}
+
+	ctx->pid = pid;
+	ctx->flags = flags;
+
+	ctx->crid = atomic_inc_return(&cr_ctx_count);
+
+	return ctx;
+}
+
+/*
+ * During checkpoint and restart the code writes outs/reads in data
+ * to/from the chekcpoint image from/to a temporary buffer (ctx->hbuf).
+ * Because operations can be nested, one should call cr_hbuf_get() to
+ * reserve space in the buffer, and then cr_hbuf_put() when no longer
+ * needs that space.
+ */
+
+/**
+ * cr_hbuf_get - reserve space on the hbuf
+ * @ctx: checkpoint context
+ * @n: number of bytes to reserve
+ *
+ * Returns pointer to reserved space
+ */
+void *cr_hbuf_get(struct cr_ctx *ctx, int n)
+{
+	void *ptr;
+
+	BUG_ON(ctx->hpos + n > CR_HBUF_TOTAL);
+	ptr = (void *) (((char *) ctx->hbuf) + ctx->hpos);
+	ctx->hpos += n;
+	return ptr;
+}
+
+/**
+ * cr_hbuf_put - unreserve space on the hbuf
+ * @ctx: checkpoint context
+ * @n: number of bytes to reserve
+ */
+void cr_hbuf_put(struct cr_ctx *ctx, int n)
+{
+	BUG_ON(ctx->hpos < n);
+	ctx->hpos -= n;
+}
 
 /**
  * sys_checkpoint - checkpoint a container
@@ -22,9 +205,23 @@
  */
 asmlinkage long sys_checkpoint(pid_t pid, int fd, unsigned long flags)
 {
-	pr_debug("sys_checkpoint not implemented yet\n");
-	return -ENOSYS;
+	struct cr_ctx *ctx;
+	int ret;
+
+	/* no flags for now */
+	if (flags)
+		return -EINVAL;
+
+	ctx = cr_ctx_alloc(pid, fd, flags | CR_CTX_CKPT);
+	if (IS_ERR(ctx))
+		return PTR_ERR(ctx);
+
+	ret = do_checkpoint(ctx);
+
+	cr_ctx_free(ctx);
+	return ret;
 }
+
 /**
  * sys_restart - restart a container
  * @crid: checkpoint image identifier
@@ -36,6 +233,19 @@ asmlinkage long sys_checkpoint(pid_t pid, int fd, unsigned long flags)
  */
 asmlinkage long sys_restart(int crid, int fd, unsigned long flags)
 {
-	pr_debug("sys_restart not implemented yet\n");
-	return -ENOSYS;
+	struct cr_ctx *ctx;
+	int ret;
+
+	/* no flags for now */
+	if (flags)
+		return -EINVAL;
+
+	ctx = cr_ctx_alloc(crid, fd, flags | CR_CTX_RSTR);
+	if (IS_ERR(ctx))
+		return PTR_ERR(ctx);
+
+	ret = do_restart(ctx);
+
+	cr_ctx_free(ctx);
+	return ret;
 }
diff --git a/include/linux/checkpoint.h b/include/linux/checkpoint.h
new file mode 100644
index 0000000..5e53ae6
--- /dev/null
+++ b/include/linux/checkpoint.h
@@ -0,0 +1,60 @@
+#ifndef _CHECKPOINT_CKPT_H_
+#define _CHECKPOINT_CKPT_H_
+/*
+ *  Generic container checkpoint-restart
+ *
+ *  Copyright (C) 2008 Oren Laadan
+ *
+ *  This file is subject to the terms and conditions of the GNU General Public
+ *  License.  See the file COPYING in the main directory of the Linux
+ *  distribution for more details.
+ */
+
+#define CR_VERSION  1
+
+struct cr_ctx {
+	pid_t pid;		/* container identifier */
+	int crid;		/* unique checkpoint id */
+
+	unsigned long flags;
+	unsigned long oflags;	/* restart: old flags */
+
+	struct file *file;
+	int total;		/* total read/written */
+
+	void *hbuf;		/* temporary buffer for headers */
+	int hpos;		/* position in headers buffer */
+};
+
+/* cr_ctx: flags */
+#define CR_CTX_CKPT	0x1
+#define CR_CTX_RSTR	0x2
+
+/* allocation defaults */
+#define CR_HBUF_ORDER  1
+#define CR_HBUF_TOTAL  (PAGE_SIZE << CR_HBUF_ORDER)
+
+extern int cr_uwrite(struct cr_ctx *ctx, void *buf, int count);
+extern int cr_kwrite(struct cr_ctx *ctx, void *buf, int count);
+extern int cr_uread(struct cr_ctx *ctx, void *buf, int count);
+extern int cr_kread(struct cr_ctx *ctx, void *buf, int count);
+
+extern void *cr_hbuf_get(struct cr_ctx *ctx, int n);
+extern void cr_hbuf_put(struct cr_ctx *ctx, int n);
+
+struct cr_hdr;
+
+extern int cr_write_obj(struct cr_ctx *ctx, struct cr_hdr *h, void *buf);
+extern int cr_write_string(struct cr_ctx *ctx, char *str, int len);
+
+extern int cr_read_obj(struct cr_ctx *ctx, struct cr_hdr *h, void *buf, int n);
+extern int cr_read_obj_type(struct cr_ctx *ctx, void *buf, int n, int type);
+extern int cr_read_string(struct cr_ctx *ctx, void *str, int len);
+
+extern int do_checkpoint(struct cr_ctx *ctx);
+extern int do_restart(struct cr_ctx *ctx);
+
+#define cr_debug(fmt, args...)  \
+	pr_debug("[CR:%s] " fmt, __func__, ## args)
+
+#endif /* _CHECKPOINT_CKPT_H_ */
diff --git a/include/linux/checkpoint_hdr.h b/include/linux/checkpoint_hdr.h
new file mode 100644
index 0000000..79e4df2
--- /dev/null
+++ b/include/linux/checkpoint_hdr.h
@@ -0,0 +1,75 @@
+#ifndef _CHECKPOINT_CKPT_HDR_H_
+#define _CHECKPOINT_CKPT_HDR_H_
+/*
+ *  Generic container checkpoint-restart
+ *
+ *  Copyright (C) 2008 Oren Laadan
+ *
+ *  This file is subject to the terms and conditions of the GNU General Public
+ *  License.  See the file COPYING in the main directory of the Linux
+ *  distribution for more details.
+ */
+
+#include <linux/types.h>
+#include <linux/utsname.h>
+
+/*
+ * To maintain compatibility between 32-bit and 64-bit architecture flavors,
+ * keep data 64-bit aligned: use padding for structure members, and use
+ * __attribute__ ((aligned (8))) for the entire structure.
+ */
+
+/* records: generic header */
+
+struct cr_hdr {
+	__s16 type;
+	__s16 len;
+	__u32 parent;
+};
+
+/* header types */
+enum {
+	CR_HDR_HEAD = 1,
+	CR_HDR_STRING,
+
+	CR_HDR_TASK = 101,
+	CR_HDR_THREAD,
+	CR_HDR_CPU,
+
+	CR_HDR_MM = 201,
+	CR_HDR_VMA,
+	CR_HDR_MM_CONTEXT,
+
+	CR_HDR_TAIL = 5001
+};
+
+struct cr_hdr_head {
+	__u64 magic;
+
+	__u16 major;
+	__u16 minor;
+	__u16 patch;
+	__u16 rev;
+
+	__u64 time;	/* when checkpoint taken */
+	__u64 flags;	/* checkpoint options */
+
+	char release[__NEW_UTS_LEN];
+	char version[__NEW_UTS_LEN];
+	char machine[__NEW_UTS_LEN];
+} __attribute__((aligned(8)));
+
+struct cr_hdr_tail {
+	__u64 magic;
+} __attribute__((aligned(8)));
+
+struct cr_hdr_task {
+	__u32 state;
+	__u32 exit_state;
+	__u32 exit_code;
+	__u32 exit_signal;
+
+	__s32 task_comm_len;
+} __attribute__((aligned(8)));
+
+#endif /* _CHECKPOINT_CKPT_HDR_H_ */
diff --git a/include/linux/magic.h b/include/linux/magic.h
index 1fa0c2c..c2b811c 100644
--- a/include/linux/magic.h
+++ b/include/linux/magic.h
@@ -42,4 +42,7 @@
 #define FUTEXFS_SUPER_MAGIC	0xBAD1DEA
 #define INOTIFYFS_SUPER_MAGIC	0x2BAD1DEA
 
+#define CHECKPOINT_MAGIC_HEAD  0x00feed0cc0a2d200LL
+#define CHECKPOINT_MAGIC_TAIL  0x002d2a0cc0deef00LL
+
 #endif /* __LINUX_MAGIC_H__ */
-- 
1.5.4.3

[RFC v5][PATCH 3/8] x86 support for checkpoint/restart

[Oren Laadan]

(Following Dave Hansen's refactoring of the original post)

Add logic to save and restore architecture specific state, including
thread-specific state, CPU registers and FPU state.

Currently only x86-32 is supported. Compiling on x86-64 will trigger
an explicit error.

Signed-off-by: Oren Laadan <orenl-eQaUEPhvms7ENvBUuze7eA@public.gmane.org>
---
 arch/x86/mm/Makefile             |    2 +
 arch/x86/mm/checkpoint.c         |  198 ++++++++++++++++++++++++++++++++++++++
 arch/x86/mm/restart.c            |  177 ++++++++++++++++++++++++++++++++++
 checkpoint/checkpoint.c          |   13 +++-
 checkpoint/checkpoint_arch.h     |    7 ++
 checkpoint/restart.c             |   13 +++-
 include/asm-x86/checkpoint_hdr.h |   72 ++++++++++++++
 include/linux/checkpoint_hdr.h   |    1 +
 8 files changed, 481 insertions(+), 2 deletions(-)
 create mode 100644 arch/x86/mm/checkpoint.c
 create mode 100644 arch/x86/mm/restart.c
 create mode 100644 checkpoint/checkpoint_arch.h
 create mode 100644 include/asm-x86/checkpoint_hdr.h

diff --git a/arch/x86/mm/Makefile b/arch/x86/mm/Makefile
index dfb932d..58fe072 100644
--- a/arch/x86/mm/Makefile
+++ b/arch/x86/mm/Makefile
@@ -22,3 +22,5 @@ endif
 obj-$(CONFIG_ACPI_NUMA)		+= srat_$(BITS).o
 
 obj-$(CONFIG_MEMTEST)		+= memtest.o
+
+obj-$(CONFIG_CHECKPOINT_RESTART) += checkpoint.o restart.o
diff --git a/arch/x86/mm/checkpoint.c b/arch/x86/mm/checkpoint.c
new file mode 100644
index 0000000..eb60003
--- /dev/null
+++ b/arch/x86/mm/checkpoint.c
@@ -0,0 +1,198 @@
+/*
+ *  Checkpoint/restart - architecture specific support for x86
+ *
+ *  Copyright (C) 2008 Oren Laadan
+ *
+ *  This file is subject to the terms and conditions of the GNU General Public
+ *  License.  See the file COPYING in the main directory of the Linux
+ *  distribution for more details.
+ */
+
+#include <asm/desc.h>
+#include <asm/i387.h>
+
+#include <linux/checkpoint.h>
+#include <linux/checkpoint_hdr.h>
+
+/* dump the thread_struct of a given task */
+int cr_write_thread(struct cr_ctx *ctx, struct task_struct *t)
+{
+	struct cr_hdr h;
+	struct cr_hdr_thread *hh = cr_hbuf_get(ctx, sizeof(*hh));
+	struct thread_struct *thread;
+	struct desc_struct *desc;
+	int ntls = 0;
+	int n, ret;
+
+	h.type = CR_HDR_THREAD;
+	h.len = sizeof(*hh);
+	h.parent = task_pid_vnr(t);
+
+	thread = &t->thread;
+
+	/* calculate no. of TLS entries that follow */
+	desc = thread->tls_array;
+	for (n = GDT_ENTRY_TLS_ENTRIES; n > 0; n--, desc++) {
+		if (desc->a || desc->b)
+			ntls++;
+	}
+
+	hh->gdt_entry_tls_entries = GDT_ENTRY_TLS_ENTRIES;
+	hh->sizeof_tls_array = sizeof(thread->tls_array);
+	hh->ntls = ntls;
+
+	ret = cr_write_obj(ctx, &h, hh);
+	cr_hbuf_put(ctx, sizeof(*hh));
+	if (ret < 0)
+		return ret;
+
+	/* for simplicity dump the entire array, cherry-pick upon restart */
+	ret = cr_kwrite(ctx, thread->tls_array, sizeof(thread->tls_array));
+
+	cr_debug("ntls %d\n", ntls);
+
+	/* IGNORE RESTART BLOCKS FOR NOW ... */
+
+	return ret;
+}
+
+#ifdef CONFIG_X86_64
+
+#error "CONFIG_X86_64 unsupported yet."
+
+#else	/* !CONFIG_X86_64 */
+
+void cr_write_cpu_regs(struct cr_hdr_cpu *hh, struct task_struct *t)
+{
+	struct thread_struct *thread = &t->thread;
+	struct pt_regs *regs = task_pt_regs(t);
+
+	hh->bp = regs->bp;
+	hh->bx = regs->bx;
+	hh->ax = regs->ax;
+	hh->cx = regs->cx;
+	hh->dx = regs->dx;
+	hh->si = regs->si;
+	hh->di = regs->di;
+	hh->orig_ax = regs->orig_ax;
+	hh->ip = regs->ip;
+	hh->cs = regs->cs;
+	hh->flags = regs->flags;
+	hh->sp = regs->sp;
+	hh->ss = regs->ss;
+
+	hh->ds = regs->ds;
+	hh->es = regs->es;
+
+	/*
+	 * for checkpoint in process context (from within a container)
+	 * the GS and FS registers should be saved from the hardware;
+	 * otherwise they are already sabed on the thread structure
+	 */
+	if (t == current) {
+		savesegment(gs, hh->gs);
+		savesegment(fs, hh->fs);
+	} else {
+		hh->gs = thread->gs;
+		hh->fs = thread->fs;
+	}
+
+	/*
+	 * for checkpoint in process context (from within a container),
+	 * the actual syscall is taking place at this very moment; so
+	 * we (optimistically) subtitute the future return value (0) of
+	 * this syscall into the orig_eax, so that upon restart it will
+	 * succeed (or it will endlessly retry checkpoint...)
+	 */
+	if (t == current) {
+		BUG_ON(hh->orig_ax < 0);
+		hh->ax = 0;
+	}
+}
+
+void cr_write_cpu_debug(struct cr_hdr_cpu *hh, struct task_struct *t)
+{
+	struct thread_struct *thread = &t->thread;
+
+	/* debug regs */
+
+	preempt_disable();
+
+	/*
+	 * for checkpoint in process context (from within a container),
+	 * get the actual registers; otherwise get the saved values.
+	 */
+
+	if (t == current) {
+		get_debugreg(hh->debugreg0, 0);
+		get_debugreg(hh->debugreg1, 1);
+		get_debugreg(hh->debugreg2, 2);
+		get_debugreg(hh->debugreg3, 3);
+		get_debugreg(hh->debugreg6, 6);
+		get_debugreg(hh->debugreg7, 7);
+	} else {
+		hh->debugreg0 = thread->debugreg0;
+		hh->debugreg1 = thread->debugreg1;
+		hh->debugreg2 = thread->debugreg2;
+		hh->debugreg3 = thread->debugreg3;
+		hh->debugreg6 = thread->debugreg6;
+		hh->debugreg7 = thread->debugreg7;
+	}
+
+	hh->debugreg4 = 0;
+	hh->debugreg5 = 0;
+
+	hh->uses_debug = !!(task_thread_info(t)->flags & TIF_DEBUG);
+
+	preempt_enable();
+}
+
+void cr_write_cpu_fpu(struct cr_hdr_cpu *hh, struct task_struct *t)
+{
+	struct thread_struct *thread = &t->thread;
+	struct thread_info *thread_info = task_thread_info(t);
+
+	/* i387 + MMU + SSE logic */
+
+	preempt_disable();
+
+	hh->used_math = tsk_used_math(t) ? 1 : 0;
+	if (hh->used_math) {
+		/*
+		 * normally, no need to unlazy_fpu(), since TS_USEDFPU flag
+		 * have been cleared when task was conexted-switched out...
+		 * except if we are in process context, in which case we do
+		 */
+		if (thread_info->status & TS_USEDFPU)
+			unlazy_fpu(current);
+
+		hh->has_fxsr = cpu_has_fxsr;
+		memcpy(&hh->xstate, &thread->xstate, sizeof(thread->xstate));
+	}
+
+	preempt_enable();
+}
+
+#endif	/* CONFIG_X86_64 */
+
+/* dump the cpu state and registers of a given task */
+int cr_write_cpu(struct cr_ctx *ctx, struct task_struct *t)
+{
+	struct cr_hdr h;
+	struct cr_hdr_cpu *hh = cr_hbuf_get(ctx, sizeof(*hh));
+	int ret;
+
+	h.type = CR_HDR_CPU;
+	h.len = sizeof(*hh);
+	h.parent = task_pid_vnr(t);
+
+	cr_write_cpu_regs(hh, t);
+	cr_write_cpu_debug(hh, t);
+	cr_write_cpu_fpu(hh, t);
+
+	cr_debug("math %d debug %d\n", hh->used_math, hh->uses_debug);
+
+	ret = cr_write_obj(ctx, &h, hh);
+	cr_hbuf_put(ctx, sizeof(*hh));
+	return ret;
+}
diff --git a/arch/x86/mm/restart.c b/arch/x86/mm/restart.c
new file mode 100644
index 0000000..0178080
--- /dev/null
+++ b/arch/x86/mm/restart.c
@@ -0,0 +1,177 @@
+/*
+ *  Checkpoint/restart - architecture specific support for x86
+ *
+ *  Copyright (C) 2008 Oren Laadan
+ *
+ *  This file is subject to the terms and conditions of the GNU General Public
+ *  License.  See the file COPYING in the main directory of the Linux
+ *  distribution for more details.
+ */
+
+#include <asm/desc.h>
+#include <asm/i387.h>
+
+#include <linux/checkpoint.h>
+#include <linux/checkpoint_hdr.h>
+
+/* read the thread_struct into the current task */
+int cr_read_thread(struct cr_ctx *ctx)
+{
+	struct cr_hdr_thread *hh = cr_hbuf_get(ctx, sizeof(*hh));
+	struct task_struct *t = current;
+	struct thread_struct *thread = &t->thread;
+	int parent;
+
+	parent = cr_read_obj_type(ctx, hh, sizeof(*hh), CR_HDR_THREAD);
+	if (parent < 0)
+		return parent;
+#if 0	/* activate when containers are used */
+	if (parent != task_pid_vnr(t))
+		return -EINVAL;
+#endif
+	cr_debug("ntls %d\n", hh->ntls);
+
+	if (hh->gdt_entry_tls_entries != GDT_ENTRY_TLS_ENTRIES ||
+	    hh->sizeof_tls_array != sizeof(thread->tls_array) ||
+	    hh->ntls < 0 || hh->ntls > GDT_ENTRY_TLS_ENTRIES)
+		return -EINVAL;
+
+	if (hh->ntls > 0) {
+		struct desc_struct *desc;
+		int size, cpu, ret;
+
+		/*
+		 * restore TLS by hand: why convert to struct user_desc if
+		 * sys_set_thread_entry() will convert it back ?
+		 */
+
+		size = sizeof(*desc) * GDT_ENTRY_TLS_ENTRIES;
+		desc = kmalloc(size, GFP_KERNEL);
+		if (!desc)
+			return -ENOMEM;
+
+		ret = cr_kread(ctx, desc, size);
+		if (ret >= 0) {
+			/*
+			 * FIX: add sanity checks (eg. that values makes
+			 * sense, that we don't overwrite old values, etc
+			 */
+			cpu = get_cpu();
+			memcpy(thread->tls_array, desc, size);
+			load_TLS(thread, cpu);
+			put_cpu();
+		}
+		kfree(desc);
+	}
+
+	return 0;
+}
+
+#ifdef CONFIG_X86_64
+
+#error "CONFIG_X86_64 unsupported yet."
+
+#else	/* !CONFIG_X86_64 */
+
+int cr_read_cpu_regs(struct cr_hdr_cpu *hh, struct task_struct *t)
+{
+	struct thread_struct *thread = &t->thread;
+	struct pt_regs *regs = task_pt_regs(t);
+
+	regs->bx = hh->bx;
+	regs->cx = hh->cx;
+	regs->dx = hh->dx;
+	regs->si = hh->si;
+	regs->di = hh->di;
+	regs->bp = hh->bp;
+	regs->ax = hh->ax;
+	regs->ds = hh->ds;
+	regs->es = hh->es;
+	regs->orig_ax = hh->orig_ax;
+	regs->ip = hh->ip;
+	regs->cs = hh->cs;
+	regs->flags = hh->flags;
+	regs->sp = hh->sp;
+	regs->ss = hh->ss;
+
+	thread->gs = hh->gs;
+	thread->fs = hh->fs;
+	loadsegment(gs, hh->gs);
+	loadsegment(fs, hh->fs);
+
+	return 0;
+}
+
+int cr_read_cpu_debug(struct cr_hdr_cpu *hh, struct task_struct *t)
+{
+	/* debug regs */
+
+	if (hh->uses_debug) {
+		set_debugreg(hh->debugreg0, 0);
+		set_debugreg(hh->debugreg1, 1);
+		/* ignore 4, 5 */
+		set_debugreg(hh->debugreg2, 2);
+		set_debugreg(hh->debugreg3, 3);
+		set_debugreg(hh->debugreg6, 6);
+		set_debugreg(hh->debugreg7, 7);
+	}
+
+	return 0;
+}
+
+int cr_read_cpu_fpu(struct cr_hdr_cpu *hh, struct task_struct *t)
+{
+	struct thread_struct *thread = &t->thread;
+
+	/* i387 + MMU + SSE */
+
+	preempt_disable();
+
+	__clear_fpu(t);		/* in case we used FPU in user mode */
+
+	if (!hh->used_math)
+		clear_used_math();
+	else {
+		if (hh->has_fxsr != cpu_has_fxsr) {
+			force_sig(SIGFPE, t);
+			return -EINVAL;
+		}
+		memcpy(&thread->xstate, &hh->xstate, sizeof(thread->xstate));
+		set_used_math();
+	}
+
+	preempt_enable();
+
+	return 0;
+}
+
+#endif	/* CONFIG_X86_64 */
+
+/* read the cpu state and registers for the current task */
+int cr_read_cpu(struct cr_ctx *ctx)
+{
+	struct cr_hdr_cpu *hh = cr_hbuf_get(ctx, sizeof(*hh));
+	struct task_struct *t = current;
+	int parent, ret;
+
+	parent = cr_read_obj_type(ctx, hh, sizeof(*hh), CR_HDR_CPU);
+	if (parent < 0)
+		return parent;
+#if 0	/* activate when containers are used */
+	if (parent != task_pid_vnr(t))
+		return -EINVAL;
+#endif
+	/* FIX: sanity check for sensitive registers (eg. eflags) */
+
+	ret = cr_read_cpu_regs(hh, t);
+	if (ret < 0)
+		goto out;
+	ret = cr_read_cpu_debug(hh, t);
+	if (ret < 0)
+		goto out;
+	ret = cr_read_cpu_fpu(hh, t);
+
+	cr_debug("math %d debug %d\n", hh->used_math, hh->uses_debug);
+ out:
+	return ret;
+}
diff --git a/checkpoint/checkpoint.c b/checkpoint/checkpoint.c
index e5e188f..6ca26d0 100644
--- a/checkpoint/checkpoint.c
+++ b/checkpoint/checkpoint.c
@@ -20,6 +20,8 @@
 #include <linux/checkpoint.h>
 #include <linux/checkpoint_hdr.h>
 
+#include "checkpoint_arch.h"
+
 /**
  * cr_write_obj - write a record described by a cr_hdr
  * @ctx: checkpoint context
@@ -145,8 +147,17 @@ static int cr_write_task(struct cr_ctx *ctx, struct task_struct *t)
 	}
 
 	ret = cr_write_task_struct(ctx, t);
-	cr_debug("ret %d\n", ret);
+	cr_debug("task_struct: ret %d\n", ret);
+	if (ret < 0)
+		goto out;
+	ret = cr_write_thread(ctx, t);
+	cr_debug("thread: ret %d\n", ret);
+	if (ret < 0)
+		goto out;
+	ret = cr_write_cpu(ctx, t);
+	cr_debug("cpu: ret %d\n", ret);
 
+ out:
 	return ret;
 }
 
diff --git a/checkpoint/checkpoint_arch.h b/checkpoint/checkpoint_arch.h
new file mode 100644
index 0000000..bf2d21e
--- /dev/null
+++ b/checkpoint/checkpoint_arch.h
@@ -0,0 +1,7 @@
+#include <linux/checkpoint.h>
+
+extern int cr_write_thread(struct cr_ctx *ctx, struct task_struct *t);
+extern int cr_write_cpu(struct cr_ctx *ctx, struct task_struct *t);
+
+extern int cr_read_thread(struct cr_ctx *ctx);
+extern int cr_read_cpu(struct cr_ctx *ctx);
diff --git a/checkpoint/restart.c b/checkpoint/restart.c
index ef6bc37..0484de0 100644
--- a/checkpoint/restart.c
+++ b/checkpoint/restart.c
@@ -15,6 +15,8 @@
 #include <linux/checkpoint.h>
 #include <linux/checkpoint_hdr.h>
 
+#include "checkpoint_arch.h"
+
 /**
  * cr_read_obj - read a whole record (cr_hdr followed by payload)
  * @ctx: checkpoint context
@@ -164,8 +166,17 @@ static int cr_read_task(struct cr_ctx *ctx)
 	int ret;
 
 	ret = cr_read_task_struct(ctx);
-	cr_debug("ret %d\n", ret);
+	cr_debug("task_struct: ret %d\n", ret);
+	if (ret < 0)
+		goto out;
+	ret = cr_read_thread(ctx);
+	cr_debug("thread: ret %d\n", ret);
+	if (ret < 0)
+		goto out;
+	ret = cr_read_cpu(ctx);
+	cr_debug("cpu: ret %d\n", ret);
 
+ out:
 	return ret;
 }
 
diff --git a/include/asm-x86/checkpoint_hdr.h b/include/asm-x86/checkpoint_hdr.h
new file mode 100644
index 0000000..44a903c
--- /dev/null
+++ b/include/asm-x86/checkpoint_hdr.h
@@ -0,0 +1,72 @@
+#ifndef __ASM_X86_CKPT_HDR_H
+#define __ASM_X86_CKPT_HDR_H
+/*
+ *  Checkpoint/restart - architecture specific headers x86
+ *
+ *  Copyright (C) 2008 Oren Laadan
+ *
+ *  This file is subject to the terms and conditions of the GNU General Public
+ *  License.  See the file COPYING in the main directory of the Linux
+ *  distribution for more details.
+ */
+
+#include <asm/processor.h>
+
+struct cr_hdr_thread {
+	/* NEED: restart blocks */
+
+	__s16 gdt_entry_tls_entries;
+	__s16 sizeof_tls_array;
+	__s16 ntls;	/* number of TLS entries to follow */
+} __attribute__((aligned(8)));
+
+struct cr_hdr_cpu {
+	/* see struct pt_regs (x86-64) */
+	__u64 r15;
+	__u64 r14;
+	__u64 r13;
+	__u64 r12;
+	__u64 bp;
+	__u64 bx;
+	__u64 r11;
+	__u64 r10;
+	__u64 r9;
+	__u64 r8;
+	__u64 ax;
+	__u64 cx;
+	__u64 dx;
+	__u64 si;
+	__u64 di;
+	__u64 orig_ax;
+	__u64 ip;
+	__u64 cs;
+	__u64 flags;
+	__u64 sp;
+	__u64 ss;
+
+	/* segment registers */
+	__u64 ds;
+	__u64 es;
+	__u64 fs;
+	__u64 gs;
+
+	/* debug registers */
+	__u64 debugreg0;
+	__u64 debugreg1;
+	__u64 debugreg2;
+	__u64 debugreg3;
+	__u64 debugreg4;
+	__u64 debugreg5;
+	__u64 debugreg6;
+	__u64 debugreg7;
+
+	__u16 uses_debug;
+	__u16 used_math;
+	__u16 has_fxsr;
+	__u16 _padding;
+
+	union thread_xstate xstate;	/* i387 */
+
+} __attribute__((aligned(8)));
+
+#endif /* __ASM_X86_CKPT_HDR__H */
diff --git a/include/linux/checkpoint_hdr.h b/include/linux/checkpoint_hdr.h
index 79e4df2..03ec72e 100644
--- a/include/linux/checkpoint_hdr.h
+++ b/include/linux/checkpoint_hdr.h
@@ -12,6 +12,7 @@
 
 #include <linux/types.h>
 #include <linux/utsname.h>
+#include <asm/checkpoint_hdr.h>
 
 /*
  * To maintain compatibility between 32-bit and 64-bit architecture flavors,
-- 
1.5.4.3

Re: [RFC v5][PATCH 3/8] x86 support for checkpoint/restart

[Serge E. Hallyn]

Quoting Oren Laadan (orenl-eQaUEPhvms7ENvBUuze7eA@public.gmane.org):
> (Following Dave Hansen's refactoring of the original post)
> 
> Add logic to save and restore architecture specific state, including
> thread-specific state, CPU registers and FPU state.
> 
> Currently only x86-32 is supported. Compiling on x86-64 will trigger
> an explicit error.
> 
> Signed-off-by: Oren Laadan <orenl-eQaUEPhvms7ENvBUuze7eA@public.gmane.org>

Acked-by: Serge Hallyn <serue-r/Jw6+rmf7HQT0dZR+AlfA@public.gmane.org>

> ---
>  arch/x86/mm/Makefile             |    2 +
>  arch/x86/mm/checkpoint.c         |  198 ++++++++++++++++++++++++++++++++++++++
>  arch/x86/mm/restart.c            |  177 ++++++++++++++++++++++++++++++++++
>  checkpoint/checkpoint.c          |   13 +++-
>  checkpoint/checkpoint_arch.h     |    7 ++
>  checkpoint/restart.c             |   13 +++-
>  include/asm-x86/checkpoint_hdr.h |   72 ++++++++++++++
>  include/linux/checkpoint_hdr.h   |    1 +
>  8 files changed, 481 insertions(+), 2 deletions(-)
>  create mode 100644 arch/x86/mm/checkpoint.c
>  create mode 100644 arch/x86/mm/restart.c
>  create mode 100644 checkpoint/checkpoint_arch.h
>  create mode 100644 include/asm-x86/checkpoint_hdr.h
> 
> diff --git a/arch/x86/mm/Makefile b/arch/x86/mm/Makefile
> index dfb932d..58fe072 100644
> --- a/arch/x86/mm/Makefile
> +++ b/arch/x86/mm/Makefile
> @@ -22,3 +22,5 @@ endif
>  obj-$(CONFIG_ACPI_NUMA)		+= srat_$(BITS).o
> 
>  obj-$(CONFIG_MEMTEST)		+= memtest.o
> +
> +obj-$(CONFIG_CHECKPOINT_RESTART) += checkpoint.o restart.o
> diff --git a/arch/x86/mm/checkpoint.c b/arch/x86/mm/checkpoint.c
> new file mode 100644
> index 0000000..eb60003
> --- /dev/null
> +++ b/arch/x86/mm/checkpoint.c
> @@ -0,0 +1,198 @@
> +/*
> + *  Checkpoint/restart - architecture specific support for x86
> + *
> + *  Copyright (C) 2008 Oren Laadan
> + *
> + *  This file is subject to the terms and conditions of the GNU General Public
> + *  License.  See the file COPYING in the main directory of the Linux
> + *  distribution for more details.
> + */
> +
> +#include <asm/desc.h>
> +#include <asm/i387.h>
> +
> +#include <linux/checkpoint.h>
> +#include <linux/checkpoint_hdr.h>
> +
> +/* dump the thread_struct of a given task */
> +int cr_write_thread(struct cr_ctx *ctx, struct task_struct *t)
> +{
> +	struct cr_hdr h;
> +	struct cr_hdr_thread *hh = cr_hbuf_get(ctx, sizeof(*hh));
> +	struct thread_struct *thread;
> +	struct desc_struct *desc;
> +	int ntls = 0;
> +	int n, ret;
> +
> +	h.type = CR_HDR_THREAD;
> +	h.len = sizeof(*hh);
> +	h.parent = task_pid_vnr(t);
> +
> +	thread = &t->thread;
> +
> +	/* calculate no. of TLS entries that follow */
> +	desc = thread->tls_array;
> +	for (n = GDT_ENTRY_TLS_ENTRIES; n > 0; n--, desc++) {
> +		if (desc->a || desc->b)
> +			ntls++;
> +	}
> +
> +	hh->gdt_entry_tls_entries = GDT_ENTRY_TLS_ENTRIES;
> +	hh->sizeof_tls_array = sizeof(thread->tls_array);
> +	hh->ntls = ntls;
> +
> +	ret = cr_write_obj(ctx, &h, hh);
> +	cr_hbuf_put(ctx, sizeof(*hh));
> +	if (ret < 0)
> +		return ret;
> +
> +	/* for simplicity dump the entire array, cherry-pick upon restart */
> +	ret = cr_kwrite(ctx, thread->tls_array, sizeof(thread->tls_array));
> +
> +	cr_debug("ntls %d\n", ntls);
> +
> +	/* IGNORE RESTART BLOCKS FOR NOW ... */
> +
> +	return ret;
> +}
> +
> +#ifdef CONFIG_X86_64
> +
> +#error "CONFIG_X86_64 unsupported yet."
> +
> +#else	/* !CONFIG_X86_64 */
> +
> +void cr_write_cpu_regs(struct cr_hdr_cpu *hh, struct task_struct *t)
> +{
> +	struct thread_struct *thread = &t->thread;
> +	struct pt_regs *regs = task_pt_regs(t);
> +
> +	hh->bp = regs->bp;
> +	hh->bx = regs->bx;
> +	hh->ax = regs->ax;
> +	hh->cx = regs->cx;
> +	hh->dx = regs->dx;
> +	hh->si = regs->si;
> +	hh->di = regs->di;
> +	hh->orig_ax = regs->orig_ax;
> +	hh->ip = regs->ip;
> +	hh->cs = regs->cs;
> +	hh->flags = regs->flags;
> +	hh->sp = regs->sp;
> +	hh->ss = regs->ss;
> +
> +	hh->ds = regs->ds;
> +	hh->es = regs->es;
> +
> +	/*
> +	 * for checkpoint in process context (from within a container)
> +	 * the GS and FS registers should be saved from the hardware;
> +	 * otherwise they are already sabed on the thread structure
> +	 */
> +	if (t == current) {
> +		savesegment(gs, hh->gs);
> +		savesegment(fs, hh->fs);
> +	} else {
> +		hh->gs = thread->gs;
> +		hh->fs = thread->fs;
> +	}
> +
> +	/*
> +	 * for checkpoint in process context (from within a container),
> +	 * the actual syscall is taking place at this very moment; so
> +	 * we (optimistically) subtitute the future return value (0) of
> +	 * this syscall into the orig_eax, so that upon restart it will
> +	 * succeed (or it will endlessly retry checkpoint...)
> +	 */
> +	if (t == current) {
> +		BUG_ON(hh->orig_ax < 0);
> +		hh->ax = 0;
> +	}
> +}
> +
> +void cr_write_cpu_debug(struct cr_hdr_cpu *hh, struct task_struct *t)
> +{
> +	struct thread_struct *thread = &t->thread;
> +
> +	/* debug regs */
> +
> +	preempt_disable();
> +
> +	/*
> +	 * for checkpoint in process context (from within a container),
> +	 * get the actual registers; otherwise get the saved values.
> +	 */
> +
> +	if (t == current) {
> +		get_debugreg(hh->debugreg0, 0);
> +		get_debugreg(hh->debugreg1, 1);
> +		get_debugreg(hh->debugreg2, 2);
> +		get_debugreg(hh->debugreg3, 3);
> +		get_debugreg(hh->debugreg6, 6);
> +		get_debugreg(hh->debugreg7, 7);
> +	} else {
> +		hh->debugreg0 = thread->debugreg0;
> +		hh->debugreg1 = thread->debugreg1;
> +		hh->debugreg2 = thread->debugreg2;
> +		hh->debugreg3 = thread->debugreg3;
> +		hh->debugreg6 = thread->debugreg6;
> +		hh->debugreg7 = thread->debugreg7;
> +	}
> +
> +	hh->debugreg4 = 0;
> +	hh->debugreg5 = 0;
> +
> +	hh->uses_debug = !!(task_thread_info(t)->flags & TIF_DEBUG);
> +
> +	preempt_enable();
> +}
> +
> +void cr_write_cpu_fpu(struct cr_hdr_cpu *hh, struct task_struct *t)
> +{
> +	struct thread_struct *thread = &t->thread;
> +	struct thread_info *thread_info = task_thread_info(t);
> +
> +	/* i387 + MMU + SSE logic */
> +
> +	preempt_disable();
> +
> +	hh->used_math = tsk_used_math(t) ? 1 : 0;
> +	if (hh->used_math) {
> +		/*
> +		 * normally, no need to unlazy_fpu(), since TS_USEDFPU flag
> +		 * have been cleared when task was conexted-switched out...
> +		 * except if we are in process context, in which case we do
> +		 */
> +		if (thread_info->status & TS_USEDFPU)
> +			unlazy_fpu(current);
> +
> +		hh->has_fxsr = cpu_has_fxsr;
> +		memcpy(&hh->xstate, &thread->xstate, sizeof(thread->xstate));
> +	}
> +
> +	preempt_enable();
> +}
> +
> +#endif	/* CONFIG_X86_64 */
> +
> +/* dump the cpu state and registers of a given task */
> +int cr_write_cpu(struct cr_ctx *ctx, struct task_struct *t)
> +{
> +	struct cr_hdr h;
> +	struct cr_hdr_cpu *hh = cr_hbuf_get(ctx, sizeof(*hh));
> +	int ret;
> +
> +	h.type = CR_HDR_CPU;
> +	h.len = sizeof(*hh);
> +	h.parent = task_pid_vnr(t);
> +
> +	cr_write_cpu_regs(hh, t);
> +	cr_write_cpu_debug(hh, t);
> +	cr_write_cpu_fpu(hh, t);
> +
> +	cr_debug("math %d debug %d\n", hh->used_math, hh->uses_debug);
> +
> +	ret = cr_write_obj(ctx, &h, hh);
> +	cr_hbuf_put(ctx, sizeof(*hh));
> +	return ret;
> +}
> diff --git a/arch/x86/mm/restart.c b/arch/x86/mm/restart.c
> new file mode 100644
> index 0000000..0178080
> --- /dev/null
> +++ b/arch/x86/mm/restart.c
> @@ -0,0 +1,177 @@
> +/*
> + *  Checkpoint/restart - architecture specific support for x86
> + *
> + *  Copyright (C) 2008 Oren Laadan
> + *
> + *  This file is subject to the terms and conditions of the GNU General Public
> + *  License.  See the file COPYING in the main directory of the Linux
> + *  distribution for more details.
> + */
> +
> +#include <asm/desc.h>
> +#include <asm/i387.h>
> +
> +#include <linux/checkpoint.h>
> +#include <linux/checkpoint_hdr.h>
> +
> +/* read the thread_struct into the current task */
> +int cr_read_thread(struct cr_ctx *ctx)
> +{
> +	struct cr_hdr_thread *hh = cr_hbuf_get(ctx, sizeof(*hh));
> +	struct task_struct *t = current;
> +	struct thread_struct *thread = &t->thread;
> +	int parent;
> +
> +	parent = cr_read_obj_type(ctx, hh, sizeof(*hh), CR_HDR_THREAD);
> +	if (parent < 0)
> +		return parent;
> +#if 0	/* activate when containers are used */
> +	if (parent != task_pid_vnr(t))
> +		return -EINVAL;
> +#endif
> +	cr_debug("ntls %d\n", hh->ntls);
> +
> +	if (hh->gdt_entry_tls_entries != GDT_ENTRY_TLS_ENTRIES ||
> +	    hh->sizeof_tls_array != sizeof(thread->tls_array) ||
> +	    hh->ntls < 0 || hh->ntls > GDT_ENTRY_TLS_ENTRIES)
> +		return -EINVAL;
> +
> +	if (hh->ntls > 0) {
> +		struct desc_struct *desc;
> +		int size, cpu, ret;
> +
> +		/*
> +		 * restore TLS by hand: why convert to struct user_desc if
> +		 * sys_set_thread_entry() will convert it back ?
> +		 */
> +
> +		size = sizeof(*desc) * GDT_ENTRY_TLS_ENTRIES;
> +		desc = kmalloc(size, GFP_KERNEL);
> +		if (!desc)
> +			return -ENOMEM;
> +
> +		ret = cr_kread(ctx, desc, size);
> +		if (ret >= 0) {
> +			/*
> +			 * FIX: add sanity checks (eg. that values makes
> +			 * sense, that we don't overwrite old values, etc
> +			 */
> +			cpu = get_cpu();
> +			memcpy(thread->tls_array, desc, size);
> +			load_TLS(thread, cpu);
> +			put_cpu();
> +		}
> +		kfree(desc);
> +	}
> +
> +	return 0;
> +}
> +
> +#ifdef CONFIG_X86_64
> +
> +#error "CONFIG_X86_64 unsupported yet."
> +
> +#else	/* !CONFIG_X86_64 */
> +
> +int cr_read_cpu_regs(struct cr_hdr_cpu *hh, struct task_struct *t)
> +{
> +	struct thread_struct *thread = &t->thread;
> +	struct pt_regs *regs = task_pt_regs(t);
> +
> +	regs->bx = hh->bx;
> +	regs->cx = hh->cx;
> +	regs->dx = hh->dx;
> +	regs->si = hh->si;
> +	regs->di = hh->di;
> +	regs->bp = hh->bp;
> +	regs->ax = hh->ax;
> +	regs->ds = hh->ds;
> +	regs->es = hh->es;
> +	regs->orig_ax = hh->orig_ax;
> +	regs->ip = hh->ip;
> +	regs->cs = hh->cs;
> +	regs->flags = hh->flags;
> +	regs->sp = hh->sp;
> +	regs->ss = hh->ss;
> +
> +	thread->gs = hh->gs;
> +	thread->fs = hh->fs;
> +	loadsegment(gs, hh->gs);
> +	loadsegment(fs, hh->fs);
> +
> +	return 0;
> +}
> +
> +int cr_read_cpu_debug(struct cr_hdr_cpu *hh, struct task_struct *t)
> +{
> +	/* debug regs */
> +
> +	if (hh->uses_debug) {
> +		set_debugreg(hh->debugreg0, 0);
> +		set_debugreg(hh->debugreg1, 1);
> +		/* ignore 4, 5 */
> +		set_debugreg(hh->debugreg2, 2);
> +		set_debugreg(hh->debugreg3, 3);
> +		set_debugreg(hh->debugreg6, 6);
> +		set_debugreg(hh->debugreg7, 7);
> +	}
> +
> +	return 0;
> +}
> +
> +int cr_read_cpu_fpu(struct cr_hdr_cpu *hh, struct task_struct *t)
> +{
> +	struct thread_struct *thread = &t->thread;
> +
> +	/* i387 + MMU + SSE */
> +
> +	preempt_disable();
> +
> +	__clear_fpu(t);		/* in case we used FPU in user mode */
> +
> +	if (!hh->used_math)
> +		clear_used_math();
> +	else {
> +		if (hh->has_fxsr != cpu_has_fxsr) {
> +			force_sig(SIGFPE, t);
> +			return -EINVAL;
> +		}
> +		memcpy(&thread->xstate, &hh->xstate, sizeof(thread->xstate));
> +		set_used_math();
> +	}
> +
> +	preempt_enable();
> +
> +	return 0;
> +}
> +
> +#endif	/* CONFIG_X86_64 */
> +
> +/* read the cpu state and registers for the current task */
> +int cr_read_cpu(struct cr_ctx *ctx)
> +{
> +	struct cr_hdr_cpu *hh = cr_hbuf_get(ctx, sizeof(*hh));
> +	struct task_struct *t = current;
> +	int parent, ret;
> +
> +	parent = cr_read_obj_type(ctx, hh, sizeof(*hh), CR_HDR_CPU);
> +	if (parent < 0)
> +		return parent;
> +#if 0	/* activate when containers are used */
> +	if (parent != task_pid_vnr(t))
> +		return -EINVAL;
> +#endif
> +	/* FIX: sanity check for sensitive registers (eg. eflags) */
> +
> +	ret = cr_read_cpu_regs(hh, t);
> +	if (ret < 0)
> +		goto out;
> +	ret = cr_read_cpu_debug(hh, t);
> +	if (ret < 0)
> +		goto out;
> +	ret = cr_read_cpu_fpu(hh, t);
> +
> +	cr_debug("math %d debug %d\n", hh->used_math, hh->uses_debug);
> + out:
> +	return ret;
> +}
> diff --git a/checkpoint/checkpoint.c b/checkpoint/checkpoint.c
> index e5e188f..6ca26d0 100644
> --- a/checkpoint/checkpoint.c
> +++ b/checkpoint/checkpoint.c
> @@ -20,6 +20,8 @@
>  #include <linux/checkpoint.h>
>  #include <linux/checkpoint_hdr.h>
> 
> +#include "checkpoint_arch.h"
> +
>  /**
>   * cr_write_obj - write a record described by a cr_hdr
>   * @ctx: checkpoint context
> @@ -145,8 +147,17 @@ static int cr_write_task(struct cr_ctx *ctx, struct task_struct *t)
>  	}
> 
>  	ret = cr_write_task_struct(ctx, t);
> -	cr_debug("ret %d\n", ret);
> +	cr_debug("task_struct: ret %d\n", ret);
> +	if (ret < 0)
> +		goto out;
> +	ret = cr_write_thread(ctx, t);
> +	cr_debug("thread: ret %d\n", ret);
> +	if (ret < 0)
> +		goto out;
> +	ret = cr_write_cpu(ctx, t);
> +	cr_debug("cpu: ret %d\n", ret);
> 
> + out:
>  	return ret;
>  }
> 
> diff --git a/checkpoint/checkpoint_arch.h b/checkpoint/checkpoint_arch.h
> new file mode 100644
> index 0000000..bf2d21e
> --- /dev/null
> +++ b/checkpoint/checkpoint_arch.h
> @@ -0,0 +1,7 @@
> +#include <linux/checkpoint.h>
> +
> +extern int cr_write_thread(struct cr_ctx *ctx, struct task_struct *t);
> +extern int cr_write_cpu(struct cr_ctx *ctx, struct task_struct *t);
> +
> +extern int cr_read_thread(struct cr_ctx *ctx);
> +extern int cr_read_cpu(struct cr_ctx *ctx);
> diff --git a/checkpoint/restart.c b/checkpoint/restart.c
> index ef6bc37..0484de0 100644
> --- a/checkpoint/restart.c
> +++ b/checkpoint/restart.c
> @@ -15,6 +15,8 @@
>  #include <linux/checkpoint.h>
>  #include <linux/checkpoint_hdr.h>
> 
> +#include "checkpoint_arch.h"
> +
>  /**
>   * cr_read_obj - read a whole record (cr_hdr followed by payload)
>   * @ctx: checkpoint context
> @@ -164,8 +166,17 @@ static int cr_read_task(struct cr_ctx *ctx)
>  	int ret;
> 
>  	ret = cr_read_task_struct(ctx);
> -	cr_debug("ret %d\n", ret);
> +	cr_debug("task_struct: ret %d\n", ret);
> +	if (ret < 0)
> +		goto out;
> +	ret = cr_read_thread(ctx);
> +	cr_debug("thread: ret %d\n", ret);
> +	if (ret < 0)
> +		goto out;
> +	ret = cr_read_cpu(ctx);
> +	cr_debug("cpu: ret %d\n", ret);
> 
> + out:
>  	return ret;
>  }
> 
> diff --git a/include/asm-x86/checkpoint_hdr.h b/include/asm-x86/checkpoint_hdr.h
> new file mode 100644
> index 0000000..44a903c
> --- /dev/null
> +++ b/include/asm-x86/checkpoint_hdr.h
> @@ -0,0 +1,72 @@
> +#ifndef __ASM_X86_CKPT_HDR_H
> +#define __ASM_X86_CKPT_HDR_H
> +/*
> + *  Checkpoint/restart - architecture specific headers x86
> + *
> + *  Copyright (C) 2008 Oren Laadan
> + *
> + *  This file is subject to the terms and conditions of the GNU General Public
> + *  License.  See the file COPYING in the main directory of the Linux
> + *  distribution for more details.
> + */
> +
> +#include <asm/processor.h>
> +
> +struct cr_hdr_thread {
> +	/* NEED: restart blocks */
> +
> +	__s16 gdt_entry_tls_entries;
> +	__s16 sizeof_tls_array;
> +	__s16 ntls;	/* number of TLS entries to follow */
> +} __attribute__((aligned(8)));
> +
> +struct cr_hdr_cpu {
> +	/* see struct pt_regs (x86-64) */
> +	__u64 r15;
> +	__u64 r14;
> +	__u64 r13;
> +	__u64 r12;
> +	__u64 bp;
> +	__u64 bx;
> +	__u64 r11;
> +	__u64 r10;
> +	__u64 r9;
> +	__u64 r8;
> +	__u64 ax;
> +	__u64 cx;
> +	__u64 dx;
> +	__u64 si;
> +	__u64 di;
> +	__u64 orig_ax;
> +	__u64 ip;
> +	__u64 cs;
> +	__u64 flags;
> +	__u64 sp;
> +	__u64 ss;
> +
> +	/* segment registers */
> +	__u64 ds;
> +	__u64 es;
> +	__u64 fs;
> +	__u64 gs;
> +
> +	/* debug registers */
> +	__u64 debugreg0;
> +	__u64 debugreg1;
> +	__u64 debugreg2;
> +	__u64 debugreg3;
> +	__u64 debugreg4;
> +	__u64 debugreg5;
> +	__u64 debugreg6;
> +	__u64 debugreg7;
> +
> +	__u16 uses_debug;
> +	__u16 used_math;
> +	__u16 has_fxsr;
> +	__u16 _padding;
> +
> +	union thread_xstate xstate;	/* i387 */
> +
> +} __attribute__((aligned(8)));
> +
> +#endif /* __ASM_X86_CKPT_HDR__H */
> diff --git a/include/linux/checkpoint_hdr.h b/include/linux/checkpoint_hdr.h
> index 79e4df2..03ec72e 100644
> --- a/include/linux/checkpoint_hdr.h
> +++ b/include/linux/checkpoint_hdr.h
> @@ -12,6 +12,7 @@
> 
>  #include <linux/types.h>
>  #include <linux/utsname.h>
> +#include <asm/checkpoint_hdr.h>
> 
>  /*
>   * To maintain compatibility between 32-bit and 64-bit architecture flavors,
> -- 
> 1.5.4.3
> 
> _______________________________________________
> Containers mailing list
> Containers-cunTk1MwBs9QetFLy7KEm3xJsTq8ys+cHZ5vskTnxNA@public.gmane.org
> https://lists.linux-foundation.org/mailman/listinfo/containers

[RFC v5][PATCH 4/8] Dump memory address space

[Oren Laadan]

For each VMA, there is a 'struct cr_vma'; if the VMA is file-mapped,
it will be followed by the file name. Then comes the actual contents,
in one or more chunk: each chunk begins with a header that specifies
how many pages it holds, then the virtual addresses of all the dumped
pages in that chunk, followed by the actual contents of all dumped
pages. A header with zero number of pages marks the end of the contents.
Then comes the next VMA and so on.

Signed-off-by: Oren Laadan <orenl-eQaUEPhvms7ENvBUuze7eA@public.gmane.org>
---
 arch/x86/mm/checkpoint.c         |   30 +++
 arch/x86/mm/restart.c            |    1 +
 checkpoint/Makefile              |    3 +-
 checkpoint/checkpoint.c          |   53 ++++
 checkpoint/checkpoint_arch.h     |    2 +
 checkpoint/checkpoint_mem.h      |   41 ++++
 checkpoint/ckpt_mem.c            |  492 ++++++++++++++++++++++++++++++++++++++
 checkpoint/sys.c                 |   27 ++-
 include/asm-x86/checkpoint_hdr.h |    5 +
 include/linux/checkpoint.h       |   12 +
 include/linux/checkpoint_hdr.h   |   32 +++
 11 files changed, 692 insertions(+), 6 deletions(-)
 create mode 100644 checkpoint/checkpoint_mem.h
 create mode 100644 checkpoint/ckpt_mem.c

diff --git a/arch/x86/mm/checkpoint.c b/arch/x86/mm/checkpoint.c
index eb60003..fa87e4a 100644
--- a/arch/x86/mm/checkpoint.c
+++ b/arch/x86/mm/checkpoint.c
@@ -196,3 +196,33 @@ int cr_write_cpu(struct cr_ctx *ctx, struct task_struct *t)
 	cr_hbuf_put(ctx, sizeof(*hh));
 	return ret;
 }
+
+/* dump the mm->context state */
+int cr_write_mm_context(struct cr_ctx *ctx, struct mm_struct *mm, int parent)
+{
+	struct cr_hdr h;
+	struct cr_hdr_mm_context *hh = cr_hbuf_get(ctx, sizeof(*hh));
+	int ret;
+
+	h.type = CR_HDR_MM_CONTEXT;
+	h.len = sizeof(*hh);
+	h.parent = parent;
+
+	mutex_lock(&mm->context.lock);
+
+	hh->ldt_entry_size = LDT_ENTRY_SIZE;
+	hh->nldt = mm->context.size;
+
+	cr_debug("nldt %d\n", hh->nldt);
+
+	ret = cr_write_obj(ctx, &h, hh);
+	cr_hbuf_put(ctx, sizeof(*hh));
+	if (ret < 0)
+		goto out;
+
+	ret = cr_kwrite(ctx, mm->context.ldt, hh->nldt * LDT_ENTRY_SIZE);
+
+ out:
+	mutex_unlock(&mm->context.lock);
+	return ret;
+}
diff --git a/arch/x86/mm/restart.c b/arch/x86/mm/restart.c
index 0178080..79780df 100644
--- a/arch/x86/mm/restart.c
+++ b/arch/x86/mm/restart.c
@@ -8,6 +8,7 @@
  *  distribution for more details.
  */
 
+#include <linux/unistd.h>
 #include <asm/desc.h>
 #include <asm/i387.h>
 
diff --git a/checkpoint/Makefile b/checkpoint/Makefile
index d2df68c..3a0df6d 100644
--- a/checkpoint/Makefile
+++ b/checkpoint/Makefile
@@ -2,4 +2,5 @@
 # Makefile for linux checkpoint/restart.
 #
 
-obj-$(CONFIG_CHECKPOINT_RESTART) += sys.o checkpoint.o restart.o
+obj-$(CONFIG_CHECKPOINT_RESTART) += sys.o checkpoint.o restart.o \
+		ckpt_mem.o
diff --git a/checkpoint/checkpoint.c b/checkpoint/checkpoint.c
index 6ca26d0..d4c1b31 100644
--- a/checkpoint/checkpoint.c
+++ b/checkpoint/checkpoint.c
@@ -55,6 +55,55 @@ int cr_write_string(struct cr_ctx *ctx, char *str, int len)
 	return cr_write_obj(ctx, &h, str);
 }
 
+/**
+ * cr_fill_fname - return pathname of a given file
+ * @path: path name
+ * @root: relative root
+ * @buf: buffer for pathname
+ * @n: buffer length (in) and pathname length (out)
+ */
+static char *
+cr_fill_fname(struct path *path, struct path *root, char *buf, int *n)
+{
+	char *fname;
+
+	BUG_ON(!buf);
+	fname = __d_path(path, root, buf, *n);
+	if (!IS_ERR(fname))
+		*n = (buf + (*n) - fname);
+	return fname;
+}
+
+/**
+ * cr_write_fname - write a file name
+ * @ctx: checkpoint context
+ * @path: path name
+ * @root: relative root
+ */
+int cr_write_fname(struct cr_ctx *ctx, struct path *path, struct path *root)
+{
+	struct cr_hdr h;
+	char *buf, *fname;
+	int ret, flen;
+
+	flen = PATH_MAX;
+	buf = kmalloc(flen, GFP_KERNEL);
+	if (!buf)
+		return -ENOMEM;
+
+	fname = cr_fill_fname(path, root, buf, &flen);
+	if (!IS_ERR(fname)) {
+		h.type = CR_HDR_FNAME;
+		h.len = flen;
+		h.parent = 0;
+		ret = cr_write_obj(ctx, &h, fname);
+	} else
+		ret = PTR_ERR(fname);
+
+	kfree(buf);
+	return ret;
+}
+
 /* write the checkpoint header */
 static int cr_write_head(struct cr_ctx *ctx)
 {
@@ -150,6 +199,10 @@ static int cr_write_task(struct cr_ctx *ctx, struct task_struct *t)
 	cr_debug("task_struct: ret %d\n", ret);
 	if (ret < 0)
 		goto out;
+	ret = cr_write_mm(ctx, t);
+	cr_debug("memory: ret %d\n", ret);
+	if (ret < 0)
+		goto out;
 	ret = cr_write_thread(ctx, t);
 	cr_debug("thread: ret %d\n", ret);
 	if (ret < 0)
diff --git a/checkpoint/checkpoint_arch.h b/checkpoint/checkpoint_arch.h
index bf2d21e..7da4ad0 100644
--- a/checkpoint/checkpoint_arch.h
+++ b/checkpoint/checkpoint_arch.h
@@ -2,6 +2,8 @@
 
 extern int cr_write_thread(struct cr_ctx *ctx, struct task_struct *t);
 extern int cr_write_cpu(struct cr_ctx *ctx, struct task_struct *t);
+extern int cr_write_mm_context(struct cr_ctx *ctx,
+			       struct mm_struct *mm, int parent);
 
 extern int cr_read_thread(struct cr_ctx *ctx);
 extern int cr_read_cpu(struct cr_ctx *ctx);
diff --git a/checkpoint/checkpoint_mem.h b/checkpoint/checkpoint_mem.h
new file mode 100644
index 0000000..85546f4
--- /dev/null
+++ b/checkpoint/checkpoint_mem.h
@@ -0,0 +1,41 @@
+#ifndef _CHECKPOINT_CKPT_MEM_H_
+#define _CHECKPOINT_CKPT_MEM_H_
+/*
+ *  Generic container checkpoint-restart
+ *
+ *  Copyright (C) 2008 Oren Laadan
+ *
+ *  This file is subject to the terms and conditions of the GNU General Public
+ *  License.  See the file COPYING in the main directory of the Linux
+ *  distribution for more details.
+ */
+
+#include <linux/mm_types.h>
+
+/*
+ * page-array chains: each cr_pgarr describes a set of <strcut page *,vaddr>
+ * tuples (where vaddr is the virtual address of a page in a particular mm).
+ * Specifically, we use separate arrays so that all vaddrs can be written
+ * and read at once.
+ */
+
+struct cr_pgarr {
+	unsigned long *vaddrs;
+	struct page **pages;
+	unsigned int nr_used;
+	struct list_head list;
+};
+
+#define CR_PGARR_TOTAL  (PAGE_SIZE / sizeof(void *))
+#define CR_PGARR_CHUNK  (4 * CR_PGARR_TOTAL)
+
+extern void cr_pgarr_free(struct cr_ctx *ctx);
+extern struct cr_pgarr *cr_pgarr_current(struct cr_ctx *ctx);
+extern void cr_pgarr_reset_all(struct cr_ctx *ctx);
+
+static inline int cr_pgarr_is_full(struct cr_pgarr *pgarr)
+{
+	return (pgarr->nr_used == CR_PGARR_TOTAL);
+}
+
+#endif /* _CHECKPOINT_CKPT_MEM_H_ */
diff --git a/checkpoint/ckpt_mem.c b/checkpoint/ckpt_mem.c
new file mode 100644
index 0000000..e0226f4
--- /dev/null
+++ b/checkpoint/ckpt_mem.c
@@ -0,0 +1,492 @@
+/*
+ *  Checkpoint memory contents
+ *
+ *  Copyright (C) 2008 Oren Laadan
+ *
+ *  This file is subject to the terms and conditions of the GNU General Public
+ *  License.  See the file COPYING in the main directory of the Linux
+ *  distribution for more details.
+ */
+
+#include <linux/kernel.h>
+#include <linux/sched.h>
+#include <linux/slab.h>
+#include <linux/file.h>
+#include <linux/pagemap.h>
+#include <linux/mm_types.h>
+#include <linux/checkpoint.h>
+#include <linux/checkpoint_hdr.h>
+
+#include "checkpoint_arch.h"
+#include "checkpoint_mem.h"
+
+/*
+ * utilities to alloc, free, and handle 'struct cr_pgarr' (page-arrays)
+ * (common to ckpt_mem.c and rstr_mem.c).
+ *
+ * The checkpoint context structure has two members for page-arrays:
+ *   ctx->pgarr_list: list head of the page-array chain
+ *
+ * During checkpoint (and restart) the chain tracks the dirty pages (page
+ * pointer and virtual address) of each MM. For a particular MM, these are
+ * always added to the head of the page-array chain (ctx->pgarr_list).
+ * This "current" page-array advances as necessary, and new page-array
+ * descriptors are allocated on-demand. Before the next chunk of pages,
+ * the chain is reset but not freed (that is, dereference page pointers).
+ */
+
+/* return first page-array in the chain */
+static inline struct cr_pgarr *cr_pgarr_first(struct cr_ctx *ctx)
+{
+	if (list_empty(&ctx->pgarr_list))
+		return NULL;
+	return list_first_entry(&ctx->pgarr_list, struct cr_pgarr, list);
+}
+
+/* release pages referenced by a page-array */
+static void cr_pgarr_release_pages(struct cr_pgarr *pgarr)
+{
+	int i;
+
+	cr_debug("nr_used %d\n", pgarr->nr_used);
+	/*
+	 * although both checkpoint and restart use 'nr_used', we only
+	 * collect pages during checkpoint; in restart we simply return
+	 */
+	if (!pgarr->pages)
+		return;
+	for (i = pgarr->nr_used; i--; /**/)
+		page_cache_release(pgarr->pages[i]);
+}
+
+/* free a single page-array object */
+static void cr_pgarr_free_one(struct cr_pgarr *pgarr)
+{
+	cr_pgarr_release_pages(pgarr);
+	kfree(pgarr->pages);
+	kfree(pgarr->vaddrs);
+	kfree(pgarr);
+}
+
+/* free a chain of page-arrays */
+void cr_pgarr_free(struct cr_ctx *ctx)
+{
+	struct cr_pgarr *pgarr, *tmp;
+
+	list_for_each_entry_safe(pgarr, tmp, &ctx->pgarr_list, list) {
+		list_del(&pgarr->list);
+		cr_pgarr_free_one(pgarr);
+	}
+}
+
+/* allocate a single page-array object */
+static struct cr_pgarr *cr_pgarr_alloc_one(unsigned long flags)
+{
+	struct cr_pgarr *pgarr;
+
+	pgarr = kzalloc(sizeof(*pgarr), GFP_KERNEL);
+	if (!pgarr)
+		return NULL;
+
+	pgarr->vaddrs = kmalloc(CR_PGARR_TOTAL * sizeof(unsigned long),
+			       GFP_KERNEL);
+	if (!pgarr->vaddrs)
+		goto nomem;
+
+	/* pgarr->pages is needed only for checkpoint */
+	if (flags & CR_CTX_CKPT) {
+		pgarr->pages = kmalloc(CR_PGARR_TOTAL * sizeof(struct page *),
+				       GFP_KERNEL);
+		if (!pgarr->pages)
+			goto nomem;
+	}
+
+	return pgarr;
+
+ nomem:
+	cr_pgarr_free_one(pgarr);
+	return NULL;
+}
+
+/* cr_pgarr_current - return the next available page-array in the chain
+ * @ctx: checkpoint context
+ *
+ * Returns the first page-array in the list that has space. Extends the
+ * list if none has space.
+ */
+struct cr_pgarr *cr_pgarr_current(struct cr_ctx *ctx)
+{
+	struct cr_pgarr *pgarr;
+
+	pgarr = cr_pgarr_first(ctx);
+	if (pgarr && !cr_pgarr_is_full(pgarr))
+		goto out;
+	pgarr = cr_pgarr_alloc_one(ctx->flags);
+	if (!pgarr)
+		goto out;
+	list_add(&pgarr->list, &ctx->pgarr_list);
+ out:
+	return pgarr;
+}
+
+/* reset the page-array chain (dropping page references if necessary) */
+void cr_pgarr_reset_all(struct cr_ctx *ctx)
+{
+	struct cr_pgarr *pgarr;
+
+	list_for_each_entry(pgarr, &ctx->pgarr_list, list) {
+		cr_pgarr_release_pages(pgarr);
+		pgarr->nr_used = 0;
+	}
+}
+
+/*
+ * Checkpoint is outside the context of the checkpointee, so one cannot
+ * simply read pages from user-space. Instead, we scan the address space
+ * of the target to cherry-pick pages of interest. Selected pages are
+ * enlisted in a page-array chain (attached to the checkpoint context).
+ * To save their contents, each page is mapped to kernel memory and then
+ * dumped to the file descriptor.
+ */
+
+
+/**
+ * cr_follow_page - return page pointer for pages that were modified
+ * @vma - target vma
+ * @addr - page address
+ *
+ * Looks up the page that correspond to the address in the vma, and
+ * returns the page if it was modified (and grabs a reference to it),
+ * or otherwise returns NULL (or error).
+ */
+static struct page *
+cr_follow_page(struct vm_area_struct *vma, unsigned long addr)
+{
+	struct page *page;
+
+	/*
+	 * simplified version of get_user_pages(): already have vma,
+	 * only need FOLL_ANON, and (for now) ignore fault stats.
+	 *
+	 * follow_page() will return NULL if the page is not present
+	 * (swapped), ZERO_PAGE(0) if the pte wasn't allocated, and
+	 * the actual page pointer otherwise.
+	 *
+	 * FIXME: consolidate with get_user_pages()
+	 */
+
+	cond_resched();
+	while (!(page = follow_page(vma, addr, FOLL_ANON | FOLL_GET))) {
+		int ret;
+
+		/* the page is swapped out - bring it in (optimize ?) */
+		ret = handle_mm_fault(vma->vm_mm, vma, addr, 0);
+		if (ret & VM_FAULT_ERROR) {
+			if (ret & VM_FAULT_OOM)
+				return ERR_PTR(-ENOMEM);
+			else if (ret & VM_FAULT_SIGBUS)
+				return ERR_PTR(-EFAULT);
+			else
+				BUG();
+			break;
+		}
+		cond_resched();
+	}
+
+	if (IS_ERR(page))
+		return page;
+
+	if (page == ZERO_PAGE(0)) {
+		/* this is the zero page: ignore */
+		page_cache_release(page);
+		page = NULL;
+	} else if (vma->vm_file && (page_mapping(page) != NULL)) {
+		/* file backed clean cow: ignore */
+		page_cache_release(page);
+		page = NULL;
+	}
+
+	return page;
+}
+
+/**
+ * cr_private_vma_fill_pgarr - fill a page-array with addr/page tuples
+ * @ctx - checkpoint context
+ * @pgarr - page-array to fill
+ * @vma - vma to scan
+ * @start - start address (updated)
+ *
+ * Returns the number of pages collected
+ */
+static int
+cr_private_vma_fill_pgarr(struct cr_ctx *ctx, struct cr_pgarr *pgarr,
+			  struct vm_area_struct *vma, unsigned long *start)
+{
+	unsigned long end = vma->vm_end;
+	unsigned long addr = *start;
+	int orig_used = pgarr->nr_used;
+
+	/* this function is only for private memory (anon or file-mapped) */
+	BUG_ON(vma->vm_flags & (VM_SHARED | VM_MAYSHARE));
+
+	while (addr < end) {
+		struct page *page;
+
+		page = cr_follow_page(vma, addr);
+		if (IS_ERR(page))
+			return PTR_ERR(page);
+
+		if (page) {
+			pgarr->pages[pgarr->nr_used] = page;
+			pgarr->vaddrs[pgarr->nr_used] = addr;
+			pgarr->nr_used++;
+		}
+
+		addr += PAGE_SIZE;
+
+		if (cr_pgarr_is_full(pgarr))
+			break;
+	}
+
+	*start = addr;
+	return pgarr->nr_used - orig_used;
+}
+
+/* dump contents of a pages: use kmap_atomic() to avoid TLB flush */
+static int cr_page_write(struct cr_ctx *ctx, struct page *page, char *buf)
+{
+	void *ptr;
+
+	ptr = kmap_atomic(page, KM_USER1);
+	memcpy(buf, ptr, PAGE_SIZE);
+	kunmap_atomic(page, KM_USER1);
+
+	return cr_kwrite(ctx, buf, PAGE_SIZE);
+}
+
+/**
+ * cr_vma_dump_pages - dump pages listed in the ctx page-array chain
+ * @ctx - checkpoint context
+ * @total - total number of pages
+ *
+ * First dump all virtual addresses, followed by the contents of all pages
+ */
+static int cr_vma_dump_pages(struct cr_ctx *ctx, int total)
+{
+	struct cr_pgarr *pgarr;
+	char *buf;
+	int i, ret = 0;
+
+	if (!total)
+		return 0;
+
+	list_for_each_entry_reverse(pgarr, &ctx->pgarr_list, list) {
+		ret = cr_kwrite(ctx, pgarr->vaddrs,
+				pgarr->nr_used * sizeof(*pgarr->vaddrs));
+		if (ret < 0)
+			return ret;
+	}
+
+	buf = kmalloc(PAGE_SIZE, GFP_KERNEL);
+	if (!buf)
+		return -ENOMEM;
+
+	list_for_each_entry_reverse(pgarr, &ctx->pgarr_list, list) {
+		for (i = 0; i < pgarr->nr_used; i++) {
+			ret = cr_page_write(ctx, pgarr->pages[i], buf);
+			if (ret < 0)
+				goto out;
+		}
+	}
+
+ out:
+	kfree(buf);
+	return ret;
+}
+
+/**
+ * cr_write_private_vma_contents - dump contents of a VMA with private memory
+ * @ctx - checkpoint context
+ * @vma - vma to scan
+ *
+ * Collect lists of pages that needs to be dumped, and corresponding
+ * virtual addresses into ctx->pgarr_list page-array chain. Then dump
+ * the addresses, followed by the page contents.
+ */
+static int
+cr_write_private_vma_contents(struct cr_ctx *ctx, struct vm_area_struct *vma)
+{
+	struct cr_hdr h;
+	struct cr_hdr_pgarr *hh;
+	unsigned long addr = vma->vm_start;
+	struct cr_pgarr *pgarr;
+	unsigned long cnt = 0;
+	int ret;
+
+	/*
+	 * Work iteratively, collecting and dumping at most CR_PGARR_CHUNK
+	 * in each round. Each iterations is divided into two steps:
+	 *
+	 * (1) scan: scan through the PTEs of the vma to collect the pages
+	 * to dump (later we'll also make them COW), while keeping a list
+	 * of pages and their corresponding addresses on ctx->pgarr_list.
+	 *
+	 * (2) dump: write out a header specifying how many pages, followed
+	 * by the addresses of all pages in ctx->pgarr_list, followed by
+	 * the actual contents of all pages. (Then, release the references
+	 * to the pages and reset the page-array chain).
+	 *
+	 * (This split makes the logic simpler by first counting the pages
+	 * that need saving. More importantly, it allows for a future
+	 * optimization that will reduce application downtime by deferring
+	 * the actual write-out of the data to after the application is
+	 * allowed to resume execution).
+	 *
+	 * After dumpting the entire contents, conclude with a header that
+	 * specifies 0 pages to mark the end of the contents.
+	 */
+
+	h.type = CR_HDR_PGARR;
+	h.len = sizeof(*hh);
+	h.parent = 0;
+
+	while (addr < vma->vm_end) {
+		pgarr = cr_pgarr_current(ctx);
+		if (!pgarr)
+			return -ENOMEM;
+		ret = cr_private_vma_fill_pgarr(ctx, pgarr, vma, &addr);
+		if (ret < 0)
+			return ret;
+		cnt += ret;
+
+		/* did we complete a chunk, or is this the last chunk ? */
+		if (cnt >= CR_PGARR_CHUNK || (cnt && addr == vma->vm_end)) {
+			hh = cr_hbuf_get(ctx, sizeof(*hh));
+			hh->nr_pages = cnt;
+			ret = cr_write_obj(ctx, &h, hh);
+			cr_hbuf_put(ctx, sizeof(*hh));
+			if (ret < 0)
+				return ret;
+
+			ret = cr_vma_dump_pages(ctx, cnt);
+			if (ret < 0)
+				return ret;
+
+			cr_pgarr_reset_all(ctx);
+		}
+	}
+
+	/* mark end of contents with header saying "0" pages */
+	hh = cr_hbuf_get(ctx, sizeof(*hh));
+	hh->nr_pages = 0;
+	ret = cr_write_obj(ctx, &h, hh);
+	cr_hbuf_put(ctx, sizeof(*hh));
+
+	return ret;
+}
+
+static int cr_write_vma(struct cr_ctx *ctx, struct vm_area_struct *vma)
+{
+	struct cr_hdr h;
+	struct cr_hdr_vma *hh = cr_hbuf_get(ctx, sizeof(*hh));
+	int vma_type, ret;
+
+	h.type = CR_HDR_VMA;
+	h.len = sizeof(*hh);
+	h.parent = 0;
+
+	hh->vm_start = vma->vm_start;
+	hh->vm_end = vma->vm_end;
+	hh->vm_page_prot = vma->vm_page_prot.pgprot;
+	hh->vm_flags = vma->vm_flags;
+	hh->vm_pgoff = vma->vm_pgoff;
+
+	if (vma->vm_flags & (VM_SHARED | VM_IO | VM_HUGETLB | VM_NONLINEAR)) {
+		pr_warning("CR: unsupported VMA %#lx\n", vma->vm_flags);
+		return -ETXTBSY;
+	}
+
+	/* by default assume anon memory */
+	vma_type = CR_VMA_ANON;
+
+	/*
+	 * if there is a backing file, assume private-mapped
+	 * (FIXME: check if the file is unlinked)
+	 */
+	if (vma->vm_file)
+		vma_type = CR_VMA_FILE;
+
+	hh->vma_type = vma_type;
+
+	ret = cr_write_obj(ctx, &h, hh);
+	cr_hbuf_put(ctx, sizeof(*hh));
+
+	if (ret < 0)
+		return ret;
+
+	/* save the file name, if relevant */
+	if (vma->vm_file) {
+		ret = cr_write_fname(ctx, &vma->vm_file->f_path, ctx->vfsroot);
+		if (ret < 0)
+			return ret;
+	}
+
+	ret = cr_write_private_vma_contents(ctx, vma);
+
+	return ret;
+}
+
+int cr_write_mm(struct cr_ctx *ctx, struct task_struct *t)
+{
+	struct cr_hdr h;
+	struct cr_hdr_mm *hh = cr_hbuf_get(ctx, sizeof(*hh));
+	struct mm_struct *mm;
+	struct vm_area_struct *vma;
+	int objref, ret;
+
+	h.type = CR_HDR_MM;
+	h.len = sizeof(*hh);
+	h.parent = task_pid_vnr(t);
+
+	mm = get_task_mm(t);
+
+	objref = 0;	/* will be meaningful with multiple processes */
+	hh->objref = objref;
+
+	down_read(&mm->mmap_sem);
+
+	hh->start_code = mm->start_code;
+	hh->end_code = mm->end_code;
+	hh->start_data = mm->start_data;
+	hh->end_data = mm->end_data;
+	hh->start_brk = mm->start_brk;
+	hh->brk = mm->brk;
+	hh->start_stack = mm->start_stack;
+	hh->arg_start = mm->arg_start;
+	hh->arg_end = mm->arg_end;
+	hh->env_start = mm->env_start;
+	hh->env_end = mm->env_end;
+
+	hh->map_count = mm->map_count;
+
+	/* FIX: need also mm->flags */
+
+	ret = cr_write_obj(ctx, &h, hh);
+	cr_hbuf_put(ctx, sizeof(*hh));
+	if (ret < 0)
+		goto out;
+
+	/* write the vma's */
+	for (vma = mm->mmap; vma; vma = vma->vm_next) {
+		ret = cr_write_vma(ctx, vma);
+		if (ret < 0)
+			goto out;
+	}
+
+	ret = cr_write_mm_context(ctx, mm, objref);
+
+ out:
+	up_read(&mm->mmap_sem);
+	mmput(mm);
+	return ret;
+}
diff --git a/checkpoint/sys.c b/checkpoint/sys.c
index 30863c6..c4ac157 100644
--- a/checkpoint/sys.c
+++ b/checkpoint/sys.c
@@ -16,6 +16,8 @@
 #include <linux/capability.h>
 #include <linux/checkpoint.h>
 
+#include "checkpoint_mem.h"
+
 /*
  * helpers to write/read to/from the image file descriptor
  *
@@ -111,7 +113,6 @@ int cr_kread(struct cr_ctx *ctx, void *buf, int count)
 	return ret;
 }
 
-
 /*
  * helpers to manage CR contexts: allocated for each checkpoint and/or
  * restart operation, and persists until the operation is completed.
@@ -127,6 +128,11 @@ void cr_ctx_free(struct cr_ctx *ctx)
 
 	free_pages((unsigned long) ctx->hbuf, CR_HBUF_ORDER);
 
+	if (ctx->vfsroot)
+		path_put(ctx->vfsroot);
+
+	cr_pgarr_free(ctx);
+
 	kfree(ctx);
 }
 
@@ -145,10 +151,17 @@ struct cr_ctx *cr_ctx_alloc(pid_t pid, int fd, unsigned long flags)
 	}
 
 	ctx->hbuf = (void *) __get_free_pages(GFP_KERNEL, CR_HBUF_ORDER);
-	if (!ctx->hbuf) {
-		cr_ctx_free(ctx);
-		return ERR_PTR(-ENOMEM);
-	}
+	if (!ctx->hbuf)
+		goto nomem;
+
+	/*
+	 * assume checkpointer is in container's root vfs
+	 * FIXME: this works for now, but will change with real containers
+	 */
+	ctx->vfsroot = &current->fs->root;
+	path_get(ctx->vfsroot);
+
+	INIT_LIST_HEAD(&ctx->pgarr_list);
 
 	ctx->pid = pid;
 	ctx->flags = flags;
@@ -156,6 +169,10 @@ struct cr_ctx *cr_ctx_alloc(pid_t pid, int fd, unsigned long flags)
 	ctx->crid = atomic_inc_return(&cr_ctx_count);
 
 	return ctx;
+
+ nomem:
+	cr_ctx_free(ctx);
+	return ERR_PTR(-ENOMEM);
 }
 
 /*
diff --git a/include/asm-x86/checkpoint_hdr.h b/include/asm-x86/checkpoint_hdr.h
index 44a903c..6bc61ac 100644
--- a/include/asm-x86/checkpoint_hdr.h
+++ b/include/asm-x86/checkpoint_hdr.h
@@ -69,4 +69,9 @@ struct cr_hdr_cpu {
 
 } __attribute__((aligned(8)));
 
+struct cr_hdr_mm_context {
+	__s16 ldt_entry_size;
+	__s16 nldt;
+} __attribute__((aligned(8)));
+
 #endif /* __ASM_X86_CKPT_HDR__H */
diff --git a/include/linux/checkpoint.h b/include/linux/checkpoint.h
index 5e53ae6..d74e64d 100644
--- a/include/linux/checkpoint.h
+++ b/include/linux/checkpoint.h
@@ -10,6 +10,9 @@
  *  distribution for more details.
  */
 
+#include <linux/path.h>
+#include <linux/fs.h>
+
 #define CR_VERSION  1
 
 struct cr_ctx {
@@ -24,6 +27,10 @@ struct cr_ctx {
 
 	void *hbuf;		/* temporary buffer for headers */
 	int hpos;		/* position in headers buffer */
+
+	struct list_head pgarr_list;	/* page array to dump VMA contents */
+
+	struct path *vfsroot;	/* container root (FIXME) */
 };
 
 /* cr_ctx: flags */
@@ -46,11 +53,16 @@ struct cr_hdr;
 
 extern int cr_write_obj(struct cr_ctx *ctx, struct cr_hdr *h, void *buf);
 extern int cr_write_string(struct cr_ctx *ctx, char *str, int len);
+extern int cr_write_fname(struct cr_ctx *ctx,
+			  struct path *path, struct path *root);
 
 extern int cr_read_obj(struct cr_ctx *ctx, struct cr_hdr *h, void *buf, int n);
 extern int cr_read_obj_type(struct cr_ctx *ctx, void *buf, int n, int type);
 extern int cr_read_string(struct cr_ctx *ctx, void *str, int len);
 
+extern int cr_write_mm(struct cr_ctx *ctx, struct task_struct *t);
+extern int cr_read_mm(struct cr_ctx *ctx);
+
 extern int do_checkpoint(struct cr_ctx *ctx);
 extern int do_restart(struct cr_ctx *ctx);
 
diff --git a/include/linux/checkpoint_hdr.h b/include/linux/checkpoint_hdr.h
index 03ec72e..2b110f1 100644
--- a/include/linux/checkpoint_hdr.h
+++ b/include/linux/checkpoint_hdr.h
@@ -32,6 +32,7 @@ struct cr_hdr {
 enum {
 	CR_HDR_HEAD = 1,
 	CR_HDR_STRING,
+	CR_HDR_FNAME,
 
 	CR_HDR_TASK = 101,
 	CR_HDR_THREAD,
@@ -39,6 +40,7 @@ enum {
 
 	CR_HDR_MM = 201,
 	CR_HDR_VMA,
+	CR_HDR_PGARR,
 	CR_HDR_MM_CONTEXT,
 
 	CR_HDR_TAIL = 5001
@@ -73,4 +75,34 @@ struct cr_hdr_task {
 	__s32 task_comm_len;
 } __attribute__((aligned(8)));
 
+struct cr_hdr_mm {
+	__u32 objref;		/* identifier for shared objects */
+	__u32 map_count;
+
+	__u64 start_code, end_code, start_data, end_data;
+	__u64 start_brk, brk, start_stack;
+	__u64 arg_start, arg_end, env_start, env_end;
+} __attribute__((aligned(8)));
+
+/* vma subtypes */
+enum vm_type {
+	CR_VMA_ANON = 1,
+	CR_VMA_FILE
+};
+
+struct cr_hdr_vma {
+	__u32 vma_type;
+	__u32 _padding;
+
+	__u64 vm_start;
+	__u64 vm_end;
+	__u64 vm_page_prot;
+	__u64 vm_flags;
+	__u64 vm_pgoff;
+} __attribute__((aligned(8)));
+
+struct cr_hdr_pgarr {
+	__u64 nr_pages;		/* number of pages to saved */
+} __attribute__((aligned(8)));
+
 #endif /* _CHECKPOINT_CKPT_HDR_H_ */
-- 
1.5.4.3

[RFC v5][PATCH 5/8] Restore memory address space

[Oren Laadan]

Restoring the memory address space begins with nuking the existing one
of the current process, and then reading the VMA state and contents.
Call do_mmap_pgoffset() for each VMA and then read in the data.

Signed-off-by: Oren Laadan <orenl-eQaUEPhvms7ENvBUuze7eA@public.gmane.org>
---
 arch/x86/mm/restart.c            |   54 ++++++
 checkpoint/Makefile              |    2 +-
 checkpoint/checkpoint_arch.h     |    2 +
 checkpoint/checkpoint_mem.h      |    5 +
 checkpoint/restart.c             |   43 +++++
 checkpoint/rstr_mem.c            |  370 ++++++++++++++++++++++++++++++++++++++
 include/asm-x86/checkpoint_hdr.h |    4 +
 include/linux/checkpoint.h       |    3 +
 8 files changed, 482 insertions(+), 1 deletions(-)
 create mode 100644 checkpoint/rstr_mem.c

diff --git a/arch/x86/mm/restart.c b/arch/x86/mm/restart.c
index 79780df..a0cd5ab 100644
--- a/arch/x86/mm/restart.c
+++ b/arch/x86/mm/restart.c
@@ -176,3 +176,57 @@ int cr_read_cpu(struct cr_ctx *ctx)
  out:
 	return ret;
 }
+
+int cr_read_mm_context(struct cr_ctx *ctx, struct mm_struct *mm, int parent)
+{
+	struct cr_hdr_mm_context *hh = cr_hbuf_get(ctx, sizeof(*hh));
+	int n, rparent;
+
+	rparent = cr_read_obj_type(ctx, hh, sizeof(*hh), CR_HDR_MM_CONTEXT);
+	cr_debug("parent %d rparent %d nldt %d\n", parent, rparent, hh->nldt);
+	if (rparent < 0)
+		return rparent;
+	if (rparent != parent)
+		return -EINVAL;
+
+	if (hh->nldt < 0 || hh->ldt_entry_size != LDT_ENTRY_SIZE)
+		return -EINVAL;
+
+	/*
+	 * to utilize the syscall modify_ldt() we first convert the data
+	 * in the checkpoint image from 'struct desc_struct' to 'struct
+	 * user_desc' with reverse logic of include/asm/desc.h:fill_ldt()
+	 */
+
+	for (n = 0; n < hh->nldt; n++) {
+		struct user_desc info;
+		struct desc_struct desc;
+		mm_segment_t old_fs;
+		int ret;
+
+		ret = cr_kread(ctx, &desc, LDT_ENTRY_SIZE);
+		if (ret < 0)
+			return ret;
+
+		info.entry_number = n;
+		info.base_addr = desc.base0 | (desc.base1 << 16);
+		info.limit = desc.limit0;
+		info.seg_32bit = desc.d;
+		info.contents = desc.type >> 2;
+		info.read_exec_only = (desc.type >> 1) ^ 1;
+		info.limit_in_pages = desc.g;
+		info.seg_not_present = desc.p ^ 1;
+		info.useable = desc.avl;
+
+		old_fs = get_fs();
+		set_fs(get_ds());
+		ret = sys_modify_ldt(1, &info, sizeof(info));
+		set_fs(old_fs);
+
+		if (ret < 0)
+			return ret;
+	}
+
+	cr_hbuf_put(ctx, sizeof(*hh));
+	return 0;
+}
diff --git a/checkpoint/Makefile b/checkpoint/Makefile
index 3a0df6d..ac35033 100644
--- a/checkpoint/Makefile
+++ b/checkpoint/Makefile
@@ -3,4 +3,4 @@
 #
 
 obj-$(CONFIG_CHECKPOINT_RESTART) += sys.o checkpoint.o restart.o \
-		ckpt_mem.o
+		ckpt_mem.o rstr_mem.o
diff --git a/checkpoint/checkpoint_arch.h b/checkpoint/checkpoint_arch.h
index 7da4ad0..018a72e 100644
--- a/checkpoint/checkpoint_arch.h
+++ b/checkpoint/checkpoint_arch.h
@@ -7,3 +7,5 @@ extern int cr_write_mm_context(struct cr_ctx *ctx,
 
 extern int cr_read_thread(struct cr_ctx *ctx);
 extern int cr_read_cpu(struct cr_ctx *ctx);
+extern int cr_read_mm_context(struct cr_ctx *ctx,
+			      struct mm_struct *mm, int parent);
diff --git a/checkpoint/checkpoint_mem.h b/checkpoint/checkpoint_mem.h
index 85546f4..85a5cf3 100644
--- a/checkpoint/checkpoint_mem.h
+++ b/checkpoint/checkpoint_mem.h
@@ -38,4 +38,9 @@ static inline int cr_pgarr_is_full(struct cr_pgarr *pgarr)
 	return (pgarr->nr_used == CR_PGARR_TOTAL);
 }
 
+static inline int cr_pgarr_nr_free(struct cr_pgarr *pgarr)
+{
+	return CR_PGARR_TOTAL - pgarr->nr_used;
+}
+
 #endif /* _CHECKPOINT_CKPT_MEM_H_ */
diff --git a/checkpoint/restart.c b/checkpoint/restart.c
index 0484de0..a0d5e60 100644
--- a/checkpoint/restart.c
+++ b/checkpoint/restart.c
@@ -77,6 +77,45 @@ int cr_read_string(struct cr_ctx *ctx, void *str, int len)
 	return cr_read_obj_type(ctx, str, len, CR_HDR_STRING);
 }
 
+/**
+ * cr_read_fname - read a file name
+ * @ctx: checkpoint context
+ * @fname: buffer
+ * @n: buffer length
+ */
+int cr_read_fname(struct cr_ctx *ctx, void *fname, int flen)
+{
+	return cr_read_obj_type(ctx, fname, flen, CR_HDR_FNAME);
+}
+
+/**
+ * cr_read_open_fname - read a file name and open a file
+ * @ctx: checkpoint context
+ * @flags: file flags
+ * @mode: file mode
+ */
+struct file *cr_read_open_fname(struct cr_ctx *ctx, int flags, int mode)
+{
+	struct file *file;
+	char *fname;
+	int flen, ret;
+
+	flen = PATH_MAX;
+	fname = kmalloc(flen, GFP_KERNEL);
+	if (!fname)
+		return ERR_PTR(-ENOMEM);
+
+	ret = cr_read_fname(ctx, fname, flen);
+	cr_debug("fname '%s' flags %#x mode %#x\n", fname, flags, mode);
+	if (ret >= 0)
+		file = filp_open(fname, flags, mode);
+	else
+		file = ERR_PTR(ret);
+
+	kfree(fname);
+	return file;
+}
+
 /* read the checkpoint header */
 static int cr_read_head(struct cr_ctx *ctx)
 {
@@ -169,6 +208,10 @@ static int cr_read_task(struct cr_ctx *ctx)
 	cr_debug("task_struct: ret %d\n", ret);
 	if (ret < 0)
 		goto out;
+	ret = cr_read_mm(ctx);
+	cr_debug("memory: ret %d\n", ret);
+	if (ret < 0)
+		goto out;
 	ret = cr_read_thread(ctx);
 	cr_debug("thread: ret %d\n", ret);
 	if (ret < 0)
diff --git a/checkpoint/rstr_mem.c b/checkpoint/rstr_mem.c
new file mode 100644
index 0000000..c878a0d
--- /dev/null
+++ b/checkpoint/rstr_mem.c
@@ -0,0 +1,370 @@
+/*
+ *  Restart memory contents
+ *
+ *  Copyright (C) 2008 Oren Laadan
+ *
+ *  This file is subject to the terms and conditions of the GNU General Public
+ *  License.  See the file COPYING in the main directory of the Linux
+ *  distribution for more details.
+ */
+
+#include <linux/kernel.h>
+#include <linux/sched.h>
+#include <linux/fcntl.h>
+#include <linux/file.h>
+#include <linux/fs.h>
+#include <linux/pagemap.h>
+#include <linux/mm_types.h>
+#include <linux/mman.h>
+#include <linux/mm.h>
+#include <linux/err.h>
+#include <linux/checkpoint.h>
+#include <linux/checkpoint_hdr.h>
+
+#include "checkpoint_arch.h"
+#include "checkpoint_mem.h"
+
+/*
+ * Unlike checkpoint, restart is executed in the context of each restarting
+ * process: vma regions are restored via a call to mmap(), and the data is
+ * read into the address space of the current process.
+ */
+
+
+/**
+ * cr_read_pages_vaddrs - read addresses of pages to page-array chain
+ * @ctx - restart context
+ * @nr_pages - number of address to read
+ */
+static int cr_read_pages_vaddrs(struct cr_ctx *ctx, unsigned long nr_pages)
+{
+	struct cr_pgarr *pgarr;
+	unsigned long *vaddrp;
+	int nr, ret;
+
+	while (nr_pages) {
+		pgarr = cr_pgarr_current(ctx);
+		if (!pgarr)
+			return -ENOMEM;
+		nr = cr_pgarr_nr_free(pgarr);
+		if (nr > nr_pages)
+			nr = nr_pages;
+		vaddrp = &pgarr->vaddrs[pgarr->nr_used];
+		ret = cr_kread(ctx, vaddrp, nr * sizeof(unsigned long));
+		if (ret < 0)
+			return ret;
+		pgarr->nr_used += nr;
+		nr_pages -= nr;
+	}
+	return 0;
+}
+
+static int cr_page_read(struct cr_ctx *ctx, struct page *page, char *buf)
+{
+	void *ptr;
+	int ret;
+
+	ret = cr_kread(ctx, buf, PAGE_SIZE);
+	if (ret < 0)
+		return ret;
+
+	ptr = kmap_atomic(page, KM_USER1);
+	memcpy(ptr, buf, PAGE_SIZE);
+	kunmap_atomic(page, KM_USER1);
+
+	return 0;
+}
+
+/**
+ * cr_read_pages_contents - read in data of pages in page-array chain
+ * @ctx - restart context
+ */
+static int cr_read_pages_contents(struct cr_ctx *ctx)
+{
+	struct mm_struct *mm = current->mm;
+	struct cr_pgarr *pgarr;
+	unsigned long *vaddrs;
+	char *buf;
+	int i, ret = 0;
+
+	buf = kmalloc(PAGE_SIZE, GFP_KERNEL);
+	if (!buf)
+		return -ENOMEM;
+
+	down_read(&mm->mmap_sem);
+	list_for_each_entry_reverse(pgarr, &ctx->pgarr_list, list) {
+		vaddrs = pgarr->vaddrs;
+		for (i = 0; i < pgarr->nr_used; i++) {
+			struct page *page;
+
+			ret = get_user_pages(current, mm, vaddrs[i],
+					     1, 1, 1, &page, NULL);
+			if (ret < 0)
+				goto out;
+
+			ret = cr_page_read(ctx, page, buf);
+			page_cache_release(page);
+
+			if (ret < 0)
+				goto out;
+		}
+	}
+
+ out:
+	up_read(&mm->mmap_sem);
+	kfree(buf);
+	return 0;
+}
+
+/**
+ * cr_read_private_vma_contents - restore contents of a VMA with private memory
+ * @ctx - restart context
+ *
+ * Reads a header that specifies how many pages will follow, then reads
+ * a list of virtual addresses into ctx->pgarr_list page-array chain,
+ * followed by the actual contents of the corresponding pages. Iterates
+ * these steps until reaching a header specifying "0" pages, which marks
+ * the end of the contents.
+ */
+static int cr_read_private_vma_contents(struct cr_ctx *ctx)
+{
+	struct cr_hdr_pgarr *hh;
+	unsigned long nr_pages;
+	int parent, ret = 0;
+
+	while (!ret) {
+		hh = cr_hbuf_get(ctx, sizeof(*hh));
+		parent = cr_read_obj_type(ctx, hh, sizeof(*hh), CR_HDR_PGARR);
+		if (parent < 0)
+			return parent;
+		else if (parent != 0)
+			return -EINVAL;
+
+		cr_debug("nr_pages %ld\n", (unsigned long) hh->nr_pages);
+
+		nr_pages = hh->nr_pages;
+		cr_hbuf_put(ctx, sizeof(*hh));
+
+		if (!nr_pages)
+			break;
+
+		ret = cr_read_pages_vaddrs(ctx, nr_pages);
+		if (ret < 0)
+			break;
+		ret = cr_read_pages_contents(ctx);
+		if (ret < 0)
+			break;
+		cr_pgarr_reset_all(ctx);
+	}
+
+	return ret;
+}
+
+/**
+ * cr_calc_map_prot_bits - convert vm_flags to mmap protection
+ * orig_vm_flags: source vm_flags
+ */
+static unsigned long cr_calc_map_prot_bits(unsigned long orig_vm_flags)
+{
+	unsigned long vm_prot = 0;
+
+	if (orig_vm_flags & VM_READ)
+		vm_prot |= PROT_READ;
+	if (orig_vm_flags & VM_WRITE)
+		vm_prot |= PROT_WRITE;
+	if (orig_vm_flags & VM_EXEC)
+		vm_prot |= PROT_EXEC;
+	if (orig_vm_flags & PROT_SEM)   /* only (?) with IPC-SHM  */
+		vm_prot |= PROT_SEM;
+
+	return vm_prot;
+}
+
+/**
+ * cr_calc_map_flags_bits - convert vm_flags to mmap flags
+ * orig_vm_flags: source vm_flags
+ */
+static unsigned long cr_calc_map_flags_bits(unsigned long orig_vm_flags)
+{
+	unsigned long vm_flags = 0;
+
+	vm_flags = MAP_FIXED;
+	if (orig_vm_flags & VM_GROWSDOWN)
+		vm_flags |= MAP_GROWSDOWN;
+	if (orig_vm_flags & VM_DENYWRITE)
+		vm_flags |= MAP_DENYWRITE;
+	if (orig_vm_flags & VM_EXECUTABLE)
+		vm_flags |= MAP_EXECUTABLE;
+	if (orig_vm_flags & VM_MAYSHARE)
+		vm_flags |= MAP_SHARED;
+	else
+		vm_flags |= MAP_PRIVATE;
+
+	return vm_flags;
+}
+
+static int cr_read_vma(struct cr_ctx *ctx, struct mm_struct *mm)
+{
+	struct cr_hdr_vma *hh = cr_hbuf_get(ctx, sizeof(*hh));
+	unsigned long vm_size, vm_start, vm_flags, vm_prot, vm_pgoff;
+	unsigned long addr;
+	struct file *file = NULL;
+	int parent, ret = 0;
+
+	parent = cr_read_obj_type(ctx, hh, sizeof(*hh), CR_HDR_VMA);
+	if (parent < 0)
+		return parent;
+	else if (parent != 0)
+		return -EINVAL;
+
+	cr_debug("vma %#lx-%#lx type %d\n", (unsigned long) hh->vm_start,
+		 (unsigned long) hh->vm_end, (int) hh->vma_type);
+
+	if (hh->vm_end < hh->vm_start)
+		return -EINVAL;
+
+	vm_start = hh->vm_start;
+	vm_pgoff = hh->vm_pgoff;
+	vm_size = hh->vm_end - hh->vm_start;
+	vm_prot = cr_calc_map_prot_bits(hh->vm_flags);
+	vm_flags = cr_calc_map_flags_bits(hh->vm_flags);
+
+	switch (hh->vma_type) {
+
+	case CR_VMA_ANON:		/* anonymous private mapping */
+		if (vm_flags & VM_SHARED)
+			return -EINVAL;
+		/*
+		 * vm_pgoff for anonymous mapping is the "global" page
+		 * offset (namely from addr 0x0), so we force a zero
+		 */
+		vm_pgoff = 0;
+		break;
+
+	case CR_VMA_FILE:		/* private mapping from a file */
+		if (vm_flags & VM_SHARED)
+			return -EINVAL;
+		/*
+		 * for private mapping using 'read-only' is sufficient
+		 */
+		file = cr_read_open_fname(ctx, O_RDONLY, 0);
+		if (IS_ERR(file))
+			return PTR_ERR(file);
+		break;
+
+	default:
+		return -EINVAL;
+
+	}
+
+	cr_hbuf_put(ctx, sizeof(*hh));
+
+	down_write(&mm->mmap_sem);
+	addr = do_mmap_pgoff(file, vm_start, vm_size,
+			     vm_prot, vm_flags, vm_pgoff);
+	up_write(&mm->mmap_sem);
+	cr_debug("size %#lx prot %#lx flag %#lx pgoff %#lx => %#lx\n",
+		 vm_size, vm_prot, vm_flags, vm_pgoff, addr);
+
+	/* the file (if opened) is now referenced by the vma */
+	if (file)
+		filp_close(file, NULL);
+
+	if (IS_ERR((void *) addr))
+		return PTR_ERR((void *) addr);
+
+	/*
+	 * CR_VMA_ANON: read in memory as is
+	 * CR_VMA_FILE: read in memory as is
+	 * (more to follow ...)
+	 */
+
+	switch (hh->vma_type) {
+	case CR_VMA_ANON:
+	case CR_VMA_FILE:
+		/* standard case: read the data into the memory */
+		ret = cr_read_private_vma_contents(ctx);
+		break;
+	}
+
+	if (ret < 0)
+		return ret;
+
+	cr_debug("vma retval %d\n", ret);
+	return 0;
+}
+
+static int cr_destroy_mm(struct mm_struct *mm)
+{
+	struct vm_area_struct *vmnext = mm->mmap;
+	struct vm_area_struct *vma;
+	int ret;
+
+	while (vmnext) {
+		vma = vmnext;
+		vmnext = vmnext->vm_next;
+		ret = do_munmap(mm, vma->vm_start, vma->vm_end-vma->vm_start);
+		if (ret < 0) {
+			pr_debug("CR: restart failed do_munmap (%d)\n", ret);
+			return ret;
+		}
+	}
+	return 0;
+}
+
+int cr_read_mm(struct cr_ctx *ctx)
+{
+	struct cr_hdr_mm *hh = cr_hbuf_get(ctx, sizeof(*hh));
+	struct mm_struct *mm;
+	int nr, parent, ret;
+
+	parent = cr_read_obj_type(ctx, hh, sizeof(*hh), CR_HDR_MM);
+	if (parent < 0)
+		return parent;
+#if 0	/* activate when containers are used */
+	if (parent != task_pid_vnr(current))
+		return -EINVAL;
+#endif
+	cr_debug("map_count %d\n", hh->map_count);
+
+	/* XXX need more sanity checks */
+	if (hh->start_code > hh->end_code ||
+	    hh->start_data > hh->end_data || hh->map_count < 0)
+		return -EINVAL;
+
+	mm = current->mm;
+
+	/* point of no return -- destruct current mm */
+	down_write(&mm->mmap_sem);
+	ret = cr_destroy_mm(mm);
+	if (ret < 0) {
+		up_write(&mm->mmap_sem);
+		return ret;
+	}
+	mm->start_code = hh->start_code;
+	mm->end_code = hh->end_code;
+	mm->start_data = hh->start_data;
+	mm->end_data = hh->end_data;
+	mm->start_brk = hh->start_brk;
+	mm->brk = hh->brk;
+	mm->start_stack = hh->start_stack;
+	mm->arg_start = hh->arg_start;
+	mm->arg_end = hh->arg_end;
+	mm->env_start = hh->env_start;
+	mm->env_end = hh->env_end;
+	up_write(&mm->mmap_sem);
+
+
+	/* FIX: need also mm->flags */
+
+	for (nr = hh->map_count; nr; nr--) {
+		ret = cr_read_vma(ctx, mm);
+		if (ret < 0)
+			return ret;
+	}
+
+	ret = cr_read_mm_context(ctx, mm, hh->objref);
+
+	cr_hbuf_put(ctx, sizeof(*hh));
+	return ret;
+}
diff --git a/include/asm-x86/checkpoint_hdr.h b/include/asm-x86/checkpoint_hdr.h
index 6bc61ac..f8eee6a 100644
--- a/include/asm-x86/checkpoint_hdr.h
+++ b/include/asm-x86/checkpoint_hdr.h
@@ -74,4 +74,8 @@ struct cr_hdr_mm_context {
 	__s16 nldt;
 } __attribute__((aligned(8)));
 
+
+/* misc prototypes from kernel (not defined elsewhere) */
+asmlinkage int sys_modify_ldt(int func, void __user *ptr, unsigned long bytecount);
+
 #endif /* __ASM_X86_CKPT_HDR__H */
diff --git a/include/linux/checkpoint.h b/include/linux/checkpoint.h
index d74e64d..a9eafac 100644
--- a/include/linux/checkpoint.h
+++ b/include/linux/checkpoint.h
@@ -59,6 +59,9 @@ extern int cr_write_fname(struct cr_ctx *ctx,
 extern int cr_read_obj(struct cr_ctx *ctx, struct cr_hdr *h, void *buf, int n);
 extern int cr_read_obj_type(struct cr_ctx *ctx, void *buf, int n, int type);
 extern int cr_read_string(struct cr_ctx *ctx, void *str, int len);
+extern int cr_read_fname(struct cr_ctx *ctx, void *fname, int n);
+extern struct file *cr_read_open_fname(struct cr_ctx *ctx,
+				       int flags, int mode);
 
 extern int cr_write_mm(struct cr_ctx *ctx, struct task_struct *t);
 extern int cr_read_mm(struct cr_ctx *ctx);
-- 
1.5.4.3

Re: [RFC v5][PATCH 5/8] Restore memory address space

[Dave Hansen]

On Sat, 2008-09-13 at 19:06 -0400, Oren Laadan wrote:
> +struct file *cr_read_open_fname(struct cr_ctx *ctx, int flags, int mode)
> +{
> +	struct file *file;
> +	char *fname;
> +	int flen, ret;
> +
> +	flen = PATH_MAX;
> +	fname = kmalloc(flen, GFP_KERNEL);
> +	if (!fname)
> +		return ERR_PTR(-ENOMEM);
> +
> +	ret = cr_read_fname(ctx, fname, flen);
> +	cr_debug("fname '%s' flags %#x mode %#x\n", fname, flags, mode);
> +	if (ret >= 0)
> +		file = filp_open(fname, flags, mode);
> +	else
> +		file = ERR_PTR(ret);
> +
> +	kfree(fname);
> +	return file;
> +}

PATH_MAX is about as short of a global macro name as you're going to
get.  It should just be used directly.  Please kill 'flen'.


> +static int cr_read_pages_vaddrs(struct cr_ctx *ctx, unsigned long nr_pages)
> +{
> +	struct cr_pgarr *pgarr;
> +	unsigned long *vaddrp;
> +	int nr, ret;
> +
> +	while (nr_pages) {
> +		pgarr = cr_pgarr_current(ctx);
> +		if (!pgarr)
> +			return -ENOMEM;
> +		nr = cr_pgarr_nr_free(pgarr);
> +		if (nr > nr_pages)
> +			nr = nr_pages;
> +		vaddrp = &pgarr->vaddrs[pgarr->nr_used];
> +		ret = cr_kread(ctx, vaddrp, nr * sizeof(unsigned long));
> +		if (ret < 0)
> +			return ret;
> +		pgarr->nr_used += nr;
> +		nr_pages -= nr;
> +	}
> +	return 0;
> +}
> +
> +static int cr_page_read(struct cr_ctx *ctx, struct page *page, char *buf)
> +{
> +	void *ptr;
> +	int ret;
> +
> +	ret = cr_kread(ctx, buf, PAGE_SIZE);
> +	if (ret < 0)
> +		return ret;
> +
> +	ptr = kmap_atomic(page, KM_USER1);
> +	memcpy(ptr, buf, PAGE_SIZE);
> +	kunmap_atomic(page, KM_USER1);
> +
> +	return 0;
> +}
> +
> +/**
> + * cr_read_pages_contents - read in data of pages in page-array chain
> + * @ctx - restart context
> + */
> +static int cr_read_pages_contents(struct cr_ctx *ctx)
> +{
> +	struct mm_struct *mm = current->mm;
> +	struct cr_pgarr *pgarr;
> +	unsigned long *vaddrs;
> +	char *buf;
> +	int i, ret = 0;
> +
> +	buf = kmalloc(PAGE_SIZE, GFP_KERNEL);
> +	if (!buf)
> +		return -ENOMEM;
> +
> +	down_read(&mm->mmap_sem);
> +	list_for_each_entry_reverse(pgarr, &ctx->pgarr_list, list) {
> +		vaddrs = pgarr->vaddrs;
> +		for (i = 0; i < pgarr->nr_used; i++) {
> +			struct page *page;
> +
> +			ret = get_user_pages(current, mm, vaddrs[i],
> +					     1, 1, 1, &page, NULL);
> +			if (ret < 0)
> +				goto out;
> +
> +			ret = cr_page_read(ctx, page, buf);
> +			page_cache_release(page);
> +
> +			if (ret < 0)
> +				goto out;
> +		}
> +	}
> +
> + out:
> +	up_read(&mm->mmap_sem);
> +	kfree(buf);
> +	return 0;
> +}
> +
> +/**
> + * cr_read_private_vma_contents - restore contents of a VMA with private memory
> + * @ctx - restart context
> + *
> + * Reads a header that specifies how many pages will follow, then reads
> + * a list of virtual addresses into ctx->pgarr_list page-array chain,
> + * followed by the actual contents of the corresponding pages. Iterates
> + * these steps until reaching a header specifying "0" pages, which marks
> + * the end of the contents.
> + */
> +static int cr_read_private_vma_contents(struct cr_ctx *ctx)
> +{
> +	struct cr_hdr_pgarr *hh;
> +	unsigned long nr_pages;
> +	int parent, ret = 0;
> +
> +	while (!ret) {
> +		hh = cr_hbuf_get(ctx, sizeof(*hh));
> +		parent = cr_read_obj_type(ctx, hh, sizeof(*hh), CR_HDR_PGARR);
> +		if (parent < 0)
> +			return parent;
> +		else if (parent != 0)
> +			return -EINVAL;
> +
> +		cr_debug("nr_pages %ld\n", (unsigned long) hh->nr_pages);
> +
> +		nr_pages = hh->nr_pages;
> +		cr_hbuf_put(ctx, sizeof(*hh));
> +
> +		if (!nr_pages)
> +			break;
> +
> +		ret = cr_read_pages_vaddrs(ctx, nr_pages);
> +		if (ret < 0)
> +			break;
> +		ret = cr_read_pages_contents(ctx);
> +		if (ret < 0)
> +			break;
> +		cr_pgarr_reset_all(ctx);
> +	}
> +
> +	return ret;
> +}

That's an interesting loop condition, especially since it will never
actually get triggered.  while(1) would give the same functionality,
right?


-- Dave

[RFC v5][PATCH 6/8] Checkpoint/restart: initial documentation

[Oren Laadan]

Covers application checkpoint/restart, overall design, interfaces
and checkpoint image format.

Signed-off-by: Oren Laadan <orenl-eQaUEPhvms7ENvBUuze7eA@public.gmane.org>
---
 Documentation/checkpoint.txt |  207 ++++++++++++++++++++++++++++++++++++++++++
 1 files changed, 207 insertions(+), 0 deletions(-)
 create mode 100644 Documentation/checkpoint.txt

diff --git a/Documentation/checkpoint.txt b/Documentation/checkpoint.txt
new file mode 100644
index 0000000..6bf75ce
--- /dev/null
+++ b/Documentation/checkpoint.txt
@@ -0,0 +1,207 @@
+
+	=== Checkpoint-Restart support in the Linux kernel ===
+
+Copyright (C) 2008 Oren Laadan
+
+Author:		Oren Laadan <orenl-eQaUEPhvms7ENvBUuze7eA@public.gmane.org>
+
+License:	The GNU Free Documentation License, Version 1.2
+		(dual licensed under the GPL v2)
+Reviewers:
+
+Application checkpoint/restart [CR] is the ability to save the state
+of a running application so that it can later resume its execution
+from the time at which it was checkpointed. An application can be
+migrated by checkpointing it on one machine and restarting it on
+another. CR can provide many potential benefits:
+
+* Failure recovery: by rolling back an to a previous checkpoint
+
+* Improved response time: by restarting applications from checkpoints
+  instead of from scratch.
+
+* Improved system utilization: by suspending long running CPU
+  intensive jobs and resuming them when load decreases.
+
+* Fault resilience: by migrating applications off of faulty hosts.
+
+* Dynamic load balancing: by migrating applications to less loaded
+  hosts.
+
+* Improved service availability and administration: by migrating
+  applications before host maintenance so that they continue to run
+  with minimal downtime
+
+* Time-travel: by taking periodic checkpoints and restarting from
+  any previous checkpoint.
+
+
+=== Overall design
+
+Checkpoint and restart is done in the kernel as much as possible. The
+kernel exports a relative opaque 'blob' of data to userspace which can
+then be handed to the new kernel at restore time.  The 'blob' contains
+data and state of select portions of kernel structures such as VMAs
+and mm_structs, as well as copies of the actual memory that the tasks
+use. Any changes in this blob's format between kernel revisions can be
+handled by an in-userspace conversion program. The approach is similar
+to virtually all of the commercial CR products out there, as well as
+the research project Zap.
+
+Two new system calls are introduced to provide CR: sys_checkpoint and
+sys_restart.  The checkpoint code basically serializes internal kernel
+state and writes it out to a file descriptor, and the resulting image
+is stream-able. More specifically, it consists of 5 steps:
+  1. Pre-dump
+  2. Freeze the container
+  3. Dump
+  4. Thaw (or kill) the container
+  5. Post-dump
+Steps 1 and 5 are an optimization to reduce application downtime:
+"pre-dump" works before freezing the container, e.g. the pre-copy for
+live migration, and "post-dump" works after the container resumes
+execution, e.g. write-back the data to secondary storage.
+
+The restart code basically reads the saved kernel state and from a
+file descriptor, and re-creates the tasks and the resources they need
+to resume execution. The restart code is executed by each task that
+is restored in a new container to reconstruct its own state.
+
+
+=== Interfaces
+
+int sys_checkpoint(pid_t pid, int fd, unsigned long flag);
+  Checkpoint a container whose init task is identified by pid, to the
+  file designated by fd. Flags will have future meaning (should be 0
+  for now).
+  Returns: a positive integer that identifies the checkpoint image
+  (for future reference in case it is kept in memory) upon success,
+  0 if it returns from a restart, and -1 if an error occurs.
+
+int sys_restart(int crid, int fd, unsigned long flags);
+  Restart a container from a checkpoint image identified by crid, or
+  from the blob stored in the file designated by fd. Flags will have
+  future meaning (should be 0 for now).
+  Returns: 0 on success and -1 if an error occurs.
+
+Thus, if checkpoint is initiated by a process in the container, one
+can use logic similar to fork():
+	...
+	crid = checkpoint(...);
+	switch (crid) {
+	case -1:
+		perror("checkpoint failed");
+		break;
+	default:
+		fprintf(stderr, "checkpoint succeeded, CRID=%d\n", ret);
+		/* proceed with execution after checkpoint */
+		...
+		break;
+	case 0:
+		fprintf(stderr, "returned after restart\n");
+		/* proceed with action required following a restart */
+		...
+		break;
+	}
+	...
+And to initiate a restart, the process in an empty container can use
+logic similar to execve():
+	...
+	if (restart(crid, ...) < 0)
+		perror("restart failed");
+	/* only get here if restart failed */
+	...
+
+
+=== Checkpoint image format
+
+The checkpoint image format is composed of records consistings of a
+pre-header that identifies its contents, followed by a payload. (The
+idea here is to enable parallel checkpointing in the future in which
+multiple threads interleave data from multiple processes into a single
+stream).
+
+The pre-header is defined by "struct cr_hdr" as follows:
+
+struct cr_hdr {
+	__s16 type;
+	__s16 len;
+	__u32 id;
+};
+
+Here, 'type' field identifies the type of the payload, 'len' tells its
+length in bytes. The 'id' identifies the owner object instance. The
+meaning of the 'id' field varies depending on the type. For example,
+for type CR_HDR_MM, the 'id' identifies the task to which this MM
+belongs. The payload also varies depending on the type, for instance,
+the data describing a task_struct is given by a 'struct cr_hdr_task'
+(type CR_HDR_TASK) and so on.
+
+The format of the memory dump is as follows: for each VMA, there is a
+'struct cr_vma'; if the VMA is file-mapped, it is followed by the file
+name. Following comes the actual contents, in one or more chunk: each
+chunk begins with a header that specifies how many pages it holds,
+then a the virtual addresses of all the dumped pages in that chunk,
+followed by the actual contents of all the dumped pages. A header with
+zero number of pages marks the end of the contents for a particular
+VMA. Then comes the next VMA and so on.
+
+To illustrate this, consider a single simple task with two VMAs: one
+is file mapped with two dumped pages, and the other is anonymous with
+three dumped pages. The checkpoint image will look like this:
+
+cr_hdr + cr_hdr_head
+cr_hdr + cr_hdr_task
+	cr_hdr + cr_hdr_mm
+		cr_hdr + cr_hdr_vma + cr_hdr + string
+			cr_hdr_pgarr (nr_pages = 2)
+			addr1, addr2
+			page1, page2
+			cr_hdr_pgarr (nr_pages = 0)
+		cr_hdr + cr_hdr_vma
+			cr_hdr_pgarr (nr_pages = 3)
+			addr3, addr4, addr5
+			page3, page4, page5
+			cr_hdr_pgarr (nr_pages = 0)
+		cr_hdr + cr_mm_context
+	cr_hdr + cr_hdr_thread
+	cr_hdr + cr_hdr_cpu
+cr_hdr + cr_hdr_tail
+
+
+=== Changelog
+
+[2008-Sep-11] v5:
+  - Config is 'def_bool n' by default
+  - Improve memory dump/restore code (following Dave Hansen's comments)
+  - Change dump format (and code) to allow chunks of <vaddrs, pages>
+    instead of one long list of each
+  - Fix use of follow_page() to avoid faulting in non-present pages
+  - Memory restore now maps user pages explicitly to copy data into them,
+    instead of reading directly to user space; got rid of mprotect_fixup()
+  - Remove preempt_disable() when restoring debug registers
+  - Rename headers files s/ckpt/checkpoint/
+  - Fix misc bugs in files dump/restore
+  - Fix cleanup on some error paths
+  - Fix misc coding style
+
+[2008-Sep-04] v4:
+  - Fix calculation of hash table size
+  - Fix header structure alignment
+  - Use stand list_... for cr_pgarr
+
+[2008-Aug-20] v3:
+  - Various fixes and clean-ups
+  - Use standard hlist_... for hash table
+  - Better use of standard kmalloc/kfree
+
+[2008-Aug-09] v2:
+  - Added utsname->{release,version,machine} to checkpoint header
+  - Pad header structures to 64 bits to ensure compatibility
+  - Address comments from LKML and linux-containers mailing list
+
+[2008-Jul-29] v1:
+In this incarnation, CR only works on single task. The address space
+may consist of only private, simple VMAs - anonymous or file-mapped.
+Both checkpoint and restart will ignore the first argument (pid/crid)
+and instead act on themselves.
-- 
1.5.4.3

Re: [RFC v5][PATCH 6/8] Checkpoint/restart: initial documentation

[Serge E. Hallyn]

Quoting Oren Laadan (orenl-eQaUEPhvms7ENvBUuze7eA@public.gmane.org):
> Covers application checkpoint/restart, overall design, interfaces
> and checkpoint image format.
> 
> Signed-off-by: Oren Laadan <orenl-eQaUEPhvms7ENvBUuze7eA@public.gmane.org>

This really should include your demo programs from your patch 0/9
announcement.

> ---
>  Documentation/checkpoint.txt |  207 ++++++++++++++++++++++++++++++++++++++++++
>  1 files changed, 207 insertions(+), 0 deletions(-)
>  create mode 100644 Documentation/checkpoint.txt
> 
> diff --git a/Documentation/checkpoint.txt b/Documentation/checkpoint.txt
> new file mode 100644
> index 0000000..6bf75ce
> --- /dev/null
> +++ b/Documentation/checkpoint.txt
> @@ -0,0 +1,207 @@
> +
> +	=== Checkpoint-Restart support in the Linux kernel ===
> +
> +Copyright (C) 2008 Oren Laadan
> +
> +Author:		Oren Laadan <orenl-eQaUEPhvms7ENvBUuze7eA@public.gmane.org>
> +
> +License:	The GNU Free Documentation License, Version 1.2
> +		(dual licensed under the GPL v2)
> +Reviewers:
> +
> +Application checkpoint/restart [CR] is the ability to save the state
> +of a running application so that it can later resume its execution
> +from the time at which it was checkpointed. An application can be
> +migrated by checkpointing it on one machine and restarting it on
> +another. CR can provide many potential benefits:
> +
> +* Failure recovery: by rolling back an to a previous checkpoint
> +
> +* Improved response time: by restarting applications from checkpoints
> +  instead of from scratch.
> +
> +* Improved system utilization: by suspending long running CPU
> +  intensive jobs and resuming them when load decreases.
> +
> +* Fault resilience: by migrating applications off of faulty hosts.
> +
> +* Dynamic load balancing: by migrating applications to less loaded
> +  hosts.
> +
> +* Improved service availability and administration: by migrating
> +  applications before host maintenance so that they continue to run
> +  with minimal downtime
> +
> +* Time-travel: by taking periodic checkpoints and restarting from
> +  any previous checkpoint.
> +
> +
> +=== Overall design
> +
> +Checkpoint and restart is done in the kernel as much as possible. The
> +kernel exports a relative opaque 'blob' of data to userspace which can
> +then be handed to the new kernel at restore time.  The 'blob' contains
> +data and state of select portions of kernel structures such as VMAs
> +and mm_structs, as well as copies of the actual memory that the tasks
> +use. Any changes in this blob's format between kernel revisions can be
> +handled by an in-userspace conversion program. The approach is similar
> +to virtually all of the commercial CR products out there, as well as
> +the research project Zap.
> +
> +Two new system calls are introduced to provide CR: sys_checkpoint and
> +sys_restart.  The checkpoint code basically serializes internal kernel
> +state and writes it out to a file descriptor, and the resulting image
> +is stream-able. More specifically, it consists of 5 steps:
> +  1. Pre-dump
> +  2. Freeze the container
> +  3. Dump
> +  4. Thaw (or kill) the container
> +  5. Post-dump
> +Steps 1 and 5 are an optimization to reduce application downtime:
> +"pre-dump" works before freezing the container, e.g. the pre-copy for
> +live migration, and "post-dump" works after the container resumes
> +execution, e.g. write-back the data to secondary storage.
> +
> +The restart code basically reads the saved kernel state and from a
> +file descriptor, and re-creates the tasks and the resources they need
> +to resume execution. The restart code is executed by each task that
> +is restored in a new container to reconstruct its own state.
> +
> +
> +=== Interfaces
> +
> +int sys_checkpoint(pid_t pid, int fd, unsigned long flag);
> +  Checkpoint a container whose init task is identified by pid, to the
> +  file designated by fd. Flags will have future meaning (should be 0
> +  for now).
> +  Returns: a positive integer that identifies the checkpoint image
> +  (for future reference in case it is kept in memory) upon success,
> +  0 if it returns from a restart, and -1 if an error occurs.
> +
> +int sys_restart(int crid, int fd, unsigned long flags);
> +  Restart a container from a checkpoint image identified by crid, or
> +  from the blob stored in the file designated by fd. Flags will have
> +  future meaning (should be 0 for now).
> +  Returns: 0 on success and -1 if an error occurs.
> +
> +Thus, if checkpoint is initiated by a process in the container, one
> +can use logic similar to fork():
> +	...
> +	crid = checkpoint(...);
> +	switch (crid) {
> +	case -1:
> +		perror("checkpoint failed");
> +		break;
> +	default:
> +		fprintf(stderr, "checkpoint succeeded, CRID=%d\n", ret);
> +		/* proceed with execution after checkpoint */
> +		...
> +		break;
> +	case 0:
> +		fprintf(stderr, "returned after restart\n");
> +		/* proceed with action required following a restart */
> +		...
> +		break;
> +	}
> +	...
> +And to initiate a restart, the process in an empty container can use
> +logic similar to execve():
> +	...
> +	if (restart(crid, ...) < 0)
> +		perror("restart failed");
> +	/* only get here if restart failed */
> +	...
> +
> +
> +=== Checkpoint image format
> +
> +The checkpoint image format is composed of records consistings of a
> +pre-header that identifies its contents, followed by a payload. (The
> +idea here is to enable parallel checkpointing in the future in which
> +multiple threads interleave data from multiple processes into a single
> +stream).
> +
> +The pre-header is defined by "struct cr_hdr" as follows:
> +
> +struct cr_hdr {
> +	__s16 type;
> +	__s16 len;
> +	__u32 id;
> +};
> +
> +Here, 'type' field identifies the type of the payload, 'len' tells its
> +length in bytes. The 'id' identifies the owner object instance. The
> +meaning of the 'id' field varies depending on the type. For example,
> +for type CR_HDR_MM, the 'id' identifies the task to which this MM
> +belongs. The payload also varies depending on the type, for instance,
> +the data describing a task_struct is given by a 'struct cr_hdr_task'
> +(type CR_HDR_TASK) and so on.
> +
> +The format of the memory dump is as follows: for each VMA, there is a
> +'struct cr_vma'; if the VMA is file-mapped, it is followed by the file
> +name. Following comes the actual contents, in one or more chunk: each
> +chunk begins with a header that specifies how many pages it holds,
> +then a the virtual addresses of all the dumped pages in that chunk,
> +followed by the actual contents of all the dumped pages. A header with
> +zero number of pages marks the end of the contents for a particular
> +VMA. Then comes the next VMA and so on.
> +
> +To illustrate this, consider a single simple task with two VMAs: one
> +is file mapped with two dumped pages, and the other is anonymous with
> +three dumped pages. The checkpoint image will look like this:
> +
> +cr_hdr + cr_hdr_head
> +cr_hdr + cr_hdr_task
> +	cr_hdr + cr_hdr_mm
> +		cr_hdr + cr_hdr_vma + cr_hdr + string
> +			cr_hdr_pgarr (nr_pages = 2)
> +			addr1, addr2
> +			page1, page2
> +			cr_hdr_pgarr (nr_pages = 0)
> +		cr_hdr + cr_hdr_vma
> +			cr_hdr_pgarr (nr_pages = 3)
> +			addr3, addr4, addr5
> +			page3, page4, page5
> +			cr_hdr_pgarr (nr_pages = 0)
> +		cr_hdr + cr_mm_context
> +	cr_hdr + cr_hdr_thread
> +	cr_hdr + cr_hdr_cpu
> +cr_hdr + cr_hdr_tail
> +
> +
> +=== Changelog
> +
> +[2008-Sep-11] v5:
> +  - Config is 'def_bool n' by default
> +  - Improve memory dump/restore code (following Dave Hansen's comments)
> +  - Change dump format (and code) to allow chunks of <vaddrs, pages>
> +    instead of one long list of each
> +  - Fix use of follow_page() to avoid faulting in non-present pages
> +  - Memory restore now maps user pages explicitly to copy data into them,
> +    instead of reading directly to user space; got rid of mprotect_fixup()
> +  - Remove preempt_disable() when restoring debug registers
> +  - Rename headers files s/ckpt/checkpoint/
> +  - Fix misc bugs in files dump/restore
> +  - Fix cleanup on some error paths
> +  - Fix misc coding style
> +
> +[2008-Sep-04] v4:
> +  - Fix calculation of hash table size
> +  - Fix header structure alignment
> +  - Use stand list_... for cr_pgarr
> +
> +[2008-Aug-20] v3:
> +  - Various fixes and clean-ups
> +  - Use standard hlist_... for hash table
> +  - Better use of standard kmalloc/kfree
> +
> +[2008-Aug-09] v2:
> +  - Added utsname->{release,version,machine} to checkpoint header
> +  - Pad header structures to 64 bits to ensure compatibility
> +  - Address comments from LKML and linux-containers mailing list
> +
> +[2008-Jul-29] v1:
> +In this incarnation, CR only works on single task. The address space
> +may consist of only private, simple VMAs - anonymous or file-mapped.
> +Both checkpoint and restart will ignore the first argument (pid/crid)
> +and instead act on themselves.
> -- 
> 1.5.4.3
> 
> _______________________________________________
> Containers mailing list
> Containers-cunTk1MwBs9QetFLy7KEm3xJsTq8ys+cHZ5vskTnxNA@public.gmane.org
> https://lists.linux-foundation.org/mailman/listinfo/containers

Re: [RFC v5][PATCH 6/8] Checkpoint/restart: initial documentation

[MinChan Kim]

On Sun, Sep 14, 2008 at 8:06 AM, Oren Laadan <orenl-eQaUEPhvms7ENvBUuze7eA@public.gmane.org> wrote:
> Covers application checkpoint/restart, overall design, interfaces
> and checkpoint image format.
>
> Signed-off-by: Oren Laadan <orenl-eQaUEPhvms7ENvBUuze7eA@public.gmane.org>
> ---
>  Documentation/checkpoint.txt |  207 ++++++++++++++++++++++++++++++++++++++++++
>  1 files changed, 207 insertions(+), 0 deletions(-)
>  create mode 100644 Documentation/checkpoint.txt
>
> diff --git a/Documentation/checkpoint.txt b/Documentation/checkpoint.txt
> new file mode 100644
> index 0000000..6bf75ce
> --- /dev/null
> +++ b/Documentation/checkpoint.txt
> @@ -0,0 +1,207 @@
> +
> +       === Checkpoint-Restart support in the Linux kernel ===
> +
> +Copyright (C) 2008 Oren Laadan
> +
> +Author:                Oren Laadan <orenl-eQaUEPhvms7ENvBUuze7eA@public.gmane.org>
> +
> +License:       The GNU Free Documentation License, Version 1.2
> +               (dual licensed under the GPL v2)
> +Reviewers:
> +
> +Application checkpoint/restart [CR] is the ability to save the state
> +of a running application so that it can later resume its execution
> +from the time at which it was checkpointed. An application can be
> +migrated by checkpointing it on one machine and restarting it on
> +another. CR can provide many potential benefits:
> +
> +* Failure recovery: by rolling back an to a previous checkpoint
> +
> +* Improved response time: by restarting applications from checkpoints
> +  instead of from scratch.
> +
> +* Improved system utilization: by suspending long running CPU
> +  intensive jobs and resuming them when load decreases.
> +
> +* Fault resilience: by migrating applications off of faulty hosts.
> +
> +* Dynamic load balancing: by migrating applications to less loaded
> +  hosts.
> +
> +* Improved service availability and administration: by migrating
> +  applications before host maintenance so that they continue to run
> +  with minimal downtime
> +
> +* Time-travel: by taking periodic checkpoints and restarting from
> +  any previous checkpoint.
> +
> +
> +=== Overall design
> +
> +Checkpoint and restart is done in the kernel as much as possible. The
> +kernel exports a relative opaque 'blob' of data to userspace which can
> +then be handed to the new kernel at restore time.  The 'blob' contains
> +data and state of select portions of kernel structures such as VMAs
> +and mm_structs, as well as copies of the actual memory that the tasks
> +use. Any changes in this blob's format between kernel revisions can be
> +handled by an in-userspace conversion program. The approach is similar
> +to virtually all of the commercial CR products out there, as well as
> +the research project Zap.
> +
> +Two new system calls are introduced to provide CR: sys_checkpoint and
> +sys_restart.  The checkpoint code basically serializes internal kernel
> +state and writes it out to a file descriptor, and the resulting image
> +is stream-able. More specifically, it consists of 5 steps:
> +  1. Pre-dump
> +  2. Freeze the container
> +  3. Dump
> +  4. Thaw (or kill) the container
> +  5. Post-dump
> +Steps 1 and 5 are an optimization to reduce application downtime:
> +"pre-dump" works before freezing the container, e.g. the pre-copy for
> +live migration, and "post-dump" works after the container resumes
> +execution, e.g. write-back the data to secondary storage.
> +
> +The restart code basically reads the saved kernel state and from a
> +file descriptor, and re-creates the tasks and the resources they need
> +to resume execution. The restart code is executed by each task that
> +is restored in a new container to reconstruct its own state.
> +
> +
> +=== Interfaces
> +
> +int sys_checkpoint(pid_t pid, int fd, unsigned long flag);
> +  Checkpoint a container whose init task is identified by pid, to the
> +  file designated by fd. Flags will have future meaning (should be 0
> +  for now).
> +  Returns: a positive integer that identifies the checkpoint image
> +  (for future reference in case it is kept in memory) upon success,
> +  0 if it returns from a restart, and -1 if an error occurs.
> +
> +int sys_restart(int crid, int fd, unsigned long flags);
> +  Restart a container from a checkpoint image identified by crid, or
> +  from the blob stored in the file designated by fd. Flags will have
> +  future meaning (should be 0 for now).
> +  Returns: 0 on success and -1 if an error occurs.
> +
> +Thus, if checkpoint is initiated by a process in the container, one
> +can use logic similar to fork():
> +       ...
> +       crid = checkpoint(...);
> +       switch (crid) {
> +       case -1:
> +               perror("checkpoint failed");
> +               break;
> +       default:
> +               fprintf(stderr, "checkpoint succeeded, CRID=%d\n", ret);
> +               /* proceed with execution after checkpoint */
> +               ...
> +               break;
> +       case 0:
> +               fprintf(stderr, "returned after restart\n");
> +               /* proceed with action required following a restart */
> +               ...
> +               break;
> +       }
> +       ...
> +And to initiate a restart, the process in an empty container can use
> +logic similar to execve():
> +       ...
> +       if (restart(crid, ...) < 0)
> +               perror("restart failed");
> +       /* only get here if restart failed */
> +       ...
> +
> +
> +=== Checkpoint image format
> +
> +The checkpoint image format is composed of records consistings of a
> +pre-header that identifies its contents, followed by a payload. (The
> +idea here is to enable parallel checkpointing in the future in which
> +multiple threads interleave data from multiple processes into a single
> +stream).
> +
> +The pre-header is defined by "struct cr_hdr" as follows:
> +
> +struct cr_hdr {
> +       __s16 type;
> +       __s16 len;
> +       __u32 id;
> +};
> +
> +Here, 'type' field identifies the type of the payload, 'len' tells its
> +length in bytes. The 'id' identifies the owner object instance. The

which is right between id and parent?
It is confusing me :)

> +meaning of the 'id' field varies depending on the type. For example,
> +for type CR_HDR_MM, the 'id' identifies the task to which this MM
> +belongs. The payload also varies depending on the type, for instance,
> +the data describing a task_struct is given by a 'struct cr_hdr_task'
> +(type CR_HDR_TASK) and so on.
> +
> +The format of the memory dump is as follows: for each VMA, there is a
> +'struct cr_vma'; if the VMA is file-mapped, it is followed by the file
> +name. Following comes the actual contents, in one or more chunk: each
> +chunk begins with a header that specifies how many pages it holds,
> +then a the virtual addresses of all the dumped pages in that chunk,
> +followed by the actual contents of all the dumped pages. A header with
> +zero number of pages marks the end of the contents for a particular
> +VMA. Then comes the next VMA and so on.
> +
> +To illustrate this, consider a single simple task with two VMAs: one
> +is file mapped with two dumped pages, and the other is anonymous with
> +three dumped pages. The checkpoint image will look like this:
> +
> +cr_hdr + cr_hdr_head
> +cr_hdr + cr_hdr_task
> +       cr_hdr + cr_hdr_mm
> +               cr_hdr + cr_hdr_vma + cr_hdr + string
> +                       cr_hdr_pgarr (nr_pages = 2)
> +                       addr1, addr2
> +                       page1, page2
> +                       cr_hdr_pgarr (nr_pages = 0)
> +               cr_hdr + cr_hdr_vma
> +                       cr_hdr_pgarr (nr_pages = 3)
> +                       addr3, addr4, addr5
> +                       page3, page4, page5
> +                       cr_hdr_pgarr (nr_pages = 0)
> +               cr_hdr + cr_mm_context
> +       cr_hdr + cr_hdr_thread
> +       cr_hdr + cr_hdr_cpu
> +cr_hdr + cr_hdr_tail
> +
> +
> +=== Changelog
> +
> +[2008-Sep-11] v5:
> +  - Config is 'def_bool n' by default
> +  - Improve memory dump/restore code (following Dave Hansen's comments)
> +  - Change dump format (and code) to allow chunks of <vaddrs, pages>
> +    instead of one long list of each
> +  - Fix use of follow_page() to avoid faulting in non-present pages
> +  - Memory restore now maps user pages explicitly to copy data into them,
> +    instead of reading directly to user space; got rid of mprotect_fixup()
> +  - Remove preempt_disable() when restoring debug registers
> +  - Rename headers files s/ckpt/checkpoint/
> +  - Fix misc bugs in files dump/restore
> +  - Fix cleanup on some error paths
> +  - Fix misc coding style
> +
> +[2008-Sep-04] v4:
> +  - Fix calculation of hash table size
> +  - Fix header structure alignment
> +  - Use stand list_... for cr_pgarr
> +
> +[2008-Aug-20] v3:
> +  - Various fixes and clean-ups
> +  - Use standard hlist_... for hash table
> +  - Better use of standard kmalloc/kfree
> +
> +[2008-Aug-09] v2:
> +  - Added utsname->{release,version,machine} to checkpoint header
> +  - Pad header structures to 64 bits to ensure compatibility
> +  - Address comments from LKML and linux-containers mailing list
> +
> +[2008-Jul-29] v1:
> +In this incarnation, CR only works on single task. The address space
> +may consist of only private, simple VMAs - anonymous or file-mapped.
> +Both checkpoint and restart will ignore the first argument (pid/crid)
> +and instead act on themselves.
> --
> 1.5.4.3
>
> --
> To unsubscribe from this list: send the line "unsubscribe linux-kernel" in
> the body of a message to majordomo-u79uwXL29TY76Z2rM5mHXA@public.gmane.org
> More majordomo info at  http://vger.kernel.org/majordomo-info.html
> Please read the FAQ at  http://www.tux.org/lkml/
>



-- 
Kinds regards,
MinChan Kim

[RFC v5][PATCH 7/8] Infrastructure for shared objects

[Oren Laadan]

Infrastructure to handle objects that may be shared and referenced by
multiple tasks or other objects, e..g open files, memory address space
etc.

The state of shared objects is saved once. On the first encounter, the
state is dumped and the object is assigned a unique identifier (objref)
and also stored in a hash table (indexed by its physical kenrel address).
From then on the object will be found in the hash and only its identifier
is saved.

On restart the identifier is looked up in the hash table; if not found
then the state is read, the object is created, and added to the hash
table (this time indexed by its identifier). Otherwise, the object in
the hash table is used.

Signed-off-by: Oren Laadan <orenl-eQaUEPhvms7ENvBUuze7eA@public.gmane.org>
---
 Documentation/checkpoint.txt |   38 +++++++
 checkpoint/Makefile          |    2 +-
 checkpoint/objhash.c         |  237 ++++++++++++++++++++++++++++++++++++++++++
 checkpoint/sys.c             |    4 +
 include/linux/checkpoint.h   |   20 ++++
 5 files changed, 300 insertions(+), 1 deletions(-)
 create mode 100644 checkpoint/objhash.c

diff --git a/Documentation/checkpoint.txt b/Documentation/checkpoint.txt
index 6bf75ce..2929512 100644
--- a/Documentation/checkpoint.txt
+++ b/Documentation/checkpoint.txt
@@ -169,6 +169,44 @@ cr_hdr + cr_hdr_task
 cr_hdr + cr_hdr_tail
 
 
+=== Shared resources (objects)
+
+Many resources used by tasks may be shared by more than one task (e.g.
+file descriptors, memory address space, etc), or even have multiple
+references from other resources (e.g. a single inode that represents
+two ends of a pipe).
+
+Clearly, the state of shared objects need only be saved once, even if
+they occur multiple times. We use a hash table (ctx->objhash) to keep
+track of shared objects in the following manner.
+
+On the first encounter, the state is dumped and the object is assigned
+a unique identifier and also stored in the hash table (indexed by its
+physical kenrel address). From then on the object will be found in the
+hash and only its identifier is saved.
+
+On restart the identifier is looked up in the hash table; if not found
+then the state is read, the object is created, and added to the hash
+table (this time indexed by its identifier). Otherwise, the object in
+the hash table is used.
+
+The interface for the hash table is the following:
+
+cr_obj_get_by_ptr - find the unique identifier - object reference (objref)
+  of the object that is pointer to by ptr (or 0 if not found) [checkpoint]
+
+cr_obj_add_ptr - add the object pointed to by ptr to the hash table if
+  it isn't already there, and fill its unique identifier (objref); will
+  return 0 if already found in the has, or 1 otherwise [checkpoint]
+
+cr_obj_get_by_ref - return the pointer to the object whose unique identifier
+  is equal to objref [restart]
+
+cr_obj_add_ref - add the object with unique identifier objref, pointed to by
+  ptr to the hash table if it isn't already there; will return 0 if already
+  found in the has, or 1 otherwise [restart]
+
+
 === Changelog
 
 [2008-Sep-11] v5:
diff --git a/checkpoint/Makefile b/checkpoint/Makefile
index ac35033..9843fb9 100644
--- a/checkpoint/Makefile
+++ b/checkpoint/Makefile
@@ -2,5 +2,5 @@
 # Makefile for linux checkpoint/restart.
 #
 
-obj-$(CONFIG_CHECKPOINT_RESTART) += sys.o checkpoint.o restart.o \
+obj-$(CONFIG_CHECKPOINT_RESTART) += sys.o checkpoint.o restart.o objhash.o \
 		ckpt_mem.o rstr_mem.o
diff --git a/checkpoint/objhash.c b/checkpoint/objhash.c
new file mode 100644
index 0000000..0862086
--- /dev/null
+++ b/checkpoint/objhash.c
@@ -0,0 +1,237 @@
+/*
+ *  Checkpoint-restart - object hash infrastructure to manage shared objects
+ *
+ *  Copyright (C) 2008 Oren Laadan
+ *
+ *  This file is subject to the terms and conditions of the GNU General Public
+ *  License.  See the file COPYING in the main directory of the Linux
+ *  distribution for more details.
+ */
+
+#include <linux/kernel.h>
+#include <linux/file.h>
+#include <linux/hash.h>
+#include <linux/checkpoint.h>
+
+struct cr_objref {
+	int objref;
+	void *ptr;
+	unsigned short type;
+	unsigned short flags;
+	struct hlist_node hash;
+};
+
+struct cr_objhash {
+	struct hlist_head *head;
+	int objref_index;
+};
+
+#define CR_OBJHASH_NBITS  10
+#define CR_OBJHASH_TOTAL  (1UL << CR_OBJHASH_NBITS)
+
+static void cr_obj_ref_drop(struct cr_objref *obj)
+{
+	switch (obj->type) {
+	case CR_OBJ_FILE:
+		fput((struct file *) obj->ptr);
+		break;
+	default:
+		BUG();
+	}
+}
+
+static void cr_obj_ref_grab(struct cr_objref *obj)
+{
+	switch (obj->type) {
+	case CR_OBJ_FILE:
+		get_file((struct file *) obj->ptr);
+		break;
+	default:
+		BUG();
+	}
+}
+
+static void cr_objhash_clear(struct cr_objhash *objhash)
+{
+	struct hlist_head *h = objhash->head;
+	struct hlist_node *n, *t;
+	struct cr_objref *obj;
+	int i;
+
+	for (i = 0; i < CR_OBJHASH_TOTAL; i++) {
+		hlist_for_each_entry_safe(obj, n, t, &h[i], hash) {
+			cr_obj_ref_drop(obj);
+			kfree(obj);
+		}
+	}
+}
+
+void cr_objhash_free(struct cr_ctx *ctx)
+{
+	struct cr_objhash *objhash = ctx->objhash;
+
+	if (objhash) {
+		cr_objhash_clear(objhash);
+		kfree(objhash->head);
+		kfree(ctx->objhash);
+		ctx->objhash = NULL;
+	}
+}
+
+int cr_objhash_alloc(struct cr_ctx *ctx)
+{
+	struct cr_objhash *objhash;
+	struct hlist_head *head;
+
+	objhash = kzalloc(sizeof(*objhash), GFP_KERNEL);
+	if (!objhash)
+		return -ENOMEM;
+	head = kzalloc(CR_OBJHASH_TOTAL * sizeof(*head), GFP_KERNEL);
+	if (!head) {
+		kfree(objhash);
+		return -ENOMEM;
+	}
+
+	objhash->head = head;
+	objhash->objref_index = 1;
+
+	ctx->objhash = objhash;
+	return 0;
+}
+
+static struct cr_objref *cr_obj_find_by_ptr(struct cr_ctx *ctx, void *ptr)
+{
+	struct hlist_head *h;
+	struct hlist_node *n;
+	struct cr_objref *obj;
+
+	h = &ctx->objhash->head[hash_ptr(ptr, CR_OBJHASH_NBITS)];
+	hlist_for_each_entry(obj, n, h, hash)
+		if (obj->ptr == ptr)
+			return obj;
+	return NULL;
+}
+
+static struct cr_objref *cr_obj_find_by_objref(struct cr_ctx *ctx, int objref)
+{
+	struct hlist_head *h;
+	struct hlist_node *n;
+	struct cr_objref *obj;
+
+	h = &ctx->objhash->head[hash_ptr((void *) objref, CR_OBJHASH_NBITS)];
+	hlist_for_each_entry(obj, n, h, hash)
+		if (obj->objref == objref)
+			return obj;
+	return NULL;
+}
+
+static struct cr_objref *cr_obj_new(struct cr_ctx *ctx, void *ptr, int objref,
+				    unsigned short type, unsigned short flags)
+{
+	struct cr_objref *obj;
+
+	obj = kmalloc(sizeof(*obj), GFP_KERNEL);
+	if (obj) {
+		int i;
+
+		obj->ptr = ptr;
+		obj->type = type;
+		obj->flags = flags;
+
+		if (objref) {
+			/* use 'objref' to index (restart) */
+			obj->objref = objref;
+			i = hash_ptr((void *) objref, CR_OBJHASH_NBITS);
+		} else {
+			/* use 'ptr' to index, assign objref (checkpoint) */
+			obj->objref = ctx->objhash->objref_index++;;
+			i = hash_ptr(ptr, CR_OBJHASH_NBITS);
+		}
+
+		hlist_add_head(&obj->hash, &ctx->objhash->head[i]);
+		cr_obj_ref_grab(obj);
+	}
+	return obj;
+}
+
+/**
+ * cr_obj_add_ptr - add an object to the hash table if not already there
+ * @ctx: checkpoint context
+ * @ptr: pointer to object
+ * @objref: unique identifier - object reference [output]
+ * @type: object type
+ * @flags: object flags
+ *
+ * Fills the unique identifier of the object into @objref
+ *
+ * returns 0 if found, 1 if added, < 0 on error
+ */
+int cr_obj_add_ptr(struct cr_ctx *ctx, void *ptr, int *objref,
+		   unsigned short type, unsigned short flags)
+{
+	struct cr_objref *obj;
+	int ret = 0;
+
+	obj = cr_obj_find_by_ptr(ctx, ptr);
+	if (!obj) {
+		obj = cr_obj_new(ctx, ptr, 0, type, flags);
+		if (!obj)
+			return -ENOMEM;
+		else
+			ret = 1;
+	} else if (obj->type != type)	/* sanity check */
+		return -EINVAL;
+	*objref = obj->objref;
+	return ret;
+}
+
+/**
+ * cr_obj_add_ref - add an object with unique identifer to the hash table
+ * @ctx: checkpoint context
+ * @ptr: pointer to object
+ * @objref: unique identifier - object reference
+ * @type: object type
+ * @flags: object flags
+ */
+int cr_obj_add_ref(struct cr_ctx *ctx, void *ptr, int objref,
+		   unsigned short type, unsigned short flags)
+{
+	struct cr_objref *obj;
+
+	obj = cr_obj_new(ctx, ptr, objref, type, flags);
+	return obj ? 0 : -ENOMEM;
+}
+
+/**
+ * cr_obj_get_by_ptr - find the unique identifier (objref) of an object
+ * @ctx: checkpoint context
+ * @ptr: pointer to object
+ * @type: object type
+ */
+int cr_obj_get_by_ptr(struct cr_ctx *ctx, void *ptr, unsigned short type)
+{
+	struct cr_objref *obj;
+
+	obj = cr_obj_find_by_ptr(ctx, ptr);
+	if (obj)
+		return obj->type == type ? obj->objref : -EINVAL;
+	else
+		return -ESRCH;
+}
+
+/**
+ * cr_obj_get_by_ref - find an object given its unique identifier (objref)
+ * @ctx: checkpoint context
+ * @objref: unique identifier - object reference
+ * @type: object type
+ */
+void *cr_obj_get_by_ref(struct cr_ctx *ctx, int objref, unsigned short type)
+{
+	struct cr_objref *obj;
+
+	obj = cr_obj_find_by_objref(ctx, objref);
+	if (obj)
+		return obj->type == type ? obj->ptr : ERR_PTR(-EINVAL);
+	else
+		return NULL;
+}
diff --git a/checkpoint/sys.c b/checkpoint/sys.c
index c4ac157..3b17ecc 100644
--- a/checkpoint/sys.c
+++ b/checkpoint/sys.c
@@ -132,6 +132,7 @@ void cr_ctx_free(struct cr_ctx *ctx)
 		path_put(ctx->vfsroot);
 
 	cr_pgarr_free(ctx);
+	cr_objhash_free(ctx);
 
 	kfree(ctx);
 }
@@ -154,6 +155,9 @@ struct cr_ctx *cr_ctx_alloc(pid_t pid, int fd, unsigned long flags)
 	if (!ctx->hbuf)
 		goto nomem;
 
+	if (cr_objhash_alloc(ctx) < 0)
+		goto nomem;
+
 	/*
 	 * assume checkpointer is in container's root vfs
 	 * FIXME: this works for now, but will change with real containers
diff --git a/include/linux/checkpoint.h b/include/linux/checkpoint.h
index a9eafac..9990bb1 100644
--- a/include/linux/checkpoint.h
+++ b/include/linux/checkpoint.h
@@ -28,6 +28,8 @@ struct cr_ctx {
 	void *hbuf;		/* temporary buffer for headers */
 	int hpos;		/* position in headers buffer */
 
+	struct cr_objhash *objhash;	/* hash for shared objects */
+
 	struct list_head pgarr_list;	/* page array to dump VMA contents */
 
 	struct path *vfsroot;	/* container root (FIXME) */
@@ -49,6 +51,24 @@ extern int cr_kread(struct cr_ctx *ctx, void *buf, int count);
 extern void *cr_hbuf_get(struct cr_ctx *ctx, int n);
 extern void cr_hbuf_put(struct cr_ctx *ctx, int n);
 
+/* shared objects handling */
+
+enum {
+	CR_OBJ_FILE = 1,
+	CR_OBJ_MAX
+};
+
+extern void cr_objhash_free(struct cr_ctx *ctx);
+extern int cr_objhash_alloc(struct cr_ctx *ctx);
+extern void *cr_obj_get_by_ref(struct cr_ctx *ctx,
+			       int objref, unsigned short type);
+extern int cr_obj_get_by_ptr(struct cr_ctx *ctx,
+			     void *ptr, unsigned short type);
+extern int cr_obj_add_ptr(struct cr_ctx *ctx, void *ptr, int *objref,
+			  unsigned short type, unsigned short flags);
+extern int cr_obj_add_ref(struct cr_ctx *ctx, void *ptr, int objref,
+			  unsigned short type, unsigned short flags);
+
 struct cr_hdr;
 
 extern int cr_write_obj(struct cr_ctx *ctx, struct cr_hdr *h, void *buf);
-- 
1.5.4.3

Re: [RFC v5][PATCH 7/8] Infrastructure for shared objects

[Dave Hansen]

On Sat, 2008-09-13 at 19:06 -0400, Oren Laadan wrote:
> +=== Shared resources (objects)
> +
> +Many resources used by tasks may be shared by more than one task (e.g.
> +file descriptors, memory address space, etc), or even have multiple
> +references from other resources (e.g. a single inode that represents
> +two ends of a pipe).
> +
> +Clearly, the state of shared objects need only be saved once, even if
> +they occur multiple times. We use a hash table (ctx->objhash) to keep
> +track of shared objects in the following manner.
> +
> +On the first encounter, the state is dumped and the object is assigned
> +a unique identifier and also stored in the hash table (indexed by its
> +physical kenrel address). From then on the object will be found in the
> +hash and only its identifier is saved.

kernel?     ^^^^^^

> +On restart the identifier is looked up in the hash table; if not found
> +then the state is read, the object is created, and added to the hash
> +table (this time indexed by its identifier). Otherwise, the object in
> +the hash table is used.
> +
> +The interface for the hash table is the following:
> +
> +cr_obj_get_by_ptr - find the unique identifier - object reference (objref)
> +  of the object that is pointer to by ptr (or 0 if not found) [checkpoint]
> +
> +cr_obj_add_ptr - add the object pointed to by ptr to the hash table if
> +  it isn't already there, and fill its unique identifier (objref); will
> +  return 0 if already found in the has, or 1 otherwise [checkpoint]
> +
> +cr_obj_get_by_ref - return the pointer to the object whose unique identifier
> +  is equal to objref [restart]
> +
> +cr_obj_add_ref - add the object with unique identifier objref, pointed to by
> +  ptr to the hash table if it isn't already there; will return 0 if already
> +  found in the has, or 1 otherwise [restart]

I'd much rather see all this documentation put properly inline with the
source code.  I doubt anyone will actually find this stuff.

> +
>  === Changelog
> 
>  [2008-Sep-11] v5:
> diff --git a/checkpoint/Makefile b/checkpoint/Makefile
> index ac35033..9843fb9 100644
> --- a/checkpoint/Makefile
> +++ b/checkpoint/Makefile
> @@ -2,5 +2,5 @@
>  # Makefile for linux checkpoint/restart.
>  #
> 
> -obj-$(CONFIG_CHECKPOINT_RESTART) += sys.o checkpoint.o restart.o \
> +obj-$(CONFIG_CHECKPOINT_RESTART) += sys.o checkpoint.o restart.o objhash.o \
>  		ckpt_mem.o rstr_mem.o
> diff --git a/checkpoint/objhash.c b/checkpoint/objhash.c
> new file mode 100644
> index 0000000..0862086
> --- /dev/null
> +++ b/checkpoint/objhash.c
> @@ -0,0 +1,237 @@
> +/*
> + *  Checkpoint-restart - object hash infrastructure to manage shared objects
> + *
> + *  Copyright (C) 2008 Oren Laadan
> + *
> + *  This file is subject to the terms and conditions of the GNU General Public
> + *  License.  See the file COPYING in the main directory of the Linux
> + *  distribution for more details.
> + */
> +
> +#include <linux/kernel.h>
> +#include <linux/file.h>
> +#include <linux/hash.h>
> +#include <linux/checkpoint.h>
> +
> +struct cr_objref {
> +	int objref;
> +	void *ptr;
> +	unsigned short type;
> +	unsigned short flags;
> +	struct hlist_node hash;
> +};
> +
> +struct cr_objhash {
> +	struct hlist_head *head;
> +	int objref_index;
> +};
> +
> +#define CR_OBJHASH_NBITS  10
> +#define CR_OBJHASH_TOTAL  (1UL << CR_OBJHASH_NBITS)
> +
> +static void cr_obj_ref_drop(struct cr_objref *obj)
> +{
> +	switch (obj->type) {
> +	case CR_OBJ_FILE:
> +		fput((struct file *) obj->ptr);
> +		break;
> +	default:
> +		BUG();
> +	}
> +}
> +
> +static void cr_obj_ref_grab(struct cr_objref *obj)
> +{
> +	switch (obj->type) {
> +	case CR_OBJ_FILE:
> +		get_file((struct file *) obj->ptr);
> +		break;
> +	default:
> +		BUG();
> +	}
> +}
> +
> +static void cr_objhash_clear(struct cr_objhash *objhash)
> +{
> +	struct hlist_head *h = objhash->head;
> +	struct hlist_node *n, *t;
> +	struct cr_objref *obj;
> +	int i;
> +
> +	for (i = 0; i < CR_OBJHASH_TOTAL; i++) {
> +		hlist_for_each_entry_safe(obj, n, t, &h[i], hash) {
> +			cr_obj_ref_drop(obj);
> +			kfree(obj);
> +		}
> +	}
> +}
> +
> +void cr_objhash_free(struct cr_ctx *ctx)
> +{
> +	struct cr_objhash *objhash = ctx->objhash;
> +
> +	if (objhash) {
> +		cr_objhash_clear(objhash);
> +		kfree(objhash->head);
> +		kfree(ctx->objhash);
> +		ctx->objhash = NULL;
> +	}
> +}
> +int cr_objhash_alloc(struct cr_ctx *ctx)
> +{
> +	struct cr_objhash *objhash;
> +	struct hlist_head *head;
> +
> +	objhash = kzalloc(sizeof(*objhash), GFP_KERNEL);
> +	if (!objhash)
> +		return -ENOMEM;
> +	head = kzalloc(CR_OBJHASH_TOTAL * sizeof(*head), GFP_KERNEL);
> +	if (!head) {
> +		kfree(objhash);
> +		return -ENOMEM;
> +	}
> +
> +	objhash->head = head;
> +	objhash->objref_index = 1;
> +
> +	ctx->objhash = objhash;
> +	return 0;
> +}
> +
> +static struct cr_objref *cr_obj_find_by_ptr(struct cr_ctx *ctx, void *ptr)
> +{
> +	struct hlist_head *h;
> +	struct hlist_node *n;
> +	struct cr_objref *obj;
> +
> +	h = &ctx->objhash->head[hash_ptr(ptr, CR_OBJHASH_NBITS)];
> +	hlist_for_each_entry(obj, n, h, hash)
> +		if (obj->ptr == ptr)
> +			return obj;
> +	return NULL;
> +}
> +
> +static struct cr_objref *cr_obj_find_by_objref(struct cr_ctx *ctx, int objref)
> +{
> +	struct hlist_head *h;
> +	struct hlist_node *n;
> +	struct cr_objref *obj;
> +
> +	h = &ctx->objhash->head[hash_ptr((void *) objref, CR_OBJHASH_NBITS)];
> +	hlist_for_each_entry(obj, n, h, hash)
> +		if (obj->objref == objref)
> +			return obj;
> +	return NULL;
> +}
> +
> +static struct cr_objref *cr_obj_new(struct cr_ctx *ctx, void *ptr, int objref,
> +				    unsigned short type, unsigned short flags)
> +{
> +	struct cr_objref *obj;
> +
> +	obj = kmalloc(sizeof(*obj), GFP_KERNEL);
> +	if (obj) {
> +		int i;
> +
> +		obj->ptr = ptr;
> +		obj->type = type;
> +		obj->flags = flags;
> +
> +		if (objref) {
> +			/* use 'objref' to index (restart) */
> +			obj->objref = objref;
> +			i = hash_ptr((void *) objref, CR_OBJHASH_NBITS);
> +		} else {
> +			/* use 'ptr' to index, assign objref (checkpoint) */
> +			obj->objref = ctx->objhash->objref_index++;;
> +			i = hash_ptr(ptr, CR_OBJHASH_NBITS);
> +		}
> +
> +		hlist_add_head(&obj->hash, &ctx->objhash->head[i]);
> +		cr_obj_ref_grab(obj);
> +	}
> +	return obj;
> +}
> +
> +/**
> + * cr_obj_add_ptr - add an object to the hash table if not already there
> + * @ctx: checkpoint context
> + * @ptr: pointer to object
> + * @objref: unique identifier - object reference [output]
> + * @type: object type
> + * @flags: object flags
> + *
> + * Fills the unique identifier of the object into @objref
> + *
> + * returns 0 if found, 1 if added, < 0 on error
> + */
> +int cr_obj_add_ptr(struct cr_ctx *ctx, void *ptr, int *objref,
> +		   unsigned short type, unsigned short flags)
> +{
> +	struct cr_objref *obj;
> +	int ret = 0;
> +
> +	obj = cr_obj_find_by_ptr(ctx, ptr);
> +	if (!obj) {
> +		obj = cr_obj_new(ctx, ptr, 0, type, flags);
> +		if (!obj)
> +			return -ENOMEM;
> +		else
> +			ret = 1;
> +	} else if (obj->type != type)	/* sanity check */
> +		return -EINVAL;
> +	*objref = obj->objref;
> +	return ret;
> +}
> +
> +/**
> + * cr_obj_add_ref - add an object with unique identifer to the hash table
> + * @ctx: checkpoint context
> + * @ptr: pointer to object
> + * @objref: unique identifier - object reference
> + * @type: object type
> + * @flags: object flags
> + */
> +int cr_obj_add_ref(struct cr_ctx *ctx, void *ptr, int objref,
> +		   unsigned short type, unsigned short flags)
> +{
> +	struct cr_objref *obj;
> +
> +	obj = cr_obj_new(ctx, ptr, objref, type, flags);
> +	return obj ? 0 : -ENOMEM;
> +}
> +
> +/**
> + * cr_obj_get_by_ptr - find the unique identifier (objref) of an object
> + * @ctx: checkpoint context
> + * @ptr: pointer to object
> + * @type: object type
> + */
> +int cr_obj_get_by_ptr(struct cr_ctx *ctx, void *ptr, unsigned short type)
> +{
> +	struct cr_objref *obj;
> +
> +	obj = cr_obj_find_by_ptr(ctx, ptr);
> +	if (obj)
> +		return obj->type == type ? obj->objref : -EINVAL;
> +	else
> +		return -ESRCH;
> +}

I have some nits about some of that.  I'd prefer to see the main code
flow of the functions get taken out of the if(){} blocks and put at the
first indenting level (cr_obj_new() could use this).

One other nit would be to try and get rid of some of the '?:' use.  I
personally find those hard to read and we don't use them that heavily in
the core kernel.  For instance, the above can be written:

	obj = cr_obj_find_by_ptr(ctx, ptr);
	if (!obj)
		return -ESRCH;
	if (obj->type != type)
		return -EINVAL;
	return obj->objref;

Which make the error conditions and their results just pop out much
easier at the cost of a single added line of code.

-- Dave

Re: [RFC v5][PATCH 7/8] Infrastructure for shared objects

[MinChan Kim]

On Wed, Sep 17, 2008 at 1:48 AM, Dave Hansen <dave-23VcF4HTsmIX0ybBhKVfKdBPR1lH4CV8@public.gmane.org> wrote:
> On Sat, 2008-09-13 at 19:06 -0400, Oren Laadan wrote:
>> +=== Shared resources (objects)
>> +
>> +Many resources used by tasks may be shared by more than one task (e.g.
>> +file descriptors, memory address space, etc), or even have multiple
>> +references from other resources (e.g. a single inode that represents
>> +two ends of a pipe).
>> +
>> +Clearly, the state of shared objects need only be saved once, even if
>> +they occur multiple times. We use a hash table (ctx->objhash) to keep
>> +track of shared objects in the following manner.
>> +
>> +On the first encounter, the state is dumped and the object is assigned
>> +a unique identifier and also stored in the hash table (indexed by its
>> +physical kenrel address). From then on the object will be found in the
>> +hash and only its identifier is saved.
>
> kernel?     ^^^^^^

One more thing, 'physical' is rather awkward.
Many kernel developer use physical address and virtual address with
different meaning. It is confusing them.
How about omitting 'physcial'?

>
>> +On restart the identifier is looked up in the hash table; if not found
>> +then the state is read, the object is created, and added to the hash
>> +table (this time indexed by its identifier). Otherwise, the object in
>> +the hash table is used.
>> +
>> +The interface for the hash table is the following:
>> +
>> +cr_obj_get_by_ptr - find the unique identifier - object reference (objref)
>> +  of the object that is pointer to by ptr (or 0 if not found) [checkpoint]
>> +
>> +cr_obj_add_ptr - add the object pointed to by ptr to the hash table if
>> +  it isn't already there, and fill its unique identifier (objref); will
>> +  return 0 if already found in the has, or 1 otherwise [checkpoint]
>> +
>> +cr_obj_get_by_ref - return the pointer to the object whose unique identifier
>> +  is equal to objref [restart]
>> +
>> +cr_obj_add_ref - add the object with unique identifier objref, pointed to by
>> +  ptr to the hash table if it isn't already there; will return 0 if already
>> +  found in the has, or 1 otherwise [restart]
>
> I'd much rather see all this documentation put properly inline with the
> source code.  I doubt anyone will actually find this stuff.
>
>> +
>>  === Changelog
>>
>>  [2008-Sep-11] v5:
>> diff --git a/checkpoint/Makefile b/checkpoint/Makefile
>> index ac35033..9843fb9 100644
>> --- a/checkpoint/Makefile
>> +++ b/checkpoint/Makefile
>> @@ -2,5 +2,5 @@
>>  # Makefile for linux checkpoint/restart.
>>  #
>>
>> -obj-$(CONFIG_CHECKPOINT_RESTART) += sys.o checkpoint.o restart.o \
>> +obj-$(CONFIG_CHECKPOINT_RESTART) += sys.o checkpoint.o restart.o objhash.o \
>>               ckpt_mem.o rstr_mem.o
>> diff --git a/checkpoint/objhash.c b/checkpoint/objhash.c
>> new file mode 100644
>> index 0000000..0862086
>> --- /dev/null
>> +++ b/checkpoint/objhash.c
>> @@ -0,0 +1,237 @@
>> +/*
>> + *  Checkpoint-restart - object hash infrastructure to manage shared objects
>> + *
>> + *  Copyright (C) 2008 Oren Laadan
>> + *
>> + *  This file is subject to the terms and conditions of the GNU General Public
>> + *  License.  See the file COPYING in the main directory of the Linux
>> + *  distribution for more details.
>> + */
>> +
>> +#include <linux/kernel.h>
>> +#include <linux/file.h>
>> +#include <linux/hash.h>
>> +#include <linux/checkpoint.h>
>> +
>> +struct cr_objref {
>> +     int objref;
>> +     void *ptr;
>> +     unsigned short type;
>> +     unsigned short flags;
>> +     struct hlist_node hash;
>> +};
>> +
>> +struct cr_objhash {
>> +     struct hlist_head *head;
>> +     int objref_index;
>> +};
>> +
>> +#define CR_OBJHASH_NBITS  10
>> +#define CR_OBJHASH_TOTAL  (1UL << CR_OBJHASH_NBITS)
>> +
>> +static void cr_obj_ref_drop(struct cr_objref *obj)
>> +{
>> +     switch (obj->type) {
>> +     case CR_OBJ_FILE:
>> +             fput((struct file *) obj->ptr);
>> +             break;
>> +     default:
>> +             BUG();
>> +     }
>> +}
>> +
>> +static void cr_obj_ref_grab(struct cr_objref *obj)
>> +{
>> +     switch (obj->type) {
>> +     case CR_OBJ_FILE:
>> +             get_file((struct file *) obj->ptr);
>> +             break;
>> +     default:
>> +             BUG();
>> +     }
>> +}
>> +
>> +static void cr_objhash_clear(struct cr_objhash *objhash)
>> +{
>> +     struct hlist_head *h = objhash->head;
>> +     struct hlist_node *n, *t;
>> +     struct cr_objref *obj;
>> +     int i;
>> +
>> +     for (i = 0; i < CR_OBJHASH_TOTAL; i++) {
>> +             hlist_for_each_entry_safe(obj, n, t, &h[i], hash) {
>> +                     cr_obj_ref_drop(obj);
>> +                     kfree(obj);
>> +             }
>> +     }
>> +}
>> +
>> +void cr_objhash_free(struct cr_ctx *ctx)
>> +{
>> +     struct cr_objhash *objhash = ctx->objhash;
>> +
>> +     if (objhash) {
>> +             cr_objhash_clear(objhash);
>> +             kfree(objhash->head);
>> +             kfree(ctx->objhash);
>> +             ctx->objhash = NULL;
>> +     }
>> +}
>> +int cr_objhash_alloc(struct cr_ctx *ctx)
>> +{
>> +     struct cr_objhash *objhash;
>> +     struct hlist_head *head;
>> +
>> +     objhash = kzalloc(sizeof(*objhash), GFP_KERNEL);
>> +     if (!objhash)
>> +             return -ENOMEM;
>> +     head = kzalloc(CR_OBJHASH_TOTAL * sizeof(*head), GFP_KERNEL);
>> +     if (!head) {
>> +             kfree(objhash);
>> +             return -ENOMEM;
>> +     }
>> +
>> +     objhash->head = head;
>> +     objhash->objref_index = 1;
>> +
>> +     ctx->objhash = objhash;
>> +     return 0;
>> +}
>> +
>> +static struct cr_objref *cr_obj_find_by_ptr(struct cr_ctx *ctx, void *ptr)
>> +{
>> +     struct hlist_head *h;
>> +     struct hlist_node *n;
>> +     struct cr_objref *obj;
>> +
>> +     h = &ctx->objhash->head[hash_ptr(ptr, CR_OBJHASH_NBITS)];
>> +     hlist_for_each_entry(obj, n, h, hash)
>> +             if (obj->ptr == ptr)
>> +                     return obj;
>> +     return NULL;
>> +}
>> +
>> +static struct cr_objref *cr_obj_find_by_objref(struct cr_ctx *ctx, int objref)
>> +{
>> +     struct hlist_head *h;
>> +     struct hlist_node *n;
>> +     struct cr_objref *obj;
>> +
>> +     h = &ctx->objhash->head[hash_ptr((void *) objref, CR_OBJHASH_NBITS)];
>> +     hlist_for_each_entry(obj, n, h, hash)
>> +             if (obj->objref == objref)
>> +                     return obj;
>> +     return NULL;
>> +}
>> +
>> +static struct cr_objref *cr_obj_new(struct cr_ctx *ctx, void *ptr, int objref,
>> +                                 unsigned short type, unsigned short flags)
>> +{
>> +     struct cr_objref *obj;
>> +
>> +     obj = kmalloc(sizeof(*obj), GFP_KERNEL);
>> +     if (obj) {
>> +             int i;
>> +
>> +             obj->ptr = ptr;
>> +             obj->type = type;
>> +             obj->flags = flags;
>> +
>> +             if (objref) {
>> +                     /* use 'objref' to index (restart) */
>> +                     obj->objref = objref;
>> +                     i = hash_ptr((void *) objref, CR_OBJHASH_NBITS);
>> +             } else {
>> +                     /* use 'ptr' to index, assign objref (checkpoint) */
>> +                     obj->objref = ctx->objhash->objref_index++;;
>> +                     i = hash_ptr(ptr, CR_OBJHASH_NBITS);
>> +             }
>> +
>> +             hlist_add_head(&obj->hash, &ctx->objhash->head[i]);
>> +             cr_obj_ref_grab(obj);
>> +     }
>> +     return obj;
>> +}
>> +
>> +/**
>> + * cr_obj_add_ptr - add an object to the hash table if not already there
>> + * @ctx: checkpoint context
>> + * @ptr: pointer to object
>> + * @objref: unique identifier - object reference [output]
>> + * @type: object type
>> + * @flags: object flags
>> + *
>> + * Fills the unique identifier of the object into @objref
>> + *
>> + * returns 0 if found, 1 if added, < 0 on error
>> + */
>> +int cr_obj_add_ptr(struct cr_ctx *ctx, void *ptr, int *objref,
>> +                unsigned short type, unsigned short flags)
>> +{
>> +     struct cr_objref *obj;
>> +     int ret = 0;
>> +
>> +     obj = cr_obj_find_by_ptr(ctx, ptr);
>> +     if (!obj) {
>> +             obj = cr_obj_new(ctx, ptr, 0, type, flags);
>> +             if (!obj)
>> +                     return -ENOMEM;
>> +             else
>> +                     ret = 1;
>> +     } else if (obj->type != type)   /* sanity check */
>> +             return -EINVAL;
>> +     *objref = obj->objref;
>> +     return ret;
>> +}
>> +
>> +/**
>> + * cr_obj_add_ref - add an object with unique identifer to the hash table
>> + * @ctx: checkpoint context
>> + * @ptr: pointer to object
>> + * @objref: unique identifier - object reference
>> + * @type: object type
>> + * @flags: object flags
>> + */
>> +int cr_obj_add_ref(struct cr_ctx *ctx, void *ptr, int objref,
>> +                unsigned short type, unsigned short flags)
>> +{
>> +     struct cr_objref *obj;
>> +
>> +     obj = cr_obj_new(ctx, ptr, objref, type, flags);
>> +     return obj ? 0 : -ENOMEM;
>> +}
>> +
>> +/**
>> + * cr_obj_get_by_ptr - find the unique identifier (objref) of an object
>> + * @ctx: checkpoint context
>> + * @ptr: pointer to object
>> + * @type: object type
>> + */
>> +int cr_obj_get_by_ptr(struct cr_ctx *ctx, void *ptr, unsigned short type)
>> +{
>> +     struct cr_objref *obj;
>> +
>> +     obj = cr_obj_find_by_ptr(ctx, ptr);
>> +     if (obj)
>> +             return obj->type == type ? obj->objref : -EINVAL;
>> +     else
>> +             return -ESRCH;
>> +}
>
> I have some nits about some of that.  I'd prefer to see the main code
> flow of the functions get taken out of the if(){} blocks and put at the
> first indenting level (cr_obj_new() could use this).
>
> One other nit would be to try and get rid of some of the '?:' use.  I
> personally find those hard to read and we don't use them that heavily in
> the core kernel.  For instance, the above can be written:
>
>        obj = cr_obj_find_by_ptr(ctx, ptr);
>        if (!obj)
>                return -ESRCH;
>        if (obj->type != type)
>                return -EINVAL;
>        return obj->objref;
>
> Which make the error conditions and their results just pop out much
> easier at the cost of a single added line of code.
>
> -- Dave
>
> --
> To unsubscribe from this list: send the line "unsubscribe linux-kernel" in
> the body of a message to majordomo-u79uwXL29TY76Z2rM5mHXA@public.gmane.org
> More majordomo info at  http://vger.kernel.org/majordomo-info.html
> Please read the FAQ at  http://www.tux.org/lkml/
>



-- 
Kinds regards,
MinChan Kim

Re: [RFC v5][PATCH 7/8] Infrastructure for shared objects

[Serge E. Hallyn]

Quoting Oren Laadan (orenl-eQaUEPhvms7ENvBUuze7eA@public.gmane.org):
> Infrastructure to handle objects that may be shared and referenced by
> multiple tasks or other objects, e..g open files, memory address space
> etc.
> 
> The state of shared objects is saved once. On the first encounter, the
> state is dumped and the object is assigned a unique identifier (objref)
> and also stored in a hash table (indexed by its physical kenrel address).
> >From then on the object will be found in the hash and only its identifier
> is saved.
> 
> On restart the identifier is looked up in the hash table; if not found
> then the state is read, the object is created, and added to the hash
> table (this time indexed by its identifier). Otherwise, the object in
> the hash table is used.
> 
> Signed-off-by: Oren Laadan <orenl-eQaUEPhvms7ENvBUuze7eA@public.gmane.org>

Acked-by: Serge Hallyn <serue-r/Jw6+rmf7HQT0dZR+AlfA@public.gmane.org>

Thanks, Oren, I actually think this is quite nice and readable.

Though three questions below.  First one is, since you've mentioned
having multiple threads doing checkpoint, won't you need some locking?
I assume that's coming in later patches if/when needed?

> ---
>  Documentation/checkpoint.txt |   38 +++++++
>  checkpoint/Makefile          |    2 +-
>  checkpoint/objhash.c         |  237 ++++++++++++++++++++++++++++++++++++++++++
>  checkpoint/sys.c             |    4 +
>  include/linux/checkpoint.h   |   20 ++++
>  5 files changed, 300 insertions(+), 1 deletions(-)
>  create mode 100644 checkpoint/objhash.c
> 
> diff --git a/Documentation/checkpoint.txt b/Documentation/checkpoint.txt
> index 6bf75ce..2929512 100644
> --- a/Documentation/checkpoint.txt
> +++ b/Documentation/checkpoint.txt
> @@ -169,6 +169,44 @@ cr_hdr + cr_hdr_task
>  cr_hdr + cr_hdr_tail
> 
> 
> +=== Shared resources (objects)
> +
> +Many resources used by tasks may be shared by more than one task (e.g.
> +file descriptors, memory address space, etc), or even have multiple
> +references from other resources (e.g. a single inode that represents
> +two ends of a pipe).
> +
> +Clearly, the state of shared objects need only be saved once, even if
> +they occur multiple times. We use a hash table (ctx->objhash) to keep
> +track of shared objects in the following manner.
> +
> +On the first encounter, the state is dumped and the object is assigned
> +a unique identifier and also stored in the hash table (indexed by its
> +physical kenrel address). From then on the object will be found in the
> +hash and only its identifier is saved.
> +
> +On restart the identifier is looked up in the hash table; if not found
> +then the state is read, the object is created, and added to the hash
> +table (this time indexed by its identifier). Otherwise, the object in
> +the hash table is used.
> +
> +The interface for the hash table is the following:
> +
> +cr_obj_get_by_ptr - find the unique identifier - object reference (objref)
> +  of the object that is pointer to by ptr (or 0 if not found) [checkpoint]
> +
> +cr_obj_add_ptr - add the object pointed to by ptr to the hash table if
> +  it isn't already there, and fill its unique identifier (objref); will
> +  return 0 if already found in the has, or 1 otherwise [checkpoint]
> +
> +cr_obj_get_by_ref - return the pointer to the object whose unique identifier
> +  is equal to objref [restart]
> +
> +cr_obj_add_ref - add the object with unique identifier objref, pointed to by
> +  ptr to the hash table if it isn't already there; will return 0 if already
> +  found in the has, or 1 otherwise [restart]
> +
> +
>  === Changelog
> 
>  [2008-Sep-11] v5:
> diff --git a/checkpoint/Makefile b/checkpoint/Makefile
> index ac35033..9843fb9 100644
> --- a/checkpoint/Makefile
> +++ b/checkpoint/Makefile
> @@ -2,5 +2,5 @@
>  # Makefile for linux checkpoint/restart.
>  #
> 
> -obj-$(CONFIG_CHECKPOINT_RESTART) += sys.o checkpoint.o restart.o \
> +obj-$(CONFIG_CHECKPOINT_RESTART) += sys.o checkpoint.o restart.o objhash.o \
>  		ckpt_mem.o rstr_mem.o
> diff --git a/checkpoint/objhash.c b/checkpoint/objhash.c
> new file mode 100644
> index 0000000..0862086
> --- /dev/null
> +++ b/checkpoint/objhash.c
> @@ -0,0 +1,237 @@
> +/*
> + *  Checkpoint-restart - object hash infrastructure to manage shared objects
> + *
> + *  Copyright (C) 2008 Oren Laadan
> + *
> + *  This file is subject to the terms and conditions of the GNU General Public
> + *  License.  See the file COPYING in the main directory of the Linux
> + *  distribution for more details.
> + */
> +
> +#include <linux/kernel.h>
> +#include <linux/file.h>
> +#include <linux/hash.h>
> +#include <linux/checkpoint.h>
> +
> +struct cr_objref {
> +	int objref;
> +	void *ptr;
> +	unsigned short type;

What is the point of the 'type'?

By that I mean: is it meant to catch bugs in the implementation, or bad
checkpoint images?

> +	unsigned short flags;
> +	struct hlist_node hash;
> +};
> +
> +struct cr_objhash {
> +	struct hlist_head *head;
> +	int objref_index;

What exactly will objref_index be used for?  I don't see any real
usage here or in your later patches.

> +};
> +
> +#define CR_OBJHASH_NBITS  10
> +#define CR_OBJHASH_TOTAL  (1UL << CR_OBJHASH_NBITS)
> +
> +static void cr_obj_ref_drop(struct cr_objref *obj)
> +{
> +	switch (obj->type) {
> +	case CR_OBJ_FILE:
> +		fput((struct file *) obj->ptr);
> +		break;
> +	default:
> +		BUG();
> +	}
> +}
> +
> +static void cr_obj_ref_grab(struct cr_objref *obj)
> +{
> +	switch (obj->type) {
> +	case CR_OBJ_FILE:
> +		get_file((struct file *) obj->ptr);
> +		break;
> +	default:
> +		BUG();
> +	}
> +}
> +
> +static void cr_objhash_clear(struct cr_objhash *objhash)
> +{
> +	struct hlist_head *h = objhash->head;
> +	struct hlist_node *n, *t;
> +	struct cr_objref *obj;
> +	int i;
> +
> +	for (i = 0; i < CR_OBJHASH_TOTAL; i++) {
> +		hlist_for_each_entry_safe(obj, n, t, &h[i], hash) {
> +			cr_obj_ref_drop(obj);
> +			kfree(obj);
> +		}
> +	}
> +}
> +
> +void cr_objhash_free(struct cr_ctx *ctx)
> +{
> +	struct cr_objhash *objhash = ctx->objhash;
> +
> +	if (objhash) {
> +		cr_objhash_clear(objhash);
> +		kfree(objhash->head);
> +		kfree(ctx->objhash);
> +		ctx->objhash = NULL;
> +	}
> +}
> +
> +int cr_objhash_alloc(struct cr_ctx *ctx)
> +{
> +	struct cr_objhash *objhash;
> +	struct hlist_head *head;
> +
> +	objhash = kzalloc(sizeof(*objhash), GFP_KERNEL);
> +	if (!objhash)
> +		return -ENOMEM;
> +	head = kzalloc(CR_OBJHASH_TOTAL * sizeof(*head), GFP_KERNEL);
> +	if (!head) {
> +		kfree(objhash);
> +		return -ENOMEM;
> +	}
> +
> +	objhash->head = head;
> +	objhash->objref_index = 1;
> +
> +	ctx->objhash = objhash;
> +	return 0;
> +}
> +
> +static struct cr_objref *cr_obj_find_by_ptr(struct cr_ctx *ctx, void *ptr)
> +{
> +	struct hlist_head *h;
> +	struct hlist_node *n;
> +	struct cr_objref *obj;
> +
> +	h = &ctx->objhash->head[hash_ptr(ptr, CR_OBJHASH_NBITS)];
> +	hlist_for_each_entry(obj, n, h, hash)
> +		if (obj->ptr == ptr)
> +			return obj;
> +	return NULL;
> +}
> +
> +static struct cr_objref *cr_obj_find_by_objref(struct cr_ctx *ctx, int objref)
> +{
> +	struct hlist_head *h;
> +	struct hlist_node *n;
> +	struct cr_objref *obj;
> +
> +	h = &ctx->objhash->head[hash_ptr((void *) objref, CR_OBJHASH_NBITS)];
> +	hlist_for_each_entry(obj, n, h, hash)
> +		if (obj->objref == objref)
> +			return obj;
> +	return NULL;
> +}
> +
> +static struct cr_objref *cr_obj_new(struct cr_ctx *ctx, void *ptr, int objref,
> +				    unsigned short type, unsigned short flags)
> +{
> +	struct cr_objref *obj;
> +
> +	obj = kmalloc(sizeof(*obj), GFP_KERNEL);
> +	if (obj) {
> +		int i;
> +
> +		obj->ptr = ptr;
> +		obj->type = type;
> +		obj->flags = flags;
> +
> +		if (objref) {
> +			/* use 'objref' to index (restart) */
> +			obj->objref = objref;
> +			i = hash_ptr((void *) objref, CR_OBJHASH_NBITS);
> +		} else {
> +			/* use 'ptr' to index, assign objref (checkpoint) */
> +			obj->objref = ctx->objhash->objref_index++;;
> +			i = hash_ptr(ptr, CR_OBJHASH_NBITS);
> +		}
> +
> +		hlist_add_head(&obj->hash, &ctx->objhash->head[i]);
> +		cr_obj_ref_grab(obj);
> +	}
> +	return obj;
> +}
> +
> +/**
> + * cr_obj_add_ptr - add an object to the hash table if not already there
> + * @ctx: checkpoint context
> + * @ptr: pointer to object
> + * @objref: unique identifier - object reference [output]
> + * @type: object type
> + * @flags: object flags
> + *
> + * Fills the unique identifier of the object into @objref
> + *
> + * returns 0 if found, 1 if added, < 0 on error
> + */
> +int cr_obj_add_ptr(struct cr_ctx *ctx, void *ptr, int *objref,
> +		   unsigned short type, unsigned short flags)
> +{
> +	struct cr_objref *obj;
> +	int ret = 0;
> +
> +	obj = cr_obj_find_by_ptr(ctx, ptr);
> +	if (!obj) {
> +		obj = cr_obj_new(ctx, ptr, 0, type, flags);
> +		if (!obj)
> +			return -ENOMEM;
> +		else
> +			ret = 1;
> +	} else if (obj->type != type)	/* sanity check */
> +		return -EINVAL;
> +	*objref = obj->objref;
> +	return ret;
> +}
> +
> +/**
> + * cr_obj_add_ref - add an object with unique identifer to the hash table
> + * @ctx: checkpoint context
> + * @ptr: pointer to object
> + * @objref: unique identifier - object reference
> + * @type: object type
> + * @flags: object flags
> + */
> +int cr_obj_add_ref(struct cr_ctx *ctx, void *ptr, int objref,
> +		   unsigned short type, unsigned short flags)
> +{
> +	struct cr_objref *obj;
> +
> +	obj = cr_obj_new(ctx, ptr, objref, type, flags);
> +	return obj ? 0 : -ENOMEM;
> +}
> +
> +/**
> + * cr_obj_get_by_ptr - find the unique identifier (objref) of an object
> + * @ctx: checkpoint context
> + * @ptr: pointer to object
> + * @type: object type
> + */
> +int cr_obj_get_by_ptr(struct cr_ctx *ctx, void *ptr, unsigned short type)
> +{
> +	struct cr_objref *obj;
> +
> +	obj = cr_obj_find_by_ptr(ctx, ptr);
> +	if (obj)
> +		return obj->type == type ? obj->objref : -EINVAL;
> +	else
> +		return -ESRCH;
> +}
> +
> +/**
> + * cr_obj_get_by_ref - find an object given its unique identifier (objref)
> + * @ctx: checkpoint context
> + * @objref: unique identifier - object reference
> + * @type: object type
> + */
> +void *cr_obj_get_by_ref(struct cr_ctx *ctx, int objref, unsigned short type)
> +{
> +	struct cr_objref *obj;
> +
> +	obj = cr_obj_find_by_objref(ctx, objref);
> +	if (obj)
> +		return obj->type == type ? obj->ptr : ERR_PTR(-EINVAL);
> +	else
> +		return NULL;
> +}
> diff --git a/checkpoint/sys.c b/checkpoint/sys.c
> index c4ac157..3b17ecc 100644
> --- a/checkpoint/sys.c
> +++ b/checkpoint/sys.c
> @@ -132,6 +132,7 @@ void cr_ctx_free(struct cr_ctx *ctx)
>  		path_put(ctx->vfsroot);
> 
>  	cr_pgarr_free(ctx);
> +	cr_objhash_free(ctx);
> 
>  	kfree(ctx);
>  }
> @@ -154,6 +155,9 @@ struct cr_ctx *cr_ctx_alloc(pid_t pid, int fd, unsigned long flags)
>  	if (!ctx->hbuf)
>  		goto nomem;
> 
> +	if (cr_objhash_alloc(ctx) < 0)
> +		goto nomem;
> +
>  	/*
>  	 * assume checkpointer is in container's root vfs
>  	 * FIXME: this works for now, but will change with real containers
> diff --git a/include/linux/checkpoint.h b/include/linux/checkpoint.h
> index a9eafac..9990bb1 100644
> --- a/include/linux/checkpoint.h
> +++ b/include/linux/checkpoint.h
> @@ -28,6 +28,8 @@ struct cr_ctx {
>  	void *hbuf;		/* temporary buffer for headers */
>  	int hpos;		/* position in headers buffer */
> 
> +	struct cr_objhash *objhash;	/* hash for shared objects */
> +
>  	struct list_head pgarr_list;	/* page array to dump VMA contents */
> 
>  	struct path *vfsroot;	/* container root (FIXME) */
> @@ -49,6 +51,24 @@ extern int cr_kread(struct cr_ctx *ctx, void *buf, int count);
>  extern void *cr_hbuf_get(struct cr_ctx *ctx, int n);
>  extern void cr_hbuf_put(struct cr_ctx *ctx, int n);
> 
> +/* shared objects handling */
> +
> +enum {
> +	CR_OBJ_FILE = 1,
> +	CR_OBJ_MAX
> +};
> +
> +extern void cr_objhash_free(struct cr_ctx *ctx);
> +extern int cr_objhash_alloc(struct cr_ctx *ctx);
> +extern void *cr_obj_get_by_ref(struct cr_ctx *ctx,
> +			       int objref, unsigned short type);
> +extern int cr_obj_get_by_ptr(struct cr_ctx *ctx,
> +			     void *ptr, unsigned short type);
> +extern int cr_obj_add_ptr(struct cr_ctx *ctx, void *ptr, int *objref,
> +			  unsigned short type, unsigned short flags);
> +extern int cr_obj_add_ref(struct cr_ctx *ctx, void *ptr, int objref,
> +			  unsigned short type, unsigned short flags);
> +
>  struct cr_hdr;
> 
>  extern int cr_write_obj(struct cr_ctx *ctx, struct cr_hdr *h, void *buf);
> -- 
> 1.5.4.3
> 
> _______________________________________________
> Containers mailing list
> Containers-cunTk1MwBs9QetFLy7KEm3xJsTq8ys+cHZ5vskTnxNA@public.gmane.org
> https://lists.linux-foundation.org/mailman/listinfo/containers

Re: [RFC v5][PATCH 7/8] Infrastructure for shared objects

[Oren Laadan]

Serge E. Hallyn wrote:
> Quoting Oren Laadan (orenl-eQaUEPhvms7ENvBUuze7eA@public.gmane.org):
>> Infrastructure to handle objects that may be shared and referenced by
>> multiple tasks or other objects, e..g open files, memory address space
>> etc.
>>
>> The state of shared objects is saved once. On the first encounter, the
>> state is dumped and the object is assigned a unique identifier (objref)
>> and also stored in a hash table (indexed by its physical kenrel address).
>> >From then on the object will be found in the hash and only its identifier
>> is saved.
>>
>> On restart the identifier is looked up in the hash table; if not found
>> then the state is read, the object is created, and added to the hash
>> table (this time indexed by its identifier). Otherwise, the object in
>> the hash table is used.
>>
>> Signed-off-by: Oren Laadan <orenl-eQaUEPhvms7ENvBUuze7eA@public.gmane.org>
> 
> Acked-by: Serge Hallyn <serue-r/Jw6+rmf7HQT0dZR+AlfA@public.gmane.org>
> 
> Thanks, Oren, I actually think this is quite nice and readable.
> 
> Though three questions below.  First one is, since you've mentioned
> having multiple threads doing checkpoint, won't you need some locking?

yes.

> I assume that's coming in later patches if/when needed?

yes.

[...]

>> +++ b/checkpoint/objhash.c
>> @@ -0,0 +1,237 @@
>> +/*
>> + *  Checkpoint-restart - object hash infrastructure to manage shared objects
>> + *
>> + *  Copyright (C) 2008 Oren Laadan
>> + *
>> + *  This file is subject to the terms and conditions of the GNU General Public
>> + *  License.  See the file COPYING in the main directory of the Linux
>> + *  distribution for more details.
>> + */
>> +
>> +#include <linux/kernel.h>
>> +#include <linux/file.h>
>> +#include <linux/hash.h>
>> +#include <linux/checkpoint.h>
>> +
>> +struct cr_objref {
>> +	int objref;
>> +	void *ptr;
>> +	unsigned short type;
> 
> What is the point of the 'type'?
> 
> By that I mean: is it meant to catch bugs in the implementation, or bad
> checkpoint images?

The hash table is maintained in the kernel during checkpoint/restart,
adding shared objects to the hash table when they are first seen. An
object can be a 'struct file', 'struct mm_struct', etc.

When an object is added, it's reference count is incremented to ensure
that the pointer is still valid for as long as it's in the hash table.
Similarly, when we remove an object from the hash, we need to decrement
the reference count. This is done in cr_obj_ref_grab(), cr_obj_ref_drop().

There are different functions to inc/dec the reference count of objects
of different types. '->type' keeps track of the type of the object, so
we know which function to call. (At this point, we only track shared
'struct file' so it isn't that clear from the code).

> 
>> +	unsigned short flags;
>> +	struct hlist_node hash;
>> +};
>> +
>> +struct cr_objhash {
>> +	struct hlist_head *head;
>> +	int objref_index;
> 
> What exactly will objref_index be used for?  I don't see any real
> usage here or in your later patches.
> 

'objref_index' is a counter used to assign unique identifiers (objref)
to objects as they are added to the hash table. It is incremented with
every object that joins the hash table (and the old value is used as
the objref of that object). It is used in cr_obj_new().

>> +};
>> +
>> +#define CR_OBJHASH_NBITS  10
>> +#define CR_OBJHASH_TOTAL  (1UL << CR_OBJHASH_NBITS)
>> +
>> +static void cr_obj_ref_drop(struct cr_objref *obj)
>> +{
>> +	switch (obj->type) {
>> +	case CR_OBJ_FILE:
>> +		fput((struct file *) obj->ptr);
>> +		break;
>> +	default:
>> +		BUG();
>> +	}
>> +}
>> +
>> +static void cr_obj_ref_grab(struct cr_objref *obj)
>> +{
>> +	switch (obj->type) {
>> +	case CR_OBJ_FILE:
>> +		get_file((struct file *) obj->ptr);
>> +		break;
>> +	default:
>> +		BUG();
>> +	}
>> +}
>> +

[...]

>> +static struct cr_objref *cr_obj_new(struct cr_ctx *ctx, void *ptr, int objref,
>> +				    unsigned short type, unsigned short flags)
>> +{
>> +	struct cr_objref *obj;
>> +
>> +	obj = kmalloc(sizeof(*obj), GFP_KERNEL);
>> +	if (obj) {
>> +		int i;
>> +
>> +		obj->ptr = ptr;
>> +		obj->type = type;
>> +		obj->flags = flags;
>> +
>> +		if (objref) {
>> +			/* use 'objref' to index (restart) */
>> +			obj->objref = objref;
>> +			i = hash_ptr((void *) objref, CR_OBJHASH_NBITS);
>> +		} else {
>> +			/* use 'ptr' to index, assign objref (checkpoint) */
>> +			obj->objref = ctx->objhash->objref_index++;;
>> +			i = hash_ptr(ptr, CR_OBJHASH_NBITS);
>> +		}
>> +
>> +		hlist_add_head(&obj->hash, &ctx->objhash->head[i]);
>> +		cr_obj_ref_grab(obj);
>> +	}
>> +	return obj;
>> +}

[...]

Oren.

Re: [RFC v5][PATCH 7/8] Infrastructure for shared objects

[Serge E. Hallyn]

Quoting Oren Laadan (orenl-eQaUEPhvms7ENvBUuze7eA@public.gmane.org):
> 
> 
> Serge E. Hallyn wrote:
> > Quoting Oren Laadan (orenl-eQaUEPhvms7ENvBUuze7eA@public.gmane.org):
> >> Infrastructure to handle objects that may be shared and referenced by
> >> multiple tasks or other objects, e..g open files, memory address space
> >> etc.
> >>
> >> The state of shared objects is saved once. On the first encounter, the
> >> state is dumped and the object is assigned a unique identifier (objref)
> >> and also stored in a hash table (indexed by its physical kenrel address).
> >> >From then on the object will be found in the hash and only its identifier
> >> is saved.
> >>
> >> On restart the identifier is looked up in the hash table; if not found
> >> then the state is read, the object is created, and added to the hash
> >> table (this time indexed by its identifier). Otherwise, the object in
> >> the hash table is used.
> >>
> >> Signed-off-by: Oren Laadan <orenl-eQaUEPhvms7ENvBUuze7eA@public.gmane.org>
> > 
> > Acked-by: Serge Hallyn <serue-r/Jw6+rmf7HQT0dZR+AlfA@public.gmane.org>
> > 
> > Thanks, Oren, I actually think this is quite nice and readable.
> > 
> > Though three questions below.  First one is, since you've mentioned
> > having multiple threads doing checkpoint, won't you need some locking?
> 
> yes.
> 
> > I assume that's coming in later patches if/when needed?
> 
> yes.
> 
> [...]
> 
> >> +++ b/checkpoint/objhash.c
> >> @@ -0,0 +1,237 @@
> >> +/*
> >> + *  Checkpoint-restart - object hash infrastructure to manage shared objects
> >> + *
> >> + *  Copyright (C) 2008 Oren Laadan
> >> + *
> >> + *  This file is subject to the terms and conditions of the GNU General Public
> >> + *  License.  See the file COPYING in the main directory of the Linux
> >> + *  distribution for more details.
> >> + */
> >> +
> >> +#include <linux/kernel.h>
> >> +#include <linux/file.h>
> >> +#include <linux/hash.h>
> >> +#include <linux/checkpoint.h>
> >> +
> >> +struct cr_objref {
> >> +	int objref;
> >> +	void *ptr;
> >> +	unsigned short type;
> > 
> > What is the point of the 'type'?
> > 
> > By that I mean: is it meant to catch bugs in the implementation, or bad
> > checkpoint images?
> 

...

> There are different functions to inc/dec the reference count of objects
> of different types. '->type' keeps track of the type of the object, so
> we know which function to call. (At this point, we only track shared
> 'struct file' so it isn't that clear from the code).

Doh, right.

> 
> > 
> >> +	unsigned short flags;
> >> +	struct hlist_node hash;
> >> +};
> >> +
> >> +struct cr_objhash {
> >> +	struct hlist_head *head;
> >> +	int objref_index;
> > 
> > What exactly will objref_index be used for?  I don't see any real
> > usage here or in your later patches.
> > 
> 
> 'objref_index' is a counter used to assign unique identifiers (objref)
> to objects as they are added to the hash table. It is incremented with
> every object that joins the hash table (and the old value is used as
> the objref of that object). It is used in cr_obj_new().

If you were to rename objref_index to next_free_ref, I think that'd
be helpful.  (I also don't think 'obj' should in the name at all, and
you should just s/objref/id/, but that's really too much pain)

thanks,
-serge

[RFC v5][PATCH 8/8] Dump open file descriptors

[Oren Laadan]

Dump the files_struct of a task with 'struct cr_hdr_files', followed by
all open file descriptors. Since FDs can be shared, they are assigned an
objref and registered in the object hash.

For each open FD there is a 'struct cr_hdr_fd_ent' with the FD, its objref
and its close-on-exec property. If the FD is to be saved (first time)
then this is followed by a 'struct cr_hdr_fd_data' with the FD state.
Then will come the next FD and so on.

This patch only handles basic FDs - regular files, directories and also
symbolic links.

Signed-off-by: Oren Laadan <orenl-eQaUEPhvms7ENvBUuze7eA@public.gmane.org>
---
 checkpoint/Makefile            |    2 +-
 checkpoint/checkpoint.c        |    4 +
 checkpoint/checkpoint_file.h   |   17 +++
 checkpoint/ckpt_file.c         |  221 ++++++++++++++++++++++++++++++++++++++++
 include/linux/checkpoint.h     |    7 +-
 include/linux/checkpoint_hdr.h |   32 ++++++-
 6 files changed, 278 insertions(+), 5 deletions(-)
 create mode 100644 checkpoint/checkpoint_file.h
 create mode 100644 checkpoint/ckpt_file.c

diff --git a/checkpoint/Makefile b/checkpoint/Makefile
index 9843fb9..7496695 100644
--- a/checkpoint/Makefile
+++ b/checkpoint/Makefile
@@ -3,4 +3,4 @@
 #
 
 obj-$(CONFIG_CHECKPOINT_RESTART) += sys.o checkpoint.o restart.o objhash.o \
-		ckpt_mem.o rstr_mem.o
+		ckpt_mem.o rstr_mem.o ckpt_file.o
diff --git a/checkpoint/checkpoint.c b/checkpoint/checkpoint.c
index d4c1b31..87420dc 100644
--- a/checkpoint/checkpoint.c
+++ b/checkpoint/checkpoint.c
@@ -203,6 +203,10 @@ static int cr_write_task(struct cr_ctx *ctx, struct task_struct *t)
 	cr_debug("memory: ret %d\n", ret);
 	if (ret < 0)
 		goto out;
+	ret = cr_write_files(ctx, t);
+	cr_debug("files: ret %d\n", ret);
+	if (ret < 0)
+		goto out;
 	ret = cr_write_thread(ctx, t);
 	cr_debug("thread: ret %d\n", ret);
 	if (ret < 0)
diff --git a/checkpoint/checkpoint_file.h b/checkpoint/checkpoint_file.h
new file mode 100644
index 0000000..9dc3eba
--- /dev/null
+++ b/checkpoint/checkpoint_file.h
@@ -0,0 +1,17 @@
+#ifndef _CHECKPOINT_CKPT_FILE_H_
+#define _CHECKPOINT_CKPT_FILE_H_
+/*
+ *  Checkpoint file descriptors
+ *
+ *  Copyright (C) 2008 Oren Laadan
+ *
+ *  This file is subject to the terms and conditions of the GNU General Public
+ *  License.  See the file COPYING in the main directory of the Linux
+ *  distribution for more details.
+ */
+
+#include <linux/fdtable.h>
+
+int cr_scan_fds(struct files_struct *files, int **fdtable);
+
+#endif /* _CHECKPOINT_CKPT_FILE_H_ */
diff --git a/checkpoint/ckpt_file.c b/checkpoint/ckpt_file.c
new file mode 100644
index 0000000..bf049d1
--- /dev/null
+++ b/checkpoint/ckpt_file.c
@@ -0,0 +1,221 @@
+/*
+ *  Checkpoint file descriptors
+ *
+ *  Copyright (C) 2008 Oren Laadan
+ *
+ *  This file is subject to the terms and conditions of the GNU General Public
+ *  License.  See the file COPYING in the main directory of the Linux
+ *  distribution for more details.
+ */
+
+#include <linux/kernel.h>
+#include <linux/sched.h>
+#include <linux/file.h>
+#include <linux/fdtable.h>
+#include <linux/checkpoint.h>
+#include <linux/checkpoint_hdr.h>
+
+#include "checkpoint_file.h"
+
+#define CR_DEFAULT_FDTABLE  256		/* an initial guess */
+
+/**
+ * cr_scan_fds - scan file table and construct array of open fds
+ * @files: files_struct pointer
+ * @fdtable: (output) array of open fds
+ *
+ * Returns the number of open fds found, and also the file table
+ * array via *fdtable. The caller should free the array.
+ */
+int cr_scan_fds(struct files_struct *files, int **fdtable)
+{
+	struct fdtable *fdt;
+	int *fds;
+	int i, n, tot;
+
+	n = 0;
+	tot = CR_DEFAULT_FDTABLE;
+
+	fds = kmalloc(tot * sizeof(*fds), GFP_KERNEL);
+	if (!fds)
+		return -ENOMEM;
+
+	spin_lock(&files->file_lock);
+	fdt = files_fdtable(files);
+	for (i = 0; i < fdt->max_fds; i++) {
+		if (!fcheck_files(files, i))
+			continue;
+		if (n == tot) {
+			/*
+			 * fcheck_files() is safe with drop/re-acquire
+			 * of the lock, because it tests:  fd < max_fds
+			 */
+			spin_unlock(&files->file_lock);
+			tot *= 2;
+			if (tot < 0) {		/* overflow ? */
+				kfree(fds);
+				return -EMFILE;
+			}
+			fds = krealloc(fds, tot * sizeof(*fds), GFP_KERNEL);
+			if (!fds)
+				return -ENOMEM;
+			spin_lock(&files->file_lock);
+		}
+		fds[n++] = i;
+	}
+	spin_unlock(&files->file_lock);
+
+	*fdtable = fds;
+	return n;
+}
+
+/* cr_write_fd_data - dump the state of a given file pointer */
+static int cr_write_fd_data(struct cr_ctx *ctx, struct file *file, int parent)
+{
+	struct cr_hdr h;
+	struct cr_hdr_fd_data *hh = cr_hbuf_get(ctx, sizeof(*hh));
+	struct dentry *dent = file->f_dentry;
+	struct inode *inode = dent->d_inode;
+	enum fd_type fd_type;
+	int ret;
+
+	h.type = CR_HDR_FD_DATA;
+	h.len = sizeof(*hh);
+	h.parent = parent;
+
+	hh->f_flags = file->f_flags;
+	hh->f_mode = file->f_mode;
+	hh->f_pos = file->f_pos;
+	hh->f_version = file->f_version;
+	/* FIX: need also file->uid, file->gid, file->f_owner, etc */
+
+	switch (inode->i_mode & S_IFMT) {
+	case S_IFREG:
+		fd_type = CR_FD_FILE;
+		break;
+	case S_IFDIR:
+		fd_type = CR_FD_DIR;
+		break;
+	case S_IFLNK:
+		fd_type = CR_FD_LINK;
+		break;
+	default:
+		return -EBADF;
+	}
+
+	/* FIX: check if the file/dir/link is unlinked */
+	hh->fd_type = fd_type;
+
+	ret = cr_write_obj(ctx, &h, hh);
+	cr_hbuf_put(ctx, sizeof(*hh));
+	if (ret < 0)
+		return ret;
+
+	return cr_write_fname(ctx, &file->f_path, ctx->vfsroot);
+}
+
+/**
+ * cr_write_fd_ent - dump the state of a given file descriptor
+ * @ctx: checkpoint context
+ * @files: files_struct pointer
+ * @fd: file descriptor
+ *
+ * Saves the state of the file descriptor; looks up the actual file
+ * pointer in the hash table, and if found saves the matching objref,
+ * otherwise calls cr_write_fd_data to dump the file pointer too.
+ */
+static int
+cr_write_fd_ent(struct cr_ctx *ctx, struct files_struct *files, int fd)
+{
+	struct cr_hdr h;
+	struct cr_hdr_fd_ent *hh = cr_hbuf_get(ctx, sizeof(*hh));
+	struct file *file = NULL;
+	struct fdtable *fdt;
+	int coe, objref, new, ret;
+
+	rcu_read_lock();
+	fdt = files_fdtable(files);
+	file = fcheck_files(files, fd);
+	if (file) {
+		coe = FD_ISSET(fd, fdt->close_on_exec);
+		get_file(file);
+	}
+	rcu_read_unlock();
+
+	/* sanity check (although this shouldn't happen) */
+	if (!file)
+		return -EBADF;
+
+	new = cr_obj_add_ptr(ctx, (void *) file, &objref, CR_OBJ_FILE, 0);
+	cr_debug("fd %d objref %d file %p c-o-e %d)\n", fd, objref, file, coe);
+
+	if (new < 0) {
+		ret = new;
+		goto out;
+	}
+
+	h.type = CR_HDR_FD_ENT;
+	h.len = sizeof(*hh);
+	h.parent = 0;
+
+	hh->objref = objref;
+	hh->fd = fd;
+	hh->close_on_exec = coe;
+
+	ret = cr_write_obj(ctx, &h, hh);
+	cr_hbuf_put(ctx, sizeof(*hh));
+	if (ret < 0)
+		goto out;
+
+	/* new==1 if-and-only-if file was newly added to hash */
+	if (new)
+		ret = cr_write_fd_data(ctx, file, objref);
+
+out:
+	fput(file);
+	return ret;
+}
+
+int cr_write_files(struct cr_ctx *ctx, struct task_struct *t)
+{
+	struct cr_hdr h;
+	struct cr_hdr_files *hh = cr_hbuf_get(ctx, sizeof(*hh));
+	struct files_struct *files;
+	int *fdtable;
+	int nfds, n, ret;
+
+	h.type = CR_HDR_FILES;
+	h.len = sizeof(*hh);
+	h.parent = task_pid_vnr(t);
+
+	files = get_files_struct(t);
+
+	hh->objref = 0;	/* will be meaningful with multiple processes */
+
+	nfds = cr_scan_fds(files, &fdtable);
+	if (nfds < 0) {
+		ret = nfds;
+		goto out;
+	}
+
+	hh->nfds = nfds;
+
+	ret = cr_write_obj(ctx, &h, hh);
+	cr_hbuf_put(ctx, sizeof(*hh));
+	if (ret < 0)
+		goto clean;
+
+	cr_debug("nfds %d\n", nfds);
+	for (n = 0; n < nfds; n++) {
+		ret = cr_write_fd_ent(ctx, files, fdtable[n]);
+		if (ret < 0)
+			break;
+	}
+
+ clean:
+	kfree(fdtable);
+ out:
+	put_files_struct(files);
+
+	return ret;
+}
diff --git a/include/linux/checkpoint.h b/include/linux/checkpoint.h
index 9990bb1..a202c54 100644
--- a/include/linux/checkpoint.h
+++ b/include/linux/checkpoint.h
@@ -13,7 +13,7 @@
 #include <linux/path.h>
 #include <linux/fs.h>
 
-#define CR_VERSION  1
+#define CR_VERSION  2
 
 struct cr_ctx {
 	pid_t pid;		/* container identifier */
@@ -83,11 +83,12 @@ extern int cr_read_fname(struct cr_ctx *ctx, void *fname, int n);
 extern struct file *cr_read_open_fname(struct cr_ctx *ctx,
 				       int flags, int mode);
 
+extern int do_checkpoint(struct cr_ctx *ctx);
 extern int cr_write_mm(struct cr_ctx *ctx, struct task_struct *t);
-extern int cr_read_mm(struct cr_ctx *ctx);
+extern int cr_write_files(struct cr_ctx *ctx, struct task_struct *t);
 
-extern int do_checkpoint(struct cr_ctx *ctx);
 extern int do_restart(struct cr_ctx *ctx);
+extern int cr_read_mm(struct cr_ctx *ctx);
 
 #define cr_debug(fmt, args...)  \
 	pr_debug("[CR:%s] " fmt, __func__, ## args)
diff --git a/include/linux/checkpoint_hdr.h b/include/linux/checkpoint_hdr.h
index 2b110f1..cbb920f 100644
--- a/include/linux/checkpoint_hdr.h
+++ b/include/linux/checkpoint_hdr.h
@@ -17,7 +17,7 @@
 /*
  * To maintain compatibility between 32-bit and 64-bit architecture flavors,
  * keep data 64-bit aligned: use padding for structure members, and use
- * __attribute__ ((aligned (8))) for the entire structure.
+ * __attribute__((aligned(8))) for the entire structure.
  */
 
 /* records: generic header */
@@ -43,6 +43,10 @@ enum {
 	CR_HDR_PGARR,
 	CR_HDR_MM_CONTEXT,
 
+	CR_HDR_FILES = 301,
+	CR_HDR_FD_ENT,
+	CR_HDR_FD_DATA,
+
 	CR_HDR_TAIL = 5001
 };
 
@@ -105,4 +109,30 @@ struct cr_hdr_pgarr {
 	__u64 nr_pages;		/* number of pages to saved */
 } __attribute__((aligned(8)));
 
+struct cr_hdr_files {
+	__u32 objref;		/* identifier for shared objects */
+	__u32 nfds;
+} __attribute__((aligned(8)));
+
+struct cr_hdr_fd_ent {
+	__u32 objref;		/* identifier for shared objects */
+	__s32 fd;
+	__u32 close_on_exec;
+} __attribute__((aligned(8)));
+
+/* fd types */
+enum  fd_type {
+	CR_FD_FILE = 1,
+	CR_FD_DIR,
+	CR_FD_LINK
+};
+
+struct cr_hdr_fd_data {
+	__u16 fd_type;
+	__u16 f_mode;
+	__u32 f_flags;
+	__u64 f_pos;
+	__u64 f_version;
+} __attribute__((aligned(8)));
+
 #endif /* _CHECKPOINT_CKPT_HDR_H_ */
-- 
1.5.4.3

Re: [RFC v5][PATCH 8/8] Dump open file descriptors

[Oren Laadan]

Bastian Blank wrote:
> On Sat, Sep 13, 2008 at 07:06:06PM -0400, Oren Laadan wrote:
>> +int cr_scan_fds(struct files_struct *files, int **fdtable)
>> +{
>> +	struct fdtable *fdt;
>> +	int *fds;
>> +	int i, n, tot;
>> +
>> +	n = 0;
>> +	tot = CR_DEFAULT_FDTABLE;
> 
> Why not?
> | int i;
> | int n = 0;
> | int tot = CR_DEFAULT_FDTABLE;
> 
> IHMO easier readable.

Ok.

> 
>> +	spin_lock(&files->file_lock);
>> +	fdt = files_fdtable(files);
>> +	for (i = 0; i < fdt->max_fds; i++) {
> 
> The process is suspended at this state?

Yes, the assumption is that the process is frozen (or that we checkpoint
ourselves).

With this assumption, it is possible to (a) leave out RCU locking, and also
(b) continue after the krealloc() from where we left off. Also, it means that
(c) the contents of our 'fds' array remain valid by the time the caller makes
use of it.

This certainly deserves a comment in the code, will add.

If the assumption doesn't hold, then I'll have to add the RCU locking. Cases
(b) and (c) are already safe because the caller(s) use fcheck_files() to
access the file-descriptors collected in the array.

While in my mind a task should never be unfrozen while being checkpointed, in
reality future code may allow it e.g. if a OOM kicks in a kills it. So I tend
to add the RCU lock for safety. It can always be optimized out later.

> 
>> +		if (n == tot) {
>> +			/*
>> +			 * fcheck_files() is safe with drop/re-acquire
>> +			 * of the lock, because it tests:  fd < max_fds
>> +			 */
>> +			spin_unlock(&files->file_lock);
>> +			tot *= 2;
>> +			if (tot < 0) {		/* overflow ? */
> 
> _NO_. tot is signed, this does not have documented overflow behaviour.
> You need to restrict this to a sane number.

Ok. (btw, krealloc() will fail much earlier anyway).

> 
>> +				kfree(fds);
>> +				return -EMFILE;
>> +			}
>> +			fds = krealloc(fds, tot * sizeof(*fds), GFP_KERNEL);
>> +			if (!fds)
> 
> krealloc does not free the memory on error, so this is a leak.

Ok.

Thanks,

Oren.

Re: [RFC v5][PATCH 8/8] Dump open file descriptors

[Serge E. Hallyn]

Quoting Oren Laadan (orenl-eQaUEPhvms7ENvBUuze7eA@public.gmane.org):
> 
> 
> Bastian Blank wrote:
> > On Sat, Sep 13, 2008 at 07:06:06PM -0400, Oren Laadan wrote:
> >> +int cr_scan_fds(struct files_struct *files, int **fdtable)
> >> +{
> >> +	struct fdtable *fdt;
> >> +	int *fds;
> >> +	int i, n, tot;
> >> +
> >> +	n = 0;
> >> +	tot = CR_DEFAULT_FDTABLE;
> > 
> > Why not?
> > | int i;
> > | int n = 0;
> > | int tot = CR_DEFAULT_FDTABLE;
> > 
> > IHMO easier readable.
> 
> Ok.
> 
> > 
> >> +	spin_lock(&files->file_lock);
> >> +	fdt = files_fdtable(files);
> >> +	for (i = 0; i < fdt->max_fds; i++) {
> > 
> > The process is suspended at this state?
> 
> Yes, the assumption is that the process is frozen (or that we checkpoint
> ourselves).
> 
> With this assumption, it is possible to (a) leave out RCU locking, and also
> (b) continue after the krealloc() from where we left off. Also, it means that
> (c) the contents of our 'fds' array remain valid by the time the caller makes
> use of it.
> 
> This certainly deserves a comment in the code, will add.
> 
> If the assumption doesn't hold, then I'll have to add the RCU locking. Cases
> (b) and (c) are already safe because the caller(s) use fcheck_files() to
> access the file-descriptors collected in the array.
> 
> While in my mind a task should never be unfrozen while being checkpointed, in
> reality future code may allow it e.g. if a OOM kicks in a kills it. So I tend
> to add the RCU lock for safety. It can always be optimized out later.

More to the point, you're not preventing them being unfrozen, so I think
the locking needs to stay.

> > 
> >> +		if (n == tot) {
> >> +			/*
> >> +			 * fcheck_files() is safe with drop/re-acquire
> >> +			 * of the lock, because it tests:  fd < max_fds
> >> +			 */
> >> +			spin_unlock(&files->file_lock);
> >> +			tot *= 2;
> >> +			if (tot < 0) {		/* overflow ? */
> > 
> > _NO_. tot is signed, this does not have documented overflow behaviour.
> > You need to restrict this to a sane number.
> 
> Ok. (btw, krealloc() will fail much earlier anyway).

Right, so you may as well drop this.  You're not protecting from
userspace here, right?  You're protecting against a bogus max_fds.
Not worthwhile.

> >> +				kfree(fds);
> >> +				return -EMFILE;
> >> +			}
> >> +			fds = krealloc(fds, tot * sizeof(*fds), GFP_KERNEL);
> >> +			if (!fds)
> > 
> > krealloc does not free the memory on error, so this is a leak.
> 
> Ok.
> 
> Thanks,
> 
> Oren.
> _______________________________________________
> Containers mailing list
> Containers-cunTk1MwBs9QetFLy7KEm3xJsTq8ys+cHZ5vskTnxNA@public.gmane.org
> https://lists.linux-foundation.org/mailman/listinfo/containers

Re: [RFC v5][PATCH 8/8] Dump open file descriptors

[Dave Hansen]

Oren,

You should probably also cc the x86 maintainers on at least the
x86-specific bits.


-- Dave

Re: [RFC v5][PATCH 8/8] Dump open file descriptors

[Bastian Blank]

On Sat, Sep 13, 2008 at 07:06:06PM -0400, Oren Laadan wrote:
> +int cr_scan_fds(struct files_struct *files, int **fdtable)
> +{
> +	struct fdtable *fdt;
> +	int *fds;
> +	int i, n, tot;
> +
> +	n = 0;
> +	tot = CR_DEFAULT_FDTABLE;

Why not?
| int i;
| int n = 0;
| int tot = CR_DEFAULT_FDTABLE;

IHMO easier readable.

> +	spin_lock(&files->file_lock);
> +	fdt = files_fdtable(files);
> +	for (i = 0; i < fdt->max_fds; i++) {

The process is suspended at this state?

> +		if (n == tot) {
> +			/*
> +			 * fcheck_files() is safe with drop/re-acquire
> +			 * of the lock, because it tests:  fd < max_fds
> +			 */
> +			spin_unlock(&files->file_lock);
> +			tot *= 2;
> +			if (tot < 0) {		/* overflow ? */

_NO_. tot is signed, this does not have documented overflow behaviour.
You need to restrict this to a sane number.

> +				kfree(fds);
> +				return -EMFILE;
> +			}
> +			fds = krealloc(fds, tot * sizeof(*fds), GFP_KERNEL);
> +			if (!fds)

krealloc does not free the memory on error, so this is a leak.

Bastian

-- 
The more complex the mind, the greater the need for the simplicity of play.
		-- Kirk, "Shore Leave", stardate 3025.8

Re: [RFC v5][PATCH 8/8] Dump open file descriptors

[Dave Hansen]

On Sat, 2008-09-13 at 19:06 -0400, Oren Laadan wrote:
> 
> +static int
> +cr_write_fd_ent(struct cr_ctx *ctx, struct files_struct *files, int
> fd)
> +{
> +       struct cr_hdr h;
> +       struct cr_hdr_fd_ent *hh = cr_hbuf_get(ctx, sizeof(*hh));
> +       struct file *file = NULL;
> +       struct fdtable *fdt;
> +       int coe, objref, new, ret;
> +
> +       rcu_read_lock();
> +       fdt = files_fdtable(files);
> +       file = fcheck_files(files, fd);
> +       if (file) {
> +               coe = FD_ISSET(fd, fdt->close_on_exec);
> +               get_file(file);
> +       }
> +       rcu_read_unlock();
> +
> +       /* sanity check (although this shouldn't happen) */
> +       if (!file)
> +               return -EBADF;
> +
> +       new = cr_obj_add_ptr(ctx, (void *) file, &objref, CR_OBJ_FILE, 0);

Dude, you don't need to cast to void*.  

-- Dave

Re: [RFC v5][PATCH 8/8] Dump open file descriptors

[Dave Hansen]

On Sat, 2008-09-13 at 19:06 -0400, Oren Laadan wrote:
> +/* cr_write_fd_data - dump the state of a given file pointer */
> +static int cr_write_fd_data(struct cr_ctx *ctx, struct file *file, int parent)
> +{
> +	struct cr_hdr h;
> +	struct cr_hdr_fd_data *hh = cr_hbuf_get(ctx, sizeof(*hh));
> +	struct dentry *dent = file->f_dentry;
> +	struct inode *inode = dent->d_inode;
> +	enum fd_type fd_type;
> +	int ret;
> +
> +	h.type = CR_HDR_FD_DATA;
> +	h.len = sizeof(*hh);
> +	h.parent = parent;
> +
> +	hh->f_flags = file->f_flags;
> +	hh->f_mode = file->f_mode;
> +	hh->f_pos = file->f_pos;
> +	hh->f_version = file->f_version;
> +	/* FIX: need also file->uid, file->gid, file->f_owner, etc */
> +
> +	switch (inode->i_mode & S_IFMT) {
> +	case S_IFREG:
> +		fd_type = CR_FD_FILE;
> +		break;
> +	case S_IFDIR:
> +		fd_type = CR_FD_DIR;
> +		break;
> +	case S_IFLNK:
> +		fd_type = CR_FD_LINK;
> +		break;
> +	default:
> +		return -EBADF;
> +	}
> +
> +	/* FIX: check if the file/dir/link is unlinked */
> +	hh->fd_type = fd_type;
> +
> +	ret = cr_write_obj(ctx, &h, hh);
> +	cr_hbuf_put(ctx, sizeof(*hh));
> +	if (ret < 0)
> +		return ret;
> +
> +	return cr_write_fname(ctx, &file->f_path, ctx->vfsroot);
> +}

One of the big things I'm looking for in these file patches is to make
sure that we can expand some day to lots of mounts and lots of
filesystem namespaces.

This tells me that we probably need a per-process vfsroot (which is also
a shared object).  We probably need to make a note in the task structure
as well as here.  

-- Dave

[RFC v5][PATCH 9/9] Restore open file descriprtors

[Oren Laadan]

Restore open file descriptors: for each FD read 'struct cr_hdr_fd_ent'
and lookup objref in the hash table; if not found (first occurence), read
in 'struct cr_hdr_fd_data', create a new FD and register in the hash.
Otherwise attach the file pointer from the hash as an FD.

This patch only handles basic FDs - regular files, directories and also
symbolic links.

Signed-off-by: Oren Laadan <orenl-eQaUEPhvms7ENvBUuze7eA@public.gmane.org>
---
 checkpoint/Makefile        |    2 +-
 checkpoint/restart.c       |    4 +
 checkpoint/rstr_file.c     |  202 ++++++++++++++++++++++++++++++++++++++++++++
 include/linux/checkpoint.h |    1 +
 4 files changed, 208 insertions(+), 1 deletions(-)
 create mode 100644 checkpoint/rstr_file.c

diff --git a/checkpoint/Makefile b/checkpoint/Makefile
index 7496695..88bbc10 100644
--- a/checkpoint/Makefile
+++ b/checkpoint/Makefile
@@ -3,4 +3,4 @@
 #
 
 obj-$(CONFIG_CHECKPOINT_RESTART) += sys.o checkpoint.o restart.o objhash.o \
-		ckpt_mem.o rstr_mem.o ckpt_file.o
+		ckpt_mem.o rstr_mem.o ckpt_file.o rstr_file.o
diff --git a/checkpoint/restart.c b/checkpoint/restart.c
index a0d5e60..956e274 100644
--- a/checkpoint/restart.c
+++ b/checkpoint/restart.c
@@ -212,6 +212,10 @@ static int cr_read_task(struct cr_ctx *ctx)
 	cr_debug("memory: ret %d\n", ret);
 	if (ret < 0)
 		goto out;
+	ret = cr_read_files(ctx);
+	cr_debug("files: ret %d\n", ret);
+	if (ret < 0)
+		goto out;
 	ret = cr_read_thread(ctx);
 	cr_debug("thread: ret %d\n", ret);
 	if (ret < 0)
diff --git a/checkpoint/rstr_file.c b/checkpoint/rstr_file.c
new file mode 100644
index 0000000..780c0fc
--- /dev/null
+++ b/checkpoint/rstr_file.c
@@ -0,0 +1,202 @@
+/*
+ *  Checkpoint file descriptors
+ *
+ *  Copyright (C) 2008 Oren Laadan
+ *
+ *  This file is subject to the terms and conditions of the GNU General Public
+ *  License.  See the file COPYING in the main directory of the Linux
+ *  distribution for more details.
+ */
+
+#include <linux/kernel.h>
+#include <linux/sched.h>
+#include <linux/fs.h>
+#include <linux/file.h>
+#include <linux/fdtable.h>
+#include <linux/fsnotify.h>
+#include <linux/syscalls.h>
+#include <linux/checkpoint.h>
+#include <linux/checkpoint_hdr.h>
+
+#include "checkpoint_file.h"
+
+static int cr_close_all_fds(struct files_struct *files)
+{
+	int *fdtable;
+	int nfds;
+
+	nfds = cr_scan_fds(files, &fdtable);
+	if (nfds < 0)
+		return nfds;
+	while (nfds--)
+		sys_close(fdtable[nfds]);
+	kfree(fdtable);
+	return 0;
+}
+
+/**
+ * cr_attach_file - attach a lonely file ptr to a file descriptor
+ * @file: lonely file pointer
+ */
+static int cr_attach_file(struct file *file)
+{
+	int fd = get_unused_fd_flags(0);
+
+	if (fd >= 0) {
+		fsnotify_open(file->f_path.dentry);
+		fd_install(fd, file);
+	}
+	return fd;
+}
+
+#define CR_SETFL_MASK (O_APPEND|O_NONBLOCK|O_NDELAY|FASYNC|O_DIRECT|O_NOATIME)
+
+/* cr_read_fd_data - restore the state of a given file pointer */
+static int
+cr_read_fd_data(struct cr_ctx *ctx, struct files_struct *files, int parent)
+{
+	struct cr_hdr_fd_data *hh = cr_hbuf_get(ctx, sizeof(*hh));
+	struct file *file;
+	int fd, rparent, ret;
+
+	rparent = cr_read_obj_type(ctx, hh, sizeof(*hh), CR_HDR_FD_DATA);
+	cr_debug("rparent %d parent %d flags %#x mode %#x how %d\n",
+		 rparent, parent, hh->f_flags, hh->f_mode, hh->fd_type);
+	if (rparent < 0)
+		return rparent;
+	if (rparent != parent)
+		return -EINVAL;
+	/* FIX: more sanity checks on f_flags, f_mode etc */
+
+	switch (hh->fd_type) {
+	case CR_FD_FILE:
+	case CR_FD_DIR:
+	case CR_FD_LINK:
+		file = cr_read_open_fname(ctx, hh->f_flags, hh->f_mode);
+		break;
+	default:
+		file = ERR_PTR(-EINVAL);
+		break;
+	}
+
+	if (IS_ERR(file))
+		return PTR_ERR(file);
+
+	/* FIX: need to restore uid, gid, owner etc */
+
+	fd = cr_attach_file(file);	/* no need to cleanup 'file' below */
+	if (fd < 0) {
+		filp_close(file, NULL);
+		return fd;
+	}
+
+	/* register new <objref, file> tuple in hash table */
+	ret = cr_obj_add_ref(ctx, (void *) file, parent, CR_OBJ_FILE, 0);
+	if (ret < 0)
+		goto out;
+	ret = sys_fcntl(fd, F_SETFL, hh->f_flags & CR_SETFL_MASK);
+	if (ret < 0)
+		goto out;
+	ret = vfs_llseek(file, hh->f_pos, SEEK_SET);
+	if (ret == -ESPIPE)	/* ignore error on non-seekable files */
+		ret = 0;
+
+ out:
+	cr_hbuf_put(ctx, sizeof(*hh));
+	return ret < 0 ? ret : fd;
+}
+
+/**
+ * cr_read_fd_ent - restore the state of a given file descriptor
+ * @ctx: checkpoint context
+ * @files: files_struct pointer
+ * @parent: parent objref
+ *
+ * Restores the state of a file descriptor; looks up the objref (in the
+ * header) in the hash table, and if found picks the matching file and
+ * use it; otherwise calls cr_read_fd_data to restore the file too.
+ */
+static int
+cr_read_fd_ent(struct cr_ctx *ctx, struct files_struct *files, int parent)
+{
+	struct cr_hdr_fd_ent *hh = cr_hbuf_get(ctx, sizeof(*hh));
+	struct file *file;
+	int newfd, rparent;
+
+	rparent = cr_read_obj_type(ctx, hh, sizeof(*hh), CR_HDR_FD_ENT);
+	cr_debug("rparent %d parent %d ref %d\n", rparent, parent, hh->objref);
+	if (rparent < 0)
+		return rparent;
+	if (rparent != parent)
+		return -EINVAL;
+	cr_debug("fd %d coe %d\n", hh->fd, hh->close_on_exec);
+	if (hh->objref <= 0)
+		return -EINVAL;
+
+	file = cr_obj_get_by_ref(ctx, hh->objref, CR_OBJ_FILE);
+	if (IS_ERR(file))
+		return PTR_ERR(file);
+
+	if (file) {
+		newfd = cr_attach_file(file);
+		if (newfd < 0)
+			return newfd;
+		get_file(file);
+	} else {
+		/* create new file pointer (and register in hash table) */
+		newfd = cr_read_fd_data(ctx, files, hh->objref);
+		if (newfd < 0)
+			return newfd;
+	}
+
+	cr_debug("newfd got %d wanted %d\n", newfd, hh->fd);
+
+	/* if newfd isn't desired fd then reposition it */
+	if (newfd != hh->fd) {
+		int ret = sys_dup2(newfd, hh->fd);
+		if (ret < 0)
+			return ret;
+		sys_close(newfd);
+	}
+
+	if (hh->close_on_exec)
+		set_close_on_exec(hh->fd, 1);
+
+	cr_hbuf_put(ctx, sizeof(*hh));
+	return 0;
+}
+
+int cr_read_files(struct cr_ctx *ctx)
+{
+	struct cr_hdr_files *hh = cr_hbuf_get(ctx, sizeof(*hh));
+	struct files_struct *files = current->files;
+	int i, parent, ret;
+
+	parent = cr_read_obj_type(ctx, hh, sizeof(*hh), CR_HDR_FILES);
+	if (parent < 0)
+		return parent;
+#if 0	/* activate when containers are used */
+	if (parent != task_pid_vnr(current))
+		return -EINVAL;
+#endif
+	cr_debug("objref %d nfds %d\n", hh->objref, hh->nfds);
+	if (hh->objref < 0 || hh->nfds < 0)
+		return -EINVAL;
+
+	if (hh->nfds > sysctl_nr_open)
+		return -EMFILE;
+
+	/* point of no return -- close all file descriptors */
+	ret = cr_close_all_fds(files);
+	if (ret < 0)
+		return ret;
+
+	for (i = 0; i < hh->nfds; i++) {
+		ret = cr_read_fd_ent(ctx, files, hh->objref);
+		if (ret < 0)
+			break;
+	}
+
+	cr_hbuf_put(ctx, sizeof(*hh));
+	return ret;
+}
diff --git a/include/linux/checkpoint.h b/include/linux/checkpoint.h
index a202c54..fd3baac 100644
--- a/include/linux/checkpoint.h
+++ b/include/linux/checkpoint.h
@@ -89,6 +89,7 @@ extern int cr_write_files(struct cr_ctx *ctx, struct task_struct *t);
 
 extern int do_restart(struct cr_ctx *ctx);
 extern int cr_read_mm(struct cr_ctx *ctx);
+extern int cr_read_files(struct cr_ctx *ctx);
 
 #define cr_debug(fmt, args...)  \
 	pr_debug("[CR:%s] " fmt, __func__, ## args)
-- 
1.5.4.3

[RFC v5][PATCH 9/9] Restore open file descriprtors

[Oren Laadan]

Restore open file descriptors: for each FD read 'struct cr_hdr_fd_ent'
and lookup objref in the hash table; if not found (first occurence), read
in 'struct cr_hdr_fd_data', create a new FD and register in the hash.
Otherwise attach the file pointer from the hash as an FD.

This patch only handles basic FDs - regular files, directories and also
symbolic links.

Signed-off-by: Oren Laadan <orenl-eQaUEPhvms7ENvBUuze7eA@public.gmane.org>
---
 checkpoint/Makefile        |    2 +-
 checkpoint/restart.c       |    4 +
 checkpoint/rstr_file.c     |  202 ++++++++++++++++++++++++++++++++++++++++++++
 include/linux/checkpoint.h |    1 +
 4 files changed, 208 insertions(+), 1 deletions(-)
 create mode 100644 checkpoint/rstr_file.c

diff --git a/checkpoint/Makefile b/checkpoint/Makefile
index 7496695..88bbc10 100644
--- a/checkpoint/Makefile
+++ b/checkpoint/Makefile
@@ -3,4 +3,4 @@
 #
 
 obj-$(CONFIG_CHECKPOINT_RESTART) += sys.o checkpoint.o restart.o objhash.o \
-		ckpt_mem.o rstr_mem.o ckpt_file.o
+		ckpt_mem.o rstr_mem.o ckpt_file.o rstr_file.o
diff --git a/checkpoint/restart.c b/checkpoint/restart.c
index a0d5e60..956e274 100644
--- a/checkpoint/restart.c
+++ b/checkpoint/restart.c
@@ -212,6 +212,10 @@ static int cr_read_task(struct cr_ctx *ctx)
 	cr_debug("memory: ret %d\n", ret);
 	if (ret < 0)
 		goto out;
+	ret = cr_read_files(ctx);
+	cr_debug("files: ret %d\n", ret);
+	if (ret < 0)
+		goto out;
 	ret = cr_read_thread(ctx);
 	cr_debug("thread: ret %d\n", ret);
 	if (ret < 0)
diff --git a/checkpoint/rstr_file.c b/checkpoint/rstr_file.c
new file mode 100644
index 0000000..780c0fc
--- /dev/null
+++ b/checkpoint/rstr_file.c
@@ -0,0 +1,202 @@
+/*
+ *  Checkpoint file descriptors
+ *
+ *  Copyright (C) 2008 Oren Laadan
+ *
+ *  This file is subject to the terms and conditions of the GNU General Public
+ *  License.  See the file COPYING in the main directory of the Linux
+ *  distribution for more details.
+ */
+
+#include <linux/kernel.h>
+#include <linux/sched.h>
+#include <linux/fs.h>
+#include <linux/file.h>
+#include <linux/fdtable.h>
+#include <linux/fsnotify.h>
+#include <linux/syscalls.h>
+#include <linux/checkpoint.h>
+#include <linux/checkpoint_hdr.h>
+
+#include "checkpoint_file.h"
+
+static int cr_close_all_fds(struct files_struct *files)
+{
+	int *fdtable;
+	int nfds;
+
+	nfds = cr_scan_fds(files, &fdtable);
+	if (nfds < 0)
+		return nfds;
+	while (nfds--)
+		sys_close(fdtable[nfds]);
+	kfree(fdtable);
+	return 0;
+}
+
+/**
+ * cr_attach_file - attach a lonely file ptr to a file descriptor
+ * @file: lonely file pointer
+ */
+static int cr_attach_file(struct file *file)
+{
+	int fd = get_unused_fd_flags(0);
+
+	if (fd >= 0) {
+		fsnotify_open(file->f_path.dentry);
+		fd_install(fd, file);
+	}
+	return fd;
+}
+
+#define CR_SETFL_MASK (O_APPEND|O_NONBLOCK|O_NDELAY|FASYNC|O_DIRECT|O_NOATIME)
+
+/* cr_read_fd_data - restore the state of a given file pointer */
+static int
+cr_read_fd_data(struct cr_ctx *ctx, struct files_struct *files, int parent)
+{
+	struct cr_hdr_fd_data *hh = cr_hbuf_get(ctx, sizeof(*hh));
+	struct file *file;
+	int fd, rparent, ret;
+
+	rparent = cr_read_obj_type(ctx, hh, sizeof(*hh), CR_HDR_FD_DATA);
+	cr_debug("rparent %d parent %d flags %#x mode %#x how %d\n",
+		 rparent, parent, hh->f_flags, hh->f_mode, hh->fd_type);
+	if (rparent < 0)
+		return rparent;
+	if (rparent != parent)
+		return -EINVAL;
+	/* FIX: more sanity checks on f_flags, f_mode etc */
+
+	switch (hh->fd_type) {
+	case CR_FD_FILE:
+	case CR_FD_DIR:
+	case CR_FD_LINK:
+		file = cr_read_open_fname(ctx, hh->f_flags, hh->f_mode);
+		break;
+	default:
+		file = ERR_PTR(-EINVAL);
+		break;
+	}
+
+	if (IS_ERR(file))
+		return PTR_ERR(file);
+
+	/* FIX: need to restore uid, gid, owner etc */
+
+	fd = cr_attach_file(file);	/* no need to cleanup 'file' below */
+	if (fd < 0) {
+		filp_close(file, NULL);
+		return fd;
+	}
+
+	/* register new <objref, file> tuple in hash table */
+	ret = cr_obj_add_ref(ctx, (void *) file, parent, CR_OBJ_FILE, 0);
+	if (ret < 0)
+		goto out;
+	ret = sys_fcntl(fd, F_SETFL, hh->f_flags & CR_SETFL_MASK);
+	if (ret < 0)
+		goto out;
+	ret = vfs_llseek(file, hh->f_pos, SEEK_SET);
+	if (ret == -ESPIPE)	/* ignore error on non-seekable files */
+		ret = 0;
+
+ out:
+	cr_hbuf_put(ctx, sizeof(*hh));
+	return ret < 0 ? ret : fd;
+}
+
+/**
+ * cr_read_fd_ent - restore the state of a given file descriptor
+ * @ctx: checkpoint context
+ * @files: files_struct pointer
+ * @parent: parent objref
+ *
+ * Restores the state of a file descriptor; looks up the objref (in the
+ * header) in the hash table, and if found picks the matching file and
+ * use it; otherwise calls cr_read_fd_data to restore the file too.
+ */
+static int
+cr_read_fd_ent(struct cr_ctx *ctx, struct files_struct *files, int parent)
+{
+	struct cr_hdr_fd_ent *hh = cr_hbuf_get(ctx, sizeof(*hh));
+	struct file *file;
+	int newfd, rparent;
+
+	rparent = cr_read_obj_type(ctx, hh, sizeof(*hh), CR_HDR_FD_ENT);
+	cr_debug("rparent %d parent %d ref %d\n", rparent, parent, hh->objref);
+	if (rparent < 0)
+		return rparent;
+	if (rparent != parent)
+		return -EINVAL;
+	cr_debug("fd %d coe %d\n", hh->fd, hh->close_on_exec);
+	if (hh->objref <= 0)
+		return -EINVAL;
+
+	file = cr_obj_get_by_ref(ctx, hh->objref, CR_OBJ_FILE);
+	if (IS_ERR(file))
+		return PTR_ERR(file);
+
+	if (file) {
+		newfd = cr_attach_file(file);
+		if (newfd < 0)
+			return newfd;
+		get_file(file);
+	} else {
+		/* create new file pointer (and register in hash table) */
+		newfd = cr_read_fd_data(ctx, files, hh->objref);
+		if (newfd < 0)
+			return newfd;
+	}
+
+	cr_debug("newfd got %d wanted %d\n", newfd, hh->fd);
+
+	/* if newfd isn't desired fd then reposition it */
+	if (newfd != hh->fd) {
+		int ret = sys_dup2(newfd, hh->fd);
+		if (ret < 0)
+			return ret;
+		sys_close(newfd);
+	}
+
+	if (hh->close_on_exec)
+		set_close_on_exec(hh->fd, 1);
+
+	cr_hbuf_put(ctx, sizeof(*hh));
+	return 0;
+}
+
+int cr_read_files(struct cr_ctx *ctx)
+{
+	struct cr_hdr_files *hh = cr_hbuf_get(ctx, sizeof(*hh));
+	struct files_struct *files = current->files;
+	int i, parent, ret;
+
+	parent = cr_read_obj_type(ctx, hh, sizeof(*hh), CR_HDR_FILES);
+	if (parent < 0)
+		return parent;
+#if 0	/* activate when containers are used */
+	if (parent != task_pid_vnr(current))
+		return -EINVAL;
+#endif
+	cr_debug("objref %d nfds %d\n", hh->objref, hh->nfds);
+	if (hh->objref < 0 || hh->nfds < 0)
+		return -EINVAL;
+
+	if (hh->nfds > sysctl_nr_open)
+		return -EMFILE;
+
+	/* point of no return -- close all file descriptors */
+	ret = cr_close_all_fds(files);
+	if (ret < 0)
+		return ret;
+
+	for (i = 0; i < hh->nfds; i++) {
+		ret = cr_read_fd_ent(ctx, files, hh->objref);
+		if (ret < 0)
+			break;
+	}
+
+	cr_hbuf_put(ctx, sizeof(*hh));
+	return ret;
+}
diff --git a/include/linux/checkpoint.h b/include/linux/checkpoint.h
index a202c54..fd3baac 100644
--- a/include/linux/checkpoint.h
+++ b/include/linux/checkpoint.h
@@ -89,6 +89,7 @@ extern int cr_write_files(struct cr_ctx *ctx, struct task_struct *t);
 
 extern int do_restart(struct cr_ctx *ctx);
 extern int cr_read_mm(struct cr_ctx *ctx);
+extern int cr_read_files(struct cr_ctx *ctx);
 
 #define cr_debug(fmt, args...)  \
 	pr_debug("[CR:%s] " fmt, __func__, ## args)
-- 
1.5.4.3

Re: [RFC v5][PATCH 0/9] Kernel based checkpoint/restart

[Serge E. Hallyn]

Quoting Oren Laadan (orenl-eQaUEPhvms7ENvBUuze7eA@public.gmane.org):
> These patches implement basic checkpoint-restart [CR]. This version (v5)
> supports basic tasks with simple private memory, and open files (regular
> files and directories only). The main changes include rework of memory
> restart code and response to feedback. See original announcements below.
> 
> Oren.

At the kernel summit, it was asked whether at some point this CR work
could be used as the basis for a new suspend to disk implementation.
I answered that we don't plan to handle kernel threads.  In your
experience, are there any reasons why eventually we couldn't eventually
handle kernel threads?

-serge

Re: [RFC v5][PATCH 0/9] Kernel based checkpoint/restart

[Serge E. Hallyn]

Quoting Oren Laadan (orenl-eQaUEPhvms7ENvBUuze7eA@public.gmane.org):
> These patches implement basic checkpoint-restart [CR]. This version (v5)
> supports basic tasks with simple private memory, and open files (regular
> files and directories only). The main changes include rework of memory
> restart code and response to feedback. See original announcements below.
> 
> Oren.

Playing around a bit, all seems to work as advertised.

Has anyone playing with this run into any oopses with the latest
version?

Cedric, could you send out your patch (or a pointer to them) to make
ckpt/restart on another task work, so we can pound the core code a bit
harder with liblxc?

thanks,
-serge

Re: [RFC v5][PATCH 2/8] General infrastructure for checkpoint restart

[Dave Hansen]

On Sat, 2008-09-13 at 19:06 -0400, Oren Laadan wrote:
> 
> +int cr_read_obj_type(struct cr_ctx *ctx, void *buf, int n, int type)
> +{
> +       struct cr_hdr h;
> +       int ret;
> +
> +       ret = cr_read_obj(ctx, &h, buf, n);
> +       if (!ret) {
> +               if (h.type == type)
> +                       ret = h.parent;
> +               else
> +                       ret = -EINVAL;
> +       }
> +       return ret;
> +}

This one has been bugging me a bit.  This adds one net line of code and
I think it's much easier to read:

{
	struct cr_hdr h;
	int ret;

	ret = cr_read_obj(ctx, &h, buf, n);
	if (ret)
		return ret;

	ret = -EINVAL;	
	if (h.type == type)
		ret = h.parent;

	return ret;
}                     

-- Dave

Re: [RFC v5][PATCH 2/8] General infrastructure for checkpoint restart

[Dave Hansen]

On Sat, 2008-09-13 at 19:06 -0400, Oren Laadan wrote:
> +/*
> + * FIXME: (temporarily added file_pos_read() and file_pos_write() because
> + * they are static in fs/read_write.c... should cleanup and remove later)
> + */
> +static inline loff_t file_pos_read(struct file *file)
> +{
> +       return file->f_pos;
> +}
> +
> +static inline void file_pos_write(struct file *file, loff_t pos)
> +{
> +       file->f_pos = pos;
> +}

It's probably time to address this fixme.  

-- Dave

Re: [RFC v5][PATCH 2/8] General infrastructure for checkpoint restart

[Dave Hansen]

On Sat, 2008-09-13 at 19:06 -0400, Oren Laadan wrote:
> 
> +
> +struct cr_ctx *cr_ctx_alloc(pid_t pid, int fd, unsigned long flags)
> +{
> +       struct cr_ctx *ctx;
> +
> +       ctx = kzalloc(sizeof(*ctx), GFP_KERNEL);
> +       if (!ctx)
> +               return ERR_PTR(-ENOMEM);
> +
> +       ctx->file = fget(fd);
> +       if (!ctx->file) {
> +               cr_ctx_free(ctx);
> +               return ERR_PTR(-EBADF);
> +       }
> +
> +       ctx->hbuf = (void *) __get_free_pages(GFP_KERNEL, CR_HBUF_ORDER);
> +       if (!ctx->hbuf) {
> +               cr_ctx_free(ctx);
> +               return ERR_PTR(-ENOMEM);
> +       }

Could you explain why you're not using the slab here?

-- Dave

Re: [RFC v5][PATCH 2/8] General infrastructure for checkpoint restart

[Dave Hansen]

On Sat, 2008-09-13 at 19:06 -0400, Oren Laadan wrote:
> 
> +void *cr_hbuf_get(struct cr_ctx *ctx, int n)
> +{
> +       void *ptr;
> +
> +       BUG_ON(ctx->hpos + n > CR_HBUF_TOTAL);
> +       ptr = (void *) (((char *) ctx->hbuf) + ctx->hpos);
> +       ctx->hpos += n;
> +       return ptr;
> +}

All of the casting here is unnecessary.  'void *' behaves like 'char *'
when you do arithmetic on it.  

I really do detest having a memory allocator BUG_ON() when it runs out
of space.  

-- Dave

Re: [RFC v5][PATCH 2/8] General infrastructure for checkpoint restart

[Serge E. Hallyn]

Quoting Oren Laadan (orenl-eQaUEPhvms7ENvBUuze7eA@public.gmane.org):
> Add those interfaces, as well as helpers needed to easily manage the
> file format. The code is roughly broken out as follows:
> 
> checkpoint/sys.c - user/kernel data transfer, as well as setup of the
> checkpoint/restart context (a per-checkpoint data structure for
> housekeeping)
> 
> checkpoint/checkpoint.c - output wrappers and basic checkpoint handling
> 
> checkpoint/restart.c - input wrappers and basic restart handling
> 
> Patches to add the per-architecture support as well as the actual
> work to do the memory checkpoint follow in subsequent patches.
> 
> Signed-off-by: Oren Laadan <orenl-eQaUEPhvms7ENvBUuze7eA@public.gmane.org>
> ---
>  Makefile                       |    2 +-
>  checkpoint/Makefile            |    2 +-
>  checkpoint/checkpoint.c        |  174 ++++++++++++++++++++++++++++++++
>  checkpoint/restart.c           |  189 ++++++++++++++++++++++++++++++++++
>  checkpoint/sys.c               |  218 +++++++++++++++++++++++++++++++++++++++-
>  include/linux/checkpoint.h     |   60 +++++++++++
>  include/linux/checkpoint_hdr.h |   75 ++++++++++++++
>  include/linux/magic.h          |    3 +
>  8 files changed, 717 insertions(+), 6 deletions(-)
>  create mode 100644 checkpoint/checkpoint.c
>  create mode 100644 checkpoint/restart.c
>  create mode 100644 include/linux/checkpoint.h
>  create mode 100644 include/linux/checkpoint_hdr.h
> 
> diff --git a/Makefile b/Makefile
> index f448e00..a558ad2 100644
> --- a/Makefile
> +++ b/Makefile
> @@ -619,7 +619,7 @@ export mod_strip_cmd
> 
> 
>  ifeq ($(KBUILD_EXTMOD),)
> -core-y		+= kernel/ mm/ fs/ ipc/ security/ crypto/ block/
> +core-y		+= kernel/ mm/ fs/ ipc/ security/ crypto/ block/ checkpoint/
> 
>  vmlinux-dirs	:= $(patsubst %/,%,$(filter %/, $(init-y) $(init-m) \
>  		     $(core-y) $(core-m) $(drivers-y) $(drivers-m) \
> diff --git a/checkpoint/Makefile b/checkpoint/Makefile
> index 07d018b..d2df68c 100644
> --- a/checkpoint/Makefile
> +++ b/checkpoint/Makefile
> @@ -2,4 +2,4 @@
>  # Makefile for linux checkpoint/restart.
>  #
> 
> -obj-$(CONFIG_CHECKPOINT_RESTART) += sys.o
> +obj-$(CONFIG_CHECKPOINT_RESTART) += sys.o checkpoint.o restart.o
> diff --git a/checkpoint/checkpoint.c b/checkpoint/checkpoint.c
> new file mode 100644
> index 0000000..e5e188f
> --- /dev/null
> +++ b/checkpoint/checkpoint.c
> @@ -0,0 +1,174 @@
> +/*
> + *  Checkpoint logic and helpers
> + *
> + *  Copyright (C) 2008 Oren Laadan
> + *
> + *  This file is subject to the terms and conditions of the GNU General Public
> + *  License.  See the file COPYING in the main directory of the Linux
> + *  distribution for more details.
> + */
> +
> +#include <linux/version.h>
> +#include <linux/sched.h>
> +#include <linux/time.h>
> +#include <linux/fs.h>
> +#include <linux/file.h>
> +#include <linux/dcache.h>
> +#include <linux/mount.h>
> +#include <linux/utsname.h>
> +#include <linux/magic.h>
> +#include <linux/checkpoint.h>
> +#include <linux/checkpoint_hdr.h>
> +
> +/**
> + * cr_write_obj - write a record described by a cr_hdr
> + * @ctx: checkpoint context
> + * @h: record descriptor
> + * @buf: record buffer
> + */
> +int cr_write_obj(struct cr_ctx *ctx, struct cr_hdr *h, void *buf)
> +{
> +	int ret;
> +
> +	ret = cr_kwrite(ctx, h, sizeof(*h));
> +	if (ret < 0)
> +		return ret;
> +	return cr_kwrite(ctx, buf, h->len);
> +}
> +
> +/**
> + * cr_write_string - write a string
> + * @ctx: checkpoint context
> + * @str: string pointer
> + * @len: string length
> + */
> +int cr_write_string(struct cr_ctx *ctx, char *str, int len)
> +{
> +	struct cr_hdr h;
> +
> +	h.type = CR_HDR_STRING;
> +	h.len = len;
> +	h.parent = 0;
> +
> +	return cr_write_obj(ctx, &h, str);
> +}
> +
> +/* write the checkpoint header */
> +static int cr_write_head(struct cr_ctx *ctx)
> +{
> +	struct cr_hdr h;
> +	struct cr_hdr_head *hh = cr_hbuf_get(ctx, sizeof(*hh));
> +	struct new_utsname *uts;
> +	struct timeval ktv;
> +	int ret;
> +
> +	h.type = CR_HDR_HEAD;
> +	h.len = sizeof(*hh);
> +	h.parent = 0;
> +
> +	do_gettimeofday(&ktv);
> +
> +	hh->magic = CHECKPOINT_MAGIC_HEAD;
> +	hh->major = (LINUX_VERSION_CODE >> 16) & 0xff;
> +	hh->minor = (LINUX_VERSION_CODE >> 8) & 0xff;
> +	hh->patch = (LINUX_VERSION_CODE) & 0xff;
> +
> +	hh->rev = CR_VERSION;
> +
> +	hh->flags = ctx->flags;
> +	hh->time = ktv.tv_sec;
> +
> +	uts = utsname();
> +	memcpy(hh->release, uts->release, __NEW_UTS_LEN);
> +	memcpy(hh->version, uts->version, __NEW_UTS_LEN);
> +	memcpy(hh->machine, uts->machine, __NEW_UTS_LEN);
> +
> +	ret = cr_write_obj(ctx, &h, hh);
> +	cr_hbuf_put(ctx, sizeof(*hh));
> +	return ret;
> +}
> +
> +/* write the checkpoint trailer */
> +static int cr_write_tail(struct cr_ctx *ctx)
> +{
> +	struct cr_hdr h;
> +	struct cr_hdr_tail *hh = cr_hbuf_get(ctx, sizeof(*hh));
> +	int ret;
> +
> +	h.type = CR_HDR_TAIL;
> +	h.len = sizeof(*hh);
> +	h.parent = 0;
> +
> +	hh->magic = CHECKPOINT_MAGIC_TAIL;
> +
> +	ret = cr_write_obj(ctx, &h, hh);
> +	cr_hbuf_put(ctx, sizeof(*hh));
> +	return ret;
> +}
> +
> +/* dump the task_struct of a given task */
> +static int cr_write_task_struct(struct cr_ctx *ctx, struct task_struct *t)
> +{
> +	struct cr_hdr h;
> +	struct cr_hdr_task *hh = cr_hbuf_get(ctx, sizeof(*hh));
> +	int ret;
> +
> +	h.type = CR_HDR_TASK;
> +	h.len = sizeof(*hh);
> +	h.parent = 0;
> +
> +	hh->state = t->state;
> +	hh->exit_state = t->exit_state;
> +	hh->exit_code = t->exit_code;
> +	hh->exit_signal = t->exit_signal;
> +
> +	hh->task_comm_len = TASK_COMM_LEN;
> +
> +	/* FIXME: save remaining relevant task_struct fields */
> +
> +	ret = cr_write_obj(ctx, &h, hh);
> +	cr_hbuf_put(ctx, sizeof(*hh));
> +	if (ret < 0)
> +		return ret;
> +
> +	return cr_write_string(ctx, t->comm, TASK_COMM_LEN);
> +}
> +
> +/* dump the entire state of a given task */
> +static int cr_write_task(struct cr_ctx *ctx, struct task_struct *t)
> +{
> +	int ret ;
> +
> +	if (t->state == TASK_DEAD) {
> +		pr_warning("CR: task may not be in state TASK_DEAD\n");
> +		return -EAGAIN;
> +	}
> +
> +	ret = cr_write_task_struct(ctx, t);
> +	cr_debug("ret %d\n", ret);
> +
> +	return ret;
> +}
> +
> +int do_checkpoint(struct cr_ctx *ctx)
> +{
> +	int ret;
> +
> +	/* FIX: need to test whether container is checkpointable */
> +
> +	ret = cr_write_head(ctx);
> +	if (ret < 0)
> +		goto out;
> +	ret = cr_write_task(ctx, current);
> +	if (ret < 0)
> +		goto out;
> +	ret = cr_write_tail(ctx);
> +	if (ret < 0)
> +		goto out;
> +
> +	/* on success, return (unique) checkpoint identifier */
> +	ret = ctx->crid;

This feels a litlte weird, since the ctx->crid actually was calculated
in ctx_alloc() in sys_checkpoint().  I'd almost prefer do_checkpoint()
return 0 on success.

It may seem silly, but it seems to make the flow and layering clearer.

> +
> + out:
> +	return ret;
> +}
> diff --git a/checkpoint/restart.c b/checkpoint/restart.c
> new file mode 100644
> index 0000000..ef6bc37
> --- /dev/null
> +++ b/checkpoint/restart.c
> @@ -0,0 +1,189 @@
> +/*
> + *  Restart logic and helpers
> + *
> + *  Copyright (C) 2008 Oren Laadan
> + *
> + *  This file is subject to the terms and conditions of the GNU General Public
> + *  License.  See the file COPYING in the main directory of the Linux
> + *  distribution for more details.
> + */
> +
> +#include <linux/version.h>
> +#include <linux/sched.h>
> +#include <linux/file.h>
> +#include <linux/magic.h>
> +#include <linux/checkpoint.h>
> +#include <linux/checkpoint_hdr.h>
> +
> +/**
> + * cr_read_obj - read a whole record (cr_hdr followed by payload)
> + * @ctx: checkpoint context
> + * @h: record descriptor
> + * @buf: record buffer
> + * @n: available buffer size
> + *
> + * Returns size of payload
> + */
> +int cr_read_obj(struct cr_ctx *ctx, struct cr_hdr *h, void *buf, int n)
> +{
> +	int ret;
> +
> +	ret = cr_kread(ctx, h, sizeof(*h));
> +	if (ret < 0)
> +		return ret;
> +
> +	cr_debug("type %d len %d parent %d\n", h->type, h->len, h->parent);
> +
> +	if (h->len < 0 || h->len > n)
> +		return -EINVAL;

If h->len > n should we return -ENOSPC so the caller can theoretically
deal with it?

> +
> +	return cr_kread(ctx, buf, h->len);
> +}
> +
> +/**
> + * cr_read_obj_type - read a whole record of expected type
> + * @ctx: checkpoint context
> + * @buf: record buffer
> + * @n: available buffer size
> + * @type: expected record type
> + *
> + * Returns object reference of the parent object
> + */
> +int cr_read_obj_type(struct cr_ctx *ctx, void *buf, int n, int type)
> +{
> +	struct cr_hdr h;
> +	int ret;
> +
> +	ret = cr_read_obj(ctx, &h, buf, n);
> +	if (!ret) {
> +		if (h.type == type)
> +			ret = h.parent;
> +		else
> +			ret = -EINVAL;
> +	}
> +	return ret;
> +}
> +
> +/**
> + * cr_read_string - read a string
> + * @ctx: checkpoint context
> + * @str: string buffer
> + * @len: buffer buffer length
> + */
> +int cr_read_string(struct cr_ctx *ctx, void *str, int len)
> +{
> +	return cr_read_obj_type(ctx, str, len, CR_HDR_STRING);
> +}
> +
> +/* read the checkpoint header */
> +static int cr_read_head(struct cr_ctx *ctx)
> +{
> +	struct cr_hdr_head *hh = cr_hbuf_get(ctx, sizeof(*hh));
> +	int parent;
> +
> +	parent = cr_read_obj_type(ctx, hh, sizeof(*hh), CR_HDR_HEAD);
> +	if (parent < 0)
> +		return parent;
> +	else if (parent != 0)
> +		return -EINVAL;
> +
> +	if (hh->magic != CHECKPOINT_MAGIC_HEAD || hh->rev != CR_VERSION ||
> +	    hh->major != ((LINUX_VERSION_CODE >> 16) & 0xff) ||
> +	    hh->minor != ((LINUX_VERSION_CODE >> 8) & 0xff) ||
> +	    hh->patch != ((LINUX_VERSION_CODE) & 0xff))
> +		return -EINVAL;
> +
> +	if (hh->flags & ~CR_CTX_CKPT)
> +		return -EINVAL;
> +
> +	ctx->oflags = hh->flags;
> +
> +	/* FIX: verify compatibility of release, version and machine */
> +
> +	cr_hbuf_put(ctx, sizeof(*hh));
> +	return 0;
> +}
> +
> +/* read the checkpoint trailer */
> +static int cr_read_tail(struct cr_ctx *ctx)
> +{
> +	struct cr_hdr_tail *hh = cr_hbuf_get(ctx, sizeof(*hh));
> +	int parent;
> +
> +	parent = cr_read_obj_type(ctx, hh, sizeof(*hh), CR_HDR_TAIL);
> +	if (parent < 0)
> +		return parent;
> +	else if (parent != 0)
> +		return -EINVAL;
> +
> +	if (hh->magic != CHECKPOINT_MAGIC_TAIL)
> +		return -EINVAL;
> +
> +	cr_hbuf_put(ctx, sizeof(*hh));
> +	return 0;
> +}
> +
> +/* read the task_struct into the current task */
> +static int cr_read_task_struct(struct cr_ctx *ctx)
> +{
> +	struct cr_hdr_task *hh = cr_hbuf_get(ctx, sizeof(*hh));
> +	struct task_struct *t = current;
> +	char *buf;
> +	int parent, ret;
> +
> +	parent = cr_read_obj_type(ctx, hh, sizeof(*hh), CR_HDR_TASK);
> +	if (parent < 0)
> +		return parent;
> +	else if (parent != 0)
> +		return -EINVAL;
> +
> +	/* upper limit for task_comm_len to prevent DoS */
> +	if (hh->task_comm_len < 0 || hh->task_comm_len > PAGE_SIZE)
> +		return -EINVAL;
> +
> +	buf = kmalloc(hh->task_comm_len, GFP_KERNEL);
> +	if (!buf)
> +		return -ENOMEM;
> +	ret = cr_read_string(ctx, buf, hh->task_comm_len);

Here I am getting confused about return values.  

cr_read_obj() says it returns size of payload.

cr_read_obj_type() says it returns "object reference of the parent
object", though if cr_read_obj() returned non-zero then it returns its
return value, which could be size of object read.

cr_read_string() returns cr_read_obj_type()'s return value.

Now here you only copy buf into t->comm if return value was 0.

Am I misreading?  Or is this code wrong?

Annoying as it may be (causing patch conflicts in patches 3-9), could
you please put the return values of all these functions in the comments
above them?

> +	if (!ret) {
> +		/* if t->comm is too long, silently truncate */
> +		memset(t->comm, 0, TASK_COMM_LEN);
> +		memcpy(t->comm, buf, min(hh->task_comm_len, TASK_COMM_LEN));
> +	}
> +	kfree(buf);
> +
> +	/* FIXME: restore remaining relevant task_struct fields */
> +
> +	cr_hbuf_put(ctx, sizeof(*hh));
> +	return ret;
> +}
> +
> +/* read the entire state of the current task */
> +static int cr_read_task(struct cr_ctx *ctx)
> +{
> +	int ret;
> +
> +	ret = cr_read_task_struct(ctx);
> +	cr_debug("ret %d\n", ret);
> +
> +	return ret;
> +}
> +
> +int do_restart(struct cr_ctx *ctx)
> +{
> +	int ret;
> +
> +	ret = cr_read_head(ctx);
> +	if (ret < 0)
> +		goto out;
> +	ret = cr_read_task(ctx);
> +	if (ret < 0)
> +		goto out;
> +	ret = cr_read_tail(ctx);
> +	if (ret < 0)
> +		goto out;
> +
> +	/* on success, adjust the return value if needed [TODO] */
> + out:
> +	return ret;
> +}
> diff --git a/checkpoint/sys.c b/checkpoint/sys.c
> index 375129c..30863c6 100644
> --- a/checkpoint/sys.c
> +++ b/checkpoint/sys.c
> @@ -10,6 +10,189 @@
> 
>  #include <linux/sched.h>
>  #include <linux/kernel.h>
> +#include <linux/fs.h>
> +#include <linux/file.h>
> +#include <linux/uaccess.h>
> +#include <linux/capability.h>
> +#include <linux/checkpoint.h>
> +
> +/*
> + * helpers to write/read to/from the image file descriptor
> + *
> + *   cr_uwrite() - write a user-space buffer to the checkpoint image
> + *   cr_kwrite() - write a kernel-space buffer to the checkpoint image
> + *   cr_uread() - read from the checkpoint image to a user-space buffer
> + *   cr_kread() - read from the checkpoint image to a kernel-space buffer
> + */
> +
> +/*
> + * FIXME: (temporarily added file_pos_read() and file_pos_write() because
> + * they are static in fs/read_write.c... should cleanup and remove later)
> + */
> +static inline loff_t file_pos_read(struct file *file)
> +{
> +	return file->f_pos;
> +}
> +
> +static inline void file_pos_write(struct file *file, loff_t pos)
> +{
> +	file->f_pos = pos;
> +}
> +
> +int cr_uwrite(struct cr_ctx *ctx, void *buf, int count)
> +{
> +	struct file *file = ctx->file;
> +	ssize_t nwrite;
> +	int nleft;
> +
> +	for (nleft = count; nleft; nleft -= nwrite) {
> +		loff_t pos = file_pos_read(file);
> +		nwrite = vfs_write(file, (char __user *) buf, nleft, &pos);
> +		file_pos_write(file, pos);
> +		if (nwrite <= 0) {
> +			if (nwrite == -EAGAIN)
> +				nwrite = 0;
> +			else
> +				return nwrite;
> +		}
> +		buf += nwrite;
> +	}
> +
> +	ctx->total += count;
> +	return 0;
> +}
> +
> +int cr_kwrite(struct cr_ctx *ctx, void *buf, int count)
> +{
> +	mm_segment_t oldfs;
> +	int ret;
> +
> +	oldfs = get_fs();
> +	set_fs(KERNEL_DS);
> +	ret = cr_uwrite(ctx, buf, count);
> +	set_fs(oldfs);
> +
> +	return ret;
> +}
> +
> +int cr_uread(struct cr_ctx *ctx, void *buf, int count)
> +{
> +	struct file *file = ctx->file;
> +	ssize_t nread;
> +	int nleft;
> +
> +	for (nleft = count; nleft; nleft -= nread) {
> +		loff_t pos = file_pos_read(file);
> +		nread = vfs_read(file, (char __user *) buf, nleft, &pos);
> +		file_pos_write(file, pos);
> +		if (nread <= 0) {
> +			if (nread == -EAGAIN)
> +				nread = 0;
> +			else
> +				return nread;
> +		}
> +		buf += nread;
> +	}
> +
> +	ctx->total += count;
> +	return 0;
> +}
> +
> +int cr_kread(struct cr_ctx *ctx, void *buf, int count)
> +{
> +	mm_segment_t oldfs;
> +	int ret;
> +
> +	oldfs = get_fs();
> +	set_fs(KERNEL_DS);
> +	ret = cr_uread(ctx, buf, count);
> +	set_fs(oldfs);
> +
> +	return ret;
> +}
> +
> +
> +/*
> + * helpers to manage CR contexts: allocated for each checkpoint and/or
> + * restart operation, and persists until the operation is completed.
> + */
> +
> +/* unique checkpoint identifier (FIXME: should be per-container) */
> +static atomic_t cr_ctx_count;
> +
> +void cr_ctx_free(struct cr_ctx *ctx)
> +{
> +	if (ctx->file)
> +		fput(ctx->file);
> +
> +	free_pages((unsigned long) ctx->hbuf, CR_HBUF_ORDER);
> +
> +	kfree(ctx);
> +}
> +
> +struct cr_ctx *cr_ctx_alloc(pid_t pid, int fd, unsigned long flags)
> +{
> +	struct cr_ctx *ctx;
> +
> +	ctx = kzalloc(sizeof(*ctx), GFP_KERNEL);
> +	if (!ctx)
> +		return ERR_PTR(-ENOMEM);
> +
> +	ctx->file = fget(fd);
> +	if (!ctx->file) {
> +		cr_ctx_free(ctx);
> +		return ERR_PTR(-EBADF);
> +	}
> +
> +	ctx->hbuf = (void *) __get_free_pages(GFP_KERNEL, CR_HBUF_ORDER);
> +	if (!ctx->hbuf) {
> +		cr_ctx_free(ctx);
> +		return ERR_PTR(-ENOMEM);
> +	}
> +
> +	ctx->pid = pid;
> +	ctx->flags = flags;
> +
> +	ctx->crid = atomic_inc_return(&cr_ctx_count);
> +
> +	return ctx;
> +}
> +
> +/*
> + * During checkpoint and restart the code writes outs/reads in data
> + * to/from the chekcpoint image from/to a temporary buffer (ctx->hbuf).
> + * Because operations can be nested, one should call cr_hbuf_get() to
> + * reserve space in the buffer, and then cr_hbuf_put() when no longer
> + * needs that space.
> + */
> +
> +/**
> + * cr_hbuf_get - reserve space on the hbuf
> + * @ctx: checkpoint context
> + * @n: number of bytes to reserve
> + *
> + * Returns pointer to reserved space
> + */
> +void *cr_hbuf_get(struct cr_ctx *ctx, int n)
> +{
> +	void *ptr;
> +
> +	BUG_ON(ctx->hpos + n > CR_HBUF_TOTAL);
> +	ptr = (void *) (((char *) ctx->hbuf) + ctx->hpos);
> +	ctx->hpos += n;
> +	return ptr;
> +}
> +
> +/**
> + * cr_hbuf_put - unreserve space on the hbuf
> + * @ctx: checkpoint context
> + * @n: number of bytes to reserve
> + */
> +void cr_hbuf_put(struct cr_ctx *ctx, int n)
> +{
> +	BUG_ON(ctx->hpos < n);
> +	ctx->hpos -= n;
> +}
> 
>  /**
>   * sys_checkpoint - checkpoint a container
> @@ -22,9 +205,23 @@
>   */
>  asmlinkage long sys_checkpoint(pid_t pid, int fd, unsigned long flags)
>  {
> -	pr_debug("sys_checkpoint not implemented yet\n");
> -	return -ENOSYS;
> +	struct cr_ctx *ctx;
> +	int ret;
> +
> +	/* no flags for now */
> +	if (flags)
> +		return -EINVAL;
> +
> +	ctx = cr_ctx_alloc(pid, fd, flags | CR_CTX_CKPT);
> +	if (IS_ERR(ctx))
> +		return PTR_ERR(ctx);
> +
> +	ret = do_checkpoint(ctx);
> +
> +	cr_ctx_free(ctx);
> +	return ret;
>  }
> +
>  /**
>   * sys_restart - restart a container
>   * @crid: checkpoint image identifier
> @@ -36,6 +233,19 @@ asmlinkage long sys_checkpoint(pid_t pid, int fd, unsigned long flags)
>   */
>  asmlinkage long sys_restart(int crid, int fd, unsigned long flags)
>  {
> -	pr_debug("sys_restart not implemented yet\n");
> -	return -ENOSYS;
> +	struct cr_ctx *ctx;
> +	int ret;
> +
> +	/* no flags for now */
> +	if (flags)
> +		return -EINVAL;
> +
> +	ctx = cr_ctx_alloc(crid, fd, flags | CR_CTX_RSTR);
> +	if (IS_ERR(ctx))
> +		return PTR_ERR(ctx);
> +
> +	ret = do_restart(ctx);
> +
> +	cr_ctx_free(ctx);
> +	return ret;
>  }
> diff --git a/include/linux/checkpoint.h b/include/linux/checkpoint.h
> new file mode 100644
> index 0000000..5e53ae6
> --- /dev/null
> +++ b/include/linux/checkpoint.h
> @@ -0,0 +1,60 @@
> +#ifndef _CHECKPOINT_CKPT_H_
> +#define _CHECKPOINT_CKPT_H_
> +/*
> + *  Generic container checkpoint-restart
> + *
> + *  Copyright (C) 2008 Oren Laadan
> + *
> + *  This file is subject to the terms and conditions of the GNU General Public
> + *  License.  See the file COPYING in the main directory of the Linux
> + *  distribution for more details.
> + */
> +
> +#define CR_VERSION  1
> +
> +struct cr_ctx {
> +	pid_t pid;		/* container identifier */
> +	int crid;		/* unique checkpoint id */
> +
> +	unsigned long flags;
> +	unsigned long oflags;	/* restart: old flags */
> +
> +	struct file *file;
> +	int total;		/* total read/written */
> +
> +	void *hbuf;		/* temporary buffer for headers */
> +	int hpos;		/* position in headers buffer */
> +};
> +
> +/* cr_ctx: flags */
> +#define CR_CTX_CKPT	0x1
> +#define CR_CTX_RSTR	0x2
> +
> +/* allocation defaults */
> +#define CR_HBUF_ORDER  1
> +#define CR_HBUF_TOTAL  (PAGE_SIZE << CR_HBUF_ORDER)
> +
> +extern int cr_uwrite(struct cr_ctx *ctx, void *buf, int count);
> +extern int cr_kwrite(struct cr_ctx *ctx, void *buf, int count);
> +extern int cr_uread(struct cr_ctx *ctx, void *buf, int count);
> +extern int cr_kread(struct cr_ctx *ctx, void *buf, int count);
> +
> +extern void *cr_hbuf_get(struct cr_ctx *ctx, int n);
> +extern void cr_hbuf_put(struct cr_ctx *ctx, int n);
> +
> +struct cr_hdr;
> +
> +extern int cr_write_obj(struct cr_ctx *ctx, struct cr_hdr *h, void *buf);
> +extern int cr_write_string(struct cr_ctx *ctx, char *str, int len);
> +
> +extern int cr_read_obj(struct cr_ctx *ctx, struct cr_hdr *h, void *buf, int n);
> +extern int cr_read_obj_type(struct cr_ctx *ctx, void *buf, int n, int type);
> +extern int cr_read_string(struct cr_ctx *ctx, void *str, int len);
> +
> +extern int do_checkpoint(struct cr_ctx *ctx);
> +extern int do_restart(struct cr_ctx *ctx);
> +
> +#define cr_debug(fmt, args...)  \
> +	pr_debug("[CR:%s] " fmt, __func__, ## args)
> +
> +#endif /* _CHECKPOINT_CKPT_H_ */
> diff --git a/include/linux/checkpoint_hdr.h b/include/linux/checkpoint_hdr.h
> new file mode 100644
> index 0000000..79e4df2
> --- /dev/null
> +++ b/include/linux/checkpoint_hdr.h
> @@ -0,0 +1,75 @@
> +#ifndef _CHECKPOINT_CKPT_HDR_H_
> +#define _CHECKPOINT_CKPT_HDR_H_
> +/*
> + *  Generic container checkpoint-restart
> + *
> + *  Copyright (C) 2008 Oren Laadan
> + *
> + *  This file is subject to the terms and conditions of the GNU General Public
> + *  License.  See the file COPYING in the main directory of the Linux
> + *  distribution for more details.
> + */
> +
> +#include <linux/types.h>
> +#include <linux/utsname.h>
> +
> +/*
> + * To maintain compatibility between 32-bit and 64-bit architecture flavors,
> + * keep data 64-bit aligned: use padding for structure members, and use
> + * __attribute__ ((aligned (8))) for the entire structure.
> + */
> +
> +/* records: generic header */
> +
> +struct cr_hdr {
> +	__s16 type;
> +	__s16 len;
> +	__u32 parent;
> +};
> +
> +/* header types */
> +enum {
> +	CR_HDR_HEAD = 1,
> +	CR_HDR_STRING,
> +
> +	CR_HDR_TASK = 101,
> +	CR_HDR_THREAD,
> +	CR_HDR_CPU,
> +
> +	CR_HDR_MM = 201,
> +	CR_HDR_VMA,
> +	CR_HDR_MM_CONTEXT,
> +
> +	CR_HDR_TAIL = 5001
> +};
> +
> +struct cr_hdr_head {
> +	__u64 magic;
> +
> +	__u16 major;
> +	__u16 minor;
> +	__u16 patch;
> +	__u16 rev;
> +
> +	__u64 time;	/* when checkpoint taken */
> +	__u64 flags;	/* checkpoint options */
> +
> +	char release[__NEW_UTS_LEN];
> +	char version[__NEW_UTS_LEN];
> +	char machine[__NEW_UTS_LEN];
> +} __attribute__((aligned(8)));
> +
> +struct cr_hdr_tail {
> +	__u64 magic;
> +} __attribute__((aligned(8)));
> +
> +struct cr_hdr_task {
> +	__u32 state;
> +	__u32 exit_state;
> +	__u32 exit_code;
> +	__u32 exit_signal;
> +
> +	__s32 task_comm_len;
> +} __attribute__((aligned(8)));
> +
> +#endif /* _CHECKPOINT_CKPT_HDR_H_ */
> diff --git a/include/linux/magic.h b/include/linux/magic.h
> index 1fa0c2c..c2b811c 100644
> --- a/include/linux/magic.h
> +++ b/include/linux/magic.h
> @@ -42,4 +42,7 @@
>  #define FUTEXFS_SUPER_MAGIC	0xBAD1DEA
>  #define INOTIFYFS_SUPER_MAGIC	0x2BAD1DEA
> 
> +#define CHECKPOINT_MAGIC_HEAD  0x00feed0cc0a2d200LL
> +#define CHECKPOINT_MAGIC_TAIL  0x002d2a0cc0deef00LL
> +
>  #endif /* __LINUX_MAGIC_H__ */
> -- 
> 1.5.4.3
> 
> _______________________________________________
> Containers mailing list
> Containers-cunTk1MwBs9QetFLy7KEm3xJsTq8ys+cHZ5vskTnxNA@public.gmane.org
> https://lists.linux-foundation.org/mailman/listinfo/containers

Re: [RFC v5][PATCH 2/8] General infrastructure for checkpoint restart

[Oren Laadan]

Dave Hansen wrote:
> On Sat, 2008-09-13 at 19:06 -0400, Oren Laadan wrote:
>> +void *cr_hbuf_get(struct cr_ctx *ctx, int n)
>> +{
>> +       void *ptr;
>> +
>> +       BUG_ON(ctx->hpos + n > CR_HBUF_TOTAL);
>> +       ptr = (void *) (((char *) ctx->hbuf) + ctx->hpos);
>> +       ctx->hpos += n;
>> +       return ptr;
>> +}
> 
> All of the casting here is unnecessary.  'void *' behaves like 'char *'
> when you do arithmetic on it.  

ok.

> 
> I really do detest having a memory allocator BUG_ON() when it runs out
> of space.  

The BUG_ON() statement asserts that we don't run out of buffer space.
Buffer usage is a function of the checkpoint/restart logic, and does
not depend on user input, hence not susceptible to DoS.

In other words, if the code is correct, this should never happen (much
like a kernel stack overflow), and if it happens it's a kernel bug. I
think it was Arnd who recommended with regard to this to crash loudly
if there is a bug in the kernel ...

Oren.

Re: [RFC v5][PATCH 2/8] General infrastructure for checkpoint restart

[Dave Hansen]

On Mon, 2008-09-15 at 14:52 -0400, Oren Laadan wrote:
> Dave Hansen wrote:
> > On Sat, 2008-09-13 at 19:06 -0400, Oren Laadan wrote:
> >> +void *cr_hbuf_get(struct cr_ctx *ctx, int n)
> >> +{
> >> +       void *ptr;
> >> +
> >> +       BUG_ON(ctx->hpos + n > CR_HBUF_TOTAL);
> >> +       ptr = (void *) (((char *) ctx->hbuf) + ctx->hpos);
> >> +       ctx->hpos += n;
> >> +       return ptr;
> >> +}
...
> > I really do detest having a memory allocator BUG_ON() when it runs out
> > of space.  
> 
> The BUG_ON() statement asserts that we don't run out of buffer space.
> Buffer usage is a function of the checkpoint/restart logic, and does
> not depend on user input, hence not susceptible to DoS.

OK, that's fair enough.  But, can we document it as such?  "Only headers
and things of known, static sizes can go in here.  We don't use
kmalloc() instead of this because..."

> In other words, if the code is correct, this should never happen (much
> like a kernel stack overflow), and if it happens it's a kernel bug. I
> think it was Arnd who recommended with regard to this to crash loudly
> if there is a bug in the kernel ...

Yes, it does mean that there is a bug because someone either made a
structure bigger, PAGE_SIZE smaller, or a call path got deeper than we
expected.  I'm just having visions of the email hitting my inbox in 18
months. :)

The structures are sized consistently across all architectures and
configurations.  However, PAGE_SIZE and the size of that buffer are not.
The buffer will be 8k on x86, but 128k on most ppc64 configurations.

Can we at least make it sized in numbers of bytes rather than pages?
Also, please remember that using kmalloc() buys you lots of fun stuff on
top of get_free_pages(), like redzones and easier debugging.

-- Dave

Re: [RFC v5][PATCH 2/8] General infrastructure for checkpoint restart

[Bastian Blank]

On Mon, Sep 15, 2008 at 11:02:30AM -0700, Dave Hansen wrote:
> All of the casting here is unnecessary.  'void *' behaves like 'char *'
> when you do arithmetic on it.  

No. Arithmetic on void pointer is not allowed by the standard. This is a
gcc extension. It's okay to be used in the Linux kernel but not in
general.

The second cast (from char * to void *) however is useless.

Bastian

-- 
Prepare for tomorrow -- get ready.
		-- Edith Keeler, "The City On the Edge of Forever",
		   stardate unknown

Re: [RFC v5][PATCH 9/9] Restore open file descriprtors

[Serge E. Hallyn]

Quoting Oren Laadan (orenl-eQaUEPhvms7ENvBUuze7eA@public.gmane.org):
> Restore open file descriptors: for each FD read 'struct cr_hdr_fd_ent'
> and lookup objref in the hash table; if not found (first occurence), read
> in 'struct cr_hdr_fd_data', create a new FD and register in the hash.
> Otherwise attach the file pointer from the hash as an FD.
> 
> This patch only handles basic FDs - regular files, directories and also
> symbolic links.
> 
> Signed-off-by: Oren Laadan <orenl-eQaUEPhvms7ENvBUuze7eA@public.gmane.org>
> ---
>  checkpoint/Makefile        |    2 +-
>  checkpoint/restart.c       |    4 +
>  checkpoint/rstr_file.c     |  202 ++++++++++++++++++++++++++++++++++++++++++++
>  include/linux/checkpoint.h |    1 +
>  4 files changed, 208 insertions(+), 1 deletions(-)
>  create mode 100644 checkpoint/rstr_file.c
> 
> diff --git a/checkpoint/Makefile b/checkpoint/Makefile
> index 7496695..88bbc10 100644
> --- a/checkpoint/Makefile
> +++ b/checkpoint/Makefile
> @@ -3,4 +3,4 @@
>  #
> 
>  obj-$(CONFIG_CHECKPOINT_RESTART) += sys.o checkpoint.o restart.o objhash.o \
> -		ckpt_mem.o rstr_mem.o ckpt_file.o
> +		ckpt_mem.o rstr_mem.o ckpt_file.o rstr_file.o
> diff --git a/checkpoint/restart.c b/checkpoint/restart.c
> index a0d5e60..956e274 100644
> --- a/checkpoint/restart.c
> +++ b/checkpoint/restart.c
> @@ -212,6 +212,10 @@ static int cr_read_task(struct cr_ctx *ctx)
>  	cr_debug("memory: ret %d\n", ret);
>  	if (ret < 0)
>  		goto out;
> +	ret = cr_read_files(ctx);
> +	cr_debug("files: ret %d\n", ret);
> +	if (ret < 0)
> +		goto out;
>  	ret = cr_read_thread(ctx);
>  	cr_debug("thread: ret %d\n", ret);
>  	if (ret < 0)
> diff --git a/checkpoint/rstr_file.c b/checkpoint/rstr_file.c
> new file mode 100644
> index 0000000..780c0fc
> --- /dev/null
> +++ b/checkpoint/rstr_file.c
> @@ -0,0 +1,202 @@
> +/*
> + *  Checkpoint file descriptors
> + *
> + *  Copyright (C) 2008 Oren Laadan
> + *
> + *  This file is subject to the terms and conditions of the GNU General Public
> + *  License.  See the file COPYING in the main directory of the Linux
> + *  distribution for more details.
> + */
> +
> +#include <linux/kernel.h>
> +#include <linux/sched.h>
> +#include <linux/fs.h>
> +#include <linux/file.h>
> +#include <linux/fdtable.h>
> +#include <linux/fsnotify.h>
> +#include <linux/syscalls.h>
> +#include <linux/checkpoint.h>
> +#include <linux/checkpoint_hdr.h>
> +
> +#include "checkpoint_file.h"
> +
> +static int cr_close_all_fds(struct files_struct *files)
> +{
> +	int *fdtable;
> +	int nfds;
> +
> +	nfds = cr_scan_fds(files, &fdtable);
> +	if (nfds < 0)
> +		return nfds;
> +	while (nfds--)
> +		sys_close(fdtable[nfds]);
> +	kfree(fdtable);
> +	return 0;
> +}
> +
> +/**
> + * cr_attach_file - attach a lonely file ptr to a file descriptor
> + * @file: lonely file pointer
> + */
> +static int cr_attach_file(struct file *file)
> +{
> +	int fd = get_unused_fd_flags(0);
> +
> +	if (fd >= 0) {
> +		fsnotify_open(file->f_path.dentry);
> +		fd_install(fd, file);
> +	}
> +	return fd;
> +}
> +
> +#define CR_SETFL_MASK (O_APPEND|O_NONBLOCK|O_NDELAY|FASYNC|O_DIRECT|O_NOATIME)
> +
> +/* cr_read_fd_data - restore the state of a given file pointer */
> +static int
> +cr_read_fd_data(struct cr_ctx *ctx, struct files_struct *files, int parent)
> +{
> +	struct cr_hdr_fd_data *hh = cr_hbuf_get(ctx, sizeof(*hh));

You're leaking hh in a whole slew of error paths.

> +	struct file *file;
> +	int fd, rparent, ret;
> +
> +	rparent = cr_read_obj_type(ctx, hh, sizeof(*hh), CR_HDR_FD_DATA);
> +	cr_debug("rparent %d parent %d flags %#x mode %#x how %d\n",
> +		 rparent, parent, hh->f_flags, hh->f_mode, hh->fd_type);
> +	if (rparent < 0)
> +		return rparent;
> +	if (rparent != parent)
> +		return -EINVAL;
> +	/* FIX: more sanity checks on f_flags, f_mode etc */
> +
> +	switch (hh->fd_type) {
> +	case CR_FD_FILE:
> +	case CR_FD_DIR:
> +	case CR_FD_LINK:
> +		file = cr_read_open_fname(ctx, hh->f_flags, hh->f_mode);
> +		break;
> +	default:
> +		file = ERR_PTR(-EINVAL);
> +		break;
> +	}
> +
> +	if (IS_ERR(file))
> +		return PTR_ERR(file);
> +
> +	/* FIX: need to restore uid, gid, owner etc */
> +
> +	fd = cr_attach_file(file);	/* no need to cleanup 'file' below */
> +	if (fd < 0) {
> +		filp_close(file, NULL);
> +		return fd;
> +	}
> +
> +	/* register new <objref, file> tuple in hash table */
> +	ret = cr_obj_add_ref(ctx, (void *) file, parent, CR_OBJ_FILE, 0);
> +	if (ret < 0)
> +		goto out;
> +	ret = sys_fcntl(fd, F_SETFL, hh->f_flags & CR_SETFL_MASK);
> +	if (ret < 0)
> +		goto out;
> +	ret = vfs_llseek(file, hh->f_pos, SEEK_SET);
> +	if (ret == -ESPIPE)	/* ignore error on non-seekable files */
> +		ret = 0;
> +
> + out:
> +	cr_hbuf_put(ctx, sizeof(*hh));
> +	return ret < 0 ? ret : fd;
> +}
> +
> +/**
> + * cr_read_fd_ent - restore the state of a given file descriptor
> + * @ctx: checkpoint context
> + * @files: files_struct pointer
> + * @parent: parent objref
> + *
> + * Restores the state of a file descriptor; looks up the objref (in the
> + * header) in the hash table, and if found picks the matching file and
> + * use it; otherwise calls cr_read_fd_data to restore the file too.
> + */
> +static int
> +cr_read_fd_ent(struct cr_ctx *ctx, struct files_struct *files, int parent)
> +{
> +	struct cr_hdr_fd_ent *hh = cr_hbuf_get(ctx, sizeof(*hh));

Again.

> +	struct file *file;
> +	int newfd, rparent;
> +
> +	rparent = cr_read_obj_type(ctx, hh, sizeof(*hh), CR_HDR_FD_ENT);
> +	cr_debug("rparent %d parent %d ref %d\n", rparent, parent, hh->objref);
> +	if (rparent < 0)
> +		return rparent;
> +	if (rparent != parent)
> +		return -EINVAL;
> +	cr_debug("fd %d coe %d\n", hh->fd, hh->close_on_exec);
> +	if (hh->objref <= 0)
> +		return -EINVAL;
> +
> +	file = cr_obj_get_by_ref(ctx, hh->objref, CR_OBJ_FILE);
> +	if (IS_ERR(file))
> +		return PTR_ERR(file);
> +
> +	if (file) {
> +		newfd = cr_attach_file(file);
> +		if (newfd < 0)
> +			return newfd;
> +		get_file(file);
> +	} else {
> +		/* create new file pointer (and register in hash table) */
> +		newfd = cr_read_fd_data(ctx, files, hh->objref);
> +		if (newfd < 0)
> +			return newfd;
> +	}
> +
> +	cr_debug("newfd got %d wanted %d\n", newfd, hh->fd);
> +
> +	/* if newfd isn't desired fd then reposition it */
> +	if (newfd != hh->fd) {
> +		int ret = sys_dup2(newfd, hh->fd);
> +		if (ret < 0)
> +			return ret;
> +		sys_close(newfd);
> +	}
> +
> +	if (hh->close_on_exec)
> +		set_close_on_exec(hh->fd, 1);
> +
> +	cr_hbuf_put(ctx, sizeof(*hh));
> +	return 0;
> +}
> +
> +int cr_read_files(struct cr_ctx *ctx)
> +{
> +	struct cr_hdr_files *hh = cr_hbuf_get(ctx, sizeof(*hh));

Again...

> +	struct files_struct *files = current->files;
> +	int i, parent, ret;
> +
> +	parent = cr_read_obj_type(ctx, hh, sizeof(*hh), CR_HDR_FILES);
> +	if (parent < 0)
> +		return parent;
> +#if 0	/* activate when containers are used */
> +	if (parent != task_pid_vnr(current))
> +		return -EINVAL;
> +#endif
> +	cr_debug("objref %d nfds %d\n", hh->objref, hh->nfds);
> +	if (hh->objref < 0 || hh->nfds < 0)
> +		return -EINVAL;
> +
> +	if (hh->nfds > sysctl_nr_open)
> +		return -EMFILE;
> +
> +	/* point of no return -- close all file descriptors */
> +	ret = cr_close_all_fds(files);
> +	if (ret < 0)
> +		return ret;
> +
> +	for (i = 0; i < hh->nfds; i++) {
> +		ret = cr_read_fd_ent(ctx, files, hh->objref);
> +		if (ret < 0)
> +			break;
> +	}
> +
> +	cr_hbuf_put(ctx, sizeof(*hh));
> +	return ret;
> +}
> diff --git a/include/linux/checkpoint.h b/include/linux/checkpoint.h
> index a202c54..fd3baac 100644
> --- a/include/linux/checkpoint.h
> +++ b/include/linux/checkpoint.h
> @@ -89,6 +89,7 @@ extern int cr_write_files(struct cr_ctx *ctx, struct task_struct *t);
> 
>  extern int do_restart(struct cr_ctx *ctx);
>  extern int cr_read_mm(struct cr_ctx *ctx);
> +extern int cr_read_files(struct cr_ctx *ctx);
> 
>  #define cr_debug(fmt, args...)  \
>  	pr_debug("[CR:%s] " fmt, __func__, ## args)
> -- 
> 1.5.4.3
> 
> _______________________________________________
> Containers mailing list
> Containers-cunTk1MwBs9QetFLy7KEm3xJsTq8ys+cHZ5vskTnxNA@public.gmane.org
> https://lists.linux-foundation.org/mailman/listinfo/containers

Re: [RFC v5][PATCH 9/9] Restore open file descriprtors

[Oren Laadan]

Serge E. Hallyn wrote:
> Quoting Oren Laadan (orenl-eQaUEPhvms7ENvBUuze7eA@public.gmane.org):
>> Restore open file descriptors: for each FD read 'struct cr_hdr_fd_ent'
>> and lookup objref in the hash table; if not found (first occurence), read
>> in 'struct cr_hdr_fd_data', create a new FD and register in the hash.
>> Otherwise attach the file pointer from the hash as an FD.
>>
>> This patch only handles basic FDs - regular files, directories and also
>> symbolic links.
>>
>> Signed-off-by: Oren Laadan <orenl-eQaUEPhvms7ENvBUuze7eA@public.gmane.org>
>> ---
>>  checkpoint/Makefile        |    2 +-
>>  checkpoint/restart.c       |    4 +
>>  checkpoint/rstr_file.c     |  202 ++++++++++++++++++++++++++++++++++++++++++++
>>  include/linux/checkpoint.h |    1 +
>>  4 files changed, 208 insertions(+), 1 deletions(-)
>>  create mode 100644 checkpoint/rstr_file.c
>>
>> diff --git a/checkpoint/Makefile b/checkpoint/Makefile
>> index 7496695..88bbc10 100644
>> --- a/checkpoint/Makefile
>> +++ b/checkpoint/Makefile
>> @@ -3,4 +3,4 @@
>>  #
>>
>>  obj-$(CONFIG_CHECKPOINT_RESTART) += sys.o checkpoint.o restart.o objhash.o \
>> -		ckpt_mem.o rstr_mem.o ckpt_file.o
>> +		ckpt_mem.o rstr_mem.o ckpt_file.o rstr_file.o
>> diff --git a/checkpoint/restart.c b/checkpoint/restart.c
>> index a0d5e60..956e274 100644
>> --- a/checkpoint/restart.c
>> +++ b/checkpoint/restart.c
>> @@ -212,6 +212,10 @@ static int cr_read_task(struct cr_ctx *ctx)
>>  	cr_debug("memory: ret %d\n", ret);
>>  	if (ret < 0)
>>  		goto out;
>> +	ret = cr_read_files(ctx);
>> +	cr_debug("files: ret %d\n", ret);
>> +	if (ret < 0)
>> +		goto out;
>>  	ret = cr_read_thread(ctx);
>>  	cr_debug("thread: ret %d\n", ret);
>>  	if (ret < 0)
>> diff --git a/checkpoint/rstr_file.c b/checkpoint/rstr_file.c
>> new file mode 100644
>> index 0000000..780c0fc
>> --- /dev/null
>> +++ b/checkpoint/rstr_file.c
>> @@ -0,0 +1,202 @@
>> +/*
>> + *  Checkpoint file descriptors
>> + *
>> + *  Copyright (C) 2008 Oren Laadan
>> + *
>> + *  This file is subject to the terms and conditions of the GNU General Public
>> + *  License.  See the file COPYING in the main directory of the Linux
>> + *  distribution for more details.
>> + */
>> +
>> +#include <linux/kernel.h>
>> +#include <linux/sched.h>
>> +#include <linux/fs.h>
>> +#include <linux/file.h>
>> +#include <linux/fdtable.h>
>> +#include <linux/fsnotify.h>
>> +#include <linux/syscalls.h>
>> +#include <linux/checkpoint.h>
>> +#include <linux/checkpoint_hdr.h>
>> +
>> +#include "checkpoint_file.h"
>> +
>> +static int cr_close_all_fds(struct files_struct *files)
>> +{
>> +	int *fdtable;
>> +	int nfds;
>> +
>> +	nfds = cr_scan_fds(files, &fdtable);
>> +	if (nfds < 0)
>> +		return nfds;
>> +	while (nfds--)
>> +		sys_close(fdtable[nfds]);
>> +	kfree(fdtable);
>> +	return 0;
>> +}
>> +
>> +/**
>> + * cr_attach_file - attach a lonely file ptr to a file descriptor
>> + * @file: lonely file pointer
>> + */
>> +static int cr_attach_file(struct file *file)
>> +{
>> +	int fd = get_unused_fd_flags(0);
>> +
>> +	if (fd >= 0) {
>> +		fsnotify_open(file->f_path.dentry);
>> +		fd_install(fd, file);
>> +	}
>> +	return fd;
>> +}
>> +
>> +#define CR_SETFL_MASK (O_APPEND|O_NONBLOCK|O_NDELAY|FASYNC|O_DIRECT|O_NOATIME)
>> +
>> +/* cr_read_fd_data - restore the state of a given file pointer */
>> +static int
>> +cr_read_fd_data(struct cr_ctx *ctx, struct files_struct *files, int parent)
>> +{
>> +	struct cr_hdr_fd_data *hh = cr_hbuf_get(ctx, sizeof(*hh));
> 
> You're leaking hh in a whole slew of error paths.

No. (this was discussed earlier already).

cr_hbuf_get() "allocates" space inside a dedicated buffer for headers
in the checkpoint context (ctx->hbuf). It does not allocate new kernel
memory. Instead, it returns the current position in that buffer
ctx->hbuf[ctx->hpos], and advances ctx->hpos appropriately. On the
other side, cr_hbuf_put() reverses that effect, reducing ctx->hpos
accordingly.

If an error occurs, the checkpoint (or restart) operation is aborted,
and eventually the context (ctx) will be cleaned up; at that point the
special purpose buffer will be freed.

[...]

Oren.

Re: [RFC v5][PATCH 9/9] Restore open file descriprtors

[Serge E. Hallyn]

Quoting Oren Laadan (orenl-eQaUEPhvms7ENvBUuze7eA@public.gmane.org):
>
>
> Serge E. Hallyn wrote:
>> Quoting Oren Laadan (orenl-eQaUEPhvms7ENvBUuze7eA@public.gmane.org):
>>> Restore open file descriptors: for each FD read 'struct cr_hdr_fd_ent'
>>> and lookup objref in the hash table; if not found (first occurence), read
>>> in 'struct cr_hdr_fd_data', create a new FD and register in the hash.
>>> Otherwise attach the file pointer from the hash as an FD.
>>>
>>> This patch only handles basic FDs - regular files, directories and also
>>> symbolic links.
>>>
>>> Signed-off-by: Oren Laadan <orenl-eQaUEPhvms7ENvBUuze7eA@public.gmane.org>
>>> ---
>>>  checkpoint/Makefile        |    2 +-
>>>  checkpoint/restart.c       |    4 +
>>>  checkpoint/rstr_file.c     |  202 ++++++++++++++++++++++++++++++++++++++++++++
>>>  include/linux/checkpoint.h |    1 +
>>>  4 files changed, 208 insertions(+), 1 deletions(-)
>>>  create mode 100644 checkpoint/rstr_file.c
>>>
>>> diff --git a/checkpoint/Makefile b/checkpoint/Makefile
>>> index 7496695..88bbc10 100644
>>> --- a/checkpoint/Makefile
>>> +++ b/checkpoint/Makefile
>>> @@ -3,4 +3,4 @@
>>>  #
>>>
>>>  obj-$(CONFIG_CHECKPOINT_RESTART) += sys.o checkpoint.o restart.o objhash.o \
>>> -		ckpt_mem.o rstr_mem.o ckpt_file.o
>>> +		ckpt_mem.o rstr_mem.o ckpt_file.o rstr_file.o
>>> diff --git a/checkpoint/restart.c b/checkpoint/restart.c
>>> index a0d5e60..956e274 100644
>>> --- a/checkpoint/restart.c
>>> +++ b/checkpoint/restart.c
>>> @@ -212,6 +212,10 @@ static int cr_read_task(struct cr_ctx *ctx)
>>>  	cr_debug("memory: ret %d\n", ret);
>>>  	if (ret < 0)
>>>  		goto out;
>>> +	ret = cr_read_files(ctx);
>>> +	cr_debug("files: ret %d\n", ret);
>>> +	if (ret < 0)
>>> +		goto out;
>>>  	ret = cr_read_thread(ctx);
>>>  	cr_debug("thread: ret %d\n", ret);
>>>  	if (ret < 0)
>>> diff --git a/checkpoint/rstr_file.c b/checkpoint/rstr_file.c
>>> new file mode 100644
>>> index 0000000..780c0fc
>>> --- /dev/null
>>> +++ b/checkpoint/rstr_file.c
>>> @@ -0,0 +1,202 @@
>>> +/*
>>> + *  Checkpoint file descriptors
>>> + *
>>> + *  Copyright (C) 2008 Oren Laadan
>>> + *
>>> + *  This file is subject to the terms and conditions of the GNU General Public
>>> + *  License.  See the file COPYING in the main directory of the Linux
>>> + *  distribution for more details.
>>> + */
>>> +
>>> +#include <linux/kernel.h>
>>> +#include <linux/sched.h>
>>> +#include <linux/fs.h>
>>> +#include <linux/file.h>
>>> +#include <linux/fdtable.h>
>>> +#include <linux/fsnotify.h>
>>> +#include <linux/syscalls.h>
>>> +#include <linux/checkpoint.h>
>>> +#include <linux/checkpoint_hdr.h>
>>> +
>>> +#include "checkpoint_file.h"
>>> +
>>> +static int cr_close_all_fds(struct files_struct *files)
>>> +{
>>> +	int *fdtable;
>>> +	int nfds;
>>> +
>>> +	nfds = cr_scan_fds(files, &fdtable);
>>> +	if (nfds < 0)
>>> +		return nfds;
>>> +	while (nfds--)
>>> +		sys_close(fdtable[nfds]);
>>> +	kfree(fdtable);
>>> +	return 0;
>>> +}
>>> +
>>> +/**
>>> + * cr_attach_file - attach a lonely file ptr to a file descriptor
>>> + * @file: lonely file pointer
>>> + */
>>> +static int cr_attach_file(struct file *file)
>>> +{
>>> +	int fd = get_unused_fd_flags(0);
>>> +
>>> +	if (fd >= 0) {
>>> +		fsnotify_open(file->f_path.dentry);
>>> +		fd_install(fd, file);
>>> +	}
>>> +	return fd;
>>> +}
>>> +
>>> +#define CR_SETFL_MASK (O_APPEND|O_NONBLOCK|O_NDELAY|FASYNC|O_DIRECT|O_NOATIME)
>>> +
>>> +/* cr_read_fd_data - restore the state of a given file pointer */
>>> +static int
>>> +cr_read_fd_data(struct cr_ctx *ctx, struct files_struct *files, int parent)
>>> +{
>>> +	struct cr_hdr_fd_data *hh = cr_hbuf_get(ctx, sizeof(*hh));
>>
>> You're leaking hh in a whole slew of error paths.
>
> No. (this was discussed earlier already).
>
> cr_hbuf_get() "allocates" space inside a dedicated buffer for headers
> in the checkpoint context (ctx->hbuf). It does not allocate new kernel
> memory. Instead, it returns the current position in that buffer
> ctx->hbuf[ctx->hpos], and advances ctx->hpos appropriately. On the
> other side, cr_hbuf_put() reverses that effect, reducing ctx->hpos
> accordingly.
>
> If an error occurs, the checkpoint (or restart) operation is aborted,
> and eventually the context (ctx) will be cleaned up; at that point the
> special purpose buffer will be freed.
>
> [...]
>
> Oren.

Yes I realize you're not doing a real allocation here and so, especially
if the whole thing is about to fail anyway, there may seem to be little
point in bothering to _put().  The thing is it's an unbalanced
operation, and the behind-the-scenes implementation may change at some
later point so IMO it's definately worth balancing these things now.

-serge

Re: [RFC v5][PATCH 9/9] Restore open file descriprtors

[Dave Hansen]

On Tue, 2008-09-16 at 20:11 -0400, Oren Laadan wrote:
> 
> No. (this was discussed earlier already).
> 
> cr_hbuf_get() "allocates" space inside a dedicated buffer for headers
> in the checkpoint context (ctx->hbuf). It does not allocate new kernel
> memory. Instead, it returns the current position in that buffer
> ctx->hbuf[ctx->hpos], and advances ctx->hpos appropriately. On the
> other side, cr_hbuf_put() reverses that effect, reducing ctx->hpos
> accordingly.
> 
> If an error occurs, the checkpoint (or restart) operation is aborted,
> and eventually the context (ctx) will be cleaned up; at that point the
> special purpose buffer will be freed.

I think this is like claiming that my malloc() will get freed if my
applications exits, so I don't have to worry about free().  It is messy,
it makes lifetime rules and use less explicit, and it makes bugs harder
to find.  If I were a good programmer (which I'm not) I probably
wouldn't do that.

-- Dave

Re: [RFC v5][PATCH 4/8] Dump memory address space

[MinChan Kim]

On Sun, Sep 14, 2008 at 8:06 AM, Oren Laadan <orenl-eQaUEPhvms7ENvBUuze7eA@public.gmane.org> wrote:
> For each VMA, there is a 'struct cr_vma'; if the VMA is file-mapped,
> it will be followed by the file name. Then comes the actual contents,
> in one or more chunk: each chunk begins with a header that specifies
> how many pages it holds, then the virtual addresses of all the dumped
> pages in that chunk, followed by the actual contents of all dumped
> pages. A header with zero number of pages marks the end of the contents.
> Then comes the next VMA and so on.
>
> Signed-off-by: Oren Laadan <orenl-eQaUEPhvms7ENvBUuze7eA@public.gmane.org>
> ---
>  arch/x86/mm/checkpoint.c         |   30 +++
>  arch/x86/mm/restart.c            |    1 +
>  checkpoint/Makefile              |    3 +-
>  checkpoint/checkpoint.c          |   53 ++++
>  checkpoint/checkpoint_arch.h     |    2 +
>  checkpoint/checkpoint_mem.h      |   41 ++++
>  checkpoint/ckpt_mem.c            |  492 ++++++++++++++++++++++++++++++++++++++
>  checkpoint/sys.c                 |   27 ++-
>  include/asm-x86/checkpoint_hdr.h |    5 +
>  include/linux/checkpoint.h       |   12 +
>  include/linux/checkpoint_hdr.h   |   32 +++
>  11 files changed, 692 insertions(+), 6 deletions(-)
>  create mode 100644 checkpoint/checkpoint_mem.h
>  create mode 100644 checkpoint/ckpt_mem.c
>
> diff --git a/arch/x86/mm/checkpoint.c b/arch/x86/mm/checkpoint.c
> index eb60003..fa87e4a 100644
> --- a/arch/x86/mm/checkpoint.c
> +++ b/arch/x86/mm/checkpoint.c
> @@ -196,3 +196,33 @@ int cr_write_cpu(struct cr_ctx *ctx, struct task_struct *t)
>        cr_hbuf_put(ctx, sizeof(*hh));
>        return ret;
>  }
> +
> +/* dump the mm->context state */
> +int cr_write_mm_context(struct cr_ctx *ctx, struct mm_struct *mm, int parent)
> +{
> +       struct cr_hdr h;
> +       struct cr_hdr_mm_context *hh = cr_hbuf_get(ctx, sizeof(*hh));
> +       int ret;
> +
> +       h.type = CR_HDR_MM_CONTEXT;
> +       h.len = sizeof(*hh);
> +       h.parent = parent;
> +
> +       mutex_lock(&mm->context.lock);
> +
> +       hh->ldt_entry_size = LDT_ENTRY_SIZE;
> +       hh->nldt = mm->context.size;
> +
> +       cr_debug("nldt %d\n", hh->nldt);
> +
> +       ret = cr_write_obj(ctx, &h, hh);
> +       cr_hbuf_put(ctx, sizeof(*hh));
> +       if (ret < 0)
> +               goto out;
> +
> +       ret = cr_kwrite(ctx, mm->context.ldt, hh->nldt * LDT_ENTRY_SIZE);
> +
> + out:
> +       mutex_unlock(&mm->context.lock);
> +       return ret;
> +}
> diff --git a/arch/x86/mm/restart.c b/arch/x86/mm/restart.c
> index 0178080..79780df 100644
> --- a/arch/x86/mm/restart.c
> +++ b/arch/x86/mm/restart.c
> @@ -8,6 +8,7 @@
>  *  distribution for more details.
>  */
>
> +#include <linux/unistd.h>
>  #include <asm/desc.h>
>  #include <asm/i387.h>
>
> diff --git a/checkpoint/Makefile b/checkpoint/Makefile
> index d2df68c..3a0df6d 100644
> --- a/checkpoint/Makefile
> +++ b/checkpoint/Makefile
> @@ -2,4 +2,5 @@
>  # Makefile for linux checkpoint/restart.
>  #
>
> -obj-$(CONFIG_CHECKPOINT_RESTART) += sys.o checkpoint.o restart.o
> +obj-$(CONFIG_CHECKPOINT_RESTART) += sys.o checkpoint.o restart.o \
> +               ckpt_mem.o
> diff --git a/checkpoint/checkpoint.c b/checkpoint/checkpoint.c
> index 6ca26d0..d4c1b31 100644
> --- a/checkpoint/checkpoint.c
> +++ b/checkpoint/checkpoint.c
> @@ -55,6 +55,55 @@ int cr_write_string(struct cr_ctx *ctx, char *str, int len)
>        return cr_write_obj(ctx, &h, str);
>  }
>
> +/**
> + * cr_fill_fname - return pathname of a given file
> + * @path: path name
> + * @root: relative root
> + * @buf: buffer for pathname
> + * @n: buffer length (in) and pathname length (out)
> + */
> +static char *
> +cr_fill_fname(struct path *path, struct path *root, char *buf, int *n)
> +{
> +       char *fname;
> +
> +       BUG_ON(!buf);
> +       fname = __d_path(path, root, buf, *n);
> +       if (!IS_ERR(fname))
> +               *n = (buf + (*n) - fname);
> +       return fname;
> +}
> +
> +/**
> + * cr_write_fname - write a file name
> + * @ctx: checkpoint context
> + * @path: path name
> + * @root: relative root
> + */
> +int cr_write_fname(struct cr_ctx *ctx, struct path *path, struct path *root)
> +{
> +       struct cr_hdr h;
> +       char *buf, *fname;
> +       int ret, flen;
> +
> +       flen = PATH_MAX;
> +       buf = kmalloc(flen, GFP_KERNEL);
> +       if (!buf)
> +               return -ENOMEM;
> +
> +       fname = cr_fill_fname(path, root, buf, &flen);
> +       if (!IS_ERR(fname)) {
> +               h.type = CR_HDR_FNAME;
> +               h.len = flen;
> +               h.parent = 0;
> +               ret = cr_write_obj(ctx, &h, fname);
> +       } else
> +               ret = PTR_ERR(fname);
> +
> +       kfree(buf);
> +       return ret;
> +}
> +
>  /* write the checkpoint header */
>  static int cr_write_head(struct cr_ctx *ctx)
>  {
> @@ -150,6 +199,10 @@ static int cr_write_task(struct cr_ctx *ctx, struct task_struct *t)
>        cr_debug("task_struct: ret %d\n", ret);
>        if (ret < 0)
>                goto out;
> +       ret = cr_write_mm(ctx, t);
> +       cr_debug("memory: ret %d\n", ret);
> +       if (ret < 0)
> +               goto out;
>        ret = cr_write_thread(ctx, t);
>        cr_debug("thread: ret %d\n", ret);
>        if (ret < 0)
> diff --git a/checkpoint/checkpoint_arch.h b/checkpoint/checkpoint_arch.h
> index bf2d21e..7da4ad0 100644
> --- a/checkpoint/checkpoint_arch.h
> +++ b/checkpoint/checkpoint_arch.h
> @@ -2,6 +2,8 @@
>
>  extern int cr_write_thread(struct cr_ctx *ctx, struct task_struct *t);
>  extern int cr_write_cpu(struct cr_ctx *ctx, struct task_struct *t);
> +extern int cr_write_mm_context(struct cr_ctx *ctx,
> +                              struct mm_struct *mm, int parent);
>
>  extern int cr_read_thread(struct cr_ctx *ctx);
>  extern int cr_read_cpu(struct cr_ctx *ctx);
> diff --git a/checkpoint/checkpoint_mem.h b/checkpoint/checkpoint_mem.h
> new file mode 100644
> index 0000000..85546f4
> --- /dev/null
> +++ b/checkpoint/checkpoint_mem.h
> @@ -0,0 +1,41 @@
> +#ifndef _CHECKPOINT_CKPT_MEM_H_
> +#define _CHECKPOINT_CKPT_MEM_H_
> +/*
> + *  Generic container checkpoint-restart
> + *
> + *  Copyright (C) 2008 Oren Laadan
> + *
> + *  This file is subject to the terms and conditions of the GNU General Public
> + *  License.  See the file COPYING in the main directory of the Linux
> + *  distribution for more details.
> + */
> +
> +#include <linux/mm_types.h>
> +
> +/*
> + * page-array chains: each cr_pgarr describes a set of <strcut page *,vaddr>
> + * tuples (where vaddr is the virtual address of a page in a particular mm).
> + * Specifically, we use separate arrays so that all vaddrs can be written
> + * and read at once.
> + */
> +
> +struct cr_pgarr {
> +       unsigned long *vaddrs;
> +       struct page **pages;
> +       unsigned int nr_used;
> +       struct list_head list;
> +};
> +
> +#define CR_PGARR_TOTAL  (PAGE_SIZE / sizeof(void *))
> +#define CR_PGARR_CHUNK  (4 * CR_PGARR_TOTAL)
> +
> +extern void cr_pgarr_free(struct cr_ctx *ctx);
> +extern struct cr_pgarr *cr_pgarr_current(struct cr_ctx *ctx);
> +extern void cr_pgarr_reset_all(struct cr_ctx *ctx);
> +
> +static inline int cr_pgarr_is_full(struct cr_pgarr *pgarr)
> +{
> +       return (pgarr->nr_used == CR_PGARR_TOTAL);
> +}
> +
> +#endif /* _CHECKPOINT_CKPT_MEM_H_ */
> diff --git a/checkpoint/ckpt_mem.c b/checkpoint/ckpt_mem.c
> new file mode 100644
> index 0000000..e0226f4
> --- /dev/null
> +++ b/checkpoint/ckpt_mem.c
> @@ -0,0 +1,492 @@
> +/*
> + *  Checkpoint memory contents
> + *
> + *  Copyright (C) 2008 Oren Laadan
> + *
> + *  This file is subject to the terms and conditions of the GNU General Public
> + *  License.  See the file COPYING in the main directory of the Linux
> + *  distribution for more details.
> + */
> +
> +#include <linux/kernel.h>
> +#include <linux/sched.h>
> +#include <linux/slab.h>
> +#include <linux/file.h>
> +#include <linux/pagemap.h>
> +#include <linux/mm_types.h>
> +#include <linux/checkpoint.h>
> +#include <linux/checkpoint_hdr.h>
> +
> +#include "checkpoint_arch.h"
> +#include "checkpoint_mem.h"
> +
> +/*
> + * utilities to alloc, free, and handle 'struct cr_pgarr' (page-arrays)
> + * (common to ckpt_mem.c and rstr_mem.c).
> + *
> + * The checkpoint context structure has two members for page-arrays:
> + *   ctx->pgarr_list: list head of the page-array chain
> + *
> + * During checkpoint (and restart) the chain tracks the dirty pages (page
> + * pointer and virtual address) of each MM. For a particular MM, these are
> + * always added to the head of the page-array chain (ctx->pgarr_list).
> + * This "current" page-array advances as necessary, and new page-array
> + * descriptors are allocated on-demand. Before the next chunk of pages,
> + * the chain is reset but not freed (that is, dereference page pointers).
> + */
> +
> +/* return first page-array in the chain */
> +static inline struct cr_pgarr *cr_pgarr_first(struct cr_ctx *ctx)
> +{
> +       if (list_empty(&ctx->pgarr_list))
> +               return NULL;
> +       return list_first_entry(&ctx->pgarr_list, struct cr_pgarr, list);
> +}
> +
> +/* release pages referenced by a page-array */
> +static void cr_pgarr_release_pages(struct cr_pgarr *pgarr)
> +{
> +       int i;
> +
> +       cr_debug("nr_used %d\n", pgarr->nr_used);
> +       /*
> +        * although both checkpoint and restart use 'nr_used', we only
> +        * collect pages during checkpoint; in restart we simply return
> +        */
> +       if (!pgarr->pages)
> +               return;
> +       for (i = pgarr->nr_used; i--; /**/)
> +               page_cache_release(pgarr->pages[i]);
> +}
> +
> +/* free a single page-array object */
> +static void cr_pgarr_free_one(struct cr_pgarr *pgarr)
> +{
> +       cr_pgarr_release_pages(pgarr);
> +       kfree(pgarr->pages);
> +       kfree(pgarr->vaddrs);
> +       kfree(pgarr);
> +}
> +
> +/* free a chain of page-arrays */
> +void cr_pgarr_free(struct cr_ctx *ctx)
> +{
> +       struct cr_pgarr *pgarr, *tmp;
> +
> +       list_for_each_entry_safe(pgarr, tmp, &ctx->pgarr_list, list) {
> +               list_del(&pgarr->list);
> +               cr_pgarr_free_one(pgarr);
> +       }
> +}
> +
> +/* allocate a single page-array object */
> +static struct cr_pgarr *cr_pgarr_alloc_one(unsigned long flags)
> +{
> +       struct cr_pgarr *pgarr;
> +
> +       pgarr = kzalloc(sizeof(*pgarr), GFP_KERNEL);
> +       if (!pgarr)
> +               return NULL;
> +
> +       pgarr->vaddrs = kmalloc(CR_PGARR_TOTAL * sizeof(unsigned long),
> +                              GFP_KERNEL);
> +       if (!pgarr->vaddrs)
> +               goto nomem;
> +
> +       /* pgarr->pages is needed only for checkpoint */
> +       if (flags & CR_CTX_CKPT) {
> +               pgarr->pages = kmalloc(CR_PGARR_TOTAL * sizeof(struct page *),
> +                                      GFP_KERNEL);
> +               if (!pgarr->pages)
> +                       goto nomem;
> +       }
> +
> +       return pgarr;
> +
> + nomem:
> +       cr_pgarr_free_one(pgarr);
> +       return NULL;
> +}
> +
> +/* cr_pgarr_current - return the next available page-array in the chain
> + * @ctx: checkpoint context
> + *
> + * Returns the first page-array in the list that has space. Extends the
> + * list if none has space.
> + */
> +struct cr_pgarr *cr_pgarr_current(struct cr_ctx *ctx)
> +{
> +       struct cr_pgarr *pgarr;
> +
> +       pgarr = cr_pgarr_first(ctx);
> +       if (pgarr && !cr_pgarr_is_full(pgarr))
> +               goto out;
> +       pgarr = cr_pgarr_alloc_one(ctx->flags);
> +       if (!pgarr)
> +               goto out;
> +       list_add(&pgarr->list, &ctx->pgarr_list);
> + out:
> +       return pgarr;
> +}
> +
> +/* reset the page-array chain (dropping page references if necessary) */
> +void cr_pgarr_reset_all(struct cr_ctx *ctx)
> +{
> +       struct cr_pgarr *pgarr;
> +
> +       list_for_each_entry(pgarr, &ctx->pgarr_list, list) {
> +               cr_pgarr_release_pages(pgarr);
> +               pgarr->nr_used = 0;
> +       }
> +}
> +
> +/*
> + * Checkpoint is outside the context of the checkpointee, so one cannot
> + * simply read pages from user-space. Instead, we scan the address space
> + * of the target to cherry-pick pages of interest. Selected pages are
> + * enlisted in a page-array chain (attached to the checkpoint context).
> + * To save their contents, each page is mapped to kernel memory and then
> + * dumped to the file descriptor.
> + */
> +
> +
> +/**
> + * cr_follow_page - return page pointer for pages that were modified
> + * @vma - target vma
> + * @addr - page address
> + *
> + * Looks up the page that correspond to the address in the vma, and
> + * returns the page if it was modified (and grabs a reference to it),
> + * or otherwise returns NULL (or error).
> + */
> +static struct page *
> +cr_follow_page(struct vm_area_struct *vma, unsigned long addr)
> +{
> +       struct page *page;
> +
> +       /*
> +        * simplified version of get_user_pages(): already have vma,
> +        * only need FOLL_ANON, and (for now) ignore fault stats.
> +        *
> +        * follow_page() will return NULL if the page is not present
> +        * (swapped), ZERO_PAGE(0) if the pte wasn't allocated, and
> +        * the actual page pointer otherwise.
> +        *
> +        * FIXME: consolidate with get_user_pages()
> +        */
> +
> +       cond_resched();
> +       while (!(page = follow_page(vma, addr, FOLL_ANON | FOLL_GET))) {
> +               int ret;
> +
> +               /* the page is swapped out - bring it in (optimize ?) */
> +               ret = handle_mm_fault(vma->vm_mm, vma, addr, 0);
> +               if (ret & VM_FAULT_ERROR) {
> +                       if (ret & VM_FAULT_OOM)
> +                               return ERR_PTR(-ENOMEM);
> +                       else if (ret & VM_FAULT_SIGBUS)
> +                               return ERR_PTR(-EFAULT);
> +                       else
> +                               BUG();
> +                       break;
> +               }
> +               cond_resched();
> +       }
> +
> +       if (IS_ERR(page))
> +               return page;
> +
> +       if (page == ZERO_PAGE(0)) {
> +               /* this is the zero page: ignore */
> +               page_cache_release(page);
> +               page = NULL;
> +       } else if (vma->vm_file && (page_mapping(page) != NULL)) {
> +               /* file backed clean cow: ignore */


If COW occur in file-backed pages, Is page_mapping is modified ?
I don't understand why this condition represent "file backed clean cow"?
Could you elaborate please?

> +               page_cache_release(page);
> +               page = NULL;
> +       }
> +
> +       return page;
> +}
> +
> +/**
> + * cr_private_vma_fill_pgarr - fill a page-array with addr/page tuples
> + * @ctx - checkpoint context
> + * @pgarr - page-array to fill
> + * @vma - vma to scan
> + * @start - start address (updated)
> + *
> + * Returns the number of pages collected
> + */
> +static int
> +cr_private_vma_fill_pgarr(struct cr_ctx *ctx, struct cr_pgarr *pgarr,
> +                         struct vm_area_struct *vma, unsigned long *start)
> +{
> +       unsigned long end = vma->vm_end;
> +       unsigned long addr = *start;
> +       int orig_used = pgarr->nr_used;
> +
> +       /* this function is only for private memory (anon or file-mapped) */
> +       BUG_ON(vma->vm_flags & (VM_SHARED | VM_MAYSHARE));
> +
> +       while (addr < end) {
> +               struct page *page;
> +
> +               page = cr_follow_page(vma, addr);
> +               if (IS_ERR(page))
> +                       return PTR_ERR(page);
> +
> +               if (page) {
> +                       pgarr->pages[pgarr->nr_used] = page;
> +                       pgarr->vaddrs[pgarr->nr_used] = addr;
> +                       pgarr->nr_used++;
> +               }
> +
> +               addr += PAGE_SIZE;
> +
> +               if (cr_pgarr_is_full(pgarr))
> +                       break;
> +       }
> +
> +       *start = addr;
> +       return pgarr->nr_used - orig_used;
> +}
> +
> +/* dump contents of a pages: use kmap_atomic() to avoid TLB flush */
> +static int cr_page_write(struct cr_ctx *ctx, struct page *page, char *buf)
> +{
> +       void *ptr;
> +
> +       ptr = kmap_atomic(page, KM_USER1);
> +       memcpy(buf, ptr, PAGE_SIZE);
> +       kunmap_atomic(page, KM_USER1);
> +
> +       return cr_kwrite(ctx, buf, PAGE_SIZE);
> +}
> +
> +/**
> + * cr_vma_dump_pages - dump pages listed in the ctx page-array chain
> + * @ctx - checkpoint context
> + * @total - total number of pages
> + *
> + * First dump all virtual addresses, followed by the contents of all pages
> + */
> +static int cr_vma_dump_pages(struct cr_ctx *ctx, int total)
> +{
> +       struct cr_pgarr *pgarr;
> +       char *buf;
> +       int i, ret = 0;
> +
> +       if (!total)
> +               return 0;
> +
> +       list_for_each_entry_reverse(pgarr, &ctx->pgarr_list, list) {
> +               ret = cr_kwrite(ctx, pgarr->vaddrs,
> +                               pgarr->nr_used * sizeof(*pgarr->vaddrs));
> +               if (ret < 0)
> +                       return ret;
> +       }
> +
> +       buf = kmalloc(PAGE_SIZE, GFP_KERNEL);
> +       if (!buf)
> +               return -ENOMEM;
> +
> +       list_for_each_entry_reverse(pgarr, &ctx->pgarr_list, list) {
> +               for (i = 0; i < pgarr->nr_used; i++) {
> +                       ret = cr_page_write(ctx, pgarr->pages[i], buf);
> +                       if (ret < 0)
> +                               goto out;
> +               }
> +       }
> +
> + out:
> +       kfree(buf);
> +       return ret;
> +}
> +
> +/**
> + * cr_write_private_vma_contents - dump contents of a VMA with private memory
> + * @ctx - checkpoint context
> + * @vma - vma to scan
> + *
> + * Collect lists of pages that needs to be dumped, and corresponding
> + * virtual addresses into ctx->pgarr_list page-array chain. Then dump
> + * the addresses, followed by the page contents.
> + */
> +static int
> +cr_write_private_vma_contents(struct cr_ctx *ctx, struct vm_area_struct *vma)
> +{
> +       struct cr_hdr h;
> +       struct cr_hdr_pgarr *hh;
> +       unsigned long addr = vma->vm_start;
> +       struct cr_pgarr *pgarr;
> +       unsigned long cnt = 0;
> +       int ret;
> +
> +       /*
> +        * Work iteratively, collecting and dumping at most CR_PGARR_CHUNK
> +        * in each round. Each iterations is divided into two steps:
> +        *
> +        * (1) scan: scan through the PTEs of the vma to collect the pages
> +        * to dump (later we'll also make them COW), while keeping a list
> +        * of pages and their corresponding addresses on ctx->pgarr_list.
> +        *
> +        * (2) dump: write out a header specifying how many pages, followed
> +        * by the addresses of all pages in ctx->pgarr_list, followed by
> +        * the actual contents of all pages. (Then, release the references
> +        * to the pages and reset the page-array chain).
> +        *
> +        * (This split makes the logic simpler by first counting the pages
> +        * that need saving. More importantly, it allows for a future
> +        * optimization that will reduce application downtime by deferring
> +        * the actual write-out of the data to after the application is
> +        * allowed to resume execution).
> +        *
> +        * After dumpting the entire contents, conclude with a header that
> +        * specifies 0 pages to mark the end of the contents.
> +        */
> +
> +       h.type = CR_HDR_PGARR;
> +       h.len = sizeof(*hh);
> +       h.parent = 0;
> +
> +       while (addr < vma->vm_end) {
> +               pgarr = cr_pgarr_current(ctx);
> +               if (!pgarr)
> +                       return -ENOMEM;
> +               ret = cr_private_vma_fill_pgarr(ctx, pgarr, vma, &addr);
> +               if (ret < 0)
> +                       return ret;
> +               cnt += ret;
> +
> +               /* did we complete a chunk, or is this the last chunk ? */
> +               if (cnt >= CR_PGARR_CHUNK || (cnt && addr == vma->vm_end)) {
> +                       hh = cr_hbuf_get(ctx, sizeof(*hh));
> +                       hh->nr_pages = cnt;
> +                       ret = cr_write_obj(ctx, &h, hh);
> +                       cr_hbuf_put(ctx, sizeof(*hh));
> +                       if (ret < 0)
> +                               return ret;
> +
> +                       ret = cr_vma_dump_pages(ctx, cnt);
> +                       if (ret < 0)
> +                               return ret;
> +
> +                       cr_pgarr_reset_all(ctx);
> +               }
> +       }
> +
> +       /* mark end of contents with header saying "0" pages */
> +       hh = cr_hbuf_get(ctx, sizeof(*hh));
> +       hh->nr_pages = 0;
> +       ret = cr_write_obj(ctx, &h, hh);
> +       cr_hbuf_put(ctx, sizeof(*hh));
> +
> +       return ret;
> +}
> +
> +static int cr_write_vma(struct cr_ctx *ctx, struct vm_area_struct *vma)
> +{
> +       struct cr_hdr h;
> +       struct cr_hdr_vma *hh = cr_hbuf_get(ctx, sizeof(*hh));
> +       int vma_type, ret;
> +
> +       h.type = CR_HDR_VMA;
> +       h.len = sizeof(*hh);
> +       h.parent = 0;
> +
> +       hh->vm_start = vma->vm_start;
> +       hh->vm_end = vma->vm_end;
> +       hh->vm_page_prot = vma->vm_page_prot.pgprot;
> +       hh->vm_flags = vma->vm_flags;
> +       hh->vm_pgoff = vma->vm_pgoff;
> +
> +       if (vma->vm_flags & (VM_SHARED | VM_IO | VM_HUGETLB | VM_NONLINEAR)) {
> +               pr_warning("CR: unsupported VMA %#lx\n", vma->vm_flags);
> +               return -ETXTBSY;
> +       }
> +
> +       /* by default assume anon memory */
> +       vma_type = CR_VMA_ANON;
> +
> +       /*
> +        * if there is a backing file, assume private-mapped
> +        * (FIXME: check if the file is unlinked)
> +        */
> +       if (vma->vm_file)
> +               vma_type = CR_VMA_FILE;
> +
> +       hh->vma_type = vma_type;
> +
> +       ret = cr_write_obj(ctx, &h, hh);
> +       cr_hbuf_put(ctx, sizeof(*hh));
> +
> +       if (ret < 0)
> +               return ret;
> +
> +       /* save the file name, if relevant */
> +       if (vma->vm_file) {
> +               ret = cr_write_fname(ctx, &vma->vm_file->f_path, ctx->vfsroot);
> +               if (ret < 0)
> +                       return ret;
> +       }
> +
> +       ret = cr_write_private_vma_contents(ctx, vma);
> +
> +       return ret;
> +}
> +
> +int cr_write_mm(struct cr_ctx *ctx, struct task_struct *t)
> +{
> +       struct cr_hdr h;
> +       struct cr_hdr_mm *hh = cr_hbuf_get(ctx, sizeof(*hh));
> +       struct mm_struct *mm;
> +       struct vm_area_struct *vma;
> +       int objref, ret;
> +
> +       h.type = CR_HDR_MM;
> +       h.len = sizeof(*hh);
> +       h.parent = task_pid_vnr(t);
> +
> +       mm = get_task_mm(t);
> +
> +       objref = 0;     /* will be meaningful with multiple processes */
> +       hh->objref = objref;
> +
> +       down_read(&mm->mmap_sem);
> +
> +       hh->start_code = mm->start_code;
> +       hh->end_code = mm->end_code;
> +       hh->start_data = mm->start_data;
> +       hh->end_data = mm->end_data;
> +       hh->start_brk = mm->start_brk;
> +       hh->brk = mm->brk;
> +       hh->start_stack = mm->start_stack;
> +       hh->arg_start = mm->arg_start;
> +       hh->arg_end = mm->arg_end;
> +       hh->env_start = mm->env_start;
> +       hh->env_end = mm->env_end;
> +
> +       hh->map_count = mm->map_count;
> +
> +       /* FIX: need also mm->flags */
> +
> +       ret = cr_write_obj(ctx, &h, hh);
> +       cr_hbuf_put(ctx, sizeof(*hh));
> +       if (ret < 0)
> +               goto out;
> +
> +       /* write the vma's */
> +       for (vma = mm->mmap; vma; vma = vma->vm_next) {
> +               ret = cr_write_vma(ctx, vma);
> +               if (ret < 0)
> +                       goto out;
> +       }
> +
> +       ret = cr_write_mm_context(ctx, mm, objref);
> +
> + out:
> +       up_read(&mm->mmap_sem);
> +       mmput(mm);
> +       return ret;
> +}
> diff --git a/checkpoint/sys.c b/checkpoint/sys.c
> index 30863c6..c4ac157 100644
> --- a/checkpoint/sys.c
> +++ b/checkpoint/sys.c
> @@ -16,6 +16,8 @@
>  #include <linux/capability.h>
>  #include <linux/checkpoint.h>
>
> +#include "checkpoint_mem.h"
> +
>  /*
>  * helpers to write/read to/from the image file descriptor
>  *
> @@ -111,7 +113,6 @@ int cr_kread(struct cr_ctx *ctx, void *buf, int count)
>        return ret;
>  }
>
> -
>  /*
>  * helpers to manage CR contexts: allocated for each checkpoint and/or
>  * restart operation, and persists until the operation is completed.
> @@ -127,6 +128,11 @@ void cr_ctx_free(struct cr_ctx *ctx)
>
>        free_pages((unsigned long) ctx->hbuf, CR_HBUF_ORDER);
>
> +       if (ctx->vfsroot)
> +               path_put(ctx->vfsroot);
> +
> +       cr_pgarr_free(ctx);
> +
>        kfree(ctx);
>  }
>
> @@ -145,10 +151,17 @@ struct cr_ctx *cr_ctx_alloc(pid_t pid, int fd, unsigned long flags)
>        }
>
>        ctx->hbuf = (void *) __get_free_pages(GFP_KERNEL, CR_HBUF_ORDER);
> -       if (!ctx->hbuf) {
> -               cr_ctx_free(ctx);
> -               return ERR_PTR(-ENOMEM);
> -       }
> +       if (!ctx->hbuf)
> +               goto nomem;
> +
> +       /*
> +        * assume checkpointer is in container's root vfs
> +        * FIXME: this works for now, but will change with real containers
> +        */
> +       ctx->vfsroot = &current->fs->root;
> +       path_get(ctx->vfsroot);
> +
> +       INIT_LIST_HEAD(&ctx->pgarr_list);
>
>        ctx->pid = pid;
>        ctx->flags = flags;
> @@ -156,6 +169,10 @@ struct cr_ctx *cr_ctx_alloc(pid_t pid, int fd, unsigned long flags)
>        ctx->crid = atomic_inc_return(&cr_ctx_count);
>
>        return ctx;
> +
> + nomem:
> +       cr_ctx_free(ctx);
> +       return ERR_PTR(-ENOMEM);
>  }
>
>  /*
> diff --git a/include/asm-x86/checkpoint_hdr.h b/include/asm-x86/checkpoint_hdr.h
> index 44a903c..6bc61ac 100644
> --- a/include/asm-x86/checkpoint_hdr.h
> +++ b/include/asm-x86/checkpoint_hdr.h
> @@ -69,4 +69,9 @@ struct cr_hdr_cpu {
>
>  } __attribute__((aligned(8)));
>
> +struct cr_hdr_mm_context {
> +       __s16 ldt_entry_size;
> +       __s16 nldt;
> +} __attribute__((aligned(8)));
> +
>  #endif /* __ASM_X86_CKPT_HDR__H */
> diff --git a/include/linux/checkpoint.h b/include/linux/checkpoint.h
> index 5e53ae6..d74e64d 100644
> --- a/include/linux/checkpoint.h
> +++ b/include/linux/checkpoint.h
> @@ -10,6 +10,9 @@
>  *  distribution for more details.
>  */
>
> +#include <linux/path.h>
> +#include <linux/fs.h>
> +
>  #define CR_VERSION  1
>
>  struct cr_ctx {
> @@ -24,6 +27,10 @@ struct cr_ctx {
>
>        void *hbuf;             /* temporary buffer for headers */
>        int hpos;               /* position in headers buffer */
> +
> +       struct list_head pgarr_list;    /* page array to dump VMA contents */
> +
> +       struct path *vfsroot;   /* container root (FIXME) */
>  };
>
>  /* cr_ctx: flags */
> @@ -46,11 +53,16 @@ struct cr_hdr;
>
>  extern int cr_write_obj(struct cr_ctx *ctx, struct cr_hdr *h, void *buf);
>  extern int cr_write_string(struct cr_ctx *ctx, char *str, int len);
> +extern int cr_write_fname(struct cr_ctx *ctx,
> +                         struct path *path, struct path *root);
>
>  extern int cr_read_obj(struct cr_ctx *ctx, struct cr_hdr *h, void *buf, int n);
>  extern int cr_read_obj_type(struct cr_ctx *ctx, void *buf, int n, int type);
>  extern int cr_read_string(struct cr_ctx *ctx, void *str, int len);
>
> +extern int cr_write_mm(struct cr_ctx *ctx, struct task_struct *t);
> +extern int cr_read_mm(struct cr_ctx *ctx);
> +
>  extern int do_checkpoint(struct cr_ctx *ctx);
>  extern int do_restart(struct cr_ctx *ctx);
>
> diff --git a/include/linux/checkpoint_hdr.h b/include/linux/checkpoint_hdr.h
> index 03ec72e..2b110f1 100644
> --- a/include/linux/checkpoint_hdr.h
> +++ b/include/linux/checkpoint_hdr.h
> @@ -32,6 +32,7 @@ struct cr_hdr {
>  enum {
>        CR_HDR_HEAD = 1,
>        CR_HDR_STRING,
> +       CR_HDR_FNAME,
>
>        CR_HDR_TASK = 101,
>        CR_HDR_THREAD,
> @@ -39,6 +40,7 @@ enum {
>
>        CR_HDR_MM = 201,
>        CR_HDR_VMA,
> +       CR_HDR_PGARR,
>        CR_HDR_MM_CONTEXT,
>
>        CR_HDR_TAIL = 5001
> @@ -73,4 +75,34 @@ struct cr_hdr_task {
>        __s32 task_comm_len;
>  } __attribute__((aligned(8)));
>
> +struct cr_hdr_mm {
> +       __u32 objref;           /* identifier for shared objects */
> +       __u32 map_count;
> +
> +       __u64 start_code, end_code, start_data, end_data;
> +       __u64 start_brk, brk, start_stack;
> +       __u64 arg_start, arg_end, env_start, env_end;
> +} __attribute__((aligned(8)));
> +
> +/* vma subtypes */
> +enum vm_type {
> +       CR_VMA_ANON = 1,
> +       CR_VMA_FILE
> +};
> +
> +struct cr_hdr_vma {
> +       __u32 vma_type;
> +       __u32 _padding;
> +
> +       __u64 vm_start;
> +       __u64 vm_end;
> +       __u64 vm_page_prot;
> +       __u64 vm_flags;
> +       __u64 vm_pgoff;
> +} __attribute__((aligned(8)));
> +
> +struct cr_hdr_pgarr {
> +       __u64 nr_pages;         /* number of pages to saved */
> +} __attribute__((aligned(8)));
> +
>  #endif /* _CHECKPOINT_CKPT_HDR_H_ */
> --
> 1.5.4.3
>
> --
> To unsubscribe from this list: send the line "unsubscribe linux-kernel" in
> the body of a message to majordomo-u79uwXL29TY76Z2rM5mHXA@public.gmane.org
> More majordomo info at  http://vger.kernel.org/majordomo-info.html
> Please read the FAQ at  http://www.tux.org/lkml/
>



-- 
Kinds regards,
MinChan Kim

Re: [RFC v5][PATCH 0/9] Kernel based checkpoint/restart

[Cedric Le Goater]

Serge E. Hallyn wrote:
> Quoting Oren Laadan (orenl-eQaUEPhvms7ENvBUuze7eA@public.gmane.org):
>> These patches implement basic checkpoint-restart [CR]. This version (v5)
>> supports basic tasks with simple private memory, and open files (regular
>> files and directories only). The main changes include rework of memory
>> restart code and response to feedback. See original announcements below.
>>
>> Oren.
> 
> Playing around a bit, all seems to work as advertised.
> 
> Has anyone playing with this run into any oopses with the latest
> version?
> 
> Cedric, could you send out your patch (or a pointer to them) to make
> ckpt/restart on another task work, so we can pound the core code a bit
> harder with liblxc?

sure. the user tools are in CVS for the moment :

	http://lxc.cvs.sourceforge.net/lxc/

but Daniel will probably release something soon. the lxc-checkpoint and 
lxc-restart tools are a bit too optimistic on the availability of the 
checkpoint and restart syscalls, a cleanup might be needed.

the patchset is here :

	http://legoater.free.fr/patches/2.6.27/2.6.27-rc7-lxc2/

it's based on top of current mainline and includes :

	. sysfs patches for net namespace
	. freezer subsystem
	. oren's V5 checkpoint and restart patches
	. a personal hack to do external checkpoint and restart using
	  the lxc-checkpoint and lxc-restart tools
	. some fixes
	
and also

	an old mq namespace implementation (being reworked currently)

restart is still bogus because some state is not well captured at
checkpoint time. 

I'm working on it right now !

Thanks,

C.

Re: [RFC v5][PATCH 0/9] Kernel based checkpoint/restart

[Oren Laadan]

Serge E. Hallyn wrote:
> Quoting Oren Laadan (orenl-eQaUEPhvms7ENvBUuze7eA@public.gmane.org):
>> These patches implement basic checkpoint-restart [CR]. This version (v5)
>> supports basic tasks with simple private memory, and open files (regular
>> files and directories only). The main changes include rework of memory
>> restart code and response to feedback. See original announcements below.
>>
>> Oren.
> 
> At the kernel summit, it was asked whether at some point this CR work
> could be used as the basis for a new suspend to disk implementation.
> I answered that we don't plan to handle kernel threads.  In your
> experience, are there any reasons why eventually we couldn't eventually
> handle kernel threads?
> 
> -serge

As far as I can see, there isn't an inherent reason not to handle
kernel threads. However, I never looked deep into the problem.

The main issue that I can see with it is similar to what the
hibernation developers must tackle anyway - how to freeze kernel
threads when some of them may still be needed to take the system
down.

Assuming that is solved, then we're left with how to freeze the
kernel threads in a state that makes sense for a restart; for
regular tasks this is right before going back to user-land (*),
for kernel threads it may not be the best place :)

(*) however, tasks that are ptraced or in the middle of a vfork
will require special treatment - since upon freezing they cannot
be forced to that convenient position; so upon restart there must
be special code to make their behavior after restart compatible
with what they would have done originally - probably by emulation
as opposed to rebuilding their old kernel stack. For instance,
if a task was stopped due to ptrace before return from a syscall,
then upon restart it should return to that exact state.

So I'd assume that kernel threads could be treated in a similar
manner by special-casing, if necessary.

Question: I'd assume that at least for some of the kernel threads
a simple re-launch at restart will do; how many really require that
we save and restore their state ?

Oren