[PATCH 0/7] notifier error injection

[PATCH 0/7] notifier error injection

[Akinobu Mita]

This provides the ability to inject artifical errors to the following
notifier chain callbacks. It is useful to test the error handling of
notifier call chain failures.

 * CPU notifier
 * PM notifier
 * memory hotplug notifier
 * powerpc pSeries reconfig notifier

We already have cpu-notifier-error-inject.ko module providing such
functionality for the CPU notifier.  However, this patch set adds
common basic functions and consistent debugfs interface for them.

Akinobu Mita (7):
  pm: improve error code of pm_notifier_call_chain()
  debugfs: add debugfs_create_int
  fault-injection: notifier error injection
  cpu: CPU notifier error injection
  PM: PM notifier error injection
  memory: memory notifier error injection
  powerpc: pSeries reconfig notifier error injection

 Documentation/feature-removal-schedule.txt |    8 ++++
 arch/powerpc/platforms/pseries/reconfig.c  |   31 +++++++++++++++
 drivers/base/memory.c                      |   29 ++++++++++++++
 drivers/char/apm-emulation.c               |    2 +-
 drivers/s390/char/vmwatchdog.c             |    4 +-
 drivers/s390/cio/css.c                     |    8 ++--
 fs/debugfs/file.c                          |   56 +++++++++++++++++++++++++++
 include/linux/debugfs.h                    |    9 ++++
 include/linux/notifier.h                   |   25 ++++++++++++
 kernel/cpu.c                               |   31 +++++++++++++++
 kernel/notifier.c                          |   57 ++++++++++++++++++++++++++++
 kernel/power/main.c                        |   35 ++++++++++++++++-
 lib/Kconfig.debug                          |   50 ++++++++++++++++++++++++
 13 files changed, 336 insertions(+), 9 deletions(-)

-- 
1.7.4.4

[PATCH 1/7] pm: improve error code of pm_notifier_call_chain()

[Akinobu Mita]

This enables pm_notifier_call_chain() to get the actual error code
in the callback rather than always assume -EINVAL by converting all
PM notifier calls to return encapsulate error code with
notifier_from_errno().

Signed-off-by: Akinobu Mita <akinobu.mita@gmail.com>
Cc: Pavel Machek <pavel@ucw.cz>
Cc: "Rafael J. Wysocki" <rjw@sisk.pl>
Cc: linux-pm@lists.linux-foundation.org
Cc: Jiri Kosina <jkosina@suse.cz>
Cc: Martin Schwidefsky <schwidefsky@de.ibm.com>
Cc: Heiko Carstens <heiko.carstens@de.ibm.com>
Cc: linux390@de.ibm.com
Cc: linux-s390@vger.kernel.org
---
 drivers/char/apm-emulation.c   |    2 +-
 drivers/s390/char/vmwatchdog.c |    4 ++--
 drivers/s390/cio/css.c         |    8 ++++----
 kernel/power/main.c            |    5 +++--
 4 files changed, 10 insertions(+), 9 deletions(-)

diff --git a/drivers/char/apm-emulation.c b/drivers/char/apm-emulation.c
index 548708c..a7346ab 100644
--- a/drivers/char/apm-emulation.c
+++ b/drivers/char/apm-emulation.c
@@ -606,7 +606,7 @@ static int apm_suspend_notifier(struct notifier_block *nb,
 			return NOTIFY_OK;
 
 		/* interrupted by signal */
-		return NOTIFY_BAD;
+		return notifier_from_errno(err);
 
 	case PM_POST_SUSPEND:
 		/*
diff --git a/drivers/s390/char/vmwatchdog.c b/drivers/s390/char/vmwatchdog.c
index 12ef912..11312f4 100644
--- a/drivers/s390/char/vmwatchdog.c
+++ b/drivers/s390/char/vmwatchdog.c
@@ -258,13 +258,13 @@ static int vmwdt_suspend(void)
 	if (test_and_set_bit(VMWDT_OPEN, &vmwdt_is_open)) {
 		pr_err("The system cannot be suspended while the watchdog"
 			" is in use\n");
-		return NOTIFY_BAD;
+		return notifier_from_errno(-EBUSY);
 	}
 	if (test_bit(VMWDT_RUNNING, &vmwdt_is_open)) {
 		clear_bit(VMWDT_OPEN, &vmwdt_is_open);
 		pr_err("The system cannot be suspended while the watchdog"
 			" is running\n");
-		return NOTIFY_BAD;
+		return notifier_from_errno(-EBUSY);
 	}
 	return NOTIFY_DONE;
 }
diff --git a/drivers/s390/cio/css.c b/drivers/s390/cio/css.c
index c47b25f..92d7324 100644
--- a/drivers/s390/cio/css.c
+++ b/drivers/s390/cio/css.c
@@ -814,8 +814,8 @@ static int css_power_event(struct notifier_block *this, unsigned long event,
 				mutex_unlock(&css->mutex);
 				continue;
 			}
-			if (__chsc_do_secm(css, 0))
-				ret = NOTIFY_BAD;
+			ret = __chsc_do_secm(css, 0);
+			ret = notifier_from_errno(ret);
 			mutex_unlock(&css->mutex);
 		}
 		break;
@@ -831,8 +831,8 @@ static int css_power_event(struct notifier_block *this, unsigned long event,
 				mutex_unlock(&css->mutex);
 				continue;
 			}
-			if (__chsc_do_secm(css, 1))
-				ret = NOTIFY_BAD;
+			ret = __chsc_do_secm(css, 1);
+			ret = notifier_from_errno(ret);
 			mutex_unlock(&css->mutex);
 		}
 		/* search for subchannels, which appeared during hibernation */
diff --git a/kernel/power/main.c b/kernel/power/main.c
index 2981af4..6c601f8 100644
--- a/kernel/power/main.c
+++ b/kernel/power/main.c
@@ -37,8 +37,9 @@ EXPORT_SYMBOL_GPL(unregister_pm_notifier);
 
 int pm_notifier_call_chain(unsigned long val)
 {
-	return (blocking_notifier_call_chain(&pm_chain_head, val, NULL)
-			== NOTIFY_BAD) ? -EINVAL : 0;
+	int ret = blocking_notifier_call_chain(&pm_chain_head, val, NULL);
+
+	return notifier_to_errno(ret);
 }
 
 /* If set, devices may be suspended and resumed asynchronously. */
-- 
1.7.4.4

Re: [PATCH 1/7] pm: improve error code of pm_notifier_call_chain()

[Pavel Machek]

On Sun 2011-07-03 23:16:15, Akinobu Mita wrote:
> This enables pm_notifier_call_chain() to get the actual error code
> in the callback rather than always assume -EINVAL by converting all
> PM notifier calls to return encapsulate error code with
> notifier_from_errno().
> 
> Signed-off-by: Akinobu Mita <akinobu.mita@gmail.com>
> Cc: Pavel Machek <pavel@ucw.cz>

Nothing obviously wrong here.
-- 
(english) http://www.livejournal.com/~pavelmachek
(cesky, pictures) http://atrey.karlin.mff.cuni.cz/~pavel/picture/horses/blog.html

Re: [PATCH 1/7] pm: improve error code of pm_notifier_call_chain()

[Akinobu Mita]

2011/7/4 Pavel Machek <pavel@ucw.cz>:
> On Sun 2011-07-03 23:16:15, Akinobu Mita wrote:
>> This enables pm_notifier_call_chain() to get the actual error code
>> in the callback rather than always assume -EINVAL by converting all
>> PM notifier calls to return encapsulate error code with
>> notifier_from_errno().
>>
>> Signed-off-by: Akinobu Mita <akinobu.mita@gmail.com>
>> Cc: Pavel Machek <pavel@ucw.cz>
>
> Nothing obviously wrong here.

Thanks. Can I add your ACK for this patch?

Re: [PATCH 1/7] pm: improve error code of pm_notifier_call_chain()

[Rafael J. Wysocki]

On Monday, July 04, 2011, Akinobu Mita wrote:
> 2011/7/4 Pavel Machek <pavel@ucw.cz>:
> > On Sun 2011-07-03 23:16:15, Akinobu Mita wrote:
> >> This enables pm_notifier_call_chain() to get the actual error code
> >> in the callback rather than always assume -EINVAL by converting all
> >> PM notifier calls to return encapsulate error code with
> >> notifier_from_errno().
> >>
> >> Signed-off-by: Akinobu Mita <akinobu.mita@gmail.com>
> >> Cc: Pavel Machek <pavel@ucw.cz>
> >
> > Nothing obviously wrong here.
> 
> Thanks. Can I add your ACK for this patch?

Do you want me to take this patch?

Rafael

Re: [PATCH 1/7] pm: improve error code of pm_notifier_call_chain()

[Akinobu Mita]

2011/7/8 Rafael J. Wysocki <rjw@sisk.pl>:
> On Monday, July 04, 2011, Akinobu Mita wrote:
>> 2011/7/4 Pavel Machek <pavel@ucw.cz>:
>> > On Sun 2011-07-03 23:16:15, Akinobu Mita wrote:
>> >> This enables pm_notifier_call_chain() to get the actual error code
>> >> in the callback rather than always assume -EINVAL by converting all
>> >> PM notifier calls to return encapsulate error code with
>> >> notifier_from_errno().
>> >>
>> >> Signed-off-by: Akinobu Mita <akinobu.mita@gmail.com>
>> >> Cc: Pavel Machek <pavel@ucw.cz>
>> >
>> > Nothing obviously wrong here.
>>
>> Thanks. Can I add your ACK for this patch?
>
> Do you want me to take this patch?

Yes, please merge it to your tree.

Re: [PATCH 1/7] pm: improve error code of pm_notifier_call_chain()

[Rafael J. Wysocki]

On Friday, July 08, 2011, Akinobu Mita wrote:
> 2011/7/8 Rafael J. Wysocki <rjw@sisk.pl>:
> > On Monday, July 04, 2011, Akinobu Mita wrote:
> >> 2011/7/4 Pavel Machek <pavel@ucw.cz>:
> >> > On Sun 2011-07-03 23:16:15, Akinobu Mita wrote:
> >> >> This enables pm_notifier_call_chain() to get the actual error code
> >> >> in the callback rather than always assume -EINVAL by converting all
> >> >> PM notifier calls to return encapsulate error code with
> >> >> notifier_from_errno().
> >> >>
> >> >> Signed-off-by: Akinobu Mita <akinobu.mita@gmail.com>
> >> >> Cc: Pavel Machek <pavel@ucw.cz>
> >> >
> >> > Nothing obviously wrong here.
> >>
> >> Thanks. Can I add your ACK for this patch?
> >
> > Do you want me to take this patch?
> 
> Yes, please merge it to your tree.

Applied.

Thanks,
Rafael

[PATCH 2/7] debugfs: add debugfs_create_int

[Akinobu Mita]

Introduce debugfs_create_int() for creating a debugfs file that is used to
read and write an int value.

It will be used by notifier error injection and there are two potential
users of it (ceph/writeback_cngestion_kb and ocd/count on avr32)

Signed-off-by: Akinobu Mita <akinobu.mita@gmail.com>
Cc: Greg Kroah-Hartman <gregkh@suse.de>
---
 fs/debugfs/file.c       |   56 +++++++++++++++++++++++++++++++++++++++++++++++
 include/linux/debugfs.h |    9 +++++++
 2 files changed, 65 insertions(+), 0 deletions(-)

diff --git a/fs/debugfs/file.c b/fs/debugfs/file.c
index 90f7657..05cb1c2 100644
--- a/fs/debugfs/file.c
+++ b/fs/debugfs/file.c
@@ -213,6 +213,62 @@ struct dentry *debugfs_create_u32(const char *name, mode_t mode,
 }
 EXPORT_SYMBOL_GPL(debugfs_create_u32);
 
+static int debugfs_int_set(void *data, u64 val)
+{
+	*(int *)data = val;
+	return 0;
+}
+
+static int debugfs_int_get(void *data, u64 *val)
+{
+	*val = *(int *)data;
+	return 0;
+}
+
+DEFINE_SIMPLE_ATTRIBUTE(fops_int, debugfs_int_get, debugfs_int_set, "%lld\n");
+DEFINE_SIMPLE_ATTRIBUTE(fops_int_ro, debugfs_int_get, NULL, "%lld\n");
+DEFINE_SIMPLE_ATTRIBUTE(fops_int_wo, NULL, debugfs_int_set, "%lld\n");
+
+/**
+ * debugfs_create_int - create a debugfs file that is used to read and write an int value
+ * @name: a pointer to a string containing the name of the file to create.
+ * @mode: the permission that the file should have
+ * @parent: a pointer to the parent dentry for this file.  This should be a
+ *          directory dentry if set.  If this parameter is %NULL, then the
+ *          file will be created in the root of the debugfs filesystem.
+ * @value: a pointer to the variable that the file should read to and write
+ *         from.
+ *
+ * This function creates a file in debugfs with the given name that
+ * contains the value of the variable @value.  If the @mode variable is so
+ * set, it can be read from, and written to.
+ *
+ * This function will return a pointer to a dentry if it succeeds.  This
+ * pointer must be passed to the debugfs_remove() function when the file is
+ * to be removed (no automatic cleanup happens if your module is unloaded,
+ * you are responsible here.)  If an error occurs, %NULL will be returned.
+ *
+ * If debugfs is not enabled in the kernel, the value -%ENODEV will be
+ * returned.  It is not wise to check for this value, but rather, check for
+ * %NULL or !%NULL instead as to eliminate the need for #ifdef in the calling
+ * code.
+ */
+struct dentry *debugfs_create_int(const char *name, mode_t mode,
+				struct dentry *parent, int *value)
+{
+	const struct file_operations *fops = &fops_int;
+
+	/* if there are no write bits set, make read only */
+	if (!(mode & S_IWUGO))
+		fops = &fops_int_ro;
+	/* if there are no read bits set, make write only */
+	if (!(mode & S_IRUGO))
+		fops = &fops_u32_wo;
+
+	return debugfs_create_file(name, mode, parent, value, fops);
+}
+EXPORT_SYMBOL_GPL(debugfs_create_int);
+
 static int debugfs_u64_set(void *data, u64 val)
 {
 	*(u64 *)data = val;
diff --git a/include/linux/debugfs.h b/include/linux/debugfs.h
index e7d9b20..6300792 100644
--- a/include/linux/debugfs.h
+++ b/include/linux/debugfs.h
@@ -55,6 +55,8 @@ struct dentry *debugfs_create_u16(const char *name, mode_t mode,
 				  struct dentry *parent, u16 *value);
 struct dentry *debugfs_create_u32(const char *name, mode_t mode,
 				  struct dentry *parent, u32 *value);
+struct dentry *debugfs_create_int(const char *name, mode_t mode,
+				  struct dentry *parent, int *value);
 struct dentry *debugfs_create_u64(const char *name, mode_t mode,
 				  struct dentry *parent, u64 *value);
 struct dentry *debugfs_create_x8(const char *name, mode_t mode,
@@ -139,6 +141,13 @@ static inline struct dentry *debugfs_create_u32(const char *name, mode_t mode,
 	return ERR_PTR(-ENODEV);
 }
 
+static inline struct dentry *debugfs_create_int(const char *name, mode_t mode,
+						struct dentry *parent,
+						int *value);
+{
+	return ERR_PTR(-ENODEV);
+}
+
 static inline struct dentry *debugfs_create_u64(const char *name, mode_t mode,
 						struct dentry *parent,
 						u64 *value)
-- 
1.7.4.4

Re: [PATCH 2/7] debugfs: add debugfs_create_int

[Greg KH]

On Sun, Jul 03, 2011 at 11:16:16PM +0900, Akinobu Mita wrote:
> Introduce debugfs_create_int() for creating a debugfs file that is used to
> read and write an int value.

Um, what's wrong with the existing s32 and s64 versions that debugfs
already provides?

userspace doesn't know what "int" means for the kernel as it might be
running in 32bit mode with a 64bit kernel.  That is why this isn't a
good idea, so please convert your other patch to use the existing api
and change the variable type in your patch.

greg k-h

Re: [PATCH 2/7] debugfs: add debugfs_create_int

[Akinobu Mita]

2011/7/4 Greg KH <gregkh@suse.de>:
> On Sun, Jul 03, 2011 at 11:16:16PM +0900, Akinobu Mita wrote:
>> Introduce debugfs_create_int() for creating a debugfs file that is used to
>> read and write an int value.
>
> Um, what's wrong with the existing s32 and s64 versions that debugfs
> already provides?

I couldn't find s32 version in linux/debugfs.h

> userspace doesn't know what "int" means for the kernel as it might be
> running in 32bit mode with a 64bit kernel.  That is why this isn't a
> good idea, so please convert your other patch to use the existing api
> and change the variable type in your patch.

Shoud I add debugfs_create_s32() instead of debugfs_create_int() for
this?

Re: [PATCH 2/7] debugfs: add debugfs_create_int

[Greg KH]

On Mon, Jul 04, 2011 at 02:31:25PM +0900, Akinobu Mita wrote:
> 2011/7/4 Greg KH <gregkh@suse.de>:
> > On Sun, Jul 03, 2011 at 11:16:16PM +0900, Akinobu Mita wrote:
> >> Introduce debugfs_create_int() for creating a debugfs file that is used to
> >> read and write an int value.
> >
> > Um, what's wrong with the existing s32 and s64 versions that debugfs
> > already provides?
> 
> I couldn't find s32 version in linux/debugfs.h
> 
> > userspace doesn't know what "int" means for the kernel as it might be
> > running in 32bit mode with a 64bit kernel.  That is why this isn't a
> > good idea, so please convert your other patch to use the existing api
> > and change the variable type in your patch.
> 
> Shoud I add debugfs_create_s32() instead of debugfs_create_int() for
> this?

Does it have to be a signed value?  If not, just use the
debugfs_create_u32() function.  Or use that and just cast the result,
right?

thanks,

greg k-h

Re: [PATCH 2/7] debugfs: add debugfs_create_int

[Akinobu Mita]

2011/7/5 Greg KH <gregkh@suse.de>:
> On Mon, Jul 04, 2011 at 02:31:25PM +0900, Akinobu Mita wrote:
>> 2011/7/4 Greg KH <gregkh@suse.de>:
>> > On Sun, Jul 03, 2011 at 11:16:16PM +0900, Akinobu Mita wrote:
>> >> Introduce debugfs_create_int() for creating a debugfs file that is used to
>> >> read and write an int value.
>> >
>> > Um, what's wrong with the existing s32 and s64 versions that debugfs
>> > already provides?
>>
>> I couldn't find s32 version in linux/debugfs.h
>>
>> > userspace doesn't know what "int" means for the kernel as it might be
>> > running in 32bit mode with a 64bit kernel.  That is why this isn't a
>> > good idea, so please convert your other patch to use the existing api
>> > and change the variable type in your patch.
>>
>> Shoud I add debugfs_create_s32() instead of debugfs_create_int() for
>> this?
>
> Does it have to be a signed value?  If not, just use the
> debugfs_create_u32() function.  Or use that and just cast the result,
> right?

The debugfs file that I want to use needs to contain an error code
like -ENOMEM.

Now I realized that I should create a custom simple attribute that
is only allowed to store a valid errno (-MAX_ERRNO <= errno <= 0).

So I don't add new function like debugfs_create_s32 for now.

[PATCH 3/7] fault-injection: notifier error injection

[Akinobu Mita]

The notifier error injection provides the ability to inject artifical
errors to specified notifier chain callbacks. It is useful to test
the error handling of notifier call chain failures.

This adds common basic functions to define which type of events can be
fail and to initialize the debugfs interface to control what error code
should be returned and which event should be failed.

Signed-off-by: Akinobu Mita <akinobu.mita@gmail.com>
Cc: Pavel Machek <pavel@ucw.cz>
Cc: "Rafael J. Wysocki" <rjw@sisk.pl>
Cc: Greg Kroah-Hartman <gregkh@suse.de>
Cc: Benjamin Herrenschmidt <benh@kernel.crashing.org>
Cc: Paul Mackerras <paulus@samba.org>
Cc: linux-pm@lists.linux-foundation.org
Cc: linux-mm@kvack.org
Cc: linuxppc-dev@lists.ozlabs.org
---
 include/linux/notifier.h |   25 ++++++++++++++++++++
 kernel/notifier.c        |   57 ++++++++++++++++++++++++++++++++++++++++++++++
 lib/Kconfig.debug        |   11 +++++++++
 3 files changed, 93 insertions(+), 0 deletions(-)

diff --git a/include/linux/notifier.h b/include/linux/notifier.h
index c0688b0..51882d6 100644
--- a/include/linux/notifier.h
+++ b/include/linux/notifier.h
@@ -278,5 +278,30 @@ extern struct blocking_notifier_head reboot_notifier_list;
 #define VT_UPDATE		0x0004 /* A bigger update occurred */
 #define VT_PREWRITE		0x0005 /* A char is about to be written to the console */
 
+#ifdef CONFIG_NOTIFIER_ERROR_INJECTION
+
+struct err_inject_notifier_action {
+	unsigned long val;
+	int error;
+	const char *name;
+};
+
+#define ERR_INJECT_NOTIFIER_ACTION(action)	\
+	.name = #action, .val = (action),
+
+struct err_inject_notifier_block {
+	struct notifier_block nb;
+	struct dentry *dir;
+	struct err_inject_notifier_action actions[];
+	/* The last slot must be terminated with zero sentinel */
+};
+
+extern int err_inject_notifier_block_init(struct err_inject_notifier_block *enb,
+				const char *name, int priority);
+extern void err_inject_notifier_block_cleanup(
+				struct err_inject_notifier_block *enb);
+
+#endif /* CONFIG_NOTIFIER_ERROR_INJECTION */
+
 #endif /* __KERNEL__ */
 #endif /* _LINUX_NOTIFIER_H */
diff --git a/kernel/notifier.c b/kernel/notifier.c
index 2488ba7..8dcc485 100644
--- a/kernel/notifier.c
+++ b/kernel/notifier.c
@@ -5,6 +5,7 @@
 #include <linux/rcupdate.h>
 #include <linux/vmalloc.h>
 #include <linux/reboot.h>
+#include <linux/debugfs.h>
 
 /*
  *	Notifier list for kernel code which wants to be called
@@ -584,3 +585,59 @@ int unregister_die_notifier(struct notifier_block *nb)
 	return atomic_notifier_chain_unregister(&die_chain, nb);
 }
 EXPORT_SYMBOL_GPL(unregister_die_notifier);
+
+#ifdef CONFIG_NOTIFIER_ERROR_INJECTION
+
+static int err_inject_notifier_callback(struct notifier_block *nb,
+				unsigned long val, void *p)
+{
+	int err = 0;
+	struct err_inject_notifier_block *enb =
+		container_of(nb, struct err_inject_notifier_block, nb);
+	struct err_inject_notifier_action *action;
+
+	for (action = enb->actions; action->name; action++) {
+		if (action->val == val) {
+			err = action->error;
+			break;
+		}
+	}
+	if (err) {
+		printk(KERN_INFO "Injecting error (%d) to %s\n",
+			err, action->name);
+	}
+
+	return notifier_from_errno(err);
+}
+
+int err_inject_notifier_block_init(struct err_inject_notifier_block *enb,
+				const char *name, int priority)
+{
+	struct err_inject_notifier_action *action;
+	mode_t mode = S_IFREG | S_IRUSR | S_IWUSR;
+
+	enb->nb.notifier_call = err_inject_notifier_callback;
+	enb->nb.priority = priority;
+
+	enb->dir = debugfs_create_dir(name, NULL);
+	if (!enb->dir)
+		return -ENOMEM;
+
+	for (action = enb->actions; action->name; action++) {
+		struct dentry *file = debugfs_create_int(action->name, mode,
+						enb->dir, &action->error);
+
+		if (!file) {
+			debugfs_remove_recursive(enb->dir);
+			return -ENOMEM;
+		}
+	}
+	return 0;
+}
+
+void err_inject_notifier_block_cleanup(struct err_inject_notifier_block *enb)
+{
+	debugfs_remove_recursive(enb->dir);
+}
+
+#endif /* CONFIG_NOTIFIER_ERROR_INJECTION */
diff --git a/lib/Kconfig.debug b/lib/Kconfig.debug
index dd373c8..8c6ce7e 100644
--- a/lib/Kconfig.debug
+++ b/lib/Kconfig.debug
@@ -1018,6 +1018,17 @@ config LKDTM
 	Documentation on how to use the module can be found in
 	Documentation/fault-injection/provoke-crashes.txt
 
+config NOTIFIER_ERROR_INJECTION
+	bool "Notifier error injection"
+	depends on DEBUG_KERNEL
+	select DEBUG_FS
+	help
+	  This option provides the ability to inject artifical errors to
+	  specified notifier chain callbacks. It is useful to test the error
+	  handling of notifier call chain failures.
+
+	  Say N if unsure.
+
 config CPU_NOTIFIER_ERROR_INJECT
 	tristate "CPU notifier error injection module"
 	depends on HOTPLUG_CPU && DEBUG_KERNEL
-- 
1.7.4.4

Re: [PATCH 3/7] fault-injection: notifier error injection

[Pavel Machek]

> +	for (action = enb->actions; action->name; action++) {
> +		struct dentry *file = debugfs_create_int(action->name, mode,
> +						enb->dir, &action->error);
> +
> +		if (!file) {
> +			debugfs_remove_recursive(enb->dir);
> +			return -ENOMEM;
> +		}

Few lines how this work would be welcome...?
-- 
(english) http://www.livejournal.com/~pavelmachek
(cesky, pictures) http://atrey.karlin.mff.cuni.cz/~pavel/picture/horses/blog.html

Re: [PATCH 3/7] fault-injection: notifier error injection

[Akinobu Mita]

2011/7/4 Pavel Machek <pavel@ucw.cz>:
>
>> +     for (action = enb->actions; action->name; action++) {
>> +             struct dentry *file = debugfs_create_int(action->name, mode,
>> +                                             enb->dir, &action->error);
>> +
>> +             if (!file) {
>> +                     debugfs_remove_recursive(enb->dir);
>> +                     return -ENOMEM;
>> +             }
>
> Few lines how this work would be welcome...?

OK, I'll add a comment like below.

/*
 * Create debugfs r/w file containing action->error. If notifier call
 * chain is called with action->val, it will fail with the error code
 */

[PATCH 4/7] cpu: CPU notifier error injection

[Akinobu Mita]

This provides the ability to inject artifical errors to CPU notifier
chain callbacks.  It is controlled through debugfs interface under
/sys/kernel/debug/cpu-notifier-error-inject/

Each of the files in the directory represents an event which can be
failed and contains the error code.  If the notifier call chain should
be failed with some events notified, write the error code to the files.

Example1: inject CPU offline error (-1 == -EPERM)

	# echo -1 > /sys/kernel/debug/cpu-notifier-error-inject/CPU_UP_PREPARE
	# echo 0 > /sys/devices/system/cpu/cpu1/online
	bash: echo: write error: Operation not permitted

Example2: inject CPU online error (-2 == -ENOENT)

	# echo -2 > /sys/kernel/debug/cpu-notifier-error-inject/CPU_DOWN_PREPARE
	# echo 1 > /sys/devices/system/cpu/cpu1/online
	bash: echo: write error: No such file or directory

Signed-off-by: Akinobu Mita <akinobu.mita@gmail.com>
---
 Documentation/feature-removal-schedule.txt |    8 +++++++
 kernel/cpu.c                               |   31 ++++++++++++++++++++++++++++
 lib/Kconfig.debug                          |   14 ++++++++++++
 3 files changed, 53 insertions(+), 0 deletions(-)

diff --git a/Documentation/feature-removal-schedule.txt b/Documentation/feature-removal-schedule.txt
index 72e2384..fe5d2b5 100644
--- a/Documentation/feature-removal-schedule.txt
+++ b/Documentation/feature-removal-schedule.txt
@@ -583,3 +583,11 @@ Why:	Superseded by the UVCIOC_CTRL_QUERY ioctl.
 Who:	Laurent Pinchart <laurent.pinchart@ideasonboard.com>
 
 ----------------------------
+
+What:	Support for CPU_NOTIFIER_ERROR_INJECT (cpu-notifier-error-inejct.ko)
+When:	3.3
+Why:	Replaced with debugfs interface in
+	/sys/kernel/debug/cpu-notifier-error-inject/
+Who:	Akinobu Mita <akinobu.mita@gmail.com>
+
+----------------------------
diff --git a/kernel/cpu.c b/kernel/cpu.c
index 12b7458..c326f4d 100644
--- a/kernel/cpu.c
+++ b/kernel/cpu.c
@@ -170,6 +170,37 @@ void __ref unregister_cpu_notifier(struct notifier_block *nb)
 }
 EXPORT_SYMBOL(unregister_cpu_notifier);
 
+#ifdef CONFIG_CPU_NOTIFIER_ERROR_INJECTION
+
+static struct err_inject_notifier_block err_inject_cpu_notifier = {
+	.actions = {
+		{ ERR_INJECT_NOTIFIER_ACTION(CPU_UP_PREPARE) },
+		{ ERR_INJECT_NOTIFIER_ACTION(CPU_UP_PREPARE_FROZEN) },
+		{ ERR_INJECT_NOTIFIER_ACTION(CPU_DOWN_PREPARE) },
+		{ ERR_INJECT_NOTIFIER_ACTION(CPU_DOWN_PREPARE_FROZEN) },
+		{}
+	}
+};
+
+static int __init err_inject_cpu_notifier_init(void)
+{
+	int err;
+
+	err = err_inject_notifier_block_init(&err_inject_cpu_notifier,
+				"cpu-notifier-error-inject", -1);
+	if (err)
+		return err;
+
+	err = register_hotcpu_notifier(&err_inject_cpu_notifier.nb);
+	if (err)
+		err_inject_notifier_block_cleanup(&err_inject_cpu_notifier);
+
+	return err;
+}
+late_initcall(err_inject_cpu_notifier_init);
+
+#endif /* CONFIG_CPU_NOTIFIER_ERROR_INJECTION */
+
 static inline void check_for_tasks(int cpu)
 {
 	struct task_struct *p;
diff --git a/lib/Kconfig.debug b/lib/Kconfig.debug
index 8c6ce7e..d944b32 100644
--- a/lib/Kconfig.debug
+++ b/lib/Kconfig.debug
@@ -1029,6 +1029,20 @@ config NOTIFIER_ERROR_INJECTION
 
 	  Say N if unsure.
 
+config CPU_NOTIFIER_ERROR_INJECTION
+	bool "CPU notifier error injection"
+	depends on HOTPLUG_CPU && NOTIFIER_ERROR_INJECTION
+	help
+	  This option provides the ability to inject artifical errors to
+	  CPU notifier chain callbacks.  It is controlled through debugfs
+	  interface under /sys/kernel/debug/cpu-notifier-error-inject/
+
+	  Example1: inject CPU offline error (-1 == -EPERM)
+
+	  # echo -1 > /sys/kernel/debug/cpu-notifier-error-inject/CPU_UP_PREPARE
+	  # echo 0 > /sys/devices/system/cpu/cpu1/online
+	  bash: echo: write error: Operation not permitted
+
 config CPU_NOTIFIER_ERROR_INJECT
 	tristate "CPU notifier error injection module"
 	depends on HOTPLUG_CPU && DEBUG_KERNEL
-- 
1.7.4.4

[PATCH 5/7] PM: PM notifier error injection

[Akinobu Mita]

This provides the ability to inject artifical errors to PM notifier
chain callbacks.  It is controlled through debugfs interface under
/sys/kernel/debug/pm-notifier-error-inject/

Each of the files in the directory represents an event which can be
failed and contains the error code.  If the notifier call chain should
be failed with some events notified, write the error code to the files.

Signed-off-by: Akinobu Mita <akinobu.mita@gmail.com>
Cc: Pavel Machek <pavel@ucw.cz>
Cc: "Rafael J. Wysocki" <rjw@sisk.pl>
Cc: linux-pm@lists.linux-foundation.org
---
 kernel/power/main.c |   30 ++++++++++++++++++++++++++++++
 lib/Kconfig.debug   |    8 ++++++++
 2 files changed, 38 insertions(+), 0 deletions(-)

diff --git a/kernel/power/main.c b/kernel/power/main.c
index 6c601f8..04b3774 100644
--- a/kernel/power/main.c
+++ b/kernel/power/main.c
@@ -42,6 +42,36 @@ int pm_notifier_call_chain(unsigned long val)
 	return notifier_to_errno(ret);
 }
 
+#ifdef CONFIG_PM_NOTIFIER_ERROR_INJECTION
+
+static struct err_inject_notifier_block err_inject_pm_notifier = {
+	.actions = {
+		{ ERR_INJECT_NOTIFIER_ACTION(PM_HIBERNATION_PREPARE) },
+		{ ERR_INJECT_NOTIFIER_ACTION(PM_SUSPEND_PREPARE) },
+		{ ERR_INJECT_NOTIFIER_ACTION(PM_RESTORE_PREPARE) },
+		{}
+	}
+};
+
+static int __init err_inject_pm_notifier_init(void)
+{
+	int err;
+
+	err = err_inject_notifier_block_init(&err_inject_pm_notifier,
+				"pm-notifier-error-inject", -1);
+	if (err)
+		return err;
+
+	err = register_pm_notifier(&err_inject_pm_notifier.nb);
+	if (err)
+		err_inject_notifier_block_cleanup(&err_inject_pm_notifier);
+
+	return err;
+}
+late_initcall(err_inject_pm_notifier_init);
+
+#endif /* CONFIG_PM_NOTIFIER_ERROR_INJECTION */
+
 /* If set, devices may be suspended and resumed asynchronously. */
 int pm_async_enabled = 1;
 
diff --git a/lib/Kconfig.debug b/lib/Kconfig.debug
index d944b32..3ffb38b 100644
--- a/lib/Kconfig.debug
+++ b/lib/Kconfig.debug
@@ -1043,6 +1043,14 @@ config CPU_NOTIFIER_ERROR_INJECTION
 	  # echo 0 > /sys/devices/system/cpu/cpu1/online
 	  bash: echo: write error: Operation not permitted
 
+config PM_NOTIFIER_ERROR_INJECTION
+	bool "PM notifier error injection"
+	depends on PM_DEBUG && NOTIFIER_ERROR_INJECTION
+	help
+	  This option provides the ability to inject artifical errors to
+	  PM notifier chain callbacks.  It is controlled through debugfs
+	  interface under /sys/kernel/debug/pm-notifier-error-inject/
+
 config CPU_NOTIFIER_ERROR_INJECT
 	tristate "CPU notifier error injection module"
 	depends on HOTPLUG_CPU && DEBUG_KERNEL
-- 
1.7.4.4

Re: [PATCH 5/7] PM: PM notifier error injection

[Rafael J. Wysocki]

On Sunday, July 03, 2011, Akinobu Mita wrote:
> This provides the ability to inject artifical errors to PM notifier
> chain callbacks.  It is controlled through debugfs interface under
> /sys/kernel/debug/pm-notifier-error-inject/
> 
> Each of the files in the directory represents an event which can be
> failed and contains the error code.  If the notifier call chain should
> be failed with some events notified, write the error code to the files.
> 
> Signed-off-by: Akinobu Mita <akinobu.mita@gmail.com>
> Cc: Pavel Machek <pavel@ucw.cz>
> Cc: "Rafael J. Wysocki" <rjw@sisk.pl>
> Cc: linux-pm@lists.linux-foundation.org
> ---
>  kernel/power/main.c |   30 ++++++++++++++++++++++++++++++
>  lib/Kconfig.debug   |    8 ++++++++
>  2 files changed, 38 insertions(+), 0 deletions(-)
> 
> diff --git a/kernel/power/main.c b/kernel/power/main.c
> index 6c601f8..04b3774 100644
> --- a/kernel/power/main.c
> +++ b/kernel/power/main.c
> @@ -42,6 +42,36 @@ int pm_notifier_call_chain(unsigned long val)
>  	return notifier_to_errno(ret);
>  }
>  
> +#ifdef CONFIG_PM_NOTIFIER_ERROR_INJECTION
> +
> +static struct err_inject_notifier_block err_inject_pm_notifier = {
> +	.actions = {
> +		{ ERR_INJECT_NOTIFIER_ACTION(PM_HIBERNATION_PREPARE) },
> +		{ ERR_INJECT_NOTIFIER_ACTION(PM_SUSPEND_PREPARE) },
> +		{ ERR_INJECT_NOTIFIER_ACTION(PM_RESTORE_PREPARE) },
> +		{}

Why have you omitted the PM_POST_* actions?

> +	}
> +};
> +
> +static int __init err_inject_pm_notifier_init(void)
> +{
> +	int err;
> +
> +	err = err_inject_notifier_block_init(&err_inject_pm_notifier,
> +				"pm-notifier-error-inject", -1);
> +	if (err)
> +		return err;
> +
> +	err = register_pm_notifier(&err_inject_pm_notifier.nb);
> +	if (err)
> +		err_inject_notifier_block_cleanup(&err_inject_pm_notifier);
> +
> +	return err;
> +}
> +late_initcall(err_inject_pm_notifier_init);
> +
> +#endif /* CONFIG_PM_NOTIFIER_ERROR_INJECTION */
> +
>  /* If set, devices may be suspended and resumed asynchronously. */
>  int pm_async_enabled = 1;
>  
> diff --git a/lib/Kconfig.debug b/lib/Kconfig.debug
> index d944b32..3ffb38b 100644
> --- a/lib/Kconfig.debug
> +++ b/lib/Kconfig.debug
> @@ -1043,6 +1043,14 @@ config CPU_NOTIFIER_ERROR_INJECTION
>  	  # echo 0 > /sys/devices/system/cpu/cpu1/online
>  	  bash: echo: write error: Operation not permitted
>  
> +config PM_NOTIFIER_ERROR_INJECTION
> +	bool "PM notifier error injection"
> +	depends on PM_DEBUG && NOTIFIER_ERROR_INJECTION
> +	help
> +	  This option provides the ability to inject artifical errors to
> +	  PM notifier chain callbacks.  It is controlled through debugfs
> +	  interface under /sys/kernel/debug/pm-notifier-error-inject/

I'm not sure the help is necessary.  I think it should be selected
automatically if both PM_DEBUG and NOTIFIER_ERROR_INJECTION are set.

> +
>  config CPU_NOTIFIER_ERROR_INJECT
>  	tristate "CPU notifier error injection module"
>  	depends on HOTPLUG_CPU && DEBUG_KERNEL

Thanks,
Rafael

Re: [PATCH 5/7] PM: PM notifier error injection

[Akinobu Mita]

>> --- a/kernel/power/main.c
>> +++ b/kernel/power/main.c
>> @@ -42,6 +42,36 @@ int pm_notifier_call_chain(unsigned long val)
>>       return notifier_to_errno(ret);
>>  }
>>
>> +#ifdef CONFIG_PM_NOTIFIER_ERROR_INJECTION
>> +
>> +static struct err_inject_notifier_block err_inject_pm_notifier = {
>> +     .actions = {
>> +             { ERR_INJECT_NOTIFIER_ACTION(PM_HIBERNATION_PREPARE) },
>> +             { ERR_INJECT_NOTIFIER_ACTION(PM_SUSPEND_PREPARE) },
>> +             { ERR_INJECT_NOTIFIER_ACTION(PM_RESTORE_PREPARE) },
>> +             {}
>
> Why have you omitted the PM_POST_* actions?

Because the callbacks of PM_POST_* are not supposed to fail (i.e.
the return value of pm_notifier_call() is ignored).

>> --- a/lib/Kconfig.debug
>> +++ b/lib/Kconfig.debug
>> @@ -1043,6 +1043,14 @@ config CPU_NOTIFIER_ERROR_INJECTION
>>         # echo 0 > /sys/devices/system/cpu/cpu1/online
>>         bash: echo: write error: Operation not permitted
>>
>> +config PM_NOTIFIER_ERROR_INJECTION
>> +     bool "PM notifier error injection"
>> +     depends on PM_DEBUG && NOTIFIER_ERROR_INJECTION
>> +     help
>> +       This option provides the ability to inject artifical errors to
>> +       PM notifier chain callbacks.  It is controlled through debugfs
>> +       interface under /sys/kernel/debug/pm-notifier-error-inject/
>
> I'm not sure the help is necessary.  I think it should be selected
> automatically if both PM_DEBUG and NOTIFIER_ERROR_INJECTION are set.

OK, I'll try to tweak the option to be better.

Re: [PATCH 5/7] PM: PM notifier error injection

[Rafael J. Wysocki]

On Friday, July 08, 2011, Akinobu Mita wrote:
> >> --- a/kernel/power/main.c
> >> +++ b/kernel/power/main.c
> >> @@ -42,6 +42,36 @@ int pm_notifier_call_chain(unsigned long val)
> >>       return notifier_to_errno(ret);
> >>  }
> >>
> >> +#ifdef CONFIG_PM_NOTIFIER_ERROR_INJECTION
> >> +
> >> +static struct err_inject_notifier_block err_inject_pm_notifier = {
> >> +     .actions = {
> >> +             { ERR_INJECT_NOTIFIER_ACTION(PM_HIBERNATION_PREPARE) },
> >> +             { ERR_INJECT_NOTIFIER_ACTION(PM_SUSPEND_PREPARE) },
> >> +             { ERR_INJECT_NOTIFIER_ACTION(PM_RESTORE_PREPARE) },
> >> +             {}
> >
> > Why have you omitted the PM_POST_* actions?
> 
> Because the callbacks of PM_POST_* are not supposed to fail (i.e.
> the return value of pm_notifier_call() is ignored).

OK, fair enough.

> >> --- a/lib/Kconfig.debug
> >> +++ b/lib/Kconfig.debug
> >> @@ -1043,6 +1043,14 @@ config CPU_NOTIFIER_ERROR_INJECTION
> >>         # echo 0 > /sys/devices/system/cpu/cpu1/online
> >>         bash: echo: write error: Operation not permitted
> >>
> >> +config PM_NOTIFIER_ERROR_INJECTION
> >> +     bool "PM notifier error injection"
> >> +     depends on PM_DEBUG && NOTIFIER_ERROR_INJECTION
> >> +     help
> >> +       This option provides the ability to inject artifical errors to
> >> +       PM notifier chain callbacks.  It is controlled through debugfs
> >> +       interface under /sys/kernel/debug/pm-notifier-error-inject/
> >
> > I'm not sure the help is necessary.  I think it should be selected
> > automatically if both PM_DEBUG and NOTIFIER_ERROR_INJECTION are set.
> 
> OK, I'll try to tweak the option to be better.

Thanks!

Rafael

[PATCH 6/7] memory: memory notifier error injection

[Akinobu Mita]

This provides the ability to inject artifical errors to memory hotplug
notifier chain callbacks.  It is controlled through debugfs interface
under /sys/kernel/debug/memory-notifier-error-inject/

Each of the files in the directory represents an event which can be
failed and contains the error code.  If the notifier call chain should
be failed with some events notified, write the error code to the files.

Signed-off-by: Akinobu Mita <akinobu.mita@gmail.com>
Cc: Greg Kroah-Hartman <gregkh@suse.de>
Cc: linux-mm@kvack.org
---
 drivers/base/memory.c |   29 +++++++++++++++++++++++++++++
 lib/Kconfig.debug     |    8 ++++++++
 2 files changed, 37 insertions(+), 0 deletions(-)

diff --git a/drivers/base/memory.c b/drivers/base/memory.c
index 9f9b235..5b7430f 100644
--- a/drivers/base/memory.c
+++ b/drivers/base/memory.c
@@ -89,6 +89,35 @@ void unregister_memory_isolate_notifier(struct notifier_block *nb)
 }
 EXPORT_SYMBOL(unregister_memory_isolate_notifier);
 
+#ifdef CONFIG_MEMORY_NOTIFIER_ERROR_INJECTION
+
+static struct err_inject_notifier_block err_inject_memory_notifier = {
+	.actions = {
+		{ ERR_INJECT_NOTIFIER_ACTION(MEM_GOING_ONLINE) },
+		{ ERR_INJECT_NOTIFIER_ACTION(MEM_GOING_OFFLINE) },
+		{}
+	}
+};
+
+static int __init err_inject_memory_notifier_init(void)
+{
+	int err;
+
+	err = err_inject_notifier_block_init(&err_inject_memory_notifier,
+				"memory-notifier-error-inject", -1);
+	if (err)
+		return err;
+
+	err = register_memory_notifier(&err_inject_memory_notifier.nb);
+	if (err)
+		err_inject_notifier_block_cleanup(&err_inject_memory_notifier);
+
+	return err;
+}
+late_initcall(err_inject_memory_notifier_init);
+
+#endif /* CONFIG_MEMORY_NOTIFIER_ERROR_INJECTION */
+
 /*
  * register_memory - Setup a sysfs device for a memory block
  */
diff --git a/lib/Kconfig.debug b/lib/Kconfig.debug
index 3ffb38b..52f0b0e 100644
--- a/lib/Kconfig.debug
+++ b/lib/Kconfig.debug
@@ -1051,6 +1051,14 @@ config PM_NOTIFIER_ERROR_INJECTION
 	  PM notifier chain callbacks.  It is controlled through debugfs
 	  interface under /sys/kernel/debug/pm-notifier-error-inject/
 
+config MEMORY_NOTIFIER_ERROR_INJECTION
+	bool "Memory hotplug notifier error injection"
+	depends on MEMORY_HOTPLUG_SPARSE && NOTIFIER_ERROR_INJECTION
+	help
+	  This option provides the ability to inject artifical errors to
+	  memory hotplug notifier chain callbacks.  It is controlled through
+	  debugfs interface under /sys/kernel/debug/memory-notifier-error-inject/
+
 config CPU_NOTIFIER_ERROR_INJECT
 	tristate "CPU notifier error injection module"
 	depends on HOTPLUG_CPU && DEBUG_KERNEL
-- 
1.7.4.4

[PATCH 7/7] powerpc: pSeries reconfig notifier error injection

[Akinobu Mita]

This provides the ability to inject artifical errors to pSeries reconfig
notifier chain callbacks.  It is controlled through debugfs interface
under /sys/kernel/debug/pSeries-reconfig-notifier-error-inject/

Each of the files in the directory represents an event which can be
failed and contains the error code.  If the notifier call chain should
be failed with some events notified, write the error code to the files.

Signed-off-by: Akinobu Mita <akinobu.mita@gmail.com>
Cc: Benjamin Herrenschmidt <benh@kernel.crashing.org>
Cc: Paul Mackerras <paulus@samba.org>
Cc: linuxppc-dev@lists.ozlabs.org
---
 arch/powerpc/platforms/pseries/reconfig.c |   31 +++++++++++++++++++++++++++++
 lib/Kconfig.debug                         |    9 ++++++++
 2 files changed, 40 insertions(+), 0 deletions(-)

diff --git a/arch/powerpc/platforms/pseries/reconfig.c b/arch/powerpc/platforms/pseries/reconfig.c
index 168651a..31d9b0f 100644
--- a/arch/powerpc/platforms/pseries/reconfig.c
+++ b/arch/powerpc/platforms/pseries/reconfig.c
@@ -117,6 +117,37 @@ int pSeries_reconfig_notify(unsigned long action, void *p)
 	return notifier_to_errno(err);
 }
 
+#ifdef CONFIG_PSERIES_RECONFIG_NOTIFIER_ERROR_INJECTION
+
+static struct err_inject_notifier_block err_inject_reconfig_nb = {
+	.actions = {
+		{ ERR_INJECT_NOTIFIER_ACTION(PSERIES_RECONFIG_ADD) },
+		{ ERR_INJECT_NOTIFIER_ACTION(PSERIES_RECONFIG_REMOVE) },
+		{ ERR_INJECT_NOTIFIER_ACTION(PSERIES_DRCONF_MEM_ADD) },
+		{ ERR_INJECT_NOTIFIER_ACTION(PSERIES_DRCONF_MEM_REMOVE) },
+		{}
+	}
+};
+
+static int __init err_inject_reconfig_notifier_init(void)
+{
+	int err;
+
+	err = err_inject_notifier_block_init(&err_inject_reconfig_nb,
+			"pSeries-reconfig-notifier-error-inject", -1);
+	if (err)
+		return err;
+
+	err = pSeries_reconfig_notifier_register(&err_inject_reconfig_nb.nb);
+	if (err)
+		err_inject_notifier_block_cleanup(&err_inject_reconfig_nb);
+
+	return err;
+}
+late_initcall(err_inject_reconfig_notifier_init);
+
+#endif /* CONFIG_PSERIES_RECONFIG_NOTIFIER_ERROR_INJECTION */
+
 static int pSeries_reconfig_add_node(const char *path, struct property *proplist)
 {
 	struct device_node *np;
diff --git a/lib/Kconfig.debug b/lib/Kconfig.debug
index 52f0b0e..2becf8c 100644
--- a/lib/Kconfig.debug
+++ b/lib/Kconfig.debug
@@ -1059,6 +1059,15 @@ config MEMORY_NOTIFIER_ERROR_INJECTION
 	  memory hotplug notifier chain callbacks.  It is controlled through
 	  debugfs interface under /sys/kernel/debug/memory-notifier-error-inject/
 
+config PSERIES_RECONFIG_NOTIFIER_ERROR_INJECTION
+	bool "pSeries reconfig notifier error injection"
+	depends on PPC_PSERIES && NOTIFIER_ERROR_INJECTION
+	help
+	  This option provides the ability to inject artifical errors to
+	  pSeries reconfig notifier chain callbacks.  It is controlled
+	  through debugfs interface under
+	  /sys/kernel/debug/pSeries-reconfig-notifier-error-inject/
+
 config CPU_NOTIFIER_ERROR_INJECT
 	tristate "CPU notifier error injection module"
 	depends on HOTPLUG_CPU && DEBUG_KERNEL
-- 
1.7.4.4