[PATCH 1/2] OProfile - Enable SPU switch notification to detect currently active SPU tasks - update

[PATCH 1/2] OProfile - Enable SPU switch notification to detect currently active SPU tasks - update

[Bob Nelson]

From: Maynard Johnson <mpjohn@us.ibm.com>

This patch adds to the capability of spu_switch_event_register so that
the caller is also notified of currently active SPU tasks.
Exports spu_switch_event_register and spu_switch_event_unregister so
that OProfile can get access to the notifications provided.

Signed-off-by: Maynard Johnson <mpjohn@us.ibm.com>
Signed-off-by: Carl Love <carll@us.ibm.com>
Signed-off-by: Bob Nelson <rrnelson@us.ibm.com>
Acked-by: Arnd Bergmann <arnd.bergmann@de.ibm.com>
Acked-by: Paul Mackerras <paulus@samba.org>

---

We would like this patch included in -mm and 2.6.23

Changed "for (node = 0; node < MAX_NUMNODES; node++)" loop to 
for_each_online_node(node).
Added comment to memory barrier.
Better info in changelog.

Index: powerpc.git/arch/powerpc/platforms/cell/spufs/sched.c
===================================================================
--- powerpc.git.orig/arch/powerpc/platforms/cell/spufs/sched.c
+++ powerpc.git/arch/powerpc/platforms/cell/spufs/sched.c
@@ -204,21 +204,51 @@ static void spu_remove_from_active_list(
 
 static BLOCKING_NOTIFIER_HEAD(spu_switch_notifier);
 
-static void spu_switch_notify(struct spu *spu, struct spu_context *ctx)
+void spu_switch_notify(struct spu *spu, struct spu_context *ctx)
 {
 	blocking_notifier_call_chain(&spu_switch_notifier,
 			    ctx ? ctx->object_id : 0, spu);
 }
 
+static void notify_spus_active(void)
+{
+	int node;
+
+	/*
+	 * Wake up the active spu_contexts.
+	 *
+	 * When the awakened processes see their "notify_active" flag is set,
+	 * they will call spu_switch_notify();
+	 */
+	for_each_online_node(node) {
+		struct spu *spu;
+		mutex_lock(&spu_prio->active_mutex[node]);
+		list_for_each_entry(spu, &spu_prio->active_list[node], list) {
+			struct spu_context *ctx = spu->ctx;
+			set_bit(SPU_SCHED_NOTIFY_ACTIVE, &ctx->sched_flags);
+			mb();	/* make sure any tasks woken up below */
+				/* can see the bit(s) set above */
+			wake_up_all(&ctx->stop_wq);
+		}
+		mutex_unlock(&spu_prio->active_mutex[node]);
+	}
+}
+
 int spu_switch_event_register(struct notifier_block * n)
 {
-	return blocking_notifier_chain_register(&spu_switch_notifier, n);
+	int ret;
+	ret = blocking_notifier_chain_register(&spu_switch_notifier, n);
+	if (!ret)
+		notify_spus_active();
+	return ret;
 }
+EXPORT_SYMBOL_GPL(spu_switch_event_register);
 
 int spu_switch_event_unregister(struct notifier_block * n)
 {
 	return blocking_notifier_chain_unregister(&spu_switch_notifier, n);
 }
+EXPORT_SYMBOL_GPL(spu_switch_event_unregister);
 
 /**
  * spu_bind_context - bind spu context to physical spu
Index: powerpc.git/arch/powerpc/platforms/cell/spufs/spufs.h
===================================================================
--- powerpc.git.orig/arch/powerpc/platforms/cell/spufs/spufs.h
+++ powerpc.git/arch/powerpc/platforms/cell/spufs/spufs.h
@@ -53,6 +53,11 @@ enum spuctx_execution_state {
 	SPUCTX_UTIL_MAX
 };
 
+/* ctx->sched_flags */
+enum {
+	SPU_SCHED_NOTIFY_ACTIVE,
+};
+
 struct spu_context {
 	struct spu *spu;		  /* pointer to a physical SPU */
 	struct spu_state csa;		  /* SPU context save area. */
@@ -231,6 +236,7 @@ void spu_acquire_saved(struct spu_contex
 int spu_activate(struct spu_context *ctx, unsigned long flags);
 void spu_deactivate(struct spu_context *ctx);
 void spu_yield(struct spu_context *ctx);
+void spu_switch_notify(struct spu *spu, struct spu_context *ctx);
 void spu_set_timeslice(struct spu_context *ctx);
 void spu_update_sched_info(struct spu_context *ctx);
 void __spu_update_sched_info(struct spu_context *ctx);
Index: powerpc.git/arch/powerpc/platforms/cell/spufs/run.c
===================================================================
--- powerpc.git.orig/arch/powerpc/platforms/cell/spufs/run.c
+++ powerpc.git/arch/powerpc/platforms/cell/spufs/run.c
@@ -18,15 +18,17 @@ void spufs_stop_callback(struct spu *spu
 	wake_up_all(&ctx->stop_wq);
 }
 
-static inline int spu_stopped(struct spu_context *ctx, u32 * stat)
+static inline int spu_stopped(struct spu_context *ctx, u32 *stat)
 {
 	struct spu *spu;
 	u64 pte_fault;
 
 	*stat = ctx->ops->status_read(ctx);
-	if (ctx->state != SPU_STATE_RUNNABLE)
-		return 1;
+
 	spu = ctx->spu;
+	if (ctx->state != SPU_STATE_RUNNABLE ||
+	    test_bit(SPU_SCHED_NOTIFY_ACTIVE, &ctx->sched_flags))
+		return 1;
 	pte_fault = spu->dsisr &
 	    (MFC_DSISR_PTE_NOT_FOUND | MFC_DSISR_ACCESS_DENIED);
 	return (!(*stat & SPU_STATUS_RUNNING) || pte_fault || spu->class_0_pending) ?
@@ -124,7 +126,7 @@ out:
 	return ret;
 }
 
-static int spu_run_init(struct spu_context *ctx, u32 * npc)
+static int spu_run_init(struct spu_context *ctx, u32 *npc)
 {
 	if (ctx->flags & SPU_CREATE_ISOLATE) {
 		unsigned long runcntl;
@@ -154,8 +156,8 @@ static int spu_run_init(struct spu_conte
 	return 0;
 }
 
-static int spu_run_fini(struct spu_context *ctx, u32 * npc,
-			       u32 * status)
+static int spu_run_fini(struct spu_context *ctx, u32 *npc,
+			       u32 *status)
 {
 	int ret = 0;
 
@@ -293,6 +295,7 @@ long spufs_run_spu(struct file *file, st
 		   u32 *npc, u32 *event)
 {
 	int ret;
+	struct spu *spu;
 	u32 status;
 
 	if (mutex_lock_interruptible(&ctx->run_mutex))
@@ -326,8 +329,16 @@ long spufs_run_spu(struct file *file, st
 
 	do {
 		ret = spufs_wait(ctx->stop_wq, spu_stopped(ctx, &status));
+		spu = ctx->spu;
 		if (unlikely(ret))
 			break;
+		if (unlikely(test_and_clear_bit(SPU_SCHED_NOTIFY_ACTIVE,
+						&ctx->sched_flags))) {
+			if (!(status & SPU_STATUS_STOPPED_BY_STOP)) {
+				spu_switch_notify(spu, ctx);
+				continue;
+			}
+		}
 		if ((status & SPU_STATUS_STOPPED_BY_STOP) &&
 		    (status >> SPU_STOP_STATUS_SHIFT == 0x2104)) {
 			ret = spu_process_callback(ctx);

Re: [PATCH 1/2] OProfile - Enable SPU switch notification to detect currently active SPU tasks - update

[Andrew Morton]

On Fri, 20 Jul 2007 14:24:07 -0500
Bob Nelson <rrnelson@linux.vnet.ibm.com> wrote:

> From: Maynard Johnson <mpjohn@us.ibm.com>
> 
> This patch adds to the capability of spu_switch_event_register so that
> the caller is also notified of currently active SPU tasks.
> Exports spu_switch_event_register and spu_switch_event_unregister so
> that OProfile can get access to the notifications provided.
> 
> Signed-off-by: Maynard Johnson <mpjohn@us.ibm.com>
> Signed-off-by: Carl Love <carll@us.ibm.com>
> Signed-off-by: Bob Nelson <rrnelson@us.ibm.com>
> Acked-by: Arnd Bergmann <arnd.bergmann@de.ibm.com>
> Acked-by: Paul Mackerras <paulus@samba.org>
> 
> ---
> 
> We would like this patch included in -mm and 2.6.23
> 
> Changed "for (node = 0; node < MAX_NUMNODES; node++)" loop to 
> for_each_online_node(node).
> Added comment to memory barrier.
> Better info in changelog.

here it is:

--- a/arch/powerpc/platforms/cell/spufs/sched.c~oprofile-enable-spu-switch-notification-to-detect-currently-active-spu-tasks-update
+++ a/arch/powerpc/platforms/cell/spufs/sched.c
@@ -220,13 +220,14 @@ static void notify_spus_active(void)
 	 * When the awakened processes see their "notify_active" flag is set,
 	 * they will call spu_switch_notify();
 	 */
-	for (node = 0; node < MAX_NUMNODES; node++) {
+	for_each_online_node(node) {
 		struct spu *spu;
 		mutex_lock(&spu_prio->active_mutex[node]);
 		list_for_each_entry(spu, &spu_prio->active_list[node], list) {
 			struct spu_context *ctx = spu->ctx;
 			set_bit(SPU_SCHED_NOTIFY_ACTIVE, &ctx->sched_flags);
-			mb();
+			mb();	/* make sure any tasks woken up below */
+				/* can see the bit(s) set above */
 			wake_up_all(&ctx->stop_wq);
 		}
 		mutex_unlock(&spu_prio->active_mutex[node]);
_

I still wonder about that barrier.  At the least it should be smp_mb(). 
But aren't our set_bit() semantics _alone_ sufficient to make this barrier
unneeded?

If it _is_ possible for the effects of a set_bit() to not be visible to a
woken-up thread then I suspect we'll have nasty little problems in quite a
few places.  Maybe wake_up() should itself have a barrier to prevent such
things?

Doing that would be a documentation-only change, I suspect, given that the
current implementation of wake_up() starts out with a spin_lock_irqsave().

Re: [PATCH 1/2] OProfile - Enable SPU switch notification to detect currently active SPU tasks - update

[Arnd Bergmann]

T24gRnJpZGF5IDIwIEp1bHkgMjAwNywgQW5kcmV3IE1vcnRvbiB3cm90ZToKPiAroKCgoKCgoGZv
cl9lYWNoX29ubGluZV9ub2RlKG5vZGUpIHsKPiCgoKCgoKCgoKCgoKCgoKCgc3RydWN0IHNwdSAq
c3B1Owo+IKCgoKCgoKCgoKCgoKCgoKBtdXRleF9sb2NrKCZzcHVfcHJpby0+YWN0aXZlX211dGV4
W25vZGVdKTsKPiCgoKCgoKCgoKCgoKCgoKCgbGlzdF9mb3JfZWFjaF9lbnRyeShzcHUsICZzcHVf
cHJpby0+YWN0aXZlX2xpc3Rbbm9kZV0sIGxpc3QpIHsKPiCgoKCgoKCgoKCgoKCgoKCgoKCgoKCg
oKBzdHJ1Y3Qgc3B1X2NvbnRleHQgKmN0eCA9IHNwdS0+Y3R4Owo+IKCgoKCgoKCgoKCgoKCgoKCg
oKCgoKCgoHNldF9iaXQoU1BVX1NDSEVEX05PVElGWV9BQ1RJVkUsICZjdHgtPnNjaGVkX2ZsYWdz
KTsKPiAtoKCgoKCgoKCgoKCgoKCgoKCgoKCgoKBtYigpOwo+ICugoKCgoKCgoKCgoKCgoKCgoKCg
oKCgoG1iKCk7oKCgLyogbWFrZSBzdXJlIGFueSB0YXNrcyB3b2tlbiB1cCBiZWxvdyAqLwo+ICug
oKCgoKCgoKCgoKCgoKCgoKCgoKCgoKCgoKCgoKCgLyogY2FuIHNlZSB0aGUgYml0KHMpIHNldCBh
Ym92ZSAqLwo+IKCgoKCgoKCgoKCgoKCgoKCgoKCgoKCgoHdha2VfdXBfYWxsKCZjdHgtPnN0b3Bf
d3EpOwo+IKCgoKCgoKCgoKCgoKCgoKB9Cj4goKCgoKCgoKCgoKCgoKCgoG11dGV4X3VubG9jaygm
c3B1X3ByaW8tPmFjdGl2ZV9tdXRleFtub2RlXSk7Cj4gXwo+IAo+IEkgc3RpbGwgd29uZGVyIGFi
b3V0IHRoYXQgYmFycmllci4goEF0IHRoZSBsZWFzdCBpdCBzaG91bGQgYmUgc21wX21iKCkuIAo+
IEJ1dCBhcmVuJ3Qgb3VyIHNldF9iaXQoKSBzZW1hbnRpY3MgX2Fsb25lXyBzdWZmaWNpZW50IHRv
IG1ha2UgdGhpcyBiYXJyaWVyCj4gdW5uZWVkZWQ/CgpJIGJlbGlldmUgdGhleSBhcmUgc3VmZmlj
aWVudCBvbiBwb3dlcnBjLCBhbmQgc2hvdWxkIGJlIGV2ZXJ5d2hlcmUsIHNvCnRoZSBtYigpIGNh
biBwcm9iYWJseSBnbyBhd2F5IGVudGlyZWx5LiBzZXRfYml0KCkgZG9lcyBhIGxkYXJ4L3N0ZGN4
LAp3aGljaCBpcyBndWFyYW50ZWVkIHRvIGJlIGF0b21pYyBhY3Jvc3MgQ1BVcy4KCglBcm5kIDw+
PAo=

Re: [PATCH 1/2] OProfile - Enable SPU switch notification to detect currently active SPU tasks - update

[Paul Mackerras]

Andrew Morton writes:

> I still wonder about that barrier.  At the least it should be smp_mb(). 
> But aren't our set_bit() semantics _alone_ sufficient to make this barrier
> unneeded?

No, but our wake_up semantics certainly ought to be, if they aren't
already.  It's the wake_up which implies synchronization with other
tasks, not the set_bit.

Paul.