* [PATCH 2/5] spufs: switchable spu contexts
2005-11-15 20:53 [PATCH 0/5] SPU file system for 2.6.15-rc-mm Arnd Bergmann
@ 2005-11-15 20:53 ` Arnd Bergmann
2005-11-15 20:53 ` [PATCH 3/5] kernel-side context switch code for spufs Arnd Bergmann
` (3 subsequent siblings)
4 siblings, 0 replies; 10+ messages in thread
From: Arnd Bergmann @ 2005-11-15 20:53 UTC (permalink / raw)
To: akpm; +Cc: Paul Mackerras, linuxppc64-dev, linux-kernel, mnutter,
Arnd Bergmann
[-- Attachment #1: spufs-context-4.diff --]
[-- Type: text/plain, Size: 43043 bytes --]
Add some infrastructure for saving and restoring the context of an
SPE. This patch creates a new structure that can hold the whole
state of a physical SPE in memory. It also contains code that
avoids races during the context switch and the binary code that
is loaded to the SPU in order to access its registers.
The actual PPE- and SPE-side context switch code are two separate
patches.
From: Mark Nutter <mnutter@us.ibm.com>
Signed-off-by: Arnd Bergmann <arndb@de.ibm.com>
---
arch/ppc64/kernel/spu_base.c | 27 ++
fs/spufs/Makefile | 4
fs/spufs/context.c | 18 +
fs/spufs/spu_restore_dump.h_shipped | 342 ++++++++++++++++++++++++++++++++++++
fs/spufs/spu_save_dump.h_shipped | 290 ++++++++++++++++++++++++++++++
fs/spufs/spufs.h | 2
fs/spufs/switch.c | 174 ++++++++++++++++++
include/asm-ppc64/spu.h | 76 ++++++++
include/asm-ppc64/spu_csa.h | 256 ++++++++++++++++++++++++++
9 files changed, 1185 insertions(+), 4 deletions(-)
Index: linux-cg/arch/ppc64/kernel/spu_base.c
===================================================================
--- linux-cg.orig/arch/powerpc/platforms/cell/spu_base.c
+++ linux-cg/arch/powerpc/platforms/cell/spu_base.c
@@ -62,7 +62,9 @@ static int __spu_trap_error(struct spu *
static void spu_restart_dma(struct spu *spu)
{
struct spu_priv2 __iomem *priv2 = spu->priv2;
- out_be64(&priv2->mfc_control_RW, MFC_CNTL_RESTART_DMA_COMMAND);
+
+ if (!test_bit(SPU_CONTEXT_SWITCH_PENDING_nr, &spu->flags))
+ out_be64(&priv2->mfc_control_RW, MFC_CNTL_RESTART_DMA_COMMAND);
}
static int __spu_trap_data_seg(struct spu *spu, unsigned long ea)
@@ -72,6 +74,11 @@ static int __spu_trap_data_seg(struct sp
pr_debug("%s\n", __FUNCTION__);
+ if (test_bit(SPU_CONTEXT_SWITCH_ACTIVE_nr, &spu->flags)) {
+ printk("%s: invalid access during switch!\n", __func__);
+ return 1;
+ }
+
if (REGION_ID(ea) != USER_REGION_ID) {
pr_debug("invalid region access at %016lx\n", ea);
return 1;
@@ -98,6 +105,7 @@ static int __spu_trap_data_seg(struct sp
return 0;
}
+extern int hash_page(unsigned long ea, unsigned long access, unsigned long trap); //XXX
static int __spu_trap_data_map(struct spu *spu, unsigned long ea)
{
unsigned long dsisr;
@@ -107,8 +115,21 @@ static int __spu_trap_data_map(struct sp
priv1 = spu->priv1;
dsisr = in_be64(&priv1->mfc_dsisr_RW);
- wake_up(&spu->stop_wq);
+ /* Handle kernel space hash faults immediately.
+ User hash faults need to be deferred to process context. */
+ if ((dsisr & MFC_DSISR_PTE_NOT_FOUND)
+ && REGION_ID(ea) != USER_REGION_ID
+ && hash_page(ea, _PAGE_PRESENT, 0x300) == 0) {
+ spu_restart_dma(spu);
+ return 0;
+ }
+
+ if (test_bit(SPU_CONTEXT_SWITCH_ACTIVE_nr, &spu->flags)) {
+ printk("%s: invalid access during switch!\n", __func__);
+ return 1;
+ }
+ wake_up(&spu->stop_wq);
return 0;
}
@@ -382,7 +403,6 @@ void spu_free(struct spu *spu)
}
EXPORT_SYMBOL(spu_free);
-extern int hash_page(unsigned long ea, unsigned long access, unsigned long trap); //XXX
static int spu_handle_mm_fault(struct spu *spu)
{
struct spu_priv1 __iomem *priv1;
@@ -650,6 +670,7 @@ static int __init create_spu(struct devi
spu->slb_replace = 0;
spu->mm = NULL;
spu->class_0_pending = 0;
+ spu->flags = 0UL;
spin_lock_init(&spu->register_lock);
out_be64(&spu->priv1->mfc_sdr_RW, mfspr(SPRN_SDR1));
Index: linux-cg/arch/powerpc/platforms/cell/spufs/Makefile
===================================================================
--- linux-cg.orig/arch/powerpc/platforms/cell/spufs/Makefile
+++ linux-cg/arch/powerpc/platforms/cell/spufs/Makefile
@@ -1,3 +1,5 @@
obj-$(CONFIG_SPU_FS) += spufs.o
-spufs-y += inode.o file.o context.o syscalls.o
+spufs-y += inode.o file.o context.o switch.o syscalls.o
+
+$(obj)/switch.o: $(obj)/spu_save_dump.h $(obj)/spu_restore_dump.h
Index: linux-cg/arch/powerpc/platforms/cell/spufs/context.c
===================================================================
--- linux-cg.orig/arch/powerpc/platforms/cell/spufs/context.c
+++ linux-cg/arch/powerpc/platforms/cell/spufs/context.c
@@ -22,6 +22,7 @@
#include <linux/slab.h>
#include <asm/spu.h>
+#include <asm/spu_csa.h>
#include "spufs.h"
struct spu_context *alloc_spu_context(void)
@@ -30,9 +31,25 @@ struct spu_context *alloc_spu_context(vo
ctx = kmalloc(sizeof *ctx, GFP_KERNEL);
if (!ctx)
goto out;
+ /* Future enhancement: do not call spu_alloc()
+ * here. This step should be deferred until
+ * spu_run()!!
+ *
+ * More work needs to be done to read(),
+ * write(), mmap(), etc., so that operations
+ * are performed on CSA when the context is
+ * not currently being run. In this way we
+ * can support arbitrarily large number of
+ * entries in /spu, allow state queries, etc.
+ */
ctx->spu = spu_alloc();
if (!ctx->spu)
goto out_free;
+ spu_init_csa(&ctx->csa);
+ if (!ctx->csa.lscsa) {
+ spu_free(ctx->spu);
+ goto out_free;
+ }
init_rwsem(&ctx->backing_sema);
spin_lock_init(&ctx->mmio_lock);
kref_init(&ctx->kref);
@@ -50,6 +67,7 @@ void destroy_spu_context(struct kref *kr
ctx = container_of(kref, struct spu_context, kref);
if (ctx->spu)
spu_free(ctx->spu);
+ spu_fini_csa(&ctx->csa);
kfree(ctx);
}
Index: linux-cg/arch/powerpc/platforms/cell/spufs/spu_restore_dump.h_shipped
===================================================================
--- /dev/null
+++ linux-cg/arch/powerpc/platforms/cell/spufs/spu_restore_dump.h_shipped
@@ -0,0 +1,231 @@
+/*
+ * spu_restore_dump.h: Copyright (C) 2005 IBM.
+ * Hex-dump auto generated from spu_restore.c.
+ * Do not edit!
+ */
+static unsigned int spu_restore_code[] __page_aligned = {
+0x40800000, 0x409ff801, 0x24000080, 0x24fd8081,
+0x1cd80081, 0x33001180, 0x42030003, 0x33800284,
+0x1c010204, 0x40200000, 0x40200000, 0x40200000,
+0x34000190, 0x34004191, 0x34008192, 0x3400c193,
+0x141fc205, 0x23fffd84, 0x1c100183, 0x217ffa85,
+0x3080a000, 0x3080a201, 0x3080a402, 0x3080a603,
+0x3080a804, 0x3080aa05, 0x3080ac06, 0x3080ae07,
+0x3080b008, 0x3080b209, 0x3080b40a, 0x3080b60b,
+0x3080b80c, 0x3080ba0d, 0x3080bc0e, 0x3080be0f,
+0x00003ffc, 0x00000000, 0x00000000, 0x00000000,
+0x01a00182, 0x3ec00083, 0xb0a14103, 0x01a00204,
+0x3ec10082, 0x4202800e, 0x04000703, 0xb0a14202,
+0x21a00803, 0x3fbf028d, 0x3f20068d, 0x3fbe0682,
+0x3fe30102, 0x21a00882, 0x3f82028f, 0x3fe3078f,
+0x3fbf0784, 0x3f200204, 0x3fbe0204, 0x3fe30204,
+0x04000203, 0x21a00903, 0x40848002, 0x21a00982,
+0x40800003, 0x21a00a03, 0x40802002, 0x21a00a82,
+0x21a00083, 0x40800082, 0x21a00b02, 0x10002818,
+0x40a80002, 0x32800007, 0x4207000c, 0x18008208,
+0x40a0000b, 0x4080020a, 0x40800709, 0x00200000,
+0x42070002, 0x3ac30384, 0x1cffc489, 0x00200000,
+0x18008383, 0x38830382, 0x4cffc486, 0x3ac28185,
+0xb0408584, 0x28830382, 0x1c020387, 0x38828182,
+0xb0408405, 0x1802c408, 0x28828182, 0x217ff886,
+0x04000583, 0x21a00803, 0x3fbe0682, 0x3fe30102,
+0x04000106, 0x21a00886, 0x04000603, 0x21a00903,
+0x40803c02, 0x21a00982, 0x40800003, 0x04000184,
+0x21a00a04, 0x40802202, 0x21a00a82, 0x42028005,
+0x34208702, 0x21002282, 0x21a00804, 0x21a00886,
+0x3fbf0782, 0x3f200102, 0x3fbe0102, 0x3fe30102,
+0x21a00902, 0x40804003, 0x21a00983, 0x21a00a04,
+0x40805a02, 0x21a00a82, 0x40800083, 0x21a00b83,
+0x01a00c02, 0x01a00d83, 0x3420c282, 0x21a00e02,
+0x34210283, 0x21a00f03, 0x34200284, 0x77400200,
+0x3421c282, 0x21a00702, 0x34218283, 0x21a00083,
+0x34214282, 0x21a00b02, 0x4200480c, 0x00200000,
+0x1c010286, 0x34220284, 0x34220302, 0x0f608203,
+0x5c024204, 0x3b81810b, 0x42013c02, 0x00200000,
+0x18008185, 0x38808183, 0x3b814182, 0x21004e84,
+0x4020007f, 0x35000100, 0x000004e0, 0x000002a0,
+0x000002e8, 0x00000428, 0x00000360, 0x000002e8,
+0x000004a0, 0x00000468, 0x000003c8, 0x00000360,
+0x409ffe02, 0x30801203, 0x40800204, 0x3ec40085,
+0x10009c09, 0x3ac10606, 0xb060c105, 0x4020007f,
+0x4020007f, 0x20801203, 0x38810602, 0xb0408586,
+0x28810602, 0x32004180, 0x34204702, 0x21a00382,
+0x4020007f, 0x327fdc80, 0x409ffe02, 0x30801203,
+0x40800204, 0x3ec40087, 0x40800405, 0x00200000,
+0x40800606, 0x3ac10608, 0x3ac14609, 0x3ac1860a,
+0xb060c107, 0x20801203, 0x41004003, 0x38810602,
+0x4020007f, 0xb0408188, 0x4020007f, 0x28810602,
+0x41201002, 0x38814603, 0x10009c09, 0xb060c109,
+0x4020007f, 0x28814603, 0x41193f83, 0x38818602,
+0x60ffc003, 0xb040818a, 0x28818602, 0x32003080,
+0x409ffe02, 0x30801203, 0x40800204, 0x3ec40087,
+0x41201008, 0x10009c14, 0x40800405, 0x3ac10609,
+0x40800606, 0x3ac1460a, 0xb060c107, 0x3ac1860b,
+0x20801203, 0x38810602, 0xb0408409, 0x28810602,
+0x38814603, 0xb060c40a, 0x4020007f, 0x28814603,
+0x41193f83, 0x38818602, 0x60ffc003, 0xb040818b,
+0x28818602, 0x32002380, 0x409ffe02, 0x30801204,
+0x40800205, 0x3ec40083, 0x40800406, 0x3ac14607,
+0x3ac18608, 0xb0810103, 0x41004002, 0x20801204,
+0x4020007f, 0x38814603, 0x10009c0b, 0xb060c107,
+0x4020007f, 0x4020007f, 0x28814603, 0x38818602,
+0x4020007f, 0x4020007f, 0xb0408588, 0x28818602,
+0x4020007f, 0x32001780, 0x409ffe02, 0x1000640e,
+0x40800204, 0x30801203, 0x40800405, 0x3ec40087,
+0x40800606, 0x3ac10608, 0x3ac14609, 0x3ac1860a,
+0xb060c107, 0x20801203, 0x413d8003, 0x38810602,
+0x4020007f, 0x327fd780, 0x409ffe02, 0x10007f0c,
+0x40800205, 0x30801204, 0x40800406, 0x3ec40083,
+0x3ac14607, 0x3ac18608, 0xb0810103, 0x413d8002,
+0x20801204, 0x38814603, 0x4020007f, 0x327feb80,
+0x409ffe02, 0x30801203, 0x40800204, 0x3ec40087,
+0x40800405, 0x1000650a, 0x40800606, 0x3ac10608,
+0x3ac14609, 0x3ac1860a, 0xb060c107, 0x20801203,
+0x38810602, 0xb0408588, 0x4020007f, 0x327fc980,
+0x00400000, 0x40800003, 0x4020007f, 0x35000000,
+0x00000000, 0x00000000, 0x00000000, 0x00000000,
+0x00000000, 0x00000000, 0x00000000, 0x00000000,
+0x00000000, 0x00000000, 0x00000000, 0x00000000,
+0x00000000, 0x00000000, 0x00000000, 0x00000000,
+0x00000000, 0x00000000, 0x00000000, 0x00000000,
+0x00000000, 0x00000000, 0x00000000, 0x00000000,
+0x00000000, 0x00000000, 0x00000000, 0x00000000,
+0x00000000, 0x00000000, 0x00000000, 0x00000000,
+0x00000000, 0x00000000, 0x00000000, 0x00000000,
+0x00000000, 0x00000000, 0x00000000, 0x00000000,
+0x00000000, 0x00000000, 0x00000000, 0x00000000,
+0x00000000, 0x00000000, 0x00000000, 0x00000000,
+0x00000000, 0x00000000, 0x00000000, 0x00000000,
+0x00000000, 0x00000000, 0x00000000, 0x00000000,
+0x00000000, 0x00000000, 0x00000000, 0x00000000,
+0x00000000, 0x00000000, 0x00000000, 0x00000000,
+0x00000000, 0x00000000, 0x00000000, 0x00000000,
+0x00000000, 0x00000000, 0x00000000, 0x00000000,
+0x00000000, 0x00000000, 0x00000000, 0x00000000,
+0x00000000, 0x00000000, 0x00000000, 0x00000000,
+0x00000000, 0x00000000, 0x00000000, 0x00000000,
+0x00000000, 0x00000000, 0x00000000, 0x00000000,
+0x00000000, 0x00000000, 0x00000000, 0x00000000,
+0x00000000, 0x00000000, 0x00000000, 0x00000000,
+0x00000000, 0x00000000, 0x00000000, 0x00000000,
+0x00000000, 0x00000000, 0x00000000, 0x00000000,
+0x00000000, 0x00000000, 0x00000000, 0x00000000,
+0x00000000, 0x00000000, 0x00000000, 0x00000000,
+0x00000000, 0x00000000, 0x00000000, 0x00000000,
+0x00000000, 0x00000000, 0x00000000, 0x00000000,
+0x00000000, 0x00000000, 0x00000000, 0x00000000,
+0x00000000, 0x00000000, 0x00000000, 0x00000000,
+0x00000000, 0x00000000, 0x00000000, 0x00000000,
+0x00000000, 0x00000000, 0x00000000, 0x00000000,
+0x00000000, 0x00000000, 0x00000000, 0x00000000,
+0x00000000, 0x00000000, 0x00000000, 0x00000000,
+0x00000000, 0x00000000, 0x00000000, 0x00000000,
+0x00000000, 0x00000000, 0x00000000, 0x00000000,
+0x00000000, 0x00000000, 0x00000000, 0x00000000,
+0x00000000, 0x00000000, 0x00000000, 0x00000000,
+0x00000000, 0x00000000, 0x00000000, 0x00000000,
+0x00000000, 0x00000000, 0x00000000, 0x00000000,
+0x00000000, 0x00000000, 0x00000000, 0x00000000,
+0x00000000, 0x00000000, 0x00000000, 0x00000000,
+0x00000000, 0x00000000, 0x00000000, 0x00000000,
+0x00000000, 0x00000000, 0x00000000, 0x00000000,
+0x00000000, 0x00000000, 0x00000000, 0x00000000,
+0x00000000, 0x00000000, 0x00000000, 0x00000000,
+0x00000000, 0x00000000, 0x00000000, 0x00000000,
+0x00000000, 0x00000000, 0x00000000, 0x00000000,
+0x00000000, 0x00000000, 0x00000000, 0x00000000,
+0x00000000, 0x00000000, 0x00000000, 0x00000000,
+0x00000000, 0x00000000, 0x00000000, 0x00000000,
+0x00000000, 0x00000000, 0x00000000, 0x00000000,
+0x00000000, 0x00000000, 0x00000000, 0x00000000,
+0x00000000, 0x00000000, 0x00000000, 0x00000000,
+0x00000000, 0x00000000, 0x00000000, 0x00000000,
+0x00000000, 0x00000000, 0x00000000, 0x00000000,
+0x00000000, 0x00000000, 0x00000000, 0x00000000,
+0x00000000, 0x00000000, 0x00000000, 0x00000000,
+0x00000000, 0x00000000, 0x00000000, 0x00000000,
+0x00000000, 0x00000000, 0x00000000, 0x00000000,
+0x00000000, 0x00000000, 0x00000000, 0x00000000,
+0x00000000, 0x00000000, 0x00000000, 0x00000000,
+0x00000000, 0x00000000, 0x00000000, 0x00000000,
+0x00000000, 0x00000000, 0x00000000, 0x00000000,
+0x00000000, 0x00000000, 0x00000000, 0x00000000,
+0x00000000, 0x00000000, 0x00000000, 0x00000000,
+0x00000000, 0x00000000, 0x00000000, 0x00000000,
+0x00000000, 0x00000000, 0x00000000, 0x00000000,
+0x00000000, 0x00000000, 0x00000000, 0x00000000,
+0x00000000, 0x00000000, 0x00000000, 0x00000000,
+0x00000000, 0x00000000, 0x00000000, 0x00000000,
+0x00000000, 0x00000000, 0x00000000, 0x00000000,
+0x00000000, 0x00000000, 0x00000000, 0x00000000,
+0x00000000, 0x00000000, 0x00000000, 0x00000000,
+0x00000000, 0x00000000, 0x00000000, 0x00000000,
+0x00000000, 0x00000000, 0x00000000, 0x00000000,
+0x00000000, 0x00000000, 0x00000000, 0x00000000,
+0x00000000, 0x00000000, 0x00000000, 0x00000000,
+0x00000000, 0x00000000, 0x00000000, 0x00000000,
+0x00000000, 0x00000000, 0x00000000, 0x00000000,
+0x00000000, 0x00000000, 0x00000000, 0x00000000,
+0x00000000, 0x00000000, 0x00000000, 0x00000000,
+0x00000000, 0x00000000, 0x00000000, 0x00000000,
+0x00000000, 0x00000000, 0x00000000, 0x00000000,
+0x00000000, 0x00000000, 0x00000000, 0x00000000,
+0x00000000, 0x00000000, 0x00000000, 0x00000000,
+0x00000000, 0x00000000, 0x00000000, 0x00000000,
+0x00000000, 0x00000000, 0x00000000, 0x00000000,
+0x00000000, 0x00000000, 0x00000000, 0x00000000,
+0x00000000, 0x00000000, 0x00000000, 0x00000000,
+0x00000000, 0x00000000, 0x00000000, 0x00000000,
+0x00000000, 0x00000000, 0x00000000, 0x00000000,
+0x00000000, 0x00000000, 0x00000000, 0x00000000,
+0x00000000, 0x00000000, 0x00000000, 0x00000000,
+0x00000000, 0x00000000, 0x00000000, 0x00000000,
+0x00000000, 0x00000000, 0x00000000, 0x00000000,
+0x00000000, 0x00000000, 0x00000000, 0x00000000,
+0x00000000, 0x00000000, 0x00000000, 0x00000000,
+0x00000000, 0x00000000, 0x00000000, 0x00000000,
+0x00000000, 0x00000000, 0x00000000, 0x00000000,
+0x00000000, 0x00000000, 0x00000000, 0x00000000,
+0x00000000, 0x00000000, 0x00000000, 0x00000000,
+0x00000000, 0x00000000, 0x00000000, 0x00000000,
+0x00000000, 0x00000000, 0x00000000, 0x00000000,
+0x00000000, 0x00000000, 0x00000000, 0x00000000,
+0x00000000, 0x00000000, 0x00000000, 0x00000000,
+0x00000000, 0x00000000, 0x00000000, 0x00000000,
+0x00000000, 0x00000000, 0x00000000, 0x00000000,
+0x00000000, 0x00000000, 0x00000000, 0x00000000,
+0x00000000, 0x00000000, 0x00000000, 0x00000000,
+0x00000000, 0x00000000, 0x00000000, 0x00000000,
+0x00000000, 0x00000000, 0x00000000, 0x00000000,
+0x00000000, 0x00000000, 0x00000000, 0x00000000,
+0x00000000, 0x00000000, 0x00000000, 0x00000000,
+0x00000000, 0x00000000, 0x00000000, 0x00000000,
+0x00000000, 0x00000000, 0x00000000, 0x00000000,
+0x00000000, 0x00000000, 0x00000000, 0x00000000,
+0x00000000, 0x00000000, 0x00000000, 0x00000000,
+0x00000000, 0x00000000, 0x00000000, 0x00000000,
+0x00000000, 0x00000000, 0x00000000, 0x00000000,
+0x00000000, 0x00000000, 0x00000000, 0x00000000,
+0x00000000, 0x00000000, 0x00000000, 0x00000000,
+0x00000000, 0x00000000, 0x00000000, 0x00000000,
+0x00000000, 0x00000000, 0x00000000, 0x00000000,
+0x00000000, 0x00000000, 0x00000000, 0x00000000,
+0x00000000, 0x00000000, 0x00000000, 0x00000000,
+0x00000000, 0x00000000, 0x00000000, 0x00000000,
+0x00000000, 0x00000000, 0x00000000, 0x00000000,
+0x00000000, 0x00000000, 0x00000000, 0x00000000,
+0x00000000, 0x00000000, 0x00000000, 0x00000000,
+0x00000000, 0x00000000, 0x00000000, 0x00000000,
+0x00000000, 0x00000000, 0x00000000, 0x00000000,
+0x00000000, 0x00000000, 0x00000000, 0x00000000,
+0x00000000, 0x00000000, 0x00000000, 0x00000000,
+0x00000000, 0x00000000, 0x00000000, 0x00000000,
+0x00000000, 0x00000000, 0x00000000, 0x00000000,
+0x00000000, 0x00000000, 0x00000000, 0x00000000,
+0x00000000, 0x00000000, 0x00000000, 0x00000000,
+0x00000000, 0x00000000, 0x00000000, 0x00000000,
+0x00000000, 0x00000000, 0x00000000, 0x00000000,
+0x00000000, 0x00000000, 0x00000000, 0x00000000,
+0x00000000, 0x00000000, 0x00000000, 0x00000000,
+0x00000000, 0x00000000, 0x00000000, 0x00000000,
+};
Index: linux-cg/arch/powerpc/platforms/cell/spufs/spu_save_dump.h_shipped
===================================================================
--- /dev/null
+++ linux-cg/arch/powerpc/platforms/cell/spufs/spu_save_dump.h_shipped
@@ -0,0 +1,191 @@
+/*
+ * spu_save_dump.h: Copyright (C) 2005 IBM.
+ * Hex-dump auto generated from spu_save.c.
+ * Do not edit!
+ */
+static unsigned int spu_save_code[] __page_aligned = {
+0x20805000, 0x20805201, 0x20805402, 0x20805603,
+0x20805804, 0x20805a05, 0x20805c06, 0x20805e07,
+0x20806008, 0x20806209, 0x2080640a, 0x2080660b,
+0x2080680c, 0x20806a0d, 0x20806c0e, 0x20806e0f,
+0x4201c003, 0x33800184, 0x1c010204, 0x40200000,
+0x24000190, 0x24004191, 0x24008192, 0x2400c193,
+0x141fc205, 0x23fffd84, 0x1c100183, 0x217ffb85,
+0x40800000, 0x409ff801, 0x24000080, 0x24fd8081,
+0x1cd80081, 0x33000180, 0x00000000, 0x00000000,
+0x01a00182, 0x3ec00083, 0xb1c38103, 0x01a00204,
+0x3ec10082, 0x4201400d, 0xb1c38202, 0x01a00583,
+0x34218682, 0x3ed80684, 0xb0408184, 0x24218682,
+0x01a00603, 0x00200000, 0x34214682, 0x3ed40684,
+0xb0408184, 0x40800003, 0x24214682, 0x21a00083,
+0x40800082, 0x21a00b02, 0x4020007f, 0x1000251e,
+0x40a80002, 0x32800008, 0x4205c00c, 0x00200000,
+0x40a0000b, 0x3f82070f, 0x4080020a, 0x40800709,
+0x3fe3078f, 0x3fbf0783, 0x3f200183, 0x3fbe0183,
+0x3fe30187, 0x18008387, 0x4205c002, 0x3ac30404,
+0x1cffc489, 0x00200000, 0x18008403, 0x38830402,
+0x4cffc486, 0x3ac28185, 0xb0408584, 0x28830402,
+0x1c020408, 0x38828182, 0xb0408385, 0x1802c387,
+0x28828182, 0x217ff886, 0x04000582, 0x32800007,
+0x21a00802, 0x3fbf0705, 0x3f200285, 0x3fbe0285,
+0x3fe30285, 0x21a00885, 0x04000603, 0x21a00903,
+0x40803c02, 0x21a00982, 0x04000386, 0x21a00a06,
+0x40801202, 0x21a00a82, 0x73000003, 0x24200683,
+0x01a00404, 0x00200000, 0x34204682, 0x3ec40683,
+0xb0408203, 0x24204682, 0x01a00783, 0x00200000,
+0x3421c682, 0x3edc0684, 0xb0408184, 0x2421c682,
+0x21a00806, 0x21a00885, 0x3fbf0784, 0x3f200204,
+0x3fbe0204, 0x3fe30204, 0x21a00904, 0x40804002,
+0x21a00982, 0x21a00a06, 0x40805a02, 0x21a00a82,
+0x04000683, 0x21a00803, 0x21a00885, 0x21a00904,
+0x40848002, 0x21a00982, 0x21a00a06, 0x40801002,
+0x21a00a82, 0x21a00a06, 0x40806602, 0x00200000,
+0x35800009, 0x21a00a82, 0x40800083, 0x21a00b83,
+0x01a00c02, 0x01a00d83, 0x00003ffb, 0x40800003,
+0x4020007f, 0x35000000, 0x00000000, 0x00000000,
+0x00000000, 0x00000000, 0x00000000, 0x00000000,
+0x00000000, 0x00000000, 0x00000000, 0x00000000,
+0x00000000, 0x00000000, 0x00000000, 0x00000000,
+0x00000000, 0x00000000, 0x00000000, 0x00000000,
+0x00000000, 0x00000000, 0x00000000, 0x00000000,
+0x00000000, 0x00000000, 0x00000000, 0x00000000,
+0x00000000, 0x00000000, 0x00000000, 0x00000000,
+0x00000000, 0x00000000, 0x00000000, 0x00000000,
+0x00000000, 0x00000000, 0x00000000, 0x00000000,
+0x00000000, 0x00000000, 0x00000000, 0x00000000,
+0x00000000, 0x00000000, 0x00000000, 0x00000000,
+0x00000000, 0x00000000, 0x00000000, 0x00000000,
+0x00000000, 0x00000000, 0x00000000, 0x00000000,
+0x00000000, 0x00000000, 0x00000000, 0x00000000,
+0x00000000, 0x00000000, 0x00000000, 0x00000000,
+0x00000000, 0x00000000, 0x00000000, 0x00000000,
+0x00000000, 0x00000000, 0x00000000, 0x00000000,
+0x00000000, 0x00000000, 0x00000000, 0x00000000,
+0x00000000, 0x00000000, 0x00000000, 0x00000000,
+0x00000000, 0x00000000, 0x00000000, 0x00000000,
+0x00000000, 0x00000000, 0x00000000, 0x00000000,
+0x00000000, 0x00000000, 0x00000000, 0x00000000,
+0x00000000, 0x00000000, 0x00000000, 0x00000000,
+0x00000000, 0x00000000, 0x00000000, 0x00000000,
+0x00000000, 0x00000000, 0x00000000, 0x00000000,
+0x00000000, 0x00000000, 0x00000000, 0x00000000,
+0x00000000, 0x00000000, 0x00000000, 0x00000000,
+0x00000000, 0x00000000, 0x00000000, 0x00000000,
+0x00000000, 0x00000000, 0x00000000, 0x00000000,
+0x00000000, 0x00000000, 0x00000000, 0x00000000,
+0x00000000, 0x00000000, 0x00000000, 0x00000000,
+0x00000000, 0x00000000, 0x00000000, 0x00000000,
+0x00000000, 0x00000000, 0x00000000, 0x00000000,
+0x00000000, 0x00000000, 0x00000000, 0x00000000,
+0x00000000, 0x00000000, 0x00000000, 0x00000000,
+0x00000000, 0x00000000, 0x00000000, 0x00000000,
+0x00000000, 0x00000000, 0x00000000, 0x00000000,
+0x00000000, 0x00000000, 0x00000000, 0x00000000,
+0x00000000, 0x00000000, 0x00000000, 0x00000000,
+0x00000000, 0x00000000, 0x00000000, 0x00000000,
+0x00000000, 0x00000000, 0x00000000, 0x00000000,
+0x00000000, 0x00000000, 0x00000000, 0x00000000,
+0x00000000, 0x00000000, 0x00000000, 0x00000000,
+0x00000000, 0x00000000, 0x00000000, 0x00000000,
+0x00000000, 0x00000000, 0x00000000, 0x00000000,
+0x00000000, 0x00000000, 0x00000000, 0x00000000,
+0x00000000, 0x00000000, 0x00000000, 0x00000000,
+0x00000000, 0x00000000, 0x00000000, 0x00000000,
+0x00000000, 0x00000000, 0x00000000, 0x00000000,
+0x00000000, 0x00000000, 0x00000000, 0x00000000,
+0x00000000, 0x00000000, 0x00000000, 0x00000000,
+0x00000000, 0x00000000, 0x00000000, 0x00000000,
+0x00000000, 0x00000000, 0x00000000, 0x00000000,
+0x00000000, 0x00000000, 0x00000000, 0x00000000,
+0x00000000, 0x00000000, 0x00000000, 0x00000000,
+0x00000000, 0x00000000, 0x00000000, 0x00000000,
+0x00000000, 0x00000000, 0x00000000, 0x00000000,
+0x00000000, 0x00000000, 0x00000000, 0x00000000,
+0x00000000, 0x00000000, 0x00000000, 0x00000000,
+0x00000000, 0x00000000, 0x00000000, 0x00000000,
+0x00000000, 0x00000000, 0x00000000, 0x00000000,
+0x00000000, 0x00000000, 0x00000000, 0x00000000,
+0x00000000, 0x00000000, 0x00000000, 0x00000000,
+0x00000000, 0x00000000, 0x00000000, 0x00000000,
+0x00000000, 0x00000000, 0x00000000, 0x00000000,
+0x00000000, 0x00000000, 0x00000000, 0x00000000,
+0x00000000, 0x00000000, 0x00000000, 0x00000000,
+0x00000000, 0x00000000, 0x00000000, 0x00000000,
+0x00000000, 0x00000000, 0x00000000, 0x00000000,
+0x00000000, 0x00000000, 0x00000000, 0x00000000,
+0x00000000, 0x00000000, 0x00000000, 0x00000000,
+0x00000000, 0x00000000, 0x00000000, 0x00000000,
+0x00000000, 0x00000000, 0x00000000, 0x00000000,
+0x00000000, 0x00000000, 0x00000000, 0x00000000,
+0x00000000, 0x00000000, 0x00000000, 0x00000000,
+0x00000000, 0x00000000, 0x00000000, 0x00000000,
+0x00000000, 0x00000000, 0x00000000, 0x00000000,
+0x00000000, 0x00000000, 0x00000000, 0x00000000,
+0x00000000, 0x00000000, 0x00000000, 0x00000000,
+0x00000000, 0x00000000, 0x00000000, 0x00000000,
+0x00000000, 0x00000000, 0x00000000, 0x00000000,
+0x00000000, 0x00000000, 0x00000000, 0x00000000,
+0x00000000, 0x00000000, 0x00000000, 0x00000000,
+0x00000000, 0x00000000, 0x00000000, 0x00000000,
+0x00000000, 0x00000000, 0x00000000, 0x00000000,
+0x00000000, 0x00000000, 0x00000000, 0x00000000,
+0x00000000, 0x00000000, 0x00000000, 0x00000000,
+0x00000000, 0x00000000, 0x00000000, 0x00000000,
+0x00000000, 0x00000000, 0x00000000, 0x00000000,
+0x00000000, 0x00000000, 0x00000000, 0x00000000,
+0x00000000, 0x00000000, 0x00000000, 0x00000000,
+0x00000000, 0x00000000, 0x00000000, 0x00000000,
+0x00000000, 0x00000000, 0x00000000, 0x00000000,
+0x00000000, 0x00000000, 0x00000000, 0x00000000,
+0x00000000, 0x00000000, 0x00000000, 0x00000000,
+0x00000000, 0x00000000, 0x00000000, 0x00000000,
+0x00000000, 0x00000000, 0x00000000, 0x00000000,
+0x00000000, 0x00000000, 0x00000000, 0x00000000,
+0x00000000, 0x00000000, 0x00000000, 0x00000000,
+0x00000000, 0x00000000, 0x00000000, 0x00000000,
+0x00000000, 0x00000000, 0x00000000, 0x00000000,
+0x00000000, 0x00000000, 0x00000000, 0x00000000,
+0x00000000, 0x00000000, 0x00000000, 0x00000000,
+0x00000000, 0x00000000, 0x00000000, 0x00000000,
+0x00000000, 0x00000000, 0x00000000, 0x00000000,
+0x00000000, 0x00000000, 0x00000000, 0x00000000,
+0x00000000, 0x00000000, 0x00000000, 0x00000000,
+0x00000000, 0x00000000, 0x00000000, 0x00000000,
+0x00000000, 0x00000000, 0x00000000, 0x00000000,
+0x00000000, 0x00000000, 0x00000000, 0x00000000,
+0x00000000, 0x00000000, 0x00000000, 0x00000000,
+0x00000000, 0x00000000, 0x00000000, 0x00000000,
+0x00000000, 0x00000000, 0x00000000, 0x00000000,
+0x00000000, 0x00000000, 0x00000000, 0x00000000,
+0x00000000, 0x00000000, 0x00000000, 0x00000000,
+0x00000000, 0x00000000, 0x00000000, 0x00000000,
+0x00000000, 0x00000000, 0x00000000, 0x00000000,
+0x00000000, 0x00000000, 0x00000000, 0x00000000,
+0x00000000, 0x00000000, 0x00000000, 0x00000000,
+0x00000000, 0x00000000, 0x00000000, 0x00000000,
+0x00000000, 0x00000000, 0x00000000, 0x00000000,
+0x00000000, 0x00000000, 0x00000000, 0x00000000,
+0x00000000, 0x00000000, 0x00000000, 0x00000000,
+0x00000000, 0x00000000, 0x00000000, 0x00000000,
+0x00000000, 0x00000000, 0x00000000, 0x00000000,
+0x00000000, 0x00000000, 0x00000000, 0x00000000,
+0x00000000, 0x00000000, 0x00000000, 0x00000000,
+0x00000000, 0x00000000, 0x00000000, 0x00000000,
+0x00000000, 0x00000000, 0x00000000, 0x00000000,
+0x00000000, 0x00000000, 0x00000000, 0x00000000,
+0x00000000, 0x00000000, 0x00000000, 0x00000000,
+0x00000000, 0x00000000, 0x00000000, 0x00000000,
+0x00000000, 0x00000000, 0x00000000, 0x00000000,
+0x00000000, 0x00000000, 0x00000000, 0x00000000,
+0x00000000, 0x00000000, 0x00000000, 0x00000000,
+0x00000000, 0x00000000, 0x00000000, 0x00000000,
+0x00000000, 0x00000000, 0x00000000, 0x00000000,
+0x00000000, 0x00000000, 0x00000000, 0x00000000,
+0x00000000, 0x00000000, 0x00000000, 0x00000000,
+0x00000000, 0x00000000, 0x00000000, 0x00000000,
+0x00000000, 0x00000000, 0x00000000, 0x00000000,
+0x00000000, 0x00000000, 0x00000000, 0x00000000,
+0x00000000, 0x00000000, 0x00000000, 0x00000000,
+0x00000000, 0x00000000, 0x00000000, 0x00000000,
+0x00000000, 0x00000000, 0x00000000, 0x00000000,
+};
Index: linux-cg/arch/powerpc/platforms/cell/spufs/spufs.h
===================================================================
--- linux-cg.orig/arch/powerpc/platforms/cell/spufs/spufs.h
+++ linux-cg/arch/powerpc/platforms/cell/spufs/spufs.h
@@ -28,6 +28,7 @@
#include <linux/fs.h>
#include <asm/spu.h>
+#include <asm/spu_csa.h>
/* The magic number for our file system */
enum {
@@ -36,6 +37,7 @@ enum {
struct spu_context {
struct spu *spu; /* pointer to a physical SPU */
+ struct spu_state csa; /* SPU context save area. */
struct rw_semaphore backing_sema; /* protects the above */
spinlock_t mmio_lock; /* protects mmio access */
Index: linux-cg/arch/powerpc/platforms/cell/spufs/switch.c
===================================================================
--- /dev/null
+++ linux-cg/arch/powerpc/platforms/cell/spufs/switch.c
@@ -0,0 +1,174 @@
+/*
+ * spu_switch.c
+ *
+ * (C) Copyright IBM Corp. 2005
+ *
+ * Author: Mark Nutter <mnutter@us.ibm.com>
+ *
+ * Host-side part of SPU context switch sequence outlined in
+ * Synergistic Processor Element, Book IV.
+ *
+ * A fully premptive switch of an SPE is very expensive in terms
+ * of time and system resources. SPE Book IV indicates that SPE
+ * allocation should follow a "serially reusable device" model,
+ * in which the SPE is assigned a task until it completes. When
+ * this is not possible, this sequence may be used to premptively
+ * save, and then later (optionally) restore the context of a
+ * program executing on an SPE.
+ *
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2, or (at your option)
+ * any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ */
+
+#include <linux/config.h>
+#include <linux/module.h>
+#include <linux/errno.h>
+#include <linux/sched.h>
+#include <linux/kernel.h>
+#include <linux/mm.h>
+#include <linux/vmalloc.h>
+#include <linux/smp.h>
+#include <linux/smp_lock.h>
+#include <linux/stddef.h>
+#include <linux/unistd.h>
+
+#include <asm/io.h>
+#include <asm/spu.h>
+#include <asm/spu_csa.h>
+#include <asm/mmu_context.h>
+
+#include "spu_save_dump.h"
+#include "spu_restore_dump.h"
+
+/**
+ * spu_save - SPU context save, with locking.
+ * @prev: pointer to SPU context save area, to be saved.
+ * @spu: pointer to SPU iomem structure.
+ *
+ * Acquire locks, perform the save operation then return.
+ */
+int spu_save(struct spu_state *prev, struct spu *spu)
+{
+ /* XXX missing */
+
+ return 0;
+}
+
+/**
+ * spu_restore - SPU context restore, with harvest and locking.
+ * @new: pointer to SPU context save area, to be restored.
+ * @spu: pointer to SPU iomem structure.
+ *
+ * Perform harvest + restore, as we may not be coming
+ * from a previous succesful save operation, and the
+ * hardware state is unknown.
+ */
+int spu_restore(struct spu_state *new, struct spu *spu)
+{
+ /* XXX missing */
+
+ return 0;
+}
+
+/**
+ * spu_switch - SPU context switch (save + restore).
+ * @prev: pointer to SPU context save area, to be saved.
+ * @new: pointer to SPU context save area, to be restored.
+ * @spu: pointer to SPU iomem structure.
+ *
+ * Perform save, then restore. Only harvest if the
+ * save fails, as cleanup is otherwise not needed.
+ */
+int spu_switch(struct spu_state *prev, struct spu_state *new, struct spu *spu)
+{
+ /* XXX missing */
+
+ return 0;
+}
+
+static void init_prob(struct spu_state *csa)
+{
+ csa->spu_chnlcnt_RW[9] = 1;
+ csa->spu_chnlcnt_RW[21] = 16;
+ csa->spu_chnlcnt_RW[23] = 1;
+ csa->spu_chnlcnt_RW[28] = 1;
+ csa->spu_chnlcnt_RW[30] = 1;
+ csa->prob.spu_runcntl_RW = SPU_RUNCNTL_STOP;
+}
+
+static void init_priv1(struct spu_state *csa)
+{
+ /* Enable decode, relocate, tlbie response, master runcntl. */
+ csa->priv1.mfc_sr1_RW = MFC_STATE1_LOCAL_STORAGE_DECODE_MASK |
+ MFC_STATE1_MASTER_RUN_CONTROL_MASK |
+ MFC_STATE1_PROBLEM_STATE_MASK |
+ MFC_STATE1_RELOCATE_MASK | MFC_STATE1_BUS_TLBIE_MASK;
+
+ /* Set storage description. */
+ csa->priv1.mfc_sdr_RW = mfspr(SPRN_SDR1);
+
+ /* Enable OS-specific set of interrupts. */
+ csa->priv1.int_mask_class0_RW = CLASS0_ENABLE_DMA_ALIGNMENT_INTR |
+ CLASS0_ENABLE_INVALID_DMA_COMMAND_INTR |
+ CLASS0_ENABLE_SPU_ERROR_INTR;
+ csa->priv1.int_mask_class1_RW = CLASS1_ENABLE_SEGMENT_FAULT_INTR |
+ CLASS1_ENABLE_STORAGE_FAULT_INTR;
+ csa->priv1.int_mask_class2_RW = CLASS2_ENABLE_MAILBOX_INTR |
+ CLASS2_ENABLE_SPU_STOP_INTR | CLASS2_ENABLE_SPU_HALT_INTR;
+}
+
+static void init_priv2(struct spu_state *csa)
+{
+ csa->priv2.spu_lslr_RW = LS_ADDR_MASK;
+ csa->priv2.mfc_control_RW = MFC_CNTL_RESUME_DMA_QUEUE |
+ MFC_CNTL_NORMAL_DMA_QUEUE_OPERATION |
+ MFC_CNTL_DMA_QUEUES_EMPTY_MASK;
+}
+
+/**
+ * spu_alloc_csa - allocate and initialize an SPU context save area.
+ *
+ * Allocate and initialize the contents of an SPU context save area.
+ * This includes enabling address translation, interrupt masks, etc.,
+ * as appropriate for the given OS environment.
+ *
+ * Note that storage for the 'lscsa' is allocated separately,
+ * as it is by far the largest of the context save regions,
+ * and may need to be pinned or otherwise specially aligned.
+ */
+void spu_init_csa(struct spu_state *csa)
+{
+ struct spu_lscsa *lscsa;
+
+ if (!csa)
+ return;
+ memset(csa, 0, sizeof(struct spu_state));
+
+ lscsa = vmalloc(sizeof(struct spu_lscsa));
+ if (!lscsa)
+ return;
+
+ memset(lscsa, 0, sizeof(struct spu_lscsa));
+ csa->lscsa = lscsa;
+
+ init_prob(csa);
+ init_priv1(csa);
+ init_priv2(csa);
+}
+
+void spu_fini_csa(struct spu_state *csa)
+{
+ vfree(csa->lscsa);
+}
Index: linux-cg/include/asm-powerpc/spu.h
===================================================================
--- linux-cg.orig/include/asm-powerpc/spu.h
+++ linux-cg/include/asm-powerpc/spu.h
@@ -29,6 +29,81 @@
#define LS_ORDER (6) /* 256 kb */
#define LS_SIZE (PAGE_SIZE << LS_ORDER)
+#define LS_ADDR_MASK (LS_SIZE - 1)
+
+#define MFC_PUT_CMD 0x20
+#define MFC_PUTS_CMD 0x28
+#define MFC_PUTR_CMD 0x30
+#define MFC_PUTF_CMD 0x22
+#define MFC_PUTB_CMD 0x21
+#define MFC_PUTFS_CMD 0x2A
+#define MFC_PUTBS_CMD 0x29
+#define MFC_PUTRF_CMD 0x32
+#define MFC_PUTRB_CMD 0x31
+#define MFC_PUTL_CMD 0x24
+#define MFC_PUTRL_CMD 0x34
+#define MFC_PUTLF_CMD 0x26
+#define MFC_PUTLB_CMD 0x25
+#define MFC_PUTRLF_CMD 0x36
+#define MFC_PUTRLB_CMD 0x35
+
+#define MFC_GET_CMD 0x40
+#define MFC_GETS_CMD 0x48
+#define MFC_GETF_CMD 0x42
+#define MFC_GETB_CMD 0x41
+#define MFC_GETFS_CMD 0x4A
+#define MFC_GETBS_CMD 0x49
+#define MFC_GETL_CMD 0x44
+#define MFC_GETLF_CMD 0x46
+#define MFC_GETLB_CMD 0x45
+
+#define MFC_SDCRT_CMD 0x80
+#define MFC_SDCRTST_CMD 0x81
+#define MFC_SDCRZ_CMD 0x89
+#define MFC_SDCRS_CMD 0x8D
+#define MFC_SDCRF_CMD 0x8F
+
+#define MFC_GETLLAR_CMD 0xD0
+#define MFC_PUTLLC_CMD 0xB4
+#define MFC_PUTLLUC_CMD 0xB0
+#define MFC_PUTQLLUC_CMD 0xB8
+#define MFC_SNDSIG_CMD 0xA0
+#define MFC_SNDSIGB_CMD 0xA1
+#define MFC_SNDSIGF_CMD 0xA2
+#define MFC_BARRIER_CMD 0xC0
+#define MFC_EIEIO_CMD 0xC8
+#define MFC_SYNC_CMD 0xCC
+
+#define MFC_MIN_DMA_SIZE_SHIFT 4 /* 16 bytes */
+#define MFC_MAX_DMA_SIZE_SHIFT 14 /* 16384 bytes */
+#define MFC_MIN_DMA_SIZE (1 << MFC_MIN_DMA_SIZE_SHIFT)
+#define MFC_MAX_DMA_SIZE (1 << MFC_MAX_DMA_SIZE_SHIFT)
+#define MFC_MIN_DMA_SIZE_MASK (MFC_MIN_DMA_SIZE - 1)
+#define MFC_MAX_DMA_SIZE_MASK (MFC_MAX_DMA_SIZE - 1)
+#define MFC_MIN_DMA_LIST_SIZE 0x0008 /* 8 bytes */
+#define MFC_MAX_DMA_LIST_SIZE 0x4000 /* 16K bytes */
+
+#define MFC_TAGID_TO_TAGMASK(tag_id) (1 << (tag_id & 0x1F))
+
+/* Events for Channels 0-2 */
+#define MFC_DMA_TAG_STATUS_UPDATE_EVENT 0x00000001
+#define MFC_DMA_TAG_CMD_STALL_NOTIFY_EVENT 0x00000002
+#define MFC_DMA_QUEUE_AVAILABLE_EVENT 0x00000008
+#define MFC_SPU_MAILBOX_WRITTEN_EVENT 0x00000010
+#define MFC_DECREMENTER_EVENT 0x00000020
+#define MFC_PU_INT_MAILBOX_AVAILABLE_EVENT 0x00000040
+#define MFC_PU_MAILBOX_AVAILABLE_EVENT 0x00000080
+#define MFC_SIGNAL_2_EVENT 0x00000100
+#define MFC_SIGNAL_1_EVENT 0x00000200
+#define MFC_LLR_LOST_EVENT 0x00000400
+#define MFC_PRIV_ATTN_EVENT 0x00000800
+#define MFC_MULTI_SRC_EVENT 0x00001000
+
+/* Flags indicating progress during context switch. */
+#define SPU_CONTEXT_SWITCH_PENDING_nr 0UL
+#define SPU_CONTEXT_SWITCH_ACTIVE_nr 1UL
+#define SPU_CONTEXT_SWITCH_PENDING (1UL << SPU_CONTEXT_SWITCH_PENDING_nr)
+#define SPU_CONTEXT_SWITCH_ACTIVE (1UL << SPU_CONTEXT_SWITCH_ACTIVE_nr)
struct spu {
char *name;
@@ -41,6 +116,7 @@ struct spu {
int number;
u32 isrc;
u32 node;
+ u64 flags;
struct kref kref;
size_t ls_size;
unsigned int slb_replace;
Index: linux-cg/include/asm-powerpc/spu_csa.h
===================================================================
--- /dev/null
+++ linux-cg/include/asm-powerpc/spu_csa.h
@@ -0,0 +1,256 @@
+/*
+ * spu_csa.h: Definitions for SPU context save area (CSA).
+ *
+ * (C) Copyright IBM 2005
+ *
+ * Author: Mark Nutter <mnutter@us.ibm.com>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2, or (at your option)
+ * any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ */
+
+#ifndef _SPU_CSA_H_
+#define _SPU_CSA_H_
+
+/*
+ * Total number of 128-bit registers.
+ */
+#define NR_SPU_GPRS 128
+#define NR_SPU_SPRS 9
+#define NR_SPU_REGS_PAD 7
+#define NR_SPU_SPILL_REGS 144 /* GPRS + SPRS + PAD */
+#define SIZEOF_SPU_SPILL_REGS NR_SPU_SPILL_REGS * 16
+
+#define SPU_SAVE_COMPLETE 0x3FFB
+#define SPU_RESTORE_COMPLETE 0x3FFC
+
+/*
+ * Definitions for various 'stopped' status conditions,
+ * to be recreated during context restore.
+ */
+#define SPU_STOPPED_STATUS_P 1
+#define SPU_STOPPED_STATUS_I 2
+#define SPU_STOPPED_STATUS_H 3
+#define SPU_STOPPED_STATUS_S 4
+#define SPU_STOPPED_STATUS_S_I 5
+#define SPU_STOPPED_STATUS_S_P 6
+#define SPU_STOPPED_STATUS_P_H 7
+#define SPU_STOPPED_STATUS_P_I 8
+#define SPU_STOPPED_STATUS_R 9
+
+#ifndef __ASSEMBLY__
+/**
+ * spu_reg128 - generic 128-bit register definition.
+ */
+struct spu_reg128 {
+ u32 slot[4];
+};
+
+/**
+ * struct spu_lscsa - Local Store Context Save Area.
+ * @gprs: Array of saved registers.
+ * @fpcr: Saved floating point status control register.
+ * @decr: Saved decrementer value.
+ * @decr_status: Indicates decrementer run status.
+ * @ppu_mb: Saved PPU mailbox data.
+ * @ppuint_mb: Saved PPU interrupting mailbox data.
+ * @tag_mask: Saved tag group mask.
+ * @event_mask: Saved event mask.
+ * @srr0: Saved SRR0.
+ * @stopped_status: Conditions to be recreated by restore.
+ * @ls: Saved contents of Local Storage Area.
+ *
+ * The LSCSA represents state that is primarily saved and
+ * restored by SPU-side code.
+ */
+struct spu_lscsa {
+ struct spu_reg128 gprs[128];
+ struct spu_reg128 fpcr;
+ struct spu_reg128 decr;
+ struct spu_reg128 decr_status;
+ struct spu_reg128 ppu_mb;
+ struct spu_reg128 ppuint_mb;
+ struct spu_reg128 tag_mask;
+ struct spu_reg128 event_mask;
+ struct spu_reg128 srr0;
+ struct spu_reg128 stopped_status;
+ struct spu_reg128 pad[119]; /* 'ls' must be page-aligned. */
+ unsigned char ls[LS_SIZE];
+};
+
+#ifdef __KERNEL__
+
+/*
+ * struct spu_problem_collapsed - condensed problem state area, w/o pads.
+ */
+struct spu_problem_collapsed {
+ u64 spc_mssync_RW;
+ u32 mfc_lsa_W;
+ u32 unused_pad0;
+ u64 mfc_ea_W;
+ union mfc_tag_size_class_cmd mfc_union_W;
+ u32 dma_qstatus_R;
+ u32 dma_querytype_RW;
+ u32 dma_querymask_RW;
+ u32 dma_tagstatus_R;
+ u32 pu_mb_R;
+ u32 spu_mb_W;
+ u32 mb_stat_R;
+ u32 spu_runcntl_RW;
+ u32 spu_status_R;
+ u32 spu_spc_R;
+ u32 spu_npc_RW;
+ u32 signal_notify1;
+ u32 signal_notify2;
+ u32 unused_pad1;
+};
+
+/*
+ * struct spu_priv1_collapsed - condensed privileged 1 area, w/o pads.
+ */
+struct spu_priv1_collapsed {
+ u64 mfc_sr1_RW;
+ u64 mfc_lpid_RW;
+ u64 spu_idr_RW;
+ u64 mfc_vr_RO;
+ u64 spu_vr_RO;
+ u64 int_mask_class0_RW;
+ u64 int_mask_class1_RW;
+ u64 int_mask_class2_RW;
+ u64 int_stat_class0_RW;
+ u64 int_stat_class1_RW;
+ u64 int_stat_class2_RW;
+ u64 int_route_RW;
+ u64 mfc_atomic_flush_RW;
+ u64 resource_allocation_groupID_RW;
+ u64 resource_allocation_enable_RW;
+ u64 mfc_fir_R;
+ u64 mfc_fir_status_or_W;
+ u64 mfc_fir_status_and_W;
+ u64 mfc_fir_mask_R;
+ u64 mfc_fir_mask_or_W;
+ u64 mfc_fir_mask_and_W;
+ u64 mfc_fir_chkstp_enable_RW;
+ u64 smf_sbi_signal_sel;
+ u64 smf_ato_signal_sel;
+ u64 mfc_sdr_RW;
+ u64 tlb_index_hint_RO;
+ u64 tlb_index_W;
+ u64 tlb_vpn_RW;
+ u64 tlb_rpn_RW;
+ u64 tlb_invalidate_entry_W;
+ u64 tlb_invalidate_all_W;
+ u64 smm_hid;
+ u64 mfc_accr_RW;
+ u64 mfc_dsisr_RW;
+ u64 mfc_dar_RW;
+ u64 rmt_index_RW;
+ u64 rmt_data1_RW;
+ u64 mfc_dsir_R;
+ u64 mfc_lsacr_RW;
+ u64 mfc_lscrr_R;
+ u64 mfc_tclass_id_RW;
+ u64 mfc_rm_boundary;
+ u64 smf_dma_signal_sel;
+ u64 smm_signal_sel;
+ u64 mfc_cer_R;
+ u64 pu_ecc_cntl_RW;
+ u64 pu_ecc_stat_RW;
+ u64 spu_ecc_addr_RW;
+ u64 spu_err_mask_RW;
+ u64 spu_trig0_sel;
+ u64 spu_trig1_sel;
+ u64 spu_trig2_sel;
+ u64 spu_trig3_sel;
+ u64 spu_trace_sel;
+ u64 spu_event0_sel;
+ u64 spu_event1_sel;
+ u64 spu_event2_sel;
+ u64 spu_event3_sel;
+ u64 spu_trace_cntl;
+};
+
+/*
+ * struct spu_priv2_collapsed - condensed priviliged 2 area, w/o pads.
+ */
+struct spu_priv2_collapsed {
+ u64 slb_index_W;
+ u64 slb_esid_RW;
+ u64 slb_vsid_RW;
+ u64 slb_invalidate_entry_W;
+ u64 slb_invalidate_all_W;
+ struct mfc_cq_sr spuq[16];
+ struct mfc_cq_sr puq[8];
+ u64 mfc_control_RW;
+ u64 puint_mb_R;
+ u64 spu_privcntl_RW;
+ u64 spu_lslr_RW;
+ u64 spu_chnlcntptr_RW;
+ u64 spu_chnlcnt_RW;
+ u64 spu_chnldata_RW;
+ u64 spu_cfg_RW;
+ u64 spu_pm_trace_tag_status_RW;
+ u64 spu_tag_status_query_RW;
+ u64 spu_cmd_buf1_RW;
+ u64 spu_cmd_buf2_RW;
+ u64 spu_atomic_status_RW;
+};
+
+/**
+ * struct spu_state
+ * @lscsa: Local Store Context Save Area.
+ * @prob: Collapsed Problem State Area, w/o pads.
+ * @priv1: Collapsed Privileged 1 Area, w/o pads.
+ * @priv2: Collapsed Privileged 2 Area, w/o pads.
+ * @spu_chnlcnt_RW: Array of saved channel counts.
+ * @spu_chnldata_RW: Array of saved channel data.
+ * @suspend_time: Time stamp when decrementer disabled.
+ * @slb_esid_RW: Array of saved SLB esid entries.
+ * @slb_vsid_RW: Array of saved SLB vsid entries.
+ *
+ * Structure representing the whole of the SPU
+ * context save area (CSA). This struct contains
+ * all of the state necessary to suspend and then
+ * later optionally resume execution of an SPU
+ * context.
+ *
+ * The @lscsa region is by far the largest, and is
+ * allocated separately so that it may either be
+ * pinned or mapped to/from application memory, as
+ * appropriate for the OS environment.
+ */
+struct spu_state {
+ struct spu_lscsa *lscsa;
+ struct spu_problem_collapsed prob;
+ struct spu_priv1_collapsed priv1;
+ struct spu_priv2_collapsed priv2;
+ u64 spu_chnlcnt_RW[32];
+ u64 spu_chnldata_RW[32];
+ u32 spu_mailbox_data[4];
+ u32 pu_mailbox_data[1];
+ unsigned long suspend_time;
+ u64 slb_esid_RW[8];
+ u64 slb_vsid_RW[8];
+};
+
+extern void spu_init_csa(struct spu_state *csa);
+extern void spu_fini_csa(struct spu_state *csa);
+extern int spu_save(struct spu_state *prev, struct spu *spu);
+extern int spu_restore(struct spu_state *new, struct spu *spu);
+extern int spu_switch(struct spu_state *prev, struct spu_state *new,
+ struct spu *spu);
+
+#endif /* __KERNEL__ */
+#endif /* !__ASSEMBLY__ */
+#endif /* _SPU_CSA_H_ */
--
^ permalink raw reply [flat|nested] 10+ messages in thread* [PATCH 3/5] kernel-side context switch code for spufs
2005-11-15 20:53 [PATCH 0/5] SPU file system for 2.6.15-rc-mm Arnd Bergmann
2005-11-15 20:53 ` [PATCH 2/5] spufs: switchable spu contexts Arnd Bergmann
@ 2005-11-15 20:53 ` Arnd Bergmann
2005-11-15 20:53 ` [PATCH 4/5] spufs: add spu-side context switch code Arnd Bergmann
` (2 subsequent siblings)
4 siblings, 0 replies; 10+ messages in thread
From: Arnd Bergmann @ 2005-11-15 20:53 UTC (permalink / raw)
To: akpm; +Cc: Paul Mackerras, linuxppc64-dev, linux-kernel, mnutter,
Arnd Bergmann
[-- Attachment #1: spufs-context-part2-4.diff --]
[-- Type: text/plain, Size: 62635 bytes --]
This adds the code needed to perform a context switch from
spufs, following the recommended 76-step sequence.
From: Mark Nutter <mnutter@us.ibm.com>
Signed-off-by: Arnd Bergmann <arndb@de.ibm.com>
---
switch.c | 2052 ++++++++++++++++++++++++++++++++++++++++++++++++++++++-
1 files changed, 2046 insertions(+), 6 deletions(-)
Index: linux-cg/arch/powerpc/platforms/cell/spufs/switch.c
===================================================================
--- linux-cg.orig/arch/powerpc/platforms/cell/spufs/switch.c
+++ linux-cg/arch/powerpc/platforms/cell/spufs/switch.c
@@ -52,6 +52,2019 @@
#include "spu_save_dump.h"
#include "spu_restore_dump.h"
+#if 0
+#define POLL_WHILE_TRUE(_c) { \
+ do { \
+ } while (_c); \
+ }
+#else
+#define RELAX_SPIN_COUNT 1000
+#define POLL_WHILE_TRUE(_c) { \
+ do { \
+ int _i; \
+ for (_i=0; _i<RELAX_SPIN_COUNT && (_c); _i++) { \
+ cpu_relax(); \
+ } \
+ if (unlikely(_c)) yield(); \
+ else break; \
+ } while (_c); \
+ }
+#endif /* debug */
+
+#define POLL_WHILE_FALSE(_c) POLL_WHILE_TRUE(!(_c))
+
+static inline void acquire_spu_lock(struct spu *spu)
+{
+ /* Save, Step 1:
+ * Restore, Step 1:
+ * Acquire SPU-specific mutual exclusion lock.
+ * TBD.
+ */
+}
+
+static inline void release_spu_lock(struct spu *spu)
+{
+ /* Restore, Step 76:
+ * Release SPU-specific mutual exclusion lock.
+ * TBD.
+ */
+}
+
+static inline int check_spu_isolate(struct spu_state *csa, struct spu *spu)
+{
+ struct spu_problem __iomem *prob = spu->problem;
+ u32 isolate_state;
+
+ /* Save, Step 2:
+ * Save, Step 6:
+ * If SPU_Status[E,L,IS] any field is '1', this
+ * SPU is in isolate state and cannot be context
+ * saved at this time.
+ */
+ isolate_state = SPU_STATUS_ISOLATED_STATE |
+ SPU_STATUS_ISOLATED_LOAD_STAUTUS | SPU_STATUS_ISOLATED_EXIT_STAUTUS;
+ return (in_be32(&prob->spu_status_R) & isolate_state) ? 1 : 0;
+}
+
+static inline void disable_interrupts(struct spu_state *csa, struct spu *spu)
+{
+ struct spu_priv1 __iomem *priv1 = spu->priv1;
+
+ /* Save, Step 3:
+ * Restore, Step 2:
+ * Save INT_Mask_class0 in CSA.
+ * Write INT_MASK_class0 with value of 0.
+ * Save INT_Mask_class1 in CSA.
+ * Write INT_MASK_class1 with value of 0.
+ * Save INT_Mask_class2 in CSA.
+ * Write INT_MASK_class2 with value of 0.
+ */
+ spin_lock_irq(&spu->register_lock);
+ if (csa) {
+ csa->priv1.int_mask_class0_RW =
+ in_be64(&priv1->int_mask_class0_RW);
+ csa->priv1.int_mask_class1_RW =
+ in_be64(&priv1->int_mask_class1_RW);
+ csa->priv1.int_mask_class2_RW =
+ in_be64(&priv1->int_mask_class2_RW);
+ }
+ out_be64(&priv1->int_mask_class0_RW, 0UL);
+ out_be64(&priv1->int_mask_class1_RW, 0UL);
+ out_be64(&priv1->int_mask_class2_RW, 0UL);
+ eieio();
+ spin_unlock_irq(&spu->register_lock);
+}
+
+static inline void set_watchdog_timer(struct spu_state *csa, struct spu *spu)
+{
+ /* Save, Step 4:
+ * Restore, Step 25.
+ * Set a software watchdog timer, which specifies the
+ * maximum allowable time for a context save sequence.
+ *
+ * For present, this implementation will not set a global
+ * watchdog timer, as virtualization & variable system load
+ * may cause unpredictable execution times.
+ */
+}
+
+static inline void inhibit_user_access(struct spu_state *csa, struct spu *spu)
+{
+ /* Save, Step 5:
+ * Restore, Step 3:
+ * Inhibit user-space access (if provided) to this
+ * SPU by unmapping the virtual pages assigned to
+ * the SPU memory-mapped I/O (MMIO) for problem
+ * state. TBD.
+ */
+}
+
+static inline void set_switch_pending(struct spu_state *csa, struct spu *spu)
+{
+ /* Save, Step 7:
+ * Restore, Step 5:
+ * Set a software context switch pending flag.
+ */
+ set_bit(SPU_CONTEXT_SWITCH_PENDING_nr, &spu->flags);
+ mb();
+}
+
+static inline void save_mfc_cntl(struct spu_state *csa, struct spu *spu)
+{
+ struct spu_priv2 __iomem *priv2 = spu->priv2;
+
+ /* Save, Step 8:
+ * Read and save MFC_CNTL[Ss].
+ */
+ if (csa) {
+ csa->priv2.mfc_control_RW = in_be64(&priv2->mfc_control_RW) &
+ MFC_CNTL_SUSPEND_DMA_STATUS_MASK;
+ }
+}
+
+static inline void save_spu_runcntl(struct spu_state *csa, struct spu *spu)
+{
+ struct spu_problem __iomem *prob = spu->problem;
+
+ /* Save, Step 9:
+ * Save SPU_Runcntl in the CSA. This value contains
+ * the "Application Desired State".
+ */
+ csa->prob.spu_runcntl_RW = in_be32(&prob->spu_runcntl_RW);
+}
+
+static inline void save_mfc_sr1(struct spu_state *csa, struct spu *spu)
+{
+ struct spu_priv1 __iomem *priv1 = spu->priv1;
+
+ /* Save, Step 10:
+ * Save MFC_SR1 in the CSA.
+ */
+ csa->priv1.mfc_sr1_RW = in_be64(&priv1->mfc_sr1_RW);
+}
+
+static inline void save_spu_status(struct spu_state *csa, struct spu *spu)
+{
+ struct spu_problem __iomem *prob = spu->problem;
+
+ /* Save, Step 11:
+ * Read SPU_Status[R], and save to CSA.
+ */
+ if ((in_be32(&prob->spu_status_R) & SPU_STATUS_RUNNING) == 0) {
+ csa->prob.spu_status_R = in_be32(&prob->spu_status_R);
+ } else {
+ u32 stopped;
+
+ out_be32(&prob->spu_runcntl_RW, SPU_RUNCNTL_STOP);
+ eieio();
+ POLL_WHILE_TRUE(in_be32(&prob->spu_status_R) &
+ SPU_STATUS_RUNNING);
+ stopped =
+ SPU_STATUS_INVALID_INSTR | SPU_STATUS_SINGLE_STEP |
+ SPU_STATUS_STOPPED_BY_HALT | SPU_STATUS_STOPPED_BY_STOP;
+ if ((in_be32(&prob->spu_status_R) & stopped) == 0)
+ csa->prob.spu_status_R = SPU_STATUS_RUNNING;
+ else
+ csa->prob.spu_status_R = in_be32(&prob->spu_status_R);
+ }
+}
+
+static inline void save_mfc_decr(struct spu_state *csa, struct spu *spu)
+{
+ struct spu_priv2 __iomem *priv2 = spu->priv2;
+
+ /* Save, Step 12:
+ * Read MFC_CNTL[Ds]. Update saved copy of
+ * CSA.MFC_CNTL[Ds].
+ */
+ if (in_be64(&priv2->mfc_control_RW) & MFC_CNTL_DECREMENTER_RUNNING) {
+ csa->priv2.mfc_control_RW |= MFC_CNTL_DECREMENTER_RUNNING;
+ csa->suspend_time = get_cycles();
+ out_be64(&priv2->spu_chnlcntptr_RW, 7ULL);
+ eieio();
+ csa->spu_chnldata_RW[7] = in_be64(&priv2->spu_chnldata_RW);
+ eieio();
+ }
+}
+
+static inline void halt_mfc_decr(struct spu_state *csa, struct spu *spu)
+{
+ struct spu_priv2 __iomem *priv2 = spu->priv2;
+
+ /* Save, Step 13:
+ * Write MFC_CNTL[Dh] set to a '1' to halt
+ * the decrementer.
+ */
+ out_be64(&priv2->mfc_control_RW, MFC_CNTL_DECREMENTER_HALTED);
+ eieio();
+}
+
+static inline void save_timebase(struct spu_state *csa, struct spu *spu)
+{
+ /* Save, Step 14:
+ * Read PPE Timebase High and Timebase low registers
+ * and save in CSA. TBD.
+ */
+ csa->suspend_time = get_cycles();
+}
+
+static inline void remove_other_spu_access(struct spu_state *csa,
+ struct spu *spu)
+{
+ /* Save, Step 15:
+ * Remove other SPU access to this SPU by unmapping
+ * this SPU's pages from their address space. TBD.
+ */
+}
+
+static inline void do_mfc_mssync(struct spu_state *csa, struct spu *spu)
+{
+ struct spu_problem __iomem *prob = spu->problem;
+
+ /* Save, Step 16:
+ * Restore, Step 11.
+ * Write SPU_MSSync register. Poll SPU_MSSync[P]
+ * for a value of 0.
+ */
+ out_be64(&prob->spc_mssync_RW, 1UL);
+ POLL_WHILE_TRUE(in_be64(&prob->spc_mssync_RW) & MS_SYNC_PENDING);
+}
+
+static inline void issue_mfc_tlbie(struct spu_state *csa, struct spu *spu)
+{
+ struct spu_priv1 __iomem *priv1 = spu->priv1;
+
+ /* Save, Step 17:
+ * Restore, Step 12.
+ * Restore, Step 48.
+ * Write TLB_Invalidate_Entry[IS,VPN,L,Lp]=0 register.
+ * Then issue a PPE sync instruction.
+ */
+ out_be64(&priv1->tlb_invalidate_entry_W, 0UL);
+ mb();
+}
+
+static inline void handle_pending_interrupts(struct spu_state *csa,
+ struct spu *spu)
+{
+ /* Save, Step 18:
+ * Handle any pending interrupts from this SPU
+ * here. This is OS or hypervisor specific. One
+ * option is to re-enable interrupts to handle any
+ * pending interrupts, with the interrupt handlers
+ * recognizing the software Context Switch Pending
+ * flag, to ensure the SPU execution or MFC command
+ * queue is not restarted. TBD.
+ */
+}
+
+static inline void save_mfc_queues(struct spu_state *csa, struct spu *spu)
+{
+ struct spu_priv2 __iomem *priv2 = spu->priv2;
+ int i;
+
+ /* Save, Step 19:
+ * If MFC_Cntl[Se]=0 then save
+ * MFC command queues.
+ */
+ if ((in_be64(&priv2->mfc_control_RW) & MFC_CNTL_DMA_QUEUES_EMPTY) == 0) {
+ for (i = 0; i < 8; i++) {
+ csa->priv2.puq[i].mfc_cq_data0_RW =
+ in_be64(&priv2->puq[i].mfc_cq_data0_RW);
+ csa->priv2.puq[i].mfc_cq_data1_RW =
+ in_be64(&priv2->puq[i].mfc_cq_data1_RW);
+ csa->priv2.puq[i].mfc_cq_data2_RW =
+ in_be64(&priv2->puq[i].mfc_cq_data2_RW);
+ csa->priv2.puq[i].mfc_cq_data3_RW =
+ in_be64(&priv2->puq[i].mfc_cq_data3_RW);
+ }
+ for (i = 0; i < 16; i++) {
+ csa->priv2.spuq[i].mfc_cq_data0_RW =
+ in_be64(&priv2->spuq[i].mfc_cq_data0_RW);
+ csa->priv2.spuq[i].mfc_cq_data1_RW =
+ in_be64(&priv2->spuq[i].mfc_cq_data1_RW);
+ csa->priv2.spuq[i].mfc_cq_data2_RW =
+ in_be64(&priv2->spuq[i].mfc_cq_data2_RW);
+ csa->priv2.spuq[i].mfc_cq_data3_RW =
+ in_be64(&priv2->spuq[i].mfc_cq_data3_RW);
+ }
+ }
+}
+
+static inline void save_ppu_querymask(struct spu_state *csa, struct spu *spu)
+{
+ struct spu_problem __iomem *prob = spu->problem;
+
+ /* Save, Step 20:
+ * Save the PPU_QueryMask register
+ * in the CSA.
+ */
+ csa->prob.dma_querymask_RW = in_be32(&prob->dma_querymask_RW);
+}
+
+static inline void save_ppu_querytype(struct spu_state *csa, struct spu *spu)
+{
+ struct spu_problem __iomem *prob = spu->problem;
+
+ /* Save, Step 21:
+ * Save the PPU_QueryType register
+ * in the CSA.
+ */
+ csa->prob.dma_querytype_RW = in_be32(&prob->dma_querytype_RW);
+}
+
+static inline void save_mfc_csr_tsq(struct spu_state *csa, struct spu *spu)
+{
+ struct spu_priv2 __iomem *priv2 = spu->priv2;
+
+ /* Save, Step 22:
+ * Save the MFC_CSR_TSQ register
+ * in the LSCSA.
+ */
+ csa->priv2.spu_tag_status_query_RW =
+ in_be64(&priv2->spu_tag_status_query_RW);
+}
+
+static inline void save_mfc_csr_cmd(struct spu_state *csa, struct spu *spu)
+{
+ struct spu_priv2 __iomem *priv2 = spu->priv2;
+
+ /* Save, Step 23:
+ * Save the MFC_CSR_CMD1 and MFC_CSR_CMD2
+ * registers in the CSA.
+ */
+ csa->priv2.spu_cmd_buf1_RW = in_be64(&priv2->spu_cmd_buf1_RW);
+ csa->priv2.spu_cmd_buf2_RW = in_be64(&priv2->spu_cmd_buf2_RW);
+}
+
+static inline void save_mfc_csr_ato(struct spu_state *csa, struct spu *spu)
+{
+ struct spu_priv2 __iomem *priv2 = spu->priv2;
+
+ /* Save, Step 24:
+ * Save the MFC_CSR_ATO register in
+ * the CSA.
+ */
+ csa->priv2.spu_atomic_status_RW = in_be64(&priv2->spu_atomic_status_RW);
+}
+
+static inline void save_mfc_tclass_id(struct spu_state *csa, struct spu *spu)
+{
+ struct spu_priv1 __iomem *priv1 = spu->priv1;
+
+ /* Save, Step 25:
+ * Save the MFC_TCLASS_ID register in
+ * the CSA.
+ */
+ csa->priv1.mfc_tclass_id_RW = in_be64(&priv1->mfc_tclass_id_RW);
+}
+
+static inline void set_mfc_tclass_id(struct spu_state *csa, struct spu *spu)
+{
+ struct spu_priv1 __iomem *priv1 = spu->priv1;
+
+ /* Save, Step 26:
+ * Restore, Step 23.
+ * Write the MFC_TCLASS_ID register with
+ * the value 0x10000000.
+ */
+ out_be64(&priv1->mfc_tclass_id_RW, 0x10000000);
+ eieio();
+}
+
+static inline void purge_mfc_queue(struct spu_state *csa, struct spu *spu)
+{
+ struct spu_priv2 __iomem *priv2 = spu->priv2;
+
+ /* Save, Step 27:
+ * Restore, Step 14.
+ * Write MFC_CNTL[Pc]=1 (purge queue).
+ */
+ out_be64(&priv2->mfc_control_RW, MFC_CNTL_PURGE_DMA_REQUEST);
+ eieio();
+}
+
+static inline void wait_purge_complete(struct spu_state *csa, struct spu *spu)
+{
+ struct spu_priv2 __iomem *priv2 = spu->priv2;
+
+ /* Save, Step 28:
+ * Poll MFC_CNTL[Ps] until value '11' is read
+ * (purge complete).
+ */
+ POLL_WHILE_FALSE(in_be64(&priv2->mfc_control_RW) &
+ MFC_CNTL_PURGE_DMA_COMPLETE);
+}
+
+static inline void save_mfc_slbs(struct spu_state *csa, struct spu *spu)
+{
+ struct spu_priv1 __iomem *priv1 = spu->priv1;
+ struct spu_priv2 __iomem *priv2 = spu->priv2;
+ int i;
+
+ /* Save, Step 29:
+ * If MFC_SR1[R]='1', save SLBs in CSA.
+ */
+ if (in_be64(&priv1->mfc_sr1_RW) & MFC_STATE1_RELOCATE_MASK) {
+ csa->priv2.slb_index_W = in_be64(&priv2->slb_index_W);
+ for (i = 0; i < 8; i++) {
+ out_be64(&priv2->slb_index_W, i);
+ eieio();
+ csa->slb_esid_RW[i] = in_be64(&priv2->slb_esid_RW);
+ csa->slb_vsid_RW[i] = in_be64(&priv2->slb_vsid_RW);
+ eieio();
+ }
+ }
+}
+
+static inline void setup_mfc_sr1(struct spu_state *csa, struct spu *spu)
+{
+ struct spu_priv1 __iomem *priv1 = spu->priv1;
+
+ /* Save, Step 30:
+ * Restore, Step 18:
+ * Write MFC_SR1 with MFC_SR1[D=0,S=1] and
+ * MFC_SR1[TL,R,Pr,T] set correctly for the
+ * OS specific environment.
+ *
+ * Implementation note: The SPU-side code
+ * for save/restore is privileged, so the
+ * MFC_SR1[Pr] bit is not set.
+ *
+ */
+ out_be64(&priv1->mfc_sr1_RW, (MFC_STATE1_MASTER_RUN_CONTROL_MASK |
+ MFC_STATE1_RELOCATE_MASK |
+ MFC_STATE1_BUS_TLBIE_MASK));
+}
+
+static inline void save_spu_npc(struct spu_state *csa, struct spu *spu)
+{
+ struct spu_problem __iomem *prob = spu->problem;
+
+ /* Save, Step 31:
+ * Save SPU_NPC in the CSA.
+ */
+ csa->prob.spu_npc_RW = in_be32(&prob->spu_npc_RW);
+}
+
+static inline void save_spu_privcntl(struct spu_state *csa, struct spu *spu)
+{
+ struct spu_priv2 __iomem *priv2 = spu->priv2;
+
+ /* Save, Step 32:
+ * Save SPU_PrivCntl in the CSA.
+ */
+ csa->priv2.spu_privcntl_RW = in_be64(&priv2->spu_privcntl_RW);
+}
+
+static inline void reset_spu_privcntl(struct spu_state *csa, struct spu *spu)
+{
+ struct spu_priv2 __iomem *priv2 = spu->priv2;
+
+ /* Save, Step 33:
+ * Restore, Step 16:
+ * Write SPU_PrivCntl[S,Le,A] fields reset to 0.
+ */
+ out_be64(&priv2->spu_privcntl_RW, 0UL);
+ eieio();
+}
+
+static inline void save_spu_lslr(struct spu_state *csa, struct spu *spu)
+{
+ struct spu_priv2 __iomem *priv2 = spu->priv2;
+
+ /* Save, Step 34:
+ * Save SPU_LSLR in the CSA.
+ */
+ csa->priv2.spu_lslr_RW = in_be64(&priv2->spu_lslr_RW);
+}
+
+static inline void reset_spu_lslr(struct spu_state *csa, struct spu *spu)
+{
+ struct spu_priv2 __iomem *priv2 = spu->priv2;
+
+ /* Save, Step 35:
+ * Restore, Step 17.
+ * Reset SPU_LSLR.
+ */
+ out_be64(&priv2->spu_lslr_RW, LS_ADDR_MASK);
+ eieio();
+}
+
+static inline void save_spu_cfg(struct spu_state *csa, struct spu *spu)
+{
+ struct spu_priv2 __iomem *priv2 = spu->priv2;
+
+ /* Save, Step 36:
+ * Save SPU_Cfg in the CSA.
+ */
+ csa->priv2.spu_cfg_RW = in_be64(&priv2->spu_cfg_RW);
+}
+
+static inline void save_pm_trace(struct spu_state *csa, struct spu *spu)
+{
+ /* Save, Step 37:
+ * Save PM_Trace_Tag_Wait_Mask in the CSA.
+ * Not performed by this implementation.
+ */
+}
+
+static inline void save_mfc_rag(struct spu_state *csa, struct spu *spu)
+{
+ struct spu_priv1 __iomem *priv1 = spu->priv1;
+
+ /* Save, Step 38:
+ * Save RA_GROUP_ID register and the
+ * RA_ENABLE reigster in the CSA.
+ */
+ csa->priv1.resource_allocation_groupID_RW =
+ in_be64(&priv1->resource_allocation_groupID_RW);
+ csa->priv1.resource_allocation_enable_RW =
+ in_be64(&priv1->resource_allocation_enable_RW);
+}
+
+static inline void save_ppu_mb_stat(struct spu_state *csa, struct spu *spu)
+{
+ struct spu_problem __iomem *prob = spu->problem;
+
+ /* Save, Step 39:
+ * Save MB_Stat register in the CSA.
+ */
+ csa->prob.mb_stat_R = in_be32(&prob->mb_stat_R);
+}
+
+static inline void save_ppu_mb(struct spu_state *csa, struct spu *spu)
+{
+ struct spu_problem __iomem *prob = spu->problem;
+
+ /* Save, Step 40:
+ * Save the PPU_MB register in the CSA.
+ */
+ csa->prob.pu_mb_R = in_be32(&prob->pu_mb_R);
+}
+
+static inline void save_ppuint_mb(struct spu_state *csa, struct spu *spu)
+{
+ struct spu_priv2 __iomem *priv2 = spu->priv2;
+
+ /* Save, Step 41:
+ * Save the PPUINT_MB register in the CSA.
+ */
+ csa->priv2.puint_mb_R = in_be64(&priv2->puint_mb_R);
+}
+
+static inline void save_ch_part1(struct spu_state *csa, struct spu *spu)
+{
+ struct spu_priv2 __iomem *priv2 = spu->priv2;
+ u64 idx, ch_indices[7] = { 0UL, 1UL, 3UL, 4UL, 24UL, 25UL, 27UL };
+ int i;
+
+ /* Save, Step 42:
+ * Save the following CH: [0,1,3,4,24,25,27]
+ */
+ for (i = 0; i < 7; i++) {
+ idx = ch_indices[i];
+ out_be64(&priv2->spu_chnlcntptr_RW, idx);
+ eieio();
+ csa->spu_chnldata_RW[idx] = in_be64(&priv2->spu_chnldata_RW);
+ csa->spu_chnlcnt_RW[idx] = in_be64(&priv2->spu_chnlcnt_RW);
+ out_be64(&priv2->spu_chnldata_RW, 0UL);
+ out_be64(&priv2->spu_chnlcnt_RW, 0UL);
+ eieio();
+ }
+}
+
+static inline void save_spu_mb(struct spu_state *csa, struct spu *spu)
+{
+ struct spu_priv2 __iomem *priv2 = spu->priv2;
+ int i;
+
+ /* Save, Step 43:
+ * Save SPU Read Mailbox Channel.
+ */
+ out_be64(&priv2->spu_chnlcntptr_RW, 29UL);
+ eieio();
+ csa->spu_chnlcnt_RW[29] = in_be64(&priv2->spu_chnlcnt_RW);
+ for (i = 0; i < 4; i++) {
+ csa->pu_mailbox_data[i] = in_be64(&priv2->spu_chnldata_RW);
+ }
+ out_be64(&priv2->spu_chnlcnt_RW, 0UL);
+ eieio();
+}
+
+static inline void save_mfc_cmd(struct spu_state *csa, struct spu *spu)
+{
+ struct spu_priv2 __iomem *priv2 = spu->priv2;
+
+ /* Save, Step 44:
+ * Save MFC_CMD Channel.
+ */
+ out_be64(&priv2->spu_chnlcntptr_RW, 21UL);
+ eieio();
+ csa->spu_chnlcnt_RW[21] = in_be64(&priv2->spu_chnlcnt_RW);
+ eieio();
+}
+
+static inline void reset_ch(struct spu_state *csa, struct spu *spu)
+{
+ struct spu_priv2 __iomem *priv2 = spu->priv2;
+ u64 ch_indices[4] = { 21UL, 23UL, 28UL, 30UL };
+ u64 ch_counts[4] = { 16UL, 1UL, 1UL, 1UL };
+ u64 idx;
+ int i;
+
+ /* Save, Step 45:
+ * Reset the following CH: [21, 23, 28, 30]
+ */
+ for (i = 0; i < 4; i++) {
+ idx = ch_indices[i];
+ out_be64(&priv2->spu_chnlcntptr_RW, idx);
+ eieio();
+ out_be64(&priv2->spu_chnlcnt_RW, ch_counts[i]);
+ eieio();
+ }
+}
+
+static inline void resume_mfc_queue(struct spu_state *csa, struct spu *spu)
+{
+ struct spu_priv2 __iomem *priv2 = spu->priv2;
+
+ /* Save, Step 46:
+ * Restore, Step 25.
+ * Write MFC_CNTL[Sc]=0 (resume queue processing).
+ */
+ out_be64(&priv2->mfc_control_RW, MFC_CNTL_RESUME_DMA_QUEUE);
+}
+
+static inline void invalidate_slbs(struct spu_state *csa, struct spu *spu)
+{
+ struct spu_priv1 __iomem *priv1 = spu->priv1;
+ struct spu_priv2 __iomem *priv2 = spu->priv2;
+
+ /* Save, Step 45:
+ * Restore, Step 19:
+ * If MFC_SR1[R]=1, write 0 to SLB_Invalidate_All.
+ */
+ if (in_be64(&priv1->mfc_sr1_RW) & MFC_STATE1_RELOCATE_MASK) {
+ out_be64(&priv2->slb_invalidate_all_W, 0UL);
+ eieio();
+ }
+}
+
+static inline void get_kernel_slb(u64 ea, u64 slb[2])
+{
+ slb[0] = (get_kernel_vsid(ea) << SLB_VSID_SHIFT) | SLB_VSID_KERNEL;
+ slb[1] = (ea & ESID_MASK) | SLB_ESID_V;
+
+ /* Large pages are used for kernel text/data, but not vmalloc. */
+ if (cpu_has_feature(CPU_FTR_16M_PAGE)
+ && REGION_ID(ea) == KERNEL_REGION_ID)
+ slb[0] |= SLB_VSID_L;
+}
+
+static inline void load_mfc_slb(struct spu *spu, u64 slb[2], int slbe)
+{
+ struct spu_priv2 __iomem *priv2 = spu->priv2;
+
+ out_be64(&priv2->slb_index_W, slbe);
+ eieio();
+ out_be64(&priv2->slb_vsid_RW, slb[0]);
+ out_be64(&priv2->slb_esid_RW, slb[1]);
+ eieio();
+}
+
+static inline void setup_mfc_slbs(struct spu_state *csa, struct spu *spu)
+{
+ u64 code_slb[2];
+ u64 lscsa_slb[2];
+
+ /* Save, Step 47:
+ * Restore, Step 30.
+ * If MFC_SR1[R]=1, write 0 to SLB_Invalidate_All
+ * register, then initialize SLB_VSID and SLB_ESID
+ * to provide access to SPU context save code and
+ * LSCSA.
+ *
+ * This implementation places both the context
+ * switch code and LSCSA in kernel address space.
+ *
+ * Further this implementation assumes that the
+ * MFC_SR1[R]=1 (in other words, assume that
+ * translation is desired by OS environment).
+ */
+ invalidate_slbs(csa, spu);
+ get_kernel_slb((unsigned long)&spu_save_code[0], code_slb);
+ get_kernel_slb((unsigned long)csa->lscsa, lscsa_slb);
+ load_mfc_slb(spu, code_slb, 0);
+ if ((lscsa_slb[0] != code_slb[0]) || (lscsa_slb[1] != code_slb[1]))
+ load_mfc_slb(spu, lscsa_slb, 1);
+}
+
+static inline void set_switch_active(struct spu_state *csa, struct spu *spu)
+{
+ /* Save, Step 48:
+ * Restore, Step 23.
+ * Change the software context switch pending flag
+ * to context switch active.
+ */
+ set_bit(SPU_CONTEXT_SWITCH_ACTIVE_nr, &spu->flags);
+ clear_bit(SPU_CONTEXT_SWITCH_PENDING_nr, &spu->flags);
+ mb();
+}
+
+static inline void enable_interrupts(struct spu_state *csa, struct spu *spu)
+{
+ struct spu_priv1 __iomem *priv1 = spu->priv1;
+ unsigned long class1_mask = CLASS1_ENABLE_SEGMENT_FAULT_INTR |
+ CLASS1_ENABLE_STORAGE_FAULT_INTR;
+
+ /* Save, Step 49:
+ * Restore, Step 22:
+ * Reset and then enable interrupts, as
+ * needed by OS.
+ *
+ * This implementation enables only class1
+ * (translation) interrupts.
+ */
+ spin_lock_irq(&spu->register_lock);
+ out_be64(&priv1->int_stat_class0_RW, ~(0UL));
+ out_be64(&priv1->int_stat_class1_RW, ~(0UL));
+ out_be64(&priv1->int_stat_class2_RW, ~(0UL));
+ out_be64(&priv1->int_mask_class0_RW, 0UL);
+ out_be64(&priv1->int_mask_class1_RW, class1_mask);
+ out_be64(&priv1->int_mask_class2_RW, 0UL);
+ spin_unlock_irq(&spu->register_lock);
+}
+
+static inline int send_mfc_dma(struct spu *spu, unsigned long ea,
+ unsigned int ls_offset, unsigned int size,
+ unsigned int tag, unsigned int rclass,
+ unsigned int cmd)
+{
+ struct spu_problem __iomem *prob = spu->problem;
+ union mfc_tag_size_class_cmd command;
+ unsigned int transfer_size;
+ volatile unsigned int status = 0x0;
+
+ while (size > 0) {
+ transfer_size =
+ (size > MFC_MAX_DMA_SIZE) ? MFC_MAX_DMA_SIZE : size;
+ command.u.mfc_size = transfer_size;
+ command.u.mfc_tag = tag;
+ command.u.mfc_rclassid = rclass;
+ command.u.mfc_cmd = cmd;
+ do {
+ out_be32(&prob->mfc_lsa_W, ls_offset);
+ out_be64(&prob->mfc_ea_W, ea);
+ out_be64(&prob->mfc_union_W.all64, command.all64);
+ status =
+ in_be32(&prob->mfc_union_W.by32.mfc_class_cmd32);
+ if (unlikely(status & 0x2)) {
+ cpu_relax();
+ }
+ } while (status & 0x3);
+ size -= transfer_size;
+ ea += transfer_size;
+ ls_offset += transfer_size;
+ }
+ return 0;
+}
+
+static inline void save_ls_16kb(struct spu_state *csa, struct spu *spu)
+{
+ unsigned long addr = (unsigned long)&csa->lscsa->ls[0];
+ unsigned int ls_offset = 0x0;
+ unsigned int size = 16384;
+ unsigned int tag = 0;
+ unsigned int rclass = 0;
+ unsigned int cmd = MFC_PUT_CMD;
+
+ /* Save, Step 50:
+ * Issue a DMA command to copy the first 16K bytes
+ * of local storage to the CSA.
+ */
+ send_mfc_dma(spu, addr, ls_offset, size, tag, rclass, cmd);
+}
+
+static inline void set_spu_npc(struct spu_state *csa, struct spu *spu)
+{
+ struct spu_problem __iomem *prob = spu->problem;
+
+ /* Save, Step 51:
+ * Restore, Step 31.
+ * Write SPU_NPC[IE]=0 and SPU_NPC[LSA] to entry
+ * point address of context save code in local
+ * storage.
+ *
+ * This implementation uses SPU-side save/restore
+ * programs with entry points at LSA of 0.
+ */
+ out_be32(&prob->spu_npc_RW, 0);
+ eieio();
+}
+
+static inline void set_signot1(struct spu_state *csa, struct spu *spu)
+{
+ struct spu_problem __iomem *prob = spu->problem;
+ union {
+ u64 ull;
+ u32 ui[2];
+ } addr64;
+
+ /* Save, Step 52:
+ * Restore, Step 32:
+ * Write SPU_Sig_Notify_1 register with upper 32-bits
+ * of the CSA.LSCSA effective address.
+ */
+ addr64.ull = (u64) csa->lscsa;
+ out_be32(&prob->signal_notify1, addr64.ui[0]);
+ eieio();
+}
+
+static inline void set_signot2(struct spu_state *csa, struct spu *spu)
+{
+ struct spu_problem __iomem *prob = spu->problem;
+ union {
+ u64 ull;
+ u32 ui[2];
+ } addr64;
+
+ /* Save, Step 53:
+ * Restore, Step 33:
+ * Write SPU_Sig_Notify_2 register with lower 32-bits
+ * of the CSA.LSCSA effective address.
+ */
+ addr64.ull = (u64) csa->lscsa;
+ out_be32(&prob->signal_notify2, addr64.ui[1]);
+ eieio();
+}
+
+static inline void send_save_code(struct spu_state *csa, struct spu *spu)
+{
+ unsigned long addr = (unsigned long)&spu_save_code[0];
+ unsigned int ls_offset = 0x0;
+ unsigned int size = sizeof(spu_save_code);
+ unsigned int tag = 0;
+ unsigned int rclass = 0;
+ unsigned int cmd = MFC_GETFS_CMD;
+
+ /* Save, Step 54:
+ * Issue a DMA command to copy context save code
+ * to local storage and start SPU.
+ */
+ send_mfc_dma(spu, addr, ls_offset, size, tag, rclass, cmd);
+}
+
+static inline void set_ppu_querymask(struct spu_state *csa, struct spu *spu)
+{
+ struct spu_problem __iomem *prob = spu->problem;
+
+ /* Save, Step 55:
+ * Restore, Step 38.
+ * Write PPU_QueryMask=1 (enable Tag Group 0)
+ * and issue eieio instruction.
+ */
+ out_be32(&prob->dma_querymask_RW, MFC_TAGID_TO_TAGMASK(0));
+ eieio();
+}
+
+static inline void wait_tag_complete(struct spu_state *csa, struct spu *spu)
+{
+ struct spu_priv1 __iomem *priv1 = spu->priv1;
+ struct spu_problem __iomem *prob = spu->problem;
+ u32 mask = MFC_TAGID_TO_TAGMASK(0);
+ unsigned long flags;
+
+ /* Save, Step 56:
+ * Restore, Step 39.
+ * Restore, Step 39.
+ * Restore, Step 46.
+ * Poll PPU_TagStatus[gn] until 01 (Tag group 0 complete)
+ * or write PPU_QueryType[TS]=01 and wait for Tag Group
+ * Complete Interrupt. Write INT_Stat_Class0 or
+ * INT_Stat_Class2 with value of 'handled'.
+ */
+ POLL_WHILE_FALSE(in_be32(&prob->dma_tagstatus_R) & mask);
+
+ local_irq_save(flags);
+ out_be64(&priv1->int_stat_class0_RW, ~(0UL));
+ out_be64(&priv1->int_stat_class2_RW, ~(0UL));
+ local_irq_restore(flags);
+}
+
+static inline void wait_spu_stopped(struct spu_state *csa, struct spu *spu)
+{
+ struct spu_priv1 __iomem *priv1 = spu->priv1;
+ struct spu_problem __iomem *prob = spu->problem;
+ unsigned long flags;
+
+ /* Save, Step 57:
+ * Restore, Step 40.
+ * Poll until SPU_Status[R]=0 or wait for SPU Class 0
+ * or SPU Class 2 interrupt. Write INT_Stat_class0
+ * or INT_Stat_class2 with value of handled.
+ */
+ POLL_WHILE_TRUE(in_be32(&prob->spu_status_R) & SPU_STATUS_RUNNING);
+
+ local_irq_save(flags);
+ out_be64(&priv1->int_stat_class0_RW, ~(0UL));
+ out_be64(&priv1->int_stat_class2_RW, ~(0UL));
+ local_irq_restore(flags);
+}
+
+static inline int check_save_status(struct spu_state *csa, struct spu *spu)
+{
+ struct spu_problem __iomem *prob = spu->problem;
+ u32 complete;
+
+ /* Save, Step 54:
+ * If SPU_Status[P]=1 and SPU_Status[SC] = "success",
+ * context save succeeded, otherwise context save
+ * failed.
+ */
+ complete = ((SPU_SAVE_COMPLETE << SPU_STOP_STATUS_SHIFT) |
+ SPU_STATUS_STOPPED_BY_STOP);
+ return (in_be32(&prob->spu_status_R) != complete) ? 1 : 0;
+}
+
+static inline void terminate_spu_app(struct spu_state *csa, struct spu *spu)
+{
+ /* Restore, Step 4:
+ * If required, notify the "using application" that
+ * the SPU task has been terminated. TBD.
+ */
+}
+
+static inline void suspend_mfc(struct spu_state *csa, struct spu *spu)
+{
+ struct spu_priv2 __iomem *priv2 = spu->priv2;
+
+ /* Restore, Step 7:
+ * Restore, Step 47.
+ * Write MFC_Cntl[Dh,Sc]='1','1' to suspend
+ * the queue and halt the decrementer.
+ */
+ out_be64(&priv2->mfc_control_RW, MFC_CNTL_SUSPEND_DMA_QUEUE |
+ MFC_CNTL_DECREMENTER_HALTED);
+ eieio();
+}
+
+static inline void wait_suspend_mfc_complete(struct spu_state *csa,
+ struct spu *spu)
+{
+ struct spu_priv2 __iomem *priv2 = spu->priv2;
+
+ /* Restore, Step 8:
+ * Restore, Step 47.
+ * Poll MFC_CNTL[Ss] until 11 is returned.
+ */
+ POLL_WHILE_FALSE(in_be64(&priv2->mfc_control_RW) &
+ MFC_CNTL_SUSPEND_COMPLETE);
+}
+
+static inline int suspend_spe(struct spu_state *csa, struct spu *spu)
+{
+ struct spu_problem __iomem *prob = spu->problem;
+
+ /* Restore, Step 9:
+ * If SPU_Status[R]=1, stop SPU execution
+ * and wait for stop to complete.
+ *
+ * Returns 1 if SPU_Status[R]=1 on entry.
+ * 0 otherwise
+ */
+ if (in_be32(&prob->spu_status_R) & SPU_STATUS_RUNNING) {
+ if (in_be32(&prob->spu_status_R) &
+ SPU_STATUS_ISOLATED_EXIT_STAUTUS) {
+ POLL_WHILE_TRUE(in_be32(&prob->spu_status_R) &
+ SPU_STATUS_RUNNING);
+ }
+ if ((in_be32(&prob->spu_status_R) &
+ SPU_STATUS_ISOLATED_LOAD_STAUTUS)
+ || (in_be32(&prob->spu_status_R) &
+ SPU_STATUS_ISOLATED_STATE)) {
+ out_be32(&prob->spu_runcntl_RW, SPU_RUNCNTL_STOP);
+ eieio();
+ POLL_WHILE_TRUE(in_be32(&prob->spu_status_R) &
+ SPU_STATUS_RUNNING);
+ out_be32(&prob->spu_runcntl_RW, 0x2);
+ eieio();
+ POLL_WHILE_TRUE(in_be32(&prob->spu_status_R) &
+ SPU_STATUS_RUNNING);
+ }
+ if (in_be32(&prob->spu_status_R) &
+ SPU_STATUS_WAITING_FOR_CHANNEL) {
+ out_be32(&prob->spu_runcntl_RW, SPU_RUNCNTL_STOP);
+ eieio();
+ POLL_WHILE_TRUE(in_be32(&prob->spu_status_R) &
+ SPU_STATUS_RUNNING);
+ }
+ return 1;
+ }
+ return 0;
+}
+
+static inline void clear_spu_status(struct spu_state *csa, struct spu *spu)
+{
+ struct spu_problem __iomem *prob = spu->problem;
+ struct spu_priv1 __iomem *priv1 = spu->priv1;
+
+ /* Restore, Step 10:
+ * If SPU_Status[R]=0 and SPU_Status[E,L,IS]=1,
+ * release SPU from isolate state.
+ */
+ if (!(in_be32(&prob->spu_status_R) & SPU_STATUS_RUNNING)) {
+ if (in_be32(&prob->spu_status_R) &
+ SPU_STATUS_ISOLATED_EXIT_STAUTUS) {
+ out_be64(&priv1->mfc_sr1_RW,
+ MFC_STATE1_MASTER_RUN_CONTROL_MASK);
+ eieio();
+ out_be32(&prob->spu_runcntl_RW, SPU_RUNCNTL_RUNNABLE);
+ eieio();
+ POLL_WHILE_TRUE(in_be32(&prob->spu_status_R) &
+ SPU_STATUS_RUNNING);
+ }
+ if ((in_be32(&prob->spu_status_R) &
+ SPU_STATUS_ISOLATED_LOAD_STAUTUS)
+ || (in_be32(&prob->spu_status_R) &
+ SPU_STATUS_ISOLATED_STATE)) {
+ out_be64(&priv1->mfc_sr1_RW,
+ MFC_STATE1_MASTER_RUN_CONTROL_MASK);
+ eieio();
+ out_be32(&prob->spu_runcntl_RW, 0x2);
+ eieio();
+ POLL_WHILE_TRUE(in_be32(&prob->spu_status_R) &
+ SPU_STATUS_RUNNING);
+ }
+ }
+}
+
+static inline void reset_ch_part1(struct spu_state *csa, struct spu *spu)
+{
+ struct spu_priv2 __iomem *priv2 = spu->priv2;
+ u64 ch_indices[7] = { 0UL, 1UL, 3UL, 4UL, 24UL, 25UL, 27UL };
+ u64 idx;
+ int i;
+
+ /* Restore, Step 20:
+ * Reset the following CH: [0,1,3,4,24,25,27]
+ */
+ for (i = 0; i < 7; i++) {
+ idx = ch_indices[i];
+ out_be64(&priv2->spu_chnlcntptr_RW, idx);
+ eieio();
+ out_be64(&priv2->spu_chnldata_RW, 0UL);
+ out_be64(&priv2->spu_chnlcnt_RW, 0UL);
+ eieio();
+ }
+}
+
+static inline void reset_ch_part2(struct spu_state *csa, struct spu *spu)
+{
+ struct spu_priv2 __iomem *priv2 = spu->priv2;
+ u64 ch_indices[5] = { 21UL, 23UL, 28UL, 29UL, 30UL };
+ u64 ch_counts[5] = { 16UL, 1UL, 1UL, 0UL, 1UL };
+ u64 idx;
+ int i;
+
+ /* Restore, Step 21:
+ * Reset the following CH: [21, 23, 28, 29, 30]
+ */
+ for (i = 0; i < 5; i++) {
+ idx = ch_indices[i];
+ out_be64(&priv2->spu_chnlcntptr_RW, idx);
+ eieio();
+ out_be64(&priv2->spu_chnlcnt_RW, ch_counts[i]);
+ eieio();
+ }
+}
+
+static inline void setup_spu_status_part1(struct spu_state *csa,
+ struct spu *spu)
+{
+ u32 status_P = SPU_STATUS_STOPPED_BY_STOP;
+ u32 status_I = SPU_STATUS_INVALID_INSTR;
+ u32 status_H = SPU_STATUS_STOPPED_BY_HALT;
+ u32 status_S = SPU_STATUS_SINGLE_STEP;
+ u32 status_S_I = SPU_STATUS_SINGLE_STEP | SPU_STATUS_INVALID_INSTR;
+ u32 status_S_P = SPU_STATUS_SINGLE_STEP | SPU_STATUS_STOPPED_BY_STOP;
+ u32 status_P_H = SPU_STATUS_STOPPED_BY_HALT |SPU_STATUS_STOPPED_BY_STOP;
+ u32 status_P_I = SPU_STATUS_STOPPED_BY_STOP |SPU_STATUS_INVALID_INSTR;
+ u32 status_code;
+
+ /* Restore, Step 27:
+ * If the CSA.SPU_Status[I,S,H,P]=1 then add the correct
+ * instruction sequence to the end of the SPU based restore
+ * code (after the "context restored" stop and signal) to
+ * restore the correct SPU status.
+ *
+ * NOTE: Rather than modifying the SPU executable, we
+ * instead add a new 'stopped_status' field to the
+ * LSCSA. The SPU-side restore reads this field and
+ * takes the appropriate action when exiting.
+ */
+
+ status_code =
+ (csa->prob.spu_status_R >> SPU_STOP_STATUS_SHIFT) & 0xFFFF;
+ if ((csa->prob.spu_status_R & status_P_I) == status_P_I) {
+
+ /* SPU_Status[P,I]=1 - Illegal Instruction followed
+ * by Stop and Signal instruction, followed by 'br -4'.
+ *
+ */
+ csa->lscsa->stopped_status.slot[0] = SPU_STOPPED_STATUS_P_I;
+ csa->lscsa->stopped_status.slot[1] = status_code;
+
+ } else if ((csa->prob.spu_status_R & status_P_H) == status_P_H) {
+
+ /* SPU_Status[P,H]=1 - Halt Conditional, followed
+ * by Stop and Signal instruction, followed by
+ * 'br -4'.
+ */
+ csa->lscsa->stopped_status.slot[0] = SPU_STOPPED_STATUS_P_H;
+ csa->lscsa->stopped_status.slot[1] = status_code;
+
+ } else if ((csa->prob.spu_status_R & status_S_P) == status_S_P) {
+
+ /* SPU_Status[S,P]=1 - Stop and Signal instruction
+ * followed by 'br -4'.
+ */
+ csa->lscsa->stopped_status.slot[0] = SPU_STOPPED_STATUS_S_P;
+ csa->lscsa->stopped_status.slot[1] = status_code;
+
+ } else if ((csa->prob.spu_status_R & status_S_I) == status_S_I) {
+
+ /* SPU_Status[S,I]=1 - Illegal instruction followed
+ * by 'br -4'.
+ */
+ csa->lscsa->stopped_status.slot[0] = SPU_STOPPED_STATUS_S_I;
+ csa->lscsa->stopped_status.slot[1] = status_code;
+
+ } else if ((csa->prob.spu_status_R & status_P) == status_P) {
+
+ /* SPU_Status[P]=1 - Stop and Signal instruction
+ * followed by 'br -4'.
+ */
+ csa->lscsa->stopped_status.slot[0] = SPU_STOPPED_STATUS_P;
+ csa->lscsa->stopped_status.slot[1] = status_code;
+
+ } else if ((csa->prob.spu_status_R & status_H) == status_H) {
+
+ /* SPU_Status[H]=1 - Halt Conditional, followed
+ * by 'br -4'.
+ */
+ csa->lscsa->stopped_status.slot[0] = SPU_STOPPED_STATUS_H;
+
+ } else if ((csa->prob.spu_status_R & status_S) == status_S) {
+
+ /* SPU_Status[S]=1 - Two nop instructions.
+ */
+ csa->lscsa->stopped_status.slot[0] = SPU_STOPPED_STATUS_S;
+
+ } else if ((csa->prob.spu_status_R & status_I) == status_I) {
+
+ /* SPU_Status[I]=1 - Illegal instruction followed
+ * by 'br -4'.
+ */
+ csa->lscsa->stopped_status.slot[0] = SPU_STOPPED_STATUS_I;
+
+ }
+}
+
+static inline void setup_spu_status_part2(struct spu_state *csa,
+ struct spu *spu)
+{
+ u32 mask;
+
+ /* Restore, Step 28:
+ * If the CSA.SPU_Status[I,S,H,P,R]=0 then
+ * add a 'br *' instruction to the end of
+ * the SPU based restore code.
+ *
+ * NOTE: Rather than modifying the SPU executable, we
+ * instead add a new 'stopped_status' field to the
+ * LSCSA. The SPU-side restore reads this field and
+ * takes the appropriate action when exiting.
+ */
+ mask = SPU_STATUS_INVALID_INSTR |
+ SPU_STATUS_SINGLE_STEP |
+ SPU_STATUS_STOPPED_BY_HALT |
+ SPU_STATUS_STOPPED_BY_STOP | SPU_STATUS_RUNNING;
+ if (!(csa->prob.spu_status_R & mask)) {
+ csa->lscsa->stopped_status.slot[0] = SPU_STOPPED_STATUS_R;
+ }
+}
+
+static inline void restore_mfc_rag(struct spu_state *csa, struct spu *spu)
+{
+ struct spu_priv1 __iomem *priv1 = spu->priv1;
+
+ /* Restore, Step 29:
+ * Restore RA_GROUP_ID register and the
+ * RA_ENABLE reigster from the CSA.
+ */
+ out_be64(&priv1->resource_allocation_groupID_RW,
+ csa->priv1.resource_allocation_groupID_RW);
+ out_be64(&priv1->resource_allocation_enable_RW,
+ csa->priv1.resource_allocation_enable_RW);
+}
+
+static inline void send_restore_code(struct spu_state *csa, struct spu *spu)
+{
+ unsigned long addr = (unsigned long)&spu_restore_code[0];
+ unsigned int ls_offset = 0x0;
+ unsigned int size = sizeof(spu_restore_code);
+ unsigned int tag = 0;
+ unsigned int rclass = 0;
+ unsigned int cmd = MFC_GETFS_CMD;
+
+ /* Restore, Step 37:
+ * Issue MFC DMA command to copy context
+ * restore code to local storage.
+ */
+ send_mfc_dma(spu, addr, ls_offset, size, tag, rclass, cmd);
+}
+
+static inline void setup_decr(struct spu_state *csa, struct spu *spu)
+{
+ /* Restore, Step 34:
+ * If CSA.MFC_CNTL[Ds]=1 (decrementer was
+ * running) then adjust decrementer, set
+ * decrementer running status in LSCSA,
+ * and set decrementer "wrapped" status
+ * in LSCSA.
+ */
+ if (csa->priv2.mfc_control_RW & MFC_CNTL_DECREMENTER_RUNNING) {
+ cycles_t resume_time = get_cycles();
+ cycles_t delta_time = resume_time - csa->suspend_time;
+
+ csa->lscsa->decr.slot[0] = delta_time;
+ }
+}
+
+static inline void setup_ppu_mb(struct spu_state *csa, struct spu *spu)
+{
+ /* Restore, Step 35:
+ * Copy the CSA.PU_MB data into the LSCSA.
+ */
+ csa->lscsa->ppu_mb.slot[0] = csa->prob.pu_mb_R;
+}
+
+static inline void setup_ppuint_mb(struct spu_state *csa, struct spu *spu)
+{
+ /* Restore, Step 36:
+ * Copy the CSA.PUINT_MB data into the LSCSA.
+ */
+ csa->lscsa->ppuint_mb.slot[0] = csa->priv2.puint_mb_R;
+}
+
+static inline int check_restore_status(struct spu_state *csa, struct spu *spu)
+{
+ struct spu_problem __iomem *prob = spu->problem;
+ u32 complete;
+
+ /* Restore, Step 40:
+ * If SPU_Status[P]=1 and SPU_Status[SC] = "success",
+ * context restore succeeded, otherwise context restore
+ * failed.
+ */
+ complete = ((SPU_RESTORE_COMPLETE << SPU_STOP_STATUS_SHIFT) |
+ SPU_STATUS_STOPPED_BY_STOP);
+ return (in_be32(&prob->spu_status_R) != complete) ? 1 : 0;
+}
+
+static inline void restore_spu_privcntl(struct spu_state *csa, struct spu *spu)
+{
+ struct spu_priv2 __iomem *priv2 = spu->priv2;
+
+ /* Restore, Step 41:
+ * Restore SPU_PrivCntl from the CSA.
+ */
+ out_be64(&priv2->spu_privcntl_RW, csa->priv2.spu_privcntl_RW);
+ eieio();
+}
+
+static inline void restore_status_part1(struct spu_state *csa, struct spu *spu)
+{
+ struct spu_problem __iomem *prob = spu->problem;
+ u32 mask;
+
+ /* Restore, Step 42:
+ * If any CSA.SPU_Status[I,S,H,P]=1, then
+ * restore the error or single step state.
+ */
+ mask = SPU_STATUS_INVALID_INSTR |
+ SPU_STATUS_SINGLE_STEP |
+ SPU_STATUS_STOPPED_BY_HALT | SPU_STATUS_STOPPED_BY_STOP;
+ if (csa->prob.spu_status_R & mask) {
+ out_be32(&prob->spu_runcntl_RW, SPU_RUNCNTL_RUNNABLE);
+ eieio();
+ POLL_WHILE_TRUE(in_be32(&prob->spu_status_R) &
+ SPU_STATUS_RUNNING);
+ }
+}
+
+static inline void restore_status_part2(struct spu_state *csa, struct spu *spu)
+{
+ struct spu_problem __iomem *prob = spu->problem;
+ u32 mask;
+
+ /* Restore, Step 43:
+ * If all CSA.SPU_Status[I,S,H,P,R]=0 then write
+ * SPU_RunCntl[R0R1]='01', wait for SPU_Status[R]=1,
+ * then write '00' to SPU_RunCntl[R0R1] and wait
+ * for SPU_Status[R]=0.
+ */
+ mask = SPU_STATUS_INVALID_INSTR |
+ SPU_STATUS_SINGLE_STEP |
+ SPU_STATUS_STOPPED_BY_HALT |
+ SPU_STATUS_STOPPED_BY_STOP | SPU_STATUS_RUNNING;
+ if (!(csa->prob.spu_status_R & mask)) {
+ out_be32(&prob->spu_runcntl_RW, SPU_RUNCNTL_RUNNABLE);
+ eieio();
+ POLL_WHILE_FALSE(in_be32(&prob->spu_status_R) &
+ SPU_STATUS_RUNNING);
+ out_be32(&prob->spu_runcntl_RW, SPU_RUNCNTL_STOP);
+ eieio();
+ POLL_WHILE_TRUE(in_be32(&prob->spu_status_R) &
+ SPU_STATUS_RUNNING);
+ }
+}
+
+static inline void restore_ls_16kb(struct spu_state *csa, struct spu *spu)
+{
+ unsigned long addr = (unsigned long)&csa->lscsa->ls[0];
+ unsigned int ls_offset = 0x0;
+ unsigned int size = 16384;
+ unsigned int tag = 0;
+ unsigned int rclass = 0;
+ unsigned int cmd = MFC_GET_CMD;
+
+ /* Restore, Step 44:
+ * Issue a DMA command to restore the first
+ * 16kb of local storage from CSA.
+ */
+ send_mfc_dma(spu, addr, ls_offset, size, tag, rclass, cmd);
+}
+
+static inline void clear_interrupts(struct spu_state *csa, struct spu *spu)
+{
+ struct spu_priv1 __iomem *priv1 = spu->priv1;
+
+ /* Restore, Step 49:
+ * Write INT_MASK_class0 with value of 0.
+ * Write INT_MASK_class1 with value of 0.
+ * Write INT_MASK_class2 with value of 0.
+ * Write INT_STAT_class0 with value of -1.
+ * Write INT_STAT_class1 with value of -1.
+ * Write INT_STAT_class2 with value of -1.
+ */
+ spin_lock_irq(&spu->register_lock);
+ out_be64(&priv1->int_mask_class0_RW, 0UL);
+ out_be64(&priv1->int_mask_class1_RW, 0UL);
+ out_be64(&priv1->int_mask_class2_RW, 0UL);
+ out_be64(&priv1->int_stat_class0_RW, ~(0UL));
+ out_be64(&priv1->int_stat_class1_RW, ~(0UL));
+ out_be64(&priv1->int_stat_class2_RW, ~(0UL));
+ spin_unlock_irq(&spu->register_lock);
+}
+
+static inline void restore_mfc_queues(struct spu_state *csa, struct spu *spu)
+{
+ struct spu_priv2 __iomem *priv2 = spu->priv2;
+ int i;
+
+ /* Restore, Step 50:
+ * If MFC_Cntl[Se]!=0 then restore
+ * MFC command queues.
+ */
+ if ((csa->priv2.mfc_control_RW & MFC_CNTL_DMA_QUEUES_EMPTY_MASK) == 0) {
+ for (i = 0; i < 8; i++) {
+ out_be64(&priv2->puq[i].mfc_cq_data0_RW,
+ csa->priv2.puq[i].mfc_cq_data0_RW);
+ out_be64(&priv2->puq[i].mfc_cq_data1_RW,
+ csa->priv2.puq[i].mfc_cq_data1_RW);
+ out_be64(&priv2->puq[i].mfc_cq_data2_RW,
+ csa->priv2.puq[i].mfc_cq_data2_RW);
+ out_be64(&priv2->puq[i].mfc_cq_data3_RW,
+ csa->priv2.puq[i].mfc_cq_data3_RW);
+ }
+ for (i = 0; i < 16; i++) {
+ out_be64(&priv2->spuq[i].mfc_cq_data0_RW,
+ csa->priv2.spuq[i].mfc_cq_data0_RW);
+ out_be64(&priv2->spuq[i].mfc_cq_data1_RW,
+ csa->priv2.spuq[i].mfc_cq_data1_RW);
+ out_be64(&priv2->spuq[i].mfc_cq_data2_RW,
+ csa->priv2.spuq[i].mfc_cq_data2_RW);
+ out_be64(&priv2->spuq[i].mfc_cq_data3_RW,
+ csa->priv2.spuq[i].mfc_cq_data3_RW);
+ }
+ }
+ eieio();
+}
+
+static inline void restore_ppu_querymask(struct spu_state *csa, struct spu *spu)
+{
+ struct spu_problem __iomem *prob = spu->problem;
+
+ /* Restore, Step 51:
+ * Restore the PPU_QueryMask register from CSA.
+ */
+ out_be32(&prob->dma_querymask_RW, csa->prob.dma_querymask_RW);
+ eieio();
+}
+
+static inline void restore_ppu_querytype(struct spu_state *csa, struct spu *spu)
+{
+ struct spu_problem __iomem *prob = spu->problem;
+
+ /* Restore, Step 52:
+ * Restore the PPU_QueryType register from CSA.
+ */
+ out_be32(&prob->dma_querytype_RW, csa->prob.dma_querytype_RW);
+ eieio();
+}
+
+static inline void restore_mfc_csr_tsq(struct spu_state *csa, struct spu *spu)
+{
+ struct spu_priv2 __iomem *priv2 = spu->priv2;
+
+ /* Restore, Step 53:
+ * Restore the MFC_CSR_TSQ register from CSA.
+ */
+ out_be64(&priv2->spu_tag_status_query_RW,
+ csa->priv2.spu_tag_status_query_RW);
+ eieio();
+}
+
+static inline void restore_mfc_csr_cmd(struct spu_state *csa, struct spu *spu)
+{
+ struct spu_priv2 __iomem *priv2 = spu->priv2;
+
+ /* Restore, Step 54:
+ * Restore the MFC_CSR_CMD1 and MFC_CSR_CMD2
+ * registers from CSA.
+ */
+ out_be64(&priv2->spu_cmd_buf1_RW, csa->priv2.spu_cmd_buf1_RW);
+ out_be64(&priv2->spu_cmd_buf2_RW, csa->priv2.spu_cmd_buf2_RW);
+ eieio();
+}
+
+static inline void restore_mfc_csr_ato(struct spu_state *csa, struct spu *spu)
+{
+ struct spu_priv2 __iomem *priv2 = spu->priv2;
+
+ /* Restore, Step 55:
+ * Restore the MFC_CSR_ATO register from CSA.
+ */
+ out_be64(&priv2->spu_atomic_status_RW, csa->priv2.spu_atomic_status_RW);
+}
+
+static inline void restore_mfc_tclass_id(struct spu_state *csa, struct spu *spu)
+{
+ struct spu_priv1 __iomem *priv1 = spu->priv1;
+
+ /* Restore, Step 56:
+ * Restore the MFC_TCLASS_ID register from CSA.
+ */
+ out_be64(&priv1->mfc_tclass_id_RW, csa->priv1.mfc_tclass_id_RW);
+ eieio();
+}
+
+static inline void set_llr_event(struct spu_state *csa, struct spu *spu)
+{
+ u64 ch0_cnt, ch0_data;
+ u64 ch1_data;
+
+ /* Restore, Step 57:
+ * Set the Lock Line Reservation Lost Event by:
+ * 1. OR CSA.SPU_Event_Status with bit 21 (Lr) set to 1.
+ * 2. If CSA.SPU_Channel_0_Count=0 and
+ * CSA.SPU_Wr_Event_Mask[Lr]=1 and
+ * CSA.SPU_Event_Status[Lr]=0 then set
+ * CSA.SPU_Event_Status_Count=1.
+ */
+ ch0_cnt = csa->spu_chnlcnt_RW[0];
+ ch0_data = csa->spu_chnldata_RW[0];
+ ch1_data = csa->spu_chnldata_RW[1];
+ csa->spu_chnldata_RW[0] |= MFC_LLR_LOST_EVENT;
+ if ((ch0_cnt == 0) && !(ch0_data & MFC_LLR_LOST_EVENT) &&
+ (ch1_data & MFC_LLR_LOST_EVENT)) {
+ csa->spu_chnlcnt_RW[0] = 1;
+ }
+}
+
+static inline void restore_decr_wrapped(struct spu_state *csa, struct spu *spu)
+{
+ /* Restore, Step 58:
+ * If the status of the CSA software decrementer
+ * "wrapped" flag is set, OR in a '1' to
+ * CSA.SPU_Event_Status[Tm].
+ */
+ if (csa->lscsa->decr_status.slot[0] == 1) {
+ csa->spu_chnldata_RW[0] |= 0x20;
+ }
+ if ((csa->lscsa->decr_status.slot[0] == 1) &&
+ (csa->spu_chnlcnt_RW[0] == 0 &&
+ ((csa->spu_chnldata_RW[2] & 0x20) == 0x0) &&
+ ((csa->spu_chnldata_RW[0] & 0x20) != 0x1))) {
+ csa->spu_chnlcnt_RW[0] = 1;
+ }
+}
+
+static inline void restore_ch_part1(struct spu_state *csa, struct spu *spu)
+{
+ struct spu_priv2 __iomem *priv2 = spu->priv2;
+ u64 idx, ch_indices[7] = { 0UL, 1UL, 3UL, 4UL, 24UL, 25UL, 27UL };
+ int i;
+
+ /* Restore, Step 59:
+ * Restore the following CH: [0,1,3,4,24,25,27]
+ */
+ for (i = 0; i < 7; i++) {
+ idx = ch_indices[i];
+ out_be64(&priv2->spu_chnlcntptr_RW, idx);
+ eieio();
+ out_be64(&priv2->spu_chnldata_RW, csa->spu_chnldata_RW[idx]);
+ out_be64(&priv2->spu_chnlcnt_RW, csa->spu_chnlcnt_RW[idx]);
+ eieio();
+ }
+}
+
+static inline void restore_ch_part2(struct spu_state *csa, struct spu *spu)
+{
+ struct spu_priv2 __iomem *priv2 = spu->priv2;
+ u64 ch_indices[3] = { 9UL, 21UL, 23UL };
+ u64 ch_counts[3] = { 1UL, 16UL, 1UL };
+ u64 idx;
+ int i;
+
+ /* Restore, Step 60:
+ * Restore the following CH: [9,21,23].
+ */
+ ch_counts[0] = 1UL;
+ ch_counts[1] = csa->spu_chnlcnt_RW[21];
+ ch_counts[2] = 1UL;
+ for (i = 0; i < 3; i++) {
+ idx = ch_indices[i];
+ out_be64(&priv2->spu_chnlcntptr_RW, idx);
+ eieio();
+ out_be64(&priv2->spu_chnlcnt_RW, ch_counts[i]);
+ eieio();
+ }
+}
+
+static inline void restore_spu_lslr(struct spu_state *csa, struct spu *spu)
+{
+ struct spu_priv2 __iomem *priv2 = spu->priv2;
+
+ /* Restore, Step 61:
+ * Restore the SPU_LSLR register from CSA.
+ */
+ out_be64(&priv2->spu_lslr_RW, csa->priv2.spu_lslr_RW);
+ eieio();
+}
+
+static inline void restore_spu_cfg(struct spu_state *csa, struct spu *spu)
+{
+ struct spu_priv2 __iomem *priv2 = spu->priv2;
+
+ /* Restore, Step 62:
+ * Restore the SPU_Cfg register from CSA.
+ */
+ out_be64(&priv2->spu_cfg_RW, csa->priv2.spu_cfg_RW);
+ eieio();
+}
+
+static inline void restore_pm_trace(struct spu_state *csa, struct spu *spu)
+{
+ /* Restore, Step 63:
+ * Restore PM_Trace_Tag_Wait_Mask from CSA.
+ * Not performed by this implementation.
+ */
+}
+
+static inline void restore_spu_npc(struct spu_state *csa, struct spu *spu)
+{
+ struct spu_problem __iomem *prob = spu->problem;
+
+ /* Restore, Step 64:
+ * Restore SPU_NPC from CSA.
+ */
+ out_be32(&prob->spu_npc_RW, csa->prob.spu_npc_RW);
+ eieio();
+}
+
+static inline void restore_spu_mb(struct spu_state *csa, struct spu *spu)
+{
+ struct spu_priv2 __iomem *priv2 = spu->priv2;
+ int i;
+
+ /* Restore, Step 65:
+ * Restore MFC_RdSPU_MB from CSA.
+ */
+ out_be64(&priv2->spu_chnlcntptr_RW, 29UL);
+ eieio();
+ out_be64(&priv2->spu_chnlcnt_RW, csa->spu_chnlcnt_RW[29]);
+ for (i = 0; i < 4; i++) {
+ out_be64(&priv2->spu_chnldata_RW, csa->pu_mailbox_data[i]);
+ }
+ eieio();
+}
+
+static inline void check_ppu_mb_stat(struct spu_state *csa, struct spu *spu)
+{
+ struct spu_problem __iomem *prob = spu->problem;
+ u32 dummy = 0;
+
+ /* Restore, Step 66:
+ * If CSA.MB_Stat[P]=0 (mailbox empty) then
+ * read from the PPU_MB register.
+ */
+ if ((csa->prob.mb_stat_R & 0xFF) == 0) {
+ dummy = in_be32(&prob->pu_mb_R);
+ eieio();
+ }
+}
+
+static inline void check_ppuint_mb_stat(struct spu_state *csa, struct spu *spu)
+{
+ struct spu_priv1 __iomem *priv1 = spu->priv1;
+ struct spu_priv2 __iomem *priv2 = spu->priv2;
+ u64 dummy = 0UL;
+
+ /* Restore, Step 66:
+ * If CSA.MB_Stat[I]=0 (mailbox empty) then
+ * read from the PPUINT_MB register.
+ */
+ if ((csa->prob.mb_stat_R & 0xFF0000) == 0) {
+ dummy = in_be64(&priv2->puint_mb_R);
+ eieio();
+ out_be64(&priv1->int_stat_class2_RW,
+ CLASS2_ENABLE_MAILBOX_INTR);
+ eieio();
+ }
+}
+
+static inline void restore_mfc_slbs(struct spu_state *csa, struct spu *spu)
+{
+ struct spu_priv2 __iomem *priv2 = spu->priv2;
+ int i;
+
+ /* Restore, Step 68:
+ * If MFC_SR1[R]='1', restore SLBs from CSA.
+ */
+ if (csa->priv1.mfc_sr1_RW & MFC_STATE1_RELOCATE_MASK) {
+ for (i = 0; i < 8; i++) {
+ out_be64(&priv2->slb_index_W, i);
+ eieio();
+ out_be64(&priv2->slb_esid_RW, csa->slb_esid_RW[i]);
+ out_be64(&priv2->slb_vsid_RW, csa->slb_vsid_RW[i]);
+ eieio();
+ }
+ out_be64(&priv2->slb_index_W, csa->priv2.slb_index_W);
+ eieio();
+ }
+}
+
+static inline void restore_mfc_sr1(struct spu_state *csa, struct spu *spu)
+{
+ struct spu_priv1 __iomem *priv1 = spu->priv1;
+
+ /* Restore, Step 69:
+ * Restore the MFC_SR1 register from CSA.
+ */
+ out_be64(&priv1->mfc_sr1_RW, csa->priv1.mfc_sr1_RW);
+ eieio();
+}
+
+static inline void restore_other_spu_access(struct spu_state *csa,
+ struct spu *spu)
+{
+ /* Restore, Step 70:
+ * Restore other SPU mappings to this SPU. TBD.
+ */
+}
+
+static inline void restore_spu_runcntl(struct spu_state *csa, struct spu *spu)
+{
+ struct spu_problem __iomem *prob = spu->problem;
+
+ /* Restore, Step 71:
+ * If CSA.SPU_Status[R]=1 then write
+ * SPU_RunCntl[R0R1]='01'.
+ */
+ if (csa->prob.spu_status_R & SPU_STATUS_RUNNING) {
+ out_be32(&prob->spu_runcntl_RW, SPU_RUNCNTL_RUNNABLE);
+ eieio();
+ }
+}
+
+static inline void restore_mfc_cntl(struct spu_state *csa, struct spu *spu)
+{
+ struct spu_priv2 __iomem *priv2 = spu->priv2;
+
+ /* Restore, Step 72:
+ * Restore the MFC_CNTL register for the CSA.
+ */
+ out_be64(&priv2->mfc_control_RW, csa->priv2.mfc_control_RW);
+ eieio();
+}
+
+static inline void enable_user_access(struct spu_state *csa, struct spu *spu)
+{
+ /* Restore, Step 73:
+ * Enable user-space access (if provided) to this
+ * SPU by mapping the virtual pages assigned to
+ * the SPU memory-mapped I/O (MMIO) for problem
+ * state. TBD.
+ */
+}
+
+static inline void reset_switch_active(struct spu_state *csa, struct spu *spu)
+{
+ /* Restore, Step 74:
+ * Reset the "context switch active" flag.
+ */
+ clear_bit(SPU_CONTEXT_SWITCH_ACTIVE_nr, &spu->flags);
+ mb();
+}
+
+static inline void reenable_interrupts(struct spu_state *csa, struct spu *spu)
+{
+ struct spu_priv1 __iomem *priv1 = spu->priv1;
+
+ /* Restore, Step 75:
+ * Re-enable SPU interrupts.
+ */
+ spin_lock_irq(&spu->register_lock);
+ out_be64(&priv1->int_mask_class0_RW, csa->priv1.int_mask_class0_RW);
+ out_be64(&priv1->int_mask_class1_RW, csa->priv1.int_mask_class1_RW);
+ out_be64(&priv1->int_mask_class2_RW, csa->priv1.int_mask_class2_RW);
+ spin_unlock_irq(&spu->register_lock);
+}
+
+static int quiece_spu(struct spu_state *prev, struct spu *spu)
+{
+ /*
+ * Combined steps 2-18 of SPU context save sequence, which
+ * quiesce the SPU state (disable SPU execution, MFC command
+ * queues, decrementer, SPU interrupts, etc.).
+ *
+ * Returns 0 on success.
+ * 2 if failed step 2.
+ * 6 if failed step 6.
+ */
+
+ if (check_spu_isolate(prev, spu)) { /* Step 2. */
+ return 2;
+ }
+ disable_interrupts(prev, spu); /* Step 3. */
+ set_watchdog_timer(prev, spu); /* Step 4. */
+ inhibit_user_access(prev, spu); /* Step 5. */
+ if (check_spu_isolate(prev, spu)) { /* Step 6. */
+ return 6;
+ }
+ set_switch_pending(prev, spu); /* Step 7. */
+ save_mfc_cntl(prev, spu); /* Step 8. */
+ save_spu_runcntl(prev, spu); /* Step 9. */
+ save_mfc_sr1(prev, spu); /* Step 10. */
+ save_spu_status(prev, spu); /* Step 11. */
+ save_mfc_decr(prev, spu); /* Step 12. */
+ halt_mfc_decr(prev, spu); /* Step 13. */
+ save_timebase(prev, spu); /* Step 14. */
+ remove_other_spu_access(prev, spu); /* Step 15. */
+ do_mfc_mssync(prev, spu); /* Step 16. */
+ issue_mfc_tlbie(prev, spu); /* Step 17. */
+ handle_pending_interrupts(prev, spu); /* Step 18. */
+
+ return 0;
+}
+
+static void save_csa(struct spu_state *prev, struct spu *spu)
+{
+ /*
+ * Combine steps 19-44 of SPU context save sequence, which
+ * save regions of the privileged & problem state areas.
+ */
+
+ save_mfc_queues(prev, spu); /* Step 19. */
+ save_ppu_querymask(prev, spu); /* Step 20. */
+ save_ppu_querytype(prev, spu); /* Step 21. */
+ save_mfc_csr_tsq(prev, spu); /* Step 22. */
+ save_mfc_csr_cmd(prev, spu); /* Step 23. */
+ save_mfc_csr_ato(prev, spu); /* Step 24. */
+ save_mfc_tclass_id(prev, spu); /* Step 25. */
+ set_mfc_tclass_id(prev, spu); /* Step 26. */
+ purge_mfc_queue(prev, spu); /* Step 27. */
+ wait_purge_complete(prev, spu); /* Step 28. */
+ save_mfc_slbs(prev, spu); /* Step 29. */
+ setup_mfc_sr1(prev, spu); /* Step 30. */
+ save_spu_npc(prev, spu); /* Step 31. */
+ save_spu_privcntl(prev, spu); /* Step 32. */
+ reset_spu_privcntl(prev, spu); /* Step 33. */
+ save_spu_lslr(prev, spu); /* Step 34. */
+ reset_spu_lslr(prev, spu); /* Step 35. */
+ save_spu_cfg(prev, spu); /* Step 36. */
+ save_pm_trace(prev, spu); /* Step 37. */
+ save_mfc_rag(prev, spu); /* Step 38. */
+ save_ppu_mb_stat(prev, spu); /* Step 39. */
+ save_ppu_mb(prev, spu); /* Step 40. */
+ save_ppuint_mb(prev, spu); /* Step 41. */
+ save_ch_part1(prev, spu); /* Step 42. */
+ save_spu_mb(prev, spu); /* Step 43. */
+ save_mfc_cmd(prev, spu); /* Step 44. */
+ reset_ch(prev, spu); /* Step 45. */
+}
+
+static void save_lscsa(struct spu_state *prev, struct spu *spu)
+{
+ /*
+ * Perform steps 46-57 of SPU context save sequence,
+ * which save regions of the local store and register
+ * file.
+ */
+
+ resume_mfc_queue(prev, spu); /* Step 46. */
+ setup_mfc_slbs(prev, spu); /* Step 47. */
+ set_switch_active(prev, spu); /* Step 48. */
+ enable_interrupts(prev, spu); /* Step 49. */
+ save_ls_16kb(prev, spu); /* Step 50. */
+ set_spu_npc(prev, spu); /* Step 51. */
+ set_signot1(prev, spu); /* Step 52. */
+ set_signot2(prev, spu); /* Step 53. */
+ send_save_code(prev, spu); /* Step 54. */
+ set_ppu_querymask(prev, spu); /* Step 55. */
+ wait_tag_complete(prev, spu); /* Step 56. */
+ wait_spu_stopped(prev, spu); /* Step 57. */
+}
+
+static void harvest(struct spu_state *prev, struct spu *spu)
+{
+ /*
+ * Perform steps 2-25 of SPU context restore sequence,
+ * which resets an SPU either after a failed save, or
+ * when using SPU for first time.
+ */
+
+ disable_interrupts(prev, spu); /* Step 2. */
+ inhibit_user_access(prev, spu); /* Step 3. */
+ terminate_spu_app(prev, spu); /* Step 4. */
+ set_switch_pending(prev, spu); /* Step 5. */
+ remove_other_spu_access(prev, spu); /* Step 6. */
+ suspend_mfc(prev, spu); /* Step 7. */
+ wait_suspend_mfc_complete(prev, spu); /* Step 8. */
+ if (!suspend_spe(prev, spu)) /* Step 9. */
+ clear_spu_status(prev, spu); /* Step 10. */
+ do_mfc_mssync(prev, spu); /* Step 11. */
+ issue_mfc_tlbie(prev, spu); /* Step 12. */
+ handle_pending_interrupts(prev, spu); /* Step 13. */
+ purge_mfc_queue(prev, spu); /* Step 14. */
+ wait_purge_complete(prev, spu); /* Step 15. */
+ reset_spu_privcntl(prev, spu); /* Step 16. */
+ reset_spu_lslr(prev, spu); /* Step 17. */
+ setup_mfc_sr1(prev, spu); /* Step 18. */
+ invalidate_slbs(prev, spu); /* Step 19. */
+ reset_ch_part1(prev, spu); /* Step 20. */
+ reset_ch_part2(prev, spu); /* Step 21. */
+ enable_interrupts(prev, spu); /* Step 22. */
+ set_switch_active(prev, spu); /* Step 23. */
+ set_mfc_tclass_id(prev, spu); /* Step 24. */
+ resume_mfc_queue(prev, spu); /* Step 25. */
+}
+
+static void restore_lscsa(struct spu_state *next, struct spu *spu)
+{
+ /*
+ * Perform steps 26-40 of SPU context restore sequence,
+ * which restores regions of the local store and register
+ * file.
+ */
+
+ set_watchdog_timer(next, spu); /* Step 26. */
+ setup_spu_status_part1(next, spu); /* Step 27. */
+ setup_spu_status_part2(next, spu); /* Step 28. */
+ restore_mfc_rag(next, spu); /* Step 29. */
+ setup_mfc_slbs(next, spu); /* Step 30. */
+ set_spu_npc(next, spu); /* Step 31. */
+ set_signot1(next, spu); /* Step 32. */
+ set_signot2(next, spu); /* Step 33. */
+ setup_decr(next, spu); /* Step 34. */
+ setup_ppu_mb(next, spu); /* Step 35. */
+ setup_ppuint_mb(next, spu); /* Step 36. */
+ send_restore_code(next, spu); /* Step 37. */
+ set_ppu_querymask(next, spu); /* Step 38. */
+ wait_tag_complete(next, spu); /* Step 39. */
+ wait_spu_stopped(next, spu); /* Step 40. */
+}
+
+static void restore_csa(struct spu_state *next, struct spu *spu)
+{
+ /*
+ * Combine steps 41-76 of SPU context restore sequence, which
+ * restore regions of the privileged & problem state areas.
+ */
+
+ restore_spu_privcntl(next, spu); /* Step 41. */
+ restore_status_part1(next, spu); /* Step 42. */
+ restore_status_part2(next, spu); /* Step 43. */
+ restore_ls_16kb(next, spu); /* Step 44. */
+ wait_tag_complete(next, spu); /* Step 45. */
+ suspend_mfc(next, spu); /* Step 46. */
+ wait_suspend_mfc_complete(next, spu); /* Step 47. */
+ issue_mfc_tlbie(next, spu); /* Step 48. */
+ clear_interrupts(next, spu); /* Step 49. */
+ restore_mfc_queues(next, spu); /* Step 50. */
+ restore_ppu_querymask(next, spu); /* Step 51. */
+ restore_ppu_querytype(next, spu); /* Step 52. */
+ restore_mfc_csr_tsq(next, spu); /* Step 53. */
+ restore_mfc_csr_cmd(next, spu); /* Step 54. */
+ restore_mfc_csr_ato(next, spu); /* Step 55. */
+ restore_mfc_tclass_id(next, spu); /* Step 56. */
+ set_llr_event(next, spu); /* Step 57. */
+ restore_decr_wrapped(next, spu); /* Step 58. */
+ restore_ch_part1(next, spu); /* Step 59. */
+ restore_ch_part2(next, spu); /* Step 60. */
+ restore_spu_lslr(next, spu); /* Step 61. */
+ restore_spu_cfg(next, spu); /* Step 62. */
+ restore_pm_trace(next, spu); /* Step 63. */
+ restore_spu_npc(next, spu); /* Step 64. */
+ restore_spu_mb(next, spu); /* Step 65. */
+ check_ppu_mb_stat(next, spu); /* Step 66. */
+ check_ppuint_mb_stat(next, spu); /* Step 67. */
+ restore_mfc_slbs(next, spu); /* Step 68. */
+ restore_mfc_sr1(next, spu); /* Step 69. */
+ restore_other_spu_access(next, spu); /* Step 70. */
+ restore_spu_runcntl(next, spu); /* Step 71. */
+ restore_mfc_cntl(next, spu); /* Step 72. */
+ enable_user_access(next, spu); /* Step 73. */
+ reset_switch_active(next, spu); /* Step 74. */
+ reenable_interrupts(next, spu); /* Step 75. */
+}
+
+static int __do_spu_save(struct spu_state *prev, struct spu *spu)
+{
+ int rc;
+
+ /*
+ * SPU context save can be broken into three phases:
+ *
+ * (a) quiesce [steps 2-16].
+ * (b) save of CSA, performed by PPE [steps 17-42]
+ * (c) save of LSCSA, mostly performed by SPU [steps 43-52].
+ *
+ * Returns 0 on success.
+ * 2,6 if failed to quiece SPU
+ * 53 if SPU-side of save failed.
+ */
+
+ rc = quiece_spu(prev, spu); /* Steps 2-16. */
+ switch (rc) {
+ default:
+ case 2:
+ case 6:
+ harvest(prev, spu);
+ return rc;
+ break;
+ case 0:
+ break;
+ }
+ save_csa(prev, spu); /* Steps 17-43. */
+ save_lscsa(prev, spu); /* Steps 44-53. */
+ return check_save_status(prev, spu); /* Step 54. */
+}
+
+static int __do_spu_restore(struct spu_state *next, struct spu *spu)
+{
+ int rc;
+
+ /*
+ * SPU context restore can be broken into three phases:
+ *
+ * (a) harvest (or reset) SPU [steps 2-24].
+ * (b) restore LSCSA [steps 25-40], mostly performed by SPU.
+ * (c) restore CSA [steps 41-76], performed by PPE.
+ *
+ * The 'harvest' step is not performed here, but rather
+ * as needed below.
+ */
+
+ restore_lscsa(next, spu); /* Steps 24-39. */
+ rc = check_restore_status(next, spu); /* Step 40. */
+ switch (rc) {
+ default:
+ /* Failed. Return now. */
+ return rc;
+ break;
+ case 0:
+ /* Fall through to next step. */
+ break;
+ }
+ restore_csa(next, spu);
+
+ return 0;
+}
+
/**
* spu_save - SPU context save, with locking.
* @prev: pointer to SPU context save area, to be saved.
@@ -61,9 +2074,13 @@
*/
int spu_save(struct spu_state *prev, struct spu *spu)
{
- /* XXX missing */
+ int rc;
- return 0;
+ acquire_spu_lock(spu); /* Step 1. */
+ rc = __do_spu_save(prev, spu); /* Steps 2-53. */
+ release_spu_lock(spu);
+
+ return rc;
}
/**
@@ -77,9 +2094,14 @@ int spu_save(struct spu_state *prev, str
*/
int spu_restore(struct spu_state *new, struct spu *spu)
{
- /* XXX missing */
+ int rc;
- return 0;
+ acquire_spu_lock(spu);
+ harvest(NULL, spu);
+ rc = __do_spu_restore(new, spu);
+ release_spu_lock(spu);
+
+ return rc;
}
/**
@@ -93,9 +2115,17 @@ int spu_restore(struct spu_state *new, s
*/
int spu_switch(struct spu_state *prev, struct spu_state *new, struct spu *spu)
{
- /* XXX missing */
+ int rc;
- return 0;
+ acquire_spu_lock(spu); /* Save, Step 1. */
+ rc = __do_spu_save(prev, spu); /* Save, Steps 2-53. */
+ if (rc != 0) {
+ harvest(prev, spu);
+ }
+ rc = __do_spu_restore(new, spu);
+ release_spu_lock(spu);
+
+ return rc;
}
static void init_prob(struct spu_state *csa)
Index: linux-cg/include/asm-powerpc/spu_csa.h
===================================================================
--- linux-cg.orig/include/asm-powerpc/spu_csa.h
+++ linux-cg/include/asm-powerpc/spu_csa.h
@@ -200,7 +200,6 @@ struct spu_priv2_collapsed {
u64 spu_chnlcnt_RW;
u64 spu_chnldata_RW;
u64 spu_cfg_RW;
- u64 spu_pm_trace_tag_status_RW;
u64 spu_tag_status_query_RW;
u64 spu_cmd_buf1_RW;
u64 spu_cmd_buf2_RW;
--
^ permalink raw reply [flat|nested] 10+ messages in thread* [PATCH 5/5] spufs: cooperative scheduler support
2005-11-15 20:53 [PATCH 0/5] SPU file system for 2.6.15-rc-mm Arnd Bergmann
` (2 preceding siblings ...)
2005-11-15 20:53 ` [PATCH 4/5] spufs: add spu-side context switch code Arnd Bergmann
@ 2005-11-15 20:53 ` Arnd Bergmann
[not found] ` <20051115210408.327453000@localhost>
4 siblings, 0 replies; 10+ messages in thread
From: Arnd Bergmann @ 2005-11-15 20:53 UTC (permalink / raw)
To: akpm; +Cc: Paul Mackerras, linuxppc64-dev, linux-kernel, mnutter,
Arnd Bergmann
[-- Attachment #1: spufs-scheduler-2.diff --]
[-- Type: text/plain, Size: 76384 bytes --]
This adds a scheduler for SPUs to make it possible to use
more logical SPUs than physical ones are present in the
system.
Currently, there is no support for preempting a running
SPU thread, they have to leave the SPU by either triggering
an event on the SPU that causes it to return to the
owning thread or by sending a signal to it.
This patch also adds operations that enable accessing an SPU
in either runnable or saved state. We use an RW semaphore
to protect the state of the SPU from changing underneath
us, while we are holding it readable. In order to change
the state, it is acquired writeable and a context save
or restore is executed before downgrading the semaphore
to read-only.
From: Mark Nutter <mnutter@us.ibm.com>,
Uli Weigand <Ulrich.Weigand@de.ibm.com>
Signed-off-by: Arnd Bergmann <arndb@de.ibm.com>
Index: linux-2.6.15-rc/arch/powerpc/platforms/cell/spufs/context.c
===================================================================
--- linux-2.6.15-rc.orig/arch/powerpc/platforms/cell/spufs/context.c
+++ linux-2.6.15-rc/arch/powerpc/platforms/cell/spufs/context.c
@@ -20,39 +20,38 @@
* Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
*/
+#include <linux/fs.h>
+#include <linux/mm.h>
#include <linux/slab.h>
#include <asm/spu.h>
#include <asm/spu_csa.h>
#include "spufs.h"
-struct spu_context *alloc_spu_context(void)
+struct spu_context *alloc_spu_context(struct address_space *local_store)
{
struct spu_context *ctx;
ctx = kmalloc(sizeof *ctx, GFP_KERNEL);
if (!ctx)
goto out;
- /* Future enhancement: do not call spu_alloc()
- * here. This step should be deferred until
- * spu_run()!!
- *
- * More work needs to be done to read(),
- * write(), mmap(), etc., so that operations
- * are performed on CSA when the context is
- * not currently being run. In this way we
- * can support arbitrarily large number of
- * entries in /spu, allow state queries, etc.
+ /* Binding to physical processor deferred
+ * until spu_activate().
*/
- ctx->spu = spu_alloc();
- if (!ctx->spu)
- goto out_free;
spu_init_csa(&ctx->csa);
if (!ctx->csa.lscsa) {
- spu_free(ctx->spu);
goto out_free;
}
- init_rwsem(&ctx->backing_sema);
spin_lock_init(&ctx->mmio_lock);
kref_init(&ctx->kref);
+ init_rwsem(&ctx->state_sema);
+ init_waitqueue_head(&ctx->ibox_wq);
+ init_waitqueue_head(&ctx->wbox_wq);
+ ctx->ibox_fasync = NULL;
+ ctx->wbox_fasync = NULL;
+ ctx->state = SPU_STATE_SAVED;
+ ctx->local_store = local_store;
+ ctx->spu = NULL;
+ ctx->ops = &spu_backing_ops;
+ ctx->owner = get_task_mm(current);
goto out;
out_free:
kfree(ctx);
@@ -65,8 +64,11 @@ void destroy_spu_context(struct kref *kr
{
struct spu_context *ctx;
ctx = container_of(kref, struct spu_context, kref);
- if (ctx->spu)
- spu_free(ctx->spu);
+ down_write(&ctx->state_sema);
+ spu_deactivate(ctx);
+ ctx->ibox_fasync = NULL;
+ ctx->wbox_fasync = NULL;
+ up_write(&ctx->state_sema);
spu_fini_csa(&ctx->csa);
kfree(ctx);
}
@@ -82,4 +84,80 @@ int put_spu_context(struct spu_context *
return kref_put(&ctx->kref, &destroy_spu_context);
}
+/* give up the mm reference when the context is about to be destroyed */
+void spu_forget(struct spu_context *ctx)
+{
+ struct mm_struct *mm;
+ spu_acquire_saved(ctx);
+ mm = ctx->owner;
+ ctx->owner = NULL;
+ mmput(mm);
+ spu_release(ctx);
+}
+
+void spu_acquire(struct spu_context *ctx)
+{
+ down_read(&ctx->state_sema);
+}
+
+void spu_release(struct spu_context *ctx)
+{
+ up_read(&ctx->state_sema);
+}
+
+static void spu_unmap_mappings(struct spu_context *ctx)
+{
+ unmap_mapping_range(ctx->local_store, 0, LS_SIZE, 1);
+}
+
+int spu_acquire_runnable(struct spu_context *ctx)
+{
+ int ret = 0;
+ down_read(&ctx->state_sema);
+ if (ctx->state == SPU_STATE_RUNNABLE)
+ return 0;
+ /* ctx is about to be freed, can't acquire any more */
+ if (!ctx->owner) {
+ ret = -EINVAL;
+ goto out;
+ }
+ up_read(&ctx->state_sema);
+
+ down_write(&ctx->state_sema);
+ if (ctx->state == SPU_STATE_SAVED) {
+ spu_unmap_mappings(ctx);
+ ret = spu_activate(ctx, 0);
+ ctx->state = SPU_STATE_RUNNABLE;
+ }
+ downgrade_write(&ctx->state_sema);
+ if (ret)
+ goto out;
+
+ /* On success, we return holding the lock */
+ return ret;
+out:
+ /* Release here, to simplify calling code. */
+ up_read(&ctx->state_sema);
+
+ return ret;
+}
+
+void spu_acquire_saved(struct spu_context *ctx)
+{
+ down_read(&ctx->state_sema);
+
+ if (ctx->state == SPU_STATE_SAVED)
+ return;
+
+ up_read(&ctx->state_sema);
+ down_write(&ctx->state_sema);
+
+ if (ctx->state == SPU_STATE_RUNNABLE) {
+ spu_unmap_mappings(ctx);
+ spu_deactivate(ctx);
+ ctx->state = SPU_STATE_SAVED;
+ }
+
+ downgrade_write(&ctx->state_sema);
+}
Index: linux-2.6.15-rc/arch/powerpc/platforms/cell/spufs/file.c
===================================================================
--- linux-2.6.15-rc.orig/arch/powerpc/platforms/cell/spufs/file.c
+++ linux-2.6.15-rc/arch/powerpc/platforms/cell/spufs/file.c
@@ -32,11 +32,13 @@
#include "spufs.h"
+
static int
spufs_mem_open(struct inode *inode, struct file *file)
{
struct spufs_inode_info *i = SPUFS_I(inode);
file->private_data = i->i_ctx;
+ file->f_mapping = i->i_ctx->local_store;
return 0;
}
@@ -44,23 +46,16 @@ static ssize_t
spufs_mem_read(struct file *file, char __user *buffer,
size_t size, loff_t *pos)
{
- struct spu *spu;
- struct spu_context *ctx;
+ struct spu_context *ctx = file->private_data;
+ char *local_store;
int ret;
- ctx = file->private_data;
- spu = ctx->spu;
+ spu_acquire(ctx);
- down_read(&ctx->backing_sema);
- if (spu->number & 0/*1*/) {
- ret = generic_file_read(file, buffer, size, pos);
- goto out;
- }
+ local_store = ctx->ops->get_ls(ctx);
+ ret = simple_read_from_buffer(buffer, size, pos, local_store, LS_SIZE);
- ret = simple_read_from_buffer(buffer, size, pos,
- spu->local_store, LS_SIZE);
-out:
- up_read(&ctx->backing_sema);
+ spu_release(ctx);
return ret;
}
@@ -69,50 +64,181 @@ spufs_mem_write(struct file *file, const
size_t size, loff_t *pos)
{
struct spu_context *ctx = file->private_data;
- struct spu *spu = ctx->spu;
-
- if (spu->number & 0) //1)
- return generic_file_write(file, buffer, size, pos);
+ char *local_store;
+ int ret;
size = min_t(ssize_t, LS_SIZE - *pos, size);
if (size <= 0)
return -EFBIG;
*pos += size;
- return copy_from_user(spu->local_store + *pos - size,
- buffer, size) ? -EFAULT : size;
+
+ spu_acquire(ctx);
+
+ local_store = ctx->ops->get_ls(ctx);
+ ret = copy_from_user(local_store + *pos - size,
+ buffer, size) ? -EFAULT : size;
+
+ spu_release(ctx);
+ return ret;
+}
+
+#ifdef CONFIG_SPARSEMEM
+static struct page *
+spufs_mem_mmap_nopage(struct vm_area_struct *vma,
+ unsigned long address, int *type)
+{
+ struct page *page = NOPAGE_SIGBUS;
+
+ struct spu_context *ctx = vma->vm_file->private_data;
+ unsigned long offset = address - vma->vm_start;
+ offset += vma->vm_pgoff << PAGE_SHIFT;
+
+ spu_acquire(ctx);
+
+ if (ctx->state == SPU_STATE_SAVED)
+ page = vmalloc_to_page(ctx->csa.lscsa->ls + offset);
+ else
+ page = pfn_to_page((ctx->spu->local_store_phys + offset)
+ >> PAGE_SHIFT);
+
+ spu_release(ctx);
+
+ if (type)
+ *type = VM_FAULT_MINOR;
+
+ return page;
}
+static struct vm_operations_struct spufs_mem_mmap_vmops = {
+ .nopage = spufs_mem_mmap_nopage,
+};
+
static int
spufs_mem_mmap(struct file *file, struct vm_area_struct *vma)
{
- struct spu_context *ctx = file->private_data;
- struct spu *spu = ctx->spu;
- unsigned long pfn;
-
- if (spu->number & 0) //1)
- return generic_file_mmap(file, vma);
+ if (!(vma->vm_flags & VM_SHARED))
+ return -EINVAL;
+ /* FIXME: */
vma->vm_flags |= VM_RESERVED;
- vma->vm_page_prot = __pgprot(pgprot_val (vma->vm_page_prot)
- | _PAGE_NO_CACHE);
- pfn = spu->local_store_phys >> PAGE_SHIFT;
- /*
- * This will work for actual SPUs, but not for vmalloc memory:
- */
- if (remap_pfn_range(vma, vma->vm_start, pfn,
- vma->vm_end-vma->vm_start, vma->vm_page_prot))
- return -EAGAIN;
+ vma->vm_page_prot = __pgprot(pgprot_val(vma->vm_page_prot)
+ | _PAGE_NO_CACHE);
+
+ vma->vm_ops = &spufs_mem_mmap_vmops;
return 0;
}
+#endif
static struct file_operations spufs_mem_fops = {
.open = spufs_mem_open,
.read = spufs_mem_read,
.write = spufs_mem_write,
+ .llseek = generic_file_llseek,
+#ifdef CONFIG_SPARSEMEM
.mmap = spufs_mem_mmap,
+#endif
+};
+
+static int
+spufs_regs_open(struct inode *inode, struct file *file)
+{
+ struct spufs_inode_info *i = SPUFS_I(inode);
+ file->private_data = i->i_ctx;
+ return 0;
+}
+
+static ssize_t
+spufs_regs_read(struct file *file, char __user *buffer,
+ size_t size, loff_t *pos)
+{
+ struct spu_context *ctx = file->private_data;
+ struct spu_lscsa *lscsa = ctx->csa.lscsa;
+ int ret;
+
+ spu_acquire_saved(ctx);
+
+ ret = simple_read_from_buffer(buffer, size, pos,
+ lscsa->gprs, sizeof lscsa->gprs);
+
+ spu_release(ctx);
+ return ret;
+}
+
+static ssize_t
+spufs_regs_write(struct file *file, const char __user *buffer,
+ size_t size, loff_t *pos)
+{
+ struct spu_context *ctx = file->private_data;
+ struct spu_lscsa *lscsa = ctx->csa.lscsa;
+ int ret;
+
+ size = min_t(ssize_t, sizeof lscsa->gprs - *pos, size);
+ if (size <= 0)
+ return -EFBIG;
+ *pos += size;
+
+ spu_acquire_saved(ctx);
+
+ ret = copy_from_user(lscsa->gprs + *pos - size,
+ buffer, size) ? -EFAULT : size;
+
+ spu_release(ctx);
+ return ret;
+}
+
+static struct file_operations spufs_regs_fops = {
+ .open = spufs_regs_open,
+ .read = spufs_regs_read,
+ .write = spufs_regs_write,
.llseek = generic_file_llseek,
};
+static ssize_t
+spufs_fpcr_read(struct file *file, char __user * buffer,
+ size_t size, loff_t * pos)
+{
+ struct spu_context *ctx = file->private_data;
+ struct spu_lscsa *lscsa = ctx->csa.lscsa;
+ int ret;
+
+ spu_acquire_saved(ctx);
+
+ ret = simple_read_from_buffer(buffer, size, pos,
+ &lscsa->fpcr, sizeof(lscsa->fpcr));
+
+ spu_release(ctx);
+ return ret;
+}
+
+static ssize_t
+spufs_fpcr_write(struct file *file, const char __user * buffer,
+ size_t size, loff_t * pos)
+{
+ struct spu_context *ctx = file->private_data;
+ struct spu_lscsa *lscsa = ctx->csa.lscsa;
+ int ret;
+
+ size = min_t(ssize_t, sizeof(lscsa->fpcr) - *pos, size);
+ if (size <= 0)
+ return -EFBIG;
+ *pos += size;
+
+ spu_acquire_saved(ctx);
+
+ ret = copy_from_user((char *)&lscsa->fpcr + *pos - size,
+ buffer, size) ? -EFAULT : size;
+
+ spu_release(ctx);
+ return ret;
+}
+
+static struct file_operations spufs_fpcr_fops = {
+ .open = spufs_regs_open,
+ .read = spufs_fpcr_read,
+ .write = spufs_fpcr_write,
+ .llseek = generic_file_llseek,
+};
+
/* generic open function for all pipe-like files */
static int spufs_pipe_open(struct inode *inode, struct file *file)
{
@@ -125,21 +251,19 @@ static int spufs_pipe_open(struct inode
static ssize_t spufs_mbox_read(struct file *file, char __user *buf,
size_t len, loff_t *pos)
{
- struct spu_context *ctx;
- struct spu_problem __iomem *prob;
- u32 mbox_stat;
+ struct spu_context *ctx = file->private_data;
u32 mbox_data;
+ int ret;
if (len < 4)
return -EINVAL;
- ctx = file->private_data;
- prob = ctx->spu->problem;
- mbox_stat = in_be32(&prob->mb_stat_R);
- if (!(mbox_stat & 0x0000ff))
- return -EAGAIN;
+ spu_acquire(ctx);
+ ret = ctx->ops->mbox_read(ctx, &mbox_data);
+ spu_release(ctx);
- mbox_data = in_be32(&prob->pu_mb_R);
+ if (!ret)
+ return -EAGAIN;
if (copy_to_user(buf, &mbox_data, sizeof mbox_data))
return -EFAULT;
@@ -155,14 +279,17 @@ static struct file_operations spufs_mbox
static ssize_t spufs_mbox_stat_read(struct file *file, char __user *buf,
size_t len, loff_t *pos)
{
- struct spu_context *ctx;
+ struct spu_context *ctx = file->private_data;
u32 mbox_stat;
if (len < 4)
return -EINVAL;
- ctx = file->private_data;
- mbox_stat = in_be32(&ctx->spu->problem->mb_stat_R) & 0xff;
+ spu_acquire(ctx);
+
+ mbox_stat = ctx->ops->mbox_stat_read(ctx) & 0xff;
+
+ spu_release(ctx);
if (copy_to_user(buf, &mbox_stat, sizeof mbox_stat))
return -EFAULT;
@@ -175,57 +302,78 @@ static struct file_operations spufs_mbox
.read = spufs_mbox_stat_read,
};
+/*
+ * spufs_wait
+ * Same as wait_event_interruptible(), except that here
+ * we need to call spu_release(ctx) before sleeping, and
+ * then spu_acquire(ctx) when awoken.
+ */
+
+#define spufs_wait(wq, condition) \
+({ \
+ int __ret = 0; \
+ DEFINE_WAIT(__wait); \
+ for (;;) { \
+ prepare_to_wait(&(wq), &__wait, TASK_INTERRUPTIBLE); \
+ if (condition) \
+ break; \
+ if (!signal_pending(current)) { \
+ spu_release(ctx); \
+ schedule(); \
+ spu_acquire(ctx); \
+ continue; \
+ } \
+ __ret = -ERESTARTSYS; \
+ break; \
+ } \
+ finish_wait(&(wq), &__wait); \
+ __ret; \
+})
+
/* low-level ibox access function */
-size_t spu_ibox_read(struct spu *spu, u32 *data)
+size_t spu_ibox_read(struct spu_context *ctx, u32 *data)
{
- int ret;
-
- spin_lock_irq(&spu->register_lock);
+ return ctx->ops->ibox_read(ctx, data);
+}
- if (in_be32(&spu->problem->mb_stat_R) & 0xff0000) {
- /* read the first available word */
- *data = in_be64(&spu->priv2->puint_mb_R);
- ret = 4;
- } else {
- /* make sure we get woken up by the interrupt */
- out_be64(&spu->priv1->int_mask_class2_RW,
- in_be64(&spu->priv1->int_mask_class2_RW) | 0x1);
- ret = 0;
- }
+static int spufs_ibox_fasync(int fd, struct file *file, int on)
+{
+ struct spu_context *ctx = file->private_data;
- spin_unlock_irq(&spu->register_lock);
- return ret;
+ return fasync_helper(fd, file, on, &ctx->ibox_fasync);
}
-EXPORT_SYMBOL(spu_ibox_read);
-static int spufs_ibox_fasync(int fd, struct file *file, int on)
+/* interrupt-level ibox callback function. */
+void spufs_ibox_callback(struct spu *spu)
{
- struct spu_context *ctx;
- ctx = file->private_data;
- return fasync_helper(fd, file, on, &ctx->spu->ibox_fasync);
+ struct spu_context *ctx = spu->ctx;
+
+ wake_up_all(&ctx->ibox_wq);
+ kill_fasync(&ctx->ibox_fasync, SIGIO, POLLIN);
}
static ssize_t spufs_ibox_read(struct file *file, char __user *buf,
size_t len, loff_t *pos)
{
- struct spu_context *ctx;
+ struct spu_context *ctx = file->private_data;
u32 ibox_data;
ssize_t ret;
if (len < 4)
return -EINVAL;
- ctx = file->private_data;
+ spu_acquire(ctx);
ret = 0;
if (file->f_flags & O_NONBLOCK) {
- if (!spu_ibox_read(ctx->spu, &ibox_data))
+ if (!spu_ibox_read(ctx, &ibox_data))
ret = -EAGAIN;
} else {
- ret = wait_event_interruptible(ctx->spu->ibox_wq,
- spu_ibox_read(ctx->spu, &ibox_data));
+ ret = spufs_wait(ctx->ibox_wq, spu_ibox_read(ctx, &ibox_data));
}
+ spu_release(ctx);
+
if (ret)
return ret;
@@ -238,16 +386,17 @@ static ssize_t spufs_ibox_read(struct fi
static unsigned int spufs_ibox_poll(struct file *file, poll_table *wait)
{
- struct spu_context *ctx;
- struct spu_problem __iomem *prob;
+ struct spu_context *ctx = file->private_data;
u32 mbox_stat;
unsigned int mask;
- ctx = file->private_data;
- prob = ctx->spu->problem;
- mbox_stat = in_be32(&prob->mb_stat_R);
+ spu_acquire(ctx);
+
+ mbox_stat = ctx->ops->mbox_stat_read(ctx);
+
+ spu_release(ctx);
- poll_wait(file, &ctx->spu->ibox_wq, wait);
+ poll_wait(file, &ctx->ibox_wq, wait);
mask = 0;
if (mbox_stat & 0xff0000)
@@ -266,14 +415,15 @@ static struct file_operations spufs_ibox
static ssize_t spufs_ibox_stat_read(struct file *file, char __user *buf,
size_t len, loff_t *pos)
{
- struct spu_context *ctx;
+ struct spu_context *ctx = file->private_data;
u32 ibox_stat;
if (len < 4)
return -EINVAL;
- ctx = file->private_data;
- ibox_stat = (in_be32(&ctx->spu->problem->mb_stat_R) >> 16) & 0xff;
+ spu_acquire(ctx);
+ ibox_stat = (ctx->ops->mbox_stat_read(ctx) >> 16) & 0xff;
+ spu_release(ctx);
if (copy_to_user(buf, &ibox_stat, sizeof ibox_stat))
return -EFAULT;
@@ -287,75 +437,69 @@ static struct file_operations spufs_ibox
};
/* low-level mailbox write */
-size_t spu_wbox_write(struct spu *spu, u32 data)
+size_t spu_wbox_write(struct spu_context *ctx, u32 data)
{
- int ret;
+ return ctx->ops->wbox_write(ctx, data);
+}
- spin_lock_irq(&spu->register_lock);
+static int spufs_wbox_fasync(int fd, struct file *file, int on)
+{
+ struct spu_context *ctx = file->private_data;
+ int ret;
- if (in_be32(&spu->problem->mb_stat_R) & 0x00ff00) {
- /* we have space to write wbox_data to */
- out_be32(&spu->problem->spu_mb_W, data);
- ret = 4;
- } else {
- /* make sure we get woken up by the interrupt when space
- becomes available */
- out_be64(&spu->priv1->int_mask_class2_RW,
- in_be64(&spu->priv1->int_mask_class2_RW) | 0x10);
- ret = 0;
- }
+ ret = fasync_helper(fd, file, on, &ctx->wbox_fasync);
- spin_unlock_irq(&spu->register_lock);
return ret;
}
-EXPORT_SYMBOL(spu_wbox_write);
-static int spufs_wbox_fasync(int fd, struct file *file, int on)
+/* interrupt-level wbox callback function. */
+void spufs_wbox_callback(struct spu *spu)
{
- struct spu_context *ctx;
- ctx = file->private_data;
- return fasync_helper(fd, file, on, &ctx->spu->wbox_fasync);
+ struct spu_context *ctx = spu->ctx;
+
+ wake_up_all(&ctx->wbox_wq);
+ kill_fasync(&ctx->wbox_fasync, SIGIO, POLLOUT);
}
static ssize_t spufs_wbox_write(struct file *file, const char __user *buf,
size_t len, loff_t *pos)
{
- struct spu_context *ctx;
+ struct spu_context *ctx = file->private_data;
u32 wbox_data;
int ret;
if (len < 4)
return -EINVAL;
- ctx = file->private_data;
-
if (copy_from_user(&wbox_data, buf, sizeof wbox_data))
return -EFAULT;
+ spu_acquire(ctx);
+
ret = 0;
if (file->f_flags & O_NONBLOCK) {
- if (!spu_wbox_write(ctx->spu, wbox_data))
+ if (!spu_wbox_write(ctx, wbox_data))
ret = -EAGAIN;
} else {
- ret = wait_event_interruptible(ctx->spu->wbox_wq,
- spu_wbox_write(ctx->spu, wbox_data));
+ ret = spufs_wait(ctx->wbox_wq, spu_wbox_write(ctx, wbox_data));
}
+ spu_release(ctx);
+
return ret ? ret : sizeof wbox_data;
}
static unsigned int spufs_wbox_poll(struct file *file, poll_table *wait)
{
- struct spu_context *ctx;
- struct spu_problem __iomem *prob;
+ struct spu_context *ctx = file->private_data;
u32 mbox_stat;
unsigned int mask;
- ctx = file->private_data;
- prob = ctx->spu->problem;
- mbox_stat = in_be32(&prob->mb_stat_R);
+ spu_acquire(ctx);
+ mbox_stat = ctx->ops->mbox_stat_read(ctx);
+ spu_release(ctx);
- poll_wait(file, &ctx->spu->wbox_wq, wait);
+ poll_wait(file, &ctx->wbox_wq, wait);
mask = 0;
if (mbox_stat & 0x00ff00)
@@ -374,14 +518,15 @@ static struct file_operations spufs_wbox
static ssize_t spufs_wbox_stat_read(struct file *file, char __user *buf,
size_t len, loff_t *pos)
{
- struct spu_context *ctx;
+ struct spu_context *ctx = file->private_data;
u32 wbox_stat;
if (len < 4)
return -EINVAL;
- ctx = file->private_data;
- wbox_stat = (in_be32(&ctx->spu->problem->mb_stat_R) >> 8) & 0xff;
+ spu_acquire(ctx);
+ wbox_stat = (ctx->ops->mbox_stat_read(ctx) >> 8) & 0xff;
+ spu_release(ctx);
if (copy_to_user(buf, &wbox_stat, sizeof wbox_stat))
return -EFAULT;
@@ -395,47 +540,41 @@ static struct file_operations spufs_wbox
};
long spufs_run_spu(struct file *file, struct spu_context *ctx,
- u32 *npc, u32 *status)
+ u32 *npc, u32 *status)
{
- struct spu_problem __iomem *prob;
int ret;
- if (file->f_flags & O_NONBLOCK) {
- ret = -EAGAIN;
- if (!down_write_trylock(&ctx->backing_sema))
- goto out;
- } else {
- down_write(&ctx->backing_sema);
- }
+ ret = spu_acquire_runnable(ctx);
+ if (ret)
+ return ret;
- prob = ctx->spu->problem;
- out_be32(&prob->spu_npc_RW, *npc);
+ ctx->ops->npc_write(ctx, *npc);
ret = spu_run(ctx->spu);
- *status = in_be32(&prob->spu_status_R);
- *npc = in_be32(&prob->spu_npc_RW);
+ if (!ret)
+ ret = ctx->ops->status_read(ctx);
- up_write(&ctx->backing_sema);
+ *npc = ctx->ops->npc_read(ctx);
-out:
+ spu_release(ctx);
+ spu_yield(ctx);
return ret;
}
static ssize_t spufs_signal1_read(struct file *file, char __user *buf,
size_t len, loff_t *pos)
{
- struct spu_context *ctx;
- struct spu_problem *prob;
+ struct spu_context *ctx = file->private_data;
u32 data;
- ctx = file->private_data;
- prob = ctx->spu->problem;
-
if (len < 4)
return -EINVAL;
- data = in_be32(&prob->signal_notify1);
+ spu_acquire(ctx);
+ data = ctx->ops->signal1_read(ctx);
+ spu_release(ctx);
+
if (copy_to_user(buf, &data, 4))
return -EFAULT;
@@ -446,11 +585,9 @@ static ssize_t spufs_signal1_write(struc
size_t len, loff_t *pos)
{
struct spu_context *ctx;
- struct spu_problem *prob;
u32 data;
ctx = file->private_data;
- prob = ctx->spu->problem;
if (len < 4)
return -EINVAL;
@@ -458,7 +595,9 @@ static ssize_t spufs_signal1_write(struc
if (copy_from_user(&data, buf, 4))
return -EFAULT;
- out_be32(&prob->signal_notify1, data);
+ spu_acquire(ctx);
+ ctx->ops->signal1_write(ctx, data);
+ spu_release(ctx);
return 4;
}
@@ -473,16 +612,17 @@ static ssize_t spufs_signal2_read(struct
size_t len, loff_t *pos)
{
struct spu_context *ctx;
- struct spu_problem *prob;
u32 data;
ctx = file->private_data;
- prob = ctx->spu->problem;
if (len < 4)
return -EINVAL;
- data = in_be32(&prob->signal_notify2);
+ spu_acquire(ctx);
+ data = ctx->ops->signal2_read(ctx);
+ spu_release(ctx);
+
if (copy_to_user(buf, &data, 4))
return -EFAULT;
@@ -493,11 +633,9 @@ static ssize_t spufs_signal2_write(struc
size_t len, loff_t *pos)
{
struct spu_context *ctx;
- struct spu_problem *prob;
u32 data;
ctx = file->private_data;
- prob = ctx->spu->problem;
if (len < 4)
return -EINVAL;
@@ -505,7 +643,9 @@ static ssize_t spufs_signal2_write(struc
if (copy_from_user(&data, buf, 4))
return -EFAULT;
- out_be32(&prob->signal_notify2, data);
+ spu_acquire(ctx);
+ ctx->ops->signal2_write(ctx, data);
+ spu_release(ctx);
return 4;
}
@@ -519,23 +659,22 @@ static struct file_operations spufs_sign
static void spufs_signal1_type_set(void *data, u64 val)
{
struct spu_context *ctx = data;
- struct spu_priv2 *priv2 = ctx->spu->priv2;
- u64 tmp;
- spin_lock_irq(&ctx->spu->register_lock);
- tmp = in_be64(&priv2->spu_cfg_RW);
- if (val)
- tmp |= 1;
- else
- tmp &= ~1;
- out_be64(&priv2->spu_cfg_RW, tmp);
- spin_unlock_irq(&ctx->spu->register_lock);
+ spu_acquire(ctx);
+ ctx->ops->signal1_type_set(ctx, val);
+ spu_release(ctx);
}
static u64 spufs_signal1_type_get(void *data)
{
struct spu_context *ctx = data;
- return (in_be64(&ctx->spu->priv2->spu_cfg_RW) & 1) != 0;
+ u64 ret;
+
+ spu_acquire(ctx);
+ ret = ctx->ops->signal1_type_get(ctx);
+ spu_release(ctx);
+
+ return ret;
}
DEFINE_SIMPLE_ATTRIBUTE(spufs_signal1_type, spufs_signal1_type_get,
spufs_signal1_type_set, "%llu");
@@ -543,23 +682,22 @@ DEFINE_SIMPLE_ATTRIBUTE(spufs_signal1_ty
static void spufs_signal2_type_set(void *data, u64 val)
{
struct spu_context *ctx = data;
- struct spu_priv2 *priv2 = ctx->spu->priv2;
- u64 tmp;
- spin_lock_irq(&ctx->spu->register_lock);
- tmp = in_be64(&priv2->spu_cfg_RW);
- if (val)
- tmp |= 2;
- else
- tmp &= ~2;
- out_be64(&priv2->spu_cfg_RW, tmp);
- spin_unlock_irq(&ctx->spu->register_lock);
+ spu_acquire(ctx);
+ ctx->ops->signal2_type_set(ctx, val);
+ spu_release(ctx);
}
static u64 spufs_signal2_type_get(void *data)
{
struct spu_context *ctx = data;
- return (in_be64(&ctx->spu->priv2->spu_cfg_RW) & 2) != 0;
+ u64 ret;
+
+ spu_acquire(ctx);
+ ret = ctx->ops->signal2_type_get(ctx);
+ spu_release(ctx);
+
+ return ret;
}
DEFINE_SIMPLE_ATTRIBUTE(spufs_signal2_type, spufs_signal2_type_get,
spufs_signal2_type_set, "%llu");
@@ -567,20 +705,135 @@ DEFINE_SIMPLE_ATTRIBUTE(spufs_signal2_ty
static void spufs_npc_set(void *data, u64 val)
{
struct spu_context *ctx = data;
- out_be32(&ctx->spu->problem->spu_npc_RW, val);
+ spu_acquire(ctx);
+ ctx->ops->npc_write(ctx, val);
+ spu_release(ctx);
}
static u64 spufs_npc_get(void *data)
{
struct spu_context *ctx = data;
u64 ret;
- ret = in_be32(&ctx->spu->problem->spu_npc_RW);
+ spu_acquire(ctx);
+ ret = ctx->ops->npc_read(ctx);
+ spu_release(ctx);
return ret;
}
DEFINE_SIMPLE_ATTRIBUTE(spufs_npc_ops, spufs_npc_get, spufs_npc_set, "%llx\n")
+static void spufs_decr_set(void *data, u64 val)
+{
+ struct spu_context *ctx = data;
+ struct spu_lscsa *lscsa = ctx->csa.lscsa;
+ spu_acquire_saved(ctx);
+ lscsa->decr.slot[0] = (u32) val;
+ spu_release(ctx);
+}
+
+static u64 spufs_decr_get(void *data)
+{
+ struct spu_context *ctx = data;
+ struct spu_lscsa *lscsa = ctx->csa.lscsa;
+ u64 ret;
+ spu_acquire_saved(ctx);
+ ret = lscsa->decr.slot[0];
+ spu_release(ctx);
+ return ret;
+}
+DEFINE_SIMPLE_ATTRIBUTE(spufs_decr_ops, spufs_decr_get, spufs_decr_set,
+ "%llx\n")
+
+static void spufs_decr_status_set(void *data, u64 val)
+{
+ struct spu_context *ctx = data;
+ struct spu_lscsa *lscsa = ctx->csa.lscsa;
+ spu_acquire_saved(ctx);
+ lscsa->decr_status.slot[0] = (u32) val;
+ spu_release(ctx);
+}
+
+static u64 spufs_decr_status_get(void *data)
+{
+ struct spu_context *ctx = data;
+ struct spu_lscsa *lscsa = ctx->csa.lscsa;
+ u64 ret;
+ spu_acquire_saved(ctx);
+ ret = lscsa->decr_status.slot[0];
+ spu_release(ctx);
+ return ret;
+}
+DEFINE_SIMPLE_ATTRIBUTE(spufs_decr_status_ops, spufs_decr_status_get,
+ spufs_decr_status_set, "%llx\n")
+
+static void spufs_spu_tag_mask_set(void *data, u64 val)
+{
+ struct spu_context *ctx = data;
+ struct spu_lscsa *lscsa = ctx->csa.lscsa;
+ spu_acquire_saved(ctx);
+ lscsa->tag_mask.slot[0] = (u32) val;
+ spu_release(ctx);
+}
+
+static u64 spufs_spu_tag_mask_get(void *data)
+{
+ struct spu_context *ctx = data;
+ struct spu_lscsa *lscsa = ctx->csa.lscsa;
+ u64 ret;
+ spu_acquire_saved(ctx);
+ ret = lscsa->tag_mask.slot[0];
+ spu_release(ctx);
+ return ret;
+}
+DEFINE_SIMPLE_ATTRIBUTE(spufs_spu_tag_mask_ops, spufs_spu_tag_mask_get,
+ spufs_spu_tag_mask_set, "%llx\n")
+
+static void spufs_event_mask_set(void *data, u64 val)
+{
+ struct spu_context *ctx = data;
+ struct spu_lscsa *lscsa = ctx->csa.lscsa;
+ spu_acquire_saved(ctx);
+ lscsa->event_mask.slot[0] = (u32) val;
+ spu_release(ctx);
+}
+
+static u64 spufs_event_mask_get(void *data)
+{
+ struct spu_context *ctx = data;
+ struct spu_lscsa *lscsa = ctx->csa.lscsa;
+ u64 ret;
+ spu_acquire_saved(ctx);
+ ret = lscsa->event_mask.slot[0];
+ spu_release(ctx);
+ return ret;
+}
+DEFINE_SIMPLE_ATTRIBUTE(spufs_event_mask_ops, spufs_event_mask_get,
+ spufs_event_mask_set, "%llx\n")
+
+static void spufs_srr0_set(void *data, u64 val)
+{
+ struct spu_context *ctx = data;
+ struct spu_lscsa *lscsa = ctx->csa.lscsa;
+ spu_acquire_saved(ctx);
+ lscsa->srr0.slot[0] = (u32) val;
+ spu_release(ctx);
+}
+
+static u64 spufs_srr0_get(void *data)
+{
+ struct spu_context *ctx = data;
+ struct spu_lscsa *lscsa = ctx->csa.lscsa;
+ u64 ret;
+ spu_acquire_saved(ctx);
+ ret = lscsa->srr0.slot[0];
+ spu_release(ctx);
+ return ret;
+}
+DEFINE_SIMPLE_ATTRIBUTE(spufs_srr0_ops, spufs_srr0_get, spufs_srr0_set,
+ "%llx\n")
+
struct tree_descr spufs_dir_contents[] = {
{ "mem", &spufs_mem_fops, 0666, },
+ { "regs", &spufs_regs_fops, 0666, },
{ "mbox", &spufs_mbox_fops, 0444, },
{ "ibox", &spufs_ibox_fops, 0444, },
{ "wbox", &spufs_wbox_fops, 0222, },
@@ -592,5 +845,11 @@ struct tree_descr spufs_dir_contents[] =
{ "signal1_type", &spufs_signal1_type, 0666, },
{ "signal2_type", &spufs_signal2_type, 0666, },
{ "npc", &spufs_npc_ops, 0666, },
+ { "fpcr", &spufs_fpcr_fops, 0666, },
+ { "decr", &spufs_decr_ops, 0666, },
+ { "decr_status", &spufs_decr_status_ops, 0666, },
+ { "spu_tag_mask", &spufs_spu_tag_mask_ops, 0666, },
+ { "event_mask", &spufs_event_mask_ops, 0666, },
+ { "srr0", &spufs_srr0_ops, 0666, },
{},
};
Index: linux-2.6.15-rc/arch/powerpc/platforms/cell/spufs/spufs.h
===================================================================
--- linux-2.6.15-rc.orig/arch/powerpc/platforms/cell/spufs/spufs.h
+++ linux-2.6.15-rc/arch/powerpc/platforms/cell/spufs/spufs.h
@@ -35,15 +35,50 @@ enum {
SPUFS_MAGIC = 0x23c9b64e,
};
+struct spu_context_ops;
+
struct spu_context {
struct spu *spu; /* pointer to a physical SPU */
struct spu_state csa; /* SPU context save area. */
- struct rw_semaphore backing_sema; /* protects the above */
spinlock_t mmio_lock; /* protects mmio access */
+ struct address_space *local_store;/* local store backing store */
+
+ enum { SPU_STATE_RUNNABLE, SPU_STATE_SAVED } state;
+ struct rw_semaphore state_sema;
+
+ struct mm_struct *owner;
struct kref kref;
+ wait_queue_head_t ibox_wq;
+ wait_queue_head_t wbox_wq;
+ struct fasync_struct *ibox_fasync;
+ struct fasync_struct *wbox_fasync;
+ struct spu_context_ops *ops;
};
+/* SPU context query/set operations. */
+struct spu_context_ops {
+ int (*mbox_read) (struct spu_context * ctx, u32 * data);
+ u32(*mbox_stat_read) (struct spu_context * ctx);
+ int (*ibox_read) (struct spu_context * ctx, u32 * data);
+ int (*wbox_write) (struct spu_context * ctx, u32 data);
+ u32(*signal1_read) (struct spu_context * ctx);
+ void (*signal1_write) (struct spu_context * ctx, u32 data);
+ u32(*signal2_read) (struct spu_context * ctx);
+ void (*signal2_write) (struct spu_context * ctx, u32 data);
+ void (*signal1_type_set) (struct spu_context * ctx, u64 val);
+ u64(*signal1_type_get) (struct spu_context * ctx);
+ void (*signal2_type_set) (struct spu_context * ctx, u64 val);
+ u64(*signal2_type_get) (struct spu_context * ctx);
+ u32(*npc_read) (struct spu_context * ctx);
+ void (*npc_write) (struct spu_context * ctx, u32 data);
+ u32(*status_read) (struct spu_context * ctx);
+ char*(*get_ls) (struct spu_context * ctx);
+};
+
+extern struct spu_context_ops spu_hw_ops;
+extern struct spu_context_ops spu_backing_ops;
+
struct spufs_inode_info {
struct spu_context *i_ctx;
struct inode vfs_inode;
@@ -60,14 +95,28 @@ long spufs_create_thread(struct nameidat
unsigned int flags, mode_t mode);
/* context management */
-struct spu_context * alloc_spu_context(void);
+struct spu_context * alloc_spu_context(struct address_space *local_store);
void destroy_spu_context(struct kref *kref);
struct spu_context * get_spu_context(struct spu_context *ctx);
int put_spu_context(struct spu_context *ctx);
+void spu_forget(struct spu_context *ctx);
void spu_acquire(struct spu_context *ctx);
void spu_release(struct spu_context *ctx);
-void spu_acquire_runnable(struct spu_context *ctx);
+int spu_acquire_runnable(struct spu_context *ctx);
void spu_acquire_saved(struct spu_context *ctx);
+int spu_activate(struct spu_context *ctx, u64 flags);
+void spu_deactivate(struct spu_context *ctx);
+void spu_yield(struct spu_context *ctx);
+int __init spu_sched_init(void);
+void __exit spu_sched_exit(void);
+
+size_t spu_wbox_write(struct spu_context *ctx, u32 data);
+size_t spu_ibox_read(struct spu_context *ctx, u32 *data);
+
+/* irq callback funcs. */
+void spufs_ibox_callback(struct spu *spu);
+void spufs_wbox_callback(struct spu *spu);
+
#endif
Index: linux-2.6.15-rc/arch/powerpc/platforms/cell/setup.c
===================================================================
--- linux-2.6.15-rc.orig/arch/powerpc/platforms/cell/setup.c
+++ linux-2.6.15-rc/arch/powerpc/platforms/cell/setup.c
@@ -67,6 +67,77 @@ void cell_show_cpuinfo(struct seq_file *
of_node_put(root);
}
+#ifdef CONFIG_SPARSEMEM
+static int __init find_spu_node_id(struct device_node *spe)
+{
+ unsigned int *id;
+#ifdef CONFIG_NUMA
+ struct device_node *cpu;
+ cpu = spe->parent->parent;
+ id = (unsigned int *)get_property(cpu, "node-id", NULL);
+#else
+ id = NULL;
+#endif
+ return id ? *id : 0;
+}
+
+static void __init cell_spuprop_present(struct device_node *spe,
+ const char *prop, int early)
+{
+ struct address_prop {
+ unsigned long address;
+ unsigned int len;
+ } __attribute__((packed)) *p;
+ int proplen;
+
+ unsigned long start_pfn, end_pfn, pfn;
+ int node_id;
+
+ p = (void*)get_property(spe, prop, &proplen);
+ WARN_ON(proplen != sizeof (*p));
+
+ node_id = find_spu_node_id(spe);
+
+ start_pfn = p->address >> PAGE_SHIFT;
+ end_pfn = (p->address + p->len + PAGE_SIZE - 1) >> PAGE_SHIFT;
+
+ /* We need to call memory_present *before* the call to sparse_init,
+ but we can initialize the page structs only *after* that call.
+ Thus, we're being called twice. */
+ if (early)
+ memory_present(node_id, start_pfn, end_pfn);
+ else {
+ /* As the pages backing SPU LS and I/O are outside the range
+ of regular memory, their page structs were not initialized
+ by free_area_init. Do it here instead. */
+ for (pfn = start_pfn; pfn < end_pfn; pfn++) {
+ struct page *page = pfn_to_page(pfn);
+ set_page_links(page, ZONE_DMA, node_id, pfn);
+ set_page_count(page, 0);
+ reset_page_mapcount(page);
+ SetPageReserved(page);
+ INIT_LIST_HEAD(&page->lru);
+ }
+ }
+}
+
+static void __init cell_spumem_init(int early)
+{
+ struct device_node *node;
+ for (node = of_find_node_by_type(NULL, "spe");
+ node; node = of_find_node_by_type(node, "spe")) {
+ cell_spuprop_present(node, "local-store", early);
+ cell_spuprop_present(node, "problem", early);
+ cell_spuprop_present(node, "priv1", early);
+ cell_spuprop_present(node, "priv2", early);
+ }
+}
+#else
+static void __init cell_spumem_init(int early)
+{
+}
+#endif
+
static void cell_progress(char *s, unsigned short hex)
{
printk("*** %04x : %s\n", hex, s ? s : "");
@@ -98,6 +169,8 @@ static void __init cell_setup_arch(void)
#endif
mmio_nvram_init();
+
+ cell_spumem_init(0);
}
/*
@@ -113,6 +186,8 @@ static void __init cell_init_early(void)
ppc64_interrupt_controller = IC_CELL_PIC;
+ cell_spumem_init(1);
+
DBG(" <- cell_init_early()\n");
}
Index: linux-2.6.15-rc/arch/powerpc/platforms/cell/spufs/inode.c
===================================================================
--- linux-2.6.15-rc.orig/arch/powerpc/platforms/cell/spufs/inode.c
+++ linux-2.6.15-rc/arch/powerpc/platforms/cell/spufs/inode.c
@@ -41,24 +41,6 @@
static kmem_cache_t *spufs_inode_cache;
-/* Information about the backing dev, same as ramfs */
-#if 0
-static struct backing_dev_info spufs_backing_dev_info = {
- .ra_pages = 0, /* No readahead */
- .capabilities = BDI_CAP_NO_ACCT_DIRTY | BDI_CAP_NO_WRITEBACK |
- BDI_CAP_MAP_DIRECT | BDI_CAP_MAP_COPY | BDI_CAP_READ_MAP |
- BDI_CAP_WRITE_MAP,
-};
-
-static struct address_space_operations spufs_aops = {
- .readpage = simple_readpage,
- .prepare_write = simple_prepare_write,
- .commit_write = simple_commit_write,
-};
-#endif
-
-/* Inode operations */
-
static struct inode *
spufs_alloc_inode(struct super_block *sb)
{
@@ -111,9 +93,6 @@ spufs_setattr(struct dentry *dentry, str
{
struct inode *inode = dentry->d_inode;
-/* dump_stack();
- pr_debug("ia_size %lld, i_size:%lld\n", attr->ia_size, inode->i_size);
-*/
if ((attr->ia_valid & ATTR_SIZE) &&
(attr->ia_size != inode->i_size))
return -EINVAL;
@@ -127,9 +106,7 @@ spufs_new_file(struct super_block *sb, s
struct spu_context *ctx)
{
static struct inode_operations spufs_file_iops = {
- .getattr = simple_getattr,
.setattr = spufs_setattr,
- .unlink = simple_unlink,
};
struct inode *inode;
int ret;
@@ -183,21 +160,32 @@ out:
static int spufs_rmdir(struct inode *root, struct dentry *dir_dentry)
{
- struct dentry *dentry;
+ struct dentry *dentry, *tmp;
+ struct spu_context *ctx;
int err;
- spin_lock(&dcache_lock);
/* remove all entries */
err = 0;
- list_for_each_entry(dentry, &dir_dentry->d_subdirs, d_child) {
- if (d_unhashed(dentry) || !dentry->d_inode)
- continue;
- atomic_dec(&dentry->d_count);
+ list_for_each_entry_safe(dentry, tmp, &dir_dentry->d_subdirs, d_child) {
+ spin_lock(&dcache_lock);
spin_lock(&dentry->d_lock);
- __d_drop(dentry);
- spin_unlock(&dentry->d_lock);
+ if (!(d_unhashed(dentry)) && dentry->d_inode) {
+ dget_locked(dentry);
+ __d_drop(dentry);
+ spin_unlock(&dentry->d_lock);
+ simple_unlink(dir_dentry->d_inode, dentry);
+ spin_unlock(&dcache_lock);
+ dput(dentry);
+ } else {
+ spin_unlock(&dentry->d_lock);
+ spin_unlock(&dcache_lock);
+ }
}
- spin_unlock(&dcache_lock);
+
+ /* We have to give up the mm_struct */
+ ctx = SPUFS_I(dir_dentry->d_inode)->i_ctx;
+ spu_forget(ctx);
+
if (!err) {
shrink_dcache_parent(dir_dentry);
err = simple_rmdir(root, dir_dentry);
@@ -249,7 +237,7 @@ spufs_mkdir(struct inode *dir, struct de
inode->i_gid = dir->i_gid;
inode->i_mode &= S_ISGID;
}
- ctx = alloc_spu_context();
+ ctx = alloc_spu_context(inode->i_mapping);
SPUFS_I(inode)->i_ctx = ctx;
if (!ctx)
goto out_iput;
@@ -368,7 +356,8 @@ spufs_parse_options(char *options, struc
}
static int
-spufs_create_root(struct super_block *sb, void *data) {
+spufs_create_root(struct super_block *sb, void *data)
+{
struct inode *inode;
int ret;
@@ -441,6 +430,10 @@ static int spufs_init(void)
if (!spufs_inode_cache)
goto out;
+ if (spu_sched_init() != 0) {
+ kmem_cache_destroy(spufs_inode_cache);
+ goto out;
+ }
ret = register_filesystem(&spufs_type);
if (ret)
goto out_cache;
@@ -459,6 +452,7 @@ module_init(spufs_init);
static void spufs_exit(void)
{
+ spu_sched_exit();
unregister_spu_syscalls(&spufs_calls);
unregister_filesystem(&spufs_type);
kmem_cache_destroy(spufs_inode_cache);
Index: linux-2.6.15-rc/arch/powerpc/platforms/cell/spufs/switch.c
===================================================================
--- linux-2.6.15-rc.orig/arch/powerpc/platforms/cell/spufs/switch.c
+++ linux-2.6.15-rc/arch/powerpc/platforms/cell/spufs/switch.c
@@ -646,7 +646,7 @@ static inline void save_spu_mb(struct sp
eieio();
csa->spu_chnlcnt_RW[29] = in_be64(&priv2->spu_chnlcnt_RW);
for (i = 0; i < 4; i++) {
- csa->pu_mailbox_data[i] = in_be64(&priv2->spu_chnldata_RW);
+ csa->spu_mailbox_data[i] = in_be64(&priv2->spu_chnldata_RW);
}
out_be64(&priv2->spu_chnlcnt_RW, 0UL);
eieio();
@@ -1667,7 +1667,7 @@ static inline void restore_spu_mb(struct
eieio();
out_be64(&priv2->spu_chnlcnt_RW, csa->spu_chnlcnt_RW[29]);
for (i = 0; i < 4; i++) {
- out_be64(&priv2->spu_chnldata_RW, csa->pu_mailbox_data[i]);
+ out_be64(&priv2->spu_chnldata_RW, csa->spu_mailbox_data[i]);
}
eieio();
}
@@ -2079,7 +2079,10 @@ int spu_save(struct spu_state *prev, str
acquire_spu_lock(spu); /* Step 1. */
rc = __do_spu_save(prev, spu); /* Steps 2-53. */
release_spu_lock(spu);
-
+ if (rc) {
+ panic("%s failed on SPU[%d], rc=%d.\n",
+ __func__, spu->number, rc);
+ }
return rc;
}
@@ -2098,34 +2101,31 @@ int spu_restore(struct spu_state *new, s
acquire_spu_lock(spu);
harvest(NULL, spu);
+ spu->stop_code = 0;
+ spu->dar = 0;
+ spu->dsisr = 0;
+ spu->slb_replace = 0;
+ spu->class_0_pending = 0;
rc = __do_spu_restore(new, spu);
release_spu_lock(spu);
-
+ if (rc) {
+ panic("%s failed on SPU[%d] rc=%d.\n",
+ __func__, spu->number, rc);
+ }
return rc;
}
/**
- * spu_switch - SPU context switch (save + restore).
- * @prev: pointer to SPU context save area, to be saved.
- * @new: pointer to SPU context save area, to be restored.
+ * spu_harvest - SPU harvest (reset) operation
* @spu: pointer to SPU iomem structure.
*
- * Perform save, then restore. Only harvest if the
- * save fails, as cleanup is otherwise not needed.
+ * Perform SPU harvest (reset) operation.
*/
-int spu_switch(struct spu_state *prev, struct spu_state *new, struct spu *spu)
+void spu_harvest(struct spu *spu)
{
- int rc;
-
- acquire_spu_lock(spu); /* Save, Step 1. */
- rc = __do_spu_save(prev, spu); /* Save, Steps 2-53. */
- if (rc != 0) {
- harvest(prev, spu);
- }
- rc = __do_spu_restore(new, spu);
+ acquire_spu_lock(spu);
+ harvest(NULL, spu);
release_spu_lock(spu);
-
- return rc;
}
static void init_prob(struct spu_state *csa)
@@ -2181,6 +2181,7 @@ static void init_priv2(struct spu_state
void spu_init_csa(struct spu_state *csa)
{
struct spu_lscsa *lscsa;
+ unsigned char *p;
if (!csa)
return;
@@ -2192,6 +2193,11 @@ void spu_init_csa(struct spu_state *csa)
memset(lscsa, 0, sizeof(struct spu_lscsa));
csa->lscsa = lscsa;
+ csa->register_lock = SPIN_LOCK_UNLOCKED;
+
+ /* Set LS pages reserved to allow for user-space mapping. */
+ for (p = lscsa->ls; p < lscsa->ls + LS_SIZE; p += PAGE_SIZE)
+ SetPageReserved(vmalloc_to_page(p));
init_prob(csa);
init_priv1(csa);
@@ -2200,5 +2206,10 @@ void spu_init_csa(struct spu_state *csa)
void spu_fini_csa(struct spu_state *csa)
{
+ /* Clear reserved bit before vfree. */
+ unsigned char *p;
+ for (p = csa->lscsa->ls; p < csa->lscsa->ls + LS_SIZE; p += PAGE_SIZE)
+ ClearPageReserved(vmalloc_to_page(p));
+
vfree(csa->lscsa);
}
Index: linux-2.6.15-rc/arch/powerpc/platforms/cell/spu_base.c
===================================================================
--- linux-2.6.15-rc.orig/arch/powerpc/platforms/cell/spu_base.c
+++ linux-2.6.15-rc/arch/powerpc/platforms/cell/spu_base.c
@@ -69,51 +69,49 @@ static void spu_restart_dma(struct spu *
static int __spu_trap_data_seg(struct spu *spu, unsigned long ea)
{
- struct spu_priv2 __iomem *priv2;
- struct mm_struct *mm;
+ struct spu_priv2 __iomem *priv2 = spu->priv2;
+ struct mm_struct *mm = spu->mm;
+ u64 esid, vsid;
pr_debug("%s\n", __FUNCTION__);
if (test_bit(SPU_CONTEXT_SWITCH_ACTIVE_nr, &spu->flags)) {
+ /* SLBs are pre-loaded for context switch, so
+ * we should never get here!
+ */
printk("%s: invalid access during switch!\n", __func__);
return 1;
}
-
- if (REGION_ID(ea) != USER_REGION_ID) {
+ if (!mm || (REGION_ID(ea) != USER_REGION_ID)) {
+ /* Future: support kernel segments so that drivers
+ * can use SPUs.
+ */
pr_debug("invalid region access at %016lx\n", ea);
return 1;
}
- priv2 = spu->priv2;
- mm = spu->mm;
+ esid = (ea & ESID_MASK) | SLB_ESID_V;
+ vsid = (get_vsid(mm->context.id, ea) << SLB_VSID_SHIFT) | SLB_VSID_USER;
+ if (in_hugepage_area(mm->context, ea))
+ vsid |= SLB_VSID_L;
+ out_be64(&priv2->slb_index_W, spu->slb_replace);
+ out_be64(&priv2->slb_vsid_RW, vsid);
+ out_be64(&priv2->slb_esid_RW, esid);
+
+ spu->slb_replace++;
if (spu->slb_replace >= 8)
spu->slb_replace = 0;
- out_be64(&priv2->slb_index_W, spu->slb_replace);
- out_be64(&priv2->slb_vsid_RW,
- (get_vsid(mm->context.id, ea) << SLB_VSID_SHIFT)
- | SLB_VSID_USER);
- out_be64(&priv2->slb_esid_RW, (ea & ESID_MASK) | SLB_ESID_V);
-
spu_restart_dma(spu);
- pr_debug("set slb %d context %lx, ea %016lx, vsid %016lx, esid %016lx\n",
- spu->slb_replace, mm->context.id, ea,
- (get_vsid(mm->context.id, ea) << SLB_VSID_SHIFT)| SLB_VSID_USER,
- (ea & ESID_MASK) | SLB_ESID_V);
return 0;
}
extern int hash_page(unsigned long ea, unsigned long access, unsigned long trap); //XXX
-static int __spu_trap_data_map(struct spu *spu, unsigned long ea)
+static int __spu_trap_data_map(struct spu *spu, unsigned long ea, u64 dsisr)
{
- unsigned long dsisr;
- struct spu_priv1 __iomem *priv1;
-
pr_debug("%s\n", __FUNCTION__);
- priv1 = spu->priv1;
- dsisr = in_be64(&priv1->mfc_dsisr_RW);
/* Handle kernel space hash faults immediately.
User hash faults need to be deferred to process context. */
@@ -129,14 +127,17 @@ static int __spu_trap_data_map(struct sp
return 1;
}
+ spu->dar = ea;
+ spu->dsisr = dsisr;
+ mb();
wake_up(&spu->stop_wq);
return 0;
}
static int __spu_trap_mailbox(struct spu *spu)
{
- wake_up_all(&spu->ibox_wq);
- kill_fasync(&spu->ibox_fasync, SIGIO, POLLIN);
+ if (spu->ibox_callback)
+ spu->ibox_callback(spu);
/* atomically disable SPU mailbox interrupts */
spin_lock(&spu->register_lock);
@@ -171,8 +172,8 @@ static int __spu_trap_tag_group(struct s
static int __spu_trap_spubox(struct spu *spu)
{
- wake_up_all(&spu->wbox_wq);
- kill_fasync(&spu->wbox_fasync, SIGIO, POLLOUT);
+ if (spu->wbox_callback)
+ spu->wbox_callback(spu);
/* atomically disable SPU mailbox interrupts */
spin_lock(&spu->register_lock);
@@ -220,17 +221,25 @@ static irqreturn_t
spu_irq_class_1(int irq, void *data, struct pt_regs *regs)
{
struct spu *spu;
- unsigned long stat, dar;
+ unsigned long stat, mask, dar, dsisr;
spu = data;
- stat = in_be64(&spu->priv1->int_stat_class1_RW);
+
+ /* atomically read & clear class1 status. */
+ spin_lock(&spu->register_lock);
+ mask = in_be64(&spu->priv1->int_mask_class1_RW);
+ stat = in_be64(&spu->priv1->int_stat_class1_RW) & mask;
dar = in_be64(&spu->priv1->mfc_dar_RW);
+ dsisr = in_be64(&spu->priv1->mfc_dsisr_RW);
+ out_be64(&spu->priv1->mfc_dsisr_RW, 0UL);
+ out_be64(&spu->priv1->int_stat_class1_RW, stat);
+ spin_unlock(&spu->register_lock);
if (stat & 1) /* segment fault */
__spu_trap_data_seg(spu, dar);
if (stat & 2) { /* mapping fault */
- __spu_trap_data_map(spu, dar);
+ __spu_trap_data_map(spu, dar, dsisr);
}
if (stat & 4) /* ls compare & suspend on get */
@@ -239,7 +248,6 @@ spu_irq_class_1(int irq, void *data, str
if (stat & 8) /* ls compare & suspend on put */
;
- out_be64(&spu->priv1->int_stat_class1_RW, stat);
return stat ? IRQ_HANDLED : IRQ_NONE;
}
@@ -396,8 +404,6 @@ EXPORT_SYMBOL(spu_alloc);
void spu_free(struct spu *spu)
{
down(&spu_mutex);
- spu->ibox_fasync = NULL;
- spu->wbox_fasync = NULL;
list_add_tail(&spu->list, &spu_list);
up(&spu_mutex);
}
@@ -405,15 +411,13 @@ EXPORT_SYMBOL(spu_free);
static int spu_handle_mm_fault(struct spu *spu)
{
- struct spu_priv1 __iomem *priv1;
struct mm_struct *mm = spu->mm;
struct vm_area_struct *vma;
u64 ea, dsisr, is_write;
int ret;
- priv1 = spu->priv1;
- ea = in_be64(&priv1->mfc_dar_RW);
- dsisr = in_be64(&priv1->mfc_dsisr_RW);
+ ea = spu->dar;
+ dsisr = spu->dsisr;
#if 0
if (!IS_VALID_EA(ea)) {
return -EFAULT;
@@ -476,15 +480,14 @@ bad_area:
static int spu_handle_pte_fault(struct spu *spu)
{
- struct spu_priv1 __iomem *priv1;
u64 ea, dsisr, access, error = 0UL;
int ret = 0;
- priv1 = spu->priv1;
- ea = in_be64(&priv1->mfc_dar_RW);
- dsisr = in_be64(&priv1->mfc_dsisr_RW);
- access = (_PAGE_PRESENT | _PAGE_USER);
+ ea = spu->dar;
+ dsisr = spu->dsisr;
if (dsisr & MFC_DSISR_PTE_NOT_FOUND) {
+ access = (_PAGE_PRESENT | _PAGE_USER);
+ access |= (dsisr & MFC_DSISR_ACCESS_PUT) ? _PAGE_RW : 0UL;
if (hash_page(ea, access, 0x300) != 0)
error |= CLASS1_ENABLE_STORAGE_FAULT_INTR;
}
@@ -495,18 +498,33 @@ static int spu_handle_pte_fault(struct s
else
error &= ~CLASS1_ENABLE_STORAGE_FAULT_INTR;
}
- if (!error)
+ spu->dar = 0UL;
+ spu->dsisr = 0UL;
+ if (!error) {
spu_restart_dma(spu);
-
+ } else {
+ __spu_trap_invalid_dma(spu);
+ }
return ret;
}
+static inline int spu_pending(struct spu *spu, u32 * stat)
+{
+ struct spu_problem __iomem *prob = spu->problem;
+ u64 pte_fault;
+
+ *stat = in_be32(&prob->spu_status_R);
+ pte_fault = spu->dsisr &
+ (MFC_DSISR_PTE_NOT_FOUND | MFC_DSISR_ACCESS_DENIED);
+ return (!(*stat & 0x1) || pte_fault || spu->class_0_pending) ? 1 : 0;
+}
+
int spu_run(struct spu *spu)
{
struct spu_problem __iomem *prob;
struct spu_priv1 __iomem *priv1;
struct spu_priv2 __iomem *priv2;
- unsigned long status;
+ u32 status;
int ret;
prob = spu->problem;
@@ -514,21 +532,15 @@ int spu_run(struct spu *spu)
priv2 = spu->priv2;
/* Let SPU run. */
- spu->mm = current->mm;
eieio();
out_be32(&prob->spu_runcntl_RW, SPU_RUNCNTL_RUNNABLE);
do {
ret = wait_event_interruptible(spu->stop_wq,
- (!((status = in_be32(&prob->spu_status_R)) & 0x1))
- || (in_be64(&priv1->mfc_dsisr_RW) & MFC_DSISR_PTE_NOT_FOUND)
- || spu->class_0_pending);
-
- if (status & SPU_STATUS_STOPPED_BY_STOP)
- ret = -EAGAIN;
- else if (status & SPU_STATUS_STOPPED_BY_HALT)
- ret = -EIO;
- else if (in_be64(&priv1->mfc_dsisr_RW) & MFC_DSISR_PTE_NOT_FOUND)
+ spu_pending(spu, &status));
+
+ if (spu->dsisr &
+ (MFC_DSISR_PTE_NOT_FOUND | MFC_DSISR_ACCESS_DENIED))
ret = spu_handle_pte_fault(spu);
if (spu->class_0_pending)
@@ -537,7 +549,9 @@ int spu_run(struct spu *spu)
if (!ret && signal_pending(current))
ret = -ERESTARTSYS;
- } while (!ret);
+ } while (!ret && !(status &
+ (SPU_STATUS_STOPPED_BY_STOP |
+ SPU_STATUS_STOPPED_BY_HALT)));
/* Ensure SPU is stopped. */
out_be32(&prob->spu_runcntl_RW, SPU_RUNCNTL_STOP);
@@ -549,8 +563,6 @@ int spu_run(struct spu *spu)
out_be64(&priv1->tlb_invalidate_entry_W, 0UL);
eieio();
- spu->mm = NULL;
-
/* Check for SPU breakpoint. */
if (unlikely(current->ptrace & PT_PTRACED)) {
status = in_be32(&prob->spu_status_R);
@@ -669,19 +681,21 @@ static int __init create_spu(struct devi
spu->stop_code = 0;
spu->slb_replace = 0;
spu->mm = NULL;
+ spu->ctx = NULL;
+ spu->rq = NULL;
+ spu->pid = 0;
spu->class_0_pending = 0;
spu->flags = 0UL;
+ spu->dar = 0UL;
+ spu->dsisr = 0UL;
spin_lock_init(&spu->register_lock);
out_be64(&spu->priv1->mfc_sdr_RW, mfspr(SPRN_SDR1));
out_be64(&spu->priv1->mfc_sr1_RW, 0x33);
init_waitqueue_head(&spu->stop_wq);
- init_waitqueue_head(&spu->wbox_wq);
- init_waitqueue_head(&spu->ibox_wq);
-
- spu->ibox_fasync = NULL;
- spu->wbox_fasync = NULL;
+ spu->ibox_callback = NULL;
+ spu->wbox_callback = NULL;
down(&spu_mutex);
spu->number = number++;
Index: linux-2.6.15-rc/arch/powerpc/platforms/cell/spufs/Makefile
===================================================================
--- linux-2.6.15-rc.orig/arch/powerpc/platforms/cell/spufs/Makefile
+++ linux-2.6.15-rc/arch/powerpc/platforms/cell/spufs/Makefile
@@ -1,6 +1,6 @@
obj-$(CONFIG_SPU_FS) += spufs.o
-
spufs-y += inode.o file.o context.o switch.o syscalls.o
+spufs-y += sched.o backing_ops.o hw_ops.o
# Rules to build switch.o with the help of SPU tool chain
SPU_CROSS := spu-
Index: linux-2.6.15-rc/arch/powerpc/platforms/cell/spufs/backing_ops.c
===================================================================
--- /dev/null
+++ linux-2.6.15-rc/arch/powerpc/platforms/cell/spufs/backing_ops.c
@@ -0,0 +1,252 @@
+/* backing_ops.c - query/set operations on saved SPU context.
+ *
+ * Copyright (C) IBM 2005
+ * Author: Mark Nutter <mnutter@us.ibm.com>
+ *
+ * These register operations allow SPUFS to operate on saved
+ * SPU contexts rather than hardware.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2, or (at your option)
+ * any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ */
+
+#include <linux/config.h>
+#include <linux/module.h>
+#include <linux/errno.h>
+#include <linux/sched.h>
+#include <linux/kernel.h>
+#include <linux/mm.h>
+#include <linux/vmalloc.h>
+#include <linux/smp.h>
+#include <linux/smp_lock.h>
+#include <linux/stddef.h>
+#include <linux/unistd.h>
+
+#include <asm/io.h>
+#include <asm/spu.h>
+#include <asm/spu_csa.h>
+#include <asm/mmu_context.h>
+#include "spufs.h"
+
+/*
+ * Reads/writes to various problem and priv2 registers require
+ * state changes, i.e. generate SPU events, modify channel
+ * counts, etc.
+ */
+
+static void gen_spu_event(struct spu_context *ctx, u32 event)
+{
+ u64 ch0_cnt;
+ u64 ch0_data;
+ u64 ch1_data;
+
+ ch0_cnt = ctx->csa.spu_chnlcnt_RW[0];
+ ch0_data = ctx->csa.spu_chnldata_RW[0];
+ ch1_data = ctx->csa.spu_chnldata_RW[1];
+ ctx->csa.spu_chnldata_RW[0] |= event;
+ if ((ch0_cnt == 0) && !(ch0_data & event) && (ch1_data & event)) {
+ ctx->csa.spu_chnlcnt_RW[0] = 1;
+ }
+}
+
+static int spu_backing_mbox_read(struct spu_context *ctx, u32 * data)
+{
+ u32 mbox_stat;
+ int ret = 0;
+
+ spin_lock(&ctx->csa.register_lock);
+ mbox_stat = ctx->csa.prob.mb_stat_R;
+ if (mbox_stat & 0x0000ff) {
+ /* Read the first available word.
+ * Implementation note: the depth
+ * of pu_mb_R is currently 1.
+ */
+ *data = ctx->csa.prob.pu_mb_R;
+ ctx->csa.prob.mb_stat_R &= ~(0x0000ff);
+ ctx->csa.spu_chnlcnt_RW[28] = 1;
+ gen_spu_event(ctx, MFC_PU_MAILBOX_AVAILABLE_EVENT);
+ ret = 4;
+ }
+ spin_unlock(&ctx->csa.register_lock);
+ return ret;
+}
+
+static u32 spu_backing_mbox_stat_read(struct spu_context *ctx)
+{
+ return ctx->csa.prob.mb_stat_R;
+}
+
+static int spu_backing_ibox_read(struct spu_context *ctx, u32 * data)
+{
+ int ret;
+
+ spin_lock(&ctx->csa.register_lock);
+ if (ctx->csa.prob.mb_stat_R & 0xff0000) {
+ /* Read the first available word.
+ * Implementation note: the depth
+ * of puint_mb_R is currently 1.
+ */
+ *data = ctx->csa.priv2.puint_mb_R;
+ ctx->csa.prob.mb_stat_R &= ~(0xff0000);
+ ctx->csa.spu_chnlcnt_RW[30] = 1;
+ gen_spu_event(ctx, MFC_PU_INT_MAILBOX_AVAILABLE_EVENT);
+ ret = 4;
+ } else {
+ /* make sure we get woken up by the interrupt */
+ ctx->csa.priv1.int_mask_class2_RW |= 0x1UL;
+ ret = 0;
+ }
+ spin_unlock(&ctx->csa.register_lock);
+ return ret;
+}
+
+static int spu_backing_wbox_write(struct spu_context *ctx, u32 data)
+{
+ int ret;
+
+ spin_lock(&ctx->csa.register_lock);
+ if ((ctx->csa.prob.mb_stat_R) & 0x00ff00) {
+ int slot = ctx->csa.spu_chnlcnt_RW[29];
+ int avail = (ctx->csa.prob.mb_stat_R & 0x00ff00) >> 8;
+
+ /* We have space to write wbox_data.
+ * Implementation note: the depth
+ * of spu_mb_W is currently 4.
+ */
+ BUG_ON(avail != (4 - slot));
+ ctx->csa.spu_mailbox_data[slot] = data;
+ ctx->csa.spu_chnlcnt_RW[29] = ++slot;
+ ctx->csa.prob.mb_stat_R = (((4 - slot) & 0xff) << 8);
+ gen_spu_event(ctx, MFC_SPU_MAILBOX_WRITTEN_EVENT);
+ ret = 4;
+ } else {
+ /* make sure we get woken up by the interrupt when space
+ becomes available */
+ ctx->csa.priv1.int_mask_class2_RW |= 0x10;
+ ret = 0;
+ }
+ spin_unlock(&ctx->csa.register_lock);
+ return ret;
+}
+
+static u32 spu_backing_signal1_read(struct spu_context *ctx)
+{
+ return ctx->csa.spu_chnldata_RW[3];
+}
+
+static void spu_backing_signal1_write(struct spu_context *ctx, u32 data)
+{
+ spin_lock(&ctx->csa.register_lock);
+ if (ctx->csa.priv2.spu_cfg_RW & 0x1)
+ ctx->csa.spu_chnldata_RW[3] |= data;
+ else
+ ctx->csa.spu_chnldata_RW[3] = data;
+ ctx->csa.spu_chnlcnt_RW[3] = 1;
+ gen_spu_event(ctx, MFC_SIGNAL_1_EVENT);
+ spin_unlock(&ctx->csa.register_lock);
+}
+
+static u32 spu_backing_signal2_read(struct spu_context *ctx)
+{
+ return ctx->csa.spu_chnldata_RW[4];
+}
+
+static void spu_backing_signal2_write(struct spu_context *ctx, u32 data)
+{
+ spin_lock(&ctx->csa.register_lock);
+ if (ctx->csa.priv2.spu_cfg_RW & 0x2)
+ ctx->csa.spu_chnldata_RW[4] |= data;
+ else
+ ctx->csa.spu_chnldata_RW[4] = data;
+ ctx->csa.spu_chnlcnt_RW[4] = 1;
+ gen_spu_event(ctx, MFC_SIGNAL_2_EVENT);
+ spin_unlock(&ctx->csa.register_lock);
+}
+
+static void spu_backing_signal1_type_set(struct spu_context *ctx, u64 val)
+{
+ u64 tmp;
+
+ spin_lock(&ctx->csa.register_lock);
+ tmp = ctx->csa.priv2.spu_cfg_RW;
+ if (val)
+ tmp |= 1;
+ else
+ tmp &= ~1;
+ ctx->csa.priv2.spu_cfg_RW = tmp;
+ spin_unlock(&ctx->csa.register_lock);
+}
+
+static u64 spu_backing_signal1_type_get(struct spu_context *ctx)
+{
+ return ((ctx->csa.priv2.spu_cfg_RW & 1) != 0);
+}
+
+static void spu_backing_signal2_type_set(struct spu_context *ctx, u64 val)
+{
+ u64 tmp;
+
+ spin_lock(&ctx->csa.register_lock);
+ tmp = ctx->csa.priv2.spu_cfg_RW;
+ if (val)
+ tmp |= 2;
+ else
+ tmp &= ~2;
+ ctx->csa.priv2.spu_cfg_RW = tmp;
+ spin_unlock(&ctx->csa.register_lock);
+}
+
+static u64 spu_backing_signal2_type_get(struct spu_context *ctx)
+{
+ return ((ctx->csa.priv2.spu_cfg_RW & 2) != 0);
+}
+
+static u32 spu_backing_npc_read(struct spu_context *ctx)
+{
+ return ctx->csa.prob.spu_npc_RW;
+}
+
+static void spu_backing_npc_write(struct spu_context *ctx, u32 val)
+{
+ ctx->csa.prob.spu_npc_RW = val;
+}
+
+static u32 spu_backing_status_read(struct spu_context *ctx)
+{
+ return ctx->csa.prob.spu_status_R;
+}
+
+static char *spu_backing_get_ls(struct spu_context *ctx)
+{
+ return ctx->csa.lscsa->ls;
+}
+
+struct spu_context_ops spu_backing_ops = {
+ .mbox_read = spu_backing_mbox_read,
+ .mbox_stat_read = spu_backing_mbox_stat_read,
+ .ibox_read = spu_backing_ibox_read,
+ .wbox_write = spu_backing_wbox_write,
+ .signal1_read = spu_backing_signal1_read,
+ .signal1_write = spu_backing_signal1_write,
+ .signal2_read = spu_backing_signal2_read,
+ .signal2_write = spu_backing_signal2_write,
+ .signal1_type_set = spu_backing_signal1_type_set,
+ .signal1_type_get = spu_backing_signal1_type_get,
+ .signal2_type_set = spu_backing_signal2_type_set,
+ .signal2_type_get = spu_backing_signal2_type_get,
+ .npc_read = spu_backing_npc_read,
+ .npc_write = spu_backing_npc_write,
+ .status_read = spu_backing_status_read,
+ .get_ls = spu_backing_get_ls,
+};
Index: linux-2.6.15-rc/arch/powerpc/platforms/cell/spufs/hw_ops.c
===================================================================
--- /dev/null
+++ linux-2.6.15-rc/arch/powerpc/platforms/cell/spufs/hw_ops.c
@@ -0,0 +1,206 @@
+/* hw_ops.c - query/set operations on active SPU context.
+ *
+ * Copyright (C) IBM 2005
+ * Author: Mark Nutter <mnutter@us.ibm.com>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2, or (at your option)
+ * any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ */
+
+#include <linux/config.h>
+#include <linux/module.h>
+#include <linux/errno.h>
+#include <linux/sched.h>
+#include <linux/kernel.h>
+#include <linux/mm.h>
+#include <linux/vmalloc.h>
+#include <linux/smp.h>
+#include <linux/smp_lock.h>
+#include <linux/stddef.h>
+#include <linux/unistd.h>
+
+#include <asm/io.h>
+#include <asm/spu.h>
+#include <asm/spu_csa.h>
+#include <asm/mmu_context.h>
+#include "spufs.h"
+
+static int spu_hw_mbox_read(struct spu_context *ctx, u32 * data)
+{
+ struct spu *spu = ctx->spu;
+ struct spu_problem __iomem *prob = spu->problem;
+ u32 mbox_stat;
+ int ret = 0;
+
+ spin_lock_irq(&spu->register_lock);
+ mbox_stat = in_be32(&prob->mb_stat_R);
+ if (mbox_stat & 0x0000ff) {
+ *data = in_be32(&prob->pu_mb_R);
+ ret = 4;
+ }
+ spin_unlock_irq(&spu->register_lock);
+ return ret;
+}
+
+static u32 spu_hw_mbox_stat_read(struct spu_context *ctx)
+{
+ return in_be32(&ctx->spu->problem->mb_stat_R);
+}
+
+static int spu_hw_ibox_read(struct spu_context *ctx, u32 * data)
+{
+ struct spu *spu = ctx->spu;
+ struct spu_problem __iomem *prob = spu->problem;
+ struct spu_priv1 __iomem *priv1 = spu->priv1;
+ struct spu_priv2 __iomem *priv2 = spu->priv2;
+ int ret;
+
+ spin_lock_irq(&spu->register_lock);
+ if (in_be32(&prob->mb_stat_R) & 0xff0000) {
+ /* read the first available word */
+ *data = in_be64(&priv2->puint_mb_R);
+ ret = 4;
+ } else {
+ /* make sure we get woken up by the interrupt */
+ out_be64(&priv1->int_mask_class2_RW,
+ in_be64(&priv1->int_mask_class2_RW) | 0x1);
+ ret = 0;
+ }
+ spin_unlock_irq(&spu->register_lock);
+ return ret;
+}
+
+static int spu_hw_wbox_write(struct spu_context *ctx, u32 data)
+{
+ struct spu *spu = ctx->spu;
+ struct spu_problem __iomem *prob = spu->problem;
+ struct spu_priv1 __iomem *priv1 = spu->priv1;
+ int ret;
+
+ spin_lock_irq(&spu->register_lock);
+ if (in_be32(&prob->mb_stat_R) & 0x00ff00) {
+ /* we have space to write wbox_data to */
+ out_be32(&prob->spu_mb_W, data);
+ ret = 4;
+ } else {
+ /* make sure we get woken up by the interrupt when space
+ becomes available */
+ out_be64(&priv1->int_mask_class2_RW,
+ in_be64(&priv1->int_mask_class2_RW) | 0x10);
+ ret = 0;
+ }
+ spin_unlock_irq(&spu->register_lock);
+ return ret;
+}
+
+static u32 spu_hw_signal1_read(struct spu_context *ctx)
+{
+ return in_be32(&ctx->spu->problem->signal_notify1);
+}
+
+static void spu_hw_signal1_write(struct spu_context *ctx, u32 data)
+{
+ out_be32(&ctx->spu->problem->signal_notify1, data);
+}
+
+static u32 spu_hw_signal2_read(struct spu_context *ctx)
+{
+ return in_be32(&ctx->spu->problem->signal_notify1);
+}
+
+static void spu_hw_signal2_write(struct spu_context *ctx, u32 data)
+{
+ out_be32(&ctx->spu->problem->signal_notify2, data);
+}
+
+static void spu_hw_signal1_type_set(struct spu_context *ctx, u64 val)
+{
+ struct spu *spu = ctx->spu;
+ struct spu_priv2 __iomem *priv2 = spu->priv2;
+ u64 tmp;
+
+ spin_lock_irq(&spu->register_lock);
+ tmp = in_be64(&priv2->spu_cfg_RW);
+ if (val)
+ tmp |= 1;
+ else
+ tmp &= ~1;
+ out_be64(&priv2->spu_cfg_RW, tmp);
+ spin_unlock_irq(&spu->register_lock);
+}
+
+static u64 spu_hw_signal1_type_get(struct spu_context *ctx)
+{
+ return ((in_be64(&ctx->spu->priv2->spu_cfg_RW) & 1) != 0);
+}
+
+static void spu_hw_signal2_type_set(struct spu_context *ctx, u64 val)
+{
+ struct spu *spu = ctx->spu;
+ struct spu_priv2 __iomem *priv2 = spu->priv2;
+ u64 tmp;
+
+ spin_lock_irq(&spu->register_lock);
+ tmp = in_be64(&priv2->spu_cfg_RW);
+ if (val)
+ tmp |= 2;
+ else
+ tmp &= ~2;
+ out_be64(&priv2->spu_cfg_RW, tmp);
+ spin_unlock_irq(&spu->register_lock);
+}
+
+static u64 spu_hw_signal2_type_get(struct spu_context *ctx)
+{
+ return ((in_be64(&ctx->spu->priv2->spu_cfg_RW) & 2) != 0);
+}
+
+static u32 spu_hw_npc_read(struct spu_context *ctx)
+{
+ return in_be32(&ctx->spu->problem->spu_npc_RW);
+}
+
+static void spu_hw_npc_write(struct spu_context *ctx, u32 val)
+{
+ out_be32(&ctx->spu->problem->spu_npc_RW, val);
+}
+
+static u32 spu_hw_status_read(struct spu_context *ctx)
+{
+ return in_be32(&ctx->spu->problem->spu_status_R);
+}
+
+static char *spu_hw_get_ls(struct spu_context *ctx)
+{
+ return ctx->spu->local_store;
+}
+
+struct spu_context_ops spu_hw_ops = {
+ .mbox_read = spu_hw_mbox_read,
+ .mbox_stat_read = spu_hw_mbox_stat_read,
+ .ibox_read = spu_hw_ibox_read,
+ .wbox_write = spu_hw_wbox_write,
+ .signal1_read = spu_hw_signal1_read,
+ .signal1_write = spu_hw_signal1_write,
+ .signal2_read = spu_hw_signal2_read,
+ .signal2_write = spu_hw_signal2_write,
+ .signal1_type_set = spu_hw_signal1_type_set,
+ .signal1_type_get = spu_hw_signal1_type_get,
+ .signal2_type_set = spu_hw_signal2_type_set,
+ .signal2_type_get = spu_hw_signal2_type_get,
+ .npc_read = spu_hw_npc_read,
+ .npc_write = spu_hw_npc_write,
+ .status_read = spu_hw_status_read,
+ .get_ls = spu_hw_get_ls,
+};
Index: linux-2.6.15-rc/arch/powerpc/platforms/cell/spufs/sched.c
===================================================================
--- /dev/null
+++ linux-2.6.15-rc/arch/powerpc/platforms/cell/spufs/sched.c
@@ -0,0 +1,419 @@
+/* sched.c - SPU scheduler.
+ *
+ * Copyright (C) IBM 2005
+ * Author: Mark Nutter <mnutter@us.ibm.com>
+ *
+ * SPU scheduler, based on Linux thread priority. For now use
+ * a simple "cooperative" yield model with no preemption. SPU
+ * scheduling will eventually be preemptive: When a thread with
+ * a higher static priority gets ready to run, then an active SPU
+ * context will be preempted and returned to the waitq.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2, or (at your option)
+ * any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ */
+
+#define DEBUG 1
+#include <linux/config.h>
+#include <linux/module.h>
+#include <linux/errno.h>
+#include <linux/sched.h>
+#include <linux/kernel.h>
+#include <linux/mm.h>
+#include <linux/completion.h>
+#include <linux/vmalloc.h>
+#include <linux/smp.h>
+#include <linux/smp_lock.h>
+#include <linux/stddef.h>
+#include <linux/unistd.h>
+
+#include <asm/io.h>
+#include <asm/mmu_context.h>
+#include <asm/spu.h>
+#include <asm/spu_csa.h>
+#include "spufs.h"
+
+#define SPU_BITMAP_SIZE (((MAX_PRIO+BITS_PER_LONG)/BITS_PER_LONG)+1)
+struct spu_prio_array {
+ atomic_t nr_blocked;
+ unsigned long bitmap[SPU_BITMAP_SIZE];
+ wait_queue_head_t waitq[MAX_PRIO];
+};
+
+/* spu_runqueue - This is the main runqueue data structure for SPUs. */
+struct spu_runqueue {
+ struct semaphore sem;
+ unsigned long nr_active;
+ unsigned long nr_idle;
+ unsigned long nr_switches;
+ struct list_head active_list;
+ struct list_head idle_list;
+ struct spu_prio_array prio;
+};
+
+static struct spu_runqueue *spu_runqueues = NULL;
+
+static inline struct spu_runqueue *spu_rq(void)
+{
+ /* Future: make this a per-NODE array,
+ * and use cpu_to_node(smp_processor_id())
+ */
+ return spu_runqueues;
+}
+
+static inline struct spu *del_idle(struct spu_runqueue *rq)
+{
+ struct spu *spu;
+
+ BUG_ON(rq->nr_idle <= 0);
+ BUG_ON(list_empty(&rq->idle_list));
+ /* Future: Move SPU out of low-power SRI state. */
+ spu = list_entry(rq->idle_list.next, struct spu, sched_list);
+ list_del_init(&spu->sched_list);
+ rq->nr_idle--;
+ return spu;
+}
+
+static inline void del_active(struct spu_runqueue *rq, struct spu *spu)
+{
+ BUG_ON(rq->nr_active <= 0);
+ BUG_ON(list_empty(&rq->active_list));
+ list_del_init(&spu->sched_list);
+ rq->nr_active--;
+}
+
+static inline void add_idle(struct spu_runqueue *rq, struct spu *spu)
+{
+ /* Future: Put SPU into low-power SRI state. */
+ list_add_tail(&spu->sched_list, &rq->idle_list);
+ rq->nr_idle++;
+}
+
+static inline void add_active(struct spu_runqueue *rq, struct spu *spu)
+{
+ rq->nr_active++;
+ rq->nr_switches++;
+ list_add_tail(&spu->sched_list, &rq->active_list);
+}
+
+static void prio_wakeup(struct spu_runqueue *rq)
+{
+ if (atomic_read(&rq->prio.nr_blocked) && rq->nr_idle) {
+ int best = sched_find_first_bit(rq->prio.bitmap);
+ if (best < MAX_PRIO) {
+ wait_queue_head_t *wq = &rq->prio.waitq[best];
+ wake_up_interruptible_nr(wq, 1);
+ }
+ }
+}
+
+static void prio_wait(struct spu_runqueue *rq, u64 flags)
+{
+ int prio = current->prio;
+ wait_queue_head_t *wq = &rq->prio.waitq[prio];
+ DEFINE_WAIT(wait);
+
+ __set_bit(prio, rq->prio.bitmap);
+ atomic_inc(&rq->prio.nr_blocked);
+ prepare_to_wait_exclusive(wq, &wait, TASK_INTERRUPTIBLE);
+ if (!signal_pending(current)) {
+ up(&rq->sem);
+ pr_debug("%s: pid=%d prio=%d\n", __FUNCTION__,
+ current->pid, current->prio);
+ schedule();
+ down(&rq->sem);
+ }
+ finish_wait(wq, &wait);
+ atomic_dec(&rq->prio.nr_blocked);
+ if (!waitqueue_active(wq))
+ __clear_bit(prio, rq->prio.bitmap);
+}
+
+static inline int is_best_prio(struct spu_runqueue *rq)
+{
+ int best_prio;
+
+ best_prio = sched_find_first_bit(rq->prio.bitmap);
+ return (current->prio < best_prio) ? 1 : 0;
+}
+
+static inline void mm_needs_global_tlbie(struct mm_struct *mm)
+{
+ /* Global TLBIE broadcast required with SPEs. */
+#if (NR_CPUS > 1)
+ __cpus_setall(&mm->cpu_vm_mask, NR_CPUS);
+#else
+ __cpus_setall(&mm->cpu_vm_mask, NR_CPUS+1); /* is this ok? */
+#endif
+}
+
+static inline void bind_context(struct spu *spu, struct spu_context *ctx)
+{
+ pr_debug("%s: pid=%d SPU=%d\n", __FUNCTION__, current->pid,
+ spu->number);
+ spu->ctx = ctx;
+ spu->flags = 0;
+ ctx->spu = spu;
+ ctx->ops = &spu_hw_ops;
+ spu->pid = current->pid;
+ spu->prio = current->prio;
+ spu->mm = ctx->owner;
+ mm_needs_global_tlbie(spu->mm);
+ spu->ibox_callback = spufs_ibox_callback;
+ spu->wbox_callback = spufs_wbox_callback;
+ mb();
+ spu_restore(&ctx->csa, spu);
+}
+
+static inline void unbind_context(struct spu *spu, struct spu_context *ctx)
+{
+ pr_debug("%s: unbind pid=%d SPU=%d\n", __FUNCTION__,
+ spu->pid, spu->number);
+ spu_save(&ctx->csa, spu);
+ ctx->state = SPU_STATE_SAVED;
+ spu->ibox_callback = NULL;
+ spu->wbox_callback = NULL;
+ spu->mm = NULL;
+ spu->pid = 0;
+ spu->prio = MAX_PRIO;
+ ctx->ops = &spu_backing_ops;
+ ctx->spu = NULL;
+ spu->ctx = NULL;
+}
+
+static struct spu *preempt_active(struct spu_runqueue *rq)
+{
+ struct list_head *p;
+ struct spu_context *ctx;
+ struct spu *spu;
+
+ /* Future: implement real preemption. For now just
+ * boot a lower priority ctx that is in "detached"
+ * state, i.e. on a processor but not currently in
+ * spu_run().
+ */
+ list_for_each(p, &rq->active_list) {
+ spu = list_entry(p, struct spu, sched_list);
+ if (current->prio < spu->prio) {
+ ctx = spu->ctx;
+ if (down_write_trylock(&ctx->state_sema)) {
+ if (ctx->state != SPU_STATE_RUNNABLE) {
+ up_write(&ctx->state_sema);
+ continue;
+ }
+ pr_debug("%s: booting pid=%d from SPU %d\n",
+ __FUNCTION__, spu->pid, spu->number);
+ del_active(rq, spu);
+ up(&rq->sem);
+ unbind_context(spu, ctx);
+ up_write(&ctx->state_sema);
+ return spu;
+ }
+ }
+ }
+ return NULL;
+}
+
+static struct spu *get_idle_spu(u64 flags)
+{
+ struct spu_runqueue *rq;
+ struct spu *spu = NULL;
+
+ rq = spu_rq();
+ down(&rq->sem);
+ for (;;) {
+ if (rq->nr_idle > 0) {
+ if (is_best_prio(rq)) {
+ /* Fall through. */
+ spu = del_idle(rq);
+ break;
+ } else {
+ prio_wakeup(rq);
+ up(&rq->sem);
+ yield();
+ if (signal_pending(current)) {
+ return NULL;
+ }
+ rq = spu_rq();
+ down(&rq->sem);
+ continue;
+ }
+ } else {
+ if (is_best_prio(rq)) {
+ if ((spu = preempt_active(rq)) != NULL)
+ return spu;
+ }
+ prio_wait(rq, flags);
+ if (signal_pending(current)) {
+ prio_wakeup(rq);
+ spu = NULL;
+ break;
+ }
+ continue;
+ }
+ }
+ up(&rq->sem);
+ return spu;
+}
+
+static void put_idle_spu(struct spu *spu)
+{
+ struct spu_runqueue *rq = spu->rq;
+
+ down(&rq->sem);
+ add_idle(rq, spu);
+ prio_wakeup(rq);
+ up(&rq->sem);
+}
+
+static int get_active_spu(struct spu *spu)
+{
+ struct spu_runqueue *rq = spu->rq;
+ struct list_head *p;
+ struct spu *tmp;
+ int rc = 0;
+
+ down(&rq->sem);
+ list_for_each(p, &rq->active_list) {
+ tmp = list_entry(p, struct spu, sched_list);
+ if (tmp == spu) {
+ del_active(rq, spu);
+ rc = 1;
+ break;
+ }
+ }
+ up(&rq->sem);
+ return rc;
+}
+
+static void put_active_spu(struct spu *spu)
+{
+ struct spu_runqueue *rq = spu->rq;
+
+ down(&rq->sem);
+ add_active(rq, spu);
+ up(&rq->sem);
+}
+
+/* Lock order:
+ * spu_activate() & spu_deactivate() require the
+ * caller to have down_write(&ctx->state_sema).
+ *
+ * The rq->sem is breifly held (inside or outside a
+ * given ctx lock) for list management, but is never
+ * held during save/restore.
+ */
+
+int spu_activate(struct spu_context *ctx, u64 flags)
+{
+ struct spu *spu;
+
+ if (ctx->spu)
+ return 0;
+ spu = get_idle_spu(flags);
+ if (!spu)
+ return (signal_pending(current)) ? -ERESTARTSYS : -EAGAIN;
+ bind_context(spu, ctx);
+ put_active_spu(spu);
+ return 0;
+}
+
+void spu_deactivate(struct spu_context *ctx)
+{
+ struct spu *spu;
+ int needs_idle;
+
+ spu = ctx->spu;
+ if (!spu)
+ return;
+ needs_idle = get_active_spu(spu);
+ unbind_context(spu, ctx);
+ if (needs_idle)
+ put_idle_spu(spu);
+}
+
+void spu_yield(struct spu_context *ctx)
+{
+ struct spu *spu;
+
+ if (!down_write_trylock(&ctx->state_sema))
+ return;
+ spu = ctx->spu;
+ if ((ctx->state == SPU_STATE_RUNNABLE) &&
+ (sched_find_first_bit(spu->rq->prio.bitmap) <= current->prio)) {
+ pr_debug("%s: yielding SPU %d\n", __FUNCTION__, spu->number);
+ spu_deactivate(ctx);
+ ctx->state = SPU_STATE_SAVED;
+ }
+ up_write(&ctx->state_sema);
+}
+
+int __init spu_sched_init(void)
+{
+ struct spu_runqueue *rq;
+ struct spu *spu;
+ int i;
+
+ rq = spu_runqueues = kmalloc(sizeof(struct spu_runqueue), GFP_KERNEL);
+ if (!rq) {
+ printk(KERN_WARNING "%s: Unable to allocate runqueues.\n",
+ __FUNCTION__);
+ return 1;
+ }
+ memset(rq, 0, sizeof(struct spu_runqueue));
+ init_MUTEX(&rq->sem);
+ INIT_LIST_HEAD(&rq->active_list);
+ INIT_LIST_HEAD(&rq->idle_list);
+ rq->nr_active = 0;
+ rq->nr_idle = 0;
+ rq->nr_switches = 0;
+ atomic_set(&rq->prio.nr_blocked, 0);
+ for (i = 0; i < MAX_PRIO; i++) {
+ init_waitqueue_head(&rq->prio.waitq[i]);
+ __clear_bit(i, rq->prio.bitmap);
+ }
+ __set_bit(MAX_PRIO, rq->prio.bitmap);
+ for (;;) {
+ spu = spu_alloc();
+ if (!spu)
+ break;
+ pr_debug("%s: adding SPU[%d]\n", __FUNCTION__, spu->number);
+ add_idle(rq, spu);
+ spu->rq = rq;
+ }
+ if (!rq->nr_idle) {
+ printk(KERN_WARNING "%s: No available SPUs.\n", __FUNCTION__);
+ kfree(rq);
+ return 1;
+ }
+ return 0;
+}
+
+void __exit spu_sched_exit(void)
+{
+ struct spu_runqueue *rq = spu_rq();
+ struct spu *spu;
+
+ if (!rq) {
+ printk(KERN_WARNING "%s: no runqueues!\n", __FUNCTION__);
+ return;
+ }
+ while (rq->nr_idle > 0) {
+ spu = del_idle(rq);
+ if (!spu)
+ break;
+ spu_free(spu);
+ }
+ kfree(rq);
+}
Index: linux-2.6.15-rc/arch/powerpc/platforms/cell/spufs/syscalls.c
===================================================================
--- linux-2.6.15-rc.orig/arch/powerpc/platforms/cell/spufs/syscalls.c
+++ linux-2.6.15-rc/arch/powerpc/platforms/cell/spufs/syscalls.c
@@ -36,7 +36,7 @@ long do_spu_run(struct file *filp, __u32
u32 npc, status;
ret = -EFAULT;
- if (get_user(npc, unpc))
+ if (get_user(npc, unpc) || get_user(status, ustatus))
goto out;
ret = -EINVAL;
@@ -46,13 +46,7 @@ long do_spu_run(struct file *filp, __u32
i = SPUFS_I(filp->f_dentry->d_inode);
ret = spufs_run_spu(filp, i->i_ctx, &npc, &status);
- if (ret ==-EAGAIN || ret == -EIO)
- ret = status;
-
- if (put_user(npc, unpc))
- ret = -EFAULT;
-
- if (ustatus && put_user(status, ustatus))
+ if (put_user(npc, unpc) || put_user(status, ustatus))
ret = -EFAULT;
out:
return ret;
Index: linux-2.6.15-rc/include/asm-powerpc/spu.h
===================================================================
--- linux-2.6.15-rc.orig/include/asm-powerpc/spu.h
+++ linux-2.6.15-rc/include/asm-powerpc/spu.h
@@ -105,6 +105,9 @@
#define SPU_CONTEXT_SWITCH_PENDING (1UL << SPU_CONTEXT_SWITCH_PENDING_nr)
#define SPU_CONTEXT_SWITCH_ACTIVE (1UL << SPU_CONTEXT_SWITCH_ACTIVE_nr)
+struct spu_context;
+struct spu_runqueue;
+
struct spu {
char *name;
unsigned long local_store_phys;
@@ -113,23 +116,28 @@ struct spu {
struct spu_priv1 __iomem *priv1;
struct spu_priv2 __iomem *priv2;
struct list_head list;
+ struct list_head sched_list;
int number;
u32 isrc;
u32 node;
u64 flags;
+ u64 dar;
+ u64 dsisr;
struct kref kref;
size_t ls_size;
unsigned int slb_replace;
struct mm_struct *mm;
+ struct spu_context *ctx;
+ struct spu_runqueue *rq;
+ pid_t pid;
+ int prio;
int class_0_pending;
spinlock_t register_lock;
u32 stop_code;
wait_queue_head_t stop_wq;
- wait_queue_head_t ibox_wq;
- wait_queue_head_t wbox_wq;
- struct fasync_struct *ibox_fasync;
- struct fasync_struct *wbox_fasync;
+ void (* wbox_callback)(struct spu *spu);
+ void (* ibox_callback)(struct spu *spu);
char irq_c0[8];
char irq_c1[8];
@@ -140,9 +148,6 @@ struct spu *spu_alloc(void);
void spu_free(struct spu *spu);
int spu_run(struct spu *spu);
-size_t spu_wbox_write(struct spu *spu, u32 data);
-size_t spu_ibox_read(struct spu *spu, u32 *data);
-
extern struct spufs_calls {
asmlinkage long (*create_thread)(const char __user *name,
unsigned int flags, mode_t mode);
Index: linux-2.6.15-rc/include/asm-powerpc/spu_csa.h
===================================================================
--- linux-2.6.15-rc.orig/include/asm-powerpc/spu_csa.h
+++ linux-2.6.15-rc/include/asm-powerpc/spu_csa.h
@@ -241,6 +241,7 @@ struct spu_state {
unsigned long suspend_time;
u64 slb_esid_RW[8];
u64 slb_vsid_RW[8];
+ spinlock_t register_lock;
};
extern void spu_init_csa(struct spu_state *csa);
--
^ permalink raw reply [flat|nested] 10+ messages in thread