[patch] mspec

[patch] mspec

[Jes Sorensen]

Hi,

Since today is the day of do_no_pfn, it is obviously also the time for
mspec! Here we go :)

Cheers,
Jes

This patch implements the special memory driver (mspec) based on the
do_no_pfn approach. The driver is currently used only on SN2 hardware
with special fetchop support but could be beneficial on other
architectures using the uncached mode.

Signed-off-by: Jes Sorensen <jes@sgi.com>

----

 drivers/char/Kconfig  |    8 
 drivers/char/Makefile |    1 
 drivers/char/mspec.c  |  422 ++++++++++++++++++++++++++++++++++++++++++++++++++
 3 files changed, 431 insertions(+)

Index: linux-2.6/drivers/char/Kconfig
===================================================================
--- linux-2.6.orig/drivers/char/Kconfig
+++ linux-2.6/drivers/char/Kconfig
@@ -422,6 +422,14 @@
          If you have an SGI Altix with an attached SABrick
          say Y or M here, otherwise say N.
 
+config MSPEC
+	tristate "Memory special operations driver"
+	depends on IA64
+	help
+	  If you have an ia64 and you want to enable memory special
+	  operations support (formerly known as fetchop), say Y here,
+	  otherwise say N.
+
 source "drivers/serial/Kconfig"
 
 config UNIX98_PTYS
Index: linux-2.6/drivers/char/Makefile
===================================================================
--- linux-2.6.orig/drivers/char/Makefile
+++ linux-2.6/drivers/char/Makefile
@@ -51,6 +51,7 @@
 obj-$(CONFIG_VIOTAPE)		+= viotape.o
 obj-$(CONFIG_HVCS)		+= hvcs.o
 obj-$(CONFIG_SGI_MBCS)		+= mbcs.o
+obj-$(CONFIG_MSPEC)		+= mspec.o
 
 obj-$(CONFIG_PRINTER)		+= lp.o
 obj-$(CONFIG_TIPAR)		+= tipar.o
Index: linux-2.6/drivers/char/mspec.c
===================================================================
--- /dev/null
+++ linux-2.6/drivers/char/mspec.c
@@ -0,0 +1,422 @@
+/*
+ * Copyright (C) 2001-2006 Silicon Graphics, Inc.  All rights
+ * reserved.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License
+ * as published by the Free Software Foundation.
+ */
+
+/*
+ * SN Platform Special Memory (mspec) Support
+ *
+ * This driver exports the SN special memory (mspec) facility to user
+ * processes.
+ * There are three types of memory made available thru this driver:
+ * fetchops, uncached and cached.
+ *
+ * Fetchops are atomic memory operations that are implemented in the
+ * memory controller on SGI SN hardware.
+ *
+ * Uncached are used for memory write combining feature of the ia64
+ * cpu.
+ *
+ * Cached are used for areas of memory that are used as cached addresses
+ * on our partition and used as uncached addresses from other partitions.
+ * Due to a design constraint of the SN2 Shub, you can not have processors
+ * on the same FSB perform both a cached and uncached reference to the
+ * same cache line.  These special memory cached regions prevent the
+ * kernel from ever dropping in a TLB entry and therefore prevent the
+ * processor from ever speculating a cache line from this page.
+ */
+
+#include <linux/config.h>
+#include <linux/types.h>
+#include <linux/kernel.h>
+#include <linux/module.h>
+#include <linux/init.h>
+#include <linux/errno.h>
+#include <linux/miscdevice.h>
+#include <linux/spinlock.h>
+#include <linux/mm.h>
+#include <linux/vmalloc.h>
+#include <linux/string.h>
+#include <linux/slab.h>
+#include <linux/numa.h>
+#include <asm/page.h>
+#include <asm/system.h>
+#include <asm/pgtable.h>
+#include <asm/atomic.h>
+#include <asm/tlbflush.h>
+#include <asm/uncached.h>
+#include <asm/sn/addrs.h>
+#include <asm/sn/arch.h>
+#include <asm/sn/mspec.h>
+#include <asm/sn/sn_cpuid.h>
+#include <asm/sn/io.h>
+#include <asm/sn/bte.h>
+#include <asm/sn/shubio.h>
+
+
+#define FETCHOP_ID	"SGI Fetchop,"
+#define CACHED_ID	"Cached,"
+#define UNCACHED_ID	"Uncached"
+#define REVISION	"4.0"
+#define MSPEC_BASENAME	"mspec"
+
+/*
+ * Page types allocated by the device.
+ */
+enum {
+	MSPEC_FETCHOP = 1,
+	MSPEC_CACHED,
+	MSPEC_UNCACHED
+};
+
+static int is_sn2;
+
+/*
+ * One of these structures is allocated when an mspec region is mmaped. The
+ * structure is pointed to by the vma->vm_private_data field in the vma struct.
+ * This structure is used to record the addresses of the mspec pages.
+ */
+struct vma_data {
+	atomic_t refcnt;	/* Number of vmas sharing the data. */
+	spinlock_t lock;	/* Serialize access to the vma. */
+	int count;		/* Number of pages allocated. */
+	int type;		/* Type of pages allocated. */
+	unsigned long maddr[0];	/* Array of MSPEC addresses. */
+};
+
+/* used on shub2 to clear FOP cache in the HUB */
+static unsigned long scratch_page[MAX_NUMNODES];
+#define SH2_AMO_CACHE_ENTRIES	4
+
+static inline int
+mspec_zero_block(unsigned long addr, int len)
+{
+	int status;
+
+	if (is_sn2) {
+		if (is_shub2()) {
+			int nid;
+			void *p;
+			int i;
+
+			nid = nasid_to_cnodeid(get_node_number(__pa(addr)));
+			p = (void *)TO_AMO(scratch_page[nid]);
+
+			for (i=0; i < SH2_AMO_CACHE_ENTRIES; i++) {
+				FETCHOP_LOAD_OP(p, FETCHOP_LOAD);
+				p += FETCHOP_VAR_SIZE;
+			}
+		}
+
+		status = bte_copy(0, addr & ~__IA64_UNCACHED_OFFSET, len,
+				  BTE_WACQUIRE | BTE_ZERO_FILL, NULL);
+	} else {
+		memset((char *) addr, 0, len);
+		status = 0;
+	}
+	return status;
+}
+
+/*
+ * mspec_open
+ *
+ * Called when a device mapping is created by a means other than mmap
+ * (via fork, etc.).  Increments the reference count on the underlying
+ * mspec data so it is not freed prematurely.
+ */
+static void
+mspec_open(struct vm_area_struct *vma)
+{
+	struct vma_data *vdata;
+
+	vdata = vma->vm_private_data;
+	atomic_inc(&vdata->refcnt);
+}
+
+/*
+ * mspec_close
+ *
+ * Called when unmapping a device mapping. Frees all mspec pages
+ * belonging to the vma.
+ */
+static void
+mspec_close(struct vm_area_struct *vma)
+{
+	struct vma_data *vdata;
+	int i, pages, result, vdata_size;
+
+	vdata = vma->vm_private_data;
+	if (!atomic_dec_and_test(&vdata->refcnt))
+		return;
+
+	pages = (vma->vm_end - vma->vm_start) >> PAGE_SHIFT;
+	vdata_size = sizeof(struct vma_data) + pages * sizeof(long);
+	for (i = 0; i < pages; i++) {
+		if (vdata->maddr[i] == 0)
+			continue;
+		/*
+		 * Clear the page before sticking it back
+		 * into the pool.
+		 */
+		result = mspec_zero_block(vdata->maddr[i], PAGE_SIZE);
+		if (!result)
+			uncached_free_page(vdata->maddr[i]);
+		else
+			printk(KERN_WARNING "mspec_close(): "
+			       "failed to zero page %i\n",
+			       result);
+	}
+
+	if (vdata_size <= PAGE_SIZE)
+		kfree(vdata);
+	else
+		vfree(vdata);
+}
+
+
+/*
+ * mspec_nopfn
+ *
+ * Creates a mspec page and maps it to user space.
+ */
+static unsigned long
+mspec_nopfn(struct vm_area_struct *vma, unsigned long address)
+{
+	unsigned long paddr, maddr;
+	unsigned long pfn;
+	int index;
+	struct vma_data *vdata = vma->vm_private_data;
+
+	index = (address - vma->vm_start) >> PAGE_SHIFT;
+	maddr = (volatile unsigned long) vdata->maddr[index];
+	if (maddr == 0) {
+		maddr = uncached_alloc_page(numa_node_id());
+		if (maddr == 0)
+			return NOPFN_OOM;
+
+		spin_lock(&vdata->lock);
+		if (vdata->maddr[index] == 0) {
+			vdata->count++;
+			vdata->maddr[index] = maddr;
+		} else {
+			uncached_free_page(maddr);
+			maddr = vdata->maddr[index];
+		}
+		spin_unlock(&vdata->lock);
+	}
+
+	if (vdata->type == MSPEC_FETCHOP)
+		paddr = TO_AMO(maddr);
+	else
+		paddr = __pa(TO_CAC(maddr));
+
+	pfn = paddr >> PAGE_SHIFT;
+
+	return pfn;
+}
+
+static struct vm_operations_struct mspec_vm_ops = {
+	.open = mspec_open,
+	.close = mspec_close,
+	.nopfn = mspec_nopfn
+};
+
+/*
+ * mspec_mmap
+ *
+ * Called when mmaping the device.  Initializes the vma with a fault handler
+ * and private data structure necessary to allocate, track, and free the
+ * underlying pages.
+ */
+static int
+mspec_mmap(struct file *file, struct vm_area_struct *vma, int type)
+{
+	struct vma_data *vdata;
+	int pages, vdata_size;
+
+	if (vma->vm_pgoff != 0)
+		return -EINVAL;
+
+	if ((vma->vm_flags & VM_SHARED) == 0)
+		return -EINVAL;
+
+	if ((vma->vm_flags & VM_WRITE) == 0)
+		return -EPERM;
+
+	pages = (vma->vm_end - vma->vm_start) >> PAGE_SHIFT;
+	vdata_size = sizeof(struct vma_data) + pages * sizeof(long);
+	if (vdata_size <= PAGE_SIZE)
+		vdata = kmalloc(vdata_size, GFP_KERNEL);
+	else
+		vdata = vmalloc(vdata_size);
+	if (!vdata)
+		return -ENOMEM;
+	memset(vdata, 0, vdata_size);
+
+	vdata->type = type;
+	spin_lock_init(&vdata->lock);
+	vdata->refcnt = ATOMIC_INIT(1);
+	vma->vm_private_data = vdata;
+
+	vma->vm_flags |= (VM_IO | VM_LOCKED | VM_RESERVED | VM_PFNMAP);
+	if (vdata->type == MSPEC_FETCHOP || vdata->type == MSPEC_UNCACHED)
+		vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot);
+	vma->vm_ops = &mspec_vm_ops;
+
+	return 0;
+}
+
+static int
+fetchop_mmap(struct file *file, struct vm_area_struct *vma)
+{
+	return mspec_mmap(file, vma, MSPEC_FETCHOP);
+}
+
+static int
+cached_mmap(struct file *file, struct vm_area_struct *vma)
+{
+	return mspec_mmap(file, vma, MSPEC_CACHED);
+}
+
+static int
+uncached_mmap(struct file *file, struct vm_area_struct *vma)
+{
+	return mspec_mmap(file, vma, MSPEC_UNCACHED);
+}
+
+static struct file_operations fetchop_fops = {
+	.owner = THIS_MODULE,
+	.mmap = fetchop_mmap
+};
+
+static struct miscdevice fetchop_miscdev = {
+	.minor = MISC_DYNAMIC_MINOR,
+	.name = "sgi_fetchop",
+	.fops = &fetchop_fops
+};
+
+static struct file_operations cached_fops = {
+	.owner = THIS_MODULE,
+	.mmap = cached_mmap
+};
+
+static struct miscdevice cached_miscdev = {
+	.minor = MISC_DYNAMIC_MINOR,
+	.name = "mspec_cached",
+	.fops = &cached_fops
+};
+
+static struct file_operations uncached_fops = {
+	.owner = THIS_MODULE,
+	.mmap = uncached_mmap
+};
+
+static struct miscdevice uncached_miscdev = {
+	.minor = MISC_DYNAMIC_MINOR,
+	.name = "mspec_uncached",
+	.fops = &uncached_fops
+};
+
+/*
+ * mspec_init
+ *
+ * Called at boot time to initialize the mspec facility.
+ */
+static int __init
+mspec_init(void)
+{
+	int ret;
+	int nid;
+
+	/*
+	 * The fetchop device only works on SN2 hardware, uncached and cached
+	 * memory drivers should both be valid on all ia64 hardware
+	 */
+	if (ia64_platform_is("sn2")) {
+		is_sn2 = 1;
+		if (is_shub2()) {
+			ret = -ENOMEM;
+			for_each_online_node(nid) {
+				int actual_nid;
+				int nasid;
+				unsigned long phys;
+
+				scratch_page[nid] = uncached_alloc_page(nid);
+				if (scratch_page[nid] == 0)
+					goto free_scratch_pages;
+				phys = __pa(scratch_page[nid]);
+				nasid = get_node_number(phys);
+				actual_nid = nasid_to_cnodeid(nasid);
+				if (actual_nid != nid)
+					goto free_scratch_pages;
+			}
+		}
+
+		ret = misc_register(&fetchop_miscdev);
+		if (ret) {
+			printk(KERN_ERR
+			       "%s: failed to register device %i\n",
+			       FETCHOP_ID, ret);
+			goto free_scratch_pages;
+		}
+	}
+	ret = misc_register(&cached_miscdev);
+	if (ret) {
+		printk(KERN_ERR "%s: failed to register device %i\n",
+		       CACHED_ID, ret);
+		if (is_sn2)
+			misc_deregister(&fetchop_miscdev);
+		goto free_scratch_pages;
+	}
+	ret = misc_register(&uncached_miscdev);
+	if (ret) {
+		printk(KERN_ERR "%s: failed to register device %i\n",
+		       UNCACHED_ID, ret);
+		misc_deregister(&cached_miscdev);
+		if (is_sn2)
+			misc_deregister(&fetchop_miscdev);
+		goto free_scratch_pages;
+	}
+
+	printk(KERN_INFO "%s %s initialized devices: %s %s %s\n",
+	       MSPEC_BASENAME, REVISION, is_sn2 ? FETCHOP_ID : "",
+	       CACHED_ID, UNCACHED_ID);
+
+	return 0;
+
+free_scratch_pages:
+	for_each_node(nid) {
+		if (scratch_page[nid] != 0)
+			uncached_free_page(scratch_page[nid]);
+	}
+	return ret;
+}
+
+static void __exit
+mspec_exit(void)
+{
+	int nid;
+
+	misc_deregister(&uncached_miscdev);
+	misc_deregister(&cached_miscdev);
+	if (is_sn2) {
+		misc_deregister(&fetchop_miscdev);
+
+		for_each_node(nid) {
+			if (scratch_page[nid] != 0)
+				uncached_free_page(scratch_page[nid]);
+		}
+	}
+}
+
+module_init(mspec_init);
+module_exit(mspec_exit);
+
+MODULE_AUTHOR("Silicon Graphics, Inc.");
+MODULE_DESCRIPTION("Driver for SGI SN special memory operations");
+MODULE_LICENSE("GPL");
+MODULE_INFO(supported, "external");

Re: [patch] mspec

[Randy.Dunlap]

On 19 Jun 2006 05:20:23 -0400 Jes Sorensen wrote:

> Hi,
> 
> Since today is the day of do_no_pfn, it is obviously also the time for
> mspec! Here we go :)

Just minor stuff:

> +MODULE_AUTHOR("Silicon Graphics, Inc.");
> +MODULE_DESCRIPTION("Driver for SGI SN special memory operations");
> +MODULE_LICENSE("GPL");
> +MODULE_INFO(supported, "external");

What does that last line mean?  what does "external" mean?
There are no other source files in the 2.6.17 tree that say
anything like that.

And of the 1800+ MODULE_AUTHOR() lines, SGI seems to be
dominant is using company instead of a person for AUTHOR.
Yes, there are a few others as well.
module.h says:
/* Author, ideally of form NAME <EMAIL>[, NAME <EMAIL>]*[ and NAME <EMAIL>] */

---
~Randy

Re: [patch] mspec

[Jes Sorensen]

Randy.Dunlap wrote:
> On 19 Jun 2006 05:20:23 -0400 Jes Sorensen wrote:

>> +MODULE_AUTHOR("Silicon Graphics, Inc.");
>> +MODULE_DESCRIPTION("Driver for SGI SN special memory operations");
>> +MODULE_LICENSE("GPL");
>> +MODULE_INFO(supported, "external");
> 
> What does that last line mean?  what does "external" mean?
> There are no other source files in the 2.6.17 tree that say
> anything like that.

Hi Randy,

I actually don't know where that comes from, we can just say supported.

> And of the 1800+ MODULE_AUTHOR() lines, SGI seems to be
> dominant is using company instead of a person for AUTHOR.
> Yes, there are a few others as well.

I inherited the code from someone else who probably inherited it parts
of it from somewhere else. However the point is that SGI as a company is
willing to take responsibility for the code.

Cheers,
Jes

Re: [patch] mspec

[Jes Sorensen]

>>>>> "Randy" == Randy Dunlap <rdunlap@xenotime.net> writes:

Randy> On 19 Jun 2006 05:20:23 -0400 Jes Sorensen wrote:
>> +MODULE_AUTHOR("Silicon Graphics, Inc.");
>> +MODULE_DESCRIPTION("Driver for SGI SN special memory operations");
>> +MODULE_LICENSE("GPL"); +MODULE_INFO(supported, "external");

Randy> What does that last line mean?  what does "external" mean?
Randy> There are no other source files in the 2.6.17 tree that say
Randy> anything like that.

Randy> And of the 1800+ MODULE_AUTHOR() lines, SGI seems to be
Randy> dominant is using company instead of a person for AUTHOR.  Yes,
Randy> there are a few others as well.  module.h says: /* Author,
Randy> ideally of form NAME <EMAIL>[, NAME <EMAIL>]*[ and NAME
Randy> <EMAIL>] */

Randy,

I went back and looked through the usages of these macros and it does
indeed seem that nobody else tries to state 'supported' in MODULE_INFO
so I have removed that and added an email address to the author line,
which should always stay valid.

Cheers,
Jes

This patch implements the special memory driver (mspec) based on the
do_no_pfn approach. The driver is currently used only on SN2 hardware
with special fetchop support but could be beneficial on other
architectures using the uncached mode.

Signed-off-by: Jes Sorensen <jes@sgi.com>

----

 drivers/char/Kconfig  |    8 
 drivers/char/Makefile |    1 
 drivers/char/mspec.c  |  421 ++++++++++++++++++++++++++++++++++++++++++++++++++
 3 files changed, 430 insertions(+)

Index: linux-2.6/drivers/char/Kconfig
===================================================================
--- linux-2.6.orig/drivers/char/Kconfig
+++ linux-2.6/drivers/char/Kconfig
@@ -422,6 +422,14 @@ config SGI_MBCS
          If you have an SGI Altix with an attached SABrick
          say Y or M here, otherwise say N.
 
+config MSPEC
+	tristate "Memory special operations driver"
+	depends on IA64
+	help
+	  If you have an ia64 and you want to enable memory special
+	  operations support (formerly known as fetchop), say Y here,
+	  otherwise say N.
+
 source "drivers/serial/Kconfig"
 
 config UNIX98_PTYS
Index: linux-2.6/drivers/char/Makefile
===================================================================
--- linux-2.6.orig/drivers/char/Makefile
+++ linux-2.6/drivers/char/Makefile
@@ -51,6 +51,7 @@ obj-$(CONFIG_VIOCONS)		+= viocons.o
 obj-$(CONFIG_VIOTAPE)		+= viotape.o
 obj-$(CONFIG_HVCS)		+= hvcs.o
 obj-$(CONFIG_SGI_MBCS)		+= mbcs.o
+obj-$(CONFIG_MSPEC)		+= mspec.o
 
 obj-$(CONFIG_PRINTER)		+= lp.o
 obj-$(CONFIG_TIPAR)		+= tipar.o
Index: linux-2.6/drivers/char/mspec.c
===================================================================
--- /dev/null
+++ linux-2.6/drivers/char/mspec.c
@@ -0,0 +1,421 @@
+/*
+ * Copyright (C) 2001-2006 Silicon Graphics, Inc.  All rights
+ * reserved.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License
+ * as published by the Free Software Foundation.
+ */
+
+/*
+ * SN Platform Special Memory (mspec) Support
+ *
+ * This driver exports the SN special memory (mspec) facility to user
+ * processes.
+ * There are three types of memory made available thru this driver:
+ * fetchops, uncached and cached.
+ *
+ * Fetchops are atomic memory operations that are implemented in the
+ * memory controller on SGI SN hardware.
+ *
+ * Uncached are used for memory write combining feature of the ia64
+ * cpu.
+ *
+ * Cached are used for areas of memory that are used as cached addresses
+ * on our partition and used as uncached addresses from other partitions.
+ * Due to a design constraint of the SN2 Shub, you can not have processors
+ * on the same FSB perform both a cached and uncached reference to the
+ * same cache line.  These special memory cached regions prevent the
+ * kernel from ever dropping in a TLB entry and therefore prevent the
+ * processor from ever speculating a cache line from this page.
+ */
+
+#include <linux/config.h>
+#include <linux/types.h>
+#include <linux/kernel.h>
+#include <linux/module.h>
+#include <linux/init.h>
+#include <linux/errno.h>
+#include <linux/miscdevice.h>
+#include <linux/spinlock.h>
+#include <linux/mm.h>
+#include <linux/vmalloc.h>
+#include <linux/string.h>
+#include <linux/slab.h>
+#include <linux/numa.h>
+#include <asm/page.h>
+#include <asm/system.h>
+#include <asm/pgtable.h>
+#include <asm/atomic.h>
+#include <asm/tlbflush.h>
+#include <asm/uncached.h>
+#include <asm/sn/addrs.h>
+#include <asm/sn/arch.h>
+#include <asm/sn/mspec.h>
+#include <asm/sn/sn_cpuid.h>
+#include <asm/sn/io.h>
+#include <asm/sn/bte.h>
+#include <asm/sn/shubio.h>
+
+
+#define FETCHOP_ID	"SGI Fetchop,"
+#define CACHED_ID	"Cached,"
+#define UNCACHED_ID	"Uncached"
+#define REVISION	"4.0"
+#define MSPEC_BASENAME	"mspec"
+
+/*
+ * Page types allocated by the device.
+ */
+enum {
+	MSPEC_FETCHOP = 1,
+	MSPEC_CACHED,
+	MSPEC_UNCACHED
+};
+
+static int is_sn2;
+
+/*
+ * One of these structures is allocated when an mspec region is mmaped. The
+ * structure is pointed to by the vma->vm_private_data field in the vma struct.
+ * This structure is used to record the addresses of the mspec pages.
+ */
+struct vma_data {
+	atomic_t refcnt;	/* Number of vmas sharing the data. */
+	spinlock_t lock;	/* Serialize access to the vma. */
+	int count;		/* Number of pages allocated. */
+	int type;		/* Type of pages allocated. */
+	unsigned long maddr[0];	/* Array of MSPEC addresses. */
+};
+
+/* used on shub2 to clear FOP cache in the HUB */
+static unsigned long scratch_page[MAX_NUMNODES];
+#define SH2_AMO_CACHE_ENTRIES	4
+
+static inline int
+mspec_zero_block(unsigned long addr, int len)
+{
+	int status;
+
+	if (is_sn2) {
+		if (is_shub2()) {
+			int nid;
+			void *p;
+			int i;
+
+			nid = nasid_to_cnodeid(get_node_number(__pa(addr)));
+			p = (void *)TO_AMO(scratch_page[nid]);
+
+			for (i=0; i < SH2_AMO_CACHE_ENTRIES; i++) {
+				FETCHOP_LOAD_OP(p, FETCHOP_LOAD);
+				p += FETCHOP_VAR_SIZE;
+			}
+		}
+
+		status = bte_copy(0, addr & ~__IA64_UNCACHED_OFFSET, len,
+				  BTE_WACQUIRE | BTE_ZERO_FILL, NULL);
+	} else {
+		memset((char *) addr, 0, len);
+		status = 0;
+	}
+	return status;
+}
+
+/*
+ * mspec_open
+ *
+ * Called when a device mapping is created by a means other than mmap
+ * (via fork, etc.).  Increments the reference count on the underlying
+ * mspec data so it is not freed prematurely.
+ */
+static void
+mspec_open(struct vm_area_struct *vma)
+{
+	struct vma_data *vdata;
+
+	vdata = vma->vm_private_data;
+	atomic_inc(&vdata->refcnt);
+}
+
+/*
+ * mspec_close
+ *
+ * Called when unmapping a device mapping. Frees all mspec pages
+ * belonging to the vma.
+ */
+static void
+mspec_close(struct vm_area_struct *vma)
+{
+	struct vma_data *vdata;
+	int i, pages, result, vdata_size;
+
+	vdata = vma->vm_private_data;
+	if (!atomic_dec_and_test(&vdata->refcnt))
+		return;
+
+	pages = (vma->vm_end - vma->vm_start) >> PAGE_SHIFT;
+	vdata_size = sizeof(struct vma_data) + pages * sizeof(long);
+	for (i = 0; i < pages; i++) {
+		if (vdata->maddr[i] == 0)
+			continue;
+		/*
+		 * Clear the page before sticking it back
+		 * into the pool.
+		 */
+		result = mspec_zero_block(vdata->maddr[i], PAGE_SIZE);
+		if (!result)
+			uncached_free_page(vdata->maddr[i]);
+		else
+			printk(KERN_WARNING "mspec_close(): "
+			       "failed to zero page %i\n",
+			       result);
+	}
+
+	if (vdata_size <= PAGE_SIZE)
+		kfree(vdata);
+	else
+		vfree(vdata);
+}
+
+
+/*
+ * mspec_nopfn
+ *
+ * Creates a mspec page and maps it to user space.
+ */
+static unsigned long
+mspec_nopfn(struct vm_area_struct *vma, unsigned long address)
+{
+	unsigned long paddr, maddr;
+	unsigned long pfn;
+	int index;
+	struct vma_data *vdata = vma->vm_private_data;
+
+	index = (address - vma->vm_start) >> PAGE_SHIFT;
+	maddr = (volatile unsigned long) vdata->maddr[index];
+	if (maddr == 0) {
+		maddr = uncached_alloc_page(numa_node_id());
+		if (maddr == 0)
+			return NOPFN_OOM;
+
+		spin_lock(&vdata->lock);
+		if (vdata->maddr[index] == 0) {
+			vdata->count++;
+			vdata->maddr[index] = maddr;
+		} else {
+			uncached_free_page(maddr);
+			maddr = vdata->maddr[index];
+		}
+		spin_unlock(&vdata->lock);
+	}
+
+	if (vdata->type == MSPEC_FETCHOP)
+		paddr = TO_AMO(maddr);
+	else
+		paddr = __pa(TO_CAC(maddr));
+
+	pfn = paddr >> PAGE_SHIFT;
+
+	return pfn;
+}
+
+static struct vm_operations_struct mspec_vm_ops = {
+	.open = mspec_open,
+	.close = mspec_close,
+	.nopfn = mspec_nopfn
+};
+
+/*
+ * mspec_mmap
+ *
+ * Called when mmaping the device.  Initializes the vma with a fault handler
+ * and private data structure necessary to allocate, track, and free the
+ * underlying pages.
+ */
+static int
+mspec_mmap(struct file *file, struct vm_area_struct *vma, int type)
+{
+	struct vma_data *vdata;
+	int pages, vdata_size;
+
+	if (vma->vm_pgoff != 0)
+		return -EINVAL;
+
+	if ((vma->vm_flags & VM_SHARED) == 0)
+		return -EINVAL;
+
+	if ((vma->vm_flags & VM_WRITE) == 0)
+		return -EPERM;
+
+	pages = (vma->vm_end - vma->vm_start) >> PAGE_SHIFT;
+	vdata_size = sizeof(struct vma_data) + pages * sizeof(long);
+	if (vdata_size <= PAGE_SIZE)
+		vdata = kmalloc(vdata_size, GFP_KERNEL);
+	else
+		vdata = vmalloc(vdata_size);
+	if (!vdata)
+		return -ENOMEM;
+	memset(vdata, 0, vdata_size);
+
+	vdata->type = type;
+	spin_lock_init(&vdata->lock);
+	vdata->refcnt = ATOMIC_INIT(1);
+	vma->vm_private_data = vdata;
+
+	vma->vm_flags |= (VM_IO | VM_LOCKED | VM_RESERVED | VM_PFNMAP);
+	if (vdata->type == MSPEC_FETCHOP || vdata->type == MSPEC_UNCACHED)
+		vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot);
+	vma->vm_ops = &mspec_vm_ops;
+
+	return 0;
+}
+
+static int
+fetchop_mmap(struct file *file, struct vm_area_struct *vma)
+{
+	return mspec_mmap(file, vma, MSPEC_FETCHOP);
+}
+
+static int
+cached_mmap(struct file *file, struct vm_area_struct *vma)
+{
+	return mspec_mmap(file, vma, MSPEC_CACHED);
+}
+
+static int
+uncached_mmap(struct file *file, struct vm_area_struct *vma)
+{
+	return mspec_mmap(file, vma, MSPEC_UNCACHED);
+}
+
+static struct file_operations fetchop_fops = {
+	.owner = THIS_MODULE,
+	.mmap = fetchop_mmap
+};
+
+static struct miscdevice fetchop_miscdev = {
+	.minor = MISC_DYNAMIC_MINOR,
+	.name = "sgi_fetchop",
+	.fops = &fetchop_fops
+};
+
+static struct file_operations cached_fops = {
+	.owner = THIS_MODULE,
+	.mmap = cached_mmap
+};
+
+static struct miscdevice cached_miscdev = {
+	.minor = MISC_DYNAMIC_MINOR,
+	.name = "mspec_cached",
+	.fops = &cached_fops
+};
+
+static struct file_operations uncached_fops = {
+	.owner = THIS_MODULE,
+	.mmap = uncached_mmap
+};
+
+static struct miscdevice uncached_miscdev = {
+	.minor = MISC_DYNAMIC_MINOR,
+	.name = "mspec_uncached",
+	.fops = &uncached_fops
+};
+
+/*
+ * mspec_init
+ *
+ * Called at boot time to initialize the mspec facility.
+ */
+static int __init
+mspec_init(void)
+{
+	int ret;
+	int nid;
+
+	/*
+	 * The fetchop device only works on SN2 hardware, uncached and cached
+	 * memory drivers should both be valid on all ia64 hardware
+	 */
+	if (ia64_platform_is("sn2")) {
+		is_sn2 = 1;
+		if (is_shub2()) {
+			ret = -ENOMEM;
+			for_each_online_node(nid) {
+				int actual_nid;
+				int nasid;
+				unsigned long phys;
+
+				scratch_page[nid] = uncached_alloc_page(nid);
+				if (scratch_page[nid] == 0)
+					goto free_scratch_pages;
+				phys = __pa(scratch_page[nid]);
+				nasid = get_node_number(phys);
+				actual_nid = nasid_to_cnodeid(nasid);
+				if (actual_nid != nid)
+					goto free_scratch_pages;
+			}
+		}
+
+		ret = misc_register(&fetchop_miscdev);
+		if (ret) {
+			printk(KERN_ERR
+			       "%s: failed to register device %i\n",
+			       FETCHOP_ID, ret);
+			goto free_scratch_pages;
+		}
+	}
+	ret = misc_register(&cached_miscdev);
+	if (ret) {
+		printk(KERN_ERR "%s: failed to register device %i\n",
+		       CACHED_ID, ret);
+		if (is_sn2)
+			misc_deregister(&fetchop_miscdev);
+		goto free_scratch_pages;
+	}
+	ret = misc_register(&uncached_miscdev);
+	if (ret) {
+		printk(KERN_ERR "%s: failed to register device %i\n",
+		       UNCACHED_ID, ret);
+		misc_deregister(&cached_miscdev);
+		if (is_sn2)
+			misc_deregister(&fetchop_miscdev);
+		goto free_scratch_pages;
+	}
+
+	printk(KERN_INFO "%s %s initialized devices: %s %s %s\n",
+	       MSPEC_BASENAME, REVISION, is_sn2 ? FETCHOP_ID : "",
+	       CACHED_ID, UNCACHED_ID);
+
+	return 0;
+
+ free_scratch_pages:
+	for_each_node(nid) {
+		if (scratch_page[nid] != 0)
+			uncached_free_page(scratch_page[nid]);
+	}
+	return ret;
+}
+
+static void __exit
+mspec_exit(void)
+{
+	int nid;
+
+	misc_deregister(&uncached_miscdev);
+	misc_deregister(&cached_miscdev);
+	if (is_sn2) {
+		misc_deregister(&fetchop_miscdev);
+
+		for_each_node(nid) {
+			if (scratch_page[nid] != 0)
+				uncached_free_page(scratch_page[nid]);
+		}
+	}
+}
+
+module_init(mspec_init);
+module_exit(mspec_exit);
+
+MODULE_AUTHOR("Silicon Graphics, Inc. <linux-altix@sgi.com>");
+MODULE_DESCRIPTION("Driver for SGI SN special memory operations");
+MODULE_LICENSE("GPL");

Re: [patch] mspec

[Robin Holt]

On Tue, Jun 20, 2006 at 10:06:37AM +0200, Jes Sorensen wrote:
> Randy.Dunlap wrote:
> > On 19 Jun 2006 05:20:23 -0400 Jes Sorensen wrote:
> 
> >> +MODULE_INFO(supported, "external");

That is a SuSE thing that keeps the tainted flag from being set.

Thanks,
Robin

Re: [patch] mspec

[Randy.Dunlap]

On 20 Jun 2006 06:23:16 -0400 Jes Sorensen wrote:

> >>>>> "Randy" == Randy Dunlap <rdunlap@xenotime.net> writes:
> 
> Randy> On 19 Jun 2006 05:20:23 -0400 Jes Sorensen wrote:
> >> +MODULE_AUTHOR("Silicon Graphics, Inc.");
> >> +MODULE_DESCRIPTION("Driver for SGI SN special memory operations");
> >> +MODULE_LICENSE("GPL"); +MODULE_INFO(supported, "external");
> 
> Randy> What does that last line mean?  what does "external" mean?
> Randy> There are no other source files in the 2.6.17 tree that say
> Randy> anything like that.
> 
> Randy> And of the 1800+ MODULE_AUTHOR() lines, SGI seems to be
> Randy> dominant is using company instead of a person for AUTHOR.  Yes,
> Randy> there are a few others as well.  module.h says: /* Author,
> Randy> ideally of form NAME <EMAIL>[, NAME <EMAIL>]*[ and NAME
> Randy> <EMAIL>] */
> 
> Randy,
> 
> I went back and looked through the usages of these macros and it does
> indeed seem that nobody else tries to state 'supported' in MODULE_INFO
> so I have removed that and added an email address to the author line,
> which should always stay valid.

Hi Jes,
Thanks for doing that, esp. the 'supported' part.


> +MODULE_AUTHOR("Silicon Graphics, Inc. <linux-altix@sgi.com>");
> +MODULE_DESCRIPTION("Driver for SGI SN special memory operations");
> +MODULE_LICENSE("GPL");

---
~Randy

Re: [patch] mspec

[Jes Sorensen]

Hi,

Latest mspec driver patch to go with do_no_pfn.

Cheers,
Jes

This patch implements the special memory driver (mspec) based on the
do_no_pfn approach. The driver is currently used only on SN2 hardware
with special fetchop support but could be beneficial on other
architectures using the uncached mode.

Signed-off-by: Jes Sorensen <jes@sgi.com>

----

 drivers/char/Kconfig  |    8 
 drivers/char/Makefile |    1 
 drivers/char/mspec.c  |  421 ++++++++++++++++++++++++++++++++++++++++++++++++++
 3 files changed, 430 insertions(+)

Index: linux-2.6/drivers/char/Kconfig
===================================================================
--- linux-2.6.orig/drivers/char/Kconfig
+++ linux-2.6/drivers/char/Kconfig
@@ -439,6 +439,14 @@ config SGI_MBCS
          If you have an SGI Altix with an attached SABrick
          say Y or M here, otherwise say N.
 
+config MSPEC
+	tristate "Memory special operations driver"
+	depends on IA64
+	help
+	  If you have an ia64 and you want to enable memory special
+	  operations support (formerly known as fetchop), say Y here,
+	  otherwise say N.
+
 source "drivers/serial/Kconfig"
 
 config UNIX98_PTYS
Index: linux-2.6/drivers/char/Makefile
===================================================================
--- linux-2.6.orig/drivers/char/Makefile
+++ linux-2.6/drivers/char/Makefile
@@ -51,6 +51,7 @@ obj-$(CONFIG_VIOCONS)		+= viocons.o
 obj-$(CONFIG_VIOTAPE)		+= viotape.o
 obj-$(CONFIG_HVCS)		+= hvcs.o
 obj-$(CONFIG_SGI_MBCS)		+= mbcs.o
+obj-$(CONFIG_MSPEC)		+= mspec.o
 
 obj-$(CONFIG_PRINTER)		+= lp.o
 obj-$(CONFIG_TIPAR)		+= tipar.o
Index: linux-2.6/drivers/char/mspec.c
===================================================================
--- /dev/null
+++ linux-2.6/drivers/char/mspec.c
@@ -0,0 +1,421 @@
+/*
+ * Copyright (C) 2001-2006 Silicon Graphics, Inc.  All rights
+ * reserved.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License
+ * as published by the Free Software Foundation.
+ */
+
+/*
+ * SN Platform Special Memory (mspec) Support
+ *
+ * This driver exports the SN special memory (mspec) facility to user
+ * processes.
+ * There are three types of memory made available thru this driver:
+ * fetchops, uncached and cached.
+ *
+ * Fetchops are atomic memory operations that are implemented in the
+ * memory controller on SGI SN hardware.
+ *
+ * Uncached are used for memory write combining feature of the ia64
+ * cpu.
+ *
+ * Cached are used for areas of memory that are used as cached addresses
+ * on our partition and used as uncached addresses from other partitions.
+ * Due to a design constraint of the SN2 Shub, you can not have processors
+ * on the same FSB perform both a cached and uncached reference to the
+ * same cache line.  These special memory cached regions prevent the
+ * kernel from ever dropping in a TLB entry and therefore prevent the
+ * processor from ever speculating a cache line from this page.
+ */
+
+#include <linux/config.h>
+#include <linux/types.h>
+#include <linux/kernel.h>
+#include <linux/module.h>
+#include <linux/init.h>
+#include <linux/errno.h>
+#include <linux/miscdevice.h>
+#include <linux/spinlock.h>
+#include <linux/mm.h>
+#include <linux/vmalloc.h>
+#include <linux/string.h>
+#include <linux/slab.h>
+#include <linux/numa.h>
+#include <asm/page.h>
+#include <asm/system.h>
+#include <asm/pgtable.h>
+#include <asm/atomic.h>
+#include <asm/tlbflush.h>
+#include <asm/uncached.h>
+#include <asm/sn/addrs.h>
+#include <asm/sn/arch.h>
+#include <asm/sn/mspec.h>
+#include <asm/sn/sn_cpuid.h>
+#include <asm/sn/io.h>
+#include <asm/sn/bte.h>
+#include <asm/sn/shubio.h>
+
+
+#define FETCHOP_ID	"SGI Fetchop,"
+#define CACHED_ID	"Cached,"
+#define UNCACHED_ID	"Uncached"
+#define REVISION	"4.0"
+#define MSPEC_BASENAME	"mspec"
+
+/*
+ * Page types allocated by the device.
+ */
+enum {
+	MSPEC_FETCHOP = 1,
+	MSPEC_CACHED,
+	MSPEC_UNCACHED
+};
+
+static int is_sn2;
+
+/*
+ * One of these structures is allocated when an mspec region is mmaped. The
+ * structure is pointed to by the vma->vm_private_data field in the vma struct.
+ * This structure is used to record the addresses of the mspec pages.
+ */
+struct vma_data {
+	atomic_t refcnt;	/* Number of vmas sharing the data. */
+	spinlock_t lock;	/* Serialize access to the vma. */
+	int count;		/* Number of pages allocated. */
+	int type;		/* Type of pages allocated. */
+	unsigned long maddr[0];	/* Array of MSPEC addresses. */
+};
+
+/* used on shub2 to clear FOP cache in the HUB */
+static unsigned long scratch_page[MAX_NUMNODES];
+#define SH2_AMO_CACHE_ENTRIES	4
+
+static inline int
+mspec_zero_block(unsigned long addr, int len)
+{
+	int status;
+
+	if (is_sn2) {
+		if (is_shub2()) {
+			int nid;
+			void *p;
+			int i;
+
+			nid = nasid_to_cnodeid(get_node_number(__pa(addr)));
+			p = (void *)TO_AMO(scratch_page[nid]);
+
+			for (i=0; i < SH2_AMO_CACHE_ENTRIES; i++) {
+				FETCHOP_LOAD_OP(p, FETCHOP_LOAD);
+				p += FETCHOP_VAR_SIZE;
+			}
+		}
+
+		status = bte_copy(0, addr & ~__IA64_UNCACHED_OFFSET, len,
+				  BTE_WACQUIRE | BTE_ZERO_FILL, NULL);
+	} else {
+		memset((char *) addr, 0, len);
+		status = 0;
+	}
+	return status;
+}
+
+/*
+ * mspec_open
+ *
+ * Called when a device mapping is created by a means other than mmap
+ * (via fork, etc.).  Increments the reference count on the underlying
+ * mspec data so it is not freed prematurely.
+ */
+static void
+mspec_open(struct vm_area_struct *vma)
+{
+	struct vma_data *vdata;
+
+	vdata = vma->vm_private_data;
+	atomic_inc(&vdata->refcnt);
+}
+
+/*
+ * mspec_close
+ *
+ * Called when unmapping a device mapping. Frees all mspec pages
+ * belonging to the vma.
+ */
+static void
+mspec_close(struct vm_area_struct *vma)
+{
+	struct vma_data *vdata;
+	int i, pages, result, vdata_size;
+
+	vdata = vma->vm_private_data;
+	if (!atomic_dec_and_test(&vdata->refcnt))
+		return;
+
+	pages = (vma->vm_end - vma->vm_start) >> PAGE_SHIFT;
+	vdata_size = sizeof(struct vma_data) + pages * sizeof(long);
+	for (i = 0; i < pages; i++) {
+		if (vdata->maddr[i] == 0)
+			continue;
+		/*
+		 * Clear the page before sticking it back
+		 * into the pool.
+		 */
+		result = mspec_zero_block(vdata->maddr[i], PAGE_SIZE);
+		if (!result)
+			uncached_free_page(vdata->maddr[i]);
+		else
+			printk(KERN_WARNING "mspec_close(): "
+			       "failed to zero page %i\n",
+			       result);
+	}
+
+	if (vdata_size <= PAGE_SIZE)
+		kfree(vdata);
+	else
+		vfree(vdata);
+}
+
+
+/*
+ * mspec_nopfn
+ *
+ * Creates a mspec page and maps it to user space.
+ */
+static unsigned long
+mspec_nopfn(struct vm_area_struct *vma, unsigned long address)
+{
+	unsigned long paddr, maddr;
+	unsigned long pfn;
+	int index;
+	struct vma_data *vdata = vma->vm_private_data;
+
+	index = (address - vma->vm_start) >> PAGE_SHIFT;
+	maddr = (volatile unsigned long) vdata->maddr[index];
+	if (maddr == 0) {
+		maddr = uncached_alloc_page(numa_node_id());
+		if (maddr == 0)
+			return NOPFN_OOM;
+
+		spin_lock(&vdata->lock);
+		if (vdata->maddr[index] == 0) {
+			vdata->count++;
+			vdata->maddr[index] = maddr;
+		} else {
+			uncached_free_page(maddr);
+			maddr = vdata->maddr[index];
+		}
+		spin_unlock(&vdata->lock);
+	}
+
+	if (vdata->type == MSPEC_FETCHOP)
+		paddr = TO_AMO(maddr);
+	else
+		paddr = __pa(TO_CAC(maddr));
+
+	pfn = paddr >> PAGE_SHIFT;
+
+	return pfn;
+}
+
+static struct vm_operations_struct mspec_vm_ops = {
+	.open = mspec_open,
+	.close = mspec_close,
+	.nopfn = mspec_nopfn
+};
+
+/*
+ * mspec_mmap
+ *
+ * Called when mmaping the device.  Initializes the vma with a fault handler
+ * and private data structure necessary to allocate, track, and free the
+ * underlying pages.
+ */
+static int
+mspec_mmap(struct file *file, struct vm_area_struct *vma, int type)
+{
+	struct vma_data *vdata;
+	int pages, vdata_size;
+
+	if (vma->vm_pgoff != 0)
+		return -EINVAL;
+
+	if ((vma->vm_flags & VM_SHARED) == 0)
+		return -EINVAL;
+
+	if ((vma->vm_flags & VM_WRITE) == 0)
+		return -EPERM;
+
+	pages = (vma->vm_end - vma->vm_start) >> PAGE_SHIFT;
+	vdata_size = sizeof(struct vma_data) + pages * sizeof(long);
+	if (vdata_size <= PAGE_SIZE)
+		vdata = kmalloc(vdata_size, GFP_KERNEL);
+	else
+		vdata = vmalloc(vdata_size);
+	if (!vdata)
+		return -ENOMEM;
+	memset(vdata, 0, vdata_size);
+
+	vdata->type = type;
+	spin_lock_init(&vdata->lock);
+	vdata->refcnt = ATOMIC_INIT(1);
+	vma->vm_private_data = vdata;
+
+	vma->vm_flags |= (VM_IO | VM_LOCKED | VM_RESERVED | VM_PFNMAP);
+	if (vdata->type == MSPEC_FETCHOP || vdata->type == MSPEC_UNCACHED)
+		vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot);
+	vma->vm_ops = &mspec_vm_ops;
+
+	return 0;
+}
+
+static int
+fetchop_mmap(struct file *file, struct vm_area_struct *vma)
+{
+	return mspec_mmap(file, vma, MSPEC_FETCHOP);
+}
+
+static int
+cached_mmap(struct file *file, struct vm_area_struct *vma)
+{
+	return mspec_mmap(file, vma, MSPEC_CACHED);
+}
+
+static int
+uncached_mmap(struct file *file, struct vm_area_struct *vma)
+{
+	return mspec_mmap(file, vma, MSPEC_UNCACHED);
+}
+
+static struct file_operations fetchop_fops = {
+	.owner = THIS_MODULE,
+	.mmap = fetchop_mmap
+};
+
+static struct miscdevice fetchop_miscdev = {
+	.minor = MISC_DYNAMIC_MINOR,
+	.name = "sgi_fetchop",
+	.fops = &fetchop_fops
+};
+
+static struct file_operations cached_fops = {
+	.owner = THIS_MODULE,
+	.mmap = cached_mmap
+};
+
+static struct miscdevice cached_miscdev = {
+	.minor = MISC_DYNAMIC_MINOR,
+	.name = "mspec_cached",
+	.fops = &cached_fops
+};
+
+static struct file_operations uncached_fops = {
+	.owner = THIS_MODULE,
+	.mmap = uncached_mmap
+};
+
+static struct miscdevice uncached_miscdev = {
+	.minor = MISC_DYNAMIC_MINOR,
+	.name = "mspec_uncached",
+	.fops = &uncached_fops
+};
+
+/*
+ * mspec_init
+ *
+ * Called at boot time to initialize the mspec facility.
+ */
+static int __init
+mspec_init(void)
+{
+	int ret;
+	int nid;
+
+	/*
+	 * The fetchop device only works on SN2 hardware, uncached and cached
+	 * memory drivers should both be valid on all ia64 hardware
+	 */
+	if (ia64_platform_is("sn2")) {
+		is_sn2 = 1;
+		if (is_shub2()) {
+			ret = -ENOMEM;
+			for_each_online_node(nid) {
+				int actual_nid;
+				int nasid;
+				unsigned long phys;
+
+				scratch_page[nid] = uncached_alloc_page(nid);
+				if (scratch_page[nid] == 0)
+					goto free_scratch_pages;
+				phys = __pa(scratch_page[nid]);
+				nasid = get_node_number(phys);
+				actual_nid = nasid_to_cnodeid(nasid);
+				if (actual_nid != nid)
+					goto free_scratch_pages;
+			}
+		}
+
+		ret = misc_register(&fetchop_miscdev);
+		if (ret) {
+			printk(KERN_ERR
+			       "%s: failed to register device %i\n",
+			       FETCHOP_ID, ret);
+			goto free_scratch_pages;
+		}
+	}
+	ret = misc_register(&cached_miscdev);
+	if (ret) {
+		printk(KERN_ERR "%s: failed to register device %i\n",
+		       CACHED_ID, ret);
+		if (is_sn2)
+			misc_deregister(&fetchop_miscdev);
+		goto free_scratch_pages;
+	}
+	ret = misc_register(&uncached_miscdev);
+	if (ret) {
+		printk(KERN_ERR "%s: failed to register device %i\n",
+		       UNCACHED_ID, ret);
+		misc_deregister(&cached_miscdev);
+		if (is_sn2)
+			misc_deregister(&fetchop_miscdev);
+		goto free_scratch_pages;
+	}
+
+	printk(KERN_INFO "%s %s initialized devices: %s %s %s\n",
+	       MSPEC_BASENAME, REVISION, is_sn2 ? FETCHOP_ID : "",
+	       CACHED_ID, UNCACHED_ID);
+
+	return 0;
+
+ free_scratch_pages:
+	for_each_node(nid) {
+		if (scratch_page[nid] != 0)
+			uncached_free_page(scratch_page[nid]);
+	}
+	return ret;
+}
+
+static void __exit
+mspec_exit(void)
+{
+	int nid;
+
+	misc_deregister(&uncached_miscdev);
+	misc_deregister(&cached_miscdev);
+	if (is_sn2) {
+		misc_deregister(&fetchop_miscdev);
+
+		for_each_node(nid) {
+			if (scratch_page[nid] != 0)
+				uncached_free_page(scratch_page[nid]);
+		}
+	}
+}
+
+module_init(mspec_init);
+module_exit(mspec_exit);
+
+MODULE_AUTHOR("Silicon Graphics, Inc. <linux-altix@sgi.com>");
+MODULE_DESCRIPTION("Driver for SGI SN special memory operations");
+MODULE_LICENSE("GPL");