Re: [PATCH 5/6] xen-gntalloc: Userspace grant allocation driver

[Daniel De Graaf <dgdegra@tycho.nsa.gov>]

[-- Attachment #1: Type: text/plain, Size: 4320 bytes --]

On 01/27/2011 02:23 PM, Konrad Rzeszutek Wilk wrote:
> On Thu, Jan 27, 2011 at 01:52:10PM -0500, Konrad Rzeszutek Wilk wrote:
>> On Fri, Jan 21, 2011 at 10:59:07AM -0500, Daniel De Graaf wrote:
>>> This allows a userspace application to allocate a shared page for
>>> implementing inter-domain communication or device drivers. These
>>> shared pages can be mapped using the gntdev device or by the kernel
>>> in another domain.
>>>
>>> Signed-off-by: Daniel De Graaf <dgdegra@tycho.nsa.gov>
>>> ---
>>>  drivers/xen/Kconfig    |    8 +
>>>  drivers/xen/Makefile   |    2 +
>>>  drivers/xen/gntalloc.c |  477 ++++++++++++++++++++++++++++++++++++++++++++++++
>>>  include/xen/gntalloc.h |   70 +++++++
>>>  4 files changed, 557 insertions(+), 0 deletions(-)
>>>  create mode 100644 drivers/xen/gntalloc.c
>>>  create mode 100644 include/xen/gntalloc.h
>>>
>>> diff --git a/drivers/xen/Kconfig b/drivers/xen/Kconfig
>>> index 07bec09..a3d7afb 100644
>>> --- a/drivers/xen/Kconfig
>>> +++ b/drivers/xen/Kconfig
>>> @@ -80,6 +80,14 @@ config XEN_GNTDEV
>>>  	help
>>>  	  Allows userspace processes to use grants.
>>>  
>>> +config XEN_GRANT_DEV_ALLOC
>>> +	tristate "User-space grant reference allocator driver"
>>> +	depends on XEN
>>> +	help
>>> +	  Allows userspace processes to create pages with access granted
>>> +	  to other domains. This can be used to implement frontend drivers
>>> +	  or as part of an inter-domain shared memory channel.
> 
> 
>>> +
>>>  config XEN_PLATFORM_PCI
>>>  	tristate "xen platform pci device driver"
>>>  	depends on XEN_PVHVM && PCI
>>> diff --git a/drivers/xen/Makefile b/drivers/xen/Makefile
>>> index 5088cc2..9585a1d 100644
>>> --- a/drivers/xen/Makefile
>>> +++ b/drivers/xen/Makefile
>>> @@ -10,6 +10,7 @@ obj-$(CONFIG_XEN_XENCOMM)	+= xencomm.o
>>>  obj-$(CONFIG_XEN_BALLOON)	+= balloon.o
>>>  obj-$(CONFIG_XEN_DEV_EVTCHN)	+= xen-evtchn.o
>>>  obj-$(CONFIG_XEN_GNTDEV)	+= xen-gntdev.o
>>> +obj-$(CONFIG_XEN_GRANT_DEV_ALLOC)	+= xen-gntalloc.o
>>>  obj-$(CONFIG_XENFS)		+= xenfs/
>>>  obj-$(CONFIG_XEN_SYS_HYPERVISOR)	+= sys-hypervisor.o
>>>  obj-$(CONFIG_XEN_PLATFORM_PCI)	+= xen-platform-pci.o
>>> @@ -18,5 +19,6 @@ obj-$(CONFIG_XEN_DOM0)		+= pci.o
>>>  
>>>  xen-evtchn-y			:= evtchn.o
>>>  xen-gntdev-y				:= gntdev.o
>>> +xen-gntalloc-y				:= gntalloc.o
>>>  
>>>  xen-platform-pci-y		:= platform-pci.o
>>> diff --git a/drivers/xen/gntalloc.c b/drivers/xen/gntalloc.c
>>> new file mode 100644
>>> index 0000000..a230dc4
>>> --- /dev/null
>>> +++ b/drivers/xen/gntalloc.c
>>> @@ -0,0 +1,477 @@
>>> +/******************************************************************************
>>> + * gntalloc.c
>>> + *
>>> + * Device for creating grant references (in user-space) that may be shared
>>> + * with other domains.
>>> + *
>>> + * This program is distributed in the hope that it will be useful,
>>> + * but WITHOUT ANY WARRANTY; without even the implied warranty of
>>> + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
>>> + * GNU General Public License for more details.
>>> + *
>>> + * You should have received a copy of the GNU General Public License
>>> + * along with this program; if not, write to the Free Software
>>> + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
>>> + */
>>> +
>>> +/*
>>> + * This driver exists to allow userspace programs in Linux to allocate kernel
>>> + * memory that will later be shared with another domain.  Without this device,
>>> + * Linux userspace programs cannot create grant references.
> 
> Why can't this be done in gntdev? Is there a simple test program for this?
> 

I didn't do this in gntdev so that it would be possible to allow programs to map
grants without also giving them rights to allocate (or vice versa). The actions
taken on the pages are also different (gntdev remaps them, gntalloc just shares)
so much of the surrounding map/unmap logic is different.

If it's preferred, I could try to add the ioctls to gntdev; there is some overlap
in functionality, especially when using the HVM code in gntdev. The limits for
gntalloc need to remain different, because a page allocated by gntalloc is
impossible to reclaim using the OOM killer (unlike pages allocated by gntdev).

Simple test program (for gntdev and gntalloc) attached.

-- 
Daniel De Graaf
National Security Agency

[-- Attachment #2: test_gnt.c --]
[-- Type: text/plain, Size: 5181 bytes --]

#include <errno.h>
#include <fcntl.h>
#include <unistd.h>
#include <stdio.h>
#include <string.h>
#include <stdint.h>
#include <sys/ioctl.h>
#include <sys/mman.h>
struct ioctl_gntdev_grant_ref {
	/* The domain ID of the grant to be mapped. */
	uint32_t domid;
	/* The grant reference of the grant to be mapped. */
	uint32_t ref;
};

/*
 * Allocates a new page and creates a new grant reference.
 */
#define IOCTL_GNTALLOC_ALLOC_GREF \
_IOC(_IOC_NONE, 'G', 5, sizeof(struct ioctl_gntalloc_alloc_gref))
struct ioctl_gntalloc_alloc_gref {
	/* IN parameters */
	/* The ID of the domain to be given access to the grants. */
	uint16_t domid;
	/* Flags for this mapping */
	uint16_t flags;
	/* Number of pages to map */
	uint32_t count;
	/* OUT parameters */
	/* The offset to be used on a subsequent call to mmap(). */
	uint64_t index;
	/* The grant references of the newly created grant, one per page */
	/* Variable size, depending on count */
	uint32_t gref_ids[1];
};

#define GNTALLOC_FLAG_WRITABLE 1

/*
 * Deallocates the grant reference, allowing the associated page to be freed if
 * no other domains are using it.
 */
#define IOCTL_GNTALLOC_DEALLOC_GREF \
_IOC(_IOC_NONE, 'G', 6, sizeof(struct ioctl_gntalloc_dealloc_gref))
struct ioctl_gntalloc_dealloc_gref {
	/* IN parameters */
	/* The offset returned in the map operation */
	uint64_t index;
	/* Number of references to unmap */
	uint32_t count;
};

#define IOCTL_GNTDEV_MAP_GRANT_REF \
_IOC(_IOC_NONE, 'G', 0, sizeof(struct ioctl_gntdev_map_grant_ref))
struct ioctl_gntdev_map_grant_ref {
    /* IN parameters */
    /* The number of grants to be mapped. */
    uint32_t count;
    uint32_t pad;
    /* OUT parameters */
    /* The offset to be used on a subsequent call to mmap(). */
    uint64_t index;
    /* Variable IN parameter. */
    /* Array of grant references, of size @count. */
    struct ioctl_gntdev_grant_ref refs[1];
};
#define GNTDEV_MAP_WRITABLE 0x1

#define IOCTL_GNTDEV_UNMAP_GRANT_REF \
_IOC(_IOC_NONE, 'G', 1, sizeof(struct ioctl_gntdev_unmap_grant_ref))       
struct ioctl_gntdev_unmap_grant_ref {
    /* IN parameters */
    /* The offset was returned by the corresponding map operation. */
    uint64_t index;
    /* The number of pages to be unmapped. */
    uint32_t count;
    uint32_t pad;
};


int a_fd;
int d_fd;

struct data {
	uint64_t* mem;
	int handle;
} items[128];

void sa(int id) {
	struct ioctl_gntalloc_alloc_gref arg = {
		.domid = id,
		.flags = GNTALLOC_FLAG_WRITABLE,
		.count = 1
	};
	int rv = ioctl(a_fd, IOCTL_GNTALLOC_ALLOC_GREF, &arg);
	if (rv)
		printf("src-add error: %s (rv=%d)\n", strerror(errno), rv);
	else {
		int i=0;
		while (items[i].mem) i++;
		items[i].mem = mmap(0, 4096, PROT_READ|PROT_WRITE, MAP_SHARED, a_fd, arg.index);
		items[i].handle = arg.index;
		printf("src-add mapped %d at %d=%ld\n", arg.gref_ids[0], arg.index, items[i].mem);
	}
}

void sd(int ref) {
	struct ioctl_gntalloc_dealloc_gref arg = {
		.index = ref,
		.count = 1
	};

	int rv = ioctl(a_fd, IOCTL_GNTALLOC_DEALLOC_GREF, &arg);
	if (rv)
		printf("src-del error: %s (rv=%d)\n", strerror(errno), rv);
}

void mm(int domid, int refid) {
	struct ioctl_gntdev_map_grant_ref arg = {
		.count = 1,
		.refs[0].domid = domid,
		.refs[0].ref = refid,
	};
	int rv = ioctl(d_fd, IOCTL_GNTDEV_MAP_GRANT_REF, &arg);
	if (rv)
		printf("mm error: %s (rv=%d)\n", strerror(errno), rv);
	else {
		int i=0;
		while (items[i].mem) i++;
		items[i].mem = mmap(0, 4096, PROT_READ|PROT_WRITE, MAP_SHARED, d_fd, arg.index);
		items[i].handle = arg.index;
		printf("mapped index %d at %ld\n", arg.index, items[i].mem);
	}
}

void gu(int index) {
	struct ioctl_gntdev_unmap_grant_ref arg = {
		.index = index,
		.count = 1,
	};
	int rv = ioctl(d_fd, IOCTL_GNTDEV_UNMAP_GRANT_REF, &arg);
	if (rv)
		printf("gu error: %s (rv=%d)\n", strerror(errno), rv);
}

void mu(void* addr) {
	int i = 0;
	munmap(addr, 4096);
	while (i < 128)
	{
		if (items[i].mem == addr)
			items[i].mem = 0;
		i++;
	}
}

int bump;

void show() {
	int i;
	for(i=0; i < 128; i++) {
		if (!items[i].mem)
			continue;
		uint64_t val = items[i].mem[0];
		uint64_t repl = val + bump;
		printf("%02d(%ld,%d): current %16lx   new %16lx\n", i, items[i].mem, items[i].handle, val, repl);
		items[i].mem[0] = repl;
	}
	printf("END\n");
}

int main(int argc, char** argv) {
	a_fd = open("/dev/xen/gntalloc", O_RDWR);
	d_fd = open("/dev/xen/gntdev", O_RDWR);
	bump = 1 << (2 * (getpid() % 12));
	printf(
		"src-add <domid>       return gntref, address\n"
		"map <domid> <ref>     return index, address\n"
		"src-del <gntref>      no rv\n"
		"gu <index>            no rv\n"
		"unmap <address>       no rv\n"
		"show                  print and change mapped items\n"
		" This process bumps by %x\n",
		bump
	);
	while (1) {
		char line[80];
		char word[30];
		long a, b;
		printf("\n> ");
		fflush(stdout);
		fgets(line, 80, stdin);
		sscanf(line, "%s %ld %ld", word, &a, &b);
		if (!strcmp(word, "src-add")) {
			sa(a);
		} else if (!strcmp(word, "src-del")) {
			sd(a);
		} else if (!strcmp(word, "map")) {
			mm(a, b);
		} else if (!strcmp(word, "gu")) {
			gu(a);
		} else if (!strcmp(word, "unmap")) {
			mu((void*)a);
		} else {
			show();
		}
	}
}

[-- Attachment #3: Type: text/plain, Size: 138 bytes --]

_______________________________________________
Xen-devel mailing list
Xen-devel@lists.xensource.com
http://lists.xensource.com/xen-devel