[PATCH] selfttest/powerpc: Add memory page migration tests

[PATCH] selfttest/powerpc: Add memory page migration tests

[Anshuman Khandual]

This adds two tests for memory page migration. One for normal page
migration which works for both 4K or 64K base page size kernel and
the other one is for 16MB huge page migration which will work both
4K or 64K base page sized 16MB huge pages as and when we support
huge page migration.

Signed-off-by: Anshuman Khandual <khandual@linux.vnet.ibm.com>
---
- Works for normal page migration on both 64K and 4K base pages
- Works for 16MB huge page migration (64K) on Aneesh's V2 PTE changes

 tools/testing/selftests/powerpc/mm/Makefile        |  14 +-
 .../selftests/powerpc/mm/hugepage-migration.c      |  30 ++++
 tools/testing/selftests/powerpc/mm/migration.h     | 196 +++++++++++++++++++++
 .../testing/selftests/powerpc/mm/page-migration.c  |  33 ++++
 tools/testing/selftests/powerpc/mm/run_mmtests     |  21 +++
 5 files changed, 289 insertions(+), 5 deletions(-)
 create mode 100644 tools/testing/selftests/powerpc/mm/hugepage-migration.c
 create mode 100644 tools/testing/selftests/powerpc/mm/migration.h
 create mode 100644 tools/testing/selftests/powerpc/mm/page-migration.c
 create mode 100755 tools/testing/selftests/powerpc/mm/run_mmtests

diff --git a/tools/testing/selftests/powerpc/mm/Makefile b/tools/testing/selftests/powerpc/mm/Makefile
index ee179e2..c482614 100644
--- a/tools/testing/selftests/powerpc/mm/Makefile
+++ b/tools/testing/selftests/powerpc/mm/Makefile
@@ -1,12 +1,16 @@
 noarg:
 	$(MAKE) -C ../
 
-TEST_PROGS := hugetlb_vs_thp_test subpage_prot
-TEST_FILES := tempfile
+TEST_PROGS := run_mmtests
+TEST_FILES := hugetlb_vs_thp_test
+TEST_FILES += subpage_prot
+TEST_FILES += tempfile
+TEST_FILES += hugepage-migration
+TEST_FILES += page-migration
 
-all: $(TEST_PROGS) $(TEST_FILES)
+all: $(TEST_FILES)
 
-$(TEST_PROGS): ../harness.c
+$(TEST_FILES): ../harness.c
 
 include ../../lib.mk
 
@@ -14,4 +18,4 @@ tempfile:
 	dd if=/dev/zero of=tempfile bs=64k count=1
 
 clean:
-	rm -f $(TEST_PROGS) tempfile
+	rm -f $(TEST_FILES)
diff --git a/tools/testing/selftests/powerpc/mm/hugepage-migration.c b/tools/testing/selftests/powerpc/mm/hugepage-migration.c
new file mode 100644
index 0000000..b60bc10
--- /dev/null
+++ b/tools/testing/selftests/powerpc/mm/hugepage-migration.c
@@ -0,0 +1,30 @@
+/*
+ * Copyright (C) 2015, Anshuman Khandual, IBM Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 as published
+ * by the Free Software Foundation.
+ */
+#include "migration.h"
+
+static int hugepage_migration(void)
+{
+	int ret = 0;
+
+	if ((unsigned long)getpagesize() == 0x1000)
+		printf("Running on base page size 4K\n");
+
+	if ((unsigned long)getpagesize() == 0x10000)
+		printf("Running on base page size 64K\n");
+
+	ret = test_huge_migration(16 * MEM_MB);
+	ret = test_huge_migration(256 * MEM_MB);
+	ret = test_huge_migration(512 * MEM_MB);
+
+	return ret;
+}
+
+int main(void)
+{
+	return test_harness(hugepage_migration, "hugepage_migration");
+}
diff --git a/tools/testing/selftests/powerpc/mm/migration.h b/tools/testing/selftests/powerpc/mm/migration.h
new file mode 100644
index 0000000..2f9e3f9
--- /dev/null
+++ b/tools/testing/selftests/powerpc/mm/migration.h
@@ -0,0 +1,196 @@
+/*
+ * Copyright (C) 2015, Anshuman Khandual, IBM Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 as published
+ * by the Free Software Foundation.
+ */
+#include <stdlib.h>
+#include <stdio.h>
+#include <string.h>
+#include <unistd.h>
+#include <sys/mman.h>
+#include <fcntl.h>
+
+#include "utils.h"
+
+#define HPAGE_OFF	0
+#define HPAGE_ON	1
+
+#define PAGE_SHIFT_4K	12
+#define PAGE_SHIFT_64K	16
+#define PAGE_SIZE_4K	0x1000
+#define PAGE_SIZE_64K	0x10000
+#define PAGE_SIZE_HUGE	16UL * 1024 * 1024
+
+#define MEM_GB		1024UL * 1024 * 1024
+#define MEM_MB		1024UL * 1024
+#define MME_KB		1024UL
+
+#define PMAP_FILE	"/proc/self/pagemap"
+#define PMAP_PFN	0x007FFFFFFFFFFFFFUL
+#define PMAP_SIZE	8
+
+#define SOFT_OFFLINE	"/sys/devices/system/memory/soft_offline_page"
+#define HARD_OFFLINE	"/sys/devices/system/memory/hard_offline_page"
+
+#define MMAP_LENGTH	(256 * MEM_MB)
+#define MMAP_ADDR	(void *)(0x0UL)
+#define MMAP_PROT	(PROT_READ | PROT_WRITE)
+#define MMAP_FLAGS	(MAP_PRIVATE | MAP_ANONYMOUS)
+#define MMAP_FLAGS_HUGE	(MAP_SHARED)
+
+#define FILE_NAME	"huge/hugepagefile"
+
+static void write_buffer(char *addr, unsigned long length)
+{
+	unsigned long i;
+
+	for (i = 0; i < length; i++)
+		*(addr + i) = (char)i;
+}
+
+static int read_buffer(char *addr, unsigned long length)
+{
+	unsigned long i;
+
+	for (i = 0; i < length; i++) {
+		if (*(addr + i) != (char)i) {
+			printf("Data miscompare at addr[%lu]\n", i);
+			return 1;
+		}
+	}
+	return 0;
+}
+
+static unsigned long get_npages(unsigned long length, unsigned long size)
+{
+	unsigned int tmp1 = length, tmp2 = size;
+
+	return tmp1/tmp2;
+}
+
+static void soft_offline_pages(int hugepage, void *addr,
+				unsigned long npages, unsigned long *failed)
+{
+	unsigned long psize, offset, pfn, pa, count;
+	void *tmp;
+	int i, fd1, fd2;
+	char buf[20];
+
+	fd1 = open(PMAP_FILE, O_RDONLY);
+	if (fd1 == -1) {
+		perror("open() failed");
+		exit(-1);
+	}
+
+	fd2 = open(SOFT_OFFLINE, O_WRONLY);
+	if (fd2 == -1) {
+		perror("open() failed");
+		exit(-1);
+	}
+
+	count = 0;
+	psize = getpagesize();
+	for (i = 0; i < npages; i++) {
+		if (hugepage)
+			tmp = addr + i * PAGE_SIZE_HUGE;
+		else
+			tmp = addr + i * psize;
+
+		offset = ((unsigned long) tmp / psize) * PMAP_SIZE;
+
+		if (lseek(fd1, offset, SEEK_SET) == -1) {
+			perror("lseek() failed");
+			exit(-1);
+		}
+
+		if (read(fd1, &pfn, sizeof(pfn)) == -1) {
+			perror("read() failed");
+			exit(-1);
+		}
+
+		pfn = pfn & PMAP_PFN;
+		if (psize == PAGE_SIZE_4K)
+			pa = pfn << PAGE_SHIFT_4K;
+
+		if (psize == PAGE_SIZE_64K)
+			pa = pfn << PAGE_SHIFT_64K;
+
+		sprintf(buf, "0x%lx\n", pa);
+
+		if (write(fd2, buf, strlen(buf)) == -1) {
+			perror("write() failed");
+			printf("%d PFN: %lx BUF: %s\n",i, pfn, buf);
+			count++;
+			exit(-1);
+		}
+
+	}
+
+	if (failed)
+		*failed = count;
+
+	close(fd1);
+	close(fd2);
+}
+
+int test_migration(unsigned long length)
+{
+	unsigned long failed;
+	void *addr;
+	int ret;
+
+	addr = mmap(MMAP_ADDR, length, MMAP_PROT, MMAP_FLAGS, -1, 0);
+	if (addr == MAP_FAILED) {
+		perror("mmap() failed");
+		exit(-1);
+	}
+
+	write_buffer(addr, length);
+	soft_offline_pages(HPAGE_OFF, addr, length/getpagesize(), &failed);
+	ret = read_buffer(addr, length);
+
+	printf("%ld pages moved, %ld pages failed\n", (length/getpagesize() - failed), failed);
+
+	munmap(addr, length);
+	return ret;
+}
+
+int test_huge_migration(unsigned long length)
+{
+	unsigned long failed, npages;
+	void *addr;
+	int fd, ret;
+
+	fd = open(FILE_NAME, O_CREAT | O_RDWR, 0755);
+	if (fd < 0) {
+		perror("open() failed");
+		exit(-1);
+	}
+
+	addr = mmap(MMAP_ADDR, length, MMAP_PROT, MMAP_FLAGS_HUGE, fd, 0);
+	if (addr == MAP_FAILED) {
+		perror("mmap() failed");
+		unlink(FILE_NAME);
+		exit(-1);
+	}
+
+        if (mlock(addr, length) == -1) {
+                perror("mlock() failed");
+		munmap(addr, length);
+                unlink(FILE_NAME);
+                exit(-1);
+        }
+
+	write_buffer(addr, length);
+	npages = get_npages(length, PAGE_SIZE_HUGE);
+	soft_offline_pages(HPAGE_ON, addr, npages, &failed);
+	ret = read_buffer(addr, length);
+
+	printf("%ld pages moved, %ld pages failed\n", (npages - failed), failed);
+
+	munmap(addr, length);
+	unlink(FILE_NAME);
+	return ret;
+}
diff --git a/tools/testing/selftests/powerpc/mm/page-migration.c b/tools/testing/selftests/powerpc/mm/page-migration.c
new file mode 100644
index 0000000..fc6e472
--- /dev/null
+++ b/tools/testing/selftests/powerpc/mm/page-migration.c
@@ -0,0 +1,33 @@
+/*
+ * Copyright (C) 2015, Anshuman Khandual, IBM Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 as published
+ * by the Free Software Foundation.
+ */
+#include "migration.h"
+
+static int page_migration(void)
+{
+	int ret = 0;
+
+	if ((unsigned long)getpagesize() == 0x1000)
+		printf("Running on base page size 4K\n");
+
+	if ((unsigned long)getpagesize() == 0x10000)
+		printf("Running on base page size 64K\n");
+
+	ret = test_migration(4 * MEM_MB);
+	ret = test_migration(64 * MEM_MB);
+	ret = test_migration(256 * MEM_MB);
+	ret = test_migration(512 * MEM_MB);
+	ret = test_migration(1 * MEM_GB);
+	ret = test_migration(2 * MEM_GB);
+
+	return ret;
+}
+
+int main(void)
+{
+	return test_harness(page_migration, "page_migration");
+}
diff --git a/tools/testing/selftests/powerpc/mm/run_mmtests b/tools/testing/selftests/powerpc/mm/run_mmtests
new file mode 100755
index 0000000..c537f4e
--- /dev/null
+++ b/tools/testing/selftests/powerpc/mm/run_mmtests
@@ -0,0 +1,21 @@
+#!/bin/bash
+mnt=./huge
+mkdir $mnt
+mount -t hugetlbfs none $mnt
+
+echo 256 > /proc/sys/vm/nr_hugepages
+echo "HugeTLB vs THP"
+./hugetlb_vs_thp_test
+
+echo "Subpage Protection"
+./subpage_prot
+
+echo "Test normal page migration"
+./page-migration
+
+echo "Test huge page migration"
+./hugepage-migration
+
+#cleanup
+umount $mnt
+rm -rf $mnt
-- 
2.1.0

Re: [PATCH] selfttest/powerpc: Add memory page migration tests

[Anshuman Khandual]

On 10/15/2015 09:16 PM, Anshuman Khandual wrote:
> This adds two tests for memory page migration. One for normal page
> migration which works for both 4K or 64K base page size kernel and
> the other one is for 16MB huge page migration which will work both
> 4K or 64K base page sized 16MB huge pages as and when we support
> huge page migration.
> 
> Signed-off-by: Anshuman Khandual <khandual@linux.vnet.ibm.com>
> ---
> - Works for normal page migration on both 64K and 4K base pages
> - Works for 16MB huge page migration (64K) on Aneesh's V2 PTE changes
> 
> +
> +int test_migration(unsigned long length)
> +{
> +	unsigned long failed;
> +	void *addr;
> +	int ret;
> +
> +	addr = mmap(MMAP_ADDR, length, MMAP_PROT, MMAP_FLAGS, -1, 0);
> +	if (addr == MAP_FAILED) {
> +		perror("mmap() failed");
> +		exit(-1);
> +	}

Will add a mlock() call here as well. Some times soft offline
fails while trying to move a huge chunk of memory on a system
because of swapping. With a mlock() in there the problem gets
resolved.

Re: [PATCH] selfttest/powerpc: Add memory page migration tests

[Aneesh Kumar K.V]

Anshuman Khandual <khandual@linux.vnet.ibm.com> writes:

> On 10/15/2015 09:16 PM, Anshuman Khandual wrote:
>> This adds two tests for memory page migration. One for normal page
>> migration which works for both 4K or 64K base page size kernel and
>> the other one is for 16MB huge page migration which will work both
>> 4K or 64K base page sized 16MB huge pages as and when we support
>> huge page migration.
>> 
>> Signed-off-by: Anshuman Khandual <khandual@linux.vnet.ibm.com>
>> ---
>> - Works for normal page migration on both 64K and 4K base pages
>> - Works for 16MB huge page migration (64K) on Aneesh's V2 PTE changes
>> 
>> +
>> +int test_migration(unsigned long length)
>> +{
>> +	unsigned long failed;
>> +	void *addr;
>> +	int ret;
>> +
>> +	addr = mmap(MMAP_ADDR, length, MMAP_PROT, MMAP_FLAGS, -1, 0);
>> +	if (addr == MAP_FAILED) {
>> +		perror("mmap() failed");
>> +		exit(-1);
>> +	}
>
> Will add a mlock() call here as well. Some times soft offline
> fails while trying to move a huge chunk of memory on a system
> because of swapping. With a mlock() in there the problem gets
> resolved.

Is that with respect to hugetlb pages ? I doubt, because hugetlb pages
are not swapped out.

-aneesh

Re: [PATCH] selfttest/powerpc: Add memory page migration tests

[Anshuman Khandual]

On 10/16/2015 05:09 PM, Aneesh Kumar K.V wrote:
> Anshuman Khandual <khandual@linux.vnet.ibm.com> writes:
> 
>> On 10/15/2015 09:16 PM, Anshuman Khandual wrote:
>>> This adds two tests for memory page migration. One for normal page
>>> migration which works for both 4K or 64K base page size kernel and
>>> the other one is for 16MB huge page migration which will work both
>>> 4K or 64K base page sized 16MB huge pages as and when we support
>>> huge page migration.
>>>
>>> Signed-off-by: Anshuman Khandual <khandual@linux.vnet.ibm.com>
>>> ---
>>> - Works for normal page migration on both 64K and 4K base pages
>>> - Works for 16MB huge page migration (64K) on Aneesh's V2 PTE changes
>>>
>>> +
>>> +int test_migration(unsigned long length)
>>> +{
>>> +	unsigned long failed;
>>> +	void *addr;
>>> +	int ret;
>>> +
>>> +	addr = mmap(MMAP_ADDR, length, MMAP_PROT, MMAP_FLAGS, -1, 0);
>>> +	if (addr == MAP_FAILED) {
>>> +		perror("mmap() failed");
>>> +		exit(-1);
>>> +	}
>>
>> Will add a mlock() call here as well. Some times soft offline
>> fails while trying to move a huge chunk of memory on a system
>> because of swapping. With a mlock() in there the problem gets
>> resolved.
> 
> Is that with respect to hugetlb pages ? I doubt, because hugetlb pages
> are not swapped out.

Thats correct. It is related to normal pages (4K or 64K). Have not
seen this swap problem of HugeTLB pages.

Re: [PATCH] selfttest/powerpc: Add memory page migration tests

[Michael Ellerman]

On Thu, 2015-10-15 at 21:16 +0530, Anshuman Khandual wrote:

> This adds two tests for memory page migration. One for normal page
> migration which works for both 4K or 64K base page size kernel and
> the other one is for 16MB huge page migration which will work both
> 4K or 64K base page sized 16MB huge pages as and when we support
> huge page migration.

> diff --git a/tools/testing/selftests/powerpc/mm/run_mmtests b/tools/testing/selftests/powerpc/mm/run_mmtests
> new file mode 100755
> index 0000000..c537f4e
> --- /dev/null
> +++ b/tools/testing/selftests/powerpc/mm/run_mmtests
> @@ -0,0 +1,21 @@
> +#!/bin/bash
> +mnt=./huge
> +mkdir $mnt
> +mount -t hugetlbfs none $mnt
> +
> +echo 256 > /proc/sys/vm/nr_hugepages
> +echo "HugeTLB vs THP"
> +./hugetlb_vs_thp_test
> +
> +echo "Subpage Protection"
> +./subpage_prot
> +
> +echo "Test normal page migration"
> +./page-migration
> +
> +echo "Test huge page migration"
> +./hugepage-migration
> +
> +#cleanup
> +umount $mnt
> +rm -rf $mnt

I'm not a fan of this script at all.

It looks like you borrowed the logic from selftests/vm/run_vmtests, but
stripped out all the error handling.

Please write it so that it attempts to do the hugepage setup, but if it fails,
prints a warning and then tries to run the tests anyway. That way if a regular
user runs the test, but hugepages are already setup, some of the tests can
still run.

It also needs to print if any of the tests fail, and the whole test needs to
return an error if any test fails.

cheers