* [PATCH 2/3] selftests: add membarrier syscall test
From: Mathieu Desnoyers @ 2015-07-10 20:58 UTC (permalink / raw)
To: Andrew Morton
Cc: linux-kernel, linux-api, Pranith Kumar, Michael Ellerman,
Mathieu Desnoyers
In-Reply-To: <1436561912-24365-1-git-send-email-mathieu.desnoyers@efficios.com>
From: Pranith Kumar <bobby.prani@gmail.com>
This patch adds a self test for the membarrier system call.
CC: Michael Ellerman <mpe@ellerman.id.au>
Signed-off-by: Pranith Kumar <bobby.prani@gmail.com>
Signed-off-by: Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
---
tools/testing/selftests/Makefile | 1 +
tools/testing/selftests/membarrier/.gitignore | 1 +
tools/testing/selftests/membarrier/Makefile | 11 ++++
.../testing/selftests/membarrier/membarrier_test.c | 71 ++++++++++++++++++++++
4 files changed, 84 insertions(+)
create mode 100644 tools/testing/selftests/membarrier/.gitignore
create mode 100644 tools/testing/selftests/membarrier/Makefile
create mode 100644 tools/testing/selftests/membarrier/membarrier_test.c
diff --git a/tools/testing/selftests/Makefile b/tools/testing/selftests/Makefile
index 24ae9e8..df577a4 100644
--- a/tools/testing/selftests/Makefile
+++ b/tools/testing/selftests/Makefile
@@ -6,6 +6,7 @@ TARGETS += firmware
TARGETS += ftrace
TARGETS += futex
TARGETS += kcmp
+TARGETS += membarrier
TARGETS += memfd
TARGETS += memory-hotplug
TARGETS += mount
diff --git a/tools/testing/selftests/membarrier/.gitignore b/tools/testing/selftests/membarrier/.gitignore
new file mode 100644
index 0000000..020c44f4
--- /dev/null
+++ b/tools/testing/selftests/membarrier/.gitignore
@@ -0,0 +1 @@
+membarrier_test
diff --git a/tools/testing/selftests/membarrier/Makefile b/tools/testing/selftests/membarrier/Makefile
new file mode 100644
index 0000000..877a503
--- /dev/null
+++ b/tools/testing/selftests/membarrier/Makefile
@@ -0,0 +1,11 @@
+CFLAGS += -g -I../../../../usr/include/
+
+all:
+ $(CC) $(CFLAGS) membarrier_test.c -o membarrier_test
+
+TEST_PROGS := membarrier_test
+
+include ../lib.mk
+
+clean:
+ $(RM) membarrier_test
diff --git a/tools/testing/selftests/membarrier/membarrier_test.c b/tools/testing/selftests/membarrier/membarrier_test.c
new file mode 100644
index 0000000..3c9f217
--- /dev/null
+++ b/tools/testing/selftests/membarrier/membarrier_test.c
@@ -0,0 +1,71 @@
+#define _GNU_SOURCE
+#define __EXPORTED_HEADERS__
+
+#include <linux/membarrier.h>
+#include <asm-generic/unistd.h>
+#include <sys/syscall.h>
+#include <stdio.h>
+#include <errno.h>
+#include <string.h>
+
+#include "../kselftest.h"
+
+static int sys_membarrier(int cmd, int flags)
+{
+ return syscall(__NR_membarrier, cmd, flags);
+}
+
+static void test_membarrier_fail(void)
+{
+ int cmd = -1, flags = 0;
+
+ if (sys_membarrier(cmd, flags) != -1) {
+ printf("membarrier: Should fail but passed\n");
+ ksft_exit_fail();
+ }
+}
+
+static void test_membarrier_success(void)
+{
+ int flags = 0;
+
+ if (sys_membarrier(MEMBARRIER_CMD_SHARED, flags) != 0) {
+ printf("membarrier: Executing MEMBARRIER failed, %s\n",
+ strerror(errno));
+ ksft_exit_fail();
+ }
+
+ printf("membarrier: MEMBARRIER_CMD_SHARED success\n");
+}
+
+static void test_membarrier(void)
+{
+ test_membarrier_fail();
+ test_membarrier_success();
+}
+
+static int test_membarrier_exists(void)
+{
+ int flags = 0;
+
+ if (sys_membarrier(MEMBARRIER_CMD_QUERY, flags))
+ return 0;
+
+ return 1;
+}
+
+int main(int argc, char **argv)
+{
+ printf("membarrier: MEMBARRIER_CMD_QUERY ");
+ if (test_membarrier_exists()) {
+ printf("syscall implemented\n");
+ test_membarrier();
+ } else {
+ printf("syscall not implemented!\n");
+ return ksft_exit_fail();
+ }
+
+ printf("membarrier: tests done!\n");
+
+ return ksft_exit_pass();
+}
--
2.1.4
^ permalink raw reply related
* [PATCH 3/3] selftests: enhance membarrier syscall test
From: Mathieu Desnoyers @ 2015-07-10 20:58 UTC (permalink / raw)
To: Andrew Morton
Cc: linux-kernel, linux-api, Mathieu Desnoyers, Michael Ellerman,
Pranith Kumar
In-Reply-To: <1436561912-24365-1-git-send-email-mathieu.desnoyers@efficios.com>
Update the membarrier syscall self-test to match the membarrier
interface. Extend coverage of the interface. Consider ENOSYS as a "SKIP"
test, since it is a valid configuration, but does not allow testing the
system call.
CC: Michael Ellerman <mpe@ellerman.id.au>
CC: Pranith Kumar <bobby.prani@gmail.com>
Signed-off-by: Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
---
.../testing/selftests/membarrier/membarrier_test.c | 100 +++++++++++++++------
1 file changed, 75 insertions(+), 25 deletions(-)
diff --git a/tools/testing/selftests/membarrier/membarrier_test.c b/tools/testing/selftests/membarrier/membarrier_test.c
index 3c9f217..dde3125 100644
--- a/tools/testing/selftests/membarrier/membarrier_test.c
+++ b/tools/testing/selftests/membarrier/membarrier_test.c
@@ -10,62 +10,112 @@
#include "../kselftest.h"
+enum test_membarrier_status {
+ TEST_MEMBARRIER_PASS = 0,
+ TEST_MEMBARRIER_FAIL,
+ TEST_MEMBARRIER_SKIP,
+};
+
static int sys_membarrier(int cmd, int flags)
{
return syscall(__NR_membarrier, cmd, flags);
}
-static void test_membarrier_fail(void)
+static enum test_membarrier_status test_membarrier_cmd_fail(void)
{
int cmd = -1, flags = 0;
if (sys_membarrier(cmd, flags) != -1) {
- printf("membarrier: Should fail but passed\n");
- ksft_exit_fail();
+ printf("membarrier: Wrong command should fail but passed.\n");
+ return TEST_MEMBARRIER_FAIL;
+ }
+ return TEST_MEMBARRIER_PASS;
+}
+
+static enum test_membarrier_status test_membarrier_flags_fail(void)
+{
+ int cmd = MEMBARRIER_CMD_QUERY, flags = 1;
+
+ if (sys_membarrier(cmd, flags) != -1) {
+ printf("membarrier: Wrong flags should fail but passed.\n");
+ return TEST_MEMBARRIER_FAIL;
}
+ return TEST_MEMBARRIER_PASS;
}
-static void test_membarrier_success(void)
+static enum test_membarrier_status test_membarrier_success(void)
{
- int flags = 0;
+ int cmd = MEMBARRIER_CMD_SHARED, flags = 0;
- if (sys_membarrier(MEMBARRIER_CMD_SHARED, flags) != 0) {
- printf("membarrier: Executing MEMBARRIER failed, %s\n",
+ if (sys_membarrier(cmd, flags) != 0) {
+ printf("membarrier: Executing MEMBARRIER_CMD_SHARED failed. %s.\n",
strerror(errno));
- ksft_exit_fail();
+ return TEST_MEMBARRIER_FAIL;
}
- printf("membarrier: MEMBARRIER_CMD_SHARED success\n");
+ printf("membarrier: MEMBARRIER_CMD_SHARED success.\n");
+ return TEST_MEMBARRIER_PASS;
}
-static void test_membarrier(void)
+static enum test_membarrier_status test_membarrier(void)
{
- test_membarrier_fail();
- test_membarrier_success();
+ enum test_membarrier_status status;
+
+ status = test_membarrier_cmd_fail();
+ if (status)
+ return status;
+ status = test_membarrier_flags_fail();
+ if (status)
+ return status;
+ status = test_membarrier_success();
+ if (status)
+ return status;
+ return TEST_MEMBARRIER_PASS;
}
-static int test_membarrier_exists(void)
+static enum test_membarrier_status test_membarrier_query(void)
{
- int flags = 0;
-
- if (sys_membarrier(MEMBARRIER_CMD_QUERY, flags))
- return 0;
+ int flags = 0, ret;
- return 1;
+ printf("membarrier MEMBARRIER_CMD_QUERY ");
+ ret = sys_membarrier(MEMBARRIER_CMD_QUERY, flags);
+ if (ret < 0) {
+ printf("failed. %s.\n", strerror(errno));
+ switch (errno) {
+ case ENOSYS:
+ /*
+ * It is valid to build a kernel with
+ * CONFIG_MEMBARRIER=n. However, this skips the tests.
+ */
+ return TEST_MEMBARRIER_SKIP;
+ case EINVAL:
+ default:
+ return TEST_MEMBARRIER_FAIL;
+ }
+ }
+ if (!(ret & MEMBARRIER_CMD_SHARED)) {
+ printf("command MEMBARRIER_CMD_SHARED is not supported.\n");
+ return TEST_MEMBARRIER_FAIL;
+ }
+ printf("syscall available.\n");
+ return TEST_MEMBARRIER_PASS;
}
int main(int argc, char **argv)
{
- printf("membarrier: MEMBARRIER_CMD_QUERY ");
- if (test_membarrier_exists()) {
- printf("syscall implemented\n");
- test_membarrier();
- } else {
- printf("syscall not implemented!\n");
+ switch (test_membarrier_query()) {
+ case TEST_MEMBARRIER_FAIL:
return ksft_exit_fail();
+ case TEST_MEMBARRIER_SKIP:
+ return ksft_exit_skip();
+ }
+ switch (test_membarrier()) {
+ case TEST_MEMBARRIER_FAIL:
+ return ksft_exit_fail();
+ case TEST_MEMBARRIER_SKIP:
+ return ksft_exit_skip();
}
printf("membarrier: tests done!\n");
-
return ksft_exit_pass();
}
--
2.1.4
^ permalink raw reply related
* Good Day
From: abdullahmohad02 @ 2015-07-10 22:20 UTC (permalink / raw)
Good Day
I know that this mail will come to you as a surprise as we have never
met before, but need not to worry as I am contacting you independently
of my Investigation and no one is informed of this communication. I
need your Urgent assistance in transferring the sum of $7.5 million
immediately to your Private account. The money has been here in our
Bank lying dormant for years now without anybody coming for the claim
of it.
I want to release the money to you as the relative to our deceased
customer (The account owner) MR. ANDREAS SCHRANNER who died a long
with his supposed NEXT OF KIN since 2003. The Banking law here does
not allow such money to stay more than 13 Years, because the money
will be recalled to the Bank treasury account as unclaimed fund.
By indicating your interest contact me through my private email
address ( abdullahmohad02-Re5JQEeQqe8AvxtiuMwx3w@public.gmane.org ) I will send you the full
details on how the Business will be executed. Please respond urgently
and delete if you are not interested
Regards,
Mr. Abdullah Mohad
^ permalink raw reply
* [PATCH 2/1] ipc,mqueue: Delete bogus overflow check
From: Davidlohr Bueso @ 2015-07-11 0:48 UTC (permalink / raw)
To: Marcus Gelderie
Cc: mtk.manpages-Re5JQEeQqe8AvxtiuMwx3w, Doug Ledford, lkml,
David Howells, Alexander Viro, John Duffy, Arto Bendiken,
Linux API, akpm-de/tnXTf+JLsfHDXvbKv3WD2FQJk+8+b
In-Reply-To: <20150706154928.GA19828-W7fNxlbxG8VSq9BJjBFyUp/QNRX+jHPU@public.gmane.org>
Mathematically, returning -EOVERFLOW in mq_attr_ok()
cannot occur under this condition:
mq_treesize = attr->mq_maxmsg * sizeof(struct msg_msg) +
min_t(unsigned int, attr->mq_maxmsg, MQ_PRIO_MAX) *
sizeof(struct posix_msg_tree_node);
total_size = attr->mq_maxmsg * attr->mq_msgsize;
if (total_size + mq_treesize < total_size)
return -EOVERFLOW;
Thus remove the check and simplify code around calculating
total queue overhead by introducing a mqueue_sizeof() helper.
Signed-off-by: Davidlohr Bueso <dbueso-l3A5Bk7waGM@public.gmane.org>
---
Passes ipc stresser and ltp tests.
ipc/mqueue.c | 65 +++++++++++++++++++++++++++---------------------------------
1 file changed, 29 insertions(+), 36 deletions(-)
diff --git a/ipc/mqueue.c b/ipc/mqueue.c
index 161a180..a5d0c9e 100644
--- a/ipc/mqueue.c
+++ b/ipc/mqueue.c
@@ -209,6 +209,31 @@ try_again:
return msg;
}
+/*
+ * We used to allocate a static array of pointers and account
+ * the size of that array as well as one msg_msg struct per
+ * possible message into the queue size. That's no longer
+ * accurate as the queue is now an rbtree and will grow and
+ * shrink depending on usage patterns. We can, however, still
+ * account one msg_msg struct per message, but the nodes are
+ * allocated depending on priority usage, and most programs
+ * only use one, or a handful, of priorities. However, since
+ * this is pinned memory, we need to assume worst case, so
+ * that means the min(mq_maxmsg, max_priorities) * struct
+ * posix_msg_tree_node.
+ */
+static inline unsigned long mqueue_sizeof(struct mqueue_inode_info *info)
+{
+ unsigned long mq_treesize, mq_max_msgsize;
+
+ mq_treesize = info->attr.mq_maxmsg * sizeof(struct msg_msg) +
+ min_t(unsigned int, info->attr.mq_maxmsg, MQ_PRIO_MAX) *
+ sizeof(struct posix_msg_tree_node);
+
+ mq_max_msgsize = info->attr.mq_maxmsg * info->attr.mq_msgsize;
+ return mq_treesize + mq_max_msgsize; /* bytes */
+}
+
static struct inode *mqueue_get_inode(struct super_block *sb,
struct ipc_namespace *ipc_ns, umode_t mode,
struct mq_attr *attr)
@@ -229,7 +254,7 @@ static struct inode *mqueue_get_inode(struct super_block *sb,
if (S_ISREG(mode)) {
struct mqueue_inode_info *info;
- unsigned long mq_bytes, mq_treesize;
+ unsigned long mq_bytes;
inode->i_fop = &mqueue_file_operations;
inode->i_size = FILENT_SIZE;
@@ -254,25 +279,8 @@ static struct inode *mqueue_get_inode(struct super_block *sb,
info->attr.mq_maxmsg = attr->mq_maxmsg;
info->attr.mq_msgsize = attr->mq_msgsize;
}
- /*
- * We used to allocate a static array of pointers and account
- * the size of that array as well as one msg_msg struct per
- * possible message into the queue size. That's no longer
- * accurate as the queue is now an rbtree and will grow and
- * shrink depending on usage patterns. We can, however, still
- * account one msg_msg struct per message, but the nodes are
- * allocated depending on priority usage, and most programs
- * only use one, or a handful, of priorities. However, since
- * this is pinned memory, we need to assume worst case, so
- * that means the min(mq_maxmsg, max_priorities) * struct
- * posix_msg_tree_node.
- */
- mq_treesize = info->attr.mq_maxmsg * sizeof(struct msg_msg) +
- min_t(unsigned int, info->attr.mq_maxmsg, MQ_PRIO_MAX) *
- sizeof(struct posix_msg_tree_node);
- mq_bytes = mq_treesize + (info->attr.mq_maxmsg *
- info->attr.mq_msgsize);
+ mq_bytes = mqueue_sizeof(info);
spin_lock(&mq_lock);
if (u->mq_bytes + mq_bytes < u->mq_bytes ||
@@ -371,7 +379,7 @@ static void mqueue_evict_inode(struct inode *inode)
{
struct mqueue_inode_info *info;
struct user_struct *user;
- unsigned long mq_bytes, mq_treesize;
+ unsigned long mq_bytes;
struct ipc_namespace *ipc_ns;
struct msg_msg *msg;
@@ -388,13 +396,7 @@ static void mqueue_evict_inode(struct inode *inode)
kfree(info->node_cache);
spin_unlock(&info->lock);
- /* Total amount of bytes accounted for the mqueue */
- mq_treesize = info->attr.mq_maxmsg * sizeof(struct msg_msg) +
- min_t(unsigned int, info->attr.mq_maxmsg, MQ_PRIO_MAX) *
- sizeof(struct posix_msg_tree_node);
-
- mq_bytes = mq_treesize + (info->attr.mq_maxmsg *
- info->attr.mq_msgsize);
+ mq_bytes = mqueue_sizeof(info);
user = info->user;
if (user) {
@@ -692,9 +694,6 @@ static void remove_notification(struct mqueue_inode_info *info)
static int mq_attr_ok(struct ipc_namespace *ipc_ns, struct mq_attr *attr)
{
- int mq_treesize;
- unsigned long total_size;
-
if (attr->mq_maxmsg <= 0 || attr->mq_msgsize <= 0)
return -EINVAL;
if (capable(CAP_SYS_RESOURCE)) {
@@ -709,12 +708,6 @@ static int mq_attr_ok(struct ipc_namespace *ipc_ns, struct mq_attr *attr)
/* check for overflow */
if (attr->mq_msgsize > ULONG_MAX/attr->mq_maxmsg)
return -EOVERFLOW;
- mq_treesize = attr->mq_maxmsg * sizeof(struct msg_msg) +
- min_t(unsigned int, attr->mq_maxmsg, MQ_PRIO_MAX) *
- sizeof(struct posix_msg_tree_node);
- total_size = attr->mq_maxmsg * attr->mq_msgsize;
- if (total_size + mq_treesize < total_size)
- return -EOVERFLOW;
return 0;
}
--
2.1.4
^ permalink raw reply related
* Re: Re: [PATCH tip/master ] ftracetest: Update kprobe-tracer testcases because of renaming do_fork
From: Masami Hiramatsu @ 2015-07-11 1:13 UTC (permalink / raw)
To: Steven Rostedt
Cc: Vince Weaver, linux-api-u79uwXL29TY76Z2rM5mHXA, Heiko Carstens,
Shuah Khan, Linux Kernel Mailing List, Ingo Molnar, Namhyung Kim,
Ingo Molnar
In-Reply-To: <20150709080311.37ae53c3-2kNGR76GQU9OHLTnHDQRgA@public.gmane.org>
On 2015/07/09 21:03, Steven Rostedt wrote:
> On Thu, 09 Jul 2015 19:10:12 +0900
> Masami Hiramatsu <masami.hiramatsu.pt-FCd8Q96Dh0JBDgjK7y7TUQ@public.gmane.org> wrote:
>
>> echo 0 > events/enable
>> echo > kprobe_events
>> -echo p:myevent do_fork > kprobe_events
>> +echo p:myevent _do_fork > kprobe_events
>> grep myevent kprobe_events
>> test -d events/kprobes/myevent
>> echo > kprobe_events
>> diff --git a/tools/testing/selftests/ftrace/test.d/kprobe/busy_check.tc b/tools/testing/selftests/ftrace/test.d/kprobe/busy_check.tc
>> index d8c7bb6..74507db 100644
>> --- a/tools/testing/selftests/ftrace/test.d/kprobe/busy_check.tc
>> +++ b/tools/testing/selftests/ftrace/test.d/kprobe/busy_check.tc
>> @@ -5,7 +5,7 @@
>>
>> echo 0 > events/enable
>> echo > kprobe_events
>> -echo p:myevent do_fork > kprobe_events
>> +echo p:myevent _do_fork > kprobe_events
>> test -d events/kprobes/myevent
>> echo 1 > events/kprobes/myevent/enable
>> echo > kprobe_events && exit 1 # this must fail
>> diff --git a/tools/testing/selftests/ftrace/test.d/kprobe/kprobe_args.tc b/tools/testing/selftests/ftrace/test.d/kprobe/kprobe_args.tc
>> index c45ee27..64949d4 100644
>> --- a/tools/testing/selftests/ftrace/test.d/kprobe/kprobe_args.tc
>> +++ b/tools/testing/selftests/ftrace/test.d/kprobe/kprobe_args.tc
>> @@ -5,7 +5,7 @@
>>
>> echo 0 > events/enable
>> echo > kprobe_events
>> -echo 'p:testprobe do_fork $stack $stack0 +0($stack)' > kprobe_events
>> +echo 'p:testprobe _do_fork $stack $stack0 +0($stack)' > kprobe_events
>> grep testprobe kprobe_events
>> test -d events/kprobes/testprobe
>> echo 1 > events/kprobes/testprobe/enable
>> diff --git a/tools/testing/selftests/ftrace/test.d/kprobe/kprobe_ftrace.tc b/tools/testing/selftests/ftrace/test.d/kprobe/kprobe_ftrace.tc
>> index ab41d2b..d6f2f49 100644
>> --- a/tools/testing/selftests/ftrace/test.d/kprobe/kprobe_ftrace.tc
>> +++ b/tools/testing/selftests/ftrace/test.d/kprobe/kprobe_ftrace.tc
>> @@ -6,31 +6,31 @@ grep function available_tracers || exit_unsupported # this is configurable
>>
>> # prepare
>> echo nop > current_tracer
>> -echo do_fork > set_ftrace_filter
>> +echo _do_fork > set_ftrace_filter
>> echo 0 > events/enable
>> echo > kprobe_events
>> -echo 'p:testprobe do_fork' > kprobe_events
>> +echo 'p:testprobe _do_fork' > kprobe_events
>>
>> # kprobe on / ftrace off
>> echo 1 > events/kprobes/testprobe/enable
>> echo > trace
>> ( echo "forked")
>> grep testprobe trace
>> -! grep 'do_fork <-' trace
>> +! grep '_do_fork <-' trace
>>
>> # kprobe on / ftrace on
>> echo function > current_tracer
>> echo > trace
>> ( echo "forked")
>> grep testprobe trace
>> -grep 'do_fork <-' trace
>> +grep '_do_fork <-' trace
>>
>> # kprobe off / ftrace on
>> echo 0 > events/kprobes/testprobe/enable
>> echo > trace
>> ( echo "forked")
>> ! grep testprobe trace
>> -grep 'do_fork <-' trace
>> +grep '_do_fork <-' trace
>>
>> # kprobe on / ftrace on
>> echo 1 > events/kprobes/testprobe/enable
>> @@ -38,14 +38,14 @@ echo function > current_tracer
>> echo > trace
>> ( echo "forked")
>> grep testprobe trace
>> -grep 'do_fork <-' trace
>> +grep '_do_fork <-' trace
>>
>> # kprobe on / ftrace off
>> echo nop > current_tracer
>> echo > trace
>> ( echo "forked")
>> grep testprobe trace
>> -! grep 'do_fork <-' trace
>> +! grep '_do_fork <-' trace
>>
>> # cleanup
>> echo nop > current_tracer
>> diff --git a/tools/testing/selftests/ftrace/test.d/kprobe/kretprobe_args.tc b/tools/testing/selftests/ftrace/test.d/kprobe/kretprobe_args.tc
>> index 3171798..0d09546 100644
>> --- a/tools/testing/selftests/ftrace/test.d/kprobe/kretprobe_args.tc
>> +++ b/tools/testing/selftests/ftrace/test.d/kprobe/kretprobe_args.tc
>> @@ -5,7 +5,7 @@
>>
>> echo 0 > events/enable
>> echo > kprobe_events
>> -echo 'r:testprobe2 do_fork $retval' > kprobe_events
>> +echo 'r:testprobe2 _do_fork $retval' > kprobe_events
>> grep testprobe2 kprobe_events
>> test -d events/kprobes/testprobe2
>> echo 1 > events/kprobes/testprobe2/enable
>
>
> Instead of hard coding the name, what about adding a global variable
> for the function, in case we need to change it again, it would only
> need to be changed in one place?
OK, I'll introduce FORK_SYMBOL="_do_fork" :)
Thank you,
--
Masami HIRAMATSU
Linux Technology Research Center, System Productivity Research Dept.
Center for Technology Innovation - Systems Engineering
Hitachi, Ltd., Research & Development Group
E-mail: masami.hiramatsu.pt-FCd8Q96Dh0JBDgjK7y7TUQ@public.gmane.org
^ permalink raw reply
* Re: [PATCH 2/1] ipc,mqueue: Delete bogus overflow check
From: Al Viro @ 2015-07-11 2:03 UTC (permalink / raw)
To: Davidlohr Bueso
Cc: Marcus Gelderie, mtk.manpages-Re5JQEeQqe8AvxtiuMwx3w,
Doug Ledford, lkml, David Howells, John Duffy, Arto Bendiken,
Linux API, akpm-de/tnXTf+JLsfHDXvbKv3WD2FQJk+8+b
In-Reply-To: <1436575691.27924.53.camel-h16yJtLeMjHk1uMJSBkQmQ@public.gmane.org>
On Fri, Jul 10, 2015 at 05:48:11PM -0700, Davidlohr Bueso wrote:
> Mathematically, returning -EOVERFLOW in mq_attr_ok()
> cannot occur under this condition:
>
> mq_treesize = attr->mq_maxmsg * sizeof(struct msg_msg) +
> min_t(unsigned int, attr->mq_maxmsg, MQ_PRIO_MAX) *
> sizeof(struct posix_msg_tree_node);
> total_size = attr->mq_maxmsg * attr->mq_msgsize;
> if (total_size + mq_treesize < total_size)
> return -EOVERFLOW;
A proof would be nice. More detailed one than "cannot occur", that is.
Condition in question is basically mq_treesize < 0 or
total_size + mq_treesize (in natural numbers) > 2^BITS_PER_LONG.
Now, the maximal values of ->mq_maxmsg and ->mq_msgsize are 2^16 and
2^24 resp. and we are guaranteed that their product is below 2^BITS_PER_LONG.
For mq_treesize we are guaranteed that it's below 2^31. Now, on a 64bit
box that would suffice to avoid overflow - the product is at most 2^40 and
its sum with mq_treesize can't wrap around.
For 32bit system, though... Suppose attr->mq_maxmsg == 65535 and
attr->mq_msgsize == 65537. Their product *is* below 2^BITS_PER_LONG - it's
exactly 1 less than that. _Any_ non-zero value for mq_tresize (and it
will be non-zero in the above) will lead to wraparound.
Looks like a counterexample to your assertion above...
^ permalink raw reply
* Re: [PATCH 2/1] ipc,mqueue: Delete bogus overflow check
From: Doug Ledford @ 2015-07-11 2:59 UTC (permalink / raw)
To: Al Viro, Davidlohr Bueso
Cc: Marcus Gelderie, mtk.manpages-Re5JQEeQqe8AvxtiuMwx3w, lkml,
David Howells, John Duffy, Arto Bendiken, Linux API,
akpm-de/tnXTf+JLsfHDXvbKv3WD2FQJk+8+b
In-Reply-To: <20150711020300.GH17109-3bDd1+5oDREiFSDQTTA3OLVCufUGDwFn@public.gmane.org>
[-- Attachment #1: Type: text/plain, Size: 1872 bytes --]
On 07/10/2015 10:03 PM, Al Viro wrote:
> On Fri, Jul 10, 2015 at 05:48:11PM -0700, Davidlohr Bueso wrote:
>> Mathematically, returning -EOVERFLOW in mq_attr_ok()
>> cannot occur under this condition:
>>
>> mq_treesize = attr->mq_maxmsg * sizeof(struct msg_msg) +
>> min_t(unsigned int, attr->mq_maxmsg, MQ_PRIO_MAX) *
>> sizeof(struct posix_msg_tree_node);
>> total_size = attr->mq_maxmsg * attr->mq_msgsize;
>> if (total_size + mq_treesize < total_size)
>> return -EOVERFLOW;
>
> A proof would be nice. More detailed one than "cannot occur", that is.
>
> Condition in question is basically mq_treesize < 0 or
> total_size + mq_treesize (in natural numbers) > 2^BITS_PER_LONG.
> Now, the maximal values of ->mq_maxmsg and ->mq_msgsize are 2^16 and
> 2^24 resp. and we are guaranteed that their product is below 2^BITS_PER_LONG.
> For mq_treesize we are guaranteed that it's below 2^31. Now, on a 64bit
> box that would suffice to avoid overflow - the product is at most 2^40 and
> its sum with mq_treesize can't wrap around.
>
> For 32bit system, though... Suppose attr->mq_maxmsg == 65535 and
> attr->mq_msgsize == 65537. Their product *is* below 2^BITS_PER_LONG - it's
> exactly 1 less than that. _Any_ non-zero value for mq_tresize (and it
> will be non-zero in the above) will lead to wraparound.
>
> Looks like a counterexample to your assertion above...
>
I'm pretty sure you're right. The above looks like an example of "Gee,
we need to protect against signed wrap around. Wait, it's unsigned, no
worries." when in fact unsigned will wrap around too if the total
exceeds the maximum (it just wraps to a small positive value instead of
a large negative value).
--
Doug Ledford <dledford-H+wXaHxf7aLQT0dZR+AlfA@public.gmane.org>
GPG KeyID: 0E572FDD
[-- Attachment #2: OpenPGP digital signature --]
[-- Type: application/pgp-signature, Size: 884 bytes --]
^ permalink raw reply
* [PATCH -mm v7 0/6] idle memory tracking
From: Vladimir Davydov @ 2015-07-11 14:48 UTC (permalink / raw)
To: Andrew Morton
Cc: Andres Lagar-Cavilla, Minchan Kim, Raghavendra K T,
Johannes Weiner, Michal Hocko, Greg Thelen, Michel Lespinasse,
David Rientjes, Pavel Emelyanov, Cyrill Gorcunov, Jonathan Corbet,
linux-api, linux-doc, linux-mm, cgroups, linux-kernel
Hi,
This patch set introduces a new user API for tracking user memory pages
that have not been used for a given period of time. The purpose of this
is to provide the userspace with the means of tracking a workload's
working set, i.e. the set of pages that are actively used by the
workload. Knowing the working set size can be useful for partitioning
the system more efficiently, e.g. by tuning memory cgroup limits
appropriately, or for job placement within a compute cluster.
It is based on top of v4.2-rc1-mmotm-2015-07-06-16-25
---- USE CASES ----
The unified cgroup hierarchy has memory.low and memory.high knobs, which
are defined as the low and high boundaries for the workload working set
size. However, the working set size of a workload may be unknown or
change in time. With this patch set, one can periodically estimate the
amount of memory unused by each cgroup and tune their memory.low and
memory.high parameters accordingly, therefore optimizing the overall
memory utilization.
Another use case is balancing workloads within a compute cluster.
Knowing how much memory is not really used by a workload unit may help
take a more optimal decision when considering migrating the unit to
another node within the cluster.
Also, as noted by Minchan, this would be useful for per-process reclaim
(https://lwn.net/Articles/545668/). With idle tracking, we could reclaim idle
pages only by smart user memory manager.
---- USER API ----
The user API consists of two new proc files:
* /proc/kpageidle. This file implements a bitmap where each bit corresponds
to a page, indexed by PFN. When the bit is set, the corresponding page is
idle. A page is considered idle if it has not been accessed since it was
marked idle. To mark a page idle one should set the bit corresponding to the
page by writing to the file. A value written to the file is OR-ed with the
current bitmap value. Only user memory pages can be marked idle, for other
page types input is silently ignored. Writing to this file beyond max PFN
results in the ENXIO error. Only available when CONFIG_IDLE_PAGE_TRACKING is
set.
This file can be used to estimate the amount of pages that are not
used by a particular workload as follows:
1. mark all pages of interest idle by setting corresponding bits in the
/proc/kpageidle bitmap
2. wait until the workload accesses its working set
3. read /proc/kpageidle and count the number of bits set
* /proc/kpagecgroup. This file contains a 64-bit inode number of the
memory cgroup each page is charged to, indexed by PFN. Only available when
CONFIG_MEMCG is set.
This file can be used to find all pages (including unmapped file
pages) accounted to a particular cgroup. Using /proc/kpageidle, one
can then estimate the cgroup working set size.
For an example of using these files for estimating the amount of unused
memory pages per each memory cgroup, please see the script attached
below.
---- REASONING ----
The reason to introduce the new user API instead of using
/proc/PID/{clear_refs,smaps} is that the latter has two serious
drawbacks:
- it does not count unmapped file pages
- it affects the reclaimer logic
The new API attempts to overcome them both. For more details on how it
is achieved, please see the comment to patch 5.
---- CHANGE LOG ----
Changes in v7:
This iteration addresses Andres's comments to v6:
- do not reuse page_referenced for clearing idle flag, introduce a
separate function instead; this way we won't issue expensive tlb
flushes on /proc/kpageidle read/write
- propagate young/idle flags from head to tail pages on thp split
- skip compound tail pages while reading/writing /proc/kpageidle
- cleanup page_referenced_one
Changes in v6:
- Split the patch introducing page_cgroup_ino helper to ease review.
- Rebase on top of v4.1-rc7-mmotm-2015-06-09-16-55
Changes in v5:
- Fix possible race between kpageidle_clear_pte_refs() and
__page_set_anon_rmap() by checking that a page is on an LRU list
under zone->lru_lock (Minchan).
- Export idle flag via /proc/kpageflags (Minchan).
- Rebase on top of 4.1-rc3.
Changes in v4:
This iteration primarily addresses Minchan's comments to v3:
- Implement /proc/kpageidle as a bitmap instead of using u64 per each page,
because there does not seem to be any future uses for the other 63 bits.
- Do not double-increase pra->referenced in page_referenced_one() if the page
was young and referenced recently.
- Remove the pointless (page_count == 0) check from kpageidle_get_page().
- Rename kpageidle_clear_refs() to kpageidle_clear_pte_refs().
- Improve comments to kpageidle-related functions.
- Rebase on top of 4.1-rc2.
Note it does not address Minchan's concern of possible __page_set_anon_rmap vs
page_referenced race (see https://lkml.org/lkml/2015/5/3/220) since it is still
unclear if this race can really happen (see https://lkml.org/lkml/2015/5/4/160)
Changes in v3:
- Enable CONFIG_IDLE_PAGE_TRACKING for 32 bit. Since this feature
requires two extra page flags and there is no space for them on 32
bit, page ext is used (thanks to Minchan Kim).
- Minor code cleanups and comments improved.
- Rebase on top of 4.1-rc1.
Changes in v2:
- The main difference from v1 is the API change. In v1 the user can
only set the idle flag for all pages at once, and for clearing the
Idle flag on pages accessed via page tables /proc/PID/clear_refs
should be used.
The main drawback of the v1 approach, as noted by Minchan, is that on
big machines setting the idle flag for each pages can result in CPU
bursts, which would be especially frustrating if the user only wanted
to estimate the amount of idle pages for a particular process or VMA.
With the new API a more fine-grained approach is possible: one can
read a process's /proc/PID/pagemap and set/check the Idle flag only
for those pages of the process's address space he or she is
interested in.
Another good point about the v2 API is that it is possible to limit
/proc/kpage* scanning rate when the user wants to estimate the total
number of idle pages, which is unachievable with the v1 approach.
- Make /proc/kpagecgroup return the ino of the closest online ancestor
in case the cgroup a page is charged to is offline.
- Fix /proc/PID/clear_refs not clearing Young page flag.
- Rebase on top of v4.0-rc6-mmotm-2015-04-01-14-54
v6: https://lkml.org/lkml/2015/6/12/301
v5: https://lkml.org/lkml/2015/5/12/449
v4: https://lkml.org/lkml/2015/5/7/580
v3: https://lkml.org/lkml/2015/4/28/224
v2: https://lkml.org/lkml/2015/4/7/260
v1: https://lkml.org/lkml/2015/3/18/794
---- PATCH SET STRUCTURE ----
The patch set is organized as follows:
- patch 1 adds page_cgroup_ino() helper for the sake of
/proc/kpagecgroup and patches 2-3 do related cleanup
- patch 4 adds /proc/kpagecgroup, which reports cgroup ino each page is
charged to
- patch 5 implements the idle page tracking feature, including the
userspace API, /proc/kpageidle
- patch 6 exports idle flag via /proc/kpageflags
---- SIMILAR WORKS ----
Originally, the patch for tracking idle memory was proposed back in 2011
by Michel Lespinasse (see http://lwn.net/Articles/459269/). The main
difference between Michel's patch and this one is that Michel
implemented a kernel space daemon for estimating idle memory size per
cgroup while this patch only provides the userspace with the minimal API
for doing the job, leaving the rest up to the userspace. However, they
both share the same idea of Idle/Young page flags to avoid affecting the
reclaimer logic.
---- SCRIPT FOR COUNTING IDLE PAGES PER CGROUP ----
#! /usr/bin/python
#
import os
import stat
import errno
import struct
CGROUP_MOUNT = "/sys/fs/cgroup/memory"
BUFSIZE = 8 * 1024 # must be multiple of 8
def get_hugepage_size():
with open("/proc/meminfo", "r") as f:
for s in f:
k, v = s.split(":")
if k == "Hugepagesize":
return int(v.split()[0]) * 1024
PAGE_SIZE = os.sysconf("SC_PAGE_SIZE")
HUGEPAGE_SIZE = get_hugepage_size()
def set_idle():
f = open("/proc/kpageidle", "wb", BUFSIZE)
while True:
try:
f.write(struct.pack("Q", pow(2, 64) - 1))
except IOError as err:
if err.errno == errno.ENXIO:
break
raise
f.close()
def count_idle():
f_flags = open("/proc/kpageflags", "rb", BUFSIZE)
f_cgroup = open("/proc/kpagecgroup", "rb", BUFSIZE)
with open("/proc/kpageidle", "rb", BUFSIZE) as f:
while f.read(BUFSIZE): pass # update idle flag
idlememsz = {}
while True:
s1, s2 = f_flags.read(8), f_cgroup.read(8)
if not s1 or not s2:
break
flags, = struct.unpack('Q', s1)
cgino, = struct.unpack('Q', s2)
unevictable = (flags >> 18) & 1
huge = (flags >> 22) & 1
idle = (flags >> 25) & 1
if idle and not unevictable:
idlememsz[cgino] = idlememsz.get(cgino, 0) + \
(HUGEPAGE_SIZE if huge else PAGE_SIZE)
f_flags.close()
f_cgroup.close()
return idlememsz
if __name__ == "__main__":
print "Setting the idle flag for each page..."
set_idle()
raw_input("Wait until the workload accesses its working set, "
"then press Enter")
print "Counting idle pages..."
idlememsz = count_idle()
for dir, subdirs, files in os.walk(CGROUP_MOUNT):
ino = os.stat(dir)[stat.ST_INO]
print dir + ": " + str(idlememsz.get(ino, 0) / 1024) + " kB"
---- END SCRIPT ----
Comments are more than welcome.
Thanks,
Vladimir Davydov (6):
memcg: add page_cgroup_ino helper
hwpoison: use page_cgroup_ino for filtering by memcg
memcg: zap try_get_mem_cgroup_from_page
proc: add kpagecgroup file
proc: add kpageidle file
proc: export idle flag via kpageflags
Documentation/vm/pagemap.txt | 22 ++-
fs/proc/page.c | 278 +++++++++++++++++++++++++++++++++
fs/proc/task_mmu.c | 4 +-
include/linux/memcontrol.h | 7 +-
include/linux/mm.h | 98 ++++++++++++
include/linux/page-flags.h | 11 ++
include/linux/page_ext.h | 4 +
include/uapi/linux/kernel-page-flags.h | 1 +
mm/Kconfig | 12 ++
mm/debug.c | 4 +
mm/huge_memory.c | 5 +
mm/hwpoison-inject.c | 5 +-
mm/memcontrol.c | 71 +++++----
mm/memory-failure.c | 16 +-
mm/page_ext.c | 3 +
mm/rmap.c | 5 +
mm/swap.c | 2 +
17 files changed, 486 insertions(+), 62 deletions(-)
--
2.1.4
^ permalink raw reply
* [PATCH -mm v7 1/6] memcg: add page_cgroup_ino helper
From: Vladimir Davydov @ 2015-07-11 14:48 UTC (permalink / raw)
To: Andrew Morton
Cc: Andres Lagar-Cavilla, Minchan Kim, Raghavendra K T,
Johannes Weiner, Michal Hocko, Greg Thelen, Michel Lespinasse,
David Rientjes, Pavel Emelyanov, Cyrill Gorcunov, Jonathan Corbet,
linux-api, linux-doc, linux-mm, cgroups, linux-kernel
In-Reply-To: <cover.1436623799.git.vdavydov@parallels.com>
This function returns the inode number of the closest online ancestor of
the memory cgroup a page is charged to. It is required for exporting
information about which page is charged to which cgroup to userspace,
which will be introduced by a following patch.
Signed-off-by: Vladimir Davydov <vdavydov@parallels.com>
---
include/linux/memcontrol.h | 1 +
mm/memcontrol.c | 23 +++++++++++++++++++++++
2 files changed, 24 insertions(+)
diff --git a/include/linux/memcontrol.h b/include/linux/memcontrol.h
index 73b02b0a8f60..50069abebc3c 100644
--- a/include/linux/memcontrol.h
+++ b/include/linux/memcontrol.h
@@ -116,6 +116,7 @@ static inline bool mm_match_cgroup(struct mm_struct *mm,
extern struct cgroup_subsys_state *mem_cgroup_css(struct mem_cgroup *memcg);
extern struct cgroup_subsys_state *mem_cgroup_css_from_page(struct page *page);
+extern unsigned long page_cgroup_ino(struct page *page);
struct mem_cgroup *mem_cgroup_iter(struct mem_cgroup *,
struct mem_cgroup *,
diff --git a/mm/memcontrol.c b/mm/memcontrol.c
index acb93c554f6e..894dc2169979 100644
--- a/mm/memcontrol.c
+++ b/mm/memcontrol.c
@@ -631,6 +631,29 @@ struct cgroup_subsys_state *mem_cgroup_css_from_page(struct page *page)
return &memcg->css;
}
+/**
+ * page_cgroup_ino - return inode number of the memcg a page is charged to
+ * @page: the page
+ *
+ * Look up the closest online ancestor of the memory cgroup @page is charged to
+ * and return its inode number or 0 if @page is not charged to any cgroup. It
+ * is safe to call this function without holding a reference to @page.
+ */
+unsigned long page_cgroup_ino(struct page *page)
+{
+ struct mem_cgroup *memcg;
+ unsigned long ino = 0;
+
+ rcu_read_lock();
+ memcg = READ_ONCE(page->mem_cgroup);
+ while (memcg && !(memcg->css.flags & CSS_ONLINE))
+ memcg = parent_mem_cgroup(memcg);
+ if (memcg)
+ ino = cgroup_ino(memcg->css.cgroup);
+ rcu_read_unlock();
+ return ino;
+}
+
static struct mem_cgroup_per_zone *
mem_cgroup_page_zoneinfo(struct mem_cgroup *memcg, struct page *page)
{
--
2.1.4
^ permalink raw reply related
* [PATCH -mm v7 2/6] hwpoison: use page_cgroup_ino for filtering by memcg
From: Vladimir Davydov @ 2015-07-11 14:48 UTC (permalink / raw)
To: Andrew Morton
Cc: Andres Lagar-Cavilla, Minchan Kim, Raghavendra K T,
Johannes Weiner, Michal Hocko, Greg Thelen, Michel Lespinasse,
David Rientjes, Pavel Emelyanov, Cyrill Gorcunov, Jonathan Corbet,
linux-api, linux-doc, linux-mm, cgroups, linux-kernel
In-Reply-To: <cover.1436623799.git.vdavydov@parallels.com>
Hwpoison allows to filter pages by memory cgroup ino. Currently, it
calls try_get_mem_cgroup_from_page to obtain the cgroup from a page and
then its ino using cgroup_ino, but now we have an apter method for that,
page_cgroup_ino, so use it instead.
Signed-off-by: Vladimir Davydov <vdavydov@parallels.com>
---
mm/hwpoison-inject.c | 5 +----
mm/memory-failure.c | 16 ++--------------
2 files changed, 3 insertions(+), 18 deletions(-)
diff --git a/mm/hwpoison-inject.c b/mm/hwpoison-inject.c
index bf73ac17dad4..5015679014c1 100644
--- a/mm/hwpoison-inject.c
+++ b/mm/hwpoison-inject.c
@@ -45,12 +45,9 @@ static int hwpoison_inject(void *data, u64 val)
/*
* do a racy check with elevated page count, to make sure PG_hwpoison
* will only be set for the targeted owner (or on a free page).
- * We temporarily take page lock for try_get_mem_cgroup_from_page().
* memory_failure() will redo the check reliably inside page lock.
*/
- lock_page(hpage);
err = hwpoison_filter(hpage);
- unlock_page(hpage);
if (err)
goto put_out;
@@ -126,7 +123,7 @@ static int pfn_inject_init(void)
if (!dentry)
goto fail;
-#ifdef CONFIG_MEMCG_SWAP
+#ifdef CONFIG_MEMCG
dentry = debugfs_create_u64("corrupt-filter-memcg", 0600,
hwpoison_dir, &hwpoison_filter_memcg);
if (!dentry)
diff --git a/mm/memory-failure.c b/mm/memory-failure.c
index 1cf7f2988422..97005396a507 100644
--- a/mm/memory-failure.c
+++ b/mm/memory-failure.c
@@ -130,27 +130,15 @@ static int hwpoison_filter_flags(struct page *p)
* can only guarantee that the page either belongs to the memcg tasks, or is
* a freed page.
*/
-#ifdef CONFIG_MEMCG_SWAP
+#ifdef CONFIG_MEMCG
u64 hwpoison_filter_memcg;
EXPORT_SYMBOL_GPL(hwpoison_filter_memcg);
static int hwpoison_filter_task(struct page *p)
{
- struct mem_cgroup *mem;
- struct cgroup_subsys_state *css;
- unsigned long ino;
-
if (!hwpoison_filter_memcg)
return 0;
- mem = try_get_mem_cgroup_from_page(p);
- if (!mem)
- return -EINVAL;
-
- css = mem_cgroup_css(mem);
- ino = cgroup_ino(css->cgroup);
- css_put(css);
-
- if (ino != hwpoison_filter_memcg)
+ if (page_cgroup_ino(p) != hwpoison_filter_memcg)
return -EINVAL;
return 0;
--
2.1.4
--
To unsubscribe, send a message with 'unsubscribe linux-mm' in
the body to majordomo@kvack.org. For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"dont@kvack.org"> email@kvack.org </a>
^ permalink raw reply related
* [PATCH -mm v7 3/6] memcg: zap try_get_mem_cgroup_from_page
From: Vladimir Davydov @ 2015-07-11 14:48 UTC (permalink / raw)
To: Andrew Morton
Cc: Andres Lagar-Cavilla, Minchan Kim, Raghavendra K T,
Johannes Weiner, Michal Hocko, Greg Thelen, Michel Lespinasse,
David Rientjes, Pavel Emelyanov, Cyrill Gorcunov, Jonathan Corbet,
linux-api, linux-doc, linux-mm, cgroups, linux-kernel
In-Reply-To: <cover.1436623799.git.vdavydov@parallels.com>
It is only used in mem_cgroup_try_charge, so fold it in and zap it.
Signed-off-by: Vladimir Davydov <vdavydov@parallels.com>
---
include/linux/memcontrol.h | 6 ------
mm/memcontrol.c | 48 ++++++++++++----------------------------------
2 files changed, 12 insertions(+), 42 deletions(-)
diff --git a/include/linux/memcontrol.h b/include/linux/memcontrol.h
index 50069abebc3c..635edfe06bac 100644
--- a/include/linux/memcontrol.h
+++ b/include/linux/memcontrol.h
@@ -94,7 +94,6 @@ bool mem_cgroup_is_descendant(struct mem_cgroup *memcg,
struct mem_cgroup *root);
bool task_in_mem_cgroup(struct task_struct *task, struct mem_cgroup *memcg);
-extern struct mem_cgroup *try_get_mem_cgroup_from_page(struct page *page);
extern struct mem_cgroup *mem_cgroup_from_task(struct task_struct *p);
extern struct mem_cgroup *parent_mem_cgroup(struct mem_cgroup *memcg);
@@ -259,11 +258,6 @@ static inline struct lruvec *mem_cgroup_page_lruvec(struct page *page,
return &zone->lruvec;
}
-static inline struct mem_cgroup *try_get_mem_cgroup_from_page(struct page *page)
-{
- return NULL;
-}
-
static inline bool mm_match_cgroup(struct mm_struct *mm,
struct mem_cgroup *memcg)
{
diff --git a/mm/memcontrol.c b/mm/memcontrol.c
index 894dc2169979..fa1447fcba33 100644
--- a/mm/memcontrol.c
+++ b/mm/memcontrol.c
@@ -2378,40 +2378,6 @@ static void cancel_charge(struct mem_cgroup *memcg, unsigned int nr_pages)
css_put_many(&memcg->css, nr_pages);
}
-/*
- * try_get_mem_cgroup_from_page - look up page's memcg association
- * @page: the page
- *
- * Look up, get a css reference, and return the memcg that owns @page.
- *
- * The page must be locked to prevent racing with swap-in and page
- * cache charges. If coming from an unlocked page table, the caller
- * must ensure the page is on the LRU or this can race with charging.
- */
-struct mem_cgroup *try_get_mem_cgroup_from_page(struct page *page)
-{
- struct mem_cgroup *memcg;
- unsigned short id;
- swp_entry_t ent;
-
- VM_BUG_ON_PAGE(!PageLocked(page), page);
-
- memcg = page->mem_cgroup;
- if (memcg) {
- if (!css_tryget_online(&memcg->css))
- memcg = NULL;
- } else if (PageSwapCache(page)) {
- ent.val = page_private(page);
- id = lookup_swap_cgroup_id(ent);
- rcu_read_lock();
- memcg = mem_cgroup_from_id(id);
- if (memcg && !css_tryget_online(&memcg->css))
- memcg = NULL;
- rcu_read_unlock();
- }
- return memcg;
-}
-
static void lock_page_lru(struct page *page, int *isolated)
{
struct zone *zone = page_zone(page);
@@ -5628,8 +5594,20 @@ int mem_cgroup_try_charge(struct page *page, struct mm_struct *mm,
* the page lock, which serializes swap cache removal, which
* in turn serializes uncharging.
*/
+ VM_BUG_ON_PAGE(!PageLocked(page), page);
if (page->mem_cgroup)
goto out;
+
+ if (do_swap_account) {
+ swp_entry_t ent = { .val = page_private(page), };
+ unsigned short id = lookup_swap_cgroup_id(ent);
+
+ rcu_read_lock();
+ memcg = mem_cgroup_from_id(id);
+ if (memcg && !css_tryget_online(&memcg->css))
+ memcg = NULL;
+ rcu_read_unlock();
+ }
}
if (PageTransHuge(page)) {
@@ -5637,8 +5615,6 @@ int mem_cgroup_try_charge(struct page *page, struct mm_struct *mm,
VM_BUG_ON_PAGE(!PageTransHuge(page), page);
}
- if (do_swap_account && PageSwapCache(page))
- memcg = try_get_mem_cgroup_from_page(page);
if (!memcg)
memcg = get_mem_cgroup_from_mm(mm);
--
2.1.4
--
To unsubscribe, send a message with 'unsubscribe linux-mm' in
the body to majordomo@kvack.org. For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"dont@kvack.org"> email@kvack.org </a>
^ permalink raw reply related
* [PATCH -mm v7 4/6] proc: add kpagecgroup file
From: Vladimir Davydov @ 2015-07-11 14:48 UTC (permalink / raw)
To: Andrew Morton
Cc: Andres Lagar-Cavilla, Minchan Kim, Raghavendra K T,
Johannes Weiner, Michal Hocko, Greg Thelen, Michel Lespinasse,
David Rientjes, Pavel Emelyanov, Cyrill Gorcunov, Jonathan Corbet,
linux-api, linux-doc, linux-mm, cgroups, linux-kernel
In-Reply-To: <cover.1436623799.git.vdavydov@parallels.com>
/proc/kpagecgroup contains a 64-bit inode number of the memory cgroup
each page is charged to, indexed by PFN. Having this information is
useful for estimating a cgroup working set size.
The file is present if CONFIG_PROC_PAGE_MONITOR && CONFIG_MEMCG.
Signed-off-by: Vladimir Davydov <vdavydov@parallels.com>
---
Documentation/vm/pagemap.txt | 6 ++++-
fs/proc/page.c | 53 ++++++++++++++++++++++++++++++++++++++++++++
2 files changed, 58 insertions(+), 1 deletion(-)
diff --git a/Documentation/vm/pagemap.txt b/Documentation/vm/pagemap.txt
index 6bfbc172cdb9..a9b7afc8fbc6 100644
--- a/Documentation/vm/pagemap.txt
+++ b/Documentation/vm/pagemap.txt
@@ -5,7 +5,7 @@ pagemap is a new (as of 2.6.25) set of interfaces in the kernel that allow
userspace programs to examine the page tables and related information by
reading files in /proc.
-There are three components to pagemap:
+There are four components to pagemap:
* /proc/pid/pagemap. This file lets a userspace process find out which
physical frame each virtual page is mapped to. It contains one 64-bit
@@ -65,6 +65,10 @@ There are three components to pagemap:
23. BALLOON
24. ZERO_PAGE
+ * /proc/kpagecgroup. This file contains a 64-bit inode number of the
+ memory cgroup each page is charged to, indexed by PFN. Only available when
+ CONFIG_MEMCG is set.
+
Short descriptions to the page flags:
0. LOCKED
diff --git a/fs/proc/page.c b/fs/proc/page.c
index 7eee2d8b97d9..70d23245dd43 100644
--- a/fs/proc/page.c
+++ b/fs/proc/page.c
@@ -9,6 +9,7 @@
#include <linux/proc_fs.h>
#include <linux/seq_file.h>
#include <linux/hugetlb.h>
+#include <linux/memcontrol.h>
#include <linux/kernel-page-flags.h>
#include <asm/uaccess.h>
#include "internal.h"
@@ -225,10 +226,62 @@ static const struct file_operations proc_kpageflags_operations = {
.read = kpageflags_read,
};
+#ifdef CONFIG_MEMCG
+static ssize_t kpagecgroup_read(struct file *file, char __user *buf,
+ size_t count, loff_t *ppos)
+{
+ u64 __user *out = (u64 __user *)buf;
+ struct page *ppage;
+ unsigned long src = *ppos;
+ unsigned long pfn;
+ ssize_t ret = 0;
+ u64 ino;
+
+ pfn = src / KPMSIZE;
+ count = min_t(unsigned long, count, (max_pfn * KPMSIZE) - src);
+ if (src & KPMMASK || count & KPMMASK)
+ return -EINVAL;
+
+ while (count > 0) {
+ if (pfn_valid(pfn))
+ ppage = pfn_to_page(pfn);
+ else
+ ppage = NULL;
+
+ if (ppage)
+ ino = page_cgroup_ino(ppage);
+ else
+ ino = 0;
+
+ if (put_user(ino, out)) {
+ ret = -EFAULT;
+ break;
+ }
+
+ pfn++;
+ out++;
+ count -= KPMSIZE;
+ }
+
+ *ppos += (char __user *)out - buf;
+ if (!ret)
+ ret = (char __user *)out - buf;
+ return ret;
+}
+
+static const struct file_operations proc_kpagecgroup_operations = {
+ .llseek = mem_lseek,
+ .read = kpagecgroup_read,
+};
+#endif /* CONFIG_MEMCG */
+
static int __init proc_page_init(void)
{
proc_create("kpagecount", S_IRUSR, NULL, &proc_kpagecount_operations);
proc_create("kpageflags", S_IRUSR, NULL, &proc_kpageflags_operations);
+#ifdef CONFIG_MEMCG
+ proc_create("kpagecgroup", S_IRUSR, NULL, &proc_kpagecgroup_operations);
+#endif
return 0;
}
fs_initcall(proc_page_init);
--
2.1.4
--
To unsubscribe, send a message with 'unsubscribe linux-mm' in
the body to majordomo@kvack.org. For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"dont@kvack.org"> email@kvack.org </a>
^ permalink raw reply related
* [PATCH -mm v7 5/6] proc: add kpageidle file
From: Vladimir Davydov @ 2015-07-11 14:48 UTC (permalink / raw)
To: Andrew Morton
Cc: Andres Lagar-Cavilla, Minchan Kim, Raghavendra K T,
Johannes Weiner, Michal Hocko, Greg Thelen, Michel Lespinasse,
David Rientjes, Pavel Emelyanov, Cyrill Gorcunov, Jonathan Corbet,
linux-api, linux-doc, linux-mm, cgroups, linux-kernel
In-Reply-To: <cover.1436623799.git.vdavydov@parallels.com>
Knowing the portion of memory that is not used by a certain application
or memory cgroup (idle memory) can be useful for partitioning the system
efficiently, e.g. by setting memory cgroup limits appropriately.
Currently, the only means to estimate the amount of idle memory provided
by the kernel is /proc/PID/{clear_refs,smaps}: the user can clear the
access bit for all pages mapped to a particular process by writing 1 to
clear_refs, wait for some time, and then count smaps:Referenced.
However, this method has two serious shortcomings:
- it does not count unmapped file pages
- it affects the reclaimer logic
To overcome these drawbacks, this patch introduces two new page flags,
Idle and Young, and a new proc file, /proc/kpageidle. A page's Idle flag
can only be set from userspace by setting bit in /proc/kpageidle at the
offset corresponding to the page, and it is cleared whenever the page is
accessed either through page tables (it is cleared in page_referenced()
in this case) or using the read(2) system call (mark_page_accessed()).
Thus by setting the Idle flag for pages of a particular workload, which
can be found e.g. by reading /proc/PID/pagemap, waiting for some time to
let the workload access its working set, and then reading the kpageidle
file, one can estimate the amount of pages that are not used by the
workload.
The Young page flag is used to avoid interference with the memory
reclaimer. A page's Young flag is set whenever the Access bit of a page
table entry pointing to the page is cleared by writing to kpageidle. If
page_referenced() is called on a Young page, it will add 1 to its return
value, therefore concealing the fact that the Access bit was cleared.
Note, since there is no room for extra page flags on 32 bit, this
feature uses extended page flags when compiled on 32 bit.
Signed-off-by: Vladimir Davydov <vdavydov@parallels.com>
---
Documentation/vm/pagemap.txt | 12 ++-
fs/proc/page.c | 222 +++++++++++++++++++++++++++++++++++++++++++
fs/proc/task_mmu.c | 4 +-
include/linux/mm.h | 98 +++++++++++++++++++
include/linux/page-flags.h | 11 +++
include/linux/page_ext.h | 4 +
mm/Kconfig | 12 +++
mm/debug.c | 4 +
mm/huge_memory.c | 5 +
mm/page_ext.c | 3 +
mm/rmap.c | 5 +
mm/swap.c | 2 +
12 files changed, 380 insertions(+), 2 deletions(-)
diff --git a/Documentation/vm/pagemap.txt b/Documentation/vm/pagemap.txt
index a9b7afc8fbc6..c9266340852c 100644
--- a/Documentation/vm/pagemap.txt
+++ b/Documentation/vm/pagemap.txt
@@ -5,7 +5,7 @@ pagemap is a new (as of 2.6.25) set of interfaces in the kernel that allow
userspace programs to examine the page tables and related information by
reading files in /proc.
-There are four components to pagemap:
+There are five components to pagemap:
* /proc/pid/pagemap. This file lets a userspace process find out which
physical frame each virtual page is mapped to. It contains one 64-bit
@@ -69,6 +69,16 @@ There are four components to pagemap:
memory cgroup each page is charged to, indexed by PFN. Only available when
CONFIG_MEMCG is set.
+ * /proc/kpageidle. This file implements a bitmap where each bit corresponds
+ to a page, indexed by PFN. When the bit is set, the corresponding page is
+ idle. A page is considered idle if it has not been accessed since it was
+ marked idle. To mark a page idle one should set the bit corresponding to the
+ page by writing to the file. A value written to the file is OR-ed with the
+ current bitmap value. Only user memory pages can be marked idle, for other
+ page types input is silently ignored. Writing to this file beyond max PFN
+ results in the ENXIO error. Only available when CONFIG_IDLE_PAGE_TRACKING is
+ set.
+
Short descriptions to the page flags:
0. LOCKED
diff --git a/fs/proc/page.c b/fs/proc/page.c
index 70d23245dd43..e51690c5f173 100644
--- a/fs/proc/page.c
+++ b/fs/proc/page.c
@@ -5,6 +5,7 @@
#include <linux/ksm.h>
#include <linux/mm.h>
#include <linux/mmzone.h>
+#include <linux/rmap.h>
#include <linux/huge_mm.h>
#include <linux/proc_fs.h>
#include <linux/seq_file.h>
@@ -16,6 +17,7 @@
#define KPMSIZE sizeof(u64)
#define KPMMASK (KPMSIZE - 1)
+#define KPMBITS (KPMSIZE * BITS_PER_BYTE)
/* /proc/kpagecount - an array exposing page counts
*
@@ -275,6 +277,222 @@ static const struct file_operations proc_kpagecgroup_operations = {
};
#endif /* CONFIG_MEMCG */
+#ifdef CONFIG_IDLE_PAGE_TRACKING
+/*
+ * Idle page tracking only considers user memory pages, for other types of
+ * pages the idle flag is always unset and an attempt to set it is silently
+ * ignored.
+ *
+ * We treat a page as a user memory page if it is on an LRU list, because it is
+ * always safe to pass such a page to rmap_walk(), which is essential for idle
+ * page tracking. With such an indicator of user pages we can skip isolated
+ * pages, but since there are not usually many of them, it will hardly affect
+ * the overall result.
+ *
+ * This function tries to get a user memory page by pfn as described above.
+ */
+static struct page *kpageidle_get_page(unsigned long pfn)
+{
+ struct page *page;
+ struct zone *zone;
+
+ if (!pfn_valid(pfn))
+ return NULL;
+
+ page = pfn_to_page(pfn);
+ if (!page || PageTail(page) || !PageLRU(page) ||
+ !get_page_unless_zero(page))
+ return NULL;
+
+ if (unlikely(PageTail(page))) {
+ put_page(page);
+ return NULL;
+ }
+
+ zone = page_zone(page);
+ spin_lock_irq(&zone->lru_lock);
+ if (unlikely(!PageLRU(page))) {
+ put_page(page);
+ page = NULL;
+ }
+ spin_unlock_irq(&zone->lru_lock);
+ return page;
+}
+
+static int kpageidle_clear_pte_refs_one(struct page *page,
+ struct vm_area_struct *vma,
+ unsigned long addr, void *arg)
+{
+ struct mm_struct *mm = vma->vm_mm;
+ spinlock_t *ptl;
+ pmd_t *pmd;
+ pte_t *pte;
+ bool referenced = false;
+
+ if (unlikely(PageTransHuge(page))) {
+ pmd = page_check_address_pmd(page, mm, addr,
+ PAGE_CHECK_ADDRESS_PMD_FLAG, &ptl);
+ if (pmd) {
+ referenced = pmdp_test_and_clear_young(vma, addr, pmd);
+ spin_unlock(ptl);
+ }
+ } else {
+ pte = page_check_address(page, mm, addr, &ptl, 0);
+ if (pte) {
+ referenced = ptep_test_and_clear_young(vma, addr, pte);
+ pte_unmap_unlock(pte, ptl);
+ }
+ }
+ if (referenced) {
+ clear_page_idle(page);
+ /*
+ * We cleared the referenced bit in a mapping to this page. To
+ * avoid interference with page reclaim, mark it young so that
+ * page_referenced() will return > 0.
+ */
+ set_page_young(page);
+ }
+ return SWAP_AGAIN;
+}
+
+static void kpageidle_clear_pte_refs(struct page *page)
+{
+ struct rmap_walk_control rwc = {
+ .rmap_one = kpageidle_clear_pte_refs_one,
+ .anon_lock = page_lock_anon_vma_read,
+ };
+ bool need_lock;
+
+ if (!page_mapped(page) ||
+ !page_rmapping(page))
+ return;
+
+ need_lock = !PageAnon(page) || PageKsm(page);
+ if (need_lock && !trylock_page(page))
+ return;
+
+ rmap_walk(page, &rwc);
+
+ if (need_lock)
+ unlock_page(page);
+}
+
+static ssize_t kpageidle_read(struct file *file, char __user *buf,
+ size_t count, loff_t *ppos)
+{
+ u64 __user *out = (u64 __user *)buf;
+ struct page *page;
+ unsigned long pfn, end_pfn;
+ ssize_t ret = 0;
+ u64 idle_bitmap = 0;
+ int bit;
+
+ if (*ppos & KPMMASK || count & KPMMASK)
+ return -EINVAL;
+
+ pfn = *ppos * BITS_PER_BYTE;
+ if (pfn >= max_pfn)
+ return 0;
+
+ end_pfn = pfn + count * BITS_PER_BYTE;
+ if (end_pfn > max_pfn)
+ end_pfn = ALIGN(max_pfn, KPMBITS);
+
+ for (; pfn < end_pfn; pfn++) {
+ bit = pfn % KPMBITS;
+ page = kpageidle_get_page(pfn);
+ if (page) {
+ if (page_is_idle(page)) {
+ /*
+ * The page might have been referenced via a
+ * pte, in which case it is not idle. Clear
+ * refs and recheck.
+ */
+ kpageidle_clear_pte_refs(page);
+ if (page_is_idle(page))
+ idle_bitmap |= 1ULL << bit;
+ }
+ put_page(page);
+ }
+ if (bit == KPMBITS - 1) {
+ if (put_user(idle_bitmap, out)) {
+ ret = -EFAULT;
+ break;
+ }
+ idle_bitmap = 0;
+ out++;
+ }
+ }
+
+ *ppos += (char __user *)out - buf;
+ if (!ret)
+ ret = (char __user *)out - buf;
+ return ret;
+}
+
+static ssize_t kpageidle_write(struct file *file, const char __user *buf,
+ size_t count, loff_t *ppos)
+{
+ const u64 __user *in = (const u64 __user *)buf;
+ struct page *page;
+ unsigned long pfn, end_pfn;
+ ssize_t ret = 0;
+ u64 idle_bitmap = 0;
+ int bit;
+
+ if (*ppos & KPMMASK || count & KPMMASK)
+ return -EINVAL;
+
+ pfn = *ppos * BITS_PER_BYTE;
+ if (pfn >= max_pfn)
+ return -ENXIO;
+
+ end_pfn = pfn + count * BITS_PER_BYTE;
+ if (end_pfn > max_pfn)
+ end_pfn = ALIGN(max_pfn, KPMBITS);
+
+ for (; pfn < end_pfn; pfn++) {
+ bit = pfn % KPMBITS;
+ if (bit == 0) {
+ if (get_user(idle_bitmap, in)) {
+ ret = -EFAULT;
+ break;
+ }
+ in++;
+ }
+ if (idle_bitmap >> bit & 1) {
+ page = kpageidle_get_page(pfn);
+ if (page) {
+ kpageidle_clear_pte_refs(page);
+ set_page_idle(page);
+ put_page(page);
+ }
+ }
+ }
+
+ *ppos += (const char __user *)in - buf;
+ if (!ret)
+ ret = (const char __user *)in - buf;
+ return ret;
+}
+
+static const struct file_operations proc_kpageidle_operations = {
+ .llseek = mem_lseek,
+ .read = kpageidle_read,
+ .write = kpageidle_write,
+};
+
+#ifndef CONFIG_64BIT
+static bool need_page_idle(void)
+{
+ return true;
+}
+struct page_ext_operations page_idle_ops = {
+ .need = need_page_idle,
+};
+#endif
+#endif /* CONFIG_IDLE_PAGE_TRACKING */
+
static int __init proc_page_init(void)
{
proc_create("kpagecount", S_IRUSR, NULL, &proc_kpagecount_operations);
@@ -282,6 +500,10 @@ static int __init proc_page_init(void)
#ifdef CONFIG_MEMCG
proc_create("kpagecgroup", S_IRUSR, NULL, &proc_kpagecgroup_operations);
#endif
+#ifdef CONFIG_IDLE_PAGE_TRACKING
+ proc_create("kpageidle", S_IRUSR | S_IWUSR, NULL,
+ &proc_kpageidle_operations);
+#endif
return 0;
}
fs_initcall(proc_page_init);
diff --git a/fs/proc/task_mmu.c b/fs/proc/task_mmu.c
index 3b4d8255e806..3efd7f641f92 100644
--- a/fs/proc/task_mmu.c
+++ b/fs/proc/task_mmu.c
@@ -458,7 +458,7 @@ static void smaps_account(struct mem_size_stats *mss, struct page *page,
mss->resident += size;
/* Accumulate the size in pages that have been accessed. */
- if (young || PageReferenced(page))
+ if (young || page_is_young(page) || PageReferenced(page))
mss->referenced += size;
mapcount = page_mapcount(page);
if (mapcount >= 2) {
@@ -810,6 +810,7 @@ static int clear_refs_pte_range(pmd_t *pmd, unsigned long addr,
/* Clear accessed and referenced bits. */
pmdp_test_and_clear_young(vma, addr, pmd);
+ test_and_clear_page_young(page);
ClearPageReferenced(page);
out:
spin_unlock(ptl);
@@ -837,6 +838,7 @@ out:
/* Clear accessed and referenced bits. */
ptep_test_and_clear_young(vma, addr, pte);
+ test_and_clear_page_young(page);
ClearPageReferenced(page);
}
pte_unmap_unlock(pte - 1, ptl);
diff --git a/include/linux/mm.h b/include/linux/mm.h
index 7f471789781a..de450c1191b9 100644
--- a/include/linux/mm.h
+++ b/include/linux/mm.h
@@ -2205,5 +2205,103 @@ void __init setup_nr_node_ids(void);
static inline void setup_nr_node_ids(void) {}
#endif
+#ifdef CONFIG_IDLE_PAGE_TRACKING
+#ifdef CONFIG_64BIT
+static inline bool page_is_young(struct page *page)
+{
+ return PageYoung(page);
+}
+
+static inline void set_page_young(struct page *page)
+{
+ SetPageYoung(page);
+}
+
+static inline bool test_and_clear_page_young(struct page *page)
+{
+ return TestClearPageYoung(page);
+}
+
+static inline bool page_is_idle(struct page *page)
+{
+ return PageIdle(page);
+}
+
+static inline void set_page_idle(struct page *page)
+{
+ SetPageIdle(page);
+}
+
+static inline void clear_page_idle(struct page *page)
+{
+ ClearPageIdle(page);
+}
+#else /* !CONFIG_64BIT */
+/*
+ * If there is not enough space to store Idle and Young bits in page flags, use
+ * page ext flags instead.
+ */
+extern struct page_ext_operations page_idle_ops;
+
+static inline bool page_is_young(struct page *page)
+{
+ return test_bit(PAGE_EXT_YOUNG, &lookup_page_ext(page)->flags);
+}
+
+static inline void set_page_young(struct page *page)
+{
+ set_bit(PAGE_EXT_YOUNG, &lookup_page_ext(page)->flags);
+}
+
+static inline bool test_and_clear_page_young(struct page *page)
+{
+ return test_and_clear_bit(PAGE_EXT_YOUNG,
+ &lookup_page_ext(page)->flags);
+}
+
+static inline bool page_is_idle(struct page *page)
+{
+ return test_bit(PAGE_EXT_IDLE, &lookup_page_ext(page)->flags);
+}
+
+static inline void set_page_idle(struct page *page)
+{
+ set_bit(PAGE_EXT_IDLE, &lookup_page_ext(page)->flags);
+}
+
+static inline void clear_page_idle(struct page *page)
+{
+ clear_bit(PAGE_EXT_IDLE, &lookup_page_ext(page)->flags);
+}
+#endif /* CONFIG_64BIT */
+#else /* !CONFIG_IDLE_PAGE_TRACKING */
+static inline bool page_is_young(struct page *page)
+{
+ return false;
+}
+
+static inline void set_page_young(struct page *page)
+{
+}
+
+static inline bool test_and_clear_page_young(struct page *page)
+{
+ return false;
+}
+
+static inline bool page_is_idle(struct page *page)
+{
+ return false;
+}
+
+static inline void set_page_idle(struct page *page)
+{
+}
+
+static inline void clear_page_idle(struct page *page)
+{
+}
+#endif /* CONFIG_IDLE_PAGE_TRACKING */
+
#endif /* __KERNEL__ */
#endif /* _LINUX_MM_H */
diff --git a/include/linux/page-flags.h b/include/linux/page-flags.h
index 91b7f9b2b774..478f2241f284 100644
--- a/include/linux/page-flags.h
+++ b/include/linux/page-flags.h
@@ -109,6 +109,10 @@ enum pageflags {
#ifdef CONFIG_TRANSPARENT_HUGEPAGE
PG_compound_lock,
#endif
+#if defined(CONFIG_IDLE_PAGE_TRACKING) && defined(CONFIG_64BIT)
+ PG_young,
+ PG_idle,
+#endif
__NR_PAGEFLAGS,
/* Filesystems */
@@ -363,6 +367,13 @@ PAGEFLAG_FALSE(HWPoison)
#define __PG_HWPOISON 0
#endif
+#if defined(CONFIG_IDLE_PAGE_TRACKING) && defined(CONFIG_64BIT)
+TESTPAGEFLAG(Young, young, PF_ANY)
+SETPAGEFLAG(Young, young, PF_ANY)
+TESTCLEARFLAG(Young, young, PF_ANY)
+PAGEFLAG(Idle, idle, PF_ANY)
+#endif
+
/*
* On an anonymous page mapped into a user virtual memory area,
* page->mapping points to its anon_vma, not to a struct address_space;
diff --git a/include/linux/page_ext.h b/include/linux/page_ext.h
index c42981cd99aa..17f118a82854 100644
--- a/include/linux/page_ext.h
+++ b/include/linux/page_ext.h
@@ -26,6 +26,10 @@ enum page_ext_flags {
PAGE_EXT_DEBUG_POISON, /* Page is poisoned */
PAGE_EXT_DEBUG_GUARD,
PAGE_EXT_OWNER,
+#if defined(CONFIG_IDLE_PAGE_TRACKING) && !defined(CONFIG_64BIT)
+ PAGE_EXT_YOUNG,
+ PAGE_EXT_IDLE,
+#endif
};
/*
diff --git a/mm/Kconfig b/mm/Kconfig
index e79de2bd12cd..db817e2c2ec8 100644
--- a/mm/Kconfig
+++ b/mm/Kconfig
@@ -654,3 +654,15 @@ config DEFERRED_STRUCT_PAGE_INIT
when kswapd starts. This has a potential performance impact on
processes running early in the lifetime of the systemm until kswapd
finishes the initialisation.
+
+config IDLE_PAGE_TRACKING
+ bool "Enable idle page tracking"
+ select PROC_PAGE_MONITOR
+ select PAGE_EXTENSION if !64BIT
+ help
+ This feature allows to estimate the amount of user pages that have
+ not been touched during a given period of time. This information can
+ be useful to tune memory cgroup limits and/or for job placement
+ within a compute cluster.
+
+ See Documentation/vm/pagemap.txt for more details.
diff --git a/mm/debug.c b/mm/debug.c
index 76089ddf99ea..6c1b3ea61bfd 100644
--- a/mm/debug.c
+++ b/mm/debug.c
@@ -48,6 +48,10 @@ static const struct trace_print_flags pageflag_names[] = {
#ifdef CONFIG_TRANSPARENT_HUGEPAGE
{1UL << PG_compound_lock, "compound_lock" },
#endif
+#if defined(CONFIG_IDLE_PAGE_TRACKING) && defined(CONFIG_64BIT)
+ {1UL << PG_young, "young" },
+ {1UL << PG_idle, "idle" },
+#endif
};
static void dump_flags(unsigned long flags,
diff --git a/mm/huge_memory.c b/mm/huge_memory.c
index 9671f51e954d..db404966faf4 100644
--- a/mm/huge_memory.c
+++ b/mm/huge_memory.c
@@ -1754,6 +1754,11 @@ static void __split_huge_page_refcount(struct page *page,
/* clear PageTail before overwriting first_page */
smp_wmb();
+ if (page_is_young(page))
+ set_page_young(page_tail);
+ if (page_is_idle(page))
+ set_page_idle(page_tail);
+
/*
* __split_huge_page_splitting() already set the
* splitting bit in all pmd that could map this
diff --git a/mm/page_ext.c b/mm/page_ext.c
index d86fd2f5353f..e4b3af054bf2 100644
--- a/mm/page_ext.c
+++ b/mm/page_ext.c
@@ -59,6 +59,9 @@ static struct page_ext_operations *page_ext_ops[] = {
#ifdef CONFIG_PAGE_OWNER
&page_owner_ops,
#endif
+#if defined(CONFIG_IDLE_PAGE_TRACKING) && !defined(CONFIG_64BIT)
+ &page_idle_ops,
+#endif
};
static unsigned long total_usage;
diff --git a/mm/rmap.c b/mm/rmap.c
index 49b244b1f18c..c96677ade3d1 100644
--- a/mm/rmap.c
+++ b/mm/rmap.c
@@ -798,6 +798,11 @@ static int page_referenced_one(struct page *page, struct vm_area_struct *vma,
pte_unmap_unlock(pte, ptl);
}
+ if (referenced)
+ clear_page_idle(page);
+ if (test_and_clear_page_young(page))
+ referenced++;
+
if (referenced) {
pra->referenced++;
pra->vm_flags |= vma->vm_flags;
diff --git a/mm/swap.c b/mm/swap.c
index ab7c338eda87..db43c9b4891d 100644
--- a/mm/swap.c
+++ b/mm/swap.c
@@ -623,6 +623,8 @@ void mark_page_accessed(struct page *page)
} else if (!PageReferenced(page)) {
SetPageReferenced(page);
}
+ if (page_is_idle(page))
+ clear_page_idle(page);
}
EXPORT_SYMBOL(mark_page_accessed);
--
2.1.4
--
To unsubscribe, send a message with 'unsubscribe linux-mm' in
the body to majordomo@kvack.org. For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"dont@kvack.org"> email@kvack.org </a>
^ permalink raw reply related
* [PATCH -mm v7 6/6] proc: export idle flag via kpageflags
From: Vladimir Davydov @ 2015-07-11 14:48 UTC (permalink / raw)
To: Andrew Morton
Cc: Andres Lagar-Cavilla, Minchan Kim, Raghavendra K T,
Johannes Weiner, Michal Hocko, Greg Thelen, Michel Lespinasse,
David Rientjes, Pavel Emelyanov, Cyrill Gorcunov, Jonathan Corbet,
linux-api, linux-doc, linux-mm, cgroups, linux-kernel
In-Reply-To: <cover.1436623799.git.vdavydov@parallels.com>
As noted by Minchan, a benefit of reading idle flag from
/proc/kpageflags is that one can easily filter dirty and/or unevictable
pages while estimating the size of unused memory.
Note that idle flag read from /proc/kpageflags may be stale in case the
page was accessed via a PTE, because it would be too costly to iterate
over all page mappings on each /proc/kpageflags read to provide an
up-to-date value. To make sure the flag is up-to-date one has to read
/proc/kpageidle first.
Signed-off-by: Vladimir Davydov <vdavydov@parallels.com>
---
Documentation/vm/pagemap.txt | 6 ++++++
fs/proc/page.c | 3 +++
include/uapi/linux/kernel-page-flags.h | 1 +
3 files changed, 10 insertions(+)
diff --git a/Documentation/vm/pagemap.txt b/Documentation/vm/pagemap.txt
index c9266340852c..5896b7d7fd74 100644
--- a/Documentation/vm/pagemap.txt
+++ b/Documentation/vm/pagemap.txt
@@ -64,6 +64,7 @@ There are five components to pagemap:
22. THP
23. BALLOON
24. ZERO_PAGE
+ 25. IDLE
* /proc/kpagecgroup. This file contains a 64-bit inode number of the
memory cgroup each page is charged to, indexed by PFN. Only available when
@@ -124,6 +125,11 @@ Short descriptions to the page flags:
24. ZERO_PAGE
zero page for pfn_zero or huge_zero page
+25. IDLE
+ page has not been accessed since it was marked idle (see /proc/kpageidle)
+ Note that this flag may be stale in case the page was accessed via a PTE.
+ To make sure the flag is up-to-date one has to read /proc/kpageidle first.
+
[IO related page flags]
1. ERROR IO error occurred
3. UPTODATE page has up-to-date data
diff --git a/fs/proc/page.c b/fs/proc/page.c
index e51690c5f173..4fa4eadfd30e 100644
--- a/fs/proc/page.c
+++ b/fs/proc/page.c
@@ -149,6 +149,9 @@ u64 stable_page_flags(struct page *page)
if (PageBalloon(page))
u |= 1 << KPF_BALLOON;
+ if (page_is_idle(page))
+ u |= 1 << KPF_IDLE;
+
u |= kpf_copy_bit(k, KPF_LOCKED, PG_locked);
u |= kpf_copy_bit(k, KPF_SLAB, PG_slab);
diff --git a/include/uapi/linux/kernel-page-flags.h b/include/uapi/linux/kernel-page-flags.h
index a6c4962e5d46..5da5f8751ce7 100644
--- a/include/uapi/linux/kernel-page-flags.h
+++ b/include/uapi/linux/kernel-page-flags.h
@@ -33,6 +33,7 @@
#define KPF_THP 22
#define KPF_BALLOON 23
#define KPF_ZERO_PAGE 24
+#define KPF_IDLE 25
#endif /* _UAPILINUX_KERNEL_PAGE_FLAGS_H */
--
2.1.4
--
To unsubscribe, send a message with 'unsubscribe linux-mm' in
the body to majordomo@kvack.org. For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"dont@kvack.org"> email@kvack.org </a>
^ permalink raw reply related
* Re: [PATCH -mm v6 5/6] proc: add kpageidle file
From: Vladimir Davydov @ 2015-07-11 14:53 UTC (permalink / raw)
To: Andres Lagar-Cavilla
Cc: Andrew Morton, Minchan Kim, Raghavendra K T, Johannes Weiner,
Michal Hocko, Greg Thelen, Michel Lespinasse, David Rientjes,
Pavel Emelyanov, Cyrill Gorcunov, Jonathan Corbet, linux-api,
linux-doc, linux-mm, cgroups, linux-kernel
In-Reply-To: <20150709131900.GK2436@esperanza>
On Thu, Jul 09, 2015 at 04:19:00PM +0300, Vladimir Davydov wrote:
> On Wed, Jul 08, 2015 at 04:01:13PM -0700, Andres Lagar-Cavilla wrote:
> > On Fri, Jun 12, 2015 at 2:52 AM, Vladimir Davydov
> > > +#ifdef CONFIG_IDLE_PAGE_TRACKING
> > > +/*
> > > + * Idle page tracking only considers user memory pages, for other types of
> > > + * pages the idle flag is always unset and an attempt to set it is silently
> > > + * ignored.
> > > + *
> > > + * We treat a page as a user memory page if it is on an LRU list, because it is
> > > + * always safe to pass such a page to page_referenced(), which is essential for
> > > + * idle page tracking. With such an indicator of user pages we can skip
> > > + * isolated pages, but since there are not usually many of them, it will hardly
> > > + * affect the overall result.
> > > + *
> > > + * This function tries to get a user memory page by pfn as described above.
> > > + */
> > > +static struct page *kpageidle_get_page(unsigned long pfn)
> > > +{
> > > + struct page *page;
> > > + struct zone *zone;
> > > +
> > > + if (!pfn_valid(pfn))
> > > + return NULL;
> > > +
> > > + page = pfn_to_page(pfn);
> > > + if (!page || !PageLRU(page))
> >
> > Isolation can race in while you're processing the page, after these
> > checks. This is ok, but worth a small comment.
>
> Agree, will add one.
Oh, the comment is already present - it's in the description to this
function. Minchan asked me to add it long time ago, and so I did.
Completely forgot about it.
Thanks,
Vladimir
^ permalink raw reply
* [RFC v4 24/25] m68k/mac: Fix PRAM accessors
From: Finn Thain @ 2015-07-12 10:25 UTC (permalink / raw)
To: linux-kernel, linux-m68k, linuxppc-dev, Geert Uytterhoeven,
linux-api
In-Reply-To: <20150712102527.356151908@telegraphics.com.au>
[-- Attachment #1: mac68k-fix-pram-accessors --]
[-- Type: text/plain, Size: 3451 bytes --]
Signed-off-by: Finn Thain <fthain@telegraphics.com.au>
---
Tested on a PowerBook 520 and Quadra 650.
Changes since v2:
- Make use of the RTC_* macros from the previous patch and add a few more
besides.
---
arch/m68k/mac/misc.c | 39 +++++++++++++++++++++++++++++++++------
include/uapi/linux/pmu.h | 2 ++
2 files changed, 35 insertions(+), 6 deletions(-)
Index: linux/arch/m68k/mac/misc.c
===================================================================
--- linux.orig/arch/m68k/mac/misc.c 2015-07-12 20:25:22.000000000 +1000
+++ linux/arch/m68k/mac/misc.c 2015-07-12 20:25:23.000000000 +1000
@@ -119,19 +119,22 @@ static void pmu_write_time(long data)
static unsigned char pmu_pram_read_byte(int offset)
{
struct adb_request req;
- if (pmu_request(&req, NULL, 3, PMU_READ_NVRAM,
- (offset >> 8) & 0xFF, offset & 0xFF) < 0)
+
+ if (pmu_request(&req, NULL, 3, PMU_READ_XPRAM,
+ offset & 0xFF, 1) < 0)
return 0;
while (!req.complete)
pmu_poll();
- return req.reply[3];
+
+ return req.reply[1];
}
static void pmu_pram_write_byte(unsigned char data, int offset)
{
struct adb_request req;
- if (pmu_request(&req, NULL, 4, PMU_WRITE_NVRAM,
- (offset >> 8) & 0xFF, offset & 0xFF, data) < 0)
+
+ if (pmu_request(&req, NULL, 4, PMU_WRITE_XPRAM,
+ offset & 0xFF, 1, data) < 0)
return;
while (!req.complete)
pmu_poll();
@@ -257,6 +260,16 @@ static void via_rtc_send(__u8 data)
#define RTC_REG_WRITE_PROTECT 13
/*
+ * Inside Mac has no information about two-byte RTC commands but
+ * the MESS source code has the essentials.
+ */
+
+#define RTC_REG_XPRAM 14
+#define RTC_CMD_XPRAM_READ (RTC_CMD_READ(RTC_REG_XPRAM) << 8)
+#define RTC_CMD_XPRAM_WRITE (RTC_CMD_WRITE(RTC_REG_XPRAM) << 8)
+#define RTC_CMD_XPRAM_ARG(a) (((a & 0xE0) << 3) | ((a & 0x1F) << 2))
+
+/*
* Execute a VIA PRAM/RTC command. For read commands
* data should point to a one-byte buffer for the
* resulting data. For write commands it should point
@@ -303,11 +316,25 @@ static void via_rtc_command(int command,
static unsigned char via_pram_read_byte(int offset)
{
- return 0;
+ unsigned char temp;
+
+ via_rtc_command(RTC_CMD_XPRAM_READ | RTC_CMD_XPRAM_ARG(offset), &temp);
+
+ return temp;
}
static void via_pram_write_byte(unsigned char data, int offset)
{
+ unsigned char temp;
+
+ temp = 0x55;
+ via_rtc_command(RTC_CMD_WRITE(RTC_REG_WRITE_PROTECT), &temp);
+
+ temp = data;
+ via_rtc_command(RTC_CMD_XPRAM_WRITE | RTC_CMD_XPRAM_ARG(offset), &temp);
+
+ temp = 0x55 | RTC_FLG_WRITE_PROTECT;
+ via_rtc_command(RTC_CMD_WRITE(RTC_REG_WRITE_PROTECT), &temp);
}
/*
Index: linux/include/uapi/linux/pmu.h
===================================================================
--- linux.orig/include/uapi/linux/pmu.h 2015-07-12 20:24:53.000000000 +1000
+++ linux/include/uapi/linux/pmu.h 2015-07-12 20:25:23.000000000 +1000
@@ -18,7 +18,9 @@
#define PMU_POWER_CTRL 0x11 /* control power of some devices */
#define PMU_ADB_CMD 0x20 /* send ADB packet */
#define PMU_ADB_POLL_OFF 0x21 /* disable ADB auto-poll */
+#define PMU_WRITE_XPRAM 0x32 /* write eXtended Parameter RAM */
#define PMU_WRITE_NVRAM 0x33 /* write non-volatile RAM */
+#define PMU_READ_XPRAM 0x3a /* read eXtended Parameter RAM */
#define PMU_READ_NVRAM 0x3b /* read non-volatile RAM */
#define PMU_SET_RTC 0x30 /* set real-time clock */
#define PMU_READ_RTC 0x38 /* read real-time clock */
^ permalink raw reply
* [RFC PATCH] getcpu_cache system call: caching current CPU number (x86)
From: Mathieu Desnoyers @ 2015-07-12 18:06 UTC (permalink / raw)
To: Paul Turner
Cc: Mathieu Desnoyers, Andrew Hunter, Peter Zijlstra, Ingo Molnar,
Ben Maurer, Steven Rostedt, Paul E. McKenney, Josh Triplett,
Lai Jiangshan, Linus Torvalds, Andrew Morton,
linux-api-u79uwXL29TY76Z2rM5mHXA
Expose a new system call allowing threads to register a userspace memory
area where to store the current CPU number. Scheduler migration sets the
TIF_NOTIFY_RESUME flag on the current thread. Upon return to user-space,
a notify-resume handler updates the current CPU value within that
user-space memory area.
This getcpu cache is an alternative to the sched_getcpu() vdso which has
a few benefits:
- It is faster to do a memory read that to call a vDSO,
- This cache value can be read from within an inline assembly, which
makes it a useful building block for restartable sequences.
This approach is inspired by Paul Turner and Andrew Hunter's work
on percpu atomics, which lets the kernel handle restart of critical
sections:
Ref.:
* https://lkml.org/lkml/2015/6/24/665
* https://lwn.net/Articles/650333/
* http://www.linuxplumbersconf.org/2013/ocw/system/presentations/1695/original/LPC%20-%20PerCpu%20Atomics.pdf
Benchmarking sched_getcpu() vs tls cache approach. Getting the
current CPU number:
- With Linux vdso: 12.7 ns
- With TLS-cached cpu number: 0.3 ns
Signed-off-by: Mathieu Desnoyers <mathieu.desnoyers-vg+e7yoeK/dWk0Htik3J/w@public.gmane.org>
CC: Paul Turner <pjt-hpIqsD4AKlfQT0dZR+AlfA@public.gmane.org>
CC: Andrew Hunter <ahh-hpIqsD4AKlfQT0dZR+AlfA@public.gmane.org>
CC: Peter Zijlstra <peterz-wEGCiKHe2LqWVfeAwA7xHQ@public.gmane.org>
CC: Ingo Molnar <mingo-H+wXaHxf7aLQT0dZR+AlfA@public.gmane.org>
CC: Ben Maurer <bmaurer-b10kYP2dOMg@public.gmane.org>
CC: Steven Rostedt <rostedt-nx8X9YLhiw1AfugRpC6u6w@public.gmane.org>
CC: "Paul E. McKenney" <paulmck-23VcF4HTsmIX0ybBhKVfKdBPR1lH4CV8@public.gmane.org>
CC: Josh Triplett <josh-iaAMLnmF4UmaiuxdJuQwMA@public.gmane.org>
CC: Lai Jiangshan <laijs-BthXqXjhjHXQFUHtdCDX3A@public.gmane.org>
CC: Linus Torvalds <torvalds-de/tnXTf+JLsfHDXvbKv3WD2FQJk+8+b@public.gmane.org>
CC: Andrew Morton <akpm-de/tnXTf+JLsfHDXvbKv3WD2FQJk+8+b@public.gmane.org>
CC: linux-api-u79uwXL29TY76Z2rM5mHXA@public.gmane.org
---
arch/x86/kernel/signal.c | 2 ++
arch/x86/syscalls/syscall_64.tbl | 1 +
fs/exec.c | 1 +
include/linux/sched.h | 27 +++++++++++++++
include/uapi/asm-generic/unistd.h | 4 ++-
init/Kconfig | 9 +++++
kernel/Makefile | 1 +
kernel/fork.c | 2 ++
kernel/getcpu-cache.c | 70 +++++++++++++++++++++++++++++++++++++++
kernel/sched/core.c | 3 ++
kernel/sched/sched.h | 2 ++
kernel/sys_ni.c | 3 ++
12 files changed, 124 insertions(+), 1 deletion(-)
create mode 100644 kernel/getcpu-cache.c
diff --git a/arch/x86/kernel/signal.c b/arch/x86/kernel/signal.c
index e504246..157cec0 100644
--- a/arch/x86/kernel/signal.c
+++ b/arch/x86/kernel/signal.c
@@ -750,6 +750,8 @@ do_notify_resume(struct pt_regs *regs, void *unused, __u32 thread_info_flags)
if (thread_info_flags & _TIF_NOTIFY_RESUME) {
clear_thread_flag(TIF_NOTIFY_RESUME);
tracehook_notify_resume(regs);
+ if (getcpu_cache_active(current))
+ getcpu_cache_handle_notify_resume(current);
}
if (thread_info_flags & _TIF_USER_RETURN_NOTIFY)
fire_user_return_notifiers();
diff --git a/arch/x86/syscalls/syscall_64.tbl b/arch/x86/syscalls/syscall_64.tbl
index 8d656fb..cfcf8e7 100644
--- a/arch/x86/syscalls/syscall_64.tbl
+++ b/arch/x86/syscalls/syscall_64.tbl
@@ -329,6 +329,7 @@
320 common kexec_file_load sys_kexec_file_load
321 common bpf sys_bpf
322 64 execveat stub_execveat
+323 common getcpu_cache sys_getcpu_cache
#
# x32-specific system call numbers start at 512 to avoid cache impact
diff --git a/fs/exec.c b/fs/exec.c
index c7f9b73..20ef2e6 100644
--- a/fs/exec.c
+++ b/fs/exec.c
@@ -1555,6 +1555,7 @@ static int do_execveat_common(int fd, struct filename *filename,
/* execve succeeded */
current->fs->in_exec = 0;
current->in_execve = 0;
+ getcpu_cache_execve(current);
acct_update_integrals(current);
task_numa_free(current);
free_bprm(bprm);
diff --git a/include/linux/sched.h b/include/linux/sched.h
index a419b65..0654cc2 100644
--- a/include/linux/sched.h
+++ b/include/linux/sched.h
@@ -1710,6 +1710,9 @@ struct task_struct {
#ifdef CONFIG_DEBUG_ATOMIC_SLEEP
unsigned long task_state_change;
#endif
+#ifdef CONFIG_GETCPU_CACHE
+ int32_t __user *getcpu_cache;
+#endif
};
/* Future-safe accessor for struct task_struct's cpus_allowed. */
@@ -3090,4 +3093,28 @@ static inline unsigned long rlimit_max(unsigned int limit)
return task_rlimit_max(current, limit);
}
+#ifdef CONFIG_GETCPU_CACHE
+void getcpu_cache_fork(struct task_struct *t);
+void getcpu_cache_execve(struct task_struct *t);
+void getcpu_cache_handle_notify_resume(struct task_struct *t);
+static inline bool getcpu_cache_active(struct task_struct *t)
+{
+ return t->getcpu_cache;
+}
+#else
+static inline void getcpu_cache_fork(struct task_struct *t)
+{
+}
+static inline void getcpu_cache_execve(struct task_struct *t)
+{
+}
+static inline void getcpu_cache_handle_notify_resume(struct task_struct *t)
+{
+}
+static inline bool getcpu_cache_active(struct task_struct *t)
+{
+ return false;
+}
+#endif
+
#endif
diff --git a/include/uapi/asm-generic/unistd.h b/include/uapi/asm-generic/unistd.h
index e016bd9..f82b70d 100644
--- a/include/uapi/asm-generic/unistd.h
+++ b/include/uapi/asm-generic/unistd.h
@@ -709,9 +709,11 @@ __SYSCALL(__NR_memfd_create, sys_memfd_create)
__SYSCALL(__NR_bpf, sys_bpf)
#define __NR_execveat 281
__SC_COMP(__NR_execveat, sys_execveat, compat_sys_execveat)
+#define __NR_getcpu_cache 282
+__SYSCALL(__NR_getcpu_cache, sys_getcpu_cache)
#undef __NR_syscalls
-#define __NR_syscalls 282
+#define __NR_syscalls 283
/*
* All syscalls below here should go away really,
diff --git a/init/Kconfig b/init/Kconfig
index f5dbc6d..fac919b 100644
--- a/init/Kconfig
+++ b/init/Kconfig
@@ -1559,6 +1559,15 @@ config PCI_QUIRKS
bugs/quirks. Disable this only if your target machine is
unaffected by PCI quirks.
+config GETCPU_CACHE
+ bool "Enable getcpu_cache() system call" if EXPERT
+ default y
+ help
+ Enable the getcpu_cache() system call which provides a
+ user-space cache for the current CPU number value.
+
+ If unsure, say Y.
+
config EMBEDDED
bool "Embedded system"
option allnoconfig_y
diff --git a/kernel/Makefile b/kernel/Makefile
index 1408b33..3350ba1 100644
--- a/kernel/Makefile
+++ b/kernel/Makefile
@@ -96,6 +96,7 @@ obj-$(CONFIG_CRASH_DUMP) += crash_dump.o
obj-$(CONFIG_JUMP_LABEL) += jump_label.o
obj-$(CONFIG_CONTEXT_TRACKING) += context_tracking.o
obj-$(CONFIG_TORTURE_TEST) += torture.o
+obj-$(CONFIG_GETCPU_CACHE) += getcpu-cache.o
$(obj)/configs.o: $(obj)/config_data.h
diff --git a/kernel/fork.c b/kernel/fork.c
index cf65139..334e62d 100644
--- a/kernel/fork.c
+++ b/kernel/fork.c
@@ -1549,6 +1549,8 @@ static struct task_struct *copy_process(unsigned long clone_flags,
cgroup_post_fork(p);
if (clone_flags & CLONE_THREAD)
threadgroup_change_end(current);
+ if (!(clone_flags & CLONE_THREAD))
+ getcpu_cache_fork(p);
perf_event_fork(p);
trace_task_newtask(p, clone_flags);
diff --git a/kernel/getcpu-cache.c b/kernel/getcpu-cache.c
new file mode 100644
index 0000000..b4e5c77
--- /dev/null
+++ b/kernel/getcpu-cache.c
@@ -0,0 +1,70 @@
+/*
+ * Copyright (C) 2015 Mathieu Desnoyers <mathieu.desnoyers-vg+e7yoeK/dWk0Htik3J/w@public.gmane.org>
+ *
+ * getcpu_cache system call
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ */
+
+#include <linux/init.h>
+#include <linux/sched.h>
+#include <linux/uaccess.h>
+#include <linux/syscalls.h>
+
+/*
+ * This resume handler should always be executed between a migration
+ * triggered by preemption and return to user-space.
+ */
+void getcpu_cache_handle_notify_resume(struct task_struct *t)
+{
+ int32_t __user *gcp = t->getcpu_cache;
+
+ if (gcp == NULL)
+ return;
+ if (unlikely(t->flags & PF_EXITING))
+ return;
+ /*
+ * access_ok() of gcp_user has already been checked by
+ * sys_getcpu_cache().
+ */
+ if (__put_user(raw_smp_processor_id(), gcp))
+ force_sig(SIGSEGV, current);
+}
+
+/*
+ * If parent process has a getcpu_cache, the child inherits. Only
+ * applies when forking a process, not a thread.
+ */
+void getcpu_cache_fork(struct task_struct *t)
+{
+ t->getcpu_cache = current->getcpu_cache;
+}
+
+void getcpu_cache_execve(struct task_struct *t)
+{
+ t->getcpu_cache = NULL;
+}
+
+/*
+ * sys_getcpu_cache - setup getcpu cache for caller thread
+ */
+SYSCALL_DEFINE2(getcpu_cache, int32_t __user *, gcp, int, flags)
+{
+ if (flags)
+ return -EINVAL;
+ if (gcp != NULL && !access_ok(VERIFY_WRITE, gcp, sizeof(int32_t)))
+ return -EFAULT;
+ current->getcpu_cache = gcp;
+ /* Will update *gcp on resume */
+ if (gcp)
+ set_thread_flag(TIF_NOTIFY_RESUME);
+ return 0;
+}
diff --git a/kernel/sched/core.c b/kernel/sched/core.c
index 62671f5..a9009d4 100644
--- a/kernel/sched/core.c
+++ b/kernel/sched/core.c
@@ -1823,6 +1823,9 @@ static void __sched_fork(unsigned long clone_flags, struct task_struct *p)
p->numa_group = NULL;
#endif /* CONFIG_NUMA_BALANCING */
+#ifdef CONFIG_GETCPU_CACHE
+ p->getcpu_cache = NULL;
+#endif
}
#ifdef CONFIG_NUMA_BALANCING
diff --git a/kernel/sched/sched.h b/kernel/sched/sched.h
index dc0f435..bf3e346 100644
--- a/kernel/sched/sched.h
+++ b/kernel/sched/sched.h
@@ -921,6 +921,8 @@ static inline void __set_task_cpu(struct task_struct *p, unsigned int cpu)
{
set_task_rq(p, cpu);
#ifdef CONFIG_SMP
+ if (getcpu_cache_active(p))
+ set_tsk_thread_flag(p, TIF_NOTIFY_RESUME);
/*
* After ->cpu is set up to a new value, task_rq_lock(p, ...) can be
* successfuly executed on another CPU. We must ensure that updates of
diff --git a/kernel/sys_ni.c b/kernel/sys_ni.c
index 5adcb0a..3691dc8 100644
--- a/kernel/sys_ni.c
+++ b/kernel/sys_ni.c
@@ -229,3 +229,6 @@ cond_syscall(sys_bpf);
/* execveat */
cond_syscall(sys_execveat);
+
+/* current CPU number cache */
+cond_syscall(sys_getcpu_cache);
--
2.1.4
^ permalink raw reply related
* Re: [RFC PATCH] getcpu_cache system call: caching current CPU number (x86)
From: Josh Triplett @ 2015-07-12 18:47 UTC (permalink / raw)
To: Mathieu Desnoyers
Cc: Paul Turner, Andrew Hunter, Peter Zijlstra, Ingo Molnar,
Ben Maurer, Steven Rostedt, Paul E. McKenney, Lai Jiangshan,
Linus Torvalds, Andrew Morton, linux-api-u79uwXL29TY76Z2rM5mHXA
In-Reply-To: <1436724386-30909-1-git-send-email-mathieu.desnoyers-vg+e7yoeK/dWk0Htik3J/w@public.gmane.org>
On Sun, Jul 12, 2015 at 02:06:26PM -0400, Mathieu Desnoyers wrote:
> Expose a new system call allowing threads to register a userspace memory
> area where to store the current CPU number. Scheduler migration sets the
> TIF_NOTIFY_RESUME flag on the current thread. Upon return to user-space,
> a notify-resume handler updates the current CPU value within that
> user-space memory area.
>
> This getcpu cache is an alternative to the sched_getcpu() vdso which has
> a few benefits:
> - It is faster to do a memory read that to call a vDSO,
> - This cache value can be read from within an inline assembly, which
> makes it a useful building block for restartable sequences.
>
> This approach is inspired by Paul Turner and Andrew Hunter's work
> on percpu atomics, which lets the kernel handle restart of critical
> sections:
> Ref.:
> * https://lkml.org/lkml/2015/6/24/665
> * https://lwn.net/Articles/650333/
> * http://www.linuxplumbersconf.org/2013/ocw/system/presentations/1695/original/LPC%20-%20PerCpu%20Atomics.pdf
>
> Benchmarking sched_getcpu() vs tls cache approach. Getting the
> current CPU number:
>
> - With Linux vdso: 12.7 ns
> - With TLS-cached cpu number: 0.3 ns
Nice. One comment below about an interesting assumption that needs
confirmation.
> --- /dev/null
> +++ b/kernel/getcpu-cache.c
[...]
> +void getcpu_cache_handle_notify_resume(struct task_struct *t)
> +{
> + int32_t __user *gcp = t->getcpu_cache;
> +
> + if (gcp == NULL)
> + return;
> + if (unlikely(t->flags & PF_EXITING))
> + return;
> + /*
> + * access_ok() of gcp_user has already been checked by
> + * sys_getcpu_cache().
> + */
> + if (__put_user(raw_smp_processor_id(), gcp))
> + force_sig(SIGSEGV, current);
> +}
> +
> +/*
> + * If parent process has a getcpu_cache, the child inherits. Only
> + * applies when forking a process, not a thread.
> + */
> +void getcpu_cache_fork(struct task_struct *t)
> +{
> + t->getcpu_cache = current->getcpu_cache;
> +}
> +
> +void getcpu_cache_execve(struct task_struct *t)
> +{
> + t->getcpu_cache = NULL;
> +}
> +
> +/*
> + * sys_getcpu_cache - setup getcpu cache for caller thread
> + */
> +SYSCALL_DEFINE2(getcpu_cache, int32_t __user *, gcp, int, flags)
> +{
> + if (flags)
> + return -EINVAL;
> + if (gcp != NULL && !access_ok(VERIFY_WRITE, gcp, sizeof(int32_t)))
> + return -EFAULT;
> + current->getcpu_cache = gcp;
So, you store a userspace address, and intentionally only validate it
when initially set, not when used. You clear it on exec, though not on
fork. Could any cases other than exec could make this problematic? In
particular, what about unusual personality flags, such as
ADDR_LIMIT_32BIT or ADDR_LIMIT_3GB?
> + /* Will update *gcp on resume */
> + if (gcp)
Minor nit: you're using the pointer as a boolean here, but comparing it
to NULL elsewhere; you should be consistent. I'd suggest consistently
using gcp and !gcp, without the comparison to NULL.
- Josh Triplett
^ permalink raw reply
* Re: [RFC PATCH] getcpu_cache system call: caching current CPU number (x86)
From: Andy Lutomirski @ 2015-07-13 3:38 UTC (permalink / raw)
To: Mathieu Desnoyers
Cc: Paul E. McKenney, Ben Maurer, Ingo Molnar, Andrew Morton,
Josh Triplett, Lai Jiangshan, Paul Turner, Steven Rostedt,
Andrew Hunter, Linux API, Linus Torvalds, Peter Zijlstra
In-Reply-To: <1436724386-30909-1-git-send-email-mathieu.desnoyers-vg+e7yoeK/dWk0Htik3J/w@public.gmane.org>
On Jul 12, 2015 12:06 PM, "Mathieu Desnoyers"
<mathieu.desnoyers-vg+e7yoeK/dWk0Htik3J/w@public.gmane.org> wrote:
>
> Expose a new system call allowing threads to register a userspace memory
> area where to store the current CPU number. Scheduler migration sets the
> TIF_NOTIFY_RESUME flag on the current thread. Upon return to user-space,
> a notify-resume handler updates the current CPU value within that
> user-space memory area.
>
> This getcpu cache is an alternative to the sched_getcpu() vdso which has
> a few benefits:
> - It is faster to do a memory read that to call a vDSO,
> - This cache value can be read from within an inline assembly, which
> makes it a useful building block for restartable sequences.
>
Let's wait and see what the final percpu atomic solution is. If it
involves percpu segments, then this is unnecessary.
Also, this will need to be rebased onto -tip, and that should wait
until the big exit rewrite is farther along.
> This approach is inspired by Paul Turner and Andrew Hunter's work
> on percpu atomics, which lets the kernel handle restart of critical
> sections:
> Ref.:
> * https://lkml.org/lkml/2015/6/24/665
> * https://lwn.net/Articles/650333/
> * http://www.linuxplumbersconf.org/2013/ocw/system/presentations/1695/original/LPC%20-%20PerCpu%20Atomics.pdf
>
> Benchmarking sched_getcpu() vs tls cache approach. Getting the
> current CPU number:
>
> - With Linux vdso: 12.7 ns
This is a bit unfair, because the glibc wrapper sucks and the
__vdso_getcpu interface is overcomplicated. We can fix it with a
better API. It won't make it *that* much faster, though.
>
> diff --git a/arch/x86/kernel/signal.c b/arch/x86/kernel/signal.c
> index e504246..157cec0 100644
> --- a/arch/x86/kernel/signal.c
> +++ b/arch/x86/kernel/signal.c
> @@ -750,6 +750,8 @@ do_notify_resume(struct pt_regs *regs, void *unused, __u32 thread_info_flags)
> if (thread_info_flags & _TIF_NOTIFY_RESUME) {
> clear_thread_flag(TIF_NOTIFY_RESUME);
> tracehook_notify_resume(regs);
> + if (getcpu_cache_active(current))
> + getcpu_cache_handle_notify_resume(current);
We need to disentangle this stuff. This is buried way too deeply here.
Fortunately, do_notify_resume is going away. It's already unused on
64-bit kernels in -next.
> +/*
> + * This resume handler should always be executed between a migration
> + * triggered by preemption and return to user-space.
> + */
> +void getcpu_cache_handle_notify_resume(struct task_struct *t)
> +{
> + int32_t __user *gcp = t->getcpu_cache;
> +
> + if (gcp == NULL)
> + return;
> + if (unlikely(t->flags & PF_EXITING))
> + return;
> + /*
> + * access_ok() of gcp_user has already been checked by
> + * sys_getcpu_cache().
> + */
> + if (__put_user(raw_smp_processor_id(), gcp))
> + force_sig(SIGSEGV, current);
We're preemptible here, although I think it's okay. But I'd at least
clear the getcpu_cache state if __put_user fails, because otherwise
it's not entirely obvious to me that we can't infinite loop.
> +/*
> + * sys_getcpu_cache - setup getcpu cache for caller thread
> + */
> +SYSCALL_DEFINE2(getcpu_cache, int32_t __user *, gcp, int, flags)
> +{
> + if (flags)
> + return -EINVAL;
> + if (gcp != NULL && !access_ok(VERIFY_WRITE, gcp, sizeof(int32_t)))
> + return -EFAULT;
> + current->getcpu_cache = gcp;
> + /* Will update *gcp on resume */
> + if (gcp)
> + set_thread_flag(TIF_NOTIFY_RESUME);
> + return 0;
> +}
IMO this is impolite. If the pointer is bad, we should return -EFAULT
rather than sending SIGSEGV.
--Andy
^ permalink raw reply
* Re: [RFC PATCH] getcpu_cache system call: caching current CPU number (x86)
From: Andy Lutomirski @ 2015-07-13 3:40 UTC (permalink / raw)
To: Josh Triplett
Cc: Mathieu Desnoyers, Paul Turner, Andrew Hunter, Peter Zijlstra,
Ingo Molnar, Ben Maurer, Steven Rostedt, Paul E. McKenney,
Lai Jiangshan, Linus Torvalds, Andrew Morton, Linux API
In-Reply-To: <20150712184730.GD18191@x>
On Sun, Jul 12, 2015 at 11:47 AM, Josh Triplett <josh-iaAMLnmF4UmaiuxdJuQwMA@public.gmane.org> wrote:
> On Sun, Jul 12, 2015 at 02:06:26PM -0400, Mathieu Desnoyers wrote:
>> Expose a new system call allowing threads to register a userspace memory
>> area where to store the current CPU number. Scheduler migration sets the
>> TIF_NOTIFY_RESUME flag on the current thread. Upon return to user-space,
>> a notify-resume handler updates the current CPU value within that
>> user-space memory area.
>>
>> +
>> +/*
>> + * sys_getcpu_cache - setup getcpu cache for caller thread
>> + */
>> +SYSCALL_DEFINE2(getcpu_cache, int32_t __user *, gcp, int, flags)
>> +{
>> + if (flags)
>> + return -EINVAL;
>> + if (gcp != NULL && !access_ok(VERIFY_WRITE, gcp, sizeof(int32_t)))
>> + return -EFAULT;
>> + current->getcpu_cache = gcp;
>
> So, you store a userspace address, and intentionally only validate it
> when initially set, not when used. You clear it on exec, though not on
> fork. Could any cases other than exec could make this problematic? In
> particular, what about unusual personality flags, such as
> ADDR_LIMIT_32BIT or ADDR_LIMIT_3GB?
>
On x86, this is safe, although it does violate the rules.
That being said, I've never understood why access_ok cares at all what
kind of task we're in.
^ permalink raw reply
* [PATCH v3 00/10] hugetlbfs: add fallocate support
From: Mike Kravetz @ 2015-07-13 4:20 UTC (permalink / raw)
To: linux-mm, linux-kernel, linux-api
Cc: Dave Hansen, Naoya Horiguchi, David Rientjes, Hugh Dickins,
Davidlohr Bueso, Aneesh Kumar, Hillf Danton, Christoph Hellwig,
Andrew Morton, Michal Hocko, Mike Kravetz
Only change in this revision is the fix to the self-discovered
issue in region_chg(). Functional and stress tests passing.
Full changelog below.
As suggested during the RFC process, tests have been proposed to
libhugetlbfs as described at:
http://librelist.com/browser//libhugetlbfs/2015/6/25/patch-tests-add-tests-for-fallocate-system-call/
fallocate(2) man page modifications are also necessary to specify
that fallocate for hugetlbfs only operates on whole pages. This
change will be submitted once the code has stabilized and been
proposed for merging.
hugetlbfs is used today by applications that want a high degree of
control over huge page usage. Often, large hugetlbfs files are used
to map a large number huge pages into the application processes.
The applications know when page ranges within these large files will
no longer be used, and ideally would like to release them back to
the subpool or global pools for other uses. The fallocate() system
call provides an interface for preallocation and hole punching within
files. This patch set adds fallocate functionality to hugetlbfs.
v3:
Fixed issue with region_chg to recheck if there are sufficient
entries in the cache after acquiring lock.
v2:
Fixed leak in resv_map_release discovered by Hillf Danton.
Used LONG_MAX as indicator of truncate function for region_del.
v1:
Add a cache of region descriptors to the resv_map for use by
region_add in case hole punch deletes entries necessary for
a successful operation.
RFC v4:
Removed alloc_huge_page/hugetlb_reserve_pages race patches as already
in mmotm
Moved hugetlb_fix_reserve_counts in series as suggested by Naoya Horiguchi
Inline'ed hugetlb_fault_mutex routines as suggested by Davidlohr Bueso and
existing code changed to use new interfaces as suggested by Naoya
fallocate preallocation code cleaned up and made simpler
Modified alloc_huge_page to handle special case where allocation is
for a hole punched area with spool reserves
RFC v3:
Folded in patch for alloc_huge_page/hugetlb_reserve_pages race
in current code
fallocate allocation and hole punch is synchronized with page
faults via existing mutex table
hole punch uses existing hugetlb_vmtruncate_list instead of more
generic unmap_mapping_range for unmapping
Error handling for the case when region_del() fauils
RFC v2:
Addressed alignment and error handling issues noticed by Hillf Danton
New region_del() routine for region tracking/resv_map of ranges
Fixed several issues found during more extensive testing
Error handling in region_del() when kmalloc() fails stills needs
to be addressed
madvise remove support remains
Mike Kravetz (10):
mm/hugetlb: add cache of descriptors to resv_map for region_add
mm/hugetlb: add region_del() to delete a specific range of entries
mm/hugetlb: expose hugetlb fault mutex for use by fallocate
hugetlbfs: hugetlb_vmtruncate_list() needs to take a range to delete
hugetlbfs: truncate_hugepages() takes a range of pages
mm/hugetlb: vma_has_reserves() needs to handle fallocate hole punch
mm/hugetlb: alloc_huge_page handle areas hole punched by fallocate
hugetlbfs: New huge_add_to_page_cache helper routine
hugetlbfs: add hugetlbfs_fallocate()
mm: madvise allow remove operation for hugetlbfs
fs/hugetlbfs/inode.c | 281 +++++++++++++++++++++++++++++---
include/linux/hugetlb.h | 17 +-
mm/hugetlb.c | 423 ++++++++++++++++++++++++++++++++++++++----------
mm/madvise.c | 2 +-
4 files changed, 619 insertions(+), 104 deletions(-)
--
2.1.0
--
To unsubscribe, send a message with 'unsubscribe linux-mm' in
the body to majordomo@kvack.org. For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"dont@kvack.org"> email@kvack.org </a>
^ permalink raw reply
* [PATCH v3 01/10] mm/hugetlb: add cache of descriptors to resv_map for region_add
From: Mike Kravetz @ 2015-07-13 4:20 UTC (permalink / raw)
To: linux-mm, linux-kernel, linux-api
Cc: Dave Hansen, Naoya Horiguchi, David Rientjes, Hugh Dickins,
Davidlohr Bueso, Aneesh Kumar, Hillf Danton, Christoph Hellwig,
Andrew Morton, Michal Hocko, Mike Kravetz
In-Reply-To: <1436761268-6397-1-git-send-email-mike.kravetz@oracle.com>
fallocate hole punch will want to remove a specific range of
pages. When pages are removed, their associated entries in
the region/reserve map will also be removed. This will break
an assumption in the region_chg/region_add calling sequence.
If a new region descriptor must be allocated, it is done as
part of the region_chg processing. In this way, region_add
can not fail because it does not need to attempt an allocation.
To prepare for fallocate hole punch, create a "cache" of
descriptors that can be used by region_add if necessary.
region_chg will ensure there are sufficient entries in the
cache. It will be necessary to track the number of in progress
add operations to know a sufficient number of descriptors
reside in the cache. A new routine region_abort is added to
adjust this in progress count when add operations are aborted.
vma_abort_reservation is also added for callers creating
reservations with vma_needs_reservation/vma_commit_reservation.
Signed-off-by: Mike Kravetz <mike.kravetz@oracle.com>
---
include/linux/hugetlb.h | 3 +
mm/hugetlb.c | 169 ++++++++++++++++++++++++++++++++++++++++++------
2 files changed, 153 insertions(+), 19 deletions(-)
diff --git a/include/linux/hugetlb.h b/include/linux/hugetlb.h
index d891f94..667cf44 100644
--- a/include/linux/hugetlb.h
+++ b/include/linux/hugetlb.h
@@ -35,6 +35,9 @@ struct resv_map {
struct kref refs;
spinlock_t lock;
struct list_head regions;
+ long adds_in_progress;
+ struct list_head rgn_cache;
+ long rgn_cache_count;
};
extern struct resv_map *resv_map_alloc(void);
void resv_map_release(struct kref *ref);
diff --git a/mm/hugetlb.c b/mm/hugetlb.c
index a8c3087..241d16d 100644
--- a/mm/hugetlb.c
+++ b/mm/hugetlb.c
@@ -240,11 +240,14 @@ struct file_region {
/*
* Add the huge page range represented by [f, t) to the reserve
- * map. Existing regions will be expanded to accommodate the
- * specified range. We know only existing regions need to be
- * expanded, because region_add is only called after region_chg
- * with the same range. If a new file_region structure must
- * be allocated, it is done in region_chg.
+ * map. In the normal case, existing regions will be expanded
+ * to accommodate the specified range. Sufficient regions should
+ * exist for expansion due to the previous call to region_chg
+ * with the same range. However, it is possible that region_del
+ * could have been called after region_chg and modifed the map
+ * in such a way that no region exists to be expanded. In this
+ * case, pull a region descriptor from the cache associated with
+ * the map and use that for the new range.
*
* Return the number of new huge pages added to the map. This
* number is greater than or equal to zero.
@@ -261,6 +264,27 @@ static long region_add(struct resv_map *resv, long f, long t)
if (f <= rg->to)
break;
+ if (&rg->link == head || t < rg->from) {
+ /*
+ * No region exists which can be expanded to include the
+ * specified range. Pull a region descriptor from the
+ * cache, and use it for this range.
+ */
+ VM_BUG_ON(!resv->rgn_cache_count);
+
+ resv->rgn_cache_count--;
+ nrg = list_first_entry(&resv->rgn_cache, struct file_region,
+ link);
+ list_del(&nrg->link);
+
+ nrg->from = f;
+ nrg->to = t;
+ list_add(&nrg->link, rg->link.prev);
+
+ add += t - f;
+ goto out_locked;
+ }
+
/* Round our left edge to the current segment if it encloses us. */
if (f > rg->from)
f = rg->from;
@@ -294,6 +318,8 @@ static long region_add(struct resv_map *resv, long f, long t)
add += t - nrg->to; /* Added to end of region */
nrg->to = t;
+out_locked:
+ resv->adds_in_progress--;
spin_unlock(&resv->lock);
VM_BUG_ON(add < 0);
return add;
@@ -312,11 +338,16 @@ static long region_add(struct resv_map *resv, long f, long t)
* so that the subsequent region_add call will have all the
* regions it needs and will not fail.
*
+ * Upon entry, region_chg will also examine the cache of
+ * region descriptors associated with the map. If there
+ * not enough descriptors cached, one will be allocated
+ * for the in progress add operation.
+ *
* Returns the number of huge pages that need to be added
* to the existing reservation map for the range [f, t).
* This number is greater or equal to zero. -ENOMEM is
- * returned if a new file_region structure is needed and can
- * not be allocated.
+ * returned if a new file_region structure or cache entry
+ * is needed and can not be allocated.
*/
static long region_chg(struct resv_map *resv, long f, long t)
{
@@ -326,6 +357,31 @@ static long region_chg(struct resv_map *resv, long f, long t)
retry:
spin_lock(&resv->lock);
+retry_locked:
+ resv->adds_in_progress++;
+
+ /*
+ * Check for sufficient descriptors in the cache to accommodate
+ * the number of in progress add operations.
+ */
+ if (resv->adds_in_progress > resv->rgn_cache_count) {
+ struct file_region *trg;
+
+ VM_BUG_ON(resv->adds_in_progress - resv->rgn_cache_count > 1);
+ /* Must drop lock to allocate a new descriptor. */
+ resv->adds_in_progress--;
+ spin_unlock(&resv->lock);
+
+ trg = kmalloc(sizeof(*trg), GFP_KERNEL);
+ if (!trg)
+ return -ENOMEM;
+
+ spin_lock(&resv->lock);
+ list_add(&trg->link, &resv->rgn_cache);
+ resv->rgn_cache_count++;
+ goto retry_locked;
+ }
+
/* Locate the region we are before or in. */
list_for_each_entry(rg, head, link)
if (f <= rg->to)
@@ -336,6 +392,7 @@ retry:
* size such that we can guarantee to record the reservation. */
if (&rg->link == head || t < rg->from) {
if (!nrg) {
+ resv->adds_in_progress--;
spin_unlock(&resv->lock);
nrg = kmalloc(sizeof(*nrg), GFP_KERNEL);
if (!nrg)
@@ -385,6 +442,25 @@ out_nrg:
}
/*
+ * Abort the in progress add operation. The adds_in_progress field
+ * of the resv_map keeps track of the operations in progress between
+ * calls to region_chg and region_add. Operations are sometimes
+ * aborted after the call to region_chg. In such cases, region_abort
+ * is called to decrement the adds_in_progress counter.
+ *
+ * NOTE: The range arguments [f, t) are not needed or used in this
+ * routine. They are kept to make reading the calling code easier as
+ * arguments will match the associated region_chg call.
+ */
+static void region_abort(struct resv_map *resv, long f, long t)
+{
+ spin_lock(&resv->lock);
+ VM_BUG_ON(!resv->rgn_cache_count);
+ resv->adds_in_progress--;
+ spin_unlock(&resv->lock);
+}
+
+/*
* Truncate the reserve map at index 'end'. Modify/truncate any
* region which contains end. Delete any regions past end.
* Return the number of huge pages removed from the map.
@@ -544,22 +620,44 @@ static void set_vma_private_data(struct vm_area_struct *vma,
struct resv_map *resv_map_alloc(void)
{
struct resv_map *resv_map = kmalloc(sizeof(*resv_map), GFP_KERNEL);
- if (!resv_map)
+ struct file_region *rg = kmalloc(sizeof(*rg), GFP_KERNEL);
+
+ if (!resv_map || !rg) {
+ kfree(resv_map);
+ kfree(rg);
return NULL;
+ }
kref_init(&resv_map->refs);
spin_lock_init(&resv_map->lock);
INIT_LIST_HEAD(&resv_map->regions);
+ resv_map->adds_in_progress = 0;
+
+ INIT_LIST_HEAD(&resv_map->rgn_cache);
+ list_add(&rg->link, &resv_map->rgn_cache);
+ resv_map->rgn_cache_count = 1;
+
return resv_map;
}
void resv_map_release(struct kref *ref)
{
struct resv_map *resv_map = container_of(ref, struct resv_map, refs);
+ struct list_head *head = &resv_map->rgn_cache;
+ struct file_region *rg, *trg;
/* Clear out any active regions before we release the map. */
region_truncate(resv_map, 0);
+
+ /* ... and any entries left in the cache */
+ list_for_each_entry_safe(rg, trg, head, link) {
+ list_del(&rg->link);
+ kfree(rg);
+ }
+
+ VM_BUG_ON(resv_map->adds_in_progress);
+
kfree(resv_map);
}
@@ -1473,16 +1571,18 @@ static void return_unused_surplus_pages(struct hstate *h,
}
}
+
/*
- * vma_needs_reservation and vma_commit_reservation are used by the huge
- * page allocation routines to manage reservations.
+ * vma_needs_reservation, vma_commit_reservation and vma_abort_reservation
+ * are used by the huge page allocation routines to manage reservations.
*
* vma_needs_reservation is called to determine if the huge page at addr
* within the vma has an associated reservation. If a reservation is
* needed, the value 1 is returned. The caller is then responsible for
* managing the global reservation and subpool usage counts. After
* the huge page has been allocated, vma_commit_reservation is called
- * to add the page to the reservation map.
+ * to add the page to the reservation map. If the reservation must be
+ * aborted instead of committed, vma_abort_reservation is called.
*
* In the normal case, vma_commit_reservation returns the same value
* as the preceding vma_needs_reservation call. The only time this
@@ -1490,9 +1590,14 @@ static void return_unused_surplus_pages(struct hstate *h,
* is the responsibility of the caller to notice the difference and
* take appropriate action.
*/
+enum vma_resv_mode {
+ VMA_NEEDS_RESV,
+ VMA_COMMIT_RESV,
+ VMA_ABORT_RESV,
+};
static long __vma_reservation_common(struct hstate *h,
struct vm_area_struct *vma, unsigned long addr,
- bool commit)
+ enum vma_resv_mode mode)
{
struct resv_map *resv;
pgoff_t idx;
@@ -1503,10 +1608,20 @@ static long __vma_reservation_common(struct hstate *h,
return 1;
idx = vma_hugecache_offset(h, vma, addr);
- if (commit)
- ret = region_add(resv, idx, idx + 1);
- else
+ switch (mode) {
+ case VMA_NEEDS_RESV:
ret = region_chg(resv, idx, idx + 1);
+ break;
+ case VMA_COMMIT_RESV:
+ ret = region_add(resv, idx, idx + 1);
+ break;
+ case VMA_ABORT_RESV:
+ region_abort(resv, idx, idx + 1);
+ ret = 0;
+ break;
+ default:
+ BUG();
+ }
if (vma->vm_flags & VM_MAYSHARE)
return ret;
@@ -1517,13 +1632,19 @@ static long __vma_reservation_common(struct hstate *h,
static long vma_needs_reservation(struct hstate *h,
struct vm_area_struct *vma, unsigned long addr)
{
- return __vma_reservation_common(h, vma, addr, false);
+ return __vma_reservation_common(h, vma, addr, VMA_NEEDS_RESV);
}
static long vma_commit_reservation(struct hstate *h,
struct vm_area_struct *vma, unsigned long addr)
{
- return __vma_reservation_common(h, vma, addr, true);
+ return __vma_reservation_common(h, vma, addr, VMA_COMMIT_RESV);
+}
+
+static void vma_abort_reservation(struct hstate *h,
+ struct vm_area_struct *vma, unsigned long addr)
+{
+ (void)__vma_reservation_common(h, vma, addr, VMA_ABORT_RESV);
}
static struct page *alloc_huge_page(struct vm_area_struct *vma,
@@ -1549,8 +1670,10 @@ static struct page *alloc_huge_page(struct vm_area_struct *vma,
if (chg < 0)
return ERR_PTR(-ENOMEM);
if (chg || avoid_reserve)
- if (hugepage_subpool_get_pages(spool, 1) < 0)
+ if (hugepage_subpool_get_pages(spool, 1) < 0) {
+ vma_abort_reservation(h, vma, addr);
return ERR_PTR(-ENOSPC);
+ }
ret = hugetlb_cgroup_charge_cgroup(idx, pages_per_huge_page(h), &h_cg);
if (ret)
@@ -1596,6 +1719,7 @@ out_uncharge_cgroup:
out_subpool_put:
if (chg || avoid_reserve)
hugepage_subpool_put_pages(spool, 1);
+ vma_abort_reservation(h, vma, addr);
return ERR_PTR(-ENOSPC);
}
@@ -3236,11 +3360,14 @@ retry:
* any allocations necessary to record that reservation occur outside
* the spinlock.
*/
- if ((flags & FAULT_FLAG_WRITE) && !(vma->vm_flags & VM_SHARED))
+ if ((flags & FAULT_FLAG_WRITE) && !(vma->vm_flags & VM_SHARED)) {
if (vma_needs_reservation(h, vma, address) < 0) {
ret = VM_FAULT_OOM;
goto backout_unlocked;
}
+ /* Just decrements count, does not deallocate */
+ vma_abort_reservation(h, vma, address);
+ }
ptl = huge_pte_lockptr(h, mm, ptep);
spin_lock(ptl);
@@ -3387,6 +3514,8 @@ int hugetlb_fault(struct mm_struct *mm, struct vm_area_struct *vma,
ret = VM_FAULT_OOM;
goto out_mutex;
}
+ /* Just decrements count, does not deallocate */
+ vma_abort_reservation(h, vma, address);
if (!(vma->vm_flags & VM_MAYSHARE))
pagecache_page = hugetlbfs_pagecache_page(h,
@@ -3726,6 +3855,8 @@ int hugetlb_reserve_pages(struct inode *inode,
}
return 0;
out_err:
+ if (!vma || vma->vm_flags & VM_MAYSHARE)
+ region_abort(resv_map, from, to);
if (vma && is_vma_resv_set(vma, HPAGE_RESV_OWNER))
kref_put(&resv_map->refs, resv_map_release);
return ret;
--
2.1.0
^ permalink raw reply related
* [PATCH v3 02/10] mm/hugetlb: add region_del() to delete a specific range of entries
From: Mike Kravetz @ 2015-07-13 4:21 UTC (permalink / raw)
To: linux-mm, linux-kernel, linux-api
Cc: Dave Hansen, Naoya Horiguchi, David Rientjes, Hugh Dickins,
Davidlohr Bueso, Aneesh Kumar, Hillf Danton, Christoph Hellwig,
Andrew Morton, Michal Hocko, Mike Kravetz
In-Reply-To: <1436761268-6397-1-git-send-email-mike.kravetz@oracle.com>
fallocate hole punch will want to remove a specific range of pages.
The existing region_truncate() routine deletes all region/reserve
map entries after a specified offset. region_del() will provide
this same functionality if the end of region is specified as LONG_MAX.
Hence, region_del() can replace region_truncate().
Unlike region_truncate(), region_del() can return an error in the
rare case where it can not allocate memory for a region descriptor.
This ONLY happens in the case where an existing region must be split.
Current callers passing LONG_MAX as end of range will never experience
this error and do not need to deal with error handling. Future
callers of region_del() (such as fallocate hole punch) will need to
handle this error.
Signed-off-by: Mike Kravetz <mike.kravetz@oracle.com>
---
mm/hugetlb.c | 99 ++++++++++++++++++++++++++++++++++++++++++++----------------
1 file changed, 73 insertions(+), 26 deletions(-)
diff --git a/mm/hugetlb.c b/mm/hugetlb.c
index 241d16d..a5c8b3c 100644
--- a/mm/hugetlb.c
+++ b/mm/hugetlb.c
@@ -461,43 +461,90 @@ static void region_abort(struct resv_map *resv, long f, long t)
}
/*
- * Truncate the reserve map at index 'end'. Modify/truncate any
- * region which contains end. Delete any regions past end.
- * Return the number of huge pages removed from the map.
+ * Delete the specified range [f, t) from the reserve map. If the
+ * t parameter is LONG_MAX, this indicates that ALL regions after f
+ * should be deleted. Locate the regions which intersect [f, t)
+ * and either trim, delete or split the existing regions.
+ *
+ * Returns the number of huge pages deleted from the reserve map.
+ * In the normal case, the return value is zero or more. In the
+ * case where a region must be split, a new region descriptor must
+ * be allocated. If the allocation fails, -ENOMEM will be returned.
+ * NOTE: If the parameter t == LONG_MAX, then we will never split
+ * a region and possibly return -ENOMEM. Callers specifying
+ * t == LONG_MAX do not need to check for -ENOMEM error.
*/
-static long region_truncate(struct resv_map *resv, long end)
+static long region_del(struct resv_map *resv, long f, long t)
{
struct list_head *head = &resv->regions;
struct file_region *rg, *trg;
- long chg = 0;
+ struct file_region *nrg = NULL;
+ long del = 0;
+retry:
spin_lock(&resv->lock);
- /* Locate the region we are either in or before. */
- list_for_each_entry(rg, head, link)
- if (end <= rg->to)
+ list_for_each_entry_safe(rg, trg, head, link) {
+ if (rg->to <= f)
+ continue;
+ if (rg->from >= t)
break;
- if (&rg->link == head)
- goto out;
- /* If we are in the middle of a region then adjust it. */
- if (end > rg->from) {
- chg = rg->to - end;
- rg->to = end;
- rg = list_entry(rg->link.next, typeof(*rg), link);
- }
+ if (f > rg->from && t < rg->to) { /* Must split region */
+ /*
+ * Check for an entry in the cache before dropping
+ * lock and attempting allocation.
+ */
+ if (!nrg &&
+ resv->rgn_cache_count > resv->adds_in_progress) {
+ nrg = list_first_entry(&resv->rgn_cache,
+ struct file_region,
+ link);
+ list_del(&nrg->link);
+ resv->rgn_cache_count--;
+ }
- /* Drop any remaining regions. */
- list_for_each_entry_safe(rg, trg, rg->link.prev, link) {
- if (&rg->link == head)
+ if (!nrg) {
+ spin_unlock(&resv->lock);
+ nrg = kmalloc(sizeof(*nrg), GFP_KERNEL);
+ if (!nrg)
+ return -ENOMEM;
+ goto retry;
+ }
+
+ del += t - f;
+
+ /* New entry for end of split region */
+ nrg->from = t;
+ nrg->to = rg->to;
+ INIT_LIST_HEAD(&nrg->link);
+
+ /* Original entry is trimmed */
+ rg->to = f;
+
+ list_add(&nrg->link, &rg->link);
+ nrg = NULL;
break;
- chg += rg->to - rg->from;
- list_del(&rg->link);
- kfree(rg);
+ }
+
+ if (f <= rg->from && t >= rg->to) { /* Remove entire region */
+ del += rg->to - rg->from;
+ list_del(&rg->link);
+ kfree(rg);
+ continue;
+ }
+
+ if (f <= rg->from) { /* Trim beginning of region */
+ del += t - rg->from;
+ rg->from = t;
+ } else { /* Trim end of region */
+ del += rg->to - f;
+ rg->to = f;
+ }
}
-out:
spin_unlock(&resv->lock);
- return chg;
+ kfree(nrg);
+ return del;
}
/*
@@ -648,7 +695,7 @@ void resv_map_release(struct kref *ref)
struct file_region *rg, *trg;
/* Clear out any active regions before we release the map. */
- region_truncate(resv_map, 0);
+ region_del(resv_map, 0, LONG_MAX);
/* ... and any entries left in the cache */
list_for_each_entry_safe(rg, trg, head, link) {
@@ -3871,7 +3918,7 @@ void hugetlb_unreserve_pages(struct inode *inode, long offset, long freed)
long gbl_reserve;
if (resv_map)
- chg = region_truncate(resv_map, offset);
+ chg = region_del(resv_map, offset, LONG_MAX);
spin_lock(&inode->i_lock);
inode->i_blocks -= (blocks_per_huge_page(h) * freed);
spin_unlock(&inode->i_lock);
--
2.1.0
--
To unsubscribe, send a message with 'unsubscribe linux-mm' in
the body to majordomo@kvack.org. For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"dont@kvack.org"> email@kvack.org </a>
^ permalink raw reply related
* [PATCH v3 03/10] mm/hugetlb: expose hugetlb fault mutex for use by fallocate
From: Mike Kravetz @ 2015-07-13 4:21 UTC (permalink / raw)
To: linux-mm, linux-kernel, linux-api
Cc: Dave Hansen, Naoya Horiguchi, David Rientjes, Hugh Dickins,
Davidlohr Bueso, Aneesh Kumar, Hillf Danton, Christoph Hellwig,
Andrew Morton, Michal Hocko, Mike Kravetz
In-Reply-To: <1436761268-6397-1-git-send-email-mike.kravetz@oracle.com>
hugetlb page faults are currently synchronized by the table of
mutexes (htlb_fault_mutex_table). fallocate code will need to
synchronize with the page fault code when it allocates or
deletes pages. Expose interfaces so that fallocate operations
can be synchronized with page faults. Minor name changes to
be more consistent with other global hugetlb symbols.
Signed-off-by: Mike Kravetz <mike.kravetz@oracle.com>
---
include/linux/hugetlb.h | 5 +++++
mm/hugetlb.c | 20 ++++++++++----------
2 files changed, 15 insertions(+), 10 deletions(-)
diff --git a/include/linux/hugetlb.h b/include/linux/hugetlb.h
index 667cf44..933da39 100644
--- a/include/linux/hugetlb.h
+++ b/include/linux/hugetlb.h
@@ -88,6 +88,11 @@ int dequeue_hwpoisoned_huge_page(struct page *page);
bool isolate_huge_page(struct page *page, struct list_head *list);
void putback_active_hugepage(struct page *page);
void free_huge_page(struct page *page);
+extern struct mutex *hugetlb_fault_mutex_table;
+u32 hugetlb_fault_mutex_hash(struct hstate *h, struct mm_struct *mm,
+ struct vm_area_struct *vma,
+ struct address_space *mapping,
+ pgoff_t idx, unsigned long address);
#ifdef CONFIG_ARCH_WANT_HUGE_PMD_SHARE
pte_t *huge_pmd_share(struct mm_struct *mm, unsigned long addr, pud_t *pud);
diff --git a/mm/hugetlb.c b/mm/hugetlb.c
index a5c8b3c..52c2801 100644
--- a/mm/hugetlb.c
+++ b/mm/hugetlb.c
@@ -64,7 +64,7 @@ DEFINE_SPINLOCK(hugetlb_lock);
* prevent spurious OOMs when the hugepage pool is fully utilized.
*/
static int num_fault_mutexes;
-static struct mutex *htlb_fault_mutex_table ____cacheline_aligned_in_smp;
+struct mutex *hugetlb_fault_mutex_table ____cacheline_aligned_in_smp;
/* Forward declaration */
static int hugetlb_acct_memory(struct hstate *h, long delta);
@@ -2482,7 +2482,7 @@ static void __exit hugetlb_exit(void)
}
kobject_put(hugepages_kobj);
- kfree(htlb_fault_mutex_table);
+ kfree(hugetlb_fault_mutex_table);
}
module_exit(hugetlb_exit);
@@ -2515,12 +2515,12 @@ static int __init hugetlb_init(void)
#else
num_fault_mutexes = 1;
#endif
- htlb_fault_mutex_table =
+ hugetlb_fault_mutex_table =
kmalloc(sizeof(struct mutex) * num_fault_mutexes, GFP_KERNEL);
- BUG_ON(!htlb_fault_mutex_table);
+ BUG_ON(!hugetlb_fault_mutex_table);
for (i = 0; i < num_fault_mutexes; i++)
- mutex_init(&htlb_fault_mutex_table[i]);
+ mutex_init(&hugetlb_fault_mutex_table[i]);
return 0;
}
module_init(hugetlb_init);
@@ -3454,7 +3454,7 @@ backout_unlocked:
}
#ifdef CONFIG_SMP
-static u32 fault_mutex_hash(struct hstate *h, struct mm_struct *mm,
+u32 hugetlb_fault_mutex_hash(struct hstate *h, struct mm_struct *mm,
struct vm_area_struct *vma,
struct address_space *mapping,
pgoff_t idx, unsigned long address)
@@ -3479,7 +3479,7 @@ static u32 fault_mutex_hash(struct hstate *h, struct mm_struct *mm,
* For uniprocesor systems we always use a single mutex, so just
* return 0 and avoid the hashing overhead.
*/
-static u32 fault_mutex_hash(struct hstate *h, struct mm_struct *mm,
+u32 hugetlb_fault_mutex_hash(struct hstate *h, struct mm_struct *mm,
struct vm_area_struct *vma,
struct address_space *mapping,
pgoff_t idx, unsigned long address)
@@ -3527,8 +3527,8 @@ int hugetlb_fault(struct mm_struct *mm, struct vm_area_struct *vma,
* get spurious allocation failures if two CPUs race to instantiate
* the same page in the page cache.
*/
- hash = fault_mutex_hash(h, mm, vma, mapping, idx, address);
- mutex_lock(&htlb_fault_mutex_table[hash]);
+ hash = hugetlb_fault_mutex_hash(h, mm, vma, mapping, idx, address);
+ mutex_lock(&hugetlb_fault_mutex_table[hash]);
entry = huge_ptep_get(ptep);
if (huge_pte_none(entry)) {
@@ -3613,7 +3613,7 @@ out_ptl:
put_page(pagecache_page);
}
out_mutex:
- mutex_unlock(&htlb_fault_mutex_table[hash]);
+ mutex_unlock(&hugetlb_fault_mutex_table[hash]);
/*
* Generally it's safe to hold refcount during waiting page lock. But
* here we just wait to defer the next page fault to avoid busy loop and
--
2.1.0
--
To unsubscribe, send a message with 'unsubscribe linux-mm' in
the body to majordomo@kvack.org. For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"dont@kvack.org"> email@kvack.org </a>
^ permalink raw reply related
* [PATCH v3 04/10] hugetlbfs: hugetlb_vmtruncate_list() needs to take a range to delete
From: Mike Kravetz @ 2015-07-13 4:21 UTC (permalink / raw)
To: linux-mm, linux-kernel, linux-api
Cc: Dave Hansen, Naoya Horiguchi, David Rientjes, Hugh Dickins,
Davidlohr Bueso, Aneesh Kumar, Hillf Danton, Christoph Hellwig,
Andrew Morton, Michal Hocko, Mike Kravetz
In-Reply-To: <1436761268-6397-1-git-send-email-mike.kravetz@oracle.com>
fallocate hole punch will want to unmap a specific range of pages.
Modify the existing hugetlb_vmtruncate_list() routine to take a
start/end range. If end is 0, this indicates all pages after start
should be unmapped. This is the same as the existing truncate
functionality. Modify existing callers to add 0 as end of range.
Since the routine will be used in hole punch as well as truncate
operations, it is more appropriately renamed to hugetlb_vmdelete_list().
Signed-off-by: Mike Kravetz <mike.kravetz@oracle.com>
---
fs/hugetlbfs/inode.c | 25 ++++++++++++++++++-------
1 file changed, 18 insertions(+), 7 deletions(-)
diff --git a/fs/hugetlbfs/inode.c b/fs/hugetlbfs/inode.c
index 0cf74df..ed40f56 100644
--- a/fs/hugetlbfs/inode.c
+++ b/fs/hugetlbfs/inode.c
@@ -349,11 +349,15 @@ static void hugetlbfs_evict_inode(struct inode *inode)
}
static inline void
-hugetlb_vmtruncate_list(struct rb_root *root, pgoff_t pgoff)
+hugetlb_vmdelete_list(struct rb_root *root, pgoff_t start, pgoff_t end)
{
struct vm_area_struct *vma;
- vma_interval_tree_foreach(vma, root, pgoff, ULONG_MAX) {
+ /*
+ * end == 0 indicates that the entire range after
+ * start should be unmapped.
+ */
+ vma_interval_tree_foreach(vma, root, start, end ? end : ULONG_MAX) {
unsigned long v_offset;
/*
@@ -362,13 +366,20 @@ hugetlb_vmtruncate_list(struct rb_root *root, pgoff_t pgoff)
* which overlap the truncated area starting at pgoff,
* and no vma on a 32-bit arch can span beyond the 4GB.
*/
- if (vma->vm_pgoff < pgoff)
- v_offset = (pgoff - vma->vm_pgoff) << PAGE_SHIFT;
+ if (vma->vm_pgoff < start)
+ v_offset = (start - vma->vm_pgoff) << PAGE_SHIFT;
else
v_offset = 0;
- unmap_hugepage_range(vma, vma->vm_start + v_offset,
- vma->vm_end, NULL);
+ if (end) {
+ end = ((end - start) << PAGE_SHIFT) +
+ vma->vm_start + v_offset;
+ if (end > vma->vm_end)
+ end = vma->vm_end;
+ } else
+ end = vma->vm_end;
+
+ unmap_hugepage_range(vma, vma->vm_start + v_offset, end, NULL);
}
}
@@ -384,7 +395,7 @@ static int hugetlb_vmtruncate(struct inode *inode, loff_t offset)
i_size_write(inode, offset);
i_mmap_lock_write(mapping);
if (!RB_EMPTY_ROOT(&mapping->i_mmap))
- hugetlb_vmtruncate_list(&mapping->i_mmap, pgoff);
+ hugetlb_vmdelete_list(&mapping->i_mmap, pgoff, 0);
i_mmap_unlock_write(mapping);
truncate_hugepages(inode, offset);
return 0;
--
2.1.0
--
To unsubscribe, send a message with 'unsubscribe linux-mm' in
the body to majordomo@kvack.org. For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"dont@kvack.org"> email@kvack.org </a>
^ permalink raw reply related
page: next (older) | prev (newer) | latest
- recent:[subjects (threaded)|topics (new)|topics (active)]
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox