* [PATCH] virtio_balloon: fix PFN format for virtio-1
From: Michael S. Tsirkin @ 2016-05-18 12:38 UTC (permalink / raw)
To: linux-kernel; +Cc: virtualization
Everything should be LE when using virtio-1, but
the linux balloon driver does not seem to care about that.
Cc: stable@vger.kernel.org
Reported-by: Cornelia Huck <cornelia.huck@de.ibm.com>
Signed-off-by: Michael S. Tsirkin <mst@redhat.com>
---
drivers/virtio/virtio_balloon.c | 20 ++++++++++++--------
1 file changed, 12 insertions(+), 8 deletions(-)
diff --git a/drivers/virtio/virtio_balloon.c b/drivers/virtio/virtio_balloon.c
index 7b6d74f..476c0e3 100644
--- a/drivers/virtio/virtio_balloon.c
+++ b/drivers/virtio/virtio_balloon.c
@@ -75,7 +75,7 @@ struct virtio_balloon {
/* The array of pfns we tell the Host about. */
unsigned int num_pfns;
- u32 pfns[VIRTIO_BALLOON_ARRAY_PFNS_MAX];
+ __virtio32 pfns[VIRTIO_BALLOON_ARRAY_PFNS_MAX];
/* Memory statistics */
struct virtio_balloon_stat stats[VIRTIO_BALLOON_S_NR];
@@ -127,14 +127,16 @@ static void tell_host(struct virtio_balloon *vb, struct virtqueue *vq)
}
-static void set_page_pfns(u32 pfns[], struct page *page)
+static void set_page_pfns(struct virtio_balloon *vb,
+ __virtio32 pfns[], struct page *page)
{
unsigned int i;
/* Set balloon pfns pointing at this page.
* Note that the first pfn points at start of the page. */
for (i = 0; i < VIRTIO_BALLOON_PAGES_PER_PAGE; i++)
- pfns[i] = page_to_balloon_pfn(page) + i;
+ pfns[i] = cpu_to_virtio32(vb->vdev,
+ page_to_balloon_pfn(page) + i);
}
static unsigned fill_balloon(struct virtio_balloon *vb, size_t num)
@@ -158,7 +160,7 @@ static unsigned fill_balloon(struct virtio_balloon *vb, size_t num)
msleep(200);
break;
}
- set_page_pfns(vb->pfns + vb->num_pfns, page);
+ set_page_pfns(vb, vb->pfns + vb->num_pfns, page);
vb->num_pages += VIRTIO_BALLOON_PAGES_PER_PAGE;
if (!virtio_has_feature(vb->vdev,
VIRTIO_BALLOON_F_DEFLATE_ON_OOM))
@@ -177,10 +179,12 @@ static unsigned fill_balloon(struct virtio_balloon *vb, size_t num)
static void release_pages_balloon(struct virtio_balloon *vb)
{
unsigned int i;
+ struct page *page;
/* Find pfns pointing at start of each page, get pages and free them. */
for (i = 0; i < vb->num_pfns; i += VIRTIO_BALLOON_PAGES_PER_PAGE) {
- struct page *page = balloon_pfn_to_page(vb->pfns[i]);
+ page = balloon_pfn_to_page(virtio32_to_cpu(vb->vdev,
+ vb->pfns[i]));
if (!virtio_has_feature(vb->vdev,
VIRTIO_BALLOON_F_DEFLATE_ON_OOM))
adjust_managed_page_count(page, 1);
@@ -203,7 +207,7 @@ static unsigned leak_balloon(struct virtio_balloon *vb, size_t num)
page = balloon_page_dequeue(vb_dev_info);
if (!page)
break;
- set_page_pfns(vb->pfns + vb->num_pfns, page);
+ set_page_pfns(vb, vb->pfns + vb->num_pfns, page);
vb->num_pages -= VIRTIO_BALLOON_PAGES_PER_PAGE;
}
@@ -471,13 +475,13 @@ static int virtballoon_migratepage(struct balloon_dev_info *vb_dev_info,
__count_vm_event(BALLOON_MIGRATE);
spin_unlock_irqrestore(&vb_dev_info->pages_lock, flags);
vb->num_pfns = VIRTIO_BALLOON_PAGES_PER_PAGE;
- set_page_pfns(vb->pfns, newpage);
+ set_page_pfns(vb, vb->pfns, newpage);
tell_host(vb, vb->inflate_vq);
/* balloon's page migration 2nd step -- deflate "page" */
balloon_page_delete(page);
vb->num_pfns = VIRTIO_BALLOON_PAGES_PER_PAGE;
- set_page_pfns(vb->pfns, page);
+ set_page_pfns(vb, vb->pfns, page);
tell_host(vb, vb->deflate_vq);
mutex_unlock(&vb->balloon_lock);
--
MST
^ permalink raw reply related
* Re: [PATCH RFC 1/1] virtio-balloon: handle virtio-1 endianness
From: Michael S. Tsirkin @ 2016-05-18 12:33 UTC (permalink / raw)
To: Cornelia Huck; +Cc: borntraeger, linux-kernel, virtualization
In-Reply-To: <1463573563-34081-2-git-send-email-cornelia.huck@de.ibm.com>
On Wed, May 18, 2016 at 02:12:43PM +0200, Cornelia Huck wrote:
> As virtio-1 devices use little endian on their queues, we need
> to make sure the pfns are in virtio32 format before we put
> them on the queue.
>
> Signed-off-by: Cornelia Huck <cornelia.huck@de.ibm.com>
I have a less hacky version that I was testing.
Will post now.
> ---
> drivers/virtio/virtio_balloon.c | 6 +++++-
> 1 file changed, 5 insertions(+), 1 deletion(-)
>
> diff --git a/drivers/virtio/virtio_balloon.c b/drivers/virtio/virtio_balloon.c
> index 7b6d74f..89da06c 100644
> --- a/drivers/virtio/virtio_balloon.c
> +++ b/drivers/virtio/virtio_balloon.c
> @@ -76,6 +76,7 @@ struct virtio_balloon {
> /* The array of pfns we tell the Host about. */
> unsigned int num_pfns;
> u32 pfns[VIRTIO_BALLOON_ARRAY_PFNS_MAX];
> + __virtio32 vpfns[VIRTIO_BALLOON_ARRAY_PFNS_MAX];
>
> /* Memory statistics */
> struct virtio_balloon_stat stats[VIRTIO_BALLOON_S_NR];
> @@ -115,8 +116,11 @@ static void tell_host(struct virtio_balloon *vb, struct virtqueue *vq)
> {
> struct scatterlist sg;
> unsigned int len;
> + int i;
>
> - sg_init_one(&sg, vb->pfns, sizeof(vb->pfns[0]) * vb->num_pfns);
> + for (i = 0; i < vb->num_pfns; i++)
> + vb->vpfns[i] = cpu_to_virtio32(vb->vdev, vb->pfns[i]);
> + sg_init_one(&sg, vb->vpfns, sizeof(vb->vpfns[0]) * vb->num_pfns);
>
> /* We should always be able to add one buffer to an empty queue. */
> virtqueue_add_outbuf(vq, &sg, 1, vb, GFP_KERNEL);
> --
> 2.6.6
^ permalink raw reply
* [PATCH RFC 1/1] virtio-balloon: handle virtio-1 endianness
From: Cornelia Huck @ 2016-05-18 12:12 UTC (permalink / raw)
To: mst; +Cc: borntraeger, linux-kernel, virtualization
In-Reply-To: <1463573563-34081-1-git-send-email-cornelia.huck@de.ibm.com>
As virtio-1 devices use little endian on their queues, we need
to make sure the pfns are in virtio32 format before we put
them on the queue.
Signed-off-by: Cornelia Huck <cornelia.huck@de.ibm.com>
---
drivers/virtio/virtio_balloon.c | 6 +++++-
1 file changed, 5 insertions(+), 1 deletion(-)
diff --git a/drivers/virtio/virtio_balloon.c b/drivers/virtio/virtio_balloon.c
index 7b6d74f..89da06c 100644
--- a/drivers/virtio/virtio_balloon.c
+++ b/drivers/virtio/virtio_balloon.c
@@ -76,6 +76,7 @@ struct virtio_balloon {
/* The array of pfns we tell the Host about. */
unsigned int num_pfns;
u32 pfns[VIRTIO_BALLOON_ARRAY_PFNS_MAX];
+ __virtio32 vpfns[VIRTIO_BALLOON_ARRAY_PFNS_MAX];
/* Memory statistics */
struct virtio_balloon_stat stats[VIRTIO_BALLOON_S_NR];
@@ -115,8 +116,11 @@ static void tell_host(struct virtio_balloon *vb, struct virtqueue *vq)
{
struct scatterlist sg;
unsigned int len;
+ int i;
- sg_init_one(&sg, vb->pfns, sizeof(vb->pfns[0]) * vb->num_pfns);
+ for (i = 0; i < vb->num_pfns; i++)
+ vb->vpfns[i] = cpu_to_virtio32(vb->vdev, vb->pfns[i]);
+ sg_init_one(&sg, vb->vpfns, sizeof(vb->vpfns[0]) * vb->num_pfns);
/* We should always be able to add one buffer to an empty queue. */
virtqueue_add_outbuf(vq, &sg, 1, vb, GFP_KERNEL);
--
2.6.6
^ permalink raw reply related
* [PATCH RFC 0/1] virtio-balloon vs. endianness
From: Cornelia Huck @ 2016-05-18 12:12 UTC (permalink / raw)
To: mst; +Cc: borntraeger, linux-kernel, virtualization
Hi Michael,
this patch (against your vhost branch) should fix the endianness issues
we saw on s390 that I mentioned on irc yesterday.
Both the config space and the stats seem to be fine endianness-wise,
but the pfns for inflate/deflate were not converted to little endian
for virtio-1 (the qemu code is correct).
Without the patch, I get an immediate crash on qemu master when
started via libvirt with currentMemory=0.5*Memory (thanks to
Christian for the hint) - as virtio-ccw is enabling virtio-1 by
default:
[ 3.273419] ------------[ cut here ]------------
[ 3.273424] Kernel BUG at 0000000000300df2 [verbose debug info unavailable]
[ 3.273617] illegal operation: 0001 ilc:1 [#1] PREEMPT SMP DEBUG_PAGEALLOC
[ 3.273623] Modules linked in: autofs4
[ 3.273627] CPU: 2 PID: 1 Comm: systemd Not tainted 4.5.0-00798-g1181f16 #6
[ 3.273629] task: 000000003def8000 ti: 000000003deec000 task.ti: 000000003deec000
[ 3.273631] Krnl PSW : 0704d00180000000 0000000000300df2 (do_iter_readv_writev+0x2/0x90)
[ 3.273640] R:0 T:1 IO:1 EX:1 Key:0 M:1 W:0 P:0 AS:3 CC:1 PM:0 EA:3
Krnl GPRS: 000000003deefe30 0000000000008000 000000003bba8c00 000000003deefdb0
[ 3.273644] 000000003deeff08 00000000001b3e60 000000003deefda8 000003ffc18fde60
[ 3.273646] 0000000000000248 0000000000000000 000000003deeff08 0000000000000001
[ 3.273647] 000000003bba8c00 00000000001b3e60 0000000000301f08 000000003deefd00
[ 3.273655] Krnl Code:>0000000000300df2: 000 unknown
0000000000300df4: 0000 unknown
0000000000300df6: 0000 unknown
0000000000300df8: 0000 unknown
0000000000300dfa: 0000 unknown
0000000000300dfc: 0000 unknown
0000000000300dfe: 0000 unknown
0000000000300e00: 0000 unknown
[ 3.273665] Call Trace:
[ 3.273667] ([<0000000000301e86>] do_readv_writev+0x86/0x260)
[ 3.273669] [<0000000000302132>] vfs_writev+0x5a/0x78
[ 3.273671] [<000000000030305e>] SyS_writev+0x66/0xe8
[ 3.273677] [<000000000076955e>] system_call+0xd6/0x270
[ 3.273679] [<000003ff9d2f82c4>] 0x3ff9d2f82c4
[ 3.273680] INFO: lockdep is turned off.
[ 3.273681] Last Breaking-Event-Address:
[ 3.273683] [<0000000000769a60>] io_int_handler+0x17c/0x298
[ 3.273686]
[ 3.273688] Kernel panic - not syncing: Fatal exception: panic_on_oops
The crash is gone by either forcing the device to legacy (max_revision=0)
or by applying the patch below in the guest.
[There also have been reports of people getting immediate "Out of puff!"
messages, but I don't know how to reproduce that.]
Problems should presumably also arise with virtio-pci on big endian
platforms, but given that it took us some time to hit this in tests
with the always-modern virtio-ccw environment, I'm not surprised if
nobody hit that yet.
[As an aside: Should the virtio spec be a bit more clear on how the
queues for the balloon operate, or do we want to avoid spending more
time on the legacy balloon?]
The fix (this patch or something different) needs to be cc:stable,
I guess.
Cornelia Huck (1):
virtio-balloon: handle virtio-1 endianness
drivers/virtio/virtio_balloon.c | 6 +++++-
1 file changed, 5 insertions(+), 1 deletion(-)
--
2.6.6
^ permalink raw reply
* Re: [PATCH v5 0/6] Support calling functions on dedicated physical cpu
From: Juergen Gross @ 2016-05-17 11:21 UTC (permalink / raw)
To: Juergen Gross, linux-kernel, xen-devel
Cc: jeremy, jdelvare, peterz, hpa, akataria, x86, virtualization,
chrisw, mingo, david.vrabel, Douglas_Warzecha, pali.rohar,
boris.ostrovsky, tglx, linux
In-Reply-To: <570E07FD.4030403@suse.com>
On 13/04/16 10:49, Juergen Gross wrote:
> On 06/04/16 16:17, Juergen Gross wrote:
>> Some hardware (e.g. Dell Studio laptops) require special functions to
>> be called on physical cpu 0 in order to avoid occasional hangs. When
>> running as dom0 under Xen this could be achieved only via special boot
>> parameters (vcpu pinning) limiting the hypervisor in it's scheduling
>> decisions.
>>
>> This patch series is adding a generic function to be able to temporarily
>> pin a (virtual) cpu to a dedicated physical cpu for executing above
>> mentioned functions on that specific cpu. The drivers (dcdbas and i8k)
>> requiring this functionality are modified accordingly.
>>
>> Changes in V5:
>> - patch 3: rename and reshuffle parameters of smp_call_on_cpu() as requested
>> by Peter Zijlstra
>> - patch 3: test target cpu to be online as requested by Peter Zijlstra
>> - patch 4: less wordy messages as requested by David Vrabel
>>
>> Changes in V4:
>> - move patches 5 and 6 further up in the series
>> - patch 2 (was 5): WARN_ONCE in case platform doesn't support pinning
>> as requested by Peter Zijlstra
>> - patch 3 (was 2): change return value in case of illegal cpu as
>> requested by Peter Zijlstra
>> - patch 3 (was 2): make pinning of vcpu an option as suggested by
>> Peter Zijlstra
>> - patches 5 and 6 (were 3 and 4): add call to get_online_cpus()
>>
>> Changes in V3:
>> - use get_cpu()/put_cpu() as suggested by David Vrabel
>>
>> Changes in V2:
>> - instead of manipulating the allowed set of cpus use cpu specific
>> workqueue as requested by Peter Zijlstra
>> - add include/linux/hypervisor.h to hide architecture specific stuff
>> from generic kernel code
>>
>> Juergen Gross (6):
>> xen: sync xen header
>> virt, sched: add generic vcpu pinning support
>> smp: add function to execute a function synchronously on a cpu
>> xen: add xen_pin_vcpu() to support calling functions on a dedicated
>> pcpu
>> dcdbas: make use of smp_call_on_cpu()
>> hwmon: use smp_call_on_cpu() for dell-smm i8k
>>
>> MAINTAINERS | 1 +
>> arch/x86/include/asm/hypervisor.h | 4 ++
>> arch/x86/kernel/cpu/hypervisor.c | 11 +++++
>> arch/x86/xen/enlighten.c | 40 +++++++++++++++
>> drivers/firmware/dcdbas.c | 51 +++++++++----------
>> drivers/hwmon/dell-smm-hwmon.c | 35 +++++++------
>> include/linux/hypervisor.h | 17 +++++++
>> include/linux/smp.h | 3 ++
>> include/xen/interface/sched.h | 100 +++++++++++++++++++++++++++++++-------
>> kernel/smp.c | 51 +++++++++++++++++++
>> kernel/up.c | 18 +++++++
>> 11 files changed, 273 insertions(+), 58 deletions(-)
>> create mode 100644 include/linux/hypervisor.h
>>
>
> So patches 1, 3 and 4 are acked. Any comments regarding patches 2, 5
> and 6?
Patch 6 gained another Ack. What about 2 and 5?
Juergen
^ permalink raw reply
* [PATCH v2 6/6] powerpc: pseries: pv-qspinlock build config/make
From: Pan Xinhui @ 2016-05-17 7:49 UTC (permalink / raw)
To: linux-kernel, virtualization, linuxppc-dev
Cc: mnipxh, Pan Xinhui, peterz, benh, boqun.feng, waiman.long, mingo,
paulus, mpe, paulmck
In-Reply-To: <1463471387-5115-1-git-send-email-xinhui.pan@linux.vnet.ibm.com>
pseries can use pv-qspinlock.
Signed-off-by: Pan Xinhui <xinhui.pan@linux.vnet.ibm.com>
---
arch/powerpc/kernel/Makefile | 1 +
arch/powerpc/platforms/pseries/Kconfig | 8 ++++++++
2 files changed, 9 insertions(+)
diff --git a/arch/powerpc/kernel/Makefile b/arch/powerpc/kernel/Makefile
index 2da380f..ae7c2f1 100644
--- a/arch/powerpc/kernel/Makefile
+++ b/arch/powerpc/kernel/Makefile
@@ -50,6 +50,7 @@ obj-$(CONFIG_PPC_970_NAP) += idle_power4.o
obj-$(CONFIG_PPC_P7_NAP) += idle_power7.o
procfs-y := proc_powerpc.o
obj-$(CONFIG_PROC_FS) += $(procfs-y)
+obj-$(CONFIG_PARAVIRT_SPINLOCKS) += paravirt.o
rtaspci-$(CONFIG_PPC64)-$(CONFIG_PCI) := rtas_pci.o
obj-$(CONFIG_PPC_RTAS) += rtas.o rtas-rtc.o $(rtaspci-y-y)
obj-$(CONFIG_PPC_RTAS_DAEMON) += rtasd.o
diff --git a/arch/powerpc/platforms/pseries/Kconfig b/arch/powerpc/platforms/pseries/Kconfig
index f669323..46632e4 100644
--- a/arch/powerpc/platforms/pseries/Kconfig
+++ b/arch/powerpc/platforms/pseries/Kconfig
@@ -128,3 +128,11 @@ config HV_PERF_CTRS
systems. 24x7 is available on Power 8 systems.
If unsure, select Y.
+
+config PARAVIRT_SPINLOCKS
+ bool "Paravirtialization support for qspinlock"
+ depends on PPC_SPLPAR && QUEUED_SPINLOCKS
+ default y
+ help
+ If platform supports virtualization, for example PowerVM, this option
+ can let guest have a better performace.
--
1.9.1
^ permalink raw reply related
* [PATCH v2 5/6] pv-qspinlock: use cmpxchg_release in __pv_queued_spin_unlock
From: Pan Xinhui @ 2016-05-17 7:49 UTC (permalink / raw)
To: linux-kernel, virtualization, linuxppc-dev
Cc: mnipxh, Pan Xinhui, peterz, benh, boqun.feng, waiman.long, mingo,
paulus, mpe, paulmck
In-Reply-To: <1463471387-5115-1-git-send-email-xinhui.pan@linux.vnet.ibm.com>
cmpxchg_release is lighter, we can gain a better performace then.
Suggested-by: Boqun Feng <boqun.feng@gmail.com>
Signed-off-by: Pan Xinhui <xinhui.pan@linux.vnet.ibm.com>
---
kernel/locking/qspinlock_paravirt.h | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/kernel/locking/qspinlock_paravirt.h b/kernel/locking/qspinlock_paravirt.h
index a5b1248..2bbffe4 100644
--- a/kernel/locking/qspinlock_paravirt.h
+++ b/kernel/locking/qspinlock_paravirt.h
@@ -614,7 +614,7 @@ __visible void __pv_queued_spin_unlock(struct qspinlock *lock)
* unhash. Otherwise it would be possible to have multiple @lock
* entries, which would be BAD.
*/
- locked = cmpxchg(&l->locked, _Q_LOCKED_VAL, 0);
+ locked = cmpxchg_release(&l->locked, _Q_LOCKED_VAL, 0);
if (likely(locked == _Q_LOCKED_VAL))
return;
--
1.9.1
^ permalink raw reply related
* [PATCH v2 4/6] pv-qspinlock: powerpc support pv-qspinlock
From: Pan Xinhui @ 2016-05-17 7:49 UTC (permalink / raw)
To: linux-kernel, virtualization, linuxppc-dev
Cc: mnipxh, Pan Xinhui, peterz, benh, boqun.feng, waiman.long, mingo,
paulus, mpe, paulmck
In-Reply-To: <1463471387-5115-1-git-send-email-xinhui.pan@linux.vnet.ibm.com>
As we need let pv-qspinlock-kernel run on all environment which might
have no powervm, we should runtime choose which qspinlock version to
use. The default pv-qspinlock use native version. pv_lock initialization
should be done in bootstage with irq disabled. And if possible, restore
pv_lock_ops callbacks to pv version.
Signed-off-by: Pan Xinhui <xinhui.pan@linux.vnet.ibm.com>
---
arch/powerpc/include/asm/qspinlock.h | 17 +++++++++
arch/powerpc/include/asm/qspinlock_paravirt.h | 38 +++++++++++++++++++
.../powerpc/include/asm/qspinlock_paravirt_types.h | 13 +++++++
arch/powerpc/kernel/paravirt.c | 44 ++++++++++++++++++++++
arch/powerpc/platforms/pseries/setup.c | 5 +++
5 files changed, 117 insertions(+)
create mode 100644 arch/powerpc/include/asm/qspinlock_paravirt.h
create mode 100644 arch/powerpc/include/asm/qspinlock_paravirt_types.h
create mode 100644 arch/powerpc/kernel/paravirt.c
diff --git a/arch/powerpc/include/asm/qspinlock.h b/arch/powerpc/include/asm/qspinlock.h
index 5883954..4728f6e 100644
--- a/arch/powerpc/include/asm/qspinlock.h
+++ b/arch/powerpc/include/asm/qspinlock.h
@@ -12,10 +12,27 @@ static inline void native_queued_spin_unlock(struct qspinlock *lock)
smp_store_release((u8 *)lock, 0);
}
+#ifdef CONFIG_PARAVIRT_SPINLOCKS
+
+#define __ARCH_NEED_PV_HASH_LOOKUP
+
+#include <asm/qspinlock_paravirt.h>
+
+static inline void queued_spin_lock_slowpath(struct qspinlock *lock, u32 val)
+{
+ pv_queued_spin_lock(lock, val);
+}
+
+static inline void queued_spin_unlock(struct qspinlock *lock)
+{
+ pv_queued_spin_unlock(lock);
+}
+#else
static inline void queued_spin_unlock(struct qspinlock *lock)
{
native_queued_spin_unlock(lock);
}
+#endif
#include <asm-generic/qspinlock.h>
diff --git a/arch/powerpc/include/asm/qspinlock_paravirt.h b/arch/powerpc/include/asm/qspinlock_paravirt.h
new file mode 100644
index 0000000..cd17a79
--- /dev/null
+++ b/arch/powerpc/include/asm/qspinlock_paravirt.h
@@ -0,0 +1,38 @@
+#ifndef CONFIG_PARAVIRT_SPINLOCKS
+#error "do not include this file"
+#endif
+
+#ifndef _ASM_QSPINLOCK_PARAVIRT_H
+#define _ASM_QSPINLOCK_PARAVIRT_H
+
+#include <asm/qspinlock_paravirt_types.h>
+
+extern void pv_lock_init(void);
+extern void native_queued_spin_lock_slowpath(struct qspinlock *lock, u32 val);
+extern void __pv_init_lock_hash(void);
+extern void __pv_queued_spin_lock_slowpath(struct qspinlock *lock, u32 val);
+extern void __pv_queued_spin_unlock(struct qspinlock *lock);
+
+static inline void pv_queued_spin_lock(struct qspinlock *lock, u32 val)
+{
+ CLEAR_IO_SYNC;
+ pv_lock_op.lock(lock, val);
+}
+
+static inline void pv_queued_spin_unlock(struct qspinlock *lock)
+{
+ SYNC_IO;
+ pv_lock_op.unlock(lock);
+}
+
+static inline void pv_wait(u8 *ptr, u8 val, int lockcpu)
+{
+ pv_lock_op.wait(ptr, val, lockcpu);
+}
+
+static inline void pv_kick(int cpu)
+{
+ pv_lock_op.kick(cpu);
+}
+
+#endif
diff --git a/arch/powerpc/include/asm/qspinlock_paravirt_types.h b/arch/powerpc/include/asm/qspinlock_paravirt_types.h
new file mode 100644
index 0000000..e1fdeb0
--- /dev/null
+++ b/arch/powerpc/include/asm/qspinlock_paravirt_types.h
@@ -0,0 +1,13 @@
+#ifndef _ASM_QSPINLOCK_PARAVIRT_TYPES_H
+#define _ASM_QSPINLOCK_PARAVIRT_TYPES_H
+
+struct pv_lock_ops {
+ void (*lock)(struct qspinlock *lock, u32 val);
+ void (*unlock)(struct qspinlock *lock);
+ void (*wait)(u8 *ptr, u8 val, int cpu);
+ void (*kick)(int cpu);
+};
+
+extern struct pv_lock_ops pv_lock_op;
+
+#endif
diff --git a/arch/powerpc/kernel/paravirt.c b/arch/powerpc/kernel/paravirt.c
new file mode 100644
index 0000000..ea56c5f
--- /dev/null
+++ b/arch/powerpc/kernel/paravirt.c
@@ -0,0 +1,44 @@
+/*
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ */
+
+#include <linux/spinlock.h>
+
+static void __native_queued_spin_unlock(struct qspinlock *lock)
+{
+ native_queued_spin_unlock(lock);
+}
+
+static void __pv_wait(u8 *ptr, u8 val, int cpu)
+{
+ HMT_low();
+ __spin_yield_cpu(cpu);
+ HMT_medium();
+}
+
+static void __pv_kick(int cpu)
+{
+ __spin_wake_cpu(cpu);
+}
+
+struct pv_lock_ops pv_lock_op = {
+ .lock = native_queued_spin_lock_slowpath,
+ .unlock = __native_queued_spin_unlock,
+ .wait = NULL,
+ .kick = NULL,
+};
+EXPORT_SYMBOL(pv_lock_op);
+
+void __init pv_lock_init(void)
+{
+ if (SHARED_PROCESSOR) {
+ __pv_init_lock_hash();
+ pv_lock_op.lock = __pv_queued_spin_lock_slowpath;
+ pv_lock_op.unlock = __pv_queued_spin_unlock;
+ pv_lock_op.wait = __pv_wait;
+ pv_lock_op.kick = __pv_kick;
+ }
+}
diff --git a/arch/powerpc/platforms/pseries/setup.c b/arch/powerpc/platforms/pseries/setup.c
index 6e944fc..c9f056e 100644
--- a/arch/powerpc/platforms/pseries/setup.c
+++ b/arch/powerpc/platforms/pseries/setup.c
@@ -547,6 +547,11 @@ static void __init pSeries_setup_arch(void)
"%ld\n", rc);
}
}
+
+#ifdef CONFIG_PARAVIRT_SPINLOCKS
+ pv_lock_init();
+#endif
+
}
static int __init pSeries_init_panel(void)
--
1.9.1
^ permalink raw reply related
* [PATCH v2 3/6] powerpc: lib/locks.c: cpu yield/wake helper function
From: Pan Xinhui @ 2016-05-17 7:49 UTC (permalink / raw)
To: linux-kernel, virtualization, linuxppc-dev
Cc: mnipxh, Pan Xinhui, peterz, benh, boqun.feng, waiman.long, mingo,
paulus, mpe, paulmck
In-Reply-To: <1463471387-5115-1-git-send-email-xinhui.pan@linux.vnet.ibm.com>
pv-qspinlock core has pv_wait/pv_kick which will give a better
performace by yielding and kicking cpu at some cases.
lets support them by adding two corresponding helper functions.
Signed-off-by: Pan Xinhui <xinhui.pan@linux.vnet.ibm.com>
---
arch/powerpc/include/asm/spinlock.h | 4 ++++
arch/powerpc/lib/locks.c | 32 ++++++++++++++++++++++++++++++++
2 files changed, 36 insertions(+)
diff --git a/arch/powerpc/include/asm/spinlock.h b/arch/powerpc/include/asm/spinlock.h
index 4359ee6..3b65372 100644
--- a/arch/powerpc/include/asm/spinlock.h
+++ b/arch/powerpc/include/asm/spinlock.h
@@ -56,9 +56,13 @@
/* We only yield to the hypervisor if we are in shared processor mode */
#define SHARED_PROCESSOR (lppaca_shared_proc(local_paca->lppaca_ptr))
extern void __spin_yield(arch_spinlock_t *lock);
+extern void __spin_yield_cpu(int cpu);
+extern void __spin_wake_cpu(int cpu);
extern void __rw_yield(arch_rwlock_t *lock);
#else /* SPLPAR */
#define __spin_yield(x) barrier()
+#define __spin_yield_cpu(x) barrier()
+#define __spin_wake_cpu(x) barrier()
#define __rw_yield(x) barrier()
#define SHARED_PROCESSOR 0
#endif
diff --git a/arch/powerpc/lib/locks.c b/arch/powerpc/lib/locks.c
index a9ebd71..4109679a 100644
--- a/arch/powerpc/lib/locks.c
+++ b/arch/powerpc/lib/locks.c
@@ -23,6 +23,38 @@
#include <asm/hvcall.h>
#include <asm/smp.h>
+void __spin_yield_cpu(int cpu)
+{
+ unsigned int holder_cpu = cpu, yield_count;
+
+ if (cpu == -1) {
+ plpar_hcall_norets(H_CEDE);
+ return;
+ }
+ BUG_ON(holder_cpu >= nr_cpu_ids);
+ yield_count = be32_to_cpu(lppaca_of(holder_cpu).yield_count);
+ if ((yield_count & 1) == 0)
+ return; /* virtual cpu is currently running */
+ rmb();
+ plpar_hcall_norets(H_CONFER,
+ get_hard_smp_processor_id(holder_cpu), yield_count);
+}
+EXPORT_SYMBOL_GPL(__spin_yield_cpu);
+
+void __spin_wake_cpu(int cpu)
+{
+ unsigned int holder_cpu = cpu, yield_count;
+
+ BUG_ON(holder_cpu >= nr_cpu_ids);
+ yield_count = be32_to_cpu(lppaca_of(holder_cpu).yield_count);
+ if ((yield_count & 1) == 0)
+ return; /* virtual cpu is currently running */
+ rmb();
+ plpar_hcall_norets(H_PROD,
+ get_hard_smp_processor_id(holder_cpu));
+}
+EXPORT_SYMBOL_GPL(__spin_wake_cpu);
+
#ifndef CONFIG_QUEUED_SPINLOCKS
void __spin_yield(arch_spinlock_t *lock)
{
--
1.9.1
^ permalink raw reply related
* [PATCH v2 2/6] powerpc: pseries/Kconfig: qspinlock build config
From: Pan Xinhui @ 2016-05-17 7:49 UTC (permalink / raw)
To: linux-kernel, virtualization, linuxppc-dev
Cc: mnipxh, Pan Xinhui, peterz, benh, boqun.feng, waiman.long, mingo,
paulus, mpe, paulmck
In-Reply-To: <1463471387-5115-1-git-send-email-xinhui.pan@linux.vnet.ibm.com>
pseries will use qspinlock by default.
Signed-off-by: Pan Xinhui <xinhui.pan@linux.vnet.ibm.com>
---
arch/powerpc/platforms/pseries/Kconfig | 1 +
1 file changed, 1 insertion(+)
diff --git a/arch/powerpc/platforms/pseries/Kconfig b/arch/powerpc/platforms/pseries/Kconfig
index bec90fb..f669323 100644
--- a/arch/powerpc/platforms/pseries/Kconfig
+++ b/arch/powerpc/platforms/pseries/Kconfig
@@ -21,6 +21,7 @@ config PPC_PSERIES
select HOTPLUG_CPU if SMP
select ARCH_RANDOM
select PPC_DOORBELL
+ select ARCH_USE_QUEUED_SPINLOCKS
default y
config PPC_SPLPAR
--
1.9.1
^ permalink raw reply related
* [PATCH v2 1/6] qspinlock: powerpc support qspinlock
From: Pan Xinhui @ 2016-05-17 7:49 UTC (permalink / raw)
To: linux-kernel, virtualization, linuxppc-dev
Cc: mnipxh, Pan Xinhui, peterz, benh, boqun.feng, waiman.long, mingo,
paulus, mpe, paulmck
In-Reply-To: <1463471387-5115-1-git-send-email-xinhui.pan@linux.vnet.ibm.com>
Base code to enable qspinlock on powerpc. this patch add some #ifdef
here and there. Although there is no paravirt related code, we can
successfully build a qspinlock kernel after apply this patch.
Signed-off-by: Pan Xinhui <xinhui.pan@linux.vnet.ibm.com>
---
arch/powerpc/include/asm/qspinlock.h | 22 ++++++++++++++++++++++
arch/powerpc/include/asm/spinlock.h | 27 +++++++++++++++------------
arch/powerpc/include/asm/spinlock_types.h | 4 ++++
arch/powerpc/lib/locks.c | 4 ++++
4 files changed, 45 insertions(+), 12 deletions(-)
create mode 100644 arch/powerpc/include/asm/qspinlock.h
diff --git a/arch/powerpc/include/asm/qspinlock.h b/arch/powerpc/include/asm/qspinlock.h
new file mode 100644
index 0000000..5883954
--- /dev/null
+++ b/arch/powerpc/include/asm/qspinlock.h
@@ -0,0 +1,22 @@
+#ifndef _ASM_POWERPC_QSPINLOCK_H
+#define _ASM_POWERPC_QSPINLOCK_H
+
+#include <asm-generic/qspinlock_types.h>
+
+#define SPIN_THRESHOLD (1 << 15)
+#define queued_spin_unlock queued_spin_unlock
+
+static inline void native_queued_spin_unlock(struct qspinlock *lock)
+{
+ /* no load/store can be across the unlock()*/
+ smp_store_release((u8 *)lock, 0);
+}
+
+static inline void queued_spin_unlock(struct qspinlock *lock)
+{
+ native_queued_spin_unlock(lock);
+}
+
+#include <asm-generic/qspinlock.h>
+
+#endif /* _ASM_POWERPC_QSPINLOCK_H */
diff --git a/arch/powerpc/include/asm/spinlock.h b/arch/powerpc/include/asm/spinlock.h
index 523673d..4359ee6 100644
--- a/arch/powerpc/include/asm/spinlock.h
+++ b/arch/powerpc/include/asm/spinlock.h
@@ -52,6 +52,20 @@
#define SYNC_IO
#endif
+#if defined(CONFIG_PPC_SPLPAR)
+/* We only yield to the hypervisor if we are in shared processor mode */
+#define SHARED_PROCESSOR (lppaca_shared_proc(local_paca->lppaca_ptr))
+extern void __spin_yield(arch_spinlock_t *lock);
+extern void __rw_yield(arch_rwlock_t *lock);
+#else /* SPLPAR */
+#define __spin_yield(x) barrier()
+#define __rw_yield(x) barrier()
+#define SHARED_PROCESSOR 0
+#endif
+
+#ifdef CONFIG_QUEUED_SPINLOCKS
+#include <asm/qspinlock.h>
+#else
static __always_inline int arch_spin_value_unlocked(arch_spinlock_t lock)
{
return lock.slock == 0;
@@ -106,18 +120,6 @@ static inline int arch_spin_trylock(arch_spinlock_t *lock)
* held. Conveniently, we have a word in the paca that holds this
* value.
*/
-
-#if defined(CONFIG_PPC_SPLPAR)
-/* We only yield to the hypervisor if we are in shared processor mode */
-#define SHARED_PROCESSOR (lppaca_shared_proc(local_paca->lppaca_ptr))
-extern void __spin_yield(arch_spinlock_t *lock);
-extern void __rw_yield(arch_rwlock_t *lock);
-#else /* SPLPAR */
-#define __spin_yield(x) barrier()
-#define __rw_yield(x) barrier()
-#define SHARED_PROCESSOR 0
-#endif
-
static inline void arch_spin_lock(arch_spinlock_t *lock)
{
CLEAR_IO_SYNC;
@@ -169,6 +171,7 @@ extern void arch_spin_unlock_wait(arch_spinlock_t *lock);
do { while (arch_spin_is_locked(lock)) cpu_relax(); } while (0)
#endif
+#endif /* !CONFIG_QUEUED_SPINLOCKS */
/*
* Read-write spinlocks, allowing multiple readers
* but only one writer.
diff --git a/arch/powerpc/include/asm/spinlock_types.h b/arch/powerpc/include/asm/spinlock_types.h
index 2351adc..bd7144e 100644
--- a/arch/powerpc/include/asm/spinlock_types.h
+++ b/arch/powerpc/include/asm/spinlock_types.h
@@ -5,11 +5,15 @@
# error "please don't include this file directly"
#endif
+#ifdef CONFIG_QUEUED_SPINLOCKS
+#include <asm-generic/qspinlock_types.h>
+#else
typedef struct {
volatile unsigned int slock;
} arch_spinlock_t;
#define __ARCH_SPIN_LOCK_UNLOCKED { 0 }
+#endif
typedef struct {
volatile signed int lock;
diff --git a/arch/powerpc/lib/locks.c b/arch/powerpc/lib/locks.c
index f7deebd..a9ebd71 100644
--- a/arch/powerpc/lib/locks.c
+++ b/arch/powerpc/lib/locks.c
@@ -23,6 +23,7 @@
#include <asm/hvcall.h>
#include <asm/smp.h>
+#ifndef CONFIG_QUEUED_SPINLOCKS
void __spin_yield(arch_spinlock_t *lock)
{
unsigned int lock_value, holder_cpu, yield_count;
@@ -42,6 +43,7 @@ void __spin_yield(arch_spinlock_t *lock)
get_hard_smp_processor_id(holder_cpu), yield_count);
}
EXPORT_SYMBOL_GPL(__spin_yield);
+#endif
/*
* Waiting for a read lock or a write lock on a rwlock...
@@ -69,6 +71,7 @@ void __rw_yield(arch_rwlock_t *rw)
}
#endif
+#ifndef CONFIG_QUEUED_SPINLOCKS
void arch_spin_unlock_wait(arch_spinlock_t *lock)
{
smp_mb();
@@ -84,3 +87,4 @@ void arch_spin_unlock_wait(arch_spinlock_t *lock)
}
EXPORT_SYMBOL(arch_spin_unlock_wait);
+#endif
--
1.9.1
^ permalink raw reply related
* [PATCH v3 0/6] powerpc use pv-qpsinlock instead of spinlock
From: Pan Xinhui @ 2016-05-17 7:49 UTC (permalink / raw)
To: linux-kernel, virtualization, linuxppc-dev
Cc: mnipxh, Pan Xinhui, peterz, benh, boqun.feng, waiman.long, mingo,
paulus, mpe, paulmck
change fome v1:
separate into 6 pathes from one patch
some minor code changes.
benchmark test results are below.
run 3 tests on pseries IBM,8408-E8E with 32cpus, 64GB memory
perf bench futex hash
perf bench futex lock-pi
perf record -advRT || perf bench sched messaging -g 1000 || perf report
summary:
_____test________________spinlcok______________pv-qspinlcok_____
|futex hash | 556370 ops | 629634 ops |
|futex lock-pi | 362 ops | 367 ops |
|workloads | 23% | 13% |
details:
spinlock:
test 1)
# Running futex/hash benchmark...
Run summary [PID 9962]: 32 threads, each operating on 1024 [private] futexes for 10 secs.
Averaged 556370 operations/sec (+- 0.61%), total secs = 10
test 2)
# Running futex/lock-pi benchmark...
Run summary [PID 9962]: 32 threads doing pi lock/unlock pairing for 10 secs.
Averaged 362 operations/sec (+- 0.00%), total secs = 10
test 3)
perf bench sched messaging -g 1000
perf record -avdRT and perf report
# Samples: 2M of event 'cycles:ppp'
# Event count (approx.): 2045582241213
#
# Overhead Command Shared Object Symbol
# ........ ............... ................... ........................................................
#
22.96% sched-messaging [kernel.kallsyms] [k] _raw_spin_lock_irqsave
19.76% sched-messaging [kernel.kallsyms] [k] __spin_yield
2.09% sched-messaging [kernel.kallsyms] [k] __slab_free
2.07% sched-messaging [kernel.kallsyms] [k] unix_stream_read_generic
pv-qspinlock:
test 1)
# Running futex/hash benchmark...
Run summary [PID 3219]: 32 threads, each operating on 1024 [private] futexes for 10 secs.
Averaged 629634 operations/sec (+- 0.38%), total secs = 10
test 2)
# Running futex/lock-pi benchmark...
Run summary [PID 3219]: 32 threads doing pi lock/unlock pairing for 10 secs.
Averaged 367 operations/sec (+- 0.00%), total secs = 10
test 3)
perf bench sched messaging -g 1000
perf record -avdRT and perf report
# Samples: 1M of event 'cycles:ppp'
# Event count (approx.): 1250040606393
#
# Overhead Command Shared Object Symbol
# ........ ............... .............................. ........................................................
#
9.87% sched-messaging [kernel.vmlinux] [k] __pv_queued_spin_lock_slowpath
3.66% sched-messaging [kernel.vmlinux] [k] __pv_queued_spin_unlock
3.37% sched-messaging [kernel.vmlinux] [k] __slab_free
3.06% sched-messaging [kernel.vmlinux] [k] unix_stream_read_generic
Pan Xinhui (6):
qspinlock: powerpc support qspinlock
powerpc: pseries/Kconfig: qspinlock build config
powerpc: lib/locks.c: cpu yield/wake helper function
pv-qspinlock: powerpc support pv-qspinlock
pv-qspinlock: use cmpxchg_release in __pv_queued_spin_unlock
powerpc: pseries: pv-qspinlock build config/make
arch/powerpc/include/asm/qspinlock.h | 39 +++++++++++++++++++
arch/powerpc/include/asm/qspinlock_paravirt.h | 38 +++++++++++++++++++
.../powerpc/include/asm/qspinlock_paravirt_types.h | 13 +++++++
arch/powerpc/include/asm/spinlock.h | 31 +++++++++------
arch/powerpc/include/asm/spinlock_types.h | 4 ++
arch/powerpc/kernel/Makefile | 1 +
arch/powerpc/kernel/paravirt.c | 44 ++++++++++++++++++++++
arch/powerpc/lib/locks.c | 36 ++++++++++++++++++
arch/powerpc/platforms/pseries/Kconfig | 9 +++++
arch/powerpc/platforms/pseries/setup.c | 5 +++
kernel/locking/qspinlock_paravirt.h | 2 +-
11 files changed, 209 insertions(+), 13 deletions(-)
create mode 100644 arch/powerpc/include/asm/qspinlock.h
create mode 100644 arch/powerpc/include/asm/qspinlock_paravirt.h
create mode 100644 arch/powerpc/include/asm/qspinlock_paravirt_types.h
create mode 100644 arch/powerpc/kernel/paravirt.c
--
1.9.1
^ permalink raw reply
* Re: [PATCH v5 02/12] mm: migrate: support non-lru movable page migration
From: Sergey Senozhatsky @ 2016-05-17 1:30 UTC (permalink / raw)
To: Minchan Kim
Cc: Rik van Riel, Sergey Senozhatsky, Sergey Senozhatsky,
Jonathan Corbet, Hugh Dickins, linux-kernel, dri-devel,
virtualization, John Einar Reitan, linux-mm, Gioh Kim, Mel Gorman,
Andrew Morton, Joonsoo Kim, Vlastimil Babka, Rafael Aquini
In-Reply-To: <20160517011850.GD31335@bbox>
On (05/17/16 10:18), Minchan Kim wrote:
[..]
> > > #ifdef CONFIG_MIGRATION
> > >
> > > +extern int PageMovable(struct page *page);
> > > +extern void __SetPageMovable(struct page *page, struct address_space *mapping);
> > > +extern void __ClearPageMovable(struct page *page);
> > > extern void putback_movable_pages(struct list_head *l);
> > > extern int migrate_page(struct address_space *,
> > > struct page *, struct page *, enum migrate_mode);
> > > extern int migrate_pages(struct list_head *l, new_page_t new, free_page_t free,
> > > unsigned long private, enum migrate_mode mode, int reason);
> > > +extern bool isolate_movable_page(struct page *page, isolate_mode_t mode);
> > > +extern void putback_movable_page(struct page *page);
> > >
> > > extern int migrate_prep(void);
> > > extern int migrate_prep_local(void);
> >
> > given that some of Movable users can be built as modules, shouldn't
> > at least some of those symbols be exported via EXPORT_SYMBOL?
>
> Those functions aim for VM compaction so driver shouldn't use it.
> Only driver should be aware of are __SetPageMovable and __CleraPageMovable.
> I will export them.
>
and PageMovable(), probably?
zsmalloc():
VM_BUG_ON_PAGE(!PageMovable(page), page);
> Thanks for the review, Sergey!
no prob! sorry, I'm a bit slow and late here.
-ss
^ permalink raw reply
* Re: [PATCH v5 02/12] mm: migrate: support non-lru movable page migration
From: Minchan Kim @ 2016-05-17 1:18 UTC (permalink / raw)
To: Sergey Senozhatsky
Cc: Rik van Riel, Sergey Senozhatsky, Rafael Aquini, Jonathan Corbet,
Hugh Dickins, linux-kernel, dri-devel, virtualization,
John Einar Reitan, linux-mm, Gioh Kim, Mel Gorman, Andrew Morton,
Joonsoo Kim, Vlastimil Babka
In-Reply-To: <20160516071751.GA32079@swordfish>
On Mon, May 16, 2016 at 04:17:51PM +0900, Sergey Senozhatsky wrote:
> On (05/09/16 11:20), Minchan Kim wrote:
> [..]
> > +++ b/include/linux/migrate.h
> > @@ -32,11 +32,16 @@ extern char *migrate_reason_names[MR_TYPES];
> >
> > #ifdef CONFIG_MIGRATION
> >
> > +extern int PageMovable(struct page *page);
> > +extern void __SetPageMovable(struct page *page, struct address_space *mapping);
> > +extern void __ClearPageMovable(struct page *page);
> > extern void putback_movable_pages(struct list_head *l);
> > extern int migrate_page(struct address_space *,
> > struct page *, struct page *, enum migrate_mode);
> > extern int migrate_pages(struct list_head *l, new_page_t new, free_page_t free,
> > unsigned long private, enum migrate_mode mode, int reason);
> > +extern bool isolate_movable_page(struct page *page, isolate_mode_t mode);
> > +extern void putback_movable_page(struct page *page);
> >
> > extern int migrate_prep(void);
> > extern int migrate_prep_local(void);
>
> given that some of Movable users can be built as modules, shouldn't
> at least some of those symbols be exported via EXPORT_SYMBOL?
Those functions aim for VM compaction so driver shouldn't use it.
Only driver should be aware of are __SetPageMovable and __CleraPageMovable.
I will export them.
Thanks for the review, Sergey!
>
> -ss
^ permalink raw reply
* Re: [PATCH v5 02/12] mm: migrate: support non-lru movable page migration
From: Minchan Kim @ 2016-05-17 1:16 UTC (permalink / raw)
To: Sergey Senozhatsky
Cc: Rik van Riel, Sergey Senozhatsky, Rafael Aquini, Jonathan Corbet,
Hugh Dickins, linux-kernel, dri-devel, virtualization,
John Einar Reitan, linux-mm, Gioh Kim, Mel Gorman, Andrew Morton,
Joonsoo Kim, Vlastimil Babka
In-Reply-To: <20160516070455.GA28813@swordfish>
On Mon, May 16, 2016 at 04:04:55PM +0900, Sergey Senozhatsky wrote:
> On (05/09/16 11:20), Minchan Kim wrote:
> > +++ b/include/linux/migrate.h
> > @@ -32,11 +32,16 @@ extern char *migrate_reason_names[MR_TYPES];
> >
> > #ifdef CONFIG_MIGRATION
> >
> > +extern int PageMovable(struct page *page);
> > +extern void __SetPageMovable(struct page *page, struct address_space *mapping);
> > +extern void __ClearPageMovable(struct page *page);
> > extern void putback_movable_pages(struct list_head *l);
> > extern int migrate_page(struct address_space *,
> > struct page *, struct page *, enum migrate_mode);
> > extern int migrate_pages(struct list_head *l, new_page_t new, free_page_t free,
> > unsigned long private, enum migrate_mode mode, int reason);
> > +extern bool isolate_movable_page(struct page *page, isolate_mode_t mode);
> > +extern void putback_movable_page(struct page *page);
> >
> > extern int migrate_prep(void);
> > extern int migrate_prep_local(void);
>
> __ClearPageMovable() is under CONFIG_MIGRATION in include/linux/migrate.h,
> but zsmalloc checks for CONFIG_COMPACTION.
Thanks!
PageMovable check function should be in compact.c, I think.
I will fix it.
>
> can we have stub declarations of movable functions for !CONFIG_MIGRATION builds?
> otherwise the users (zsmalloc, for example) have to do things like
>
> static void reset_page(struct page *page)
> {
> #ifdef CONFIG_COMPACTION
> __ClearPageMovable(page);
> #endif
> clear_bit(PG_private, &page->flags);
> clear_bit(PG_private_2, &page->flags);
> set_page_private(page, 0);
> ClearPageHugeObject(page);
> page->freelist = NULL;
> }
>
> -ss
^ permalink raw reply
* Re: [PATCH v5 03/12] mm: balloon: use general non-lru movable page feature
From: kbuild test robot @ 2016-05-16 19:16 UTC (permalink / raw)
Cc: Rafael Aquini, Minchan Kim, linux-kernel, virtualization,
linux-mm, Gioh Kim, kbuild-all, Andrew Morton,
Konstantin Khlebnikov
In-Reply-To: <1462760433-32357-4-git-send-email-minchan@kernel.org>
[-- Attachment #1: Type: text/plain, Size: 880 bytes --]
Hi,
[auto build test ERROR on next-20160506]
[cannot apply to v4.6-rc7 v4.6-rc6 v4.6-rc5 v4.6]
[if your patch is applied to the wrong git tree, please drop us a note to help improve the system]
url: https://github.com/0day-ci/linux/commits/Minchan-Kim/mm-use-put_page-to-free-page-instead-of-putback_lru_page/20160509-102355
config: i386-randconfig-s0-05161922 (attached as .config)
compiler: gcc-6 (Debian 6.1.1-1) 6.1.1 20160430
reproduce:
# save the attached .config to linux build tree
make ARCH=i386
All errors (new ones prefixed by >>):
>> ERROR: "__ClearPageMovable" [drivers/virtio/virtio_balloon.ko] undefined!
>> ERROR: "__SetPageMovable" [drivers/virtio/virtio_balloon.ko] undefined!
---
0-DAY kernel test infrastructure Open Source Technology Center
https://lists.01.org/pipermail/kbuild-all Intel Corporation
[-- Attachment #2: .config.gz --]
[-- Type: application/octet-stream, Size: 25703 bytes --]
[-- Attachment #3: Type: text/plain, Size: 183 bytes --]
_______________________________________________
Virtualization mailing list
Virtualization@lists.linux-foundation.org
https://lists.linuxfoundation.org/mailman/listinfo/virtualization
^ permalink raw reply
* Re: [PATCH v5 02/12] mm: migrate: support non-lru movable page migration
From: Sergey Senozhatsky @ 2016-05-16 7:17 UTC (permalink / raw)
To: Minchan Kim
Cc: Rik van Riel, Sergey Senozhatsky, Rafael Aquini, Jonathan Corbet,
Hugh Dickins, linux-kernel, dri-devel, virtualization,
John Einar Reitan, linux-mm, Gioh Kim, Mel Gorman, Andrew Morton,
Joonsoo Kim, Vlastimil Babka
In-Reply-To: <1462760433-32357-3-git-send-email-minchan@kernel.org>
On (05/09/16 11:20), Minchan Kim wrote:
[..]
> +++ b/include/linux/migrate.h
> @@ -32,11 +32,16 @@ extern char *migrate_reason_names[MR_TYPES];
>
> #ifdef CONFIG_MIGRATION
>
> +extern int PageMovable(struct page *page);
> +extern void __SetPageMovable(struct page *page, struct address_space *mapping);
> +extern void __ClearPageMovable(struct page *page);
> extern void putback_movable_pages(struct list_head *l);
> extern int migrate_page(struct address_space *,
> struct page *, struct page *, enum migrate_mode);
> extern int migrate_pages(struct list_head *l, new_page_t new, free_page_t free,
> unsigned long private, enum migrate_mode mode, int reason);
> +extern bool isolate_movable_page(struct page *page, isolate_mode_t mode);
> +extern void putback_movable_page(struct page *page);
>
> extern int migrate_prep(void);
> extern int migrate_prep_local(void);
given that some of Movable users can be built as modules, shouldn't
at least some of those symbols be exported via EXPORT_SYMBOL?
-ss
^ permalink raw reply
* Re: [PATCH v5 02/12] mm: migrate: support non-lru movable page migration
From: Sergey Senozhatsky @ 2016-05-16 7:04 UTC (permalink / raw)
To: Minchan Kim
Cc: Rik van Riel, Sergey Senozhatsky, Rafael Aquini, Jonathan Corbet,
Hugh Dickins, linux-kernel, dri-devel, virtualization,
John Einar Reitan, linux-mm, Gioh Kim, Mel Gorman, Andrew Morton,
Joonsoo Kim, Vlastimil Babka
In-Reply-To: <1462760433-32357-3-git-send-email-minchan@kernel.org>
On (05/09/16 11:20), Minchan Kim wrote:
> +++ b/include/linux/migrate.h
> @@ -32,11 +32,16 @@ extern char *migrate_reason_names[MR_TYPES];
>
> #ifdef CONFIG_MIGRATION
>
> +extern int PageMovable(struct page *page);
> +extern void __SetPageMovable(struct page *page, struct address_space *mapping);
> +extern void __ClearPageMovable(struct page *page);
> extern void putback_movable_pages(struct list_head *l);
> extern int migrate_page(struct address_space *,
> struct page *, struct page *, enum migrate_mode);
> extern int migrate_pages(struct list_head *l, new_page_t new, free_page_t free,
> unsigned long private, enum migrate_mode mode, int reason);
> +extern bool isolate_movable_page(struct page *page, isolate_mode_t mode);
> +extern void putback_movable_page(struct page *page);
>
> extern int migrate_prep(void);
> extern int migrate_prep_local(void);
__ClearPageMovable() is under CONFIG_MIGRATION in include/linux/migrate.h,
but zsmalloc checks for CONFIG_COMPACTION.
can we have stub declarations of movable functions for !CONFIG_MIGRATION builds?
otherwise the users (zsmalloc, for example) have to do things like
static void reset_page(struct page *page)
{
#ifdef CONFIG_COMPACTION
__ClearPageMovable(page);
#endif
clear_bit(PG_private, &page->flags);
clear_bit(PG_private_2, &page->flags);
set_page_private(page, 0);
ClearPageHugeObject(page);
page->freelist = NULL;
}
-ss
^ permalink raw reply
* [PATCH] tools/virtio: add inorder option
From: Michael S. Tsirkin @ 2016-05-15 10:03 UTC (permalink / raw)
To: linux-kernel; +Cc: Jesper Dangaard Brouer, virtualization
skips ring accesses but drops out of order support
At high level this follows alf design
https://github.com/netoptimizer/prototype-kernel/blob/master/kernel/lib/alf_queue.c
but on most (single-threaded) benchmarks this seems to be slower than the layout
without index use.
Signed-off-by: Michael S. Tsirkin <mst@redhat.com>
---
tools/virtio/ringtest/virtio_ring_0_9.c | 49 ++++++++++++++++++++++++++++-
tools/virtio/ringtest/virtio_ring_inorder.c | 2 ++
tools/virtio/ringtest/Makefile | 4 ++-
3 files changed, 53 insertions(+), 2 deletions(-)
create mode 100644 tools/virtio/ringtest/virtio_ring_inorder.c
diff --git a/tools/virtio/ringtest/virtio_ring_0_9.c b/tools/virtio/ringtest/virtio_ring_0_9.c
index 47c9a1a..7618662 100644
--- a/tools/virtio/ringtest/virtio_ring_0_9.c
+++ b/tools/virtio/ringtest/virtio_ring_0_9.c
@@ -26,6 +26,14 @@ struct vring ring;
* high bits of ring id ^ 0x8000).
*/
/* #ifdef RING_POLL */
+/* enabling the below activates experimental in-order code
+ * (which skips ring updates and reads and writes len in descriptor).
+ */
+/* #ifdef INORDER */
+
+#if defined(RING_POLL) && defined(INORDER)
+#error "RING_POLL and INORDER are mutually exclusive"
+#endif
/* how much padding is needed to avoid false cache sharing */
#define HOST_GUEST_PADDING 0x80
@@ -35,7 +43,11 @@ struct guest {
unsigned short last_used_idx;
unsigned short num_free;
unsigned short kicked_avail_idx;
+#ifndef INORDER
unsigned short free_head;
+#else
+ unsigned short reserved_free_head;
+#endif
unsigned char reserved[HOST_GUEST_PADDING - 10];
} guest;
@@ -66,8 +78,10 @@ void alloc_ring(void)
guest.avail_idx = 0;
guest.kicked_avail_idx = -1;
guest.last_used_idx = 0;
+#ifndef INORDER
/* Put everything in free lists. */
guest.free_head = 0;
+#endif
for (i = 0; i < ring_size - 1; i++)
ring.desc[i].next = i + 1;
host.used_idx = 0;
@@ -84,13 +98,20 @@ void alloc_ring(void)
/* guest side */
int add_inbuf(unsigned len, void *buf, void *datap)
{
- unsigned head, avail;
+ unsigned head;
+#ifndef INORDER
+ unsigned avail;
+#endif
struct vring_desc *desc;
if (!guest.num_free)
return -1;
+#ifdef INORDER
+ head = (ring_size - 1) & (guest.avail_idx++);
+#else
head = guest.free_head;
+#endif
guest.num_free--;
desc = ring.desc;
@@ -102,7 +123,9 @@ int add_inbuf(unsigned len, void *buf, void *datap)
* descriptors.
*/
desc[head].flags &= ~VRING_DESC_F_NEXT;
+#ifndef INORDER
guest.free_head = desc[head].next;
+#endif
data[head].data = datap;
@@ -113,8 +136,12 @@ int add_inbuf(unsigned len, void *buf, void *datap)
ring.avail->ring[avail & (ring_size - 1)] =
(head | (avail & ~(ring_size - 1))) ^ 0x8000;
#else
+#ifndef INORDER
+ /* Barrier A (for pairing) */
+ smp_release();
avail = (ring_size - 1) & (guest.avail_idx++);
ring.avail->ring[avail] = head;
+#endif
/* Barrier A (for pairing) */
smp_release();
#endif
@@ -141,15 +168,27 @@ void *get_buf(unsigned *lenp, void **bufp)
return NULL;
/* Barrier B (for pairing) */
smp_acquire();
+#ifdef INORDER
+ head = (ring_size - 1) & guest.last_used_idx;
+ index = head;
+#else
head = (ring_size - 1) & guest.last_used_idx;
index = ring.used->ring[head].id;
#endif
+
+#endif
+#ifdef INORDER
+ *lenp = ring.desc[index].len;
+#else
*lenp = ring.used->ring[head].len;
+#endif
datap = data[index].data;
*bufp = (void*)(unsigned long)ring.desc[index].addr;
data[index].data = NULL;
+#ifndef INORDER
ring.desc[index].next = guest.free_head;
guest.free_head = index;
+#endif
guest.num_free++;
guest.last_used_idx++;
return datap;
@@ -283,16 +322,24 @@ bool use_buf(unsigned *lenp, void **bufp)
smp_acquire();
used_idx &= ring_size - 1;
+#ifdef INORDER
+ head = used_idx;
+#else
head = ring.avail->ring[used_idx];
+#endif
desc = &ring.desc[head];
#endif
*lenp = desc->len;
*bufp = (void *)(unsigned long)desc->addr;
+#ifdef INORDER
+ desc->len = desc->len - 1;
+#else
/* now update used ring */
ring.used->ring[used_idx].id = head;
ring.used->ring[used_idx].len = desc->len - 1;
+#endif
/* Barrier B (for pairing) */
smp_release();
host.used_idx++;
diff --git a/tools/virtio/ringtest/virtio_ring_inorder.c b/tools/virtio/ringtest/virtio_ring_inorder.c
new file mode 100644
index 0000000..2438ca5
--- /dev/null
+++ b/tools/virtio/ringtest/virtio_ring_inorder.c
@@ -0,0 +1,2 @@
+#define INORDER 1
+#include "virtio_ring_0_9.c"
diff --git a/tools/virtio/ringtest/Makefile b/tools/virtio/ringtest/Makefile
index feaa64a..8037971 100644
--- a/tools/virtio/ringtest/Makefile
+++ b/tools/virtio/ringtest/Makefile
@@ -1,6 +1,6 @@
all:
-all: ring virtio_ring_0_9 virtio_ring_poll
+all: ring virtio_ring_0_9 virtio_ring_poll virtio_ring_inorder
CFLAGS += -Wall
CFLAGS += -pthread -O2 -ggdb
@@ -13,10 +13,12 @@ virtio_ring_poll.o: virtio_ring_poll.c virtio_ring_0_9.c main.h
ring: ring.o main.o
virtio_ring_0_9: virtio_ring_0_9.o main.o
virtio_ring_poll: virtio_ring_poll.o main.o
+virtio_ring_inorder: virtio_ring_inorder.o main.o
clean:
-rm main.o
-rm ring.o ring
-rm virtio_ring_0_9.o virtio_ring_0_9
-rm virtio_ring_poll.o virtio_ring_poll
+ -rm virtio_ring_inorder.o virtio_ring_inorder
.PHONY: all clean
--
MST
^ permalink raw reply related
* Re: [PATCH v4 6/6] hwmon: use smp_call_on_cpu() for dell-smm i8k
From: Pali Rohár @ 2016-05-12 8:51 UTC (permalink / raw)
To: Juergen Gross
Cc: x86, jeremy, jdelvare, peterz, akataria, linux-kernel,
virtualization, chrisw, mingo, david.vrabel, Douglas_Warzecha,
hpa, xen-devel, boris.ostrovsky, tglx, Guenter Roeck
In-Reply-To: <5730A08D.60100@suse.com>
On Monday 09 May 2016 16:37:01 Juergen Gross wrote:
> On 21/04/16 15:27, Pali Rohár wrote:
> > On Thursday 21 April 2016 15:12:52 Juergen Gross wrote:
> >> On 21/04/16 12:57, Pali Rohár wrote:
> >>> On Tuesday 05 April 2016 21:31:52 Pali Rohár wrote:
> >>>> On Tuesday 05 April 2016 16:54:14 Guenter Roeck wrote:
> >>>>> On Tue, Apr 05, 2016 at 07:10:07AM +0200, Juergen Gross wrote:
> >>>>>> Use the smp_call_on_cpu() function to call system management
> >>>>>> mode on cpu 0.
> >>>>>> Make call secure by adding get_online_cpus() to avoid e.g. suspend
> >>>>>> resume cycles in between.
> >>>>>>
> >>>>>> Signed-off-by: Juergen Gross <jgross@suse.com>
> >>>>>> ---
> >>>>>> V4: add call to get_online_cpus()
> >>>>>
> >>>>> Pali, any chance to test this ?
> >>>>
> >>>> I can test it, but just on machine where (probably) smm calls can be
> >>>> send from any cpu... Need some time for testing and I believe I can do
> >>>> that at the end of the week.
> >>>
> >>> Sorry I had absolutely no more free time last weekend :-( And same
> >>> prediction is for this weekend and also next one...
> >>
> >> Pali, I've got a Dell laptop (Latitude E6440) here. Would this device be
> >> okay for a test?
> >
> > Hi!
> >
> > Proper regression test should check if this patch does not break any
> > function or drivers dependent on dcdbas.ko. And should be done on both
> > notebook devices: which needs to issue that smm call on cpu 0 and also
> > on which it is not needed.
>
> Hmm, couldn't get one which needs smm to be called on cpu 0.
> OTOH I've done various tests and added a printk() in raise_smm()
> and i8k_smm_func() issuing the cpu number it was called on.
Understood, those machines are old and probably rare now.
> > Some notebooks which needs smm call to issued from cpu 0 can be found in
> > git commit messages of i8k, dell-laptop or dcdbas kernel drivers.
> >
> >> What would you do for testing? In case you can give me
> >> some hints how to do a sensible test I'd do it.
> >
> > Test e.g. dell-laptop.ko driver. It provides /sys interface for changing
> > keyboard backlight or changing rfkill switches (bluetooth wifi).
>
> Done.
>
> > Also test tools from libsmbios (userspace) package.
>
> Done.
>
> > There must be no difference in output/functionality with or without your
> > patches.
>
> Verified.
>
> >> I've verified by adding a printk() to smp_call_on_cpu() that at least
> >> one of the modified drivers has been used during system boot.
> >
> > Also you can patch i8k/dcdbas smm function to print cpu number on which
> > is code running (to verify that it was really called on cpu 0 as
> > needed).
>
> Done.
>
> I tested suspend/resume, too, as adding get_online_cpus() might have
> changed behavior. Worked like a charm. :-)
Ok, I think this should be enough. You can add my Acked-by.
--
Pali Rohár
pali.rohar@gmail.com
_______________________________________________
Virtualization mailing list
Virtualization@lists.linux-foundation.org
https://lists.linuxfoundation.org/mailman/listinfo/virtualization
^ permalink raw reply
* Re: [PATCH v4 6/6] hwmon: use smp_call_on_cpu() for dell-smm i8k
From: Juergen Gross @ 2016-05-09 14:37 UTC (permalink / raw)
To: Pali Rohár
Cc: x86, jeremy, jdelvare, peterz, akataria, linux-kernel,
virtualization, chrisw, mingo, david.vrabel, Douglas_Warzecha,
hpa, xen-devel, boris.ostrovsky, tglx, Guenter Roeck
In-Reply-To: <20160421132735.GR29406@pali>
On 21/04/16 15:27, Pali Rohár wrote:
> On Thursday 21 April 2016 15:12:52 Juergen Gross wrote:
>> On 21/04/16 12:57, Pali Rohár wrote:
>>> On Tuesday 05 April 2016 21:31:52 Pali Rohár wrote:
>>>> On Tuesday 05 April 2016 16:54:14 Guenter Roeck wrote:
>>>>> On Tue, Apr 05, 2016 at 07:10:07AM +0200, Juergen Gross wrote:
>>>>>> Use the smp_call_on_cpu() function to call system management
>>>>>> mode on cpu 0.
>>>>>> Make call secure by adding get_online_cpus() to avoid e.g. suspend
>>>>>> resume cycles in between.
>>>>>>
>>>>>> Signed-off-by: Juergen Gross <jgross@suse.com>
>>>>>> ---
>>>>>> V4: add call to get_online_cpus()
>>>>>
>>>>> Pali, any chance to test this ?
>>>>
>>>> I can test it, but just on machine where (probably) smm calls can be
>>>> send from any cpu... Need some time for testing and I believe I can do
>>>> that at the end of the week.
>>>
>>> Sorry I had absolutely no more free time last weekend :-( And same
>>> prediction is for this weekend and also next one...
>>
>> Pali, I've got a Dell laptop (Latitude E6440) here. Would this device be
>> okay for a test?
>
> Hi!
>
> Proper regression test should check if this patch does not break any
> function or drivers dependent on dcdbas.ko. And should be done on both
> notebook devices: which needs to issue that smm call on cpu 0 and also
> on which it is not needed.
Hmm, couldn't get one which needs smm to be called on cpu 0.
OTOH I've done various tests and added a printk() in raise_smm()
and i8k_smm_func() issuing the cpu number it was called on.
> Some notebooks which needs smm call to issued from cpu 0 can be found in
> git commit messages of i8k, dell-laptop or dcdbas kernel drivers.
>
>> What would you do for testing? In case you can give me
>> some hints how to do a sensible test I'd do it.
>
> Test e.g. dell-laptop.ko driver. It provides /sys interface for changing
> keyboard backlight or changing rfkill switches (bluetooth wifi).
Done.
> Also test tools from libsmbios (userspace) package.
Done.
> There must be no difference in output/functionality with or without your
> patches.
Verified.
>> I've verified by adding a printk() to smp_call_on_cpu() that at least
>> one of the modified drivers has been used during system boot.
>
> Also you can patch i8k/dcdbas smm function to print cpu number on which
> is code running (to verify that it was really called on cpu 0 as
> needed).
Done.
I tested suspend/resume, too, as adding get_online_cpus() might have
changed behavior. Worked like a charm. :-)
Juergen
_______________________________________________
Virtualization mailing list
Virtualization@lists.linux-foundation.org
https://lists.linuxfoundation.org/mailman/listinfo/virtualization
^ permalink raw reply
* Re: [PATCH V2 RFC] fixup! virtio: convert to use DMA api
From: Paolo Bonzini @ 2016-05-09 11:09 UTC (permalink / raw)
To: Michael S. Tsirkin, David Woodhouse
Cc: Wei Liu, Stefan Hajnoczi, qemu-block, Stefano Stabellini,
Joerg Roedel, qemu-devel, peterx, linux-kernel,
Christian Borntraeger, iommu, Andy Lutomirski, kvm, Amit Shah,
virtualization, Anthony PERARD
In-Reply-To: <20160428182341-mutt-send-email-mst@redhat.com>
On 28/04/2016 17:37, Michael S. Tsirkin wrote:
> > All the internally-emulated devices *can* be either translated or
> > untranslated. That's just a matter of software. Surely, you currently
> > *can't* have translated assigned devices (until someone implements the
> > whole VT-d page table shadowing or whatever), so you'll be barred from
> > assigning a device to a slot which *previously* had an untranslated
> > device. But so what? Put it in a different slot instead.
>
> Unfortunately people got used to be able to put any device
> in any slot, and built external tools around that ability.
> It's rather painful to break this assumption.
Once you move to PCIe, a lot of things become more complicated. This is
just one of them; instead of needing half a dozen PCI bridges, you'll
need half a dozen plus one.
Paolo
^ permalink raw reply
* [PATCH v5 03/12] mm: balloon: use general non-lru movable page feature
From: Minchan Kim @ 2016-05-09 2:20 UTC (permalink / raw)
To: Andrew Morton
Cc: Rafael Aquini, Minchan Kim, linux-kernel, virtualization,
linux-mm, Gioh Kim, Konstantin Khlebnikov
In-Reply-To: <1462760433-32357-1-git-send-email-minchan@kernel.org>
Now, VM has a feature to migrate non-lru movable pages so
balloon doesn't need custom migration hooks in migrate.c
and compaction.c. Instead, this patch implements
page->mapping->a_ops->{isolate|migrate|putback} functions.
With that, we could remove hooks for ballooning in general
migration functions and make balloon compaction simple.
Cc: virtualization@lists.linux-foundation.org
Cc: Rafael Aquini <aquini@redhat.com>
Cc: Konstantin Khlebnikov <koct9i@gmail.com>
Signed-off-by: Gioh Kim <gi-oh.kim@profitbricks.com>
Signed-off-by: Minchan Kim <minchan@kernel.org>
---
drivers/virtio/virtio_balloon.c | 52 +++++++++++++++++++--
include/linux/balloon_compaction.h | 51 ++++++---------------
include/uapi/linux/magic.h | 1 +
mm/balloon_compaction.c | 94 +++++++-------------------------------
mm/compaction.c | 7 ---
mm/migrate.c | 19 +-------
mm/vmscan.c | 2 +-
7 files changed, 83 insertions(+), 143 deletions(-)
diff --git a/drivers/virtio/virtio_balloon.c b/drivers/virtio/virtio_balloon.c
index 7b6d74f0c72f..04fc63b4a735 100644
--- a/drivers/virtio/virtio_balloon.c
+++ b/drivers/virtio/virtio_balloon.c
@@ -30,6 +30,7 @@
#include <linux/oom.h>
#include <linux/wait.h>
#include <linux/mm.h>
+#include <linux/mount.h>
/*
* Balloon device works in 4K page units. So each page is pointed to by
@@ -45,6 +46,10 @@ static int oom_pages = OOM_VBALLOON_DEFAULT_PAGES;
module_param(oom_pages, int, S_IRUSR | S_IWUSR);
MODULE_PARM_DESC(oom_pages, "pages to free on OOM");
+#ifdef CONFIG_BALLOON_COMPACTION
+static struct vfsmount *balloon_mnt;
+#endif
+
struct virtio_balloon {
struct virtio_device *vdev;
struct virtqueue *inflate_vq, *deflate_vq, *stats_vq;
@@ -486,6 +491,24 @@ static int virtballoon_migratepage(struct balloon_dev_info *vb_dev_info,
return MIGRATEPAGE_SUCCESS;
}
+
+static struct dentry *balloon_mount(struct file_system_type *fs_type,
+ int flags, const char *dev_name, void *data)
+{
+ static const struct dentry_operations ops = {
+ .d_dname = simple_dname,
+ };
+
+ return mount_pseudo(fs_type, "balloon-kvm:", NULL, &ops,
+ BALLOON_KVM_MAGIC);
+}
+
+static struct file_system_type balloon_fs = {
+ .name = "balloon-kvm",
+ .mount = balloon_mount,
+ .kill_sb = kill_anon_super,
+};
+
#endif /* CONFIG_BALLOON_COMPACTION */
static int virtballoon_probe(struct virtio_device *vdev)
@@ -515,9 +538,6 @@ static int virtballoon_probe(struct virtio_device *vdev)
vb->vdev = vdev;
balloon_devinfo_init(&vb->vb_dev_info);
-#ifdef CONFIG_BALLOON_COMPACTION
- vb->vb_dev_info.migratepage = virtballoon_migratepage;
-#endif
err = init_vqs(vb);
if (err)
@@ -527,13 +547,33 @@ static int virtballoon_probe(struct virtio_device *vdev)
vb->nb.priority = VIRTBALLOON_OOM_NOTIFY_PRIORITY;
err = register_oom_notifier(&vb->nb);
if (err < 0)
- goto out_oom_notify;
+ goto out_del_vqs;
+
+#ifdef CONFIG_BALLOON_COMPACTION
+ balloon_mnt = kern_mount(&balloon_fs);
+ if (IS_ERR(balloon_mnt)) {
+ err = PTR_ERR(balloon_mnt);
+ unregister_oom_notifier(&vb->nb);
+ goto out_del_vqs;
+ }
+
+ vb->vb_dev_info.migratepage = virtballoon_migratepage;
+ vb->vb_dev_info.inode = alloc_anon_inode(balloon_mnt->mnt_sb);
+ if (IS_ERR(vb->vb_dev_info.inode)) {
+ err = PTR_ERR(vb->vb_dev_info.inode);
+ kern_unmount(balloon_mnt);
+ unregister_oom_notifier(&vb->nb);
+ vb->vb_dev_info.inode = NULL;
+ goto out_del_vqs;
+ }
+ vb->vb_dev_info.inode->i_mapping->a_ops = &balloon_aops;
+#endif
virtio_device_ready(vdev);
return 0;
-out_oom_notify:
+out_del_vqs:
vdev->config->del_vqs(vdev);
out_free_vb:
kfree(vb);
@@ -567,6 +607,8 @@ static void virtballoon_remove(struct virtio_device *vdev)
cancel_work_sync(&vb->update_balloon_stats_work);
remove_common(vb);
+ if (vb->vb_dev_info.inode)
+ iput(vb->vb_dev_info.inode);
kfree(vb);
}
diff --git a/include/linux/balloon_compaction.h b/include/linux/balloon_compaction.h
index 9b0a15d06a4f..79542b2698ec 100644
--- a/include/linux/balloon_compaction.h
+++ b/include/linux/balloon_compaction.h
@@ -48,6 +48,7 @@
#include <linux/migrate.h>
#include <linux/gfp.h>
#include <linux/err.h>
+#include <linux/fs.h>
/*
* Balloon device information descriptor.
@@ -62,6 +63,7 @@ struct balloon_dev_info {
struct list_head pages; /* Pages enqueued & handled to Host */
int (*migratepage)(struct balloon_dev_info *, struct page *newpage,
struct page *page, enum migrate_mode mode);
+ struct inode *inode;
};
extern struct page *balloon_page_enqueue(struct balloon_dev_info *b_dev_info);
@@ -73,45 +75,19 @@ static inline void balloon_devinfo_init(struct balloon_dev_info *balloon)
spin_lock_init(&balloon->pages_lock);
INIT_LIST_HEAD(&balloon->pages);
balloon->migratepage = NULL;
+ balloon->inode = NULL;
}
#ifdef CONFIG_BALLOON_COMPACTION
-extern bool balloon_page_isolate(struct page *page);
+extern const struct address_space_operations balloon_aops;
+extern bool balloon_page_isolate(struct page *page,
+ isolate_mode_t mode);
extern void balloon_page_putback(struct page *page);
-extern int balloon_page_migrate(struct page *newpage,
+extern int balloon_page_migrate(struct address_space *mapping,
+ struct page *newpage,
struct page *page, enum migrate_mode mode);
/*
- * __is_movable_balloon_page - helper to perform @page PageBalloon tests
- */
-static inline bool __is_movable_balloon_page(struct page *page)
-{
- return PageBalloon(page);
-}
-
-/*
- * balloon_page_movable - test PageBalloon to identify balloon pages
- * and PagePrivate to check that the page is not
- * isolated and can be moved by compaction/migration.
- *
- * As we might return false positives in the case of a balloon page being just
- * released under us, this need to be re-tested later, under the page lock.
- */
-static inline bool balloon_page_movable(struct page *page)
-{
- return PageBalloon(page) && PagePrivate(page);
-}
-
-/*
- * isolated_balloon_page - identify an isolated balloon page on private
- * compaction/migration page lists.
- */
-static inline bool isolated_balloon_page(struct page *page)
-{
- return PageBalloon(page);
-}
-
-/*
* balloon_page_insert - insert a page into the balloon's page list and make
* the page->private assignment accordingly.
* @balloon : pointer to balloon device
@@ -124,7 +100,7 @@ static inline void balloon_page_insert(struct balloon_dev_info *balloon,
struct page *page)
{
__SetPageBalloon(page);
- SetPagePrivate(page);
+ __SetPageMovable(page, balloon->inode->i_mapping);
set_page_private(page, (unsigned long)balloon);
list_add(&page->lru, &balloon->pages);
}
@@ -140,11 +116,14 @@ static inline void balloon_page_insert(struct balloon_dev_info *balloon,
static inline void balloon_page_delete(struct page *page)
{
__ClearPageBalloon(page);
+ __ClearPageMovable(page);
set_page_private(page, 0);
- if (PagePrivate(page)) {
- ClearPagePrivate(page);
+ /*
+ * No touch page.lru field once @page has been isolated
+ * because VM is using the field.
+ */
+ if (!PageIsolated(page))
list_del(&page->lru);
- }
}
/*
diff --git a/include/uapi/linux/magic.h b/include/uapi/linux/magic.h
index 546b38886e11..d829ce63529d 100644
--- a/include/uapi/linux/magic.h
+++ b/include/uapi/linux/magic.h
@@ -80,5 +80,6 @@
#define BPF_FS_MAGIC 0xcafe4a11
/* Since UDF 2.01 is ISO 13346 based... */
#define UDF_SUPER_MAGIC 0x15013346
+#define BALLOON_KVM_MAGIC 0x13661366
#endif /* __LINUX_MAGIC_H__ */
diff --git a/mm/balloon_compaction.c b/mm/balloon_compaction.c
index 57b3e9bd6bc5..da91df50ba31 100644
--- a/mm/balloon_compaction.c
+++ b/mm/balloon_compaction.c
@@ -70,7 +70,7 @@ struct page *balloon_page_dequeue(struct balloon_dev_info *b_dev_info)
*/
if (trylock_page(page)) {
#ifdef CONFIG_BALLOON_COMPACTION
- if (!PagePrivate(page)) {
+ if (PageIsolated(page)) {
/* raced with isolation */
unlock_page(page);
continue;
@@ -106,110 +106,50 @@ EXPORT_SYMBOL_GPL(balloon_page_dequeue);
#ifdef CONFIG_BALLOON_COMPACTION
-static inline void __isolate_balloon_page(struct page *page)
+bool balloon_page_isolate(struct page *page, isolate_mode_t mode)
+
{
struct balloon_dev_info *b_dev_info = balloon_page_device(page);
unsigned long flags;
spin_lock_irqsave(&b_dev_info->pages_lock, flags);
- ClearPagePrivate(page);
list_del(&page->lru);
b_dev_info->isolated_pages++;
spin_unlock_irqrestore(&b_dev_info->pages_lock, flags);
+
+ return true;
}
-static inline void __putback_balloon_page(struct page *page)
+void balloon_page_putback(struct page *page)
{
struct balloon_dev_info *b_dev_info = balloon_page_device(page);
unsigned long flags;
spin_lock_irqsave(&b_dev_info->pages_lock, flags);
- SetPagePrivate(page);
list_add(&page->lru, &b_dev_info->pages);
b_dev_info->isolated_pages--;
spin_unlock_irqrestore(&b_dev_info->pages_lock, flags);
}
-/* __isolate_lru_page() counterpart for a ballooned page */
-bool balloon_page_isolate(struct page *page)
-{
- /*
- * Avoid burning cycles with pages that are yet under __free_pages(),
- * or just got freed under us.
- *
- * In case we 'win' a race for a balloon page being freed under us and
- * raise its refcount preventing __free_pages() from doing its job
- * the put_page() at the end of this block will take care of
- * release this page, thus avoiding a nasty leakage.
- */
- if (likely(get_page_unless_zero(page))) {
- /*
- * As balloon pages are not isolated from LRU lists, concurrent
- * compaction threads can race against page migration functions
- * as well as race against the balloon driver releasing a page.
- *
- * In order to avoid having an already isolated balloon page
- * being (wrongly) re-isolated while it is under migration,
- * or to avoid attempting to isolate pages being released by
- * the balloon driver, lets be sure we have the page lock
- * before proceeding with the balloon page isolation steps.
- */
- if (likely(trylock_page(page))) {
- /*
- * A ballooned page, by default, has PagePrivate set.
- * Prevent concurrent compaction threads from isolating
- * an already isolated balloon page by clearing it.
- */
- if (balloon_page_movable(page)) {
- __isolate_balloon_page(page);
- unlock_page(page);
- return true;
- }
- unlock_page(page);
- }
- put_page(page);
- }
- return false;
-}
-
-/* putback_lru_page() counterpart for a ballooned page */
-void balloon_page_putback(struct page *page)
-{
- /*
- * 'lock_page()' stabilizes the page and prevents races against
- * concurrent isolation threads attempting to re-isolate it.
- */
- lock_page(page);
-
- if (__is_movable_balloon_page(page)) {
- __putback_balloon_page(page);
- /* drop the extra ref count taken for page isolation */
- put_page(page);
- } else {
- WARN_ON(1);
- dump_page(page, "not movable balloon page");
- }
- unlock_page(page);
-}
/* move_to_new_page() counterpart for a ballooned page */
-int balloon_page_migrate(struct page *newpage,
- struct page *page, enum migrate_mode mode)
+int balloon_page_migrate(struct address_space *mapping,
+ struct page *newpage, struct page *page,
+ enum migrate_mode mode)
{
struct balloon_dev_info *balloon = balloon_page_device(page);
- int rc = -EAGAIN;
VM_BUG_ON_PAGE(!PageLocked(page), page);
VM_BUG_ON_PAGE(!PageLocked(newpage), newpage);
- if (WARN_ON(!__is_movable_balloon_page(page))) {
- dump_page(page, "not movable balloon page");
- return rc;
- }
+ return balloon->migratepage(balloon, newpage, page, mode);
+}
- if (balloon && balloon->migratepage)
- rc = balloon->migratepage(balloon, newpage, page, mode);
+const struct address_space_operations balloon_aops = {
+ .migratepage = balloon_page_migrate,
+ .isolate_page = balloon_page_isolate,
+ .putback_page = balloon_page_putback,
+};
+EXPORT_SYMBOL_GPL(balloon_aops);
- return rc;
-}
#endif /* CONFIG_BALLOON_COMPACTION */
diff --git a/mm/compaction.c b/mm/compaction.c
index e9eeb8eb3b10..d7e2212064e4 100644
--- a/mm/compaction.c
+++ b/mm/compaction.c
@@ -757,13 +757,6 @@ isolate_migratepages_block(struct compact_control *cc, unsigned long low_pfn,
*/
is_lru = PageLRU(page);
if (!is_lru) {
- if (unlikely(balloon_page_movable(page))) {
- if (balloon_page_isolate(page)) {
- /* Successfully isolated */
- goto isolate_success;
- }
- }
-
/*
* __PageMovable can return false positive so we need
* to verify it under page_lock.
diff --git a/mm/migrate.c b/mm/migrate.c
index 24205e1f9204..5478ea1c809f 100644
--- a/mm/migrate.c
+++ b/mm/migrate.c
@@ -199,14 +199,12 @@ void putback_movable_pages(struct list_head *l)
list_del(&page->lru);
dec_zone_page_state(page, NR_ISOLATED_ANON +
page_is_file_cache(page));
- if (unlikely(isolated_balloon_page(page))) {
- balloon_page_putback(page);
/*
* We isolated non-lru movable page so here we can use
* __PageMovable because LRU page's mapping cannot have
* PAGE_MAPPING_MOVABLE.
*/
- } else if (unlikely(__PageMovable(page))) {
+ if (unlikely(__PageMovable(page))) {
VM_BUG_ON_PAGE(!PageIsolated(page), page);
lock_page(page);
if (PageMovable(page))
@@ -1016,18 +1014,6 @@ static int __unmap_and_move(struct page *page, struct page *newpage,
if (unlikely(!trylock_page(newpage)))
goto out_unlock;
- if (unlikely(isolated_balloon_page(page))) {
- /*
- * A ballooned page does not need any special attention from
- * physical to virtual reverse mapping procedures.
- * Skip any attempt to unmap PTEs or to remap swap cache,
- * in order to avoid burning cycles at rmap level, and perform
- * the page migration right away (proteced by page lock).
- */
- rc = balloon_page_migrate(newpage, page, mode);
- goto out_unlock_both;
- }
-
if (unlikely(!is_lru)) {
rc = move_to_new_page(newpage, page, mode);
goto out_unlock_both;
@@ -1082,8 +1068,7 @@ static int __unmap_and_move(struct page *page, struct page *newpage,
* list in here.
*/
if (rc == MIGRATEPAGE_SUCCESS) {
- if (unlikely(__is_movable_balloon_page(newpage) ||
- __PageMovable(newpage)))
+ if (unlikely(__PageMovable(newpage)))
put_page(newpage);
else
putback_lru_page(newpage);
diff --git a/mm/vmscan.c b/mm/vmscan.c
index c4a2f4512fca..93ba33789ac6 100644
--- a/mm/vmscan.c
+++ b/mm/vmscan.c
@@ -1254,7 +1254,7 @@ unsigned long reclaim_clean_pages_from_list(struct zone *zone,
list_for_each_entry_safe(page, next, page_list, lru) {
if (page_is_file_cache(page) && !PageDirty(page) &&
- !isolated_balloon_page(page)) {
+ !__PageMovable(page)) {
ClearPageActive(page);
list_move(&page->lru, &clean_pages);
}
--
1.9.1
^ permalink raw reply related
* [PATCH v5 02/12] mm: migrate: support non-lru movable page migration
From: Minchan Kim @ 2016-05-09 2:20 UTC (permalink / raw)
To: Andrew Morton
Cc: Rik van Riel, Sergey Senozhatsky, Rafael Aquini, Minchan Kim,
Jonathan Corbet, Hugh Dickins, linux-kernel, dri-devel,
virtualization, John Einar Reitan, linux-mm, Gioh Kim, Mel Gorman,
Joonsoo Kim, Vlastimil Babka
In-Reply-To: <1462760433-32357-1-git-send-email-minchan@kernel.org>
We have allowed migration for only LRU pages until now and it was
enough to make high-order pages. But recently, embedded system(e.g.,
webOS, android) uses lots of non-movable pages(e.g., zram, GPU memory)
so we have seen several reports about troubles of small high-order
allocation. For fixing the problem, there were several efforts
(e,g,. enhance compaction algorithm, SLUB fallback to 0-order page,
reserved memory, vmalloc and so on) but if there are lots of
non-movable pages in system, their solutions are void in the long run.
So, this patch is to support facility to change non-movable pages
with movable. For the feature, this patch introduces functions related
to migration to address_space_operations as well as some page flags.
If a driver want to make own pages movable, it should define three functions
which are function pointers of struct address_space_operations.
1. bool (*isolate_page) (struct page *page, isolate_mode_t mode);
What VM expects on isolate_page function of driver is to return *true*
if driver isolates page successfully. On returing true, VM marks the page
as PG_isolated so concurrent isolation in several CPUs skip the page
for isolation. If a driver cannot isolate the page, it should return *false*.
Once page is successfully isolated, VM uses page.lru fields so driver
shouldn't expect to preserve values in that fields.
2. int (*migratepage) (struct address_space *mapping,
struct page *newpage, struct page *oldpage, enum migrate_mode);
After isolation, VM calls migratepage of driver with isolated page.
The function of migratepage is to move content of the old page to new page
and set up fields of struct page newpage. Keep in mind that you should
clear PG_movable of oldpage via __ClearPageMovable under page_lock if you
migrated the oldpage successfully and returns MIGRATEPAGE_SUCCESS.
If driver cannot migrate the page at the moment, driver can return -EAGAIN.
On -EAGAIN, VM will retry page migration in a short time because VM interprets
-EAGAIN as "temporal migration failure". On returning any error except -EAGAIN,
VM will give up the page migration without retrying in this time.
Driver shouldn't touch page.lru field VM using in the functions.
3. void (*putback_page)(struct page *);
If migration fails on isolated page, VM should return the isolated page
to the driver so VM calls driver's putback_page with migration failed page.
In this function, driver should put the isolated page back to the own data
structure.
4. non-lru movable page flags
There are two page flags for supporting non-lru movable page.
* PG_movable
Driver should use the below function to make page movable under page_lock.
void __SetPageMovable(struct page *page, struct address_space *mapping)
It needs argument of address_space for registering migration family functions
which will be called by VM. Exactly speaking, PG_movable is not a real flag of
struct page. Rather than, VM reuses page->mapping's lower bits to represent it.
#define PAGE_MAPPING_MOVABLE 0x2
page->mapping = page->mapping | PAGE_MAPPING_MOVABLE;
so driver shouldn't access page->mapping directly. Instead, driver should
use page_mapping which mask off the low two bits of page->mapping so it can get
right struct address_space.
For testing of non-lru movable page, VM supports __PageMovable function.
However, it doesn't guarantee to identify non-lru movable page because
page->mapping field is unified with other variables in struct page.
As well, if driver releases the page after isolation by VM, page->mapping
doesn't have stable value although it has PAGE_MAPPING_MOVABLE
(Look at __ClearPageMovable). But __PageMovable is cheap to catch whether
page is LRU or non-lru movable once the page has been isolated. Because
LRU pages never can have PAGE_MAPPING_MOVABLE in page->mapping. It is also
good for just peeking to test non-lru movable pages before more expensive
checking with lock_page in pfn scanning to select victim.
For guaranteeing non-lru movable page, VM provides PageMovable function.
Unlike __PageMovable, PageMovable functions validates page->mapping and
mapping->a_ops->isolate_page under lock_page. The lock_page prevents sudden
destroying of page->mapping.
Driver using __SetPageMovable should clear the flag via __ClearMovablePage
under page_lock before the releasing the page.
* PG_isolated
To prevent concurrent isolation among several CPUs, VM marks isolated page
as PG_isolated under lock_page. So if a CPU encounters PG_isolated non-lru
movable page, it can skip it. Driver doesn't need to manipulate the flag
because VM will set/clear it automatically. Keep in mind that if driver
sees PG_isolated page, it means the page have been isolated by VM so it
shouldn't touch page.lru field.
PG_isolated is alias with PG_reclaim flag so driver shouldn't use the flag
for own purpose.
Cc: Rik van Riel <riel@redhat.com>
Cc: Vlastimil Babka <vbabka@suse.cz>
Cc: Joonsoo Kim <iamjoonsoo.kim@lge.com>
Cc: Mel Gorman <mgorman@suse.de>
Cc: Hugh Dickins <hughd@google.com>
Cc: Rafael Aquini <aquini@redhat.com>
Cc: virtualization@lists.linux-foundation.org
Cc: Jonathan Corbet <corbet@lwn.net>
Cc: John Einar Reitan <john.reitan@foss.arm.com>
Cc: dri-devel@lists.freedesktop.org
Cc: Sergey Senozhatsky <sergey.senozhatsky@gmail.com>
Signed-off-by: Gioh Kim <gi-oh.kim@profitbricks.com>
Signed-off-by: Minchan Kim <minchan@kernel.org>
---
Documentation/filesystems/Locking | 4 +
Documentation/filesystems/vfs.txt | 11 ++
Documentation/vm/page_migration | 107 +++++++++++++++++-
include/linux/fs.h | 2 +
include/linux/ksm.h | 3 +-
include/linux/migrate.h | 5 +
include/linux/mm.h | 1 +
include/linux/page-flags.h | 29 +++--
mm/compaction.c | 47 +++++---
mm/ksm.c | 4 +-
mm/migrate.c | 222 ++++++++++++++++++++++++++++++++++----
mm/page_alloc.c | 2 +-
mm/util.c | 6 +-
13 files changed, 391 insertions(+), 52 deletions(-)
diff --git a/Documentation/filesystems/Locking b/Documentation/filesystems/Locking
index 75eea7ce3d7c..dda6e3f8e203 100644
--- a/Documentation/filesystems/Locking
+++ b/Documentation/filesystems/Locking
@@ -195,7 +195,9 @@ unlocks and drops the reference.
int (*releasepage) (struct page *, int);
void (*freepage)(struct page *);
int (*direct_IO)(struct kiocb *, struct iov_iter *iter);
+ bool (*isolate_page) (struct page *, isolate_mode_t);
int (*migratepage)(struct address_space *, struct page *, struct page *);
+ void (*putback_page) (struct page *);
int (*launder_page)(struct page *);
int (*is_partially_uptodate)(struct page *, unsigned long, unsigned long);
int (*error_remove_page)(struct address_space *, struct page *);
@@ -219,7 +221,9 @@ invalidatepage: yes
releasepage: yes
freepage: yes
direct_IO:
+isolate_page: yes
migratepage: yes (both)
+putback_page: yes
launder_page: yes
is_partially_uptodate: yes
error_remove_page: yes
diff --git a/Documentation/filesystems/vfs.txt b/Documentation/filesystems/vfs.txt
index c61a223ef3ff..900360cbcdae 100644
--- a/Documentation/filesystems/vfs.txt
+++ b/Documentation/filesystems/vfs.txt
@@ -592,9 +592,14 @@ struct address_space_operations {
int (*releasepage) (struct page *, int);
void (*freepage)(struct page *);
ssize_t (*direct_IO)(struct kiocb *, struct iov_iter *iter);
+ /* isolate a page for migration */
+ bool (*isolate_page) (struct page *, isolate_mode_t);
/* migrate the contents of a page to the specified target */
int (*migratepage) (struct page *, struct page *);
+ /* put migration-failed page back to right list */
+ void (*putback_page) (struct page *);
int (*launder_page) (struct page *);
+
int (*is_partially_uptodate) (struct page *, unsigned long,
unsigned long);
void (*is_dirty_writeback) (struct page *, bool *, bool *);
@@ -747,6 +752,10 @@ struct address_space_operations {
and transfer data directly between the storage and the
application's address space.
+ isolate_page: Called by the VM when isolating a movable non-lru page.
+ If page is successfully isolated, VM marks the page as PG_isolated
+ via __SetPageIsolated.
+
migrate_page: This is used to compact the physical memory usage.
If the VM wants to relocate a page (maybe off a memory card
that is signalling imminent failure) it will pass a new page
@@ -754,6 +763,8 @@ struct address_space_operations {
transfer any private data across and update any references
that it has to the page.
+ putback_page: Called by the VM when isolated page's migration fails.
+
launder_page: Called before freeing a page - it writes back the dirty page. To
prevent redirtying the page, it is kept locked during the whole
operation.
diff --git a/Documentation/vm/page_migration b/Documentation/vm/page_migration
index fea5c0864170..b89d1de026df 100644
--- a/Documentation/vm/page_migration
+++ b/Documentation/vm/page_migration
@@ -142,5 +142,110 @@ is increased so that the page cannot be freed while page migration occurs.
20. The new page is moved to the LRU and can be scanned by the swapper
etc again.
-Christoph Lameter, May 8, 2006.
+C. Non-LRU page migration
+-------------------------
+
+Although original migration aimed for reducing the latency of memory access
+for NUMA, compaction who want to create high-order page is also main customer.
+
+Current problem of the implementation is that it is designed to migrate only
+*LRU* pages. However, there are potential non-lru pages which can be migrated
+in drivers, for example, zsmalloc, virtio-balloon pages.
+
+For virtio-balloon pages, some parts of migration code path have been hooked
+up and added virtio-balloon specific functions to intercept migration logics.
+It's too specific to a driver so other drivers who want to make their pages
+movable would have to add own specific hooks in migration path.
+
+To overclome the problem, VM supports non-LRU page migration which provides
+generic functions for non-LRU movable pages without driver specific hooks
+migration path.
+
+If a driver want to make own pages movable, it should define three functions
+which are function pointers of struct address_space_operations.
+
+1. bool (*isolate_page) (struct page *page, isolate_mode_t mode);
+
+What VM expects on isolate_page function of driver is to return *true*
+if driver isolates page successfully. On returing true, VM marks the page
+as PG_isolated so concurrent isolation in several CPUs skip the page
+for isolation. If a driver cannot isolate the page, it should return *false*.
+
+Once page is successfully isolated, VM uses page.lru fields so driver
+shouldn't expect to preserve values in that fields.
+
+2. int (*migratepage) (struct address_space *mapping,
+ struct page *newpage, struct page *oldpage, enum migrate_mode);
+
+After isolation, VM calls migratepage of driver with isolated page.
+The function of migratepage is to move content of the old page to new page
+and set up fields of struct page newpage. Keep in mind that you should
+clear PG_movable of oldpage via __ClearPageMovable under page_lock if you
+migrated the oldpage successfully and returns MIGRATEPAGE_SUCCESS.
+If driver cannot migrate the page at the moment, driver can return -EAGAIN.
+On -EAGAIN, VM will retry page migration in a short time because VM interprets
+-EAGAIN as "temporal migration failure". On returning any error except -EAGAIN,
+VM will give up the page migration without retrying in this time.
+
+Driver shouldn't touch page.lru field VM using in the functions.
+
+3. void (*putback_page)(struct page *);
+
+If migration fails on isolated page, VM should return the isolated page
+to the driver so VM calls driver's putback_page with migration failed page.
+In this function, driver should put the isolated page back to the own data
+structure.
+4. non-lru movable page flags
+
+There are two page flags for supporting non-lru movable page.
+
+* PG_movable
+
+Driver should use the below function to make page movable under page_lock.
+
+ void __SetPageMovable(struct page *page, struct address_space *mapping)
+
+It needs argument of address_space for registering migration family functions
+which will be called by VM. Exactly speaking, PG_movable is not a real flag of
+struct page. Rather than, VM reuses page->mapping's lower bits to represent it.
+
+ #define PAGE_MAPPING_MOVABLE 0x2
+ page->mapping = page->mapping | PAGE_MAPPING_MOVABLE;
+
+so driver shouldn't access page->mapping directly. Instead, driver should
+use page_mapping which mask off the low two bits of page->mapping so it can get
+right struct address_space.
+
+For testing of non-lru movable page, VM supports __PageMovable function.
+However, it doesn't guarantee to identify non-lru movable page because
+page->mapping field is unified with other variables in struct page.
+As well, if driver releases the page after isolation by VM, page->mapping
+doesn't have stable value although it has PAGE_MAPPING_MOVABLE
+(Look at __ClearPageMovable). But __PageMovable is cheap to catch whether
+page is LRU or non-lru movable once the page has been isolated. Because
+LRU pages never can have PAGE_MAPPING_MOVABLE in page->mapping. It is also
+good for just peeking to test non-lru movable pages before more expensive
+checking with lock_page in pfn scanning to select victim.
+
+For guaranteeing non-lru movable page, VM provides PageMovable function.
+Unlike __PageMovable, PageMovable functions validates page->mapping and
+mapping->a_ops->isolate_page under lock_page. The lock_page prevents sudden
+destroying of page->mapping.
+
+Driver using __SetPageMovable should clear the flag via __ClearMovablePage
+under page_lock before the releasing the page.
+
+* PG_isolated
+
+To prevent concurrent isolation among several CPUs, VM marks isolated page
+as PG_isolated under lock_page. So if a CPU encounters PG_isolated non-lru
+movable page, it can skip it. Driver doesn't need to manipulate the flag
+because VM will set/clear it automatically. Keep in mind that if driver
+sees PG_isolated page, it means the page have been isolated by VM so it
+shouldn't touch page.lru field.
+PG_isolated is alias with PG_reclaim flag so driver shouldn't use the flag
+for own purpose.
+
+Christoph Lameter, May 8, 2006.
+Minchan Kim, Mar 28, 2016.
diff --git a/include/linux/fs.h b/include/linux/fs.h
index d61cd084987e..92029377b9e0 100644
--- a/include/linux/fs.h
+++ b/include/linux/fs.h
@@ -403,6 +403,8 @@ struct address_space_operations {
*/
int (*migratepage) (struct address_space *,
struct page *, struct page *, enum migrate_mode);
+ bool (*isolate_page)(struct page *, isolate_mode_t);
+ void (*putback_page)(struct page *);
int (*launder_page) (struct page *);
int (*is_partially_uptodate) (struct page *, unsigned long,
unsigned long);
diff --git a/include/linux/ksm.h b/include/linux/ksm.h
index 7ae216a39c9e..481c8c4627ca 100644
--- a/include/linux/ksm.h
+++ b/include/linux/ksm.h
@@ -43,8 +43,7 @@ static inline struct stable_node *page_stable_node(struct page *page)
static inline void set_page_stable_node(struct page *page,
struct stable_node *stable_node)
{
- page->mapping = (void *)stable_node +
- (PAGE_MAPPING_ANON | PAGE_MAPPING_KSM);
+ page->mapping = (void *)((unsigned long)stable_node | PAGE_MAPPING_KSM);
}
/*
diff --git a/include/linux/migrate.h b/include/linux/migrate.h
index 9b50325e4ddf..31a748f43fa9 100644
--- a/include/linux/migrate.h
+++ b/include/linux/migrate.h
@@ -32,11 +32,16 @@ extern char *migrate_reason_names[MR_TYPES];
#ifdef CONFIG_MIGRATION
+extern int PageMovable(struct page *page);
+extern void __SetPageMovable(struct page *page, struct address_space *mapping);
+extern void __ClearPageMovable(struct page *page);
extern void putback_movable_pages(struct list_head *l);
extern int migrate_page(struct address_space *,
struct page *, struct page *, enum migrate_mode);
extern int migrate_pages(struct list_head *l, new_page_t new, free_page_t free,
unsigned long private, enum migrate_mode mode, int reason);
+extern bool isolate_movable_page(struct page *page, isolate_mode_t mode);
+extern void putback_movable_page(struct page *page);
extern int migrate_prep(void);
extern int migrate_prep_local(void);
diff --git a/include/linux/mm.h b/include/linux/mm.h
index 7b52750caf9e..91e9ebc44e37 100644
--- a/include/linux/mm.h
+++ b/include/linux/mm.h
@@ -1021,6 +1021,7 @@ static inline pgoff_t page_file_index(struct page *page)
}
bool page_mapped(struct page *page);
+struct address_space *page_mapping(struct page *page);
/*
* Return true only if the page has been allocated with
diff --git a/include/linux/page-flags.h b/include/linux/page-flags.h
index e5a32445f930..ccb46bc97f81 100644
--- a/include/linux/page-flags.h
+++ b/include/linux/page-flags.h
@@ -129,6 +129,9 @@ enum pageflags {
/* Compound pages. Stored in first tail page's flags */
PG_double_map = PG_private_2,
+
+ /* non-lru isolated movable page */
+ PG_isolated = PG_reclaim,
};
#ifndef __GENERATING_BOUNDS_H
@@ -361,25 +364,33 @@ PAGEFLAG(Idle, idle, PF_ANY)
* and then page->mapping points, not to an anon_vma, but to a private
* structure which KSM associates with that merged page. See ksm.h.
*
- * PAGE_MAPPING_KSM without PAGE_MAPPING_ANON is currently never used.
+ * PAGE_MAPPING_KSM without PAGE_MAPPING_ANON is used for non-lru movable
+ * page and then page->mapping points a struct address_space.
*
* Please note that, confusingly, "page_mapping" refers to the inode
* address_space which maps the page from disk; whereas "page_mapped"
* refers to user virtual address space into which the page is mapped.
*/
-#define PAGE_MAPPING_ANON 1
-#define PAGE_MAPPING_KSM 2
-#define PAGE_MAPPING_FLAGS (PAGE_MAPPING_ANON | PAGE_MAPPING_KSM)
+#define PAGE_MAPPING_ANON 0x1
+#define PAGE_MAPPING_MOVABLE 0x2
+#define PAGE_MAPPING_KSM (PAGE_MAPPING_ANON | PAGE_MAPPING_MOVABLE)
+#define PAGE_MAPPING_FLAGS (PAGE_MAPPING_ANON | PAGE_MAPPING_MOVABLE)
-static __always_inline int PageAnonHead(struct page *page)
+static __always_inline int PageMappingFlag(struct page *page)
{
- return ((unsigned long)page->mapping & PAGE_MAPPING_ANON) != 0;
+ return ((unsigned long)page->mapping & PAGE_MAPPING_FLAGS) != 0;
}
static __always_inline int PageAnon(struct page *page)
{
page = compound_head(page);
- return PageAnonHead(page);
+ return ((unsigned long)page->mapping & PAGE_MAPPING_ANON) != 0;
+}
+
+static __always_inline int __PageMovable(struct page *page)
+{
+ return ((unsigned long)page->mapping & PAGE_MAPPING_FLAGS) ==
+ PAGE_MAPPING_MOVABLE;
}
#ifdef CONFIG_KSM
@@ -393,7 +404,7 @@ static __always_inline int PageKsm(struct page *page)
{
page = compound_head(page);
return ((unsigned long)page->mapping & PAGE_MAPPING_FLAGS) ==
- (PAGE_MAPPING_ANON | PAGE_MAPPING_KSM);
+ PAGE_MAPPING_KSM;
}
#else
TESTPAGEFLAG_FALSE(Ksm)
@@ -641,6 +652,8 @@ static inline void __ClearPageBalloon(struct page *page)
atomic_set(&page->_mapcount, -1);
}
+__PAGEFLAG(Isolated, isolated, PF_ANY);
+
/*
* If network-based swap is enabled, sl*b must keep track of whether pages
* were allocated from pfmemalloc reserves.
diff --git a/mm/compaction.c b/mm/compaction.c
index d8a20fcf8678..e9eeb8eb3b10 100644
--- a/mm/compaction.c
+++ b/mm/compaction.c
@@ -735,21 +735,6 @@ isolate_migratepages_block(struct compact_control *cc, unsigned long low_pfn,
}
/*
- * Check may be lockless but that's ok as we recheck later.
- * It's possible to migrate LRU pages and balloon pages
- * Skip any other type of page
- */
- is_lru = PageLRU(page);
- if (!is_lru) {
- if (unlikely(balloon_page_movable(page))) {
- if (balloon_page_isolate(page)) {
- /* Successfully isolated */
- goto isolate_success;
- }
- }
- }
-
- /*
* Regardless of being on LRU, compound pages such as THP and
* hugetlbfs are not to be compacted. We can potentially save
* a lot of iterations if we skip them at once. The check is
@@ -765,8 +750,38 @@ isolate_migratepages_block(struct compact_control *cc, unsigned long low_pfn,
goto isolate_fail;
}
- if (!is_lru)
+ /*
+ * Check may be lockless but that's ok as we recheck later.
+ * It's possible to migrate LRU and non-lru movable pages.
+ * Skip any other type of page
+ */
+ is_lru = PageLRU(page);
+ if (!is_lru) {
+ if (unlikely(balloon_page_movable(page))) {
+ if (balloon_page_isolate(page)) {
+ /* Successfully isolated */
+ goto isolate_success;
+ }
+ }
+
+ /*
+ * __PageMovable can return false positive so we need
+ * to verify it under page_lock.
+ */
+ if (unlikely(__PageMovable(page)) &&
+ !PageIsolated(page)) {
+ if (locked) {
+ spin_unlock_irqrestore(&zone->lru_lock,
+ flags);
+ locked = false;
+ }
+
+ if (isolate_movable_page(page, isolate_mode))
+ goto isolate_success;
+ }
+
goto isolate_fail;
+ }
/*
* Migration will fail if an anonymous page is pinned in memory,
diff --git a/mm/ksm.c b/mm/ksm.c
index 53bf3ba176ec..60bd202dd9f4 100644
--- a/mm/ksm.c
+++ b/mm/ksm.c
@@ -532,8 +532,8 @@ static struct page *get_ksm_page(struct stable_node *stable_node, bool lock_it)
void *expected_mapping;
unsigned long kpfn;
- expected_mapping = (void *)stable_node +
- (PAGE_MAPPING_ANON | PAGE_MAPPING_KSM);
+ expected_mapping = (void *)((unsigned long)stable_node |
+ PAGE_MAPPING_KSM);
again:
kpfn = READ_ONCE(stable_node->kpfn);
page = pfn_to_page(kpfn);
diff --git a/mm/migrate.c b/mm/migrate.c
index f2932498ded2..24205e1f9204 100644
--- a/mm/migrate.c
+++ b/mm/migrate.c
@@ -47,6 +47,38 @@
#include "internal.h"
+int PageMovable(struct page *page)
+{
+ struct address_space *mapping;
+
+ WARN_ON(!PageLocked(page));
+ if (!__PageMovable(page))
+ goto out;
+
+ mapping = page_mapping(page);
+ if (mapping && mapping->a_ops && mapping->a_ops->isolate_page)
+ return 1;
+out:
+ return 0;
+}
+
+void __SetPageMovable(struct page *page, struct address_space *mapping)
+{
+ VM_BUG_ON_PAGE(!PageLocked(page), page);
+ VM_BUG_ON_PAGE((unsigned long)mapping & PAGE_MAPPING_MOVABLE, page);
+ page->mapping = (void *)((unsigned long)mapping | PAGE_MAPPING_MOVABLE);
+}
+
+void __ClearPageMovable(struct page *page)
+{
+ VM_BUG_ON_PAGE(!PageLocked(page), page);
+ VM_BUG_ON_PAGE(!PageMovable(page), page);
+ VM_BUG_ON_PAGE(!((unsigned long)page->mapping & PAGE_MAPPING_MOVABLE),
+ page);
+ page->mapping = (void *)((unsigned long)page->mapping &
+ PAGE_MAPPING_MOVABLE);
+}
+
/*
* migrate_prep() needs to be called before we start compiling a list of pages
* to be migrated using isolate_lru_page(). If scheduling work on other CPUs is
@@ -73,6 +105,79 @@ int migrate_prep_local(void)
return 0;
}
+bool isolate_movable_page(struct page *page, isolate_mode_t mode)
+{
+ struct address_space *mapping;
+
+ /*
+ * Avoid burning cycles with pages that are yet under __free_pages(),
+ * or just got freed under us.
+ *
+ * In case we 'win' a race for a movable page being freed under us and
+ * raise its refcount preventing __free_pages() from doing its job
+ * the put_page() at the end of this block will take care of
+ * release this page, thus avoiding a nasty leakage.
+ */
+ if (unlikely(!get_page_unless_zero(page)))
+ goto out;
+
+ /*
+ * Check PageMovable before holding a PG_lock because page's owner
+ * assumes anybody doesn't touch PG_lock of newly allocated page
+ * so unconditionally grapping the lock ruins page's owner side.
+ */
+ if (unlikely(!__PageMovable(page)))
+ goto out_putpage;
+ /*
+ * As movable pages are not isolated from LRU lists, concurrent
+ * compaction threads can race against page migration functions
+ * as well as race against the releasing a page.
+ *
+ * In order to avoid having an already isolated movable page
+ * being (wrongly) re-isolated while it is under migration,
+ * or to avoid attempting to isolate pages being released,
+ * lets be sure we have the page lock
+ * before proceeding with the movable page isolation steps.
+ */
+ if (unlikely(!trylock_page(page)))
+ goto out_putpage;
+
+ if (!PageMovable(page) || PageIsolated(page))
+ goto out_no_isolated;
+
+ mapping = page_mapping(page);
+ if (!mapping->a_ops->isolate_page(page, mode))
+ goto out_no_isolated;
+
+ /* Driver shouldn't use PG_isolated bit of page->flags */
+ WARN_ON_ONCE(PageIsolated(page));
+ __SetPageIsolated(page);
+ unlock_page(page);
+
+ return true;
+
+out_no_isolated:
+ unlock_page(page);
+out_putpage:
+ put_page(page);
+out:
+ return false;
+}
+
+/* It should be called on page which is PG_movable */
+void putback_movable_page(struct page *page)
+{
+ struct address_space *mapping;
+
+ VM_BUG_ON_PAGE(!PageLocked(page), page);
+ VM_BUG_ON_PAGE(!PageMovable(page), page);
+ VM_BUG_ON_PAGE(!PageIsolated(page), page);
+
+ mapping = page_mapping(page);
+ mapping->a_ops->putback_page(page);
+ __ClearPageIsolated(page);
+}
+
/*
* Put previously isolated pages back onto the appropriate lists
* from where they were once taken off for compaction/migration.
@@ -94,10 +199,25 @@ void putback_movable_pages(struct list_head *l)
list_del(&page->lru);
dec_zone_page_state(page, NR_ISOLATED_ANON +
page_is_file_cache(page));
- if (unlikely(isolated_balloon_page(page)))
+ if (unlikely(isolated_balloon_page(page))) {
balloon_page_putback(page);
- else
+ /*
+ * We isolated non-lru movable page so here we can use
+ * __PageMovable because LRU page's mapping cannot have
+ * PAGE_MAPPING_MOVABLE.
+ */
+ } else if (unlikely(__PageMovable(page))) {
+ VM_BUG_ON_PAGE(!PageIsolated(page), page);
+ lock_page(page);
+ if (PageMovable(page))
+ putback_movable_page(page);
+ else
+ __ClearPageIsolated(page);
+ unlock_page(page);
+ put_page(page);
+ } else {
putback_lru_page(page);
+ }
}
}
@@ -592,7 +712,7 @@ void migrate_page_copy(struct page *newpage, struct page *page)
***********************************************************/
/*
- * Common logic to directly migrate a single page suitable for
+ * Common logic to directly migrate a single LRU page suitable for
* pages that do not use PagePrivate/PagePrivate2.
*
* Pages are locked upon entry and exit.
@@ -755,33 +875,69 @@ static int move_to_new_page(struct page *newpage, struct page *page,
enum migrate_mode mode)
{
struct address_space *mapping;
- int rc;
+ int rc = -EAGAIN;
+ bool is_lru = !__PageMovable(page);
VM_BUG_ON_PAGE(!PageLocked(page), page);
VM_BUG_ON_PAGE(!PageLocked(newpage), newpage);
mapping = page_mapping(page);
- if (!mapping)
- rc = migrate_page(mapping, newpage, page, mode);
- else if (mapping->a_ops->migratepage)
- /*
- * Most pages have a mapping and most filesystems provide a
- * migratepage callback. Anonymous pages are part of swap
- * space which also has its own migratepage callback. This
- * is the most common path for page migration.
- */
- rc = mapping->a_ops->migratepage(mapping, newpage, page, mode);
- else
- rc = fallback_migrate_page(mapping, newpage, page, mode);
+ /*
+ * In case of non-lru page, it could be released after
+ * isolation step. In that case, we shouldn't try
+ * fallback migration which is designed for LRU pages.
+ */
+ if (unlikely(!is_lru)) {
+ VM_BUG_ON_PAGE(!PageIsolated(page), page);
+ if (!PageMovable(page)) {
+ rc = MIGRATEPAGE_SUCCESS;
+ __ClearPageIsolated(page);
+ goto out;
+ }
+ }
+
+ if (likely(is_lru)) {
+ if (!mapping)
+ rc = migrate_page(mapping, newpage, page, mode);
+ else if (mapping->a_ops->migratepage)
+ /*
+ * Most pages have a mapping and most filesystems
+ * provide a migratepage callback. Anonymous pages
+ * are part of swap space which also has its own
+ * migratepage callback. This is the most common path
+ * for page migration.
+ */
+ rc = mapping->a_ops->migratepage(mapping, newpage,
+ page, mode);
+ else
+ rc = fallback_migrate_page(mapping, newpage,
+ page, mode);
+ } else {
+ rc = mapping->a_ops->migratepage(mapping, newpage,
+ page, mode);
+ WARN_ON_ONCE(rc == MIGRATEPAGE_SUCCESS &&
+ !PageIsolated(page));
+ }
/*
* When successful, old pagecache page->mapping must be cleared before
* page is freed; but stats require that PageAnon be left as PageAnon.
*/
if (rc == MIGRATEPAGE_SUCCESS) {
- if (!PageAnon(page))
+ if (__PageMovable(page)) {
+ VM_BUG_ON_PAGE(!PageIsolated(page), page);
+
+ /*
+ * We clear PG_movable under page_lock so any compactor
+ * cannot try to migrate this page.
+ */
+ __ClearPageIsolated(page);
+ }
+
+ if (!((unsigned long)page->mapping & PAGE_MAPPING_FLAGS))
page->mapping = NULL;
}
+out:
return rc;
}
@@ -791,6 +947,7 @@ static int __unmap_and_move(struct page *page, struct page *newpage,
int rc = -EAGAIN;
int page_was_mapped = 0;
struct anon_vma *anon_vma = NULL;
+ bool is_lru = !__PageMovable(page);
if (!trylock_page(page)) {
if (!force || mode == MIGRATE_ASYNC)
@@ -871,6 +1028,11 @@ static int __unmap_and_move(struct page *page, struct page *newpage,
goto out_unlock_both;
}
+ if (unlikely(!is_lru)) {
+ rc = move_to_new_page(newpage, page, mode);
+ goto out_unlock_both;
+ }
+
/*
* Corner case handling:
* 1. When a new swap-cache page is read into, it is added to the LRU
@@ -920,7 +1082,8 @@ static int __unmap_and_move(struct page *page, struct page *newpage,
* list in here.
*/
if (rc == MIGRATEPAGE_SUCCESS) {
- if (unlikely(__is_movable_balloon_page(newpage)))
+ if (unlikely(__is_movable_balloon_page(newpage) ||
+ __PageMovable(newpage)))
put_page(newpage);
else
putback_lru_page(newpage);
@@ -961,6 +1124,12 @@ static ICE_noinline int unmap_and_move(new_page_t get_new_page,
/* page was freed from under us. So we are done. */
ClearPageActive(page);
ClearPageUnevictable(page);
+ if (unlikely(__PageMovable(page))) {
+ lock_page(page);
+ if (!PageMovable(page))
+ __ClearPageIsolated(page);
+ unlock_page(page);
+ }
if (put_new_page)
put_new_page(newpage, private);
else
@@ -1010,8 +1179,21 @@ static ICE_noinline int unmap_and_move(new_page_t get_new_page,
num_poisoned_pages_inc();
}
} else {
- if (rc != -EAGAIN)
- putback_lru_page(page);
+ if (rc != -EAGAIN) {
+ if (likely(!__PageMovable(page))) {
+ putback_lru_page(page);
+ goto put_new;
+ }
+
+ lock_page(page);
+ if (PageMovable(page))
+ putback_movable_page(page);
+ else
+ __ClearPageIsolated(page);
+ unlock_page(page);
+ put_page(page);
+ }
+put_new:
if (put_new_page)
put_new_page(newpage, private);
else
diff --git a/mm/page_alloc.c b/mm/page_alloc.c
index 6663d1736bce..c37472f66566 100644
--- a/mm/page_alloc.c
+++ b/mm/page_alloc.c
@@ -1015,7 +1015,7 @@ static __always_inline bool free_pages_prepare(struct page *page,
(page + i)->flags &= ~PAGE_FLAGS_CHECK_AT_PREP;
}
}
- if (PageAnonHead(page))
+ if (PageMappingFlag(page))
page->mapping = NULL;
if (check_free)
bad += free_pages_check(page);
diff --git a/mm/util.c b/mm/util.c
index 917e0e3d0f8e..b756ee36f7f0 100644
--- a/mm/util.c
+++ b/mm/util.c
@@ -399,10 +399,12 @@ struct address_space *page_mapping(struct page *page)
}
mapping = page->mapping;
- if ((unsigned long)mapping & PAGE_MAPPING_FLAGS)
+ if ((unsigned long)mapping & PAGE_MAPPING_ANON)
return NULL;
- return mapping;
+
+ return (void *)((unsigned long)mapping & ~PAGE_MAPPING_FLAGS);
}
+EXPORT_SYMBOL(page_mapping);
/* Slow path of page_mapcount() for compound pages */
int __page_mapcount(struct page *page)
--
1.9.1
^ permalink raw reply related
* [PATCH v5 00/13] Support non-lru page migration
From: Minchan Kim @ 2016-05-09 2:20 UTC (permalink / raw)
To: Andrew Morton
Cc: Rik van Riel, Sergey Senozhatsky, Rafael Aquini, Minchan Kim,
Chan Gyun Jeong, Jonathan Corbet, Hugh Dickins, linux-kernel,
dri-devel, virtualization, John Einar Reitan, linux-mm,
Chulmin Kim, Gioh Kim, Konstantin Khlebnikov, Sangseok Lee,
Joonsoo Kim, Kyeongdon Kim, Naoya Horiguchi, Vlastimil Babka,
Mel Gorman
Recently, I got many reports about perfermance degradation in embedded
system(Android mobile phone, webOS TV and so on) and easy fork fail.
The problem was fragmentation caused by zram and GPU driver mainly.
With memory pressure, their pages were spread out all of pageblock and
it cannot be migrated with current compaction algorithm which supports
only LRU pages. In the end, compaction cannot work well so reclaimer
shrinks all of working set pages. It made system very slow and even to
fail to fork easily which requires order-[2 or 3] allocations.
Other pain point is that they cannot use CMA memory space so when OOM
kill happens, I can see many free pages in CMA area, which is not
memory efficient. In our product which has big CMA memory, it reclaims
zones too exccessively to allocate GPU and zram page although there are
lots of free space in CMA so system becomes very slow easily.
To solve these problem, this patch tries to add facility to migrate
non-lru pages via introducing new functions and page flags to help
migration.
struct address_space_operations {
..
..
bool (*isolate_page)(struct page *, isolate_mode_t);
void (*putback_page)(struct page *);
..
}
new page flags
PG_movable
PG_isolated
For details, please read description in "mm: migrate: support non-lru
movable page migration".
Originally, Gioh Kim had tried to support this feature but he moved so
I took over the work. I took many code from his work and changed a little
bit and Konstantin Khlebnikov helped Gioh a lot so he should deserve to have
many credit, too.
And I should mention Chulmin who have tested this patchset heavily
so I can find many bugs from him. :)
Thanks, Gioh, Konstantin and Chulmin!
This patchset consists of five parts.
1. clean up migration
mm: use put_page to free page instead of putback_lru_page
2. add non-lru page migration feature
mm: migrate: support non-lru movable page migration
3. rework KVM memory-ballooning
mm: balloon: use general non-lru movable page feature
4. zsmalloc refactoring for preparing page migration
zsmalloc: keep max_object in size_class
zsmalloc: use bit_spin_lock
zsmalloc: use accessor
zsmalloc: factor page chain functionality out
zsmalloc: introduce zspage structure
zsmalloc: separate free_zspage from putback_zspage
zsmalloc: use freeobj for index
5. zsmalloc page migration
zsmalloc: page migration support
zram: use __GFP_MOVABLE for memory allocation
* From v4
* rebase on mmotm-2016-05-05-17-19
* fix huge object migration - Chulmin
* !CONFIG_COMPACTION support for zsmalloc
* From v3
* rebase on mmotm-2016-04-06-20-40
* fix swap_info deadlock - Chulmin
* race without page_lock - Vlastimil
* no use page._mapcount for potential user-mapped page driver - Vlastimil
* fix and enhance doc/description - Vlastimil
* use page->mapping lower bits to represent PG_movable
* make driver side's rule simple.
* From v2
* rebase on mmotm-2016-03-29-15-54-16
* check PageMovable before lock_page - Joonsoo
* check PageMovable before PageIsolated checking - Joonsoo
* add more description about rule
* From v1
* rebase on v4.5-mmotm-2016-03-17-15-04
* reordering patches to merge clean-up patches first
* add Acked-by/Reviewed-by from Vlastimil and Sergey
* use each own mount model instead of reusing anon_inode_fs - Al Viro
* small changes - YiPing, Gioh
Cc: Vlastimil Babka <vbabka@suse.cz>
Cc: dri-devel@lists.freedesktop.org
Cc: Hugh Dickins <hughd@google.com>
Cc: John Einar Reitan <john.reitan@foss.arm.com>
Cc: Jonathan Corbet <corbet@lwn.net>
Cc: Joonsoo Kim <iamjoonsoo.kim@lge.com>
Cc: Konstantin Khlebnikov <koct9i@gmail.com>
Cc: Mel Gorman <mgorman@suse.de>
Cc: Naoya Horiguchi <n-horiguchi@ah.jp.nec.com>
Cc: Rafael Aquini <aquini@redhat.com>
Cc: Rik van Riel <riel@redhat.com>
Cc: Sergey Senozhatsky <sergey.senozhatsky@gmail.com>
Cc: virtualization@lists.linux-foundation.org
Cc: Gioh Kim <gi-oh.kim@profitbricks.com>
Cc: Chan Gyun Jeong <chan.jeong@lge.com>
Cc: Sangseok Lee <sangseok.lee@lge.com>
Cc: Kyeongdon Kim <kyeongdon.kim@lge.com>
Cc: Chulmin Kim <cmlaika.kim@samsung.com>
Minchan Kim (12):
mm: use put_page to free page instead of putback_lru_page
mm: migrate: support non-lru movable page migration
mm: balloon: use general non-lru movable page feature
zsmalloc: keep max_object in size_class
zsmalloc: use bit_spin_lock
zsmalloc: use accessor
zsmalloc: factor page chain functionality out
zsmalloc: introduce zspage structure
zsmalloc: separate free_zspage from putback_zspage
zsmalloc: use freeobj for index
zsmalloc: page migration support
zram: use __GFP_MOVABLE for memory allocation
Documentation/filesystems/Locking | 4 +
Documentation/filesystems/vfs.txt | 11 +
Documentation/vm/page_migration | 107 ++-
drivers/block/zram/zram_drv.c | 6 +-
drivers/virtio/virtio_balloon.c | 52 +-
include/linux/balloon_compaction.h | 51 +-
include/linux/fs.h | 2 +
include/linux/ksm.h | 3 +-
include/linux/migrate.h | 5 +
include/linux/mm.h | 1 +
include/linux/page-flags.h | 29 +-
include/uapi/linux/magic.h | 2 +
mm/balloon_compaction.c | 94 +--
mm/compaction.c | 40 +-
mm/ksm.c | 4 +-
mm/migrate.c | 287 ++++++--
mm/page_alloc.c | 2 +-
mm/util.c | 6 +-
mm/vmscan.c | 2 +-
mm/zsmalloc.c | 1351 +++++++++++++++++++++++++-----------
20 files changed, 1453 insertions(+), 606 deletions(-)
--
1.9.1
^ permalink raw reply
page: next (older) | prev (newer) | latest
- recent:[subjects (threaded)|topics (new)|topics (active)]
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox