* Re: [PATCH] Align VM locks, new spinlock patch
@ 2001-09-21 13:15 Juergen Doelle
2001-09-22 23:07 ` Andrew Morton
2001-09-24 14:02 ` Stephen C. Tweedie
0 siblings, 2 replies; 5+ messages in thread
From: Juergen Doelle @ 2001-09-21 13:15 UTC (permalink / raw)
To: Andrea Arcangeli
Cc: Andrew Morton, linux-kernel, Mark Hemment, lse-tech, Steve Fox
Andrea Arcangeli wrote:
> On Thu, Aug 16, 2001 at 12:14:23PM -0700, Andrew Morton wrote:
> > Problem with this approach is that it doesn't prevent the linker
> > from placing other data in the same cacheline as the aligned
> > lock, at higher addresses.
>
> that was partly intentional, but ok we can be more aggressive on that
> side ;).
>
> > Juergen, I'd suggest you dust off that patch, add the conditionals
> > which make it a no-op on uniprocessor and submit it. It's such a
>
> agreed, btw it is just a noop on up but it is undefined for __GNUC__ >
> 2, also it would be nice if he could do it in linux/ instead of asm/, it
> should not need special arch trick (spinlock_t and SMP_CACHE_SIZE are
> the only thing it needs).
Sorry, it tooks some time, because of my vacation and then the disk drive
I used for testing gets damaged.
I changed the spinlock patch according to your suggestions.
Description
-----------
The patch introduces a new type spinlock_cacheline_t which
o aligns the spinlock to a cacheline and
o avoids that this spinlock shares the cacheline with other data
This type is now hardware independent.
It declares following spinlocks from type spinlock_cacheline_t:
o kmap_lock
o pagecache_lock
o pagemap_lru_lock
o lru_list_lock
o kernel_flag
Where the kernel_flag is modified only for i386 architecture, because
the spinlock itself is hardware dependent implemented.
I tested the patch on 2.4.5, 2.4.8, and 2.4.10-pre7.
Kernel 2.4.10-pre7 requires a separate version of the patch, because
the pagecache_lock and the lru_list_lock are already aligned.
The peak throughput results with dbench:
2.4.10 2.4.10 + improvement
spinlock patch by patch
U 103.77 102.14 - 1.6%
1 96.82 96.77 - 0.1%
2 155.32 155.62 0.2%
4 209.45 222.11 6.0%
8 208.06 234.82 12.9%
The improvement is less than in previous posted results, because the
pagemap_lru_lock and the lru_list_lock are already cacheline aligned
in 2.4.10 (2.4.9).
I attached two versions of the patch, one which applies for 2.4.x-2.4.8
and below a version for 2.4.10 (heading: Spinlock patch for kernel 2.4.10),
which applies also to 2.4.9, but I didn't test it.
Juergen
______________________________________________________________
Juergen Doelle
IBM Linux Technology Center - kernel performance
jdoelle@de.ibm.com
= = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = =
Spinlock patch for kernel 2.4.x-2.4.8
= = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = =
--- linux/include/asm-i386/smplock.h.orig Thu Sep 6 14:46:25 2001
+++ linux/include/asm-i386/smplock.h Thu Sep 6 16:38:02 2001
@@ -8,7 +8,13 @@
#include <linux/sched.h>
#include <asm/current.h>
-extern spinlock_t kernel_flag;
+
+#ifdef CONFIG_SMP
+ extern spinlock_cacheline_t kernel_flag_cacheline;
+ #define kernel_flag kernel_flag_cacheline.lock
+#else
+ extern spinlock_t kernel_flag;
+#endif
#define kernel_locked() spin_is_locked(&kernel_flag)
--- linux/arch/i386/kernel/i386_ksyms.c.orig Thu Sep 6 16:00:29 2001
+++ linux/arch/i386/kernel/i386_ksyms.c Thu Sep 6 16:03:07 2001
@@ -120,7 +120,7 @@
#ifdef CONFIG_SMP
EXPORT_SYMBOL(cpu_data);
-EXPORT_SYMBOL(kernel_flag);
+EXPORT_SYMBOL(kernel_flag_cacheline);
EXPORT_SYMBOL(smp_num_cpus);
EXPORT_SYMBOL(cpu_online_map);
EXPORT_SYMBOL_NOVERS(__write_lock_failed);
--- linux/include/linux/spinlock.h.orig Thu Sep 6 14:30:41 2001
+++ linux/include/linux/spinlock.h Thu Sep 6 16:38:02 2001
@@ -34,6 +34,13 @@
#ifdef CONFIG_SMP
#include <asm/spinlock.h>
+#include <linux/cache.h>
+typedef union {
+ spinlock_t lock;
+ char fill_up[(SMP_CACHE_BYTES)];
+} spinlock_cacheline_t __attribute__ ((aligned(SMP_CACHE_BYTES)));
+
+
#elif !defined(spin_lock_init) /* !SMP and spin_lock_init not previously
defined (e.g. by including asm/spinlock.h */
--- linux/include/linux/swap.h.orig Thu Sep 6 14:38:01 2001
+++ linux/include/linux/swap.h Thu Sep 6 16:38:02 2001
@@ -88,7 +88,13 @@
extern struct address_space swapper_space;
extern atomic_t page_cache_size;
extern atomic_t buffermem_pages;
-extern spinlock_t pagecache_lock;
+#ifdef CONFIG_SMP
+ extern spinlock_cacheline_t pagecache_lock_cacheline;
+ #define pagecache_lock (pagecache_lock_cacheline.lock)
+#else
+ extern spinlock_t pagecache_lock;
+#endif
+
extern void __remove_inode_page(struct page *);
/* Incomplete types for prototype declarations: */
@@ -179,7 +185,12 @@
extern unsigned long swap_cache_find_success;
#endif
-extern spinlock_t pagemap_lru_lock;
+#ifdef CONFIG_SMP
+ extern spinlock_cacheline_t pagemap_lru_lock_cacheline;
+ #define pagemap_lru_lock pagemap_lru_lock_cacheline.lock
+#else
+ extern spinlock_t pagemap_lru_lock;
+#endif
/*
* Page aging defines.
--- linux/fs/buffer.c.orig Thu Sep 6 15:10:07 2001
+++ linux/fs/buffer.c Thu Sep 6 15:54:00 2001
@@ -83,13 +83,22 @@
static rwlock_t hash_table_lock = RW_LOCK_UNLOCKED;
static struct buffer_head *lru_list[NR_LIST];
-static spinlock_t lru_list_lock = SPIN_LOCK_UNLOCKED;
+
+#ifdef CONFIG_SMP
+ static spinlock_cacheline_t lru_list_lock_cacheline = {SPIN_LOCK_UNLOCKED};
+ #ifndef lru_list_lock
+ #define lru_list_lock lru_list_lock_cacheline.lock
+ #endif
+#else
+ static spinlock_t lru_list_lock = SPIN_LOCK_UNLOCKED;
+#endif
static int nr_buffers_type[NR_LIST];
static unsigned long size_buffers_type[NR_LIST];
static struct buffer_head * unused_list;
static int nr_unused_buffer_heads;
static spinlock_t unused_list_lock = SPIN_LOCK_UNLOCKED;
+
static DECLARE_WAIT_QUEUE_HEAD(buffer_wait);
struct bh_free_head {
--- linux/mm/filemap.c.orig Thu Sep 6 14:55:44 2001
+++ linux/mm/filemap.c Thu Sep 6 15:06:21 2001
@@ -45,12 +45,28 @@
unsigned int page_hash_bits;
struct page **page_hash_table;
-spinlock_t pagecache_lock = SPIN_LOCK_UNLOCKED;
+#ifdef CONFIG_SMP
+ spinlock_cacheline_t pagecache_lock_cacheline = {SPIN_LOCK_UNLOCKED};
+ #ifndef pagecache_lock
+ #define pagecache_lock pagecache_lock_cacheline.lock
+ #endif
+#else
+ spinlock_t pagecache_lock = SPIN_LOCK_UNLOCKED;
+#endif
+
/*
* NOTE: to avoid deadlocking you must never acquire the pagecache_lock with
* the pagemap_lru_lock held.
*/
-spinlock_t pagemap_lru_lock = SPIN_LOCK_UNLOCKED;
+#ifdef CONFIG_SMP
+ spinlock_cacheline_t pagemap_lru_lock_cacheline = {SPIN_LOCK_UNLOCKED};
+ #ifndef pagemap_lru_lock
+ #define pagemap_lru_lock pagemap_lru_lock_cacheline.lock
+ #endif
+#else
+ spinlock_t pagemap_lru_lock = SPIN_LOCK_UNLOCKED;
+#endif
+
#define CLUSTER_PAGES (1 << page_cluster)
#define CLUSTER_OFFSET(x) (((x) >> page_cluster) << page_cluster)
--- linux/mm/highmem.c.orig Thu Sep 6 14:55:54 2001
+++ linux/mm/highmem.c Thu Sep 6 15:08:22 2001
@@ -32,7 +32,14 @@
*/
static int pkmap_count[LAST_PKMAP];
static unsigned int last_pkmap_nr;
-static spinlock_t kmap_lock = SPIN_LOCK_UNLOCKED;
+#ifdef CONFIG_SMP
+ static spinlock_cacheline_t kmap_lock_cacheline = {SPIN_LOCK_UNLOCKED};
+ #ifndef kmap_lock
+ #define kmap_lock kmap_lock_cacheline.lock
+ #endif
+#else
+ static spinlock_t kmap_lock = SPIN_LOCK_UNLOCKED;
+#endif
pte_t * pkmap_page_table;
--- linux/arch/i386/kernel/smp.c.orig Thu Sep 6 14:52:01 2001
+++ linux/arch/i386/kernel/smp.c Thu Sep 6 16:04:38 2001
@@ -101,7 +101,14 @@
*/
/* The 'big kernel lock' */
+#ifdef CONFIG_SMP
+ spinlock_cacheline_t kernel_flag_cacheline = {SPIN_LOCK_UNLOCKED};
+ #ifndef kernel_flag
+ #define kernel_flag kernel_flag_cacheline.lock
+ #endif
+#else
spinlock_t kernel_flag = SPIN_LOCK_UNLOCKED;
+#endif
struct tlb_state cpu_tlbstate[NR_CPUS] = {[0 ... NR_CPUS-1] = { &init_mm, 0 }};
= = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = =
Spinlock patch for kernel 2.4.10 (2.4.9)
= = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = =
--- linux/include/asm-i386/smplock.h.orig Thu Sep 6 14:46:25 2001
+++ linux/include/asm-i386/smplock.h Thu Sep 6 16:38:02 2001
@@ -8,7 +8,13 @@
#include <linux/sched.h>
#include <asm/current.h>
-extern spinlock_t kernel_flag;
+
+#ifdef CONFIG_SMP
+ extern spinlock_cacheline_t kernel_flag_cacheline;
+ #define kernel_flag kernel_flag_cacheline.lock
+#else
+ extern spinlock_t kernel_flag;
+#endif
#define kernel_locked() spin_is_locked(&kernel_flag)
--- linux/arch/i386/kernel/i386_ksyms.c.orig Thu Sep 6 16:00:29 2001
+++ linux/arch/i386/kernel/i386_ksyms.c Thu Sep 6 16:03:07 2001
@@ -120,7 +120,7 @@
#ifdef CONFIG_SMP
EXPORT_SYMBOL(cpu_data);
-EXPORT_SYMBOL(kernel_flag);
+EXPORT_SYMBOL(kernel_flag_cacheline);
EXPORT_SYMBOL(smp_num_cpus);
EXPORT_SYMBOL(cpu_online_map);
EXPORT_SYMBOL_NOVERS(__write_lock_failed);
--- linux/include/linux/spinlock.h.orig Thu Sep 6 14:30:41 2001
+++ linux/include/linux/spinlock.h Thu Sep 6 16:38:02 2001
@@ -34,6 +34,13 @@
#ifdef CONFIG_SMP
#include <asm/spinlock.h>
+#include <linux/cache.h>
+typedef union {
+ spinlock_t lock;
+ char fill_up[(SMP_CACHE_BYTES)];
+} spinlock_cacheline_t __attribute__ ((aligned(SMP_CACHE_BYTES)));
+
+
#elif !defined(spin_lock_init) /* !SMP and spin_lock_init not previously
defined (e.g. by including asm/spinlock.h */
--- linux/include/linux/swap.h.orig Thu Sep 6 14:38:01 2001
+++ linux/include/linux/swap.h Thu Sep 6 16:38:02 2001
@@ -88,7 +88,13 @@
extern struct address_space swapper_space;
extern atomic_t page_cache_size;
extern atomic_t buffermem_pages;
-extern spinlock_t pagecache_lock;
+#ifdef CONFIG_SMP
+ extern spinlock_cacheline_t pagecache_lock_cacheline;
+ #define pagecache_lock (pagecache_lock_cacheline.lock)
+#else
+ extern spinlock_t pagecache_lock;
+#endif
+
extern void __remove_inode_page(struct page *);
/* Incomplete types for prototype declarations: */
@@ -179,7 +185,12 @@
extern unsigned long swap_cache_find_success;
#endif
-extern spinlock_t pagemap_lru_lock;
+#ifdef CONFIG_SMP
+ extern spinlock_cacheline_t pagemap_lru_lock_cacheline;
+ #define pagemap_lru_lock pagemap_lru_lock_cacheline.lock
+#else
+ extern spinlock_t pagemap_lru_lock;
+#endif
/*
* Page aging defines.
--- linux/fs/buffer.c.orig Thu Sep 6 15:10:07 2001
+++ linux/fs/buffer.c Thu Sep 6 15:54:00 2001
@@ -83,13 +83,22 @@
static rwlock_t hash_table_lock = RW_LOCK_UNLOCKED;
static struct buffer_head *lru_list[NR_LIST];
-static spinlock_t lru_list_lock = SPIN_LOCK_UNLOCKED;
+
+#ifdef CONFIG_SMP
+ static spinlock_cacheline_t lru_list_lock_cacheline = {SPIN_LOCK_UNLOCKED};
+ #ifndef lru_list_lock
+ #define lru_list_lock lru_list_lock_cacheline.lock
+ #endif
+#else
+ static spinlock_t lru_list_lock = SPIN_LOCK_UNLOCKED;
+#endif
static int nr_buffers_type[NR_LIST];
static unsigned long size_buffers_type[NR_LIST];
static struct buffer_head * unused_list;
static int nr_unused_buffer_heads;
static spinlock_t unused_list_lock = SPIN_LOCK_UNLOCKED;
+
static DECLARE_WAIT_QUEUE_HEAD(buffer_wait);
struct bh_free_head {
--- linux/mm/filemap.c.orig Thu Sep 6 14:55:44 2001
+++ linux/mm/filemap.c Thu Sep 6 15:06:21 2001
@@ -45,12 +45,28 @@
unsigned int page_hash_bits;
struct page **page_hash_table;
-spinlock_t __cacheline_aligned pagecache_lock = SPIN_LOCK_UNLOCKED;
+#ifdef CONFIG_SMP
+ spinlock_cacheline_t pagecache_lock_cacheline = {SPIN_LOCK_UNLOCKED};
+ #ifndef pagecache_lock
+ #define pagecache_lock pagecache_lock_cacheline.lock
+ #endif
+#else
+ spinlock_t pagecache_lock = SPIN_LOCK_UNLOCKED;
+#endif
+
/*
* NOTE: to avoid deadlocking you must never acquire the pagecache_lock with
* the pagemap_lru_lock held.
*/
-spinlock_t __cacheline_aligned pagemap_lru_lock = SPIN_LOCK_UNLOCKED;
+#ifdef CONFIG_SMP
+ spinlock_cacheline_t pagemap_lru_lock_cacheline = {SPIN_LOCK_UNLOCKED};
+ #ifndef pagemap_lru_lock
+ #define pagemap_lru_lock pagemap_lru_lock_cacheline.lock
+ #endif
+#else
+ spinlock_t pagemap_lru_lock = SPIN_LOCK_UNLOCKED;
+#endif
+
#define CLUSTER_PAGES (1 << page_cluster)
#define CLUSTER_OFFSET(x) (((x) >> page_cluster) << page_cluster)
--- linux/mm/highmem.c.orig Thu Sep 6 14:55:54 2001
+++ linux/mm/highmem.c Thu Sep 6 15:08:22 2001
@@ -32,7 +32,14 @@
*/
static int pkmap_count[LAST_PKMAP];
static unsigned int last_pkmap_nr;
-static spinlock_t kmap_lock = SPIN_LOCK_UNLOCKED;
+#ifdef CONFIG_SMP
+ static spinlock_cacheline_t kmap_lock_cacheline = {SPIN_LOCK_UNLOCKED};
+ #ifndef kmap_lock
+ #define kmap_lock kmap_lock_cacheline.lock
+ #endif
+#else
+ static spinlock_t kmap_lock = SPIN_LOCK_UNLOCKED;
+#endif
pte_t * pkmap_page_table;
--- linux/arch/i386/kernel/smp.c.orig Thu Sep 6 14:52:01 2001
+++ linux/arch/i386/kernel/smp.c Thu Sep 6 16:04:38 2001
@@ -101,7 +101,14 @@
*/
/* The 'big kernel lock' */
+#ifdef CONFIG_SMP
+ spinlock_cacheline_t kernel_flag_cacheline = {SPIN_LOCK_UNLOCKED};
+ #ifndef kernel_flag
+ #define kernel_flag kernel_flag_cacheline.lock
+ #endif
+#else
spinlock_t kernel_flag = SPIN_LOCK_UNLOCKED;
+#endif
struct tlb_state cpu_tlbstate[NR_CPUS] = {[0 ... NR_CPUS-1] = { &init_mm, 0 }};
^ permalink raw reply [flat|nested] 5+ messages in thread
* Re: [PATCH] Align VM locks, new spinlock patch
2001-09-21 13:15 Juergen Doelle
@ 2001-09-22 23:07 ` Andrew Morton
2001-09-24 14:02 ` Stephen C. Tweedie
1 sibling, 0 replies; 5+ messages in thread
From: Andrew Morton @ 2001-09-22 23:07 UTC (permalink / raw)
To: Juergen Doelle
Cc: Andrea Arcangeli, linux-kernel, Mark Hemment, lse-tech, Steve Fox
Juergen Doelle wrote:
>
> ...
> The patch introduces a new type spinlock_cacheline_t which
> o aligns the spinlock to a cacheline and
> o avoids that this spinlock shares the cacheline with other data
> This type is now hardware independent.
>
> It declares following spinlocks from type spinlock_cacheline_t:
> o kmap_lock
> o pagecache_lock
> o pagemap_lru_lock
> o lru_list_lock
> o kernel_flag
> Where the kernel_flag is modified only for i386 architecture, because
> the spinlock itself is hardware dependent implemented.
>
> I tested the patch on 2.4.5, 2.4.8, and 2.4.10-pre7.
> Kernel 2.4.10-pre7 requires a separate version of the patch, because
> the pagecache_lock and the lru_list_lock are already aligned.
>
> The peak throughput results with dbench:
>
> 2.4.10 2.4.10 + improvement
> spinlock patch by patch
>
> U 103.77 102.14 - 1.6%
> 1 96.82 96.77 - 0.1%
> 2 155.32 155.62 0.2%
> 4 209.45 222.11 6.0%
> 8 208.06 234.82 12.9%
>
I guess the uniprocessor case is measurement error - spin_lock()
is a no-op on UP.
I think the coding can be simplified somewhat. How does this
look?
--- linux-2.4.10-pre14/include/linux/spinlock.h Mon Jul 30 20:55:14 2001
+++ linux-akpm/include/linux/spinlock.h Sat Sep 22 14:15:25 2001
@@ -133,4 +133,20 @@ typedef struct {
extern int atomic_dec_and_lock(atomic_t *atomic, spinlock_t *lock);
#endif
+#ifdef CONFIG_SMP
+#include <linux/cache.h>
+
+typedef union {
+ spinlock_t lock;
+ char fill_up[(SMP_CACHE_BYTES)];
+} spinlock_cacheline_t __attribute__ ((aligned(SMP_CACHE_BYTES)));
+
+#else /* SMP */
+
+typedef struct {
+ spinlock_t lock;
+} spinlock_cacheline_t;
+
+
+#endif
#endif /* __LINUX_SPINLOCK_H */
--- linux-2.4.10-pre14/include/linux/swap.h Sat Sep 22 13:58:05 2001
+++ linux-akpm/include/linux/swap.h Sat Sep 22 14:15:25 2001
@@ -87,7 +87,10 @@ extern atomic_t nr_async_pages;
extern struct address_space swapper_space;
extern atomic_t page_cache_size;
extern atomic_t buffermem_pages;
-extern spinlock_t pagecache_lock;
+
+extern spinlock_cacheline_t pagecache_lock_cacheline;
+#define pagecache_lock (pagecache_lock_cacheline.lock)
+
extern void __remove_inode_page(struct page *);
/* Incomplete types for prototype declarations: */
@@ -168,7 +171,8 @@ extern unsigned long swap_cache_find_tot
extern unsigned long swap_cache_find_success;
#endif
-extern spinlock_t pagemap_lru_lock;
+extern spinlock_cacheline_t pagemap_lru_lock_cacheline;
+#define pagemap_lru_lock pagemap_lru_lock_cacheline.lock
extern void FASTCALL(mark_page_accessed(struct page *));
--- linux-2.4.10-pre14/arch/i386/kernel/i386_ksyms.c Sat Sep 22 13:57:52 2001
+++ linux-akpm/arch/i386/kernel/i386_ksyms.c Sat Sep 22 13:42:22 2001
@@ -120,7 +120,7 @@ EXPORT_SYMBOL(mmx_copy_page);
#ifdef CONFIG_SMP
EXPORT_SYMBOL(cpu_data);
-EXPORT_SYMBOL(kernel_flag);
+EXPORT_SYMBOL(kernel_flag_cacheline);
EXPORT_SYMBOL(smp_num_cpus);
EXPORT_SYMBOL(cpu_online_map);
EXPORT_SYMBOL_NOVERS(__write_lock_failed);
--- linux-2.4.10-pre14/arch/i386/kernel/smp.c Sat Sep 22 13:57:52 2001
+++ linux-akpm/arch/i386/kernel/smp.c Sat Sep 22 13:42:22 2001
@@ -101,7 +101,7 @@
*/
/* The 'big kernel lock' */
-spinlock_t kernel_flag = SPIN_LOCK_UNLOCKED;
+spinlock_cacheline_t kernel_flag_cacheline = {SPIN_LOCK_UNLOCKED};
struct tlb_state cpu_tlbstate[NR_CPUS] = {[0 ... NR_CPUS-1] = { &init_mm, 0 }};
--- linux-2.4.10-pre14/fs/buffer.c Sat Sep 22 13:58:00 2001
+++ linux-akpm/fs/buffer.c Sat Sep 22 13:42:22 2001
@@ -81,13 +81,17 @@ static struct buffer_head **hash_table;
static rwlock_t hash_table_lock = RW_LOCK_UNLOCKED;
static struct buffer_head *lru_list[NR_LIST];
-static spinlock_t lru_list_lock = SPIN_LOCK_UNLOCKED;
+
+static spinlock_cacheline_t lru_list_lock_cacheline = {SPIN_LOCK_UNLOCKED};
+#define lru_list_lock lru_list_lock_cacheline.lock
+
static int nr_buffers_type[NR_LIST];
static unsigned long size_buffers_type[NR_LIST];
static struct buffer_head * unused_list;
static int nr_unused_buffer_heads;
static spinlock_t unused_list_lock = SPIN_LOCK_UNLOCKED;
+
static DECLARE_WAIT_QUEUE_HEAD(buffer_wait);
struct bh_free_head {
--- linux-2.4.10-pre14/mm/filemap.c Sat Sep 22 13:58:05 2001
+++ linux-akpm/mm/filemap.c Sat Sep 22 13:42:22 2001
@@ -46,12 +46,13 @@ atomic_t page_cache_size = ATOMIC_INIT(0
unsigned int page_hash_bits;
struct page **page_hash_table;
-spinlock_t pagecache_lock ____cacheline_aligned_in_smp = SPIN_LOCK_UNLOCKED;
+spinlock_cacheline_t pagecache_lock_cacheline = {SPIN_LOCK_UNLOCKED};
+
/*
* NOTE: to avoid deadlocking you must never acquire the pagecache_lock with
* the pagemap_lru_lock held.
*/
-spinlock_t pagemap_lru_lock ____cacheline_aligned_in_smp = SPIN_LOCK_UNLOCKED;
+spinlock_cacheline_t pagemap_lru_lock_cacheline = {SPIN_LOCK_UNLOCKED};
#define CLUSTER_PAGES (1 << page_cluster)
#define CLUSTER_OFFSET(x) (((x) >> page_cluster) << page_cluster)
--- linux-2.4.10-pre14/mm/highmem.c Sat Sep 22 13:58:05 2001
+++ linux-akpm/mm/highmem.c Sat Sep 22 13:42:22 2001
@@ -32,7 +32,7 @@
*/
static int pkmap_count[LAST_PKMAP];
static unsigned int last_pkmap_nr;
-static spinlock_t kmap_lock = SPIN_LOCK_UNLOCKED;
+static spinlock_cacheline_t kmap_lock_cacheline = {SPIN_LOCK_UNLOCKED};
pte_t * pkmap_page_table;
^ permalink raw reply [flat|nested] 5+ messages in thread
* Re: [PATCH] Align VM locks, new spinlock patch
2001-09-21 13:15 Juergen Doelle
2001-09-22 23:07 ` Andrew Morton
@ 2001-09-24 14:02 ` Stephen C. Tweedie
1 sibling, 0 replies; 5+ messages in thread
From: Stephen C. Tweedie @ 2001-09-24 14:02 UTC (permalink / raw)
To: Juergen Doelle
Cc: Andrea Arcangeli, Andrew Morton, linux-kernel, Mark Hemment,
lse-tech, Steve Fox, Stephen Tweedie
Hi,
On Fri, Sep 21, 2001 at 03:15:17PM +0200, Juergen Doelle wrote:
> 2.4.10 2.4.10 + improvement
> spinlock patch by patch
>
> U 103.77 102.14 - 1.6%
> 1 96.82 96.77 - 0.1%
> 2 155.32 155.62 0.2%
> 4 209.45 222.11 6.0%
> 8 208.06 234.82 12.9%
>
> The improvement is less than in previous posted results, because the
> pagemap_lru_lock and the lru_list_lock are already cacheline aligned
> in 2.4.10 (2.4.9).
Do you have CPU utilisation differences for these cases, as well as
pure IO bandwidth differences? It would be interesting to see just
how much the IO code's internal latency impacts on the final dbench
numbers.
--Stephen
^ permalink raw reply [flat|nested] 5+ messages in thread
* Re: [PATCH] Align VM locks, new spinlock patch
@ 2001-09-26 15:15 Juergen Doelle
0 siblings, 0 replies; 5+ messages in thread
From: Juergen Doelle @ 2001-09-26 15:15 UTC (permalink / raw)
To: Stephen Tweedie, Andrea Arcangeli, Andrew Morton, linux-kernel,
Mark Hemment, lse-tech, Steve Fox
Stephen C. Tweedie wrote
> Do you have CPU utilisation differences for these cases, as well as
> pure IO bandwidth differences? It would be interesting to see just
> how much the IO code's internal latency impacts on the final dbench
> numbers.
I created vmstat traces for dbench 22
+-----------------------------------+------------------------------+
| 2.4.10 | 2.4.10 + spinlock patch |
+----+----------+-------+-----------+----------+-------+-----------+
|time|procs | IO | cpu |procs | IO | cpu |
+ +----------+-------+-----------+----------+-------+-----------+
|[s] | r b w | bo/s | us sy id | r b w | bo/s | us sy id |
+----+----------+-------+-----------+----------+-------+-----------+
| 1 | 23 0 0 | 0 | 0 14 85 | 22 0 0 | 0 | 1 47 52 |
| 2 | 22 0 0 | 0 | 2 98 0 | 22 0 0 | 0 | 2 99 0 |
| 3 | 22 0 0 | 0 | 2 98 0 | 22 0 0 | 0 | 4 97 0 |
| 4 | 8 14 2 | 11052 | 12 89 0 | 5 17 1 | 20465 | 11 84 5 |
| 5 | 3 19 1 | 1616 | 1 4 94 | 20 2 6 | 1788 | 16 83 0 |
| 6 | 1 21 1 | 1760 | 2 4 94 | 22 0 4 | 0 | 18 82 0 |
| 7 | 23 0 3 | 2852 | 12 46 42 | 22 0 1 | 0 | 19 80 0 |
| 8 | 22 0 3 | 0 | 15 85 0 | 22 0 0 | 0 | 19 82 0 |
| 9 | 22 0 2 | 0 | 17 83 0 | 23 0 0 | 0 | 18 82 0 |
| 10 | 23 0 1 | 0 | 16 84 0 | 22 0 0 | 0 | 17 82 1 |
| 11 | 22 0 0 | 0 | 14 86 0 | 22 0 0 | 0 | 18 83 |
| 12 | 22 0 0 | 0 | 16 84 0 | 19 0 0 | 0 | 18 82 0 |
| 13 | 22 0 0 | 0 | 19 81 1 | 9 0 0 | 0 | 7 94 |
| 14 | 20 0 0 | 0 | 17 84 0 | 0 0 0 | 0 | 0 30 70 |
| 15 | 17 0 0 | 0 | 15 85 0 |----------+-------+-----------+
| 16 | 13 0 0 | 0 | 4 97 0 |
| 17 | 12 0 0 | 0 | 0 100 0 |
| 18 | 7 0 0 | 0 | 0 99 0 |
| 19 | 0 0 0 | 0 | 0 15 85 |
+----+----------+-------+-----------+
*the empty idle columns originally containing 5315553, treat as '0'
The patch
o reduces significantly the idle times, the user process wait time
is much shorter
o reduces the I/O phase to the half of the time, by much higher rates
o increases the user and decreases the system CPU utilization
I prior posted lockmeter results on 2.4.5, where this patch reduced
for 8 CPUs the average spin hold time by about 47% and the total CPU
utilization spent for spinning by 45%.
I think it does not influence the device layer directly. It speeds up
the time spent in critical phases protected with spin locks, when the
number of competitors increases. In case of high competitions caused by
many parallel working processors, the buffer cache and pagecache
handling gets faster and this increases the number of changed pages per
second.
Therefore the mechanism used here can also be used to improve
competitive scenarios for other spin locks.
Juergen
______________________________________________________________
Juergen Doelle
IBM Linux Technology Center - kernel performance
jdoelle@de.ibm.com
^ permalink raw reply [flat|nested] 5+ messages in thread
* Re: [PATCH] Align VM locks, new spinlock patch
@ 2001-09-26 15:15 Juergen Doelle
0 siblings, 0 replies; 5+ messages in thread
From: Juergen Doelle @ 2001-09-26 15:15 UTC (permalink / raw)
To: Andrew Morton
Cc: Andrea Arcangeli, linux-kernel, linux-kernel-owner, lse-tech,
Mark Hemment, Steve Fox, Alan Cox
Andrew Morton wrote:
> I think the coding can be simplified somewhat. How does this
> look?
> ...
Looks fine, it makes the code more common between UP and SMP.
I consolidated the changes and appended the new version below.
The peak throughput results (MB/sec) for dbench on 2.4.10:
CPU 2.4.10 2.4.10 + improvement
spinlock patch by patch
U 102,9 103,0 0,0%
1 95,1 96,9 1,9%
2 155,5 156,0 0,3%
4 206,9 222,1 7,3%
8 194,9 233,7 19,9%
Throughput (MB/sec) for dbench with 8 clients on 8 CPUs:
CPU 2.4.10 2.4.10 + improvement
spinlock patch by patch
8 173,9 223,0 28,2%
Juergen
______________________________________________________________
Juergen Doelle
IBM Linux Technology Center - kernel performance
jdoelle@de.ibm.com
= = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = =
--- linux/include/linux/spinlock.h.orig Tue Sep 25 09:05:09 2001
+++ linux/include/linux/spinlock.h Tue Sep 25 11:42:51 2001
@@ -133,4 +133,20 @@
extern int atomic_dec_and_lock(atomic_t *atomic, spinlock_t *lock);
#endif
+#ifdef CONFIG_SMP
+#include <linux/cache.h>
+
+typedef union {
+ spinlock_t lock;
+ char fill_up[(SMP_CACHE_BYTES)];
+} spinlock_cacheline_t __attribute__ ((aligned(SMP_CACHE_BYTES)));
+
+#else /* SMP */
+
+typedef struct {
+ spinlock_t lock;
+} spinlock_cacheline_t;
+
+
+#endif
#endif /* __LINUX_SPINLOCK_H */
--- linux/include/linux/swap.h.orig Tue Sep 25 09:59:15 2001
+++ linux/include/linux/swap.h Tue Sep 25 11:42:51 2001
@@ -86,7 +86,10 @@
extern atomic_t nr_async_pages;
extern atomic_t page_cache_size;
extern atomic_t buffermem_pages;
-extern spinlock_t pagecache_lock;
+
+extern spinlock_cacheline_t pagecache_lock_cacheline;
+#define pagecache_lock (pagecache_lock_cacheline.lock)
+
extern void __remove_inode_page(struct page *);
/* Incomplete types for prototype declarations: */
@@ -159,7 +162,8 @@
extern unsigned long swap_cache_find_success;
#endif
-extern spinlock_t pagemap_lru_lock;
+extern spinlock_cacheline_t pagemap_lru_lock_cacheline;
+#define pagemap_lru_lock pagemap_lru_lock_cacheline.lock
extern void FASTCALL(mark_page_accessed(struct page *));
--- linux/include/asm-i386/smplock.h.orig Tue Sep 25 11:29:51 2001
+++ linux/include/asm-i386/smplock.h Tue Sep 25 11:42:52 2001
@@ -8,7 +8,8 @@
#include <linux/sched.h>
#include <asm/current.h>
-extern spinlock_t kernel_flag;
+extern spinlock_cacheline_t kernel_flag_cacheline;
+#define kernel_flag kernel_flag_cacheline.lock
#define kernel_locked() spin_is_locked(&kernel_flag)
--- linux/arch/i386/kernel/i386_ksyms.c.orig Tue Sep 25 09:14:57 2001
+++ linux/arch/i386/kernel/i386_ksyms.c Tue Sep 25 10:13:06 2001
@@ -120,7 +120,7 @@
#ifdef CONFIG_SMP
EXPORT_SYMBOL(cpu_data);
-EXPORT_SYMBOL(kernel_flag);
+EXPORT_SYMBOL(kernel_flag_cacheline);
EXPORT_SYMBOL(smp_num_cpus);
EXPORT_SYMBOL(cpu_online_map);
EXPORT_SYMBOL_NOVERS(__write_lock_failed);
--- linux/arch/i386/kernel/smp.c.orig Tue Sep 25 09:15:16 2001
+++ linux/arch/i386/kernel/smp.c Tue Sep 25 10:13:06 2001
@@ -101,7 +101,7 @@
*/
/* The 'big kernel lock' */
-spinlock_t kernel_flag = SPIN_LOCK_UNLOCKED;
+spinlock_cacheline_t kernel_flag_cacheline = {SPIN_LOCK_UNLOCKED};
struct tlb_state cpu_tlbstate[NR_CPUS] = {[0 ... NR_CPUS-1] = { &init_mm, 0 }};
--- linux/fs/buffer.c.orig Tue Sep 25 09:15:47 2001
+++ linux/fs/buffer.c Wed Sep 26 12:17:29 2001
@@ -81,7 +81,10 @@
static rwlock_t hash_table_lock = RW_LOCK_UNLOCKED;
static struct buffer_head *lru_list[NR_LIST];
-static spinlock_t lru_list_lock = SPIN_LOCK_UNLOCKED;
+
+static spinlock_cacheline_t lru_list_lock_cacheline = {SPIN_LOCK_UNLOCKED};
+#define lru_list_lock lru_list_lock_cacheline.lock
+
static int nr_buffers_type[NR_LIST];
static unsigned long size_buffers_type[NR_LIST];
--- linux/mm/filemap.c.orig Tue Sep 25 09:16:06 2001
+++ linux/mm/filemap.c Tue Sep 25 10:13:06 2001
@@ -46,12 +46,13 @@
unsigned int page_hash_bits;
struct page **page_hash_table;
-spinlock_t pagecache_lock ____cacheline_aligned_in_smp = SPIN_LOCK_UNLOCKED;
+spinlock_cacheline_t pagecache_lock_cacheline = {SPIN_LOCK_UNLOCKED};
+
/*
* NOTE: to avoid deadlocking you must never acquire the pagecache_lock with
* the pagemap_lru_lock held.
*/
-spinlock_t pagemap_lru_lock ____cacheline_aligned_in_smp = SPIN_LOCK_UNLOCKED;
+spinlock_cacheline_t pagemap_lru_lock_cacheline = {SPIN_LOCK_UNLOCKED};
#define CLUSTER_PAGES (1 << page_cluster)
#define CLUSTER_OFFSET(x) (((x) >> page_cluster) << page_cluster)
--- linux/mm/highmem.c.orig Tue Sep 25 09:29:49 2001
+++ linux/mm/highmem.c Tue Sep 25 10:21:14 2001
@@ -32,7 +32,8 @@
*/
static int pkmap_count[LAST_PKMAP];
static unsigned int last_pkmap_nr;
-static spinlock_t kmap_lock = SPIN_LOCK_UNLOCKED;
+static spinlock_cacheline_t kmap_lock_cacheline = {SPIN_LOCK_UNLOCKED};
+#define kmap_lock kmap_lock_cacheline.lock
pte_t * pkmap_page_table;
^ permalink raw reply [flat|nested] 5+ messages in thread
end of thread, other threads:[~2001-09-26 15:18 UTC | newest]
Thread overview: 5+ messages (download: mbox.gz follow: Atom feed
-- links below jump to the message on this page --
2001-09-26 15:15 [PATCH] Align VM locks, new spinlock patch Juergen Doelle
-- strict thread matches above, loose matches on Subject: below --
2001-09-26 15:15 Juergen Doelle
2001-09-21 13:15 Juergen Doelle
2001-09-22 23:07 ` Andrew Morton
2001-09-24 14:02 ` Stephen C. Tweedie
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox