From: Konrad Rzeszutek Wilk <konrad.wilk@oracle.com>
To: Andrew Cooper <andrew.cooper3@citrix.com>
Cc: xen-devel@lists.xenproject.org, JBeulich@suse.com
Subject: Re: [PATCH] asm, x86: Set max CPUs to 512 instead of 256.
Date: Thu, 22 Jan 2015 14:03:55 -0500 [thread overview]
Message-ID: <20150122190355.GA18286@l.oracle.com> (raw)
In-Reply-To: <54C12D8C.50604@citrix.com>
[-- Attachment #1: Type: text/plain, Size: 1676 bytes --]
On Thu, Jan 22, 2015 at 05:04:12PM +0000, Andrew Cooper wrote:
> On 22/01/15 16:52, Konrad Rzeszutek Wilk wrote:
> > Contemporary servers sport now 480 CPUs or such. We should crank
> > up the default amount of CPUs to a higher level to take advantage
> > of this without having the distro to use 'max_phys_cpus' override.
> >
> > Signed-off-by: Konrad Rzeszutek Wilk <konrad.wilk@oracle.com>
>
> /me would really like to try something that big out, but I have not had
> the opportunity yet to hit the 256 limit.
>
> I wonder which variables grow as a result of this change. We might want
> to see about making more things dynamically allocated after reading the
> apci tables, if we can.
I am not sure if that is possible as there is a lot of DEFINE_PER_CPU
which are cannot grow.
The structures that grow are:
struct cpumask
struct kernel_param
struct rangeset
struct csched2_runqueue_data
struct csched2_private
struct rt_vcpu
struct stopmachine_data
struct free_ptr
struct rcu_data
struct physid_mask
struct acpi_table_header
struct calibration_rendezvous
struct bug_frame
(for fun see attached diff of pahole between 256 and 512 CPUs)
>
> ~Andrew
>
> > ---
> > xen/include/asm-x86/config.h | 2 +-
> > 1 file changed, 1 insertion(+), 1 deletion(-)
> >
> > diff --git a/xen/include/asm-x86/config.h b/xen/include/asm-x86/config.h
> > index 2fbd68d..d450696 100644
> > --- a/xen/include/asm-x86/config.h
> > +++ b/xen/include/asm-x86/config.h
> > @@ -64,7 +64,7 @@
> > #ifdef MAX_PHYS_CPUS
> > #define NR_CPUS MAX_PHYS_CPUS
> > #else
> > -#define NR_CPUS 256
> > +#define NR_CPUS 512
> > #endif
> >
> > /* Linkage for x86 */
>
[-- Attachment #2: 256vs512 --]
[-- Type: text/plain, Size: 9672 bytes --]
--- 256 2015-01-22 14:01:55.200283080 -0500
+++ 512 2015-01-22 13:59:08.588811566 -0500
@@ -1,8 +1,8 @@
struct cpumask {
- long unsigned int bits[4]; /* 0 32 */
+ long unsigned int bits[8]; /* 0 64 */
+ /* --- cacheline 1 boundary (64 bytes) --- */
- /* size: 32, cachelines: 1, members: 1 */
- /* last cacheline: 32 bytes */
+ /* size: 64, cachelines: 1, members: 1 */
};
struct kernel_param {
const char * name; /* 0 8 */
@@ -4853,10 +4853,11 @@
long unsigned int start; /* 8 8 */
long unsigned int per_cpu_sz; /* 16 8 */
long unsigned int rem; /* 24 8 */
- cpumask_t cpus; /* 32 32 */
- /* --- cacheline 1 boundary (64 bytes) --- */
+ cpumask_t cpus; /* 32 64 */
+ /* --- cacheline 1 boundary (64 bytes) was 32 bytes ago --- */
- /* size: 64, cachelines: 1, members: 5 */
+ /* size: 96, cachelines: 2, members: 5 */
+ /* last cacheline: 32 bytes */
};
struct rangeset {
struct list_head rangeset_list; /* 0 16 */
@@ -4968,46 +4969,49 @@
struct csched2_runqueue_data {
int id; /* 0 4 */
spinlock_t lock; /* 4 4 */
- cpumask_t active; /* 8 32 */
- struct list_head runq; /* 40 16 */
- struct list_head svc; /* 56 16 */
+ cpumask_t active; /* 8 64 */
/* --- cacheline 1 boundary (64 bytes) was 8 bytes ago --- */
- unsigned int max_weight; /* 72 4 */
+ struct list_head runq; /* 72 16 */
+ struct list_head svc; /* 88 16 */
+ unsigned int max_weight; /* 104 4 */
/* XXX 4 bytes hole, try to pack */
- cpumask_t idle; /* 80 32 */
- cpumask_t tickled; /* 112 32 */
- /* --- cacheline 2 boundary (128 bytes) was 16 bytes ago --- */
- int load; /* 144 4 */
+ cpumask_t idle; /* 112 64 */
+ /* --- cacheline 2 boundary (128 bytes) was 48 bytes ago --- */
+ cpumask_t tickled; /* 176 64 */
+ /* --- cacheline 3 boundary (192 bytes) was 48 bytes ago --- */
+ int load; /* 240 4 */
/* XXX 4 bytes hole, try to pack */
- s_time_t load_last_update; /* 152 8 */
- s_time_t avgload; /* 160 8 */
- s_time_t b_avgload; /* 168 8 */
+ s_time_t load_last_update; /* 248 8 */
+ /* --- cacheline 4 boundary (256 bytes) --- */
+ s_time_t avgload; /* 256 8 */
+ s_time_t b_avgload; /* 264 8 */
- /* size: 176, cachelines: 3, members: 12 */
- /* sum members: 168, holes: 2, sum holes: 8 */
- /* last cacheline: 48 bytes */
+ /* size: 272, cachelines: 5, members: 12 */
+ /* sum members: 264, holes: 2, sum holes: 8 */
+ /* last cacheline: 16 bytes */
};
struct csched2_private {
spinlock_t lock; /* 0 4 */
/* XXX 4 bytes hole, try to pack */
- cpumask_t initialized; /* 8 32 */
- struct list_head sdom; /* 40 16 */
- int runq_map[256]; /* 56 1024 */
- /* --- cacheline 16 boundary (1024 bytes) was 56 bytes ago --- */
- cpumask_t active_queues; /* 1080 32 */
- /* --- cacheline 17 boundary (1088 bytes) was 24 bytes ago --- */
- struct csched2_runqueue_data rqd[256]; /* 1112 45056 */
- /* --- cacheline 721 boundary (46144 bytes) was 24 bytes ago --- */
- int load_window_shift; /* 46168 4 */
+ cpumask_t initialized; /* 8 64 */
+ /* --- cacheline 1 boundary (64 bytes) was 8 bytes ago --- */
+ struct list_head sdom; /* 72 16 */
+ int runq_map[512]; /* 88 2048 */
+ /* --- cacheline 33 boundary (2112 bytes) was 24 bytes ago --- */
+ cpumask_t active_queues; /* 2136 64 */
+ /* --- cacheline 34 boundary (2176 bytes) was 24 bytes ago --- */
+ struct csched2_runqueue_data rqd[512]; /* 2200 139264 */
+ /* --- cacheline 2210 boundary (141440 bytes) was 24 bytes ago --- */
+ int load_window_shift; /* 141464 4 */
- /* size: 46176, cachelines: 722, members: 7 */
- /* sum members: 46168, holes: 1, sum holes: 4 */
+ /* size: 141472, cachelines: 2211, members: 7 */
+ /* sum members: 141464, holes: 1, sum holes: 4 */
/* padding: 4 */
/* last cacheline: 32 bytes */
};
@@ -5149,12 +5153,12 @@
struct list_head sdom; /* 8 16 */
struct list_head runq; /* 24 16 */
struct list_head depletedq; /* 40 16 */
- cpumask_t tickled; /* 56 32 */
- /* --- cacheline 1 boundary (64 bytes) was 24 bytes ago --- */
+ cpumask_t tickled; /* 56 64 */
+ /* --- cacheline 1 boundary (64 bytes) was 56 bytes ago --- */
- /* size: 88, cachelines: 2, members: 5 */
- /* sum members: 84, holes: 1, sum holes: 4 */
- /* last cacheline: 24 bytes */
+ /* size: 120, cachelines: 2, members: 5 */
+ /* sum members: 116, holes: 1, sum holes: 4 */
+ /* last cacheline: 56 bytes */
};
struct rt_vcpu {
struct list_head q_elem; /* 0 16 */
@@ -5232,11 +5236,12 @@
/* XXX 4 bytes hole, try to pack */
- cpumask_t selected; /* 24 32 */
+ cpumask_t selected; /* 24 64 */
+ /* --- cacheline 1 boundary (64 bytes) was 24 bytes ago --- */
- /* size: 56, cachelines: 1, members: 4 */
- /* sum members: 52, holes: 1, sum holes: 4 */
- /* last cacheline: 56 bytes */
+ /* size: 88, cachelines: 2, members: 4 */
+ /* sum members: 84, holes: 1, sum holes: 4 */
+ /* last cacheline: 24 bytes */
};
struct stopmachine_data {
unsigned int nr_cpus; /* 0 4 */
@@ -5702,13 +5707,13 @@
struct vcpu * vcpu; /* 16 8 */
void * esp; /* 24 8 */
char * stack; /* 32 8 */
- cpumask_t saved_affinity; /* 40 32 */
- /* --- cacheline 1 boundary (64 bytes) was 8 bytes ago --- */
- unsigned int wakeup_cpu; /* 72 4 */
+ cpumask_t saved_affinity; /* 40 64 */
+ /* --- cacheline 1 boundary (64 bytes) was 40 bytes ago --- */
+ unsigned int wakeup_cpu; /* 104 4 */
- /* size: 80, cachelines: 2, members: 6 */
+ /* size: 112, cachelines: 2, members: 6 */
/* padding: 4 */
- /* last cacheline: 16 bytes */
+ /* last cacheline: 48 bytes */
};
struct free_ptr {
struct bhdr * prev; /* 0 8 */
@@ -5774,11 +5779,12 @@
/* XXX 4 bytes hole, try to pack */
- cpumask_t cpumask; /* 136 32 */
+ cpumask_t cpumask; /* 136 64 */
+ /* --- cacheline 3 boundary (192 bytes) was 8 bytes ago --- */
/* size: 256, cachelines: 4, members: 5 */
- /* sum members: 56, holes: 2, sum holes: 112 */
- /* padding: 88 */
+ /* sum members: 88, holes: 2, sum holes: 112 */
+ /* padding: 56 */
};
struct rcu_data {
long int quiescbatch; /* 0 8 */
@@ -7630,10 +7636,10 @@
/* last cacheline: 8 bytes */
};
struct physid_mask {
- long unsigned int mask[16]; /* 0 128 */
- /* --- cacheline 2 boundary (128 bytes) --- */
+ long unsigned int mask[32]; /* 0 256 */
+ /* --- cacheline 4 boundary (256 bytes) --- */
- /* size: 128, cachelines: 2, members: 1 */
+ /* size: 256, cachelines: 4, members: 1 */
};
struct acpi_table_header {
char signature[4]; /* 0 4 */
@@ -10672,17 +10678,18 @@
/* last cacheline: 24 bytes */
};
struct calibration_rendezvous {
- cpumask_t cpu_calibration_map; /* 0 32 */
- atomic_t semaphore; /* 32 4 */
+ cpumask_t cpu_calibration_map; /* 0 64 */
+ /* --- cacheline 1 boundary (64 bytes) --- */
+ atomic_t semaphore; /* 64 4 */
/* XXX 4 bytes hole, try to pack */
- s_time_t master_stime; /* 40 8 */
- u64 master_tsc_stamp; /* 48 8 */
+ s_time_t master_stime; /* 72 8 */
+ u64 master_tsc_stamp; /* 80 8 */
- /* size: 56, cachelines: 1, members: 4 */
- /* sum members: 52, holes: 1, sum holes: 4 */
- /* last cacheline: 56 bytes */
+ /* size: 88, cachelines: 2, members: 4 */
+ /* sum members: 84, holes: 1, sum holes: 4 */
+ /* last cacheline: 24 bytes */
};
struct bug_frame {
int loc_disp:24; /* 0: 8 4 */
[-- Attachment #3: Type: text/plain, Size: 126 bytes --]
_______________________________________________
Xen-devel mailing list
Xen-devel@lists.xen.org
http://lists.xen.org/xen-devel
next prev parent reply other threads:[~2015-01-22 19:04 UTC|newest]
Thread overview: 5+ messages / expand[flat|nested] mbox.gz Atom feed top
2015-01-22 16:52 [PATCH] asm,x86: Set max CPUs to 512 instead of 256 Konrad Rzeszutek Wilk
2015-01-22 17:04 ` [PATCH] asm, x86: " Andrew Cooper
2015-01-22 19:03 ` Konrad Rzeszutek Wilk [this message]
2015-01-22 20:04 ` Konrad Rzeszutek Wilk
2015-01-23 11:25 ` Jan Beulich
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=20150122190355.GA18286@l.oracle.com \
--to=konrad.wilk@oracle.com \
--cc=JBeulich@suse.com \
--cc=andrew.cooper3@citrix.com \
--cc=xen-devel@lists.xenproject.org \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is an external index of several public inboxes,
see mirroring instructions on how to clone and mirror
all data and code used by this external index.