[PATCH] Fixes to x86/xen: Provide a Xen PV APIC driver to support patch.

[PATCH] Fixes to x86/xen: Provide a Xen PV APIC driver to support patch.

[Konrad Rzeszutek Wilk]

Boris spotted an compile bug (32-bit) and asked some hard questions
that lead me to dig in the 32-bit code in more details.

The patch:
 [PATCH 1/2] x86/xen/apic: Work with UP, non-SMP 32-bit kernels.

fixes the compile issue and also allows us to boot on 32-bit, UP kernels
(dom0 and PV) without any warnings due to the new APIC driver.

The
 [PATCH 2/2] x86/xen/apic: WARN with details.

is more an diagnostic patch that can help in weed out other
bugs in the future.


 arch/x86/xen/apic.c | 17 +++++++++++++----
 1 file changed, 13 insertions(+), 4 deletions(-)

Konrad Rzeszutek Wilk (2):
      x86/xen/apic: Work with UP, non-SMP 32-bit kernels.
      x86/xen/apic: WARN with details.

[PATCH 1/2] x86/xen/apic: Work with UP, non-SMP 32-bit kernels.

[Konrad Rzeszutek Wilk]

Most of the APIC code that use APIC_LDR is not used on
64-bit. On 32-bit it is bit of an hack - and the mechanism
it is uses is to "setup" the APIC_LDR via apci->init_apic_ldr
(which we set to NULL) and then use apic->x86_32_early_logical_apicid
to get an CPU to APIC ID mapping and also apic->read to match
the APIC_LDR.

The 'x86_32_early_logical_apicid' is called from 'setup_local_APIC'
which is called from the following functions:
 - start_secondary ->smp_callin -> apic_ap_setup [not called on PV]
 - native_smp_prepare_cpus -> apci_bsp_setup [not called on PV]
 - up_late_init -> APIC_init_uniprocessor -> apic_bsp_setup ->setup_local_APIC
   [called on PV with CONFIG_SMP not defined, CONFIG_X86_32 and UP kernel]

This patch fixes the build issue and also allows the bootup
to continue without warnings.

Reported-by: Boris Ostrovsky <boris.ostrovsky@oracle.com>
Signed-off-by: Konrad Rzeszutek Wilk <konrad.wilk@oracle.com>
---
 arch/x86/xen/apic.c | 15 ++++++++++++---
 1 file changed, 12 insertions(+), 3 deletions(-)

diff --git a/arch/x86/xen/apic.c b/arch/x86/xen/apic.c
index 86bea3e..de1c74d 100644
--- a/arch/x86/xen/apic.c
+++ b/arch/x86/xen/apic.c
@@ -56,7 +56,10 @@ static u32 xen_apic_read(u32 reg)
 
 	if (reg == APIC_LVR)
 		return 0x10;
-
+#ifdef CONFIG_X86_32
+	if (reg == APIC_LDR)
+		return SET_APIC_LOGICAL_ID(1UL << smp_processor_id());
+#endif
 	if (reg != APIC_ID)
 		return 0;
 
@@ -117,6 +120,12 @@ static int xen_phys_pkg_id(int initial_apic_id, int index_msb)
 	return initial_apic_id >> index_msb;
 }
 
+static int xen_x86_32_early_logical_apicid(int cpu)
+{
+	/* Match with APIC_LDR read. Otherwise setup_local_APIC complains. */
+	return 1 << cpu;
+}
+
 static void xen_noop(void)
 {
 }
@@ -176,8 +185,8 @@ static struct apic xen_pv_apic = {
 	.safe_wait_icr_idle 		= xen_safe_apic_wait_icr_idle,
 
 #ifdef CONFIG_X86_32
-	/* generic_processor_info */
-	.x86_32_early_logical_apicid	= default_x86_32_early_logical_apicid,
+	/* generic_processor_info and setup_local_APIC. */
+	.x86_32_early_logical_apicid	= xen_x86_32_early_logical_apicid,
 #endif
 };
 
-- 
1.8.4.2

[PATCH 1/2] x86/xen/apic: Work with UP, non-SMP 32-bit kernels.

[Konrad Rzeszutek Wilk]

Most of the APIC code that use APIC_LDR is not used on
64-bit. On 32-bit it is bit of an hack - and the mechanism
it is uses is to "setup" the APIC_LDR via apci->init_apic_ldr
(which we set to NULL) and then use apic->x86_32_early_logical_apicid
to get an CPU to APIC ID mapping and also apic->read to match
the APIC_LDR.

The 'x86_32_early_logical_apicid' is called from 'setup_local_APIC'
which is called from the following functions:
 - start_secondary ->smp_callin -> apic_ap_setup [not called on PV]
 - native_smp_prepare_cpus -> apci_bsp_setup [not called on PV]
 - up_late_init -> APIC_init_uniprocessor -> apic_bsp_setup ->setup_local_APIC
   [called on PV with CONFIG_SMP not defined, CONFIG_X86_32 and UP kernel]

This patch fixes the build issue and also allows the bootup
to continue without warnings.

Reported-by: Boris Ostrovsky <boris.ostrovsky@oracle.com>
Signed-off-by: Konrad Rzeszutek Wilk <konrad.wilk@oracle.com>
---
 arch/x86/xen/apic.c | 15 ++++++++++++---
 1 file changed, 12 insertions(+), 3 deletions(-)

diff --git a/arch/x86/xen/apic.c b/arch/x86/xen/apic.c
index 86bea3e..de1c74d 100644
--- a/arch/x86/xen/apic.c
+++ b/arch/x86/xen/apic.c
@@ -56,7 +56,10 @@ static u32 xen_apic_read(u32 reg)
 
 	if (reg == APIC_LVR)
 		return 0x10;
-
+#ifdef CONFIG_X86_32
+	if (reg == APIC_LDR)
+		return SET_APIC_LOGICAL_ID(1UL << smp_processor_id());
+#endif
 	if (reg != APIC_ID)
 		return 0;
 
@@ -117,6 +120,12 @@ static int xen_phys_pkg_id(int initial_apic_id, int index_msb)
 	return initial_apic_id >> index_msb;
 }
 
+static int xen_x86_32_early_logical_apicid(int cpu)
+{
+	/* Match with APIC_LDR read. Otherwise setup_local_APIC complains. */
+	return 1 << cpu;
+}
+
 static void xen_noop(void)
 {
 }
@@ -176,8 +185,8 @@ static struct apic xen_pv_apic = {
 	.safe_wait_icr_idle 		= xen_safe_apic_wait_icr_idle,
 
 #ifdef CONFIG_X86_32
-	/* generic_processor_info */
-	.x86_32_early_logical_apicid	= default_x86_32_early_logical_apicid,
+	/* generic_processor_info and setup_local_APIC. */
+	.x86_32_early_logical_apicid	= xen_x86_32_early_logical_apicid,
 #endif
 };
 
-- 
1.8.4.2

Re: [PATCH 1/2] x86/xen/apic: Work with UP, non-SMP 32-bit kernels.

[Konrad Rzeszutek Wilk]

On Tue, Mar 03, 2015 at 03:20:41PM -0500, Konrad Rzeszutek Wilk wrote:
> Most of the APIC code that use APIC_LDR is not used on
> 64-bit. On 32-bit it is bit of an hack - and the mechanism
> it is uses is to "setup" the APIC_LDR via apci->init_apic_ldr
> (which we set to NULL) and then use apic->x86_32_early_logical_apicid
> to get an CPU to APIC ID mapping and also apic->read to match
> the APIC_LDR.
> 
> The 'x86_32_early_logical_apicid' is called from 'setup_local_APIC'
> which is called from the following functions:
>  - start_secondary ->smp_callin -> apic_ap_setup [not called on PV]
>  - native_smp_prepare_cpus -> apci_bsp_setup [not called on PV]
>  - up_late_init -> APIC_init_uniprocessor -> apic_bsp_setup ->setup_local_APIC
>    [called on PV with CONFIG_SMP not defined, CONFIG_X86_32 and UP kernel]
> 
> This patch fixes the build issue and also allows the bootup
> to continue without warnings.
> 
> Reported-by: Boris Ostrovsky <boris.ostrovsky@oracle.com>
> Signed-off-by: Konrad Rzeszutek Wilk <konrad.wilk@oracle.com>
> ---
>  arch/x86/xen/apic.c | 15 ++++++++++++---
>  1 file changed, 12 insertions(+), 3 deletions(-)
> 
> diff --git a/arch/x86/xen/apic.c b/arch/x86/xen/apic.c
> index 86bea3e..de1c74d 100644
> --- a/arch/x86/xen/apic.c
> +++ b/arch/x86/xen/apic.c
> @@ -56,7 +56,10 @@ static u32 xen_apic_read(u32 reg)
>  
>  	if (reg == APIC_LVR)
>  		return 0x10;
> -
> +#ifdef CONFIG_X86_32
> +	if (reg == APIC_LDR)
> +		return SET_APIC_LOGICAL_ID(1UL << smp_processor_id());
> +#endif
>  	if (reg != APIC_ID)
>  		return 0;
>  
> @@ -117,6 +120,12 @@ static int xen_phys_pkg_id(int initial_apic_id, int index_msb)
>  	return initial_apic_id >> index_msb;
>  }
>  
> +static int xen_x86_32_early_logical_apicid(int cpu)
> +{
> +	/* Match with APIC_LDR read. Otherwise setup_local_APIC complains. */
> +	return 1 << cpu;
> +}
> +

Argh. That should have been wrapped with CONFIG_X86_32 otherwise it will complain
on 64-bit (not used.. etc).

David, I can commit it in and fix it up.

>  static void xen_noop(void)
>  {
>  }
> @@ -176,8 +185,8 @@ static struct apic xen_pv_apic = {
>  	.safe_wait_icr_idle 		= xen_safe_apic_wait_icr_idle,
>  
>  #ifdef CONFIG_X86_32
> -	/* generic_processor_info */
> -	.x86_32_early_logical_apicid	= default_x86_32_early_logical_apicid,
> +	/* generic_processor_info and setup_local_APIC. */
> +	.x86_32_early_logical_apicid	= xen_x86_32_early_logical_apicid,
>  #endif
>  };
>  
> -- 
> 1.8.4.2
> 

Re: [PATCH 1/2] x86/xen/apic: Work with UP, non-SMP 32-bit kernels.

[David Vrabel]

On 03/03/15 20:29, Konrad Rzeszutek Wilk wrote:
> On Tue, Mar 03, 2015 at 03:20:41PM -0500, Konrad Rzeszutek Wilk wrote:
>> Most of the APIC code that use APIC_LDR is not used on
>> 64-bit. On 32-bit it is bit of an hack - and the mechanism
>> it is uses is to "setup" the APIC_LDR via apci->init_apic_ldr
>> (which we set to NULL) and then use apic->x86_32_early_logical_apicid
>> to get an CPU to APIC ID mapping and also apic->read to match
>> the APIC_LDR.
>>
>> The 'x86_32_early_logical_apicid' is called from 'setup_local_APIC'
>> which is called from the following functions:
>>  - start_secondary ->smp_callin -> apic_ap_setup [not called on PV]
>>  - native_smp_prepare_cpus -> apci_bsp_setup [not called on PV]
>>  - up_late_init -> APIC_init_uniprocessor -> apic_bsp_setup ->setup_local_APIC
>>    [called on PV with CONFIG_SMP not defined, CONFIG_X86_32 and UP kernel]
>>
>> This patch fixes the build issue and also allows the bootup
>> to continue without warnings.
>>
>> Reported-by: Boris Ostrovsky <boris.ostrovsky@oracle.com>
>> Signed-off-by: Konrad Rzeszutek Wilk <konrad.wilk@oracle.com>
>> ---
>>  arch/x86/xen/apic.c | 15 ++++++++++++---
>>  1 file changed, 12 insertions(+), 3 deletions(-)
>>
>> diff --git a/arch/x86/xen/apic.c b/arch/x86/xen/apic.c
>> index 86bea3e..de1c74d 100644
>> --- a/arch/x86/xen/apic.c
>> +++ b/arch/x86/xen/apic.c
>> @@ -56,7 +56,10 @@ static u32 xen_apic_read(u32 reg)
>>  
>>  	if (reg == APIC_LVR)
>>  		return 0x10;
>> -
>> +#ifdef CONFIG_X86_32
>> +	if (reg == APIC_LDR)
>> +		return SET_APIC_LOGICAL_ID(1UL << smp_processor_id());
>> +#endif
>>  	if (reg != APIC_ID)
>>  		return 0;
>>  
>> @@ -117,6 +120,12 @@ static int xen_phys_pkg_id(int initial_apic_id, int index_msb)
>>  	return initial_apic_id >> index_msb;
>>  }
>>  
>> +static int xen_x86_32_early_logical_apicid(int cpu)
>> +{
>> +	/* Match with APIC_LDR read. Otherwise setup_local_APIC complains. */
>> +	return 1 << cpu;
>> +}
>> +
> 
> Argh. That should have been wrapped with CONFIG_X86_32 otherwise it will complain
> on 64-bit (not used.. etc).
> 
> David, I can commit it in and fix it up.

Can you fold all fixes these into the original patch?  No need to
unnecessary break bisection.

David

Re: [PATCH 1/2] x86/xen/apic: Work with UP, non-SMP 32-bit kernels.

[David Vrabel]

On 03/03/15 20:29, Konrad Rzeszutek Wilk wrote:
> On Tue, Mar 03, 2015 at 03:20:41PM -0500, Konrad Rzeszutek Wilk wrote:
>> Most of the APIC code that use APIC_LDR is not used on
>> 64-bit. On 32-bit it is bit of an hack - and the mechanism
>> it is uses is to "setup" the APIC_LDR via apci->init_apic_ldr
>> (which we set to NULL) and then use apic->x86_32_early_logical_apicid
>> to get an CPU to APIC ID mapping and also apic->read to match
>> the APIC_LDR.
>>
>> The 'x86_32_early_logical_apicid' is called from 'setup_local_APIC'
>> which is called from the following functions:
>>  - start_secondary ->smp_callin -> apic_ap_setup [not called on PV]
>>  - native_smp_prepare_cpus -> apci_bsp_setup [not called on PV]
>>  - up_late_init -> APIC_init_uniprocessor -> apic_bsp_setup ->setup_local_APIC
>>    [called on PV with CONFIG_SMP not defined, CONFIG_X86_32 and UP kernel]
>>
>> This patch fixes the build issue and also allows the bootup
>> to continue without warnings.
>>
>> Reported-by: Boris Ostrovsky <boris.ostrovsky@oracle.com>
>> Signed-off-by: Konrad Rzeszutek Wilk <konrad.wilk@oracle.com>
>> ---
>>  arch/x86/xen/apic.c | 15 ++++++++++++---
>>  1 file changed, 12 insertions(+), 3 deletions(-)
>>
>> diff --git a/arch/x86/xen/apic.c b/arch/x86/xen/apic.c
>> index 86bea3e..de1c74d 100644
>> --- a/arch/x86/xen/apic.c
>> +++ b/arch/x86/xen/apic.c
>> @@ -56,7 +56,10 @@ static u32 xen_apic_read(u32 reg)
>>  
>>  	if (reg == APIC_LVR)
>>  		return 0x10;
>> -
>> +#ifdef CONFIG_X86_32
>> +	if (reg == APIC_LDR)
>> +		return SET_APIC_LOGICAL_ID(1UL << smp_processor_id());
>> +#endif
>>  	if (reg != APIC_ID)
>>  		return 0;
>>  
>> @@ -117,6 +120,12 @@ static int xen_phys_pkg_id(int initial_apic_id, int index_msb)
>>  	return initial_apic_id >> index_msb;
>>  }
>>  
>> +static int xen_x86_32_early_logical_apicid(int cpu)
>> +{
>> +	/* Match with APIC_LDR read. Otherwise setup_local_APIC complains. */
>> +	return 1 << cpu;
>> +}
>> +
> 
> Argh. That should have been wrapped with CONFIG_X86_32 otherwise it will complain
> on 64-bit (not used.. etc).
> 
> David, I can commit it in and fix it up.

Can you fold all fixes these into the original patch?  No need to
unnecessary break bisection.

David

Re: [PATCH 1/2] x86/xen/apic: Work with UP, non-SMP 32-bit kernels.

[Konrad Rzeszutek Wilk]

On Tue, Mar 03, 2015 at 03:20:41PM -0500, Konrad Rzeszutek Wilk wrote:
> Most of the APIC code that use APIC_LDR is not used on
> 64-bit. On 32-bit it is bit of an hack - and the mechanism
> it is uses is to "setup" the APIC_LDR via apci->init_apic_ldr
> (which we set to NULL) and then use apic->x86_32_early_logical_apicid
> to get an CPU to APIC ID mapping and also apic->read to match
> the APIC_LDR.
> 
> The 'x86_32_early_logical_apicid' is called from 'setup_local_APIC'
> which is called from the following functions:
>  - start_secondary ->smp_callin -> apic_ap_setup [not called on PV]
>  - native_smp_prepare_cpus -> apci_bsp_setup [not called on PV]
>  - up_late_init -> APIC_init_uniprocessor -> apic_bsp_setup ->setup_local_APIC
>    [called on PV with CONFIG_SMP not defined, CONFIG_X86_32 and UP kernel]
> 
> This patch fixes the build issue and also allows the bootup
> to continue without warnings.
> 
> Reported-by: Boris Ostrovsky <boris.ostrovsky@oracle.com>
> Signed-off-by: Konrad Rzeszutek Wilk <konrad.wilk@oracle.com>
> ---
>  arch/x86/xen/apic.c | 15 ++++++++++++---
>  1 file changed, 12 insertions(+), 3 deletions(-)
> 
> diff --git a/arch/x86/xen/apic.c b/arch/x86/xen/apic.c
> index 86bea3e..de1c74d 100644
> --- a/arch/x86/xen/apic.c
> +++ b/arch/x86/xen/apic.c
> @@ -56,7 +56,10 @@ static u32 xen_apic_read(u32 reg)
>  
>  	if (reg == APIC_LVR)
>  		return 0x10;
> -
> +#ifdef CONFIG_X86_32
> +	if (reg == APIC_LDR)
> +		return SET_APIC_LOGICAL_ID(1UL << smp_processor_id());
> +#endif
>  	if (reg != APIC_ID)
>  		return 0;
>  
> @@ -117,6 +120,12 @@ static int xen_phys_pkg_id(int initial_apic_id, int index_msb)
>  	return initial_apic_id >> index_msb;
>  }
>  
> +static int xen_x86_32_early_logical_apicid(int cpu)
> +{
> +	/* Match with APIC_LDR read. Otherwise setup_local_APIC complains. */
> +	return 1 << cpu;
> +}
> +

Argh. That should have been wrapped with CONFIG_X86_32 otherwise it will complain
on 64-bit (not used.. etc).

David, I can commit it in and fix it up.

>  static void xen_noop(void)
>  {
>  }
> @@ -176,8 +185,8 @@ static struct apic xen_pv_apic = {
>  	.safe_wait_icr_idle 		= xen_safe_apic_wait_icr_idle,
>  
>  #ifdef CONFIG_X86_32
> -	/* generic_processor_info */
> -	.x86_32_early_logical_apicid	= default_x86_32_early_logical_apicid,
> +	/* generic_processor_info and setup_local_APIC. */
> +	.x86_32_early_logical_apicid	= xen_x86_32_early_logical_apicid,
>  #endif
>  };
>  
> -- 
> 1.8.4.2
> 

[PATCH 2/2] x86/xen/apic: WARN with details.

[Konrad Rzeszutek Wilk]

We should not be writting to the APIC registers under
Xen PV. But if we do instead of just giving an blanket
warning - include some details to help troubleshoot.

Signed-off-by: Konrad Rzeszutek Wilk <konrad.wilk@oracle.com>
---
 arch/x86/xen/apic.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/arch/x86/xen/apic.c b/arch/x86/xen/apic.c
index de1c74d..28d6531 100644
--- a/arch/x86/xen/apic.c
+++ b/arch/x86/xen/apic.c
@@ -73,7 +73,7 @@ static u32 xen_apic_read(u32 reg)
 static void xen_apic_write(u32 reg, u32 val)
 {
 	/* Warn to see if there's any stray references */
-	WARN_ON(1);
+	WARN(1,"register: %x, value: %x\n", reg, val);
 }
 
 static u64 xen_apic_icr_read(void)
-- 
1.8.4.2

[PATCH 2/2] x86/xen/apic: WARN with details.

[Konrad Rzeszutek Wilk]

We should not be writting to the APIC registers under
Xen PV. But if we do instead of just giving an blanket
warning - include some details to help troubleshoot.

Signed-off-by: Konrad Rzeszutek Wilk <konrad.wilk@oracle.com>
---
 arch/x86/xen/apic.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/arch/x86/xen/apic.c b/arch/x86/xen/apic.c
index de1c74d..28d6531 100644
--- a/arch/x86/xen/apic.c
+++ b/arch/x86/xen/apic.c
@@ -73,7 +73,7 @@ static u32 xen_apic_read(u32 reg)
 static void xen_apic_write(u32 reg, u32 val)
 {
 	/* Warn to see if there's any stray references */
-	WARN_ON(1);
+	WARN(1,"register: %x, value: %x\n", reg, val);
 }
 
 static u64 xen_apic_icr_read(void)
-- 
1.8.4.2