[kvm-unit-tests PATCH v3 0/6] s390x: Add multiboot and smp

[kvm-unit-tests PATCH v3 0/6] s390x: Add multiboot and smp

[Janosch Frank]

Cross testing emulated instructions has in the past brought up some
issues on all available IBM Z hypervisors. So let's upstream the last
part of multiboot: sclp interrupts and line mode console.

SMP tests are a great way to excercise external interruptions, cpu
resets and sigp. The smp library is always initialized and provides
very rudimentary CPU management for now.

v3:
 * Simplified linemode buffering
 * Removed error constant use
 * Smaller fixes

Janosch Frank (6):
  s390x: Use interrupts in SCLP and add locking
  s390x: Add linemode console
  s390x: Add linemode buffer to fix newline on every print
  s390x: Add initial smp code
  s390x: Prepare for external calls
  s390x: SMP test

 lib/s390x/asm/arch_def.h  |  13 ++
 lib/s390x/asm/interrupt.h |   5 +
 lib/s390x/asm/sigp.h      |  28 +++-
 lib/s390x/interrupt.c     |  27 +++-
 lib/s390x/io.c            |   5 +-
 lib/s390x/sclp-console.c  | 218 ++++++++++++++++++++++++++++---
 lib/s390x/sclp.c          |  55 +++++++-
 lib/s390x/sclp.h          |  59 ++++++++-
 lib/s390x/smp.c           | 263 ++++++++++++++++++++++++++++++++++++++
 lib/s390x/smp.h           |  51 ++++++++
 s390x/Makefile            |   2 +
 s390x/cstart64.S          |   7 +
 s390x/smp.c               | 242 +++++++++++++++++++++++++++++++++++
 s390x/unittests.cfg       |   4 +
 14 files changed, 952 insertions(+), 27 deletions(-)
 create mode 100644 lib/s390x/smp.c
 create mode 100644 lib/s390x/smp.h
 create mode 100644 s390x/smp.c

-- 
2.17.2

[kvm-unit-tests PATCH v3 1/6] s390x: Use interrupts in SCLP and add locking

[Janosch Frank]

We need to properly implement interrupt handling for SCLP, because on
z/VM and LPAR SCLP calls are not synchronous!

Also with smp CPUs have to compete for sclp. Let's add some locking,
so they execute sclp calls in an orderly fashion and don't compete for
the data buffer.

Signed-off-by: Janosch Frank <frankja@linux.ibm.com>
Reviewed-by: Thomas Huth <thuth@redhat.com>
---
 lib/s390x/asm/interrupt.h |  2 ++
 lib/s390x/interrupt.c     | 12 +++++++--
 lib/s390x/sclp-console.c  |  2 ++
 lib/s390x/sclp.c          | 55 +++++++++++++++++++++++++++++++++++++--
 lib/s390x/sclp.h          |  3 +++
 5 files changed, 70 insertions(+), 4 deletions(-)

diff --git a/lib/s390x/asm/interrupt.h b/lib/s390x/asm/interrupt.h
index 013709f..f485e96 100644
--- a/lib/s390x/asm/interrupt.h
+++ b/lib/s390x/asm/interrupt.h
@@ -11,6 +11,8 @@
 #define _ASMS390X_IRQ_H_
 #include <asm/arch_def.h>
 
+#define EXT_IRQ_SERVICE_SIG	0x2401
+
 void handle_pgm_int(void);
 void handle_ext_int(void);
 void handle_mcck_int(void);
diff --git a/lib/s390x/interrupt.c b/lib/s390x/interrupt.c
index cf0a794..7832711 100644
--- a/lib/s390x/interrupt.c
+++ b/lib/s390x/interrupt.c
@@ -12,6 +12,7 @@
 #include <libcflat.h>
 #include <asm/interrupt.h>
 #include <asm/barrier.h>
+#include <sclp.h>
 
 static bool pgm_int_expected;
 static struct lowcore *lc;
@@ -107,8 +108,15 @@ void handle_pgm_int(void)
 
 void handle_ext_int(void)
 {
-	report_abort("Unexpected external call interrupt: at %#lx",
-		     lc->ext_old_psw.addr);
+	if (lc->ext_int_code != EXT_IRQ_SERVICE_SIG) {
+		report_abort("Unexpected external call interrupt: at %#lx",
+			     lc->ext_old_psw.addr);
+	} else {
+		lc->ext_old_psw.mask &= ~PSW_MASK_EXT;
+		lc->sw_int_cr0 &= ~(1UL << 9);
+		sclp_handle_ext();
+		lc->ext_int_code = 0;
+	}
 }
 
 void handle_mcck_int(void)
diff --git a/lib/s390x/sclp-console.c b/lib/s390x/sclp-console.c
index bc01f41..a5ef45f 100644
--- a/lib/s390x/sclp-console.c
+++ b/lib/s390x/sclp-console.c
@@ -17,6 +17,7 @@ static void sclp_set_write_mask(void)
 {
 	WriteEventMask *sccb = (void *)_sccb;
 
+	sclp_mark_busy();
 	sccb->h.length = sizeof(WriteEventMask);
 	sccb->mask_length = sizeof(unsigned int);
 	sccb->receive_mask = SCLP_EVENT_MASK_MSG_ASCII;
@@ -37,6 +38,7 @@ void sclp_print(const char *str)
 	int len = strlen(str);
 	WriteEventData *sccb = (void *)_sccb;
 
+	sclp_mark_busy();
 	sccb->h.length = sizeof(WriteEventData) + len;
 	sccb->h.function_code = SCLP_FC_NORMAL_WRITE;
 	sccb->ebh.length = sizeof(EventBufferHeader) + len;
diff --git a/lib/s390x/sclp.c b/lib/s390x/sclp.c
index b60f7a4..56fca0c 100644
--- a/lib/s390x/sclp.c
+++ b/lib/s390x/sclp.c
@@ -14,6 +14,8 @@
 #include <asm/page.h>
 #include <asm/arch_def.h>
 #include <asm/interrupt.h>
+#include <asm/barrier.h>
+#include <asm/spinlock.h>
 #include "sclp.h"
 #include <alloc_phys.h>
 #include <alloc_page.h>
@@ -25,6 +27,8 @@ static uint64_t max_ram_size;
 static uint64_t ram_size;
 
 char _sccb[PAGE_SIZE] __attribute__((__aligned__(4096)));
+static volatile bool sclp_busy;
+static struct spinlock sclp_lock;
 
 static void mem_init(phys_addr_t mem_end)
 {
@@ -41,17 +45,62 @@ static void mem_init(phys_addr_t mem_end)
 	page_alloc_ops_enable();
 }
 
+static void sclp_setup_int(void)
+{
+	uint64_t mask;
+
+	ctl_set_bit(0, 9);
+
+	mask = extract_psw_mask();
+	mask |= PSW_MASK_EXT;
+	load_psw_mask(mask);
+}
+
+void sclp_handle_ext(void)
+{
+	ctl_clear_bit(0, 9);
+	spin_lock(&sclp_lock);
+	sclp_busy = false;
+	spin_unlock(&sclp_lock);
+}
+
+void sclp_wait_busy(void)
+{
+	while (sclp_busy)
+		mb();
+}
+
+void sclp_mark_busy(void)
+{
+	/*
+	 * With multiple CPUs we might need to wait for another CPU's
+	 * request before grabbing the busy indication.
+	 */
+	while (true) {
+		sclp_wait_busy();
+		spin_lock(&sclp_lock);
+		if (!sclp_busy) {
+			sclp_busy = true;
+			spin_unlock(&sclp_lock);
+			return;
+		}
+		spin_unlock(&sclp_lock);
+	}
+}
+
 static void sclp_read_scp_info(ReadInfo *ri, int length)
 {
 	unsigned int commands[] = { SCLP_CMDW_READ_SCP_INFO_FORCED,
 				    SCLP_CMDW_READ_SCP_INFO };
-	int i;
+	int i, cc;
 
 	for (i = 0; i < ARRAY_SIZE(commands); i++) {
+		sclp_mark_busy();
 		memset(&ri->h, 0, sizeof(ri->h));
 		ri->h.length = length;
 
-		if (sclp_service_call(commands[i], ri))
+		cc = sclp_service_call(commands[i], ri);
+		if (cc)
 			break;
 		if (ri->h.response_code == SCLP_RC_NORMAL_READ_COMPLETION)
 			return;
@@ -66,12 +115,14 @@ int sclp_service_call(unsigned int command, void *sccb)
 {
 	int cc;
 
+	sclp_setup_int();
 	asm volatile(
 		"       .insn   rre,0xb2200000,%1,%2\n"  /* servc %1,%2 */
 		"       ipm     %0\n"
 		"       srl     %0,28"
 		: "=&d" (cc) : "d" (command), "a" (__pa(sccb))
 		: "cc", "memory");
+	sclp_wait_busy();
 	if (cc == 3)
 		return -1;
 	if (cc == 2)
diff --git a/lib/s390x/sclp.h b/lib/s390x/sclp.h
index 583c4e5..63cf609 100644
--- a/lib/s390x/sclp.h
+++ b/lib/s390x/sclp.h
@@ -213,6 +213,9 @@ typedef struct ReadEventData {
 } __attribute__((packed)) ReadEventData;
 
 extern char _sccb[];
+void sclp_handle_ext(void);
+void sclp_wait_busy(void);
+void sclp_mark_busy(void);
 void sclp_console_setup(void);
 void sclp_print(const char *str);
 int sclp_service_call(unsigned int command, void *sccb);
-- 
2.17.2

Re: [kvm-unit-tests PATCH v3 1/6] s390x: Use interrupts in SCLP and add locking

[David Hildenbrand]

On 20.09.19 10:03, Janosch Frank wrote:
> We need to properly implement interrupt handling for SCLP, because on
> z/VM and LPAR SCLP calls are not synchronous!
> 
> Also with smp CPUs have to compete for sclp. Let's add some locking,
> so they execute sclp calls in an orderly fashion and don't compete for
> the data buffer.
> 
> Signed-off-by: Janosch Frank <frankja@linux.ibm.com>
> Reviewed-by: Thomas Huth <thuth@redhat.com>
> ---
>  lib/s390x/asm/interrupt.h |  2 ++
>  lib/s390x/interrupt.c     | 12 +++++++--
>  lib/s390x/sclp-console.c  |  2 ++
>  lib/s390x/sclp.c          | 55 +++++++++++++++++++++++++++++++++++++--
>  lib/s390x/sclp.h          |  3 +++
>  5 files changed, 70 insertions(+), 4 deletions(-)
> 
> diff --git a/lib/s390x/asm/interrupt.h b/lib/s390x/asm/interrupt.h
> index 013709f..f485e96 100644
> --- a/lib/s390x/asm/interrupt.h
> +++ b/lib/s390x/asm/interrupt.h
> @@ -11,6 +11,8 @@
>  #define _ASMS390X_IRQ_H_
>  #include <asm/arch_def.h>
>  
> +#define EXT_IRQ_SERVICE_SIG	0x2401
> +
>  void handle_pgm_int(void);
>  void handle_ext_int(void);
>  void handle_mcck_int(void);
> diff --git a/lib/s390x/interrupt.c b/lib/s390x/interrupt.c
> index cf0a794..7832711 100644
> --- a/lib/s390x/interrupt.c
> +++ b/lib/s390x/interrupt.c
> @@ -12,6 +12,7 @@
>  #include <libcflat.h>
>  #include <asm/interrupt.h>
>  #include <asm/barrier.h>
> +#include <sclp.h>
>  
>  static bool pgm_int_expected;
>  static struct lowcore *lc;
> @@ -107,8 +108,15 @@ void handle_pgm_int(void)
>  
>  void handle_ext_int(void)
>  {
> -	report_abort("Unexpected external call interrupt: at %#lx",
> -		     lc->ext_old_psw.addr);
> +	if (lc->ext_int_code != EXT_IRQ_SERVICE_SIG) {
> +		report_abort("Unexpected external call interrupt: at %#lx",
> +			     lc->ext_old_psw.addr);
> +	} else {
> +		lc->ext_old_psw.mask &= ~PSW_MASK_EXT;
> +		lc->sw_int_cr0 &= ~(1UL << 9);
> +		sclp_handle_ext();
> +		lc->ext_int_code = 0;
> +	}
>  }
>  
>  void handle_mcck_int(void)
> diff --git a/lib/s390x/sclp-console.c b/lib/s390x/sclp-console.c
> index bc01f41..a5ef45f 100644
> --- a/lib/s390x/sclp-console.c
> +++ b/lib/s390x/sclp-console.c
> @@ -17,6 +17,7 @@ static void sclp_set_write_mask(void)
>  {
>  	WriteEventMask *sccb = (void *)_sccb;
>  
> +	sclp_mark_busy();
>  	sccb->h.length = sizeof(WriteEventMask);
>  	sccb->mask_length = sizeof(unsigned int);
>  	sccb->receive_mask = SCLP_EVENT_MASK_MSG_ASCII;
> @@ -37,6 +38,7 @@ void sclp_print(const char *str)
>  	int len = strlen(str);
>  	WriteEventData *sccb = (void *)_sccb;
>  
> +	sclp_mark_busy();
>  	sccb->h.length = sizeof(WriteEventData) + len;
>  	sccb->h.function_code = SCLP_FC_NORMAL_WRITE;
>  	sccb->ebh.length = sizeof(EventBufferHeader) + len;
> diff --git a/lib/s390x/sclp.c b/lib/s390x/sclp.c
> index b60f7a4..56fca0c 100644
> --- a/lib/s390x/sclp.c
> +++ b/lib/s390x/sclp.c
> @@ -14,6 +14,8 @@
>  #include <asm/page.h>
>  #include <asm/arch_def.h>
>  #include <asm/interrupt.h>
> +#include <asm/barrier.h>
> +#include <asm/spinlock.h>
>  #include "sclp.h"
>  #include <alloc_phys.h>
>  #include <alloc_page.h>
> @@ -25,6 +27,8 @@ static uint64_t max_ram_size;
>  static uint64_t ram_size;
>  
>  char _sccb[PAGE_SIZE] __attribute__((__aligned__(4096)));
> +static volatile bool sclp_busy;
> +static struct spinlock sclp_lock;
>  
>  static void mem_init(phys_addr_t mem_end)
>  {
> @@ -41,17 +45,62 @@ static void mem_init(phys_addr_t mem_end)
>  	page_alloc_ops_enable();
>  }
>  
> +static void sclp_setup_int(void)
> +{
> +	uint64_t mask;
> +
> +	ctl_set_bit(0, 9);
> +
> +	mask = extract_psw_mask();
> +	mask |= PSW_MASK_EXT;
> +	load_psw_mask(mask);
> +}
> +
> +void sclp_handle_ext(void)
> +{
> +	ctl_clear_bit(0, 9);
> +	spin_lock(&sclp_lock);
> +	sclp_busy = false;
> +	spin_unlock(&sclp_lock);
> +}
> +
> +void sclp_wait_busy(void)
> +{
> +	while (sclp_busy)
> +		mb();
> +}
> +
> +void sclp_mark_busy(void)
> +{
> +	/*
> +	 * With multiple CPUs we might need to wait for another CPU's
> +	 * request before grabbing the busy indication.
> +	 */
> +	while (true) {
> +		sclp_wait_busy();
> +		spin_lock(&sclp_lock);
> +		if (!sclp_busy) {
> +			sclp_busy = true;
> +			spin_unlock(&sclp_lock);
> +			return;
> +		}
> +		spin_unlock(&sclp_lock);
> +	}
> +}
> +
>  static void sclp_read_scp_info(ReadInfo *ri, int length)
>  {
>  	unsigned int commands[] = { SCLP_CMDW_READ_SCP_INFO_FORCED,
>  				    SCLP_CMDW_READ_SCP_INFO };
> -	int i;
> +	int i, cc;
>  
>  	for (i = 0; i < ARRAY_SIZE(commands); i++) {
> +		sclp_mark_busy();
>  		memset(&ri->h, 0, sizeof(ri->h));
>  		ri->h.length = length;
>  
> -		if (sclp_service_call(commands[i], ri))
> +		cc = sclp_service_call(commands[i], ri);
> +		if (cc)
>  			break;
>  		if (ri->h.response_code == SCLP_RC_NORMAL_READ_COMPLETION)
>  			return;
> @@ -66,12 +115,14 @@ int sclp_service_call(unsigned int command, void *sccb)
>  {
>  	int cc;
>  
> +	sclp_setup_int();
>  	asm volatile(
>  		"       .insn   rre,0xb2200000,%1,%2\n"  /* servc %1,%2 */
>  		"       ipm     %0\n"
>  		"       srl     %0,28"
>  		: "=&d" (cc) : "d" (command), "a" (__pa(sccb))
>  		: "cc", "memory");
> +	sclp_wait_busy();
>  	if (cc == 3)
>  		return -1;
>  	if (cc == 2)
> diff --git a/lib/s390x/sclp.h b/lib/s390x/sclp.h
> index 583c4e5..63cf609 100644
> --- a/lib/s390x/sclp.h
> +++ b/lib/s390x/sclp.h
> @@ -213,6 +213,9 @@ typedef struct ReadEventData {
>  } __attribute__((packed)) ReadEventData;
>  
>  extern char _sccb[];
> +void sclp_handle_ext(void);
> +void sclp_wait_busy(void);
> +void sclp_mark_busy(void);
>  void sclp_console_setup(void);
>  void sclp_print(const char *str);
>  int sclp_service_call(unsigned int command, void *sccb);
> 

Acked-by: David Hildenbrand <david@redhat.com>

-- 

Thanks,

David / dhildenb

[kvm-unit-tests PATCH v3 2/6] s390x: Add linemode console

[Janosch Frank]

z/VM isn't fond of vt220, so we need line mode when running under
z/VM.

Signed-off-by: Janosch Frank <frankja@linux.ibm.com>
Acked-by: David Hildenbrand <david@redhat.com>
Tested-by: Thomas Huth <thuth@redhat.com>
---
 lib/s390x/sclp-console.c | 181 +++++++++++++++++++++++++++++++++++----
 lib/s390x/sclp.h         |  55 +++++++++++-
 2 files changed, 218 insertions(+), 18 deletions(-)

diff --git a/lib/s390x/sclp-console.c b/lib/s390x/sclp-console.c
index a5ef45f..19416b5 100644
--- a/lib/s390x/sclp-console.c
+++ b/lib/s390x/sclp-console.c
@@ -11,21 +11,165 @@
 #include <libcflat.h>
 #include <string.h>
 #include <asm/page.h>
+#include <asm/arch_def.h>
+#include <asm/io.h>
 #include "sclp.h"
 
+/*
+ * ASCII (IBM PC 437) -> EBCDIC 037
+ */
+static uint8_t _ascebc[256] = {
+ /*00 NUL   SOH   STX   ETX   EOT   ENQ   ACK   BEL */
+     0x00, 0x01, 0x02, 0x03, 0x37, 0x2D, 0x2E, 0x2F,
+ /*08  BS    HT    LF    VT    FF    CR    SO    SI */
+ /*              ->NL                               */
+     0x16, 0x05, 0x15, 0x0B, 0x0C, 0x0D, 0x0E, 0x0F,
+ /*10 DLE   DC1   DC2   DC3   DC4   NAK   SYN   ETB */
+     0x10, 0x11, 0x12, 0x13, 0x3C, 0x3D, 0x32, 0x26,
+ /*18 CAN    EM   SUB   ESC    FS    GS    RS    US */
+ /*                               ->IGS ->IRS ->IUS */
+     0x18, 0x19, 0x3F, 0x27, 0x22, 0x1D, 0x1E, 0x1F,
+ /*20  SP     !     "     #     $     %     &     ' */
+     0x40, 0x5A, 0x7F, 0x7B, 0x5B, 0x6C, 0x50, 0x7D,
+ /*28   (     )     *     +     ,     -    .      / */
+     0x4D, 0x5D, 0x5C, 0x4E, 0x6B, 0x60, 0x4B, 0x61,
+ /*30   0     1     2     3     4     5     6     7 */
+     0xF0, 0xF1, 0xF2, 0xF3, 0xF4, 0xF5, 0xF6, 0xF7,
+ /*38   8     9     :     ;     <     =     >     ? */
+     0xF8, 0xF9, 0x7A, 0x5E, 0x4C, 0x7E, 0x6E, 0x6F,
+ /*40   @     A     B     C     D     E     F     G */
+     0x7C, 0xC1, 0xC2, 0xC3, 0xC4, 0xC5, 0xC6, 0xC7,
+ /*48   H     I     J     K     L     M     N     O */
+     0xC8, 0xC9, 0xD1, 0xD2, 0xD3, 0xD4, 0xD5, 0xD6,
+ /*50   P     Q     R     S     T     U     V     W */
+     0xD7, 0xD8, 0xD9, 0xE2, 0xE3, 0xE4, 0xE5, 0xE6,
+ /*58   X     Y     Z     [     \     ]     ^     _ */
+     0xE7, 0xE8, 0xE9, 0xBA, 0xE0, 0xBB, 0xB0, 0x6D,
+ /*60   `     a     b     c     d     e     f     g */
+     0x79, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87,
+ /*68   h     i     j     k     l     m     n     o */
+     0x88, 0x89, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96,
+ /*70   p     q     r     s     t     u     v     w */
+     0x97, 0x98, 0x99, 0xA2, 0xA3, 0xA4, 0xA5, 0xA6,
+ /*78   x     y     z     {     |     }     ~    DL */
+     0xA7, 0xA8, 0xA9, 0xC0, 0x4F, 0xD0, 0xA1, 0x07,
+ /*80*/
+     0x3F, 0x3F, 0x3F, 0x3F, 0x3F, 0x3F, 0x3F, 0x3F,
+ /*88*/
+     0x3F, 0x3F, 0x3F, 0x3F, 0x3F, 0x3F, 0x3F, 0x3F,
+ /*90*/
+     0x3F, 0x3F, 0x3F, 0x3F, 0x3F, 0x3F, 0x3F, 0x3F,
+ /*98*/
+     0x3F, 0x3F, 0x3F, 0x3F, 0x3F, 0x3F, 0x3F, 0x3F,
+ /*A0*/
+     0x3F, 0x3F, 0x3F, 0x3F, 0x3F, 0x3F, 0x3F, 0x3F,
+ /*A8*/
+     0x3F, 0x3F, 0x3F, 0x3F, 0x3F, 0x3F, 0x3F, 0x3F,
+ /*B0*/
+     0x3F, 0x3F, 0x3F, 0x3F, 0x3F, 0x3F, 0x3F, 0x3F,
+ /*B8*/
+     0x3F, 0x3F, 0x3F, 0x3F, 0x3F, 0x3F, 0x3F, 0x3F,
+ /*C0*/
+     0x3F, 0x3F, 0x3F, 0x3F, 0x3F, 0x3F, 0x3F, 0x3F,
+ /*C8*/
+     0x3F, 0x3F, 0x3F, 0x3F, 0x3F, 0x3F, 0x3F, 0x3F,
+ /*D0*/
+     0x3F, 0x3F, 0x3F, 0x3F, 0x3F, 0x3F, 0x3F, 0x3F,
+ /*D8*/
+     0x3F, 0x3F, 0x3F, 0x3F, 0x3F, 0x3F, 0x3F, 0x3F,
+ /*E0        sz	*/
+     0x3F, 0x59, 0x3F, 0x3F, 0x3F, 0x3F, 0x3F, 0x3F,
+ /*E8*/
+     0x3F, 0x3F, 0x3F, 0x3F, 0x3F, 0x3F, 0x3F, 0x3F,
+ /*F0*/
+     0x3F, 0x3F, 0x3F, 0x3F, 0x3F, 0x3F, 0x3F, 0x3F,
+ /*F8*/
+     0x90, 0x3F, 0x3F, 0x3F, 0x3F, 0xEA, 0x3F, 0xFF
+};
+
+static void sclp_print_ascii(const char *str)
+{
+	int len = strlen(str);
+	WriteEventData *sccb = (void *)_sccb;
+
+	sclp_mark_busy();
+	memset(sccb, 0, sizeof(*sccb));
+	sccb->h.length = offsetof(WriteEventData, msg) + len;
+	sccb->h.function_code = SCLP_FC_NORMAL_WRITE;
+	sccb->ebh.length = sizeof(EventBufferHeader) + len;
+	sccb->ebh.type = SCLP_EVENT_ASCII_CONSOLE_DATA;
+	memcpy(&sccb->msg, str, len);
+
+	sclp_service_call(SCLP_CMD_WRITE_EVENT_DATA, sccb);
+}
+
+static void sclp_print_lm(const char *str)
+{
+	unsigned char *ptr, *end, ch;
+	unsigned int count, offset, len;
+	struct WriteEventData *sccb;
+	struct mdb *mdb;
+	struct mto *mto;
+	struct go *go;
+
+	sclp_mark_busy();
+	sccb = (struct WriteEventData *) _sccb;
+	end = (unsigned char *) sccb + 4096 - 1;
+	memset(sccb, 0, sizeof(*sccb));
+	ptr = (unsigned char *) &sccb->msg.mdb.mto;
+	len = strlen(str);
+	offset = 0;
+	do {
+		for (count = sizeof(*mto); offset < len; count++) {
+			ch = str[offset++];
+			if (ch == 0x0a || ptr + count > end)
+				break;
+			ptr[count] = _ascebc[ch];
+		}
+		mto = (struct mto *) ptr;
+		mto->length = count;
+		mto->type = 4;
+		mto->line_type_flags = LNTPFLGS_ENDTEXT;
+		ptr += count;
+	} while (offset < len && ptr + sizeof(*mto) <= end);
+	len = ptr - (unsigned char *) sccb;
+	sccb->h.length = len - offsetof(struct WriteEventData, h);
+	sccb->h.function_code = SCLP_FC_NORMAL_WRITE;
+	sccb->ebh.type = EVTYP_MSG;
+	sccb->ebh.length = len - offsetof(struct WriteEventData, ebh);
+	mdb = &sccb->msg.mdb;
+	mdb->header.type = 1;
+	mdb->header.tag = 0xD4C4C240;
+	mdb->header.revision_code = 1;
+	mdb->header.length = len - offsetof(struct WriteEventData, msg.mdb.header);
+	go = &mdb->go;
+	go->length = sizeof(*go);
+	go->type = 1;
+	sclp_service_call(SCLP_CMD_WRITE_EVENT_DATA, sccb);
+}
+
+/*
+ * SCLP needs to be initialized by setting a send and receive mask,
+ * indicating which messages the control program (we) want(s) to
+ * send/receive.
+ */
 static void sclp_set_write_mask(void)
 {
 	WriteEventMask *sccb = (void *)_sccb;
 
 	sclp_mark_busy();
+	memset(_sccb, 0, sizeof(*sccb));
 	sccb->h.length = sizeof(WriteEventMask);
-	sccb->mask_length = sizeof(unsigned int);
-	sccb->receive_mask = SCLP_EVENT_MASK_MSG_ASCII;
-	sccb->cp_receive_mask = SCLP_EVENT_MASK_MSG_ASCII;
-	sccb->send_mask = SCLP_EVENT_MASK_MSG_ASCII;
-	sccb->cp_send_mask = SCLP_EVENT_MASK_MSG_ASCII;
+	sccb->h.function_code = SCLP_FC_NORMAL_WRITE;
+	sccb->mask_length = sizeof(sccb_mask_t);
+
+	/* For now we don't process sclp input. */
+	sccb->cp_receive_mask = 0;
+	/* We send ASCII and line mode. */
+	sccb->cp_send_mask = SCLP_EVENT_MASK_MSG_ASCII | SCLP_EVENT_MASK_MSG;
 
 	sclp_service_call(SCLP_CMD_WRITE_EVENT_MASK, sccb);
+	assert(sccb->h.response_code == SCLP_RC_NORMAL_COMPLETION);
 }
 
 void sclp_console_setup(void)
@@ -35,16 +179,19 @@ void sclp_console_setup(void)
 
 void sclp_print(const char *str)
 {
-	int len = strlen(str);
-	WriteEventData *sccb = (void *)_sccb;
-
-	sclp_mark_busy();
-	sccb->h.length = sizeof(WriteEventData) + len;
-	sccb->h.function_code = SCLP_FC_NORMAL_WRITE;
-	sccb->ebh.length = sizeof(EventBufferHeader) + len;
-	sccb->ebh.type = SCLP_EVENT_ASCII_CONSOLE_DATA;
-	sccb->ebh.flags = 0;
-	memcpy(sccb->data, str, len);
-
-	sclp_service_call(SCLP_CMD_WRITE_EVENT_DATA, sccb);
+	/*
+	 * z/VM advertises a vt220 console which is not functional:
+	 * (response code 05F0, "not active because of the state of
+	 * the machine"). Hence testing the masks would only work if
+	 * we also use stsi data to distinguish z/VM.
+	 *
+	 * Let's rather print on all available consoles.
+	 */
+	if (strlen(str) > (PAGE_SIZE / 2)) {
+		sclp_print_ascii("Warning: Printing is limited to 2KB of data.");
+		sclp_print_lm("Warning: Printing is limited to 2KB of data.");
+		return;
+	}
+	sclp_print_ascii(str);
+	sclp_print_lm(str);
 }
diff --git a/lib/s390x/sclp.h b/lib/s390x/sclp.h
index 63cf609..98c482a 100644
--- a/lib/s390x/sclp.h
+++ b/lib/s390x/sclp.h
@@ -179,6 +179,7 @@ typedef struct SCCB {
 /* SCLP event masks */
 #define SCLP_EVENT_MASK_SIGNAL_QUIESCE          0x00000008
 #define SCLP_EVENT_MASK_MSG_ASCII               0x00000040
+#define SCLP_EVENT_MASK_MSG          		0x40000000
 
 #define SCLP_UNCONDITIONAL_READ                 0x00
 #define SCLP_SELECTIVE_READ                     0x01
@@ -193,6 +194,55 @@ typedef struct WriteEventMask {
     uint32_t receive_mask;
 } __attribute__((packed)) WriteEventMask;
 
+#define MDBTYP_GO               0x0001
+#define MDBTYP_MTO              0x0004
+#define EVTYP_MSG               0x02
+#define LNTPFLGS_CNTLTEXT       0x8000
+#define LNTPFLGS_LABELTEXT      0x4000
+#define LNTPFLGS_DATATEXT       0x2000
+#define LNTPFLGS_ENDTEXT        0x1000
+#define LNTPFLGS_PROMPTTEXT     0x0800
+
+typedef uint32_t sccb_mask_t;
+
+/* SCLP line mode console related structures. */
+
+struct mto {
+	u16 length;
+	u16 type;
+	u16 line_type_flags;
+	u8  alarm_control;
+	u8  _reserved[3];
+} __attribute__((packed));
+
+struct go {
+	u16 length;
+	u16 type;
+	u32 domid;
+	u8  hhmmss_time[8];
+	u8  th_time[3];
+	u8  reserved_0;
+	u8  dddyyyy_date[7];
+	u8  _reserved_1;
+	u16 general_msg_flags;
+	u8  _reserved_2[10];
+	u8  originating_system_name[8];
+	u8  job_guest_name[8];
+} __attribute__((packed));
+
+struct mdb_header {
+	u16 length;
+	u16 type;
+	u32 tag;
+	u32 revision_code;
+} __attribute__((packed));
+
+struct mdb {
+	struct mdb_header header;
+	struct go go;
+	struct mto mto;
+} __attribute__((packed));
+
 typedef struct EventBufferHeader {
     uint16_t length;
     uint8_t  type;
@@ -203,7 +253,10 @@ typedef struct EventBufferHeader {
 typedef struct WriteEventData {
     SCCBHeader h;
     EventBufferHeader ebh;
-    char data[0];
+    union {
+	char data[0];
+	struct mdb mdb;
+    } msg;
 } __attribute__((packed)) WriteEventData;
 
 typedef struct ReadEventData {
-- 
2.17.2

[kvm-unit-tests PATCH v3 3/6] s390x: Add linemode buffer to fix newline on every print

[Janosch Frank]

Linemode seems to add a newline for each sent message which makes
reading rather hard. Hence we add a small buffer and only print if
it's full or a newline is encountered. Except for when the string is
longer than the buffer, then we flush the buffer and print directly.

Signed-off-by: Janosch Frank <frankja@linux.ibm.com>
Reviewed-by: Thomas Huth <thuth@redhat.com>
Acked-by: David Hildenbrand <david@redhat.com>
---
 lib/s390x/sclp-console.c | 45 ++++++++++++++++++++++++++++++++++++----
 1 file changed, 41 insertions(+), 4 deletions(-)

diff --git a/lib/s390x/sclp-console.c b/lib/s390x/sclp-console.c
index 19416b5..e1b9000 100644
--- a/lib/s390x/sclp-console.c
+++ b/lib/s390x/sclp-console.c
@@ -13,6 +13,7 @@
 #include <asm/page.h>
 #include <asm/arch_def.h>
 #include <asm/io.h>
+#include <asm/spinlock.h>
 #include "sclp.h"
 
 /*
@@ -87,6 +88,10 @@ static uint8_t _ascebc[256] = {
      0x90, 0x3F, 0x3F, 0x3F, 0x3F, 0xEA, 0x3F, 0xFF
 };
 
+static char lm_buff[120];
+static unsigned char lm_buff_off;
+static struct spinlock lm_buff_lock;
+
 static void sclp_print_ascii(const char *str)
 {
 	int len = strlen(str);
@@ -103,10 +108,10 @@ static void sclp_print_ascii(const char *str)
 	sclp_service_call(SCLP_CMD_WRITE_EVENT_DATA, sccb);
 }
 
-static void sclp_print_lm(const char *str)
+static void lm_print(const char *buff, int len)
 {
 	unsigned char *ptr, *end, ch;
-	unsigned int count, offset, len;
+	unsigned int count, offset;
 	struct WriteEventData *sccb;
 	struct mdb *mdb;
 	struct mto *mto;
@@ -117,11 +122,10 @@ static void sclp_print_lm(const char *str)
 	end = (unsigned char *) sccb + 4096 - 1;
 	memset(sccb, 0, sizeof(*sccb));
 	ptr = (unsigned char *) &sccb->msg.mdb.mto;
-	len = strlen(str);
 	offset = 0;
 	do {
 		for (count = sizeof(*mto); offset < len; count++) {
-			ch = str[offset++];
+			ch = buff[offset++];
 			if (ch == 0x0a || ptr + count > end)
 				break;
 			ptr[count] = _ascebc[ch];
@@ -148,6 +152,39 @@ static void sclp_print_lm(const char *str)
 	sclp_service_call(SCLP_CMD_WRITE_EVENT_DATA, sccb);
 }
 
+
+/*
+ * In contrast to the ascii console, linemode produces a new
+ * line with every write of data. The report() function uses
+ * several printf() calls to generate a line of data which
+ * would all end up on different lines.
+ *
+ * Hence we buffer here until we encounter a \n or the buffer
+ * is full. That means that linemode output can look a bit
+ * different from ascii and that it takes a bit longer for
+ * lines to appear.
+ */
+static void sclp_print_lm(const char *str)
+{
+	int len = strlen(str), i = 0;
+
+	spin_lock(&lm_buff_lock);
+
+	while (len) {
+		lm_buff[lm_buff_off] = str[i];
+		lm_buff_off++;
+		len--;
+		/* Buffer full or newline? */
+		if (str[i] == '\n' || lm_buff_off == (sizeof(lm_buff) - 1)) {
+			lm_print(lm_buff, lm_buff_off);
+			memset(lm_buff, 0 , sizeof(lm_buff));
+			lm_buff_off = 0;
+		}
+		i++;
+	}
+	spin_unlock(&lm_buff_lock);
+}
+
 /*
  * SCLP needs to be initialized by setting a send and receive mask,
  * indicating which messages the control program (we) want(s) to
-- 
2.17.2

Re: [kvm-unit-tests PATCH v3 3/6] s390x: Add linemode buffer to fix newline on every print

[David Hildenbrand]

On 20.09.19 10:03, Janosch Frank wrote:
> Linemode seems to add a newline for each sent message which makes
> reading rather hard. Hence we add a small buffer and only print if
> it's full or a newline is encountered. Except for when the string is
> longer than the buffer, then we flush the buffer and print directly.

I think that last sentence is not correct anymore.

> 
> Signed-off-by: Janosch Frank <frankja@linux.ibm.com>
> Reviewed-by: Thomas Huth <thuth@redhat.com>
> Acked-by: David Hildenbrand <david@redhat.com>
> ---
>  lib/s390x/sclp-console.c | 45 ++++++++++++++++++++++++++++++++++++----
>  1 file changed, 41 insertions(+), 4 deletions(-)
> 
> diff --git a/lib/s390x/sclp-console.c b/lib/s390x/sclp-console.c
> index 19416b5..e1b9000 100644
> --- a/lib/s390x/sclp-console.c
> +++ b/lib/s390x/sclp-console.c
> @@ -13,6 +13,7 @@
>  #include <asm/page.h>
>  #include <asm/arch_def.h>
>  #include <asm/io.h>
> +#include <asm/spinlock.h>
>  #include "sclp.h"
>  
>  /*
> @@ -87,6 +88,10 @@ static uint8_t _ascebc[256] = {
>       0x90, 0x3F, 0x3F, 0x3F, 0x3F, 0xEA, 0x3F, 0xFF
>  };
>  
> +static char lm_buff[120];
> +static unsigned char lm_buff_off;
> +static struct spinlock lm_buff_lock;
> +
>  static void sclp_print_ascii(const char *str)
>  {
>  	int len = strlen(str);
> @@ -103,10 +108,10 @@ static void sclp_print_ascii(const char *str)
>  	sclp_service_call(SCLP_CMD_WRITE_EVENT_DATA, sccb);
>  }
>  
> -static void sclp_print_lm(const char *str)
> +static void lm_print(const char *buff, int len)
>  {
>  	unsigned char *ptr, *end, ch;
> -	unsigned int count, offset, len;
> +	unsigned int count, offset;
>  	struct WriteEventData *sccb;
>  	struct mdb *mdb;
>  	struct mto *mto;
> @@ -117,11 +122,10 @@ static void sclp_print_lm(const char *str)
>  	end = (unsigned char *) sccb + 4096 - 1;
>  	memset(sccb, 0, sizeof(*sccb));
>  	ptr = (unsigned char *) &sccb->msg.mdb.mto;
> -	len = strlen(str);
>  	offset = 0;
>  	do {
>  		for (count = sizeof(*mto); offset < len; count++) {
> -			ch = str[offset++];
> +			ch = buff[offset++];
>  			if (ch == 0x0a || ptr + count > end)
>  				break;
>  			ptr[count] = _ascebc[ch];
> @@ -148,6 +152,39 @@ static void sclp_print_lm(const char *str)
>  	sclp_service_call(SCLP_CMD_WRITE_EVENT_DATA, sccb);
>  }
>  
> +
> +/*
> + * In contrast to the ascii console, linemode produces a new
> + * line with every write of data. The report() function uses
> + * several printf() calls to generate a line of data which
> + * would all end up on different lines.
> + *
> + * Hence we buffer here until we encounter a \n or the buffer
> + * is full. That means that linemode output can look a bit
> + * different from ascii and that it takes a bit longer for
> + * lines to appear.
> + */
> +static void sclp_print_lm(const char *str)
> +{
> +	int len = strlen(str), i = 0;
> +
> +	spin_lock(&lm_buff_lock);
> +
> +	while (len) {

for (i = 0; i < len; i++)

Then make len const and drop "len--" and "i++" from the body.

> +		lm_buff[lm_buff_off] = str[i];
> +		lm_buff_off++;

lm_buff[lm_buff_off++] = str[i];

if you feel like saving one LOC :)

> +		len--;
> +		/* Buffer full or newline? */
> +		if (str[i] == '\n' || lm_buff_off == (sizeof(lm_buff) - 1)) {

I still prefer ARRAY_SIZE(), but this is also fine.

> +			lm_print(lm_buff, lm_buff_off);
> +			memset(lm_buff, 0 , sizeof(lm_buff));

Is the memset really necessary? (I would have assumed it would only
print until the last "\n" ?)

> +			lm_buff_off = 0;
> +		}
> +		i++;
> +	}

Guess we don't care about performance, so the simple byte-based approach
should be just fine.

> +	spin_unlock(&lm_buff_lock);
> +}
> +
>  /*
>   * SCLP needs to be initialized by setting a send and receive mask,
>   * indicating which messages the control program (we) want(s) to
> 


I wonder if we have to implement some kind of fflush(), so we will flush
the buffer on any abort ... but I assume we will always end printing
with a "\n", so we don't really care.

-- 

Thanks,

David / dhildenb

Re: [kvm-unit-tests PATCH v3 3/6] s390x: Add linemode buffer to fix newline on every print

[Janosch Frank]

[-- Attachment #1.1: Type: text/plain, Size: 4552 bytes --]

On 9/20/19 10:24 AM, David Hildenbrand wrote:
> On 20.09.19 10:03, Janosch Frank wrote:
>> Linemode seems to add a newline for each sent message which makes
>> reading rather hard. Hence we add a small buffer and only print if
>> it's full or a newline is encountered. Except for when the string is
>> longer than the buffer, then we flush the buffer and print directly.
> 
> I think that last sentence is not correct anymore.
> 
>>
>> Signed-off-by: Janosch Frank <frankja@linux.ibm.com>
>> Reviewed-by: Thomas Huth <thuth@redhat.com>
>> Acked-by: David Hildenbrand <david@redhat.com>
>> ---
>>  lib/s390x/sclp-console.c | 45 ++++++++++++++++++++++++++++++++++++----
>>  1 file changed, 41 insertions(+), 4 deletions(-)
>>
>> diff --git a/lib/s390x/sclp-console.c b/lib/s390x/sclp-console.c
>> index 19416b5..e1b9000 100644
>> --- a/lib/s390x/sclp-console.c
>> +++ b/lib/s390x/sclp-console.c
>> @@ -13,6 +13,7 @@
>>  #include <asm/page.h>
>>  #include <asm/arch_def.h>
>>  #include <asm/io.h>
>> +#include <asm/spinlock.h>
>>  #include "sclp.h"
>>  
>>  /*
>> @@ -87,6 +88,10 @@ static uint8_t _ascebc[256] = {
>>       0x90, 0x3F, 0x3F, 0x3F, 0x3F, 0xEA, 0x3F, 0xFF
>>  };
>>  
>> +static char lm_buff[120];
>> +static unsigned char lm_buff_off;
>> +static struct spinlock lm_buff_lock;
>> +
>>  static void sclp_print_ascii(const char *str)
>>  {
>>  	int len = strlen(str);
>> @@ -103,10 +108,10 @@ static void sclp_print_ascii(const char *str)
>>  	sclp_service_call(SCLP_CMD_WRITE_EVENT_DATA, sccb);
>>  }
>>  
>> -static void sclp_print_lm(const char *str)
>> +static void lm_print(const char *buff, int len)
>>  {
>>  	unsigned char *ptr, *end, ch;
>> -	unsigned int count, offset, len;
>> +	unsigned int count, offset;
>>  	struct WriteEventData *sccb;
>>  	struct mdb *mdb;
>>  	struct mto *mto;
>> @@ -117,11 +122,10 @@ static void sclp_print_lm(const char *str)
>>  	end = (unsigned char *) sccb + 4096 - 1;
>>  	memset(sccb, 0, sizeof(*sccb));
>>  	ptr = (unsigned char *) &sccb->msg.mdb.mto;
>> -	len = strlen(str);
>>  	offset = 0;
>>  	do {
>>  		for (count = sizeof(*mto); offset < len; count++) {
>> -			ch = str[offset++];
>> +			ch = buff[offset++];
>>  			if (ch == 0x0a || ptr + count > end)
>>  				break;
>>  			ptr[count] = _ascebc[ch];
>> @@ -148,6 +152,39 @@ static void sclp_print_lm(const char *str)
>>  	sclp_service_call(SCLP_CMD_WRITE_EVENT_DATA, sccb);
>>  }
>>  
>> +
>> +/*
>> + * In contrast to the ascii console, linemode produces a new
>> + * line with every write of data. The report() function uses
>> + * several printf() calls to generate a line of data which
>> + * would all end up on different lines.
>> + *
>> + * Hence we buffer here until we encounter a \n or the buffer
>> + * is full. That means that linemode output can look a bit
>> + * different from ascii and that it takes a bit longer for
>> + * lines to appear.
>> + */
>> +static void sclp_print_lm(const char *str)
>> +{
>> +	int len = strlen(str), i = 0;
>> +
>> +	spin_lock(&lm_buff_lock);
>> +
>> +	while (len) {
> 
> for (i = 0; i < len; i++)
> 
> Then make len const and drop "len--" and "i++" from the body.

Sure, the loop was a first draft anyway.
I'm asking myself how I ended up writing something so complicated in the
first place :-)

> 
>> +		lm_buff[lm_buff_off] = str[i];
>> +		lm_buff_off++;
> 
> lm_buff[lm_buff_off++] = str[i];
> 
> if you feel like saving one LOC :)
> 
>> +		len--;
>> +		/* Buffer full or newline? */
>> +		if (str[i] == '\n' || lm_buff_off == (sizeof(lm_buff) - 1)) {
> 
> I still prefer ARRAY_SIZE(), but this is also fine.
> 
>> +			lm_print(lm_buff, lm_buff_off);
>> +			memset(lm_buff, 0 , sizeof(lm_buff));
> 
> Is the memset really necessary? (I would have assumed it would only
> print until the last "\n" ?)

It's not, I just like clearing things

> 
>> +			lm_buff_off = 0;
>> +		}
>> +		i++;
>> +	}
> 
> Guess we don't care about performance, so the simple byte-based approach
> should be just fine.

;-)

> 
>> +	spin_unlock(&lm_buff_lock);
>> +}
>> +
>>  /*
>>   * SCLP needs to be initialized by setting a send and receive mask,
>>   * indicating which messages the control program (we) want(s) to
>>
> 
> 
> I wonder if we have to implement some kind of fflush(), so we will flush
> the buffer on any abort ... but I assume we will always end printing
> with a "\n", so we don't really care.

I haven't encountered incomplete output up to now.


[-- Attachment #2: OpenPGP digital signature --]
[-- Type: application/pgp-signature, Size: 833 bytes --]

[kvm-unit-tests PATCH] s390x: Add linemode buffer to fix newline on every print

[Janosch Frank]

Linemode seems to add a newline for each sent message which makes
reading rather hard. Hence we add a small buffer and only print if
it's full or a newline is encountered.

Signed-off-by: Janosch Frank <frankja@linux.ibm.com>
Reviewed-by: Thomas Huth <thuth@redhat.com>
Acked-by: David Hildenbrand <david@redhat.com>
---
 lib/s390x/sclp-console.c | 43 ++++++++++++++++++++++++++++++++++++----
 1 file changed, 39 insertions(+), 4 deletions(-)

diff --git a/lib/s390x/sclp-console.c b/lib/s390x/sclp-console.c
index 19416b5..6067a1a 100644
--- a/lib/s390x/sclp-console.c
+++ b/lib/s390x/sclp-console.c
@@ -13,6 +13,7 @@
 #include <asm/page.h>
 #include <asm/arch_def.h>
 #include <asm/io.h>
+#include <asm/spinlock.h>
 #include "sclp.h"
 
 /*
@@ -87,6 +88,10 @@ static uint8_t _ascebc[256] = {
      0x90, 0x3F, 0x3F, 0x3F, 0x3F, 0xEA, 0x3F, 0xFF
 };
 
+static char lm_buff[120];
+static unsigned char lm_buff_off;
+static struct spinlock lm_buff_lock;
+
 static void sclp_print_ascii(const char *str)
 {
 	int len = strlen(str);
@@ -103,10 +108,10 @@ static void sclp_print_ascii(const char *str)
 	sclp_service_call(SCLP_CMD_WRITE_EVENT_DATA, sccb);
 }
 
-static void sclp_print_lm(const char *str)
+static void lm_print(const char *buff, int len)
 {
 	unsigned char *ptr, *end, ch;
-	unsigned int count, offset, len;
+	unsigned int count, offset;
 	struct WriteEventData *sccb;
 	struct mdb *mdb;
 	struct mto *mto;
@@ -117,11 +122,10 @@ static void sclp_print_lm(const char *str)
 	end = (unsigned char *) sccb + 4096 - 1;
 	memset(sccb, 0, sizeof(*sccb));
 	ptr = (unsigned char *) &sccb->msg.mdb.mto;
-	len = strlen(str);
 	offset = 0;
 	do {
 		for (count = sizeof(*mto); offset < len; count++) {
-			ch = str[offset++];
+			ch = buff[offset++];
 			if (ch == 0x0a || ptr + count > end)
 				break;
 			ptr[count] = _ascebc[ch];
@@ -148,6 +152,37 @@ static void sclp_print_lm(const char *str)
 	sclp_service_call(SCLP_CMD_WRITE_EVENT_DATA, sccb);
 }
 
+
+/*
+ * In contrast to the ascii console, linemode produces a new
+ * line with every write of data. The report() function uses
+ * several printf() calls to generate a line of data which
+ * would all end up on different lines.
+ *
+ * Hence we buffer here until we encounter a \n or the buffer
+ * is full. That means that linemode output can look a bit
+ * different from ascii and that it takes a bit longer for
+ * lines to appear.
+ */
+static void sclp_print_lm(const char *str)
+{
+	int i;
+	const int len = strlen(str);
+
+	spin_lock(&lm_buff_lock);
+
+	for (i = 0; i < len; i++) {
+		lm_buff[lm_buff_off++] = str[i];
+
+		/* Buffer full or newline? */
+		if (str[i] == '\n' || lm_buff_off == (ARRAY_SIZE(lm_buff) - 1)) {
+			lm_print(lm_buff, lm_buff_off);
+			lm_buff_off = 0;
+		}
+	}
+	spin_unlock(&lm_buff_lock);
+}
+
 /*
  * SCLP needs to be initialized by setting a send and receive mask,
  * indicating which messages the control program (we) want(s) to
-- 
2.17.2

[kvm-unit-tests PATCH v3 4/6] s390x: Add initial smp code

[Janosch Frank]

Let's add a rudimentary SMP library, which will scan for cpus and has
helper functions that manage the cpu state.

Signed-off-by: Janosch Frank <frankja@linux.ibm.com>
---
 lib/s390x/asm/arch_def.h |   8 ++
 lib/s390x/asm/sigp.h     |  28 ++++-
 lib/s390x/io.c           |   5 +-
 lib/s390x/sclp.h         |   1 +
 lib/s390x/smp.c          | 263 +++++++++++++++++++++++++++++++++++++++
 lib/s390x/smp.h          |  51 ++++++++
 s390x/Makefile           |   1 +
 s390x/cstart64.S         |   7 ++
 8 files changed, 358 insertions(+), 6 deletions(-)
 create mode 100644 lib/s390x/smp.c
 create mode 100644 lib/s390x/smp.h

diff --git a/lib/s390x/asm/arch_def.h b/lib/s390x/asm/arch_def.h
index 5f8f45e..d5a7f51 100644
--- a/lib/s390x/asm/arch_def.h
+++ b/lib/s390x/asm/arch_def.h
@@ -157,6 +157,14 @@ struct cpuid {
 	uint64_t reserved : 15;
 };
 
+static inline unsigned short stap(void)
+{
+	unsigned short cpu_address;
+
+	asm volatile("stap %0" : "=Q" (cpu_address));
+	return cpu_address;
+}
+
 static inline int tprot(unsigned long addr)
 {
 	int cc;
diff --git a/lib/s390x/asm/sigp.h b/lib/s390x/asm/sigp.h
index fbd94fc..2d52313 100644
--- a/lib/s390x/asm/sigp.h
+++ b/lib/s390x/asm/sigp.h
@@ -46,14 +46,32 @@
 
 #ifndef __ASSEMBLER__
 
-static inline void sigp_stop(void)
+
+static inline int sigp(uint16_t addr, uint8_t order, unsigned long parm,
+		       uint32_t *status)
 {
-	register unsigned long status asm ("1") = 0;
-	register unsigned long cpu asm ("2") = 0;
+	register unsigned long reg1 asm ("1") = parm;
+	int cc;
 
 	asm volatile(
-		"	sigp %0,%1,0(%2)\n"
-		: "+d" (status)  : "d" (cpu), "d" (SIGP_STOP) : "cc");
+		"	sigp	%1,%2,0(%3)\n"
+		"	ipm	%0\n"
+		"	srl	%0,28\n"
+		: "=d" (cc), "+d" (reg1) : "d" (addr), "a" (order) : "cc");
+	if (status)
+		*status = reg1;
+	return cc;
+}
+
+static inline int sigp_retry(uint16_t addr, uint8_t order, unsigned long parm,
+			     uint32_t *status)
+{
+	int cc;
+
+	do {
+		cc = sigp(addr, order, parm, status);
+	} while (cc == 2);
+	return cc;
 }
 
 #endif /* __ASSEMBLER__ */
diff --git a/lib/s390x/io.c b/lib/s390x/io.c
index becadfc..32f09b5 100644
--- a/lib/s390x/io.c
+++ b/lib/s390x/io.c
@@ -16,6 +16,7 @@
 #include <asm/facility.h>
 #include <asm/sigp.h>
 #include "sclp.h"
+#include "smp.h"
 
 extern char ipl_args[];
 uint8_t stfl_bytes[NR_STFL_BYTES] __attribute__((aligned(8)));
@@ -37,12 +38,14 @@ void setup(void)
 	setup_facilities();
 	sclp_console_setup();
 	sclp_memory_setup();
+	smp_setup();
 }
 
 void exit(int code)
 {
+	smp_teardown();
 	printf("\nEXIT: STATUS=%d\n", ((code) << 1) | 1);
 	while (1) {
-		sigp_stop();
+		sigp(0, SIGP_STOP, 0, NULL);
 	}
 }
diff --git a/lib/s390x/sclp.h b/lib/s390x/sclp.h
index 98c482a..4e69845 100644
--- a/lib/s390x/sclp.h
+++ b/lib/s390x/sclp.h
@@ -19,6 +19,7 @@
 #define SCLP_CMD_CODE_MASK                      0xffff00ff
 
 /* SCLP command codes */
+#define SCLP_READ_CPU_INFO			0x00010001
 #define SCLP_CMDW_READ_SCP_INFO                 0x00020001
 #define SCLP_CMDW_READ_SCP_INFO_FORCED          0x00120001
 #define SCLP_READ_STORAGE_ELEMENT_INFO          0x00040001
diff --git a/lib/s390x/smp.c b/lib/s390x/smp.c
new file mode 100644
index 0000000..05379b0
--- /dev/null
+++ b/lib/s390x/smp.c
@@ -0,0 +1,263 @@
+/*
+ * s390x smp
+ * Based on Linux's arch/s390/kernel/smp.c and
+ * arch/s390/include/asm/sigp.h
+ *
+ * Copyright (c) 2019 IBM Corp
+ *
+ * Authors:
+ *  Janosch Frank <frankja@linux.ibm.com>
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2.
+ */
+#include <libcflat.h>
+#include <asm/arch_def.h>
+#include <asm/sigp.h>
+#include <asm/page.h>
+#include <asm/barrier.h>
+#include <asm/spinlock.h>
+#include <asm/asm-offsets.h>
+
+#include <alloc.h>
+#include <alloc_page.h>
+
+#include "smp.h"
+#include "sclp.h"
+
+static char cpu_info_buffer[PAGE_SIZE] __attribute__((__aligned__(4096)));
+static struct cpu *cpus;
+static struct cpu *cpu0;
+static struct spinlock lock;
+
+extern void smp_cpu_setup_state(void);
+
+int smp_query_num_cpus(void)
+{
+	struct ReadCpuInfo *info = (void *)cpu_info_buffer;
+	return info->nr_configured;
+}
+
+struct cpu *smp_cpu_from_addr(uint16_t addr)
+{
+	int i, num = smp_query_num_cpus();
+	struct cpu *cpu = NULL;
+
+	for (i = 0; i < num; i++) {
+		if (cpus[i].addr == addr)
+			cpu = &cpus[i];
+	}
+	return cpu;
+}
+
+bool smp_cpu_stopped(uint16_t addr)
+{
+	uint32_t status;
+
+	if (sigp(addr, SIGP_SENSE, 0, &status) != SIGP_CC_STATUS_STORED)
+		return false;
+	return !!(status & (SIGP_STATUS_CHECK_STOP|SIGP_STATUS_STOPPED));
+}
+
+bool smp_cpu_running(uint16_t addr)
+{
+	if (sigp(addr, SIGP_SENSE_RUNNING, 0, NULL) != SIGP_CC_STATUS_STORED)
+		return true;
+	/* Status stored condition code is equivalent to cpu not running. */
+	return false;
+}
+
+static int smp_cpu_stop_nolock(uint16_t addr, bool store)
+{
+	struct cpu *cpu;
+	uint8_t order = store ? SIGP_STOP_AND_STORE_STATUS : SIGP_STOP;
+
+	cpu = smp_cpu_from_addr(addr);
+	if (!cpu || cpu == cpu0)
+		return -1;
+
+	if (sigp_retry(addr, order, 0, NULL))
+		return -1;
+
+	while (!smp_cpu_stopped(addr))
+		mb();
+	cpu->active = false;
+	return 0;
+}
+
+int smp_cpu_stop(uint16_t addr)
+{
+	int rc = 0;
+
+	spin_lock(&lock);
+	rc = smp_cpu_stop_nolock(addr, false);
+	spin_unlock(&lock);
+	return rc;
+}
+
+int smp_cpu_stop_store_status(uint16_t addr)
+{
+	int rc = 0;
+
+	spin_lock(&lock);
+	rc = smp_cpu_stop_nolock(addr, true);
+	spin_unlock(&lock);
+	return rc;
+}
+
+int smp_cpu_restart(uint16_t addr)
+{
+	int rc = -1;
+	struct cpu *cpu;
+
+	spin_lock(&lock);
+	cpu = smp_cpu_from_addr(addr);
+	if (!cpu)
+		goto out;
+
+	rc = sigp(addr, SIGP_RESTART, 0, NULL);
+	cpu->active = true;
+out:
+	spin_unlock(&lock);
+	return rc;
+}
+
+int smp_cpu_start(uint16_t addr, struct psw psw)
+{
+	int rc = -1;
+	struct cpu *cpu;
+	struct lowcore *lc;
+
+	spin_lock(&lock);
+	cpu = smp_cpu_from_addr(addr);
+	if (!cpu)
+		goto out;
+
+	lc = cpu->lowcore;
+	lc->restart_new_psw.mask = psw.mask;
+	lc->restart_new_psw.addr = psw.addr;
+	rc = sigp(addr, SIGP_RESTART, 0, NULL);
+out:
+	spin_unlock(&lock);
+	return rc;
+}
+
+int smp_cpu_destroy(uint16_t addr)
+{
+	struct cpu *cpu;
+	int rc = 0;
+
+	spin_lock(&lock);
+	rc = smp_cpu_stop_nolock(addr, false);
+	if (rc)
+		goto out;
+
+	cpu = smp_cpu_from_addr(addr);
+	free_pages(cpu->lowcore, 2 * PAGE_SIZE);
+	free_pages(cpu->stack, 4 * PAGE_SIZE);
+	cpu->lowcore = (void *)-1UL;
+	cpu->stack = (void *)-1UL;
+
+out:
+	spin_unlock(&lock);
+	return rc;
+}
+
+int smp_cpu_setup(uint16_t addr, struct psw psw)
+{
+	struct lowcore *lc;
+	struct cpu *cpu;
+	int rc = -1;
+
+	spin_lock(&lock);
+
+	if (!cpus)
+		goto out;
+
+	cpu = smp_cpu_from_addr(addr);
+
+	if (!cpu)
+		goto out;
+
+	if (cpu->active)
+		goto out;
+
+	sigp_retry(cpu->addr, SIGP_INITIAL_CPU_RESET, 0, NULL);
+
+	lc = alloc_pages(1);
+	cpu->lowcore = lc;
+	memset(lc, 0, PAGE_SIZE * 2);
+	sigp_retry(cpu->addr, SIGP_SET_PREFIX, (unsigned long )lc, NULL);
+
+	/* Copy all exception psws. */
+	memcpy(lc, cpu0->lowcore, 512);
+
+	/* Setup stack */
+	cpu->stack = (uint64_t *)alloc_pages(2);
+
+	/* Start without DAT and any other mask bits. */
+	cpu->lowcore->sw_int_grs[14] = psw.addr;
+	cpu->lowcore->sw_int_grs[15] = (uint64_t)cpu->stack + (PAGE_SIZE * 4);
+	lc->restart_new_psw.mask = 0x0000000180000000UL;
+	lc->restart_new_psw.addr = (uint64_t)smp_cpu_setup_state;
+	lc->sw_int_cr0 = 0x0000000000040000UL;
+
+	/* Start processing */
+	rc = sigp_retry(cpu->addr, SIGP_RESTART, 0, NULL);
+	if (!rc)
+		cpu->active = true;
+
+out:
+	spin_unlock(&lock);
+	return rc;
+}
+
+/*
+ * Disregarding state, stop all cpus that once were online except for
+ * calling cpu.
+ */
+void smp_teardown(void)
+{
+	int i = 0;
+	uint16_t this_cpu = stap();
+	struct ReadCpuInfo *info = (void *)cpu_info_buffer;
+
+	spin_lock(&lock);
+	for (; i < info->nr_configured; i++) {
+		if (cpus[i].active &&
+		    cpus[i].addr != this_cpu) {
+			sigp_retry(cpus[i].addr, SIGP_STOP, 0, NULL);
+		}
+	}
+	spin_unlock(&lock);
+}
+
+/*Expected to be called from boot cpu */
+extern uint64_t *stackptr;
+void smp_setup(void)
+{
+	int i = 0;
+	unsigned short cpu0_addr = stap();
+	struct ReadCpuInfo *info = (void *)cpu_info_buffer;
+
+	spin_lock(&lock);
+	sclp_mark_busy();
+	info->h.length = PAGE_SIZE;
+	sclp_service_call(SCLP_READ_CPU_INFO, cpu_info_buffer);
+
+	if (smp_query_num_cpus() > 1)
+		printf("SMP: Initializing, found %d cpus\n", info->nr_configured);
+
+	cpus = calloc(info->nr_configured, sizeof(cpus));
+	for (i = 0; i < info->nr_configured; i++) {
+		cpus[i].addr = info->entries[i].address;
+		cpus[i].active = false;
+		if (info->entries[i].address == cpu0_addr) {
+			cpu0 = &cpus[i];
+			cpu0->stack = stackptr;
+			cpu0->lowcore = (void *)0;
+			cpu0->active = true;
+		}
+	}
+	spin_unlock(&lock);
+}
diff --git a/lib/s390x/smp.h b/lib/s390x/smp.h
new file mode 100644
index 0000000..4476c31
--- /dev/null
+++ b/lib/s390x/smp.h
@@ -0,0 +1,51 @@
+/*
+ * s390x smp
+ *
+ * Copyright (c) 2019 IBM Corp
+ *
+ * Authors:
+ *  Janosch Frank <frankja@linux.ibm.com>
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2.
+ */
+#ifndef SMP_H
+#define SMP_H
+
+struct cpu {
+	struct lowcore *lowcore;
+	uint64_t *stack;
+	uint16_t addr;
+	bool active;
+};
+
+struct cpu_status {
+    uint64_t    fprs[16];                       /* 0x0000 */
+    uint64_t    grs[16];                        /* 0x0080 */
+    struct psw  psw;                            /* 0x0100 */
+    uint8_t     pad_0x0110[0x0118 - 0x0110];    /* 0x0110 */
+    uint32_t    prefix;                         /* 0x0118 */
+    uint32_t    fpc;                            /* 0x011c */
+    uint8_t     pad_0x0120[0x0124 - 0x0120];    /* 0x0120 */
+    uint32_t    todpr;                          /* 0x0124 */
+    uint64_t    cputm;                          /* 0x0128 */
+    uint64_t    ckc;                            /* 0x0130 */
+    uint8_t     pad_0x0138[0x0140 - 0x0138];    /* 0x0138 */
+    uint32_t    ars[16];                        /* 0x0140 */
+    uint64_t    crs[16];                        /* 0x0384 */
+};
+
+int smp_query_num_cpus(void);
+struct cpu *smp_cpu_from_addr(uint16_t addr);
+bool smp_cpu_stopped(uint16_t addr);
+bool smp_cpu_running(uint16_t addr);
+int smp_cpu_restart(uint16_t addr);
+int smp_cpu_start(uint16_t addr, struct psw psw);
+int smp_cpu_stop(uint16_t addr);
+int smp_cpu_stop_store_status(uint16_t addr);
+int smp_cpu_destroy(uint16_t addr);
+int smp_cpu_setup(uint16_t addr, struct psw psw);
+void smp_teardown(void);
+void smp_setup(void);
+
+#endif
diff --git a/s390x/Makefile b/s390x/Makefile
index 96033dd..d83dd0b 100644
--- a/s390x/Makefile
+++ b/s390x/Makefile
@@ -48,6 +48,7 @@ cflatobjs += lib/s390x/sclp.o
 cflatobjs += lib/s390x/sclp-console.o
 cflatobjs += lib/s390x/interrupt.o
 cflatobjs += lib/s390x/mmu.o
+cflatobjs += lib/s390x/smp.o
 
 OBJDIRS += lib/s390x
 
diff --git a/s390x/cstart64.S b/s390x/cstart64.S
index 36f7cab..5dc1577 100644
--- a/s390x/cstart64.S
+++ b/s390x/cstart64.S
@@ -172,6 +172,13 @@ diag308_load_reset:
 	lhi	%r2, 1
 	br	%r14
 
+.globl smp_cpu_setup_state
+smp_cpu_setup_state:
+	xgr	%r1, %r1
+	lmg     %r0, %r15, GEN_LC_SW_INT_GRS
+	lctlg   %c0, %c0, GEN_LC_SW_INT_CR0
+	br	%r14
+
 pgm_int:
 	SAVE_REGS
 	brasl	%r14, handle_pgm_int
-- 
2.17.2

Re: [kvm-unit-tests PATCH v3 4/6] s390x: Add initial smp code

[Thomas Huth]

On 20/09/2019 10.03, Janosch Frank wrote:
> Let's add a rudimentary SMP library, which will scan for cpus and has
> helper functions that manage the cpu state.
> 
> Signed-off-by: Janosch Frank <frankja@linux.ibm.com>
> ---
[...]
> diff --git a/lib/s390x/smp.c b/lib/s390x/smp.c
> new file mode 100644
> index 0000000..05379b0
> --- /dev/null
> +++ b/lib/s390x/smp.c
> @@ -0,0 +1,263 @@
> +/*
> + * s390x smp
> + * Based on Linux's arch/s390/kernel/smp.c and
> + * arch/s390/include/asm/sigp.h
> + *
> + * Copyright (c) 2019 IBM Corp
> + *
> + * Authors:
> + *  Janosch Frank <frankja@linux.ibm.com>
> + *
> + * This code is free software; you can redistribute it and/or modify it
> + * under the terms of the GNU General Public License version 2.
> + */
> +#include <libcflat.h>
> +#include <asm/arch_def.h>
> +#include <asm/sigp.h>
> +#include <asm/page.h>
> +#include <asm/barrier.h>
> +#include <asm/spinlock.h>
> +#include <asm/asm-offsets.h>
> +
> +#include <alloc.h>
> +#include <alloc_page.h>
> +
> +#include "smp.h"
> +#include "sclp.h"
> +
> +static char cpu_info_buffer[PAGE_SIZE] __attribute__((__aligned__(4096)));
> +static struct cpu *cpus;
> +static struct cpu *cpu0;
> +static struct spinlock lock;
> +
> +extern void smp_cpu_setup_state(void);
> +
> +int smp_query_num_cpus(void)
> +{
> +	struct ReadCpuInfo *info = (void *)cpu_info_buffer;
> +	return info->nr_configured;
> +}
> +
> +struct cpu *smp_cpu_from_addr(uint16_t addr)
> +{
> +	int i, num = smp_query_num_cpus();
> +	struct cpu *cpu = NULL;
> +
> +	for (i = 0; i < num; i++) {
> +		if (cpus[i].addr == addr)
> +			cpu = &cpus[i];

Small optimization: Add a "break" here. Or "return &cpus[i]" directly
and "return NULL" after the loop, getting rid of the "cpu" variable.

> +	}
> +	return cpu;
> +}
> +
> +bool smp_cpu_stopped(uint16_t addr)
> +{
> +	uint32_t status;
> +
> +	if (sigp(addr, SIGP_SENSE, 0, &status) != SIGP_CC_STATUS_STORED)
> +		return false;
> +	return !!(status & (SIGP_STATUS_CHECK_STOP|SIGP_STATUS_STOPPED));
> +}
> +
> +bool smp_cpu_running(uint16_t addr)
> +{
> +	if (sigp(addr, SIGP_SENSE_RUNNING, 0, NULL) != SIGP_CC_STATUS_STORED)
> +		return true;
> +	/* Status stored condition code is equivalent to cpu not running. */
> +	return false;
> +}
> +
> +static int smp_cpu_stop_nolock(uint16_t addr, bool store)
> +{
> +	struct cpu *cpu;
> +	uint8_t order = store ? SIGP_STOP_AND_STORE_STATUS : SIGP_STOP;
> +
> +	cpu = smp_cpu_from_addr(addr);
> +	if (!cpu || cpu == cpu0)
> +		return -1;
> +
> +	if (sigp_retry(addr, order, 0, NULL))
> +		return -1;
> +
> +	while (!smp_cpu_stopped(addr))
> +		mb();
> +	cpu->active = false;
> +	return 0;
> +}
> +
> +int smp_cpu_stop(uint16_t addr)
> +{
> +	int rc = 0;

You could drop the "= 0" here.

> +	spin_lock(&lock);
> +	rc = smp_cpu_stop_nolock(addr, false);
> +	spin_unlock(&lock);
> +	return rc;
> +}
> +
> +int smp_cpu_stop_store_status(uint16_t addr)
> +{
> +	int rc = 0;

dito.

> +	spin_lock(&lock);
> +	rc = smp_cpu_stop_nolock(addr, true);
> +	spin_unlock(&lock);
> +	return rc;
> +}
> +
> +int smp_cpu_restart(uint16_t addr)
> +{
> +	int rc = -1;
> +	struct cpu *cpu;
> +
> +	spin_lock(&lock);
> +	cpu = smp_cpu_from_addr(addr);
> +	if (!cpu)
> +		goto out;
> +
> +	rc = sigp(addr, SIGP_RESTART, 0, NULL);
> +	cpu->active = true;
> +out:

For such simple code, I'd prefer:

	if (cpu) {
		rc = sigp(addr, SIGP_RESTART, 0, NULL);
		cpu->active = true;
	}

instead of using a "goto" ... anyway, just my 0.02 €.

> +	spin_unlock(&lock);
> +	return rc;
> +}
> +
> +int smp_cpu_start(uint16_t addr, struct psw psw)
> +{
> +	int rc = -1;
> +	struct cpu *cpu;
> +	struct lowcore *lc;
> +
> +	spin_lock(&lock);
> +	cpu = smp_cpu_from_addr(addr);
> +	if (!cpu)
> +		goto out;
> +
> +	lc = cpu->lowcore;
> +	lc->restart_new_psw.mask = psw.mask;
> +	lc->restart_new_psw.addr = psw.addr;
> +	rc = sigp(addr, SIGP_RESTART, 0, NULL);
> +out:

dito, could be done without "goto".

> +	spin_unlock(&lock);
> +	return rc;
> +}
> +
> +int smp_cpu_destroy(uint16_t addr)
> +{
> +	struct cpu *cpu;
> +	int rc = 0;
> +
> +	spin_lock(&lock);
> +	rc = smp_cpu_stop_nolock(addr, false);
> +	if (rc)
> +		goto out;
> +
> +	cpu = smp_cpu_from_addr(addr);
> +	free_pages(cpu->lowcore, 2 * PAGE_SIZE);
> +	free_pages(cpu->stack, 4 * PAGE_SIZE);
> +	cpu->lowcore = (void *)-1UL;
> +	cpu->stack = (void *)-1UL;
> +
> +out:

dito. Well, it's just a matter of taste, I think. I'm also fine if you
want to keep it this way.

> +	spin_unlock(&lock);
> +	return rc;
> +}

... just cosmetic nits, patch looks fine to me now, so feel free to add:

Reviewed-by: Thomas Huth <thuth@redhat.com>

[kvm-unit-tests PATCH] s390x: Add initial smp code

[Janosch Frank]

Let's add a rudimentary SMP library, which will scan for cpus and has
helper functions that manage the cpu state.

Signed-off-by: Janosch Frank <frankja@linux.ibm.com>
---

I left the setup code untouched, as I didn't want to indent too many
lines.

---
 lib/s390x/asm/arch_def.h |   8 ++
 lib/s390x/asm/sigp.h     |  28 ++++-
 lib/s390x/io.c           |   5 +-
 lib/s390x/sclp.h         |   1 +
 lib/s390x/smp.c          | 252 +++++++++++++++++++++++++++++++++++++++
 lib/s390x/smp.h          |  51 ++++++++
 s390x/Makefile           |   1 +
 s390x/cstart64.S         |   7 ++
 8 files changed, 347 insertions(+), 6 deletions(-)
 create mode 100644 lib/s390x/smp.c
 create mode 100644 lib/s390x/smp.h

diff --git a/lib/s390x/asm/arch_def.h b/lib/s390x/asm/arch_def.h
index 5f8f45e..d5a7f51 100644
--- a/lib/s390x/asm/arch_def.h
+++ b/lib/s390x/asm/arch_def.h
@@ -157,6 +157,14 @@ struct cpuid {
 	uint64_t reserved : 15;
 };
 
+static inline unsigned short stap(void)
+{
+	unsigned short cpu_address;
+
+	asm volatile("stap %0" : "=Q" (cpu_address));
+	return cpu_address;
+}
+
 static inline int tprot(unsigned long addr)
 {
 	int cc;
diff --git a/lib/s390x/asm/sigp.h b/lib/s390x/asm/sigp.h
index fbd94fc..2d52313 100644
--- a/lib/s390x/asm/sigp.h
+++ b/lib/s390x/asm/sigp.h
@@ -46,14 +46,32 @@
 
 #ifndef __ASSEMBLER__
 
-static inline void sigp_stop(void)
+
+static inline int sigp(uint16_t addr, uint8_t order, unsigned long parm,
+		       uint32_t *status)
 {
-	register unsigned long status asm ("1") = 0;
-	register unsigned long cpu asm ("2") = 0;
+	register unsigned long reg1 asm ("1") = parm;
+	int cc;
 
 	asm volatile(
-		"	sigp %0,%1,0(%2)\n"
-		: "+d" (status)  : "d" (cpu), "d" (SIGP_STOP) : "cc");
+		"	sigp	%1,%2,0(%3)\n"
+		"	ipm	%0\n"
+		"	srl	%0,28\n"
+		: "=d" (cc), "+d" (reg1) : "d" (addr), "a" (order) : "cc");
+	if (status)
+		*status = reg1;
+	return cc;
+}
+
+static inline int sigp_retry(uint16_t addr, uint8_t order, unsigned long parm,
+			     uint32_t *status)
+{
+	int cc;
+
+	do {
+		cc = sigp(addr, order, parm, status);
+	} while (cc == 2);
+	return cc;
 }
 
 #endif /* __ASSEMBLER__ */
diff --git a/lib/s390x/io.c b/lib/s390x/io.c
index becadfc..32f09b5 100644
--- a/lib/s390x/io.c
+++ b/lib/s390x/io.c
@@ -16,6 +16,7 @@
 #include <asm/facility.h>
 #include <asm/sigp.h>
 #include "sclp.h"
+#include "smp.h"
 
 extern char ipl_args[];
 uint8_t stfl_bytes[NR_STFL_BYTES] __attribute__((aligned(8)));
@@ -37,12 +38,14 @@ void setup(void)
 	setup_facilities();
 	sclp_console_setup();
 	sclp_memory_setup();
+	smp_setup();
 }
 
 void exit(int code)
 {
+	smp_teardown();
 	printf("\nEXIT: STATUS=%d\n", ((code) << 1) | 1);
 	while (1) {
-		sigp_stop();
+		sigp(0, SIGP_STOP, 0, NULL);
 	}
 }
diff --git a/lib/s390x/sclp.h b/lib/s390x/sclp.h
index 98c482a..4e69845 100644
--- a/lib/s390x/sclp.h
+++ b/lib/s390x/sclp.h
@@ -19,6 +19,7 @@
 #define SCLP_CMD_CODE_MASK                      0xffff00ff
 
 /* SCLP command codes */
+#define SCLP_READ_CPU_INFO			0x00010001
 #define SCLP_CMDW_READ_SCP_INFO                 0x00020001
 #define SCLP_CMDW_READ_SCP_INFO_FORCED          0x00120001
 #define SCLP_READ_STORAGE_ELEMENT_INFO          0x00040001
diff --git a/lib/s390x/smp.c b/lib/s390x/smp.c
new file mode 100644
index 0000000..7602886
--- /dev/null
+++ b/lib/s390x/smp.c
@@ -0,0 +1,252 @@
+/*
+ * s390x smp
+ * Based on Linux's arch/s390/kernel/smp.c and
+ * arch/s390/include/asm/sigp.h
+ *
+ * Copyright (c) 2019 IBM Corp
+ *
+ * Authors:
+ *  Janosch Frank <frankja@linux.ibm.com>
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2.
+ */
+#include <libcflat.h>
+#include <asm/arch_def.h>
+#include <asm/sigp.h>
+#include <asm/page.h>
+#include <asm/barrier.h>
+#include <asm/spinlock.h>
+#include <asm/asm-offsets.h>
+
+#include <alloc.h>
+#include <alloc_page.h>
+
+#include "smp.h"
+#include "sclp.h"
+
+static char cpu_info_buffer[PAGE_SIZE] __attribute__((__aligned__(4096)));
+static struct cpu *cpus;
+static struct cpu *cpu0;
+static struct spinlock lock;
+
+extern void smp_cpu_setup_state(void);
+
+int smp_query_num_cpus(void)
+{
+	struct ReadCpuInfo *info = (void *)cpu_info_buffer;
+	return info->nr_configured;
+}
+
+struct cpu *smp_cpu_from_addr(uint16_t addr)
+{
+	int i, num = smp_query_num_cpus();
+
+	for (i = 0; i < num; i++) {
+		if (cpus[i].addr == addr)
+			return &cpus[i];
+	}
+	return NULL;
+}
+
+bool smp_cpu_stopped(uint16_t addr)
+{
+	uint32_t status;
+
+	if (sigp(addr, SIGP_SENSE, 0, &status) != SIGP_CC_STATUS_STORED)
+		return false;
+	return !!(status & (SIGP_STATUS_CHECK_STOP|SIGP_STATUS_STOPPED));
+}
+
+bool smp_cpu_running(uint16_t addr)
+{
+	if (sigp(addr, SIGP_SENSE_RUNNING, 0, NULL) != SIGP_CC_STATUS_STORED)
+		return true;
+	/* Status stored condition code is equivalent to cpu not running. */
+	return false;
+}
+
+static int smp_cpu_stop_nolock(uint16_t addr, bool store)
+{
+	struct cpu *cpu;
+	uint8_t order = store ? SIGP_STOP_AND_STORE_STATUS : SIGP_STOP;
+
+	cpu = smp_cpu_from_addr(addr);
+	if (!cpu || cpu == cpu0)
+		return -1;
+
+	if (sigp_retry(addr, order, 0, NULL))
+		return -1;
+
+	while (!smp_cpu_stopped(addr))
+		mb();
+	cpu->active = false;
+	return 0;
+}
+
+int smp_cpu_stop(uint16_t addr)
+{
+	int rc;
+
+	spin_lock(&lock);
+	rc = smp_cpu_stop_nolock(addr, false);
+	spin_unlock(&lock);
+	return rc;
+}
+
+int smp_cpu_stop_store_status(uint16_t addr)
+{
+	int rc;
+
+	spin_lock(&lock);
+	rc = smp_cpu_stop_nolock(addr, true);
+	spin_unlock(&lock);
+	return rc;
+}
+
+int smp_cpu_restart(uint16_t addr)
+{
+	int rc = -1;
+	struct cpu *cpu;
+
+	spin_lock(&lock);
+	cpu = smp_cpu_from_addr(addr);
+	if (cpu) {
+		rc = sigp(addr, SIGP_RESTART, 0, NULL);
+		cpu->active = true;
+	}
+	spin_unlock(&lock);
+	return rc;
+}
+
+int smp_cpu_start(uint16_t addr, struct psw psw)
+{
+	int rc = -1;
+	struct cpu *cpu;
+	struct lowcore *lc;
+
+	spin_lock(&lock);
+	cpu = smp_cpu_from_addr(addr);
+	if (cpu) {
+		lc = cpu->lowcore;
+		lc->restart_new_psw.mask = psw.mask;
+		lc->restart_new_psw.addr = psw.addr;
+		rc = sigp(addr, SIGP_RESTART, 0, NULL);
+	}
+	spin_unlock(&lock);
+	return rc;
+}
+
+int smp_cpu_destroy(uint16_t addr)
+{
+	struct cpu *cpu;
+	int rc;
+
+	spin_lock(&lock);
+	rc = smp_cpu_stop_nolock(addr, false);
+	if (!rc) {
+		cpu = smp_cpu_from_addr(addr);
+		free_pages(cpu->lowcore, 2 * PAGE_SIZE);
+		free_pages(cpu->stack, 4 * PAGE_SIZE);
+		cpu->lowcore = (void *)-1UL;
+		cpu->stack = (void *)-1UL;
+	}
+	spin_unlock(&lock);
+	return rc;
+}
+
+int smp_cpu_setup(uint16_t addr, struct psw psw)
+{
+	struct lowcore *lc;
+	struct cpu *cpu;
+	int rc = -1;
+
+	spin_lock(&lock);
+
+	if (!cpus)
+		goto out;
+
+	cpu = smp_cpu_from_addr(addr);
+
+	if (!cpu || cpu->active)
+		goto out;
+
+	sigp_retry(cpu->addr, SIGP_INITIAL_CPU_RESET, 0, NULL);
+
+	lc = alloc_pages(1);
+	cpu->lowcore = lc;
+	memset(lc, 0, PAGE_SIZE * 2);
+	sigp_retry(cpu->addr, SIGP_SET_PREFIX, (unsigned long )lc, NULL);
+
+	/* Copy all exception psws. */
+	memcpy(lc, cpu0->lowcore, 512);
+
+	/* Setup stack */
+	cpu->stack = (uint64_t *)alloc_pages(2);
+
+	/* Start without DAT and any other mask bits. */
+	cpu->lowcore->sw_int_grs[14] = psw.addr;
+	cpu->lowcore->sw_int_grs[15] = (uint64_t)cpu->stack + (PAGE_SIZE * 4);
+	lc->restart_new_psw.mask = 0x0000000180000000UL;
+	lc->restart_new_psw.addr = (uint64_t)smp_cpu_setup_state;
+	lc->sw_int_cr0 = 0x0000000000040000UL;
+
+	/* Start processing */
+	rc = sigp_retry(cpu->addr, SIGP_RESTART, 0, NULL);
+	if (!rc)
+		cpu->active = true;
+
+out:
+	spin_unlock(&lock);
+	return rc;
+}
+
+/*
+ * Disregarding state, stop all cpus that once were online except for
+ * calling cpu.
+ */
+void smp_teardown(void)
+{
+	int i = 0;
+	uint16_t this_cpu = stap();
+	struct ReadCpuInfo *info = (void *)cpu_info_buffer;
+
+	spin_lock(&lock);
+	for (; i < info->nr_configured; i++) {
+		if (cpus[i].active &&
+		    cpus[i].addr != this_cpu) {
+			sigp_retry(cpus[i].addr, SIGP_STOP, 0, NULL);
+		}
+	}
+	spin_unlock(&lock);
+}
+
+/*Expected to be called from boot cpu */
+extern uint64_t *stackptr;
+void smp_setup(void)
+{
+	int i = 0;
+	unsigned short cpu0_addr = stap();
+	struct ReadCpuInfo *info = (void *)cpu_info_buffer;
+
+	spin_lock(&lock);
+	sclp_mark_busy();
+	info->h.length = PAGE_SIZE;
+	sclp_service_call(SCLP_READ_CPU_INFO, cpu_info_buffer);
+
+	if (smp_query_num_cpus() > 1)
+		printf("SMP: Initializing, found %d cpus\n", info->nr_configured);
+
+	cpus = calloc(info->nr_configured, sizeof(cpus));
+	for (i = 0; i < info->nr_configured; i++) {
+		cpus[i].addr = info->entries[i].address;
+		cpus[i].active = false;
+		if (info->entries[i].address == cpu0_addr) {
+			cpu0 = &cpus[i];
+			cpu0->stack = stackptr;
+			cpu0->lowcore = (void *)0;
+			cpu0->active = true;
+		}
+	}
+	spin_unlock(&lock);
+}
diff --git a/lib/s390x/smp.h b/lib/s390x/smp.h
new file mode 100644
index 0000000..4476c31
--- /dev/null
+++ b/lib/s390x/smp.h
@@ -0,0 +1,51 @@
+/*
+ * s390x smp
+ *
+ * Copyright (c) 2019 IBM Corp
+ *
+ * Authors:
+ *  Janosch Frank <frankja@linux.ibm.com>
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2.
+ */
+#ifndef SMP_H
+#define SMP_H
+
+struct cpu {
+	struct lowcore *lowcore;
+	uint64_t *stack;
+	uint16_t addr;
+	bool active;
+};
+
+struct cpu_status {
+    uint64_t    fprs[16];                       /* 0x0000 */
+    uint64_t    grs[16];                        /* 0x0080 */
+    struct psw  psw;                            /* 0x0100 */
+    uint8_t     pad_0x0110[0x0118 - 0x0110];    /* 0x0110 */
+    uint32_t    prefix;                         /* 0x0118 */
+    uint32_t    fpc;                            /* 0x011c */
+    uint8_t     pad_0x0120[0x0124 - 0x0120];    /* 0x0120 */
+    uint32_t    todpr;                          /* 0x0124 */
+    uint64_t    cputm;                          /* 0x0128 */
+    uint64_t    ckc;                            /* 0x0130 */
+    uint8_t     pad_0x0138[0x0140 - 0x0138];    /* 0x0138 */
+    uint32_t    ars[16];                        /* 0x0140 */
+    uint64_t    crs[16];                        /* 0x0384 */
+};
+
+int smp_query_num_cpus(void);
+struct cpu *smp_cpu_from_addr(uint16_t addr);
+bool smp_cpu_stopped(uint16_t addr);
+bool smp_cpu_running(uint16_t addr);
+int smp_cpu_restart(uint16_t addr);
+int smp_cpu_start(uint16_t addr, struct psw psw);
+int smp_cpu_stop(uint16_t addr);
+int smp_cpu_stop_store_status(uint16_t addr);
+int smp_cpu_destroy(uint16_t addr);
+int smp_cpu_setup(uint16_t addr, struct psw psw);
+void smp_teardown(void);
+void smp_setup(void);
+
+#endif
diff --git a/s390x/Makefile b/s390x/Makefile
index 96033dd..d83dd0b 100644
--- a/s390x/Makefile
+++ b/s390x/Makefile
@@ -48,6 +48,7 @@ cflatobjs += lib/s390x/sclp.o
 cflatobjs += lib/s390x/sclp-console.o
 cflatobjs += lib/s390x/interrupt.o
 cflatobjs += lib/s390x/mmu.o
+cflatobjs += lib/s390x/smp.o
 
 OBJDIRS += lib/s390x
 
diff --git a/s390x/cstart64.S b/s390x/cstart64.S
index 36f7cab..5dc1577 100644
--- a/s390x/cstart64.S
+++ b/s390x/cstart64.S
@@ -172,6 +172,13 @@ diag308_load_reset:
 	lhi	%r2, 1
 	br	%r14
 
+.globl smp_cpu_setup_state
+smp_cpu_setup_state:
+	xgr	%r1, %r1
+	lmg     %r0, %r15, GEN_LC_SW_INT_GRS
+	lctlg   %c0, %c0, GEN_LC_SW_INT_CR0
+	br	%r14
+
 pgm_int:
 	SAVE_REGS
 	brasl	%r14, handle_pgm_int
-- 
2.17.2

Re: [kvm-unit-tests PATCH] s390x: Add initial smp code

[Thomas Huth]

On 23/09/2019 16.15, Janosch Frank wrote:
> Let's add a rudimentary SMP library, which will scan for cpus and has
> helper functions that manage the cpu state.
> 
> Signed-off-by: Janosch Frank <frankja@linux.ibm.com>
> ---
> 
> I left the setup code untouched, as I didn't want to indent too many
> lines.
> 
> ---
>  lib/s390x/asm/arch_def.h |   8 ++
>  lib/s390x/asm/sigp.h     |  28 ++++-
>  lib/s390x/io.c           |   5 +-
>  lib/s390x/sclp.h         |   1 +
>  lib/s390x/smp.c          | 252 +++++++++++++++++++++++++++++++++++++++
>  lib/s390x/smp.h          |  51 ++++++++
>  s390x/Makefile           |   1 +
>  s390x/cstart64.S         |   7 ++
>  8 files changed, 347 insertions(+), 6 deletions(-)
>  create mode 100644 lib/s390x/smp.c
>  create mode 100644 lib/s390x/smp.h

Reviewed-by: Thomas Huth <thuth@redhat.com>

[kvm-unit-tests PATCH v3 5/6] s390x: Prepare for external calls

[Janosch Frank]

With SMP we also get new external interrupts like external call and
emergency call. Let's make them known.

Signed-off-by: Janosch Frank <frankja@linux.ibm.com>
Reviewed-by: Thomas Huth <thuth@redhat.com>
---
 lib/s390x/asm/arch_def.h  |  5 +++++
 lib/s390x/asm/interrupt.h |  3 +++
 lib/s390x/interrupt.c     | 23 +++++++++++++++++++----
 3 files changed, 27 insertions(+), 4 deletions(-)

diff --git a/lib/s390x/asm/arch_def.h b/lib/s390x/asm/arch_def.h
index d5a7f51..96cca2e 100644
--- a/lib/s390x/asm/arch_def.h
+++ b/lib/s390x/asm/arch_def.h
@@ -19,6 +19,11 @@ struct psw {
 #define PSW_MASK_DAT			0x0400000000000000UL
 #define PSW_MASK_PSTATE			0x0001000000000000UL
 
+#define CR0_EXTM_SCLP			0X0000000000000200UL
+#define CR0_EXTM_EXTC			0X0000000000002000UL
+#define CR0_EXTM_EMGC			0X0000000000004000UL
+#define CR0_EXTM_MASK			0X0000000000006200UL
+
 struct lowcore {
 	uint8_t		pad_0x0000[0x0080 - 0x0000];	/* 0x0000 */
 	uint32_t	ext_int_param;			/* 0x0080 */
diff --git a/lib/s390x/asm/interrupt.h b/lib/s390x/asm/interrupt.h
index f485e96..4cfade9 100644
--- a/lib/s390x/asm/interrupt.h
+++ b/lib/s390x/asm/interrupt.h
@@ -11,6 +11,8 @@
 #define _ASMS390X_IRQ_H_
 #include <asm/arch_def.h>
 
+#define EXT_IRQ_EMERGENCY_SIG	0x1201
+#define EXT_IRQ_EXTERNAL_CALL	0x1202
 #define EXT_IRQ_SERVICE_SIG	0x2401
 
 void handle_pgm_int(void);
@@ -19,6 +21,7 @@ void handle_mcck_int(void);
 void handle_io_int(void);
 void handle_svc_int(void);
 void expect_pgm_int(void);
+void expect_ext_int(void);
 uint16_t clear_pgm_int(void);
 void check_pgm_int_code(uint16_t code);
 
diff --git a/lib/s390x/interrupt.c b/lib/s390x/interrupt.c
index 7832711..5cade23 100644
--- a/lib/s390x/interrupt.c
+++ b/lib/s390x/interrupt.c
@@ -15,6 +15,7 @@
 #include <sclp.h>
 
 static bool pgm_int_expected;
+static bool ext_int_expected;
 static struct lowcore *lc;
 
 void expect_pgm_int(void)
@@ -24,6 +25,13 @@ void expect_pgm_int(void)
 	mb();
 }
 
+void expect_ext_int(void)
+{
+	ext_int_expected = true;
+	lc->ext_int_code = 0;
+	mb();
+}
+
 uint16_t clear_pgm_int(void)
 {
 	uint16_t code;
@@ -108,15 +116,22 @@ void handle_pgm_int(void)
 
 void handle_ext_int(void)
 {
-	if (lc->ext_int_code != EXT_IRQ_SERVICE_SIG) {
+	if (!ext_int_expected &&
+	    lc->ext_int_code != EXT_IRQ_SERVICE_SIG) {
 		report_abort("Unexpected external call interrupt: at %#lx",
 			     lc->ext_old_psw.addr);
-	} else {
-		lc->ext_old_psw.mask &= ~PSW_MASK_EXT;
+		return;
+	}
+
+	if (lc->ext_int_code == EXT_IRQ_SERVICE_SIG) {
 		lc->sw_int_cr0 &= ~(1UL << 9);
 		sclp_handle_ext();
-		lc->ext_int_code = 0;
+	} else {
+		ext_int_expected = false;
 	}
+
+	if (!(lc->sw_int_cr0 & CR0_EXTM_MASK))
+		lc->ext_old_psw.mask &= ~PSW_MASK_EXT;
 }
 
 void handle_mcck_int(void)
-- 
2.17.2

Re: [kvm-unit-tests PATCH v3 5/6] s390x: Prepare for external calls

[David Hildenbrand]

On 20.09.19 10:03, Janosch Frank wrote:
> With SMP we also get new external interrupts like external call and
> emergency call. Let's make them known.
> 
> Signed-off-by: Janosch Frank <frankja@linux.ibm.com>
> Reviewed-by: Thomas Huth <thuth@redhat.com>
> ---
>  lib/s390x/asm/arch_def.h  |  5 +++++
>  lib/s390x/asm/interrupt.h |  3 +++
>  lib/s390x/interrupt.c     | 23 +++++++++++++++++++----
>  3 files changed, 27 insertions(+), 4 deletions(-)
> 
> diff --git a/lib/s390x/asm/arch_def.h b/lib/s390x/asm/arch_def.h
> index d5a7f51..96cca2e 100644
> --- a/lib/s390x/asm/arch_def.h
> +++ b/lib/s390x/asm/arch_def.h
> @@ -19,6 +19,11 @@ struct psw {
>  #define PSW_MASK_DAT			0x0400000000000000UL
>  #define PSW_MASK_PSTATE			0x0001000000000000UL
>  
> +#define CR0_EXTM_SCLP			0X0000000000000200UL
> +#define CR0_EXTM_EXTC			0X0000000000002000UL
> +#define CR0_EXTM_EMGC			0X0000000000004000UL
> +#define CR0_EXTM_MASK			0X0000000000006200UL
> +
>  struct lowcore {
>  	uint8_t		pad_0x0000[0x0080 - 0x0000];	/* 0x0000 */
>  	uint32_t	ext_int_param;			/* 0x0080 */
> diff --git a/lib/s390x/asm/interrupt.h b/lib/s390x/asm/interrupt.h
> index f485e96..4cfade9 100644
> --- a/lib/s390x/asm/interrupt.h
> +++ b/lib/s390x/asm/interrupt.h
> @@ -11,6 +11,8 @@
>  #define _ASMS390X_IRQ_H_
>  #include <asm/arch_def.h>
>  
> +#define EXT_IRQ_EMERGENCY_SIG	0x1201
> +#define EXT_IRQ_EXTERNAL_CALL	0x1202
>  #define EXT_IRQ_SERVICE_SIG	0x2401
>  
>  void handle_pgm_int(void);
> @@ -19,6 +21,7 @@ void handle_mcck_int(void);
>  void handle_io_int(void);
>  void handle_svc_int(void);
>  void expect_pgm_int(void);
> +void expect_ext_int(void);
>  uint16_t clear_pgm_int(void);
>  void check_pgm_int_code(uint16_t code);
>  
> diff --git a/lib/s390x/interrupt.c b/lib/s390x/interrupt.c
> index 7832711..5cade23 100644
> --- a/lib/s390x/interrupt.c
> +++ b/lib/s390x/interrupt.c
> @@ -15,6 +15,7 @@
>  #include <sclp.h>
>  
>  static bool pgm_int_expected;
> +static bool ext_int_expected;
>  static struct lowcore *lc;
>  
>  void expect_pgm_int(void)
> @@ -24,6 +25,13 @@ void expect_pgm_int(void)
>  	mb();
>  }
>  
> +void expect_ext_int(void)
> +{
> +	ext_int_expected = true;
> +	lc->ext_int_code = 0;
> +	mb();
> +}
> +
>  uint16_t clear_pgm_int(void)
>  {
>  	uint16_t code;
> @@ -108,15 +116,22 @@ void handle_pgm_int(void)
>  
>  void handle_ext_int(void)
>  {
> -	if (lc->ext_int_code != EXT_IRQ_SERVICE_SIG) {
> +	if (!ext_int_expected &&
> +	    lc->ext_int_code != EXT_IRQ_SERVICE_SIG) {
>  		report_abort("Unexpected external call interrupt: at %#lx",
>  			     lc->ext_old_psw.addr);
> -	} else {
> -		lc->ext_old_psw.mask &= ~PSW_MASK_EXT;
> +		return;
> +	}
> +
> +	if (lc->ext_int_code == EXT_IRQ_SERVICE_SIG) {
>  		lc->sw_int_cr0 &= ~(1UL << 9);
>  		sclp_handle_ext();
> -		lc->ext_int_code = 0;
> +	} else {
> +		ext_int_expected = false;
>  	}
> +
> +	if (!(lc->sw_int_cr0 & CR0_EXTM_MASK))
> +		lc->ext_old_psw.mask &= ~PSW_MASK_EXT;
>  }
>  
>  void handle_mcck_int(void)
> 

Reviewed-by: David Hildenbrand <david@redhat.com>

-- 

Thanks,

David / dhildenb

[kvm-unit-tests PATCH v3 6/6] s390x: SMP test

[Janosch Frank]

Testing SIGP emulation for the following order codes:
* start
* stop
* restart
* set prefix
* store status
* stop and store status
* reset
* initial reset
* external call
* emegergency call

restart and set prefix are part of the library and needed to start
other cpus.

Signed-off-by: Janosch Frank <frankja@linux.ibm.com>
---
 s390x/Makefile      |   1 +
 s390x/smp.c         | 242 ++++++++++++++++++++++++++++++++++++++++++++
 s390x/unittests.cfg |   4 +
 3 files changed, 247 insertions(+)
 create mode 100644 s390x/smp.c

diff --git a/s390x/Makefile b/s390x/Makefile
index d83dd0b..3744372 100644
--- a/s390x/Makefile
+++ b/s390x/Makefile
@@ -15,6 +15,7 @@ tests += $(TEST_DIR)/cpumodel.elf
 tests += $(TEST_DIR)/diag288.elf
 tests += $(TEST_DIR)/stsi.elf
 tests += $(TEST_DIR)/skrf.elf
+tests += $(TEST_DIR)/smp.elf
 tests_binary = $(patsubst %.elf,%.bin,$(tests))
 
 all: directories test_cases test_cases_binary
diff --git a/s390x/smp.c b/s390x/smp.c
new file mode 100644
index 0000000..7032494
--- /dev/null
+++ b/s390x/smp.c
@@ -0,0 +1,242 @@
+/*
+ * Tests sigp emulation
+ *
+ * Copyright 2019 IBM Corp.
+ *
+ * Authors:
+ *    Janosch Frank <frankja@linux.ibm.com>
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2.
+ */
+#include <libcflat.h>
+#include <asm/asm-offsets.h>
+#include <asm/interrupt.h>
+#include <asm/page.h>
+#include <asm/facility.h>
+#include <asm-generic/barrier.h>
+#include <asm/sigp.h>
+
+#include <smp.h>
+#include <alloc_page.h>
+
+static int testflag = 0;
+
+static void cpu_loop(void)
+{
+	for (;;) {}
+}
+
+static void test_func(void)
+{
+	testflag = 1;
+	mb();
+	cpu_loop();
+}
+
+static void test_start(void)
+{
+	struct psw psw;
+	psw.mask =  extract_psw_mask();
+	psw.addr = (unsigned long)test_func;
+
+	smp_cpu_setup(1, psw);
+	while (!testflag) {
+		mb();
+	}
+	report("start", 1);
+}
+
+static void test_stop(void)
+{
+	smp_cpu_stop(1);
+	/*
+	 * The smp library waits for the CPU to shut down, but let's
+	 * also do it here, so we don't rely on the library
+	 * implementation
+	 */
+	while (!smp_cpu_stopped(1)) {}
+	report("stop", 1);
+}
+
+static void test_stop_store_status(void)
+{
+	struct cpu *cpu = smp_cpu_from_addr(1);
+	struct lowcore *lc = (void *)0x0;
+
+	report_prefix_push("stop store status");
+	lc->prefix_sa = 0;
+	lc->grs_sa[15] = 0;
+	smp_cpu_stop_store_status(1);
+	mb();
+	report("prefix", lc->prefix_sa == (uint32_t)(uintptr_t)cpu->lowcore);
+	report("stack", lc->grs_sa[15]);
+	report_prefix_pop();
+}
+
+static void test_store_status(void)
+{
+	struct cpu_status *status = alloc_pages(1);
+	uint32_t r;
+
+	report_prefix_push("store status at address");
+	memset(status, 0, PAGE_SIZE * 2);
+
+	report_prefix_push("running");
+	smp_cpu_restart(1);
+	sigp(1, SIGP_STORE_STATUS_AT_ADDRESS, (uintptr_t)status, &r);
+	report("incorrect state", r == SIGP_STATUS_INCORRECT_STATE);
+	report("status not written", !memcmp(status, (void*)status + PAGE_SIZE, PAGE_SIZE));
+	report_prefix_pop();
+
+	memset(status, 0, PAGE_SIZE);
+	report_prefix_push("stopped");
+	smp_cpu_stop(1);
+	sigp(1, SIGP_STORE_STATUS_AT_ADDRESS, (uintptr_t)status, NULL);
+	while (!status->prefix) { mb(); }
+	report("status written", 1);
+	free_pages(status, PAGE_SIZE * 2);
+	report_prefix_pop();
+
+	report_prefix_pop();
+}
+
+static void ecall(void)
+{
+	unsigned long mask;
+	struct lowcore *lc = (void *)0x0;
+
+	expect_ext_int();
+	ctl_set_bit(0, 13);
+	mask = extract_psw_mask();
+	mask |= PSW_MASK_EXT;
+	load_psw_mask(mask);
+	testflag = 1;
+	while (lc->ext_int_code != 0x1202) { mb(); }
+	report("ecall", 1);
+	testflag= 1;
+}
+
+static void test_ecall(void)
+{
+	struct psw psw;
+	psw.mask =  extract_psw_mask();
+	psw.addr = (unsigned long)ecall;
+
+	report_prefix_push("ecall");
+	testflag= 0;
+	smp_cpu_destroy(1);
+
+	smp_cpu_setup(1, psw);
+	while (!testflag) { mb(); }
+	testflag= 0;
+	sigp(1, SIGP_EXTERNAL_CALL, 0, NULL);
+	while(!testflag) {mb();}
+	smp_cpu_stop(1);
+	report_prefix_pop();
+}
+
+static void emcall(void)
+{
+	unsigned long mask;
+	struct lowcore *lc = (void *)0x0;
+
+	expect_ext_int();
+	ctl_set_bit(0, 14);
+	mask = extract_psw_mask();
+	mask |= PSW_MASK_EXT;
+	load_psw_mask(mask);
+	testflag= 1;
+	while (lc->ext_int_code != 0x1201) { mb(); }
+	report("ecall", 1);
+	testflag = 1;
+}
+
+static void test_emcall(void)
+{
+	struct psw psw;
+	psw.mask =  extract_psw_mask();
+	psw.addr = (unsigned long)emcall;
+
+	report_prefix_push("emcall");
+	testflag= 0;
+	smp_cpu_destroy(1);
+
+	smp_cpu_setup(1, psw);
+	while (!testflag) { mb(); }
+	testflag= 0;
+	sigp(1, SIGP_EMERGENCY_SIGNAL, 0, NULL);
+	while(!testflag) { mb(); }
+	smp_cpu_stop(1);
+	report_prefix_pop();
+}
+
+static void test_reset_initial(void)
+{
+	struct cpu_status *status = alloc_pages(0);
+	struct psw psw;
+
+	psw.mask =  extract_psw_mask();
+	psw.addr = (unsigned long)test_func;
+
+	report_prefix_push("reset initial");
+	smp_cpu_setup(1, psw);
+
+	sigp_retry(1, SIGP_INITIAL_CPU_RESET, 0, NULL);
+	sigp(1, SIGP_STORE_STATUS_AT_ADDRESS, (uintptr_t)status, NULL);
+
+	report_prefix_push("clear");
+	report("psw", !status->psw.mask && !status->psw.addr);
+	report("prefix", !status->prefix);
+	report("fpc", !status->fpc);
+	report("cpu timer", !status->cputm);
+	report("todpr", !status->todpr);
+	report_prefix_pop();
+
+	report_prefix_push("initialized");
+	report("cr0 == 0xE0", status->crs[0] == 0xE0UL);
+	report("cr14 == 0xC2000000", status->crs[14] == 0xC2000000UL);
+	report_prefix_pop();
+
+	report("cpu stopped", smp_cpu_stopped(1));
+	free_pages(status, PAGE_SIZE);
+	report_prefix_pop();
+}
+
+static void test_reset(void)
+{
+	struct psw psw;
+
+	psw.mask =  extract_psw_mask();
+	psw.addr = (unsigned long)test_func;
+
+	report_prefix_push("cpu reset");
+	smp_cpu_setup(1, psw);
+
+	sigp_retry(1, SIGP_CPU_RESET, 0, NULL);
+	report("cpu stopped", smp_cpu_stopped(1));
+	report_prefix_pop();
+}
+
+int main(void)
+{
+	report_prefix_push("smp");
+
+	if (smp_query_num_cpus() == 1) {
+		report_abort("need at least 2 cpus for this test");
+		goto done;
+	}
+
+	test_start();
+	test_stop();
+	test_stop_store_status();
+	test_store_status();
+	test_ecall();
+	test_emcall();
+	test_reset();
+	test_reset_initial();
+
+done:
+	report_prefix_pop();
+	return report_summary();
+}
diff --git a/s390x/unittests.cfg b/s390x/unittests.cfg
index cc79a4e..f1b07cd 100644
--- a/s390x/unittests.cfg
+++ b/s390x/unittests.cfg
@@ -71,3 +71,7 @@ extra_params=-device diag288,id=watchdog0 --watchdog-action inject-nmi
 
 [stsi]
 file = stsi.elf
+
+[smp]
+file = smp.elf
+extra_params =-smp 2
-- 
2.17.2

Re: [kvm-unit-tests PATCH v3 6/6] s390x: SMP test

[Thomas Huth]

On 20/09/2019 10.03, Janosch Frank wrote:
> Testing SIGP emulation for the following order codes:
> * start
> * stop
> * restart
> * set prefix
> * store status
> * stop and store status
> * reset
> * initial reset
> * external call
> * emegergency call
> 
> restart and set prefix are part of the library and needed to start
> other cpus.
> 
> Signed-off-by: Janosch Frank <frankja@linux.ibm.com>
> ---
>  s390x/Makefile      |   1 +
>  s390x/smp.c         | 242 ++++++++++++++++++++++++++++++++++++++++++++
[...]
> +int main(void)
> +{
> +	report_prefix_push("smp");
> +
> +	if (smp_query_num_cpus() == 1) {
> +		report_abort("need at least 2 cpus for this test");

nit: report_abort() aborts immediately, so the "goto done" below is dead
code. Did you mean report_skip() instead?

> +		goto done;
> +	}
> +
> +	test_start();
> +	test_stop();
> +	test_stop_store_status();
> +	test_store_status();
> +	test_ecall();
> +	test_emcall();
> +	test_reset();
> +	test_reset_initial();
> +
> +done:
> +	report_prefix_pop();
> +	return report_summary();
> +}

Apart from the nit, the patch looks fine to me. Since this is IIRC the
only nit that is left in this series, I can also fix it up when picking
up the patch, if you like - just tell me whether you prefer report_skip
or deletion of the goto.

 Thomas

Re: [kvm-unit-tests PATCH v3 6/6] s390x: SMP test

[Janosch Frank]

[-- Attachment #1.1: Type: text/plain, Size: 1627 bytes --]

On 9/25/19 10:49 AM, Thomas Huth wrote:
> On 20/09/2019 10.03, Janosch Frank wrote:
>> Testing SIGP emulation for the following order codes:
>> * start
>> * stop
>> * restart
>> * set prefix
>> * store status
>> * stop and store status
>> * reset
>> * initial reset
>> * external call
>> * emegergency call
>>
>> restart and set prefix are part of the library and needed to start
>> other cpus.
>>
>> Signed-off-by: Janosch Frank <frankja@linux.ibm.com>
>> ---
>>  s390x/Makefile      |   1 +
>>  s390x/smp.c         | 242 ++++++++++++++++++++++++++++++++++++++++++++
> [...]
>> +int main(void)
>> +{
>> +	report_prefix_push("smp");
>> +
>> +	if (smp_query_num_cpus() == 1) {
>> +		report_abort("need at least 2 cpus for this test");
> 
> nit: report_abort() aborts immediately, so the "goto done" below is dead
> code. Did you mean report_skip() instead?

That would make more sense, yes

> 
>> +		goto done;
>> +	}
>> +
>> +	test_start();
>> +	test_stop();
>> +	test_stop_store_status();
>> +	test_store_status();
>> +	test_ecall();
>> +	test_emcall();
>> +	test_reset();
>> +	test_reset_initial();
>> +
>> +done:
>> +	report_prefix_pop();
>> +	return report_summary();
>> +}
> 
> Apart from the nit, the patch looks fine to me. Since this is IIRC the
> only nit that is left in this series, I can also fix it up when picking
> up the patch, if you like - just tell me whether you prefer report_skip
> or deletion of the goto.

Well, that might depend on how much weight David's latest comment has.
:-) If you want to, you can pick and fix.

> 
>  Thomas
> 



[-- Attachment #2: OpenPGP digital signature --]
[-- Type: application/pgp-signature, Size: 833 bytes --]

Re: [kvm-unit-tests PATCH v3 6/6] s390x: SMP test

[David Hildenbrand]

On 20.09.19 10:03, Janosch Frank wrote:
> Testing SIGP emulation for the following order codes:
> * start
> * stop
> * restart
> * set prefix
> * store status
> * stop and store status
> * reset
> * initial reset
> * external call
> * emegergency call
> 
> restart and set prefix are part of the library and needed to start
> other cpus.
> 
> Signed-off-by: Janosch Frank <frankja@linux.ibm.com>
> ---
>  s390x/Makefile      |   1 +
>  s390x/smp.c         | 242 ++++++++++++++++++++++++++++++++++++++++++++
>  s390x/unittests.cfg |   4 +
>  3 files changed, 247 insertions(+)
>  create mode 100644 s390x/smp.c
> 
> diff --git a/s390x/Makefile b/s390x/Makefile
> index d83dd0b..3744372 100644
> --- a/s390x/Makefile
> +++ b/s390x/Makefile
> @@ -15,6 +15,7 @@ tests += $(TEST_DIR)/cpumodel.elf
>  tests += $(TEST_DIR)/diag288.elf
>  tests += $(TEST_DIR)/stsi.elf
>  tests += $(TEST_DIR)/skrf.elf
> +tests += $(TEST_DIR)/smp.elf
>  tests_binary = $(patsubst %.elf,%.bin,$(tests))
>  
>  all: directories test_cases test_cases_binary
> diff --git a/s390x/smp.c b/s390x/smp.c
> new file mode 100644
> index 0000000..7032494
> --- /dev/null
> +++ b/s390x/smp.c
> @@ -0,0 +1,242 @@
> +/*
> + * Tests sigp emulation
> + *
> + * Copyright 2019 IBM Corp.
> + *
> + * Authors:
> + *    Janosch Frank <frankja@linux.ibm.com>
> + *
> + * This code is free software; you can redistribute it and/or modify it
> + * under the terms of the GNU General Public License version 2.
> + */
> +#include <libcflat.h>
> +#include <asm/asm-offsets.h>
> +#include <asm/interrupt.h>
> +#include <asm/page.h>
> +#include <asm/facility.h>
> +#include <asm-generic/barrier.h>
> +#include <asm/sigp.h>
> +
> +#include <smp.h>
> +#include <alloc_page.h>
> +
> +static int testflag = 0;
> +
> +static void cpu_loop(void)
> +{
> +	for (;;) {}
> +}
> +
> +static void test_func(void)
> +{
> +	testflag = 1;
> +	mb();
> +	cpu_loop();
> +}
> +
> +static void test_start(void)
> +{
> +	struct psw psw;
> +	psw.mask =  extract_psw_mask();
> +	psw.addr = (unsigned long)test_func;
> +
> +	smp_cpu_setup(1, psw);
> +	while (!testflag) {
> +		mb();
> +	}
> +	report("start", 1);
> +}
> +
> +static void test_stop(void)
> +{
> +	smp_cpu_stop(1);
> +	/*
> +	 * The smp library waits for the CPU to shut down, but let's
> +	 * also do it here, so we don't rely on the library
> +	 * implementation
> +	 */
> +	while (!smp_cpu_stopped(1)) {}
> +	report("stop", 1);
> +}
> +
> +static void test_stop_store_status(void)
> +{
> +	struct cpu *cpu = smp_cpu_from_addr(1);
> +	struct lowcore *lc = (void *)0x0;
> +
> +	report_prefix_push("stop store status");
> +	lc->prefix_sa = 0;
> +	lc->grs_sa[15] = 0;
> +	smp_cpu_stop_store_status(1);
> +	mb();
> +	report("prefix", lc->prefix_sa == (uint32_t)(uintptr_t)cpu->lowcore);
> +	report("stack", lc->grs_sa[15]);
> +	report_prefix_pop();
> +}
> +
> +static void test_store_status(void)
> +{
> +	struct cpu_status *status = alloc_pages(1);
> +	uint32_t r;
> +
> +	report_prefix_push("store status at address");
> +	memset(status, 0, PAGE_SIZE * 2);
> +
> +	report_prefix_push("running");
> +	smp_cpu_restart(1);
> +	sigp(1, SIGP_STORE_STATUS_AT_ADDRESS, (uintptr_t)status, &r);
> +	report("incorrect state", r == SIGP_STATUS_INCORRECT_STATE);
> +	report("status not written", !memcmp(status, (void*)status + PAGE_SIZE, PAGE_SIZE));
> +	report_prefix_pop();
> +
> +	memset(status, 0, PAGE_SIZE);
> +	report_prefix_push("stopped");
> +	smp_cpu_stop(1);
> +	sigp(1, SIGP_STORE_STATUS_AT_ADDRESS, (uintptr_t)status, NULL);
> +	while (!status->prefix) { mb(); }
> +	report("status written", 1);
> +	free_pages(status, PAGE_SIZE * 2);
> +	report_prefix_pop();
> +
> +	report_prefix_pop();
> +}
> +
> +static void ecall(void)
> +{
> +	unsigned long mask;
> +	struct lowcore *lc = (void *)0x0;
> +
> +	expect_ext_int();
> +	ctl_set_bit(0, 13);
> +	mask = extract_psw_mask();
> +	mask |= PSW_MASK_EXT;
> +	load_psw_mask(mask);
> +	testflag = 1;
> +	while (lc->ext_int_code != 0x1202) { mb(); }
> +	report("ecall", 1);
> +	testflag= 1;
> +}
> +
> +static void test_ecall(void)
> +{
> +	struct psw psw;
> +	psw.mask =  extract_psw_mask();
> +	psw.addr = (unsigned long)ecall;
> +
> +	report_prefix_push("ecall");
> +	testflag= 0;
> +	smp_cpu_destroy(1);
> +
> +	smp_cpu_setup(1, psw);
> +	while (!testflag) { mb(); }
> +	testflag= 0;
> +	sigp(1, SIGP_EXTERNAL_CALL, 0, NULL);
> +	while(!testflag) {mb();}
> +	smp_cpu_stop(1);
> +	report_prefix_pop();
> +}
> +
> +static void emcall(void)
> +{
> +	unsigned long mask;
> +	struct lowcore *lc = (void *)0x0;
> +
> +	expect_ext_int();
> +	ctl_set_bit(0, 14);
> +	mask = extract_psw_mask();
> +	mask |= PSW_MASK_EXT;
> +	load_psw_mask(mask);
> +	testflag= 1;
> +	while (lc->ext_int_code != 0x1201) { mb(); }
> +	report("ecall", 1);
> +	testflag = 1;
> +}
> +
> +static void test_emcall(void)
> +{
> +	struct psw psw;
> +	psw.mask =  extract_psw_mask();
> +	psw.addr = (unsigned long)emcall;
> +
> +	report_prefix_push("emcall");
> +	testflag= 0;
> +	smp_cpu_destroy(1);
> +
> +	smp_cpu_setup(1, psw);
> +	while (!testflag) { mb(); }
> +	testflag= 0;
> +	sigp(1, SIGP_EMERGENCY_SIGNAL, 0, NULL);
> +	while(!testflag) { mb(); }
> +	smp_cpu_stop(1);
> +	report_prefix_pop();
> +}
> +
> +static void test_reset_initial(void)
> +{
> +	struct cpu_status *status = alloc_pages(0);
> +	struct psw psw;
> +
> +	psw.mask =  extract_psw_mask();
> +	psw.addr = (unsigned long)test_func;
> +
> +	report_prefix_push("reset initial");
> +	smp_cpu_setup(1, psw);
> +
> +	sigp_retry(1, SIGP_INITIAL_CPU_RESET, 0, NULL);
> +	sigp(1, SIGP_STORE_STATUS_AT_ADDRESS, (uintptr_t)status, NULL);
> +
> +	report_prefix_push("clear");
> +	report("psw", !status->psw.mask && !status->psw.addr);
> +	report("prefix", !status->prefix);
> +	report("fpc", !status->fpc);
> +	report("cpu timer", !status->cputm);
> +	report("todpr", !status->todpr);
> +	report_prefix_pop();
> +
> +	report_prefix_push("initialized");
> +	report("cr0 == 0xE0", status->crs[0] == 0xE0UL);
> +	report("cr14 == 0xC2000000", status->crs[14] == 0xC2000000UL);
> +	report_prefix_pop();
> +
> +	report("cpu stopped", smp_cpu_stopped(1));
> +	free_pages(status, PAGE_SIZE);
> +	report_prefix_pop();
> +}
> +
> +static void test_reset(void)
> +{
> +	struct psw psw;
> +
> +	psw.mask =  extract_psw_mask();
> +	psw.addr = (unsigned long)test_func;
> +
> +	report_prefix_push("cpu reset");
> +	smp_cpu_setup(1, psw);
> +
> +	sigp_retry(1, SIGP_CPU_RESET, 0, NULL);
> +	report("cpu stopped", smp_cpu_stopped(1));
> +	report_prefix_pop();
> +}
> +
> +int main(void)
> +{
> +	report_prefix_push("smp");
> +
> +	if (smp_query_num_cpus() == 1) {
> +		report_abort("need at least 2 cpus for this test");
> +		goto done;
> +	}

You should rather sense if CPU with the addr 1 is available. AFAIR, the
number of CPUs and the CPU addresses are two types of shoes.



-- 

Thanks,

David / dhildenb

Re: [kvm-unit-tests PATCH v3 6/6] s390x: SMP test

[Janosch Frank]

[-- Attachment #1.1: Type: text/plain, Size: 582 bytes --]

On 9/25/19 11:03 AM, David Hildenbrand wrote:
> On 20.09.19 10:03, Janosch Frank wrote:

>> +
>> +int main(void)
>> +{
>> +	report_prefix_push("smp");
>> +
>> +	if (smp_query_num_cpus() == 1) {
>> +		report_abort("need at least 2 cpus for this test");
>> +		goto done;
>> +	}
> 
> You should rather sense if CPU with the addr 1 is available. AFAIR, the
> number of CPUs and the CPU addresses are two types of shoes.
> 

Yes and no, if we don't have more than one cpu we certainly don't have
cpu 1. And for what I've seen up to now QEMU assigns cpus linearly.



[-- Attachment #2: OpenPGP digital signature --]
[-- Type: application/pgp-signature, Size: 833 bytes --]

Re: [kvm-unit-tests PATCH v3 6/6] s390x: SMP test

[David Hildenbrand]

On 20.09.19 10:03, Janosch Frank wrote:
> Testing SIGP emulation for the following order codes:
> * start
> * stop
> * restart
> * set prefix
> * store status
> * stop and store status
> * reset
> * initial reset
> * external call
> * emegergency call
> 
> restart and set prefix are part of the library and needed to start
> other cpus.
> 
> Signed-off-by: Janosch Frank <frankja@linux.ibm.com>
> ---
>  s390x/Makefile      |   1 +
>  s390x/smp.c         | 242 ++++++++++++++++++++++++++++++++++++++++++++
>  s390x/unittests.cfg |   4 +
>  3 files changed, 247 insertions(+)
>  create mode 100644 s390x/smp.c
> 
> diff --git a/s390x/Makefile b/s390x/Makefile
> index d83dd0b..3744372 100644
> --- a/s390x/Makefile
> +++ b/s390x/Makefile
> @@ -15,6 +15,7 @@ tests += $(TEST_DIR)/cpumodel.elf
>  tests += $(TEST_DIR)/diag288.elf
>  tests += $(TEST_DIR)/stsi.elf
>  tests += $(TEST_DIR)/skrf.elf
> +tests += $(TEST_DIR)/smp.elf
>  tests_binary = $(patsubst %.elf,%.bin,$(tests))
>  
>  all: directories test_cases test_cases_binary
> diff --git a/s390x/smp.c b/s390x/smp.c
> new file mode 100644
> index 0000000..7032494
> --- /dev/null
> +++ b/s390x/smp.c
> @@ -0,0 +1,242 @@
> +/*
> + * Tests sigp emulation
> + *
> + * Copyright 2019 IBM Corp.
> + *
> + * Authors:
> + *    Janosch Frank <frankja@linux.ibm.com>
> + *
> + * This code is free software; you can redistribute it and/or modify it
> + * under the terms of the GNU General Public License version 2.
> + */
> +#include <libcflat.h>
> +#include <asm/asm-offsets.h>
> +#include <asm/interrupt.h>
> +#include <asm/page.h>
> +#include <asm/facility.h>
> +#include <asm-generic/barrier.h>
> +#include <asm/sigp.h>
> +
> +#include <smp.h>
> +#include <alloc_page.h>
> +
> +static int testflag = 0;
> +
> +static void cpu_loop(void)
> +{
> +	for (;;) {}

Won't that be optimized out completely?

> +}
> +
> +static void test_func(void)
> +{
> +	testflag = 1;
> +	mb();
> +	cpu_loop();
> +}
> +
> +static void test_start(void)
> +{
> +	struct psw psw;
> +	psw.mask =  extract_psw_mask();
> +	psw.addr = (unsigned long)test_func;
> +
> +	smp_cpu_setup(1, psw);
> +	while (!testflag) {
> +		mb();
> +	}
> +	report("start", 1);
> +}
> +
> +static void test_stop(void)
> +{
> +	smp_cpu_stop(1);
> +	/*
> +	 * The smp library waits for the CPU to shut down, but let's
> +	 * also do it here, so we don't rely on the library
> +	 * implementation
> +	 */
> +	while (!smp_cpu_stopped(1)) {}
> +	report("stop", 1);
> +}
> +
> +static void test_stop_store_status(void)
> +{
> +	struct cpu *cpu = smp_cpu_from_addr(1);
> +	struct lowcore *lc = (void *)0x0;
> +
> +	report_prefix_push("stop store status");
> +	lc->prefix_sa = 0;
> +	lc->grs_sa[15] = 0;
> +	smp_cpu_stop_store_status(1);
> +	mb();
> +	report("prefix", lc->prefix_sa == (uint32_t)(uintptr_t)cpu->lowcore);
> +	report("stack", lc->grs_sa[15]);
> +	report_prefix_pop();
> +}
> +
> +static void test_store_status(void)
> +{
> +	struct cpu_status *status = alloc_pages(1);
> +	uint32_t r;
> +
> +	report_prefix_push("store status at address");
> +	memset(status, 0, PAGE_SIZE * 2);
> +
> +	report_prefix_push("running");
> +	smp_cpu_restart(1);
> +	sigp(1, SIGP_STORE_STATUS_AT_ADDRESS, (uintptr_t)status, &r);
> +	report("incorrect state", r == SIGP_STATUS_INCORRECT_STATE);
> +	report("status not written", !memcmp(status, (void*)status + PAGE_SIZE, PAGE_SIZE));
> +	report_prefix_pop();
> +
> +	memset(status, 0, PAGE_SIZE);
> +	report_prefix_push("stopped");
> +	smp_cpu_stop(1);
> +	sigp(1, SIGP_STORE_STATUS_AT_ADDRESS, (uintptr_t)status, NULL);
> +	while (!status->prefix) { mb(); }
> +	report("status written", 1);
> +	free_pages(status, PAGE_SIZE * 2);
> +	report_prefix_pop();
> +
> +	report_prefix_pop();
> +}
> +
> +static void ecall(void)
> +{
> +	unsigned long mask;
> +	struct lowcore *lc = (void *)0x0;
> +
> +	expect_ext_int();
> +	ctl_set_bit(0, 13);
> +	mask = extract_psw_mask();
> +	mask |= PSW_MASK_EXT;
> +	load_psw_mask(mask);
> +	testflag = 1;
> +	while (lc->ext_int_code != 0x1202) { mb(); }
> +	report("ecall", 1);
> +	testflag= 1;
> +}
> +
> +static void test_ecall(void)
> +{
> +	struct psw psw;
> +	psw.mask =  extract_psw_mask();
> +	psw.addr = (unsigned long)ecall;
> +
> +	report_prefix_push("ecall");
> +	testflag= 0;
> +	smp_cpu_destroy(1);
> +
> +	smp_cpu_setup(1, psw);
> +	while (!testflag) { mb(); }
> +	testflag= 0;
> +	sigp(1, SIGP_EXTERNAL_CALL, 0, NULL);
> +	while(!testflag) {mb();}
> +	smp_cpu_stop(1);
> +	report_prefix_pop();
> +}
> +
> +static void emcall(void)
> +{
> +	unsigned long mask;
> +	struct lowcore *lc = (void *)0x0;
> +
> +	expect_ext_int();
> +	ctl_set_bit(0, 14);
> +	mask = extract_psw_mask();
> +	mask |= PSW_MASK_EXT;
> +	load_psw_mask(mask);
> +	testflag= 1;
> +	while (lc->ext_int_code != 0x1201) { mb(); }
> +	report("ecall", 1);
> +	testflag = 1;
> +}
> +
> +static void test_emcall(void)
> +{
> +	struct psw psw;
> +	psw.mask =  extract_psw_mask();
> +	psw.addr = (unsigned long)emcall;
> +
> +	report_prefix_push("emcall");
> +	testflag= 0;
> +	smp_cpu_destroy(1);
> +
> +	smp_cpu_setup(1, psw);
> +	while (!testflag) { mb(); }
> +	testflag= 0;
> +	sigp(1, SIGP_EMERGENCY_SIGNAL, 0, NULL);
> +	while(!testflag) { mb(); }
> +	smp_cpu_stop(1);
> +	report_prefix_pop();
> +}
> +
> +static void test_reset_initial(void)
> +{
> +	struct cpu_status *status = alloc_pages(0);
> +	struct psw psw;
> +
> +	psw.mask =  extract_psw_mask();
> +	psw.addr = (unsigned long)test_func;
> +
> +	report_prefix_push("reset initial");
> +	smp_cpu_setup(1, psw);
> +
> +	sigp_retry(1, SIGP_INITIAL_CPU_RESET, 0, NULL);
> +	sigp(1, SIGP_STORE_STATUS_AT_ADDRESS, (uintptr_t)status, NULL);
> +
> +	report_prefix_push("clear");
> +	report("psw", !status->psw.mask && !status->psw.addr);
> +	report("prefix", !status->prefix);
> +	report("fpc", !status->fpc);
> +	report("cpu timer", !status->cputm);
> +	report("todpr", !status->todpr);
> +	report_prefix_pop();
> +
> +	report_prefix_push("initialized");
> +	report("cr0 == 0xE0", status->crs[0] == 0xE0UL);
> +	report("cr14 == 0xC2000000", status->crs[14] == 0xC2000000UL);
> +	report_prefix_pop();
> +
> +	report("cpu stopped", smp_cpu_stopped(1));
> +	free_pages(status, PAGE_SIZE);
> +	report_prefix_pop();
> +}
> +
> +static void test_reset(void)
> +{
> +	struct psw psw;
> +
> +	psw.mask =  extract_psw_mask();
> +	psw.addr = (unsigned long)test_func;
> +
> +	report_prefix_push("cpu reset");
> +	smp_cpu_setup(1, psw);
> +
> +	sigp_retry(1, SIGP_CPU_RESET, 0, NULL);
> +	report("cpu stopped", smp_cpu_stopped(1));
> +	report_prefix_pop();
> +}
> +
> +int main(void)
> +{
> +	report_prefix_push("smp");
> +
> +	if (smp_query_num_cpus() == 1) {
> +		report_abort("need at least 2 cpus for this test");
> +		goto done;
> +	}
> +
> +	test_start();
> +	test_stop();
> +	test_stop_store_status();
> +	test_store_status();
> +	test_ecall();
> +	test_emcall();
> +	test_reset();
> +	test_reset_initial();
> +
> +done:
> +	report_prefix_pop();
> +	return report_summary();
> +}
> diff --git a/s390x/unittests.cfg b/s390x/unittests.cfg
> index cc79a4e..f1b07cd 100644
> --- a/s390x/unittests.cfg
> +++ b/s390x/unittests.cfg
> @@ -71,3 +71,7 @@ extra_params=-device diag288,id=watchdog0 --watchdog-action inject-nmi
>  
>  [stsi]
>  file = stsi.elf
> +
> +[smp]
> +file = smp.elf
> +extra_params =-smp 2
> 


-- 

Thanks,

David / dhildenb

Re: [kvm-unit-tests PATCH v3 6/6] s390x: SMP test

[Thomas Huth]

On 25/09/2019 15.27, David Hildenbrand wrote:
> On 20.09.19 10:03, Janosch Frank wrote:
>> Testing SIGP emulation for the following order codes:
>> * start
>> * stop
>> * restart
>> * set prefix
>> * store status
>> * stop and store status
>> * reset
>> * initial reset
>> * external call
>> * emegergency call
>>
>> restart and set prefix are part of the library and needed to start
>> other cpus.
>>
>> Signed-off-by: Janosch Frank <frankja@linux.ibm.com>
>> ---
>>  s390x/Makefile      |   1 +
>>  s390x/smp.c         | 242 ++++++++++++++++++++++++++++++++++++++++++++
>>  s390x/unittests.cfg |   4 +
>>  3 files changed, 247 insertions(+)
>>  create mode 100644 s390x/smp.c
>>
>> diff --git a/s390x/Makefile b/s390x/Makefile
>> index d83dd0b..3744372 100644
>> --- a/s390x/Makefile
>> +++ b/s390x/Makefile
>> @@ -15,6 +15,7 @@ tests += $(TEST_DIR)/cpumodel.elf
>>  tests += $(TEST_DIR)/diag288.elf
>>  tests += $(TEST_DIR)/stsi.elf
>>  tests += $(TEST_DIR)/skrf.elf
>> +tests += $(TEST_DIR)/smp.elf
>>  tests_binary = $(patsubst %.elf,%.bin,$(tests))
>>  
>>  all: directories test_cases test_cases_binary
>> diff --git a/s390x/smp.c b/s390x/smp.c
>> new file mode 100644
>> index 0000000..7032494
>> --- /dev/null
>> +++ b/s390x/smp.c
>> @@ -0,0 +1,242 @@
>> +/*
>> + * Tests sigp emulation
>> + *
>> + * Copyright 2019 IBM Corp.
>> + *
>> + * Authors:
>> + *    Janosch Frank <frankja@linux.ibm.com>
>> + *
>> + * This code is free software; you can redistribute it and/or modify it
>> + * under the terms of the GNU General Public License version 2.
>> + */
>> +#include <libcflat.h>
>> +#include <asm/asm-offsets.h>
>> +#include <asm/interrupt.h>
>> +#include <asm/page.h>
>> +#include <asm/facility.h>
>> +#include <asm-generic/barrier.h>
>> +#include <asm/sigp.h>
>> +
>> +#include <smp.h>
>> +#include <alloc_page.h>
>> +
>> +static int testflag = 0;
>> +
>> +static void cpu_loop(void)
>> +{
>> +	for (;;) {}
> 
> Won't that be optimized out completely?

Why? AFAIK this is the standard way to write and endless loop ... how
can a compiler optimize that away?

 Thomas

Re: [kvm-unit-tests PATCH v3 6/6] s390x: SMP test

[David Hildenbrand]

On 25.09.19 15:30, Thomas Huth wrote:
> On 25/09/2019 15.27, David Hildenbrand wrote:
>> On 20.09.19 10:03, Janosch Frank wrote:
>>> Testing SIGP emulation for the following order codes:
>>> * start
>>> * stop
>>> * restart
>>> * set prefix
>>> * store status
>>> * stop and store status
>>> * reset
>>> * initial reset
>>> * external call
>>> * emegergency call
>>>
>>> restart and set prefix are part of the library and needed to start
>>> other cpus.
>>>
>>> Signed-off-by: Janosch Frank <frankja@linux.ibm.com>
>>> ---
>>>  s390x/Makefile      |   1 +
>>>  s390x/smp.c         | 242 ++++++++++++++++++++++++++++++++++++++++++++
>>>  s390x/unittests.cfg |   4 +
>>>  3 files changed, 247 insertions(+)
>>>  create mode 100644 s390x/smp.c
>>>
>>> diff --git a/s390x/Makefile b/s390x/Makefile
>>> index d83dd0b..3744372 100644
>>> --- a/s390x/Makefile
>>> +++ b/s390x/Makefile
>>> @@ -15,6 +15,7 @@ tests += $(TEST_DIR)/cpumodel.elf
>>>  tests += $(TEST_DIR)/diag288.elf
>>>  tests += $(TEST_DIR)/stsi.elf
>>>  tests += $(TEST_DIR)/skrf.elf
>>> +tests += $(TEST_DIR)/smp.elf
>>>  tests_binary = $(patsubst %.elf,%.bin,$(tests))
>>>  
>>>  all: directories test_cases test_cases_binary
>>> diff --git a/s390x/smp.c b/s390x/smp.c
>>> new file mode 100644
>>> index 0000000..7032494
>>> --- /dev/null
>>> +++ b/s390x/smp.c
>>> @@ -0,0 +1,242 @@
>>> +/*
>>> + * Tests sigp emulation
>>> + *
>>> + * Copyright 2019 IBM Corp.
>>> + *
>>> + * Authors:
>>> + *    Janosch Frank <frankja@linux.ibm.com>
>>> + *
>>> + * This code is free software; you can redistribute it and/or modify it
>>> + * under the terms of the GNU General Public License version 2.
>>> + */
>>> +#include <libcflat.h>
>>> +#include <asm/asm-offsets.h>
>>> +#include <asm/interrupt.h>
>>> +#include <asm/page.h>
>>> +#include <asm/facility.h>
>>> +#include <asm-generic/barrier.h>
>>> +#include <asm/sigp.h>
>>> +
>>> +#include <smp.h>
>>> +#include <alloc_page.h>
>>> +
>>> +static int testflag = 0;
>>> +
>>> +static void cpu_loop(void)
>>> +{
>>> +	for (;;) {}
>>
>> Won't that be optimized out completely?
> 
> Why? AFAIK this is the standard way to write and endless loop ... how
> can a compiler optimize that away?

Was messing it up with "just" an empty loop body, I think you're right.

-- 

Thanks,

David / dhildenb

Re: [kvm-unit-tests PATCH v3 6/6] s390x: SMP test

[David Hildenbrand]

On 25.09.19 15:32, David Hildenbrand wrote:
> On 25.09.19 15:30, Thomas Huth wrote:
>> On 25/09/2019 15.27, David Hildenbrand wrote:
>>> On 20.09.19 10:03, Janosch Frank wrote:
>>>> Testing SIGP emulation for the following order codes:
>>>> * start
>>>> * stop
>>>> * restart
>>>> * set prefix
>>>> * store status
>>>> * stop and store status
>>>> * reset
>>>> * initial reset
>>>> * external call
>>>> * emegergency call
>>>>
>>>> restart and set prefix are part of the library and needed to start
>>>> other cpus.
>>>>
>>>> Signed-off-by: Janosch Frank <frankja@linux.ibm.com>
>>>> ---
>>>>  s390x/Makefile      |   1 +
>>>>  s390x/smp.c         | 242 ++++++++++++++++++++++++++++++++++++++++++++
>>>>  s390x/unittests.cfg |   4 +
>>>>  3 files changed, 247 insertions(+)
>>>>  create mode 100644 s390x/smp.c
>>>>
>>>> diff --git a/s390x/Makefile b/s390x/Makefile
>>>> index d83dd0b..3744372 100644
>>>> --- a/s390x/Makefile
>>>> +++ b/s390x/Makefile
>>>> @@ -15,6 +15,7 @@ tests += $(TEST_DIR)/cpumodel.elf
>>>>  tests += $(TEST_DIR)/diag288.elf
>>>>  tests += $(TEST_DIR)/stsi.elf
>>>>  tests += $(TEST_DIR)/skrf.elf
>>>> +tests += $(TEST_DIR)/smp.elf
>>>>  tests_binary = $(patsubst %.elf,%.bin,$(tests))
>>>>  
>>>>  all: directories test_cases test_cases_binary
>>>> diff --git a/s390x/smp.c b/s390x/smp.c
>>>> new file mode 100644
>>>> index 0000000..7032494
>>>> --- /dev/null
>>>> +++ b/s390x/smp.c
>>>> @@ -0,0 +1,242 @@
>>>> +/*
>>>> + * Tests sigp emulation
>>>> + *
>>>> + * Copyright 2019 IBM Corp.
>>>> + *
>>>> + * Authors:
>>>> + *    Janosch Frank <frankja@linux.ibm.com>
>>>> + *
>>>> + * This code is free software; you can redistribute it and/or modify it
>>>> + * under the terms of the GNU General Public License version 2.
>>>> + */
>>>> +#include <libcflat.h>
>>>> +#include <asm/asm-offsets.h>
>>>> +#include <asm/interrupt.h>
>>>> +#include <asm/page.h>
>>>> +#include <asm/facility.h>
>>>> +#include <asm-generic/barrier.h>
>>>> +#include <asm/sigp.h>
>>>> +
>>>> +#include <smp.h>
>>>> +#include <alloc_page.h>
>>>> +
>>>> +static int testflag = 0;
>>>> +
>>>> +static void cpu_loop(void)
>>>> +{
>>>> +	for (;;) {}
>>>
>>> Won't that be optimized out completely?
>>
>> Why? AFAIK this is the standard way to write and endless loop ... how
>> can a compiler optimize that away?
> 
> Was messing it up with "just" an empty loop body, I think you're right.
> 

However

https://stackoverflow.com/questions/2178115/are-compilers-allowed-to-eliminate-infinite-loops

"This is intended to allow compiler transformations such as removal of
empty loops even when termination cannot be proven."

I think this might get optimized out.

-- 

Thanks,

David / dhildenb

Re: [kvm-unit-tests PATCH v3 6/6] s390x: SMP test

[David Hildenbrand]

On 25.09.19 15:35, David Hildenbrand wrote:
> On 25.09.19 15:32, David Hildenbrand wrote:
>> On 25.09.19 15:30, Thomas Huth wrote:
>>> On 25/09/2019 15.27, David Hildenbrand wrote:
>>>> On 20.09.19 10:03, Janosch Frank wrote:
>>>>> Testing SIGP emulation for the following order codes:
>>>>> * start
>>>>> * stop
>>>>> * restart
>>>>> * set prefix
>>>>> * store status
>>>>> * stop and store status
>>>>> * reset
>>>>> * initial reset
>>>>> * external call
>>>>> * emegergency call
>>>>>
>>>>> restart and set prefix are part of the library and needed to start
>>>>> other cpus.
>>>>>
>>>>> Signed-off-by: Janosch Frank <frankja@linux.ibm.com>
>>>>> ---
>>>>>  s390x/Makefile      |   1 +
>>>>>  s390x/smp.c         | 242 ++++++++++++++++++++++++++++++++++++++++++++
>>>>>  s390x/unittests.cfg |   4 +
>>>>>  3 files changed, 247 insertions(+)
>>>>>  create mode 100644 s390x/smp.c
>>>>>
>>>>> diff --git a/s390x/Makefile b/s390x/Makefile
>>>>> index d83dd0b..3744372 100644
>>>>> --- a/s390x/Makefile
>>>>> +++ b/s390x/Makefile
>>>>> @@ -15,6 +15,7 @@ tests += $(TEST_DIR)/cpumodel.elf
>>>>>  tests += $(TEST_DIR)/diag288.elf
>>>>>  tests += $(TEST_DIR)/stsi.elf
>>>>>  tests += $(TEST_DIR)/skrf.elf
>>>>> +tests += $(TEST_DIR)/smp.elf
>>>>>  tests_binary = $(patsubst %.elf,%.bin,$(tests))
>>>>>  
>>>>>  all: directories test_cases test_cases_binary
>>>>> diff --git a/s390x/smp.c b/s390x/smp.c
>>>>> new file mode 100644
>>>>> index 0000000..7032494
>>>>> --- /dev/null
>>>>> +++ b/s390x/smp.c
>>>>> @@ -0,0 +1,242 @@
>>>>> +/*
>>>>> + * Tests sigp emulation
>>>>> + *
>>>>> + * Copyright 2019 IBM Corp.
>>>>> + *
>>>>> + * Authors:
>>>>> + *    Janosch Frank <frankja@linux.ibm.com>
>>>>> + *
>>>>> + * This code is free software; you can redistribute it and/or modify it
>>>>> + * under the terms of the GNU General Public License version 2.
>>>>> + */
>>>>> +#include <libcflat.h>
>>>>> +#include <asm/asm-offsets.h>
>>>>> +#include <asm/interrupt.h>
>>>>> +#include <asm/page.h>
>>>>> +#include <asm/facility.h>
>>>>> +#include <asm-generic/barrier.h>
>>>>> +#include <asm/sigp.h>
>>>>> +
>>>>> +#include <smp.h>
>>>>> +#include <alloc_page.h>
>>>>> +
>>>>> +static int testflag = 0;
>>>>> +
>>>>> +static void cpu_loop(void)
>>>>> +{
>>>>> +	for (;;) {}
>>>>
>>>> Won't that be optimized out completely?
>>>
>>> Why? AFAIK this is the standard way to write and endless loop ... how
>>> can a compiler optimize that away?
>>
>> Was messing it up with "just" an empty loop body, I think you're right.
>>
> 
> However
> 
> https://stackoverflow.com/questions/2178115/are-compilers-allowed-to-eliminate-infinite-loops
> 
> "This is intended to allow compiler transformations such as removal of
> empty loops even when termination cannot be proven."
> 
> I think this might get optimized out.
> 

... but then, everybody does it, so it is most probably fine with GCC.

-- 

Thanks,

David / dhildenb

[kvm-unit-tests PATCH] s390x: Add linemode buffer to fix newline on every print

[Janosch Frank]

Linemode seems to add a newline for each sent message which makes
reading rather hard. Hence we add a small buffer and only print if
it's full or a newline is encountered. Except for when the string is
longer than the buffer, then we flush the buffer and print directly.

Signed-off-by: Janosch Frank <frankja@linux.ibm.com>
Reviewed-by: Thomas Huth <thuth@redhat.com>
---

How about something like this, splitting buffer handling and actual
printing, while adding buffer that's separate from the sclp locking. I
hope that all offsets and ptrs are correct despite the weekday and
hour...

---
 lib/s390x/sclp-console.c | 71 +++++++++++++++++++++++++++++++++++++++++++++---
 1 file changed, 67 insertions(+), 4 deletions(-)

diff --git a/lib/s390x/sclp-console.c b/lib/s390x/sclp-console.c
index 19416b5..4db09c4 100644
--- a/lib/s390x/sclp-console.c
+++ b/lib/s390x/sclp-console.c
@@ -13,6 +13,7 @@
 #include <asm/page.h>
 #include <asm/arch_def.h>
 #include <asm/io.h>
+#include <asm/spinlock.h>
 #include "sclp.h"
 
 /*
@@ -87,6 +88,11 @@ static uint8_t _ascebc[256] = {
      0x90, 0x3F, 0x3F, 0x3F, 0x3F, 0xEA, 0x3F, 0xFF
 };
 
+static char lm_buff[120];
+static unsigned char lm_buff_off;
+static struct spinlock lm_buff_lock;
+
+
 static void sclp_print_ascii(const char *str)
 {
 	int len = strlen(str);
@@ -103,10 +109,10 @@ static void sclp_print_ascii(const char *str)
 	sclp_service_call(SCLP_CMD_WRITE_EVENT_DATA, sccb);
 }
 
-static void sclp_print_lm(const char *str)
+static void lm_print(const char *buff, int len)
 {
 	unsigned char *ptr, *end, ch;
-	unsigned int count, offset, len;
+	unsigned int count, offset;
 	struct WriteEventData *sccb;
 	struct mdb *mdb;
 	struct mto *mto;
@@ -117,11 +123,10 @@ static void sclp_print_lm(const char *str)
 	end = (unsigned char *) sccb + 4096 - 1;
 	memset(sccb, 0, sizeof(*sccb));
 	ptr = (unsigned char *) &sccb->msg.mdb.mto;
-	len = strlen(str);
 	offset = 0;
 	do {
 		for (count = sizeof(*mto); offset < len; count++) {
-			ch = str[offset++];
+			ch = buff[offset++];
 			if (ch == 0x0a || ptr + count > end)
 				break;
 			ptr[count] = _ascebc[ch];
@@ -148,6 +153,64 @@ static void sclp_print_lm(const char *str)
 	sclp_service_call(SCLP_CMD_WRITE_EVENT_DATA, sccb);
 }
 
+
+/*
+ * In contrast to the ascii console, linemode produces a new
+ * line with every write of data. The report() function uses
+ * several printf() calls to generate a line of data which
+ * would all end up on different lines.
+ *
+ * Hence we buffer here until we encounter a \n or the buffer
+ * is full. That means that linemode output can look a bit
+ * different from ascii and that it takes a bit longer for
+ * lines to appear.
+ */
+static void sclp_print_lm(const char *str)
+{
+	int len;
+	char *nl;
+
+	spin_lock(&lm_buff_lock);
+
+	len = strlen(str);
+	/*
+	 * No use in copying into lm_buff, its time to flush the
+	 * buffer and print str until finished.
+	 */
+	if (len > sizeof(lm_buff)) {
+		if (lm_buff_off)
+			lm_print(lm_buff, lm_buff_off);
+		lm_print(str, len);
+		memset(lm_buff, 0 , sizeof(lm_buff));
+		lm_buff_off = 0;
+		goto out;
+	}
+
+fill:
+	len = len < (sizeof(lm_buff) - lm_buff_off) ? len : (sizeof(lm_buff) - lm_buff_off);
+	if ((lm_buff_off < sizeof(lm_buff) - 1)) {
+		memcpy(&lm_buff[lm_buff_off], str, len);
+		lm_buff_off += len;
+	}
+	/* Buffer not full and no newline */
+	nl = strchr(lm_buff, '\n');
+	if (lm_buff_off != sizeof(lm_buff) - 1 && !nl)
+		goto out;
+
+	lm_print(lm_buff, lm_buff_off);
+	memset(lm_buff, 0 , sizeof(lm_buff));
+	lm_buff_off = 0;
+
+	if (len < strlen(str)) {
+		str = &str[len];
+		len = strlen(str);
+		goto fill;
+	}
+
+out:
+	spin_unlock(&lm_buff_lock);
+}
+
 /*
  * SCLP needs to be initialized by setting a send and receive mask,
  * indicating which messages the control program (we) want(s) to
-- 
2.14.3