[PATCH 20/30] x86emul: support APX+AVX512 insns

[Jan Beulich <jbeulich@suse.com>]

It's KMOV{B,W,D,Q} which gain straight EVEX-encoded counterparts. Like
the VEX forms require special treatment, so do the EVEX ones. Respective
code is being cloned, with mode_64bit() conditionals dropped.

While adjusting the testharness'es Makefile, drop -Os from the non-APX
opmask test as well - for an assembly source that option is meaningless.

Signed-off-by: Jan Beulich <jbeulich@suse.com>
---
The spec says EVEX.B4 is ignored for opcode 0x93. While that matches
the fact that EVEX.B3 is also ignored, other similar statments in spec
version 005 (especially as to EVEX.R4 vs EVEX.R3) aren't consistent.
Hence this may need adjustment.

The checking of the 4th EVEX byte is going to re-appear for a number of
further insns. It may therefore want folding into a wrapper macro. Just
that I don't seem to be able to think of a good name for such a macro.

Is simply forcing EVEX encodings (without also involving eGPR-s) enough
of a testing approach?

--- a/tools/tests/x86_emulator/Makefile
+++ b/tools/tests/x86_emulator/Makefile
@@ -172,7 +172,8 @@ $(1)-cflags := $(foreach vec,$($(1:-gf=)
 	         "-D_$(vec) -mgfni -m$(1:-gf=) $(call non-sse,$(1)) -Os -DVEC_SIZE=$(vec)")
 endef
 define opmask-defs
-$(1)-opmask-cflags := $(foreach vec,$($(1)-opmask-vecs), "-D_$(vec) -m$(1) -Os -DSIZE=$(vec)")
+$(1)-opmask-cflags := $(foreach vec,$($(1)-opmask-vecs), "-D_$(vec) -m$(1) -DSIZE=$(vec)")
+$(1)-opmask-cflags-x86_64 := $(foreach vec,$($(1)-opmask-vecs), "-D_$(vec) -mapxf -m$(1) -DSIZE=$(vec)")
 endef
 
 $(foreach flavor,$(SIMD) $(FMA),$(eval $(call simd-defs,$(flavor))))
@@ -214,6 +215,9 @@ $(addsuffix -opmask.h,$(OPMASK)): %.h: o
 	    for cflags in $($*-cflags) $($*-cflags-$(arch)); do \
 		$(MAKE) -f testcase.mk TESTCASE=$* XEN_TARGET_ARCH=$(arch) $*-cflags="$$cflags" all; \
 		prefix=$(shell echo $(subst -,_,$*) | sed -e 's,^\([0-9]\),_\1,'); \
+		if [ "$$cflags" != "$${cflags%apx*}" ]; then \
+			prefix=$${prefix%opmask}apx; \
+		fi; \
 		flavor=$$(echo $${cflags} | sed -e 's, .*,,' -e 'y,-=,__,') ; \
 		(echo 'static const unsigned int __attribute__((section(".test, \"ax\", @progbits #")))' \
 		      "$${prefix}_$(arch)$${flavor}[] = {"; \
--- a/tools/tests/x86_emulator/evex-disp8.c
+++ b/tools/tests/x86_emulator/evex-disp8.c
@@ -1103,6 +1103,29 @@ static const struct test adx[] = {
     INSN(adox,      f3, map4, 66,    el,   dq, no),
 };
 
+static const struct test avx512bw[] = {
+//       kmov,      f2,   0f, 92,          dq
+//       kmov,      f2,   0f, 93,          dq
+    INSN(kmovd,     66,   0f, 90,    el,    d, no),
+    INSN(kmovd,     66,   0f, 91,    el,    d, no),
+    INSN(kmovq,       ,   0f, 90,    el,    q, no),
+    INSN(kmovq,       ,   0f, 91,    el,    q, no),
+};
+
+static const struct test avx512dq[] = {
+    INSN(kmovb,     66,   0f, 90,    el,    b, no),
+    INSN(kmovb,     66,   0f, 91,    el,    b, no),
+//       kmovb,     66,   0f, 92,           b
+//       kmovb,     66,   0f, 93,           b
+};
+
+static const struct test avx512f[] = {
+    INSN(kmovw,       ,   0f, 90,    el,    w, no),
+    INSN(kmovw,       ,   0f, 91,    el,    w, no),
+//       kmovw,       ,   0f, 92,           w
+//       kmovw,       ,   0f, 93,           w
+};
+
 static const unsigned char vl_all[] = { VL_512, VL_128, VL_256 };
 static const unsigned char vl_128[] = { VL_128 };
 static const unsigned char vl_no128[] = { VL_512, VL_256 };
@@ -1261,7 +1284,17 @@ static void test_one(const struct test *
         evex.w = 1;
         /* fall through */
 #endif
-    case ESZ_d: case ESZ_d_nb:
+    case ESZ_d:
+#ifdef __x86_64__
+        /*
+         * KMOVD don't follow the general pattern: They have EVEX.W set despite
+         * having a DWORD operand.
+         */
+        if ( test->spc == SPC_0f && (test->opc | 1) == 0x91 )
+            evex.w = 1;
+        /* fall through */
+#endif
+    case ESZ_d_nb:
         esz = 4;
         break;
 
@@ -1678,6 +1711,9 @@ void evex_disp8_test(void *instr, struct
     } while ( false )
 
         RUN(adx);
+        RUN(avx512f);
+        RUN(avx512dq);
+        RUN(avx512bw);
 
 #undef RUN
 
--- a/tools/tests/x86_emulator/opmask.S
+++ b/tools/tests/x86_emulator/opmask.S
@@ -8,6 +8,12 @@
 #  define R(x) e##x
 # endif
 # define DATA(x) x(%rip)
+# ifdef __APX_F__
+#  define kmovb {evex} kmovb
+#  define kmovd {evex} kmovd
+#  define kmovq {evex} kmovq
+#  define kmovw {evex} kmovw
+# endif
 #endif
 
 #if SIZE == 1
--- a/tools/tests/x86_emulator/predicates.c
+++ b/tools/tests/x86_emulator/predicates.c
@@ -1787,6 +1787,18 @@ static const struct evex {
     { { 0x7f }, 2, T, W, pfx_66, Wn, Ln }, /* vmovdqa{32,64} */
     { { 0x7f }, 2, T, W, pfx_f3, Wn, Ln }, /* vmovdqu{32,64} */
     { { 0x7f }, 2, T, W, pfx_f2, Wn, Ln }, /* vmovdqu{8,16} */
+#ifdef __x86_64__
+    { { 0x90 }, 2, T, R, pfx_no, Wn, L0 }, /* kmov{w,q} */
+    { { 0x90 }, 2, T, R, pfx_66, Wn, L0 }, /* kmov{b,d} */
+    { { 0x91 }, 2, N, W, pfx_no, Wn, L0 }, /* kmov{w,q} */
+    { { 0x91 }, 2, N, W, pfx_66, Wn, L0 }, /* kmov{b,d} */
+    { { 0x92, 0xc0 }, 2, F, N, pfx_no, W0, L0 }, /* kmovw */
+    { { 0x92, 0xc0 }, 2, F, N, pfx_66, W0, L0 }, /* kmovb */
+    { { 0x92, 0xc0 }, 2, F, N, pfx_f2, Wn, L0 }, /* kmov{d,q} */
+    { { 0x93, 0xc0 }, 2, F, N, pfx_no, W0, L0 }, /* kmovw */
+    { { 0x93, 0xc0 }, 2, F, N, pfx_66, W0, L0 }, /* kmovb */
+    { { 0x93, 0xc0 }, 2, F, N, pfx_f2, Wn, L0 }, /* kmov{d,q} */
+#endif
     { { 0xc2 }, 3, T, R, pfx_no, W0, Ln }, /* vcmpps */
     { { 0xc2 }, 3, T, R, pfx_66, W1, Ln }, /* vcmppd */
     { { 0xc2 }, 3, T, R, pfx_f3, W0, LIG }, /* vcmpss */
--- a/tools/tests/x86_emulator/test_x86_emulator.c
+++ b/tools/tests/x86_emulator/test_x86_emulator.c
@@ -348,6 +348,23 @@ static bool simd_check_avx512fp16_vl(voi
            cpu_has_avx10;
 }
 
+#ifdef __x86_64__
+static bool simd_check_avx512f_apx(void)
+{
+    return simd_check_avx512f_opmask() && check_apx_f();
+}
+
+static bool simd_check_avx512bw_apx(void)
+{
+    return simd_check_avx512bw_opmask() && check_apx_f();
+}
+
+static bool simd_check_avx512dq_apx(void)
+{
+    return simd_check_avx512dq_opmask() && check_apx_f();
+}
+#endif /* __x86_64__ */
+
 static void simd_set_regs(struct cpu_user_regs *regs)
 {
     if ( cpu_has_mmx )
@@ -532,6 +549,13 @@ static const struct {
     SIMD(OPMASK+DQ/w, avx512dq_opmask,         2),
     SIMD(OPMASK+BW/d, avx512bw_opmask,         4),
     SIMD(OPMASK+BW/q, avx512bw_opmask,         8),
+#ifdef __x86_64__
+    SIMD_(64, OPMASK+APX/w,     avx512f_apx,   2),
+    SIMD_(64, OPMASK+DQ+APX/b, avx512dq_apx,   1),
+    SIMD_(64, OPMASK+DQ+APX/w, avx512dq_apx,   2),
+    SIMD_(64, OPMASK+BW+APX/d, avx512bw_apx,   4),
+    SIMD_(64, OPMASK+BW+APX/q, avx512bw_apx,   8),
+#endif
     SIMD(AVX512F f32 scalar,  avx512f,        f4),
     SIMD(AVX512F f32x16,      avx512f,      64f4),
     SIMD(AVX512F f64 scalar,  avx512f,        f8),
--- a/xen/arch/x86/x86_emulate/decode.c
+++ b/xen/arch/x86/x86_emulate/decode.c
@@ -1100,18 +1100,35 @@ decode_twobyte(struct x86_emulate_state
         }
         break;
 
+    case X86EMUL_OPC_EVEX(0, 0x90):    /* kmov{w,q} */
+    case X86EMUL_OPC_EVEX_66(0, 0x90): /* kmov{b,d} */
+        s->promoted_evex = true;
+        fallthrough;
     case X86EMUL_OPC_VEX(0, 0x90):    /* kmov{w,q} */
     case X86EMUL_OPC_VEX_66(0, 0x90): /* kmov{b,d} */
         s->desc = DstReg | SrcMem | Mov;
         s->simd_size = simd_other;
         break;
 
+    case X86EMUL_OPC_EVEX(0, 0x91):    /* kmov{w,q} */
+    case X86EMUL_OPC_EVEX_66(0, 0x91): /* kmov{b,d} */
+        s->promoted_evex = true;
+        fallthrough;
     case X86EMUL_OPC_VEX(0, 0x91):    /* kmov{w,q} */
     case X86EMUL_OPC_VEX_66(0, 0x91): /* kmov{b,d} */
         s->desc = DstMem | SrcReg | Mov;
         s->simd_size = simd_other;
         break;
 
+    case X86EMUL_OPC_EVEX(0, 0x92):    /* kmovw */
+    case X86EMUL_OPC_EVEX_66(0, 0x92): /* kmovb */
+    case X86EMUL_OPC_EVEX_F2(0, 0x92): /* kmov{d,q} */
+    case X86EMUL_OPC_EVEX(0, 0x93):    /* kmovw */
+    case X86EMUL_OPC_EVEX_66(0, 0x93): /* kmovb */
+    case X86EMUL_OPC_EVEX_F2(0, 0x93): /* kmov{d,q} */
+        s->promoted_evex = true;
+        break;
+
     case 0xae:
         ctxt->opcode |= MASK_INSR(s->vex.pfx, X86EMUL_OPC_PFX_MASK);
         /* fall through */
--- a/xen/arch/x86/x86_emulate/x86_emulate.c
+++ b/xen/arch/x86/x86_emulate/x86_emulate.c
@@ -5316,6 +5316,16 @@ x86_emulate(
 
 #ifndef X86EMUL_NO_SIMD
 
+    case X86EMUL_OPC_EVEX(0x0f, 0x91):    /* kmov{w,q} k,mem */
+    case X86EMUL_OPC_EVEX_66(0x0f, 0x91): /* kmov{b,d} k,mem */
+        generate_exception_if(ea.type != OP_MEM, X86_EXC_UD);
+        fallthrough;
+    case X86EMUL_OPC_EVEX(0x0f, 0x90):    /* kmov{w,q} k/mem,k */
+    case X86EMUL_OPC_EVEX_66(0x0f, 0x90): /* kmov{b,d} k/mem,k */
+        generate_exception_if((evex.lr || evex.brs || evex.opmsk ||
+                               !evex.r3 || !evex.r4),
+                              X86_EXC_UD);
+        goto kmov;
     case X86EMUL_OPC_VEX(0x0f, 0x91):    /* kmov{w,q} k,mem */
     case X86EMUL_OPC_VEX_66(0x0f, 0x91): /* kmov{b,d} k,mem */
         generate_exception_if(ea.type != OP_MEM, X86_EXC_UD);
@@ -5323,6 +5333,7 @@ x86_emulate(
     case X86EMUL_OPC_VEX(0x0f, 0x90):    /* kmov{w,q} k/mem,k */
     case X86EMUL_OPC_VEX_66(0x0f, 0x90): /* kmov{b,d} k/mem,k */
         generate_exception_if(vex.l || !vex.r, X86_EXC_UD);
+    kmov:
         if ( vex.w )
         {
             visa_check(bw);
@@ -5353,6 +5364,13 @@ x86_emulate(
         insn_bytes = PFX_BYTES + 2;
         break;
 
+    case X86EMUL_OPC_EVEX(0x0f, 0x92):    /* kmovw r32,k */
+    case X86EMUL_OPC_EVEX_66(0x0f, 0x92): /* kmovb r32,k */
+    case X86EMUL_OPC_EVEX_F2(0x0f, 0x92): /* kmov{d,q} reg,k */
+        generate_exception_if((evex.lr || evex.brs || evex.opmsk ||
+                               !evex.r4 || !evex.RX),
+                              X86_EXC_UD);
+        fallthrough;
     case X86EMUL_OPC_VEX(0x0f, 0x92):    /* kmovw r32,k */
     case X86EMUL_OPC_VEX_66(0x0f, 0x92): /* kmovb r32,k */
     case X86EMUL_OPC_VEX_F2(0x0f, 0x92): /* kmov{d,q} reg,k */
@@ -5381,8 +5399,8 @@ x86_emulate(
         opc[1] = modrm & 0xf8;
         place_ret(&opc[2]);
 
-        copy_VEX(opc, vex, false);
-        ea.reg = decode_gpr(&_regs, NULL, modrm_rm);
+        copy_VEX(opc, vex, state->promoted_evex);
+        ea.reg = decode_gpr(&_regs, &state->egpr, modrm_rm);
         invoke_stub("", "", "=m" (dummy) : "a" (*ea.reg));
 
         put_stub(stub);
@@ -5391,13 +5409,19 @@ x86_emulate(
         dst.type = OP_NONE;
         break;
 
+    case X86EMUL_OPC_EVEX(0x0f, 0x93):    /* kmovw k,r32 */
+    case X86EMUL_OPC_EVEX_66(0x0f, 0x93): /* kmovb k,r32 */
+    case X86EMUL_OPC_EVEX_F2(0x0f, 0x93): /* kmov{d,q} k,reg */
+        generate_exception_if((evex.lr || evex.brs || evex.opmsk || !evex.RX),
+                              X86_EXC_UD);
+        fallthrough;
     case X86EMUL_OPC_VEX(0x0f, 0x93):    /* kmovw k,r32 */
     case X86EMUL_OPC_VEX_66(0x0f, 0x93): /* kmovb k,r32 */
     case X86EMUL_OPC_VEX_F2(0x0f, 0x93): /* kmov{d,q} k,reg */
         generate_exception_if(vex.l || vex.reg != 0xf || ea.type != OP_REG,
                               X86_EXC_UD);
         dst = ea;
-        dst.reg = decode_gpr(&_regs, NULL, modrm_reg);
+        dst.reg = decode_gpr(&_regs, &state->egpr, modrm_reg);
 
         if ( vex.pfx == vex_f2 )
         {
@@ -5425,7 +5449,7 @@ x86_emulate(
         opc[1] = modrm & 0xc7;
         place_ret(&opc[2]);
 
-        copy_VEX(opc, vex, false);
+        copy_VEX(opc, vex, state->promoted_evex);
         invoke_stub("", "", "=a" (dst.val) : [dummy] "i" (0));
 
         put_stub(stub);
@@ -8950,7 +8974,12 @@ x86_emulate(
         EXPECT(op_bytes);
         EXPECT(opc);
 
-        if ( evex_encoded() )
+        if ( state->promoted_evex )
+        {
+            place_ret(&opc[insn_bytes - PFX_BYTES]);
+            copy_VEX(opc, vex, true);
+        }
+        else if ( evex_encoded() )
         {
             place_ret(&opc[insn_bytes - EVEX_PFX_BYTES]);
             copy_EVEX(opc, evex);