Re: [PATCH 1/2] kvm-unit-tests: Add a func to run instruction in emulator

[Gleb Natapov <gleb@redhat.com>]

[-- Warning: decoded text below may be mangled, UTF-8 assumed --]
[-- Attachment #1: Type: text/plain; charset=utf-8, Size: 8765 bytes --]

On Tue, Jun 18, 2013 at 10:28:59PM +0800, æ??æ?¥å¥? <Arthur Chunqi Li> wrote:
> On Tue, Jun 18, 2013 at 8:45 PM, Gleb Natapov <gleb@redhat.com> wrote:
> > On Thu, Jun 13, 2013 at 05:30:03PM +0800, 李春奇 <Arthur Chunqi Li> wrote:
> >> Hi Gleb,
> >> I'm trying to solve these problems in the past days and meet many
> >> difficulties. You want to save all the general registers in calling
> >> insn_page, so registers should be saved to (save) in insn_page.
> >> Because all the instructions should be generated outside and copy to
> >> insn_page, and the instructions generated outside is RIP-relative, so
> >> inside insn_page (save) will be wrong pointed with RIP-relative code.
> >>
> > They do not have to be generated outside. You can write code into
> > insn_page directly. Something like this outside of any functions:
> >
> > asm(".align 4096\n\t"
> >     ".global insn_page\n\t"
> >     ".global insn_page_end\n\t"
> >     ".global test_insn\n\t"
> >     ".global test_insn_end\n\t"
> >     "insn_page:"
> >     "mov %%rax, outregs \n\t"
> >     ...
> >     "test_insn:\n\t"
> >     "in (%ds), %al\n\t"
> >     ". = . + 31\n\t"
> >     "test_insn_end:\n\t"
> >     "mov outregs, %%rax\n\t"
> >     ...
> >     "ret\n\t"
> >     ".align 4096\n\t"
> >     "insn_page_end:\n\t");
> >
> > Now you copy that into alt_insn_page, put instruction you want to test
> > into test_insn offset and remap alt_insn_page into "insn_page" virtual address.
> I used such codes:
> 
> invlpg((void *)virt_to_phys(insn_page));
virt_to_phys?

> asm volatile("call *%0" : : "r"(insn_page));
> install_page(cr3, virt_to_phys(alt_insn_page), insn_page);
> asm volatile("call *%0": : "r"(insn_page+1));
+1?

> 
> But it seems that alt_insn_page are not remapped to insn_page. Here
> insn_page and alt_insn_page are all declared statically with
> "asm(...)".
> 
> Arthur
> >
> >> I have tried to move (save) into insn_page. But when calling
> >> insn_page, data in it can only be read and any instructions like "xchg
> >> %%rax, 0+%[save]" may cause error, because at this time read is from
> >> TLB but write will cause inconsistent.
> >>
> >> Another way is disabling RIP-relative code, but I failed when using
> >> "-mcmodel-large -fno-pic", the binary is also using RIP-relative mode.
> >> Is there any way to totally disable RIP-relative code? Besides, using
> >> this feature may specified to some newer C compiler. This may not be a
> >> good solution.
> >>
> >> If we don't set %rsp and %rbp when executing emulator code, we can
> >> just use “push/pop" to save other general registers.
> >>
> >> If you have any better solutions, please let me know.
> >>
> >> Thanks,
> >> Arthur
> >>
> >> On Thu, Jun 13, 2013 at 12:50 PM, 李春奇 <Arthur Chunqi Li>
> >> <yzt356@gmail.com> wrote:
> >> > On Thu, Jun 13, 2013 at 4:50 AM, Paolo Bonzini <pbonzini@redhat.com> wrote:
> >> >> Il 06/06/2013 11:24, Arthur Chunqi Li ha scritto:
> >> >>> Add a function trap_emulator to run an instruction in emulator.
> >> >>> Set inregs first (%rax is invalid because it is used as return
> >> >>> address), put instruction codec in alt_insn and call func with
> >> >>> alt_insn_length. Get results in outregs.
> >> >>>
> >> >>> Signed-off-by: Arthur Chunqi Li <yzt356@gmail.com>
> >> >>> ---
> >> >>>  x86/emulator.c |   81 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++
> >> >>>  1 file changed, 81 insertions(+)
> >> >>>
> >> >>> diff --git a/x86/emulator.c b/x86/emulator.c
> >> >>> index 96576e5..8ab9904 100644
> >> >>> --- a/x86/emulator.c
> >> >>> +++ b/x86/emulator.c
> >> >>> @@ -11,6 +11,14 @@ int fails, tests;
> >> >>>
> >> >>>  static int exceptions;
> >> >>>
> >> >>> +struct regs {
> >> >>> +     u64 rax, rbx, rcx, rdx;
> >> >>> +     u64 rsi, rdi, rsp, rbp;
> >> >>> +     u64 rip, rflags;
> >> >>> +};
> >> >>> +
> >> >>> +static struct regs inregs, outregs;
> >> >>> +
> >> >>>  void report(const char *name, int result)
> >> >>>  {
> >> >>>       ++tests;
> >> >>> @@ -685,6 +693,79 @@ static void test_shld_shrd(u32 *mem)
> >> >>>      report("shrd (cl)", *mem == ((0x12345678 >> 3) | (5u << 29)));
> >> >>>  }
> >> >>>
> >> >>> +static void trap_emulator(uint64_t *mem, uint8_t *insn_page,
> >> >>> +                          uint8_t *alt_insn_page, void *insn_ram,
> >> >>> +                          uint8_t *alt_insn, int alt_insn_length)
> >> >>> +{
> >> >>> +     ulong *cr3 = (ulong *)read_cr3();
> >> >>> +     int i;
> >> >>> +
> >> >>> +     // Pad with RET instructions
> >> >>> +     memset(insn_page, 0xc3, 4096);
> >> >>> +     memset(alt_insn_page, 0xc3, 4096);
> >> >>> +
> >> >>> +     // Place a trapping instruction in the page to trigger a VMEXIT
> >> >>> +     insn_page[0] = 0x89; // mov %eax, (%rax)
> >> >>> +     insn_page[1] = 0x00;
> >> >>> +     insn_page[2] = 0x90; // nop
> >> >>> +     insn_page[3] = 0xc3; // ret
> >> >>> +
> >> >>> +     // Place the instruction we want the hypervisor to see in the alternate page
> >> >>> +     for (i=0; i<alt_insn_length; i++)
> >> >>> +             alt_insn_page[i] = alt_insn[i];
> >> >>> +
> >> >>> +     // Save general registers
> >> >>> +     asm volatile(
> >> >>> +             "push %rax\n\r"
> >> >>> +             "push %rbx\n\r"
> >> >>> +             "push %rcx\n\r"
> >> >>> +             "push %rdx\n\r"
> >> >>> +             "push %rsi\n\r"
> >> >>> +             "push %rdi\n\r"
> >> >>> +             );
> >> >>
> >> >> This will not work if GCC is using rsp-relative addresses to access
> >> >> local variables.  You need to use mov instructions to load from inregs,
> >> >> and put the push/pop sequences inside the "main" asm that does the "call
> >> >> *%1".
> >> > Is there any way to let gcc use absolute address to access variables?
> >> > I move variant "save" to the global and use "xchg %%rax, 0+%[save]"
> >> > and it seems that addressing for "save" is wrong.
> >> >
> >> > Arthur
> >> >>
> >> >> Paolo
> >> >>
> >> >>> +     // Load the code TLB with insn_page, but point the page tables at
> >> >>> +     // alt_insn_page (and keep the data TLB clear, for AMD decode assist).
> >> >>> +     // This will make the CPU trap on the insn_page instruction but the
> >> >>> +     // hypervisor will see alt_insn_page.
> >> >>> +     install_page(cr3, virt_to_phys(insn_page), insn_ram);
> >> >>> +     invlpg(insn_ram);
> >> >>> +     // Load code TLB
> >> >>> +     asm volatile("call *%0" : : "r"(insn_ram + 3));
> >> >>> +     install_page(cr3, virt_to_phys(alt_insn_page), insn_ram);
> >> >>> +     // Trap, let hypervisor emulate at alt_insn_page
> >> >>> +     asm volatile(
> >> >>> +             "call *%1\n\r"
> >> >>> +
> >> >>> +             "mov %%rax, 0+%[outregs] \n\t"
> >> >>> +             "mov %%rbx, 8+%[outregs] \n\t"
> >> >>> +             "mov %%rcx, 16+%[outregs] \n\t"
> >> >>> +             "mov %%rdx, 24+%[outregs] \n\t"
> >> >>> +             "mov %%rsi, 32+%[outregs] \n\t"
> >> >>> +             "mov %%rdi, 40+%[outregs] \n\t"
> >> >>> +             "mov %%rsp,48+ %[outregs] \n\t"
> >> >>> +             "mov %%rbp, 56+%[outregs] \n\t"
> >> >>> +
> >> >>> +             /* Save RFLAGS in outregs*/
> >> >>> +             "pushf \n\t"
> >> >>> +             "popq 72+%[outregs] \n\t"
> >> >>> +             : [outregs]"+m"(outregs)
> >> >>> +             : "r"(insn_ram),
> >> >>> +                     "a"(mem), "b"(inregs.rbx),
> >> >>> +                     "c"(inregs.rcx), "d"(inregs.rdx),
> >> >>> +                     "S"(inregs.rsi), "D"(inregs.rdi)
> >> >>> +             : "memory", "cc"
> >> >>> +             );
> >> >>> +     // Restore general registers
> >> >>> +     asm volatile(
> >> >>> +             "pop %rax\n\r"
> >> >>> +             "pop %rbx\n\r"
> >> >>> +             "pop %rcx\n\r"
> >> >>> +             "pop %rdx\n\r"
> >> >>> +             "pop %rsi\n\r"
> >> >>> +             "pop %rdi\n\r"
> >> >>> +             );
> >> >>> +}
> >> >>> +
> >> >>>  static void advance_rip_by_3_and_note_exception(struct ex_regs *regs)
> >> >>>  {
> >> >>>      ++exceptions;
> >> >>>
> >> >>
> >> >
> >> >
> >> >
> >> > --
> >> > Arthur Chunqi Li
> >> > Department of Computer Science
> >> > School of EECS
> >> > Peking University
> >> > Beijing, China
> >>
> >>
> >>
> >> --
> >> Arthur Chunqi Li
> >> Department of Computer Science
> >> School of EECS
> >> Peking University
> >> Beijing, China
> >
> > --
> >                         Gleb.
> 
> 
> 
> --
> Arthur Chunqi Li
> Department of Computer Science
> School of EECS
> Peking University
> Beijing, China

--
			Gleb.