From mboxrd@z Thu Jan 1 00:00:00 1970 Received: from eggs.gnu.org ([2001:4830:134:3::10]:42926) by lists.gnu.org with esmtp (Exim 4.71) (envelope-from ) id 1ZCPkw-0004wz-ML for qemu-devel@nongnu.org; Tue, 07 Jul 2015 06:05:40 -0400 Received: from Debian-exim by eggs.gnu.org with spam-scanned (Exim 4.71) (envelope-from ) id 1ZCPks-0004p9-He for qemu-devel@nongnu.org; Tue, 07 Jul 2015 06:05:38 -0400 Received: from mail-pd0-f169.google.com ([209.85.192.169]:34579) by eggs.gnu.org with esmtp (Exim 4.71) (envelope-from ) id 1ZCPks-0004oX-1R for qemu-devel@nongnu.org; Tue, 07 Jul 2015 06:05:34 -0400 Received: by pdbep18 with SMTP id ep18so123248245pdb.1 for ; Tue, 07 Jul 2015 03:05:33 -0700 (PDT) References: <1436148670-6592-1-git-send-email-aik@ozlabs.ru> <1436148670-6592-14-git-send-email-aik@ozlabs.ru> <20150707092311.728e2cd7@thh440s> From: Alexey Kardashevskiy Message-ID: <559BA465.50009@ozlabs.ru> Date: Tue, 7 Jul 2015 20:05:25 +1000 MIME-Version: 1.0 In-Reply-To: <20150707092311.728e2cd7@thh440s> Content-Type: text/plain; charset=koi8-r; format=flowed Content-Transfer-Encoding: 7bit Subject: Re: [Qemu-devel] [PATCH qemu v10 13/14] vfio: spapr: Add SPAPR IOMMU v2 support (DMA memory preregistering) List-Id: List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , To: Thomas Huth Cc: Michael Roth , qemu-devel@nongnu.org, Gavin Shan , Alex Williamson , qemu-ppc@nongnu.org, David Gibson On 07/07/2015 05:23 PM, Thomas Huth wrote: > On Mon, 6 Jul 2015 12:11:09 +1000 > Alexey Kardashevskiy wrote: > >> This makes use of the new "memory registering" feature. The idea is >> to provide the userspace ability to notify the host kernel about pages >> which are going to be used for DMA. Having this information, the host >> kernel can pin them all once per user process, do locked pages >> accounting (once) and not spent time on doing that in real time with >> possible failures which cannot be handled nicely in some cases. >> >> This adds a guest RAM memory listener which notifies a VFIO container >> about memory which needs to be pinned/unpinned. VFIO MMIO regions >> (i.e. "skip dump" regions) are skipped. >> >> The feature is only enabled for SPAPR IOMMU v2. The host kernel changes >> are required. Since v2 does not need/support VFIO_IOMMU_ENABLE, this does >> not call it when v2 is detected and enabled. >> >> This does not change the guest visible interface. >> >> Signed-off-by: Alexey Kardashevskiy >> Reviewed-by: David Gibson >> --- >> Changes: >> v9: >> * since there is no more SPAPR-specific data in container::iommu_data, >> the memory preregistration fields are common and potentially can be used >> by other architectures >> >> v7: >> * in vfio_spapr_ram_listener_region_del(), do unref() after ioctl() >> * s'ramlistener'register_listener' >> >> v6: >> * fixed commit log (s/guest/userspace/), added note about no guest visible >> change >> * fixed error checking if ram registration failed >> * added alignment check for section->offset_within_region >> >> v5: >> * simplified the patch >> * added trace points >> * added round_up() for the size >> * SPAPR IOMMU v2 used >> --- >> hw/vfio/common.c | 109 ++++++++++++++++++++++++++++++++++++++---- >> include/hw/vfio/vfio-common.h | 3 ++ >> trace-events | 1 + >> 3 files changed, 104 insertions(+), 9 deletions(-) >> >> diff --git a/hw/vfio/common.c b/hw/vfio/common.c >> index 8eacfd7..0c7ba8c 100644 >> --- a/hw/vfio/common.c >> +++ b/hw/vfio/common.c >> @@ -488,6 +488,76 @@ static void vfio_listener_release(VFIOContainer *container) >> memory_listener_unregister(&container->iommu_data.type1.listener); >> } >> >> +static void vfio_ram_do_region(VFIOContainer *container, >> + MemoryRegionSection *section, unsigned long req) >> +{ >> + int ret; >> + struct vfio_iommu_spapr_register_memory reg = { .argsz = sizeof(reg) }; >> + >> + if (!memory_region_is_ram(section->mr) || >> + memory_region_is_skip_dump(section->mr)) { >> + return; >> + } >> + >> + if (unlikely((section->offset_within_region & (getpagesize() - 1)))) { >> + error_report("%s received unaligned region", __func__); >> + return; >> + } >> + >> + reg.vaddr = (__u64) memory_region_get_ram_ptr(section->mr) + > > We're in usespace here ... I think it would be better to use uint64_t > instead of the kernel-type __u64. We are calling a kernel here - @reg is a kernel-defined struct. > >> + section->offset_within_region; >> + reg.size = ROUND_UP(int128_get64(section->size), TARGET_PAGE_SIZE); >> + >> + ret = ioctl(container->fd, req, ®); >> + trace_vfio_ram_register(_IOC_NR(req) - VFIO_BASE, reg.vaddr, reg.size, >> + ret ? -errno : 0); >> + if (!ret) { >> + return; >> + } >> + >> + /* >> + * On the initfn path, store the first error in the container so we >> + * can gracefully fail. Runtime, there's not much we can do other >> + * than throw a hardware error. >> + */ >> + if (!container->iommu_data.ram_reg_initialized) { >> + if (!container->iommu_data.ram_reg_error) { >> + container->iommu_data.ram_reg_error = -errno; >> + } >> + } else { >> + hw_error("vfio: RAM registering failed, unable to continue"); >> + } >> +} >> + >> +static void vfio_ram_listener_region_add(MemoryListener *listener, >> + MemoryRegionSection *section) >> +{ >> + VFIOContainer *container = container_of(listener, VFIOContainer, >> + iommu_data.register_listener); >> + memory_region_ref(section->mr); >> + vfio_ram_do_region(container, section, VFIO_IOMMU_SPAPR_REGISTER_MEMORY); >> +} >> + >> +static void vfio_ram_listener_region_del(MemoryListener *listener, >> + MemoryRegionSection *section) >> +{ >> + VFIOContainer *container = container_of(listener, VFIOContainer, >> + iommu_data.register_listener); >> + vfio_ram_do_region(container, section, VFIO_IOMMU_SPAPR_UNREGISTER_MEMORY); >> + memory_region_unref(section->mr); >> +} >> + >> +static const MemoryListener vfio_ram_memory_listener = { >> + .region_add = vfio_ram_listener_region_add, >> + .region_del = vfio_ram_listener_region_del, >> +}; >> + >> +static void vfio_spapr_listener_release_v2(VFIOContainer *container) >> +{ >> + memory_listener_unregister(&container->iommu_data.register_listener); >> + vfio_listener_release(container); >> +} >> + >> int vfio_mmap_region(Object *obj, VFIORegion *region, >> MemoryRegion *mem, MemoryRegion *submem, >> void **map, size_t size, off_t offset, >> @@ -698,14 +768,18 @@ static int vfio_connect_container(VFIOGroup *group, AddressSpace *as) >> >> container->iommu_data.type1.initialized = true; >> >> - } else if (ioctl(fd, VFIO_CHECK_EXTENSION, VFIO_SPAPR_TCE_IOMMU)) { >> + } else if (ioctl(fd, VFIO_CHECK_EXTENSION, VFIO_SPAPR_TCE_IOMMU) || >> + ioctl(fd, VFIO_CHECK_EXTENSION, VFIO_SPAPR_TCE_v2_IOMMU)) { >> + bool v2 = !!ioctl(fd, VFIO_CHECK_EXTENSION, VFIO_SPAPR_TCE_v2_IOMMU); > > That "!!" sounds somewhat wrong here. I think you either want to check > for "ioctl() == 1" (because only in this case you can be sure that v2 > is supported), or you can simply omit the "!!" because you're 100% sure > that the ioctl only returns 0 or 1 (and never a negative error code). The host kernel does not return an error on these ioctls, it returns 0 or 1. And "!!" is shorter than "(bool)". VFIO_CHECK_EXTENSION for Type1 does exactly the same already. >> ret = ioctl(group->fd, VFIO_GROUP_SET_CONTAINER, &fd); >> if (ret) { >> error_report("vfio: failed to set group container: %m"); >> ret = -errno; >> goto free_container_exit; >> } >> - ret = ioctl(fd, VFIO_SET_IOMMU, VFIO_SPAPR_TCE_IOMMU); >> + ret = ioctl(fd, VFIO_SET_IOMMU, >> + v2 ? VFIO_SPAPR_TCE_v2_IOMMU : VFIO_SPAPR_TCE_IOMMU); >> if (ret) { >> error_report("vfio: failed to set iommu for container: %m"); >> ret = -errno; >> @@ -717,19 +791,36 @@ static int vfio_connect_container(VFIOGroup *group, AddressSpace *as) >> * when container fd is closed so we do not call it explicitly >> * in this file. >> */ >> - ret = ioctl(fd, VFIO_IOMMU_ENABLE); >> - if (ret) { >> - error_report("vfio: failed to enable container: %m"); >> - ret = -errno; >> - goto free_container_exit; >> + if (!v2) { >> + ret = ioctl(fd, VFIO_IOMMU_ENABLE); >> + if (ret) { >> + error_report("vfio: failed to enable container: %m"); >> + ret = -errno; >> + goto free_container_exit; >> + } >> } >> >> container->iommu_data.type1.listener = vfio_memory_listener; >> - container->iommu_data.release = vfio_listener_release; >> - >> memory_listener_register(&container->iommu_data.type1.listener, >> container->space->as); >> >> + if (!v2) { >> + container->iommu_data.release = vfio_listener_release; >> + } else { >> + container->iommu_data.release = vfio_spapr_listener_release_v2; >> + container->iommu_data.register_listener = >> + vfio_ram_memory_listener; >> + memory_listener_register(&container->iommu_data.register_listener, >> + &address_space_memory); >> + >> + if (container->iommu_data.ram_reg_error) { >> + error_report("vfio: RAM memory listener initialization failed for container"); > > Line > 80 columns? afaik user visible strings are an exception in QEMU and kernel. > >> + goto listener_release_exit; >> + } >> + >> + container->iommu_data.ram_reg_initialized = true; >> + } >> + > > Thomas > -- Alexey