RE: [PATCH v13 00/22] Add vfio_device cdev for iommufd support

Duan, Zhenzhong
2023-06-21T09:17:17Z

>-----Original Message-----
>From: Liu, Yi L <yi.l.liu@intel.com>
>Sent: Friday, June 16, 2023 5:39 PM
>Subject: [PATCH v13 00/22] Add vfio_device cdev for iommufd support
>...

Per Yi's suggestion, wrote a selftest app to verify functions of this patchset by referencing https://github.com/awilliam/tests/

Test pass with this app, so Tested-by: Zhenzhong Duan <zhenzhong.duan@intel.com>
Test result:
# ./iommufd-pci-device-open 0 0000:81:11.0
Using PCI device 0000:81:11.0 vfio id: 0
Bind to IOMMUFD 4 with dev_id 1
Device supports 9 regions, 5 irqs
Region 0: size 0x20000, offset 0x0, flags 0x7
[▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒]
Region 1: size 0x0, offset 0x10000000000, flags 0x0
Region 2: size 0x0, offset 0x20000000000, flags 0x0
Region 3: size 0x4000, offset 0x30000000000, flags 0xf
[▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒]
Region 4: size 0x0, offset 0x40000000000, flags 0x0
Region 5: size 0x0, offset 0x50000000000, flags 0x0
Region 6: size 0x0, offset 0x60000000000, flags 0x0
Region 7: size 0x1000, offset 0x70000000000, flags 0x3
Region 8: Failed to get info
Attached IOMMUFD 4 ioas 2 hwpt 3
Mapped user_va 7f496ed00000 size 100000 to iova 0 in ioas 2
Hot reset dependent device count: 4
0: 0000:81:00.0 devid -1
1: 0000:81:00.1 devid -1
2: 0000:81:11.0 devid 1
3: 0000:81:11.1 devid -1
Cannot reset device, depends on device 0000:81:00.0 which is not owned.
Cannot reset device, depends on device 0000:81:00.1 which is not owned.
Cannot reset device, depends on device 0000:81:11.1 which is not owned.

Source code:
# cat iommufd-pci-device-open.c
#include <errno.h>
#include <libgen.h>
#include <fcntl.h>
#include <libgen.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <unistd.h>
#include <sys/ioctl.h>
#include <sys/mman.h>
#include <sys/param.h>
#include <sys/stat.h>
#include <sys/types.h>
#include <sys/vfs.h>

#include <linux/ioctl.h>
#include "iommufd.h"
#include "vfio.h"

void usage(char *name)
{
printf("usage: %s <vfio device id> <ssss:bb:dd.f>\n", name);
}

int main(int argc, char **argv)
{
int i, ret, container, group, device, vfioid, iommufd;
char path[PATH_MAX];
int seg, bus, dev, func;

struct vfio_device_info device_info = {
.argsz = sizeof(device_info)
};
struct vfio_region_info region_info = {
.argsz = sizeof(region_info)
};

if (argc < 3) {
usage(argv[0]);
return -1;
}

ret = sscanf(argv[1], "%d", &vfioid);
if (ret != 1) {
usage(argv[0]);
return -1;
}

ret = sscanf(argv[2], "%04x:%02x:%02x.%d", &seg, &bus, &dev, &func);
if (ret != 4) {
usage(argv[0]);
return -1;
}

printf("Using PCI device %04x:%02x:%02x.%d vfio id: %d\n",
seg, bus, dev, func, vfioid);

snprintf(path, sizeof(path), "/dev/vfio/devices/vfio%d", vfioid);
device = open(path, O_RDWR);
if (device < 0) {
printf("Failed to open %s, %d (%s)\n",
path, device, strerror(errno));
return device;
}

struct vfio_device_bind_iommufd bind = {
.argsz = sizeof(bind),
.flags = 0,
};
struct iommu_ioas_alloc alloc_data = {
.size = sizeof(alloc_data),
.flags = 0,
};
struct vfio_device_attach_iommufd_pt attach_data = {
.argsz = sizeof(attach_data),
.flags = 0,
};
struct iommu_ioas_map map = {
.size = sizeof(map),
.flags = IOMMU_IOAS_MAP_READABLE |
IOMMU_IOAS_MAP_WRITEABLE |
IOMMU_IOAS_MAP_FIXED_IOVA,
.__reserved = 0,
};

iommufd = open("/dev/iommu", O_RDWR);
if (iommufd < 0) {
printf("Failed to open /dev/iommufd, %d (%s)\n",
iommufd, strerror(errno));
return iommufd;
}

bind.iommufd = iommufd; // negative value means vfio-noiommu mode
ret = ioctl(device, VFIO_DEVICE_BIND_IOMMUFD, &bind);
if (ret < 0) {
printf("Failed VFIO_DEVICE_BIND_IOMMUFD %d (%s)\n",
ret, strerror(errno));
return ret;
}

printf("Bind to IOMMUFD %d with dev_id %d\n", iommufd, bind.out_devid);

if (ioctl(device, VFIO_DEVICE_GET_INFO, &device_info)) {
printf("Failed to get device info\n");
return -1;
}

printf("Device supports %d regions, %d irqs\n",
device_info.num_regions, device_info.num_irqs);

for (i = 0; i < device_info.num_regions; i++) {
printf("Region %d: ", i);
region_info.index = i;
if (ioctl(device, VFIO_DEVICE_GET_REGION_INFO, &region_info)) {
printf("Failed to get info\n");
continue;
}

printf("size 0x%lx, offset 0x%lx, flags 0x%x\n",
(unsigned long)region_info.size,
(unsigned long)region_info.offset, region_info.flags);
if (region_info.flags & VFIO_REGION_INFO_FLAG_MMAP) {
void *map = mmap(NULL, (size_t)region_info.size,
PROT_READ, MAP_SHARED, device,
(off_t)region_info.offset);
if (map == MAP_FAILED) {
printf("mmap failed\n");
continue;
}

printf("[");
fwrite(map, 1, region_info.size > 16 ? 16 :
region_info.size, stdout);
printf("]\n");
munmap(map, (size_t)region_info.size);
}
}

ret = ioctl(iommufd, IOMMU_IOAS_ALLOC, &alloc_data);
if (ret < 0) {
printf("Failed IOMMU_IOAS_ALLOC %d (%s)\n",
ret, strerror(errno));
return ret;
}

attach_data.pt_id = alloc_data.out_ioas_id;
ret = ioctl(device, VFIO_DEVICE_ATTACH_IOMMUFD_PT, &attach_data);
if (ret < 0) {
printf("Failed VFIO_DEVICE_ATTACH_IOMMUFD_PT ioas_id %d %d (%s)\n",
attach_data.pt_id, ret, strerror(errno));
return ret;
}

printf("Attached IOMMUFD %d ioas %d hwpt %d\n", iommufd, alloc_data.out_ioas_id, attach_data.pt_id);

/* Allocate some space and setup a DMA mapping */
map.user_va = (int64_t)mmap(0, 1024 * 1024, PROT_READ | PROT_WRITE,
MAP_PRIVATE | MAP_ANONYMOUS, 0, 0);
map.iova = 0; /* 1MB starting at 0x0 from device view */
map.length = 1024 * 1024;
map.ioas_id = alloc_data.out_ioas_id;;

ret = ioctl(iommufd, IOMMU_IOAS_MAP, &map);
if (ret < 0) {
printf("Failed VFIO_DEVICE_ATTACH_IOMMUFD_PT ioas_id %d %d (%s)\n",
attach_data.pt_id, ret, strerror(errno));
return ret;
}
printf("Mapped user_va %llx size %llx to iova %llx in ioas %d\n", map.user_va, map.length, map.iova, map.ioas_id);

struct vfio_pci_hot_reset_info *reset_info;
struct vfio_pci_dependent_device *devices;
struct vfio_pci_hot_reset *reset;
reset_info = malloc(sizeof(*reset_info));
if (!reset_info) {
printf("Failed to alloc info struct\n");
return -ENOMEM;
}

reset_info->argsz = sizeof(*reset_info);

ret = ioctl(device, VFIO_DEVICE_GET_PCI_HOT_RESET_INFO, reset_info);
if (ret && errno == ENODEV) {
printf("Device does not support hot reset\n");
return 0;
}
if (!ret || errno != ENOSPC) {
printf("Expected fail/-ENOSPC, got %d/%d\n", ret, -errno);
return -1;
}

printf("Hot reset dependent device count: %d\n", reset_info->count);

reset_info = realloc(reset_info, sizeof(*reset_info) +
(reset_info->count * sizeof(*devices)));
if (!reset_info) {
printf("Failed to re-alloc info struct\n");
return -ENOMEM;
}

reset_info->argsz = sizeof(*reset_info) +
(reset_info->count * sizeof(*devices));
ret = ioctl(device, VFIO_DEVICE_GET_PCI_HOT_RESET_INFO, reset_info);
if (ret) {
printf("Reset Info error\n");
return ret;
}

devices = &reset_info->devices[0];

for (i = 0; i < reset_info->count; i++)
printf("%d: %04x:%02x:%02x.%d devid %d\n", i,
devices[i].segment, devices[i].bus,
devices[i].devfn >> 3, devices[i].devfn & 7,
devices[i].devid);

if (!(reset_info->flags & VFIO_PCI_HOT_RESET_FLAG_DEV_ID)) {
printf("VFIO_PCI_HOT_RESET_FLAG_DEV_ID should be set for IOMMUFD\n");
return -1;
}

int unowned_cnt = 0;
if (!(reset_info->flags & VFIO_PCI_HOT_RESET_FLAG_DEV_ID_OWNED)) {
for (i = 0; i < reset_info->count; i++) {
if (devices[i].devid == VFIO_PCI_DEVID_NOT_OWNED) {
unowned_cnt++;
printf("Cannot reset device, "
"depends on device %04x:%02x:%02x.%x "
"which is not owned.\n",
devices[i].segment, devices[i].bus,
devices[i].devfn >> 3, devices[i].devfn & 7);
}
}
if (!unowned_cnt) {
printf("flags mismatch with data field, "
"VFIO_PCI_HOT_RESET_FLAG_DEV_ID_OWNED claimed but "
"no VFIO_PCI_DEVID_NOT_OWNED\n");
return -1;
}
return 0;
}

printf("Attempting reset: ");
fflush(stdout);

/* Use zero length array for hot reset with iommufd backend */
reset = malloc(sizeof(*reset));
reset->argsz = sizeof(*reset);

/* Bus reset! */
ret = ioctl(device, VFIO_DEVICE_PCI_HOT_RESET, reset);

ret = ioctl(device, VFIO_DEVICE_PCI_HOT_RESET, reset);
printf("Hot reset: %s\n", ret ? "Failed" : "Pass");

printf("Press any key to exit\n");
fgetc(stdin);

return 0;
}

Thanks
Zhenzhong