Re: Need to support mapping in bar1

[Patrick Mahan <mahan-5dHXHCkEAVbYtjvyW6yDsg@public.gmane.org>]

Stephen,

When did you generate this diff?  It failed to apply to 1.3.1r2 so I am hand
patching instead.  But you might want to update to the latest version.

Thanks,

Patrick

On 8/8/13 5:50 PM, Stephen Hemminger wrote:
> Still needs a little cleanup (patch is messy).
> 
> Subject: pci: support multiple PCI regions per device
>     
> Need to change PCI code to support multiple I/O regions on a single device.
> Some devices like VMXNET3 have multiple PCI memory regions, and some
> have none.
> 
> Signed-off-by: Stephen Hemminger <shemminger-ZtmgI6mnKB3QT0dZR+AlfA@public.gmane.org>
> 
> ---
>  app/test-pmd/config.c                   |    2 
>  app/test-pmd/testpmd.h                  |    8 -
>  lib/librte_eal/common/eal_common_pci.c  |   11 +
>  lib/librte_eal/common/include/rte_pci.h |    5 
>  lib/librte_eal/linuxapp/eal/eal_pci.c   |  244 +++++++++++++++++++-------------
>  lib/librte_pmd_ixgbe/ixgbe_ethdev.c     |    4 
>  6 files changed, 167 insertions(+), 107 deletions(-)
> 
> --- a/lib/librte_eal/common/eal_common_pci.c	2013-06-05 14:41:46.000000000 -0700
> +++ b/lib/librte_eal/common/eal_common_pci.c	2013-08-08 17:47:56.541720272 -0700
> @@ -121,12 +121,19 @@ rte_eal_pci_probe(void)
>  static int
>  pci_dump_one_device(struct rte_pci_device *dev)
>  {
> +	int i;
> +
>  	printf(PCI_PRI_FMT, dev->addr.domain, dev->addr.bus,
>  	       dev->addr.devid, dev->addr.function);
>  	printf(" - vendor:%x device:%x\n", dev->id.vendor_id,
>  	       dev->id.device_id);
> -	printf("   %16.16"PRIx64" %16.16"PRIx64"\n",
> -	       dev->mem_resource.phys_addr, dev->mem_resource.len);
> +
> +	for (i = 0; i < PCI_MAX_RESOURCE; i++) {
> +		printf("   %16.16"PRIx64" %16.16"PRIx64"\n",
> +		       dev->mem_resource[i].phys_addr,
> +		       dev->mem_resource[i].len);
> +	}
> +
>  	return 0;
>  }
>  
> --- a/lib/librte_eal/common/include/rte_pci.h	2013-08-08 17:41:37.866416535 -0700
> +++ b/lib/librte_eal/common/include/rte_pci.h	2013-08-08 17:42:34.585716365 -0700
> @@ -50,6 +50,7 @@ extern "C" {
>  #include <sys/queue.h>
>  #include <stdint.h>
>  #include <inttypes.h>
> +#include <limits.h>
>  #include <rte_interrupts.h>
>  
>  TAILQ_HEAD(pci_device_list, rte_pci_device); /**< PCI devices in D-linked Q. */
> @@ -110,10 +111,12 @@ struct rte_pci_device {
>  	TAILQ_ENTRY(rte_pci_device) next;       /**< Next probed PCI device. */
>  	struct rte_pci_addr addr;               /**< PCI location. */
>  	struct rte_pci_id id;                   /**< PCI ID. */
> -	struct rte_pci_resource mem_resource;   /**< PCI Memory Resource */
> +	struct rte_pci_resource mem_resource[PCI_MAX_RESOURCE];
> +						/**< PCI Memory Resource */
>  	struct rte_intr_handle intr_handle;     /**< Interrupt handle */
>  	const struct rte_pci_driver *driver;    /**< Associated driver */
>  	unsigned int blacklisted:1;             /**< Device is blacklisted */
> +	char uio_name[PATH_MAX];		/**< Associated UIO device name */
>  };
>  
>  /** Any PCI device identifier (vendor, device, ...) */
> --- a/lib/librte_eal/linuxapp/eal/eal_pci.c	2013-08-08 17:40:52.606974358 -0700
> +++ b/lib/librte_eal/linuxapp/eal/eal_pci.c	2013-08-08 17:47:28.942064203 -0700
> @@ -99,7 +99,6 @@ struct uio_resource {
>  
>  	struct rte_pci_addr pci_addr;
>  	void *addr;
> -	char path[PATH_MAX];
>  	unsigned long size;
>  	unsigned long offset;
>  };
> @@ -212,64 +211,108 @@ pci_uio_bind_device(struct rte_pci_devic
>  	return 0;
>  }
>  
> -/* map a particular resource from a file */
> -static void *
> -pci_map_resource(struct rte_pci_device *dev, void *requested_addr, const char *devname,
> -		unsigned long offset, unsigned long size)
> +/*
> + * open devname: it can take some time to
> + * appear, so we wait some time before returning an error
> + */
> +static int uio_open(const char *devname)
>  {
> -	unsigned n;
> -	int fd;
> -	void *mapaddr;
> +	int n, fd;
>  
> -	/*
> -	 * open devname, and mmap it: it can take some time to
> -	 * appear, so we wait some time before returning an error
> -	 */
> -	for (n=0; n<UIO_DEV_WAIT_TIMEOUT*10; n++) {
> +	for (n=0; n < UIO_DEV_WAIT_TIMEOUT*10; n++) {
>  		fd = open(devname, O_RDWR);
>  		if (fd >= 0)
> -			break;
> +			return fd;
> +
>  		if (errno != ENOENT)
>  			break;
>  		usleep(100000);
>  	}
> -	if (fd < 0) {
> -		RTE_LOG(ERR, EAL, "Cannot open %s: %s\n", devname, strerror(errno));
> -		goto fail;
> -	}
> +	return -1;
> +}
> +
> +/* map a particular resource from a file */
> +static void *
> +pci_mmap(int fd, void *addr, off_t offset, size_t size)
> +{
> +	void *mapaddr;
>  
>  	/* Map the PCI memory resource of device */
> -	mapaddr = mmap(requested_addr, size, PROT_READ | PROT_WRITE,
> -			MAP_SHARED, fd, offset);
> -	if (mapaddr == MAP_FAILED ||
> -			(requested_addr != NULL && mapaddr != requested_addr)) {
> -		RTE_LOG(ERR, EAL, "%s(): cannot mmap %s: %s\n", __func__,
> -			devname, strerror(errno));
> -		close(fd);
> -		goto fail;
> -	}
> -	if (rte_eal_process_type() == RTE_PROC_PRIMARY) {
> -		/* save fd if in primary process */
> -		dev->intr_handle.fd = fd;
> -		dev->intr_handle.type = RTE_INTR_HANDLE_UIO;
> -	} else {
> -		/* fd is not needed in slave process, close it */
> -		dev->intr_handle.fd = -1;
> -		dev->intr_handle.type = RTE_INTR_HANDLE_UNKNOWN;
> -		close(fd);
> +	mapaddr = mmap(addr, size, PROT_READ | PROT_WRITE, MAP_SHARED,
> +		       fd, offset);
> +	if (mapaddr == MAP_FAILED || (addr != NULL && mapaddr != addr)) {
> +		RTE_LOG(ERR, EAL, "%s(): cannot mmap %zd@0x%lx: %s\n",
> +			__func__, size, offset, strerror(errno));
> +		return NULL;
>  	}
>  
>  	RTE_LOG(DEBUG, EAL, "PCI memory mapped at %p\n", mapaddr);
> -
>  	return mapaddr;
> +}
> +
> +/* save the mapping details for secondary processes*/
> +static int pci_uio_map_save(const struct rte_pci_device *dev, void *mapaddr,
> +			    unsigned long offset, unsigned long size)
> +{
> +	struct uio_resource *uio_res;
> +
> +	uio_res = rte_malloc("UIO_RES", sizeof(*uio_res), 0);
> +	if (uio_res == NULL) {
> +		RTE_LOG(ERR, EAL, "%s(): cannot store uio mmap details\n",
> +			__func__);
> +		return -1;
> +	}
> +
> +	uio_res->addr = mapaddr;
> +	uio_res->offset = offset;
> +	uio_res->size = size;
> +	memcpy(&uio_res->pci_addr, &dev->addr, sizeof(uio_res->pci_addr));
> +
> +	TAILQ_INSERT_TAIL(uio_res_list, uio_res, next);
> +	return 0;
> +}
> +
> +static int pci_uio_map_restore(struct rte_pci_device *dev)
> +{
> +	struct uio_resource *uio_res;
> +	int i, fd;
> +	void *addr;
>  
> -fail:
>  	dev->intr_handle.fd = -1;
>  	dev->intr_handle.type = RTE_INTR_HANDLE_UNKNOWN;
>  
> -	return NULL;
> +	fd = uio_open(dev->uio_name);
> +	if (fd < 0)
> +		return -1;
> +
> +	TAILQ_FOREACH(uio_res, uio_res_list, next) {
> +		/* skip this element if it doesn't match our PCI address */
> +		if (memcmp(&uio_res->pci_addr, &dev->addr, sizeof(dev->addr)))
> +			continue;
> +
> +		for (i = 0; i < PCI_MAX_RESOURCE; i++) {
> +			if (dev->mem_resource[i].len == 0)
> +				continue;
> +
> +			addr = pci_mmap(fd, uio_res->addr,
> +					uio_res->offset, uio_res->size);
> +			if (addr != uio_res->addr) {
> +				RTE_LOG(ERR, EAL, "Cannot mmap device resource\n");
> +				close(fd);
> +				return -1;
> +			}
> +		}
> +
> +		close(fd);
> +		return 0;
> +	}
> +
> +	RTE_LOG(ERR, EAL, "Cannot find resource for device\n");
> +	close(fd);
> +	return -1;
>  }
> -/* map the PCI resource of a PCI device in virtual memory */
> +
> +/* map the PCI resources of a PCI device in virtual memory */
>  static int
>  pci_uio_map_resource(struct rte_pci_device *dev)
>  {
> @@ -278,35 +321,20 @@ pci_uio_map_resource(struct rte_pci_devi
>  	char dirname[PATH_MAX];
>  	char dirname2[PATH_MAX];
>  	char filename[PATH_MAX];
> -	char devname[PATH_MAX]; /* contains the /dev/uioX */
> +	int i, fd;
>  	void *mapaddr;
>  	unsigned uio_num;
> -	unsigned long size, offset;
> +	unsigned long size, offset, page_size;
>  	struct rte_pci_addr *loc = &dev->addr;
> -	struct uio_resource *uio_res;
> +
> +	page_size = sysconf(_SC_PAGE_SIZE);
>  
>  	RTE_LOG(DEBUG, EAL, "map PCI resource for device "PCI_PRI_FMT"\n",
>  	        loc->domain, loc->bus, loc->devid, loc->function);
>  
>  	/* secondary processes - use already recorded details */
> -	if (rte_eal_process_type() != RTE_PROC_PRIMARY) {
> -
> -			TAILQ_FOREACH(uio_res, uio_res_list, next) {
> -				/* skip this element if it doesn't match our PCI address */
> -				if (memcmp(&uio_res->pci_addr, &dev->addr, sizeof(dev->addr)))
> -					continue;
> -
> -				if (pci_map_resource(dev, uio_res->addr, uio_res->path, \
> -						uio_res->offset, uio_res->size) == uio_res->addr)
> -					return 0;
> -				else {
> -					RTE_LOG(ERR, EAL, "Cannot mmap device resource\n");
> -					return -1;
> -				}
> -			}
> -			RTE_LOG(ERR, EAL, "Cannot find resource for device\n");
> -			return -1;
> -	}
> +	if (rte_eal_process_type() != RTE_PROC_PRIMARY)
> +		return pci_uio_map_restore(dev);
>  
>  	/* depending on kernel version, uio can be located in uio/uioX
>  	 * or uio:uioX */
> @@ -362,44 +390,59 @@ pci_uio_map_resource(struct rte_pci_devi
>  	if (e == NULL)
>  		return 0;
>  
> -	/* get mapping offset */
> -	rte_snprintf(filename, sizeof(filename),
> -		 "%s/maps/map0/offset", dirname2);
> -	if (pci_parse_sysfs_value(filename, &offset) < 0) {
> -		RTE_LOG(ERR, EAL, "%s(): cannot parse offset\n",
> -			__func__);
> -		return -1;
> -	}
> +	/* open /dev/uioX */
> +	rte_snprintf(dev->uio_name, sizeof(dev->uio_name),
> +		     "/dev/uio%u", uio_num);
>  
> -	/* get mapping size */
> -	rte_snprintf(filename, sizeof(filename),
> -		 "%s/maps/map0/size", dirname2);
> -	if (pci_parse_sysfs_value(filename, &size) < 0) {
> -		RTE_LOG(ERR, EAL, "%s(): cannot parse size\n",
> -			__func__);
> +	fd = uio_open(dev->uio_name);
> +	if (fd < 0) {
> +		RTE_LOG(ERR, EAL, "Cannot open %s: %s\n",
> +			dev->uio_name, strerror(errno));
>  		return -1;
>  	}
>  
> -	/* open and mmap /dev/uioX */
> -	rte_snprintf(devname, sizeof(devname), "/dev/uio%u", uio_num);
> -	mapaddr = pci_map_resource(dev, NULL, devname, offset, size);
> -	if (mapaddr == NULL)
> -		return -1;
> -	dev->mem_resource.addr = mapaddr;
> +	/* map associated memory resources. */
> +	for (i = 0; i < PCI_MAX_RESOURCE; i++) {
> +		if (dev->mem_resource[i].len == 0)
> +			continue;
>  
> -	/* save the mapping details for secondary processes*/
> -	uio_res = rte_malloc("UIO_RES", sizeof(*uio_res), 0);
> -	if (uio_res == NULL){
> -		RTE_LOG(ERR, EAL, "%s(): cannot store uio mmap details\n", __func__);
> -		return -1;
> +		rte_snprintf(filename, sizeof(filename),
> +			     "%s/maps/map%d/offset", dirname2, i);
> +
> +		if (access(filename, F_OK) < 0)
> +			continue; /* this resource is not mapped via uio */
> +
> +		/* get mapping offset */
> +		if (pci_parse_sysfs_value(filename, &offset) < 0) {
> +			RTE_LOG(ERR, EAL, "%s(): cannot parse offset\n",
> +				__func__);
> +			return -1;
> +		}
> +
> +		/* page number indicates which resource */
> +		offset += i * page_size;
> +
> +		/* get mapping size */
> +		rte_snprintf(filename, sizeof(filename),
> +			     "%s/maps/map%d/size", dirname2, i);
> +		if (pci_parse_sysfs_value(filename, &size) < 0) {
> +			RTE_LOG(ERR, EAL, "%s(): cannot parse size\n",
> +				__func__);
> +			return -1;
> +		}
> +
> +		mapaddr = pci_mmap(fd, NULL, offset, size);
> +		if (mapaddr == NULL)
> +			return -1;
> +
> +		dev->mem_resource[i].addr = mapaddr;
> +		if (pci_uio_map_save(dev, mapaddr, offset, size) < 0)
> +			return -1;
>  	}
> -	uio_res->addr = mapaddr;
> -	uio_res->offset = offset;
> -	uio_res->size = size;
> -	rte_snprintf(uio_res->path, sizeof(uio_res->path), "%s", devname);
> -	memcpy(&uio_res->pci_addr, &dev->addr, sizeof(uio_res->pci_addr));
>  
> -	TAILQ_INSERT_TAIL(uio_res_list, uio_res, next);
> +	/* save fd if in primary process */
> +	dev->intr_handle.fd = fd;
> +	dev->intr_handle.type = RTE_INTR_HANDLE_UIO;
>  
>  	return 0;
>  }
> @@ -420,7 +463,7 @@ pci_parse_sysfs_resource(const char *fil
>  		};
>  		char *ptrs[PCI_RESOURCE_FMT_NVAL];
>  	} res_info;
> -	int i;
> +	int i, m;
>  	uint64_t phys_addr, end_addr, flags;
>  
>  	f = fopen(filename, "r");
> @@ -429,6 +472,7 @@ pci_parse_sysfs_resource(const char *fil
>  		return -1;
>  	}
>  
> +	m = 0;
>  	for (i = 0; i<PCI_MAX_RESOURCE; i++) {
>  
>  		if (fgets(buf, sizeof(buf), f) == NULL) {
> @@ -450,10 +494,16 @@ pci_parse_sysfs_resource(const char *fil
>  		}
>  
>  		if (flags & IORESOURCE_MEM) {
> -			dev->mem_resource.phys_addr = phys_addr;
> -			dev->mem_resource.len = end_addr - phys_addr + 1;
> -			dev->mem_resource.addr = NULL; /* not mapped for now */
> -			break;
> +			if (m == PCI_MAX_RESOURCE) {
> +				RTE_LOG(ERR, EAL, "%s(): too many memory resources\n",
> +					__func__);
> +				goto error;
> +			}
> +
> +			dev->mem_resource[m].phys_addr = phys_addr;
> +			dev->mem_resource[m].len = end_addr - phys_addr + 1;
> +			dev->mem_resource[m].addr = NULL; /* not mapped for now */
> +			++m;
>  		}
>  	}
>  	fclose(f);
> --- a/lib/librte_pmd_ixgbe/ixgbe_ethdev.c	2013-08-08 17:40:52.606974358 -0700
> +++ b/lib/librte_pmd_ixgbe/ixgbe_ethdev.c	2013-08-08 17:41:50.266263568 -0700
> @@ -369,7 +369,7 @@ eth_ixgbe_dev_init(__attribute__((unused
>  	/* Vendor and Device ID need to be set before init of shared code */
>  	hw->device_id = pci_dev->id.device_id;
>  	hw->vendor_id = pci_dev->id.vendor_id;
> -	hw->hw_addr = (void *)pci_dev->mem_resource.addr;
> +	hw->hw_addr = pci_dev->mem_resource[0].addr;
>  
>  	/* Initialize the shared code */
>  	diag = ixgbe_init_shared_code(hw);
> @@ -490,7 +490,7 @@ eth_ixgbevf_dev_init(__attribute__((unus
>  
>  	hw->device_id = pci_dev->id.device_id;
>  	hw->vendor_id = pci_dev->id.vendor_id;
> -	hw->hw_addr = (void *)pci_dev->mem_resource.addr;
> +	hw->hw_addr = pci_dev->mem_resource[0].addr;
>  
>  	/* Initialize the shared code */
>  	diag = ixgbe_init_shared_code(hw);
> --- a/app/test-pmd/config.c	2013-06-05 14:41:46.000000000 -0700
> +++ b/app/test-pmd/config.c	2013-08-08 17:41:50.294263224 -0700
> @@ -180,7 +180,7 @@ port_reg_off_is_invalid(portid_t port_id
>  		       (unsigned)reg_off);
>  		return 1;
>  	}
> -	pci_len = ports[port_id].dev_info.pci_dev->mem_resource.len;
> +	pci_len = ports[port_id].dev_info.pci_dev->mem_resource[0].len;
>  	if (reg_off >= pci_len) {
>  		printf("Port %d: register offset %u (0x%X) out of port PCI "
>  		       "resource (length=%"PRIu64")\n",
> --- a/app/test-pmd/testpmd.h	2013-06-05 14:41:46.000000000 -0700
> +++ b/app/test-pmd/testpmd.h	2013-08-08 17:41:50.294263224 -0700
> @@ -304,8 +304,8 @@ port_pci_reg_read(struct rte_port *port,
>  	void *reg_addr;
>  	uint32_t reg_v;
>  
> -	reg_addr = (void *)((char *)port->dev_info.pci_dev->mem_resource.addr +
> -			    reg_off);
> +	reg_addr = (char *)port->dev_info.pci_dev->mem_resource[0].addr
> +		+ reg_off;
>  	reg_v = *((volatile uint32_t *)reg_addr);
>  	return rte_le_to_cpu_32(reg_v);
>  }
> @@ -318,8 +318,8 @@ port_pci_reg_write(struct rte_port *port
>  {
>  	void *reg_addr;
>  
> -	reg_addr = (void *)((char *)port->dev_info.pci_dev->mem_resource.addr +
> -			    reg_off);
> +	reg_addr = (char *)port->dev_info.pci_dev->mem_resource[0].addr
> +		    + reg_off;
>  	*((volatile uint32_t *)reg_addr) = rte_cpu_to_le_32(reg_v);
>  }
>  
>