Re: [net-next 2/9] ice: Create and register virtual bus for RDMA

[Jason Gunthorpe <jgg@ziepe.ca>]

On Fri, Apr 17, 2020 at 10:10:27AM -0700, Jeff Kirsher wrote:

> +/**
> + * ice_peer_vdev_release - function to map to virtbus_devices release callback
> + * @vdev: pointer to virtbus_device to free
> + */
> +static void ice_peer_vdev_release(struct virtbus_device *vdev)
> +{
> +	struct iidc_virtbus_object *vbo;
> +
> +	vbo = container_of(vdev, struct iidc_virtbus_object, vdev);
> +	kfree(vbo);
> +}
> +
> +/**
> + * ice_init_peer_devices - initializes peer devices
> + * @pf: ptr to ice_pf
> + *
> + * This function initializes peer devices on the virtual bus.
> + */
> +int ice_init_peer_devices(struct ice_pf *pf)
> +{
> +	struct ice_vsi *vsi = pf->vsi[0];
> +	struct pci_dev *pdev = pf->pdev;
> +	struct device *dev = &pdev->dev;
> +	int status = 0;
> +	unsigned int i;
> +
> +	/* Reserve vector resources */
> +	status = ice_reserve_peer_qvector(pf);
> +	if (status < 0) {
> +		dev_err(dev, "failed to reserve vectors for peer drivers\n");
> +		return status;
> +	}
> +	for (i = 0; i < ARRAY_SIZE(ice_peers); i++) {
> +		struct ice_peer_dev_int *peer_dev_int;
> +		struct ice_peer_drv_int *peer_drv_int;
> +		struct iidc_qos_params *qos_info;
> +		struct iidc_virtbus_object *vbo;
> +		struct msix_entry *entry = NULL;
> +		struct iidc_peer_dev *peer_dev;
> +		struct virtbus_device *vdev;
> +		int j;
> +
> +		/* structure layout needed for container_of's looks like:
> +		 * ice_peer_dev_int (internal only ice peer superstruct)
> +		 * |--> iidc_peer_dev
> +		 * |--> *ice_peer_drv_int
> +		 *
> +		 * iidc_virtbus_object (container_of parent for vdev)
> +		 * |--> virtbus_device
> +		 * |--> *iidc_peer_dev (pointer from internal struct)
> +		 *
> +		 * ice_peer_drv_int (internal only peer_drv struct)
> +		 */
> +		peer_dev_int = kzalloc(sizeof(*peer_dev_int), GFP_KERNEL);
> +		if (!peer_dev_int)
> +			return -ENOMEM;
> +
> +		vbo = kzalloc(sizeof(*vbo), GFP_KERNEL);
> +		if (!vbo) {
> +			kfree(peer_dev_int);
> +			return -ENOMEM;
> +		}
> +
> +		peer_drv_int = kzalloc(sizeof(*peer_drv_int), GFP_KERNEL);
> +		if (!peer_drv_int) {
> +			kfree(peer_dev_int);
> +			kfree(vbo);
> +			return -ENOMEM;
> +		}

The lifetimes of all this memory look really suspect. The vbo holds a
pointer to the peer_dev but who ensures it it freed after all the vbo
kref's are released so there isn't a dangling pointer in
vbo->peer_dev?

One allocation is much simpler to understand:

struct iidc_virtbus_object {
   struct virbus_device vdev;
   [public members]
}

struct iidc_virtbus_object_private {
   struct iidc_virtbus_object vobj;
   [private members]
}

And just kzalloc a single iidc_virtbus_object_private

> +		peer_dev->msix_entries = entry;
> +		ice_peer_state_change(peer_dev_int, ICE_PEER_DEV_STATE_INIT,
> +				      false);
> +
> +		vdev = &vbo->vdev;
> +		vdev->name = ice_peers[i].name;
> +		vdev->release = ice_peer_vdev_release;
> +		vdev->dev.parent = &pdev->dev;
> +
> +		status = virtbus_register_device(vdev);
> +		if (status) {
> +			kfree(peer_dev_int);
> +			kfree(peer_drv_int);
> +			vdev = NULL;

To me this feels very unnatural, virtbus_register_device() does the
kfree for the vbo if it fails so this function can't have a the normal
goto error unwind and ends up open coding the error unwinds in each if
above.

> +/* Following APIs are implemented by peer drivers and invoked by device
> + * owner
> + */
> +struct iidc_peer_ops {
> +	void (*event_handler)(struct iidc_peer_dev *peer_dev,
> +			      struct iidc_event *event);
> +
> +	/* Why we have 'open' and when it is expected to be called:
> +	 * 1. symmetric set of API w.r.t close
> +	 * 2. To be invoked form driver initialization path
> +	 *     - call peer_driver:open once device owner is fully
> +	 *     initialized
> +	 * 3. To be invoked upon RESET complete
> +	 */
> +	int (*open)(struct iidc_peer_dev *peer_dev);
> +
> +	/* Peer's close function is to be called when the peer needs to be
> +	 * quiesced. This can be for a variety of reasons (enumerated in the
> +	 * iidc_close_reason enum struct). A call to close will only be
> +	 * followed by a call to either remove or open. No IDC calls from the
> +	 * peer should be accepted until it is re-opened.
> +	 *
> +	 * The *reason* parameter is the reason for the call to close. This
> +	 * can be for any reason enumerated in the iidc_close_reason struct.
> +	 * It's primary reason is for the peer's bookkeeping and in case the
> +	 * peer want to perform any different tasks dictated by the reason.
> +	 */
> +	void (*close)(struct iidc_peer_dev *peer_dev,
> +		      enum iidc_close_reason reason);

The open and close op looks really weird

Jason