Re: [PATCH v4 7/7] ocxl: Provide global MMIO accessors for external drivers

[Frederic Barrat <fbarrat@linux.ibm.com>]

Le 27/03/2019 à 06:31, Alastair D'Silva a écrit :
> From: Alastair D'Silva <alastair@d-silva.org>
> 
> External drivers that communicate via OpenCAPI will need to make
> MMIO calls to interact with the devices.
> 
> Signed-off-by: Alastair D'Silva <alastair@d-silva.org>
> Reviewed-by: Greg Kurz <groug@kaod.org>
> ---


Acked-by: Frederic Barrat <fbarrat@linux.ibm.com>

And that's the final one! Thanks a lot Alastair, that took some effort, 
but I'm quite happy with the result.

   Fred



>   drivers/misc/ocxl/Makefile |   2 +-
>   drivers/misc/ocxl/mmio.c   | 234 +++++++++++++++++++++++++++++++++++++
>   include/misc/ocxl.h        | 110 +++++++++++++++++
>   3 files changed, 345 insertions(+), 1 deletion(-)
>   create mode 100644 drivers/misc/ocxl/mmio.c
> 
> diff --git a/drivers/misc/ocxl/Makefile b/drivers/misc/ocxl/Makefile
> index bc4e39bfda7b..d07d1bb8e8d4 100644
> --- a/drivers/misc/ocxl/Makefile
> +++ b/drivers/misc/ocxl/Makefile
> @@ -1,7 +1,7 @@
>   # SPDX-License-Identifier: GPL-2.0+
>   ccflags-$(CONFIG_PPC_WERROR)	+= -Werror
>   
> -ocxl-y				+= main.o pci.o config.o file.o pasid.o
> +ocxl-y				+= main.o pci.o config.o file.o pasid.o mmio.o
>   ocxl-y				+= link.o context.o afu_irq.o sysfs.o trace.o
>   ocxl-y				+= core.o
>   obj-$(CONFIG_OCXL)		+= ocxl.o
> diff --git a/drivers/misc/ocxl/mmio.c b/drivers/misc/ocxl/mmio.c
> new file mode 100644
> index 000000000000..aae713db4ebe
> --- /dev/null
> +++ b/drivers/misc/ocxl/mmio.c
> @@ -0,0 +1,234 @@
> +// SPDX-License-Identifier: GPL-2.0+
> +// Copyright 2019 IBM Corp.
> +#include <linux/sched/mm.h>
> +#include "trace.h"
> +#include "ocxl_internal.h"
> +
> +int ocxl_global_mmio_read32(struct ocxl_afu *afu, size_t offset,
> +				enum ocxl_endian endian, u32 *val)
> +{
> +	if (offset > afu->config.global_mmio_size - 4)
> +		return -EINVAL;
> +
> +#ifdef __BIG_ENDIAN__
> +	if (endian == OCXL_HOST_ENDIAN)
> +		endian = OCXL_BIG_ENDIAN;
> +#endif
> +
> +	switch (endian) {
> +	case OCXL_BIG_ENDIAN:
> +		*val = readl_be((char *)afu->global_mmio_ptr + offset);
> +		break;
> +
> +	default:
> +		*val = readl((char *)afu->global_mmio_ptr + offset);
> +		break;
> +	}
> +
> +	return 0;
> +}
> +EXPORT_SYMBOL_GPL(ocxl_global_mmio_read32);
> +
> +int ocxl_global_mmio_read64(struct ocxl_afu *afu, size_t offset,
> +				enum ocxl_endian endian, u64 *val)
> +{
> +	if (offset > afu->config.global_mmio_size - 8)
> +		return -EINVAL;
> +
> +#ifdef __BIG_ENDIAN__
> +	if (endian == OCXL_HOST_ENDIAN)
> +		endian = OCXL_BIG_ENDIAN;
> +#endif
> +
> +	switch (endian) {
> +	case OCXL_BIG_ENDIAN:
> +		*val = readq_be((char *)afu->global_mmio_ptr + offset);
> +		break;
> +
> +	default:
> +		*val = readq((char *)afu->global_mmio_ptr + offset);
> +		break;
> +	}
> +
> +	return 0;
> +}
> +EXPORT_SYMBOL_GPL(ocxl_global_mmio_read64);
> +
> +int ocxl_global_mmio_write32(struct ocxl_afu *afu, size_t offset,
> +				enum ocxl_endian endian, u32 val)
> +{
> +	if (offset > afu->config.global_mmio_size - 4)
> +		return -EINVAL;
> +
> +#ifdef __BIG_ENDIAN__
> +	if (endian == OCXL_HOST_ENDIAN)
> +		endian = OCXL_BIG_ENDIAN;
> +#endif
> +
> +	switch (endian) {
> +	case OCXL_BIG_ENDIAN:
> +		writel_be(val, (char *)afu->global_mmio_ptr + offset);
> +		break;
> +
> +	default:
> +		writel(val, (char *)afu->global_mmio_ptr + offset);
> +		break;
> +	}
> +
> +
> +	return 0;
> +}
> +EXPORT_SYMBOL_GPL(ocxl_global_mmio_write32);
> +
> +int ocxl_global_mmio_write64(struct ocxl_afu *afu, size_t offset,
> +				enum ocxl_endian endian, u64 val)
> +{
> +	if (offset > afu->config.global_mmio_size - 8)
> +		return -EINVAL;
> +
> +#ifdef __BIG_ENDIAN__
> +	if (endian == OCXL_HOST_ENDIAN)
> +		endian = OCXL_BIG_ENDIAN;
> +#endif
> +
> +	switch (endian) {
> +	case OCXL_BIG_ENDIAN:
> +		writeq_be(val, (char *)afu->global_mmio_ptr + offset);
> +		break;
> +
> +	default:
> +		writeq(val, (char *)afu->global_mmio_ptr + offset);
> +		break;
> +	}
> +
> +
> +	return 0;
> +}
> +EXPORT_SYMBOL_GPL(ocxl_global_mmio_write64);
> +
> +int ocxl_global_mmio_set32(struct ocxl_afu *afu, size_t offset,
> +				enum ocxl_endian endian, u32 mask)
> +{
> +	u32 tmp;
> +
> +	if (offset > afu->config.global_mmio_size - 4)
> +		return -EINVAL;
> +
> +#ifdef __BIG_ENDIAN__
> +	if (endian == OCXL_HOST_ENDIAN)
> +		endian = OCXL_BIG_ENDIAN;
> +#endif
> +
> +	switch (endian) {
> +	case OCXL_BIG_ENDIAN:
> +		tmp = readl_be((char *)afu->global_mmio_ptr + offset);
> +		tmp |= mask;
> +		writel_be(tmp, (char *)afu->global_mmio_ptr + offset);
> +		break;
> +
> +	default:
> +		tmp = readl((char *)afu->global_mmio_ptr + offset);
> +		tmp |= mask;
> +		writel(tmp, (char *)afu->global_mmio_ptr + offset);
> +		break;
> +	}
> +
> +	return 0;
> +}
> +EXPORT_SYMBOL_GPL(ocxl_global_mmio_set32);
> +
> +int ocxl_global_mmio_set64(struct ocxl_afu *afu, size_t offset,
> +				enum ocxl_endian endian, u64 mask)
> +{
> +	u64 tmp;
> +
> +	if (offset > afu->config.global_mmio_size - 8)
> +		return -EINVAL;
> +
> +#ifdef __BIG_ENDIAN__
> +	if (endian == OCXL_HOST_ENDIAN)
> +		endian = OCXL_BIG_ENDIAN;
> +#endif
> +
> +	switch (endian) {
> +	case OCXL_BIG_ENDIAN:
> +		tmp = readq_be((char *)afu->global_mmio_ptr + offset);
> +		tmp |= mask;
> +		writeq_be(tmp, (char *)afu->global_mmio_ptr + offset);
> +		break;
> +
> +	default:
> +		tmp = readq((char *)afu->global_mmio_ptr + offset);
> +		tmp |= mask;
> +		writeq(tmp, (char *)afu->global_mmio_ptr + offset);
> +		break;
> +	}
> +
> +	return 0;
> +}
> +EXPORT_SYMBOL_GPL(ocxl_global_mmio_set64);
> +
> +int ocxl_global_mmio_clear32(struct ocxl_afu *afu, size_t offset,
> +				enum ocxl_endian endian, u32 mask)
> +{
> +	u32 tmp;
> +
> +	if (offset > afu->config.global_mmio_size - 4)
> +		return -EINVAL;
> +
> +#ifdef __BIG_ENDIAN__
> +	if (endian == OCXL_HOST_ENDIAN)
> +		endian = OCXL_BIG_ENDIAN;
> +#endif
> +
> +	switch (endian) {
> +	case OCXL_BIG_ENDIAN:
> +		tmp = readl_be((char *)afu->global_mmio_ptr + offset);
> +		tmp &= ~mask;
> +		writel_be(tmp, (char *)afu->global_mmio_ptr + offset);
> +		break;
> +
> +	default:
> +		tmp = readl((char *)afu->global_mmio_ptr + offset);
> +		tmp &= ~mask;
> +		writel(tmp, (char *)afu->global_mmio_ptr + offset);
> +		break;
> +	}
> +
> +
> +	return 0;
> +}
> +EXPORT_SYMBOL_GPL(ocxl_global_mmio_clear32);
> +
> +int ocxl_global_mmio_clear64(struct ocxl_afu *afu, size_t offset,
> +				enum ocxl_endian endian, u64 mask)
> +{
> +	u64 tmp;
> +
> +	if (offset > afu->config.global_mmio_size - 8)
> +		return -EINVAL;
> +
> +#ifdef __BIG_ENDIAN__
> +	if (endian == OCXL_HOST_ENDIAN)
> +		endian = OCXL_BIG_ENDIAN;
> +#endif
> +
> +	switch (endian) {
> +	case OCXL_BIG_ENDIAN:
> +		tmp = readq_be((char *)afu->global_mmio_ptr + offset);
> +		tmp &= ~mask;
> +		writeq_be(tmp, (char *)afu->global_mmio_ptr + offset);
> +		break;
> +
> +	default:
> +		tmp = readq((char *)afu->global_mmio_ptr + offset);
> +		tmp &= ~mask;
> +		writeq(tmp, (char *)afu->global_mmio_ptr + offset);
> +		break;
> +	}
> +
> +	writeq(tmp, (char *)afu->global_mmio_ptr + offset);
> +
> +	return 0;
> +}
> +EXPORT_SYMBOL_GPL(ocxl_global_mmio_clear64);
> diff --git a/include/misc/ocxl.h b/include/misc/ocxl.h
> index dea93885a839..5c4b4916e6be 100644
> --- a/include/misc/ocxl.h
> +++ b/include/misc/ocxl.h
> @@ -45,6 +45,12 @@ struct ocxl_fn_config {
>   	s8 max_afu_index;
>   };
>   
> +enum ocxl_endian {
> +	OCXL_BIG_ENDIAN = 0,    /**< AFU data is big-endian */
> +	OCXL_LITTLE_ENDIAN = 1, /**< AFU data is little-endian */
> +	OCXL_HOST_ENDIAN = 2,   /**< AFU data is the same endianness as the host */
> +};
> +
>   // These are opaque outside the ocxl driver
>   struct ocxl_afu;
>   struct ocxl_fn;
> @@ -230,6 +236,110 @@ void ocxl_afu_set_private(struct ocxl_afu *afu, void *private);
>    */
>   void *ocxl_afu_get_private(struct ocxl_afu *dev);
>   
> +// Global MMIO
> +/**
> + * Read a 32 bit value from global MMIO
> + *
> + * @afu: The AFU
> + * @offset: The Offset from the start of MMIO
> + * @endian: the endianness that the MMIO data is in
> + * @val: returns the value
> + *
> + * Returns 0 for success, negative on error
> + */
> +int ocxl_global_mmio_read32(struct ocxl_afu *afu, size_t offset,
> +				enum ocxl_endian endian, u32 *val);
> +
> +/**
> + * Read a 64 bit value from global MMIO
> + *
> + * @afu: The AFU
> + * @offset: The Offset from the start of MMIO
> + * @endian: the endianness that the MMIO data is in
> + * @val: returns the value
> + *
> + * Returns 0 for success, negative on error
> + */
> +int ocxl_global_mmio_read64(struct ocxl_afu *afu, size_t offset,
> +				enum ocxl_endian endian, u64 *val);
> +
> +/**
> + * Write a 32 bit value to global MMIO
> + *
> + * @afu: The AFU
> + * @offset: The Offset from the start of MMIO
> + * @endian: the endianness that the MMIO data is in
> + * @val: The value to write
> + *
> + * Returns 0 for success, negative on error
> + */
> +int ocxl_global_mmio_write32(struct ocxl_afu *afu, size_t offset,
> +				enum ocxl_endian endian, u32 val);
> +
> +/**
> + * Write a 64 bit value to global MMIO
> + *
> + * @afu: The AFU
> + * @offset: The Offset from the start of MMIO
> + * @endian: the endianness that the MMIO data is in
> + * @val: The value to write
> + *
> + * Returns 0 for success, negative on error
> + */
> +int ocxl_global_mmio_write64(struct ocxl_afu *afu, size_t offset,
> +				enum ocxl_endian endian, u64 val);
> +
> +/**
> + * Set bits in a 32 bit global MMIO register
> + *
> + * @afu: The AFU
> + * @offset: The Offset from the start of MMIO
> + * @endian: the endianness that the MMIO data is in
> + * @mask: a mask of the bits to set
> + *
> + * Returns 0 for success, negative on error
> + */
> +int ocxl_global_mmio_set32(struct ocxl_afu *afu, size_t offset,
> +				enum ocxl_endian endian, u32 mask);
> +
> +/**
> + * Set bits in a 64 bit global MMIO register
> + *
> + * @afu: The AFU
> + * @offset: The Offset from the start of MMIO
> + * @endian: the endianness that the MMIO data is in
> + * @mask: a mask of the bits to set
> + *
> + * Returns 0 for success, negative on error
> + */
> +int ocxl_global_mmio_set64(struct ocxl_afu *afu, size_t offset,
> +				enum ocxl_endian endian, u64 mask);
> +
> +/**
> + * Set bits in a 32 bit global MMIO register
> + *
> + * @afu: The AFU
> + * @offset: The Offset from the start of MMIO
> + * @endian: the endianness that the MMIO data is in
> + * @mask: a mask of the bits to set
> + *
> + * Returns 0 for success, negative on error
> + */
> +int ocxl_global_mmio_clear32(struct ocxl_afu *afu, size_t offset,
> +				enum ocxl_endian endian, u32 mask);
> +
> +/**
> + * Set bits in a 64 bit global MMIO register
> + *
> + * @afu: The AFU
> + * @offset: The Offset from the start of MMIO
> + * @endian: the endianness that the MMIO data is in
> + * @mask: a mask of the bits to set
> + *
> + * Returns 0 for success, negative on error
> + */
> +int ocxl_global_mmio_clear64(struct ocxl_afu *afu, size_t offset,
> +				enum ocxl_endian endian, u64 mask);
>   
>   // Functions left here are for compatibility with the cxlflash driver
>   
>