[RFC PATCH v5 0/4] CXL: Standalone switch CCI driver

[Jonathan Cameron <Jonathan.Cameron@huawei.com>]

Based on cxl/fixes as I'm not expecting this to land this cycle
and there was a messy clash that would make this unlikely to apply
shortly.

RFC v5 changes: Thanks to DavidLohr for review
- Renames of files etc as Dan suggested several versions back.
- Split cxlmem and cxlswd more thoroughly by getting rid of
  cxl_dev_state which was only needed for one iomem mappping.
  Handling the two users of that mapping is a little ugly but
  I can't currently think of a better way.
- Use same chardev major number and IDA as CXL memdevs.
- Move a lot more infrastructure definition into cxlmbox.h.
  Note we could definitely do more on this as a lot of the stuff in
  core/mbox.c is not applicable to all types of devices. However
  that reorg can wait.
- Thin out the callbacks. One wasn't used and another was only used
  right next to where it was defined. Also add docs.

This is getting cleaner, so I'd definitely like feedback on the
basic refactoring. There are open questions around how to control
access once we add more commands given many are footguns.

I'm sharing it at this stage to provide a test path for the QEMU
rework that enables emulation of the various different types of
CCI (PCI Mailbox / MCTP over I2C / Tunneled versions of each).
There are open questions from earlier versions around how we manage
the security of these. To a certain extent we wouldn't expect
to often see a switch-cci connected to a CXL host (except when
emulating), but we should probably not assume that.

CXL rev 3.0 introduced the option for a PCI function, intended to sit on an
upstream port of a CXL switch.  This function provides a mailbox
interface similar to that seen on CXL type 3 devices. However, the
command set is mostly different and intended for Fabric management.
Note however that as we add support for multi headed devices (MHDs)
a subset of commands will be available on selected MHD type 3 mailboxes.
(tunnelling DCD commands for example)

See: CXL rev 3.0
7.2.9 Switch Mailbox CCI
8.1.13 Switch Malibox CCI Configuration Space Layout
8.2.8.6 Switch Mailbox CCI capability 

It is probably relatively unusual that a typical host of CXL devices
will have access to the one of these devices, in many cases they will
be on a port connected to a BMC or similar. There are a few use cases
where the host might be in charge of the configuration.

These are very convenient for testing in conjunction with the QEMU
emulation though so far CXL switch and type 3 emulation is in QEMU
is not complex enough to make these particular interesting.

This initial support provides only a few commands but I'm sending it
out as an RFC to get some input on how we should refactor the CXL core
code to support these devices that use some of the provide functionality.

Example command line based on CXL QEMU emulation branch at
https://gitlab.com/jic23/qemu cxl-2023-07-17

qemu-system-aarch64 -M virt,nvdimm=on,gic-version=3,cxl=on -m 4g,maxmem=8G,slots=8 -cpu max -smp 4 \
 ...
 -object memory-backend-file,id=cxl-mem1,share=on,mem-path=/tmp/t3_cxl1.raw,size=256M,align=256M \
 -object memory-backend-file,id=cxl-lsa1,share=on,mem-path=/tmp/t3_lsa1.raw,size=1M,align=1M \
 -object memory-backend-file,id=cxl-mem2,share=on,mem-path=/tmp/t3_cxl2.raw,size=256M,align=256M \
 -object memory-backend-file,id=cxl-lsa2,share=on,mem-path=/tmp/t3_lsa2.raw,size=1M,align=1M \
 -device pxb-cxl,bus_nr=12,bus=pcie.0,id=cxl.1,hdm_for_passthrough=true \
 -device cxl-rp,port=0,bus=cxl.1,id=cxl_rp_port0,chassis=0,slot=2 \
 -device cxl-upstream,bus=cxl_rp_port0,id=us0,addr=0.0,multifunction=on, \
 -device cxl-switch-mailbox-cci,bus=cxl_rp_port0,addr=0.1,target=us0 \
 -device cxl-downstream,port=0,bus=us0,id=swport0,chassis=0,slot=4 \
 -device cxl-downstream,port=1,bus=us0,id=swport1,chassis=0,slot=5 \
 -device cxl-type3,bus=swport0,memdev=cxl-mem1,id=cxl-pmem1,lsa=cxl-lsa1,sn=3 \
 -device cxl-type3,bus=swport1,memdev=cxl-mem2,id=cxl-pmem2,lsa=cxl-lsa2,sn=4 \
 -machine cxl-fmw.0.targets.0=cxl.1,cxl-fmw.0.size=4G,cxl-fmw.0.interleave-granularity=1k

Really hacky test program:

//Testswitch.c
#include <linux/types.h>
#include <stdint.h>
#include <sys/ioctl.h>
#include <stdio.h>
#include <fcntl.h>
#include <stdlib.h>
#define __user
#include "cxl_mem.h" /* include/uapi/linux/cxl_mem.h */

/* Move to appropriate header later */
struct cxl_cmd_infostat_identify_rsp {
  uint16_t pcie_vid;
  uint16_t pcie_did;
  uint16_t pcie_subsys_vid;
  uint16_t pcie_subsys_id;
  uint64_t sn;
  uint8_t max_message_size;
  uint8_t component_type;
};

struct cxl_cmd_infostat_get_bg_cmd_sts_rsp { 
  uint8_t status;
  uint8_t rsvd;
  uint16_t opcode;
  uint16_t returncode;
  uint16_t vendor_ext_status;
};

struct cxl_cmd_identify_switch_device_rsp {
  uint8_t ingress_port_id;
  uint8_t rsvd;
  uint8_t num_physical_ports;
  uint8_t num_vcs;
  uint8_t active_port_bm[0x20];
  uint8_t vcs_bm[0x20];
  uint16_t total_num_vPPBs;
  uint16_t num_bound_vPPBs;
  uint8_t num_hdm_decoders;
} __attribute__((packed));

struct cci_message {
  uint8_t message_cat;
  uint8_t tag;
  uint8_t resv;
  uint8_t command;
  uint8_t command_set;
  uint8_t pl_length[3];
  uint16_t ret_code;
  uint16_t vendor_ext;
  uint8_t payload[];
};

struct cxl_cmd_tunnel_command_req {
  uint8_t id;
  uint8_t target_type;
  #define TUNNEL_TARGET_TYPE_PORT_OR_LD  0
  #define TUNNEL_TARGET_TYPE_LD_POOL_CCI 1
  uint16_t command_size;
  struct cci_message message;
};

struct cxl_cmd_tunnel_command_resp {
  uint16_t length;
  uint16_t resv;
  struct cci_message message;
};


int main()
{
  struct cxl_send_command cmd = {};
  struct cxl_cmd_infostat_identify_rsp is_identify;
  struct cxl_cmd_identify_switch_device_rsp switch_identify;
  struct cxl_cmd_infostat_get_bg_cmd_sts_rsp bg_cmd_status;
  struct cxl_cmd_tunnel_command_req *tunnel_req;
  struct cxl_cmd_tunnel_command_resp *tunnel_resp;
  int fd;
  int rc, i;
  int tun_cmd_size, tun_resp_size;

  printf("Starting\n");
  fd = open("/dev/cxl/switch0", O_RDWR);
  if (fd < 0) {
    printf("could not open file\n");
    return 0;
  }
  cmd.id = CXL_MEM_COMMAND_ID_RAW;
  cmd.id = CXL_MEM_COMMAND_ID_INFO_STAT_IDENTIFY;
  cmd.out.size = sizeof(is_identify);
  cmd.out.payload = (__u64)&is_identify;

  rc = ioctl(fd, CXL_MEM_SEND_COMMAND, &cmd);
  if (rc) {
    printf("rc %d\n", rc);
    if (rc < 0)
      return rc;
  } 

  printf("Identify on switch:\n");
  printf("VID:0x%04x DID:0x%04x\n", is_identify.pcie_vid, is_identify.pcie_did);
  printf("Subsys: VID:0x%04x DID:0x%04x\n", is_identify.pcie_subsys_vid, is_identify.pcie_subsys_id);

  cmd.id = CXL_MEM_COMMAND_ID_GET_BG_CMD_STATUS;
  cmd.out.size = sizeof(bg_cmd_status);
  cmd.out.payload = (__u64)&bg_cmd_status;

  rc = ioctl(fd, CXL_MEM_SEND_COMMAND, &cmd);
  if (rc) {
    printf("rc %d\n", rc);
    if (rc < 0)
      return rc;
  }

  cmd.id = CXL_MEM_COMMAND_ID_IDENTIFY_SWITCH_DEVICE;
  cmd.out.size = sizeof(switch_identify);
  cmd.out.payload = (__u64)&switch_identify;

  rc = ioctl(fd, CXL_MEM_SEND_COMMAND, &cmd);
  if (rc) {
    printf("rc %d\n", rc);
    if (rc < 0)
      return rc;
  }

  printf("Switch indent ingress=%#x #ports=%d\n",
	 switch_identify.ingress_port_id,
	 switch_identify.num_physical_ports);
  for (i = 0; i < sizeof(switch_identify.active_port_bm); i++) {
    int j;
    for (j = 0; j < 8; j++) {
      if (switch_identify.active_port_bm[i] & 1 << j) {
	printf("Port %x active\n", i * 8 + j);
      }
    }
  }

  tun_cmd_size = sizeof(*tunnel_req);
  tunnel_req = malloc(tun_cmd_size);
  tun_resp_size = sizeof(*tunnel_resp) + sizeof(struct cxl_cmd_infostat_identify_rsp);
  tunnel_resp = malloc(tun_resp_size);
  tunnel_req->target_type = TUNNEL_TARGET_TYPE_PORT_OR_LD;
  tunnel_req->command_size = sizeof(tunnel_req->message);
  tunnel_req->message.command = 0x01;
  tunnel_req->message.command_set = 0x0; // tunnels within tunnels
  
  cmd.id = CXL_MEM_COMMAND_ID_TUNNEL_MANAGEMENT_COMMAND;
  cmd.in.payload = tunnel_req;
  cmd.in.size = tun_cmd_size;
  cmd.out.size = tun_resp_size;
  cmd.out.payload = tunnel_resp;

  rc = ioctl(fd, CXL_MEM_SEND_COMMAND, &cmd);
  if (rc) {
    printf("rc from tunnel cmd is %d\n", rc);
    if (rc < 0)
      return rc;
  }
  printf("message response size %d\n", cmd.out.size);
  {
    struct cxl_cmd_infostat_identify_rsp *pl = (void *)tunnel_resp->message.payload;
  	printf("Vendor id  : %04x\n", pl->pcie_vid);
	printf("Device id  : %04x\n", pl->pcie_did);
	printf("Subsys vid : %04x\n", pl->pcie_subsys_vid);
	printf("Subsys id  : %04x\n", pl->pcie_subsys_id);

	switch (pl->component_type) {
	case 0x00:
		printf("Switch!\n");
		break;
	case 0x03:
		printf("Type3!\n");
		break;
	}
  }  
  return 0;	 
}

Jonathan Cameron (4):
  cxl: mbox: Preparatory move of functions to core/mbox.c and cxlmbox.h
  cxl: mbox: Factor out the mbox specific data for reuse in switch cci
  PCI: Add PCI_CLASS_SERIAL_CXL_SWITCH_CCI class ID to pci_ids.h
  cxl/pci: Add support for stand alone CXL Switch mailbox CCI

 drivers/cxl/Kconfig          |  14 ++
 drivers/cxl/Makefile         |   2 +
 drivers/cxl/core/Makefile    |   1 +
 drivers/cxl/core/core.h      |  12 +-
 drivers/cxl/core/mbox.c      | 455 +++++++++++++++++++++++++++--------
 drivers/cxl/core/memdev.c    |  44 ++--
 drivers/cxl/core/regs.c      |  35 ++-
 drivers/cxl/core/switchdev.c | 129 ++++++++++
 drivers/cxl/cxl.h            |   4 +-
 drivers/cxl/cxlmbox.h        | 203 ++++++++++++++++
 drivers/cxl/cxlmem.h         | 176 ++------------
 drivers/cxl/pci.c            | 436 ++++++++++-----------------------
 drivers/cxl/pmem.c           |   6 +-
 drivers/cxl/security.c       |  13 +-
 drivers/cxl/switch.h         |  19 ++
 drivers/cxl/switchdev.c      | 170 +++++++++++++
 include/linux/pci_ids.h      |   1 +
 include/uapi/linux/cxl_mem.h |   4 +
 18 files changed, 1128 insertions(+), 596 deletions(-)
 create mode 100644 drivers/cxl/core/switchdev.c
 create mode 100644 drivers/cxl/cxlmbox.h
 create mode 100644 drivers/cxl/switch.h
 create mode 100644 drivers/cxl/switchdev.c

-- 
2.39.2