cxl/pci: Create PCI DOE mailbox's for memory devices

DOE mailbox objects will be needed for various mailbox communications
with each memory device.

Iterate each DOE mailbox capability and create PCI DOE mailbox objects
as found.

It is not anticipated that this is the final resting place for the
iteration of the DOE devices.  The support of switch ports will drive
this code into the PCIe side.  In this imagined architecture the CXL
port driver would then query into the PCI device for the DOE mailbox
array.

For now creating the mailboxes in the CXL port is good enough for the
endpoints.  Later PCIe ports will need to support this to support switch
ports more generically.

Cc: Dan Williams <dan.j.williams@intel.com>
Cc: Davidlohr Bueso <dave@stgolabs.net>
Cc: Lukas Wunner <lukas@wunner.de>
Reviewed-by: Jonathan Cameron <Jonathan.Cameron@huawei.com>
Signed-off-by: Ira Weiny <ira.weiny@intel.com>
Link: https://lore.kernel.org/r/20220719205249.566684-5-ira.weiny@intel.com
Signed-off-by: Dan Williams <dan.j.williams@intel.com>

diff --git a/drivers/cxl/Kconfig b/drivers/cxl/Kconfig
index f64e3984689ffad92fcd68af4e0066ca09476604..7adaaf80b302aa08864fce15578909567130f89b 100644 (file)
--- a/drivers/cxl/Kconfig
+++ b/drivers/cxl/Kconfig
@@ -2,6 +2,7 @@
 menuconfig CXL_BUS
        tristate "CXL (Compute Express Link) Devices Support"
        depends on PCI
+       select PCI_DOE
        help
          CXL is a bus that is electrically compatible with PCI Express, but
          layers three protocols on that signalling (CXL.io, CXL.cache, and

diff --git a/drivers/cxl/cxlmem.h b/drivers/cxl/cxlmem.h
index c6d6f57856cc908df9e67824907a9c49aea1dcc5..bfa2eaf649a9ad6219ca34d9e4a868779d89b653 100644 (file)
--- a/drivers/cxl/cxlmem.h
+++ b/drivers/cxl/cxlmem.h
@@ -192,6 +192,7 @@ struct cxl_endpoint_dvsec_info {
  * @component_reg_phys: register base of component registers
  * @info: Cached DVSEC information about the device.
  * @serial: PCIe Device Serial Number
+ * @doe_mbs: PCI DOE mailbox array
  * @mbox_send: @dev specific transport for transmitting mailbox commands
  *
  * See section 8.2.9.5.2 Capacity Configuration and Label Storage for
@@ -226,6 +227,8 @@ struct cxl_dev_state {
        resource_size_t component_reg_phys;
        u64 serial;
 
+       struct xarray doe_mbs;
+
        int (*mbox_send)(struct cxl_dev_state *cxlds, struct cxl_mbox_cmd *cmd);
 };
 

diff --git a/drivers/cxl/pci.c b/drivers/cxl/pci.c
index eeff9599acda2b2c1b4f8e59c2371e04803bf338..faeb5d9d7a7ab3a753081872cbe1ec5c35b477be 100644 (file)
--- a/drivers/cxl/pci.c
+++ b/drivers/cxl/pci.c
@@ -8,6 +8,7 @@
 #include <linux/mutex.h>
 #include <linux/list.h>
 #include <linux/pci.h>
+#include <linux/pci-doe.h>
 #include <linux/io.h>
 #include "cxlmem.h"
 #include "cxlpci.h"
@@ -386,6 +387,47 @@ static int cxl_setup_regs(struct pci_dev *pdev, enum cxl_regloc_type type,
        return rc;
 }
 
+static void cxl_pci_destroy_doe(void *mbs)
+{
+       xa_destroy(mbs);
+}
+
+static void devm_cxl_pci_create_doe(struct cxl_dev_state *cxlds)
+{
+       struct device *dev = cxlds->dev;
+       struct pci_dev *pdev = to_pci_dev(dev);
+       u16 off = 0;
+
+       xa_init(&cxlds->doe_mbs);
+       if (devm_add_action(&pdev->dev, cxl_pci_destroy_doe, &cxlds->doe_mbs)) {
+               dev_err(dev, "Failed to create XArray for DOE's\n");
+               return;
+       }
+
+       /*
+        * Mailbox creation is best effort.  Higher layers must determine if
+        * the lack of a mailbox for their protocol is a device failure or not.
+        */
+       pci_doe_for_each_off(pdev, off) {
+               struct pci_doe_mb *doe_mb;
+
+               doe_mb = pcim_doe_create_mb(pdev, off);
+               if (IS_ERR(doe_mb)) {
+                       dev_err(dev, "Failed to create MB object for MB @ %x\n",
+                               off);
+                       continue;
+               }
+
+               if (xa_insert(&cxlds->doe_mbs, off, doe_mb, GFP_KERNEL)) {
+                       dev_err(dev, "xa_insert failed to insert MB @ %x\n",
+                               off);
+                       continue;
+               }
+
+               dev_dbg(dev, "Created DOE mailbox @%x\n", off);
+       }
+}
+
 static int cxl_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id)
 {
        struct cxl_register_map map;
@@ -434,6 +476,8 @@ static int cxl_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id)
 
        cxlds->component_reg_phys = cxl_regmap_to_base(pdev, &map);
 
+       devm_cxl_pci_create_doe(cxlds);
+
        rc = cxl_pci_setup_mailbox(cxlds);
        if (rc)
                return rc;