RDMA/efa: Limit EQs to available MSI-X vectors

When creating EQs we take into consideration the max number of EQs the
device reported it can support and the number of available CPUs. There
are situations where the number of EQs the device reported it can
support and the PCI configuration of MSI-X is different, take it in
account as well when creating EQs.
Also request at least 1 MSI-X vector for the management queue and allow
the kernel to return a number of vectors in a range between 1 and the
max supported MSI-X vectors according to the PCI config.

Reviewed-by: Michael Margolin <mrgolin@amazon.com>
Reviewed-by: Yonatan Goldhirsh <ygold@amazon.com>
Signed-off-by: Yonatan Nachum <ynachum@amazon.com>
Link: https://lore.kernel.org/r/20240131093403.18624-1-ynachum@amazon.com
Signed-off-by: Leon Romanovsky <leon@kernel.org>

diff --git a/drivers/infiniband/hw/efa/efa.h b/drivers/infiniband/hw/efa/efa.h
index e2bdec32ae80539011a88c41b1f5db1ba595a131..926f9ff1f60fda6df21e58f948b5243be7960c16 100644 (file)
--- a/drivers/infiniband/hw/efa/efa.h
+++ b/drivers/infiniband/hw/efa/efa.h
@@ -57,6 +57,7 @@ struct efa_dev {
        u64 db_bar_addr;
        u64 db_bar_len;
 
+       unsigned int num_irq_vectors;
        int admin_msix_vector_idx;
        struct efa_irq admin_irq;
 

diff --git a/drivers/infiniband/hw/efa/efa_main.c b/drivers/infiniband/hw/efa/efa_main.c
index 7b1910a862164599ef95bdd07f11868672440885..5fa3603c80d82f83416f9cc8f514ee32f841b38b 100644 (file)
--- a/drivers/infiniband/hw/efa/efa_main.c
+++ b/drivers/infiniband/hw/efa/efa_main.c
@@ -322,7 +322,9 @@ static int efa_create_eqs(struct efa_dev *dev)
        int err;
        int i;
 
-       neqs = min_t(unsigned int, neqs, num_online_cpus());
+       neqs = min_t(unsigned int, neqs,
+                    dev->num_irq_vectors - EFA_COMP_EQS_VEC_BASE);
+
        dev->neqs = neqs;
        dev->eqs = kcalloc(neqs, sizeof(*dev->eqs), GFP_KERNEL);
        if (!dev->eqs)
@@ -468,34 +470,30 @@ static void efa_disable_msix(struct efa_dev *dev)
 
 static int efa_enable_msix(struct efa_dev *dev)
 {
-       int msix_vecs, irq_num;
+       int max_vecs, num_vecs;
 
        /*
         * Reserve the max msix vectors we might need, one vector is reserved
         * for admin.
         */
-       msix_vecs = min_t(int, pci_msix_vec_count(dev->pdev),
-                         num_online_cpus() + 1);
+       max_vecs = min_t(int, pci_msix_vec_count(dev->pdev),
+                        num_online_cpus() + 1);
        dev_dbg(&dev->pdev->dev, "Trying to enable MSI-X, vectors %d\n",
-               msix_vecs);
+               max_vecs);
 
        dev->admin_msix_vector_idx = EFA_MGMNT_MSIX_VEC_IDX;
-       irq_num = pci_alloc_irq_vectors(dev->pdev, msix_vecs,
-                                       msix_vecs, PCI_IRQ_MSIX);
+       num_vecs = pci_alloc_irq_vectors(dev->pdev, 1,
+                                        max_vecs, PCI_IRQ_MSIX);
 
-       if (irq_num < 0) {
-               dev_err(&dev->pdev->dev, "Failed to enable MSI-X. irq_num %d\n",
-                       irq_num);
+       if (num_vecs < 0) {
+               dev_err(&dev->pdev->dev, "Failed to enable MSI-X. error %d\n",
+                       num_vecs);
                return -ENOSPC;
        }
 
-       if (irq_num != msix_vecs) {
-               efa_disable_msix(dev);
-               dev_err(&dev->pdev->dev,
-                       "Allocated %d MSI-X (out of %d requested)\n",
-                       irq_num, msix_vecs);
-               return -ENOSPC;
-       }
+       dev_dbg(&dev->pdev->dev, "Allocated %d MSI-X vectors\n", num_vecs);
+
+       dev->num_irq_vectors = num_vecs;
 
        return 0;
 }