virtio-pci: Introduce admin virtqueue

Introduce support for the admin virtqueue. By negotiating
VIRTIO_F_ADMIN_VQ feature, driver detects capability and creates one
administration virtqueue. Administration virtqueue implementation in
virtio pci generic layer, enables multiple types of upper layer
drivers such as vfio, net, blk to utilize it.

Signed-off-by: Feng Liu <feliu@nvidia.com>
Reviewed-by: Parav Pandit <parav@nvidia.com>
Reviewed-by: Jiri Pirko <jiri@nvidia.com>
Acked-by: Michael S. Tsirkin <mst@redhat.com>
Signed-off-by: Yishai Hadas <yishaih@nvidia.com>
Link: https://lore.kernel.org/r/20231219093247.170936-3-yishaih@nvidia.com
Signed-off-by: Alex Williamson <alex.williamson@redhat.com>

diff --git a/drivers/virtio/virtio.c b/drivers/virtio/virtio.c
index 3893dc29eb26338f036379c5664e249ad18f1df6..f4080692b3513bd67a6d946c355c2cfbd7496ecd 100644 (file)
--- a/drivers/virtio/virtio.c
+++ b/drivers/virtio/virtio.c
@@ -302,9 +302,15 @@ static int virtio_dev_probe(struct device *_d)
        if (err)
                goto err;
 
+       if (dev->config->create_avq) {
+               err = dev->config->create_avq(dev);
+               if (err)
+                       goto err;
+       }
+
        err = drv->probe(dev);
        if (err)
-               goto err;
+               goto err_probe;
 
        /* If probe didn't do it, mark device DRIVER_OK ourselves. */
        if (!(dev->config->get_status(dev) & VIRTIO_CONFIG_S_DRIVER_OK))
@@ -316,6 +322,10 @@ static int virtio_dev_probe(struct device *_d)
        virtio_config_enable(dev);
 
        return 0;
+
+err_probe:
+       if (dev->config->destroy_avq)
+               dev->config->destroy_avq(dev);
 err:
        virtio_add_status(dev, VIRTIO_CONFIG_S_FAILED);
        return err;
@@ -331,6 +341,9 @@ static void virtio_dev_remove(struct device *_d)
 
        drv->remove(dev);
 
+       if (dev->config->destroy_avq)
+               dev->config->destroy_avq(dev);
+
        /* Driver should have reset device. */
        WARN_ON_ONCE(dev->config->get_status(dev));
 
@@ -489,13 +502,20 @@ EXPORT_SYMBOL_GPL(unregister_virtio_device);
 int virtio_device_freeze(struct virtio_device *dev)
 {
        struct virtio_driver *drv = drv_to_virtio(dev->dev.driver);
+       int ret;
 
        virtio_config_disable(dev);
 
        dev->failed = dev->config->get_status(dev) & VIRTIO_CONFIG_S_FAILED;
 
-       if (drv && drv->freeze)
-               return drv->freeze(dev);
+       if (drv && drv->freeze) {
+               ret = drv->freeze(dev);
+               if (ret)
+                       return ret;
+       }
+
+       if (dev->config->destroy_avq)
+               dev->config->destroy_avq(dev);
 
        return 0;
 }
@@ -532,10 +552,16 @@ int virtio_device_restore(struct virtio_device *dev)
        if (ret)
                goto err;
 
+       if (dev->config->create_avq) {
+               ret = dev->config->create_avq(dev);
+               if (ret)
+                       goto err;
+       }
+
        if (drv->restore) {
                ret = drv->restore(dev);
                if (ret)
-                       goto err;
+                       goto err_restore;
        }
 
        /* If restore didn't do it, mark device DRIVER_OK ourselves. */
@@ -546,6 +572,9 @@ int virtio_device_restore(struct virtio_device *dev)
 
        return 0;
 
+err_restore:
+       if (dev->config->destroy_avq)
+               dev->config->destroy_avq(dev);
 err:
        virtio_add_status(dev, VIRTIO_CONFIG_S_FAILED);
        return ret;

diff --git a/drivers/virtio/virtio_pci_common.c b/drivers/virtio/virtio_pci_common.c
index 7a5593997e0efe64f2ce832bf0c8676bbaabf188..fafd13d0e4d46f4583934cc24b65f548bc852b35 100644 (file)
--- a/drivers/virtio/virtio_pci_common.c
+++ b/drivers/virtio/virtio_pci_common.c
@@ -236,6 +236,9 @@ void vp_del_vqs(struct virtio_device *vdev)
        int i;
 
        list_for_each_entry_safe(vq, n, &vdev->vqs, list) {
+               if (vp_dev->is_avq(vdev, vq->index))
+                       continue;
+
                if (vp_dev->per_vq_vectors) {
                        int v = vp_dev->vqs[vq->index]->msix_vector;
 

diff --git a/drivers/virtio/virtio_pci_common.h b/drivers/virtio/virtio_pci_common.h
index 4b773bd7c58cb7e42726127cdd188a181c8c04db..7306128e63e98450c7d4d01feba517b1da2f3b1e 100644 (file)
--- a/drivers/virtio/virtio_pci_common.h
+++ b/drivers/virtio/virtio_pci_common.h
@@ -41,6 +41,14 @@ struct virtio_pci_vq_info {
        unsigned int msix_vector;
 };
 
+struct virtio_pci_admin_vq {
+       /* Virtqueue info associated with this admin queue. */
+       struct virtio_pci_vq_info info;
+       /* Name of the admin queue: avq.$vq_index. */
+       char name[10];
+       u16 vq_index;
+};
+
 /* Our device structure */
 struct virtio_pci_device {
        struct virtio_device vdev;
@@ -58,9 +66,13 @@ struct virtio_pci_device {
        spinlock_t lock;
        struct list_head virtqueues;
 
-       /* array of all queues for house-keeping */
+       /* Array of all virtqueues reported in the
+        * PCI common config num_queues field
+        */
        struct virtio_pci_vq_info **vqs;
 
+       struct virtio_pci_admin_vq admin_vq;
+
        /* MSI-X support */
        int msix_enabled;
        int intx_enabled;
@@ -86,6 +98,7 @@ struct virtio_pci_device {
        void (*del_vq)(struct virtio_pci_vq_info *info);
 
        u16 (*config_vector)(struct virtio_pci_device *vp_dev, u16 vector);
+       bool (*is_avq)(struct virtio_device *vdev, unsigned int index);
 };
 
 /* Constants for MSI-X */

diff --git a/drivers/virtio/virtio_pci_modern.c b/drivers/virtio/virtio_pci_modern.c
index ee6a386d250b168bdd59153a62ddceb361c0af93..ce915018b5b0ffe773eb40b95e652d72ec418c43 100644 (file)
--- a/drivers/virtio/virtio_pci_modern.c
+++ b/drivers/virtio/virtio_pci_modern.c
@@ -19,6 +19,8 @@
 #define VIRTIO_RING_NO_LEGACY
 #include "virtio_pci_common.h"
 
+#define VIRTIO_AVQ_SGS_MAX     4
+
 static u64 vp_get_features(struct virtio_device *vdev)
 {
        struct virtio_pci_device *vp_dev = to_vp_device(vdev);
@@ -26,6 +28,16 @@ static u64 vp_get_features(struct virtio_device *vdev)
        return vp_modern_get_features(&vp_dev->mdev);
 }
 
+static bool vp_is_avq(struct virtio_device *vdev, unsigned int index)
+{
+       struct virtio_pci_device *vp_dev = to_vp_device(vdev);
+
+       if (!virtio_has_feature(vdev, VIRTIO_F_ADMIN_VQ))
+               return false;
+
+       return index == vp_dev->admin_vq.vq_index;
+}
+
 static void vp_transport_features(struct virtio_device *vdev, u64 features)
 {
        struct virtio_pci_device *vp_dev = to_vp_device(vdev);
@@ -37,6 +49,9 @@ static void vp_transport_features(struct virtio_device *vdev, u64 features)
 
        if (features & BIT_ULL(VIRTIO_F_RING_RESET))
                __virtio_set_bit(vdev, VIRTIO_F_RING_RESET);
+
+       if (features & BIT_ULL(VIRTIO_F_ADMIN_VQ))
+               __virtio_set_bit(vdev, VIRTIO_F_ADMIN_VQ);
 }
 
 static int __vp_check_common_size_one_feature(struct virtio_device *vdev, u32 fbit,
@@ -69,6 +84,9 @@ static int vp_check_common_size(struct virtio_device *vdev)
        if (vp_check_common_size_one_feature(vdev, VIRTIO_F_RING_RESET, queue_reset))
                return -EINVAL;
 
+       if (vp_check_common_size_one_feature(vdev, VIRTIO_F_ADMIN_VQ, admin_queue_num))
+               return -EINVAL;
+
        return 0;
 }
 
@@ -345,6 +363,7 @@ static struct virtqueue *setup_vq(struct virtio_pci_device *vp_dev,
        struct virtio_pci_modern_device *mdev = &vp_dev->mdev;
        bool (*notify)(struct virtqueue *vq);
        struct virtqueue *vq;
+       bool is_avq;
        u16 num;
        int err;
 
@@ -353,11 +372,13 @@ static struct virtqueue *setup_vq(struct virtio_pci_device *vp_dev,
        else
                notify = vp_notify;
 
-       if (index >= vp_modern_get_num_queues(mdev))
+       is_avq = vp_is_avq(&vp_dev->vdev, index);
+       if (index >= vp_modern_get_num_queues(mdev) && !is_avq)
                return ERR_PTR(-EINVAL);
 
+       num = is_avq ?
+               VIRTIO_AVQ_SGS_MAX : vp_modern_get_queue_size(mdev, index);
        /* Check if queue is either not available or already active. */
-       num = vp_modern_get_queue_size(mdev, index);
        if (!num || vp_modern_get_queue_enable(mdev, index))
                return ERR_PTR(-ENOENT);
 
@@ -383,6 +404,9 @@ static struct virtqueue *setup_vq(struct virtio_pci_device *vp_dev,
                goto err;
        }
 
+       if (is_avq)
+               vp_dev->admin_vq.info.vq = vq;
+
        return vq;
 
 err:
@@ -418,6 +442,9 @@ static void del_vq(struct virtio_pci_vq_info *info)
        struct virtio_pci_device *vp_dev = to_vp_device(vq->vdev);
        struct virtio_pci_modern_device *mdev = &vp_dev->mdev;
 
+       if (vp_is_avq(&vp_dev->vdev, vq->index))
+               vp_dev->admin_vq.info.vq = NULL;
+
        if (vp_dev->msix_enabled)
                vp_modern_queue_vector(mdev, vq->index,
                                       VIRTIO_MSI_NO_VECTOR);
@@ -527,6 +554,45 @@ static bool vp_get_shm_region(struct virtio_device *vdev,
        return true;
 }
 
+static int vp_modern_create_avq(struct virtio_device *vdev)
+{
+       struct virtio_pci_device *vp_dev = to_vp_device(vdev);
+       struct virtio_pci_admin_vq *avq;
+       struct virtqueue *vq;
+       u16 admin_q_num;
+
+       if (!virtio_has_feature(vdev, VIRTIO_F_ADMIN_VQ))
+               return 0;
+
+       admin_q_num = vp_modern_avq_num(&vp_dev->mdev);
+       if (!admin_q_num)
+               return -EINVAL;
+
+       avq = &vp_dev->admin_vq;
+       avq->vq_index = vp_modern_avq_index(&vp_dev->mdev);
+       sprintf(avq->name, "avq.%u", avq->vq_index);
+       vq = vp_dev->setup_vq(vp_dev, &vp_dev->admin_vq.info, avq->vq_index, NULL,
+                             avq->name, NULL, VIRTIO_MSI_NO_VECTOR);
+       if (IS_ERR(vq)) {
+               dev_err(&vdev->dev, "failed to setup admin virtqueue, err=%ld",
+                       PTR_ERR(vq));
+               return PTR_ERR(vq);
+       }
+
+       vp_modern_set_queue_enable(&vp_dev->mdev, avq->info.vq->index, true);
+       return 0;
+}
+
+static void vp_modern_destroy_avq(struct virtio_device *vdev)
+{
+       struct virtio_pci_device *vp_dev = to_vp_device(vdev);
+
+       if (!virtio_has_feature(vdev, VIRTIO_F_ADMIN_VQ))
+               return;
+
+       vp_dev->del_vq(&vp_dev->admin_vq.info);
+}
+
 static const struct virtio_config_ops virtio_pci_config_nodev_ops = {
        .get            = NULL,
        .set            = NULL,
@@ -545,6 +611,8 @@ static const struct virtio_config_ops virtio_pci_config_nodev_ops = {
        .get_shm_region  = vp_get_shm_region,
        .disable_vq_and_reset = vp_modern_disable_vq_and_reset,
        .enable_vq_after_reset = vp_modern_enable_vq_after_reset,
+       .create_avq = vp_modern_create_avq,
+       .destroy_avq = vp_modern_destroy_avq,
 };
 
 static const struct virtio_config_ops virtio_pci_config_ops = {
@@ -565,6 +633,8 @@ static const struct virtio_config_ops virtio_pci_config_ops = {
        .get_shm_region  = vp_get_shm_region,
        .disable_vq_and_reset = vp_modern_disable_vq_and_reset,
        .enable_vq_after_reset = vp_modern_enable_vq_after_reset,
+       .create_avq = vp_modern_create_avq,
+       .destroy_avq = vp_modern_destroy_avq,
 };
 
 /* the PCI probing function */
@@ -588,6 +658,7 @@ int virtio_pci_modern_probe(struct virtio_pci_device *vp_dev)
        vp_dev->config_vector = vp_config_vector;
        vp_dev->setup_vq = setup_vq;
        vp_dev->del_vq = del_vq;
+       vp_dev->is_avq = vp_is_avq;
        vp_dev->isr = mdev->isr;
        vp_dev->vdev.id = mdev->id;
 

diff --git a/drivers/virtio/virtio_pci_modern_dev.c b/drivers/virtio/virtio_pci_modern_dev.c
index 7de8b1ebabac4217b2240f6d8faaf486b4265f38..0d3dbfaf4b236910530b967a3fb9de25b68bcd36 100644 (file)
--- a/drivers/virtio/virtio_pci_modern_dev.c
+++ b/drivers/virtio/virtio_pci_modern_dev.c
@@ -207,6 +207,10 @@ static inline void check_offsets(void)
                     offsetof(struct virtio_pci_modern_common_cfg, queue_notify_data));
        BUILD_BUG_ON(VIRTIO_PCI_COMMON_Q_RESET !=
                     offsetof(struct virtio_pci_modern_common_cfg, queue_reset));
+       BUILD_BUG_ON(VIRTIO_PCI_COMMON_ADM_Q_IDX !=
+                    offsetof(struct virtio_pci_modern_common_cfg, admin_queue_index));
+       BUILD_BUG_ON(VIRTIO_PCI_COMMON_ADM_Q_NUM !=
+                    offsetof(struct virtio_pci_modern_common_cfg, admin_queue_num));
 }
 
 /*
@@ -296,7 +300,7 @@ int vp_modern_probe(struct virtio_pci_modern_device *mdev)
        mdev->common = vp_modern_map_capability(mdev, common,
                              sizeof(struct virtio_pci_common_cfg), 4, 0,
                              offsetofend(struct virtio_pci_modern_common_cfg,
-                                         queue_reset),
+                                         admin_queue_num),
                              &mdev->common_len, NULL);
        if (!mdev->common)
                goto err_map_common;
@@ -719,6 +723,24 @@ void __iomem *vp_modern_map_vq_notify(struct virtio_pci_modern_device *mdev,
 }
 EXPORT_SYMBOL_GPL(vp_modern_map_vq_notify);
 
+u16 vp_modern_avq_num(struct virtio_pci_modern_device *mdev)
+{
+       struct virtio_pci_modern_common_cfg __iomem *cfg;
+
+       cfg = (struct virtio_pci_modern_common_cfg __iomem *)mdev->common;
+       return vp_ioread16(&cfg->admin_queue_num);
+}
+EXPORT_SYMBOL_GPL(vp_modern_avq_num);
+
+u16 vp_modern_avq_index(struct virtio_pci_modern_device *mdev)
+{
+       struct virtio_pci_modern_common_cfg __iomem *cfg;
+
+       cfg = (struct virtio_pci_modern_common_cfg __iomem *)mdev->common;
+       return vp_ioread16(&cfg->admin_queue_index);
+}
+EXPORT_SYMBOL_GPL(vp_modern_avq_index);
+
 MODULE_VERSION("0.1");
 MODULE_DESCRIPTION("Modern Virtio PCI Device");
 MODULE_AUTHOR("Jason Wang <jasowang@redhat.com>");

diff --git a/include/linux/virtio_config.h b/include/linux/virtio_config.h
index 2b3438de2c4d4e887bdfbf89f5c137f3218b00e9..da9b271b54db8a82fce123befe529c1851bc5b7a 100644 (file)
--- a/include/linux/virtio_config.h
+++ b/include/linux/virtio_config.h
@@ -93,6 +93,8 @@ typedef void vq_callback_t(struct virtqueue *);
  *     Returns 0 on success or error status
  *     If disable_vq_and_reset is set, then enable_vq_after_reset must also be
  *     set.
+ * @create_avq: create admin virtqueue resource.
+ * @destroy_avq: destroy admin virtqueue resource.
  */
 struct virtio_config_ops {
        void (*get)(struct virtio_device *vdev, unsigned offset,
@@ -120,6 +122,8 @@ struct virtio_config_ops {
                               struct virtio_shm_region *region, u8 id);
        int (*disable_vq_and_reset)(struct virtqueue *vq);
        int (*enable_vq_after_reset)(struct virtqueue *vq);
+       int (*create_avq)(struct virtio_device *vdev);
+       void (*destroy_avq)(struct virtio_device *vdev);
 };
 
 /* If driver didn't advertise the feature, it will never appear. */

diff --git a/include/linux/virtio_pci_modern.h b/include/linux/virtio_pci_modern.h
index a09e13a577a99a0f91916ad121329dde11f42b1f..c0b1b1ca1163507dddca0870a46c93194ae07e75 100644 (file)
--- a/include/linux/virtio_pci_modern.h
+++ b/include/linux/virtio_pci_modern.h
@@ -125,4 +125,6 @@ int vp_modern_probe(struct virtio_pci_modern_device *mdev);
 void vp_modern_remove(struct virtio_pci_modern_device *mdev);
 int vp_modern_get_queue_reset(struct virtio_pci_modern_device *mdev, u16 index);
 void vp_modern_set_queue_reset(struct virtio_pci_modern_device *mdev, u16 index);
+u16 vp_modern_avq_num(struct virtio_pci_modern_device *mdev);
+u16 vp_modern_avq_index(struct virtio_pci_modern_device *mdev);
 #endif

diff --git a/include/uapi/linux/virtio_pci.h b/include/uapi/linux/virtio_pci.h
index 44f4dd2add188090ff3b03d859fb4d27009d5479..240ddeef7eae38974abc080af061ea5f036e9f5d 100644 (file)
--- a/include/uapi/linux/virtio_pci.h
+++ b/include/uapi/linux/virtio_pci.h
@@ -175,6 +175,9 @@ struct virtio_pci_modern_common_cfg {
 
        __le16 queue_notify_data;       /* read-write */
        __le16 queue_reset;             /* read-write */
+
+       __le16 admin_queue_index;       /* read-only */
+       __le16 admin_queue_num;         /* read-only */
 };
 
 /* Fields in VIRTIO_PCI_CAP_PCI_CFG: */
@@ -215,6 +218,8 @@ struct virtio_pci_cfg_cap {
 #define VIRTIO_PCI_COMMON_Q_USEDHI     52
 #define VIRTIO_PCI_COMMON_Q_NDATA      56
 #define VIRTIO_PCI_COMMON_Q_RESET      58
+#define VIRTIO_PCI_COMMON_ADM_Q_IDX    60
+#define VIRTIO_PCI_COMMON_ADM_Q_NUM    62
 
 #endif /* VIRTIO_PCI_NO_MODERN */