net/mlx5: E-Switch, Load/unload VF reps according to event from host PF

When host PF changes the number of VFs, the ECPF esw driver will get
a FW event. It should query the number of VFs enabled by host PF and
update the VF reps accordingly. Note that host PF can't change the
number of VFs dynamically, it has to reset the number of VFs to 0
before changing to a new positive number.

The host event is registered when driver is moving to switchdev mode,
and it's the last step to do in esw_offloads_init. It's unregistered
and the work queue is flushed when driver quits from switchdev mode.
In this way, the host event and devlink command are serialized.

When driver is enabling switchdev mode, pay attention to the following
two facts:
1. Host PF must not have VF initialized as the flow table in ECPF has
   ENCAP enabled as default. Such flow table can't be created with
   existing initialized VFs.
2. ECPF doesn't know how many VFs the host PF will enable, ECPF
   offloads flow steering shall create the flow table/groups based on
   the max number of VFs possibly supported by host PF.

Signed-off-by: Bodong Wang <bodong@mellanox.com>
Reviewed-by: Or Gerlitz <ogerlitz@mellanox.com>
Signed-off-by: Saeed Mahameed <saeedm@mellanox.com>

diff --git a/drivers/net/ethernet/mellanox/mlx5/core/eswitch.c b/drivers/net/ethernet/mellanox/mlx5/core/eswitch.c
index d2ab1ee19b2a73d8edecdaa2b41f5c243b905172..e18af31336e6c458ceea77a715353447aedf5c6b 100644 (file)
--- a/drivers/net/ethernet/mellanox/mlx5/core/eswitch.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/eswitch.c
@@ -39,6 +39,7 @@
 #include "lib/eq.h"
 #include "eswitch.h"
 #include "fs_core.h"
+#include "ecpf.h"
 
 enum {
        MLX5_ACTION_NONE = 0,
@@ -1639,6 +1640,7 @@ static int eswitch_vport_event(struct notifier_block *nb,
 
 int mlx5_eswitch_enable_sriov(struct mlx5_eswitch *esw, int nvfs, int mode)
 {
+       int vf_nvports = 0, total_nvports = 0;
        struct mlx5_vport *vport;
        int err;
        int i, enabled_events;
@@ -1657,6 +1659,18 @@ int mlx5_eswitch_enable_sriov(struct mlx5_eswitch *esw, int nvfs, int mode)
 
        esw_info(esw->dev, "E-Switch enable SRIOV: nvfs(%d) mode (%d)\n", nvfs, mode);
 
+       if (mode == SRIOV_OFFLOADS) {
+               if (mlx5_core_is_ecpf_esw_manager(esw->dev)) {
+                       err = mlx5_query_host_params_num_vfs(esw->dev, &vf_nvports);
+                       if (err)
+                               return err;
+                       total_nvports = esw->total_vports;
+               } else {
+                       vf_nvports = nvfs;
+                       total_nvports = nvfs + MLX5_SPECIAL_VPORTS(esw->dev);
+               }
+       }
+
        esw->mode = mode;
 
        mlx5_lag_update(esw->dev);
@@ -1666,8 +1680,7 @@ int mlx5_eswitch_enable_sriov(struct mlx5_eswitch *esw, int nvfs, int mode)
        } else {
                mlx5_reload_interface(esw->dev, MLX5_INTERFACE_PROTOCOL_ETH);
                mlx5_reload_interface(esw->dev, MLX5_INTERFACE_PROTOCOL_IB);
-               err = esw_offloads_init(esw, nvfs,
-                                       nvfs + MLX5_SPECIAL_VPORTS(esw->dev));
+               err = esw_offloads_init(esw, vf_nvports, total_nvports);
        }
 
        if (err)

diff --git a/drivers/net/ethernet/mellanox/mlx5/core/eswitch.h b/drivers/net/ethernet/mellanox/mlx5/core/eswitch.h
index 2baa0d71380c19d2aedff9c668e8fef27d8b8ae0..af5581a57e56cb4a29188aac0d4ed9edea9408a9 100644 (file)
--- a/drivers/net/ethernet/mellanox/mlx5/core/eswitch.h
+++ b/drivers/net/ethernet/mellanox/mlx5/core/eswitch.h
@@ -182,6 +182,16 @@ struct esw_mc_addr { /* SRIOV only */
        u32                    refcnt;
 };
 
+struct mlx5_host_work {
+       struct work_struct      work;
+       struct mlx5_eswitch     *esw;
+};
+
+struct mlx5_host_info {
+       struct mlx5_nb          nb;
+       u16                     num_vfs;
+};
+
 struct mlx5_eswitch {
        struct mlx5_core_dev    *dev;
        struct mlx5_nb          nb;
@@ -206,6 +216,7 @@ struct mlx5_eswitch {
        int                     mode;
        int                     nvports;
        u16                     manager_vport;
+       struct mlx5_host_info   host_info;
 };
 
 void esw_offloads_cleanup(struct mlx5_eswitch *esw);

diff --git a/drivers/net/ethernet/mellanox/mlx5/core/eswitch_offloads.c b/drivers/net/ethernet/mellanox/mlx5/core/eswitch_offloads.c
index 84a33f8e3350db253bdefe976102b277c2908e48..91c4095ac79eee8c0835fe7c7d01306db95e53bb 100644 (file)
--- a/drivers/net/ethernet/mellanox/mlx5/core/eswitch_offloads.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/eswitch_offloads.c
@@ -40,6 +40,8 @@
 #include "en.h"
 #include "fs_core.h"
 #include "lib/devcom.h"
+#include "ecpf.h"
+#include "lib/eq.h"
 
 enum {
        FDB_FAST_PATH = 0,
@@ -1640,6 +1642,57 @@ static void esw_offloads_steering_cleanup(struct mlx5_eswitch *esw)
        esw_destroy_offloads_fdb_tables(esw);
 }
 
+static void esw_host_params_event_handler(struct work_struct *work)
+{
+       struct mlx5_host_work *host_work;
+       struct mlx5_eswitch *esw;
+       int err, num_vf = 0;
+
+       host_work = container_of(work, struct mlx5_host_work, work);
+       esw = host_work->esw;
+
+       err = mlx5_query_host_params_num_vfs(esw->dev, &num_vf);
+       if (err || num_vf == esw->host_info.num_vfs)
+               goto out;
+
+       /* Number of VFs can only change from "0 to x" or "x to 0". */
+       if (esw->host_info.num_vfs > 0) {
+               esw_offloads_unload_vf_reps(esw, esw->host_info.num_vfs);
+       } else {
+               err = esw_offloads_load_vf_reps(esw, num_vf);
+
+               if (err)
+                       goto out;
+       }
+
+       esw->host_info.num_vfs = num_vf;
+
+out:
+       kfree(host_work);
+}
+
+static int esw_host_params_event(struct notifier_block *nb,
+                                unsigned long type, void *data)
+{
+       struct mlx5_host_work *host_work;
+       struct mlx5_host_info *host_info;
+       struct mlx5_eswitch *esw;
+
+       host_work = kzalloc(sizeof(*host_work), GFP_ATOMIC);
+       if (!host_work)
+               return NOTIFY_DONE;
+
+       host_info = mlx5_nb_cof(nb, struct mlx5_host_info, nb);
+       esw = container_of(host_info, struct mlx5_eswitch, host_info);
+
+       host_work->esw = esw;
+
+       INIT_WORK(&host_work->work, esw_host_params_event_handler);
+       queue_work(esw->work_queue, &host_work->work);
+
+       return NOTIFY_OK;
+}
+
 int esw_offloads_init(struct mlx5_eswitch *esw, int vf_nvports,
                      int total_nvports)
 {
@@ -1656,6 +1709,14 @@ int esw_offloads_init(struct mlx5_eswitch *esw, int vf_nvports,
                goto err_reps;
 
        esw_offloads_devcom_init(esw);
+
+       if (mlx5_core_is_ecpf_esw_manager(esw->dev)) {
+               MLX5_NB_INIT(&esw->host_info.nb, esw_host_params_event,
+                            HOST_PARAMS_CHANGE);
+               mlx5_eq_notifier_register(esw->dev, &esw->host_info.nb);
+               esw->host_info.num_vfs = vf_nvports;
+       }
+
        return 0;
 
 err_reps:
@@ -1684,7 +1745,15 @@ static int esw_offloads_stop(struct mlx5_eswitch *esw,
 
 void esw_offloads_cleanup(struct mlx5_eswitch *esw)
 {
-       u16 num_vfs = esw->dev->priv.sriov.num_vfs;
+       u16 num_vfs;
+
+       if (mlx5_core_is_ecpf_esw_manager(esw->dev)) {
+               mlx5_eq_notifier_unregister(esw->dev, &esw->host_info.nb);
+               flush_workqueue(esw->work_queue);
+               num_vfs = esw->host_info.num_vfs;
+       } else {
+               num_vfs = esw->dev->priv.sriov.num_vfs;
+       }
 
        esw_offloads_devcom_cleanup(esw);
        esw_offloads_unload_all_reps(esw, num_vfs);