ibmvnic: Add hotpluggable CPU callbacks to reassign affinity hints

When CPU's are added and removed, ibmvnic devices will reassign
hint values. Introduce a new cpu hotplug state CPUHP_IBMVNIC_DEAD
to signal to ibmvnic devices that the CPU has been removed and it
is time to reset affinity hint assignments. On the other hand,
when CPU's are being added, add a state instance to
CPUHP_AP_ONLINE_DYN which will trigger a reassignment of affinity
hints once the new CPU's are online. This implementation is based
on the virtio_net driver.

Signed-off-by: Thomas Falcon <tlfalcon@linux.ibm.com>
Signed-off-by: Dany Madden <drt@linux.ibm.com>
Signed-off-by: Nick Child <nnac123@linux.ibm.com>
Reviewed-by: Rick Lindsley <ricklind@linux.ibm.com>
Reviewed-by: Haren Myneni <haren@linux.ibm.com>
Signed-off-by: David S. Miller <davem@davemloft.net>

diff --git a/drivers/net/ethernet/ibm/ibmvnic.c b/drivers/net/ethernet/ibm/ibmvnic.c
index 0c969bdaf94d0585b37be3e79c4fe9799e6fb1f3..2fc0d50dbb86d849bdf795eff7104306c2c7a039 100644 (file)
--- a/drivers/net/ethernet/ibm/ibmvnic.c
+++ b/drivers/net/ethernet/ibm/ibmvnic.c
@@ -298,6 +298,57 @@ out:
        }
 }
 
+static int ibmvnic_cpu_online(unsigned int cpu, struct hlist_node *node)
+{
+       struct ibmvnic_adapter *adapter;
+
+       adapter = hlist_entry_safe(node, struct ibmvnic_adapter, node);
+       ibmvnic_set_affinity(adapter);
+       return 0;
+}
+
+static int ibmvnic_cpu_dead(unsigned int cpu, struct hlist_node *node)
+{
+       struct ibmvnic_adapter *adapter;
+
+       adapter = hlist_entry_safe(node, struct ibmvnic_adapter, node_dead);
+       ibmvnic_set_affinity(adapter);
+       return 0;
+}
+
+static int ibmvnic_cpu_down_prep(unsigned int cpu, struct hlist_node *node)
+{
+       struct ibmvnic_adapter *adapter;
+
+       adapter = hlist_entry_safe(node, struct ibmvnic_adapter, node);
+       ibmvnic_clean_affinity(adapter);
+       return 0;
+}
+
+static enum cpuhp_state ibmvnic_online;
+
+static int ibmvnic_cpu_notif_add(struct ibmvnic_adapter *adapter)
+{
+       int ret;
+
+       ret = cpuhp_state_add_instance_nocalls(ibmvnic_online, &adapter->node);
+       if (ret)
+               return ret;
+       ret = cpuhp_state_add_instance_nocalls(CPUHP_IBMVNIC_DEAD,
+                                              &adapter->node_dead);
+       if (!ret)
+               return ret;
+       cpuhp_state_remove_instance_nocalls(ibmvnic_online, &adapter->node);
+       return ret;
+}
+
+static void ibmvnic_cpu_notif_remove(struct ibmvnic_adapter *adapter)
+{
+       cpuhp_state_remove_instance_nocalls(ibmvnic_online, &adapter->node);
+       cpuhp_state_remove_instance_nocalls(CPUHP_IBMVNIC_DEAD,
+                                           &adapter->node_dead);
+}
+
 static long h_reg_sub_crq(unsigned long unit_address, unsigned long token,
                          unsigned long length, unsigned long *number,
                          unsigned long *irq)
@@ -6292,10 +6343,19 @@ static int ibmvnic_probe(struct vio_dev *dev, const struct vio_device_id *id)
        }
        dev_info(&dev->dev, "ibmvnic registered\n");
 
+       rc = ibmvnic_cpu_notif_add(adapter);
+       if (rc) {
+               netdev_err(netdev, "Registering cpu notifier failed\n");
+               goto cpu_notif_add_failed;
+       }
+
        complete(&adapter->probe_done);
 
        return 0;
 
+cpu_notif_add_failed:
+       unregister_netdev(netdev);
+
 ibmvnic_register_fail:
        device_remove_file(&dev->dev, &dev_attr_failover);
 
@@ -6346,6 +6406,8 @@ static void ibmvnic_remove(struct vio_dev *dev)
 
        spin_unlock_irqrestore(&adapter->state_lock, flags);
 
+       ibmvnic_cpu_notif_remove(adapter);
+
        flush_work(&adapter->ibmvnic_reset);
        flush_delayed_work(&adapter->ibmvnic_delayed_reset);
 
@@ -6476,15 +6538,40 @@ static struct vio_driver ibmvnic_driver = {
 /* module functions */
 static int __init ibmvnic_module_init(void)
 {
+       int ret;
+
+       ret = cpuhp_setup_state_multi(CPUHP_AP_ONLINE_DYN, "net/ibmvnic:online",
+                                     ibmvnic_cpu_online,
+                                     ibmvnic_cpu_down_prep);
+       if (ret < 0)
+               goto out;
+       ibmvnic_online = ret;
+       ret = cpuhp_setup_state_multi(CPUHP_IBMVNIC_DEAD, "net/ibmvnic:dead",
+                                     NULL, ibmvnic_cpu_dead);
+       if (ret)
+               goto err_dead;
+
+       ret = vio_register_driver(&ibmvnic_driver);
+       if (ret)
+               goto err_vio_register;
+
        pr_info("%s: %s %s\n", ibmvnic_driver_name, ibmvnic_driver_string,
                IBMVNIC_DRIVER_VERSION);
 
-       return vio_register_driver(&ibmvnic_driver);
+       return 0;
+err_vio_register:
+       cpuhp_remove_multi_state(CPUHP_IBMVNIC_DEAD);
+err_dead:
+       cpuhp_remove_multi_state(ibmvnic_online);
+out:
+       return ret;
 }
 
 static void __exit ibmvnic_module_exit(void)
 {
        vio_unregister_driver(&ibmvnic_driver);
+       cpuhp_remove_multi_state(CPUHP_IBMVNIC_DEAD);
+       cpuhp_remove_multi_state(ibmvnic_online);
 }
 
 module_init(ibmvnic_module_init);

diff --git a/drivers/net/ethernet/ibm/ibmvnic.h b/drivers/net/ethernet/ibm/ibmvnic.h
index 6720fec1ae673ef2a5b90caa1db150633996e854..b35c9b6f913b663f0aec3a0fc6c8d1fa6dc4cd87 100644 (file)
--- a/drivers/net/ethernet/ibm/ibmvnic.h
+++ b/drivers/net/ethernet/ibm/ibmvnic.h
@@ -984,6 +984,10 @@ struct ibmvnic_adapter {
        int reset_done_rc;
        bool wait_for_reset;
 
+       /* CPU hotplug instances for online & dead */
+       struct hlist_node node;
+       struct hlist_node node_dead;
+
        /* partner capabilities */
        u64 min_tx_queues;
        u64 min_rx_queues;

diff --git a/include/linux/cpuhotplug.h b/include/linux/cpuhotplug.h
index f61447913db97e5461a85b8daec0ef0e6fe4cc93..c8bc85a87b1edfe6187e5e1eb0f10063f0fd69fd 100644 (file)
--- a/include/linux/cpuhotplug.h
+++ b/include/linux/cpuhotplug.h
@@ -69,6 +69,7 @@ enum cpuhp_state {
        CPUHP_X86_APB_DEAD,
        CPUHP_X86_MCE_DEAD,
        CPUHP_VIRT_NET_DEAD,
+       CPUHP_IBMVNIC_DEAD,
        CPUHP_SLUB_DEAD,
        CPUHP_DEBUG_OBJ_DEAD,
        CPUHP_MM_WRITEBACK_DEAD,