net: move netdev_max_backlog to net_hotdata

netdev_max_backlog is used in rx fat path.

Move it to net_hodata for better cache locality.

Signed-off-by: Eric Dumazet <edumazet@google.com>
Acked-by: Soheil Hassas Yeganeh <soheil@google.com>
Reviewed-by: David Ahern <dsahern@kernel.org>
Link: https://lore.kernel.org/r/20240306160031.874438-6-edumazet@google.com
Signed-off-by: Jakub Kicinski <kuba@kernel.org>

diff --git a/include/linux/netdevice.h b/include/linux/netdevice.h
index b18ac8072f1875ce3eb4b77e6d18df1ca563a0d6..c9a671b7bb37366f6ce3fbeef8848a23829c1caa 100644 (file)
--- a/include/linux/netdevice.h
+++ b/include/linux/netdevice.h
@@ -4793,7 +4793,6 @@ void dev_fetch_sw_netstats(struct rtnl_link_stats64 *s,
                           const struct pcpu_sw_netstats __percpu *netstats);
 void dev_get_tstats64(struct net_device *dev, struct rtnl_link_stats64 *s);
 
-extern int             netdev_max_backlog;
 extern int             dev_rx_weight;
 extern int             dev_tx_weight;
 

diff --git a/include/net/hotdata.h b/include/net/hotdata.h
index d462cb8f16bad459b439c566274c01a0f18a95d0..dc50b200a94b6b935cd79d8e0406a61209fdc68e 100644 (file)
--- a/include/net/hotdata.h
+++ b/include/net/hotdata.h
@@ -12,6 +12,7 @@ struct net_hotdata {
        int                     netdev_budget;
        int                     netdev_budget_usecs;
        int                     tstamp_prequeue;
+       int                     max_backlog;
 };
 
 extern struct net_hotdata net_hotdata;

diff --git a/net/core/dev.c b/net/core/dev.c
index e1493e5584070009104b15ee4f0c8e36bf61c60c..e23edbaff392f11811adf8f6e439e622ddfed54e 100644 (file)
--- a/net/core/dev.c
+++ b/net/core/dev.c
@@ -4404,9 +4404,6 @@ EXPORT_SYMBOL(__dev_direct_xmit);
  *                     Receiver routines
  *************************************************************************/
 
-int netdev_max_backlog __read_mostly = 1000;
-EXPORT_SYMBOL(netdev_max_backlog);
-
 unsigned int sysctl_skb_defer_max __read_mostly = 64;
 int weight_p __read_mostly = 64;           /* old backlog weight */
 int dev_weight_rx_bias __read_mostly = 1;  /* bias for backlog weight */
@@ -4713,7 +4710,7 @@ static bool skb_flow_limit(struct sk_buff *skb, unsigned int qlen)
        struct softnet_data *sd;
        unsigned int old_flow, new_flow;
 
-       if (qlen < (READ_ONCE(netdev_max_backlog) >> 1))
+       if (qlen < (READ_ONCE(net_hotdata.max_backlog) >> 1))
                return false;
 
        sd = this_cpu_ptr(&softnet_data);
@@ -4761,7 +4758,8 @@ static int enqueue_to_backlog(struct sk_buff *skb, int cpu,
        if (!netif_running(skb->dev))
                goto drop;
        qlen = skb_queue_len(&sd->input_pkt_queue);
-       if (qlen <= READ_ONCE(netdev_max_backlog) && !skb_flow_limit(skb, qlen)) {
+       if (qlen <= READ_ONCE(net_hotdata.max_backlog) &&
+           !skb_flow_limit(skb, qlen)) {
                if (qlen) {
 enqueue:
                        __skb_queue_tail(&sd->input_pkt_queue, skb);

diff --git a/net/core/gro_cells.c b/net/core/gro_cells.c
index ed5ec5de47f670753924bd0c72db1e3ceb9b9e7a..ff8e5b64bf6b76451a69e3eae132b593c60ee204 100644 (file)
--- a/net/core/gro_cells.c
+++ b/net/core/gro_cells.c
@@ -3,6 +3,7 @@
 #include <linux/slab.h>
 #include <linux/netdevice.h>
 #include <net/gro_cells.h>
+#include <net/hotdata.h>
 
 struct gro_cell {
        struct sk_buff_head     napi_skbs;
@@ -26,7 +27,7 @@ int gro_cells_receive(struct gro_cells *gcells, struct sk_buff *skb)
 
        cell = this_cpu_ptr(gcells->cells);
 
-       if (skb_queue_len(&cell->napi_skbs) > READ_ONCE(netdev_max_backlog)) {
+       if (skb_queue_len(&cell->napi_skbs) > READ_ONCE(net_hotdata.max_backlog)) {
 drop:
                dev_core_stats_rx_dropped_inc(dev);
                kfree_skb(skb);

diff --git a/net/core/hotdata.c b/net/core/hotdata.c
index 29fcfe89fd9a697120f826dbe2fd36a1617581a1..35ed5a83ecc7ebda513fe4fafc596e053f0252c5 100644 (file)
--- a/net/core/hotdata.c
+++ b/net/core/hotdata.c
@@ -15,4 +15,6 @@ struct net_hotdata net_hotdata __cacheline_aligned = {
        .netdev_budget_usecs = 2 * USEC_PER_SEC / HZ,
 
        .tstamp_prequeue = 1,
+       .max_backlog = 1000,
 };
+EXPORT_SYMBOL(net_hotdata);

diff --git a/net/core/sysctl_net_core.c b/net/core/sysctl_net_core.c
index bddd07da099886f2747f2ac4ba39a482b0f4231d..8eaeeb289914258f90cf940e906d5c6be0cc0cd6 100644 (file)
--- a/net/core/sysctl_net_core.c
+++ b/net/core/sysctl_net_core.c
@@ -440,7 +440,7 @@ static struct ctl_table net_core_table[] = {
        },
        {
                .procname       = "netdev_max_backlog",
-               .data           = &netdev_max_backlog,
+               .data           = &net_hotdata.max_backlog,
                .maxlen         = sizeof(int),
                .mode           = 0644,
                .proc_handler   = proc_dointvec

diff --git a/net/xfrm/espintcp.c b/net/xfrm/espintcp.c
index d3b3f9e720b3b6c2a4ea89df4257a564100b2c4b..fe82e2d073006e5ab1b03868c851147c0422d26d 100644 (file)
--- a/net/xfrm/espintcp.c
+++ b/net/xfrm/espintcp.c
@@ -10,6 +10,7 @@
 #if IS_ENABLED(CONFIG_IPV6)
 #include <net/ipv6_stubs.h>
 #endif
+#include <net/hotdata.h>
 
 static void handle_nonesp(struct espintcp_ctx *ctx, struct sk_buff *skb,
                          struct sock *sk)
@@ -169,7 +170,8 @@ int espintcp_queue_out(struct sock *sk, struct sk_buff *skb)
 {
        struct espintcp_ctx *ctx = espintcp_getctx(sk);
 
-       if (skb_queue_len(&ctx->out_queue) >= READ_ONCE(netdev_max_backlog))
+       if (skb_queue_len(&ctx->out_queue) >=
+           READ_ONCE(net_hotdata.max_backlog))
                return -ENOBUFS;
 
        __skb_queue_tail(&ctx->out_queue, skb);

diff --git a/net/xfrm/xfrm_input.c b/net/xfrm/xfrm_input.c
index bd4ce21d76d7551d8f0a4a11f4b75705a7f634c5..161f535c8b9495b01f6d9689e14c40e5c0885968 100644 (file)
--- a/net/xfrm/xfrm_input.c
+++ b/net/xfrm/xfrm_input.c
@@ -21,6 +21,7 @@
 #include <net/ip_tunnels.h>
 #include <net/ip6_tunnel.h>
 #include <net/dst_metadata.h>
+#include <net/hotdata.h>
 
 #include "xfrm_inout.h"
 
@@ -764,7 +765,7 @@ int xfrm_trans_queue_net(struct net *net, struct sk_buff *skb,
 
        trans = this_cpu_ptr(&xfrm_trans_tasklet);
 
-       if (skb_queue_len(&trans->queue) >= READ_ONCE(netdev_max_backlog))
+       if (skb_queue_len(&trans->queue) >= READ_ONCE(net_hotdata.max_backlog))
                return -ENOBUFS;
 
        BUILD_BUG_ON(sizeof(struct xfrm_trans_cb) > sizeof(skb->cb));