net: rps: add rps_input_queue_head_add() helper

process_backlog() can batch increments of sd->input_queue_head,
saving some memory bandwidth.

Also add READ_ONCE()/WRITE_ONCE() annotations around
sd->input_queue_head accesses.

Signed-off-by: Eric Dumazet <edumazet@google.com>
Signed-off-by: David S. Miller <davem@davemloft.net>

diff --git a/include/net/rps.h b/include/net/rps.h
index 10ca25731c1ef766715fe7ee415ad0b71ec643a8..a93401d23d66e45210acc73f0326087813b69d59 100644 (file)
--- a/include/net/rps.h
+++ b/include/net/rps.h
@@ -138,11 +138,16 @@ static inline void rps_input_queue_tail_save(u32 *dest, u32 tail)
 #endif
 }
 
-static inline void rps_input_queue_head_incr(struct softnet_data *sd)
+static inline void rps_input_queue_head_add(struct softnet_data *sd, int val)
 {
 #ifdef CONFIG_RPS
-       sd->input_queue_head++;
+       WRITE_ONCE(sd->input_queue_head, sd->input_queue_head + val);
 #endif
 }
 
+static inline void rps_input_queue_head_incr(struct softnet_data *sd)
+{
+       rps_input_queue_head_add(sd, 1);
+}
+
 #endif /* _NET_RPS_H */

diff --git a/net/core/dev.c b/net/core/dev.c
index 79073bbc9a644049cacf8433310f4641745049e9..818699dea9d7040ee74532ccdebf01c4fd6887cc 100644 (file)
--- a/net/core/dev.c
+++ b/net/core/dev.c
@@ -4528,7 +4528,7 @@ set_rps_cpu(struct net_device *dev, struct sk_buff *skb,
        out:
 #endif
                rflow->last_qtail =
-                       per_cpu(softnet_data, next_cpu).input_queue_head;
+                       READ_ONCE(per_cpu(softnet_data, next_cpu).input_queue_head);
        }
 
        rflow->cpu = next_cpu;
@@ -4610,7 +4610,7 @@ static int get_rps_cpu(struct net_device *dev, struct sk_buff *skb,
                 */
                if (unlikely(tcpu != next_cpu) &&
                    (tcpu >= nr_cpu_ids || !cpu_online(tcpu) ||
-                    ((int)(per_cpu(softnet_data, tcpu).input_queue_head -
+                    ((int)(READ_ONCE(per_cpu(softnet_data, tcpu).input_queue_head) -
                      READ_ONCE(rflow->last_qtail))) >= 0)) {
                        tcpu = next_cpu;
                        rflow = set_rps_cpu(dev, skb, rflow, next_cpu);
@@ -4665,7 +4665,7 @@ bool rps_may_expire_flow(struct net_device *dev, u16 rxq_index,
                rflow = &flow_table->flows[flow_id];
                cpu = READ_ONCE(rflow->cpu);
                if (rflow->filter == filter_id && cpu < nr_cpu_ids &&
-                   ((int)(per_cpu(softnet_data, cpu).input_queue_head -
+                   ((int)(READ_ONCE(per_cpu(softnet_data, cpu).input_queue_head) -
                           READ_ONCE(rflow->last_qtail)) <
                     (int)(10 * flow_table->mask)))
                        expire = false;
@@ -6045,9 +6045,10 @@ static int process_backlog(struct napi_struct *napi, int quota)
                        rcu_read_lock();
                        __netif_receive_skb(skb);
                        rcu_read_unlock();
-                       rps_input_queue_head_incr(sd);
-                       if (++work >= quota)
+                       if (++work >= quota) {
+                               rps_input_queue_head_add(sd, work);
                                return work;
+                       }
 
                }
 
@@ -6070,6 +6071,8 @@ static int process_backlog(struct napi_struct *napi, int quota)
                backlog_unlock_irq_enable(sd);
        }
 
+       if (work)
+               rps_input_queue_head_add(sd, work);
        return work;
 }