mm/vmstat: use this_cpu_try_cmpxchg in mod_{zone,node}_state

Use this_cpu_try_cmpxchg instead of this_cpu_cmpxchg (*ptr, old, new) ==
old in mod_zone_state and mod_node_state.  x86 CMPXCHG instruction returns
success in ZF flag, so this change saves a compare after cmpxchg (and
related move instruction in front of cmpxchg).

Also, try_cmpxchg implicitly assigns old *ptr value to "old" when cmpxchg
fails.  There is no need to re-read the value in the loop.

No functional change intended.

Link: https://lkml.kernel.org/r/20230904150917.8318-1-ubizjak@gmail.com
Signed-off-by: Uros Bizjak <ubizjak@gmail.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>

diff --git a/mm/vmstat.c b/mm/vmstat.c
index 00e81e99c6ee24e419446b8c476faea0b8ef75bd..894e4c88d241d8a19509d05c69d5a44147fac783 100644 (file)
--- a/mm/vmstat.c
+++ b/mm/vmstat.c
@@ -559,8 +559,10 @@ static inline void mod_zone_state(struct zone *zone,
 {
        struct per_cpu_zonestat __percpu *pcp = zone->per_cpu_zonestats;
        s8 __percpu *p = pcp->vm_stat_diff + item;
-       long o, n, t, z;
+       long n, t, z;
+       s8 o;
 
+       o = this_cpu_read(*p);
        do {
                z = 0;  /* overflow to zone counters */
 
@@ -576,8 +578,7 @@ static inline void mod_zone_state(struct zone *zone,
                 */
                t = this_cpu_read(pcp->stat_threshold);
 
-               o = this_cpu_read(*p);
-               n = delta + o;
+               n = delta + (long)o;
 
                if (abs(n) > t) {
                        int os = overstep_mode * (t >> 1) ;
@@ -586,7 +587,7 @@ static inline void mod_zone_state(struct zone *zone,
                        z = n + os;
                        n = -os;
                }
-       } while (this_cpu_cmpxchg(*p, o, n) != o);
+       } while (!this_cpu_try_cmpxchg(*p, &o, n));
 
        if (z)
                zone_page_state_add(z, zone, item);
@@ -616,7 +617,8 @@ static inline void mod_node_state(struct pglist_data *pgdat,
 {
        struct per_cpu_nodestat __percpu *pcp = pgdat->per_cpu_nodestats;
        s8 __percpu *p = pcp->vm_node_stat_diff + item;
-       long o, n, t, z;
+       long n, t, z;
+       s8 o;
 
        if (vmstat_item_in_bytes(item)) {
                /*
@@ -629,6 +631,7 @@ static inline void mod_node_state(struct pglist_data *pgdat,
                delta >>= PAGE_SHIFT;
        }
 
+       o = this_cpu_read(*p);
        do {
                z = 0;  /* overflow to node counters */
 
@@ -644,8 +647,7 @@ static inline void mod_node_state(struct pglist_data *pgdat,
                 */
                t = this_cpu_read(pcp->stat_threshold);
 
-               o = this_cpu_read(*p);
-               n = delta + o;
+               n = delta + (long)o;
 
                if (abs(n) > t) {
                        int os = overstep_mode * (t >> 1) ;
@@ -654,7 +656,7 @@ static inline void mod_node_state(struct pglist_data *pgdat,
                        z = n + os;
                        n = -os;
                }
-       } while (this_cpu_cmpxchg(*p, o, n) != o);
+       } while (!this_cpu_try_cmpxchg(*p, &o, n));
 
        if (z)
                node_page_state_add(z, pgdat, item);