riscv/cmpxchg: Deduplicate cmpxchg() asm and macros

In this header every cmpxchg define (_relaxed, _acquire, _release,
vanilla) contain it's own asm file, both for 4-byte variables an 8-byte
variables, on a total of 8 versions of mostly the same asm.

This is usually bad, as it means any change may be done in up to 8
different places.

Unify those versions by creating a new define with enough parameters to
generate any version of the previous 8.

Then unify the result under a more general define, and simplify
arch_cmpxchg* generation

(This did not cause any change in generated asm)

Signed-off-by: Leonardo Bras <leobras@redhat.com>
Reviewed-by: Guo Ren <guoren@kernel.org>
Reviewed-by: Andrea Parri <parri.andrea@gmail.com>
Tested-by: Guo Ren <guoren@kernel.org>
Link: https://lore.kernel.org/r/20240103163203.72768-4-leobras@redhat.com
Signed-off-by: Palmer Dabbelt <palmer@rivosinc.com>

diff --git a/arch/riscv/include/asm/cmpxchg.h b/arch/riscv/include/asm/cmpxchg.h
index 48478a8eecee7462325cc1a17fd017724644252f..e3e0ac7ba061bc18749fd103877ae8b8f39f1e3d 100644 (file)
--- a/arch/riscv/include/asm/cmpxchg.h
+++ b/arch/riscv/include/asm/cmpxchg.h
@@ -71,190 +71,61 @@
  * store NEW in MEM.  Return the initial value in MEM.  Success is
  * indicated by comparing RETURN with OLD.
  */
-#define __cmpxchg_relaxed(ptr, old, new, size)                         \
-({                                                                     \
-       __typeof__(ptr) __ptr = (ptr);                                  \
-       __typeof__(*(ptr)) __old = (old);                               \
-       __typeof__(*(ptr)) __new = (new);                               \
-       __typeof__(*(ptr)) __ret;                                       \
-       register unsigned int __rc;                                     \
-       switch (size) {                                                 \
-       case 4:                                                         \
-               __asm__ __volatile__ (                                  \
-                       "0:     lr.w %0, %2\n"                          \
-                       "       bne  %0, %z3, 1f\n"                     \
-                       "       sc.w %1, %z4, %2\n"                     \
-                       "       bnez %1, 0b\n"                          \
-                       "1:\n"                                          \
-                       : "=&r" (__ret), "=&r" (__rc), "+A" (*__ptr)    \
-                       : "rJ" ((long)__old), "rJ" (__new)              \
-                       : "memory");                                    \
-               break;                                                  \
-       case 8:                                                         \
-               __asm__ __volatile__ (                                  \
-                       "0:     lr.d %0, %2\n"                          \
-                       "       bne %0, %z3, 1f\n"                      \
-                       "       sc.d %1, %z4, %2\n"                     \
-                       "       bnez %1, 0b\n"                          \
-                       "1:\n"                                          \
-                       : "=&r" (__ret), "=&r" (__rc), "+A" (*__ptr)    \
-                       : "rJ" (__old), "rJ" (__new)                    \
-                       : "memory");                                    \
-               break;                                                  \
-       default:                                                        \
-               BUILD_BUG();                                            \
-       }                                                               \
-       __ret;                                                          \
-})
 
-#define arch_cmpxchg_relaxed(ptr, o, n)                                        \
-({                                                                     \
-       __typeof__(*(ptr)) _o_ = (o);                                   \
-       __typeof__(*(ptr)) _n_ = (n);                                   \
-       (__typeof__(*(ptr))) __cmpxchg_relaxed((ptr),                   \
-                                       _o_, _n_, sizeof(*(ptr)));      \
-})
 
-#define __cmpxchg_acquire(ptr, old, new, size)                         \
+#define __arch_cmpxchg(lr_sfx, sc_sfx, prepend, append, r, p, co, o, n)        \
 ({                                                                     \
-       __typeof__(ptr) __ptr = (ptr);                                  \
-       __typeof__(*(ptr)) __old = (old);                               \
-       __typeof__(*(ptr)) __new = (new);                               \
-       __typeof__(*(ptr)) __ret;                                       \
        register unsigned int __rc;                                     \
-       switch (size) {                                                 \
-       case 4:                                                         \
-               __asm__ __volatile__ (                                  \
-                       "0:     lr.w %0, %2\n"                          \
-                       "       bne  %0, %z3, 1f\n"                     \
-                       "       sc.w %1, %z4, %2\n"                     \
-                       "       bnez %1, 0b\n"                          \
-                       RISCV_ACQUIRE_BARRIER                           \
-                       "1:\n"                                          \
-                       : "=&r" (__ret), "=&r" (__rc), "+A" (*__ptr)    \
-                       : "rJ" ((long)__old), "rJ" (__new)              \
-                       : "memory");                                    \
-               break;                                                  \
-       case 8:                                                         \
-               __asm__ __volatile__ (                                  \
-                       "0:     lr.d %0, %2\n"                          \
-                       "       bne %0, %z3, 1f\n"                      \
-                       "       sc.d %1, %z4, %2\n"                     \
-                       "       bnez %1, 0b\n"                          \
-                       RISCV_ACQUIRE_BARRIER                           \
-                       "1:\n"                                          \
-                       : "=&r" (__ret), "=&r" (__rc), "+A" (*__ptr)    \
-                       : "rJ" (__old), "rJ" (__new)                    \
-                       : "memory");                                    \
-               break;                                                  \
-       default:                                                        \
-               BUILD_BUG();                                            \
-       }                                                               \
-       __ret;                                                          \
-})
-
-#define arch_cmpxchg_acquire(ptr, o, n)                                        \
-({                                                                     \
-       __typeof__(*(ptr)) _o_ = (o);                                   \
-       __typeof__(*(ptr)) _n_ = (n);                                   \
-       (__typeof__(*(ptr))) __cmpxchg_acquire((ptr),                   \
-                                       _o_, _n_, sizeof(*(ptr)));      \
+                                                                       \
+       __asm__ __volatile__ (                                          \
+               prepend                                                 \
+               "0:     lr" lr_sfx " %0, %2\n"                          \
+               "       bne  %0, %z3, 1f\n"                             \
+               "       sc" sc_sfx " %1, %z4, %2\n"                     \
+               "       bnez %1, 0b\n"                                  \
+               append                                                  \
+               "1:\n"                                                  \
+               : "=&r" (r), "=&r" (__rc), "+A" (*(p))                  \
+               : "rJ" (co o), "rJ" (n)                                 \
+               : "memory");                                            \
 })
 
-#define __cmpxchg_release(ptr, old, new, size)                         \
+#define _arch_cmpxchg(ptr, old, new, sc_sfx, prepend, append)          \
 ({                                                                     \
        __typeof__(ptr) __ptr = (ptr);                                  \
-       __typeof__(*(ptr)) __old = (old);                               \
-       __typeof__(*(ptr)) __new = (new);                               \
-       __typeof__(*(ptr)) __ret;                                       \
-       register unsigned int __rc;                                     \
-       switch (size) {                                                 \
+       __typeof__(*(__ptr)) __old = (old);                             \
+       __typeof__(*(__ptr)) __new = (new);                             \
+       __typeof__(*(__ptr)) __ret;                                     \
+                                                                       \
+       switch (sizeof(*__ptr)) {                                       \
        case 4:                                                         \
-               __asm__ __volatile__ (                                  \
-                       RISCV_RELEASE_BARRIER                           \
-                       "0:     lr.w %0, %2\n"                          \
-                       "       bne  %0, %z3, 1f\n"                     \
-                       "       sc.w %1, %z4, %2\n"                     \
-                       "       bnez %1, 0b\n"                          \
-                       "1:\n"                                          \
-                       : "=&r" (__ret), "=&r" (__rc), "+A" (*__ptr)    \
-                       : "rJ" ((long)__old), "rJ" (__new)              \
-                       : "memory");                                    \
+               __arch_cmpxchg(".w", ".w" sc_sfx, prepend, append,      \
+                               __ret, __ptr, (long), __old, __new);    \
                break;                                                  \
        case 8:                                                         \
-               __asm__ __volatile__ (                                  \
-                       RISCV_RELEASE_BARRIER                           \
-                       "0:     lr.d %0, %2\n"                          \
-                       "       bne %0, %z3, 1f\n"                      \
-                       "       sc.d %1, %z4, %2\n"                     \
-                       "       bnez %1, 0b\n"                          \
-                       "1:\n"                                          \
-                       : "=&r" (__ret), "=&r" (__rc), "+A" (*__ptr)    \
-                       : "rJ" (__old), "rJ" (__new)                    \
-                       : "memory");                                    \
+               __arch_cmpxchg(".d", ".d" sc_sfx, prepend, append,      \
+                               __ret, __ptr, /**/, __old, __new);      \
                break;                                                  \
        default:                                                        \
                BUILD_BUG();                                            \
        }                                                               \
-       __ret;                                                          \
+       (__typeof__(*(__ptr)))__ret;                                    \
 })
 
-#define arch_cmpxchg_release(ptr, o, n)                                        \
-({                                                                     \
-       __typeof__(*(ptr)) _o_ = (o);                                   \
-       __typeof__(*(ptr)) _n_ = (n);                                   \
-       (__typeof__(*(ptr))) __cmpxchg_release((ptr),                   \
-                                       _o_, _n_, sizeof(*(ptr)));      \
-})
+#define arch_cmpxchg_relaxed(ptr, o, n)                                        \
+       _arch_cmpxchg((ptr), (o), (n), "", "", "")
 
-#define __cmpxchg(ptr, old, new, size)                                 \
-({                                                                     \
-       __typeof__(ptr) __ptr = (ptr);                                  \
-       __typeof__(*(ptr)) __old = (old);                               \
-       __typeof__(*(ptr)) __new = (new);                               \
-       __typeof__(*(ptr)) __ret;                                       \
-       register unsigned int __rc;                                     \
-       switch (size) {                                                 \
-       case 4:                                                         \
-               __asm__ __volatile__ (                                  \
-                       "0:     lr.w %0, %2\n"                          \
-                       "       bne  %0, %z3, 1f\n"                     \
-                       "       sc.w.rl %1, %z4, %2\n"                  \
-                       "       bnez %1, 0b\n"                          \
-                       "       fence rw, rw\n"                         \
-                       "1:\n"                                          \
-                       : "=&r" (__ret), "=&r" (__rc), "+A" (*__ptr)    \
-                       : "rJ" ((long)__old), "rJ" (__new)              \
-                       : "memory");                                    \
-               break;                                                  \
-       case 8:                                                         \
-               __asm__ __volatile__ (                                  \
-                       "0:     lr.d %0, %2\n"                          \
-                       "       bne %0, %z3, 1f\n"                      \
-                       "       sc.d.rl %1, %z4, %2\n"                  \
-                       "       bnez %1, 0b\n"                          \
-                       "       fence rw, rw\n"                         \
-                       "1:\n"                                          \
-                       : "=&r" (__ret), "=&r" (__rc), "+A" (*__ptr)    \
-                       : "rJ" (__old), "rJ" (__new)                    \
-                       : "memory");                                    \
-               break;                                                  \
-       default:                                                        \
-               BUILD_BUG();                                            \
-       }                                                               \
-       __ret;                                                          \
-})
+#define arch_cmpxchg_acquire(ptr, o, n)                                        \
+       _arch_cmpxchg((ptr), (o), (n), "", "", RISCV_ACQUIRE_BARRIER)
+
+#define arch_cmpxchg_release(ptr, o, n)                                        \
+       _arch_cmpxchg((ptr), (o), (n), "", RISCV_RELEASE_BARRIER, "")
 
 #define arch_cmpxchg(ptr, o, n)                                                \
-({                                                                     \
-       __typeof__(*(ptr)) _o_ = (o);                                   \
-       __typeof__(*(ptr)) _n_ = (n);                                   \
-       (__typeof__(*(ptr))) __cmpxchg((ptr),                           \
-                                      _o_, _n_, sizeof(*(ptr)));       \
-})
+       _arch_cmpxchg((ptr), (o), (n), ".rl", "", "     fence rw, rw\n")
 
 #define arch_cmpxchg_local(ptr, o, n)                                  \
-       (__cmpxchg_relaxed((ptr), (o), (n), sizeof(*(ptr))))
+       arch_cmpxchg_relaxed((ptr), (o), (n))
 
 #define arch_cmpxchg64(ptr, o, n)                                      \
 ({                                                                     \