x86: call instrumentation hooks from copy_mc.c

Memory accesses in copy_mc_to_kernel() and copy_mc_to_user() are performed
by assembly routines and are invisible to KASAN, KCSAN, and KMSAN.  Add
hooks from instrumentation.h to tell the tools these functions have
memcpy/copy_from_user semantics.

The call to copy_mc_fragile() in copy_mc_fragile_handle_tail() is left
intact, because the latter is only called from the assembly implementation
of copy_mc_fragile(), so the memory accesses in it are covered by the
instrumentation in copy_mc_to_kernel() and copy_mc_to_user().

Link: https://lore.kernel.org/all/3b7dbd88-0861-4638-b2d2-911c97a4cadf@I-love.SAKURA.ne.jp/
Link: https://lkml.kernel.org/r/20240320101851.2589698-3-glider@google.com
Signed-off-by: Alexander Potapenko <glider@google.com>
Suggested-by: Linus Torvalds <torvalds@linux-foundation.org>
Cc: Dmitry Vyukov <dvyukov@google.com>
Cc: Marco Elver <elver@google.com>
Cc: Tetsuo Handa <penguin-kernel@I-love.SAKURA.ne.jp>
Cc: Thomas Gleixner <tglx@linutronix.de>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>

diff --git a/arch/x86/lib/copy_mc.c b/arch/x86/lib/copy_mc.c
index 6e8b7e600def57913748a93a085c87a87eab6063..97e88e58567bf82c579e05fabef892b48831beae 100644 (file)
--- a/arch/x86/lib/copy_mc.c
+++ b/arch/x86/lib/copy_mc.c
@@ -4,6 +4,7 @@
 #include <linux/jump_label.h>
 #include <linux/uaccess.h>
 #include <linux/export.h>
+#include <linux/instrumented.h>
 #include <linux/string.h>
 #include <linux/types.h>
 
@@ -61,10 +62,20 @@ unsigned long copy_mc_enhanced_fast_string(void *dst, const void *src, unsigned
  */
 unsigned long __must_check copy_mc_to_kernel(void *dst, const void *src, unsigned len)
 {
-       if (copy_mc_fragile_enabled)
-               return copy_mc_fragile(dst, src, len);
-       if (static_cpu_has(X86_FEATURE_ERMS))
-               return copy_mc_enhanced_fast_string(dst, src, len);
+       unsigned long ret;
+
+       if (copy_mc_fragile_enabled) {
+               instrument_memcpy_before(dst, src, len);
+               ret = copy_mc_fragile(dst, src, len);
+               instrument_memcpy_after(dst, src, len, ret);
+               return ret;
+       }
+       if (static_cpu_has(X86_FEATURE_ERMS)) {
+               instrument_memcpy_before(dst, src, len);
+               ret = copy_mc_enhanced_fast_string(dst, src, len);
+               instrument_memcpy_after(dst, src, len, ret);
+               return ret;
+       }
        memcpy(dst, src, len);
        return 0;
 }
@@ -75,6 +86,7 @@ unsigned long __must_check copy_mc_to_user(void __user *dst, const void *src, un
        unsigned long ret;
 
        if (copy_mc_fragile_enabled) {
+               instrument_copy_to_user(dst, src, len);
                __uaccess_begin();
                ret = copy_mc_fragile((__force void *)dst, src, len);
                __uaccess_end();
@@ -82,6 +94,7 @@ unsigned long __must_check copy_mc_to_user(void __user *dst, const void *src, un
        }
 
        if (static_cpu_has(X86_FEATURE_ERMS)) {
+               instrument_copy_to_user(dst, src, len);
                __uaccess_begin();
                ret = copy_mc_enhanced_fast_string((__force void *)dst, src, len);
                __uaccess_end();