x86/modules: Add call patching

As for the builtins create call thunks and patch the call sites to call the
thunk on Intel SKL CPUs for retbleed mitigation.

Note, that module init functions are ignored for sake of simplicity because
loading modules is not something which is done in high frequent loops and
the attacker has not really a handle on when this happens in order to
launch a matching attack. The depth tracking will still work for calls into
the builtins and because the call is not accounted it will underflow faster
and overstuff, but that's mitigated by the saturating counter and the side
effect is only temporary.

Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org>
Link: https://lore.kernel.org/r/20220915111147.575673066@infradead.org

diff --git a/arch/x86/include/asm/alternative.h b/arch/x86/include/asm/alternative.h
index 6b7bbd0db24800e49e0843a8b1d45b24b6d0a167..ef007fa33dc417dc424fedb63ec163cc56b30715 100644 (file)
--- a/arch/x86/include/asm/alternative.h
+++ b/arch/x86/include/asm/alternative.h
@@ -89,8 +89,13 @@ struct callthunk_sites {
 
 #ifdef CONFIG_CALL_THUNKS
 extern void callthunks_patch_builtin_calls(void);
+extern void callthunks_patch_module_calls(struct callthunk_sites *sites,
+                                         struct module *mod);
 #else
 static __always_inline void callthunks_patch_builtin_calls(void) {}
+static __always_inline void
+callthunks_patch_module_calls(struct callthunk_sites *sites,
+                             struct module *mod) {}
 #endif
 
 #ifdef CONFIG_SMP

diff --git a/arch/x86/kernel/callthunks.c b/arch/x86/kernel/callthunks.c
index e5275d6e674d9c58d2fd7627c11645b7ca7741b5..7b9d998ebd7dc8d8ffa2a88a1ecce5389236b3a3 100644 (file)
--- a/arch/x86/kernel/callthunks.c
+++ b/arch/x86/kernel/callthunks.c
@@ -249,3 +249,22 @@ void __init callthunks_patch_builtin_calls(void)
        thunks_initialized = true;
        mutex_unlock(&text_mutex);
 }
+
+#ifdef CONFIG_MODULES
+void noinline callthunks_patch_module_calls(struct callthunk_sites *cs,
+                                           struct module *mod)
+{
+       struct core_text ct = {
+               .base = (unsigned long)mod->core_layout.base,
+               .end  = (unsigned long)mod->core_layout.base + mod->core_layout.size,
+               .name = mod->name,
+       };
+
+       if (!thunks_initialized)
+               return;
+
+       mutex_lock(&text_mutex);
+       callthunks_setup(cs, &ct);
+       mutex_unlock(&text_mutex);
+}
+#endif /* CONFIG_MODULES */

diff --git a/arch/x86/kernel/module.c b/arch/x86/kernel/module.c
index 43f01127721923fb8b2a68ebe74ac6e1c15de3ae..2fb9de2cef40e9a14938704145581edaec46a6ea 100644 (file)
--- a/arch/x86/kernel/module.c
+++ b/arch/x86/kernel/module.c
@@ -254,7 +254,8 @@ int module_finalize(const Elf_Ehdr *hdr,
 {
        const Elf_Shdr *s, *text = NULL, *alt = NULL, *locks = NULL,
                *para = NULL, *orc = NULL, *orc_ip = NULL,
-               *retpolines = NULL, *returns = NULL, *ibt_endbr = NULL;
+               *retpolines = NULL, *returns = NULL, *ibt_endbr = NULL,
+               *calls = NULL;
        char *secstrings = (void *)hdr + sechdrs[hdr->e_shstrndx].sh_offset;
 
        for (s = sechdrs; s < sechdrs + hdr->e_shnum; s++) {
@@ -274,6 +275,8 @@ int module_finalize(const Elf_Ehdr *hdr,
                        retpolines = s;
                if (!strcmp(".return_sites", secstrings + s->sh_name))
                        returns = s;
+               if (!strcmp(".call_sites", secstrings + s->sh_name))
+                       calls = s;
                if (!strcmp(".ibt_endbr_seal", secstrings + s->sh_name))
                        ibt_endbr = s;
        }
@@ -299,6 +302,21 @@ int module_finalize(const Elf_Ehdr *hdr,
                void *aseg = (void *)alt->sh_addr;
                apply_alternatives(aseg, aseg + alt->sh_size);
        }
+       if (calls || para) {
+               struct callthunk_sites cs = {};
+
+               if (calls) {
+                       cs.call_start = (void *)calls->sh_addr;
+                       cs.call_end = (void *)calls->sh_addr + calls->sh_size;
+               }
+
+               if (para) {
+                       cs.pv_start = (void *)para->sh_addr;
+                       cs.pv_end = (void *)para->sh_addr + para->sh_size;
+               }
+
+               callthunks_patch_module_calls(&cs, me);
+       }
        if (ibt_endbr) {
                void *iseg = (void *)ibt_endbr->sh_addr;
                apply_ibt_endbr(iseg, iseg + ibt_endbr->sh_size);