perf/x86/amd: Avoid taking branches before disabling LBR

In the following patches we will enable LBR capture on AMD CPUs at
arbitrary point in time, which means that LBR recording won't be frozen
by hardware automatically as part of hardware overflow event. So we need
to take care to minimize amount of branches and function calls/returns
on the path to freezing LBR, minimizing LBR snapshot altering as much as
possible.

As such, split out LBR disabling logic from the sanity checking logic
inside amd_pmu_lbr_disable_all(). This will ensure that no branches are
taken before LBR is frozen in the functionality added in the next patch.
Use __always_inline to also eliminate any possible function calls.

Signed-off-by: Andrii Nakryiko <andrii@kernel.org>
Signed-off-by: Ingo Molnar <mingo@kernel.org>
Reviewed-by: Sandipan Das <sandipan.das@amd.com>
Link: https://lore.kernel.org/r/20240402022118.1046049-3-andrii@kernel.org

diff --git a/arch/x86/events/amd/lbr.c b/arch/x86/events/amd/lbr.c
index 5149830c7c4fa61207a3a30122e07a24964374d9..33d0a45c0cd385b6d23f7731abfc228198d750f0 100644 (file)
--- a/arch/x86/events/amd/lbr.c
+++ b/arch/x86/events/amd/lbr.c
@@ -414,18 +414,11 @@ void amd_pmu_lbr_enable_all(void)
 void amd_pmu_lbr_disable_all(void)
 {
        struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
-       u64 dbg_ctl, dbg_extn_cfg;
 
        if (!cpuc->lbr_users || !x86_pmu.lbr_nr)
                return;
 
-       rdmsrl(MSR_AMD_DBG_EXTN_CFG, dbg_extn_cfg);
-       wrmsrl(MSR_AMD_DBG_EXTN_CFG, dbg_extn_cfg & ~DBG_EXTN_CFG_LBRV2EN);
-
-       if (cpu_feature_enabled(X86_FEATURE_AMD_LBR_PMC_FREEZE)) {
-               rdmsrl(MSR_IA32_DEBUGCTLMSR, dbg_ctl);
-               wrmsrl(MSR_IA32_DEBUGCTLMSR, dbg_ctl & ~DEBUGCTLMSR_FREEZE_LBRS_ON_PMI);
-       }
+       __amd_pmu_lbr_disable();
 }
 
 __init int amd_pmu_lbr_init(void)

diff --git a/arch/x86/events/perf_event.h b/arch/x86/events/perf_event.h
index fb56518356ecfbacdb8d79b426bd1c82e4012082..72b022a1e16c568d97d79ac423f7038185c9287c 100644 (file)
--- a/arch/x86/events/perf_event.h
+++ b/arch/x86/events/perf_event.h
@@ -1329,6 +1329,19 @@ void amd_pmu_lbr_enable_all(void);
 void amd_pmu_lbr_disable_all(void);
 int amd_pmu_lbr_hw_config(struct perf_event *event);
 
+static __always_inline void __amd_pmu_lbr_disable(void)
+{
+       u64 dbg_ctl, dbg_extn_cfg;
+
+       rdmsrl(MSR_AMD_DBG_EXTN_CFG, dbg_extn_cfg);
+       wrmsrl(MSR_AMD_DBG_EXTN_CFG, dbg_extn_cfg & ~DBG_EXTN_CFG_LBRV2EN);
+
+       if (cpu_feature_enabled(X86_FEATURE_AMD_LBR_PMC_FREEZE)) {
+               rdmsrl(MSR_IA32_DEBUGCTLMSR, dbg_ctl);
+               wrmsrl(MSR_IA32_DEBUGCTLMSR, dbg_ctl & ~DEBUGCTLMSR_FREEZE_LBRS_ON_PMI);
+       }
+}
+
 #ifdef CONFIG_PERF_EVENTS_AMD_BRS
 
 #define AMD_FAM19H_BRS_EVENT 0xc4 /* RETIRED_TAKEN_BRANCH_INSTRUCTIONS */