vdso/treewide: Add vdso_data pointer argument to __arch_get_hw_counter()

MIPS already uses and S390 will need the vdso data pointer in
__arch_get_hw_counter().

This works nicely as long as the architecture does not support time
namespaces in the VDSO. With time namespaces enabled the regular
accessor to the vdso data pointer __arch_get_vdso_data() will return the
namespace specific VDSO data page for tasks which are part of a
non-root time namespace. This would cause the architectures which need
the vdso data pointer in __arch_get_hw_counter() to access the wrong
vdso data page.

Add a vdso_data pointer argument to __arch_get_hw_counter() and hand it in
from the call sites in the core code. For architectures which do not need
the data pointer in their counter accessor function the compiler will just
optimize it out.

Fix up all existing architecture implementations and make MIPS utilize the
pointer instead of invoking the accessor function.

No functional change and no change in the resulting object code (except
MIPS).

Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
Link: https://lkml.kernel.org/r/draft-87wo2ekuzn.fsf@nanos.tec.linutronix.de

diff --git a/arch/arm/include/asm/vdso/gettimeofday.h b/arch/arm/include/asm/vdso/gettimeofday.h
index 1b207cf07697543365a1e8fe1cd4929307f5520f..2134cbd5469feef020ef30494b667b93972f9fac 100644 (file)
--- a/arch/arm/include/asm/vdso/gettimeofday.h
+++ b/arch/arm/include/asm/vdso/gettimeofday.h
@@ -113,7 +113,8 @@ static inline bool arm_vdso_hres_capable(void)
 }
 #define __arch_vdso_hres_capable arm_vdso_hres_capable
 
-static __always_inline u64 __arch_get_hw_counter(int clock_mode)
+static __always_inline u64 __arch_get_hw_counter(int clock_mode,
+                                                const struct vdso_data *vd)
 {
 #ifdef CONFIG_ARM_ARCH_TIMER
        u64 cycle_now;

diff --git a/arch/arm64/include/asm/vdso/compat_gettimeofday.h b/arch/arm64/include/asm/vdso/compat_gettimeofday.h
index 75cbae60455bc501c1d400d79ebe18e4ce8e3e52..7508b0ac1d21d51a3a715629a79af38fb2bd76f4 100644 (file)
--- a/arch/arm64/include/asm/vdso/compat_gettimeofday.h
+++ b/arch/arm64/include/asm/vdso/compat_gettimeofday.h
@@ -103,7 +103,8 @@ int clock_getres32_fallback(clockid_t _clkid, struct old_timespec32 *_ts)
        return ret;
 }
 
-static __always_inline u64 __arch_get_hw_counter(s32 clock_mode)
+static __always_inline u64 __arch_get_hw_counter(s32 clock_mode,
+                                                const struct vdso_data *vd)
 {
        u64 res;
 

diff --git a/arch/arm64/include/asm/vdso/gettimeofday.h b/arch/arm64/include/asm/vdso/gettimeofday.h
index 9c29ad3049f8f0e7b6b38865369c8c195bc8de94..631ab12816335ff0bed9eda6b9eb4ad56c657b72 100644 (file)
--- a/arch/arm64/include/asm/vdso/gettimeofday.h
+++ b/arch/arm64/include/asm/vdso/gettimeofday.h
@@ -64,7 +64,8 @@ int clock_getres_fallback(clockid_t _clkid, struct __kernel_timespec *_ts)
        return ret;
 }
 
-static __always_inline u64 __arch_get_hw_counter(s32 clock_mode)
+static __always_inline u64 __arch_get_hw_counter(s32 clock_mode,
+                                                const struct vdso_data *vd)
 {
        u64 res;
 

diff --git a/arch/mips/include/asm/vdso/gettimeofday.h b/arch/mips/include/asm/vdso/gettimeofday.h
index c63ddcaea54c900da06e1ebacd2154a5c6dedda1..2203e2d0ae2ad62eb687e8a890c94069c0960d65 100644 (file)
--- a/arch/mips/include/asm/vdso/gettimeofday.h
+++ b/arch/mips/include/asm/vdso/gettimeofday.h
@@ -167,7 +167,8 @@ static __always_inline u64 read_gic_count(const struct vdso_data *data)
 
 #endif
 
-static __always_inline u64 __arch_get_hw_counter(s32 clock_mode)
+static __always_inline u64 __arch_get_hw_counter(s32 clock_mode,
+                                                const struct vdso_data *vd)
 {
 #ifdef CONFIG_CSRC_R4K
        if (clock_mode == VDSO_CLOCKMODE_R4K)
@@ -175,7 +176,7 @@ static __always_inline u64 __arch_get_hw_counter(s32 clock_mode)
 #endif
 #ifdef CONFIG_CLKSRC_MIPS_GIC
        if (clock_mode == VDSO_CLOCKMODE_GIC)
-               return read_gic_count(get_vdso_data());
+               return read_gic_count(vd);
 #endif
        /*
         * Core checks mode already. So this raced against a concurrent

diff --git a/arch/riscv/include/asm/vdso/gettimeofday.h b/arch/riscv/include/asm/vdso/gettimeofday.h
index 3099362d9f2631fa5ea83548725b5697f5c2869b..f839f16e0d2a94d88dc657c0db6d8514c1a1844c 100644 (file)
--- a/arch/riscv/include/asm/vdso/gettimeofday.h
+++ b/arch/riscv/include/asm/vdso/gettimeofday.h
@@ -60,7 +60,8 @@ int clock_getres_fallback(clockid_t _clkid, struct __kernel_timespec *_ts)
        return ret;
 }
 
-static __always_inline u64 __arch_get_hw_counter(s32 clock_mode)
+static __always_inline u64 __arch_get_hw_counter(s32 clock_mode,
+                                                const struct vdso_data *vd)
 {
        /*
         * The purpose of csr_read(CSR_TIME) is to trap the system into

diff --git a/arch/x86/include/asm/vdso/gettimeofday.h b/arch/x86/include/asm/vdso/gettimeofday.h
index fb81fea99093cb07c2fac95e9812ac712d5dcefd..df01d7349d79918d6014df3118f465b18d3cdf9a 100644 (file)
--- a/arch/x86/include/asm/vdso/gettimeofday.h
+++ b/arch/x86/include/asm/vdso/gettimeofday.h
@@ -241,7 +241,8 @@ static u64 vread_hvclock(void)
 }
 #endif
 
-static inline u64 __arch_get_hw_counter(s32 clock_mode)
+static inline u64 __arch_get_hw_counter(s32 clock_mode,
+                                       const struct vdso_data *vd)
 {
        if (likely(clock_mode == VDSO_CLOCKMODE_TSC))
                return (u64)rdtsc_ordered();

diff --git a/lib/vdso/gettimeofday.c b/lib/vdso/gettimeofday.c
index bcc9a98a052461cc26e3a8dede9bcdc5537a853c..2919f16981404199f91335fed9f31c37385cc5c6 100644 (file)
--- a/lib/vdso/gettimeofday.c
+++ b/lib/vdso/gettimeofday.c
@@ -68,7 +68,7 @@ static int do_hres_timens(const struct vdso_data *vdns, clockid_t clk,
                if (unlikely(!vdso_clocksource_ok(vd)))
                        return -1;
 
-               cycles = __arch_get_hw_counter(vd->clock_mode);
+               cycles = __arch_get_hw_counter(vd->clock_mode, vd);
                if (unlikely(!vdso_cycles_ok(cycles)))
                        return -1;
                ns = vdso_ts->nsec;
@@ -138,7 +138,7 @@ static __always_inline int do_hres(const struct vdso_data *vd, clockid_t clk,
                if (unlikely(!vdso_clocksource_ok(vd)))
                        return -1;
 
-               cycles = __arch_get_hw_counter(vd->clock_mode);
+               cycles = __arch_get_hw_counter(vd->clock_mode, vd);
                if (unlikely(!vdso_cycles_ok(cycles)))
                        return -1;
                ns = vdso_ts->nsec;