x86/percpu: Move irq_stack variables next to current_task

Further extend struct pcpu_hot with the hard and soft irq stack
pointers.

Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org>
Link: https://lore.kernel.org/r/20220915111145.599170752@infradead.org

diff --git a/arch/x86/include/asm/current.h b/arch/x86/include/asm/current.h
index 2dd013128f1ef64c996716784a90a8ec699c9b44..ac3090ddf34e89792c5b033a1e2d5c352d23efb2 100644 (file)
--- a/arch/x86/include/asm/current.h
+++ b/arch/x86/include/asm/current.h
@@ -18,6 +18,12 @@ struct pcpu_hot {
                        int                     preempt_count;
                        int                     cpu_number;
                        unsigned long           top_of_stack;
+                       void                    *hardirq_stack_ptr;
+#ifdef CONFIG_X86_64
+                       bool                    hardirq_stack_inuse;
+#else
+                       void                    *softirq_stack_ptr;
+#endif
                };
                u8      pad[64];
        };

diff --git a/arch/x86/include/asm/irq_stack.h b/arch/x86/include/asm/irq_stack.h
index 147cb8fdda92e9e0a9a2e74bb876968af8ba29dc..798183867d78969b2a7fd7cc3169cf310926fa8e 100644 (file)
--- a/arch/x86/include/asm/irq_stack.h
+++ b/arch/x86/include/asm/irq_stack.h
@@ -116,7 +116,7 @@
        ASM_CALL_ARG2
 
 #define call_on_irqstack(func, asm_call, argconstr...)                 \
-       call_on_stack(__this_cpu_read(hardirq_stack_ptr),               \
+       call_on_stack(__this_cpu_read(pcpu_hot.hardirq_stack_ptr),      \
                      func, asm_call, argconstr)
 
 /* Macros to assert type correctness for run_*_on_irqstack macros */
@@ -135,7 +135,7 @@
         * User mode entry and interrupt on the irq stack do not        \
         * switch stacks. If from user mode the task stack is empty.    \
         */                                                             \
-       if (user_mode(regs) || __this_cpu_read(hardirq_stack_inuse)) {  \
+       if (user_mode(regs) || __this_cpu_read(pcpu_hot.hardirq_stack_inuse)) { \
                irq_enter_rcu();                                        \
                func(c_args);                                           \
                irq_exit_rcu();                                         \
@@ -146,9 +146,9 @@
                 * places. Invoke the stack switch macro with the call  \
                 * sequence which matches the above direct invocation.  \
                 */                                                     \
-               __this_cpu_write(hardirq_stack_inuse, true);            \
+               __this_cpu_write(pcpu_hot.hardirq_stack_inuse, true);   \
                call_on_irqstack(func, asm_call, constr);               \
-               __this_cpu_write(hardirq_stack_inuse, false);           \
+               __this_cpu_write(pcpu_hot.hardirq_stack_inuse, false);  \
        }                                                               \
 }
 
@@ -212,9 +212,9 @@
  */
 #define do_softirq_own_stack()                                         \
 {                                                                      \
-       __this_cpu_write(hardirq_stack_inuse, true);                    \
+       __this_cpu_write(pcpu_hot.hardirq_stack_inuse, true);           \
        call_on_irqstack(__do_softirq, ASM_CALL_ARG0);                  \
-       __this_cpu_write(hardirq_stack_inuse, false);                   \
+       __this_cpu_write(pcpu_hot.hardirq_stack_inuse, false);          \
 }
 
 #endif

diff --git a/arch/x86/include/asm/processor.h b/arch/x86/include/asm/processor.h
index c345f3096c80b9f7c1045d01dce5f21419b4a261..bdde68744eb3ad1f0f11fc30c96be841e080efbc 100644 (file)
--- a/arch/x86/include/asm/processor.h
+++ b/arch/x86/include/asm/processor.h
@@ -448,8 +448,6 @@ static inline unsigned long cpu_kernelmode_gs_base(int cpu)
        return (unsigned long)per_cpu(fixed_percpu_data.gs_base, cpu);
 }
 
-DECLARE_PER_CPU(void *, hardirq_stack_ptr);
-DECLARE_PER_CPU(bool, hardirq_stack_inuse);
 extern asmlinkage void ignore_sysret(void);
 
 /* Save actual FS/GS selectors and bases to current->thread */
@@ -458,8 +456,6 @@ void current_save_fsgs(void);
 #ifdef CONFIG_STACKPROTECTOR
 DECLARE_PER_CPU(unsigned long, __stack_chk_guard);
 #endif
-DECLARE_PER_CPU(struct irq_stack *, hardirq_stack_ptr);
-DECLARE_PER_CPU(struct irq_stack *, softirq_stack_ptr);
 #endif /* !X86_64 */
 
 struct perf_event;

diff --git a/arch/x86/kernel/cpu/common.c b/arch/x86/kernel/cpu/common.c
index 408245c2eead8500c160b8460792fe207d6159c0..2bec4b4b2c50d42be6a70af6e236c0214edfea17 100644 (file)
--- a/arch/x86/kernel/cpu/common.c
+++ b/arch/x86/kernel/cpu/common.c
@@ -2024,9 +2024,6 @@ DEFINE_PER_CPU_FIRST(struct fixed_percpu_data,
                     fixed_percpu_data) __aligned(PAGE_SIZE) __visible;
 EXPORT_PER_CPU_SYMBOL_GPL(fixed_percpu_data);
 
-DEFINE_PER_CPU(void *, hardirq_stack_ptr);
-DEFINE_PER_CPU(bool, hardirq_stack_inuse);
-
 static void wrmsrl_cstar(unsigned long val)
 {
        /*

diff --git a/arch/x86/kernel/dumpstack_32.c b/arch/x86/kernel/dumpstack_32.c
index 722fd712e1cf029adff9a73c73aad44992dbcef6..b4905d5173fd36d1df190a54725eced03f9170dd 100644 (file)
--- a/arch/x86/kernel/dumpstack_32.c
+++ b/arch/x86/kernel/dumpstack_32.c
@@ -37,7 +37,7 @@ const char *stack_type_name(enum stack_type type)
 
 static bool in_hardirq_stack(unsigned long *stack, struct stack_info *info)
 {
-       unsigned long *begin = (unsigned long *)this_cpu_read(hardirq_stack_ptr);
+       unsigned long *begin = (unsigned long *)this_cpu_read(pcpu_hot.hardirq_stack_ptr);
        unsigned long *end   = begin + (THREAD_SIZE / sizeof(long));
 
        /*
@@ -62,7 +62,7 @@ static bool in_hardirq_stack(unsigned long *stack, struct stack_info *info)
 
 static bool in_softirq_stack(unsigned long *stack, struct stack_info *info)
 {
-       unsigned long *begin = (unsigned long *)this_cpu_read(softirq_stack_ptr);
+       unsigned long *begin = (unsigned long *)this_cpu_read(pcpu_hot.softirq_stack_ptr);
        unsigned long *end   = begin + (THREAD_SIZE / sizeof(long));
 
        /*

diff --git a/arch/x86/kernel/dumpstack_64.c b/arch/x86/kernel/dumpstack_64.c
index 6c5defd6569a3eb867b58c37fccb93dd33decaee..f05339fee7785b1da2cc2cec9713e06a1242dbe7 100644 (file)
--- a/arch/x86/kernel/dumpstack_64.c
+++ b/arch/x86/kernel/dumpstack_64.c
@@ -134,7 +134,7 @@ static __always_inline bool in_exception_stack(unsigned long *stack, struct stac
 
 static __always_inline bool in_irq_stack(unsigned long *stack, struct stack_info *info)
 {
-       unsigned long *end = (unsigned long *)this_cpu_read(hardirq_stack_ptr);
+       unsigned long *end = (unsigned long *)this_cpu_read(pcpu_hot.hardirq_stack_ptr);
        unsigned long *begin;
 
        /*

diff --git a/arch/x86/kernel/irq_32.c b/arch/x86/kernel/irq_32.c
index 01833ebf5e8e3582992a8860460bbe555923eb30..dc1049c01f9b9aa19c6f986e975d382502bce062 100644 (file)
--- a/arch/x86/kernel/irq_32.c
+++ b/arch/x86/kernel/irq_32.c
@@ -52,9 +52,6 @@ static inline int check_stack_overflow(void) { return 0; }
 static inline void print_stack_overflow(void) { }
 #endif
 
-DEFINE_PER_CPU(struct irq_stack *, hardirq_stack_ptr);
-DEFINE_PER_CPU(struct irq_stack *, softirq_stack_ptr);
-
 static void call_on_stack(void *func, void *stack)
 {
        asm volatile("xchgl     %%ebx,%%esp     \n"
@@ -77,7 +74,7 @@ static inline int execute_on_irq_stack(int overflow, struct irq_desc *desc)
        u32 *isp, *prev_esp, arg1;
 
        curstk = (struct irq_stack *) current_stack();
-       irqstk = __this_cpu_read(hardirq_stack_ptr);
+       irqstk = __this_cpu_read(pcpu_hot.hardirq_stack_ptr);
 
        /*
         * this is where we switch to the IRQ stack. However, if we are
@@ -115,7 +112,7 @@ int irq_init_percpu_irqstack(unsigned int cpu)
        int node = cpu_to_node(cpu);
        struct page *ph, *ps;
 
-       if (per_cpu(hardirq_stack_ptr, cpu))
+       if (per_cpu(pcpu_hot.hardirq_stack_ptr, cpu))
                return 0;
 
        ph = alloc_pages_node(node, THREADINFO_GFP, THREAD_SIZE_ORDER);
@@ -127,8 +124,8 @@ int irq_init_percpu_irqstack(unsigned int cpu)
                return -ENOMEM;
        }
 
-       per_cpu(hardirq_stack_ptr, cpu) = page_address(ph);
-       per_cpu(softirq_stack_ptr, cpu) = page_address(ps);
+       per_cpu(pcpu_hot.hardirq_stack_ptr, cpu) = page_address(ph);
+       per_cpu(pcpu_hot.softirq_stack_ptr, cpu) = page_address(ps);
        return 0;
 }
 
@@ -138,7 +135,7 @@ void do_softirq_own_stack(void)
        struct irq_stack *irqstk;
        u32 *isp, *prev_esp;
 
-       irqstk = __this_cpu_read(softirq_stack_ptr);
+       irqstk = __this_cpu_read(pcpu_hot.softirq_stack_ptr);
 
        /* build the stack frame on the softirq stack */
        isp = (u32 *) ((char *)irqstk + sizeof(*irqstk));

diff --git a/arch/x86/kernel/irq_64.c b/arch/x86/kernel/irq_64.c
index 1c0fb96b9e39069d4d246f32f5e5c782e8e7be38..fe0c859873d17ef472aa92aac419bc427073c6e9 100644 (file)
--- a/arch/x86/kernel/irq_64.c
+++ b/arch/x86/kernel/irq_64.c
@@ -50,7 +50,7 @@ static int map_irq_stack(unsigned int cpu)
                return -ENOMEM;
 
        /* Store actual TOS to avoid adjustment in the hotpath */
-       per_cpu(hardirq_stack_ptr, cpu) = va + IRQ_STACK_SIZE - 8;
+       per_cpu(pcpu_hot.hardirq_stack_ptr, cpu) = va + IRQ_STACK_SIZE - 8;
        return 0;
 }
 #else
@@ -63,14 +63,14 @@ static int map_irq_stack(unsigned int cpu)
        void *va = per_cpu_ptr(&irq_stack_backing_store, cpu);
 
        /* Store actual TOS to avoid adjustment in the hotpath */
-       per_cpu(hardirq_stack_ptr, cpu) = va + IRQ_STACK_SIZE - 8;
+       per_cpu(pcpu_hot.hardirq_stack_ptr, cpu) = va + IRQ_STACK_SIZE - 8;
        return 0;
 }
 #endif
 
 int irq_init_percpu_irqstack(unsigned int cpu)
 {
-       if (per_cpu(hardirq_stack_ptr, cpu))
+       if (per_cpu(pcpu_hot.hardirq_stack_ptr, cpu))
                return 0;
        return map_irq_stack(cpu);
 }

diff --git a/arch/x86/kernel/process_64.c b/arch/x86/kernel/process_64.c
index 7f807e8bc9237d370b4cde37759b7083b1c6ab91..1312de5b76aa4a7f61c2d82136cb32cb7b82bc92 100644 (file)
--- a/arch/x86/kernel/process_64.c
+++ b/arch/x86/kernel/process_64.c
@@ -563,7 +563,7 @@ __switch_to(struct task_struct *prev_p, struct task_struct *next_p)
        int cpu = smp_processor_id();
 
        WARN_ON_ONCE(IS_ENABLED(CONFIG_DEBUG_ENTRY) &&
-                    this_cpu_read(hardirq_stack_inuse));
+                    this_cpu_read(pcpu_hot.hardirq_stack_inuse));
 
        if (!test_thread_flag(TIF_NEED_FPU_LOAD))
                switch_fpu_prepare(prev_fpu, cpu);