arm64: smp: initialize cpu offset earlier

Now that we have a consistent place to initialize CPU context registers
early in the boot path, let's also initialize the per-cpu offset here.
This makes the primary and secondary boot paths more consistent, and
allows for the use of per-cpu operations earlier, which will be
necessary for instrumentation with KCSAN.

Note that smp_prepare_boot_cpu() still needs to re-initialize CPU0's
offset as immediately prior to this the per-cpu areas may be
reallocated, and hence the boot-time offset may be stale. A comment is
added to make this clear.

Signed-off-by: Mark Rutland <mark.rutland@arm.com>
Cc: Ard Biesheuvel <ardb@kernel.org>
Cc: Catalin Marinas <catalin.marinas@arm.com>
Cc: James Morse <james.morse@arm.com>
Cc: Marc Zyngier <maz@kernel.org>
Cc: Suzuki Poulose <suzuki.poulose@arm.com>
Cc: Will Deacon <will@kernel.org>
Reviewed-by: Ard Biesheuvel <ardb@kernel.org>
Link: https://lore.kernel.org/r/20210520115031.18509-7-mark.rutland@arm.com
Signed-off-by: Will Deacon <will@kernel.org>

diff --git a/arch/arm64/kernel/asm-offsets.c b/arch/arm64/kernel/asm-offsets.c
index 4a5e204c33af70757dc35df898d5311851727a33..bd0fc23d8719cfb3d2f6b026ef31612e18be3d54 100644 (file)
--- a/arch/arm64/kernel/asm-offsets.c
+++ b/arch/arm64/kernel/asm-offsets.c
@@ -27,6 +27,7 @@
 int main(void)
 {
   DEFINE(TSK_ACTIVE_MM,                offsetof(struct task_struct, active_mm));
+  DEFINE(TSK_CPU,              offsetof(struct task_struct, cpu));
   BLANK();
   DEFINE(TSK_TI_FLAGS,         offsetof(struct task_struct, thread_info.flags));
   DEFINE(TSK_TI_PREEMPT,       offsetof(struct task_struct, thread_info.preempt_count));

diff --git a/arch/arm64/kernel/head.S b/arch/arm64/kernel/head.S
index e83b2899dce5b2d21bf5746fb776674e8c024c1d..070ed53c049d4a52353a5ef1228e3c1cff6ceffb 100644 (file)
--- a/arch/arm64/kernel/head.S
+++ b/arch/arm64/kernel/head.S
@@ -402,17 +402,22 @@ SYM_FUNC_END(__create_page_tables)
         * its location in the task stack. We reserve the entire pt_regs space
         * for consistency with user tasks and kthreads.
         */
-       .macro  init_cpu_task tsk, tmp
+       .macro  init_cpu_task tsk, tmp1, tmp2
        msr     sp_el0, \tsk
 
-       ldr     \tmp, [\tsk, #TSK_STACK]
-       add     sp, \tmp, #THREAD_SIZE
+       ldr     \tmp1, [\tsk, #TSK_STACK]
+       add     sp, \tmp1, #THREAD_SIZE
        sub     sp, sp, #PT_REGS_SIZE
 
        stp     xzr, xzr, [sp, #S_STACKFRAME]
        add     x29, sp, #S_STACKFRAME
 
-       scs_load \tsk, \tmp
+       scs_load \tsk, \tmp1
+
+       adr_l   \tmp1, __per_cpu_offset
+       ldr     w\tmp2, [\tsk, #TSK_CPU]
+       ldr     \tmp1, [\tmp1, \tmp2, lsl #3]
+       set_this_cpu_offset \tmp1
        .endm
 
 /*
@@ -422,7 +427,7 @@ SYM_FUNC_END(__create_page_tables)
  */
 SYM_FUNC_START_LOCAL(__primary_switched)
        adr_l   x4, init_task
-       init_cpu_task x4, x5
+       init_cpu_task x4, x5, x6
 
        adr_l   x8, vectors                     // load VBAR_EL1 with virtual
        msr     vbar_el1, x8                    // vector table address
@@ -650,7 +655,7 @@ SYM_FUNC_START_LOCAL(__secondary_switched)
        ldr     x2, [x0, #CPU_BOOT_TASK]
        cbz     x2, __secondary_too_slow
 
-       init_cpu_task x2, x1
+       init_cpu_task x2, x1, x3
 
 #ifdef CONFIG_ARM64_PTR_AUTH
        ptrauth_keys_init_cpu x2, x3, x4, x5

diff --git a/arch/arm64/kernel/setup.c b/arch/arm64/kernel/setup.c
index 61845c0821d9dc8533bb0eca1768da7b33d85d65..b7a35a03e9b902b691aca62d8fe9ede5c99b69c6 100644 (file)
--- a/arch/arm64/kernel/setup.c
+++ b/arch/arm64/kernel/setup.c
@@ -87,12 +87,6 @@ void __init smp_setup_processor_id(void)
        u64 mpidr = read_cpuid_mpidr() & MPIDR_HWID_BITMASK;
        set_cpu_logical_map(0, mpidr);
 
-       /*
-        * clear __my_cpu_offset on boot CPU to avoid hang caused by
-        * using percpu variable early, for example, lockdep will
-        * access percpu variable inside lock_release
-        */
-       set_my_cpu_offset(0);
        pr_info("Booting Linux on physical CPU 0x%010lx [0x%08x]\n",
                (unsigned long)mpidr, read_cpuid_id());
 }

diff --git a/arch/arm64/kernel/smp.c b/arch/arm64/kernel/smp.c
index 73625cc39574b1b36e5f40299b5d86a85b0f9cd9..2fe8fab886e2bfc38b65da442996f485823f200f 100644 (file)
--- a/arch/arm64/kernel/smp.c
+++ b/arch/arm64/kernel/smp.c
@@ -198,10 +198,7 @@ asmlinkage notrace void secondary_start_kernel(void)
        u64 mpidr = read_cpuid_mpidr() & MPIDR_HWID_BITMASK;
        struct mm_struct *mm = &init_mm;
        const struct cpu_operations *ops;
-       unsigned int cpu;
-
-       cpu = task_cpu(current);
-       set_my_cpu_offset(per_cpu_offset(cpu));
+       unsigned int cpu = smp_processor_id();
 
        /*
         * All kernel threads share the same mm context; grab a
@@ -448,6 +445,11 @@ void __init smp_cpus_done(unsigned int max_cpus)
 
 void __init smp_prepare_boot_cpu(void)
 {
+       /*
+        * The runtime per-cpu areas have been allocated by
+        * setup_per_cpu_areas(), and CPU0's boot time per-cpu area will be
+        * freed shortly, so we must move over to the runtime per-cpu area.
+        */
        set_my_cpu_offset(per_cpu_offset(smp_processor_id()));
        cpuinfo_store_boot_cpu();