bpf: preserve STACK_ZERO slots on partial reg spills

Instead of always forcing STACK_ZERO slots to STACK_MISC, preserve it in
situations where this is possible. E.g., when spilling register as
1/2/4-byte subslots on the stack, all the remaining bytes in the stack
slot do not automatically become unknown. If we knew they contained
zeroes, we can preserve those STACK_ZERO markers.

Add a helper mark_stack_slot_misc(), similar to scrub_spilled_slot(),
but that doesn't overwrite either STACK_INVALID nor STACK_ZERO. Note
that we need to take into account possibility of being in unprivileged
mode, in which case STACK_INVALID is forced to STACK_MISC for correctness,
as treating STACK_INVALID as equivalent STACK_MISC is only enabled in
privileged mode.

Acked-by: Eduard Zingerman <eddyz87@gmail.com>
Signed-off-by: Andrii Nakryiko <andrii@kernel.org>
Link: https://lore.kernel.org/r/20231205184248.1502704-5-andrii@kernel.org
Signed-off-by: Alexei Starovoitov <ast@kernel.org>

diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c
index 3edca06de9fd97e3d7bbe5c3503e7531eba5fd81..93de39a6e36e5c96845ab2abe10dc495bef0cfcc 100644 (file)
--- a/kernel/bpf/verifier.c
+++ b/kernel/bpf/verifier.c
@@ -1144,6 +1144,21 @@ static bool is_spilled_scalar_reg(const struct bpf_stack_state *stack)
               stack->spilled_ptr.type == SCALAR_VALUE;
 }
 
+/* Mark stack slot as STACK_MISC, unless it is already STACK_INVALID, in which
+ * case they are equivalent, or it's STACK_ZERO, in which case we preserve
+ * more precise STACK_ZERO.
+ * Note, in uprivileged mode leaving STACK_INVALID is wrong, so we take
+ * env->allow_ptr_leaks into account and force STACK_MISC, if necessary.
+ */
+static void mark_stack_slot_misc(struct bpf_verifier_env *env, u8 *stype)
+{
+       if (*stype == STACK_ZERO)
+               return;
+       if (env->allow_ptr_leaks && *stype == STACK_INVALID)
+               return;
+       *stype = STACK_MISC;
+}
+
 static void scrub_spilled_slot(u8 *stype)
 {
        if (*stype != STACK_INVALID)
@@ -4386,7 +4401,8 @@ static void copy_register_state(struct bpf_reg_state *dst, const struct bpf_reg_
        dst->live = live;
 }
 
-static void save_register_state(struct bpf_func_state *state,
+static void save_register_state(struct bpf_verifier_env *env,
+                               struct bpf_func_state *state,
                                int spi, struct bpf_reg_state *reg,
                                int size)
 {
@@ -4401,7 +4417,7 @@ static void save_register_state(struct bpf_func_state *state,
 
        /* size < 8 bytes spill */
        for (; i; i--)
-               scrub_spilled_slot(&state->stack[spi].slot_type[i - 1]);
+               mark_stack_slot_misc(env, &state->stack[spi].slot_type[i - 1]);
 }
 
 static bool is_bpf_st_mem(struct bpf_insn *insn)
@@ -4463,7 +4479,7 @@ static int check_stack_write_fixed_off(struct bpf_verifier_env *env,
        mark_stack_slot_scratched(env, spi);
        if (reg && !(off % BPF_REG_SIZE) && register_is_bounded(reg) &&
            !register_is_null(reg) && env->bpf_capable) {
-               save_register_state(state, spi, reg, size);
+               save_register_state(env, state, spi, reg, size);
                /* Break the relation on a narrowing spill. */
                if (fls64(reg->umax_value) > BITS_PER_BYTE * size)
                        state->stack[spi].spilled_ptr.id = 0;
@@ -4473,7 +4489,7 @@ static int check_stack_write_fixed_off(struct bpf_verifier_env *env,
 
                __mark_reg_known(&fake_reg, insn->imm);
                fake_reg.type = SCALAR_VALUE;
-               save_register_state(state, spi, &fake_reg, size);
+               save_register_state(env, state, spi, &fake_reg, size);
                insn_flags = 0; /* not a register spill */
        } else if (reg && is_spillable_regtype(reg->type)) {
                /* register containing pointer is being spilled into stack */
@@ -4486,7 +4502,7 @@ static int check_stack_write_fixed_off(struct bpf_verifier_env *env,
                        verbose(env, "cannot spill pointers to stack into stack frame of the caller\n");
                        return -EINVAL;
                }
-               save_register_state(state, spi, reg, size);
+               save_register_state(env, state, spi, reg, size);
        } else {
                u8 type = STACK_MISC;
 
@@ -4757,6 +4773,8 @@ static int check_stack_read_fixed_off(struct bpf_verifier_env *env,
                                                continue;
                                        if (type == STACK_MISC)
                                                continue;
+                                       if (type == STACK_ZERO)
+                                               continue;
                                        if (type == STACK_INVALID && env->allow_uninit_stack)
                                                continue;
                                        verbose(env, "invalid read from stack off %d+%d size %d\n",