bpf, x86: fix freeing of not-finalized bpf_prog_pack

syzbot reported a few issues with bpf_prog_pack [1], [2]. This only happens
with multiple subprogs. In jit_subprogs(), we first call bpf_int_jit_compile()
on each sub program. And then, we call it on each sub program again. jit_data
is not freed in the first call of bpf_int_jit_compile(). Similarly we don't
call bpf_jit_binary_pack_finalize() in the first call of bpf_int_jit_compile().

If bpf_int_jit_compile() failed for one sub program, we will call
bpf_jit_binary_pack_finalize() for this sub program. However, we don't have a
chance to call it for other sub programs. Then we will hit "goto out_free" in
jit_subprogs(), and call bpf_jit_free on some subprograms that haven't got
bpf_jit_binary_pack_finalize() yet.

At this point, bpf_jit_binary_pack_free() is called and the whole 2MB page is
freed erroneously.

Fix this with a custom bpf_jit_free() for x86_64, which calls
bpf_jit_binary_pack_finalize() if necessary. Also, with custom
bpf_jit_free(), bpf_prog_aux->use_bpf_prog_pack is not needed any more,
remove it.

Fixes: 1022a5498f6f ("bpf, x86_64: Use bpf_jit_binary_pack_alloc")
[1] https://syzkaller.appspot.com/bug?extid=2f649ec6d2eea1495a8f
[2] https://syzkaller.appspot.com/bug?extid=87f65c75f4a72db05445
Reported-by: syzbot+2f649ec6d2eea1495a8f@syzkaller.appspotmail.com
Reported-by: syzbot+87f65c75f4a72db05445@syzkaller.appspotmail.com
Signed-off-by: Song Liu <song@kernel.org>
Link: https://lore.kernel.org/r/20220706002612.4013790-1-song@kernel.org
Signed-off-by: Alexei Starovoitov <ast@kernel.org>

diff --git a/arch/x86/net/bpf_jit_comp.c b/arch/x86/net/bpf_jit_comp.c
index d2614f1bf838b3e95a7e6064921a97bcf27d669c..54c7f46c453fbd2ebcc66ea32de1f684e2b7738b 100644 (file)
--- a/arch/x86/net/bpf_jit_comp.c
+++ b/arch/x86/net/bpf_jit_comp.c
@@ -2486,3 +2486,28 @@ bool bpf_jit_supports_subprog_tailcalls(void)
 {
        return true;
 }
+
+void bpf_jit_free(struct bpf_prog *prog)
+{
+       if (prog->jited) {
+               struct x64_jit_data *jit_data = prog->aux->jit_data;
+               struct bpf_binary_header *hdr;
+
+               /*
+                * If we fail the final pass of JIT (from jit_subprogs),
+                * the program may not be finalized yet. Call finalize here
+                * before freeing it.
+                */
+               if (jit_data) {
+                       bpf_jit_binary_pack_finalize(prog, jit_data->header,
+                                                    jit_data->rw_header);
+                       kvfree(jit_data->addrs);
+                       kfree(jit_data);
+               }
+               hdr = bpf_jit_binary_pack_hdr(prog);
+               bpf_jit_binary_pack_free(hdr, NULL);
+               WARN_ON_ONCE(!bpf_prog_kallsyms_verify_off(prog));
+       }
+
+       bpf_prog_unlock_free(prog);
+}

diff --git a/include/linux/bpf.h b/include/linux/bpf.h
index 85a4db3e05365a21636c6c332b9094c30094e580..a5bf00649995eb4074775bee5512929f09202919 100644 (file)
--- a/include/linux/bpf.h
+++ b/include/linux/bpf.h
@@ -1044,7 +1044,6 @@ struct bpf_prog_aux {
        bool sleepable;
        bool tail_call_reachable;
        bool xdp_has_frags;
-       bool use_bpf_prog_pack;
        /* BTF_KIND_FUNC_PROTO for valid attach_btf_id */
        const struct btf_type *attach_func_proto;
        /* function name for valid attach_btf_id */

diff --git a/include/linux/filter.h b/include/linux/filter.h
index 4c1a8b247545038e6bc28abd194b4e9123f0abc2..a5f21dc3c4327105d10de6c1665955b3b1bee2f1 100644 (file)
--- a/include/linux/filter.h
+++ b/include/linux/filter.h
@@ -1027,6 +1027,14 @@ u64 bpf_jit_alloc_exec_limit(void);
 void *bpf_jit_alloc_exec(unsigned long size);
 void bpf_jit_free_exec(void *addr);
 void bpf_jit_free(struct bpf_prog *fp);
+struct bpf_binary_header *
+bpf_jit_binary_pack_hdr(const struct bpf_prog *fp);
+
+static inline bool bpf_prog_kallsyms_verify_off(const struct bpf_prog *fp)
+{
+       return list_empty(&fp->aux->ksym.lnode) ||
+              fp->aux->ksym.lnode.prev == LIST_POISON2;
+}
 
 struct bpf_binary_header *
 bpf_jit_binary_pack_alloc(unsigned int proglen, u8 **ro_image,

diff --git a/kernel/bpf/core.c b/kernel/bpf/core.c
index 805c2ad5c79370b0028b3297486cf8c1b44bc504..cfb8a50a9f12f47c68ba77db6820f1edc128e752 100644 (file)
--- a/kernel/bpf/core.c
+++ b/kernel/bpf/core.c
@@ -650,12 +650,6 @@ static bool bpf_prog_kallsyms_candidate(const struct bpf_prog *fp)
        return fp->jited && !bpf_prog_was_classic(fp);
 }
 
-static bool bpf_prog_kallsyms_verify_off(const struct bpf_prog *fp)
-{
-       return list_empty(&fp->aux->ksym.lnode) ||
-              fp->aux->ksym.lnode.prev == LIST_POISON2;
-}
-
 void bpf_prog_kallsyms_add(struct bpf_prog *fp)
 {
        if (!bpf_prog_kallsyms_candidate(fp) ||
@@ -1153,7 +1147,6 @@ int bpf_jit_binary_pack_finalize(struct bpf_prog *prog,
                bpf_prog_pack_free(ro_header);
                return PTR_ERR(ptr);
        }
-       prog->aux->use_bpf_prog_pack = true;
        return 0;
 }
 
@@ -1177,17 +1170,23 @@ void bpf_jit_binary_pack_free(struct bpf_binary_header *ro_header,
        bpf_jit_uncharge_modmem(size);
 }
 
+struct bpf_binary_header *
+bpf_jit_binary_pack_hdr(const struct bpf_prog *fp)
+{
+       unsigned long real_start = (unsigned long)fp->bpf_func;
+       unsigned long addr;
+
+       addr = real_start & BPF_PROG_CHUNK_MASK;
+       return (void *)addr;
+}
+
 static inline struct bpf_binary_header *
 bpf_jit_binary_hdr(const struct bpf_prog *fp)
 {
        unsigned long real_start = (unsigned long)fp->bpf_func;
        unsigned long addr;
 
-       if (fp->aux->use_bpf_prog_pack)
-               addr = real_start & BPF_PROG_CHUNK_MASK;
-       else
-               addr = real_start & PAGE_MASK;
-
+       addr = real_start & PAGE_MASK;
        return (void *)addr;
 }
 
@@ -1200,11 +1199,7 @@ void __weak bpf_jit_free(struct bpf_prog *fp)
        if (fp->jited) {
                struct bpf_binary_header *hdr = bpf_jit_binary_hdr(fp);
 
-               if (fp->aux->use_bpf_prog_pack)
-                       bpf_jit_binary_pack_free(hdr, NULL /* rw_buffer */);
-               else
-                       bpf_jit_binary_free(hdr);
-
+               bpf_jit_binary_free(hdr);
                WARN_ON_ONCE(!bpf_prog_kallsyms_verify_off(fp));
        }