bpf: Add BPF_KPTR_PERCPU as a field type

BPF_KPTR_PERCPU represents a percpu field type like below

  struct val_t {
    ... fields ...
  };
  struct t {
    ...
    struct val_t __percpu_kptr *percpu_data_ptr;
    ...
  };

where
  #define __percpu_kptr __attribute__((btf_type_tag("percpu_kptr")))

While BPF_KPTR_REF points to a trusted kernel object or a trusted
local object, BPF_KPTR_PERCPU points to a trusted local
percpu object.

This patch added basic support for BPF_KPTR_PERCPU
related to percpu_kptr field parsing, recording and free operations.
BPF_KPTR_PERCPU also supports the same map types
as BPF_KPTR_REF does.

Note that unlike a local kptr, it is possible that
a BPF_KTPR_PERCPU struct may not contain any
special fields like other kptr, bpf_spin_lock, bpf_list_head, etc.

Signed-off-by: Yonghong Song <yonghong.song@linux.dev>
Link: https://lore.kernel.org/r/20230827152739.1996391-1-yonghong.song@linux.dev
Signed-off-by: Alexei Starovoitov <ast@kernel.org>

diff --git a/include/linux/bpf.h b/include/linux/bpf.h
index 440dd1f59a1c3ad2d298dde38be4bf4d2837b1bb..87eeb3a46a1d52b3dcf3cccf836bb65ab5154fd5 100644 (file)
--- a/include/linux/bpf.h
+++ b/include/linux/bpf.h
@@ -180,14 +180,15 @@ enum btf_field_type {
        BPF_TIMER      = (1 << 1),
        BPF_KPTR_UNREF = (1 << 2),
        BPF_KPTR_REF   = (1 << 3),
-       BPF_KPTR       = BPF_KPTR_UNREF | BPF_KPTR_REF,
-       BPF_LIST_HEAD  = (1 << 4),
-       BPF_LIST_NODE  = (1 << 5),
-       BPF_RB_ROOT    = (1 << 6),
-       BPF_RB_NODE    = (1 << 7),
+       BPF_KPTR_PERCPU = (1 << 4),
+       BPF_KPTR       = BPF_KPTR_UNREF | BPF_KPTR_REF | BPF_KPTR_PERCPU,
+       BPF_LIST_HEAD  = (1 << 5),
+       BPF_LIST_NODE  = (1 << 6),
+       BPF_RB_ROOT    = (1 << 7),
+       BPF_RB_NODE    = (1 << 8),
        BPF_GRAPH_NODE_OR_ROOT = BPF_LIST_NODE | BPF_LIST_HEAD |
                                 BPF_RB_NODE | BPF_RB_ROOT,
-       BPF_REFCOUNT   = (1 << 8),
+       BPF_REFCOUNT   = (1 << 9),
 };
 
 typedef void (*btf_dtor_kfunc_t)(void *);
@@ -300,6 +301,8 @@ static inline const char *btf_field_type_name(enum btf_field_type type)
        case BPF_KPTR_UNREF:
        case BPF_KPTR_REF:
                return "kptr";
+       case BPF_KPTR_PERCPU:
+               return "percpu_kptr";
        case BPF_LIST_HEAD:
                return "bpf_list_head";
        case BPF_LIST_NODE:
@@ -325,6 +328,7 @@ static inline u32 btf_field_type_size(enum btf_field_type type)
                return sizeof(struct bpf_timer);
        case BPF_KPTR_UNREF:
        case BPF_KPTR_REF:
+       case BPF_KPTR_PERCPU:
                return sizeof(u64);
        case BPF_LIST_HEAD:
                return sizeof(struct bpf_list_head);
@@ -351,6 +355,7 @@ static inline u32 btf_field_type_align(enum btf_field_type type)
                return __alignof__(struct bpf_timer);
        case BPF_KPTR_UNREF:
        case BPF_KPTR_REF:
+       case BPF_KPTR_PERCPU:
                return __alignof__(u64);
        case BPF_LIST_HEAD:
                return __alignof__(struct bpf_list_head);
@@ -389,6 +394,7 @@ static inline void bpf_obj_init_field(const struct btf_field *field, void *addr)
        case BPF_TIMER:
        case BPF_KPTR_UNREF:
        case BPF_KPTR_REF:
+       case BPF_KPTR_PERCPU:
                break;
        default:
                WARN_ON_ONCE(1);

diff --git a/kernel/bpf/btf.c b/kernel/bpf/btf.c
index 1095bbe2985930f486010f955d96ee7b57d0a8a1..187b57276fec01f438868715b9e1e0845d1b5a72 100644 (file)
--- a/kernel/bpf/btf.c
+++ b/kernel/bpf/btf.c
@@ -3293,6 +3293,8 @@ static int btf_find_kptr(const struct btf *btf, const struct btf_type *t,
                type = BPF_KPTR_UNREF;
        else if (!strcmp("kptr", __btf_name_by_offset(btf, t->name_off)))
                type = BPF_KPTR_REF;
+       else if (!strcmp("percpu_kptr", __btf_name_by_offset(btf, t->name_off)))
+               type = BPF_KPTR_PERCPU;
        else
                return -EINVAL;
 
@@ -3457,6 +3459,7 @@ static int btf_find_struct_field(const struct btf *btf,
                        break;
                case BPF_KPTR_UNREF:
                case BPF_KPTR_REF:
+               case BPF_KPTR_PERCPU:
                        ret = btf_find_kptr(btf, member_type, off, sz,
                                            idx < info_cnt ? &info[idx] : &tmp);
                        if (ret < 0)
@@ -3523,6 +3526,7 @@ static int btf_find_datasec_var(const struct btf *btf, const struct btf_type *t,
                        break;
                case BPF_KPTR_UNREF:
                case BPF_KPTR_REF:
+               case BPF_KPTR_PERCPU:
                        ret = btf_find_kptr(btf, var_type, off, sz,
                                            idx < info_cnt ? &info[idx] : &tmp);
                        if (ret < 0)
@@ -3783,6 +3787,7 @@ struct btf_record *btf_parse_fields(const struct btf *btf, const struct btf_type
                        break;
                case BPF_KPTR_UNREF:
                case BPF_KPTR_REF:
+               case BPF_KPTR_PERCPU:
                        ret = btf_parse_kptr(btf, &rec->fields[i], &info_arr[i]);
                        if (ret < 0)
                                goto end;

diff --git a/kernel/bpf/syscall.c b/kernel/bpf/syscall.c
index eb01c31ed591dde57dc8463a7d701dc46132c8f7..6a692f3bea1500b3b48b2f8364e3bc9f3b04e63c 100644 (file)
--- a/kernel/bpf/syscall.c
+++ b/kernel/bpf/syscall.c
@@ -514,6 +514,7 @@ void btf_record_free(struct btf_record *rec)
                switch (rec->fields[i].type) {
                case BPF_KPTR_UNREF:
                case BPF_KPTR_REF:
+               case BPF_KPTR_PERCPU:
                        if (rec->fields[i].kptr.module)
                                module_put(rec->fields[i].kptr.module);
                        btf_put(rec->fields[i].kptr.btf);
@@ -560,6 +561,7 @@ struct btf_record *btf_record_dup(const struct btf_record *rec)
                switch (fields[i].type) {
                case BPF_KPTR_UNREF:
                case BPF_KPTR_REF:
+               case BPF_KPTR_PERCPU:
                        btf_get(fields[i].kptr.btf);
                        if (fields[i].kptr.module && !try_module_get(fields[i].kptr.module)) {
                                ret = -ENXIO;
@@ -650,6 +652,7 @@ void bpf_obj_free_fields(const struct btf_record *rec, void *obj)
                        WRITE_ONCE(*(u64 *)field_ptr, 0);
                        break;
                case BPF_KPTR_REF:
+               case BPF_KPTR_PERCPU:
                        xchgd_field = (void *)xchg((unsigned long *)field_ptr, 0);
                        if (!xchgd_field)
                                break;
@@ -1045,6 +1048,7 @@ static int map_check_btf(struct bpf_map *map, const struct btf *btf,
                                break;
                        case BPF_KPTR_UNREF:
                        case BPF_KPTR_REF:
+                       case BPF_KPTR_PERCPU:
                        case BPF_REFCOUNT:
                                if (map->map_type != BPF_MAP_TYPE_HASH &&
                                    map->map_type != BPF_MAP_TYPE_PERCPU_HASH &&