@@ -167,6 +167,23 @@ struct bpf_verifier_ops {
union bpf_attr __user *uattr);
};
+/* These bpf_feature bits are 100% internal to the bpf infrastructure
+ * They mirror some of the bpf_prog bits, related to features. Over
+ * time these features bits will get removed when the subsystem using
+ * them, like XDP, will support the feature from all call points
+ * (e.g. XDP drivers).
+ */
+struct bpf_features {
+ union {
+ struct {
+ u32 cb_access:1,
+ xdp_rxhash_needed:1;
+ };
+ u32 flags;
+ };
+};
+typedef u32 bpf_features_t;
+
struct bpf_prog_aux {
atomic_t refcnt;
u32 used_map_cnt;
@@ -193,6 +210,15 @@ struct bpf_array {
*/
enum bpf_prog_type owner_prog_type;
bool owner_jited;
+
+ /* Restrict features allowed */
+ bpf_features_t features_supported;
+ bool features_locked;
+
+ /* Members needed when traversing prog_array tail calls */
+ struct list_head traversed_node;
+ bool is_traversed;
+
union {
char value[0] __aligned(8);
void *ptrs[0] __aligned(8);
@@ -641,6 +641,11 @@ static inline int sk_filter(struct sock *sk, struct sk_buff *skb)
return sk_filter_trim_cap(sk, skb, 1);
}
+netdev_features_t bpf_get_xdp_features(struct bpf_prog *xdp_prog);
+bool bpf_lock_xdp_features(struct bpf_prog *prog,
+ netdev_features_t xdp_approved_f,
+ netdev_features_t xdp_dev_support_f);
+
struct bpf_prog *bpf_prog_select_runtime(struct bpf_prog *fp, int *err);
void bpf_prog_free(struct bpf_prog *fp);
@@ -1224,21 +1224,241 @@ static unsigned int __bpf_prog_run(void *ctx, const struct bpf_insn *insn)
}
STACK_FRAME_NON_STANDARD(__bpf_prog_run); /* jump table */
+/* convert bpf_prog bits into bpf_features bits */
+static bpf_features_t __bpf_prog_to_features(const struct bpf_prog *prog)
+{
+ struct bpf_features f = { 0 };
+
+ if (prog->cb_access)
+ f.cb_access = 1;
+
+ if (prog->xdp_rxhash_needed)
+ f.xdp_rxhash_needed = 1;
+
+ return f.flags;
+}
+
+/* convert bpf_features bits into net_device xdp_features */
+static netdev_features_t __bpf_features_to_xdp_features(bpf_features_t f)
+{
+ netdev_features_t features = XDP_DRV_FEATURES_REQUIRED;
+ struct bpf_features bf;
+
+ bf.flags = f;
+ if (bf.xdp_rxhash_needed)
+ features |= XDP_DRV_F_RXHASH;
+
+ return features;
+}
+
+/* Extend bpf_features with extra features based on xdp_features input */
+bpf_features_t __bpf_features_extend_from_xdp_features(bpf_features_t bpf_f,
+ netdev_features_t xdp_f)
+{
+ struct bpf_features bf;
+
+ bf.flags = bpf_f;
+ if (xdp_f & XDP_DRV_F_RXHASH)
+ bf.xdp_rxhash_needed = 1;
+
+ return bf.flags;
+}
+
+static DEFINE_MUTEX(prog_array_traversal_mutex);
+static LIST_HEAD(prog_array_traversal_q);
+
+static bpf_features_t
+__bpf_features_via_prog_array(const struct bpf_prog *top_prog,
+ bpf_features_t features)
+{
+ struct bpf_prog_aux *aux = top_prog->aux;
+ int i;
+
+ /* First extract features from bpf_prog's in known prog_array's */
+ for (i = 0; i < aux->used_map_cnt; i++) {
+ struct bpf_map *map = aux->used_maps[i];
+ struct bpf_array *array;
+ int j;
+
+ /* Walk all prog_array's */
+ if (map->map_type != BPF_MAP_TYPE_PROG_ARRAY)
+ continue;
+ array = container_of(map, struct bpf_array, map);
+
+ /* Look at features in each active bpf_prog in prog_array */
+ for (j = 0; j < array->map.max_entries; j++) {
+ const struct bpf_prog *prog;
+
+ prog = array->ptrs[j];
+ if (!prog)
+ continue;
+
+ features |= __bpf_prog_to_features(prog);
+ }
+ }
+
+ /* Now recursive visit bpf_prog's for containing prog_array's */
+ for (i = 0; i < aux->used_map_cnt; i++) {
+ struct bpf_map *map = aux->used_maps[i];
+ struct bpf_array *array;
+ int j;
+
+ /* Walk all prog_array's again */
+ if (map->map_type != BPF_MAP_TYPE_PROG_ARRAY)
+ continue;
+ array = container_of(map, struct bpf_array, map);
+
+ /* Avoid traversal loops and record prog_array's */
+ if (array->is_traversed)
+ continue;
+ array->is_traversed = true;
+ list_add_tail(&array->traversed_node, &prog_array_traversal_q);
+
+ /* Recurse into bpf_prog in prog_array */
+ for (j = 0; j < array->map.max_entries; j++) {
+ const struct bpf_prog *p;
+
+ p = array->ptrs[j];
+ if (!p)
+ continue;
+
+ features |= __bpf_features_via_prog_array(p, features);
+ }
+ }
+
+ return features;
+}
+
+/* Find superset of features traversing tail call maps */
+static bpf_features_t bpf_features_via_prog_array(const struct bpf_prog *prog,
+ bpf_features_t features)
+{
+ struct bpf_array *prog_array, *tmp;
+
+ features |= __bpf_features_via_prog_array(prog, features);
+ list_for_each_entry_safe(prog_array, tmp, &prog_array_traversal_q,
+ traversed_node)
+ {
+ list_del(&prog_array->traversed_node);
+ prog_array->is_traversed = false;
+ }
+
+ return features;
+}
+
+netdev_features_t bpf_get_xdp_features(struct bpf_prog *prog)
+{
+ bpf_features_t bpf_features;
+
+ mutex_lock(&prog_array_traversal_mutex);
+ bpf_features = __bpf_prog_to_features(prog);
+ bpf_features = bpf_features_via_prog_array(prog, bpf_features);
+ mutex_unlock(&prog_array_traversal_mutex);
+
+ return __bpf_features_to_xdp_features(bpf_features);
+}
+
+/* Caller have checked xdp features are approved */
+bool bpf_lock_xdp_features(struct bpf_prog *prog,
+ netdev_features_t xdp_approved_f,
+ netdev_features_t xdp_dev_support_f)
+{
+ struct bpf_array *prog_array, *tmp;
+ netdev_features_t xdp_f_in_use;
+ bpf_features_t bpf_f_in_use;
+ bool lock_features = true;
+ bpf_features_t max;
+
+ mutex_lock(&prog_array_traversal_mutex);
+
+ /* Get and detect if bpf_features changed */
+ bpf_f_in_use = __bpf_prog_to_features(prog);
+ bpf_f_in_use |= __bpf_features_via_prog_array(prog, bpf_f_in_use);
+ xdp_f_in_use = __bpf_features_to_xdp_features(bpf_f_in_use);
+ if (xdp_f_in_use != xdp_approved_f)
+ lock_features = false;
+
+ /* XDP driver might support more features than in-use, allow
+ * later added bpf_prog's to still use-these extra features
+ */
+ max = __bpf_features_extend_from_xdp_features(bpf_f_in_use,
+ xdp_dev_support_f);
+ list_for_each_entry_safe(prog_array, tmp, &prog_array_traversal_q,
+ traversed_node)
+ {
+ list_del(&prog_array->traversed_node);
+ prog_array->is_traversed = false;
+ if (lock_features) {
+ /* Handle when already locked by another driver.
+ * Find smallest common feature set (via simple AND)
+ */
+ if (prog_array->features_locked)
+ prog_array->features_supported &= max;
+ else
+ prog_array->features_supported = max;
+ prog_array->features_locked = true;
+ }
+ }
+ mutex_unlock(&prog_array_traversal_mutex);
+ return lock_features;
+}
+
bool bpf_prog_array_compatible(struct bpf_array *array,
const struct bpf_prog *fp)
{
+ bool compat = false;
+
+ mutex_lock(&prog_array_traversal_mutex);
+
if (!array->owner_prog_type) {
/* There's no owner yet where we could check for
* compatibility.
*/
array->owner_prog_type = fp->type;
array->owner_jited = fp->jited;
+ array->features_locked = false;
+ array->is_traversed = false;
+ compat = true;
+ goto out;
+ }
+
+ /* Features can be locked, e.g. when XDP prog is attach to net_device */
+ if (array->features_locked)
+ {
+ bpf_features_t f = __bpf_prog_to_features(fp);
+ struct bpf_array *pa, *tmp;
+ bpf_features_t max;
- return true;
+ f |= __bpf_features_via_prog_array(fp, f);
+
+ /* Detect any feature bit set, which is not supported */
+ if (f & ~(array->features_supported)) {
+ compat = false;
+ goto out;
+ }
+ /* If fp contained tail call's itself, they need to be
+ * locked down, to this array->features_supported.
+ */
+ max = array->features_supported;
+ list_for_each_entry_safe(pa, tmp, &prog_array_traversal_q,
+ traversed_node)
+ {
+ list_del(&pa->traversed_node);
+ pa->is_traversed = false;
+ /* Handle when already locked by another driver */
+ if (pa->features_locked)
+ pa->features_supported &= max;
+ else
+ pa->features_supported = max;
+ pa->features_locked = true;
+ }
}
- return array->owner_prog_type == fp->type &&
- array->owner_jited == fp->jited;
+ compat = (array->owner_prog_type == fp->type &&
+ array->owner_jited == fp->jited);
+out:
+ mutex_unlock(&prog_array_traversal_mutex);
+ return compat;
}
static int bpf_check_tail_call(const struct bpf_prog *fp)
@@ -3355,7 +3355,6 @@ static int fixup_bpf_calls(struct bpf_verifier_env *env)
* the program array.
*/
prog->cb_access = 1;
- prog->xdp_rxhash_needed = 1;
/* mark bpf_tail_call as different opcode to avoid
* conditional branch in the interpeter for every normal
@@ -6855,16 +6855,6 @@ int dev_change_proto_down(struct net_device *dev, bool proto_down)
}
EXPORT_SYMBOL(dev_change_proto_down);
-netdev_features_t bpf_get_xdp_features(struct bpf_prog *prog)
-{
- netdev_features_t features = XDP_DRV_FEATURES_REQUIRED;
-
- if (prog->xdp_rxhash_needed)
- features |= XDP_DRV_F_RXHASH;
-
- return features;
-}
-
bool xdp_features_check(struct net_device *dev, struct bpf_prog *xdp_prog,
struct netlink_ext_ack *extack, u32 flags)
{
@@ -6881,6 +6871,14 @@ bool xdp_features_check(struct net_device *dev, struct bpf_prog *xdp_prog,
"Required XDP feature not supported by device");
return false;
}
+ /* Ask BPF infra to limit runtime added bpf_prog's (tail calls)
+ * to features supported by XDP driver.
+ */
+ if (!bpf_lock_xdp_features(xdp_prog, req_features, dev_xdp_features)) {
+ NL_SET_ERR_MSG(extack,
+ "Couldn't lock XDP features supported by device");
+ return false;
+ }
return true;
}