From patchwork Sun May 31 21:46:46 2020 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Lorenzo Bianconi X-Patchwork-Id: 1301516 X-Patchwork-Delegate: bpf@iogearbox.net Return-Path: X-Original-To: incoming-bpf@patchwork.ozlabs.org Delivered-To: patchwork-incoming-bpf@bilbo.ozlabs.org Authentication-Results: ozlabs.org; spf=pass (sender SPF authorized) smtp.mailfrom=vger.kernel.org (client-ip=23.128.96.18; helo=vger.kernel.org; envelope-from=bpf-owner@vger.kernel.org; receiver=) Authentication-Results: ozlabs.org; dmarc=pass (p=none dis=none) header.from=kernel.org Authentication-Results: ozlabs.org; dkim=pass (1024-bit key; unprotected) header.d=kernel.org header.i=@kernel.org header.a=rsa-sha256 header.s=default header.b=n5MioE/W; dkim-atps=neutral Received: from vger.kernel.org (vger.kernel.org [23.128.96.18]) by ozlabs.org (Postfix) with ESMTP id 49ZsNZ6WZCz9sT8 for ; Mon, 1 Jun 2020 07:47:42 +1000 (AEST) Received: (majordomo@vger.kernel.org) by vger.kernel.org via listexpand id S1728352AbgEaVrY (ORCPT ); Sun, 31 May 2020 17:47:24 -0400 Received: from mail.kernel.org ([198.145.29.99]:37102 "EHLO mail.kernel.org" rhost-flags-OK-OK-OK-OK) by vger.kernel.org with ESMTP id S1725860AbgEaVrY (ORCPT ); Sun, 31 May 2020 17:47:24 -0400 Received: from lore-desk.lan (unknown [151.48.128.87]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by mail.kernel.org (Postfix) with ESMTPSA id 0994D206F1; Sun, 31 May 2020 21:47:20 +0000 (UTC) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/simple; d=kernel.org; s=default; t=1590961643; bh=Ki2F541Wbf8/Su6oBZd3PjYjymW5xaUDTo9NkkgkQcY=; h=From:To:Cc:Subject:Date:In-Reply-To:References:From; b=n5MioE/W/grqG7yBaDCw255hMjfv0guC1bTdtEwKZRAd02s2JWX3VVZQ7O4rCdO0J sC0hJJurPFNL1/cHCJE5LFflTwStpcRlgDD0K0++vE+y8vX2zX6q5hn35SzaikgMox YhHTbJPZp8VQ03GaqAhfXyvTRQpZKEIuTQYKvamQ= From: Lorenzo Bianconi To: bpf@vger.kernel.org, netdev@vger.kernel.org Cc: davem@davemloft.net, ast@kernel.org, brouer@redhat.com, toke@redhat.com, daniel@iogearbox.net, lorenzo.bianconi@redhat.com, dsahern@kernel.org, David Ahern Subject: [PATCH bpf-next 1/6] net: Refactor xdp_convert_buff_to_frame Date: Sun, 31 May 2020 23:46:46 +0200 Message-Id: <3dabeed5867243f624f5fb79c81dc9629a53677c.1590960613.git.lorenzo@kernel.org> X-Mailer: git-send-email 2.26.2 In-Reply-To: References: MIME-Version: 1.0 Sender: bpf-owner@vger.kernel.org Precedence: bulk List-ID: X-Mailing-List: bpf@vger.kernel.org From: David Ahern Move the guts of xdp_convert_buff_to_frame to a new helper, xdp_update_frame_from_buff so it can be reused removing code duplication Suggested-by: Jesper Dangaard Brouer Co-developed-by: Lorenzo Bianconi Signed-off-by: Lorenzo Bianconi Signed-off-by: David Ahern --- include/net/xdp.h | 35 ++++++++++++++++++++++------------- 1 file changed, 22 insertions(+), 13 deletions(-) diff --git a/include/net/xdp.h b/include/net/xdp.h index 609f819ed08b..ab1c503808a4 100644 --- a/include/net/xdp.h +++ b/include/net/xdp.h @@ -121,39 +121,48 @@ void xdp_convert_frame_to_buff(struct xdp_frame *frame, struct xdp_buff *xdp) xdp->frame_sz = frame->frame_sz; } -/* Convert xdp_buff to xdp_frame */ static inline -struct xdp_frame *xdp_convert_buff_to_frame(struct xdp_buff *xdp) +int xdp_update_frame_from_buff(struct xdp_buff *xdp, + struct xdp_frame *xdp_frame) { - struct xdp_frame *xdp_frame; - int metasize; - int headroom; - - if (xdp->rxq->mem.type == MEM_TYPE_XSK_BUFF_POOL) - return xdp_convert_zc_to_xdp_frame(xdp); + int metasize, headroom; /* Assure headroom is available for storing info */ headroom = xdp->data - xdp->data_hard_start; metasize = xdp->data - xdp->data_meta; metasize = metasize > 0 ? metasize : 0; if (unlikely((headroom - metasize) < sizeof(*xdp_frame))) - return NULL; + return -ENOMEM; /* Catch if driver didn't reserve tailroom for skb_shared_info */ if (unlikely(xdp->data_end > xdp_data_hard_end(xdp))) { XDP_WARN("Driver BUG: missing reserved tailroom"); - return NULL; + return -ENOMEM; } - /* Store info in top of packet */ - xdp_frame = xdp->data_hard_start; - xdp_frame->data = xdp->data; xdp_frame->len = xdp->data_end - xdp->data; xdp_frame->headroom = headroom - sizeof(*xdp_frame); xdp_frame->metasize = metasize; xdp_frame->frame_sz = xdp->frame_sz; + return 0; +} + +/* Convert xdp_buff to xdp_frame */ +static inline +struct xdp_frame *xdp_convert_buff_to_frame(struct xdp_buff *xdp) +{ + struct xdp_frame *xdp_frame; + + if (xdp->rxq->mem.type == MEM_TYPE_XSK_BUFF_POOL) + return xdp_convert_zc_to_xdp_frame(xdp); + + /* Store info in top of packet */ + xdp_frame = xdp->data_hard_start; + if (unlikely(xdp_update_frame_from_buff(xdp, xdp_frame) < 0)) + return NULL; + /* rxq only valid until napi_schedule ends, convert to xdp_mem_info */ xdp_frame->mem = xdp->rxq->mem; From patchwork Sun May 31 21:46:47 2020 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Lorenzo Bianconi X-Patchwork-Id: 1301517 X-Patchwork-Delegate: bpf@iogearbox.net Return-Path: X-Original-To: incoming-bpf@patchwork.ozlabs.org Delivered-To: patchwork-incoming-bpf@bilbo.ozlabs.org Authentication-Results: ozlabs.org; spf=pass (sender SPF authorized) smtp.mailfrom=vger.kernel.org (client-ip=23.128.96.18; helo=vger.kernel.org; envelope-from=bpf-owner@vger.kernel.org; receiver=) Authentication-Results: ozlabs.org; dmarc=pass (p=none dis=none) header.from=kernel.org Authentication-Results: ozlabs.org; dkim=pass (1024-bit key; unprotected) header.d=kernel.org header.i=@kernel.org header.a=rsa-sha256 header.s=default header.b=srMV+6+W; dkim-atps=neutral Received: from vger.kernel.org (vger.kernel.org [23.128.96.18]) by ozlabs.org (Postfix) with ESMTP id 49ZsNb2YxVz9sT9 for ; Mon, 1 Jun 2020 07:47:43 +1000 (AEST) Received: (majordomo@vger.kernel.org) by vger.kernel.org via listexpand id S1728445AbgEaVrc (ORCPT ); Sun, 31 May 2020 17:47:32 -0400 Received: from mail.kernel.org ([198.145.29.99]:37164 "EHLO mail.kernel.org" rhost-flags-OK-OK-OK-OK) by vger.kernel.org with ESMTP id S1728395AbgEaVrc (ORCPT ); Sun, 31 May 2020 17:47:32 -0400 Received: from lore-desk.lan (unknown [151.48.128.87]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by mail.kernel.org (Postfix) with ESMTPSA id 6F816206B6; Sun, 31 May 2020 21:47:28 +0000 (UTC) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/simple; d=kernel.org; s=default; t=1590961651; bh=r2dz4/6SeaD2r1KcLWTBrGCsEdT32F/buQiaLm3uwsU=; h=From:To:Cc:Subject:Date:In-Reply-To:References:From; b=srMV+6+WFMCr/TW6YHDj+qnmVjiKtzGNhVXeMJC8Ir/pwy0Z4lYzKEfOVTSZ8KIDs owx+lmzukSkxv0IBEmAs8pRbjrpavBh4IFHOtvlhnDKL2N0RyldjxiOLcKAwJ9n8Gg xkfVLlmfR6wjKAc34GSUzBkBI/t+e20P/zi53g+g= From: Lorenzo Bianconi To: bpf@vger.kernel.org, netdev@vger.kernel.org Cc: davem@davemloft.net, ast@kernel.org, brouer@redhat.com, toke@redhat.com, daniel@iogearbox.net, lorenzo.bianconi@redhat.com, dsahern@kernel.org Subject: [PATCH bpf-next 2/6] samples/bpf: xdp_redirect_cpu_user: do not update bpf maps in option loop Date: Sun, 31 May 2020 23:46:47 +0200 Message-Id: X-Mailer: git-send-email 2.26.2 In-Reply-To: References: MIME-Version: 1.0 Sender: bpf-owner@vger.kernel.org Precedence: bulk List-ID: X-Mailing-List: bpf@vger.kernel.org Do not update xdp_redirect_cpu maps running while option loop but defer it after all available options have been parsed. This is a preliminary patch to pass the program name we want to attach to the map entries as a user option Signed-off-by: Lorenzo Bianconi --- samples/bpf/xdp_redirect_cpu_user.c | 36 +++++++++++++++++++++-------- 1 file changed, 27 insertions(+), 9 deletions(-) diff --git a/samples/bpf/xdp_redirect_cpu_user.c b/samples/bpf/xdp_redirect_cpu_user.c index f3468168982e..1a054737c35a 100644 --- a/samples/bpf/xdp_redirect_cpu_user.c +++ b/samples/bpf/xdp_redirect_cpu_user.c @@ -684,6 +684,7 @@ int main(int argc, char **argv) int add_cpu = -1; int opt, err; int prog_fd; + int *cpu, i; __u32 qsize; n_cpus = get_nprocs_conf(); @@ -719,6 +720,13 @@ int main(int argc, char **argv) } mark_cpus_unavailable(); + cpu = malloc(n_cpus * sizeof(int)); + if (!cpu) { + fprintf(stderr, "failed to allocate cpu array\n"); + return EXIT_FAIL; + } + memset(cpu, 0, n_cpus * sizeof(int)); + /* Parse commands line args */ while ((opt = getopt_long(argc, argv, "hSd:s:p:q:c:xzF", long_options, &longindex)) != -1) { @@ -763,8 +771,7 @@ int main(int argc, char **argv) errno, strerror(errno)); goto error; } - create_cpu_entry(add_cpu, qsize, added_cpus, true); - added_cpus++; + cpu[added_cpus++] = add_cpu; break; case 'q': qsize = atoi(optarg); @@ -775,6 +782,7 @@ int main(int argc, char **argv) case 'h': error: default: + free(cpu); usage(argv, obj); return EXIT_FAIL_OPTION; } @@ -787,16 +795,21 @@ int main(int argc, char **argv) if (ifindex == -1) { fprintf(stderr, "ERR: required option --dev missing\n"); usage(argv, obj); - return EXIT_FAIL_OPTION; + err = EXIT_FAIL_OPTION; + goto out; } /* Required option */ if (add_cpu == -1) { fprintf(stderr, "ERR: required option --cpu missing\n"); fprintf(stderr, " Specify multiple --cpu option to add more\n"); usage(argv, obj); - return EXIT_FAIL_OPTION; + err = EXIT_FAIL_OPTION; + goto out; } + for (i = 0; i < added_cpus; i++) + create_cpu_entry(cpu[i], qsize, i, true); + /* Remove XDP program when program is interrupted or killed */ signal(SIGINT, int_exit); signal(SIGTERM, int_exit); @@ -804,27 +817,32 @@ int main(int argc, char **argv) prog = bpf_object__find_program_by_title(obj, prog_name); if (!prog) { fprintf(stderr, "bpf_object__find_program_by_title failed\n"); - return EXIT_FAIL; + err = EXIT_FAIL; + goto out; } prog_fd = bpf_program__fd(prog); if (prog_fd < 0) { fprintf(stderr, "bpf_program__fd failed\n"); - return EXIT_FAIL; + err = EXIT_FAIL; + goto out; } if (bpf_set_link_xdp_fd(ifindex, prog_fd, xdp_flags) < 0) { fprintf(stderr, "link set xdp fd failed\n"); - return EXIT_FAIL_XDP; + err = EXIT_FAIL_XDP; + goto out; } err = bpf_obj_get_info_by_fd(prog_fd, &info, &info_len); if (err) { printf("can't get prog info - %s\n", strerror(errno)); - return err; + goto out; } prog_id = info.id; stats_poll(interval, use_separators, prog_name, stress_mode); - return EXIT_OK; +out: + free(cpu); + return err; } From patchwork Sun May 31 21:46:48 2020 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Lorenzo Bianconi X-Patchwork-Id: 1301518 X-Patchwork-Delegate: bpf@iogearbox.net Return-Path: X-Original-To: incoming-bpf@patchwork.ozlabs.org Delivered-To: patchwork-incoming-bpf@bilbo.ozlabs.org Authentication-Results: ozlabs.org; spf=pass (sender SPF authorized) smtp.mailfrom=vger.kernel.org (client-ip=23.128.96.18; helo=vger.kernel.org; envelope-from=bpf-owner@vger.kernel.org; receiver=) Authentication-Results: ozlabs.org; dmarc=pass (p=none dis=none) header.from=kernel.org Authentication-Results: ozlabs.org; dkim=pass (1024-bit key; unprotected) header.d=kernel.org header.i=@kernel.org header.a=rsa-sha256 header.s=default header.b=G5+nj7e7; dkim-atps=neutral Received: from vger.kernel.org (vger.kernel.org [23.128.96.18]) by ozlabs.org (Postfix) with ESMTP id 49ZsNb5DSLz9sPF for ; Mon, 1 Jun 2020 07:47:43 +1000 (AEST) Received: (majordomo@vger.kernel.org) by vger.kernel.org via listexpand id S1728454AbgEaVrk (ORCPT ); Sun, 31 May 2020 17:47:40 -0400 Received: from mail.kernel.org ([198.145.29.99]:37188 "EHLO mail.kernel.org" rhost-flags-OK-OK-OK-OK) by vger.kernel.org with ESMTP id S1728395AbgEaVrj (ORCPT ); Sun, 31 May 2020 17:47:39 -0400 Received: from lore-desk.lan (unknown [151.48.128.87]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by mail.kernel.org (Postfix) with ESMTPSA id 18C53206F1; Sun, 31 May 2020 21:47:33 +0000 (UTC) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/simple; d=kernel.org; s=default; t=1590961658; bh=vIDBqI4LeBbmbgXUpAeR5oXjD6W4UaeuCDblUyQUXxQ=; h=From:To:Cc:Subject:Date:In-Reply-To:References:From; b=G5+nj7e7vyn/htHNL0x2tyKIonkNX/NXf4++/XrJHaz+peVQTK73xjrR2GUMjlcoZ OtsvI3XzDETdxb5LLIKyXXDOCPqkIj5gebyDmQmvjZeMDlsUk+JHxlxd51Z1n56KMT VU5CAuOq9LFXF9nK91JAR3xsvgxES4Og6hHDz+ME= From: Lorenzo Bianconi To: bpf@vger.kernel.org, netdev@vger.kernel.org Cc: davem@davemloft.net, ast@kernel.org, brouer@redhat.com, toke@redhat.com, daniel@iogearbox.net, lorenzo.bianconi@redhat.com, dsahern@kernel.org Subject: [PATCH bpf-next 3/6] cpumap: formalize map value as a named struct Date: Sun, 31 May 2020 23:46:48 +0200 Message-Id: <02fcf47c1b0dcf37b108994ba6f44266ad89bee6.1590960613.git.lorenzo@kernel.org> X-Mailer: git-send-email 2.26.2 In-Reply-To: References: MIME-Version: 1.0 Sender: bpf-owner@vger.kernel.org Precedence: bulk List-ID: X-Mailing-List: bpf@vger.kernel.org As it has been already done for devmap, introduce 'struct bpf_cpumap_val' to formalize the expected values that can be passed in for a CPUMAP. Update cpumap code to use the struct. Signed-off-by: Lorenzo Bianconi --- kernel/bpf/cpumap.c | 30 ++++++++++++++++++------------ 1 file changed, 18 insertions(+), 12 deletions(-) diff --git a/kernel/bpf/cpumap.c b/kernel/bpf/cpumap.c index 27595fc6da56..57402276d8af 100644 --- a/kernel/bpf/cpumap.c +++ b/kernel/bpf/cpumap.c @@ -48,11 +48,15 @@ struct xdp_bulk_queue { unsigned int count; }; +/* CPUMAP value */ +struct bpf_cpumap_val { + u32 qsize; /* queue size */ +}; + /* Struct for every remote "destination" CPU in map */ struct bpf_cpu_map_entry { u32 cpu; /* kthread CPU and map index */ int map_id; /* Back reference to map */ - u32 qsize; /* Queue size placeholder for map lookup */ /* XDP can run multiple RX-ring queues, need __percpu enqueue store */ struct xdp_bulk_queue __percpu *bulkq; @@ -66,6 +70,8 @@ struct bpf_cpu_map_entry { atomic_t refcnt; /* Control when this struct can be free'ed */ struct rcu_head rcu; + + struct bpf_cpumap_val value; }; struct bpf_cpu_map { @@ -307,8 +313,8 @@ static int cpu_map_kthread_run(void *data) return 0; } -static struct bpf_cpu_map_entry *__cpu_map_entry_alloc(u32 qsize, u32 cpu, - int map_id) +static struct bpf_cpu_map_entry * +__cpu_map_entry_alloc(struct bpf_cpumap_val *value, u32 cpu, int map_id) { gfp_t gfp = GFP_KERNEL | __GFP_NOWARN; struct bpf_cpu_map_entry *rcpu; @@ -338,13 +344,13 @@ static struct bpf_cpu_map_entry *__cpu_map_entry_alloc(u32 qsize, u32 cpu, if (!rcpu->queue) goto free_bulkq; - err = ptr_ring_init(rcpu->queue, qsize, gfp); + err = ptr_ring_init(rcpu->queue, value->qsize, gfp); if (err) goto free_queue; rcpu->cpu = cpu; rcpu->map_id = map_id; - rcpu->qsize = qsize; + rcpu->value.qsize = value->qsize; /* Setup kthread */ rcpu->kthread = kthread_create_on_node(cpu_map_kthread_run, rcpu, numa, @@ -437,12 +443,12 @@ static int cpu_map_update_elem(struct bpf_map *map, void *key, void *value, u64 map_flags) { struct bpf_cpu_map *cmap = container_of(map, struct bpf_cpu_map, map); + struct bpf_cpumap_val cpumap_value = {}; struct bpf_cpu_map_entry *rcpu; - /* Array index key correspond to CPU number */ u32 key_cpu = *(u32 *)key; - /* Value is the queue size */ - u32 qsize = *(u32 *)value; + + memcpy(&cpumap_value, value, map->value_size); if (unlikely(map_flags > BPF_EXIST)) return -EINVAL; @@ -450,18 +456,18 @@ static int cpu_map_update_elem(struct bpf_map *map, void *key, void *value, return -E2BIG; if (unlikely(map_flags == BPF_NOEXIST)) return -EEXIST; - if (unlikely(qsize > 16384)) /* sanity limit on qsize */ + if (unlikely(cpumap_value.qsize > 16384)) /* sanity limit on qsize */ return -EOVERFLOW; /* Make sure CPU is a valid possible cpu */ if (key_cpu >= nr_cpumask_bits || !cpu_possible(key_cpu)) return -ENODEV; - if (qsize == 0) { + if (cpumap_value.qsize == 0) { rcpu = NULL; /* Same as deleting */ } else { /* Updating qsize cause re-allocation of bpf_cpu_map_entry */ - rcpu = __cpu_map_entry_alloc(qsize, key_cpu, map->id); + rcpu = __cpu_map_entry_alloc(&cpumap_value, key_cpu, map->id); if (!rcpu) return -ENOMEM; rcpu->cmap = cmap; @@ -523,7 +529,7 @@ static void *cpu_map_lookup_elem(struct bpf_map *map, void *key) struct bpf_cpu_map_entry *rcpu = __cpu_map_lookup_elem(map, *(u32 *)key); - return rcpu ? &rcpu->qsize : NULL; + return rcpu ? &rcpu->value.qsize : NULL; } static int cpu_map_get_next_key(struct bpf_map *map, void *key, void *next_key) From patchwork Sun May 31 21:46:49 2020 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Lorenzo Bianconi X-Patchwork-Id: 1301523 X-Patchwork-Delegate: bpf@iogearbox.net Return-Path: X-Original-To: patchwork-incoming-netdev@ozlabs.org Delivered-To: patchwork-incoming-netdev@ozlabs.org Authentication-Results: ozlabs.org; spf=pass (sender SPF authorized) smtp.mailfrom=vger.kernel.org (client-ip=23.128.96.18; helo=vger.kernel.org; envelope-from=netdev-owner@vger.kernel.org; receiver=) Authentication-Results: ozlabs.org; dmarc=pass (p=none dis=none) header.from=kernel.org Authentication-Results: ozlabs.org; dkim=pass (1024-bit key; unprotected) header.d=kernel.org header.i=@kernel.org header.a=rsa-sha256 header.s=default header.b=Q0/HadfD; dkim-atps=neutral Received: from vger.kernel.org (vger.kernel.org [23.128.96.18]) by ozlabs.org (Postfix) with ESMTP id 49ZsNg5cmGz9sTG for ; Mon, 1 Jun 2020 07:47:47 +1000 (AEST) Received: (majordomo@vger.kernel.org) by vger.kernel.org via listexpand id S1728476AbgEaVrp (ORCPT ); Sun, 31 May 2020 17:47:45 -0400 Received: from mail.kernel.org ([198.145.29.99]:37248 "EHLO mail.kernel.org" rhost-flags-OK-OK-OK-OK) by vger.kernel.org with ESMTP id S1728395AbgEaVro (ORCPT ); Sun, 31 May 2020 17:47:44 -0400 Received: from lore-desk.lan (unknown [151.48.128.87]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by mail.kernel.org (Postfix) with ESMTPSA id 28EA420707; Sun, 31 May 2020 21:47:40 +0000 (UTC) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/simple; d=kernel.org; s=default; t=1590961663; bh=VDhhbsDMHu2r0TmrcmhRF+R1qdtCEYevimTO8kZzLLs=; h=From:To:Cc:Subject:Date:In-Reply-To:References:From; b=Q0/HadfDj4u7gCRNzdoCGJT5P7tpmOaf1k2c1zGG63/Xw40eq471eTU7Jt9Y7UVG1 U4JjHeahsmaWTGB1LcexT5AoHj+BpZUMBMMtgi51ovdw5oVg+hfcrLOR1SXYF4tXiP IKQczCmiRKKJyXnvwAB0VXZIVHY7yHLzKyg/e26E= From: Lorenzo Bianconi To: bpf@vger.kernel.org, netdev@vger.kernel.org Cc: davem@davemloft.net, ast@kernel.org, brouer@redhat.com, toke@redhat.com, daniel@iogearbox.net, lorenzo.bianconi@redhat.com, dsahern@kernel.org Subject: [PATCH bpf-next 4/6] bpf: cpumap: add the possibility to attach an eBPF program to cpumap Date: Sun, 31 May 2020 23:46:49 +0200 Message-Id: <2543519aa9cdb368504cb6043fad6cae6f6ec745.1590960613.git.lorenzo@kernel.org> X-Mailer: git-send-email 2.26.2 In-Reply-To: References: MIME-Version: 1.0 Sender: netdev-owner@vger.kernel.org Precedence: bulk List-ID: X-Mailing-List: netdev@vger.kernel.org Introduce the capability to attach an eBPF program to cpumap entries. The idea behind this feature is to add the possibility to define on which CPU run the eBPF program if the underlying hw does not support RSS. Current supported verdicts are XDP_DROP and XDP_PASS Signed-off-by: Lorenzo Bianconi --- include/linux/bpf.h | 6 ++ include/trace/events/xdp.h | 14 +++-- include/uapi/linux/bpf.h | 1 + kernel/bpf/cpumap.c | 106 +++++++++++++++++++++++++++++---- net/core/dev.c | 8 +++ net/core/filter.c | 7 +++ tools/include/uapi/linux/bpf.h | 1 + 7 files changed, 128 insertions(+), 15 deletions(-) diff --git a/include/linux/bpf.h b/include/linux/bpf.h index e042311f991f..d5bbcdcc8321 100644 --- a/include/linux/bpf.h +++ b/include/linux/bpf.h @@ -1256,6 +1256,7 @@ struct bpf_cpu_map_entry *__cpu_map_lookup_elem(struct bpf_map *map, u32 key); void __cpu_map_flush(void); int cpu_map_enqueue(struct bpf_cpu_map_entry *rcpu, struct xdp_buff *xdp, struct net_device *dev_rx); +bool cpu_map_prog_allowed(struct bpf_map *map); /* Return map's numa specified by userspace */ static inline int bpf_map_attr_numa_node(const union bpf_attr *attr) @@ -1416,6 +1417,11 @@ static inline int cpu_map_enqueue(struct bpf_cpu_map_entry *rcpu, return 0; } +static inline bool cpu_map_prog_allowed(struct bpf_map *map) +{ + return false; +} + static inline struct bpf_prog *bpf_prog_get_type_path(const char *name, enum bpf_prog_type type) { diff --git a/include/trace/events/xdp.h b/include/trace/events/xdp.h index b73d3e141323..06ec557c6bf5 100644 --- a/include/trace/events/xdp.h +++ b/include/trace/events/xdp.h @@ -177,9 +177,9 @@ DEFINE_EVENT(xdp_redirect_template, xdp_redirect_map_err, TRACE_EVENT(xdp_cpumap_kthread, TP_PROTO(int map_id, unsigned int processed, unsigned int drops, - int sched), + int sched, unsigned int xdp_pass, unsigned int xdp_drop), - TP_ARGS(map_id, processed, drops, sched), + TP_ARGS(map_id, processed, drops, sched, xdp_pass, xdp_drop), TP_STRUCT__entry( __field(int, map_id) @@ -188,6 +188,8 @@ TRACE_EVENT(xdp_cpumap_kthread, __field(unsigned int, drops) __field(unsigned int, processed) __field(int, sched) + __field(unsigned int, xdp_pass) + __field(unsigned int, xdp_drop) ), TP_fast_assign( @@ -197,16 +199,20 @@ TRACE_EVENT(xdp_cpumap_kthread, __entry->drops = drops; __entry->processed = processed; __entry->sched = sched; + __entry->xdp_pass = xdp_pass; + __entry->xdp_drop = xdp_drop; ), TP_printk("kthread" " cpu=%d map_id=%d action=%s" " processed=%u drops=%u" - " sched=%d", + " sched=%d" + " xdp_pass=%u xdp_drop=%u", __entry->cpu, __entry->map_id, __print_symbolic(__entry->act, __XDP_ACT_SYM_TAB), __entry->processed, __entry->drops, - __entry->sched) + __entry->sched, + __entry->xdp_pass, __entry->xdp_drop) ); TRACE_EVENT(xdp_cpumap_enqueue, diff --git a/include/uapi/linux/bpf.h b/include/uapi/linux/bpf.h index f74bc4a2385e..10158943c0db 100644 --- a/include/uapi/linux/bpf.h +++ b/include/uapi/linux/bpf.h @@ -226,6 +226,7 @@ enum bpf_attach_type { BPF_CGROUP_INET4_GETSOCKNAME, BPF_CGROUP_INET6_GETSOCKNAME, BPF_XDP_DEVMAP, + BPF_XDP_CPUMAP, __MAX_BPF_ATTACH_TYPE }; diff --git a/kernel/bpf/cpumap.c b/kernel/bpf/cpumap.c index 57402276d8af..24ab0a6b9772 100644 --- a/kernel/bpf/cpumap.c +++ b/kernel/bpf/cpumap.c @@ -51,6 +51,10 @@ struct xdp_bulk_queue { /* CPUMAP value */ struct bpf_cpumap_val { u32 qsize; /* queue size */ + union { + int fd; /* program file descriptor */ + u32 id; /* program id */ + } prog; }; /* Struct for every remote "destination" CPU in map */ @@ -72,6 +76,7 @@ struct bpf_cpu_map_entry { struct rcu_head rcu; struct bpf_cpumap_val value; + struct bpf_prog *prog; }; struct bpf_cpu_map { @@ -86,6 +91,7 @@ static int bq_flush_to_queue(struct xdp_bulk_queue *bq); static struct bpf_map *cpu_map_alloc(union bpf_attr *attr) { + u32 value_size = attr->value_size; struct bpf_cpu_map *cmap; int err = -ENOMEM; u64 cost; @@ -96,7 +102,9 @@ static struct bpf_map *cpu_map_alloc(union bpf_attr *attr) /* check sanity of attributes */ if (attr->max_entries == 0 || attr->key_size != 4 || - attr->value_size != 4 || attr->map_flags & ~BPF_F_NUMA_NODE) + (value_size != offsetofend(struct bpf_cpumap_val, qsize) && + value_size != offsetofend(struct bpf_cpumap_val, prog.fd)) || + attr->map_flags & ~BPF_F_NUMA_NODE) return ERR_PTR(-EINVAL); cmap = kzalloc(sizeof(*cmap), GFP_USER); @@ -240,11 +248,14 @@ static int cpu_map_kthread_run(void *data) * kthread_stop signal until queue is empty. */ while (!kthread_should_stop() || !__ptr_ring_empty(rcpu->queue)) { + unsigned int xdp_pass = 0, xdp_drop = 0; + gfp_t gfp = __GFP_ZERO | GFP_ATOMIC; unsigned int drops = 0, sched = 0; + void *xdp_frames[CPUMAP_BATCH]; void *frames[CPUMAP_BATCH]; void *skbs[CPUMAP_BATCH]; - gfp_t gfp = __GFP_ZERO | GFP_ATOMIC; - int i, n, m; + int i, n, m, nframes = 0; + struct bpf_prog *prog; /* Release CPU reschedule checks */ if (__ptr_ring_empty(rcpu->queue)) { @@ -265,28 +276,67 @@ static int cpu_map_kthread_run(void *data) * kthread CPU pinned. Lockless access to ptr_ring * consume side valid as no-resize allowed of queue. */ - n = ptr_ring_consume_batched(rcpu->queue, frames, CPUMAP_BATCH); + n = ptr_ring_consume_batched(rcpu->queue, xdp_frames, + CPUMAP_BATCH); + rcu_read_lock(); + + prog = READ_ONCE(rcpu->prog); for (i = 0; i < n; i++) { - void *f = frames[i]; + void *f = xdp_frames[i]; struct page *page = virt_to_page(f); + struct xdp_frame *xdpf; + struct xdp_buff xdp; + u32 act; + int err; /* Bring struct page memory area to curr CPU. Read by * build_skb_around via page_is_pfmemalloc(), and when * freed written by page_frag_free call. */ prefetchw(page); + if (!prog) { + frames[nframes++] = xdp_frames[i]; + continue; + } + + xdpf = f; + xdp_convert_frame_to_buff(xdpf, &xdp); + + act = bpf_prog_run_xdp(prog, &xdp); + switch (act) { + case XDP_PASS: + err = xdp_update_frame_from_buff(&xdp, xdpf); + if (err < 0) { + xdp_return_frame(xdpf); + drops++; + } else { + frames[nframes++] = xdpf; + xdp_pass++; + } + break; + default: + bpf_warn_invalid_xdp_action(act); + /* fallthrough */ + case XDP_DROP: + xdp_return_frame(xdpf); + xdp_drop++; + break; + } } - m = kmem_cache_alloc_bulk(skbuff_head_cache, gfp, n, skbs); + rcu_read_unlock(); + + m = kmem_cache_alloc_bulk(skbuff_head_cache, gfp, + nframes, skbs); if (unlikely(m == 0)) { - for (i = 0; i < n; i++) + for (i = 0; i < nframes; i++) skbs[i] = NULL; /* effect: xdp_return_frame */ - drops = n; + drops += nframes; } local_bh_disable(); - for (i = 0; i < n; i++) { + for (i = 0; i < nframes; i++) { struct xdp_frame *xdpf = frames[i]; struct sk_buff *skb = skbs[i]; int ret; @@ -303,7 +353,8 @@ static int cpu_map_kthread_run(void *data) drops++; } /* Feedback loop via tracepoint */ - trace_xdp_cpumap_kthread(rcpu->map_id, n, drops, sched); + trace_xdp_cpumap_kthread(rcpu->map_id, n, drops, sched, + xdp_pass, xdp_drop); local_bh_enable(); /* resched point, may call do_softirq() */ } @@ -313,11 +364,37 @@ static int cpu_map_kthread_run(void *data) return 0; } +bool cpu_map_prog_allowed(struct bpf_map *map) +{ + return map->map_type == BPF_MAP_TYPE_CPUMAP && + map->value_size != offsetofend(struct bpf_cpumap_val, qsize); +} + +static int __cpu_map_load_bpf_program(struct bpf_cpu_map_entry *rcpu, int fd) +{ + struct bpf_prog *prog; + + prog = bpf_prog_get_type_dev(fd, BPF_PROG_TYPE_XDP, false); + if (IS_ERR(prog)) + return PTR_ERR(prog); + + if (prog->expected_attach_type != BPF_XDP_CPUMAP) { + bpf_prog_put(prog); + return -EINVAL; + } + + rcpu->value.prog.id = prog->aux->id; + rcpu->prog = prog; + + return 0; +} + static struct bpf_cpu_map_entry * __cpu_map_entry_alloc(struct bpf_cpumap_val *value, u32 cpu, int map_id) { gfp_t gfp = GFP_KERNEL | __GFP_NOWARN; struct bpf_cpu_map_entry *rcpu; + int prog_fd = value->prog.fd; struct xdp_bulk_queue *bq; int numa, err, i; @@ -361,6 +438,9 @@ __cpu_map_entry_alloc(struct bpf_cpumap_val *value, u32 cpu, int map_id) get_cpu_map_entry(rcpu); /* 1-refcnt for being in cmap->cpu_map[] */ get_cpu_map_entry(rcpu); /* 1-refcnt for kthread */ + if (prog_fd >= 0 && __cpu_map_load_bpf_program(rcpu, prog_fd)) + goto free_ptr_ring; + /* Make sure kthread runs on a single CPU */ kthread_bind(rcpu->kthread, cpu); wake_up_process(rcpu->kthread); @@ -420,6 +500,8 @@ static void __cpu_map_entry_replace(struct bpf_cpu_map *cmap, old_rcpu = xchg(&cmap->cpu_map[key_cpu], rcpu); if (old_rcpu) { + if (old_rcpu->prog) + bpf_prog_put(old_rcpu->prog); call_rcu(&old_rcpu->rcu, __cpu_map_entry_free); INIT_WORK(&old_rcpu->kthread_stop_wq, cpu_map_kthread_stop); schedule_work(&old_rcpu->kthread_stop_wq); @@ -443,7 +525,9 @@ static int cpu_map_update_elem(struct bpf_map *map, void *key, void *value, u64 map_flags) { struct bpf_cpu_map *cmap = container_of(map, struct bpf_cpu_map, map); - struct bpf_cpumap_val cpumap_value = {}; + struct bpf_cpumap_val cpumap_value = { + .prog.fd = -1, + }; struct bpf_cpu_map_entry *rcpu; /* Array index key correspond to CPU number */ u32 key_cpu = *(u32 *)key; diff --git a/net/core/dev.c b/net/core/dev.c index 10684833f864..4b95adaa4641 100644 --- a/net/core/dev.c +++ b/net/core/dev.c @@ -5429,6 +5429,8 @@ static int generic_xdp_install(struct net_device *dev, struct netdev_bpf *xdp) for (i = 0; i < new->aux->used_map_cnt; i++) { if (dev_map_can_have_prog(new->aux->used_maps[i])) return -EINVAL; + if (cpu_map_prog_allowed(new->aux->used_maps[i])) + return -EINVAL; } } @@ -8853,6 +8855,12 @@ int dev_change_xdp_fd(struct net_device *dev, struct netlink_ext_ack *extack, return -EINVAL; } + if (prog->expected_attach_type == BPF_XDP_CPUMAP) { + NL_SET_ERR_MSG(extack, "BPF_XDP_CPUMAP programs can not be attached to a device"); + bpf_prog_put(prog); + return -EINVAL; + } + /* prog->aux->id may be 0 for orphaned device-bound progs */ if (prog->aux->id && prog->aux->id == prog_id) { bpf_prog_put(prog); diff --git a/net/core/filter.c b/net/core/filter.c index 2e9dbfd8e60c..0c748c95179e 100644 --- a/net/core/filter.c +++ b/net/core/filter.c @@ -7021,6 +7021,13 @@ static bool xdp_is_valid_access(int off, int size, } } + if (prog->expected_attach_type == BPF_XDP_CPUMAP) { + switch (off) { + case offsetof(struct xdp_md, ingress_ifindex): + return false; + } + } + if (type == BPF_WRITE) { if (bpf_prog_is_dev_bound(prog->aux)) { switch (off) { diff --git a/tools/include/uapi/linux/bpf.h b/tools/include/uapi/linux/bpf.h index f74bc4a2385e..10158943c0db 100644 --- a/tools/include/uapi/linux/bpf.h +++ b/tools/include/uapi/linux/bpf.h @@ -226,6 +226,7 @@ enum bpf_attach_type { BPF_CGROUP_INET4_GETSOCKNAME, BPF_CGROUP_INET6_GETSOCKNAME, BPF_XDP_DEVMAP, + BPF_XDP_CPUMAP, __MAX_BPF_ATTACH_TYPE }; From patchwork Sun May 31 21:46:50 2020 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Lorenzo Bianconi X-Patchwork-Id: 1301525 X-Patchwork-Delegate: bpf@iogearbox.net Return-Path: X-Original-To: patchwork-incoming-netdev@ozlabs.org Delivered-To: patchwork-incoming-netdev@ozlabs.org Authentication-Results: ozlabs.org; spf=pass (sender SPF authorized) smtp.mailfrom=vger.kernel.org (client-ip=23.128.96.18; helo=vger.kernel.org; envelope-from=netdev-owner@vger.kernel.org; receiver=) Authentication-Results: ozlabs.org; dmarc=pass (p=none dis=none) header.from=kernel.org Authentication-Results: ozlabs.org; dkim=pass (1024-bit key; unprotected) header.d=kernel.org header.i=@kernel.org header.a=rsa-sha256 header.s=default header.b=jCVtfsop; dkim-atps=neutral Received: from vger.kernel.org (vger.kernel.org [23.128.96.18]) by ozlabs.org (Postfix) with ESMTP id 49ZsNl1yS5z9sT8 for ; Mon, 1 Jun 2020 07:47:51 +1000 (AEST) Received: (majordomo@vger.kernel.org) by vger.kernel.org via listexpand id S1728482AbgEaVrt (ORCPT ); Sun, 31 May 2020 17:47:49 -0400 Received: from mail.kernel.org ([198.145.29.99]:37302 "EHLO mail.kernel.org" rhost-flags-OK-OK-OK-OK) by vger.kernel.org with ESMTP id S1728395AbgEaVrt (ORCPT ); Sun, 31 May 2020 17:47:49 -0400 Received: from lore-desk.lan (unknown [151.48.128.87]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by mail.kernel.org (Postfix) with ESMTPSA id 38465206B6; Sun, 31 May 2020 21:47:45 +0000 (UTC) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/simple; d=kernel.org; s=default; t=1590961668; bh=AYTwty+A3SMZzGlgrDYlynHYNbtMF4MiJ8SBIUndzsI=; h=From:To:Cc:Subject:Date:In-Reply-To:References:From; b=jCVtfsopL1+OQ06KPop4degUJN8hnDlxkasFb95yatPxuMc+UGdz4Zrgo7rOdb+Jc cnFw8tPPoOiR1vzd0m9hQIIqecYfzQfYReGrisLcjhoWgrxYnXVzktxX8AmIdQFVAP MGaTNj+1m+8JFhlsFuKBs69WgSQkg6Hn+NxzFmUE= From: Lorenzo Bianconi To: bpf@vger.kernel.org, netdev@vger.kernel.org Cc: davem@davemloft.net, ast@kernel.org, brouer@redhat.com, toke@redhat.com, daniel@iogearbox.net, lorenzo.bianconi@redhat.com, dsahern@kernel.org Subject: [PATCH bpf-next 5/6] bpf: cpumap: implement XDP_REDIRECT for eBPF programs attached to map entries Date: Sun, 31 May 2020 23:46:50 +0200 Message-Id: <605426d4fac4e5ae4e5d98afdafaf7e35625657c.1590960613.git.lorenzo@kernel.org> X-Mailer: git-send-email 2.26.2 In-Reply-To: References: MIME-Version: 1.0 Sender: netdev-owner@vger.kernel.org Precedence: bulk List-ID: X-Mailing-List: netdev@vger.kernel.org Add XDP_REDIRECT support for eBPF programs attached to cpumap entries Signed-off-by: Lorenzo Bianconi --- include/trace/events/xdp.h | 12 ++++++++---- kernel/bpf/cpumap.c | 21 +++++++++++++++++---- 2 files changed, 25 insertions(+), 8 deletions(-) diff --git a/include/trace/events/xdp.h b/include/trace/events/xdp.h index 06ec557c6bf5..162ce06c6da0 100644 --- a/include/trace/events/xdp.h +++ b/include/trace/events/xdp.h @@ -177,9 +177,11 @@ DEFINE_EVENT(xdp_redirect_template, xdp_redirect_map_err, TRACE_EVENT(xdp_cpumap_kthread, TP_PROTO(int map_id, unsigned int processed, unsigned int drops, - int sched, unsigned int xdp_pass, unsigned int xdp_drop), + int sched, unsigned int xdp_pass, unsigned int xdp_drop, + unsigned int xdp_redirect), - TP_ARGS(map_id, processed, drops, sched, xdp_pass, xdp_drop), + TP_ARGS(map_id, processed, drops, sched, xdp_pass, xdp_drop, + xdp_redirect), TP_STRUCT__entry( __field(int, map_id) @@ -190,6 +192,7 @@ TRACE_EVENT(xdp_cpumap_kthread, __field(int, sched) __field(unsigned int, xdp_pass) __field(unsigned int, xdp_drop) + __field(unsigned int, xdp_redirect) ), TP_fast_assign( @@ -201,18 +204,19 @@ TRACE_EVENT(xdp_cpumap_kthread, __entry->sched = sched; __entry->xdp_pass = xdp_pass; __entry->xdp_drop = xdp_drop; + __entry->xdp_redirect = xdp_redirect; ), TP_printk("kthread" " cpu=%d map_id=%d action=%s" " processed=%u drops=%u" " sched=%d" - " xdp_pass=%u xdp_drop=%u", + " xdp_pass=%u xdp_drop=%u xdp_redirect=%u", __entry->cpu, __entry->map_id, __print_symbolic(__entry->act, __XDP_ACT_SYM_TAB), __entry->processed, __entry->drops, __entry->sched, - __entry->xdp_pass, __entry->xdp_drop) + __entry->xdp_pass, __entry->xdp_drop, __entry->xdp_redirect) ); TRACE_EVENT(xdp_cpumap_enqueue, diff --git a/kernel/bpf/cpumap.c b/kernel/bpf/cpumap.c index 24ab0a6b9772..a45157627fbc 100644 --- a/kernel/bpf/cpumap.c +++ b/kernel/bpf/cpumap.c @@ -248,7 +248,7 @@ static int cpu_map_kthread_run(void *data) * kthread_stop signal until queue is empty. */ while (!kthread_should_stop() || !__ptr_ring_empty(rcpu->queue)) { - unsigned int xdp_pass = 0, xdp_drop = 0; + unsigned int xdp_pass = 0, xdp_drop = 0, xdp_redirect = 0; gfp_t gfp = __GFP_ZERO | GFP_ATOMIC; unsigned int drops = 0, sched = 0; void *xdp_frames[CPUMAP_BATCH]; @@ -279,7 +279,7 @@ static int cpu_map_kthread_run(void *data) n = ptr_ring_consume_batched(rcpu->queue, xdp_frames, CPUMAP_BATCH); - rcu_read_lock(); + rcu_read_lock_bh(); prog = READ_ONCE(rcpu->prog); for (i = 0; i < n; i++) { @@ -315,6 +315,16 @@ static int cpu_map_kthread_run(void *data) xdp_pass++; } break; + case XDP_REDIRECT: + err = xdp_do_redirect(xdpf->dev_rx, &xdp, + prog); + if (unlikely(err)) { + xdp_return_frame(xdpf); + drops++; + } else { + xdp_redirect++; + } + break; default: bpf_warn_invalid_xdp_action(act); /* fallthrough */ @@ -325,7 +335,10 @@ static int cpu_map_kthread_run(void *data) } } - rcu_read_unlock(); + if (xdp_redirect) + xdp_do_flush_map(); + + rcu_read_unlock_bh(); m = kmem_cache_alloc_bulk(skbuff_head_cache, gfp, nframes, skbs); @@ -354,7 +367,7 @@ static int cpu_map_kthread_run(void *data) } /* Feedback loop via tracepoint */ trace_xdp_cpumap_kthread(rcpu->map_id, n, drops, sched, - xdp_pass, xdp_drop); + xdp_pass, xdp_drop, xdp_redirect); local_bh_enable(); /* resched point, may call do_softirq() */ } From patchwork Sun May 31 21:46:51 2020 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Lorenzo Bianconi X-Patchwork-Id: 1301527 X-Patchwork-Delegate: bpf@iogearbox.net Return-Path: X-Original-To: incoming-bpf@patchwork.ozlabs.org Delivered-To: patchwork-incoming-bpf@bilbo.ozlabs.org Authentication-Results: ozlabs.org; spf=pass (sender SPF authorized) smtp.mailfrom=vger.kernel.org (client-ip=23.128.96.18; helo=vger.kernel.org; envelope-from=bpf-owner@vger.kernel.org; receiver=) Authentication-Results: ozlabs.org; dmarc=pass (p=none dis=none) header.from=kernel.org Authentication-Results: ozlabs.org; dkim=pass (1024-bit key; unprotected) header.d=kernel.org header.i=@kernel.org header.a=rsa-sha256 header.s=default header.b=RG9IAKTF; dkim-atps=neutral Received: from vger.kernel.org (vger.kernel.org [23.128.96.18]) by ozlabs.org (Postfix) with ESMTP id 49ZsNs0TSzz9sPF for ; Mon, 1 Jun 2020 07:47:57 +1000 (AEST) Received: (majordomo@vger.kernel.org) by vger.kernel.org via listexpand id S1728483AbgEaVr4 (ORCPT ); Sun, 31 May 2020 17:47:56 -0400 Received: from mail.kernel.org ([198.145.29.99]:37340 "EHLO mail.kernel.org" rhost-flags-OK-OK-OK-OK) by vger.kernel.org with ESMTP id S1728395AbgEaVr4 (ORCPT ); Sun, 31 May 2020 17:47:56 -0400 Received: from lore-desk.lan (unknown [151.48.128.87]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by mail.kernel.org (Postfix) with ESMTPSA id 31A7A206F1; Sun, 31 May 2020 21:47:51 +0000 (UTC) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/simple; d=kernel.org; s=default; t=1590961675; bh=pEkohytzk8BTk5rJBYiyBRumDJxbzzz7NsqRuJs1Tjg=; h=From:To:Cc:Subject:Date:In-Reply-To:References:From; b=RG9IAKTFP3/SRtpg6AFGh6YgFj6UBRSQa9QLvfUNSg2yo6sPies7yyaKSC7bWNx+7 WiFcsMRkAPMaiwS+dMpf8mkzDb46E6D0DJwJKU0HWNrzNnEq3vBTPqE6HmOcDCRYFG UJCLg6GdEnuRKYMpoe3ERYkQK1o1YpSp6DWjO+b8= From: Lorenzo Bianconi To: bpf@vger.kernel.org, netdev@vger.kernel.org Cc: davem@davemloft.net, ast@kernel.org, brouer@redhat.com, toke@redhat.com, daniel@iogearbox.net, lorenzo.bianconi@redhat.com, dsahern@kernel.org Subject: [PATCH bpf-next 6/6] samples/bpf: xdp_redirect_cpu: load an eBPF program on cpumap Date: Sun, 31 May 2020 23:46:51 +0200 Message-Id: <71d8945da7e190f2e56e9d6f86bef83c9625cac0.1590960613.git.lorenzo@kernel.org> X-Mailer: git-send-email 2.26.2 In-Reply-To: References: MIME-Version: 1.0 Sender: bpf-owner@vger.kernel.org Precedence: bulk List-ID: X-Mailing-List: bpf@vger.kernel.org Extend xdp_redirect_cpu_{usr,kern}.c adding the possibility to load a XDP program on cpumap entries Signed-off-by: Lorenzo Bianconi --- samples/bpf/xdp_redirect_cpu_kern.c | 34 ++++++--- samples/bpf/xdp_redirect_cpu_user.c | 106 ++++++++++++++++++++++++---- 2 files changed, 119 insertions(+), 21 deletions(-) diff --git a/samples/bpf/xdp_redirect_cpu_kern.c b/samples/bpf/xdp_redirect_cpu_kern.c index 2baf8db1f7e7..30f500237753 100644 --- a/samples/bpf/xdp_redirect_cpu_kern.c +++ b/samples/bpf/xdp_redirect_cpu_kern.c @@ -17,11 +17,20 @@ #define MAX_CPUS NR_CPUS +/* CPUMAP value */ +struct bpf_cpumap_val { + u32 qsize; + union { + int fd; + u32 id; + } prog; +}; + /* Special map type that can XDP_REDIRECT frames to another CPU */ struct { __uint(type, BPF_MAP_TYPE_CPUMAP); __uint(key_size, sizeof(u32)); - __uint(value_size, sizeof(u32)); + __uint(value_size, sizeof(struct bpf_cpumap_val)); __uint(max_entries, MAX_CPUS); } cpu_map SEC(".maps"); @@ -30,6 +39,9 @@ struct datarec { __u64 processed; __u64 dropped; __u64 issue; + __u64 xdp_pass; + __u64 xdp_drop; + __u64 xdp_redirect; }; /* Count RX packets, as XDP bpf_prog doesn't get direct TX-success @@ -692,13 +704,16 @@ int trace_xdp_cpumap_enqueue(struct cpumap_enqueue_ctx *ctx) * Code in: kernel/include/trace/events/xdp.h */ struct cpumap_kthread_ctx { - u64 __pad; // First 8 bytes are not accessible by bpf code - int map_id; // offset:8; size:4; signed:1; - u32 act; // offset:12; size:4; signed:0; - int cpu; // offset:16; size:4; signed:1; - unsigned int drops; // offset:20; size:4; signed:0; - unsigned int processed; // offset:24; size:4; signed:0; - int sched; // offset:28; size:4; signed:1; + u64 __pad; // First 8 bytes are not accessible + int map_id; // offset:8; size:4; signed:1; + u32 act; // offset:12; size:4; signed:0; + int cpu; // offset:16; size:4; signed:1; + unsigned int drops; // offset:20; size:4; signed:0; + unsigned int processed; // offset:24; size:4; signed:0; + int sched; // offset:28; size:4; signed:1; + unsigned int xdp_pass; // offset:32; size:4; signed:0; + unsigned int xdp_drop; // offset:36; size:4; signed:0; + unsigned int xdp_redirect; // offset:40; size:4; signed:0; }; SEC("tracepoint/xdp/xdp_cpumap_kthread") @@ -712,6 +727,9 @@ int trace_xdp_cpumap_kthread(struct cpumap_kthread_ctx *ctx) return 0; rec->processed += ctx->processed; rec->dropped += ctx->drops; + rec->xdp_pass += ctx->xdp_pass; + rec->xdp_drop += ctx->xdp_drop; + rec->xdp_redirect += ctx->xdp_redirect; /* Count times kthread yielded CPU via schedule call */ if (ctx->sched) diff --git a/samples/bpf/xdp_redirect_cpu_user.c b/samples/bpf/xdp_redirect_cpu_user.c index 1a054737c35a..13f598a24759 100644 --- a/samples/bpf/xdp_redirect_cpu_user.c +++ b/samples/bpf/xdp_redirect_cpu_user.c @@ -30,6 +30,15 @@ static const char *__doc__ = #include "bpf_util.h" +/* CPUMAP value */ +struct bpf_cpumap_val { + __u32 qsize; + union { + int fd; + __u32 id; + } prog; +}; + static int ifindex = -1; static char ifname_buf[IF_NAMESIZE]; static char *ifname; @@ -70,6 +79,8 @@ static const struct option long_options[] = { {"stress-mode", no_argument, NULL, 'x' }, {"no-separators", no_argument, NULL, 'z' }, {"force", no_argument, NULL, 'F' }, + {"map-prog-filename", required_argument, NULL, 'f' }, + {"map-prog-name", required_argument, NULL, 'P' }, {0, 0, NULL, 0 } }; @@ -156,6 +167,9 @@ struct datarec { __u64 processed; __u64 dropped; __u64 issue; + __u64 xdp_pass; + __u64 xdp_drop; + __u64 xdp_redirect; }; struct record { __u64 timestamp; @@ -175,6 +189,9 @@ static bool map_collect_percpu(int fd, __u32 key, struct record *rec) /* For percpu maps, userspace gets a value per possible CPU */ unsigned int nr_cpus = bpf_num_possible_cpus(); struct datarec values[nr_cpus]; + __u64 sum_xdp_redirect = 0; + __u64 sum_xdp_pass = 0; + __u64 sum_xdp_drop = 0; __u64 sum_processed = 0; __u64 sum_dropped = 0; __u64 sum_issue = 0; @@ -196,10 +213,19 @@ static bool map_collect_percpu(int fd, __u32 key, struct record *rec) sum_dropped += values[i].dropped; rec->cpu[i].issue = values[i].issue; sum_issue += values[i].issue; + rec->cpu[i].xdp_pass = values[i].xdp_pass; + sum_xdp_pass += values[i].xdp_pass; + rec->cpu[i].xdp_drop = values[i].xdp_drop; + sum_xdp_drop += values[i].xdp_drop; + rec->cpu[i].xdp_redirect = values[i].xdp_redirect; + sum_xdp_redirect += values[i].xdp_redirect; } rec->total.processed = sum_processed; rec->total.dropped = sum_dropped; rec->total.issue = sum_issue; + rec->total.xdp_pass = sum_xdp_pass; + rec->total.xdp_drop = sum_xdp_drop; + rec->total.xdp_redirect = sum_xdp_redirect; return true; } @@ -405,6 +431,9 @@ static void stats_print(struct stats_record *stats_rec, if (pps > 0) printf(fmt_k, "cpumap_kthread", i, pps, drop, err, e_str); + printf("%d xdp-pass %llu xdp-drop %llu " + "xdp-redirect %llu\n", i, r->xdp_pass, + r->xdp_drop, r->xdp_redirect); } pps = calc_pps(&rec->total, &prev->total, t); drop = calc_drop_pps(&rec->total, &prev->total, t); @@ -412,6 +441,9 @@ static void stats_print(struct stats_record *stats_rec, if (err > 0) e_str = "sched-sum"; printf(fm2_k, "cpumap_kthread", "total", pps, drop, err, e_str); + printf("xdp-pass %llu xdp-drop %llu xdp-redirect %llu\n", + rec->total.xdp_pass, rec->total.xdp_drop, + rec->total.xdp_redirect); } /* XDP redirect err tracepoints (very unlikely) */ @@ -494,7 +526,7 @@ static inline void swap(struct stats_record **a, struct stats_record **b) *b = tmp; } -static int create_cpu_entry(__u32 cpu, __u32 queue_size, +static int create_cpu_entry(__u32 cpu, struct bpf_cpumap_val *value, __u32 avail_idx, bool new) { __u32 curr_cpus_count = 0; @@ -504,7 +536,7 @@ static int create_cpu_entry(__u32 cpu, __u32 queue_size, /* Add a CPU entry to cpumap, as this allocate a cpu entry in * the kernel for the cpu. */ - ret = bpf_map_update_elem(cpu_map_fd, &cpu, &queue_size, 0); + ret = bpf_map_update_elem(cpu_map_fd, &cpu, value, 0); if (ret) { fprintf(stderr, "Create CPU entry failed (err:%d)\n", ret); exit(EXIT_FAIL_BPF); @@ -535,9 +567,9 @@ static int create_cpu_entry(__u32 cpu, __u32 queue_size, } } /* map_fd[7] = cpus_iterator */ - printf("%s CPU:%u as idx:%u queue_size:%d (total cpus_count:%u)\n", + printf("%s CPU:%u as idx:%u qsize:%d prog_fd: %d (cpus_count:%u)\n", new ? "Add-new":"Replace", cpu, avail_idx, - queue_size, curr_cpus_count); + value->qsize, value->prog.fd, curr_cpus_count); return 0; } @@ -561,19 +593,22 @@ static void mark_cpus_unavailable(void) } /* Stress cpumap management code by concurrently changing underlying cpumap */ -static void stress_cpumap(void) +static void stress_cpumap(struct bpf_cpumap_val *value) { /* Changing qsize will cause kernel to free and alloc a new * bpf_cpu_map_entry, with an associated/complicated tear-down * procedure. */ - create_cpu_entry(1, 1024, 0, false); - create_cpu_entry(1, 8, 0, false); - create_cpu_entry(1, 16000, 0, false); + value->qsize = 1024; + create_cpu_entry(1, value, 0, false); + value->qsize = 8; + create_cpu_entry(1, value, 0, false); + value->qsize = 16000; + create_cpu_entry(1, value, 0, false); } static void stats_poll(int interval, bool use_separators, char *prog_name, - bool stress_mode) + struct bpf_cpumap_val *value, bool stress_mode) { struct stats_record *record, *prev; @@ -591,7 +626,7 @@ static void stats_poll(int interval, bool use_separators, char *prog_name, stats_print(record, prev, prog_name); sleep(interval); if (stress_mode) - stress_cpumap(); + stress_cpumap(value); } free_stats_record(record); @@ -664,15 +699,47 @@ static int init_map_fds(struct bpf_object *obj) return 0; } +static int load_cpumap_prog(char *file_name, char *prog_name) +{ + struct bpf_prog_load_attr prog_load_attr = { + .prog_type = BPF_PROG_TYPE_XDP, + .expected_attach_type = BPF_XDP_CPUMAP, + .file = file_name, + }; + struct bpf_program *prog; + struct bpf_object *obj; + int fd; + + if (bpf_prog_load_xattr(&prog_load_attr, &obj, &fd)) + return -1; + + if (fd < 0) { + fprintf(stderr, "ERR: bpf_prog_load_xattr: %s\n", + strerror(errno)); + return fd; + } + + prog = bpf_object__find_program_by_title(obj, prog_name); + if (!prog) { + fprintf(stderr, "bpf_object__find_program_by_title failed\n"); + return EXIT_FAIL; + } + + return bpf_program__fd(prog); +} + int main(int argc, char **argv) { struct rlimit r = {10 * 1024 * 1024, RLIM_INFINITY}; char *prog_name = "xdp_cpu_map5_lb_hash_ip_pairs"; + char *map_prog_filename = "xdp_redirect_kern.o"; + char *map_prog_name = "xdp_redirect_dummy"; struct bpf_prog_load_attr prog_load_attr = { .prog_type = BPF_PROG_TYPE_UNSPEC, }; struct bpf_prog_info info = {}; __u32 info_len = sizeof(info); + struct bpf_cpumap_val value; bool use_separators = true; bool stress_mode = false; struct bpf_program *prog; @@ -728,7 +795,7 @@ int main(int argc, char **argv) memset(cpu, 0, n_cpus * sizeof(int)); /* Parse commands line args */ - while ((opt = getopt_long(argc, argv, "hSd:s:p:q:c:xzF", + while ((opt = getopt_long(argc, argv, "hSd:s:p:q:c:xzFf:P:", long_options, &longindex)) != -1) { switch (opt) { case 'd': @@ -762,6 +829,12 @@ int main(int argc, char **argv) /* Selecting eBPF prog to load */ prog_name = optarg; break; + case 'f': + map_prog_filename = optarg; + break; + case 'P': + map_prog_name = optarg; + break; case 'c': /* Add multiple CPUs */ add_cpu = strtoul(optarg, NULL, 0); @@ -807,8 +880,15 @@ int main(int argc, char **argv) goto out; } + value.prog.fd = load_cpumap_prog(map_prog_filename, map_prog_name); + if (value.prog.fd < 0) { + err = value.prog.fd; + goto out; + } + value.qsize = qsize; + for (i = 0; i < added_cpus; i++) - create_cpu_entry(cpu[i], qsize, i, true); + create_cpu_entry(cpu[i], &value, i, true); /* Remove XDP program when program is interrupted or killed */ signal(SIGINT, int_exit); @@ -841,7 +921,7 @@ int main(int argc, char **argv) } prog_id = info.id; - stats_poll(interval, use_separators, prog_name, stress_mode); + stats_poll(interval, use_separators, prog_name, &value, stress_mode); out: free(cpu); return err;