| Message ID | 20180515190615.23099-13-bjorn.topel@gmail.com |
|---|---|
| State | RFC, archived |
| Delegated to: | BPF Maintainers |
| Headers | show |
| Series | AF_XDP, zero-copy support | expand |
On Tue, 15 May 2018 21:06:15 +0200 Björn Töpel <bjorn.topel@gmail.com> wrote: > From: Magnus Karlsson <magnus.karlsson@intel.com> > > Here, the zero-copy ndo is implemented. As a shortcut, the existing > XDP Tx rings are used for zero-copy. This means that and XDP program > cannot redirect to an AF_XDP enabled XDP Tx ring. I've changed i40e1 to only have one queue via: $ ethtool -L i40e1 combined 1 And then, I'm sending on queue 1, which is/should not be avail... and then crash/BUG: $ sudo taskset -c 2 ./xdpsock --tx --interface=i40e1 --queue=1 [ 3799.936877] Number of in use tx queues changed invalidating tc mappings. Priority traffic classification disabled! [ 3799.972970] BUG: unable to handle kernel NULL pointer dereference at 0000000000000008 [ 3799.980790] PGD 80000007b0992067 P4D 80000007b0992067 PUD 7b62d4067 PMD 0 [ 3799.987654] Oops: 0002 [#1] PREEMPT SMP PTI [ 3799.991831] Modules linked in: nf_nat_masquerade_ipv4 tun nfnetlink bridge stp llc nf_nat nf_conntrack rpcrdma ib_ipoib rdma_ucm ib_ucm ib_umad rdma_cm ib_cm iw_cm sunrpc mlx5_ib ib _uverbs ib_core coretemp kvm_intel kvm irqbypass intel_cstate intel_uncore intel_rapl_perf p cspkr i2c_i801 shpchp wmi ipmi_si ipmi_devintf ipmi_msghandler acpi_pad sch_fq_codel i40e ml x5_core hid_generic ixgbe igb devlink mdio ptp sd_mod i2c_algo_bit i2c_core pps_core [last u nloaded: x_tables] [ 3800.033472] CPU: 2 PID: 2006 Comm: xdpsock Not tainted 4.17.0-rc3-af_xdp03_ZC_rfc+ #155 [ 3800.041465] Hardware name: Supermicro Super Server/X10SRi-F, BIOS 2.0a 08/01/2016 [ 3800.048943] RIP: 0010:i40e_xmit_frame_ring+0xd4/0x1490 [i40e] [ 3800.054683] RSP: 0018:ffffc9000407bcd0 EFLAGS: 00010293 [ 3800.059900] RAX: 0000000000000000 RBX: ffff88084f0fd200 RCX: 0000000000000000 [ 3800.067022] RDX: 0000000000000000 RSI: 0000000000000006 RDI: ffff8807b6e710c0 [ 3800.074148] RBP: ffff8807c6397800 R08: 00000000000000c0 R09: 0000000000000001 [ 3800.081270] R10: 0000000000000800 R11: 0000000000000010 R12: 0000000000000001 [ 3800.088396] R13: 0000000000000000 R14: 0000000000000001 R15: 000000000000003c [ 3800.095520] FS: 00007f1d1e00bb80(0000) GS:ffff88087fc80000(0000) knlGS:0000000000000000 [ 3800.103597] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [ 3800.109335] CR2: 0000000000000008 CR3: 000000087d542001 CR4: 00000000003606e0 [ 3800.116458] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [ 3800.123583] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [ 3800.130706] Call Trace: [ 3800.133157] ? validate_xmit_skb.isra.116+0x1c/0x270 [ 3800.138118] dev_direct_xmit+0xec/0x1d0 [ 3800.141949] xsk_sendmsg+0x1f4/0x380 [ 3800.145521] sock_sendmsg+0x30/0x40 [ 3800.149005] __sys_sendto+0x10e/0x140 [ 3800.152662] ? __do_page_fault+0x283/0x500 [ 3800.156751] __x64_sys_sendto+0x24/0x30 [ 3800.160585] do_syscall_64+0x42/0xf0 [ 3800.164156] entry_SYSCALL_64_after_hwframe+0x44/0xa9 [ 3800.169204] RIP: 0033:0x7f1d1d9db430 [ 3800.172774] RSP: 002b:00007fffb7278610 EFLAGS: 00000293 ORIG_RAX: 000000000000002c [ 3800.180333] RAX: ffffffffffffffda RBX: 0000000000000000 RCX: 00007f1d1d9db430 [ 3800.187456] RDX: 0000000000000000 RSI: 0000000000000000 RDI: 0000000000000003 [ 3800.194582] RBP: 0000000000000000 R08: 0000000000000000 R09: 0000000000000000 [ 3800.201705] R10: 0000000000000040 R11: 0000000000000293 R12: 0000000000000000 [ 3800.208830] R13: 0000000000000000 R14: 0000000000755510 R15: 00007f1d0d3fc000 [ 3800.215953] Code: d0 0f 86 db 05 00 00 01 c8 0f b7 ca 29 c8 83 e8 01 39 c6 0f 8f ea 06 00 00 48 8b 45 28 48 8d 14 92 41 b9 01 00 00 00 4c 8d 2c d0 <49> 89 5d 08 8b 83 80 00 00 00 66 45 89 4d 14 41 89 45 10 0f b7 [ 3800.234798] RIP: i40e_xmit_frame_ring+0xd4/0x1490 [i40e] RSP: ffffc9000407bcd0 [ 3800.242005] CR2: 0000000000000008 [ 3800.245320] ---[ end trace f169e36f468e0c59 ]--- [ 3801.726719] Kernel panic - not syncing: Fatal exception in interrupt [ 3801.733097] Kernel Offset: disabled [ 3801.785836] ---[ end Kernel panic - not syncing: Fatal exception in interrupt ]--- [ 3801.793403] ------------[ cut here ]------------ (gdb) list *(i40e_xmit_frame_ring)+0xd4 0x2ccd4 is in i40e_xmit_frame_ring (drivers/net/ethernet/intel/i40e/i40e_txrx.c:4048). warning: Source file is more recent than executable. 4043 return NETDEV_TX_BUSY; 4044 } 4045 4046 /* record the location of the first descriptor for this packet */ 4047 first = &tx_ring->tx_bi[tx_ring->next_to_use]; 4048 first->skb = skb; 4049 first->bytecount = skb->len; 4050 first->gso_segs = 1; 4051 4052 /* prepare the xmit flags */ (gdb) list *(xsk_sendmsg)+0x1f4 0xffffffff81800c34 is in xsk_sendmsg (net/xdp/xsk.c:251). warning: Source file is more recent than executable. 246 skb_shinfo(skb)->destructor_arg = (void *)(long)id; 247 skb->destructor = xsk_destruct_skb; 248 249 err = dev_direct_xmit(skb, xs->queue_id); 250 /* Ignore NET_XMIT_CN as packet might have been sent */ 251 if (err == NET_XMIT_DROP || err == NETDEV_TX_BUSY) { 252 err = -EAGAIN; 253 /* SKB consumed by dev_direct_xmit() */ 254 goto out; 255 }
On Wed, May 16, 2018 at 4:28 PM, Jesper Dangaard Brouer <brouer@redhat.com> wrote: > On Tue, 15 May 2018 21:06:15 +0200 > Björn Töpel <bjorn.topel@gmail.com> wrote: > >> From: Magnus Karlsson <magnus.karlsson@intel.com> >> >> Here, the zero-copy ndo is implemented. As a shortcut, the existing >> XDP Tx rings are used for zero-copy. This means that and XDP program >> cannot redirect to an AF_XDP enabled XDP Tx ring. > > I've changed i40e1 to only have one queue via: > $ ethtool -L i40e1 combined 1 > > And then, I'm sending on queue 1, which is/should not be avail... and then crash/BUG: > > $ sudo taskset -c 2 ./xdpsock --tx --interface=i40e1 --queue=1 > > [ 3799.936877] Number of in use tx queues changed invalidating tc mappings. Priority traffic > classification disabled! > [ 3799.972970] BUG: unable to handle kernel NULL pointer dereference at 0000000000000008 > [ 3799.980790] PGD 80000007b0992067 P4D 80000007b0992067 PUD 7b62d4067 PMD 0 > [ 3799.987654] Oops: 0002 [#1] PREEMPT SMP PTI > [ 3799.991831] Modules linked in: nf_nat_masquerade_ipv4 tun nfnetlink bridge stp llc nf_nat > nf_conntrack rpcrdma ib_ipoib rdma_ucm ib_ucm ib_umad rdma_cm ib_cm iw_cm sunrpc mlx5_ib ib > _uverbs ib_core coretemp kvm_intel kvm irqbypass intel_cstate intel_uncore intel_rapl_perf p > cspkr i2c_i801 shpchp wmi ipmi_si ipmi_devintf ipmi_msghandler acpi_pad sch_fq_codel i40e ml > x5_core hid_generic ixgbe igb devlink mdio ptp sd_mod i2c_algo_bit i2c_core pps_core [last u > nloaded: x_tables] > [ 3800.033472] CPU: 2 PID: 2006 Comm: xdpsock Not tainted 4.17.0-rc3-af_xdp03_ZC_rfc+ #155 > [ 3800.041465] Hardware name: Supermicro Super Server/X10SRi-F, BIOS 2.0a 08/01/2016 > [ 3800.048943] RIP: 0010:i40e_xmit_frame_ring+0xd4/0x1490 [i40e] > [ 3800.054683] RSP: 0018:ffffc9000407bcd0 EFLAGS: 00010293 > [ 3800.059900] RAX: 0000000000000000 RBX: ffff88084f0fd200 RCX: 0000000000000000 > [ 3800.067022] RDX: 0000000000000000 RSI: 0000000000000006 RDI: ffff8807b6e710c0 > [ 3800.074148] RBP: ffff8807c6397800 R08: 00000000000000c0 R09: 0000000000000001 > [ 3800.081270] R10: 0000000000000800 R11: 0000000000000010 R12: 0000000000000001 > [ 3800.088396] R13: 0000000000000000 R14: 0000000000000001 R15: 000000000000003c > [ 3800.095520] FS: 00007f1d1e00bb80(0000) GS:ffff88087fc80000(0000) knlGS:0000000000000000 > [ 3800.103597] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 > [ 3800.109335] CR2: 0000000000000008 CR3: 000000087d542001 CR4: 00000000003606e0 > [ 3800.116458] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 > [ 3800.123583] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 > [ 3800.130706] Call Trace: > [ 3800.133157] ? validate_xmit_skb.isra.116+0x1c/0x270 > [ 3800.138118] dev_direct_xmit+0xec/0x1d0 > [ 3800.141949] xsk_sendmsg+0x1f4/0x380 > [ 3800.145521] sock_sendmsg+0x30/0x40 > [ 3800.149005] __sys_sendto+0x10e/0x140 > [ 3800.152662] ? __do_page_fault+0x283/0x500 > [ 3800.156751] __x64_sys_sendto+0x24/0x30 > [ 3800.160585] do_syscall_64+0x42/0xf0 > [ 3800.164156] entry_SYSCALL_64_after_hwframe+0x44/0xa9 > [ 3800.169204] RIP: 0033:0x7f1d1d9db430 > [ 3800.172774] RSP: 002b:00007fffb7278610 EFLAGS: 00000293 ORIG_RAX: 000000000000002c > [ 3800.180333] RAX: ffffffffffffffda RBX: 0000000000000000 RCX: 00007f1d1d9db430 > [ 3800.187456] RDX: 0000000000000000 RSI: 0000000000000000 RDI: 0000000000000003 > [ 3800.194582] RBP: 0000000000000000 R08: 0000000000000000 R09: 0000000000000000 > [ 3800.201705] R10: 0000000000000040 R11: 0000000000000293 R12: 0000000000000000 > [ 3800.208830] R13: 0000000000000000 R14: 0000000000755510 R15: 00007f1d0d3fc000 > [ 3800.215953] Code: d0 0f 86 db 05 00 00 01 c8 0f b7 ca 29 c8 83 e8 01 39 c6 0f 8f ea 06 00 00 48 8b 45 28 48 8d 14 92 41 b9 01 00 00 00 4c 8d 2c d0 <49> 89 5d 08 8b 83 80 00 00 00 66 45 89 4d 14 41 89 45 10 0f b7 > [ 3800.234798] RIP: i40e_xmit_frame_ring+0xd4/0x1490 [i40e] RSP: ffffc9000407bcd0 > [ 3800.242005] CR2: 0000000000000008 > [ 3800.245320] ---[ end trace f169e36f468e0c59 ]--- > [ 3801.726719] Kernel panic - not syncing: Fatal exception in interrupt > [ 3801.733097] Kernel Offset: disabled > [ 3801.785836] ---[ end Kernel panic - not syncing: Fatal exception in interrupt ]--- > [ 3801.793403] ------------[ cut here ]------------ > > (gdb) list *(i40e_xmit_frame_ring)+0xd4 > 0x2ccd4 is in i40e_xmit_frame_ring (drivers/net/ethernet/intel/i40e/i40e_txrx.c:4048). > warning: Source file is more recent than executable. > 4043 return NETDEV_TX_BUSY; > 4044 } > 4045 > 4046 /* record the location of the first descriptor for this packet */ > 4047 first = &tx_ring->tx_bi[tx_ring->next_to_use]; > 4048 first->skb = skb; > 4049 first->bytecount = skb->len; > 4050 first->gso_segs = 1; > 4051 > 4052 /* prepare the xmit flags */ > > > (gdb) list *(xsk_sendmsg)+0x1f4 > 0xffffffff81800c34 is in xsk_sendmsg (net/xdp/xsk.c:251). > warning: Source file is more recent than executable. > 246 skb_shinfo(skb)->destructor_arg = (void *)(long)id; > 247 skb->destructor = xsk_destruct_skb; > 248 > 249 err = dev_direct_xmit(skb, xs->queue_id); > 250 /* Ignore NET_XMIT_CN as packet might have been sent */ > 251 if (err == NET_XMIT_DROP || err == NETDEV_TX_BUSY) { > 252 err = -EAGAIN; > 253 /* SKB consumed by dev_direct_xmit() */ > 254 goto out; > 255 } Thanks Jesper for reporting. I will take a look at it. /Magnus
On Wed, May 16, 2018 at 4:38 PM, Magnus Karlsson <magnus.karlsson@gmail.com> wrote: > On Wed, May 16, 2018 at 4:28 PM, Jesper Dangaard Brouer > <brouer@redhat.com> wrote: >> On Tue, 15 May 2018 21:06:15 +0200 >> Björn Töpel <bjorn.topel@gmail.com> wrote: >> >>> From: Magnus Karlsson <magnus.karlsson@intel.com> >>> >>> Here, the zero-copy ndo is implemented. As a shortcut, the existing >>> XDP Tx rings are used for zero-copy. This means that and XDP program >>> cannot redirect to an AF_XDP enabled XDP Tx ring. >> >> I've changed i40e1 to only have one queue via: >> $ ethtool -L i40e1 combined 1 >> >> And then, I'm sending on queue 1, which is/should not be avail... and then crash/BUG: >> >> $ sudo taskset -c 2 ./xdpsock --tx --interface=i40e1 --queue=1 >> >> [ 3799.936877] Number of in use tx queues changed invalidating tc mappings. Priority traffic >> classification disabled! >> [ 3799.972970] BUG: unable to handle kernel NULL pointer dereference at 0000000000000008 >> [ 3799.980790] PGD 80000007b0992067 P4D 80000007b0992067 PUD 7b62d4067 PMD 0 >> [ 3799.987654] Oops: 0002 [#1] PREEMPT SMP PTI >> [ 3799.991831] Modules linked in: nf_nat_masquerade_ipv4 tun nfnetlink bridge stp llc nf_nat >> nf_conntrack rpcrdma ib_ipoib rdma_ucm ib_ucm ib_umad rdma_cm ib_cm iw_cm sunrpc mlx5_ib ib >> _uverbs ib_core coretemp kvm_intel kvm irqbypass intel_cstate intel_uncore intel_rapl_perf p >> cspkr i2c_i801 shpchp wmi ipmi_si ipmi_devintf ipmi_msghandler acpi_pad sch_fq_codel i40e ml >> x5_core hid_generic ixgbe igb devlink mdio ptp sd_mod i2c_algo_bit i2c_core pps_core [last u >> nloaded: x_tables] >> [ 3800.033472] CPU: 2 PID: 2006 Comm: xdpsock Not tainted 4.17.0-rc3-af_xdp03_ZC_rfc+ #155 >> [ 3800.041465] Hardware name: Supermicro Super Server/X10SRi-F, BIOS 2.0a 08/01/2016 >> [ 3800.048943] RIP: 0010:i40e_xmit_frame_ring+0xd4/0x1490 [i40e] >> [ 3800.054683] RSP: 0018:ffffc9000407bcd0 EFLAGS: 00010293 >> [ 3800.059900] RAX: 0000000000000000 RBX: ffff88084f0fd200 RCX: 0000000000000000 >> [ 3800.067022] RDX: 0000000000000000 RSI: 0000000000000006 RDI: ffff8807b6e710c0 >> [ 3800.074148] RBP: ffff8807c6397800 R08: 00000000000000c0 R09: 0000000000000001 >> [ 3800.081270] R10: 0000000000000800 R11: 0000000000000010 R12: 0000000000000001 >> [ 3800.088396] R13: 0000000000000000 R14: 0000000000000001 R15: 000000000000003c >> [ 3800.095520] FS: 00007f1d1e00bb80(0000) GS:ffff88087fc80000(0000) knlGS:0000000000000000 >> [ 3800.103597] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 >> [ 3800.109335] CR2: 0000000000000008 CR3: 000000087d542001 CR4: 00000000003606e0 >> [ 3800.116458] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 >> [ 3800.123583] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 >> [ 3800.130706] Call Trace: >> [ 3800.133157] ? validate_xmit_skb.isra.116+0x1c/0x270 >> [ 3800.138118] dev_direct_xmit+0xec/0x1d0 >> [ 3800.141949] xsk_sendmsg+0x1f4/0x380 >> [ 3800.145521] sock_sendmsg+0x30/0x40 >> [ 3800.149005] __sys_sendto+0x10e/0x140 >> [ 3800.152662] ? __do_page_fault+0x283/0x500 >> [ 3800.156751] __x64_sys_sendto+0x24/0x30 >> [ 3800.160585] do_syscall_64+0x42/0xf0 >> [ 3800.164156] entry_SYSCALL_64_after_hwframe+0x44/0xa9 >> [ 3800.169204] RIP: 0033:0x7f1d1d9db430 >> [ 3800.172774] RSP: 002b:00007fffb7278610 EFLAGS: 00000293 ORIG_RAX: 000000000000002c >> [ 3800.180333] RAX: ffffffffffffffda RBX: 0000000000000000 RCX: 00007f1d1d9db430 >> [ 3800.187456] RDX: 0000000000000000 RSI: 0000000000000000 RDI: 0000000000000003 >> [ 3800.194582] RBP: 0000000000000000 R08: 0000000000000000 R09: 0000000000000000 >> [ 3800.201705] R10: 0000000000000040 R11: 0000000000000293 R12: 0000000000000000 >> [ 3800.208830] R13: 0000000000000000 R14: 0000000000755510 R15: 00007f1d0d3fc000 >> [ 3800.215953] Code: d0 0f 86 db 05 00 00 01 c8 0f b7 ca 29 c8 83 e8 01 39 c6 0f 8f ea 06 00 00 48 8b 45 28 48 8d 14 92 41 b9 01 00 00 00 4c 8d 2c d0 <49> 89 5d 08 8b 83 80 00 00 00 66 45 89 4d 14 41 89 45 10 0f b7 >> [ 3800.234798] RIP: i40e_xmit_frame_ring+0xd4/0x1490 [i40e] RSP: ffffc9000407bcd0 >> [ 3800.242005] CR2: 0000000000000008 >> [ 3800.245320] ---[ end trace f169e36f468e0c59 ]--- >> [ 3801.726719] Kernel panic - not syncing: Fatal exception in interrupt >> [ 3801.733097] Kernel Offset: disabled >> [ 3801.785836] ---[ end Kernel panic - not syncing: Fatal exception in interrupt ]--- >> [ 3801.793403] ------------[ cut here ]------------ >> >> (gdb) list *(i40e_xmit_frame_ring)+0xd4 >> 0x2ccd4 is in i40e_xmit_frame_ring (drivers/net/ethernet/intel/i40e/i40e_txrx.c:4048). >> warning: Source file is more recent than executable. >> 4043 return NETDEV_TX_BUSY; >> 4044 } >> 4045 >> 4046 /* record the location of the first descriptor for this packet */ >> 4047 first = &tx_ring->tx_bi[tx_ring->next_to_use]; >> 4048 first->skb = skb; >> 4049 first->bytecount = skb->len; >> 4050 first->gso_segs = 1; >> 4051 >> 4052 /* prepare the xmit flags */ >> >> >> (gdb) list *(xsk_sendmsg)+0x1f4 >> 0xffffffff81800c34 is in xsk_sendmsg (net/xdp/xsk.c:251). >> warning: Source file is more recent than executable. >> 246 skb_shinfo(skb)->destructor_arg = (void *)(long)id; >> 247 skb->destructor = xsk_destruct_skb; >> 248 >> 249 err = dev_direct_xmit(skb, xs->queue_id); >> 250 /* Ignore NET_XMIT_CN as packet might have been sent */ >> 251 if (err == NET_XMIT_DROP || err == NETDEV_TX_BUSY) { >> 252 err = -EAGAIN; >> 253 /* SKB consumed by dev_direct_xmit() */ >> 254 goto out; >> 255 } > Found it. Checked num_rx_queues in the xsk_bind code instead of real_num_rx_queues. The code below will solve the problem. Will post a proper patch for it tomorrow. Thanks Jesper for reporting this. Appreciated. /Magnus diff --git a/net/xdp/xsk.c b/net/xdp/xsk.c index ac97902..4b62a1e 100644 --- a/net/xdp/xsk.c +++ b/net/xdp/xsk.c @@ -391,7 +391,8 @@ static int xsk_bind(struct socket *sock, struct sockaddr *addr, int addr_len) goto out_unlock; } - if (sxdp->sxdp_queue_id >= dev->num_rx_queues) { + if ((xs->rx && sxdp->sxdp_queue_id >= dev->real_num_rx_queues) || + (xs->tx && sxdp->sxdp_queue_id >= dev->real_num_tx_queues)) { err = -EINVAL; goto out_unlock; }
On Wed, 16 May 2018 17:38:12 +0200 Magnus Karlsson <magnus.karlsson@gmail.com> wrote: > On Wed, May 16, 2018 at 4:38 PM, Magnus Karlsson > <magnus.karlsson@gmail.com> wrote: > > On Wed, May 16, 2018 at 4:28 PM, Jesper Dangaard Brouer > > <brouer@redhat.com> wrote: > >> On Tue, 15 May 2018 21:06:15 +0200 > >> Björn Töpel <bjorn.topel@gmail.com> wrote: > >> > >>> From: Magnus Karlsson <magnus.karlsson@intel.com> > >>> > >>> Here, the zero-copy ndo is implemented. As a shortcut, the existing > >>> XDP Tx rings are used for zero-copy. This means that and XDP program > >>> cannot redirect to an AF_XDP enabled XDP Tx ring. > >> > >> I've changed i40e1 to only have one queue via: > >> $ ethtool -L i40e1 combined 1 > >> > >> And then, I'm sending on queue 1, which is/should not be avail... and then crash/BUG: > >> > >> $ sudo taskset -c 2 ./xdpsock --tx --interface=i40e1 --queue=1 > >> > >> [ 3799.936877] Number of in use tx queues changed invalidating tc mappings. Priority traffic > >> classification disabled! > >> [ 3799.972970] BUG: unable to handle kernel NULL pointer dereference at 0000000000000008 > >> [ 3799.980790] PGD 80000007b0992067 P4D 80000007b0992067 PUD 7b62d4067 PMD 0 > >> [ 3799.987654] Oops: 0002 [#1] PREEMPT SMP PTI > >> [ 3799.991831] Modules linked in: nf_nat_masquerade_ipv4 tun nfnetlink bridge stp llc nf_nat > >> nf_conntrack rpcrdma ib_ipoib rdma_ucm ib_ucm ib_umad rdma_cm ib_cm iw_cm sunrpc mlx5_ib ib > >> _uverbs ib_core coretemp kvm_intel kvm irqbypass intel_cstate intel_uncore intel_rapl_perf p > >> cspkr i2c_i801 shpchp wmi ipmi_si ipmi_devintf ipmi_msghandler acpi_pad sch_fq_codel i40e ml > >> x5_core hid_generic ixgbe igb devlink mdio ptp sd_mod i2c_algo_bit i2c_core pps_core [last u > >> nloaded: x_tables] > >> [ 3800.033472] CPU: 2 PID: 2006 Comm: xdpsock Not tainted 4.17.0-rc3-af_xdp03_ZC_rfc+ #155 > >> [ 3800.041465] Hardware name: Supermicro Super Server/X10SRi-F, BIOS 2.0a 08/01/2016 > >> [ 3800.048943] RIP: 0010:i40e_xmit_frame_ring+0xd4/0x1490 [i40e] > >> [ 3800.054683] RSP: 0018:ffffc9000407bcd0 EFLAGS: 00010293 > >> [ 3800.059900] RAX: 0000000000000000 RBX: ffff88084f0fd200 RCX: 0000000000000000 > >> [ 3800.067022] RDX: 0000000000000000 RSI: 0000000000000006 RDI: ffff8807b6e710c0 > >> [ 3800.074148] RBP: ffff8807c6397800 R08: 00000000000000c0 R09: 0000000000000001 > >> [ 3800.081270] R10: 0000000000000800 R11: 0000000000000010 R12: 0000000000000001 > >> [ 3800.088396] R13: 0000000000000000 R14: 0000000000000001 R15: 000000000000003c > >> [ 3800.095520] FS: 00007f1d1e00bb80(0000) GS:ffff88087fc80000(0000) knlGS:0000000000000000 > >> [ 3800.103597] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 > >> [ 3800.109335] CR2: 0000000000000008 CR3: 000000087d542001 CR4: 00000000003606e0 > >> [ 3800.116458] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 > >> [ 3800.123583] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 > >> [ 3800.130706] Call Trace: > >> [ 3800.133157] ? validate_xmit_skb.isra.116+0x1c/0x270 > >> [ 3800.138118] dev_direct_xmit+0xec/0x1d0 > >> [ 3800.141949] xsk_sendmsg+0x1f4/0x380 > >> [ 3800.145521] sock_sendmsg+0x30/0x40 > >> [ 3800.149005] __sys_sendto+0x10e/0x140 > >> [ 3800.152662] ? __do_page_fault+0x283/0x500 > >> [ 3800.156751] __x64_sys_sendto+0x24/0x30 > >> [ 3800.160585] do_syscall_64+0x42/0xf0 > >> [ 3800.164156] entry_SYSCALL_64_after_hwframe+0x44/0xa9 > >> [ 3800.169204] RIP: 0033:0x7f1d1d9db430 > >> [ 3800.172774] RSP: 002b:00007fffb7278610 EFLAGS: 00000293 ORIG_RAX: 000000000000002c > >> [ 3800.180333] RAX: ffffffffffffffda RBX: 0000000000000000 RCX: 00007f1d1d9db430 > >> [ 3800.187456] RDX: 0000000000000000 RSI: 0000000000000000 RDI: 0000000000000003 > >> [ 3800.194582] RBP: 0000000000000000 R08: 0000000000000000 R09: 0000000000000000 > >> [ 3800.201705] R10: 0000000000000040 R11: 0000000000000293 R12: 0000000000000000 > >> [ 3800.208830] R13: 0000000000000000 R14: 0000000000755510 R15: 00007f1d0d3fc000 > >> [ 3800.215953] Code: d0 0f 86 db 05 00 00 01 c8 0f b7 ca 29 c8 83 e8 01 39 c6 0f 8f ea 06 00 00 48 8b 45 28 48 8d 14 92 41 b9 01 00 00 00 4c 8d 2c d0 <49> 89 5d 08 8b 83 80 00 00 00 66 45 89 4d 14 41 89 45 10 0f b7 > >> [ 3800.234798] RIP: i40e_xmit_frame_ring+0xd4/0x1490 [i40e] RSP: ffffc9000407bcd0 > >> [ 3800.242005] CR2: 0000000000000008 > >> [ 3800.245320] ---[ end trace f169e36f468e0c59 ]--- > >> [ 3801.726719] Kernel panic - not syncing: Fatal exception in interrupt > >> [ 3801.733097] Kernel Offset: disabled > >> [ 3801.785836] ---[ end Kernel panic - not syncing: Fatal exception in interrupt ]--- > >> [ 3801.793403] ------------[ cut here ]------------ > >> > >> (gdb) list *(i40e_xmit_frame_ring)+0xd4 > >> 0x2ccd4 is in i40e_xmit_frame_ring (drivers/net/ethernet/intel/i40e/i40e_txrx.c:4048). > >> warning: Source file is more recent than executable. > >> 4043 return NETDEV_TX_BUSY; > >> 4044 } > >> 4045 > >> 4046 /* record the location of the first descriptor for this packet */ > >> 4047 first = &tx_ring->tx_bi[tx_ring->next_to_use]; > >> 4048 first->skb = skb; > >> 4049 first->bytecount = skb->len; > >> 4050 first->gso_segs = 1; > >> 4051 > >> 4052 /* prepare the xmit flags */ > >> > >> > >> (gdb) list *(xsk_sendmsg)+0x1f4 > >> 0xffffffff81800c34 is in xsk_sendmsg (net/xdp/xsk.c:251). > >> warning: Source file is more recent than executable. > >> 246 skb_shinfo(skb)->destructor_arg = (void *)(long)id; > >> 247 skb->destructor = xsk_destruct_skb; > >> 248 > >> 249 err = dev_direct_xmit(skb, xs->queue_id); > >> 250 /* Ignore NET_XMIT_CN as packet might have been sent */ > >> 251 if (err == NET_XMIT_DROP || err == NETDEV_TX_BUSY) { > >> 252 err = -EAGAIN; > >> 253 /* SKB consumed by dev_direct_xmit() */ > >> 254 goto out; > >> 255 } > > > > Found it. Checked num_rx_queues in the xsk_bind code instead of > real_num_rx_queues. The code below will solve the problem. Will post a > proper patch for it tomorrow. Thanks Jesper for reporting this. > Appreciated. > > /Magnus > > diff --git a/net/xdp/xsk.c b/net/xdp/xsk.c > index ac97902..4b62a1e 100644 > --- a/net/xdp/xsk.c > +++ b/net/xdp/xsk.c > @@ -391,7 +391,8 @@ static int xsk_bind(struct socket *sock, struct > sockaddr *addr, int addr_len) > goto out_unlock; > } > > - if (sxdp->sxdp_queue_id >= dev->num_rx_queues) { > + if ((xs->rx && sxdp->sxdp_queue_id >= dev->real_num_rx_queues) || > + (xs->tx && sxdp->sxdp_queue_id >= dev->real_num_tx_queues)) { > err = -EINVAL; > goto out_unlock; > } Tried this patch... it fixed/catched the problem :-) $ sudo ./xdpsock --interface=i40e1 --queue=42 --txonly samples/bpf/xdpsock_user.c:xsk_configure:528: Assertion failed: bind(sfd, (struct sockaddr *)&sxdp, sizeof(sxdp)) == 0: errno: 22/"Invalid argument" Segmentation fault You can add: Tested-by: Jesper Dangaard Brouer <brouer@redhat.com> Notice: this bug is not related to this zero-copy patch, but to your previous patchset, which are in bpf-next. Thus, you need to send a fix patch to bpf-next...
On Tue, 15 May 2018 21:06:15 +0200 Björn Töpel <bjorn.topel@gmail.com> wrote: > From: Magnus Karlsson <magnus.karlsson@intel.com> > > Here, the zero-copy ndo is implemented. As a shortcut, the existing > XDP Tx rings are used for zero-copy. This means that and XDP program > cannot redirect to an AF_XDP enabled XDP Tx ring. This "shortcut" is not acceptable, and completely broken. The XDP_REDIRECT queue_index is based on smp_processor_id(), and can easily clash with the configured XSK queue_index. Provided a bit more code context below... On Tue, 15 May 2018 21:06:15 +0200 Björn Töpel <bjorn.topel@gmail.com> wrote: int i40e_xdp_xmit(struct net_device *dev, struct xdp_frame *xdpf) { struct i40e_netdev_priv *np = netdev_priv(dev); unsigned int queue_index = smp_processor_id(); struct i40e_vsi *vsi = np->vsi; int err; if (test_bit(__I40E_VSI_DOWN, vsi->state)) return -ENETDOWN; > @@ -4025,6 +4158,9 @@ int i40e_xdp_xmit(struct net_device *dev, struct xdp_frame *xdpf) > if (!i40e_enabled_xdp_vsi(vsi) || queue_index >= vsi->num_queue_pairs) > return -ENXIO; > > + if (vsi->xdp_rings[queue_index]->xsk_umem) > + return -ENXIO; > + Using the sane errno makes this impossible to debug (via the tracepoints). > err = i40e_xmit_xdp_ring(xdpf, vsi->xdp_rings[queue_index]); > if (err != I40E_XDP_TX) > return -ENOSPC; > @@ -4048,5 +4184,34 @@ void i40e_xdp_flush(struct net_device *dev) > if (!i40e_enabled_xdp_vsi(vsi) || queue_index >= vsi->num_queue_pairs) > return; > > + if (vsi->xdp_rings[queue_index]->xsk_umem) > + return; > + > i40e_xdp_ring_update_tail(vsi->xdp_rings[queue_index]); > }
2018-05-17 23:31 GMT+02:00 Jesper Dangaard Brouer <brouer@redhat.com>: > > On Tue, 15 May 2018 21:06:15 +0200 Björn Töpel <bjorn.topel@gmail.com> wrote: > >> From: Magnus Karlsson <magnus.karlsson@intel.com> >> >> Here, the zero-copy ndo is implemented. As a shortcut, the existing >> XDP Tx rings are used for zero-copy. This means that and XDP program >> cannot redirect to an AF_XDP enabled XDP Tx ring. > > This "shortcut" is not acceptable, and completely broken. The > XDP_REDIRECT queue_index is based on smp_processor_id(), and can easily > clash with the configured XSK queue_index. Provided a bit more code > context below... > Yes, and this is the reason we need to go for a solution with dedicated Tx rings. Again, we chose not to, and simply drops XDP_REDIRECT where the AF_XDP queue id clashes with the processor id. The queue id hijacked by AF_XDP's egress side. > On Tue, 15 May 2018 21:06:15 +0200 > Björn Töpel <bjorn.topel@gmail.com> wrote: > > int i40e_xdp_xmit(struct net_device *dev, struct xdp_frame *xdpf) > { > struct i40e_netdev_priv *np = netdev_priv(dev); > unsigned int queue_index = smp_processor_id(); > struct i40e_vsi *vsi = np->vsi; > int err; > > if (test_bit(__I40E_VSI_DOWN, vsi->state)) > return -ENETDOWN; > >> @@ -4025,6 +4158,9 @@ int i40e_xdp_xmit(struct net_device *dev, struct xdp_frame *xdpf) >> if (!i40e_enabled_xdp_vsi(vsi) || queue_index >= vsi->num_queue_pairs) >> return -ENXIO; >> >> + if (vsi->xdp_rings[queue_index]->xsk_umem) >> + return -ENXIO; >> + > > Using the sane errno makes this impossible to debug (via the tracepoints). > The rationale was that the situation was similar to an incorrectly configured receiving (from an XDP_REDIRECT perspective) interface. We'll rework this! Thanks for looking into this, Jesper! Björn >> err = i40e_xmit_xdp_ring(xdpf, vsi->xdp_rings[queue_index]); >> if (err != I40E_XDP_TX) >> return -ENOSPC; >> @@ -4048,5 +4184,34 @@ void i40e_xdp_flush(struct net_device *dev) >> if (!i40e_enabled_xdp_vsi(vsi) || queue_index >= vsi->num_queue_pairs) >> return; >> >> + if (vsi->xdp_rings[queue_index]->xsk_umem) >> + return; >> + >> i40e_xdp_ring_update_tail(vsi->xdp_rings[queue_index]); >> } > > > > -- > Best regards, > Jesper Dangaard Brouer > MSc.CS, Principal Kernel Engineer at Red Hat > LinkedIn: http://www.linkedin.com/in/brouer
diff --git a/drivers/net/ethernet/intel/i40e/i40e_main.c b/drivers/net/ethernet/intel/i40e/i40e_main.c index dc3d668a741e..91f8e892179a 100644 --- a/drivers/net/ethernet/intel/i40e/i40e_main.c +++ b/drivers/net/ethernet/intel/i40e/i40e_main.c @@ -3055,8 +3055,12 @@ static int i40e_configure_tx_ring(struct i40e_ring *ring) i40e_status err = 0; u32 qtx_ctl = 0; - if (ring_is_xdp(ring)) + ring->clean_tx_irq = i40e_clean_tx_irq; + if (ring_is_xdp(ring)) { ring->xsk_umem = i40e_xsk_umem(ring); + if (ring->xsk_umem) + ring->clean_tx_irq = i40e_clean_tx_irq_zc; + } /* some ATR related tx ring init */ if (vsi->back->flags & I40E_FLAG_FD_ATR_ENABLED) { @@ -12309,6 +12313,7 @@ static const struct net_device_ops i40e_netdev_ops = { .ndo_bpf = i40e_xdp, .ndo_xdp_xmit = i40e_xdp_xmit, .ndo_xdp_flush = i40e_xdp_flush, + .ndo_xsk_async_xmit = i40e_xsk_async_xmit, }; /** diff --git a/drivers/net/ethernet/intel/i40e/i40e_txrx.c b/drivers/net/ethernet/intel/i40e/i40e_txrx.c index f89ac524652c..17c067556aba 100644 --- a/drivers/net/ethernet/intel/i40e/i40e_txrx.c +++ b/drivers/net/ethernet/intel/i40e/i40e_txrx.c @@ -649,9 +649,13 @@ void i40e_clean_tx_ring(struct i40e_ring *tx_ring) if (!tx_ring->tx_bi) return; - /* Free all the Tx ring sk_buffs */ - for (i = 0; i < tx_ring->count; i++) - i40e_unmap_and_free_tx_resource(tx_ring, &tx_ring->tx_bi[i]); + /* Cleanup only needed for non XSK TX ZC rings */ + if (!tx_ring->xsk_umem) { + /* Free all the Tx ring sk_buffs */ + for (i = 0; i < tx_ring->count; i++) + i40e_unmap_and_free_tx_resource(tx_ring, + &tx_ring->tx_bi[i]); + } bi_size = sizeof(struct i40e_tx_buffer) * tx_ring->count; memset(tx_ring->tx_bi, 0, bi_size); @@ -768,8 +772,139 @@ void i40e_detect_recover_hung(struct i40e_vsi *vsi) } } +static inline void i40e_xdp_ring_update_tail(struct i40e_ring *xdp_ring) +{ + /* Force memory writes to complete before letting h/w + * know there are new descriptors to fetch. + */ + wmb(); + writel_relaxed(xdp_ring->next_to_use, xdp_ring->tail); +} + #define WB_STRIDE 4 +static void i40e_update_stats_and_arm_wb(struct i40e_ring *tx_ring, + struct i40e_vsi *vsi, + unsigned int total_packets, + unsigned int total_bytes, + int budget) +{ + u64_stats_update_begin(&tx_ring->syncp); + tx_ring->stats.bytes += total_bytes; + tx_ring->stats.packets += total_packets; + u64_stats_update_end(&tx_ring->syncp); + tx_ring->q_vector->tx.total_bytes += total_bytes; + tx_ring->q_vector->tx.total_packets += total_packets; + + if (tx_ring->flags & I40E_TXR_FLAGS_WB_ON_ITR) { + /* check to see if there are < 4 descriptors + * waiting to be written back, then kick the hardware to force + * them to be written back in case we stay in NAPI. + * In this mode on X722 we do not enable Interrupt. + */ + unsigned int j = i40e_get_tx_pending(tx_ring, false); + + if (budget && + ((j / WB_STRIDE) == 0) && (j > 0) && + !test_bit(__I40E_VSI_DOWN, vsi->state) && + (I40E_DESC_UNUSED(tx_ring) != tx_ring->count)) + tx_ring->arm_wb = true; + } +} + +/* Returns true if the work is finished */ +static bool i40e_xmit_zc(struct i40e_ring *xdp_ring, unsigned int budget) +{ + bool work_done = true, xdp_xmit = false; + struct i40e_tx_buffer *tx_bi; + struct i40e_tx_desc *tx_desc; + dma_addr_t dma; + u16 offset; + u32 len; + + while (budget-- > 0) { + if (!unlikely(I40E_DESC_UNUSED(xdp_ring))) { + xdp_ring->tx_stats.tx_busy++; + work_done = false; + break; + } + + if (!xsk_umem_consume_tx(xdp_ring->xsk_umem, &dma, &len, + &offset)) + break; + + xdp_xmit = true; + dma_sync_single_for_device(xdp_ring->dev, dma, len, + DMA_BIDIRECTIONAL); + + tx_bi = &xdp_ring->tx_bi[xdp_ring->next_to_use]; + tx_bi->bytecount = len; + tx_bi->gso_segs = 1; + + tx_desc = I40E_TX_DESC(xdp_ring, xdp_ring->next_to_use); + tx_desc->buffer_addr = cpu_to_le64(dma); + tx_desc->cmd_type_offset_bsz = build_ctob(I40E_TX_DESC_CMD_ICRC + | I40E_TX_DESC_CMD_EOP, + 0, len, 0); + + xdp_ring->next_to_use++; + if (xdp_ring->next_to_use == xdp_ring->count) + xdp_ring->next_to_use = 0; + } + + /* Request an interrupt for the last frame and bump tail ptr. */ + if (xdp_xmit) { + tx_desc->cmd_type_offset_bsz |= (I40E_TX_DESC_CMD_RS << + I40E_TXD_QW1_CMD_SHIFT); + i40e_xdp_ring_update_tail(xdp_ring); + } + + return !!budget && work_done; +} + +bool i40e_clean_tx_irq_zc(struct i40e_vsi *vsi, + struct i40e_ring *tx_ring, int napi_budget) +{ + unsigned int total_bytes = 0, total_packets = 0; + struct xdp_umem *umem = tx_ring->xsk_umem; + u32 head_idx = i40e_get_head(tx_ring); + unsigned int budget = vsi->work_limit; + bool work_done = true, xmit_done; + u32 completed_frames; + u32 frames_ready; + + if (head_idx < tx_ring->next_to_clean) + head_idx += tx_ring->count; + frames_ready = head_idx - tx_ring->next_to_clean; + + if (frames_ready == 0) { + goto out_xmit; + } else if (frames_ready > budget) { + completed_frames = budget; + work_done = false; + } else { + completed_frames = frames_ready; + } + + /* XXX Need to be calculated. */ + /*total_bytes += tx_buf->bytecount;*/ + total_packets += completed_frames; + + tx_ring->next_to_clean += completed_frames; + if (unlikely(tx_ring->next_to_clean >= tx_ring->count)) + tx_ring->next_to_clean -= tx_ring->count; + + xsk_umem_complete_tx(umem, completed_frames); + + i40e_update_stats_and_arm_wb(tx_ring, vsi, total_packets, + total_bytes, budget); + +out_xmit: + xmit_done = i40e_xmit_zc(tx_ring, budget); + + return work_done && xmit_done; +} + /** * i40e_clean_tx_irq - Reclaim resources after transmit completes * @vsi: the VSI we care about @@ -778,8 +913,8 @@ void i40e_detect_recover_hung(struct i40e_vsi *vsi) * * Returns true if there's any budget left (e.g. the clean is finished) **/ -static bool i40e_clean_tx_irq(struct i40e_vsi *vsi, - struct i40e_ring *tx_ring, int napi_budget) +bool i40e_clean_tx_irq(struct i40e_vsi *vsi, + struct i40e_ring *tx_ring, int napi_budget) { u16 i = tx_ring->next_to_clean; struct i40e_tx_buffer *tx_buf; @@ -874,27 +1009,9 @@ static bool i40e_clean_tx_irq(struct i40e_vsi *vsi, i += tx_ring->count; tx_ring->next_to_clean = i; - u64_stats_update_begin(&tx_ring->syncp); - tx_ring->stats.bytes += total_bytes; - tx_ring->stats.packets += total_packets; - u64_stats_update_end(&tx_ring->syncp); - tx_ring->q_vector->tx.total_bytes += total_bytes; - tx_ring->q_vector->tx.total_packets += total_packets; - if (tx_ring->flags & I40E_TXR_FLAGS_WB_ON_ITR) { - /* check to see if there are < 4 descriptors - * waiting to be written back, then kick the hardware to force - * them to be written back in case we stay in NAPI. - * In this mode on X722 we do not enable Interrupt. - */ - unsigned int j = i40e_get_tx_pending(tx_ring, false); - - if (budget && - ((j / WB_STRIDE) == 0) && (j > 0) && - !test_bit(__I40E_VSI_DOWN, vsi->state) && - (I40E_DESC_UNUSED(tx_ring) != tx_ring->count)) - tx_ring->arm_wb = true; - } + i40e_update_stats_and_arm_wb(tx_ring, vsi, total_packets, + total_bytes, budget); if (ring_is_xdp(tx_ring)) return !!budget; @@ -2266,15 +2383,6 @@ static void i40e_rx_buffer_flip(struct i40e_ring *rx_ring, #endif } -static inline void i40e_xdp_ring_update_tail(struct i40e_ring *xdp_ring) -{ - /* Force memory writes to complete before letting h/w - * know there are new descriptors to fetch. - */ - wmb(); - writel_relaxed(xdp_ring->next_to_use, xdp_ring->tail); -} - /** * i40e_clean_rx_irq - Clean completed descriptors from Rx ring - bounce buf * @rx_ring: rx descriptor ring to transact packets on @@ -2904,10 +3012,11 @@ int i40e_napi_poll(struct napi_struct *napi, int budget) * budget and be more aggressive about cleaning up the Tx descriptors. */ i40e_for_each_ring(ring, q_vector->tx) { - if (!i40e_clean_tx_irq(vsi, ring, budget)) { + if (!ring->clean_tx_irq(vsi, ring, budget)) { clean_complete = false; continue; } + arm_wb |= ring->arm_wb; ring->arm_wb = false; } @@ -3810,6 +3919,30 @@ static inline int i40e_tx_map(struct i40e_ring *tx_ring, struct sk_buff *skb, return -1; } +/** + * i40e_napi_is_scheduled - If napi is running, set the NAPIF_STATE_MISSED + * @n: napi context + * + * Returns true if NAPI is scheduled. + **/ +static bool i40e_napi_is_scheduled(struct napi_struct *n) +{ + unsigned long val, new; + + do { + val = READ_ONCE(n->state); + if (val & NAPIF_STATE_DISABLE) + return true; + + if (!(val & NAPIF_STATE_SCHED)) + return false; + + new = val | NAPIF_STATE_MISSED; + } while (cmpxchg(&n->state, val, new) != val); + + return true; +} + /** * i40e_xmit_xdp_ring - transmits an XDP buffer to an XDP Tx ring * @xdp: data to transmit @@ -4025,6 +4158,9 @@ int i40e_xdp_xmit(struct net_device *dev, struct xdp_frame *xdpf) if (!i40e_enabled_xdp_vsi(vsi) || queue_index >= vsi->num_queue_pairs) return -ENXIO; + if (vsi->xdp_rings[queue_index]->xsk_umem) + return -ENXIO; + err = i40e_xmit_xdp_ring(xdpf, vsi->xdp_rings[queue_index]); if (err != I40E_XDP_TX) return -ENOSPC; @@ -4048,5 +4184,34 @@ void i40e_xdp_flush(struct net_device *dev) if (!i40e_enabled_xdp_vsi(vsi) || queue_index >= vsi->num_queue_pairs) return; + if (vsi->xdp_rings[queue_index]->xsk_umem) + return; + i40e_xdp_ring_update_tail(vsi->xdp_rings[queue_index]); } + +int i40e_xsk_async_xmit(struct net_device *dev, u32 queue_id) +{ + struct i40e_netdev_priv *np = netdev_priv(dev); + struct i40e_vsi *vsi = np->vsi; + struct i40e_ring *ring; + + if (test_bit(__I40E_VSI_DOWN, vsi->state)) + return -ENETDOWN; + + if (!i40e_enabled_xdp_vsi(vsi)) + return -ENXIO; + + if (queue_id >= vsi->num_queue_pairs) + return -ENXIO; + + if (!vsi->xdp_rings[queue_id]->xsk_umem) + return -ENXIO; + + ring = vsi->xdp_rings[queue_id]; + + if (!i40e_napi_is_scheduled(&ring->q_vector->napi)) + i40e_force_wb(vsi, ring->q_vector); + + return 0; +} diff --git a/drivers/net/ethernet/intel/i40e/i40e_txrx.h b/drivers/net/ethernet/intel/i40e/i40e_txrx.h index 9d5d9862e9f1..ea1cac00cad4 100644 --- a/drivers/net/ethernet/intel/i40e/i40e_txrx.h +++ b/drivers/net/ethernet/intel/i40e/i40e_txrx.h @@ -428,6 +428,7 @@ struct i40e_ring { int (*clean_rx_irq)(struct i40e_ring *, int); bool (*alloc_rx_buffers)(struct i40e_ring *, u16); + bool (*clean_tx_irq)(struct i40e_vsi *, struct i40e_ring *, int); struct xdp_umem *xsk_umem; struct zero_copy_allocator zca; /* ZC allocator anchor */ @@ -510,6 +511,11 @@ void i40e_xdp_flush(struct net_device *dev); int i40e_clean_rx_irq(struct i40e_ring *rx_ring, int budget); int i40e_clean_rx_irq_zc(struct i40e_ring *rx_ring, int budget); void i40e_zca_free(struct zero_copy_allocator *alloc, unsigned long handle); +bool i40e_clean_tx_irq_zc(struct i40e_vsi *vsi, + struct i40e_ring *tx_ring, int napi_budget); +bool i40e_clean_tx_irq(struct i40e_vsi *vsi, + struct i40e_ring *tx_ring, int napi_budget); +int i40e_xsk_async_xmit(struct net_device *dev, u32 queue_id); /** * i40e_get_head - Retrieve head from head writeback