@@ -2580,7 +2580,7 @@ int i40e_napi_poll(struct napi_struct *napi, int budget)
*/
i40e_for_each_ring(ring, q_vector->tx) {
bool wd = ring->xsk_umem ?
- i40e_clean_xdp_tx_irq(vsi, ring, budget) :
+ i40e_clean_xdp_tx_irq(vsi, ring) :
i40e_clean_tx_irq(vsi, ring, budget);
if (!wd) {
@@ -3538,6 +3538,7 @@ static int i40e_xmit_xdp_ring(struct xdp_frame *xdpf,
*/
smp_wmb();
+ xdp_ring->xdp_tx_active++;
i++;
if (i == xdp_ring->count)
i = 0;
@@ -347,6 +347,7 @@ struct i40e_ring {
/* used in interrupt processing */
u16 next_to_use;
u16 next_to_clean;
+ u16 xdp_tx_active;
u8 atr_sample_rate;
u8 atr_count;
@@ -378,6 +378,7 @@ int i40e_clean_rx_irq_zc(struct i40e_ring *rx_ring, int budget)
**/
static bool i40e_xmit_zc(struct i40e_ring *xdp_ring, unsigned int budget)
{
+ unsigned int sent_frames = 0, total_bytes = 0;
struct i40e_tx_desc *tx_desc = NULL;
struct i40e_tx_buffer *tx_bi;
bool work_done = true;
@@ -408,6 +409,9 @@ static bool i40e_xmit_zc(struct i40e_ring *xdp_ring, unsigned int budget)
| I40E_TX_DESC_CMD_EOP,
0, desc.len, 0);
+ sent_frames++;
+ total_bytes += tx_bi->bytecount;
+
xdp_ring->next_to_use++;
if (xdp_ring->next_to_use == xdp_ring->count)
xdp_ring->next_to_use = 0;
@@ -420,6 +424,7 @@ static bool i40e_xmit_zc(struct i40e_ring *xdp_ring, unsigned int budget)
i40e_xdp_ring_update_tail(xdp_ring);
xsk_umem_consume_tx_done(xdp_ring->xsk_umem);
+ i40e_update_tx_stats(xdp_ring, sent_frames, total_bytes);
}
return !!budget && work_done;
@@ -434,6 +439,7 @@ static void i40e_clean_xdp_tx_buffer(struct i40e_ring *tx_ring,
struct i40e_tx_buffer *tx_bi)
{
xdp_return_frame(tx_bi->xdpf);
+ tx_ring->xdp_tx_active--;
dma_unmap_single(tx_ring->dev,
dma_unmap_addr(tx_bi, dma),
dma_unmap_len(tx_bi, len), DMA_TO_DEVICE);
@@ -447,27 +453,25 @@ static void i40e_clean_xdp_tx_buffer(struct i40e_ring *tx_ring,
*
* Returns true if cleanup/tranmission is done.
**/
-bool i40e_clean_xdp_tx_irq(struct i40e_vsi *vsi,
- struct i40e_ring *tx_ring, int napi_budget)
+bool i40e_clean_xdp_tx_irq(struct i40e_vsi *vsi, struct i40e_ring *tx_ring)
{
- unsigned int ntc, total_bytes = 0, budget = vsi->work_limit;
- u32 i, completed_frames, frames_ready, xsk_frames = 0;
+ unsigned int ntc, budget = vsi->work_limit;
struct xdp_umem *umem = tx_ring->xsk_umem;
+ u32 i, completed_frames, xsk_frames = 0;
u32 head_idx = i40e_get_head(tx_ring);
- bool work_done = true, xmit_done;
struct i40e_tx_buffer *tx_bi;
+ bool xmit_done;
if (head_idx < tx_ring->next_to_clean)
head_idx += tx_ring->count;
- frames_ready = head_idx - tx_ring->next_to_clean;
+ completed_frames = head_idx - tx_ring->next_to_clean;
- if (frames_ready == 0) {
+ if (completed_frames == 0)
goto out_xmit;
- } else if (frames_ready > budget) {
- completed_frames = budget;
- work_done = false;
- } else {
- completed_frames = frames_ready;
+
+ if (likely(!tx_ring->xdp_tx_active)) {
+ xsk_frames = completed_frames;
+ goto skip;
}
ntc = tx_ring->next_to_clean;
@@ -475,18 +479,18 @@ bool i40e_clean_xdp_tx_irq(struct i40e_vsi *vsi,
for (i = 0; i < completed_frames; i++) {
tx_bi = &tx_ring->tx_bi[ntc];
- if (tx_bi->xdpf)
+ if (tx_bi->xdpf) {
i40e_clean_xdp_tx_buffer(tx_ring, tx_bi);
- else
+ tx_bi->xdpf = NULL;
+ } else {
xsk_frames++;
-
- tx_bi->xdpf = NULL;
- total_bytes += tx_bi->bytecount;
+ }
if (++ntc >= tx_ring->count)
ntc = 0;
}
+skip:
tx_ring->next_to_clean += completed_frames;
if (unlikely(tx_ring->next_to_clean >= tx_ring->count))
tx_ring->next_to_clean -= tx_ring->count;
@@ -494,8 +498,7 @@ bool i40e_clean_xdp_tx_irq(struct i40e_vsi *vsi,
if (xsk_frames)
xsk_umem_complete_tx(umem, xsk_frames);
- i40e_arm_wb(tx_ring, vsi, budget);
- i40e_update_tx_stats(tx_ring, completed_frames, total_bytes);
+ i40e_arm_wb(tx_ring, vsi, completed_frames);
out_xmit:
if (xsk_umem_uses_need_wakeup(tx_ring->xsk_umem))
@@ -503,7 +506,7 @@ bool i40e_clean_xdp_tx_irq(struct i40e_vsi *vsi,
xmit_done = i40e_xmit_zc(tx_ring, budget);
- return work_done && xmit_done;
+ return xmit_done;
}
/**
@@ -15,8 +15,7 @@ int i40e_xsk_umem_setup(struct i40e_vsi *vsi, struct xdp_umem *umem,
bool i40e_alloc_rx_buffers_zc(struct i40e_ring *rx_ring, u16 cleaned_count);
int i40e_clean_rx_irq_zc(struct i40e_ring *rx_ring, int budget);
-bool i40e_clean_xdp_tx_irq(struct i40e_vsi *vsi,
- struct i40e_ring *tx_ring, int napi_budget);
+bool i40e_clean_xdp_tx_irq(struct i40e_vsi *vsi, struct i40e_ring *tx_ring);
int i40e_xsk_wakeup(struct net_device *dev, u32 queue_id, u32 flags);
int i40e_alloc_rx_bi_zc(struct i40e_ring *rx_ring);
void i40e_clear_rx_bi_zc(struct i40e_ring *rx_ring);