niansa-misc/linux
1
0
Fork 0
mirror of synced 2025-03-06 20:59:54 +01:00
Code Issues Projects Releases Packages Wiki Activity

ice: xsk: Do not convert to buff to frame for XDP_TX

Browse source 
Let us store pointer to xdp_buff that came from xsk_buff_pool on tx_buf
so that it will be possible to recycle it via xsk_buff_free() on Tx
cleaning side. This way it is not necessary to do expensive copy to
another xdp_buff backed by a newly allocated page.

Signed-off-by: Maciej Fijalkowski <maciej.fijalkowski@intel.com>
Signed-off-by: Daniel Borkmann <daniel@iogearbox.net>
Reviewed-by: Alexander Lobakin <alexandr.lobakin@intel.com>
Link: https://lore.kernel.org/bpf/20230131204506.219292-14-maciej.fijalkowski@intel.com
This commit is contained in:
Maciej Fijalkowski 2023-01-31 21:45:06 +01:00 • committed by Daniel Borkmann
parent f4db7b314d
commit a24b4c6e9a
4 changed files with 117 additions and 94 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

12
drivers/net/ethernet/intel/ice/ice_txrx.c
View file

@ -113,12 +113,16 @@ static void
ice_unmap_and_free_tx_buf(struct ice_tx_ring *ring, struct ice_tx_buf *tx_buf)
{
if (tx_buf->skb) {
if (tx_buf->tx_flags & ICE_TX_FLAGS_DUMMY_PKT)
if (tx_buf->tx_flags & ICE_TX_FLAGS_DUMMY_PKT) {
devm_kfree(ring->dev, tx_buf->raw_buf);
else if (ice_ring_is_xdp(ring))
page_frag_free(tx_buf->raw_buf);
else
} else if (ice_ring_is_xdp(ring)) {
if (ring->xsk_pool)
xsk_buff_free(tx_buf->xdp);
else
page_frag_free(tx_buf->raw_buf);
} else {
dev_kfree_skb_any(tx_buf->skb);
}
if (dma_unmap_len(tx_buf, len))
dma_unmap_single(ring->dev,
dma_unmap_addr(tx_buf, dma),

1
drivers/net/ethernet/intel/ice/ice_txrx.h
View file

@ -157,6 +157,7 @@ struct ice_tx_buf {
union {
struct sk_buff *skb;
void *raw_buf; /* used for XDP */
struct xdp_buff *xdp; /* used for XDP_TX ZC */
};
unsigned int bytecount;
union {

22
drivers/net/ethernet/intel/ice/ice_txrx_lib.c
View file

@ -275,7 +275,10 @@ static u32 ice_clean_xdp_irq(struct ice_tx_ring *xdp_ring)
/* count head + frags */
ready_frames -= frags + 1;
ice_clean_xdp_tx_buf(xdp_ring, tx_buf);
if (xdp_ring->xsk_pool)
xsk_buff_free(tx_buf->xdp);
else
ice_clean_xdp_tx_buf(xdp_ring, tx_buf);
ntc++;
if (ntc == cnt)
ntc = 0;
@ -407,23 +410,6 @@ int ice_xmit_xdp_ring(struct xdp_frame *xdpf, struct ice_tx_ring *xdp_ring)
return __ice_xmit_xdp_ring(&xdp, xdp_ring);
}
/**
* ice_xmit_xdp_buff - convert an XDP buffer to an XDP frame and send it
* @xdp: XDP buffer
* @xdp_ring: XDP Tx ring
*
* Returns negative on failure, 0 on success.
*/
int ice_xmit_xdp_buff(struct xdp_buff *xdp, struct ice_tx_ring *xdp_ring)
{
struct xdp_frame *xdpf = xdp_convert_buff_to_frame(xdp);
if (unlikely(!xdpf))
return ICE_XDP_CONSUMED;
return ice_xmit_xdp_ring(xdpf, xdp_ring);
}
/**
* ice_finalize_xdp_rx - Bump XDP Tx tail and/or flush redirect map
* @xdp_ring: XDP ring

176
drivers/net/ethernet/intel/ice/ice_xsk.c
View file

@ -597,6 +597,107 @@ ice_construct_skb_zc(struct ice_rx_ring *rx_ring, struct xdp_buff *xdp)
return skb;
}
/**
* ice_clean_xdp_irq_zc - AF_XDP ZC specific Tx cleaning routine
* @xdp_ring: XDP Tx ring
*/
static void ice_clean_xdp_irq_zc(struct ice_tx_ring *xdp_ring)
{
u16 ntc = xdp_ring->next_to_clean;
struct ice_tx_desc *tx_desc;
u16 cnt = xdp_ring->count;
struct ice_tx_buf *tx_buf;
u16 xsk_frames = 0;
u16 last_rs;
int i;
last_rs = xdp_ring->next_to_use ? xdp_ring->next_to_use - 1 : cnt - 1;
tx_desc = ICE_TX_DESC(xdp_ring, last_rs);
if (tx_desc->cmd_type_offset_bsz &
cpu_to_le64(ICE_TX_DESC_DTYPE_DESC_DONE)) {
if (last_rs >= ntc)
xsk_frames = last_rs - ntc + 1;
else
xsk_frames = last_rs + cnt - ntc + 1;
}
if (!xsk_frames)
return;
if (likely(!xdp_ring->xdp_tx_active))
goto skip;
ntc = xdp_ring->next_to_clean;
for (i = 0; i < xsk_frames; i++) {
tx_buf = &xdp_ring->tx_buf[ntc];
if (tx_buf->xdp) {
xsk_buff_free(tx_buf->xdp);
xdp_ring->xdp_tx_active--;
} else {
xsk_frames++;
}
ntc++;
if (ntc == cnt)
ntc = 0;
}
skip:
tx_desc->cmd_type_offset_bsz = 0;
xdp_ring->next_to_clean += xsk_frames;
if (xdp_ring->next_to_clean >= cnt)
xdp_ring->next_to_clean -= cnt;
if (xsk_frames)
xsk_tx_completed(xdp_ring->xsk_pool, xsk_frames);
}
/**
* ice_xmit_xdp_tx_zc - AF_XDP ZC handler for XDP_TX
* @xdp: XDP buffer to xmit
* @xdp_ring: XDP ring to produce descriptor onto
*
* note that this function works directly on xdp_buff, no need to convert
* it to xdp_frame. xdp_buff pointer is stored to ice_tx_buf so that cleaning
* side will be able to xsk_buff_free() it.
*
* Returns ICE_XDP_TX for successfully produced desc, ICE_XDP_CONSUMED if there
* was not enough space on XDP ring
*/
static int ice_xmit_xdp_tx_zc(struct xdp_buff *xdp,
struct ice_tx_ring *xdp_ring)
{
u32 size = xdp->data_end - xdp->data;
u32 ntu = xdp_ring->next_to_use;
struct ice_tx_desc *tx_desc;
struct ice_tx_buf *tx_buf;
dma_addr_t dma;
if (ICE_DESC_UNUSED(xdp_ring) < ICE_RING_QUARTER(xdp_ring)) {
ice_clean_xdp_irq_zc(xdp_ring);
if (!ICE_DESC_UNUSED(xdp_ring)) {
xdp_ring->ring_stats->tx_stats.tx_busy++;
return ICE_XDP_CONSUMED;
}
}
dma = xsk_buff_xdp_get_dma(xdp);
xsk_buff_raw_dma_sync_for_device(xdp_ring->xsk_pool, dma, size);
tx_buf = &xdp_ring->tx_buf[ntu];
tx_buf->xdp = xdp;
tx_desc = ICE_TX_DESC(xdp_ring, ntu);
tx_desc->buf_addr = cpu_to_le64(dma);
tx_desc->cmd_type_offset_bsz = ice_build_ctob(ICE_TX_DESC_CMD_EOP,
0, size, 0);
xdp_ring->xdp_tx_active++;
if (++ntu == xdp_ring->count)
ntu = 0;
xdp_ring->next_to_use = ntu;
return ICE_XDP_TX;
}
/**
* ice_run_xdp_zc - Executes an XDP program in zero-copy path
* @rx_ring: Rx ring
@ -630,7 +731,7 @@ ice_run_xdp_zc(struct ice_rx_ring *rx_ring, struct xdp_buff *xdp,
case XDP_PASS:
break;
case XDP_TX:
result = ice_xmit_xdp_buff(xdp, xdp_ring);
result = ice_xmit_xdp_tx_zc(xdp, xdp_ring);
if (result == ICE_XDP_CONSUMED)
goto out_failure;
break;
@ -775,75 +876,6 @@ construct_skb:
return failure ? budget : (int)total_rx_packets;
}
/**
* ice_clean_xdp_tx_buf - Free and unmap XDP Tx buffer
* @xdp_ring: XDP Tx ring
* @tx_buf: Tx buffer to clean
*/
static void
ice_clean_xdp_tx_buf(struct ice_tx_ring *xdp_ring, struct ice_tx_buf *tx_buf)
{
page_frag_free(tx_buf->raw_buf);
xdp_ring->xdp_tx_active--;
dma_unmap_single(xdp_ring->dev, dma_unmap_addr(tx_buf, dma),
dma_unmap_len(tx_buf, len), DMA_TO_DEVICE);
dma_unmap_len_set(tx_buf, len, 0);
}
/**
* ice_clean_xdp_irq_zc - produce AF_XDP descriptors to CQ
* @xdp_ring: XDP Tx ring
*/
static void ice_clean_xdp_irq_zc(struct ice_tx_ring *xdp_ring)
{
u16 ntc = xdp_ring->next_to_clean;
struct ice_tx_desc *tx_desc;
u16 cnt = xdp_ring->count;
struct ice_tx_buf *tx_buf;
u16 xsk_frames = 0;
u16 last_rs;
int i;
last_rs = xdp_ring->next_to_use ? xdp_ring->next_to_use - 1 : cnt - 1;
tx_desc = ICE_TX_DESC(xdp_ring, last_rs);
if ((tx_desc->cmd_type_offset_bsz &
cpu_to_le64(ICE_TX_DESC_DTYPE_DESC_DONE))) {
if (last_rs >= ntc)
xsk_frames = last_rs - ntc + 1;
else
xsk_frames = last_rs + cnt - ntc + 1;
}
if (!xsk_frames)
return;
if (likely(!xdp_ring->xdp_tx_active))
goto skip;
ntc = xdp_ring->next_to_clean;
for (i = 0; i < xsk_frames; i++) {
tx_buf = &xdp_ring->tx_buf[ntc];
if (tx_buf->raw_buf) {
ice_clean_xdp_tx_buf(xdp_ring, tx_buf);
tx_buf->raw_buf = NULL;
} else {
xsk_frames++;
}
ntc++;
if (ntc >= xdp_ring->count)
ntc = 0;
}
skip:
tx_desc->cmd_type_offset_bsz = 0;
xdp_ring->next_to_clean += xsk_frames;
if (xdp_ring->next_to_clean >= cnt)
xdp_ring->next_to_clean -= cnt;
if (xsk_frames)
xsk_tx_completed(xdp_ring->xsk_pool, xsk_frames);
}
/**
* ice_xmit_pkt - produce a single HW Tx descriptor out of AF_XDP descriptor
* @xdp_ring: XDP ring to produce the HW Tx descriptor on
@ -1051,8 +1083,8 @@ void ice_xsk_clean_xdp_ring(struct ice_tx_ring *xdp_ring)
while (ntc != ntu) {
struct ice_tx_buf *tx_buf = &xdp_ring->tx_buf[ntc];
if (tx_buf->raw_buf)
ice_clean_xdp_tx_buf(xdp_ring, tx_buf);
if (tx_buf->xdp)
xsk_buff_free(tx_buf->xdp);
else
xsk_frames++;