niansa-misc/linux
1
0
Fork 0
mirror of synced 2025-03-06 20:59:54 +01:00
Code Issues Projects Releases Packages Wiki Activity

xdp: transition into using xdp_frame for ndo_xdp_xmit

Browse source 
Changing API ndo_xdp_xmit to take a struct xdp_frame instead of struct
xdp_buff.  This brings xdp_return_frame and ndp_xdp_xmit in sync.

This builds towards changing the API further to become a bulk API,
because xdp_buff is not a queue-able object while xdp_frame is.

V4: Adjust for commit 59655a5b6c ("tuntap: XDP_TX can use native XDP")
V7: Adjust for commit d9314c474d ("i40e: add support for XDP_REDIRECT")

Signed-off-by: Jesper Dangaard Brouer <brouer@redhat.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
This commit is contained in:
Jesper Dangaard Brouer 2018-04-17 16:46:37 +02:00 • committed by David S. Miller
parent 039930945a
commit 44fa2dbd47
7 changed files with 74 additions and 46 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

30
drivers/net/ethernet/intel/i40e/i40e_txrx.c
View file

@ -2203,9 +2203,20 @@ static bool i40e_is_non_eop(struct i40e_ring *rx_ring,
#define I40E_XDP_CONSUMED 1 #define I40E_XDP_CONSUMED 1
#define I40E_XDP_TX 2 #define I40E_XDP_TX 2
static int i40e_xmit_xdp_ring(struct xdp_buff *xdp, static int i40e_xmit_xdp_ring(struct xdp_frame *xdpf,
struct i40e_ring *xdp_ring); struct i40e_ring *xdp_ring);
static int i40e_xmit_xdp_tx_ring(struct xdp_buff *xdp,
struct i40e_ring *xdp_ring)
{
struct xdp_frame *xdpf = convert_to_xdp_frame(xdp);
if (unlikely(!xdpf))
return I40E_XDP_CONSUMED;
return i40e_xmit_xdp_ring(xdpf, xdp_ring);
}
/** /**
* i40e_run_xdp - run an XDP program * i40e_run_xdp - run an XDP program
* @rx_ring: Rx ring being processed * @rx_ring: Rx ring being processed
@ -2233,7 +2244,7 @@ static struct sk_buff *i40e_run_xdp(struct i40e_ring *rx_ring,
break; break;
case XDP_TX: case XDP_TX:
xdp_ring = rx_ring->vsi->xdp_rings[rx_ring->queue_index]; xdp_ring = rx_ring->vsi->xdp_rings[rx_ring->queue_index];
result = i40e_xmit_xdp_ring(xdp, xdp_ring); result = i40e_xmit_xdp_tx_ring(xdp, xdp_ring);
break; break;
case XDP_REDIRECT: case XDP_REDIRECT:
err = xdp_do_redirect(rx_ring->netdev, xdp, xdp_prog); err = xdp_do_redirect(rx_ring->netdev, xdp, xdp_prog);
@ -3480,21 +3491,14 @@ dma_error:
* @xdp: data to transmit * @xdp: data to transmit
* @xdp_ring: XDP Tx ring * @xdp_ring: XDP Tx ring
**/ **/
static int i40e_xmit_xdp_ring(struct xdp_buff *xdp, static int i40e_xmit_xdp_ring(struct xdp_frame *xdpf,
struct i40e_ring *xdp_ring) struct i40e_ring *xdp_ring)
{ {
u16 i = xdp_ring->next_to_use; u16 i = xdp_ring->next_to_use;
struct i40e_tx_buffer *tx_bi; struct i40e_tx_buffer *tx_bi;
struct i40e_tx_desc *tx_desc; struct i40e_tx_desc *tx_desc;
struct xdp_frame *xdpf; u32 size = xdpf->len;
dma_addr_t dma; dma_addr_t dma;
u32 size;
xdpf = convert_to_xdp_frame(xdp);
if (unlikely(!xdpf))
return I40E_XDP_CONSUMED;
size = xdpf->len;
if (!unlikely(I40E_DESC_UNUSED(xdp_ring))) { if (!unlikely(I40E_DESC_UNUSED(xdp_ring))) {
xdp_ring->tx_stats.tx_busy++; xdp_ring->tx_stats.tx_busy++;
@ -3684,7 +3688,7 @@ netdev_tx_t i40e_lan_xmit_frame(struct sk_buff *skb, struct net_device *netdev)
* *
* Returns Zero if sent, else an error code * Returns Zero if sent, else an error code
**/ **/
int i40e_xdp_xmit(struct net_device *dev, struct xdp_buff *xdp) int i40e_xdp_xmit(struct net_device *dev, struct xdp_frame *xdpf)
{ {
struct i40e_netdev_priv *np = netdev_priv(dev); struct i40e_netdev_priv *np = netdev_priv(dev);
unsigned int queue_index = smp_processor_id(); unsigned int queue_index = smp_processor_id();
@ -3697,7 +3701,7 @@ int i40e_xdp_xmit(struct net_device *dev, struct xdp_buff *xdp)
if (!i40e_enabled_xdp_vsi(vsi) || queue_index >= vsi->num_queue_pairs) if (!i40e_enabled_xdp_vsi(vsi) || queue_index >= vsi->num_queue_pairs)
return -ENXIO; return -ENXIO;
err = i40e_xmit_xdp_ring(xdp, vsi->xdp_rings[queue_index]); err = i40e_xmit_xdp_ring(xdpf, vsi->xdp_rings[queue_index]);
if (err != I40E_XDP_TX) if (err != I40E_XDP_TX)
return -ENOSPC; return -ENOSPC;

2
drivers/net/ethernet/intel/i40e/i40e_txrx.h
View file

@ -511,7 +511,7 @@ u32 i40e_get_tx_pending(struct i40e_ring *ring, bool in_sw);
void i40e_detect_recover_hung(struct i40e_vsi *vsi); void i40e_detect_recover_hung(struct i40e_vsi *vsi);
int __i40e_maybe_stop_tx(struct i40e_ring *tx_ring, int size); int __i40e_maybe_stop_tx(struct i40e_ring *tx_ring, int size);
bool __i40e_chk_linearize(struct sk_buff *skb); bool __i40e_chk_linearize(struct sk_buff *skb);
int i40e_xdp_xmit(struct net_device *dev, struct xdp_buff *xdp); int i40e_xdp_xmit(struct net_device *dev, struct xdp_frame *xdpf);
void i40e_xdp_flush(struct net_device *dev); void i40e_xdp_flush(struct net_device *dev);
/** /**

24
drivers/net/ethernet/intel/ixgbe/ixgbe_main.c
View file

@ -2262,7 +2262,7 @@ static struct sk_buff *ixgbe_build_skb(struct ixgbe_ring *rx_ring,
#define IXGBE_XDP_TX 2 #define IXGBE_XDP_TX 2
static int ixgbe_xmit_xdp_ring(struct ixgbe_adapter *adapter, static int ixgbe_xmit_xdp_ring(struct ixgbe_adapter *adapter,
struct xdp_buff *xdp); struct xdp_frame *xdpf);
static struct sk_buff *ixgbe_run_xdp(struct ixgbe_adapter *adapter, static struct sk_buff *ixgbe_run_xdp(struct ixgbe_adapter *adapter,
struct ixgbe_ring *rx_ring, struct ixgbe_ring *rx_ring,
@ -2270,6 +2270,7 @@ static struct sk_buff *ixgbe_run_xdp(struct ixgbe_adapter *adapter,
{ {
int err, result = IXGBE_XDP_PASS; int err, result = IXGBE_XDP_PASS;
struct bpf_prog *xdp_prog; struct bpf_prog *xdp_prog;
struct xdp_frame *xdpf;
u32 act; u32 act;
rcu_read_lock(); rcu_read_lock();
@ -2278,12 +2279,19 @@ static struct sk_buff *ixgbe_run_xdp(struct ixgbe_adapter *adapter,
if (!xdp_prog) if (!xdp_prog)
goto xdp_out; goto xdp_out;
prefetchw(xdp->data_hard_start); /* xdp_frame write */
act = bpf_prog_run_xdp(xdp_prog, xdp); act = bpf_prog_run_xdp(xdp_prog, xdp);
switch (act) { switch (act) {
case XDP_PASS: case XDP_PASS:
break; break;
case XDP_TX: case XDP_TX:
result = ixgbe_xmit_xdp_ring(adapter, xdp); xdpf = convert_to_xdp_frame(xdp);
if (unlikely(!xdpf)) {
result = IXGBE_XDP_CONSUMED;
break;
}
result = ixgbe_xmit_xdp_ring(adapter, xdpf);
break; break;
case XDP_REDIRECT: case XDP_REDIRECT:
err = xdp_do_redirect(adapter->netdev, xdp, xdp_prog); err = xdp_do_redirect(adapter->netdev, xdp, xdp_prog);
@ -2386,7 +2394,6 @@ static int ixgbe_clean_rx_irq(struct ixgbe_q_vector *q_vector,
xdp.data_hard_start = xdp.data - xdp.data_hard_start = xdp.data -
ixgbe_rx_offset(rx_ring); ixgbe_rx_offset(rx_ring);
xdp.data_end = xdp.data + size; xdp.data_end = xdp.data + size;
prefetchw(xdp.data_hard_start); /* xdp_frame write */
skb = ixgbe_run_xdp(adapter, rx_ring, &xdp); skb = ixgbe_run_xdp(adapter, rx_ring, &xdp);
} }
@ -8344,20 +8351,15 @@ static u16 ixgbe_select_queue(struct net_device *dev, struct sk_buff *skb,
} }
static int ixgbe_xmit_xdp_ring(struct ixgbe_adapter *adapter, static int ixgbe_xmit_xdp_ring(struct ixgbe_adapter *adapter,
struct xdp_buff *xdp) struct xdp_frame *xdpf)
{ {
struct ixgbe_ring *ring = adapter->xdp_ring[smp_processor_id()]; struct ixgbe_ring *ring = adapter->xdp_ring[smp_processor_id()];
struct ixgbe_tx_buffer *tx_buffer; struct ixgbe_tx_buffer *tx_buffer;
union ixgbe_adv_tx_desc *tx_desc; union ixgbe_adv_tx_desc *tx_desc;
struct xdp_frame *xdpf;
u32 len, cmd_type; u32 len, cmd_type;
dma_addr_t dma; dma_addr_t dma;
u16 i; u16 i;
xdpf = convert_to_xdp_frame(xdp);
if (unlikely(!xdpf))
return -EOVERFLOW;
len = xdpf->len; len = xdpf->len;
if (unlikely(!ixgbe_desc_unused(ring))) if (unlikely(!ixgbe_desc_unused(ring)))
@ -10010,7 +10012,7 @@ static int ixgbe_xdp(struct net_device *dev, struct netdev_bpf *xdp)
} }
} }
static int ixgbe_xdp_xmit(struct net_device *dev, struct xdp_buff *xdp) static int ixgbe_xdp_xmit(struct net_device *dev, struct xdp_frame *xdpf)
{ {
struct ixgbe_adapter *adapter = netdev_priv(dev); struct ixgbe_adapter *adapter = netdev_priv(dev);
struct ixgbe_ring *ring; struct ixgbe_ring *ring;
@ -10026,7 +10028,7 @@ static int ixgbe_xdp_xmit(struct net_device *dev, struct xdp_buff *xdp)
if (unlikely(!ring)) if (unlikely(!ring))
return -ENXIO; return -ENXIO;
err = ixgbe_xmit_xdp_ring(adapter, xdp); err = ixgbe_xmit_xdp_ring(adapter, xdpf);
if (err != IXGBE_XDP_TX) if (err != IXGBE_XDP_TX)
return -ENOSPC; return -ENOSPC;

19
drivers/net/tun.c
View file

@ -1301,18 +1301,13 @@ static const struct net_device_ops tun_netdev_ops = {
.ndo_get_stats64 = tun_net_get_stats64, .ndo_get_stats64 = tun_net_get_stats64,
}; };
static int tun_xdp_xmit(struct net_device *dev, struct xdp_buff *xdp) static int tun_xdp_xmit(struct net_device *dev, struct xdp_frame *frame)
{ {
struct tun_struct *tun = netdev_priv(dev); struct tun_struct *tun = netdev_priv(dev);
struct xdp_frame *frame;
struct tun_file *tfile; struct tun_file *tfile;
u32 numqueues; u32 numqueues;
int ret = 0; int ret = 0;
frame = convert_to_xdp_frame(xdp);
if (unlikely(!frame))
return -EOVERFLOW;
rcu_read_lock(); rcu_read_lock();
numqueues = READ_ONCE(tun->numqueues); numqueues = READ_ONCE(tun->numqueues);
@ -1336,6 +1331,16 @@ out:
return ret; return ret;
} }
static int tun_xdp_tx(struct net_device *dev, struct xdp_buff *xdp)
{
struct xdp_frame *frame = convert_to_xdp_frame(xdp);
if (unlikely(!frame))
return -EOVERFLOW;
return tun_xdp_xmit(dev, frame);
}
static void tun_xdp_flush(struct net_device *dev) static void tun_xdp_flush(struct net_device *dev)
{ {
struct tun_struct *tun = netdev_priv(dev); struct tun_struct *tun = netdev_priv(dev);
@ -1683,7 +1688,7 @@ static struct sk_buff *tun_build_skb(struct tun_struct *tun,
case XDP_TX: case XDP_TX:
get_page(alloc_frag->page); get_page(alloc_frag->page);
alloc_frag->offset += buflen; alloc_frag->offset += buflen;
if (tun_xdp_xmit(tun->dev, &xdp)) if (tun_xdp_tx(tun->dev, &xdp))
goto err_redirect; goto err_redirect;
tun_xdp_flush(tun->dev); tun_xdp_flush(tun->dev);
rcu_read_unlock(); rcu_read_unlock();

24
drivers/net/virtio_net.c
View file

@ -416,10 +416,10 @@ static void virtnet_xdp_flush(struct net_device *dev)
} }
static int __virtnet_xdp_xmit(struct virtnet_info *vi, static int __virtnet_xdp_xmit(struct virtnet_info *vi,
struct xdp_buff *xdp) struct xdp_frame *xdpf)
{ {
struct virtio_net_hdr_mrg_rxbuf *hdr; struct virtio_net_hdr_mrg_rxbuf *hdr;
struct xdp_frame *xdpf, *xdpf_sent; struct xdp_frame *xdpf_sent;
struct send_queue *sq; struct send_queue *sq;
unsigned int len; unsigned int len;
unsigned int qp; unsigned int qp;
@ -432,10 +432,6 @@ static int __virtnet_xdp_xmit(struct virtnet_info *vi,
while ((xdpf_sent = virtqueue_get_buf(sq->vq, &len)) != NULL) while ((xdpf_sent = virtqueue_get_buf(sq->vq, &len)) != NULL)
xdp_return_frame(xdpf_sent); xdp_return_frame(xdpf_sent);
xdpf = convert_to_xdp_frame(xdp);
if (unlikely(!xdpf))
return -EOVERFLOW;
/* virtqueue want to use data area in-front of packet */ /* virtqueue want to use data area in-front of packet */
if (unlikely(xdpf->metasize > 0)) if (unlikely(xdpf->metasize > 0))
return -EOPNOTSUPP; return -EOPNOTSUPP;
@ -459,7 +455,7 @@ static int __virtnet_xdp_xmit(struct virtnet_info *vi,
return 0; return 0;
} }
static int virtnet_xdp_xmit(struct net_device *dev, struct xdp_buff *xdp) static int virtnet_xdp_xmit(struct net_device *dev, struct xdp_frame *xdpf)
{ {
struct virtnet_info *vi = netdev_priv(dev); struct virtnet_info *vi = netdev_priv(dev);
struct receive_queue *rq = vi->rq; struct receive_queue *rq = vi->rq;
@ -472,7 +468,7 @@ static int virtnet_xdp_xmit(struct net_device *dev, struct xdp_buff *xdp)
if (!xdp_prog) if (!xdp_prog)
return -ENXIO; return -ENXIO;
return __virtnet_xdp_xmit(vi, xdp); return __virtnet_xdp_xmit(vi, xdpf);
} }
static unsigned int virtnet_get_headroom(struct virtnet_info *vi) static unsigned int virtnet_get_headroom(struct virtnet_info *vi)
@ -569,6 +565,7 @@ static struct sk_buff *receive_small(struct net_device *dev,
xdp_prog = rcu_dereference(rq->xdp_prog); xdp_prog = rcu_dereference(rq->xdp_prog);
if (xdp_prog) { if (xdp_prog) {
struct virtio_net_hdr_mrg_rxbuf *hdr = buf + header_offset; struct virtio_net_hdr_mrg_rxbuf *hdr = buf + header_offset;
struct xdp_frame *xdpf;
struct xdp_buff xdp; struct xdp_buff xdp;
void *orig_data; void *orig_data;
u32 act; u32 act;
@ -611,7 +608,10 @@ static struct sk_buff *receive_small(struct net_device *dev,
delta = orig_data - xdp.data; delta = orig_data - xdp.data;
break; break;
case XDP_TX: case XDP_TX:
err = __virtnet_xdp_xmit(vi, &xdp); xdpf = convert_to_xdp_frame(&xdp);
if (unlikely(!xdpf))
goto err_xdp;
err = __virtnet_xdp_xmit(vi, xdpf);
if (unlikely(err)) { if (unlikely(err)) {
trace_xdp_exception(vi->dev, xdp_prog, act); trace_xdp_exception(vi->dev, xdp_prog, act);
goto err_xdp; goto err_xdp;
@ -702,6 +702,7 @@ static struct sk_buff *receive_mergeable(struct net_device *dev,
rcu_read_lock(); rcu_read_lock();
xdp_prog = rcu_dereference(rq->xdp_prog); xdp_prog = rcu_dereference(rq->xdp_prog);
if (xdp_prog) { if (xdp_prog) {
struct xdp_frame *xdpf;
struct page *xdp_page; struct page *xdp_page;
struct xdp_buff xdp; struct xdp_buff xdp;
void *data; void *data;
@ -766,7 +767,10 @@ static struct sk_buff *receive_mergeable(struct net_device *dev,
} }
break; break;
case XDP_TX: case XDP_TX:
err = __virtnet_xdp_xmit(vi, &xdp); xdpf = convert_to_xdp_frame(&xdp);
if (unlikely(!xdpf))
goto err_xdp;
err = __virtnet_xdp_xmit(vi, xdpf);
if (unlikely(err)) { if (unlikely(err)) {
trace_xdp_exception(vi->dev, xdp_prog, act); trace_xdp_exception(vi->dev, xdp_prog, act);
if (unlikely(xdp_page != page)) if (unlikely(xdp_page != page))

4
include/linux/netdevice.h
View file

@ -1165,7 +1165,7 @@ struct dev_ifalias {
* This function is used to set or query state related to XDP on the * This function is used to set or query state related to XDP on the
* netdevice and manage BPF offload. See definition of * netdevice and manage BPF offload. See definition of
* enum bpf_netdev_command for details. * enum bpf_netdev_command for details.
* int (*ndo_xdp_xmit)(struct net_device *dev, struct xdp_buff *xdp); * int (*ndo_xdp_xmit)(struct net_device *dev, struct xdp_frame *xdp);
* This function is used to submit a XDP packet for transmit on a * This function is used to submit a XDP packet for transmit on a
* netdevice. * netdevice.
* void (*ndo_xdp_flush)(struct net_device *dev); * void (*ndo_xdp_flush)(struct net_device *dev);
@ -1356,7 +1356,7 @@ struct net_device_ops {
int (*ndo_bpf)(struct net_device *dev, int (*ndo_bpf)(struct net_device *dev,
struct netdev_bpf *bpf); struct netdev_bpf *bpf);
int (*ndo_xdp_xmit)(struct net_device *dev, int (*ndo_xdp_xmit)(struct net_device *dev,
struct xdp_buff *xdp); struct xdp_frame *xdp);
void (*ndo_xdp_flush)(struct net_device *dev); void (*ndo_xdp_flush)(struct net_device *dev);
}; };

17
net/core/filter.c
View file

@ -2749,13 +2749,18 @@ static int __bpf_tx_xdp(struct net_device *dev,
struct xdp_buff *xdp, struct xdp_buff *xdp,
u32 index) u32 index)
{ {
struct xdp_frame *xdpf;
int err; int err;
if (!dev->netdev_ops->ndo_xdp_xmit) { if (!dev->netdev_ops->ndo_xdp_xmit) {
return -EOPNOTSUPP; return -EOPNOTSUPP;
} }
err = dev->netdev_ops->ndo_xdp_xmit(dev, xdp); xdpf = convert_to_xdp_frame(xdp);
if (unlikely(!xdpf))
return -EOVERFLOW;
err = dev->netdev_ops->ndo_xdp_xmit(dev, xdpf);
if (err) if (err)
return err; return err;
dev->netdev_ops->ndo_xdp_flush(dev); dev->netdev_ops->ndo_xdp_flush(dev);
@ -2771,11 +2776,19 @@ static int __bpf_tx_xdp_map(struct net_device *dev_rx, void *fwd,
if (map->map_type == BPF_MAP_TYPE_DEVMAP) { if (map->map_type == BPF_MAP_TYPE_DEVMAP) {
struct net_device *dev = fwd; struct net_device *dev = fwd;
struct xdp_frame *xdpf;
if (!dev->netdev_ops->ndo_xdp_xmit) if (!dev->netdev_ops->ndo_xdp_xmit)
return -EOPNOTSUPP; return -EOPNOTSUPP;
err = dev->netdev_ops->ndo_xdp_xmit(dev, xdp); xdpf = convert_to_xdp_frame(xdp);
if (unlikely(!xdpf))
return -EOVERFLOW;
/* TODO: move to inside map code instead, for bulk support
* err = dev_map_enqueue(dev, xdp);
*/
err = dev->netdev_ops->ndo_xdp_xmit(dev, xdpf);
if (err) if (err)
return err; return err;
__dev_map_insert_ctx(map, index); __dev_map_insert_ctx(map, index);