201 lines
5.4 KiB
C
201 lines
5.4 KiB
C
/* SPDX-License-Identifier: GPL-2.0 */
|
|
/* Copyright (c) Meta Platforms, Inc. and affiliates. */
|
|
|
|
#ifndef _FBNIC_TXRX_H_
|
|
#define _FBNIC_TXRX_H_
|
|
|
|
#include <linux/netdevice.h>
|
|
#include <linux/skbuff.h>
|
|
#include <linux/types.h>
|
|
#include <linux/u64_stats_sync.h>
|
|
#include <net/xdp.h>
|
|
|
|
struct fbnic_net;
|
|
|
|
/* Guarantee we have space needed for storing the buffer
|
|
* To store the buffer we need:
|
|
* 1 descriptor per page
|
|
* + 1 descriptor for skb head
|
|
* + 2 descriptors for metadata and optional metadata
|
|
* + 7 descriptors to keep tail out of the same cacheline as head
|
|
* If we cannot guarantee that then we should return TX_BUSY
|
|
*/
|
|
#define FBNIC_MAX_SKB_DESC (MAX_SKB_FRAGS + 10)
|
|
#define FBNIC_TX_DESC_WAKEUP (FBNIC_MAX_SKB_DESC * 2)
|
|
#define FBNIC_TX_DESC_MIN roundup_pow_of_two(FBNIC_TX_DESC_WAKEUP)
|
|
|
|
/* To receive the worst case packet we need:
|
|
* 1 descriptor for primary metadata
|
|
* + 1 descriptor for optional metadata
|
|
* + 1 descriptor for headers
|
|
* + 4 descriptors for payload
|
|
*/
|
|
#define FBNIC_MAX_RX_PKT_DESC 7
|
|
#define FBNIC_RX_DESC_MIN roundup_pow_of_two(FBNIC_MAX_RX_PKT_DESC * 2)
|
|
|
|
#define FBNIC_MAX_TXQS 128u
|
|
#define FBNIC_MAX_RXQS 128u
|
|
#define FBNIC_MAX_XDPQS 128u
|
|
|
|
/* These apply to TWQs, TCQ, RCQ */
|
|
#define FBNIC_QUEUE_SIZE_MIN 16u
|
|
#define FBNIC_QUEUE_SIZE_MAX SZ_64K
|
|
|
|
#define FBNIC_TXQ_SIZE_DEFAULT 1024
|
|
#define FBNIC_HPQ_SIZE_DEFAULT 256
|
|
#define FBNIC_PPQ_SIZE_DEFAULT 256
|
|
#define FBNIC_RCQ_SIZE_DEFAULT 1024
|
|
#define FBNIC_TX_USECS_DEFAULT 35
|
|
#define FBNIC_RX_USECS_DEFAULT 30
|
|
#define FBNIC_RX_FRAMES_DEFAULT 0
|
|
|
|
#define FBNIC_RX_TROOM \
|
|
SKB_DATA_ALIGN(sizeof(struct skb_shared_info))
|
|
#define FBNIC_RX_HROOM_PAD 128
|
|
#define FBNIC_RX_HROOM \
|
|
(ALIGN(FBNIC_RX_TROOM + FBNIC_RX_HROOM_PAD, 128) - FBNIC_RX_TROOM)
|
|
#define FBNIC_RX_PAD 0
|
|
#define FBNIC_RX_PAYLD_OFFSET 0
|
|
#define FBNIC_RX_PAYLD_PG_CL 0
|
|
|
|
#define FBNIC_RING_F_DISABLED BIT(0)
|
|
#define FBNIC_RING_F_CTX BIT(1)
|
|
#define FBNIC_RING_F_STATS BIT(2) /* Ring's stats may be used */
|
|
|
|
#define FBNIC_HDS_THRESH_MAX \
|
|
(4096 - FBNIC_RX_HROOM - FBNIC_RX_TROOM - FBNIC_RX_PAD)
|
|
#define FBNIC_HDS_THRESH_DEFAULT \
|
|
(1536 - FBNIC_RX_PAD)
|
|
#define FBNIC_HDR_BYTES_MIN 128
|
|
|
|
struct fbnic_pkt_buff {
|
|
struct xdp_buff buff;
|
|
ktime_t hwtstamp;
|
|
bool add_frag_failed;
|
|
};
|
|
|
|
struct fbnic_queue_stats {
|
|
u64 packets;
|
|
u64 bytes;
|
|
union {
|
|
struct {
|
|
u64 csum_partial;
|
|
u64 lso;
|
|
u64 ts_packets;
|
|
u64 ts_lost;
|
|
u64 stop;
|
|
u64 wake;
|
|
} twq;
|
|
struct {
|
|
u64 alloc_failed;
|
|
u64 csum_complete;
|
|
u64 csum_none;
|
|
u64 length_errors;
|
|
} rx;
|
|
struct {
|
|
u64 alloc_failed;
|
|
} bdq;
|
|
};
|
|
u64 dropped;
|
|
struct u64_stats_sync syncp;
|
|
};
|
|
|
|
#define FBNIC_PAGECNT_BIAS_MAX PAGE_SIZE
|
|
|
|
struct fbnic_rx_buf {
|
|
netmem_ref netmem;
|
|
long pagecnt_bias;
|
|
};
|
|
|
|
struct fbnic_ring {
|
|
/* Pointer to buffer specific info */
|
|
union {
|
|
struct fbnic_pkt_buff *pkt; /* RCQ */
|
|
struct fbnic_rx_buf *rx_buf; /* BDQ */
|
|
void **tx_buf; /* TWQ */
|
|
void *buffer; /* Generic pointer */
|
|
};
|
|
|
|
u32 __iomem *doorbell; /* Pointer to CSR space for ring */
|
|
__le64 *desc; /* Descriptor ring memory */
|
|
u16 size_mask; /* Size of ring in descriptors - 1 */
|
|
u8 q_idx; /* Logical netdev ring index */
|
|
u8 flags; /* Ring flags (FBNIC_RING_F_*) */
|
|
|
|
u32 head, tail; /* Head/Tail of ring */
|
|
|
|
union {
|
|
/* Rx BDQs only */
|
|
struct page_pool *page_pool;
|
|
|
|
/* Deferred_head is used to cache the head for TWQ1 if
|
|
* an attempt is made to clean TWQ1 with zero napi_budget.
|
|
* We do not use it for any other ring.
|
|
*/
|
|
s32 deferred_head;
|
|
};
|
|
|
|
struct fbnic_queue_stats stats;
|
|
|
|
/* Slow path fields follow */
|
|
dma_addr_t dma; /* Phys addr of descriptor memory */
|
|
size_t size; /* Size of descriptor ring in memory */
|
|
};
|
|
|
|
struct fbnic_q_triad {
|
|
struct fbnic_ring sub0, sub1, cmpl;
|
|
struct xdp_rxq_info xdp_rxq;
|
|
};
|
|
|
|
struct fbnic_napi_vector {
|
|
struct napi_struct napi;
|
|
struct device *dev; /* Device for DMA unmapping */
|
|
struct fbnic_dev *fbd;
|
|
|
|
u16 v_idx;
|
|
u8 txt_count;
|
|
u8 rxt_count;
|
|
|
|
struct fbnic_q_triad qt[];
|
|
};
|
|
|
|
extern const struct netdev_queue_mgmt_ops fbnic_queue_mgmt_ops;
|
|
|
|
netdev_tx_t fbnic_xmit_frame(struct sk_buff *skb, struct net_device *dev);
|
|
netdev_features_t
|
|
fbnic_features_check(struct sk_buff *skb, struct net_device *dev,
|
|
netdev_features_t features);
|
|
|
|
void fbnic_aggregate_ring_rx_counters(struct fbnic_net *fbn,
|
|
struct fbnic_ring *rxr);
|
|
void fbnic_aggregate_ring_bdq_counters(struct fbnic_net *fbn,
|
|
struct fbnic_ring *rxr);
|
|
void fbnic_aggregate_ring_tx_counters(struct fbnic_net *fbn,
|
|
struct fbnic_ring *txr);
|
|
void fbnic_aggregate_ring_xdp_counters(struct fbnic_net *fbn,
|
|
struct fbnic_ring *xdpr);
|
|
|
|
int fbnic_alloc_napi_vectors(struct fbnic_net *fbn);
|
|
void fbnic_free_napi_vectors(struct fbnic_net *fbn);
|
|
int fbnic_alloc_resources(struct fbnic_net *fbn);
|
|
void fbnic_free_resources(struct fbnic_net *fbn);
|
|
int fbnic_set_netif_queues(struct fbnic_net *fbn);
|
|
void fbnic_reset_netif_queues(struct fbnic_net *fbn);
|
|
irqreturn_t fbnic_msix_clean_rings(int irq, void *data);
|
|
void fbnic_napi_enable(struct fbnic_net *fbn);
|
|
void fbnic_napi_disable(struct fbnic_net *fbn);
|
|
void fbnic_enable(struct fbnic_net *fbn);
|
|
void fbnic_disable(struct fbnic_net *fbn);
|
|
void fbnic_flush(struct fbnic_net *fbn);
|
|
void fbnic_fill(struct fbnic_net *fbn);
|
|
|
|
void fbnic_napi_depletion_check(struct net_device *netdev);
|
|
int fbnic_wait_all_queues_idle(struct fbnic_dev *fbd, bool may_fail);
|
|
|
|
static inline int fbnic_napi_idx(const struct fbnic_napi_vector *nv)
|
|
{
|
|
return nv->v_idx - FBNIC_NON_NAPI_VECTORS;
|
|
}
|
|
|
|
#endif /* _FBNIC_TXRX_H_ */
|