#include <sys/cdefs.h>
#include "opt_inet6.h"
#include "opt_inet.h"
#include <sys/param.h>
#include <sys/systm.h>
#include <sys/kernel.h>
#include <sys/module.h>
#include <sys/bus.h>
#include <sys/conf.h>
#include <machine/bus.h>
#include <machine/resource.h>
#include <sys/rman.h>
#include <sys/queue.h>
#include <sys/sysctl.h>
#include <sys/taskqueue.h>
#include <sys/proc.h>
#include <sys/sbuf.h>
#include <sys/sched.h>
#include <sys/smp.h>
#include <sys/syslog.h>
#include <sys/socket.h>
#include <sys/sglist.h>
#include <net/if.h>
#include <net/if_var.h>
#include <net/bpf.h>
#include <net/ethernet.h>
#include <net/if_vlan_var.h>
#include <netinet/in_systm.h>
#include <netinet/in.h>
#include <netinet/ip.h>
#include <netinet/ip6.h>
#include <netinet/tcp.h>
#include <dev/pci/pcireg.h>
#include <dev/pci/pcivar.h>
#include <vm/vm.h>
#include <vm/pmap.h>
#include <cxgb_include.h>
#include <sys/mvec.h>

Include dependency graph for cxgb_sge.c:

Data Structures
struct	tx_desc

struct	rx_desc

struct	rsp_desc

struct	tx_sw_desc

struct	rx_sw_desc

struct	txq_state

struct	refill_fl_cb_arg

struct	coalesce_info

Macros
#define	COALESCE_START_DEFAULT TX_ETH_Q_SIZE>>1

#define	COALESCE_START_MAX (TX_ETH_Q_SIZE-(TX_ETH_Q_SIZE>>3))

#define	COALESCE_STOP_DEFAULT TX_ETH_Q_SIZE>>2

#define	COALESCE_STOP_MIN TX_ETH_Q_SIZE>>5

#define	TX_RECLAIM_DEFAULT TX_ETH_Q_SIZE>>5

#define	TX_RECLAIM_MAX TX_ETH_Q_SIZE>>2

#define	TX_RECLAIM_MIN TX_ETH_Q_SIZE>>6

#define	USE_GTS 0

#define	SGE_RX_SM_BUF_SIZE 1536

#define	SGE_RX_DROP_THRES 16

#define	SGE_RX_COPY_THRES 128

#define	TX_RECLAIM_PERIOD (hz >> 1)

#define	RX_SW_DESC_MAP_CREATED (1 << 0)

#define	TX_SW_DESC_MAP_CREATED (1 << 1)

#define	RX_SW_DESC_INUSE (1 << 3)

#define	TX_SW_DESC_MAPPED (1 << 4)

#define	RSPQ_NSOP_NEOP G_RSPD_SOP_EOP(0)

#define	RSPQ_EOP G_RSPD_SOP_EOP(F_RSPD_EOP)

#define	RSPQ_SOP G_RSPD_SOP_EOP(F_RSPD_SOP)

#define	RSPQ_SOP_EOP G_RSPD_SOP_EOP(F_RSPD_SOP\|F_RSPD_EOP)

#define	TXQ_LOCK_ASSERT(qs) mtx_assert(&(qs)->lock, MA_OWNED)

#define	TXQ_TRYLOCK(qs) mtx_trylock(&(qs)->lock)

#define	TXQ_LOCK(qs) mtx_lock(&(qs)->lock)

#define	TXQ_UNLOCK(qs) mtx_unlock(&(qs)->lock)

#define	TXQ_RING_EMPTY(qs) drbr_empty((qs)->port->ifp, (qs)->txq[TXQ_ETH].txq_mr)

#define	TXQ_RING_NEEDS_ENQUEUE(qs) drbr_needs_enqueue((qs)->port->ifp, (qs)->txq[TXQ_ETH].txq_mr)

#define	TXQ_RING_FLUSH(qs) drbr_flush((qs)->port->ifp, (qs)->txq[TXQ_ETH].txq_mr)

#define	TXQ_RING_DEQUEUE_COND(qs, func, arg) drbr_dequeue_cond((qs)->port->ifp, (qs)->txq[TXQ_ETH].txq_mr, func, arg)

#define	TXQ_RING_DEQUEUE(qs) drbr_dequeue((qs)->port->ifp, (qs)->txq[TXQ_ETH].txq_mr)

#define	SGE_PARERR

#define	SGE_FRAMINGERR (F_UC_REQ_FRAMINGERROR \| F_R_REQ_FRAMINGERROR)

#define	SGE_FATALERR

#define	TCPPKTHDRSIZE (ETHER_HDR_LEN + 20 + 20)

#define	GET_VTAG(cntrl, m)

#define	RSPD_GTS_MASK (F_RSPD_TXQ0_GTS \| F_RSPD_TXQ1_GTS)

#define	RSPD_CTRL_MASK

#define	NOMEM_INTR_DELAY 2500

#define	QDUMP_SBUF_SIZE 32 * 400

#define	CXGB_SYSCTL_ADD_QUAD(a)

#define	CXGB_SYSCTL_ADD_ULONG(a)

Enumerations
enum	{ TXQ_RUNNING = 1 << 0 , TXQ_LAST_PKT_DB = 1 << 1 }

Functions
	__FBSDID ("$FreeBSD$")

	SYSCTL_INT (_hw_cxgb, OID_AUTO, txq_mr_size, CTLFLAG_RDTUN, &cxgb_txq_buf_ring_size, 0, "size of per-queue mbuf ring")

	SYSCTL_INT (_hw_cxgb, OID_AUTO, tx_coalesce_force, CTLFLAG_RWTUN, &cxgb_tx_coalesce_force, 0, "coalesce small packets into a single work request regardless of ring state")

	SYSCTL_INT (_hw_cxgb, OID_AUTO, tx_coalesce_enable_start, CTLFLAG_RWTUN, &cxgb_tx_coalesce_enable_start, 0, "coalesce enable threshold")

	SYSCTL_INT (_hw_cxgb, OID_AUTO, tx_coalesce_enable_stop, CTLFLAG_RWTUN, &cxgb_tx_coalesce_enable_stop, 0, "coalesce disable threshold")

	SYSCTL_INT (_hw_cxgb, OID_AUTO, tx_reclaim_threshold, CTLFLAG_RWTUN, &cxgb_tx_reclaim_threshold, 0, "tx cleaning minimum threshold")

static void	sge_timer_cb (void *arg)

static void	sge_timer_reclaim (void *arg, int ncount)

static void	sge_txq_reclaim_handler (void *arg, int ncount)

static void	cxgb_start_locked (struct sge_qset *qs)

static __inline uint64_t	check_pkt_coalesce (struct sge_qset *qs)

static void	set_wr_hdr (struct work_request_hdr *wrp, uint32_t wr_hi, uint32_t wr_lo)

static int	coalesce_check (struct mbuf m, void arg)

static struct mbuf *	cxgb_dequeue (struct sge_qset *qs)

static __inline int	reclaim_completed_tx (struct sge_qset *qs, int reclaim_min, int queue)

static __inline int	should_restart_tx (const struct sge_txq *q)

void	t3_sge_init (adapter_t adap, struct sge_params p)

static __inline unsigned int	sgl_len (unsigned int n)

static int	get_imm_packet (adapter_t sc, const struct rsp_desc resp, struct mbuf *m)

static __inline u_int	flits_to_desc (u_int n)

void	t3_sge_err_intr_handler (adapter_t *adapter)

void	t3_sge_prep (adapter_t adap, struct sge_params p)

int	t3_sge_alloc (adapter_t *sc)

int	t3_sge_free (struct adapter *sc)

void	t3_update_qset_coalesce (struct sge_qset qs, const struct qset_params p)

static void	refill_fl_cb (void arg, bus_dma_segment_t segs, int nseg, int error)

static void	refill_fl (adapter_t sc, struct sge_fl q, int n)

static void	free_rx_bufs (adapter_t sc, struct sge_fl q)

static __inline void	__refill_fl (adapter_t adap, struct sge_fl fl)

static __inline void	__refill_fl_lt (adapter_t adap, struct sge_fl fl, int max)

static void	recycle_rx_buf (adapter_t adap, struct sge_fl q, unsigned int idx)

static void	alloc_ring_cb (void arg, bus_dma_segment_t segs, int nsegs, int error)

static int	alloc_ring (adapter_t sc, size_t nelem, size_t elem_size, size_t sw_size, bus_addr_t phys, void desc, void sdesc, bus_dma_tag_t tag, bus_dmamap_t map, bus_dma_tag_t parent_entry_tag, bus_dma_tag_t *entry_tag)

static void	sge_slow_intr_handler (void *arg, int ncount)

int	t3_sge_init_adapter (adapter_t *sc)

int	t3_sge_reset_adapter (adapter_t *sc)

int	t3_sge_init_port (struct port_info *pi)

static __inline void	refill_rspq (adapter_t sc, const struct sge_rspq q, u_int credits)

static void	init_qset_cntxt (struct sge_qset *qs, u_int id)

static void	txq_prod (struct sge_txq txq, unsigned int ndesc, struct txq_state txqs)

static __inline unsigned int	calc_tx_descs (const struct mbuf *m, int nsegs)

static __inline void	make_sgl (struct sg_ent sgp, bus_dma_segment_t segs, int nsegs)

static __inline void	check_ring_tx_db (adapter_t adap, struct sge_txq q, int mustring)

static __inline void	wr_gen2 (struct tx_desc *d, unsigned int gen)

static void	write_wr_hdr_sgl (unsigned int ndesc, struct tx_desc txd, struct txq_state txqs, const struct sge_txq txq, const struct sg_ent sgl, unsigned int flits, unsigned int sgl_flits, unsigned int wr_hi, unsigned int wr_lo)

static int	t3_encap (struct sge_qset qs, struct mbuf *m)

void	cxgb_tx_watchdog (void *arg)

static void	cxgb_tx_timeout (void *arg)

static int	cxgb_transmit_locked (struct ifnet ifp, struct sge_qset qs, struct mbuf *m)

int	cxgb_transmit (struct ifnet ifp, struct mbuf m)

void	cxgb_qflush (struct ifnet *ifp)

static __inline void	write_imm (struct tx_desc *d, caddr_t src, unsigned int len, unsigned int gen)

static __inline int	check_desc_avail (adapter_t adap, struct sge_txq q, struct mbuf *m, unsigned int ndesc, unsigned int qid)

static __inline void	reclaim_completed_tx_imm (struct sge_txq *q)

static int	ctrl_xmit (adapter_t adap, struct sge_qset qs, struct mbuf *m)

static void	restart_ctrlq (void *data, int npending)

int	t3_mgmt_tx (struct adapter adap, struct mbuf m)

static void	t3_free_qset (adapter_t sc, struct sge_qset q)

void	t3_free_sge_resources (adapter_t *sc, int nqsets)

void	t3_sge_start (adapter_t *sc)

void	t3_sge_stop (adapter_t *sc)

void	t3_free_tx_desc (struct sge_qset *qs, int reclaimable, int queue)

static __inline int	is_new_response (const struct rsp_desc r, const struct sge_rspq q)

static void	restart_tx (struct sge_qset *qs)

int	t3_sge_alloc_qset (adapter_t sc, u_int id, int nports, int irq_vec_idx, const struct qset_params p, int ntxq, struct port_info *pi)

void	t3_rx_eth (struct adapter adap, struct mbuf m, int ethpad)

static int	get_packet (adapter_t adap, unsigned int drop_thres, struct sge_qset qs, struct t3_mbuf_hdr mh, struct rsp_desc r)

static __inline void	handle_rsp_cntrl_info (struct sge_qset *qs, uint32_t flags)

static void	check_ring_db (adapter_t adap, struct sge_qset qs, unsigned int sleeping)

static int	process_responses (adapter_t adap, struct sge_qset qs, int budget)

static __inline int	process_responses_gts (adapter_t adap, struct sge_rspq rq)

void	t3b_intr (void *data)

void	t3_intr_msi (void *data)

void	t3_intr_msix (void *data)

static int	t3_dump_rspq (SYSCTL_HANDLER_ARGS)

static int	t3_dump_txq_eth (SYSCTL_HANDLER_ARGS)

static int	t3_dump_txq_ctrl (SYSCTL_HANDLER_ARGS)

static int	t3_set_coalesce_usecs (SYSCTL_HANDLER_ARGS)

static int	t3_pkt_timestamp (SYSCTL_HANDLER_ARGS)

void	t3_add_attach_sysctls (adapter_t *sc)

static int	sysctl_handle_macstat (SYSCTL_HANDLER_ARGS)

void	t3_add_configured_sysctls (adapter_t *sc)

int	t3_get_desc (const struct sge_qset qs, unsigned int qnum, unsigned int idx, unsigned char data)

Variables
int	txq_fills = 0

int	multiq_tx_enable = 1

struct sysctl_oid_list	sysctl__hw_cxgb_children

int	cxgb_txq_buf_ring_size = TX_ETH_Q_SIZE

static int	cxgb_tx_coalesce_force = 0

static int	cxgb_tx_coalesce_enable_start = COALESCE_START_DEFAULT

static int	cxgb_tx_coalesce_enable_stop = COALESCE_STOP_DEFAULT

static int	cxgb_tx_reclaim_threshold = TX_RECLAIM_DEFAULT

static int	recycle_enable = 0

int	cxgb_use_16k_clusters

int	nmbjumbop

int	nmbjumbo9

int	nmbjumbo16

struct tx_desc	__packed

static uint8_t	flit_desc_map []

int	cxgb_debug = 0

static const char *	rspq_name = "rspq"

static const char *	txq_names []

Macro Definition Documentation

◆ COALESCE_START_DEFAULT

#define COALESCE_START_DEFAULT TX_ETH_Q_SIZE>>1

Definition at line 97 of file cxgb_sge.c.

◆ COALESCE_START_MAX

#define COALESCE_START_MAX (TX_ETH_Q_SIZE-(TX_ETH_Q_SIZE>>3))

Definition at line 98 of file cxgb_sge.c.

◆ COALESCE_STOP_DEFAULT

#define COALESCE_STOP_DEFAULT TX_ETH_Q_SIZE>>2

Definition at line 99 of file cxgb_sge.c.

◆ COALESCE_STOP_MIN

#define COALESCE_STOP_MIN TX_ETH_Q_SIZE>>5

Definition at line 100 of file cxgb_sge.c.

◆ CXGB_SYSCTL_ADD_QUAD

#define CXGB_SYSCTL_ADD_QUAD ( a )

Value:

    SYSCTL_ADD_OID(ctx, poidlist, OID_AUTO, #a, \
    CTLTYPE_U64 | CTLFLAG_RD | CTLFLAG_NEEDGIANT, pi, \
    offsetof(struct mac_stats, a), sysctl_handle_macstat, "QU", 0)

◆ CXGB_SYSCTL_ADD_ULONG

#define CXGB_SYSCTL_ADD_ULONG ( a )

Value:

SYSCTL_ADD_ULONG(ctx, poidlist, OID_AUTO, #a, \

CTLFLAG_RD, &mstats->a, 0)

◆ GET_VTAG

#define GET_VTAG	(	cntrl,
		m
	)

Value:

do { \
        if ((m)->m_flags & M_VLANTAG)                                               \
                cntrl |= F_TXPKT_VLAN_VLD | V_TXPKT_VLAN((m)->m_pkthdr.ether_vtag); \
} while (0)

Definition at line 1353 of file cxgb_sge.c.

◆ NOMEM_INTR_DELAY

#define NOMEM_INTR_DELAY 2500

Definition at line 2187 of file cxgb_sge.c.

◆ QDUMP_SBUF_SIZE

#define QDUMP_SBUF_SIZE 32 * 400

Definition at line 3100 of file cxgb_sge.c.

◆ RSPD_CTRL_MASK

#define RSPD_CTRL_MASK

Value:

                        (RSPD_GTS_MASK | \
                        V_RSPD_TXQ0_CR(M_RSPD_TXQ0_CR) | \
                        V_RSPD_TXQ1_CR(M_RSPD_TXQ1_CR) | \
                        V_RSPD_TXQ2_CR(M_RSPD_TXQ2_CR))

Definition at line 2181 of file cxgb_sge.c.

◆ RSPD_GTS_MASK

#define RSPD_GTS_MASK (F_RSPD_TXQ0_GTS | F_RSPD_TXQ1_GTS)

Definition at line 2180 of file cxgb_sge.c.

◆ RSPQ_EOP

#define RSPQ_EOP G_RSPD_SOP_EOP(F_RSPD_EOP)

Definition at line 175 of file cxgb_sge.c.

◆ RSPQ_NSOP_NEOP

#define RSPQ_NSOP_NEOP G_RSPD_SOP_EOP(0)

Definition at line 174 of file cxgb_sge.c.

◆ RSPQ_SOP

#define RSPQ_SOP G_RSPD_SOP_EOP(F_RSPD_SOP)

Definition at line 176 of file cxgb_sge.c.

◆ RSPQ_SOP_EOP

#define RSPQ_SOP_EOP G_RSPD_SOP_EOP(F_RSPD_SOP|F_RSPD_EOP)

Definition at line 177 of file cxgb_sge.c.

◆ RX_SW_DESC_INUSE

#define RX_SW_DESC_INUSE (1 << 3)

Definition at line 171 of file cxgb_sge.c.

◆ RX_SW_DESC_MAP_CREATED

#define RX_SW_DESC_MAP_CREATED (1 << 0)

Definition at line 169 of file cxgb_sge.c.

◆ SGE_FATALERR

#define SGE_FATALERR

Value:

(SGE_PARERR | SGE_FRAMINGERR | F_RSPQCREDITOVERFOW | \

F_RSPQDISABLED)

F_RSPQCREDITOVERFOW

#define F_RSPQCREDITOVERFOW

Definition: cxgb_regs.h:529

SGE_FRAMINGERR

#define SGE_FRAMINGERR

Definition: cxgb_sge.c:511

SGE_PARERR

#define SGE_PARERR

Definition: cxgb_sge.c:506

Definition at line 512 of file cxgb_sge.c.

◆ SGE_FRAMINGERR

#define SGE_FRAMINGERR (F_UC_REQ_FRAMINGERROR | F_R_REQ_FRAMINGERROR)

Definition at line 511 of file cxgb_sge.c.

◆ SGE_PARERR

#define SGE_PARERR

Value:

                    (F_CPPARITYERROR | F_OCPARITYERROR | F_RCPARITYERROR | \
                    F_IRPARITYERROR | V_ITPARITYERROR(M_ITPARITYERROR) | \
                    V_FLPARITYERROR(M_FLPARITYERROR) | F_LODRBPARITYERROR | \
                    F_HIDRBPARITYERROR | F_LORCQPARITYERROR | \
                    F_HIRCQPARITYERROR)

Definition at line 506 of file cxgb_sge.c.

◆ SGE_RX_COPY_THRES

#define SGE_RX_COPY_THRES 128

Definition at line 134 of file cxgb_sge.c.

◆ SGE_RX_DROP_THRES

#define SGE_RX_DROP_THRES 16

Definition at line 133 of file cxgb_sge.c.

◆ SGE_RX_SM_BUF_SIZE

#define SGE_RX_SM_BUF_SIZE 1536

Definition at line 132 of file cxgb_sge.c.

◆ TCPPKTHDRSIZE

#define TCPPKTHDRSIZE (ETHER_HDR_LEN + 20 + 20)

Definition at line 1351 of file cxgb_sge.c.

◆ TX_RECLAIM_DEFAULT

#define TX_RECLAIM_DEFAULT TX_ETH_Q_SIZE>>5

Definition at line 101 of file cxgb_sge.c.

◆ TX_RECLAIM_MAX

#define TX_RECLAIM_MAX TX_ETH_Q_SIZE>>2

Definition at line 102 of file cxgb_sge.c.

◆ TX_RECLAIM_MIN

#define TX_RECLAIM_MIN TX_ETH_Q_SIZE>>6

Definition at line 103 of file cxgb_sge.c.

◆ TX_RECLAIM_PERIOD

#define TX_RECLAIM_PERIOD (hz >> 1)

Definition at line 140 of file cxgb_sge.c.

◆ TX_SW_DESC_MAP_CREATED

#define TX_SW_DESC_MAP_CREATED (1 << 1)

Definition at line 170 of file cxgb_sge.c.

◆ TX_SW_DESC_MAPPED

#define TX_SW_DESC_MAPPED (1 << 4)

Definition at line 172 of file cxgb_sge.c.

◆ TXQ_LOCK

#define TXQ_LOCK ( qs ) mtx_lock(&(qs)->lock)

Definition at line 232 of file cxgb_sge.c.

◆ TXQ_LOCK_ASSERT

#define TXQ_LOCK_ASSERT ( qs ) mtx_assert(&(qs)->lock, MA_OWNED)

Definition at line 230 of file cxgb_sge.c.

◆ TXQ_RING_DEQUEUE

#define TXQ_RING_DEQUEUE ( qs ) drbr_dequeue((qs)->port->ifp, (qs)->txq[TXQ_ETH].txq_mr)

Definition at line 240 of file cxgb_sge.c.

◆ TXQ_RING_DEQUEUE_COND

#define TXQ_RING_DEQUEUE_COND	(	qs,
		func,
		arg
	)	drbr_dequeue_cond((qs)->port->ifp, (qs)->txq[TXQ_ETH].txq_mr, func, arg)

Definition at line 238 of file cxgb_sge.c.

◆ TXQ_RING_EMPTY

#define TXQ_RING_EMPTY ( qs ) drbr_empty((qs)->port->ifp, (qs)->txq[TXQ_ETH].txq_mr)

Definition at line 234 of file cxgb_sge.c.

◆ TXQ_RING_FLUSH

#define TXQ_RING_FLUSH ( qs ) drbr_flush((qs)->port->ifp, (qs)->txq[TXQ_ETH].txq_mr)

Definition at line 237 of file cxgb_sge.c.

◆ TXQ_RING_NEEDS_ENQUEUE

#define TXQ_RING_NEEDS_ENQUEUE ( qs ) drbr_needs_enqueue((qs)->port->ifp, (qs)->txq[TXQ_ETH].txq_mr)

Definition at line 235 of file cxgb_sge.c.

◆ TXQ_TRYLOCK

#define TXQ_TRYLOCK ( qs ) mtx_trylock(&(qs)->lock)

Definition at line 231 of file cxgb_sge.c.

◆ TXQ_UNLOCK

#define TXQ_UNLOCK ( qs ) mtx_unlock(&(qs)->lock)

Definition at line 233 of file cxgb_sge.c.

◆ USE_GTS

#define USE_GTS 0

Definition at line 130 of file cxgb_sge.c.

Enumeration Type Documentation

◆ anonymous enum

anonymous enum

Enumerator
TXQ_RUNNING
TXQ_LAST_PKT_DB

Definition at line 145 of file cxgb_sge.c.

Function Documentation

◆ __FBSDID()

__FBSDID ( "$FreeBSD$" )

◆ __refill_fl()

static __inline void __refill_fl	(	adapter_t *	adap,
		struct sge_fl *	fl
	)

static

Definition at line 822 of file cxgb_sge.c.

References sge_fl::credits, refill_fl(), and sge_fl::size.

Referenced by sge_timer_reclaim().

Here is the call graph for this function:

Here is the caller graph for this function:

◆ __refill_fl_lt()

static __inline void __refill_fl_lt	(	adapter_t *	adap,
		struct sge_fl *	fl,
		int	max
	)

static

Definition at line 828 of file cxgb_sge.c.

References sge_fl::credits, refill_fl(), and sge_fl::size.

Referenced by process_responses().

Here is the call graph for this function:

Here is the caller graph for this function:

◆ alloc_ring()

static int alloc_ring	(	adapter_t *	sc,
		size_t	nelem,
		size_t	elem_size,
		size_t	sw_size,
		bus_addr_t *	phys,
		void *	desc,
		void *	sdesc,
		bus_dma_tag_t *	tag,
		bus_dmamap_t *	map,
		bus_dma_tag_t	parent_entry_tag,
		bus_dma_tag_t *	entry_tag
	)

static

Definition at line 876 of file cxgb_sge.c.

References alloc_ring_cb(), adapter::dev, adapter::parent_dmat, TX_MAX_SEGS, and TX_MAX_SIZE.

Referenced by t3_sge_alloc_qset().

Here is the call graph for this function:

Here is the caller graph for this function:

◆ alloc_ring_cb()

static void alloc_ring_cb	(	void *	arg,
		bus_dma_segment_t *	segs,
		int	nsegs,
		int	error
	)

static

Definition at line 867 of file cxgb_sge.c.

Referenced by alloc_ring().

Here is the caller graph for this function:

◆ calc_tx_descs()

static __inline unsigned int calc_tx_descs	(	const struct mbuf *	m,
		int	nsegs
	)

static

calc_tx_descs - calculate the number of Tx descriptors for a packet @m: the packet mbufs @nsegs: the number of segments

Returns the number of Tx descriptors needed for the given Ethernet packet. Ethernet packets require addition of WR and CPL headers.

Definition at line 1173 of file cxgb_sge.c.

References flits_to_desc(), PIO_LEN, and sgl_len().

Referenced by t3_encap().

Here is the call graph for this function:

Here is the caller graph for this function:

◆ check_desc_avail()

static __inline int check_desc_avail	(	adapter_t *	adap,
		struct sge_txq *	q,
		struct mbuf *	m,
		unsigned int	ndesc,
		unsigned int	qid
	)

static

check_desc_avail - check descriptor availability on a send queue @adap: the adapter @q: the TX queue @m: the packet needing the descriptors @ndesc: the number of Tx descriptors needed @qid: the Tx queue number in its queue set (TXQ_OFLD or TXQ_CTRL)

Checks if the requested number of Tx descriptors is available on an SGE send queue. If the queue is already suspended or not enough descriptors are available the packet is queued for later transmission. Must be called with the Tx queue locked.

Returns 0 if enough descriptors are available, 1 if there aren't enough descriptors and the packet has been queued, and 2 if the caller needs to retry because there weren't enough descriptors at the beginning of the call but some freed up in the mean time.

Definition at line 1837 of file cxgb_sge.c.

References sge_txq::in_use, sge_txq::sendq, should_restart_tx(), sge_txq::size, sge_txq::stops, test_and_clear_bit, sge_qset::txq_stopped, and txq_to_qset().

Referenced by ctrl_xmit().

Here is the call graph for this function:

Here is the caller graph for this function:

◆ check_pkt_coalesce()

static __inline uint64_t check_pkt_coalesce ( struct sge_qset * qs )

static

Definition at line 256 of file cxgb_sge.c.

References port_info::adapter, COALESCE_START_MAX, COALESCE_STOP_MIN, sge_qset::coalescing, cxgb_tx_coalesce_enable_start, cxgb_tx_coalesce_enable_stop, cxgb_tx_coalesce_force, sge_qset::idx, sge_txq::in_use, sge_qset::port, sge_qset::txq, TXQ_ETH, and TXQ_RING_EMPTY.

Referenced by cxgb_dequeue(), and cxgb_transmit_locked().

Here is the caller graph for this function:

◆ check_ring_db()

static void check_ring_db	(	adapter_t *	adap,
		struct sge_qset *	qs,
		unsigned int	sleeping
	)

static

Definition at line 2809 of file cxgb_sge.c.

Referenced by process_responses().

Here is the caller graph for this function:

◆ check_ring_tx_db()

static __inline void check_ring_tx_db	(	adapter_t *	adap,
		struct sge_txq *	q,
		int	mustring
	)

static

check_ring_tx_db - check and potentially ring a Tx queue's doorbell @adap: the adapter @q: the Tx queue

Ring the doorbell if a Tx queue is asleep. There is a natural race, where the HW is going to sleep just after we checked, however, then the interrupt handler will detect the outstanding TX packet and ring the doorbell for us.

When GTS is disabled we unconditionally ring the doorbell.

Definition at line 1235 of file cxgb_sge.c.

References A_SG_KDOORBELL, sge_txq::cntxt_id, sge_txq::db_pending, F_SELEGRCNTX, sge_txq::flags, t3_write_reg(), TXQ_LAST_PKT_DB, TXQ_RUNNING, and V_EGRCNTX.

Referenced by cxgb_start_locked(), cxgb_transmit_locked(), and t3_encap().

Here is the call graph for this function:

Here is the caller graph for this function:

◆ coalesce_check()

static int coalesce_check	(	struct mbuf *	m,
		void *	arg
	)

static

Definition at line 319 of file cxgb_sge.c.

References coalesce_info::count, coalesce_info::nbytes, and coalesce_info::noncoal.

Referenced by cxgb_dequeue().

Here is the caller graph for this function:

◆ ctrl_xmit()

static int ctrl_xmit	(	adapter_t *	adap,
		struct sge_qset *	qs,
		struct mbuf *	m
	)

static

ctrl_xmit - send a packet through an SGE control Tx queue @adap: the adapter @q: the control queue @m: the packet

Send a packet through an SGE control Tx queue. Packets sent through a control queue must fit entirely as immediate data in a single Tx descriptor and have no page fragments.

Definition at line 1894 of file cxgb_sge.c.

References A_SG_KDOORBELL, check_desc_avail(), sge_txq::cntxt_id, sge_txq::desc, F_SELEGRCNTX, F_WR_EOP, F_WR_SOP, sge_txq::gen, sge_txq::in_use, sge_txq::pidx, reclaim_completed_tx_imm(), sge_txq::size, t3_write_reg(), sge_txq::token, sge_qset::txq, TXQ_CTRL, TXQ_LOCK, TXQ_UNLOCK, V_EGRCNTX, V_WR_TID, WR_LEN, and write_imm().

Referenced by t3_mgmt_tx().

Here is the call graph for this function:

Here is the caller graph for this function:

◆ cxgb_dequeue()

static struct mbuf * cxgb_dequeue ( struct sge_qset * qs )

static

Definition at line 337 of file cxgb_sge.c.

References check_pkt_coalesce(), coalesce_check(), coalesce_info::count, coalesce_info::nbytes, coalesce_info::noncoal, TXQ_RING_DEQUEUE, and TXQ_RING_DEQUEUE_COND.

Referenced by cxgb_start_locked().

Here is the call graph for this function:

Here is the caller graph for this function:

◆ cxgb_qflush()

void cxgb_qflush ( struct ifnet * ifp )

Definition at line 1776 of file cxgb_sge.c.

Referenced by cxgb_port_attach().

Here is the caller graph for this function:

◆ cxgb_start_locked()

static void cxgb_start_locked ( struct sge_qset * qs )

static

Definition at line 1654 of file cxgb_sge.c.

References port_info::adapter, check_ring_tx_db(), cxgb_dequeue(), cxgb_tx_reclaim_threshold, cxgb_tx_timeout(), sge_txq::db_pending, ifp, port_info::ifp, sge_txq::in_use, port_info::link_config, link_config::link_ok, sge_qset::port, sge_qset::qs_flags, QS_FLUSHING, QS_TIMEOUT, reclaim_completed_tx(), sge_txq::size, t3_encap(), TX_MAX_DESC, sge_qset::txq, TXQ_ETH, TXQ_LOCK_ASSERT, TXQ_RING_EMPTY, TXQ_RING_FLUSH, and sge_txq::txq_timer.

Referenced by cxgb_transmit_locked(), cxgb_tx_timeout(), and cxgb_tx_watchdog().

Here is the call graph for this function:

Here is the caller graph for this function:

◆ cxgb_transmit()

int cxgb_transmit	(	struct ifnet *	ifp,
		struct mbuf *	m
	)

Definition at line 1748 of file cxgb_sge.c.

References port_info::adapter, cxgb_transmit_locked(), port_info::first_qset, ifp, port_info::link_config, link_config::link_ok, port_info::nqsets, sge::qs, adapter::sge, sge_qset::txq, TXQ_ETH, sge_txq::txq_mr, TXQ_TRYLOCK, and TXQ_UNLOCK.

Referenced by cxgb_port_attach().

Here is the call graph for this function:

Here is the caller graph for this function:

◆ cxgb_transmit_locked()

static int cxgb_transmit_locked	(	struct ifnet *	ifp,
		struct sge_qset *	qs,
		struct mbuf *	m
	)

static

Definition at line 1700 of file cxgb_sge.c.

References port_info::adapter, check_pkt_coalesce(), check_ring_tx_db(), cxgb_start_locked(), cxgb_tx_reclaim_threshold, cxgb_tx_timeout(), sge_txq::db_pending, ifp, sge_txq::in_use, port_info::link_config, link_config::link_ok, sge_qset::port, reclaim_completed_tx(), sge_txq::size, t3_encap(), TX_MAX_DESC, sge_qset::txq, sge_txq::txq_direct_bytes, sge_txq::txq_direct_packets, TXQ_ETH, TXQ_LOCK_ASSERT, sge_txq::txq_mr, TXQ_RING_EMPTY, TXQ_RING_NEEDS_ENQUEUE, and sge_txq::txq_timer.

Referenced by cxgb_transmit().

Here is the call graph for this function:

Here is the caller graph for this function:

◆ cxgb_tx_timeout()

static void cxgb_tx_timeout ( void * arg )

static

Definition at line 1638 of file cxgb_sge.c.

References sge_qset::coalescing, cxgb_start_locked(), sge_txq::in_use, sge_qset::qs_flags, QS_TIMEOUT, sge_txq::size, sge_qset::txq, TXQ_ETH, TXQ_TRYLOCK, and TXQ_UNLOCK.

Referenced by cxgb_start_locked(), and cxgb_transmit_locked().

Here is the call graph for this function:

Here is the caller graph for this function:

◆ cxgb_tx_watchdog()

void cxgb_tx_watchdog ( void * arg )

Definition at line 1614 of file cxgb_sge.c.

References sge_qset::coalescing, cxgb_start_locked(), cxgb_tx_coalesce_enable_start, cxgb_tx_coalesce_enable_stop, cxgb_tx_watchdog(), port_info::ifp, sge_txq::in_use, sge_qset::port, sge_qset::qs_flags, QS_FLUSHING, sge_qset::txq, TXQ_ETH, TXQ_RING_EMPTY, TXQ_TRYLOCK, TXQ_UNLOCK, and sge_txq::txq_watchdog.

Referenced by cxgb_init_locked(), and cxgb_tx_watchdog().

Here is the call graph for this function:

Here is the caller graph for this function:

◆ flits_to_desc()

static __inline u_int flits_to_desc ( u_int n )

static

Definition at line 501 of file cxgb_sge.c.

References flit_desc_map.

Referenced by calc_tx_descs(), and t3_sge_alloc_qset().

Here is the caller graph for this function:

◆ free_rx_bufs()

static void free_rx_bufs	(	adapter_t *	sc,
		struct sge_fl *	q
	)

static

free_rx_bufs - free the Rx buffers on an SGE free list @sc: the controle softc @q: the SGE free list to clean up

Release the buffers on an SGE free-buffer Rx queue. HW fetching from this queue should be stopped before calling this function.

Definition at line 794 of file cxgb_sge.c.

References sge_fl::cidx, sge_fl::credits, sge_fl::entry_tag, rx_sw_desc::flags, rx_sw_desc::m, rx_sw_desc::map, RX_SW_DESC_INUSE, rx_sw_desc::rxsd_cl, sge_fl::sdesc, sge_fl::size, and sge_fl::zone.

Referenced by t3_free_qset().

Here is the caller graph for this function:

◆ get_imm_packet()

static int get_imm_packet	(	adapter_t *	sc,
		const struct rsp_desc *	resp,
		struct mbuf *	m
	)

static

get_imm_packet - return the next ingress packet buffer from a response @resp: the response descriptor containing the packet data

Return a packet containing the immediate data of the given response.

Definition at line 483 of file cxgb_sge.c.

References CPL_RX_DATA, CPL_RX_PKT, rsp_desc::imm_data, IMMED_PKT_SIZE, cpl_rx_data::len, cpl_rx_pkt::len, rss_header::opcode, and rsp_desc::rss_hdr.

Referenced by process_responses().

Here is the caller graph for this function:

◆ get_packet()

static int get_packet	(	adapter_t *	adap,
		unsigned int	drop_thres,
		struct sge_qset *	qs,
		struct t3_mbuf_hdr *	mh,
		struct rsp_desc *	r
	)

static

get_packet - return the next ingress packet buffer from a free list @adap: the adapter that received the packet @drop_thres: # of remaining buffers before we start dropping packets @qs: the qset that the SGE free list holding the packet belongs to @mh: the mbuf header, contains a pointer to the head and tail of the mbuf chain @r: response descriptor

Get the next packet from a free list and complete setup of the sk_buff. If the packet is small we make a copy and recycle the original buffer, otherwise we use the original buffer itself. If a positive drop threshold is supplied packets are dropped and their buffers recycled if (a) the number of remaining buffers is under the threshold and the packet is too big to copy, or (b) the packet should be copied but there is no memory for the copy.

Definition at line 2684 of file cxgb_sge.c.

References sge_fl::cidx, sge_fl::credits, cxgb_debug, sge_fl::entry_tag, F_RSPD_FLQ, sge_qset::fl, flags, rsp_desc::flags, G_RSPD_LEN, G_RSPD_SOP_EOP, rsp_desc::len_cq, rx_sw_desc::m, t3_mbuf_hdr::mh_head, t3_mbuf_hdr::mh_tail, prefetch, recycle_enable, recycle_rx_buf(), RSPQ_EOP, RSPQ_NSOP_NEOP, RSPQ_SOP, RSPQ_SOP_EOP, rx_sw_desc::rxsd_cl, sge_fl::sdesc, SGE_RX_COPY_THRES, sge_fl::size, sge_fl::type, and sge_fl::zone.

Referenced by process_responses().

Here is the call graph for this function:

Here is the caller graph for this function:

◆ handle_rsp_cntrl_info()

static __inline void handle_rsp_cntrl_info	(	struct sge_qset *	qs,
		uint32_t	flags
	)

static

handle_rsp_cntrl_info - handles control information in a response @qs: the queue set corresponding to the response @flags: the response control flags

Handles the control information of an SGE response, such as GTS indications and completion credits for the queue set's Tx queues. HW coalesces credits, we don't do any extra SW coalescing.

Definition at line 2782 of file cxgb_sge.c.

References F_RSPD_TXQ0_GTS, F_RSPD_TXQ1_GTS, flags, sge_txq::flags, G_RSPD_TXQ0_CR, G_RSPD_TXQ1_CR, G_RSPD_TXQ2_CR, sge_txq::processed, sge_qset::txq, TXQ_CTRL, TXQ_ETH, TXQ_OFLD, and TXQ_RUNNING.

Referenced by process_responses().

Here is the caller graph for this function:

◆ init_qset_cntxt()

static void init_qset_cntxt	(	struct sge_qset *	qs,
		u_int	id
	)

static

init_qset_cntxt - initialize an SGE queue set context info @qs: the queue set @id: the queue set id

Initializes the TIDs and context ids for the queues of a queue set.

Definition at line 1113 of file cxgb_sge.c.

References sge_rspq::cntxt_id, sge_fl::cntxt_id, sge_txq::cntxt_id, sge_qset::fl, FW_CTRL_SGEEC_START, FW_CTRL_TID_START, FW_OFLD_SGEEC_START, FW_TUNNEL_SGEEC_START, FW_TUNNEL_TID_START, sge_qset::rspq, sge_txq::sendq, sge_txq::token, sge_qset::txq, TXQ_CTRL, TXQ_ETH, and TXQ_OFLD.

Referenced by t3_sge_alloc_qset().

Here is the caller graph for this function:

◆ is_new_response()

static __inline int is_new_response	(	const struct rsp_desc *	r,
		const struct sge_rspq *	q
	)

static

is_new_response - check if a response is newly written @r: the response descriptor @q: the response queue

Returns true if a response descriptor contains a yet unprocessed response.

Definition at line 2174 of file cxgb_sge.c.

References F_RSPD_GEN2, sge_rspq::gen, and rsp_desc::intr_gen.

Referenced by process_responses().

Here is the caller graph for this function:

◆ make_sgl()

static __inline void make_sgl	(	struct sg_ent *	sgp,
		bus_dma_segment_t *	segs,
		int	nsegs
	)

static

make_sgl - populate a scatter/gather list for a packet @sgp: the SGL to populate @segs: the packet dma segments @nsegs: the number of segments

Generates a scatter/gather list for the buffers that make up a packet and returns the SGL size in 8-byte words. The caller must size the SGL appropriately.

Definition at line 1198 of file cxgb_sge.c.

References sg_ent::addr, and sg_ent::len.

Referenced by t3_encap().

Here is the caller graph for this function:

◆ process_responses()

static int process_responses	(	adapter_t *	adap,
		struct sge_qset *	qs,
		int	budget
	)

static

process_responses - process responses from an SGE response queue @adap: the adapter @qs: the queue set to which the response queue belongs @budget: how many responses can be processed in this round

Process responses from an SGE response queue up to the supplied budget. Responses include received packets as well as credits and other events for the queues that belong to the response queue's queue set. A negative budget is effectively unlimited.

Additionally choose the interrupt holdoff time for the next interrupt on this queue. If the system is under memory shortage use a fairly long delay to help recovery.

Definition at line 2831 of file cxgb_sge.c.

References __refill_fl_lt(), AN_PKT_SIZE, sge_rspq::async_notif, check_ring_db(), sge_rspq::cidx, CPL_ASYNC_NOTIF, CPL_RX_PKT, sge_rspq::credits, lro_state::ctrl, cxgb_debug, sge_rspq::desc, lro_state::enabled, F_RSPD_ASYNC_NOTIF, F_RSPD_IMM_DATA_VALID, sge_qset::fl, flags, rsp_desc::flags, sge_rspq::gen, get_imm_packet(), get_packet(), handle_rsp_cntrl_info(), rss_header::hash_type, sge_rspq::holdoff_tmr, ifp, port_info::ifp, sge_rspq::imm_data, is_new_response(), rsp_desc::len_cq, sge_qset::lro, t3_mbuf_hdr::mh_head, t3_mbuf_hdr::mh_tail, sge_rspq::next_holdoff, NOMEM_INTR_DELAY, sge_rspq::offload_pkts, rss_header::opcode, sge_qset::port, sge_rspq::pure_rsps, refill_rspq(), restart_tx(), RSPD_CTRL_MASK, RSPD_GTS_MASK, sge_qset::rspq, sge_rspq::rspq_mh, rss_header::rss_hash_val, rsp_desc::rss_hdr, SGE_RX_DROP_THRES, sge_rspq::size, t3_rx_eth(), adapter::timestamp, and sge_qset::txq_stopped.

Referenced by process_responses_gts().

Here is the call graph for this function:

Here is the caller graph for this function:

◆ process_responses_gts()

static __inline int process_responses_gts	(	adapter_t *	adap,
		struct sge_rspq *	rq
	)

static

Definition at line 3004 of file cxgb_sge.c.

References A_SG_GTS, sge_rspq::cidx, sge_rspq::cntxt_id, cxgb_debug, sge_rspq::next_holdoff, process_responses(), rspq_to_qset(), t3_write_reg(), V_NEWINDEX, V_NEWTIMER, and V_RSPQ.

Referenced by t3_intr_msi(), t3_intr_msix(), and t3b_intr().

Here is the call graph for this function:

Here is the caller graph for this function:

◆ reclaim_completed_tx()

static __inline int reclaim_completed_tx	(	struct sge_qset *	qs,
		int	reclaim_min,
		int	queue
	)

static

reclaim_completed_tx - reclaims completed Tx descriptors @adapter: the adapter @q: the Tx queue to reclaim completed descriptors from

Reclaims Tx descriptors that the SGE has indicated it has processed, and frees the associated buffers if possible. Called with the Tx queue's lock held.

Definition at line 372 of file cxgb_sge.c.

References sge_txq::cleaned, cxgb_tx_reclaim_threshold, desc_reclaimable, sge_txq::in_use, sge_qset::lock, t3_free_tx_desc(), TX_RECLAIM_DEFAULT, TX_RECLAIM_MAX, TX_RECLAIM_MIN, sge_qset::txq, TXQ_ETH, and sge_qset::txq_stopped.

Referenced by cxgb_start_locked(), cxgb_transmit_locked(), sge_timer_reclaim(), sge_txq_reclaim_handler(), and t3_free_qset().

Here is the call graph for this function:

Here is the caller graph for this function:

◆ reclaim_completed_tx_imm()

static __inline void reclaim_completed_tx_imm ( struct sge_txq * q )

static

reclaim_completed_tx_imm - reclaim completed control-queue Tx descs @q: the SGE control Tx queue

This is a variant of reclaim_completed_tx() that is used for Tx queues that send only immediate data (presently just the control queues) and thus do not have any mbufs

Definition at line 1875 of file cxgb_sge.c.

References sge_txq::cleaned, sge_txq::in_use, and sge_txq::processed.

Referenced by ctrl_xmit(), and restart_ctrlq().

Here is the caller graph for this function:

◆ recycle_rx_buf()

static void recycle_rx_buf	(	adapter_t *	adap,
		struct sge_fl *	q,
		unsigned int	idx
	)

static

recycle_rx_buf - recycle a receive buffer @adapter: the adapter @q: the SGE free list @idx: index of buffer to recycle

Recycles the specified buffer on the given free list by adding it at the next available slot on the list.

Definition at line 846 of file cxgb_sge.c.

References A_SG_KDOORBELL, rx_desc::addr_hi, rx_desc::addr_lo, sge_fl::cntxt_id, sge_fl::credits, sge_fl::desc, sge_fl::gen, rx_desc::gen2, rx_desc::len_gen, sge_fl::pidx, sge_fl::sdesc, sge_fl::size, t3_write_reg(), V_EGRCNTX, V_FLD_GEN1, and V_FLD_GEN2.

Referenced by get_packet().

Here is the call graph for this function:

Here is the caller graph for this function:

◆ refill_fl()

static void refill_fl	(	adapter_t *	sc,
		struct sge_fl *	q,
		int	n
	)

static

refill_fl - refill an SGE free-buffer list @sc: the controller softc @q: the free-list to refill
: the number of new buffers to allocate

(Re)populate an SGE free-buffer list with up to
new packet buffers. The caller must assure that
does not exceed the queue's capacity.

Definition at line 708 of file cxgb_sge.c.

References A_SG_KDOORBELL, rx_desc::addr_hi, rx_desc::addr_lo, sge_fl::buf_size, sge_fl::cntxt_id, sge_fl::credits, sge_fl::db_pending, sge_fl::desc, sge_fl::entry_tag, refill_fl_cb_arg::error, rx_sw_desc::flags, sge_fl::gen, rx_desc::gen2, rx_desc::len_gen, rx_sw_desc::m, rx_sw_desc::map, sge_fl::pidx, refill_fl_cb(), RX_SW_DESC_INUSE, RX_SW_DESC_MAP_CREATED, rx_sw_desc::rxsd_cl, sge_fl::sdesc, refill_fl_cb_arg::seg, sge_fl::size, t3_write_reg(), V_EGRCNTX, V_FLD_GEN1, V_FLD_GEN2, and sge_fl::zone.

Referenced by __refill_fl(), __refill_fl_lt(), and t3_sge_alloc_qset().

Here is the call graph for this function:

Here is the caller graph for this function:

◆ refill_fl_cb()

static void refill_fl_cb	(	void *	arg,
		bus_dma_segment_t *	segs,
		int	nseg,
		int	error
	)

static

Definition at line 688 of file cxgb_sge.c.

References refill_fl_cb_arg::error, refill_fl_cb_arg::nseg, and refill_fl_cb_arg::seg.

Referenced by refill_fl().

Here is the caller graph for this function:

◆ refill_rspq()

static __inline void refill_rspq	(	adapter_t *	sc,
		const struct sge_rspq *	q,
		u_int	credits
	)

static

refill_rspq - replenish an SGE response queue @adapter: the adapter @q: the response queue to replenish @credits: how many new responses to make available

Replenishes a response queue by making the supplied number of responses available to HW.

Definition at line 1046 of file cxgb_sge.c.

References A_SG_RSPQ_CREDIT_RETURN, sge_rspq::cntxt_id, t3_write_reg(), V_CREDITS, and V_RSPQ.

Referenced by process_responses(), sge_timer_reclaim(), and t3_sge_alloc_qset().

Here is the call graph for this function:

Here is the caller graph for this function:

◆ restart_ctrlq()

static void restart_ctrlq	(	void *	data,
		int	npending
	)

static

restart_ctrlq - restart a suspended control queue @qs: the queue set cotaining the control queue

Resumes transmission on a suspended Tx control queue.

Definition at line 1940 of file cxgb_sge.c.

References A_SG_KDOORBELL, port_info::adapter, sge_txq::cntxt_id, sge_txq::desc, F_SELEGRCNTX, sge_txq::gen, sge_txq::in_use, sge_txq::pidx, sge_qset::port, reclaim_completed_tx_imm(), sge_txq::sendq, should_restart_tx(), sge_txq::size, sge_txq::stops, t3_write_reg(), test_and_clear_bit, sge_qset::txq, TXQ_CTRL, TXQ_LOCK, sge_qset::txq_stopped, TXQ_UNLOCK, V_EGRCNTX, and write_imm().

Referenced by t3_sge_alloc_qset().

Here is the call graph for this function:

Here is the caller graph for this function:

◆ restart_tx()

static void restart_tx ( struct sge_qset * qs )

static

Definition at line 2385 of file cxgb_sge.c.

References port_info::adapter, sge_qset::port, sge_txq::qresume_task, sge_txq::restarts, should_restart_tx(), test_and_clear_bit, adapter::tq, sge_qset::txq, TXQ_CTRL, TXQ_OFLD, and sge_qset::txq_stopped.

Referenced by process_responses().

Here is the call graph for this function:

Here is the caller graph for this function:

◆ set_wr_hdr()

static void set_wr_hdr	(	struct work_request_hdr *	wrp,
		uint32_t	wr_hi,
		uint32_t	wr_lo
	)

static

Definition at line 303 of file cxgb_sge.c.

Referenced by t3_encap(), write_imm(), and write_wr_hdr_sgl().

Here is the caller graph for this function:

◆ sge_slow_intr_handler()

static void sge_slow_intr_handler	(	void *	arg,
		int	ncount
	)

static

Definition at line 923 of file cxgb_sge.c.

References A_PL_INT_ENABLE0, adapter::slow_intr_mask, t3_read_reg(), t3_slow_intr_handler(), and t3_write_reg().

Referenced by t3_sge_init_adapter().

Here is the call graph for this function:

Here is the caller graph for this function:

◆ sge_timer_cb()

static void sge_timer_cb ( void * arg )

static

sge_timer_cb - perform periodic maintenance of an SGE qset @data: the SGE queue set to maintain

Runs periodically from a timer to perform maintenance of an SGE queue set. It performs two tasks:

a) Cleans up any completed Tx descriptors that may still be pending. Normal descriptor cleanup happens when new packets are added to a Tx queue so this timer is relatively infrequent and does any cleanup only if the Tx queue has not seen any new packets in a while. We make a best effort attempt to reclaim descriptors, in that we don't wait around if we cannot get a queue's lock (which most likely is because someone else is queueing new packets and so will also handle the clean up). Since control queues use immediate data exclusively we don't bother cleaning them up here.

b) Replenishes Rx queues that have run out due to memory shortage. Normally new Rx buffers are added when existing ones are consumed but when out of memory a queue can become empty. We try to add only a few buffers here, the queue will be replenished fully as these new buffers are used up if memory shortage has subsided.

c) Return coalesced response queue credits in case a response queue is starved.

d) Ring doorbells for T304 tunnel queues since we have seen doorbell fifo overflows and the FW doesn't implement any recovery scheme yet.

Definition at line 962 of file cxgb_sge.c.

References A_SG_KDOORBELL, sge_txq::cleaned, sge_fl::credits, F_SELEGRCNTX, port_info::first_qset, sge_qset::fl, adapter::flags, for_each_port, FW_TUNNEL_SGEEC_START, adapter_params::nports, port_info::nqsets, adapter::open_device_map, adapter::params, adapter::port, sge_txq::processed, sge::qs, adapter::sge, sge_timer_cb(), adapter::sge_timer_ch, sge_fl::size, t3_write_reg(), port_info::timer_reclaim_task, adapter::tq, TX_RECLAIM_PERIOD, sge_qset::txq, TXQ_OFLD, and USING_MSIX.

Referenced by sge_timer_cb(), t3_sge_init_adapter(), and t3_sge_reset_adapter().

Here is the call graph for this function:

Here is the caller graph for this function:

◆ sge_timer_reclaim()

static void sge_timer_reclaim	(	void *	arg,
		int	ncount
	)

static

Definition at line 1065 of file cxgb_sge.c.

References __refill_fl(), A_SG_RSPQ_FL_STATUS, port_info::adapter, sge_rspq::cntxt_id, sge_rspq::credits, sge_fl::credits, port_info::first_qset, sge_qset::fl, adapter::flags, lock, sge_rspq::lock, nqsets, port_info::nqsets, sge::qs, reclaim_completed_tx(), refill_rspq(), sge_qset::rspq, adapter::sge, sge_fl::size, t3_read_reg(), t3_write_reg(), TXQ_OFLD, and USING_MSIX.

Referenced by t3_sge_init_port().

Here is the call graph for this function:

Here is the caller graph for this function:

◆ sge_txq_reclaim_handler()

static void sge_txq_reclaim_handler	(	void *	arg,
		int	ncount
	)

static

Definition at line 1055 of file cxgb_sge.c.

References reclaim_completed_tx().

Referenced by t3_sge_alloc_qset().

Here is the call graph for this function:

Here is the caller graph for this function:

◆ sgl_len()

static __inline unsigned int sgl_len ( unsigned int n )

static

sgl_len - calculates the size of an SGL of the given capacity
: the number of SGL entries

Calculates the number of flits needed for a scatter/gather list that can hold the given number of entries.

Definition at line 471 of file cxgb_sge.c.

Referenced by calc_tx_descs(), t3_encap(), and t3_sge_alloc_qset().

Here is the caller graph for this function:

◆ should_restart_tx()

static __inline int should_restart_tx ( const struct sge_txq * q )

static

should_restart_tx - are there enough resources to restart a Tx queue? @q: the Tx queue

Checks if there are enough descriptors to restart a suspended Tx queue.

Definition at line 412 of file cxgb_sge.c.

References sge_txq::cleaned, sge_txq::in_use, sge_txq::processed, and sge_txq::size.

Referenced by check_desc_avail(), restart_ctrlq(), and restart_tx().

Here is the caller graph for this function:

◆ sysctl_handle_macstat()

static int sysctl_handle_macstat ( SYSCTL_HANDLER_ARGS )

static

Definition at line 3406 of file cxgb_sge.c.

References cxgb_refresh_stats(), port_info::mac, and cmac::stats.

Here is the call graph for this function:

◆ SYSCTL_INT() [1/5]

SYSCTL_INT	(	_hw_cxgb	,
		OID_AUTO	,
		tx_coalesce_enable_start	,
		CTLFLAG_RWTUN	,
		&	cxgb_tx_coalesce_enable_start,
		0	,
		"coalesce enable threshold"
	)

◆ SYSCTL_INT() [2/5]

SYSCTL_INT	(	_hw_cxgb	,
		OID_AUTO	,
		tx_coalesce_enable_stop	,
		CTLFLAG_RWTUN	,
		&	cxgb_tx_coalesce_enable_stop,
		0	,
		"coalesce disable threshold"
	)

◆ SYSCTL_INT() [3/5]

SYSCTL_INT	(	_hw_cxgb	,
		OID_AUTO	,
		tx_coalesce_force	,
		CTLFLAG_RWTUN	,
		&	cxgb_tx_coalesce_force,
		0	,
		"coalesce small packets into a single work request regardless of ring state"
	)

◆ SYSCTL_INT() [4/5]

SYSCTL_INT	(	_hw_cxgb	,
		OID_AUTO	,
		tx_reclaim_threshold	,
		CTLFLAG_RWTUN	,
		&	cxgb_tx_reclaim_threshold,
		0	,
		"tx cleaning minimum threshold"
	)

◆ SYSCTL_INT() [5/5]

SYSCTL_INT	(	_hw_cxgb	,
		OID_AUTO	,
		txq_mr_size	,
		CTLFLAG_RDTUN	,
		&	cxgb_txq_buf_ring_size,
		0	,
		"size of per-queue mbuf ring"
	)

◆ t3_add_attach_sysctls()

void t3_add_attach_sysctls ( adapter_t * sc )

Definition at line 3358 of file cxgb_sge.c.

References vpd_params::cclk, cxgb_debug, adapter::dev, adapter::fw_version, adapter::params, adapter::port_types, adapter_params::rev, txq_fills, and adapter_params::vpd.

Referenced by cxgb_controller_attach().

Here is the caller graph for this function:

◆ t3_add_configured_sysctls()

void t3_add_configured_sysctls ( adapter_t * sc )

Definition at line 3421 of file cxgb_sge.c.

References sge_rspq::cidx, sge_txq::cidx, sge_txq::cleaned, sge_rspq::credits, lro_state::ctrl, CXGB_SYSCTL_ADD_QUAD, CXGB_SYSCTL_ADD_ULONG, adapter::dev, sge_fl::empty, port_info::first_qset, sge_qset::fl, sge_txq::gen, sge_txq::in_use, sge_qset::lro, port_info::mac, port_info::namebuf, sge_qset::namebuf, adapter_params::nports, port_info::nqsets, adapter::params, sge_rspq::phys_addr, sge_txq::phys_addr, sge_txq::pidx, adapter::port, PORT_NAME_LEN, sge_txq::processed, sge::qs, QS_NAME_LEN, sge_qset::rspq, sge_rspq::rspq_dump_count, sge_rspq::rspq_dump_start, rspq_name, sge_txq::sendq, adapter::sge, sge_rspq::size, sge_rspq::starved, cmac::stats, t3_dump_rspq(), t3_dump_txq_ctrl(), t3_dump_txq_eth(), t3_pkt_timestamp(), t3_set_coalesce_usecs(), sge_qset::txq, sge_txq::txq_coalesced, TXQ_CTRL, sge_txq::txq_dump_count, sge_txq::txq_dump_start, sge_txq::txq_enqueued, TXQ_ETH, sge_txq::txq_frees, sge_txq::txq_mr, txq_names, sge_txq::txq_skipped, and sge_qset::txq_stopped.

Referenced by cxgb_up().

Here is the call graph for this function:

Here is the caller graph for this function:

◆ t3_dump_rspq()

static int t3_dump_rspq ( SYSCTL_HANDLER_ARGS )

static

Definition at line 3102 of file cxgb_sge.c.

References port_info::adapter, sge_rspq::cntxt_id, sge_rspq::desc, sge_qset::idx, sge_qset::port, QDUMP_SBUF_SIZE, sge_rspq::rspq_dump_count, sge_rspq::rspq_dump_start, RSPQ_Q_SIZE, rspq_to_qset(), and t3_sge_read_rspq().

Referenced by t3_add_configured_sysctls().

Here is the call graph for this function:

Here is the caller graph for this function:

◆ t3_dump_txq_ctrl()

static int t3_dump_txq_ctrl ( SYSCTL_HANDLER_ARGS )

static

Definition at line 3231 of file cxgb_sge.c.

References sge_txq::desc, tx_desc::flit, G_WR_GEN, sge_qset::idx, QDUMP_SBUF_SIZE, TXQ_CTRL, sge_txq::txq_dump_count, sge_txq::txq_dump_start, and txq_to_qset().

Referenced by t3_add_configured_sysctls().

Here is the call graph for this function:

Here is the caller graph for this function:

◆ t3_dump_txq_eth()

static int t3_dump_txq_eth ( SYSCTL_HANDLER_ARGS )

static

Definition at line 3164 of file cxgb_sge.c.

References port_info::adapter, sge_rspq::cntxt_id, sge_txq::desc, tx_desc::flit, G_WR_GEN, sge_qset::idx, sge_qset::port, QDUMP_SBUF_SIZE, sge_qset::rspq, t3_sge_read_ecntxt(), TX_ETH_Q_SIZE, sge_txq::txq_dump_count, sge_txq::txq_dump_start, TXQ_ETH, and txq_to_qset().

Referenced by t3_add_configured_sysctls().

Here is the call graph for this function:

Here is the caller graph for this function:

◆ t3_encap()

static int t3_encap	(	struct sge_qset *	qs,
		struct mbuf **	m
	)

static

Definition at line 1360 of file cxgb_sge.c.

References port_info::adapter, cpl_tx_pkt_batch_entry::addr, busdma_map_sg_collapse(), busdma_map_sg_vec(), calc_tx_descs(), check_ring_tx_db(), cpl_tx_pkt::cntrl, cpl_tx_pkt_lso::cntrl, cpl_tx_pkt_batch_entry::cntrl, txq_state::compl, CPL_ETH_II, CPL_ETH_II_VLAN, CPL_TX_PKT, CPL_TX_PKT_LSO, cxgb_debug, sge_txq::desc, sge_txq::entry_tag, F_LSO_IPV6, F_TXPKT_IPCSUM_DIS, F_TXPKT_L4CSUM_DIS, F_WR_EOP, F_WR_SOP, tx_desc::flit, FW_WROPCODE_TUNNEL_TX_PKT, txq_state::gen, GET_VTAG, port_info::ifp, cpl_tx_pkt::len, cpl_tx_pkt_lso::len, cpl_tx_pkt_batch_entry::len, sge_qset::lock, cpl_tx_pkt_lso::lso_info, tx_sw_desc::m, make_sgl(), tx_sw_desc::map, sge_txq::pidx, PIO_LEN, cpl_tx_pkt_batch::pkt_entry, sge_qset::port, prefetch, sge_txq::sdesc, set_wr_hdr(), sgl_len(), TCPPKTHDRSIZE, sge_txq::token, TX_MAX_SEGS, port_info::txpkt_intf, sge_qset::txq, sge_txq::txq_coalesced, TXQ_ETH, txq_prod(), sge_txq::txq_sgl, V_LSO_ETH_TYPE, V_LSO_IPHDR_WORDS, V_LSO_MSS, V_LSO_TCPHDR_WORDS, V_TXPKT_INTF, V_TXPKT_OPCODE, V_WR_BCNTLFLT, V_WR_DATATYPE, V_WR_GEN, V_WR_LEN, V_WR_OP, V_WR_SGLSFLT, V_WR_TID, wr_gen2(), work_request_hdr::wr_hi, work_request_hdr::wr_lo, and write_wr_hdr_sgl().

Referenced by cxgb_start_locked(), and cxgb_transmit_locked().

Here is the call graph for this function:

Here is the caller graph for this function:

◆ t3_free_qset()

static void t3_free_qset	(	adapter_t *	sc,
		struct sge_qset *	q
	)

static

free_qset - free the resources of an SGE queue set @sc: the controller owning the queue set @q: the queue set

Release the HW and SW resources associated with an SGE queue set, such as HW contexts, packet buffers, and descriptor rings. Traffic to the queue set must be quiesced prior to calling this.

Definition at line 1995 of file cxgb_sge.c.

References sge_rspq::cntxt_id, sge_fl::cntxt_id, sge_txq::cntxt_id, lro_state::ctrl, sge_rspq::desc, sge_fl::desc, sge_txq::desc, sge_rspq::desc_map, sge_fl::desc_map, sge_txq::desc_map, sge_rspq::desc_tag, sge_fl::desc_tag, sge_txq::desc_tag, sge_fl::entry_tag, sge_txq::entry_tag, sge_qset::fl, free_rx_bufs(), sge_rspq::lock, sge_qset::lock, sge_qset::lro, MTX_DESTROY, reclaim_completed_tx(), sge::reg_lock, sge_qset::rspq, sge_fl::sdesc, sge_txq::sdesc, adapter::sge, SGE_RXQ_PER_SET, SGE_TXQ_PER_SET, t3_sge_disable_fl(), t3_sge_disable_rspcntxt(), t3_sge_enable_ecntxt(), sge_qset::txq, TXQ_ETH, sge_txq::txq_ifq, and sge_txq::txq_mr.

Referenced by t3_free_sge_resources(), and t3_sge_alloc_qset().

Here is the call graph for this function:

Here is the caller graph for this function:

◆ t3_free_sge_resources()

void t3_free_sge_resources	(	adapter_t *	sc,
		int	nqsets
	)

t3_free_sge_resources - free SGE resources @sc: the adapter softc

Frees resources used by the SGE queue sets.

Definition at line 2069 of file cxgb_sge.c.

References nqsets, sge::qs, adapter::sge, t3_free_qset(), and TXQ_LOCK.

Referenced by cxgb_free(), and setup_sge_qsets().

Here is the call graph for this function:

Here is the caller graph for this function:

◆ t3_free_tx_desc()

void t3_free_tx_desc	(	struct sge_qset *	qs,
		int	reclaimable,
		int	queue
	)

t3_free_tx_desc - reclaims Tx descriptors and their buffers @adapter: the adapter @q: the Tx queue to reclaim descriptors from @reclaimable: the number of descriptors to reclaim @m_vec_size: maximum number of buffers to reclaim @desc_reclaimed: returns the number of descriptors reclaimed

Reclaims Tx descriptors from an SGE Tx queue and frees the associated Tx buffers. Called with the Tx queue lock held.

Returns number of buffers of reclaimed

Definition at line 2126 of file cxgb_sge.c.

References sge_txq::cidx, sge_txq::cntxt_id, sge_txq::entry_tag, tx_sw_desc::flags, sge_qset::lock, tx_sw_desc::m, m_freem_list(), tx_sw_desc::map, prefetch, sge_txq::sdesc, sge_txq::size, TX_SW_DESC_MAPPED, sge_qset::txq, and sge_txq::txq_skipped.

Referenced by reclaim_completed_tx().

Here is the call graph for this function:

Here is the caller graph for this function:

◆ t3_get_desc()

int t3_get_desc	(	const struct sge_qset *	qs,
		unsigned int	qnum,
		unsigned int	idx,
		unsigned char *	data
	)

t3_get_desc - dump an SGE descriptor for debugging purposes @qs: the queue set @qnum: identifies the specific queue (0..2: Tx, 3:response, 4..5: Rx) @idx: the descriptor index in the queue @data: where to dump the descriptor contents

Dumps the contents of a HW descriptor of an SGE queue. Returns the size of the descriptor.

Definition at line 3703 of file cxgb_sge.c.

References sge_rspq::desc, sge_fl::desc, sge_txq::desc, sge_qset::fl, sge_qset::rspq, sge_rspq::size, sge_fl::size, sge_txq::size, and sge_qset::txq.

Referenced by cxgb_extension_ioctl().

Here is the caller graph for this function:

◆ t3_intr_msi()

void t3_intr_msi ( void * data )

Definition at line 3070 of file cxgb_sge.c.

References A_PL_INT_ENABLE0, for_each_port, sge_rspq::lock, process_responses_gts(), sge::qs, sge_qset::rspq, adapter::sge, adapter::slow_intr_task, t3_read_reg(), t3_write_reg(), and adapter::tq.

Referenced by cxgb_controller_attach().

Here is the call graph for this function:

Here is the caller graph for this function:

◆ t3_intr_msix()

void t3_intr_msix ( void * data )

Definition at line 3090 of file cxgb_sge.c.

References sge_qset::adap, port_info::adapter, sge_qset::port, process_responses_gts(), sge_qset::rspq, and sge_rspq::unhandled_irqs.

Referenced by cxgb_setup_interrupts().

Here is the call graph for this function:

Here is the caller graph for this function:

◆ t3_mgmt_tx()

int t3_mgmt_tx	(	struct adapter *	adap,
		struct mbuf *	m
	)

Definition at line 1980 of file cxgb_sge.c.

References ctrl_xmit(), sge::qs, and adapter::sge.

Referenced by init_tp_parity(), send_pktsched_cmd(), and set_filter().

Here is the call graph for this function:

Here is the caller graph for this function:

◆ t3_pkt_timestamp()

static int t3_pkt_timestamp ( SYSCTL_HANDLER_ARGS )

static

Definition at line 3334 of file cxgb_sge.c.

References A_TP_PC_CONFIG2, F_ENABLERXPKTTMSTPRSS, adapter::flags, FULL_INIT_DONE, t3_set_reg_field(), and adapter::timestamp.

Referenced by t3_add_configured_sysctls().

Here is the call graph for this function:

Here is the caller graph for this function:

◆ t3_rx_eth()

void t3_rx_eth	(	struct adapter *	adap,
		struct mbuf *	m,
		int	ethpad
	)

Definition at line 2623 of file cxgb_sge.c.

References cpl_rx_pkt::csum, cpl_rx_pkt::csum_valid, cpl_rx_pkt::fragment, cpl_rx_pkt::iff, ifp, port_info::ifp, adapter::port, adapter::rxpkt_map, cpl_rx_pkt::vlan, and cpl_rx_pkt::vlan_valid.

Referenced by process_responses().

Here is the caller graph for this function:

◆ t3_set_coalesce_usecs()

static int t3_set_coalesce_usecs ( SYSCTL_HANDLER_ARGS )

static

Definition at line 3288 of file cxgb_sge.c.

References A_SG_GTS, sge_rspq::cntxt_id, qset_params::coalesce_usecs, adapter::flags, FULL_INIT_DONE, sge_rspq::holdoff_tmr, lock, sge_rspq::lock, adapter_params::nports, nqsets, port_info::nqsets, adapter::params, adapter::port, sge::qs, sge_params::qset, sge_qset::rspq, adapter_params::sge, adapter::sge, t3_update_qset_coalesce(), t3_write_reg(), USING_MSIX, V_NEWTIMER, and V_RSPQ.

Referenced by t3_add_configured_sysctls().

Here is the call graph for this function:

Here is the caller graph for this function:

◆ t3_sge_alloc()

int t3_sge_alloc ( adapter_t * sc )

Definition at line 606 of file cxgb_sge.c.

References adapter::dev, adapter::parent_dmat, adapter::rx_dmat, adapter::rx_jumbo_dmat, adapter::tx_dmat, TX_MAX_SEGS, and TX_MAX_SIZE.

Referenced by setup_sge_qsets().

Here is the caller graph for this function:

◆ t3_sge_alloc_qset()

int t3_sge_alloc_qset	(	adapter_t *	sc,
		u_int	id,
		int	nports,
		int	irq_vec_idx,
		const struct qset_params *	p,
		int	ntxq,
		struct port_info *	pi
	)

t3_sge_alloc_qset - initialize an SGE queue set @sc: the controller softc @id: the queue set id @nports: how many Ethernet ports will be using this queue set @irq_vec_idx: the IRQ vector index for response queue interrupts : configuration parameters for this queue set @ntxq: number of Tx queues for the queue set @pi: port info for queue set

Allocate resources and initialize an SGE queue set. A queue set comprises a response queue, two Rx free-buffer queues, and up to 3 Tx queues. The Tx queues are assigned roles in the order Ethernet queue, offload queue, and control queue.

Definition at line 2420 of file cxgb_sge.c.

References A_SG_GTS, sge_qset::adap, alloc_ring(), sge_fl::buf_size, sge_rspq::cidx, sge_rspq::cntxt_id, sge_fl::cntxt_id, sge_txq::cntxt_id, qset_params::cong_thres, lro_state::ctrl, cxgb_txq_buf_ring_size, sge_rspq::desc, sge_fl::desc, sge_txq::desc, sge_rspq::desc_map, sge_fl::desc_map, sge_txq::desc_map, sge_rspq::desc_tag, sge_fl::desc_tag, sge_txq::desc_tag, adapter::dev, lro_state::enabled, sge_fl::entry_tag, sge_txq::entry_tag, sge_qset::fl, qset_params::fl_size, flits_to_desc(), sge_rspq::gen, sge_fl::gen, sge_txq::gen, sge_rspq::holdoff_tmr, sge_qset::idx, port_info::ifp, init_qset_cntxt(), qset_params::jumbo_buf_size, qset_params::jumbo_size, sge_rspq::lock, sge_qset::lock, sge_rspq::lockbuf, sge_qset::lro, MTX_INIT, sge_qset::namebuf, sge_rspq::phys_addr, sge_fl::phys_addr, sge_txq::phys_addr, sge_qset::port, sge_txq::qreclaim_task, sge_txq::qresume_task, sge::qs, refill_fl(), refill_rspq(), sge::reg_lock, restart_ctrlq(), sge_qset::rspq, RSPQ_NAME_LEN, qset_params::rspq_size, adapter::rx_dmat, adapter::rx_jumbo_dmat, sge_fl::sdesc, sge_txq::sdesc, sge_txq::sendq, adapter::sge, SGE_CNTXT_CTRL, SGE_CNTXT_ETH, SGE_CNTXT_OFLD, SGE_RXQ_PER_SET, sge_txq_reclaim_handler(), sgl_len(), sge_rspq::size, sge_fl::size, sge_txq::size, sge_txq::stop_thres, t3_free_qset(), t3_sge_init_ecntxt(), t3_sge_init_flcntxt(), t3_sge_init_rspcntxt(), t3_update_qset_coalesce(), t3_write_reg(), sge_txq::token, adapter::tx_dmat, TX_MAX_SEGS, sge_qset::txq, TXQ_CTRL, TXQ_ETH, sge_txq::txq_ifq, TXQ_LOCK, sge_txq::txq_mr, TXQ_OFLD, qset_params::txq_size, sge_txq::txq_timer, sge_txq::txq_watchdog, sge_fl::type, USE_GTS, V_NEWTIMER, V_RSPQ, and sge_fl::zone.

Here is the call graph for this function:

◆ t3_sge_err_intr_handler()

void t3_sge_err_intr_handler ( adapter_t * adapter )

t3_sge_err_intr_handler - SGE async event interrupt handler @adapter: the adapter

Interrupt handler for SGE asynchronous (non-data) events.

Definition at line 522 of file cxgb_sge.c.

References A_SG_INT_CAUSE, A_SG_RSPQ_FL_STATUS, CH_ALERT, F_RSPQCREDITOVERFOW, F_RSPQDISABLED, S_RSPQ0DISABLED, SGE_FATALERR, SGE_FRAMINGERR, SGE_PARERR, t3_fatal_err(), t3_read_reg(), and t3_write_reg().

Referenced by t3_slow_intr_handler().

Here is the call graph for this function:

Here is the caller graph for this function:

◆ t3_sge_free()

int t3_sge_free ( struct adapter * sc )

Definition at line 660 of file cxgb_sge.c.

References adapter::parent_dmat, adapter::rx_dmat, adapter::rx_jumbo_dmat, and adapter::tx_dmat.

Referenced by cxgb_free().

Here is the caller graph for this function:

◆ t3_sge_init()

void t3_sge_init	(	adapter_t *	adap,
		struct sge_params *	p
	)

t3_sge_init - initialize SGE @adap: the adapter : the SGE parameters

Performs SGE initialization needed every time after a chip reset. We do not initialize any of the queue sets here, instead the driver top-level must request those individually. We also do not enable DMA here, that should be done after the queues have been set up.

Definition at line 430 of file cxgb_sge.c.

References A_SG_CMDQ_CREDIT_TH, A_SG_CONTROL, A_SG_DRB_PRI_THRESH, A_SG_EGR_RCQ_DRB_THRSH, A_SG_HI_DRB_HI_THRSH, A_SG_HI_DRB_LO_THRSH, A_SG_LO_DRB_HI_THRSH, A_SG_LO_DRB_LO_THRSH, A_SG_OCO_BASE, A_SG_TIMER_TICK, core_ticks_per_usec(), F_AVOIDCQOVFL, F_BIGENDIANINGRESS, F_CONGMODE, F_CQCRDTCTRL, F_DROPPKT, F_EGRGENCTRL, F_FATLPERREN, F_FLMODE, F_ISCSICOALESCING, F_ONEINTMULTQ, F_OPTONEINTMULTQ, F_TNLFLMODE, adapter::flags, adapter::params, adapter_params::rev, T3_REV_C, t3_write_reg(), USING_MSI, USING_MSIX, V_BASE1, V_HIRCQDRBTHRSH, V_HOSTPAGESIZE, V_LORCQDRBTHRSH, V_PKTSHIFT, V_THRESHOLD, V_TIMEOUT, and V_USERSPACESIZE.

Referenced by t3_init_hw().

Here is the call graph for this function:

Here is the caller graph for this function:

◆ t3_sge_init_adapter()

int t3_sge_init_adapter ( adapter_t * sc )

Definition at line 1014 of file cxgb_sge.c.

References sge_slow_intr_handler(), sge_timer_cb(), adapter::sge_timer_ch, adapter::slow_intr_task, and TX_RECLAIM_PERIOD.

Referenced by cxgb_controller_attach().

Here is the call graph for this function:

Here is the caller graph for this function:

◆ t3_sge_init_port()

int t3_sge_init_port ( struct port_info * pi )

Definition at line 1030 of file cxgb_sge.c.

References sge_timer_reclaim(), and port_info::timer_reclaim_task.

Referenced by cxgb_port_attach().

Here is the call graph for this function:

Here is the caller graph for this function:

◆ t3_sge_prep()

void t3_sge_prep	(	adapter_t *	adap,
		struct sge_params *	p
	)

Definition at line 550 of file cxgb_sge.c.

References qset_params::coalesce_usecs, qset_params::cong_thres, cxgb_use_16k_clusters, adapter::dev, FL_Q_SIZE, qset_params::fl_size, is_offload(), qset_params::jumbo_buf_size, JUMBO_Q_SIZE, qset_params::jumbo_size, sge_params::max_pkt_size, nmbjumbo16, nmbjumbo9, adapter_params::nports, nqsets, adapter::params, qset_params::polling, sge_params::qset, RSPQ_Q_SIZE, qset_params::rspq_size, SGE_QSETS, TX_CTRL_Q_SIZE, TX_ETH_Q_SIZE, TX_OFLD_Q_SIZE, TXQ_CTRL, TXQ_ETH, TXQ_OFLD, and qset_params::txq_size.

Referenced by t3_prep_adapter().

Here is the call graph for this function:

Here is the caller graph for this function:

◆ t3_sge_reset_adapter()

int t3_sge_reset_adapter ( adapter_t * sc )

Definition at line 1023 of file cxgb_sge.c.

References sge_timer_cb(), adapter::sge_timer_ch, and TX_RECLAIM_PERIOD.

Referenced by cxgb_up().

Here is the call graph for this function:

Here is the caller graph for this function:

◆ t3_sge_start()

void t3_sge_start ( adapter_t * sc )

t3_sge_start - enable SGE @sc: the controller softc

Enables the SGE for DMAs. This is the last step in starting packet transfers.

Definition at line 2087 of file cxgb_sge.c.

References A_SG_CONTROL, F_GLOBALENABLE, and t3_set_reg_field().

Referenced by cxgb_up().

Here is the call graph for this function:

Here is the caller graph for this function:

◆ t3_sge_stop()

void t3_sge_stop ( adapter_t * sc )

t3_sge_stop - disable SGE operation @sc: the adapter

Disables the DMA engine. This can be called in emeregencies (e.g., from error interrupts) or from normal process context. In the latter case it also disables any pending queue restart tasklets. Note that if it is called in interrupt context it cannot disable the restart tasklets as it cannot wait, however the tasklets will have no effect since the doorbells are disabled and the driver will call this again later from process context, at which time the tasklets will be stopped if they are still running.

Definition at line 2106 of file cxgb_sge.c.

References A_SG_CONTROL, F_GLOBALENABLE, and t3_set_reg_field().

Referenced by cxgb_down(), and t3_fatal_err().

Here is the call graph for this function:

Here is the caller graph for this function:

◆ t3_update_qset_coalesce()

void t3_update_qset_coalesce	(	struct sge_qset *	qs,
		const struct qset_params *	p
	)

Definition at line 679 of file cxgb_sge.c.

References qset_params::coalesce_usecs, sge_rspq::holdoff_tmr, sge_rspq::polling, and sge_qset::rspq.

Referenced by t3_set_coalesce_usecs(), and t3_sge_alloc_qset().

Here is the caller graph for this function:

◆ t3b_intr()

void t3b_intr ( void * data )

Definition at line 3038 of file cxgb_sge.c.

References A_PL_CLI, A_PL_INT_ENABLE0, A_SG_DATA_INTR, F_ERRINTR, for_each_port, sge_rspq::lock, process_responses_gts(), sge::qs, sge_qset::rspq, adapter::sge, adapter::slow_intr_task, t3_read_reg(), t3_write_reg(), and adapter::tq.

Referenced by cxgb_controller_attach().

Here is the call graph for this function:

Here is the caller graph for this function:

◆ txq_prod()

static void txq_prod	(	struct sge_txq *	txq,
		unsigned int	ndesc,
		struct txq_state *	txqs
	)

static

Definition at line 1133 of file cxgb_sge.c.

References sge_txq::cidx, txq_state::compl, sge_txq::gen, txq_state::gen, sge_txq::in_use, sge_txq::pidx, txq_state::pidx, S_WR_COMPL, sge_txq::size, and sge_txq::unacked.

Referenced by t3_encap().

Here is the caller graph for this function:

◆ wr_gen2()

static __inline void wr_gen2	(	struct tx_desc *	d,
		unsigned int	gen
	)

static

Definition at line 1259 of file cxgb_sge.c.

References tx_desc::flit, and TX_DESC_FLITS.

Referenced by t3_encap(), write_imm(), and write_wr_hdr_sgl().

Here is the caller graph for this function:

◆ write_imm()

static __inline void write_imm	(	struct tx_desc *	d,
		caddr_t	src,
		unsigned int	len,
		unsigned int	gen
	)

static

write_imm - write a packet into a Tx descriptor as immediate data @d: the Tx descriptor to write @m: the packet @len: the length of packet data to write as immediate data @gen: the generation bit value to write

Writes a packet as immediate data into a Tx descriptor. The packet contains a work request at its beginning. We must write the packet carefully so the SGE doesn't read accidentally before it's written in its entirety.

Definition at line 1799 of file cxgb_sge.c.

References F_WR_EOP, F_WR_SOP, set_wr_hdr(), V_WR_BCNTLFLT, V_WR_GEN, V_WR_LEN, wr_gen2(), work_request_hdr::wr_hi, and work_request_hdr::wr_lo.

Referenced by ctrl_xmit(), and restart_ctrlq().

Here is the call graph for this function:

Here is the caller graph for this function:

◆ write_wr_hdr_sgl()

static void write_wr_hdr_sgl	(	unsigned int	ndesc,
		struct tx_desc *	txd,
		struct txq_state *	txqs,
		const struct sge_txq *	txq,
		const struct sg_ent *	sgl,
		unsigned int	flits,
		unsigned int	sgl_flits,
		unsigned int	wr_hi,
		unsigned int	wr_lo
	)

static

write_wr_hdr_sgl - write a WR header and, optionally, SGL @ndesc: number of Tx descriptors spanned by the SGL @txd: first Tx descriptor to be written @txqs: txq state (generation and producer index) @txq: the SGE Tx queue @sgl: the SGL @flits: number of flits to the start of the SGL in the first descriptor @sgl_flits: the SGL size in flits @wr_hi: top 32 bits of WR header based on WR type (big endian) @wr_lo: low 32 bits of WR header based on WR type (big endian)

Write a work request header and an associated SGL. If the SGL is small enough to fit into one Tx descriptor it has already been written and we just need to write the WR header. Otherwise we distribute the SGL across the number of descriptors it spans.

Definition at line 1284 of file cxgb_sge.c.

References sge_txq::desc, F_WR_EOP, F_WR_SOP, tx_desc::flit, txq_state::gen, txq_state::pidx, sge_txq::sdesc, set_wr_hdr(), sge_txq::size, V_WR_DATATYPE, V_WR_GEN, V_WR_LEN, V_WR_SGLSFLT, WR_FLITS, wr_gen2(), work_request_hdr::wr_hi, and work_request_hdr::wr_lo.

Referenced by t3_encap().

Here is the call graph for this function:

Here is the caller graph for this function:

Variable Documentation

◆ __packed

struct rsp_desc __packed

◆ cxgb_debug

int cxgb_debug = 0

Definition at line 243 of file cxgb_sge.c.

Referenced by busdma_map_sg_collapse(), cxgb_extension_ioctl(), get_packet(), process_responses(), process_responses_gts(), t3_add_attach_sysctls(), t3_encap(), and t3_os_set_hw_addr().

◆ cxgb_tx_coalesce_enable_start

int cxgb_tx_coalesce_enable_start = COALESCE_START_DEFAULT

static

Definition at line 106 of file cxgb_sge.c.

Referenced by check_pkt_coalesce(), and cxgb_tx_watchdog().

◆ cxgb_tx_coalesce_enable_stop

int cxgb_tx_coalesce_enable_stop = COALESCE_STOP_DEFAULT

static

Definition at line 110 of file cxgb_sge.c.

Referenced by check_pkt_coalesce(), and cxgb_tx_watchdog().

◆ cxgb_tx_coalesce_force

int cxgb_tx_coalesce_force = 0

static

Definition at line 92 of file cxgb_sge.c.

Referenced by check_pkt_coalesce().

◆ cxgb_tx_reclaim_threshold

int cxgb_tx_reclaim_threshold = TX_RECLAIM_DEFAULT

static

Definition at line 114 of file cxgb_sge.c.

Referenced by cxgb_start_locked(), cxgb_transmit_locked(), and reclaim_completed_tx().

◆ cxgb_txq_buf_ring_size

int cxgb_txq_buf_ring_size = TX_ETH_Q_SIZE

Definition at line 88 of file cxgb_sge.c.

Referenced by t3_sge_alloc_qset().

◆ cxgb_use_16k_clusters

int cxgb_use_16k_clusters

extern

Referenced by t3_sge_prep().

◆ flit_desc_map

uint8_t flit_desc_map[]

static

Initial value:

= {
        0,
 
}

Definition at line 213 of file cxgb_sge.c.

Referenced by flits_to_desc().

◆ multiq_tx_enable

int multiq_tx_enable = 1

Definition at line 81 of file cxgb_sge.c.

◆ nmbjumbo16

int nmbjumbo16

extern

Referenced by t3_sge_prep().

◆ nmbjumbo9

int nmbjumbo9

extern

Referenced by t3_sge_prep().

◆ nmbjumbop

int nmbjumbop

extern

◆ recycle_enable

int recycle_enable = 0

static

Definition at line 123 of file cxgb_sge.c.

Referenced by get_packet().

◆ rspq_name

const char* rspq_name = "rspq"

static

Definition at line 3397 of file cxgb_sge.c.

Referenced by t3_add_configured_sysctls().

◆ sysctl__hw_cxgb_children

struct sysctl_oid_list sysctl__hw_cxgb_children

extern

◆ txq_fills

int txq_fills = 0

Definition at line 80 of file cxgb_sge.c.

Referenced by t3_add_attach_sysctls().

◆ txq_names

const char* txq_names[]

static

Initial value:

=
{
        "txq_eth",
        "txq_ofld",
        "txq_ctrl"      
}

Definition at line 3398 of file cxgb_sge.c.

Referenced by t3_add_configured_sysctls().

Data Structures

Macros

Enumerations

Functions

Variables

Macro Definition Documentation

◆ COALESCE_START_DEFAULT

◆ COALESCE_START_MAX

◆ COALESCE_STOP_DEFAULT

◆ COALESCE_STOP_MIN

◆ CXGB_SYSCTL_ADD_QUAD

◆ CXGB_SYSCTL_ADD_ULONG

◆ GET_VTAG

◆ NOMEM_INTR_DELAY

◆ QDUMP_SBUF_SIZE

◆ RSPD_CTRL_MASK

◆ RSPD_GTS_MASK

◆ RSPQ_EOP

◆ RSPQ_NSOP_NEOP

◆ RSPQ_SOP

◆ RSPQ_SOP_EOP

◆ RX_SW_DESC_INUSE

◆ RX_SW_DESC_MAP_CREATED

◆ SGE_FATALERR

◆ SGE_FRAMINGERR

◆ SGE_PARERR

◆ SGE_RX_COPY_THRES

◆ SGE_RX_DROP_THRES

◆ SGE_RX_SM_BUF_SIZE

◆ TCPPKTHDRSIZE

◆ TX_RECLAIM_DEFAULT

◆ TX_RECLAIM_MAX

◆ TX_RECLAIM_MIN

◆ TX_RECLAIM_PERIOD

◆ TX_SW_DESC_MAP_CREATED

◆ TX_SW_DESC_MAPPED

◆ TXQ_LOCK

◆ TXQ_LOCK_ASSERT

◆ TXQ_RING_DEQUEUE

◆ TXQ_RING_DEQUEUE_COND

◆ TXQ_RING_EMPTY

◆ TXQ_RING_FLUSH

◆ TXQ_RING_NEEDS_ENQUEUE

◆ TXQ_TRYLOCK

◆ TXQ_UNLOCK

◆ USE_GTS

Enumeration Type Documentation

◆ anonymous enum

Function Documentation

◆ __FBSDID()

◆ __refill_fl()

◆ __refill_fl_lt()

◆ alloc_ring()

◆ alloc_ring_cb()

◆ calc_tx_descs()

◆ check_desc_avail()

◆ check_pkt_coalesce()

◆ check_ring_db()

◆ check_ring_tx_db()

◆ coalesce_check()

◆ ctrl_xmit()

◆ cxgb_dequeue()

◆ cxgb_qflush()

◆ cxgb_start_locked()

◆ cxgb_transmit()

◆ cxgb_transmit_locked()

◆ cxgb_tx_timeout()

◆ cxgb_tx_watchdog()

◆ flits_to_desc()

◆ free_rx_bufs()

◆ get_imm_packet()

◆ get_packet()

◆ handle_rsp_cntrl_info()

◆ init_qset_cntxt()

◆ is_new_response()

◆ make_sgl()

◆ process_responses()

◆ process_responses_gts()

◆ reclaim_completed_tx()

◆ reclaim_completed_tx_imm()