37#include <sys/eventhandler.h>
40#include <sys/module.h>
41#include <sys/selinfo.h>
42#include <sys/socket.h>
43#include <sys/sockio.h>
44#include <machine/bus.h>
45#include <net/ethernet.h>
47#include <net/if_media.h>
48#include <net/if_var.h>
49#include <net/if_clone.h>
50#include <net/if_types.h>
51#include <net/netmap.h>
52#include <dev/netmap/netmap_kern.h>
66SYSCTL_INT(_hw_cxgbe, OID_AUTO, nm_black_hole, CTLFLAG_RWTUN, &black_hole, 0,
67 "Sink incoming packets.");
70SYSCTL_INT(_hw_cxgbe, OID_AUTO, nm_rx_ndesc, CTLFLAG_RWTUN,
71 &rx_ndesc, 0,
"# of rx descriptors after which the hw cidx is updated.");
74SYSCTL_INT(_hw_cxgbe, OID_AUTO, nm_rx_nframes, CTLFLAG_RWTUN,
75 &rx_nframes, 0,
"max # of frames received before waking up netmap rx.");
77int holdoff_tmr_idx = 2;
78SYSCTL_INT(_hw_cxgbe, OID_AUTO, nm_holdoff_tmr_idx, CTLFLAG_RWTUN,
79 &holdoff_tmr_idx, 0,
"Holdoff timer index for netmap rx queues.");
87static int nm_cong_drop = 1;
88SYSCTL_INT(_hw_cxgbe, OID_AUTO, nm_cong_drop, CTLFLAG_RWTUN,
90 "Congestion control for netmap rx queues (0 = backpressure, 1 = drop");
93SYSCTL_INT(_hw_cxgbe, OID_AUTO, starve_fl, CTLFLAG_RWTUN,
94 &starve_fl, 0,
"Don't ring fl db for netmap rx queues.");
100int lazy_tx_credit_flush = 1;
101SYSCTL_INT(_hw_cxgbe, OID_AUTO, lazy_tx_credit_flush, CTLFLAG_RWTUN,
102 &lazy_tx_credit_flush, 0,
"lazy credit flush for netmap tx queues.");
109static int nm_split_rss = 0;
110SYSCTL_INT(_hw_cxgbe, OID_AUTO, nm_split_rss, CTLFLAG_RWTUN,
111 &nm_split_rss, 0,
"Split the netmap rx queues into two groups.");
119static int nm_txcsum = 0;
120SYSCTL_INT(_hw_cxgbe, OID_AUTO, nm_txcsum, CTLFLAG_RWTUN,
121 &nm_txcsum, 0,
"Enable transmit checksum offloading.");
131 struct sysctl_oid *oid;
132 struct sysctl_oid_list *children;
133 struct sysctl_ctx_list *ctx;
137 struct netmap_adapter *na = NA(vi->
ifp);
167 snprintf(name,
sizeof(name),
"%d", idx);
168 oid = SYSCTL_ADD_NODE(ctx, children, OID_AUTO, name,
169 CTLFLAG_RD | CTLFLAG_MPSAFE, NULL,
"rx queue");
170 children = SYSCTL_CHILDREN(oid);
172 SYSCTL_ADD_U16(ctx, children, OID_AUTO,
"abs_id", CTLFLAG_RD,
174 SYSCTL_ADD_U16(ctx, children, OID_AUTO,
"cntxt_id", CTLFLAG_RD,
176 SYSCTL_ADD_U16(ctx, children, OID_AUTO,
"cidx", CTLFLAG_RD,
179 children = SYSCTL_CHILDREN(oid);
180 oid = SYSCTL_ADD_NODE(ctx, children, OID_AUTO,
"fl",
181 CTLFLAG_RD | CTLFLAG_MPSAFE, NULL,
"freelist");
182 children = SYSCTL_CHILDREN(oid);
184 SYSCTL_ADD_U16(ctx, children, OID_AUTO,
"cntxt_id", CTLFLAG_RD,
186 SYSCTL_ADD_UINT(ctx, children, OID_AUTO,
"cidx", CTLFLAG_RD,
187 &
nm_rxq->fl_cidx, 0,
"consumer index");
188 SYSCTL_ADD_UINT(ctx, children, OID_AUTO,
"pidx", CTLFLAG_RD,
203 free_nm_rxq_hwq(vi,
nm_rxq);
221 struct netmap_adapter *na = NA(vi->
ifp);
223 struct sysctl_oid *oid;
224 struct sysctl_oid_list *children = SYSCTL_CHILDREN(vi->
nm_txq_oid);
228 &nm_txq->
ba, (
void **)&nm_txq->
desc);
233 nm_txq->
sidx = na->num_tx_desc;
245 snprintf(name,
sizeof(name),
"%d", idx);
246 oid = SYSCTL_ADD_NODE(&vi->
ctx, children, OID_AUTO, name,
247 CTLFLAG_RD | CTLFLAG_MPSAFE, NULL,
"netmap tx queue");
248 children = SYSCTL_CHILDREN(oid);
250 SYSCTL_ADD_UINT(&vi->
ctx, children, OID_AUTO,
"cntxt_id", CTLFLAG_RD,
251 &nm_txq->
cntxt_id, 0,
"SGE context id of the queue");
252 SYSCTL_ADD_U16(&vi->
ctx, children, OID_AUTO,
"cidx", CTLFLAG_RD,
253 &nm_txq->
cidx, 0,
"consumer index");
254 SYSCTL_ADD_U16(&vi->
ctx, children, OID_AUTO,
"pidx", CTLFLAG_RD,
255 &nm_txq->
pidx, 0,
"producer index");
269 free_nm_txq_hwq(vi, nm_txq);
285 struct netmap_adapter *na = NA(vi->
ifp);
290 MPASS(
nm_rxq->fl_desc != NULL);
295 bzero(&c,
sizeof(c));
305 c.fl1id = htobe16(0xffff);
312 c.type_to_iqandstindex = htobe32(v |
327 c.iqns_to_fl0congen |=
332 c.fl0dcaen_to_fl0cidxfthresh =
342 device_printf(sc->
dev,
343 "failed to create netmap ingress queue: %d\n", rc);
354 panic (
"%s: nm_rxq->iq_cntxt_id (%d) more than the max (%d)",
367 panic(
"%s: nm_rxq->fl_cntxt_id (%d) more than the max (%d)",
385 for (i = 0; i < 4; i++) {
387 val |= 1 << (i << 2);
394 device_printf(sc->
dev,
395 "failed to set congestion manager context for "
416 device_printf(sc->
dev,
"%s: failed for iq %d, fl %d: %d\n",
428 struct netmap_adapter *na = NA(vi->
ifp);
432 MPASS(nm_txq->
desc != NULL);
435 bzero(nm_txq->
desc, len);
437 bzero(&c,
sizeof(c));
457 c.eqaddr = htobe64(nm_txq->
ba);
461 device_printf(vi->
dev,
462 "failed to create netmap egress queue: %d\n", rc);
469 panic(
"%s: nm_txq->cntxt_id (%d) more than the max (%d)", __func__,
474 MPASS(nm_txq->
sidx == na->num_tx_desc);
482 uint32_t mask = (1 << s_qpp) - 1;
483 volatile uint8_t *
udb;
494 nm_txq->
udb = (
volatile void *)
udb;
506 device_printf(vi->
dev,
507 "failed to bind netmap txq %d to class 0xff: %d\n",
524 device_printf(sc->
dev,
"%s: failed for eq %d: %d\n", __func__,
532 struct ifnet *
ifp,
struct netmap_adapter *na)
534 struct netmap_kring *kring;
536 int rc, i, j, nm_state, defq;
545 nm_state = atomic_load_int(&
nm_rxq->nm_state);
547 if ((nm_state !=
NM_OFF && !nm_kring_pending_off(kring)) ||
548 (nm_state ==
NM_OFF && nm_kring_pending_on(kring))) {
564 nm_state = atomic_load_int(&
nm_rxq->nm_state);
566 if ((nm_state !=
NM_OFF &&
567 !nm_kring_pending_off(kring)) ||
569 nm_kring_pending_on(kring))) {
584 if_printf(
ifp,
"netmap rss_config failed: %d\n", rc);
588 if_printf(
ifp,
"netmap defaultq config failed: %d\n", rc);
600 struct ifnet *
ifp,
struct netmap_adapter *na)
602 struct netmap_kring *kring;
604 int rc, i, j, nm_state, defq;
605 int nactive[2] = {0, 0};
606 int dq[2] = {-1, -1};
609 MPASS(nm_split_rss != 0);
614 nm_state = atomic_load_int(&
nm_rxq->nm_state);
616 if ((nm_state !=
NM_OFF && !nm_kring_pending_off(kring)) ||
617 (nm_state ==
NM_OFF && nm_kring_pending_on(kring))) {
627 if (nactive[0] == 0 || nactive[1] == 0)
628 return (cxgbe_netmap_simple_rss(sc,
vi,
ifp, na));
630 MPASS(dq[0] != -1 && dq[1] != -1);
631 if (nactive[0] > nactive[1]) {
634 }
else if (nactive[0] < nactive[1]) {
644 while (i < vi->rss_size / 2) {
645 for (j = 0; j < (
vi->
nnmrxq + 1) / 2; j++) {
646 nm_state = atomic_load_int(&
nm_rxq[j].nm_state);
648 if ((nm_state ==
NM_OFF &&
649 !nm_kring_pending_on(kring)) ||
650 (nm_state ==
NM_ON &&
651 nm_kring_pending_off(kring))) {
662 while (i < vi->rss_size) {
663 for (j = (
vi->
nnmrxq + 1) / 2; j < vi->nnmrxq; j++) {
664 nm_state = atomic_load_int(&
nm_rxq[j].nm_state);
666 if ((nm_state ==
NM_OFF &&
667 !nm_kring_pending_on(kring)) ||
668 (nm_state ==
NM_ON &&
669 nm_kring_pending_off(kring))) {
684 if_printf(
ifp,
"netmap split_rss_config failed: %d\n", rc);
688 if_printf(
ifp,
"netmap defaultq config failed: %d\n", rc);
695 struct netmap_adapter *na)
698 if (nm_split_rss == 0 ||
vi->
nnmrxq == 1)
699 return (cxgbe_netmap_simple_rss(sc,
vi,
ifp, na));
701 return (cxgbe_netmap_split_rss(sc,
vi,
ifp, na));
706 struct netmap_adapter *na)
708 struct netmap_slot *slot;
709 struct netmap_kring *kring;
720 (
ifp->if_drv_flags & IFF_DRV_RUNNING) == 0) {
721 if_printf(
ifp,
"cannot enable netmap operation because "
722 "interface is not UP.\n");
728 if (rxb->
size1 == NETMAP_BUF_SIZE(na)) {
732 if (rxb->
size2 == NETMAP_BUF_SIZE(na)) {
738 if_printf(
ifp,
"no hwidx for netmap buffer size %d.\n",
739 NETMAP_BUF_SIZE(na));
744 nm_set_native_flags(na);
748 if (!nm_kring_pending_on(kring))
753 slot = netmap_reset(na, NR_RX, i, 0);
757 MPASS((na->num_rx_desc & 7) == 0);
762 PNMB(na, &slot[j], &
ba);
764 nm_rxq->fl_desc[j] = htobe64(
ba | hwidx);
777 kring = na->tx_rings[nm_txq->
nid];
778 if (!nm_kring_pending_on(kring))
781 alloc_nm_txq_hwq(vi, nm_txq);
782 slot = netmap_reset(na, NR_TX, i, 0);
791 return (cxgbe_netmap_rss(sc, vi,
ifp, na));
796 struct netmap_adapter *na)
798 struct netmap_kring *kring;
799 int rc, i, nm_state, nactive;
807 if (!nm_netmap_on(na))
814 rc = cxgbe_netmap_rss(sc,
vi,
ifp, na);
819 kring = na->tx_rings[nm_txq->
nid];
820 if (!nm_kring_pending_off(kring))
826 device_printf(
vi->
dev,
827 "failed to stop nm_txq[%d]: %d.\n", i, rc);
832 kring->rhead = kring->rcur = kring->nr_hwcur = 0;
833 kring->rtail = kring->nr_hwtail = kring->nkr_num_slots - 1;
837 nm_state = atomic_load_int(&
nm_rxq->nm_state);
839 if (nm_state !=
NM_OFF && !nm_kring_pending_off(kring))
841 if (!nm_kring_pending_off(kring))
843 MPASS(nm_state !=
NM_OFF);
849 device_printf(
vi->
dev,
850 "failed to stop nm_rxq[%d]: %d.\n", i, rc);
858 kring->rhead = kring->rcur = kring->nr_hwcur = 0;
859 kring->rtail = kring->nr_hwtail = 0;
861 netmap_krings_mode_commit(na, 0);
863 nm_clear_native_flags(na);
869cxgbe_netmap_reg(
struct netmap_adapter *na,
int on)
871 struct ifnet *
ifp = na->ifp;
880 rc = cxgbe_netmap_on(sc, vi,
ifp, na);
882 rc = cxgbe_netmap_off(sc, vi,
ifp, na);
890ndesc_to_npkt(
const int n)
897#define MAX_NPKT_IN_TYPE1_WR (ndesc_to_npkt(SGE_MAX_WR_NDESC))
904npkt_to_ndesc(
const int n)
907 MPASS(n > 0 && n <= MAX_NPKT_IN_TYPE1_WR);
909 return ((n + 2) / 2);
917npkt_to_len16(
const int n)
920 MPASS(n > 0 && n <= MAX_NPKT_IN_TYPE1_WR);
925#define NMIDXDIFF(q, idx) IDXDIFF((q)->pidx, (q)->idx, (q)->sidx)
935 n = NMIDXDIFF(nm_txq,
dbidx);
940 switch (ffs(db) - 1) {
946 volatile uint64_t *dst, *src;
953 KASSERT(nm_txq->
udb_qid == 0 && n == 1,
954 (
"%s: inappropriate doorbell (0x%x, %d, %d) for nm_txq %p",
957 dst = (
volatile void *)((uintptr_t)nm_txq->
udb +
959 src = (
void *)&nm_txq->
desc[nm_txq->
dbidx];
960 while (src != (
void *)&nm_txq->
desc[nm_txq->
dbidx + 1])
985 struct netmap_kring *kring,
int npkt,
int npkt_remaining)
987 struct netmap_ring *ring = kring->ring;
988 struct netmap_slot *slot;
989 const u_int lim = kring->nkr_num_slots - 1;
998 n = min(
npkt, MAX_NPKT_IN_TYPE1_WR);
1001 wr = (
void *)&nm_txq->
desc[nm_txq->
pidx];
1007 cpl = (
void *)(wr + 1);
1009 for (i = 0; i < n; i++) {
1010 slot = &ring->slot[kring->nr_hwcur];
1011 PNMB(kring->na, slot, &
ba);
1016 cpl->len = htobe16(slot->len);
1017 cpl->ctrl1 = nm_txcsum ? 0 :
1020 usgl = (
void *)(cpl + 1);
1023 usgl->len0 = htobe32(slot->len);
1024 usgl->addr0 = htobe64(
ba);
1026 slot->flags &= ~(NS_REPORT | NS_BUF_CHANGED);
1027 cpl = (
void *)(usgl + 1);
1028 MPASS(slot->len + len <= UINT16_MAX);
1030 kring->nr_hwcur = nm_next(kring->nr_hwcur, lim);
1032 wr->
plen = htobe16(len);
1035 nm_txq->
pidx += npkt_to_ndesc(n);
1036 MPASS(nm_txq->
pidx <= nm_txq->
sidx);
1037 if (__predict_false(nm_txq->
pidx == nm_txq->
sidx)) {
1046 if (
npkt == 0 && npkt_remaining == 0) {
1048 if (lazy_tx_credit_flush == 0) {
1054 ring_nm_txq_db(sc, nm_txq);
1063 }
else if (NMIDXDIFF(nm_txq,
equeqidx) >= 64) {
1068 ring_nm_txq_db(sc, nm_txq);
1072 MPASS(npkt_remaining);
1077contiguous_ndesc_available(
struct sge_nm_txq *nm_txq)
1081 return (nm_txq->
cidx - nm_txq->
pidx - 1);
1082 else if (nm_txq->
cidx > 0)
1083 return (nm_txq->
sidx - nm_txq->
pidx);
1085 return (nm_txq->
sidx - nm_txq->
pidx - 1);
1092 uint16_t hw_cidx = spg->
cidx;
1096 hw_cidx = be16toh(hw_cidx);
1098 while (nm_txq->
cidx != hw_cidx) {
1099 wr = (
void *)&nm_txq->
desc[nm_txq->
cidx];
1103 MPASS(wr->
type == 1);
1104 MPASS(wr->
npkt > 0 && wr->
npkt <= MAX_NPKT_IN_TYPE1_WR);
1107 nm_txq->
cidx += npkt_to_ndesc(wr->
npkt);
1114 MPASS(nm_txq->
cidx <= nm_txq->
sidx);
1115 if (__predict_false(nm_txq->
cidx == nm_txq->
sidx))
1123cxgbe_netmap_txsync(
struct netmap_kring *kring,
int flags)
1125 struct netmap_adapter *na = kring->na;
1126 struct ifnet *
ifp = na->ifp;
1130 const u_int head = kring->rhead;
1131 u_int reclaimed = 0;
1132 int n, d, npkt_remaining, ndesc_remaining;
1141 npkt_remaining = head >= kring->nr_hwcur ? head - kring->nr_hwcur :
1142 kring->nkr_num_slots - kring->nr_hwcur + head;
1143 while (npkt_remaining) {
1144 reclaimed += reclaim_nm_tx_desc(nm_txq);
1145 ndesc_remaining = contiguous_ndesc_available(nm_txq);
1147 MPASS(ndesc_remaining > 0);
1151 if (npkt_remaining % MAX_NPKT_IN_TYPE1_WR)
1152 d += npkt_to_ndesc(npkt_remaining % MAX_NPKT_IN_TYPE1_WR);
1154 if (d <= ndesc_remaining)
1159 MAX_NPKT_IN_TYPE1_WR;
1165 npkt_remaining -= n;
1166 cxgbe_nm_tx(sc, nm_txq, kring, n, npkt_remaining);
1168 MPASS(npkt_remaining == 0);
1169 MPASS(kring->nr_hwcur == head);
1175 if (reclaimed || flags & NAF_FORCE_RECLAIM || nm_kr_txempty(kring)) {
1176 reclaimed += reclaim_nm_tx_desc(nm_txq);
1177 kring->nr_hwtail += reclaimed;
1178 if (kring->nr_hwtail >= kring->nkr_num_slots)
1179 kring->nr_hwtail -= kring->nkr_num_slots;
1186cxgbe_netmap_rxsync(
struct netmap_kring *kring,
int flags)
1188 struct netmap_adapter *na = kring->na;
1189 struct netmap_ring *ring = kring->ring;
1190 struct ifnet *
ifp = na->ifp;
1194 u_int
const head = kring->rhead;
1196 int force_update = (flags & NAF_FORCE_READ) || kring->nr_kflags & NKR_PENDINTR;
1201 if (netmap_no_pendintr || force_update) {
1202 kring->nr_hwtail = atomic_load_acq_32(&
nm_rxq->fl_cidx);
1203 kring->nr_kflags &= ~NKR_PENDINTR;
1214 n = head >= kring->nr_hwcur ? head - kring->nr_hwcur :
1215 kring->nkr_num_slots - kring->nr_hwcur + head;
1219 struct netmap_slot *slot = &ring->slot[fl_pidx];
1228 MPASS((fl_pidx & 7) == 0);
1229 MPASS((n & 7) == 0);
1231 IDXINCR(kring->nr_hwcur, n, kring->nkr_num_slots);
1235 for (i = 0; i < 8; i++, fl_pidx++, slot++) {
1236 PNMB(na, slot, &
ba);
1238 nm_rxq->fl_desc[fl_pidx] = htobe64(
ba | hwidx);
1239 slot->flags &= ~NS_BUF_CHANGED;
1240 MPASS(fl_pidx <= nm_rxq->fl_sidx2);
1245 slot = &ring->slot[0];
1275cxgbe_nm_attach(
struct vi_info *vi)
1279 struct netmap_adapter na;
1282 MPASS(vi->
ifp != NULL);
1287 bzero(&na,
sizeof(na));
1290 na.na_flags = NAF_BDG_MAYSLEEP;
1301 na.num_rx_desc = rounddown(vi->
qsize_rxq, 8);
1302 na.nm_txsync = cxgbe_netmap_txsync;
1303 na.nm_rxsync = cxgbe_netmap_rxsync;
1304 na.nm_register = cxgbe_netmap_reg;
1305 na.num_tx_rings = vi->
nnmtxq;
1306 na.num_rx_rings = vi->
nnmrxq;
1312cxgbe_nm_detach(
struct vi_info *vi)
1316 MPASS(vi->
ifp != NULL);
1318 netmap_detach(vi->
ifp);
1321static inline const void *
1328 return (&cpl->
data[1]);
1332handle_nm_sge_egr_update(
struct adapter *sc,
struct ifnet *
ifp,
1342 netmap_tx_irq(
ifp, nm_txq->
nid);
1350 struct ifnet *
ifp = vi->
ifp;
1351 struct netmap_adapter *na = NA(
ifp);
1352 struct netmap_kring *kring = na->rx_rings[
nm_rxq->
nid];
1353 struct netmap_ring *ring = kring->ring;
1359 uint32_t fl_cidx = atomic_load_acq_32(&
nm_rxq->fl_cidx);
1360 u_int fl_credits = fl_cidx & 7;
1383 cpl = unwrap_nm_fw6_msg(
cpl);
1386 handle_nm_sge_egr_update(sc,
ifp,
cpl);
1391 ring->slot[fl_cidx].flags = 0;
1394 MPASS(black_hole == 2);
1402 panic(
"%s: unexpected opcode 0x%x on nm_rxq %p",
1403 __func__, opcode,
nm_rxq);
1409 panic(
"%s: netmap queue received interrupt for iq %u\n",
1413 panic(
"%s: illegal response type %d on nm_rxq %p",
1424 if (__predict_false(++nframes == rx_nframes) && !black_hole) {
1425 atomic_store_rel_32(&
nm_rxq->fl_cidx, fl_cidx);
1430 if (__predict_false(++ndesc == rx_ndesc)) {
1431 if (black_hole && fl_credits >= 8) {
1437 fl_credits = fl_cidx & 7;
1447 atomic_store_rel_32(&
nm_rxq->fl_cidx, fl_cidx);
1453 }
else if (nframes > 0)
int tnl_cong(struct port_info *, int)
int begin_synchronized_op(struct adapter *, struct vi_info *, int, char *)
static void t4_write_reg(struct adapter *sc, uint32_t reg, uint32_t val)
struct sge_nm_rxq * nm_rxq
#define INVALID_NM_RXQ_CNTXT_ID
static int forwarding_intr_to_fwq(struct adapter *sc)
#define ASSERT_SYNCHRONIZED_OP(sc)
#define for_each_nm_rxq(vi, iter, q)
#define IDXINCR(idx, incr, wrap)
void end_synchronized_op(struct adapter *, int)
#define INVALID_NM_TXQ_CNTXT_ID
#define for_each_nm_txq(vi, iter, q)
int free_ring(struct adapter *, bus_dma_tag_t, bus_dmamap_t, bus_addr_t, void *)
int alloc_ring(struct adapter *, size_t, bus_dma_tag_t *, bus_dmamap_t *, bus_addr_t *, void **)
int t4_config_rss_range(struct adapter *adapter, int mbox, unsigned int viid, int start, int n, const u16 *rspq, unsigned int nrspq)
int t4_iq_free(struct adapter *adap, unsigned int mbox, unsigned int pf, unsigned int vf, unsigned int iqtype, unsigned int iqid, unsigned int fl0id, unsigned int fl1id)
int t4_eth_eq_stop(struct adapter *adap, unsigned int mbox, unsigned int pf, unsigned int vf, unsigned int eqid)
static int t4_wr_mbox(struct adapter *adap, int mbox, const void *cmd, int size, void *rpl)
int t4_config_vi_rss(struct adapter *adapter, int mbox, unsigned int viid, unsigned int flags, unsigned int defq, unsigned int skeyidx, unsigned int skey)
int t4_iq_stop(struct adapter *adap, unsigned int mbox, unsigned int pf, unsigned int vf, unsigned int iqtype, unsigned int iqid, unsigned int fl0id, unsigned int fl1id)
static int chip_id(struct adapter *adap)
int t4_set_params(struct adapter *adap, unsigned int mbox, unsigned int pf, unsigned int vf, unsigned int nparams, const u32 *params, const u32 *val)
int t4_eth_eq_free(struct adapter *adap, unsigned int mbox, unsigned int pf, unsigned int vf, unsigned int eqid)
volatile uint8_t * udbs_base
const struct chip_params * chip_params
struct adapter_params params
RSS_HDR __be32 opcode_qid
bus_dma_tag_t fl_desc_tag
bus_dma_tag_t iq_desc_tag
struct rx_buf_info rx_buf_info[SW_ZONE_SIZES]
struct sge_nm_rxq * nm_rxq
struct sge_nm_txq * nm_txq
struct sysctl_ctx_list ctx
struct sysctl_oid * nm_rxq_oid
struct sysctl_oid * nm_txq_oid
#define V_QINTR_TIMER_IDX(x)
#define F_TXPKT_L4CSUM_DIS
#define V_TXPKT_VF_VLD(x)
#define V_TXPKT_OPCODE(x)
#define F_TXPKT_IPCSUM_DIS
#define X_FETCHBURSTMIN_64B_T6
#define X_FETCHBURSTMIN_64B
#define X_RSPD_TYPE_FLBUF
#define X_HOSTFCMODE_NONE
#define X_FETCHBURSTMIN_128B
#define X_FETCHBURSTMAX_256B
#define X_UPDATEDELIVERY_INTERRUPT
#define X_TIMERREG_UPDATE_CIDX
#define X_FETCHBURSTMAX_512B
SYSCTL_INT(_hw_cxgbe, OID_AUTO, fl_pktshift, CTLFLAG_RDTUN, &fl_pktshift, 0, "payload DMA offset in rx buffer (bytes)")
#define V_FW_EQ_ETH_CMD_EQID(x)
#define V_FW_EQ_ETH_CMD_PFN(x)
#define V_FW_EQ_ETH_CMD_FBMIN(x)
#define F_FW_IQ_CMD_IQSTART
#define V_FW_EQ_ETH_CMD_PCIECHN(x)
#define F_FW_IQ_CMD_FL0FETCHRO
#define F_FW_EQ_ETH_CMD_AUTOEQUIQE
#define F_FW_IQ_CMD_FL0DATARO
#define V_FW_EQ_ETH_CMD_VFN(x)
#define V_FW_IQ_CMD_IQPCIECH(x)
#define F_FW_IQ_CMD_FL0CONGEN
#define V_FW_IQ_CMD_IQINTCNTTHRESH(x)
#define F_FW_IQ_CMD_ALLOC
#define V_FW_IQ_CMD_VIID(x)
#define V_FW_IQ_CMD_FL0HOSTFCMODE(x)
#define V_FW_PARAMS_PARAM_YZ(x)
#define F_FW_EQ_ETH_CMD_AUTOEQUEQE
#define V_FW_IQ_CMD_FL0CNGCHMAP(x)
#define F_FW_IQ_CMD_FL0CONGCIF
#define F_FW_IQ_CMD_FL0PADEN
#define F_FW_EQ_ETH_CMD_EQSTART
#define V_FW_EQ_ETH_CMD_HOSTFCMODE(x)
#define V_FW_IQ_CMD_IQANDSTINDEX(x)
#define V_FW_IQ_CMD_PFN(x)
#define V_FW_EQ_ETH_CMD_EQSIZE(x)
#define F_FW_IQ_CMD_FL0PACKEN
#define V_FW_IQ_CMD_TYPE(x)
#define V_FW_PARAMS_MNEM(x)
#define FW_VERSION32(MAJOR, MINOR, MICRO, BUILD)
#define V_FW_EQ_ETH_CMD_FBMAX(x)
#define V_FW_IQ_CMD_IQESIZE(x)
#define V_FW_EQ_ETH_CMD_VIID(x)
#define F_FW_IQ_CMD_IQGTSMODE
#define F_FW_IQ_CMD_IQFLINTCONGEN
#define F_FW_EQ_ETH_CMD_FETCHRO
#define V_FW_PARAMS_PARAM_X(x)
@ FW_PARAMS_PARAM_DMAQ_CONM_CTXT
@ FW_PARAMS_PARAM_DMAQ_EQ_SCHEDCLASS_ETH
#define F_FW_EQ_ETH_CMD_ALLOC
#define G_FW_EQ_ETH_CMD_EQID(x)
#define V_FW_EQ_ETH_CMD_IQID(x)
#define V_FW_IQ_CMD_VFN(x)
#define V_FW_IQ_CMD_FL0FBMAX(x)
#define V_FW_IQ_CMD_FL0FBMIN(x)
#define V_FW_IQ_CMD_IQANUD(x)
#define FW_LEN16(fw_struct)