FreeBSD kernel CXGBE device code
t4_sge.c
Go to the documentation of this file.
1/*-
2 * SPDX-License-Identifier: BSD-2-Clause-FreeBSD
3 *
4 * Copyright (c) 2011 Chelsio Communications, Inc.
5 * All rights reserved.
6 * Written by: Navdeep Parhar <np@FreeBSD.org>
7 *
8 * Redistribution and use in source and binary forms, with or without
9 * modification, are permitted provided that the following conditions
10 * are met:
11 * 1. Redistributions of source code must retain the above copyright
12 * notice, this list of conditions and the following disclaimer.
13 * 2. Redistributions in binary form must reproduce the above copyright
14 * notice, this list of conditions and the following disclaimer in the
15 * documentation and/or other materials provided with the distribution.
16 *
17 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
18 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
19 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
20 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
21 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
22 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
23 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
24 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
25 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
26 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
27 * SUCH DAMAGE.
28 */
29
30#include <sys/cdefs.h>
31__FBSDID("$FreeBSD$");
32
33#include "opt_inet.h"
34#include "opt_inet6.h"
35#include "opt_kern_tls.h"
36#include "opt_ratelimit.h"
37
38#include <sys/types.h>
39#include <sys/eventhandler.h>
40#include <sys/mbuf.h>
41#include <sys/socket.h>
42#include <sys/kernel.h>
43#include <sys/ktls.h>
44#include <sys/malloc.h>
45#include <sys/msan.h>
46#include <sys/queue.h>
47#include <sys/sbuf.h>
48#include <sys/taskqueue.h>
49#include <sys/time.h>
50#include <sys/sglist.h>
51#include <sys/sysctl.h>
52#include <sys/smp.h>
53#include <sys/socketvar.h>
54#include <sys/counter.h>
55#include <net/bpf.h>
56#include <net/ethernet.h>
57#include <net/if.h>
58#include <net/if_vlan_var.h>
59#include <net/if_vxlan.h>
60#include <netinet/in.h>
61#include <netinet/ip.h>
62#include <netinet/ip6.h>
63#include <netinet/tcp.h>
64#include <netinet/udp.h>
65#include <machine/in_cksum.h>
66#include <machine/md_var.h>
67#include <vm/vm.h>
68#include <vm/pmap.h>
69#ifdef DEV_NETMAP
70#include <machine/bus.h>
71#include <sys/selinfo.h>
72#include <net/if_var.h>
73#include <net/netmap.h>
74#include <dev/netmap/netmap_kern.h>
75#endif
76
77#include "common/common.h"
78#include "common/t4_regs.h"
79#include "common/t4_regs_values.h"
80#include "common/t4_msg.h"
81#include "t4_l2t.h"
82#include "t4_mp_ring.h"
83
84#ifdef T4_PKT_TIMESTAMP
85#define RX_COPY_THRESHOLD (MINCLSIZE - 8)
86#else
87#define RX_COPY_THRESHOLD MINCLSIZE
88#endif
89
90/* Internal mbuf flags stored in PH_loc.eight[1]. */
91#define MC_NOMAP 0x01
92#define MC_RAW_WR 0x02
93#define MC_TLS 0x04
94
95/*
96 * Ethernet frames are DMA'd at this byte offset into the freelist buffer.
97 * 0-7 are valid values.
98 */
99static int fl_pktshift = 0;
100SYSCTL_INT(_hw_cxgbe, OID_AUTO, fl_pktshift, CTLFLAG_RDTUN, &fl_pktshift, 0,
101 "payload DMA offset in rx buffer (bytes)");
102
103/*
104 * Pad ethernet payload up to this boundary.
105 * -1: driver should figure out a good value.
106 * 0: disable padding.
107 * Any power of 2 from 32 to 4096 (both inclusive) is also a valid value.
108 */
109int fl_pad = -1;
110SYSCTL_INT(_hw_cxgbe, OID_AUTO, fl_pad, CTLFLAG_RDTUN, &fl_pad, 0,
111 "payload pad boundary (bytes)");
112
113/*
114 * Status page length.
115 * -1: driver should figure out a good value.
116 * 64 or 128 are the only other valid values.
117 */
118static int spg_len = -1;
119SYSCTL_INT(_hw_cxgbe, OID_AUTO, spg_len, CTLFLAG_RDTUN, &spg_len, 0,
120 "status page size (bytes)");
121
122/*
123 * Congestion drops.
124 * -1: no congestion feedback (not recommended).
125 * 0: backpressure the channel instead of dropping packets right away.
126 * 1: no backpressure, drop packets for the congested queue immediately.
127 */
128static int cong_drop = 0;
129SYSCTL_INT(_hw_cxgbe, OID_AUTO, cong_drop, CTLFLAG_RDTUN, &cong_drop, 0,
130 "Congestion control for RX queues (0 = backpressure, 1 = drop");
131
132/*
133 * Deliver multiple frames in the same free list buffer if they fit.
134 * -1: let the driver decide whether to enable buffer packing or not.
135 * 0: disable buffer packing.
136 * 1: enable buffer packing.
137 */
138static int buffer_packing = -1;
139SYSCTL_INT(_hw_cxgbe, OID_AUTO, buffer_packing, CTLFLAG_RDTUN, &buffer_packing,
140 0, "Enable buffer packing");
141
142/*
143 * Start next frame in a packed buffer at this boundary.
144 * -1: driver should figure out a good value.
145 * T4: driver will ignore this and use the same value as fl_pad above.
146 * T5: 16, or a power of 2 from 64 to 4096 (both inclusive) is a valid value.
147 */
148static int fl_pack = -1;
149SYSCTL_INT(_hw_cxgbe, OID_AUTO, fl_pack, CTLFLAG_RDTUN, &fl_pack, 0,
150 "payload pack boundary (bytes)");
151
152/*
153 * Largest rx cluster size that the driver is allowed to allocate.
154 */
155static int largest_rx_cluster = MJUM16BYTES;
156SYSCTL_INT(_hw_cxgbe, OID_AUTO, largest_rx_cluster, CTLFLAG_RDTUN,
157 &largest_rx_cluster, 0, "Largest rx cluster (bytes)");
158
159/*
160 * Size of cluster allocation that's most likely to succeed. The driver will
161 * fall back to this size if it fails to allocate clusters larger than this.
162 */
163static int safest_rx_cluster = PAGE_SIZE;
164SYSCTL_INT(_hw_cxgbe, OID_AUTO, safest_rx_cluster, CTLFLAG_RDTUN,
165 &safest_rx_cluster, 0, "Safe rx cluster (bytes)");
166
167#ifdef RATELIMIT
168/*
169 * Knob to control TCP timestamp rewriting, and the granularity of the tick used
170 * for rewriting. -1 and 0-3 are all valid values.
171 * -1: hardware should leave the TCP timestamps alone.
172 * 0: 1ms
173 * 1: 100us
174 * 2: 10us
175 * 3: 1us
176 */
177static int tsclk = -1;
178SYSCTL_INT(_hw_cxgbe, OID_AUTO, tsclk, CTLFLAG_RDTUN, &tsclk, 0,
179 "Control TCP timestamp rewriting when using pacing");
180
181static int eo_max_backlog = 1024 * 1024;
182SYSCTL_INT(_hw_cxgbe, OID_AUTO, eo_max_backlog, CTLFLAG_RDTUN, &eo_max_backlog,
183 0, "Maximum backlog of ratelimited data per flow");
184#endif
185
186/*
187 * The interrupt holdoff timers are multiplied by this value on T6+.
188 * 1 and 3-17 (both inclusive) are legal values.
189 */
190static int tscale = 1;
191SYSCTL_INT(_hw_cxgbe, OID_AUTO, tscale, CTLFLAG_RDTUN, &tscale, 0,
192 "Interrupt holdoff timer scale on T6+");
193
194/*
195 * Number of LRO entries in the lro_ctrl structure per rx queue.
196 */
197static int lro_entries = TCP_LRO_ENTRIES;
198SYSCTL_INT(_hw_cxgbe, OID_AUTO, lro_entries, CTLFLAG_RDTUN, &lro_entries, 0,
199 "Number of LRO entries per RX queue");
200
201/*
202 * This enables presorting of frames before they're fed into tcp_lro_rx.
203 */
204static int lro_mbufs = 0;
205SYSCTL_INT(_hw_cxgbe, OID_AUTO, lro_mbufs, CTLFLAG_RDTUN, &lro_mbufs, 0,
206 "Enable presorting of LRO frames");
207
208static counter_u64_t pullups;
209SYSCTL_COUNTER_U64(_hw_cxgbe, OID_AUTO, pullups, CTLFLAG_RD, &pullups,
210 "Number of mbuf pullups performed");
211
212static counter_u64_t defrags;
213SYSCTL_COUNTER_U64(_hw_cxgbe, OID_AUTO, defrags, CTLFLAG_RD, &defrags,
214 "Number of mbuf defrags performed");
215
216static int t4_tx_coalesce = 1;
217SYSCTL_INT(_hw_cxgbe, OID_AUTO, tx_coalesce, CTLFLAG_RWTUN, &t4_tx_coalesce, 0,
218 "tx coalescing allowed");
219
220/*
221 * The driver will make aggressive attempts at tx coalescing if it sees these
222 * many packets eligible for coalescing in quick succession, with no more than
223 * the specified gap in between the eth_tx calls that delivered the packets.
224 */
225static int t4_tx_coalesce_pkts = 32;
226SYSCTL_INT(_hw_cxgbe, OID_AUTO, tx_coalesce_pkts, CTLFLAG_RWTUN,
227 &t4_tx_coalesce_pkts, 0,
228 "# of consecutive packets (1 - 255) that will trigger tx coalescing");
229static int t4_tx_coalesce_gap = 5;
230SYSCTL_INT(_hw_cxgbe, OID_AUTO, tx_coalesce_gap, CTLFLAG_RWTUN,
231 &t4_tx_coalesce_gap, 0, "tx gap (in microseconds)");
232
233static int service_iq(struct sge_iq *, int);
234static int service_iq_fl(struct sge_iq *, int);
235static struct mbuf *get_fl_payload(struct adapter *, struct sge_fl *, uint32_t);
236static int eth_rx(struct adapter *, struct sge_rxq *, const struct iq_desc *,
237 u_int);
238static inline void init_iq(struct sge_iq *, struct adapter *, int, int, int,
239 int, int);
240static inline void init_fl(struct adapter *, struct sge_fl *, int, int, char *);
241static inline void init_eq(struct adapter *, struct sge_eq *, int, int, uint8_t,
242 struct sge_iq *, char *);
243static int alloc_iq_fl(struct vi_info *, struct sge_iq *, struct sge_fl *,
244 struct sysctl_ctx_list *, struct sysctl_oid *);
245static void free_iq_fl(struct adapter *, struct sge_iq *, struct sge_fl *);
246static void add_iq_sysctls(struct sysctl_ctx_list *, struct sysctl_oid *,
247 struct sge_iq *);
248static void add_fl_sysctls(struct adapter *, struct sysctl_ctx_list *,
249 struct sysctl_oid *, struct sge_fl *);
250static int alloc_iq_fl_hwq(struct vi_info *, struct sge_iq *, struct sge_fl *);
251static int free_iq_fl_hwq(struct adapter *, struct sge_iq *, struct sge_fl *);
252static int alloc_fwq(struct adapter *);
253static void free_fwq(struct adapter *);
254static int alloc_ctrlq(struct adapter *, int);
255static void free_ctrlq(struct adapter *, int);
256static int alloc_rxq(struct vi_info *, struct sge_rxq *, int, int, int);
257static void free_rxq(struct vi_info *, struct sge_rxq *);
258static void add_rxq_sysctls(struct sysctl_ctx_list *, struct sysctl_oid *,
259 struct sge_rxq *);
260#ifdef TCP_OFFLOAD
261static int alloc_ofld_rxq(struct vi_info *, struct sge_ofld_rxq *, int, int,
262 int);
263static void free_ofld_rxq(struct vi_info *, struct sge_ofld_rxq *);
264static void add_ofld_rxq_sysctls(struct sysctl_ctx_list *, struct sysctl_oid *,
265 struct sge_ofld_rxq *);
266#endif
267static int ctrl_eq_alloc(struct adapter *, struct sge_eq *);
268static int eth_eq_alloc(struct adapter *, struct vi_info *, struct sge_eq *);
269#if defined(TCP_OFFLOAD) || defined(RATELIMIT)
270static int ofld_eq_alloc(struct adapter *, struct vi_info *, struct sge_eq *);
271#endif
272static int alloc_eq(struct adapter *, struct sge_eq *, struct sysctl_ctx_list *,
273 struct sysctl_oid *);
274static void free_eq(struct adapter *, struct sge_eq *);
275static void add_eq_sysctls(struct adapter *, struct sysctl_ctx_list *,
276 struct sysctl_oid *, struct sge_eq *);
277static int alloc_eq_hwq(struct adapter *, struct vi_info *, struct sge_eq *);
278static int free_eq_hwq(struct adapter *, struct vi_info *, struct sge_eq *);
279static int alloc_wrq(struct adapter *, struct vi_info *, struct sge_wrq *,
280 struct sysctl_ctx_list *, struct sysctl_oid *);
281static void free_wrq(struct adapter *, struct sge_wrq *);
282static void add_wrq_sysctls(struct sysctl_ctx_list *, struct sysctl_oid *,
283 struct sge_wrq *);
284static int alloc_txq(struct vi_info *, struct sge_txq *, int);
285static void free_txq(struct vi_info *, struct sge_txq *);
286static void add_txq_sysctls(struct vi_info *, struct sysctl_ctx_list *,
287 struct sysctl_oid *, struct sge_txq *);
288#if defined(TCP_OFFLOAD) || defined(RATELIMIT)
289static int alloc_ofld_txq(struct vi_info *, struct sge_ofld_txq *, int);
290static void free_ofld_txq(struct vi_info *, struct sge_ofld_txq *);
291static void add_ofld_txq_sysctls(struct sysctl_ctx_list *, struct sysctl_oid *,
292 struct sge_ofld_txq *);
293#endif
294static void oneseg_dma_callback(void *, bus_dma_segment_t *, int, int);
295static inline void ring_fl_db(struct adapter *, struct sge_fl *);
296static int refill_fl(struct adapter *, struct sge_fl *, int);
297static void refill_sfl(void *);
298static int find_refill_source(struct adapter *, int, bool);
299static void add_fl_to_sfl(struct adapter *, struct sge_fl *);
300
301static inline void get_pkt_gl(struct mbuf *, struct sglist *);
302static inline u_int txpkt_len16(u_int, const u_int);
303static inline u_int txpkt_vm_len16(u_int, const u_int);
304static inline void calculate_mbuf_len16(struct mbuf *, bool);
305static inline u_int txpkts0_len16(u_int);
306static inline u_int txpkts1_len16(void);
307static u_int write_raw_wr(struct sge_txq *, void *, struct mbuf *, u_int);
308static u_int write_txpkt_wr(struct adapter *, struct sge_txq *, struct mbuf *,
309 u_int);
310static u_int write_txpkt_vm_wr(struct adapter *, struct sge_txq *,
311 struct mbuf *);
312static int add_to_txpkts_vf(struct adapter *, struct sge_txq *, struct mbuf *,
313 int, bool *);
314static int add_to_txpkts_pf(struct adapter *, struct sge_txq *, struct mbuf *,
315 int, bool *);
316static u_int write_txpkts_wr(struct adapter *, struct sge_txq *);
317static u_int write_txpkts_vm_wr(struct adapter *, struct sge_txq *);
318static void write_gl_to_txd(struct sge_txq *, struct mbuf *, caddr_t *, int);
319static inline void copy_to_txd(struct sge_eq *, caddr_t, caddr_t *, int);
320static inline void ring_eq_db(struct adapter *, struct sge_eq *, u_int);
321static inline uint16_t read_hw_cidx(struct sge_eq *);
322static inline u_int reclaimable_tx_desc(struct sge_eq *);
323static inline u_int total_available_tx_desc(struct sge_eq *);
324static u_int reclaim_tx_descs(struct sge_txq *, u_int);
325static void tx_reclaim(void *, int);
326static __be64 get_flit(struct sglist_seg *, int, int);
327static int handle_sge_egr_update(struct sge_iq *, const struct rss_header *,
328 struct mbuf *);
329static int handle_fw_msg(struct sge_iq *, const struct rss_header *,
330 struct mbuf *);
331static int t4_handle_wrerr_rpl(struct adapter *, const __be64 *);
332static void wrq_tx_drain(void *, int);
333static void drain_wrq_wr_list(struct adapter *, struct sge_wrq *);
334
335static int sysctl_bufsizes(SYSCTL_HANDLER_ARGS);
336#ifdef RATELIMIT
337#if defined(INET) || defined(INET6)
338static inline u_int txpkt_eo_len16(u_int, u_int, u_int);
339#endif
340static int ethofld_fw4_ack(struct sge_iq *, const struct rss_header *,
341 struct mbuf *);
342#endif
343
344static counter_u64_t extfree_refs;
345static counter_u64_t extfree_rels;
346
347an_handler_t t4_an_handler;
348fw_msg_handler_t t4_fw_msg_handler[NUM_FW6_TYPES];
349cpl_handler_t t4_cpl_handler[NUM_CPL_CMDS];
350cpl_handler_t set_tcb_rpl_handlers[NUM_CPL_COOKIES];
351cpl_handler_t l2t_write_rpl_handlers[NUM_CPL_COOKIES];
352cpl_handler_t act_open_rpl_handlers[NUM_CPL_COOKIES];
353cpl_handler_t abort_rpl_rss_handlers[NUM_CPL_COOKIES];
354cpl_handler_t fw4_ack_handlers[NUM_CPL_COOKIES];
355
356void
357t4_register_an_handler(an_handler_t h)
358{
359 uintptr_t *loc;
360
361 MPASS(h == NULL || t4_an_handler == NULL);
362
363 loc = (uintptr_t *)&t4_an_handler;
364 atomic_store_rel_ptr(loc, (uintptr_t)h);
365}
366
367void
368t4_register_fw_msg_handler(int type, fw_msg_handler_t h)
369{
370 uintptr_t *loc;
371
372 MPASS(type < nitems(t4_fw_msg_handler));
373 MPASS(h == NULL || t4_fw_msg_handler[type] == NULL);
374 /*
375 * These are dispatched by the handler for FW{4|6}_CPL_MSG using the CPL
376 * handler dispatch table. Reject any attempt to install a handler for
377 * this subtype.
378 */
379 MPASS(type != FW_TYPE_RSSCPL);
380 MPASS(type != FW6_TYPE_RSSCPL);
381
382 loc = (uintptr_t *)&t4_fw_msg_handler[type];
383 atomic_store_rel_ptr(loc, (uintptr_t)h);
384}
385
386void
387t4_register_cpl_handler(int opcode, cpl_handler_t h)
388{
389 uintptr_t *loc;
390
391 MPASS(opcode < nitems(t4_cpl_handler));
392 MPASS(h == NULL || t4_cpl_handler[opcode] == NULL);
393
394 loc = (uintptr_t *)&t4_cpl_handler[opcode];
395 atomic_store_rel_ptr(loc, (uintptr_t)h);
396}
397
398static int
399set_tcb_rpl_handler(struct sge_iq *iq, const struct rss_header *rss,
400 struct mbuf *m)
401{
402 const struct cpl_set_tcb_rpl *cpl = (const void *)(rss + 1);
403 u_int tid;
404 int cookie;
405
406 MPASS(m == NULL);
407
408 tid = GET_TID(cpl);
409 if (is_hpftid(iq->adapter, tid) || is_ftid(iq->adapter, tid)) {
410 /*
411 * The return code for filter-write is put in the CPL cookie so
412 * we have to rely on the hardware tid (is_ftid) to determine
413 * that this is a response to a filter.
414 */
415 cookie = CPL_COOKIE_FILTER;
416 } else {
417 cookie = G_COOKIE(cpl->cookie);
418 }
419 MPASS(cookie > CPL_COOKIE_RESERVED);
420 MPASS(cookie < nitems(set_tcb_rpl_handlers));
421
422 return (set_tcb_rpl_handlers[cookie](iq, rss, m));
423}
424
425static int
426l2t_write_rpl_handler(struct sge_iq *iq, const struct rss_header *rss,
427 struct mbuf *m)
428{
429 const struct cpl_l2t_write_rpl *rpl = (const void *)(rss + 1);
430 unsigned int cookie;
431
432 MPASS(m == NULL);
433
434 cookie = GET_TID(rpl) & F_SYNC_WR ? CPL_COOKIE_TOM : CPL_COOKIE_FILTER;
435 return (l2t_write_rpl_handlers[cookie](iq, rss, m));
436}
437
438static int
439act_open_rpl_handler(struct sge_iq *iq, const struct rss_header *rss,
440 struct mbuf *m)
441{
442 const struct cpl_act_open_rpl *cpl = (const void *)(rss + 1);
443 u_int cookie = G_TID_COOKIE(G_AOPEN_ATID(be32toh(cpl->atid_status)));
444
445 MPASS(m == NULL);
446 MPASS(cookie != CPL_COOKIE_RESERVED);
447
448 return (act_open_rpl_handlers[cookie](iq, rss, m));
449}
450
451static int
452abort_rpl_rss_handler(struct sge_iq *iq, const struct rss_header *rss,
453 struct mbuf *m)
454{
455 struct adapter *sc = iq->adapter;
456 u_int cookie;
457
458 MPASS(m == NULL);
459 if (is_hashfilter(sc))
460 cookie = CPL_COOKIE_HASHFILTER;
461 else
462 cookie = CPL_COOKIE_TOM;
463
464 return (abort_rpl_rss_handlers[cookie](iq, rss, m));
465}
466
467static int
468fw4_ack_handler(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m)
469{
470 struct adapter *sc = iq->adapter;
471 const struct cpl_fw4_ack *cpl = (const void *)(rss + 1);
472 unsigned int tid = G_CPL_FW4_ACK_FLOWID(be32toh(OPCODE_TID(cpl)));
473 u_int cookie;
474
475 MPASS(m == NULL);
476 if (is_etid(sc, tid))
477 cookie = CPL_COOKIE_ETHOFLD;
478 else
479 cookie = CPL_COOKIE_TOM;
480
481 return (fw4_ack_handlers[cookie](iq, rss, m));
482}
483
484static void
485t4_init_shared_cpl_handlers(void)
486{
487
488 t4_register_cpl_handler(CPL_SET_TCB_RPL, set_tcb_rpl_handler);
489 t4_register_cpl_handler(CPL_L2T_WRITE_RPL, l2t_write_rpl_handler);
490 t4_register_cpl_handler(CPL_ACT_OPEN_RPL, act_open_rpl_handler);
491 t4_register_cpl_handler(CPL_ABORT_RPL_RSS, abort_rpl_rss_handler);
492 t4_register_cpl_handler(CPL_FW4_ACK, fw4_ack_handler);
493}
494
495void
496t4_register_shared_cpl_handler(int opcode, cpl_handler_t h, int cookie)
497{
498 uintptr_t *loc;
499
500 MPASS(opcode < nitems(t4_cpl_handler));
501 MPASS(cookie > CPL_COOKIE_RESERVED);
502 MPASS(cookie < NUM_CPL_COOKIES);
503 MPASS(t4_cpl_handler[opcode] != NULL);
504
505 switch (opcode) {
506 case CPL_SET_TCB_RPL:
507 loc = (uintptr_t *)&set_tcb_rpl_handlers[cookie];
508 break;
509 case CPL_L2T_WRITE_RPL:
510 loc = (uintptr_t *)&l2t_write_rpl_handlers[cookie];
511 break;
512 case CPL_ACT_OPEN_RPL:
513 loc = (uintptr_t *)&act_open_rpl_handlers[cookie];
514 break;
515 case CPL_ABORT_RPL_RSS:
516 loc = (uintptr_t *)&abort_rpl_rss_handlers[cookie];
517 break;
518 case CPL_FW4_ACK:
519 loc = (uintptr_t *)&fw4_ack_handlers[cookie];
520 break;
521 default:
522 MPASS(0);
523 return;
524 }
525 MPASS(h == NULL || *loc == (uintptr_t)NULL);
526 atomic_store_rel_ptr(loc, (uintptr_t)h);
527}
528
529/*
530 * Called on MOD_LOAD. Validates and calculates the SGE tunables.
531 */
532void
533t4_sge_modload(void)
534{
535
536 if (fl_pktshift < 0 || fl_pktshift > 7) {
537 printf("Invalid hw.cxgbe.fl_pktshift value (%d),"
538 " using 0 instead.\n", fl_pktshift);
539 fl_pktshift = 0;
540 }
541
542 if (spg_len != 64 && spg_len != 128) {
543 int len;
544
545#if defined(__i386__) || defined(__amd64__)
546 len = cpu_clflush_line_size > 64 ? 128 : 64;
547#else
548 len = 64;
549#endif
550 if (spg_len != -1) {
551 printf("Invalid hw.cxgbe.spg_len value (%d),"
552 " using %d instead.\n", spg_len, len);
553 }
554 spg_len = len;
555 }
556
557 if (cong_drop < -1 || cong_drop > 1) {
558 printf("Invalid hw.cxgbe.cong_drop value (%d),"
559 " using 0 instead.\n", cong_drop);
560 cong_drop = 0;
561 }
562
563 if (tscale != 1 && (tscale < 3 || tscale > 17)) {
564 printf("Invalid hw.cxgbe.tscale value (%d),"
565 " using 1 instead.\n", tscale);
566 tscale = 1;
567 }
568
569 if (largest_rx_cluster != MCLBYTES &&
570#if MJUMPAGESIZE != MCLBYTES
571 largest_rx_cluster != MJUMPAGESIZE &&
572#endif
573 largest_rx_cluster != MJUM9BYTES &&
574 largest_rx_cluster != MJUM16BYTES) {
575 printf("Invalid hw.cxgbe.largest_rx_cluster value (%d),"
576 " using %d instead.\n", largest_rx_cluster, MJUM16BYTES);
577 largest_rx_cluster = MJUM16BYTES;
578 }
579
580 if (safest_rx_cluster != MCLBYTES &&
581#if MJUMPAGESIZE != MCLBYTES
582 safest_rx_cluster != MJUMPAGESIZE &&
583#endif
584 safest_rx_cluster != MJUM9BYTES &&
585 safest_rx_cluster != MJUM16BYTES) {
586 printf("Invalid hw.cxgbe.safest_rx_cluster value (%d),"
587 " using %d instead.\n", safest_rx_cluster, MJUMPAGESIZE);
588 safest_rx_cluster = MJUMPAGESIZE;
589 }
590
591 extfree_refs = counter_u64_alloc(M_WAITOK);
592 extfree_rels = counter_u64_alloc(M_WAITOK);
593 pullups = counter_u64_alloc(M_WAITOK);
594 defrags = counter_u64_alloc(M_WAITOK);
595 counter_u64_zero(extfree_refs);
596 counter_u64_zero(extfree_rels);
597 counter_u64_zero(pullups);
598 counter_u64_zero(defrags);
599
600 t4_init_shared_cpl_handlers();
601 t4_register_cpl_handler(CPL_FW4_MSG, handle_fw_msg);
602 t4_register_cpl_handler(CPL_FW6_MSG, handle_fw_msg);
603 t4_register_cpl_handler(CPL_SGE_EGR_UPDATE, handle_sge_egr_update);
604#ifdef RATELIMIT
605 t4_register_shared_cpl_handler(CPL_FW4_ACK, ethofld_fw4_ack,
606 CPL_COOKIE_ETHOFLD);
607#endif
608 t4_register_fw_msg_handler(FW6_TYPE_CMD_RPL, t4_handle_fw_rpl);
609 t4_register_fw_msg_handler(FW6_TYPE_WRERR_RPL, t4_handle_wrerr_rpl);
610}
611
612void
613t4_sge_modunload(void)
614{
615
616 counter_u64_free(extfree_refs);
617 counter_u64_free(extfree_rels);
618 counter_u64_free(pullups);
619 counter_u64_free(defrags);
620}
621
622uint64_t
623t4_sge_extfree_refs(void)
624{
625 uint64_t refs, rels;
626
627 rels = counter_u64_fetch(extfree_rels);
628 refs = counter_u64_fetch(extfree_refs);
629
630 return (refs - rels);
631}
632
633/* max 4096 */
634#define MAX_PACK_BOUNDARY 512
635
636static inline void
637setup_pad_and_pack_boundaries(struct adapter *sc)
638{
639 uint32_t v, m;
640 int pad, pack, pad_shift;
641
642 pad_shift = chip_id(sc) > CHELSIO_T5 ? X_T6_INGPADBOUNDARY_SHIFT :
643 X_INGPADBOUNDARY_SHIFT;
644 pad = fl_pad;
645 if (fl_pad < (1 << pad_shift) ||
646 fl_pad > (1 << (pad_shift + M_INGPADBOUNDARY)) ||
647 !powerof2(fl_pad)) {
648 /*
649 * If there is any chance that we might use buffer packing and
650 * the chip is a T4, then pick 64 as the pad/pack boundary. Set
651 * it to the minimum allowed in all other cases.
652 */
653 pad = is_t4(sc) && buffer_packing ? 64 : 1 << pad_shift;
654
655 /*
656 * For fl_pad = 0 we'll still write a reasonable value to the
657 * register but all the freelists will opt out of padding.
658 * We'll complain here only if the user tried to set it to a
659 * value greater than 0 that was invalid.
660 */
661 if (fl_pad > 0) {
662 device_printf(sc->dev, "Invalid hw.cxgbe.fl_pad value"
663 " (%d), using %d instead.\n", fl_pad, pad);
664 }
665 }
666 m = V_INGPADBOUNDARY(M_INGPADBOUNDARY);
667 v = V_INGPADBOUNDARY(ilog2(pad) - pad_shift);
668 t4_set_reg_field(sc, A_SGE_CONTROL, m, v);
669
670 if (is_t4(sc)) {
671 if (fl_pack != -1 && fl_pack != pad) {
672 /* Complain but carry on. */
673 device_printf(sc->dev, "hw.cxgbe.fl_pack (%d) ignored,"
674 " using %d instead.\n", fl_pack, pad);
675 }
676 return;
677 }
678
679 pack = fl_pack;
680 if (fl_pack < 16 || fl_pack == 32 || fl_pack > 4096 ||
681 !powerof2(fl_pack)) {
682 if (sc->params.pci.mps > MAX_PACK_BOUNDARY)
683 pack = MAX_PACK_BOUNDARY;
684 else
685 pack = max(sc->params.pci.mps, CACHE_LINE_SIZE);
686 MPASS(powerof2(pack));
687 if (pack < 16)
688 pack = 16;
689 if (pack == 32)
690 pack = 64;
691 if (pack > 4096)
692 pack = 4096;
693 if (fl_pack != -1) {
694 device_printf(sc->dev, "Invalid hw.cxgbe.fl_pack value"
695 " (%d), using %d instead.\n", fl_pack, pack);
696 }
697 }
698 m = V_INGPACKBOUNDARY(M_INGPACKBOUNDARY);
699 if (pack == 16)
700 v = V_INGPACKBOUNDARY(0);
701 else
702 v = V_INGPACKBOUNDARY(ilog2(pack) - 5);
703
704 MPASS(!is_t4(sc)); /* T4 doesn't have SGE_CONTROL2 */
705 t4_set_reg_field(sc, A_SGE_CONTROL2, m, v);
706}
707
708/*
709 * adap->params.vpd.cclk must be set up before this is called.
710 */
711void
712t4_tweak_chip_settings(struct adapter *sc)
713{
714 int i, reg;
715 uint32_t v, m;
716 int intr_timer[SGE_NTIMERS] = {1, 5, 10, 50, 100, 200};
717 int timer_max = M_TIMERVALUE0 * 1000 / sc->params.vpd.cclk;
718 int intr_pktcount[SGE_NCOUNTERS] = {1, 8, 16, 32}; /* 63 max */
719 uint16_t indsz = min(RX_COPY_THRESHOLD - 1, M_INDICATESIZE);
720 static int sw_buf_sizes[] = {
721 MCLBYTES,
722#if MJUMPAGESIZE != MCLBYTES
723 MJUMPAGESIZE,
724#endif
725 MJUM9BYTES,
726 MJUM16BYTES
727 };
728
729 KASSERT(sc->flags & MASTER_PF,
730 ("%s: trying to change chip settings when not master.", __func__));
731
732 m = V_PKTSHIFT(M_PKTSHIFT) | F_RXPKTCPLMODE | F_EGRSTATUSPAGESIZE;
733 v = V_PKTSHIFT(fl_pktshift) | F_RXPKTCPLMODE |
734 V_EGRSTATUSPAGESIZE(spg_len == 128);
735 t4_set_reg_field(sc, A_SGE_CONTROL, m, v);
736
737 setup_pad_and_pack_boundaries(sc);
738
739 v = V_HOSTPAGESIZEPF0(PAGE_SHIFT - 10) |
740 V_HOSTPAGESIZEPF1(PAGE_SHIFT - 10) |
741 V_HOSTPAGESIZEPF2(PAGE_SHIFT - 10) |
742 V_HOSTPAGESIZEPF3(PAGE_SHIFT - 10) |
743 V_HOSTPAGESIZEPF4(PAGE_SHIFT - 10) |
744 V_HOSTPAGESIZEPF5(PAGE_SHIFT - 10) |
745 V_HOSTPAGESIZEPF6(PAGE_SHIFT - 10) |
746 V_HOSTPAGESIZEPF7(PAGE_SHIFT - 10);
747 t4_write_reg(sc, A_SGE_HOST_PAGE_SIZE, v);
748
749 t4_write_reg(sc, A_SGE_FL_BUFFER_SIZE0, 4096);
750 t4_write_reg(sc, A_SGE_FL_BUFFER_SIZE1, 65536);
751 reg = A_SGE_FL_BUFFER_SIZE2;
752 for (i = 0; i < nitems(sw_buf_sizes); i++) {
753 MPASS(reg <= A_SGE_FL_BUFFER_SIZE15);
754 t4_write_reg(sc, reg, sw_buf_sizes[i]);
755 reg += 4;
756 MPASS(reg <= A_SGE_FL_BUFFER_SIZE15);
757 t4_write_reg(sc, reg, sw_buf_sizes[i] - CL_METADATA_SIZE);
758 reg += 4;
759 }
760
761 v = V_THRESHOLD_0(intr_pktcount[0]) | V_THRESHOLD_1(intr_pktcount[1]) |
762 V_THRESHOLD_2(intr_pktcount[2]) | V_THRESHOLD_3(intr_pktcount[3]);
763 t4_write_reg(sc, A_SGE_INGRESS_RX_THRESHOLD, v);
764
765 KASSERT(intr_timer[0] <= timer_max,
766 ("%s: not a single usable timer (%d, %d)", __func__, intr_timer[0],
767 timer_max));
768 for (i = 1; i < nitems(intr_timer); i++) {
769 KASSERT(intr_timer[i] >= intr_timer[i - 1],
770 ("%s: timers not listed in increasing order (%d)",
771 __func__, i));
772
773 while (intr_timer[i] > timer_max) {
774 if (i == nitems(intr_timer) - 1) {
775 intr_timer[i] = timer_max;
776 break;
777 }
778 intr_timer[i] += intr_timer[i - 1];
779 intr_timer[i] /= 2;
780 }
781 }
782
783 v = V_TIMERVALUE0(us_to_core_ticks(sc, intr_timer[0])) |
784 V_TIMERVALUE1(us_to_core_ticks(sc, intr_timer[1]));
785 t4_write_reg(sc, A_SGE_TIMER_VALUE_0_AND_1, v);
786 v = V_TIMERVALUE2(us_to_core_ticks(sc, intr_timer[2])) |
787 V_TIMERVALUE3(us_to_core_ticks(sc, intr_timer[3]));
788 t4_write_reg(sc, A_SGE_TIMER_VALUE_2_AND_3, v);
789 v = V_TIMERVALUE4(us_to_core_ticks(sc, intr_timer[4])) |
790 V_TIMERVALUE5(us_to_core_ticks(sc, intr_timer[5]));
791 t4_write_reg(sc, A_SGE_TIMER_VALUE_4_AND_5, v);
792
793 if (chip_id(sc) >= CHELSIO_T6) {
794 m = V_TSCALE(M_TSCALE);
795 if (tscale == 1)
796 v = 0;
797 else
798 v = V_TSCALE(tscale - 2);
799 t4_set_reg_field(sc, A_SGE_ITP_CONTROL, m, v);
800
801 if (sc->debug_flags & DF_DISABLE_TCB_CACHE) {
802 m = V_RDTHRESHOLD(M_RDTHRESHOLD) | F_WRTHRTHRESHEN |
803 V_WRTHRTHRESH(M_WRTHRTHRESH);
804 t4_tp_pio_read(sc, &v, 1, A_TP_CMM_CONFIG, 1);
805 v &= ~m;
806 v |= V_RDTHRESHOLD(1) | F_WRTHRTHRESHEN |
807 V_WRTHRTHRESH(16);
808 t4_tp_pio_write(sc, &v, 1, A_TP_CMM_CONFIG, 1);
809 }
810 }
811
812 /* 4K, 16K, 64K, 256K DDP "page sizes" for TDDP */
813 v = V_HPZ0(0) | V_HPZ1(2) | V_HPZ2(4) | V_HPZ3(6);
814 t4_write_reg(sc, A_ULP_RX_TDDP_PSZ, v);
815
816 /*
817 * 4K, 8K, 16K, 64K DDP "page sizes" for iSCSI DDP. These have been
818 * chosen with MAXPHYS = 128K in mind. The largest DDP buffer that we
819 * may have to deal with is MAXPHYS + 1 page.
820 */
821 v = V_HPZ0(0) | V_HPZ1(1) | V_HPZ2(2) | V_HPZ3(4);
822 t4_write_reg(sc, A_ULP_RX_ISCSI_PSZ, v);
823
824 /* We use multiple DDP page sizes both in plain-TOE and ISCSI modes. */
825 m = v = F_TDDPTAGTCB | F_ISCSITAGTCB;
826 t4_set_reg_field(sc, A_ULP_RX_CTL, m, v);
827
828 m = V_INDICATESIZE(M_INDICATESIZE) | F_REARMDDPOFFSET |
829 F_RESETDDPOFFSET;
830 v = V_INDICATESIZE(indsz) | F_REARMDDPOFFSET | F_RESETDDPOFFSET;
831 t4_set_reg_field(sc, A_TP_PARA_REG5, m, v);
832}
833
834/*
835 * SGE wants the buffer to be at least 64B and then a multiple of 16. Its
836 * address mut be 16B aligned. If padding is in use the buffer's start and end
837 * need to be aligned to the pad boundary as well. We'll just make sure that
838 * the size is a multiple of the pad boundary here, it is up to the buffer
839 * allocation code to make sure the start of the buffer is aligned.
840 */
841static inline int
842hwsz_ok(struct adapter *sc, int hwsz)
843{
844 int mask = fl_pad ? sc->params.sge.pad_boundary - 1 : 16 - 1;
845
846 return (hwsz >= 64 && (hwsz & mask) == 0);
847}
848
849/*
850 * Initialize the rx buffer sizes and figure out which zones the buffers will
851 * be allocated from.
852 */
853void
854t4_init_rx_buf_info(struct adapter *sc)
855{
856 struct sge *s = &sc->sge;
857 struct sge_params *sp = &sc->params.sge;
858 int i, j, n;
859 static int sw_buf_sizes[] = { /* Sorted by size */
860 MCLBYTES,
861#if MJUMPAGESIZE != MCLBYTES
862 MJUMPAGESIZE,
863#endif
864 MJUM9BYTES,
865 MJUM16BYTES
866 };
867 struct rx_buf_info *rxb;
868
869 s->safe_zidx = -1;
870 rxb = &s->rx_buf_info[0];
871 for (i = 0; i < SW_ZONE_SIZES; i++, rxb++) {
872 rxb->size1 = sw_buf_sizes[i];
873 rxb->zone = m_getzone(rxb->size1);
874 rxb->type = m_gettype(rxb->size1);
875 rxb->size2 = 0;
876 rxb->hwidx1 = -1;
877 rxb->hwidx2 = -1;
878 for (j = 0; j < SGE_FLBUF_SIZES; j++) {
879 int hwsize = sp->sge_fl_buffer_size[j];
880
881 if (!hwsz_ok(sc, hwsize))
882 continue;
883
884 /* hwidx for size1 */
885 if (rxb->hwidx1 == -1 && rxb->size1 == hwsize)
886 rxb->hwidx1 = j;
887
888 /* hwidx for size2 (buffer packing) */
889 if (rxb->size1 - CL_METADATA_SIZE < hwsize)
890 continue;
891 n = rxb->size1 - hwsize - CL_METADATA_SIZE;
892 if (n == 0) {
893 rxb->hwidx2 = j;
894 rxb->size2 = hwsize;
895 break; /* stop looking */
896 }
897 if (rxb->hwidx2 != -1) {
898 if (n < sp->sge_fl_buffer_size[rxb->hwidx2] -
899 hwsize - CL_METADATA_SIZE) {
900 rxb->hwidx2 = j;
901 rxb->size2 = hwsize;
902 }
903 } else if (n <= 2 * CL_METADATA_SIZE) {
904 rxb->hwidx2 = j;
905 rxb->size2 = hwsize;
906 }
907 }
908 if (rxb->hwidx2 != -1)
909 sc->flags |= BUF_PACKING_OK;
910 if (s->safe_zidx == -1 && rxb->size1 == safest_rx_cluster)
911 s->safe_zidx = i;
912 }
913}
914
915/*
916 * Verify some basic SGE settings for the PF and VF driver, and other
917 * miscellaneous settings for the PF driver.
918 */
919int
920t4_verify_chip_settings(struct adapter *sc)
921{
922 struct sge_params *sp = &sc->params.sge;
923 uint32_t m, v, r;
924 int rc = 0;
925 const uint16_t indsz = min(RX_COPY_THRESHOLD - 1, M_INDICATESIZE);
926
927 m = F_RXPKTCPLMODE;
928 v = F_RXPKTCPLMODE;
929 r = sp->sge_control;
930 if ((r & m) != v) {
931 device_printf(sc->dev, "invalid SGE_CONTROL(0x%x)\n", r);
932 rc = EINVAL;
933 }
934
935 /*
936 * If this changes then every single use of PAGE_SHIFT in the driver
937 * needs to be carefully reviewed for PAGE_SHIFT vs sp->page_shift.
938 */
939 if (sp->page_shift != PAGE_SHIFT) {
940 device_printf(sc->dev, "invalid SGE_HOST_PAGE_SIZE(0x%x)\n", r);
941 rc = EINVAL;
942 }
943
944 if (sc->flags & IS_VF)
945 return (0);
946
947 v = V_HPZ0(0) | V_HPZ1(2) | V_HPZ2(4) | V_HPZ3(6);
948 r = t4_read_reg(sc, A_ULP_RX_TDDP_PSZ);
949 if (r != v) {
950 device_printf(sc->dev, "invalid ULP_RX_TDDP_PSZ(0x%x)\n", r);
951 if (sc->vres.ddp.size != 0)
952 rc = EINVAL;
953 }
954
955 m = v = F_TDDPTAGTCB;
956 r = t4_read_reg(sc, A_ULP_RX_CTL);
957 if ((r & m) != v) {
958 device_printf(sc->dev, "invalid ULP_RX_CTL(0x%x)\n", r);
959 if (sc->vres.ddp.size != 0)
960 rc = EINVAL;
961 }
962
963 m = V_INDICATESIZE(M_INDICATESIZE) | F_REARMDDPOFFSET |
964 F_RESETDDPOFFSET;
965 v = V_INDICATESIZE(indsz) | F_REARMDDPOFFSET | F_RESETDDPOFFSET;
966 r = t4_read_reg(sc, A_TP_PARA_REG5);
967 if ((r & m) != v) {
968 device_printf(sc->dev, "invalid TP_PARA_REG5(0x%x)\n", r);
969 if (sc->vres.ddp.size != 0)
970 rc = EINVAL;
971 }
972
973 return (rc);
974}
975
976int
977t4_create_dma_tag(struct adapter *sc)
978{
979 int rc;
980
981 rc = bus_dma_tag_create(bus_get_dma_tag(sc->dev), 1, 0,
982 BUS_SPACE_MAXADDR, BUS_SPACE_MAXADDR, NULL, NULL, BUS_SPACE_MAXSIZE,
983 BUS_SPACE_UNRESTRICTED, BUS_SPACE_MAXSIZE, BUS_DMA_ALLOCNOW, NULL,
984 NULL, &sc->dmat);
985 if (rc != 0) {
986 device_printf(sc->dev,
987 "failed to create main DMA tag: %d\n", rc);
988 }
989
990 return (rc);
991}
992
993void
994t4_sge_sysctls(struct adapter *sc, struct sysctl_ctx_list *ctx,
995 struct sysctl_oid_list *children)
996{
997 struct sge_params *sp = &sc->params.sge;
998
999 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "buffer_sizes",
1000 CTLTYPE_STRING | CTLFLAG_RD | CTLFLAG_MPSAFE, sc, 0,
1001 sysctl_bufsizes, "A", "freelist buffer sizes");
1002
1003 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "fl_pktshift", CTLFLAG_RD,
1004 NULL, sp->fl_pktshift, "payload DMA offset in rx buffer (bytes)");
1005
1006 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "fl_pad", CTLFLAG_RD,
1007 NULL, sp->pad_boundary, "payload pad boundary (bytes)");
1008
1009 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "spg_len", CTLFLAG_RD,
1010 NULL, sp->spg_len, "status page size (bytes)");
1011
1012 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "cong_drop", CTLFLAG_RD,
1013 NULL, cong_drop, "congestion drop setting");
1014
1015 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "fl_pack", CTLFLAG_RD,
1016 NULL, sp->pack_boundary, "payload pack boundary (bytes)");
1017}
1018
1019int
1020t4_destroy_dma_tag(struct adapter *sc)
1021{
1022 if (sc->dmat)
1023 bus_dma_tag_destroy(sc->dmat);
1024
1025 return (0);
1026}
1027
1028/*
1029 * Allocate and initialize the firmware event queue, control queues, and special
1030 * purpose rx queues owned by the adapter.
1031 *
1032 * Returns errno on failure. Resources allocated up to that point may still be
1033 * allocated. Caller is responsible for cleanup in case this function fails.
1034 */
1035int
1036t4_setup_adapter_queues(struct adapter *sc)
1037{
1038 int rc, i;
1039
1040 ADAPTER_LOCK_ASSERT_NOTOWNED(sc);
1041
1042 /*
1043 * Firmware event queue
1044 */
1045 rc = alloc_fwq(sc);
1046 if (rc != 0)
1047 return (rc);
1048
1049 /*
1050 * That's all for the VF driver.
1051 */
1052 if (sc->flags & IS_VF)
1053 return (rc);
1054
1055 /*
1056 * XXX: General purpose rx queues, one per port.
1057 */
1058
1059 /*
1060 * Control queues, one per port.
1061 */
1062 for_each_port(sc, i) {
1063 rc = alloc_ctrlq(sc, i);
1064 if (rc != 0)
1065 return (rc);
1066 }
1067
1068 return (rc);
1069}
1070
1071/*
1072 * Idempotent
1073 */
1074int
1075t4_teardown_adapter_queues(struct adapter *sc)
1076{
1077 int i;
1078
1079 ADAPTER_LOCK_ASSERT_NOTOWNED(sc);
1080
1081 if (sc->sge.ctrlq != NULL) {
1082 MPASS(!(sc->flags & IS_VF)); /* VFs don't allocate ctrlq. */
1083 for_each_port(sc, i)
1084 free_ctrlq(sc, i);
1085 }
1086 free_fwq(sc);
1087
1088 return (0);
1089}
1090
1091/* Maximum payload that could arrive with a single iq descriptor. */
1092static inline int
1093max_rx_payload(struct adapter *sc, struct ifnet *ifp, const bool ofld)
1094{
1095 int maxp;
1096
1097 /* large enough even when hw VLAN extraction is disabled */
1098 maxp = sc->params.sge.fl_pktshift + ETHER_HDR_LEN +
1099 ETHER_VLAN_ENCAP_LEN + ifp->if_mtu;
1100 if (ofld && sc->tt.tls && sc->cryptocaps & FW_CAPS_CONFIG_TLSKEYS &&
1101 maxp < sc->params.tp.max_rx_pdu)
1102 maxp = sc->params.tp.max_rx_pdu;
1103 return (maxp);
1104}
1105
1106int
1107t4_setup_vi_queues(struct vi_info *vi)
1108{
1109 int rc = 0, i, intr_idx;
1110 struct sge_rxq *rxq;
1111 struct sge_txq *txq;
1112#ifdef TCP_OFFLOAD
1113 struct sge_ofld_rxq *ofld_rxq;
1114#endif
1115#if defined(TCP_OFFLOAD) || defined(RATELIMIT)
1116 struct sge_ofld_txq *ofld_txq;
1117#endif
1118#ifdef DEV_NETMAP
1119 int saved_idx, iqidx;
1120 struct sge_nm_rxq *nm_rxq;
1121 struct sge_nm_txq *nm_txq;
1122#endif
1123 struct adapter *sc = vi->adapter;
1124 struct ifnet *ifp = vi->ifp;
1125 int maxp;
1126
1127 /* Interrupt vector to start from (when using multiple vectors) */
1128 intr_idx = vi->first_intr;
1129
1130#ifdef DEV_NETMAP
1131 saved_idx = intr_idx;
1132 if (ifp->if_capabilities & IFCAP_NETMAP) {
1133
1134 /* netmap is supported with direct interrupts only. */
1135 MPASS(!forwarding_intr_to_fwq(sc));
1136 MPASS(vi->first_intr >= 0);
1137
1138 /*
1139 * We don't have buffers to back the netmap rx queues
1140 * right now so we create the queues in a way that
1141 * doesn't set off any congestion signal in the chip.
1142 */
1143 for_each_nm_rxq(vi, i, nm_rxq) {
1144 rc = alloc_nm_rxq(vi, nm_rxq, intr_idx, i);
1145 if (rc != 0)
1146 goto done;
1147 intr_idx++;
1148 }
1149
1150 for_each_nm_txq(vi, i, nm_txq) {
1151 iqidx = vi->first_nm_rxq + (i % vi->nnmrxq);
1152 rc = alloc_nm_txq(vi, nm_txq, iqidx, i);
1153 if (rc != 0)
1154 goto done;
1155 }
1156 }
1157
1158 /* Normal rx queues and netmap rx queues share the same interrupts. */
1159 intr_idx = saved_idx;
1160#endif
1161
1162 /*
1163 * Allocate rx queues first because a default iqid is required when
1164 * creating a tx queue.
1165 */
1166 maxp = max_rx_payload(sc, ifp, false);
1167 for_each_rxq(vi, i, rxq) {
1168 rc = alloc_rxq(vi, rxq, i, intr_idx, maxp);
1169 if (rc != 0)
1170 goto done;
1171 if (!forwarding_intr_to_fwq(sc))
1172 intr_idx++;
1173 }
1174#ifdef DEV_NETMAP
1175 if (ifp->if_capabilities & IFCAP_NETMAP)
1176 intr_idx = saved_idx + max(vi->nrxq, vi->nnmrxq);
1177#endif
1178#ifdef TCP_OFFLOAD
1179 maxp = max_rx_payload(sc, ifp, true);
1180 for_each_ofld_rxq(vi, i, ofld_rxq) {
1181 rc = alloc_ofld_rxq(vi, ofld_rxq, i, intr_idx, maxp);
1182 if (rc != 0)
1183 goto done;
1184 if (!forwarding_intr_to_fwq(sc))
1185 intr_idx++;
1186 }
1187#endif
1188
1189 /*
1190 * Now the tx queues.
1191 */
1192 for_each_txq(vi, i, txq) {
1193 rc = alloc_txq(vi, txq, i);
1194 if (rc != 0)
1195 goto done;
1196 }
1197#if defined(TCP_OFFLOAD) || defined(RATELIMIT)
1198 for_each_ofld_txq(vi, i, ofld_txq) {
1199 rc = alloc_ofld_txq(vi, ofld_txq, i);
1200 if (rc != 0)
1201 goto done;
1202 }
1203#endif
1204done:
1205 if (rc)
1206 t4_teardown_vi_queues(vi);
1207
1208 return (rc);
1209}
1210
1211/*
1212 * Idempotent
1213 */
1214int
1215t4_teardown_vi_queues(struct vi_info *vi)
1216{
1217 int i;
1218 struct sge_rxq *rxq;
1219 struct sge_txq *txq;
1220#if defined(TCP_OFFLOAD) || defined(RATELIMIT)
1221 struct sge_ofld_txq *ofld_txq;
1222#endif
1223#ifdef TCP_OFFLOAD
1224 struct sge_ofld_rxq *ofld_rxq;
1225#endif
1226#ifdef DEV_NETMAP
1227 struct sge_nm_rxq *nm_rxq;
1228 struct sge_nm_txq *nm_txq;
1229#endif
1230
1231#ifdef DEV_NETMAP
1232 if (vi->ifp->if_capabilities & IFCAP_NETMAP) {
1233 for_each_nm_txq(vi, i, nm_txq) {
1234 free_nm_txq(vi, nm_txq);
1235 }
1236
1237 for_each_nm_rxq(vi, i, nm_rxq) {
1238 free_nm_rxq(vi, nm_rxq);
1239 }
1240 }
1241#endif
1242
1243 /*
1244 * Take down all the tx queues first, as they reference the rx queues
1245 * (for egress updates, etc.).
1246 */
1247
1248 for_each_txq(vi, i, txq) {
1249 free_txq(vi, txq);
1250 }
1251#if defined(TCP_OFFLOAD) || defined(RATELIMIT)
1252 for_each_ofld_txq(vi, i, ofld_txq) {
1253 free_ofld_txq(vi, ofld_txq);
1254 }
1255#endif
1256
1257 /*
1258 * Then take down the rx queues.
1259 */
1260
1261 for_each_rxq(vi, i, rxq) {
1262 free_rxq(vi, rxq);
1263 }
1264#ifdef TCP_OFFLOAD
1265 for_each_ofld_rxq(vi, i, ofld_rxq) {
1266 free_ofld_rxq(vi, ofld_rxq);
1267 }
1268#endif
1269
1270 return (0);
1271}
1272
1273/*
1274 * Interrupt handler when the driver is using only 1 interrupt. This is a very
1275 * unusual scenario.
1276 *
1277 * a) Deals with errors, if any.
1278 * b) Services firmware event queue, which is taking interrupts for all other
1279 * queues.
1280 */
1281void
1282t4_intr_all(void *arg)
1283{
1284 struct adapter *sc = arg;
1285 struct sge_iq *fwq = &sc->sge.fwq;
1286
1287 MPASS(sc->intr_count == 1);
1288
1289 if (sc->intr_type == INTR_INTX)
1290 t4_write_reg(sc, MYPF_REG(A_PCIE_PF_CLI), 0);
1291
1292 t4_intr_err(arg);
1293 t4_intr_evt(fwq);
1294}
1295
1296/*
1297 * Interrupt handler for errors (installed directly when multiple interrupts are
1298 * being used, or called by t4_intr_all).
1299 */
1300void
1301t4_intr_err(void *arg)
1302{
1303 struct adapter *sc = arg;
1304 uint32_t v;
1305 const bool verbose = (sc->debug_flags & DF_VERBOSE_SLOWINTR) != 0;
1306
1307 if (atomic_load_int(&sc->error_flags) & ADAP_FATAL_ERR)
1308 return;
1309
1310 v = t4_read_reg(sc, MYPF_REG(A_PL_PF_INT_CAUSE));
1311 if (v & F_PFSW) {
1312 sc->swintr++;
1313 t4_write_reg(sc, MYPF_REG(A_PL_PF_INT_CAUSE), v);
1314 }
1315
1316 if (t4_slow_intr_handler(sc, verbose))
1317 t4_fatal_err(sc, false);
1318}
1319
1320/*
1321 * Interrupt handler for iq-only queues. The firmware event queue is the only
1322 * such queue right now.
1323 */
1324void
1325t4_intr_evt(void *arg)
1326{
1327 struct sge_iq *iq = arg;
1328
1329 if (atomic_cmpset_int(&iq->state, IQS_IDLE, IQS_BUSY)) {
1330 service_iq(iq, 0);
1331 (void) atomic_cmpset_int(&iq->state, IQS_BUSY, IQS_IDLE);
1332 }
1333}
1334
1335/*
1336 * Interrupt handler for iq+fl queues.
1337 */
1338void
1339t4_intr(void *arg)
1340{
1341 struct sge_iq *iq = arg;
1342
1343 if (atomic_cmpset_int(&iq->state, IQS_IDLE, IQS_BUSY)) {
1344 service_iq_fl(iq, 0);
1345 (void) atomic_cmpset_int(&iq->state, IQS_BUSY, IQS_IDLE);
1346 }
1347}
1348
1349#ifdef DEV_NETMAP
1350/*
1351 * Interrupt handler for netmap rx queues.
1352 */
1353void
1354t4_nm_intr(void *arg)
1355{
1356 struct sge_nm_rxq *nm_rxq = arg;
1357
1358 if (atomic_cmpset_int(&nm_rxq->nm_state, NM_ON, NM_BUSY)) {
1359 service_nm_rxq(nm_rxq);
1360 (void) atomic_cmpset_int(&nm_rxq->nm_state, NM_BUSY, NM_ON);
1361 }
1362}
1363
1364/*
1365 * Interrupt handler for vectors shared between NIC and netmap rx queues.
1366 */
1367void
1368t4_vi_intr(void *arg)
1369{
1370 struct irq *irq = arg;
1371
1372 MPASS(irq->nm_rxq != NULL);
1373 t4_nm_intr(irq->nm_rxq);
1374
1375 MPASS(irq->rxq != NULL);
1376 t4_intr(irq->rxq);
1377}
1378#endif
1379
1380/*
1381 * Deals with interrupts on an iq-only (no freelist) queue.
1382 */
1383static int
1384service_iq(struct sge_iq *iq, int budget)
1385{
1386 struct sge_iq *q;
1387 struct adapter *sc = iq->adapter;
1388 struct iq_desc *d = &iq->desc[iq->cidx];
1389 int ndescs = 0, limit;
1390 int rsp_type;
1391 uint32_t lq;
1392 STAILQ_HEAD(, sge_iq) iql = STAILQ_HEAD_INITIALIZER(iql);
1393
1394 KASSERT(iq->state == IQS_BUSY, ("%s: iq %p not BUSY", __func__, iq));
1395 KASSERT((iq->flags & IQ_HAS_FL) == 0,
1396 ("%s: called for iq %p with fl (iq->flags 0x%x)", __func__, iq,
1397 iq->flags));
1398 MPASS((iq->flags & IQ_ADJ_CREDIT) == 0);
1399 MPASS((iq->flags & IQ_LRO_ENABLED) == 0);
1400
1401 limit = budget ? budget : iq->qsize / 16;
1402
1403 /*
1404 * We always come back and check the descriptor ring for new indirect
1405 * interrupts and other responses after running a single handler.
1406 */
1407 for (;;) {
1408 while ((d->rsp.u.type_gen & F_RSPD_GEN) == iq->gen) {
1409
1410 rmb();
1411
1412 rsp_type = G_RSPD_TYPE(d->rsp.u.type_gen);
1413 lq = be32toh(d->rsp.pldbuflen_qid);
1414
1415 switch (rsp_type) {
1416 case X_RSPD_TYPE_FLBUF:
1417 panic("%s: data for an iq (%p) with no freelist",
1418 __func__, iq);
1419
1420 /* NOTREACHED */
1421
1422 case X_RSPD_TYPE_CPL:
1423 KASSERT(d->rss.opcode < NUM_CPL_CMDS,
1424 ("%s: bad opcode %02x.", __func__,
1425 d->rss.opcode));
1426 t4_cpl_handler[d->rss.opcode](iq, &d->rss, NULL);
1427 break;
1428
1429 case X_RSPD_TYPE_INTR:
1430 /*
1431 * There are 1K interrupt-capable queues (qids 0
1432 * through 1023). A response type indicating a
1433 * forwarded interrupt with a qid >= 1K is an
1434 * iWARP async notification.
1435 */
1436 if (__predict_true(lq >= 1024)) {
1437 t4_an_handler(iq, &d->rsp);
1438 break;
1439 }
1440
1441 q = sc->sge.iqmap[lq - sc->sge.iq_start -
1442 sc->sge.iq_base];
1443 if (atomic_cmpset_int(&q->state, IQS_IDLE,
1444 IQS_BUSY)) {
1445 if (service_iq_fl(q, q->qsize / 16) == 0) {
1446 (void) atomic_cmpset_int(&q->state,
1447 IQS_BUSY, IQS_IDLE);
1448 } else {
1449 STAILQ_INSERT_TAIL(&iql, q,
1450 link);
1451 }
1452 }
1453 break;
1454
1455 default:
1456 KASSERT(0,
1457 ("%s: illegal response type %d on iq %p",
1458 __func__, rsp_type, iq));
1459 log(LOG_ERR,
1460 "%s: illegal response type %d on iq %p",
1461 device_get_nameunit(sc->dev), rsp_type, iq);
1462 break;
1463 }
1464
1465 d++;
1466 if (__predict_false(++iq->cidx == iq->sidx)) {
1467 iq->cidx = 0;
1468 iq->gen ^= F_RSPD_GEN;
1469 d = &iq->desc[0];
1470 }
1471 if (__predict_false(++ndescs == limit)) {
1472 t4_write_reg(sc, sc->sge_gts_reg,
1473 V_CIDXINC(ndescs) |
1474 V_INGRESSQID(iq->cntxt_id) |
1475 V_SEINTARM(V_QINTR_TIMER_IDX(X_TIMERREG_UPDATE_CIDX)));
1476 ndescs = 0;
1477
1478 if (budget) {
1479 return (EINPROGRESS);
1480 }
1481 }
1482 }
1483
1484 if (STAILQ_EMPTY(&iql))
1485 break;
1486
1487 /*
1488 * Process the head only, and send it to the back of the list if
1489 * it's still not done.
1490 */
1491 q = STAILQ_FIRST(&iql);
1492 STAILQ_REMOVE_HEAD(&iql, link);
1493 if (service_iq_fl(q, q->qsize / 8) == 0)
1494 (void) atomic_cmpset_int(&q->state, IQS_BUSY, IQS_IDLE);
1495 else
1496 STAILQ_INSERT_TAIL(&iql, q, link);
1497 }
1498
1499 t4_write_reg(sc, sc->sge_gts_reg, V_CIDXINC(ndescs) |
1500 V_INGRESSQID((u32)iq->cntxt_id) | V_SEINTARM(iq->intr_params));
1501
1502 return (0);
1503}
1504
1505#if defined(INET) || defined(INET6)
1506static inline int
1507sort_before_lro(struct lro_ctrl *lro)
1508{
1509
1510 return (lro->lro_mbuf_max != 0);
1511}
1512#endif
1513
1514static inline uint64_t
1515last_flit_to_ns(struct adapter *sc, uint64_t lf)
1516{
1517 uint64_t n = be64toh(lf) & 0xfffffffffffffff; /* 60b, not 64b. */
1518
1519 if (n > UINT64_MAX / 1000000)
1520 return (n / sc->params.vpd.cclk * 1000000);
1521 else
1522 return (n * 1000000 / sc->params.vpd.cclk);
1523}
1524
1525static inline void
1526move_to_next_rxbuf(struct sge_fl *fl)
1527{
1528
1529 fl->rx_offset = 0;
1530 if (__predict_false((++fl->cidx & 7) == 0)) {
1531 uint16_t cidx = fl->cidx >> 3;
1532
1533 if (__predict_false(cidx == fl->sidx))
1534 fl->cidx = cidx = 0;
1535 fl->hw_cidx = cidx;
1536 }
1537}
1538
1539/*
1540 * Deals with interrupts on an iq+fl queue.
1541 */
1542static int
1543service_iq_fl(struct sge_iq *iq, int budget)
1544{
1545 struct sge_rxq *rxq = iq_to_rxq(iq);
1546 struct sge_fl *fl;
1547 struct adapter *sc = iq->adapter;
1548 struct iq_desc *d = &iq->desc[iq->cidx];
1549 int ndescs, limit;
1550 int rsp_type, starved;
1551 uint32_t lq;
1552 uint16_t fl_hw_cidx;
1553 struct mbuf *m0;
1554#if defined(INET) || defined(INET6)
1555 const struct timeval lro_timeout = {0, sc->lro_timeout};
1556 struct lro_ctrl *lro = &rxq->lro;
1557#endif
1558
1559 KASSERT(iq->state == IQS_BUSY, ("%s: iq %p not BUSY", __func__, iq));
1560 MPASS(iq->flags & IQ_HAS_FL);
1561
1562 ndescs = 0;
1563#if defined(INET) || defined(INET6)
1564 if (iq->flags & IQ_ADJ_CREDIT) {
1565 MPASS(sort_before_lro(lro));
1566 iq->flags &= ~IQ_ADJ_CREDIT;
1567 if ((d->rsp.u.type_gen & F_RSPD_GEN) != iq->gen) {
1568 tcp_lro_flush_all(lro);
1569 t4_write_reg(sc, sc->sge_gts_reg, V_CIDXINC(1) |
1570 V_INGRESSQID((u32)iq->cntxt_id) |
1571 V_SEINTARM(iq->intr_params));
1572 return (0);
1573 }
1574 ndescs = 1;
1575 }
1576#else
1577 MPASS((iq->flags & IQ_ADJ_CREDIT) == 0);
1578#endif
1579
1580 limit = budget ? budget : iq->qsize / 16;
1581 fl = &rxq->fl;
1582 fl_hw_cidx = fl->hw_cidx; /* stable snapshot */
1583 while ((d->rsp.u.type_gen & F_RSPD_GEN) == iq->gen) {
1584
1585 rmb();
1586
1587 m0 = NULL;
1588 rsp_type = G_RSPD_TYPE(d->rsp.u.type_gen);
1589 lq = be32toh(d->rsp.pldbuflen_qid);
1590
1591 switch (rsp_type) {
1592 case X_RSPD_TYPE_FLBUF:
1593 if (lq & F_RSPD_NEWBUF) {
1594 if (fl->rx_offset > 0)
1595 move_to_next_rxbuf(fl);
1596 lq = G_RSPD_LEN(lq);
1597 }
1598 if (IDXDIFF(fl->hw_cidx, fl_hw_cidx, fl->sidx) > 4) {
1599 FL_LOCK(fl);
1600 refill_fl(sc, fl, 64);
1601 FL_UNLOCK(fl);
1602 fl_hw_cidx = fl->hw_cidx;
1603 }
1604
1605 if (d->rss.opcode == CPL_RX_PKT) {
1606 if (__predict_true(eth_rx(sc, rxq, d, lq) == 0))
1607 break;
1608 goto out;
1609 }
1610 m0 = get_fl_payload(sc, fl, lq);
1611 if (__predict_false(m0 == NULL))
1612 goto out;
1613
1614 /* fall through */
1615
1616 case X_RSPD_TYPE_CPL:
1617 KASSERT(d->rss.opcode < NUM_CPL_CMDS,
1618 ("%s: bad opcode %02x.", __func__, d->rss.opcode));
1619 t4_cpl_handler[d->rss.opcode](iq, &d->rss, m0);
1620 break;
1621
1622 case X_RSPD_TYPE_INTR:
1623
1624 /*
1625 * There are 1K interrupt-capable queues (qids 0
1626 * through 1023). A response type indicating a
1627 * forwarded interrupt with a qid >= 1K is an
1628 * iWARP async notification. That is the only
1629 * acceptable indirect interrupt on this queue.
1630 */
1631 if (__predict_false(lq < 1024)) {
1632 panic("%s: indirect interrupt on iq_fl %p "
1633 "with qid %u", __func__, iq, lq);
1634 }
1635
1636 t4_an_handler(iq, &d->rsp);
1637 break;
1638
1639 default:
1640 KASSERT(0, ("%s: illegal response type %d on iq %p",
1641 __func__, rsp_type, iq));
1642 log(LOG_ERR, "%s: illegal response type %d on iq %p",
1643 device_get_nameunit(sc->dev), rsp_type, iq);
1644 break;
1645 }
1646
1647 d++;
1648 if (__predict_false(++iq->cidx == iq->sidx)) {
1649 iq->cidx = 0;
1650 iq->gen ^= F_RSPD_GEN;
1651 d = &iq->desc[0];
1652 }
1653 if (__predict_false(++ndescs == limit)) {
1654 t4_write_reg(sc, sc->sge_gts_reg, V_CIDXINC(ndescs) |
1655 V_INGRESSQID(iq->cntxt_id) |
1656 V_SEINTARM(V_QINTR_TIMER_IDX(X_TIMERREG_UPDATE_CIDX)));
1657
1658#if defined(INET) || defined(INET6)
1659 if (iq->flags & IQ_LRO_ENABLED &&
1660 !sort_before_lro(lro) &&
1661 sc->lro_timeout != 0) {
1662 tcp_lro_flush_inactive(lro, &lro_timeout);
1663 }
1664#endif
1665 if (budget)
1666 return (EINPROGRESS);
1667 ndescs = 0;
1668 }
1669 }
1670out:
1671#if defined(INET) || defined(INET6)
1672 if (iq->flags & IQ_LRO_ENABLED) {
1673 if (ndescs > 0 && lro->lro_mbuf_count > 8) {
1674 MPASS(sort_before_lro(lro));
1675 /* hold back one credit and don't flush LRO state */
1676 iq->flags |= IQ_ADJ_CREDIT;
1677 ndescs--;
1678 } else {
1679 tcp_lro_flush_all(lro);
1680 }
1681 }
1682#endif
1683
1684 t4_write_reg(sc, sc->sge_gts_reg, V_CIDXINC(ndescs) |
1685 V_INGRESSQID((u32)iq->cntxt_id) | V_SEINTARM(iq->intr_params));
1686
1687 FL_LOCK(fl);
1688 starved = refill_fl(sc, fl, 64);
1689 FL_UNLOCK(fl);
1690 if (__predict_false(starved != 0))
1691 add_fl_to_sfl(sc, fl);
1692
1693 return (0);
1694}
1695
1696static inline struct cluster_metadata *
1697cl_metadata(struct fl_sdesc *sd)
1698{
1699
1700 return ((void *)(sd->cl + sd->moff));
1701}
1702
1703static void
1704rxb_free(struct mbuf *m)
1705{
1706 struct cluster_metadata *clm = m->m_ext.ext_arg1;
1707
1708 uma_zfree(clm->zone, clm->cl);
1709 counter_u64_add(extfree_rels, 1);
1710}
1711
1712/*
1713 * The mbuf returned comes from zone_muf and carries the payload in one of these
1714 * ways
1715 * a) complete frame inside the mbuf
1716 * b) m_cljset (for clusters without metadata)
1717 * d) m_extaddref (cluster with metadata)
1718 */
1719static struct mbuf *
1720get_scatter_segment(struct adapter *sc, struct sge_fl *fl, int fr_offset,
1721 int remaining)
1722{
1723 struct mbuf *m;
1724 struct fl_sdesc *sd = &fl->sdesc[fl->cidx];
1725 struct rx_buf_info *rxb = &sc->sge.rx_buf_info[sd->zidx];
1726 struct cluster_metadata *clm;
1727 int len, blen;
1728 caddr_t payload;
1729
1730 if (fl->flags & FL_BUF_PACKING) {
1731 u_int l, pad;
1732
1733 blen = rxb->size2 - fl->rx_offset; /* max possible in this buf */
1734 len = min(remaining, blen);
1735 payload = sd->cl + fl->rx_offset;
1736
1737 l = fr_offset + len;
1738 pad = roundup2(l, fl->buf_boundary) - l;
1739 if (fl->rx_offset + len + pad < rxb->size2)
1740 blen = len + pad;
1741 MPASS(fl->rx_offset + blen <= rxb->size2);
1742 } else {
1743 MPASS(fl->rx_offset == 0); /* not packing */
1744 blen = rxb->size1;
1745 len = min(remaining, blen);
1746 payload = sd->cl;
1747 }
1748
1749 if (fr_offset == 0) {
1750 m = m_gethdr(M_NOWAIT, MT_DATA);
1751 if (__predict_false(m == NULL))
1752 return (NULL);
1753 m->m_pkthdr.len = remaining;
1754 } else {
1755 m = m_get(M_NOWAIT, MT_DATA);
1756 if (__predict_false(m == NULL))
1757 return (NULL);
1758 }
1759 m->m_len = len;
1760 kmsan_mark(payload, len, KMSAN_STATE_INITED);
1761
1762 if (sc->sc_do_rxcopy && len < RX_COPY_THRESHOLD) {
1763 /* copy data to mbuf */
1764 bcopy(payload, mtod(m, caddr_t), len);
1765 if (fl->flags & FL_BUF_PACKING) {
1766 fl->rx_offset += blen;
1767 MPASS(fl->rx_offset <= rxb->size2);
1768 if (fl->rx_offset < rxb->size2)
1769 return (m); /* without advancing the cidx */
1770 }
1771 } else if (fl->flags & FL_BUF_PACKING) {
1772 clm = cl_metadata(sd);
1773 if (sd->nmbuf++ == 0) {
1774 clm->refcount = 1;
1775 clm->zone = rxb->zone;
1776 clm->cl = sd->cl;
1777 counter_u64_add(extfree_refs, 1);
1778 }
1779 m_extaddref(m, payload, blen, &clm->refcount, rxb_free, clm,
1780 NULL);
1781
1782 fl->rx_offset += blen;
1783 MPASS(fl->rx_offset <= rxb->size2);
1784 if (fl->rx_offset < rxb->size2)
1785 return (m); /* without advancing the cidx */
1786 } else {
1787 m_cljset(m, sd->cl, rxb->type);
1788 sd->cl = NULL; /* consumed, not a recycle candidate */
1789 }
1790
1791 move_to_next_rxbuf(fl);
1792
1793 return (m);
1794}
1795
1796static struct mbuf *
1797get_fl_payload(struct adapter *sc, struct sge_fl *fl, const u_int plen)
1798{
1799 struct mbuf *m0, *m, **pnext;
1800 u_int remaining;
1801
1802 if (__predict_false(fl->flags & FL_BUF_RESUME)) {
1803 M_ASSERTPKTHDR(fl->m0);
1804 MPASS(fl->m0->m_pkthdr.len == plen);
1805 MPASS(fl->remaining < plen);
1806
1807 m0 = fl->m0;
1808 pnext = fl->pnext;
1809 remaining = fl->remaining;
1810 fl->flags &= ~FL_BUF_RESUME;
1811 goto get_segment;
1812 }
1813
1814 /*
1815 * Payload starts at rx_offset in the current hw buffer. Its length is
1816 * 'len' and it may span multiple hw buffers.
1817 */
1818
1819 m0 = get_scatter_segment(sc, fl, 0, plen);
1820 if (m0 == NULL)
1821 return (NULL);
1822 remaining = plen - m0->m_len;
1823 pnext = &m0->m_next;
1824 while (remaining > 0) {
1825get_segment:
1826 MPASS(fl->rx_offset == 0);
1827 m = get_scatter_segment(sc, fl, plen - remaining, remaining);
1828 if (__predict_false(m == NULL)) {
1829 fl->m0 = m0;
1830 fl->pnext = pnext;
1831 fl->remaining = remaining;
1832 fl->flags |= FL_BUF_RESUME;
1833 return (NULL);
1834 }
1835 *pnext = m;
1836 pnext = &m->m_next;
1837 remaining -= m->m_len;
1838 }
1839 *pnext = NULL;
1840
1841 M_ASSERTPKTHDR(m0);
1842 return (m0);
1843}
1844
1845static int
1846skip_scatter_segment(struct adapter *sc, struct sge_fl *fl, int fr_offset,
1847 int remaining)
1848{
1849 struct fl_sdesc *sd = &fl->sdesc[fl->cidx];
1850 struct rx_buf_info *rxb = &sc->sge.rx_buf_info[sd->zidx];
1851 int len, blen;
1852
1853 if (fl->flags & FL_BUF_PACKING) {
1854 u_int l, pad;
1855
1856 blen = rxb->size2 - fl->rx_offset; /* max possible in this buf */
1857 len = min(remaining, blen);
1858
1859 l = fr_offset + len;
1860 pad = roundup2(l, fl->buf_boundary) - l;
1861 if (fl->rx_offset + len + pad < rxb->size2)
1862 blen = len + pad;
1863 fl->rx_offset += blen;
1864 MPASS(fl->rx_offset <= rxb->size2);
1865 if (fl->rx_offset < rxb->size2)
1866 return (len); /* without advancing the cidx */
1867 } else {
1868 MPASS(fl->rx_offset == 0); /* not packing */
1869 blen = rxb->size1;
1870 len = min(remaining, blen);
1871 }
1872 move_to_next_rxbuf(fl);
1873 return (len);
1874}
1875
1876static inline void
1877skip_fl_payload(struct adapter *sc, struct sge_fl *fl, int plen)
1878{
1879 int remaining, fr_offset, len;
1880
1881 fr_offset = 0;
1882 remaining = plen;
1883 while (remaining > 0) {
1884 len = skip_scatter_segment(sc, fl, fr_offset, remaining);
1885 fr_offset += len;
1886 remaining -= len;
1887 }
1888}
1889
1890static inline int
1891get_segment_len(struct adapter *sc, struct sge_fl *fl, int plen)
1892{
1893 int len;
1894 struct fl_sdesc *sd = &fl->sdesc[fl->cidx];
1895 struct rx_buf_info *rxb = &sc->sge.rx_buf_info[sd->zidx];
1896
1897 if (fl->flags & FL_BUF_PACKING)
1898 len = rxb->size2 - fl->rx_offset;
1899 else
1900 len = rxb->size1;
1901
1902 return (min(plen, len));
1903}
1904
1905static int
1906eth_rx(struct adapter *sc, struct sge_rxq *rxq, const struct iq_desc *d,
1907 u_int plen)
1908{
1909 struct mbuf *m0;
1910 struct ifnet *ifp = rxq->ifp;
1911 struct sge_fl *fl = &rxq->fl;
1912 struct vi_info *vi = ifp->if_softc;
1913 const struct cpl_rx_pkt *cpl;
1914#if defined(INET) || defined(INET6)
1915 struct lro_ctrl *lro = &rxq->lro;
1916#endif
1917 uint16_t err_vec, tnl_type, tnlhdr_len;
1918 static const int sw_hashtype[4][2] = {
1919 {M_HASHTYPE_NONE, M_HASHTYPE_NONE},
1920 {M_HASHTYPE_RSS_IPV4, M_HASHTYPE_RSS_IPV6},
1921 {M_HASHTYPE_RSS_TCP_IPV4, M_HASHTYPE_RSS_TCP_IPV6},
1922 {M_HASHTYPE_RSS_UDP_IPV4, M_HASHTYPE_RSS_UDP_IPV6},
1923 };
1924 static const int sw_csum_flags[2][2] = {
1925 {
1926 /* IP, inner IP */
1927 CSUM_ENCAP_VXLAN |
1928 CSUM_L3_CALC | CSUM_L3_VALID |
1929 CSUM_L4_CALC | CSUM_L4_VALID |
1930 CSUM_INNER_L3_CALC | CSUM_INNER_L3_VALID |
1931 CSUM_INNER_L4_CALC | CSUM_INNER_L4_VALID,
1932
1933 /* IP, inner IP6 */
1934 CSUM_ENCAP_VXLAN |
1935 CSUM_L3_CALC | CSUM_L3_VALID |
1936 CSUM_L4_CALC | CSUM_L4_VALID |
1937 CSUM_INNER_L4_CALC | CSUM_INNER_L4_VALID,
1938 },
1939 {
1940 /* IP6, inner IP */
1941 CSUM_ENCAP_VXLAN |
1942 CSUM_L4_CALC | CSUM_L4_VALID |
1943 CSUM_INNER_L3_CALC | CSUM_INNER_L3_VALID |
1944 CSUM_INNER_L4_CALC | CSUM_INNER_L4_VALID,
1945
1946 /* IP6, inner IP6 */
1947 CSUM_ENCAP_VXLAN |
1948 CSUM_L4_CALC | CSUM_L4_VALID |
1949 CSUM_INNER_L4_CALC | CSUM_INNER_L4_VALID,
1950 },
1951 };
1952
1953 MPASS(plen > sc->params.sge.fl_pktshift);
1954 if (vi->pfil != NULL && PFIL_HOOKED_IN(vi->pfil) &&
1955 __predict_true((fl->flags & FL_BUF_RESUME) == 0)) {
1956 struct fl_sdesc *sd = &fl->sdesc[fl->cidx];
1957 caddr_t frame;
1958 int rc, slen;
1959
1960 slen = get_segment_len(sc, fl, plen) -
1961 sc->params.sge.fl_pktshift;
1962 frame = sd->cl + fl->rx_offset + sc->params.sge.fl_pktshift;
1963 CURVNET_SET_QUIET(ifp->if_vnet);
1964 rc = pfil_run_hooks(vi->pfil, frame, ifp,
1965 slen | PFIL_MEMPTR | PFIL_IN, NULL);
1966 CURVNET_RESTORE();
1967 if (rc == PFIL_DROPPED || rc == PFIL_CONSUMED) {
1968 skip_fl_payload(sc, fl, plen);
1969 return (0);
1970 }
1971 if (rc == PFIL_REALLOCED) {
1972 skip_fl_payload(sc, fl, plen);
1973 m0 = pfil_mem2mbuf(frame);
1974 goto have_mbuf;
1975 }
1976 }
1977
1978 m0 = get_fl_payload(sc, fl, plen);
1979 if (__predict_false(m0 == NULL))
1980 return (ENOMEM);
1981
1982 m0->m_pkthdr.len -= sc->params.sge.fl_pktshift;
1983 m0->m_len -= sc->params.sge.fl_pktshift;
1984 m0->m_data += sc->params.sge.fl_pktshift;
1985
1986have_mbuf:
1987 m0->m_pkthdr.rcvif = ifp;
1988 M_HASHTYPE_SET(m0, sw_hashtype[d->rss.hash_type][d->rss.ipv6]);
1989 m0->m_pkthdr.flowid = be32toh(d->rss.hash_val);
1990
1991 cpl = (const void *)(&d->rss + 1);
1992 if (sc->params.tp.rx_pkt_encap) {
1993 const uint16_t ev = be16toh(cpl->err_vec);
1994
1995 err_vec = G_T6_COMPR_RXERR_VEC(ev);
1996 tnl_type = G_T6_RX_TNL_TYPE(ev);
1997 tnlhdr_len = G_T6_RX_TNLHDR_LEN(ev);
1998 } else {
1999 err_vec = be16toh(cpl->err_vec);
2000 tnl_type = 0;
2001 tnlhdr_len = 0;
2002 }
2003 if (cpl->csum_calc && err_vec == 0) {
2004 int ipv6 = !!(cpl->l2info & htobe32(F_RXF_IP6));
2005
2006 /* checksum(s) calculated and found to be correct. */
2007
2008 MPASS((cpl->l2info & htobe32(F_RXF_IP)) ^
2009 (cpl->l2info & htobe32(F_RXF_IP6)));
2010 m0->m_pkthdr.csum_data = be16toh(cpl->csum);
2011 if (tnl_type == 0) {
2012 if (!ipv6 && ifp->if_capenable & IFCAP_RXCSUM) {
2013 m0->m_pkthdr.csum_flags = CSUM_L3_CALC |
2014 CSUM_L3_VALID | CSUM_L4_CALC |
2015 CSUM_L4_VALID;
2016 } else if (ipv6 && ifp->if_capenable & IFCAP_RXCSUM_IPV6) {
2017 m0->m_pkthdr.csum_flags = CSUM_L4_CALC |
2018 CSUM_L4_VALID;
2019 }
2020 rxq->rxcsum++;
2021 } else {
2022 MPASS(tnl_type == RX_PKT_TNL_TYPE_VXLAN);
2023
2024 M_HASHTYPE_SETINNER(m0);
2025 if (__predict_false(cpl->ip_frag)) {
2026 /*
2027 * csum_data is for the inner frame (which is an
2028 * IP fragment) and is not 0xffff. There is no
2029 * way to pass the inner csum_data to the stack.
2030 * We don't want the stack to use the inner
2031 * csum_data to validate the outer frame or it
2032 * will get rejected. So we fix csum_data here
2033 * and let sw do the checksum of inner IP
2034 * fragments.
2035 *
2036 * XXX: Need 32b for csum_data2 in an rx mbuf.
2037 * Maybe stuff it into rcv_tstmp?
2038 */
2039 m0->m_pkthdr.csum_data = 0xffff;
2040 if (ipv6) {
2041 m0->m_pkthdr.csum_flags = CSUM_L4_CALC |
2042 CSUM_L4_VALID;
2043 } else {
2044 m0->m_pkthdr.csum_flags = CSUM_L3_CALC |
2045 CSUM_L3_VALID | CSUM_L4_CALC |
2046 CSUM_L4_VALID;
2047 }
2048 } else {
2049 int outer_ipv6;
2050
2051 MPASS(m0->m_pkthdr.csum_data == 0xffff);
2052
2053 outer_ipv6 = tnlhdr_len >=
2054 sizeof(struct ether_header) +
2055 sizeof(struct ip6_hdr);
2056 m0->m_pkthdr.csum_flags =
2057 sw_csum_flags[outer_ipv6][ipv6];
2058 }
2059 rxq->vxlan_rxcsum++;
2060 }
2061 }
2062
2063 if (cpl->vlan_ex) {
2064 m0->m_pkthdr.ether_vtag = be16toh(cpl->vlan);
2065 m0->m_flags |= M_VLANTAG;
2066 rxq->vlan_extraction++;
2067 }
2068
2069 if (rxq->iq.flags & IQ_RX_TIMESTAMP) {
2070 /*
2071 * Fill up rcv_tstmp but do not set M_TSTMP.
2072 * rcv_tstmp is not in the format that the
2073 * kernel expects and we don't want to mislead
2074 * it. For now this is only for custom code
2075 * that knows how to interpret cxgbe's stamp.
2076 */
2077 m0->m_pkthdr.rcv_tstmp =
2078 last_flit_to_ns(sc, d->rsp.u.last_flit);
2079#ifdef notyet
2080 m0->m_flags |= M_TSTMP;
2081#endif
2082 }
2083
2084#ifdef NUMA
2085 m0->m_pkthdr.numa_domain = ifp->if_numa_domain;
2086#endif
2087#if defined(INET) || defined(INET6)
2088 if (rxq->iq.flags & IQ_LRO_ENABLED && tnl_type == 0 &&
2089 (M_HASHTYPE_GET(m0) == M_HASHTYPE_RSS_TCP_IPV4 ||
2090 M_HASHTYPE_GET(m0) == M_HASHTYPE_RSS_TCP_IPV6)) {
2091 if (sort_before_lro(lro)) {
2092 tcp_lro_queue_mbuf(lro, m0);
2093 return (0); /* queued for sort, then LRO */
2094 }
2095 if (tcp_lro_rx(lro, m0, 0) == 0)
2096 return (0); /* queued for LRO */
2097 }
2098#endif
2099 ifp->if_input(ifp, m0);
2100
2101 return (0);
2102}
2103
2104/*
2105 * Must drain the wrq or make sure that someone else will.
2106 */
2107static void
2108wrq_tx_drain(void *arg, int n)
2109{
2110 struct sge_wrq *wrq = arg;
2111 struct sge_eq *eq = &wrq->eq;
2112
2113 EQ_LOCK(eq);
2114 if (TAILQ_EMPTY(&wrq->incomplete_wrs) && !STAILQ_EMPTY(&wrq->wr_list))
2115 drain_wrq_wr_list(wrq->adapter, wrq);
2116 EQ_UNLOCK(eq);
2117}
2118
2119static void
2120drain_wrq_wr_list(struct adapter *sc, struct sge_wrq *wrq)
2121{
2122 struct sge_eq *eq = &wrq->eq;
2123 u_int available, dbdiff; /* # of hardware descriptors */
2124 u_int n;
2125 struct wrqe *wr;
2126 struct fw_eth_tx_pkt_wr *dst; /* any fw WR struct will do */
2127
2128 EQ_LOCK_ASSERT_OWNED(eq);
2129 MPASS(TAILQ_EMPTY(&wrq->incomplete_wrs));
2130 wr = STAILQ_FIRST(&wrq->wr_list);
2131 MPASS(wr != NULL); /* Must be called with something useful to do */
2132 MPASS(eq->pidx == eq->dbidx);
2133 dbdiff = 0;
2134
2135 do {
2136 eq->cidx = read_hw_cidx(eq);
2137 if (eq->pidx == eq->cidx)
2138 available = eq->sidx - 1;
2139 else
2140 available = IDXDIFF(eq->cidx, eq->pidx, eq->sidx) - 1;
2141
2142 MPASS(wr->wrq == wrq);
2143 n = howmany(wr->wr_len, EQ_ESIZE);
2144 if (available < n)
2145 break;
2146
2147 dst = (void *)&eq->desc[eq->pidx];
2148 if (__predict_true(eq->sidx - eq->pidx > n)) {
2149 /* Won't wrap, won't end exactly at the status page. */
2150 bcopy(&wr->wr[0], dst, wr->wr_len);
2151 eq->pidx += n;
2152 } else {
2153 int first_portion = (eq->sidx - eq->pidx) * EQ_ESIZE;
2154
2155 bcopy(&wr->wr[0], dst, first_portion);
2156 if (wr->wr_len > first_portion) {
2157 bcopy(&wr->wr[first_portion], &eq->desc[0],
2158 wr->wr_len - first_portion);
2159 }
2160 eq->pidx = n - (eq->sidx - eq->pidx);
2161 }
2162 wrq->tx_wrs_copied++;
2163
2164 if (available < eq->sidx / 4 &&
2165 atomic_cmpset_int(&eq->equiq, 0, 1)) {
2166 /*
2167 * XXX: This is not 100% reliable with some
2168 * types of WRs. But this is a very unusual
2169 * situation for an ofld/ctrl queue anyway.
2170 */
2171 dst->equiq_to_len16 |= htobe32(F_FW_WR_EQUIQ |
2172 F_FW_WR_EQUEQ);
2173 }
2174
2175 dbdiff += n;
2176 if (dbdiff >= 16) {
2177 ring_eq_db(sc, eq, dbdiff);
2178 dbdiff = 0;
2179 }
2180
2181 STAILQ_REMOVE_HEAD(&wrq->wr_list, link);
2182 free_wrqe(wr);
2183 MPASS(wrq->nwr_pending > 0);
2184 wrq->nwr_pending--;
2185 MPASS(wrq->ndesc_needed >= n);
2186 wrq->ndesc_needed -= n;
2187 } while ((wr = STAILQ_FIRST(&wrq->wr_list)) != NULL);
2188
2189 if (dbdiff)
2190 ring_eq_db(sc, eq, dbdiff);
2191}
2192
2193/*
2194 * Doesn't fail. Holds on to work requests it can't send right away.
2195 */
2196void
2197t4_wrq_tx_locked(struct adapter *sc, struct sge_wrq *wrq, struct wrqe *wr)
2198{
2199#ifdef INVARIANTS
2200 struct sge_eq *eq = &wrq->eq;
2201#endif
2202
2203 EQ_LOCK_ASSERT_OWNED(eq);
2204 MPASS(wr != NULL);
2205 MPASS(wr->wr_len > 0 && wr->wr_len <= SGE_MAX_WR_LEN);
2206 MPASS((wr->wr_len & 0x7) == 0);
2207
2208 STAILQ_INSERT_TAIL(&wrq->wr_list, wr, link);
2209 wrq->nwr_pending++;
2210 wrq->ndesc_needed += howmany(wr->wr_len, EQ_ESIZE);
2211
2212 if (!TAILQ_EMPTY(&wrq->incomplete_wrs))
2213 return; /* commit_wrq_wr will drain wr_list as well. */
2214
2215 drain_wrq_wr_list(sc, wrq);
2216
2217 /* Doorbell must have caught up to the pidx. */
2218 MPASS(eq->pidx == eq->dbidx);
2219}
2220
2221void
2222t4_update_fl_bufsize(struct ifnet *ifp)
2223{
2224 struct vi_info *vi = ifp->if_softc;
2225 struct adapter *sc = vi->adapter;
2226 struct sge_rxq *rxq;
2227#ifdef TCP_OFFLOAD
2228 struct sge_ofld_rxq *ofld_rxq;
2229#endif
2230 struct sge_fl *fl;
2231 int i, maxp;
2232
2233 maxp = max_rx_payload(sc, ifp, false);
2234 for_each_rxq(vi, i, rxq) {
2235 fl = &rxq->fl;
2236
2237 FL_LOCK(fl);
2238 fl->zidx = find_refill_source(sc, maxp,
2239 fl->flags & FL_BUF_PACKING);
2240 FL_UNLOCK(fl);
2241 }
2242#ifdef TCP_OFFLOAD
2243 maxp = max_rx_payload(sc, ifp, true);
2244 for_each_ofld_rxq(vi, i, ofld_rxq) {
2245 fl = &ofld_rxq->fl;
2246
2247 FL_LOCK(fl);
2248 fl->zidx = find_refill_source(sc, maxp,
2249 fl->flags & FL_BUF_PACKING);
2250 FL_UNLOCK(fl);
2251 }
2252#endif
2253}
2254
2255static inline int
2256mbuf_nsegs(struct mbuf *m)
2257{
2258
2259 M_ASSERTPKTHDR(m);
2260 KASSERT(m->m_pkthdr.inner_l5hlen > 0,
2261 ("%s: mbuf %p missing information on # of segments.", __func__, m));
2262
2263 return (m->m_pkthdr.inner_l5hlen);
2264}
2265
2266static inline void
2267set_mbuf_nsegs(struct mbuf *m, uint8_t nsegs)
2268{
2269
2270 M_ASSERTPKTHDR(m);
2271 m->m_pkthdr.inner_l5hlen = nsegs;
2272}
2273
2274static inline int
2275mbuf_cflags(struct mbuf *m)
2276{
2277
2278 M_ASSERTPKTHDR(m);
2279 return (m->m_pkthdr.PH_loc.eight[4]);
2280}
2281
2282static inline void
2283set_mbuf_cflags(struct mbuf *m, uint8_t flags)
2284{
2285
2286 M_ASSERTPKTHDR(m);
2287 m->m_pkthdr.PH_loc.eight[4] = flags;
2288}
2289
2290static inline int
2291mbuf_len16(struct mbuf *m)
2292{
2293 int n;
2294
2295 M_ASSERTPKTHDR(m);
2296 n = m->m_pkthdr.PH_loc.eight[0];
2297 if (!(mbuf_cflags(m) & MC_TLS))
2298 MPASS(n > 0 && n <= SGE_MAX_WR_LEN / 16);
2299
2300 return (n);
2301}
2302
2303static inline void
2304set_mbuf_len16(struct mbuf *m, uint8_t len16)
2305{
2306
2307 M_ASSERTPKTHDR(m);
2308 if (!(mbuf_cflags(m) & MC_TLS))
2309 MPASS(len16 > 0 && len16 <= SGE_MAX_WR_LEN / 16);
2310 m->m_pkthdr.PH_loc.eight[0] = len16;
2311}
2312
2313#ifdef RATELIMIT
2314static inline int
2315mbuf_eo_nsegs(struct mbuf *m)
2316{
2317
2318 M_ASSERTPKTHDR(m);
2319 return (m->m_pkthdr.PH_loc.eight[1]);
2320}
2321
2322#if defined(INET) || defined(INET6)
2323static inline void
2324set_mbuf_eo_nsegs(struct mbuf *m, uint8_t nsegs)
2325{
2326
2327 M_ASSERTPKTHDR(m);
2328 m->m_pkthdr.PH_loc.eight[1] = nsegs;
2329}
2330#endif
2331
2332static inline int
2333mbuf_eo_len16(struct mbuf *m)
2334{
2335 int n;
2336
2337 M_ASSERTPKTHDR(m);
2338 n = m->m_pkthdr.PH_loc.eight[2];
2339 MPASS(n > 0 && n <= SGE_MAX_WR_LEN / 16);
2340
2341 return (n);
2342}
2343
2344#if defined(INET) || defined(INET6)
2345static inline void
2346set_mbuf_eo_len16(struct mbuf *m, uint8_t len16)
2347{
2348
2349 M_ASSERTPKTHDR(m);
2350 m->m_pkthdr.PH_loc.eight[2] = len16;
2351}
2352#endif
2353
2354static inline int
2355mbuf_eo_tsclk_tsoff(struct mbuf *m)
2356{
2357
2358 M_ASSERTPKTHDR(m);
2359 return (m->m_pkthdr.PH_loc.eight[3]);
2360}
2361
2362#if defined(INET) || defined(INET6)
2363static inline void
2364set_mbuf_eo_tsclk_tsoff(struct mbuf *m, uint8_t tsclk_tsoff)
2365{
2366
2367 M_ASSERTPKTHDR(m);
2368 m->m_pkthdr.PH_loc.eight[3] = tsclk_tsoff;
2369}
2370#endif
2371
2372static inline int
2373needs_eo(struct m_snd_tag *mst)
2374{
2375
2376 return (mst != NULL && mst->sw->type == IF_SND_TAG_TYPE_RATE_LIMIT);
2377}
2378#endif
2379
2380/*
2381 * Try to allocate an mbuf to contain a raw work request. To make it
2382 * easy to construct the work request, don't allocate a chain but a
2383 * single mbuf.
2384 */
2385struct mbuf *
2386alloc_wr_mbuf(int len, int how)
2387{
2388 struct mbuf *m;
2389
2390 if (len <= MHLEN)
2391 m = m_gethdr(how, MT_DATA);
2392 else if (len <= MCLBYTES)
2393 m = m_getcl(how, MT_DATA, M_PKTHDR);
2394 else
2395 m = NULL;
2396 if (m == NULL)
2397 return (NULL);
2398 m->m_pkthdr.len = len;
2399 m->m_len = len;
2400 set_mbuf_cflags(m, MC_RAW_WR);
2401 set_mbuf_len16(m, howmany(len, 16));
2402 return (m);
2403}
2404
2405static inline bool
2406needs_hwcsum(struct mbuf *m)
2407{
2408 const uint32_t csum_flags = CSUM_IP | CSUM_IP_UDP | CSUM_IP_TCP |
2409 CSUM_IP_TSO | CSUM_INNER_IP | CSUM_INNER_IP_UDP |
2410 CSUM_INNER_IP_TCP | CSUM_INNER_IP_TSO | CSUM_IP6_UDP |
2411 CSUM_IP6_TCP | CSUM_IP6_TSO | CSUM_INNER_IP6_UDP |
2412 CSUM_INNER_IP6_TCP | CSUM_INNER_IP6_TSO;
2413
2414 M_ASSERTPKTHDR(m);
2415
2416 return (m->m_pkthdr.csum_flags & csum_flags);
2417}
2418
2419static inline bool
2420needs_tso(struct mbuf *m)
2421{
2422 const uint32_t csum_flags = CSUM_IP_TSO | CSUM_IP6_TSO |
2423 CSUM_INNER_IP_TSO | CSUM_INNER_IP6_TSO;
2424
2425 M_ASSERTPKTHDR(m);
2426
2427 return (m->m_pkthdr.csum_flags & csum_flags);
2428}
2429
2430static inline bool
2431needs_vxlan_csum(struct mbuf *m)
2432{
2433
2434 M_ASSERTPKTHDR(m);
2435
2436 return (m->m_pkthdr.csum_flags & CSUM_ENCAP_VXLAN);
2437}
2438
2439static inline bool
2440needs_vxlan_tso(struct mbuf *m)
2441{
2442 const uint32_t csum_flags = CSUM_ENCAP_VXLAN | CSUM_INNER_IP_TSO |
2443 CSUM_INNER_IP6_TSO;
2444
2445 M_ASSERTPKTHDR(m);
2446
2447 return ((m->m_pkthdr.csum_flags & csum_flags) != 0 &&
2448 (m->m_pkthdr.csum_flags & csum_flags) != CSUM_ENCAP_VXLAN);
2449}
2450
2451#if defined(INET) || defined(INET6)
2452static inline bool
2453needs_inner_tcp_csum(struct mbuf *m)
2454{
2455 const uint32_t csum_flags = CSUM_INNER_IP_TSO | CSUM_INNER_IP6_TSO;
2456
2457 M_ASSERTPKTHDR(m);
2458
2459 return (m->m_pkthdr.csum_flags & csum_flags);
2460}
2461#endif
2462
2463static inline bool
2464needs_l3_csum(struct mbuf *m)
2465{
2466 const uint32_t csum_flags = CSUM_IP | CSUM_IP_TSO | CSUM_INNER_IP |
2467 CSUM_INNER_IP_TSO;
2468
2469 M_ASSERTPKTHDR(m);
2470
2471 return (m->m_pkthdr.csum_flags & csum_flags);
2472}
2473
2474static inline bool
2475needs_outer_tcp_csum(struct mbuf *m)
2476{
2477 const uint32_t csum_flags = CSUM_IP_TCP | CSUM_IP_TSO | CSUM_IP6_TCP |
2478 CSUM_IP6_TSO;
2479
2480 M_ASSERTPKTHDR(m);
2481
2482 return (m->m_pkthdr.csum_flags & csum_flags);
2483}
2484
2485#ifdef RATELIMIT
2486static inline bool
2487needs_outer_l4_csum(struct mbuf *m)
2488{
2489 const uint32_t csum_flags = CSUM_IP_UDP | CSUM_IP_TCP | CSUM_IP_TSO |
2490 CSUM_IP6_UDP | CSUM_IP6_TCP | CSUM_IP6_TSO;
2491
2492 M_ASSERTPKTHDR(m);
2493
2494 return (m->m_pkthdr.csum_flags & csum_flags);
2495}
2496
2497static inline bool
2498needs_outer_udp_csum(struct mbuf *m)
2499{
2500 const uint32_t csum_flags = CSUM_IP_UDP | CSUM_IP6_UDP;
2501
2502 M_ASSERTPKTHDR(m);
2503
2504 return (m->m_pkthdr.csum_flags & csum_flags);
2505}
2506#endif
2507
2508static inline bool
2509needs_vlan_insertion(struct mbuf *m)
2510{
2511
2512 M_ASSERTPKTHDR(m);
2513
2514 return (m->m_flags & M_VLANTAG);
2515}
2516
2517#if defined(INET) || defined(INET6)
2518static void *
2519m_advance(struct mbuf **pm, int *poffset, int len)
2520{
2521 struct mbuf *m = *pm;
2522 int offset = *poffset;
2523 uintptr_t p = 0;
2524
2525 MPASS(len > 0);
2526
2527 for (;;) {
2528 if (offset + len < m->m_len) {
2529 offset += len;
2530 p = mtod(m, uintptr_t) + offset;
2531 break;
2532 }
2533 len -= m->m_len - offset;
2534 m = m->m_next;
2535 offset = 0;
2536 MPASS(m != NULL);
2537 }
2538 *poffset = offset;
2539 *pm = m;
2540 return ((void *)p);
2541}
2542#endif
2543
2544static inline int
2545count_mbuf_ext_pgs(struct mbuf *m, int skip, vm_paddr_t *nextaddr)
2546{
2547 vm_paddr_t paddr;
2548 int i, len, off, pglen, pgoff, seglen, segoff;
2549 int nsegs = 0;
2550
2551 M_ASSERTEXTPG(m);
2552 off = mtod(m, vm_offset_t);
2553 len = m->m_len;
2554 off += skip;
2555 len -= skip;
2556
2557 if (m->m_epg_hdrlen != 0) {
2558 if (off >= m->m_epg_hdrlen) {
2559 off -= m->m_epg_hdrlen;
2560 } else {
2561 seglen = m->m_epg_hdrlen - off;
2562 segoff = off;
2563 seglen = min(seglen, len);
2564 off = 0;
2565 len -= seglen;
2566 paddr = pmap_kextract(
2567 (vm_offset_t)&m->m_epg_hdr[segoff]);
2568 if (*nextaddr != paddr)
2569 nsegs++;
2570 *nextaddr = paddr + seglen;
2571 }
2572 }
2573 pgoff = m->m_epg_1st_off;
2574 for (i = 0; i < m->m_epg_npgs && len > 0; i++) {
2575 pglen = m_epg_pagelen(m, i, pgoff);
2576 if (off >= pglen) {
2577 off -= pglen;
2578 pgoff = 0;
2579 continue;
2580 }
2581 seglen = pglen - off;
2582 segoff = pgoff + off;
2583 off = 0;
2584 seglen = min(seglen, len);
2585 len -= seglen;
2586 paddr = m->m_epg_pa[i] + segoff;
2587 if (*nextaddr != paddr)
2588 nsegs++;
2589 *nextaddr = paddr + seglen;
2590 pgoff = 0;
2591 };
2592 if (len != 0) {
2593 seglen = min(len, m->m_epg_trllen - off);
2594 len -= seglen;
2595 paddr = pmap_kextract((vm_offset_t)&m->m_epg_trail[off]);
2596 if (*nextaddr != paddr)
2597 nsegs++;
2598 *nextaddr = paddr + seglen;
2599 }
2600
2601 return (nsegs);
2602}
2603
2604
2605/*
2606 * Can deal with empty mbufs in the chain that have m_len = 0, but the chain
2607 * must have at least one mbuf that's not empty. It is possible for this
2608 * routine to return 0 if skip accounts for all the contents of the mbuf chain.
2609 */
2610static inline int
2611count_mbuf_nsegs(struct mbuf *m, int skip, uint8_t *cflags)
2612{
2613 vm_paddr_t nextaddr, paddr;
2614 vm_offset_t va;
2615 int len, nsegs;
2616
2617 M_ASSERTPKTHDR(m);
2618 MPASS(m->m_pkthdr.len > 0);
2619 MPASS(m->m_pkthdr.len >= skip);
2620
2621 nsegs = 0;
2622 nextaddr = 0;
2623 for (; m; m = m->m_next) {
2624 len = m->m_len;
2625 if (__predict_false(len == 0))
2626 continue;
2627 if (skip >= len) {
2628 skip -= len;
2629 continue;
2630 }
2631 if ((m->m_flags & M_EXTPG) != 0) {
2632 *cflags |= MC_NOMAP;
2633 nsegs += count_mbuf_ext_pgs(m, skip, &nextaddr);
2634 skip = 0;
2635 continue;
2636 }
2637 va = mtod(m, vm_offset_t) + skip;
2638 len -= skip;
2639 skip = 0;
2640 paddr = pmap_kextract(va);
2641 nsegs += sglist_count((void *)(uintptr_t)va, len);
2642 if (paddr == nextaddr)
2643 nsegs--;
2644 nextaddr = pmap_kextract(va + len - 1) + 1;
2645 }
2646
2647 return (nsegs);
2648}
2649
2650/*
2651 * The maximum number of segments that can fit in a WR.
2652 */
2653static int
2654max_nsegs_allowed(struct mbuf *m, bool vm_wr)
2655{
2656
2657 if (vm_wr) {
2658 if (needs_tso(m))
2659 return (TX_SGL_SEGS_VM_TSO);
2660 return (TX_SGL_SEGS_VM);
2661 }
2662
2663 if (needs_tso(m)) {
2664 if (needs_vxlan_tso(m))
2665 return (TX_SGL_SEGS_VXLAN_TSO);
2666 else
2667 return (TX_SGL_SEGS_TSO);
2668 }
2669
2670 return (TX_SGL_SEGS);
2671}
2672
2673static struct timeval txerr_ratecheck = {0};
2674static const struct timeval txerr_interval = {3, 0};
2675
2676/*
2677 * Analyze the mbuf to determine its tx needs. The mbuf passed in may change:
2678 * a) caller can assume it's been freed if this function returns with an error.
2679 * b) it may get defragged up if the gather list is too long for the hardware.
2680 */
2681int
2682parse_pkt(struct mbuf **mp, bool vm_wr)
2683{
2684 struct mbuf *m0 = *mp, *m;
2685 int rc, nsegs, defragged = 0;
2686 struct ether_header *eh;
2687#ifdef INET
2688 void *l3hdr;
2689#endif
2690#if defined(INET) || defined(INET6)
2691 int offset;
2692 struct tcphdr *tcp;
2693#endif
2694#if defined(KERN_TLS) || defined(RATELIMIT)
2695 struct m_snd_tag *mst;
2696#endif
2697 uint16_t eh_type;
2698 uint8_t cflags;
2699
2700 cflags = 0;
2701 M_ASSERTPKTHDR(m0);
2702 if (__predict_false(m0->m_pkthdr.len < ETHER_HDR_LEN)) {
2703 rc = EINVAL;
2704fail:
2705 m_freem(m0);
2706 *mp = NULL;
2707 return (rc);
2708 }
2709restart:
2710 /*
2711 * First count the number of gather list segments in the payload.
2712 * Defrag the mbuf if nsegs exceeds the hardware limit.
2713 */
2714 M_ASSERTPKTHDR(m0);
2715 MPASS(m0->m_pkthdr.len > 0);
2716 nsegs = count_mbuf_nsegs(m0, 0, &cflags);
2717#if defined(KERN_TLS) || defined(RATELIMIT)
2718 if (m0->m_pkthdr.csum_flags & CSUM_SND_TAG)
2719 mst = m0->m_pkthdr.snd_tag;
2720 else
2721 mst = NULL;
2722#endif
2723#ifdef KERN_TLS
2724 if (mst != NULL && mst->sw->type == IF_SND_TAG_TYPE_TLS) {
2725 int len16;
2726
2727 cflags |= MC_TLS;
2728 set_mbuf_cflags(m0, cflags);
2729 rc = t6_ktls_parse_pkt(m0, &nsegs, &len16);
2730 if (rc != 0)
2731 goto fail;
2732 set_mbuf_nsegs(m0, nsegs);
2733 set_mbuf_len16(m0, len16);
2734 return (0);
2735 }
2736#endif
2737 if (nsegs > max_nsegs_allowed(m0, vm_wr)) {
2738 if (defragged++ > 0) {
2739 rc = EFBIG;
2740 goto fail;
2741 }
2742 counter_u64_add(defrags, 1);
2743 if ((m = m_defrag(m0, M_NOWAIT)) == NULL) {
2744 rc = ENOMEM;
2745 goto fail;
2746 }
2747 *mp = m0 = m; /* update caller's copy after defrag */
2748 goto restart;
2749 }
2750
2751 if (__predict_false(nsegs > 2 && m0->m_pkthdr.len <= MHLEN &&
2752 !(cflags & MC_NOMAP))) {
2753 counter_u64_add(pullups, 1);
2754 m0 = m_pullup(m0, m0->m_pkthdr.len);
2755 if (m0 == NULL) {
2756 /* Should have left well enough alone. */
2757 rc = EFBIG;
2758 goto fail;
2759 }
2760 *mp = m0; /* update caller's copy after pullup */
2761 goto restart;
2762 }
2763 set_mbuf_nsegs(m0, nsegs);
2764 set_mbuf_cflags(m0, cflags);
2765 calculate_mbuf_len16(m0, vm_wr);
2766
2767#ifdef RATELIMIT
2768 /*
2769 * Ethofld is limited to TCP and UDP for now, and only when L4 hw
2770 * checksumming is enabled. needs_outer_l4_csum happens to check for
2771 * all the right things.
2772 */
2773 if (__predict_false(needs_eo(mst) && !needs_outer_l4_csum(m0))) {
2774 m_snd_tag_rele(m0->m_pkthdr.snd_tag);
2775 m0->m_pkthdr.snd_tag = NULL;
2776 m0->m_pkthdr.csum_flags &= ~CSUM_SND_TAG;
2777 mst = NULL;
2778 }
2779#endif
2780
2781 if (!needs_hwcsum(m0)
2782#ifdef RATELIMIT
2783 && !needs_eo(mst)
2784#endif
2785 )
2786 return (0);
2787
2788 m = m0;
2789 eh = mtod(m, struct ether_header *);
2790 eh_type = ntohs(eh->ether_type);
2791 if (eh_type == ETHERTYPE_VLAN) {
2792 struct ether_vlan_header *evh = (void *)eh;
2793
2794 eh_type = ntohs(evh->evl_proto);
2795 m0->m_pkthdr.l2hlen = sizeof(*evh);
2796 } else
2797 m0->m_pkthdr.l2hlen = sizeof(*eh);
2798
2799#if defined(INET) || defined(INET6)
2800 offset = 0;
2801#ifdef INET
2802 l3hdr = m_advance(&m, &offset, m0->m_pkthdr.l2hlen);
2803#else
2804 m_advance(&m, &offset, m0->m_pkthdr.l2hlen);
2805#endif
2806#endif
2807
2808 switch (eh_type) {
2809#ifdef INET6
2810 case ETHERTYPE_IPV6:
2811 m0->m_pkthdr.l3hlen = sizeof(struct ip6_hdr);
2812 break;
2813#endif
2814#ifdef INET
2815 case ETHERTYPE_IP:
2816 {
2817 struct ip *ip = l3hdr;
2818
2819 if (needs_vxlan_csum(m0)) {
2820 /* Driver will do the outer IP hdr checksum. */
2821 ip->ip_sum = 0;
2822 if (needs_vxlan_tso(m0)) {
2823 const uint16_t ipl = ip->ip_len;
2824
2825 ip->ip_len = 0;
2826 ip->ip_sum = ~in_cksum_hdr(ip);
2827 ip->ip_len = ipl;
2828 } else
2829 ip->ip_sum = in_cksum_hdr(ip);
2830 }
2831 m0->m_pkthdr.l3hlen = ip->ip_hl << 2;
2832 break;
2833 }
2834#endif
2835 default:
2836 if (ratecheck(&txerr_ratecheck, &txerr_interval)) {
2837 log(LOG_ERR, "%s: ethertype 0x%04x unknown. "
2838 "if_cxgbe must be compiled with the same "
2839 "INET/INET6 options as the kernel.\n", __func__,
2840 eh_type);
2841 }
2842 rc = EINVAL;
2843 goto fail;
2844 }
2845
2846#if defined(INET) || defined(INET6)
2847 if (needs_vxlan_csum(m0)) {
2848 m0->m_pkthdr.l4hlen = sizeof(struct udphdr);
2849 m0->m_pkthdr.l5hlen = sizeof(struct vxlan_header);
2850
2851 /* Inner headers. */
2852 eh = m_advance(&m, &offset, m0->m_pkthdr.l3hlen +
2853 sizeof(struct udphdr) + sizeof(struct vxlan_header));
2854 eh_type = ntohs(eh->ether_type);
2855 if (eh_type == ETHERTYPE_VLAN) {
2856 struct ether_vlan_header *evh = (void *)eh;
2857
2858 eh_type = ntohs(evh->evl_proto);
2859 m0->m_pkthdr.inner_l2hlen = sizeof(*evh);
2860 } else
2861 m0->m_pkthdr.inner_l2hlen = sizeof(*eh);
2862#ifdef INET
2863 l3hdr = m_advance(&m, &offset, m0->m_pkthdr.inner_l2hlen);
2864#else
2865 m_advance(&m, &offset, m0->m_pkthdr.inner_l2hlen);
2866#endif
2867
2868 switch (eh_type) {
2869#ifdef INET6
2870 case ETHERTYPE_IPV6:
2871 m0->m_pkthdr.inner_l3hlen = sizeof(struct ip6_hdr);
2872 break;
2873#endif
2874#ifdef INET
2875 case ETHERTYPE_IP:
2876 {
2877 struct ip *ip = l3hdr;
2878
2879 m0->m_pkthdr.inner_l3hlen = ip->ip_hl << 2;
2880 break;
2881 }
2882#endif
2883 default:
2884 if (ratecheck(&txerr_ratecheck, &txerr_interval)) {
2885 log(LOG_ERR, "%s: VXLAN hw offload requested"
2886 "with unknown ethertype 0x%04x. if_cxgbe "
2887 "must be compiled with the same INET/INET6 "
2888 "options as the kernel.\n", __func__,
2889 eh_type);
2890 }
2891 rc = EINVAL;
2892 goto fail;
2893 }
2894 if (needs_inner_tcp_csum(m0)) {
2895 tcp = m_advance(&m, &offset, m0->m_pkthdr.inner_l3hlen);
2896 m0->m_pkthdr.inner_l4hlen = tcp->th_off * 4;
2897 }
2898 MPASS((m0->m_pkthdr.csum_flags & CSUM_SND_TAG) == 0);
2899 m0->m_pkthdr.csum_flags &= CSUM_INNER_IP6_UDP |
2900 CSUM_INNER_IP6_TCP | CSUM_INNER_IP6_TSO | CSUM_INNER_IP |
2901 CSUM_INNER_IP_UDP | CSUM_INNER_IP_TCP | CSUM_INNER_IP_TSO |
2902 CSUM_ENCAP_VXLAN;
2903 }
2904
2905 if (needs_outer_tcp_csum(m0)) {
2906 tcp = m_advance(&m, &offset, m0->m_pkthdr.l3hlen);
2907 m0->m_pkthdr.l4hlen = tcp->th_off * 4;
2908#ifdef RATELIMIT
2909 if (tsclk >= 0 && *(uint32_t *)(tcp + 1) == ntohl(0x0101080a)) {
2910 set_mbuf_eo_tsclk_tsoff(m0,
2911 V_FW_ETH_TX_EO_WR_TSCLK(tsclk) |
2912 V_FW_ETH_TX_EO_WR_TSOFF(sizeof(*tcp) / 2 + 1));
2913 } else
2914 set_mbuf_eo_tsclk_tsoff(m0, 0);
2915 } else if (needs_outer_udp_csum(m0)) {
2916 m0->m_pkthdr.l4hlen = sizeof(struct udphdr);
2917#endif
2918 }
2919#ifdef RATELIMIT
2920 if (needs_eo(mst)) {
2921 u_int immhdrs;
2922
2923 /* EO WRs have the headers in the WR and not the GL. */
2924 immhdrs = m0->m_pkthdr.l2hlen + m0->m_pkthdr.l3hlen +
2925 m0->m_pkthdr.l4hlen;
2926 cflags = 0;
2927 nsegs = count_mbuf_nsegs(m0, immhdrs, &cflags);
2928 MPASS(cflags == mbuf_cflags(m0));
2929 set_mbuf_eo_nsegs(m0, nsegs);
2930 set_mbuf_eo_len16(m0,
2931 txpkt_eo_len16(nsegs, immhdrs, needs_tso(m0)));
2932 }
2933#endif
2934#endif
2935 MPASS(m0 == *mp);
2936 return (0);
2937}
2938
2939void *
2940start_wrq_wr(struct sge_wrq *wrq, int len16, struct wrq_cookie *cookie)
2941{
2942 struct sge_eq *eq = &wrq->eq;
2943 struct adapter *sc = wrq->adapter;
2944 int ndesc, available;
2945 struct wrqe *wr;
2946 void *w;
2947
2948 MPASS(len16 > 0);
2949 ndesc = tx_len16_to_desc(len16);
2950 MPASS(ndesc > 0 && ndesc <= SGE_MAX_WR_NDESC);
2951
2952 EQ_LOCK(eq);
2953
2954 if (TAILQ_EMPTY(&wrq->incomplete_wrs) && !STAILQ_EMPTY(&wrq->wr_list))
2955 drain_wrq_wr_list(sc, wrq);
2956
2957 if (!STAILQ_EMPTY(&wrq->wr_list)) {
2958slowpath:
2959 EQ_UNLOCK(eq);
2960 wr = alloc_wrqe(len16 * 16, wrq);
2961 if (__predict_false(wr == NULL))
2962 return (NULL);
2963 cookie->pidx = -1;
2964 cookie->ndesc = ndesc;
2965 return (&wr->wr);
2966 }
2967
2968 eq->cidx = read_hw_cidx(eq);
2969 if (eq->pidx == eq->cidx)
2970 available = eq->sidx - 1;
2971 else
2972 available = IDXDIFF(eq->cidx, eq->pidx, eq->sidx) - 1;
2973 if (available < ndesc)
2974 goto slowpath;
2975
2976 cookie->pidx = eq->pidx;
2977 cookie->ndesc = ndesc;
2978 TAILQ_INSERT_TAIL(&wrq->incomplete_wrs, cookie, link);
2979
2980 w = &eq->desc[eq->pidx];
2981 IDXINCR(eq->pidx, ndesc, eq->sidx);
2982 if (__predict_false(cookie->pidx + ndesc > eq->sidx)) {
2983 w = &wrq->ss[0];
2984 wrq->ss_pidx = cookie->pidx;
2985 wrq->ss_len = len16 * 16;
2986 }
2987
2988 EQ_UNLOCK(eq);
2989
2990 return (w);
2991}
2992
2993void
2994commit_wrq_wr(struct sge_wrq *wrq, void *w, struct wrq_cookie *cookie)
2995{
2996 struct sge_eq *eq = &wrq->eq;
2997 struct adapter *sc = wrq->adapter;
2998 int ndesc, pidx;
2999 struct wrq_cookie *prev, *next;
3000
3001 if (cookie->pidx == -1) {
3002 struct wrqe *wr = __containerof(w, struct wrqe, wr);
3003
3004 t4_wrq_tx(sc, wr);
3005 return;
3006 }
3007
3008 if (__predict_false(w == &wrq->ss[0])) {
3009 int n = (eq->sidx - wrq->ss_pidx) * EQ_ESIZE;
3010
3011 MPASS(wrq->ss_len > n); /* WR had better wrap around. */
3012 bcopy(&wrq->ss[0], &eq->desc[wrq->ss_pidx], n);
3013 bcopy(&wrq->ss[n], &eq->desc[0], wrq->ss_len - n);
3014 wrq->tx_wrs_ss++;
3015 } else
3016 wrq->tx_wrs_direct++;
3017
3018 EQ_LOCK(eq);
3019 ndesc = cookie->ndesc; /* Can be more than SGE_MAX_WR_NDESC here. */
3020 pidx = cookie->pidx;
3021 MPASS(pidx >= 0 && pidx < eq->sidx);
3022 prev = TAILQ_PREV(cookie, wrq_incomplete_wrs, link);
3023 next = TAILQ_NEXT(cookie, link);
3024 if (prev == NULL) {
3025 MPASS(pidx == eq->dbidx);
3026 if (next == NULL || ndesc >= 16) {
3027 int available;
3028 struct fw_eth_tx_pkt_wr *dst; /* any fw WR struct will do */
3029
3030 /*
3031 * Note that the WR via which we'll request tx updates
3032 * is at pidx and not eq->pidx, which has moved on
3033 * already.
3034 */
3035 dst = (void *)&eq->desc[pidx];
3036 available = IDXDIFF(eq->cidx, eq->pidx, eq->sidx) - 1;
3037 if (available < eq->sidx / 4 &&
3038 atomic_cmpset_int(&eq->equiq, 0, 1)) {
3039 /*
3040 * XXX: This is not 100% reliable with some
3041 * types of WRs. But this is a very unusual
3042 * situation for an ofld/ctrl queue anyway.
3043 */
3044 dst->equiq_to_len16 |= htobe32(F_FW_WR_EQUIQ |
3045 F_FW_WR_EQUEQ);
3046 }
3047
3048 ring_eq_db(wrq->adapter, eq, ndesc);
3049 } else {
3050 MPASS(IDXDIFF(next->pidx, pidx, eq->sidx) == ndesc);
3051 next->pidx = pidx;
3052 next->ndesc += ndesc;
3053 }
3054 } else {
3055 MPASS(IDXDIFF(pidx, prev->pidx, eq->sidx) == prev->ndesc);
3056 prev->ndesc += ndesc;
3057 }
3058 TAILQ_REMOVE(&wrq->incomplete_wrs, cookie, link);
3059
3060 if (TAILQ_EMPTY(&wrq->incomplete_wrs) && !STAILQ_EMPTY(&wrq->wr_list))
3061 drain_wrq_wr_list(sc, wrq);
3062
3063#ifdef INVARIANTS
3064 if (TAILQ_EMPTY(&wrq->incomplete_wrs)) {
3065 /* Doorbell must have caught up to the pidx. */
3066 MPASS(wrq->eq.pidx == wrq->eq.dbidx);
3067 }
3068#endif
3069 EQ_UNLOCK(eq);
3070}
3071
3072static u_int
3073can_resume_eth_tx(struct mp_ring *r)
3074{
3075 struct sge_eq *eq = r->cookie;
3076
3077 return (total_available_tx_desc(eq) > eq->sidx / 8);
3078}
3079
3080static inline bool
3081cannot_use_txpkts(struct mbuf *m)
3082{
3083 /* maybe put a GL limit too, to avoid silliness? */
3084
3085 return (needs_tso(m) || (mbuf_cflags(m) & (MC_RAW_WR | MC_TLS)) != 0);
3086}
3087
3088static inline int
3089discard_tx(struct sge_eq *eq)
3090{
3091
3092 return ((eq->flags & (EQ_ENABLED | EQ_QFLUSH)) != EQ_ENABLED);
3093}
3094
3095static inline int
3096wr_can_update_eq(void *p)
3097{
3098 struct fw_eth_tx_pkts_wr *wr = p;
3099
3100 switch (G_FW_WR_OP(be32toh(wr->op_pkd))) {
3101 case FW_ULPTX_WR:
3102 case FW_ETH_TX_PKT_WR:
3103 case FW_ETH_TX_PKTS_WR:
3104 case FW_ETH_TX_PKTS2_WR:
3105 case FW_ETH_TX_PKT_VM_WR:
3106 case FW_ETH_TX_PKTS_VM_WR:
3107 return (1);
3108 default:
3109 return (0);
3110 }
3111}
3112
3113static inline void
3114set_txupdate_flags(struct sge_txq *txq, u_int avail,
3115 struct fw_eth_tx_pkt_wr *wr)
3116{
3117 struct sge_eq *eq = &txq->eq;
3118 struct txpkts *txp = &txq->txp;
3119
3120 if ((txp->npkt > 0 || avail < eq->sidx / 2) &&
3121 atomic_cmpset_int(&eq->equiq, 0, 1)) {
3122 wr->equiq_to_len16 |= htobe32(F_FW_WR_EQUEQ | F_FW_WR_EQUIQ);
3123 eq->equeqidx = eq->pidx;
3124 } else if (IDXDIFF(eq->pidx, eq->equeqidx, eq->sidx) >= 32) {
3125 wr->equiq_to_len16 |= htobe32(F_FW_WR_EQUEQ);
3126 eq->equeqidx = eq->pidx;
3127 }
3128}
3129
3130#if defined(__i386__) || defined(__amd64__)
3131extern uint64_t tsc_freq;
3132#endif
3133
3134static inline bool
3135record_eth_tx_time(struct sge_txq *txq)
3136{
3137 const uint64_t cycles = get_cyclecount();
3138 const uint64_t last_tx = txq->last_tx;
3139#if defined(__i386__) || defined(__amd64__)
3140 const uint64_t itg = tsc_freq * t4_tx_coalesce_gap / 1000000;
3141#else
3142 const uint64_t itg = 0;
3143#endif
3144
3145 MPASS(cycles >= last_tx);
3146 txq->last_tx = cycles;
3147 return (cycles - last_tx < itg);
3148}
3149
3150/*
3151 * r->items[cidx] to r->items[pidx], with a wraparound at r->size, are ready to
3152 * be consumed. Return the actual number consumed. 0 indicates a stall.
3153 */
3154static u_int
3155eth_tx(struct mp_ring *r, u_int cidx, u_int pidx, bool *coalescing)
3156{
3157 struct sge_txq *txq = r->cookie;
3158 struct ifnet *ifp = txq->ifp;
3159 struct sge_eq *eq = &txq->eq;
3160 struct txpkts *txp = &txq->txp;
3161 struct vi_info *vi = ifp->if_softc;
3162 struct adapter *sc = vi->adapter;
3163 u_int total, remaining; /* # of packets */
3164 u_int n, avail, dbdiff; /* # of hardware descriptors */
3165 int i, rc;
3166 struct mbuf *m0;
3167 bool snd, recent_tx;
3168 void *wr; /* start of the last WR written to the ring */
3169
3170 TXQ_LOCK_ASSERT_OWNED(txq);
3171 recent_tx = record_eth_tx_time(txq);
3172
3173 remaining = IDXDIFF(pidx, cidx, r->size);
3174 if (__predict_false(discard_tx(eq))) {
3175 for (i = 0; i < txp->npkt; i++)
3176 m_freem(txp->mb[i]);
3177 txp->npkt = 0;
3178 while (cidx != pidx) {
3179 m0 = r->items[cidx];
3180 m_freem(m0);
3181 if (++cidx == r->size)
3182 cidx = 0;
3183 }
3184 reclaim_tx_descs(txq, eq->sidx);
3185 *coalescing = false;
3186 return (remaining); /* emptied */
3187 }
3188
3189 /* How many hardware descriptors do we have readily available. */
3190 if (eq->pidx == eq->cidx)
3191 avail = eq->sidx - 1;
3192 else
3193 avail = IDXDIFF(eq->cidx, eq->pidx, eq->sidx) - 1;
3194
3195 total = 0;
3196 if (remaining == 0) {
3197 txp->score = 0;
3198 txq->txpkts_flush++;
3199 goto send_txpkts;
3200 }
3201
3202 dbdiff = 0;
3203 MPASS(remaining > 0);
3204 while (remaining > 0) {
3205 m0 = r->items[cidx];
3206 M_ASSERTPKTHDR(m0);
3207 MPASS(m0->m_nextpkt == NULL);
3208
3209 if (avail < 2 * SGE_MAX_WR_NDESC)
3210 avail += reclaim_tx_descs(txq, 64);
3211
3212 if (t4_tx_coalesce == 0 && txp->npkt == 0)
3213 goto skip_coalescing;
3214 if (cannot_use_txpkts(m0))
3215 txp->score = 0;
3216 else if (recent_tx) {
3217 if (++txp->score == 0)
3218 txp->score = UINT8_MAX;
3219 } else
3220 txp->score = 1;
3221 if (txp->npkt > 0 || remaining > 1 ||
3222 txp->score >= t4_tx_coalesce_pkts ||
3223 atomic_load_int(&txq->eq.equiq) != 0) {
3224 if (vi->flags & TX_USES_VM_WR)
3225 rc = add_to_txpkts_vf(sc, txq, m0, avail, &snd);
3226 else
3227 rc = add_to_txpkts_pf(sc, txq, m0, avail, &snd);
3228 } else {
3229 snd = false;
3230 rc = EINVAL;
3231 }
3232 if (snd) {
3233 MPASS(txp->npkt > 0);
3234 for (i = 0; i < txp->npkt; i++)
3235 ETHER_BPF_MTAP(ifp, txp->mb[i]);
3236 if (txp->npkt > 1) {
3237 MPASS(avail >= tx_len16_to_desc(txp->len16));
3238 if (vi->flags & TX_USES_VM_WR)
3239 n = write_txpkts_vm_wr(sc, txq);
3240 else
3241 n = write_txpkts_wr(sc, txq);
3242 } else {
3243 MPASS(avail >=
3244 tx_len16_to_desc(mbuf_len16(txp->mb[0])));
3245 if (vi->flags & TX_USES_VM_WR)
3246 n = write_txpkt_vm_wr(sc, txq,
3247 txp->mb[0]);
3248 else
3249 n = write_txpkt_wr(sc, txq, txp->mb[0],
3250 avail);
3251 }
3252 MPASS(n <= SGE_MAX_WR_NDESC);
3253 avail -= n;
3254 dbdiff += n;
3255 wr = &eq->desc[eq->pidx];
3256 IDXINCR(eq->pidx, n, eq->sidx);
3257 txp->npkt = 0; /* emptied */
3258 }
3259 if (rc == 0) {
3260 /* m0 was coalesced into txq->txpkts. */
3261 goto next_mbuf;
3262 }
3263 if (rc == EAGAIN) {
3264 /*
3265 * m0 is suitable for tx coalescing but could not be
3266 * combined with the existing txq->txpkts, which has now
3267 * been transmitted. Start a new txpkts with m0.
3268 */
3269 MPASS(snd);
3270 MPASS(txp->npkt == 0);
3271 continue;
3272 }
3273
3274 MPASS(rc != 0 && rc != EAGAIN);
3275 MPASS(txp->npkt == 0);
3276skip_coalescing:
3277 n = tx_len16_to_desc(mbuf_len16(m0));
3278 if (__predict_false(avail < n)) {
3279 avail += reclaim_tx_descs(txq, min(n, 32));
3280 if (avail < n)
3281 break; /* out of descriptors */
3282 }
3283
3284 wr = &eq->desc[eq->pidx];
3285 if (mbuf_cflags(m0) & MC_RAW_WR) {
3286 n = write_raw_wr(txq, wr, m0, avail);
3287#ifdef KERN_TLS
3288 } else if (mbuf_cflags(m0) & MC_TLS) {
3289 ETHER_BPF_MTAP(ifp, m0);
3290 n = t6_ktls_write_wr(txq, wr, m0, mbuf_nsegs(m0),
3291 avail);
3292#endif
3293 } else {
3294 ETHER_BPF_MTAP(ifp, m0);
3295 if (vi->flags & TX_USES_VM_WR)
3296 n = write_txpkt_vm_wr(sc, txq, m0);
3297 else
3298 n = write_txpkt_wr(sc, txq, m0, avail);
3299 }
3300 MPASS(n >= 1 && n <= avail);
3301 if (!(mbuf_cflags(m0) & MC_TLS))
3302 MPASS(n <= SGE_MAX_WR_NDESC);
3303
3304 avail -= n;
3305 dbdiff += n;
3306 IDXINCR(eq->pidx, n, eq->sidx);
3307
3308 if (dbdiff >= 512 / EQ_ESIZE) { /* X_FETCHBURSTMAX_512B */
3309 if (wr_can_update_eq(wr))
3310 set_txupdate_flags(txq, avail, wr);
3311 ring_eq_db(sc, eq, dbdiff);
3312 avail += reclaim_tx_descs(txq, 32);
3313 dbdiff = 0;
3314 }
3315next_mbuf:
3316 total++;
3317 remaining--;
3318 if (__predict_false(++cidx == r->size))
3319 cidx = 0;
3320 }
3321 if (dbdiff != 0) {
3322 if (wr_can_update_eq(wr))
3323 set_txupdate_flags(txq, avail, wr);
3324 ring_eq_db(sc, eq, dbdiff);
3325 reclaim_tx_descs(txq, 32);
3326 } else if (eq->pidx == eq->cidx && txp->npkt > 0 &&
3327 atomic_load_int(&txq->eq.equiq) == 0) {
3328 /*
3329 * If nothing was submitted to the chip for tx (it was coalesced
3330 * into txpkts instead) and there is no tx update outstanding
3331 * then we need to send txpkts now.
3332 */
3333send_txpkts:
3334 MPASS(txp->npkt > 0);
3335 for (i = 0; i < txp->npkt; i++)
3336 ETHER_BPF_MTAP(ifp, txp->mb[i]);
3337 if (txp->npkt > 1) {
3338 MPASS(avail >= tx_len16_to_desc(txp->len16));
3339 if (vi->flags & TX_USES_VM_WR)
3340 n = write_txpkts_vm_wr(sc, txq);
3341 else
3342 n = write_txpkts_wr(sc, txq);
3343 } else {
3344 MPASS(avail >=
3345 tx_len16_to_desc(mbuf_len16(txp->mb[0])));
3346 if (vi->flags & TX_USES_VM_WR)
3347 n = write_txpkt_vm_wr(sc, txq, txp->mb[0]);
3348 else
3349 n = write_txpkt_wr(sc, txq, txp->mb[0], avail);
3350 }
3351 MPASS(n <= SGE_MAX_WR_NDESC);
3352 wr = &eq->desc[eq->pidx];
3353 IDXINCR(eq->pidx, n, eq->sidx);
3354 txp->npkt = 0; /* emptied */
3355
3356 MPASS(wr_can_update_eq(wr));
3357 set_txupdate_flags(txq, avail - n, wr);
3358 ring_eq_db(sc, eq, n);
3359 reclaim_tx_descs(txq, 32);
3360 }
3361 *coalescing = txp->npkt > 0;
3362
3363 return (total);
3364}
3365
3366static inline void
3367init_iq(struct sge_iq *iq, struct adapter *sc, int tmr_idx, int pktc_idx,
3368 int qsize, int intr_idx, int cong)
3369{
3370
3371 KASSERT(tmr_idx >= 0 && tmr_idx < SGE_NTIMERS,
3372 ("%s: bad tmr_idx %d", __func__, tmr_idx));
3373 KASSERT(pktc_idx < SGE_NCOUNTERS, /* -ve is ok, means don't use */
3374 ("%s: bad pktc_idx %d", __func__, pktc_idx));
3375 KASSERT(intr_idx >= -1 && intr_idx < sc->intr_count,
3376 ("%s: bad intr_idx %d", __func__, intr_idx));
3377
3378 iq->flags = 0;
3379 iq->state = IQS_DISABLED;
3380 iq->adapter = sc;
3381 iq->intr_params = V_QINTR_TIMER_IDX(tmr_idx);
3382 iq->intr_pktc_idx = SGE_NCOUNTERS - 1;
3383 if (pktc_idx >= 0) {
3384 iq->intr_params |= F_QINTR_CNT_EN;
3385 iq->intr_pktc_idx = pktc_idx;
3386 }
3387 iq->qsize = roundup2(qsize, 16); /* See FW_IQ_CMD/iqsize */
3388 iq->sidx = iq->qsize - sc->params.sge.spg_len / IQ_ESIZE;
3389 iq->intr_idx = intr_idx;
3390 iq->cong = cong;
3391}
3392
3393static inline void
3394init_fl(struct adapter *sc, struct sge_fl *fl, int qsize, int maxp, char *name)
3395{
3396 struct sge_params *sp = &sc->params.sge;
3397
3398 fl->qsize = qsize;
3399 fl->sidx = qsize - sc->params.sge.spg_len / EQ_ESIZE;
3400 strlcpy(fl->lockname, name, sizeof(fl->lockname));
3401 mtx_init(&fl->fl_lock, fl->lockname, NULL, MTX_DEF);
3402 if (sc->flags & BUF_PACKING_OK &&
3403 ((!is_t4(sc) && buffer_packing) || /* T5+: enabled unless 0 */
3404 (is_t4(sc) && buffer_packing == 1)))/* T4: disabled unless 1 */
3405 fl->flags |= FL_BUF_PACKING;
3406 fl->zidx = find_refill_source(sc, maxp, fl->flags & FL_BUF_PACKING);
3407 fl->safe_zidx = sc->sge.safe_zidx;
3408 if (fl->flags & FL_BUF_PACKING) {
3409 fl->lowat = roundup2(sp->fl_starve_threshold2, 8);
3410 fl->buf_boundary = sp->pack_boundary;
3411 } else {
3412 fl->lowat = roundup2(sp->fl_starve_threshold, 8);
3413 fl->buf_boundary = 16;
3414 }
3415 if (fl_pad && fl->buf_boundary < sp->pad_boundary)
3416 fl->buf_boundary = sp->pad_boundary;
3417}
3418
3419static inline void
3420init_eq(struct adapter *sc, struct sge_eq *eq, int eqtype, int qsize,
3421 uint8_t tx_chan, struct sge_iq *iq, char *name)
3422{
3423 KASSERT(eqtype >= EQ_CTRL && eqtype <= EQ_OFLD,
3424 ("%s: bad qtype %d", __func__, eqtype));
3425
3426 eq->type = eqtype;
3427 eq->tx_chan = tx_chan;
3428 eq->iq = iq;
3429 eq->sidx = qsize - sc->params.sge.spg_len / EQ_ESIZE;
3430 strlcpy(eq->lockname, name, sizeof(eq->lockname));
3431 mtx_init(&eq->eq_lock, eq->lockname, NULL, MTX_DEF);
3432}
3433
3434int
3435alloc_ring(struct adapter *sc, size_t len, bus_dma_tag_t *tag,
3436 bus_dmamap_t *map, bus_addr_t *pa, void **va)
3437{
3438 int rc;
3439
3440 rc = bus_dma_tag_create(sc->dmat, 512, 0, BUS_SPACE_MAXADDR,
3441 BUS_SPACE_MAXADDR, NULL, NULL, len, 1, len, 0, NULL, NULL, tag);
3442 if (rc != 0) {
3443 CH_ERR(sc, "cannot allocate DMA tag: %d\n", rc);
3444 goto done;
3445 }
3446
3447 rc = bus_dmamem_alloc(*tag, va,
3448 BUS_DMA_WAITOK | BUS_DMA_COHERENT | BUS_DMA_ZERO, map);
3449 if (rc != 0) {
3450 CH_ERR(sc, "cannot allocate DMA memory: %d\n", rc);
3451 goto done;
3452 }
3453
3454 rc = bus_dmamap_load(*tag, *map, *va, len, oneseg_dma_callback, pa, 0);
3455 if (rc != 0) {
3456 CH_ERR(sc, "cannot load DMA map: %d\n", rc);
3457 goto done;
3458 }
3459done:
3460 if (rc)
3461 free_ring(sc, *tag, *map, *pa, *va);
3462
3463 return (rc);
3464}
3465
3466int
3467free_ring(struct adapter *sc, bus_dma_tag_t tag, bus_dmamap_t map,
3468 bus_addr_t pa, void *va)
3469{
3470 if (pa)
3471 bus_dmamap_unload(tag, map);
3472 if (va)
3473 bus_dmamem_free(tag, va, map);
3474 if (tag)
3475 bus_dma_tag_destroy(tag);
3476
3477 return (0);
3478}
3479
3480/*
3481 * Allocates the software resources (mainly memory and sysctl nodes) for an
3482 * ingress queue and an optional freelist.
3483 *
3484 * Sets IQ_SW_ALLOCATED and returns 0 on success.
3485 */
3486static int
3487alloc_iq_fl(struct vi_info *vi, struct sge_iq *iq, struct sge_fl *fl,
3488 struct sysctl_ctx_list *ctx, struct sysctl_oid *oid)
3489{
3490 int rc;
3491 size_t len;
3492 struct adapter *sc = vi->adapter;
3493
3494 MPASS(!(iq->flags & IQ_SW_ALLOCATED));
3495
3496 len = iq->qsize * IQ_ESIZE;
3497 rc = alloc_ring(sc, len, &iq->desc_tag, &iq->desc_map, &iq->ba,
3498 (void **)&iq->desc);
3499 if (rc != 0)
3500 return (rc);
3501
3502 if (fl) {
3503 len = fl->qsize * EQ_ESIZE;
3504 rc = alloc_ring(sc, len, &fl->desc_tag, &fl->desc_map,
3505 &fl->ba, (void **)&fl->desc);
3506 if (rc) {
3507 free_ring(sc, iq->desc_tag, iq->desc_map, iq->ba,
3508 iq->desc);
3509 return (rc);
3510 }
3511
3512 /* Allocate space for one software descriptor per buffer. */
3513 fl->sdesc = malloc(fl->sidx * 8 * sizeof(struct fl_sdesc),
3514 M_CXGBE, M_ZERO | M_WAITOK);
3515
3516 add_fl_sysctls(sc, ctx, oid, fl);
3517 iq->flags |= IQ_HAS_FL;
3518 }
3519 add_iq_sysctls(ctx, oid, iq);
3520 iq->flags |= IQ_SW_ALLOCATED;
3521
3522 return (0);
3523}
3524
3525/*
3526 * Frees all software resources (memory and locks) associated with an ingress
3527 * queue and an optional freelist.
3528 */
3529static void
3530free_iq_fl(struct adapter *sc, struct sge_iq *iq, struct sge_fl *fl)
3531{
3532 MPASS(iq->flags & IQ_SW_ALLOCATED);
3533
3534 if (fl) {
3535 MPASS(iq->flags & IQ_HAS_FL);
3536 free_ring(sc, fl->desc_tag, fl->desc_map, fl->ba, fl->desc);
3537 free_fl_buffers(sc, fl);
3538 free(fl->sdesc, M_CXGBE);
3539 mtx_destroy(&fl->fl_lock);
3540 bzero(fl, sizeof(*fl));
3541 }
3542 free_ring(sc, iq->desc_tag, iq->desc_map, iq->ba, iq->desc);
3543 bzero(iq, sizeof(*iq));
3544}
3545
3546/*
3547 * Allocates a hardware ingress queue and an optional freelist that will be
3548 * associated with it.
3549 *
3550 * Returns errno on failure. Resources allocated up to that point may still be
3551 * allocated. Caller is responsible for cleanup in case this function fails.
3552 */
3553static int
3554alloc_iq_fl_hwq(struct vi_info *vi, struct sge_iq *iq, struct sge_fl *fl)
3555{
3556 int rc, i, cntxt_id;
3557 struct fw_iq_cmd c;
3558 struct adapter *sc = vi->adapter;
3559 __be32 v = 0;
3560
3561 MPASS (!(iq->flags & IQ_HW_ALLOCATED));
3562
3563 bzero(&c, sizeof(c));
3564 c.op_to_vfn = htobe32(V_FW_CMD_OP(FW_IQ_CMD) | F_FW_CMD_REQUEST |
3565 F_FW_CMD_WRITE | F_FW_CMD_EXEC | V_FW_IQ_CMD_PFN(sc->pf) |
3566 V_FW_IQ_CMD_VFN(0));
3567
3568 c.alloc_to_len16 = htobe32(F_FW_IQ_CMD_ALLOC | F_FW_IQ_CMD_IQSTART |
3569 FW_LEN16(c));
3570
3571 /* Special handling for firmware event queue */
3572 if (iq == &sc->sge.fwq)
3573 v |= F_FW_IQ_CMD_IQASYNCH;
3574
3575 if (iq->intr_idx < 0) {
3576 /* Forwarded interrupts, all headed to fwq */
3577 v |= F_FW_IQ_CMD_IQANDST;
3578 v |= V_FW_IQ_CMD_IQANDSTINDEX(sc->sge.fwq.cntxt_id);
3579 } else {
3580 KASSERT(iq->intr_idx < sc->intr_count,
3581 ("%s: invalid direct intr_idx %d", __func__, iq->intr_idx));
3582 v |= V_FW_IQ_CMD_IQANDSTINDEX(iq->intr_idx);
3583 }
3584
3585 bzero(iq->desc, iq->qsize * IQ_ESIZE);
3586 c.type_to_iqandstindex = htobe32(v |
3587 V_FW_IQ_CMD_TYPE(FW_IQ_TYPE_FL_INT_CAP) |
3588 V_FW_IQ_CMD_VIID(vi->viid) |
3589 V_FW_IQ_CMD_IQANUD(X_UPDATEDELIVERY_INTERRUPT));
3590 c.iqdroprss_to_iqesize = htobe16(V_FW_IQ_CMD_IQPCIECH(vi->pi->tx_chan) |
3591 F_FW_IQ_CMD_IQGTSMODE |
3592 V_FW_IQ_CMD_IQINTCNTTHRESH(iq->intr_pktc_idx) |
3593 V_FW_IQ_CMD_IQESIZE(ilog2(IQ_ESIZE) - 4));
3594 c.iqsize = htobe16(iq->qsize);
3595 c.iqaddr = htobe64(iq->ba);
3596 if (iq->cong >= 0)
3597 c.iqns_to_fl0congen = htobe32(F_FW_IQ_CMD_IQFLINTCONGEN);
3598
3599 if (fl) {
3600 bzero(fl->desc, fl->sidx * EQ_ESIZE + sc->params.sge.spg_len);
3601 c.iqns_to_fl0congen |=
3602 htobe32(V_FW_IQ_CMD_FL0HOSTFCMODE(X_HOSTFCMODE_NONE) |
3603 F_FW_IQ_CMD_FL0FETCHRO | F_FW_IQ_CMD_FL0DATARO |
3604 (fl_pad ? F_FW_IQ_CMD_FL0PADEN : 0) |
3605 (fl->flags & FL_BUF_PACKING ? F_FW_IQ_CMD_FL0PACKEN :
3606 0));
3607 if (iq->cong >= 0) {
3608 c.iqns_to_fl0congen |=
3609 htobe32(V_FW_IQ_CMD_FL0CNGCHMAP(iq->cong) |
3610 F_FW_IQ_CMD_FL0CONGCIF |
3611 F_FW_IQ_CMD_FL0CONGEN);
3612 }
3613 c.fl0dcaen_to_fl0cidxfthresh =
3614 htobe16(V_FW_IQ_CMD_FL0FBMIN(chip_id(sc) <= CHELSIO_T5 ?
3615 X_FETCHBURSTMIN_128B : X_FETCHBURSTMIN_64B_T6) |
3616 V_FW_IQ_CMD_FL0FBMAX(chip_id(sc) <= CHELSIO_T5 ?
3617 X_FETCHBURSTMAX_512B : X_FETCHBURSTMAX_256B));
3618 c.fl0size = htobe16(fl->qsize);
3619 c.fl0addr = htobe64(fl->ba);
3620 }
3621
3622 rc = -t4_wr_mbox(sc, sc->mbox, &c, sizeof(c), &c);
3623 if (rc != 0) {
3624 CH_ERR(sc, "failed to create hw ingress queue: %d\n", rc);
3625 return (rc);
3626 }
3627
3628 iq->cidx = 0;
3629 iq->gen = F_RSPD_GEN;
3630 iq->cntxt_id = be16toh(c.iqid);
3631 iq->abs_id = be16toh(c.physiqid);
3632
3633 cntxt_id = iq->cntxt_id - sc->sge.iq_start;
3634 if (cntxt_id >= sc->sge.iqmap_sz) {
3635 panic ("%s: iq->cntxt_id (%d) more than the max (%d)", __func__,
3636 cntxt_id, sc->sge.iqmap_sz - 1);
3637 }
3638 sc->sge.iqmap[cntxt_id] = iq;
3639
3640 if (fl) {
3641 u_int qid;
3642#ifdef INVARIANTS
3643 MPASS(!(fl->flags & FL_BUF_RESUME));
3644 for (i = 0; i < fl->sidx * 8; i++)
3645 MPASS(fl->sdesc[i].cl == NULL);
3646#endif
3647 fl->cntxt_id = be16toh(c.fl0id);
3648 fl->pidx = fl->cidx = fl->hw_cidx = fl->dbidx = 0;
3649 fl->rx_offset = 0;
3650 fl->flags &= ~(FL_STARVING | FL_DOOMED);
3651
3652 cntxt_id = fl->cntxt_id - sc->sge.eq_start;
3653 if (cntxt_id >= sc->sge.eqmap_sz) {
3654 panic("%s: fl->cntxt_id (%d) more than the max (%d)",
3655 __func__, cntxt_id, sc->sge.eqmap_sz - 1);
3656 }
3657 sc->sge.eqmap[cntxt_id] = (void *)fl;
3658
3659 qid = fl->cntxt_id;
3660 if (isset(&sc->doorbells, DOORBELL_UDB)) {
3661 uint32_t s_qpp = sc->params.sge.eq_s_qpp;
3662 uint32_t mask = (1 << s_qpp) - 1;
3663 volatile uint8_t *udb;
3664
3665 udb = sc->udbs_base + UDBS_DB_OFFSET;
3666 udb += (qid >> s_qpp) << PAGE_SHIFT;
3667 qid &= mask;
3668 if (qid < PAGE_SIZE / UDBS_SEG_SIZE) {
3669 udb += qid << UDBS_SEG_SHIFT;
3670 qid = 0;
3671 }
3672 fl->udb = (volatile void *)udb;
3673 }
3674 fl->dbval = V_QID(qid) | sc->chip_params->sge_fl_db;
3675
3676 FL_LOCK(fl);
3677 /* Enough to make sure the SGE doesn't think it's starved */
3678 refill_fl(sc, fl, fl->lowat);
3679 FL_UNLOCK(fl);
3680 }
3681
3682 if (chip_id(sc) >= CHELSIO_T5 && !(sc->flags & IS_VF) && iq->cong >= 0) {
3683 uint32_t param, val;
3684
3685 param = V_FW_PARAMS_MNEM(FW_PARAMS_MNEM_DMAQ) |
3686 V_FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_DMAQ_CONM_CTXT) |
3687 V_FW_PARAMS_PARAM_YZ(iq->cntxt_id);
3688 if (iq->cong == 0)
3689 val = 1 << 19;
3690 else {
3691 val = 2 << 19;
3692 for (i = 0; i < 4; i++) {
3693 if (iq->cong & (1 << i))
3694 val |= 1 << (i << 2);
3695 }
3696 }
3697
3698 rc = -t4_set_params(sc, sc->mbox, sc->pf, 0, 1, &param, &val);
3699 if (rc != 0) {
3700 /* report error but carry on */
3701 CH_ERR(sc, "failed to set congestion manager context "
3702 "for ingress queue %d: %d\n", iq->cntxt_id, rc);
3703 }
3704 }
3705
3706 /* Enable IQ interrupts */
3707 atomic_store_rel_int(&iq->state, IQS_IDLE);
3708 t4_write_reg(sc, sc->sge_gts_reg, V_SEINTARM(iq->intr_params) |
3709 V_INGRESSQID(iq->cntxt_id));
3710
3711 iq->flags |= IQ_HW_ALLOCATED;
3712
3713 return (0);
3714}
3715
3716static int
3717free_iq_fl_hwq(struct adapter *sc, struct sge_iq *iq, struct sge_fl *fl)
3718{
3719 int rc;
3720
3721 MPASS(iq->flags & IQ_HW_ALLOCATED);
3722 rc = -t4_iq_free(sc, sc->mbox, sc->pf, 0, FW_IQ_TYPE_FL_INT_CAP,
3723 iq->cntxt_id, fl ? fl->cntxt_id : 0xffff, 0xffff);
3724 if (rc != 0) {
3725 CH_ERR(sc, "failed to free iq %p: %d\n", iq, rc);
3726 return (rc);
3727 }
3728 iq->flags &= ~IQ_HW_ALLOCATED;
3729
3730 return (0);
3731}
3732
3733static void
3734add_iq_sysctls(struct sysctl_ctx_list *ctx, struct sysctl_oid *oid,
3735 struct sge_iq *iq)
3736{
3737 struct sysctl_oid_list *children;
3738
3739 if (ctx == NULL || oid == NULL)
3740 return;
3741
3742 children = SYSCTL_CHILDREN(oid);
3743 SYSCTL_ADD_UAUTO(ctx, children, OID_AUTO, "ba", CTLFLAG_RD, &iq->ba,
3744 "bus address of descriptor ring");
3745 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "dmalen", CTLFLAG_RD, NULL,
3746 iq->qsize * IQ_ESIZE, "descriptor ring size in bytes");
3747 SYSCTL_ADD_U16(ctx, children, OID_AUTO, "abs_id", CTLFLAG_RD,
3748 &iq->abs_id, 0, "absolute id of the queue");
3749 SYSCTL_ADD_U16(ctx, children, OID_AUTO, "cntxt_id", CTLFLAG_RD,
3750 &iq->cntxt_id, 0, "SGE context id of the queue");
3751 SYSCTL_ADD_U16(ctx, children, OID_AUTO, "cidx", CTLFLAG_RD, &iq->cidx,
3752 0, "consumer index");
3753}
3754
3755static void
3756add_fl_sysctls(struct adapter *sc, struct sysctl_ctx_list *ctx,
3757 struct sysctl_oid *oid, struct sge_fl *fl)
3758{
3759 struct sysctl_oid_list *children;
3760
3761 if (ctx == NULL || oid == NULL)
3762 return;
3763
3764 children = SYSCTL_CHILDREN(oid);
3765 oid = SYSCTL_ADD_NODE(ctx, children, OID_AUTO, "fl",
3766 CTLFLAG_RD | CTLFLAG_MPSAFE, NULL, "freelist");
3767 children = SYSCTL_CHILDREN(oid);
3768
3769 SYSCTL_ADD_UAUTO(ctx, children, OID_AUTO, "ba", CTLFLAG_RD,
3770 &fl->ba, "bus address of descriptor ring");
3771 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "dmalen", CTLFLAG_RD, NULL,
3772 fl->sidx * EQ_ESIZE + sc->params.sge.spg_len,
3773 "desc ring size in bytes");
3774 SYSCTL_ADD_U16(ctx, children, OID_AUTO, "cntxt_id", CTLFLAG_RD,
3775 &fl->cntxt_id, 0, "SGE context id of the freelist");
3776 SYSCTL_ADD_UINT(ctx, children, OID_AUTO, "padding", CTLFLAG_RD, NULL,
3777 fl_pad ? 1 : 0, "padding enabled");
3778 SYSCTL_ADD_UINT(ctx, children, OID_AUTO, "packing", CTLFLAG_RD, NULL,
3779 fl->flags & FL_BUF_PACKING ? 1 : 0, "packing enabled");
3780 SYSCTL_ADD_UINT(ctx, children, OID_AUTO, "cidx", CTLFLAG_RD, &fl->cidx,
3781 0, "consumer index");
3782 if (fl->flags & FL_BUF_PACKING) {
3783 SYSCTL_ADD_UINT(ctx, children, OID_AUTO, "rx_offset",
3784 CTLFLAG_RD, &fl->rx_offset, 0, "packing rx offset");
3785 }
3786 SYSCTL_ADD_UINT(ctx, children, OID_AUTO, "pidx", CTLFLAG_RD, &fl->pidx,
3787 0, "producer index");
3788 SYSCTL_ADD_UQUAD(ctx, children, OID_AUTO, "cluster_allocated",
3789 CTLFLAG_RD, &fl->cl_allocated, "# of clusters allocated");
3790 SYSCTL_ADD_UQUAD(ctx, children, OID_AUTO, "cluster_recycled",
3791 CTLFLAG_RD, &fl->cl_recycled, "# of clusters recycled");
3792 SYSCTL_ADD_UQUAD(ctx, children, OID_AUTO, "cluster_fast_recycled",
3793 CTLFLAG_RD, &fl->cl_fast_recycled, "# of clusters recycled (fast)");
3794}
3795
3796/*
3797 * Idempotent.
3798 */
3799static int
3800alloc_fwq(struct adapter *sc)
3801{
3802 int rc, intr_idx;
3803 struct sge_iq *fwq = &sc->sge.fwq;
3804 struct vi_info *vi = &sc->port[0]->vi[0];
3805
3806 if (!(fwq->flags & IQ_SW_ALLOCATED)) {
3807 MPASS(!(fwq->flags & IQ_HW_ALLOCATED));
3808
3809 if (sc->flags & IS_VF)
3810 intr_idx = 0;
3811 else
3812 intr_idx = sc->intr_count > 1 ? 1 : 0;
3813 init_iq(fwq, sc, 0, 0, FW_IQ_QSIZE, intr_idx, -1);
3814 rc = alloc_iq_fl(vi, fwq, NULL, &sc->ctx, sc->fwq_oid);
3815 if (rc != 0) {
3816 CH_ERR(sc, "failed to allocate fwq: %d\n", rc);
3817 return (rc);
3818 }
3819 MPASS(fwq->flags & IQ_SW_ALLOCATED);
3820 }
3821
3822 if (!(fwq->flags & IQ_HW_ALLOCATED)) {
3823 MPASS(fwq->flags & IQ_SW_ALLOCATED);
3824
3825 rc = alloc_iq_fl_hwq(vi, fwq, NULL);
3826 if (rc != 0) {
3827 CH_ERR(sc, "failed to create hw fwq: %d\n", rc);
3828 return (rc);
3829 }
3830 MPASS(fwq->flags & IQ_HW_ALLOCATED);
3831 }
3832
3833 return (0);
3834}
3835
3836/*
3837 * Idempotent.
3838 */
3839static void
3840free_fwq(struct adapter *sc)
3841{
3842 struct sge_iq *fwq = &sc->sge.fwq;
3843
3844 if (fwq->flags & IQ_HW_ALLOCATED) {
3845 MPASS(fwq->flags & IQ_SW_ALLOCATED);
3846 free_iq_fl_hwq(sc, fwq, NULL);
3847 MPASS(!(fwq->flags & IQ_HW_ALLOCATED));
3848 }
3849
3850 if (fwq->flags & IQ_SW_ALLOCATED) {
3851 MPASS(!(fwq->flags & IQ_HW_ALLOCATED));
3852 free_iq_fl(sc, fwq, NULL);
3853 MPASS(!(fwq->flags & IQ_SW_ALLOCATED));
3854 }
3855}
3856
3857/*
3858 * Idempotent.
3859 */
3860static int
3861alloc_ctrlq(struct adapter *sc, int idx)
3862{
3863 int rc;
3864 char name[16];
3865 struct sysctl_oid *oid;
3866 struct sge_wrq *ctrlq = &sc->sge.ctrlq[idx];
3867
3868 MPASS(idx < sc->params.nports);
3869
3870 if (!(ctrlq->eq.flags & EQ_SW_ALLOCATED)) {
3871 MPASS(!(ctrlq->eq.flags & EQ_HW_ALLOCATED));
3872
3873 snprintf(name, sizeof(name), "%d", idx);
3874 oid = SYSCTL_ADD_NODE(&sc->ctx, SYSCTL_CHILDREN(sc->ctrlq_oid),
3875 OID_AUTO, name, CTLFLAG_RD | CTLFLAG_MPSAFE, NULL,
3876 "ctrl queue");
3877
3878 snprintf(name, sizeof(name), "%s ctrlq%d",
3879 device_get_nameunit(sc->dev), idx);
3880 init_eq(sc, &ctrlq->eq, EQ_CTRL, CTRL_EQ_QSIZE,
3881 sc->port[idx]->tx_chan, &sc->sge.fwq, name);
3882 rc = alloc_wrq(sc, NULL, ctrlq, &sc->ctx, oid);
3883 if (rc != 0) {
3884 CH_ERR(sc, "failed to allocate ctrlq%d: %d\n", idx, rc);
3885 sysctl_remove_oid(oid, 1, 1);
3886 return (rc);
3887 }
3888 MPASS(ctrlq->eq.flags & EQ_SW_ALLOCATED);
3889 }
3890
3891 if (!(ctrlq->eq.flags & EQ_HW_ALLOCATED)) {
3892 MPASS(ctrlq->eq.flags & EQ_SW_ALLOCATED);
3893
3894 rc = alloc_eq_hwq(sc, NULL, &ctrlq->eq);
3895 if (rc != 0) {
3896 CH_ERR(sc, "failed to create hw ctrlq%d: %d\n", idx, rc);
3897 return (rc);
3898 }
3899 MPASS(ctrlq->eq.flags & EQ_HW_ALLOCATED);
3900 }
3901
3902 return (0);
3903}
3904
3905/*
3906 * Idempotent.
3907 */
3908static void
3909free_ctrlq(struct adapter *sc, int idx)
3910{
3911 struct sge_wrq *ctrlq = &sc->sge.ctrlq[idx];
3912
3913 if (ctrlq->eq.flags & EQ_HW_ALLOCATED) {
3914 MPASS(ctrlq->eq.flags & EQ_SW_ALLOCATED);
3915 free_eq_hwq(sc, NULL, &ctrlq->eq);
3916 MPASS(!(ctrlq->eq.flags & EQ_HW_ALLOCATED));
3917 }
3918
3919 if (ctrlq->eq.flags & EQ_SW_ALLOCATED) {
3920 MPASS(!(ctrlq->eq.flags & EQ_HW_ALLOCATED));
3921 free_wrq(sc, ctrlq);
3922 MPASS(!(ctrlq->eq.flags & EQ_SW_ALLOCATED));
3923 }
3924}
3925
3926int
3927tnl_cong(struct port_info *pi, int drop)
3928{
3929
3930 if (drop == -1)
3931 return (-1);
3932 else if (drop == 1)
3933 return (0);
3934 else
3935 return (pi->rx_e_chan_map);
3936}
3937
3938/*
3939 * Idempotent.
3940 */
3941static int
3942alloc_rxq(struct vi_info *vi, struct sge_rxq *rxq, int idx, int intr_idx,
3943 int maxp)
3944{
3945 int rc;
3946 struct adapter *sc = vi->adapter;
3947 struct ifnet *ifp = vi->ifp;
3948 struct sysctl_oid *oid;
3949 char name[16];
3950
3951 if (!(rxq->iq.flags & IQ_SW_ALLOCATED)) {
3952 MPASS(!(rxq->iq.flags & IQ_HW_ALLOCATED));
3953#if defined(INET) || defined(INET6)
3954 rc = tcp_lro_init_args(&rxq->lro, ifp, lro_entries, lro_mbufs);
3955 if (rc != 0)
3956 return (rc);
3957 MPASS(rxq->lro.ifp == ifp); /* also indicates LRO init'ed */
3958#endif
3959 rxq->ifp = ifp;
3960
3961 snprintf(name, sizeof(name), "%d", idx);
3962 oid = SYSCTL_ADD_NODE(&vi->ctx, SYSCTL_CHILDREN(vi->rxq_oid),
3963 OID_AUTO, name, CTLFLAG_RD | CTLFLAG_MPSAFE, NULL,
3964 "rx queue");
3965
3966 init_iq(&rxq->iq, sc, vi->tmr_idx, vi->pktc_idx, vi->qsize_rxq,
3967 intr_idx, tnl_cong(vi->pi, cong_drop));
3968#if defined(INET) || defined(INET6)
3969 if (ifp->if_capenable & IFCAP_LRO)
3970 rxq->iq.flags |= IQ_LRO_ENABLED;
3971#endif
3972 if (ifp->if_capenable & IFCAP_HWRXTSTMP)
3973 rxq->iq.flags |= IQ_RX_TIMESTAMP;
3974 snprintf(name, sizeof(name), "%s rxq%d-fl",
3975 device_get_nameunit(vi->dev), idx);
3976 init_fl(sc, &rxq->fl, vi->qsize_rxq / 8, maxp, name);
3977 rc = alloc_iq_fl(vi, &rxq->iq, &rxq->fl, &vi->ctx, oid);
3978 if (rc != 0) {
3979 CH_ERR(vi, "failed to allocate rxq%d: %d\n", idx, rc);
3980 sysctl_remove_oid(oid, 1, 1);
3981#if defined(INET) || defined(INET6)
3982 tcp_lro_free(&rxq->lro);
3983 rxq->lro.ifp = NULL;
3984#endif
3985 return (rc);
3986 }
3987 MPASS(rxq->iq.flags & IQ_SW_ALLOCATED);
3988 add_rxq_sysctls(&vi->ctx, oid, rxq);
3989 }
3990
3991 if (!(rxq->iq.flags & IQ_HW_ALLOCATED)) {
3992 MPASS(rxq->iq.flags & IQ_SW_ALLOCATED);
3993 rc = alloc_iq_fl_hwq(vi, &rxq->iq, &rxq->fl);
3994 if (rc != 0) {
3995 CH_ERR(vi, "failed to create hw rxq%d: %d\n", idx, rc);
3996 return (rc);
3997 }
3998 MPASS(rxq->iq.flags & IQ_HW_ALLOCATED);
3999
4000 if (idx == 0)
4001 sc->sge.iq_base = rxq->iq.abs_id - rxq->iq.cntxt_id;
4002 else
4003 KASSERT(rxq->iq.cntxt_id + sc->sge.iq_base == rxq->iq.abs_id,
4004 ("iq_base mismatch"));
4005 KASSERT(sc->sge.iq_base == 0 || sc->flags & IS_VF,
4006 ("PF with non-zero iq_base"));
4007
4008 /*
4009 * The freelist is just barely above the starvation threshold
4010 * right now, fill it up a bit more.
4011 */
4012 FL_LOCK(&rxq->fl);
4013 refill_fl(sc, &rxq->fl, 128);
4014 FL_UNLOCK(&rxq->fl);
4015 }
4016
4017 return (0);
4018}
4019
4020/*
4021 * Idempotent.
4022 */
4023static void
4024free_rxq(struct vi_info *vi, struct sge_rxq *rxq)
4025{
4026 if (rxq->iq.flags & IQ_HW_ALLOCATED) {
4027 MPASS(rxq->iq.flags & IQ_SW_ALLOCATED);
4028 free_iq_fl_hwq(vi->adapter, &rxq->iq, &rxq->fl);
4029 MPASS(!(rxq->iq.flags & IQ_HW_ALLOCATED));
4030 }
4031
4032 if (rxq->iq.flags & IQ_SW_ALLOCATED) {
4033 MPASS(!(rxq->iq.flags & IQ_HW_ALLOCATED));
4034#if defined(INET) || defined(INET6)
4035 tcp_lro_free(&rxq->lro);
4036#endif
4037 free_iq_fl(vi->adapter, &rxq->iq, &rxq->fl);
4038 MPASS(!(rxq->iq.flags & IQ_SW_ALLOCATED));
4039 bzero(rxq, sizeof(*rxq));
4040 }
4041}
4042
4043static void
4044add_rxq_sysctls(struct sysctl_ctx_list *ctx, struct sysctl_oid *oid,
4045 struct sge_rxq *rxq)
4046{
4047 struct sysctl_oid_list *children;
4048
4049 if (ctx == NULL || oid == NULL)
4050 return;
4051
4052 children = SYSCTL_CHILDREN(oid);
4053#if defined(INET) || defined(INET6)
4054 SYSCTL_ADD_U64(ctx, children, OID_AUTO, "lro_queued", CTLFLAG_RD,
4055 &rxq->lro.lro_queued, 0, NULL);
4056 SYSCTL_ADD_U64(ctx, children, OID_AUTO, "lro_flushed", CTLFLAG_RD,
4057 &rxq->lro.lro_flushed, 0, NULL);
4058#endif
4059 SYSCTL_ADD_UQUAD(ctx, children, OID_AUTO, "rxcsum", CTLFLAG_RD,
4060 &rxq->rxcsum, "# of times hardware assisted with checksum");
4061 SYSCTL_ADD_UQUAD(ctx, children, OID_AUTO, "vlan_extraction", CTLFLAG_RD,
4062 &rxq->vlan_extraction, "# of times hardware extracted 802.1Q tag");
4063 SYSCTL_ADD_UQUAD(ctx, children, OID_AUTO, "vxlan_rxcsum", CTLFLAG_RD,
4064 &rxq->vxlan_rxcsum,
4065 "# of times hardware assisted with inner checksum (VXLAN)");
4066}
4067
4068#ifdef TCP_OFFLOAD
4069/*
4070 * Idempotent.
4071 */
4072static int
4073alloc_ofld_rxq(struct vi_info *vi, struct sge_ofld_rxq *ofld_rxq, int idx,
4074 int intr_idx, int maxp)
4075{
4076 int rc;
4077 struct adapter *sc = vi->adapter;
4078 struct sysctl_oid *oid;
4079 char name[16];
4080
4081 if (!(ofld_rxq->iq.flags & IQ_SW_ALLOCATED)) {
4082 MPASS(!(ofld_rxq->iq.flags & IQ_HW_ALLOCATED));
4083
4084 snprintf(name, sizeof(name), "%d", idx);
4085 oid = SYSCTL_ADD_NODE(&vi->ctx,
4086 SYSCTL_CHILDREN(vi->ofld_rxq_oid), OID_AUTO, name,
4087 CTLFLAG_RD | CTLFLAG_MPSAFE, NULL, "offload rx queue");
4088
4089 init_iq(&ofld_rxq->iq, sc, vi->ofld_tmr_idx, vi->ofld_pktc_idx,
4090 vi->qsize_rxq, intr_idx, 0);
4091 snprintf(name, sizeof(name), "%s ofld_rxq%d-fl",
4092 device_get_nameunit(vi->dev), idx);
4093 init_fl(sc, &ofld_rxq->fl, vi->qsize_rxq / 8, maxp, name);
4094 rc = alloc_iq_fl(vi, &ofld_rxq->iq, &ofld_rxq->fl, &vi->ctx,
4095 oid);
4096 if (rc != 0) {
4097 CH_ERR(vi, "failed to allocate ofld_rxq%d: %d\n", idx,
4098 rc);
4099 sysctl_remove_oid(oid, 1, 1);
4100 return (rc);
4101 }
4102 MPASS(ofld_rxq->iq.flags & IQ_SW_ALLOCATED);
4103 ofld_rxq->rx_iscsi_ddp_setup_ok = counter_u64_alloc(M_WAITOK);
4104 ofld_rxq->rx_iscsi_ddp_setup_error =
4105 counter_u64_alloc(M_WAITOK);
4106 add_ofld_rxq_sysctls(&vi->ctx, oid, ofld_rxq);
4107 }
4108
4109 if (!(ofld_rxq->iq.flags & IQ_HW_ALLOCATED)) {
4110 MPASS(ofld_rxq->iq.flags & IQ_SW_ALLOCATED);
4111 rc = alloc_iq_fl_hwq(vi, &ofld_rxq->iq, &ofld_rxq->fl);
4112 if (rc != 0) {
4113 CH_ERR(vi, "failed to create hw ofld_rxq%d: %d\n", idx,
4114 rc);
4115 return (rc);
4116 }
4117 MPASS(ofld_rxq->iq.flags & IQ_HW_ALLOCATED);
4118 }
4119 return (rc);
4120}
4121
4122/*
4123 * Idempotent.
4124 */
4125static void
4126free_ofld_rxq(struct vi_info *vi, struct sge_ofld_rxq *ofld_rxq)
4127{
4128 if (ofld_rxq->iq.flags & IQ_HW_ALLOCATED) {
4129 MPASS(ofld_rxq->iq.flags & IQ_SW_ALLOCATED);
4130 free_iq_fl_hwq(vi->adapter, &ofld_rxq->iq, &ofld_rxq->fl);
4131 MPASS(!(ofld_rxq->iq.flags & IQ_HW_ALLOCATED));
4132 }
4133
4134 if (ofld_rxq->iq.flags & IQ_SW_ALLOCATED) {
4135 MPASS(!(ofld_rxq->iq.flags & IQ_HW_ALLOCATED));
4136 free_iq_fl(vi->adapter, &ofld_rxq->iq, &ofld_rxq->fl);
4137 MPASS(!(ofld_rxq->iq.flags & IQ_SW_ALLOCATED));
4138 counter_u64_free(ofld_rxq->rx_iscsi_ddp_setup_ok);
4139 counter_u64_free(ofld_rxq->rx_iscsi_ddp_setup_error);
4140 bzero(ofld_rxq, sizeof(*ofld_rxq));
4141 }
4142}
4143
4144static void
4145add_ofld_rxq_sysctls(struct sysctl_ctx_list *ctx, struct sysctl_oid *oid,
4146 struct sge_ofld_rxq *ofld_rxq)
4147{
4148 struct sysctl_oid_list *children;
4149
4150 if (ctx == NULL || oid == NULL)
4151 return;
4152
4153 children = SYSCTL_CHILDREN(oid);
4154 SYSCTL_ADD_ULONG(ctx, children, OID_AUTO,
4155 "rx_toe_tls_records", CTLFLAG_RD, &ofld_rxq->rx_toe_tls_records,
4156 "# of TOE TLS records received");
4157 SYSCTL_ADD_ULONG(ctx, children, OID_AUTO,
4158 "rx_toe_tls_octets", CTLFLAG_RD, &ofld_rxq->rx_toe_tls_octets,
4159 "# of payload octets in received TOE TLS records");
4160
4161 oid = SYSCTL_ADD_NODE(ctx, children, OID_AUTO, "iscsi",
4162 CTLFLAG_RD | CTLFLAG_MPSAFE, NULL, "TOE iSCSI statistics");
4163 children = SYSCTL_CHILDREN(oid);
4164
4165 SYSCTL_ADD_COUNTER_U64(ctx, children, OID_AUTO, "ddp_setup_ok",
4166 CTLFLAG_RD, &ofld_rxq->rx_iscsi_ddp_setup_ok,
4167 "# of times DDP buffer was setup successfully.");
4168 SYSCTL_ADD_COUNTER_U64(ctx, children, OID_AUTO, "ddp_setup_error",
4169 CTLFLAG_RD, &ofld_rxq->rx_iscsi_ddp_setup_error,
4170 "# of times DDP buffer setup failed.");
4171 SYSCTL_ADD_U64(ctx, children, OID_AUTO, "ddp_octets",
4172 CTLFLAG_RD, &ofld_rxq->rx_iscsi_ddp_octets, 0,
4173 "# of octets placed directly");
4174 SYSCTL_ADD_U64(ctx, children, OID_AUTO, "ddp_pdus",
4175 CTLFLAG_RD, &ofld_rxq->rx_iscsi_ddp_pdus, 0,
4176 "# of PDUs with data placed directly.");
4177 SYSCTL_ADD_U64(ctx, children, OID_AUTO, "fl_octets",
4178 CTLFLAG_RD, &ofld_rxq->rx_iscsi_fl_octets, 0,
4179 "# of data octets delivered in freelist");
4180 SYSCTL_ADD_U64(ctx, children, OID_AUTO, "fl_pdus",
4181 CTLFLAG_RD, &ofld_rxq->rx_iscsi_fl_pdus, 0,
4182 "# of PDUs with data delivered in freelist");
4183 SYSCTL_ADD_U64(ctx, children, OID_AUTO, "padding_errors",
4184 CTLFLAG_RD, &ofld_rxq->rx_iscsi_padding_errors, 0,
4185 "# of PDUs with invalid padding");
4186 SYSCTL_ADD_U64(ctx, children, OID_AUTO, "header_digest_errors",
4187 CTLFLAG_RD, &ofld_rxq->rx_iscsi_header_digest_errors, 0,
4188 "# of PDUs with invalid header digests");
4189 SYSCTL_ADD_U64(ctx, children, OID_AUTO, "data_digest_errors",
4190 CTLFLAG_RD, &ofld_rxq->rx_iscsi_data_digest_errors, 0,
4191 "# of PDUs with invalid data digests");
4192}
4193#endif
4194
4195/*
4196 * Returns a reasonable automatic cidx flush threshold for a given queue size.
4197 */
4198static u_int
4199qsize_to_fthresh(int qsize)
4200{
4201 u_int fthresh;
4202
4203 while (!powerof2(qsize))
4204 qsize++;
4205 fthresh = ilog2(qsize);
4206 if (fthresh > X_CIDXFLUSHTHRESH_128)
4207 fthresh = X_CIDXFLUSHTHRESH_128;
4208
4209 return (fthresh);
4210}
4211
4212static int
4213ctrl_eq_alloc(struct adapter *sc, struct sge_eq *eq)
4214{
4215 int rc, cntxt_id;
4216 struct fw_eq_ctrl_cmd c;
4217 int qsize = eq->sidx + sc->params.sge.spg_len / EQ_ESIZE;
4218
4219 bzero(&c, sizeof(c));
4220
4221 c.op_to_vfn = htobe32(V_FW_CMD_OP(FW_EQ_CTRL_CMD) | F_FW_CMD_REQUEST |
4222 F_FW_CMD_WRITE | F_FW_CMD_EXEC | V_FW_EQ_CTRL_CMD_PFN(sc->pf) |
4223 V_FW_EQ_CTRL_CMD_VFN(0));
4224 c.alloc_to_len16 = htobe32(F_FW_EQ_CTRL_CMD_ALLOC |
4225 F_FW_EQ_CTRL_CMD_EQSTART | FW_LEN16(c));
4226 c.cmpliqid_eqid = htonl(V_FW_EQ_CTRL_CMD_CMPLIQID(eq->iqid));
4227 c.physeqid_pkd = htobe32(0);
4228 c.fetchszm_to_iqid =
4229 htobe32(V_FW_EQ_CTRL_CMD_HOSTFCMODE(X_HOSTFCMODE_STATUS_PAGE) |
4230 V_FW_EQ_CTRL_CMD_PCIECHN(eq->tx_chan) |
4231 F_FW_EQ_CTRL_CMD_FETCHRO | V_FW_EQ_CTRL_CMD_IQID(eq->iqid));
4232 c.dcaen_to_eqsize =
4233 htobe32(V_FW_EQ_CTRL_CMD_FBMIN(chip_id(sc) <= CHELSIO_T5 ?
4234 X_FETCHBURSTMIN_64B : X_FETCHBURSTMIN_64B_T6) |
4235 V_FW_EQ_CTRL_CMD_FBMAX(X_FETCHBURSTMAX_512B) |
4236 V_FW_EQ_CTRL_CMD_CIDXFTHRESH(qsize_to_fthresh(qsize)) |
4237 V_FW_EQ_CTRL_CMD_EQSIZE(qsize));
4238 c.eqaddr = htobe64(eq->ba);
4239
4240 rc = -t4_wr_mbox(sc, sc->mbox, &c, sizeof(c), &c);
4241 if (rc != 0) {
4242 CH_ERR(sc, "failed to create hw ctrlq for tx_chan %d: %d\n",
4243 eq->tx_chan, rc);
4244 return (rc);
4245 }
4246
4247 eq->cntxt_id = G_FW_EQ_CTRL_CMD_EQID(be32toh(c.cmpliqid_eqid));
4248 eq->abs_id = G_FW_EQ_CTRL_CMD_PHYSEQID(be32toh(c.physeqid_pkd));
4249 cntxt_id = eq->cntxt_id - sc->sge.eq_start;
4250 if (cntxt_id >= sc->sge.eqmap_sz)
4251 panic("%s: eq->cntxt_id (%d) more than the max (%d)", __func__,
4252 cntxt_id, sc->sge.eqmap_sz - 1);
4253 sc->sge.eqmap[cntxt_id] = eq;
4254
4255 return (rc);
4256}
4257
4258static int
4259eth_eq_alloc(struct adapter *sc, struct vi_info *vi, struct sge_eq *eq)
4260{
4261 int rc, cntxt_id;
4262 struct fw_eq_eth_cmd c;
4263 int qsize = eq->sidx + sc->params.sge.spg_len / EQ_ESIZE;
4264
4265 bzero(&c, sizeof(c));
4266
4267 c.op_to_vfn = htobe32(V_FW_CMD_OP(FW_EQ_ETH_CMD) | F_FW_CMD_REQUEST |
4268 F_FW_CMD_WRITE | F_FW_CMD_EXEC | V_FW_EQ_ETH_CMD_PFN(sc->pf) |
4269 V_FW_EQ_ETH_CMD_VFN(0));
4270 c.alloc_to_len16 = htobe32(F_FW_EQ_ETH_CMD_ALLOC |
4271 F_FW_EQ_ETH_CMD_EQSTART | FW_LEN16(c));
4272 c.autoequiqe_to_viid = htobe32(F_FW_EQ_ETH_CMD_AUTOEQUIQE |
4273 F_FW_EQ_ETH_CMD_AUTOEQUEQE | V_FW_EQ_ETH_CMD_VIID(vi->viid));
4274 c.fetchszm_to_iqid =
4275 htobe32(V_FW_EQ_ETH_CMD_HOSTFCMODE(X_HOSTFCMODE_NONE) |
4276 V_FW_EQ_ETH_CMD_PCIECHN(eq->tx_chan) | F_FW_EQ_ETH_CMD_FETCHRO |
4277 V_FW_EQ_ETH_CMD_IQID(eq->iqid));
4278 c.dcaen_to_eqsize =
4279 htobe32(V_FW_EQ_ETH_CMD_FBMIN(chip_id(sc) <= CHELSIO_T5 ?
4280 X_FETCHBURSTMIN_64B : X_FETCHBURSTMIN_64B_T6) |
4281 V_FW_EQ_ETH_CMD_FBMAX(X_FETCHBURSTMAX_512B) |
4282 V_FW_EQ_ETH_CMD_EQSIZE(qsize));
4283 c.eqaddr = htobe64(eq->ba);
4284
4285 rc = -t4_wr_mbox(sc, sc->mbox, &c, sizeof(c), &c);
4286 if (rc != 0) {
4287 device_printf(vi->dev,
4288 "failed to create Ethernet egress queue: %d\n", rc);
4289 return (rc);
4290 }
4291
4292 eq->cntxt_id = G_FW_EQ_ETH_CMD_EQID(be32toh(c.eqid_pkd));
4293 eq->abs_id = G_FW_EQ_ETH_CMD_PHYSEQID(be32toh(c.physeqid_pkd));
4294 cntxt_id = eq->cntxt_id - sc->sge.eq_start;
4295 if (cntxt_id >= sc->sge.eqmap_sz)
4296 panic("%s: eq->cntxt_id (%d) more than the max (%d)", __func__,
4297 cntxt_id, sc->sge.eqmap_sz - 1);
4298 sc->sge.eqmap[cntxt_id] = eq;
4299
4300 return (rc);
4301}
4302
4303#if defined(TCP_OFFLOAD) || defined(RATELIMIT)
4304static int
4305ofld_eq_alloc(struct adapter *sc, struct vi_info *vi, struct sge_eq *eq)
4306{
4307 int rc, cntxt_id;
4308 struct fw_eq_ofld_cmd c;
4309 int qsize = eq->sidx + sc->params.sge.spg_len / EQ_ESIZE;
4310
4311 bzero(&c, sizeof(c));
4312
4313 c.op_to_vfn = htonl(V_FW_CMD_OP(FW_EQ_OFLD_CMD) | F_FW_CMD_REQUEST |
4314 F_FW_CMD_WRITE | F_FW_CMD_EXEC | V_FW_EQ_OFLD_CMD_PFN(sc->pf) |
4315 V_FW_EQ_OFLD_CMD_VFN(0));
4316 c.alloc_to_len16 = htonl(F_FW_EQ_OFLD_CMD_ALLOC |
4317 F_FW_EQ_OFLD_CMD_EQSTART | FW_LEN16(c));
4318 c.fetchszm_to_iqid =
4319 htonl(V_FW_EQ_OFLD_CMD_HOSTFCMODE(X_HOSTFCMODE_STATUS_PAGE) |
4320 V_FW_EQ_OFLD_CMD_PCIECHN(eq->tx_chan) |
4321 F_FW_EQ_OFLD_CMD_FETCHRO | V_FW_EQ_OFLD_CMD_IQID(eq->iqid));
4322 c.dcaen_to_eqsize =
4323 htobe32(V_FW_EQ_OFLD_CMD_FBMIN(chip_id(sc) <= CHELSIO_T5 ?
4324 X_FETCHBURSTMIN_64B : X_FETCHBURSTMIN_64B_T6) |
4325 V_FW_EQ_OFLD_CMD_FBMAX(X_FETCHBURSTMAX_512B) |
4326 V_FW_EQ_OFLD_CMD_CIDXFTHRESH(qsize_to_fthresh(qsize)) |
4327 V_FW_EQ_OFLD_CMD_EQSIZE(qsize));
4328 c.eqaddr = htobe64(eq->ba);
4329
4330 rc = -t4_wr_mbox(sc, sc->mbox, &c, sizeof(c), &c);
4331 if (rc != 0) {
4332 device_printf(vi->dev,
4333 "failed to create egress queue for TCP offload: %d\n", rc);
4334 return (rc);
4335 }
4336
4337 eq->cntxt_id = G_FW_EQ_OFLD_CMD_EQID(be32toh(c.eqid_pkd));
4338 eq->abs_id = G_FW_EQ_OFLD_CMD_PHYSEQID(be32toh(c.physeqid_pkd));
4339 cntxt_id = eq->cntxt_id - sc->sge.eq_start;
4340 if (cntxt_id >= sc->sge.eqmap_sz)
4341 panic("%s: eq->cntxt_id (%d) more than the max (%d)", __func__,
4342 cntxt_id, sc->sge.eqmap_sz - 1);
4343 sc->sge.eqmap[cntxt_id] = eq;
4344
4345 return (rc);
4346}
4347#endif
4348
4349/* SW only */
4350static int
4351alloc_eq(struct adapter *sc, struct sge_eq *eq, struct sysctl_ctx_list *ctx,
4352 struct sysctl_oid *oid)
4353{
4354 int rc, qsize;
4355 size_t len;
4356
4357 MPASS(!(eq->flags & EQ_SW_ALLOCATED));
4358
4359 qsize = eq->sidx + sc->params.sge.spg_len / EQ_ESIZE;
4360 len = qsize * EQ_ESIZE;
4361 rc = alloc_ring(sc, len, &eq->desc_tag, &eq->desc_map, &eq->ba,
4362 (void **)&eq->desc);
4363 if (rc)
4364 return (rc);
4365 if (ctx != NULL && oid != NULL)
4366 add_eq_sysctls(sc, ctx, oid, eq);
4367 eq->flags |= EQ_SW_ALLOCATED;
4368
4369 return (0);
4370}
4371
4372/* SW only */
4373static void
4374free_eq(struct adapter *sc, struct sge_eq *eq)
4375{
4376 MPASS(eq->flags & EQ_SW_ALLOCATED);
4377 if (eq->type == EQ_ETH)
4378 MPASS(eq->pidx == eq->cidx);
4379
4380 free_ring(sc, eq->desc_tag, eq->desc_map, eq->ba, eq->desc);
4381 mtx_destroy(&eq->eq_lock);
4382 bzero(eq, sizeof(*eq));
4383}
4384
4385static void
4386add_eq_sysctls(struct adapter *sc, struct sysctl_ctx_list *ctx,
4387 struct sysctl_oid *oid, struct sge_eq *eq)
4388{
4389 struct sysctl_oid_list *children = SYSCTL_CHILDREN(oid);
4390
4391 SYSCTL_ADD_UAUTO(ctx, children, OID_AUTO, "ba", CTLFLAG_RD, &eq->ba,
4392 "bus address of descriptor ring");
4393 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "dmalen", CTLFLAG_RD, NULL,
4394 eq->sidx * EQ_ESIZE + sc->params.sge.spg_len,
4395 "desc ring size in bytes");
4396 SYSCTL_ADD_UINT(ctx, children, OID_AUTO, "abs_id", CTLFLAG_RD,
4397 &eq->abs_id, 0, "absolute id of the queue");
4398 SYSCTL_ADD_UINT(ctx, children, OID_AUTO, "cntxt_id", CTLFLAG_RD,
4399 &eq->cntxt_id, 0, "SGE context id of the queue");
4400 SYSCTL_ADD_U16(ctx, children, OID_AUTO, "cidx", CTLFLAG_RD, &eq->cidx,
4401 0, "consumer index");
4402 SYSCTL_ADD_U16(ctx, children, OID_AUTO, "pidx", CTLFLAG_RD, &eq->pidx,
4403 0, "producer index");
4404 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "sidx", CTLFLAG_RD, NULL,
4405 eq->sidx, "status page index");
4406}
4407
4408static int
4409alloc_eq_hwq(struct adapter *sc, struct vi_info *vi, struct sge_eq *eq)
4410{
4411 int rc;
4412
4413 MPASS(!(eq->flags & EQ_HW_ALLOCATED));
4414
4415 eq->iqid = eq->iq->cntxt_id;
4416 eq->pidx = eq->cidx = eq->dbidx = 0;
4417 /* Note that equeqidx is not used with sge_wrq (OFLD/CTRL) queues. */
4418 eq->equeqidx = 0;
4419 eq->doorbells = sc->doorbells;
4420 bzero(eq->desc, eq->sidx * EQ_ESIZE + sc->params.sge.spg_len);
4421
4422 switch (eq->type) {
4423 case EQ_CTRL:
4424 rc = ctrl_eq_alloc(sc, eq);
4425 break;
4426
4427 case EQ_ETH:
4428 rc = eth_eq_alloc(sc, vi, eq);
4429 break;
4430
4431#if defined(TCP_OFFLOAD) || defined(RATELIMIT)
4432 case EQ_OFLD:
4433 rc = ofld_eq_alloc(sc, vi, eq);
4434 break;
4435#endif
4436
4437 default:
4438 panic("%s: invalid eq type %d.", __func__, eq->type);
4439 }
4440 if (rc != 0) {
4441 CH_ERR(sc, "failed to allocate egress queue(%d): %d\n",
4442 eq->type, rc);
4443 return (rc);
4444 }
4445
4446 if (isset(&eq->doorbells, DOORBELL_UDB) ||
4447 isset(&eq->doorbells, DOORBELL_UDBWC) ||
4448 isset(&eq->doorbells, DOORBELL_WCWR)) {
4449 uint32_t s_qpp = sc->params.sge.eq_s_qpp;
4450 uint32_t mask = (1 << s_qpp) - 1;
4451 volatile uint8_t *udb;
4452
4453 udb = sc->udbs_base + UDBS_DB_OFFSET;
4454 udb += (eq->cntxt_id >> s_qpp) << PAGE_SHIFT; /* pg offset */
4455 eq->udb_qid = eq->cntxt_id & mask; /* id in page */
4456 if (eq->udb_qid >= PAGE_SIZE / UDBS_SEG_SIZE)
4457 clrbit(&eq->doorbells, DOORBELL_WCWR);
4458 else {
4459 udb += eq->udb_qid << UDBS_SEG_SHIFT; /* seg offset */
4460 eq->udb_qid = 0;
4461 }
4462 eq->udb = (volatile void *)udb;
4463 }
4464
4465 eq->flags |= EQ_HW_ALLOCATED;
4466 return (0);
4467}
4468
4469static int
4470free_eq_hwq(struct adapter *sc, struct vi_info *vi __unused, struct sge_eq *eq)
4471{
4472 int rc;
4473
4474 MPASS(eq->flags & EQ_HW_ALLOCATED);
4475
4476 switch (eq->type) {
4477 case EQ_CTRL:
4478 rc = -t4_ctrl_eq_free(sc, sc->mbox, sc->pf, 0, eq->cntxt_id);
4479 break;
4480 case EQ_ETH:
4481 rc = -t4_eth_eq_free(sc, sc->mbox, sc->pf, 0, eq->cntxt_id);
4482 break;
4483#if defined(TCP_OFFLOAD) || defined(RATELIMIT)
4484 case EQ_OFLD:
4485 rc = -t4_ofld_eq_free(sc, sc->mbox, sc->pf, 0, eq->cntxt_id);
4486 break;
4487#endif
4488 default:
4489 panic("%s: invalid eq type %d.", __func__, eq->type);
4490 }
4491 if (rc != 0) {
4492 CH_ERR(sc, "failed to free eq (type %d): %d\n", eq->type, rc);
4493 return (rc);
4494 }
4495 eq->flags &= ~EQ_HW_ALLOCATED;
4496
4497 return (0);
4498}
4499
4500static int
4501alloc_wrq(struct adapter *sc, struct vi_info *vi, struct sge_wrq *wrq,
4502 struct sysctl_ctx_list *ctx, struct sysctl_oid *oid)
4503{
4504 struct sge_eq *eq = &wrq->eq;
4505 int rc;
4506
4507 MPASS(!(eq->flags & EQ_SW_ALLOCATED));
4508
4509 rc = alloc_eq(sc, eq, ctx, oid);
4510 if (rc)
4511 return (rc);
4512 MPASS(eq->flags & EQ_SW_ALLOCATED);
4513 /* Can't fail after this. */
4514
4515 wrq->adapter = sc;
4516 TASK_INIT(&wrq->wrq_tx_task, 0, wrq_tx_drain, wrq);
4517 TAILQ_INIT(&wrq->incomplete_wrs);
4518 STAILQ_INIT(&wrq->wr_list);
4519 wrq->nwr_pending = 0;
4520 wrq->ndesc_needed = 0;
4521 add_wrq_sysctls(ctx, oid, wrq);
4522
4523 return (0);
4524}
4525
4526static void
4527free_wrq(struct adapter *sc, struct sge_wrq *wrq)
4528{
4529 free_eq(sc, &wrq->eq);
4530 MPASS(wrq->nwr_pending == 0);
4531 MPASS(TAILQ_EMPTY(&wrq->incomplete_wrs));
4532 MPASS(STAILQ_EMPTY(&wrq->wr_list));
4533 bzero(wrq, sizeof(*wrq));
4534}
4535
4536static void
4537add_wrq_sysctls(struct sysctl_ctx_list *ctx, struct sysctl_oid *oid,
4538 struct sge_wrq *wrq)
4539{
4540 struct sysctl_oid_list *children;
4541
4542 if (ctx == NULL || oid == NULL)
4543 return;
4544
4545 children = SYSCTL_CHILDREN(oid);
4546 SYSCTL_ADD_UQUAD(ctx, children, OID_AUTO, "tx_wrs_direct", CTLFLAG_RD,
4547 &wrq->tx_wrs_direct, "# of work requests (direct)");
4548 SYSCTL_ADD_UQUAD(ctx, children, OID_AUTO, "tx_wrs_copied", CTLFLAG_RD,
4549 &wrq->tx_wrs_copied, "# of work requests (copied)");
4550 SYSCTL_ADD_UQUAD(ctx, children, OID_AUTO, "tx_wrs_sspace", CTLFLAG_RD,
4551 &wrq->tx_wrs_ss, "# of work requests (copied from scratch space)");
4552}
4553
4554/*
4555 * Idempotent.
4556 */
4557static int
4558alloc_txq(struct vi_info *vi, struct sge_txq *txq, int idx)
4559{
4560 int rc, iqidx;
4561 struct port_info *pi = vi->pi;
4562 struct adapter *sc = vi->adapter;
4563 struct sge_eq *eq = &txq->eq;
4564 struct txpkts *txp;
4565 char name[16];
4566 struct sysctl_oid *oid;
4567
4568 if (!(eq->flags & EQ_SW_ALLOCATED)) {
4569 MPASS(!(eq->flags & EQ_HW_ALLOCATED));
4570
4571 snprintf(name, sizeof(name), "%d", idx);
4572 oid = SYSCTL_ADD_NODE(&vi->ctx, SYSCTL_CHILDREN(vi->txq_oid),
4573 OID_AUTO, name, CTLFLAG_RD | CTLFLAG_MPSAFE, NULL,
4574 "tx queue");
4575
4576 iqidx = vi->first_rxq + (idx % vi->nrxq);
4577 snprintf(name, sizeof(name), "%s txq%d",
4578 device_get_nameunit(vi->dev), idx);
4579 init_eq(sc, &txq->eq, EQ_ETH, vi->qsize_txq, pi->tx_chan,
4580 &sc->sge.rxq[iqidx].iq, name);
4581
4582 rc = mp_ring_alloc(&txq->r, eq->sidx, txq, eth_tx,
4583 can_resume_eth_tx, M_CXGBE, &eq->eq_lock, M_WAITOK);
4584 if (rc != 0) {
4585 CH_ERR(vi, "failed to allocate mp_ring for txq%d: %d\n",
4586 idx, rc);
4587failed:
4588 sysctl_remove_oid(oid, 1, 1);
4589 return (rc);
4590 }
4591
4592 rc = alloc_eq(sc, eq, &vi->ctx, oid);
4593 if (rc) {
4594 CH_ERR(vi, "failed to allocate txq%d: %d\n", idx, rc);
4595 mp_ring_free(txq->r);
4596 goto failed;
4597 }
4598 MPASS(eq->flags & EQ_SW_ALLOCATED);
4599 /* Can't fail after this point. */
4600
4601 TASK_INIT(&txq->tx_reclaim_task, 0, tx_reclaim, eq);
4602 txq->ifp = vi->ifp;
4603 txq->gl = sglist_alloc(TX_SGL_SEGS, M_WAITOK);
4604 txq->sdesc = malloc(eq->sidx * sizeof(struct tx_sdesc), M_CXGBE,
4605 M_ZERO | M_WAITOK);
4606
4607 add_txq_sysctls(vi, &vi->ctx, oid, txq);
4608 }
4609
4610 if (!(eq->flags & EQ_HW_ALLOCATED)) {
4611 MPASS(eq->flags & EQ_SW_ALLOCATED);
4612 rc = alloc_eq_hwq(sc, vi, eq);
4613 if (rc != 0) {
4614 CH_ERR(vi, "failed to create hw txq%d: %d\n", idx, rc);
4615 return (rc);
4616 }
4617 MPASS(eq->flags & EQ_HW_ALLOCATED);
4618 /* Can't fail after this point. */
4619
4620 if (idx == 0)
4621 sc->sge.eq_base = eq->abs_id - eq->cntxt_id;
4622 else
4623 KASSERT(eq->cntxt_id + sc->sge.eq_base == eq->abs_id,
4624 ("eq_base mismatch"));
4625 KASSERT(sc->sge.eq_base == 0 || sc->flags & IS_VF,
4626 ("PF with non-zero eq_base"));
4627
4628 txp = &txq->txp;
4629 MPASS(nitems(txp->mb) >= sc->params.max_pkts_per_eth_tx_pkts_wr);
4630 txq->txp.max_npkt = min(nitems(txp->mb),
4631 sc->params.max_pkts_per_eth_tx_pkts_wr);
4632 if (vi->flags & TX_USES_VM_WR && !(sc->flags & IS_VF))
4633 txq->txp.max_npkt--;
4634
4635 if (vi->flags & TX_USES_VM_WR)
4636 txq->cpl_ctrl0 = htobe32(V_TXPKT_OPCODE(CPL_TX_PKT_XT) |
4637 V_TXPKT_INTF(pi->tx_chan));
4638 else
4639 txq->cpl_ctrl0 = htobe32(V_TXPKT_OPCODE(CPL_TX_PKT_XT) |
4640 V_TXPKT_INTF(pi->tx_chan) | V_TXPKT_PF(sc->pf) |
4641 V_TXPKT_VF(vi->vin) | V_TXPKT_VF_VLD(vi->vfvld));
4642
4643 txq->tc_idx = -1;
4644 }
4645
4646 return (0);
4647}
4648
4649/*
4650 * Idempotent.
4651 */
4652static void
4653free_txq(struct vi_info *vi, struct sge_txq *txq)
4654{
4655 struct adapter *sc = vi->adapter;
4656 struct sge_eq *eq = &txq->eq;
4657
4658 if (eq->flags & EQ_HW_ALLOCATED) {
4659 MPASS(eq->flags & EQ_SW_ALLOCATED);
4660 free_eq_hwq(sc, NULL, eq);
4661 MPASS(!(eq->flags & EQ_HW_ALLOCATED));
4662 }
4663
4664 if (eq->flags & EQ_SW_ALLOCATED) {
4665 MPASS(!(eq->flags & EQ_HW_ALLOCATED));
4666 sglist_free(txq->gl);
4667 free(txq->sdesc, M_CXGBE);
4668 mp_ring_free(txq->r);
4669 free_eq(sc, eq);
4670 MPASS(!(eq->flags & EQ_SW_ALLOCATED));
4671 bzero(txq, sizeof(*txq));
4672 }
4673}
4674
4675static void
4676add_txq_sysctls(struct vi_info *vi, struct sysctl_ctx_list *ctx,
4677 struct sysctl_oid *oid, struct sge_txq *txq)
4678{
4679 struct adapter *sc;
4680 struct sysctl_oid_list *children;
4681
4682 if (ctx == NULL || oid == NULL)
4683 return;
4684
4685 sc = vi->adapter;
4686 children = SYSCTL_CHILDREN(oid);
4687
4688 mp_ring_sysctls(txq->r, ctx, children);
4689
4690 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "tc",
4691 CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_MPSAFE, vi, txq - sc->sge.txq,
4692 sysctl_tc, "I", "traffic class (-1 means none)");
4693
4694 SYSCTL_ADD_UQUAD(ctx, children, OID_AUTO, "txcsum", CTLFLAG_RD,
4695 &txq->txcsum, "# of times hardware assisted with checksum");
4696 SYSCTL_ADD_UQUAD(ctx, children, OID_AUTO, "vlan_insertion", CTLFLAG_RD,
4697 &txq->vlan_insertion, "# of times hardware inserted 802.1Q tag");
4698 SYSCTL_ADD_UQUAD(ctx, children, OID_AUTO, "tso_wrs", CTLFLAG_RD,
4699 &txq->tso_wrs, "# of TSO work requests");
4700 SYSCTL_ADD_UQUAD(ctx, children, OID_AUTO, "imm_wrs", CTLFLAG_RD,
4701 &txq->imm_wrs, "# of work requests with immediate data");
4702 SYSCTL_ADD_UQUAD(ctx, children, OID_AUTO, "sgl_wrs", CTLFLAG_RD,
4703 &txq->sgl_wrs, "# of work requests with direct SGL");
4704 SYSCTL_ADD_UQUAD(ctx, children, OID_AUTO, "txpkt_wrs", CTLFLAG_RD,
4705 &txq->txpkt_wrs, "# of txpkt work requests (one pkt/WR)");
4706 SYSCTL_ADD_UQUAD(ctx, children, OID_AUTO, "txpkts0_wrs", CTLFLAG_RD,
4707 &txq->txpkts0_wrs, "# of txpkts (type 0) work requests");
4708 SYSCTL_ADD_UQUAD(ctx, children, OID_AUTO, "txpkts1_wrs", CTLFLAG_RD,
4709 &txq->txpkts1_wrs, "# of txpkts (type 1) work requests");
4710 SYSCTL_ADD_UQUAD(ctx, children, OID_AUTO, "txpkts0_pkts", CTLFLAG_RD,
4711 &txq->txpkts0_pkts,
4712 "# of frames tx'd using type0 txpkts work requests");
4713 SYSCTL_ADD_UQUAD(ctx, children, OID_AUTO, "txpkts1_pkts", CTLFLAG_RD,
4714 &txq->txpkts1_pkts,
4715 "# of frames tx'd using type1 txpkts work requests");
4716 SYSCTL_ADD_UQUAD(ctx, children, OID_AUTO, "txpkts_flush", CTLFLAG_RD,
4717 &txq->txpkts_flush,
4718 "# of times txpkts had to be flushed out by an egress-update");
4719 SYSCTL_ADD_UQUAD(ctx, children, OID_AUTO, "raw_wrs", CTLFLAG_RD,
4720 &txq->raw_wrs, "# of raw work requests (non-packets)");
4721 SYSCTL_ADD_UQUAD(ctx, children, OID_AUTO, "vxlan_tso_wrs", CTLFLAG_RD,
4722 &txq->vxlan_tso_wrs, "# of VXLAN TSO work requests");
4723 SYSCTL_ADD_UQUAD(ctx, children, OID_AUTO, "vxlan_txcsum", CTLFLAG_RD,
4724 &txq->vxlan_txcsum,
4725 "# of times hardware assisted with inner checksums (VXLAN)");
4726
4727#ifdef KERN_TLS
4728 if (is_ktls(sc)) {
4729 SYSCTL_ADD_UQUAD(ctx, children, OID_AUTO, "kern_tls_records",
4730 CTLFLAG_RD, &txq->kern_tls_records,
4731 "# of NIC TLS records transmitted");
4732 SYSCTL_ADD_UQUAD(ctx, children, OID_AUTO, "kern_tls_short",
4733 CTLFLAG_RD, &txq->kern_tls_short,
4734 "# of short NIC TLS records transmitted");
4735 SYSCTL_ADD_UQUAD(ctx, children, OID_AUTO, "kern_tls_partial",
4736 CTLFLAG_RD, &txq->kern_tls_partial,
4737 "# of partial NIC TLS records transmitted");
4738 SYSCTL_ADD_UQUAD(ctx, children, OID_AUTO, "kern_tls_full",
4739 CTLFLAG_RD, &txq->kern_tls_full,
4740 "# of full NIC TLS records transmitted");
4741 SYSCTL_ADD_UQUAD(ctx, children, OID_AUTO, "kern_tls_octets",
4742 CTLFLAG_RD, &txq->kern_tls_octets,
4743 "# of payload octets in transmitted NIC TLS records");
4744 SYSCTL_ADD_UQUAD(ctx, children, OID_AUTO, "kern_tls_waste",
4745 CTLFLAG_RD, &txq->kern_tls_waste,
4746 "# of octets DMAd but not transmitted in NIC TLS records");
4747 SYSCTL_ADD_UQUAD(ctx, children, OID_AUTO, "kern_tls_options",
4748 CTLFLAG_RD, &txq->kern_tls_options,
4749 "# of NIC TLS options-only packets transmitted");
4750 SYSCTL_ADD_UQUAD(ctx, children, OID_AUTO, "kern_tls_header",
4751 CTLFLAG_RD, &txq->kern_tls_header,
4752 "# of NIC TLS header-only packets transmitted");
4753 SYSCTL_ADD_UQUAD(ctx, children, OID_AUTO, "kern_tls_fin",
4754 CTLFLAG_RD, &txq->kern_tls_fin,
4755 "# of NIC TLS FIN-only packets transmitted");
4756 SYSCTL_ADD_UQUAD(ctx, children, OID_AUTO, "kern_tls_fin_short",
4757 CTLFLAG_RD, &txq->kern_tls_fin_short,
4758 "# of NIC TLS padded FIN packets on short TLS records");
4759 SYSCTL_ADD_UQUAD(ctx, children, OID_AUTO, "kern_tls_cbc",
4760 CTLFLAG_RD, &txq->kern_tls_cbc,
4761 "# of NIC TLS sessions using AES-CBC");
4762 SYSCTL_ADD_UQUAD(ctx, children, OID_AUTO, "kern_tls_gcm",
4763 CTLFLAG_RD, &txq->kern_tls_gcm,
4764 "# of NIC TLS sessions using AES-GCM");
4765 }
4766#endif
4767}
4768
4769#if defined(TCP_OFFLOAD) || defined(RATELIMIT)
4770/*
4771 * Idempotent.
4772 */
4773static int
4774alloc_ofld_txq(struct vi_info *vi, struct sge_ofld_txq *ofld_txq, int idx)
4775{
4776 struct sysctl_oid *oid;
4777 struct port_info *pi = vi->pi;
4778 struct adapter *sc = vi->adapter;
4779 struct sge_eq *eq = &ofld_txq->wrq.eq;
4780 int rc, iqidx;
4781 char name[16];
4782
4783 MPASS(idx >= 0);
4784 MPASS(idx < vi->nofldtxq);
4785
4786 if (!(eq->flags & EQ_SW_ALLOCATED)) {
4787 snprintf(name, sizeof(name), "%d", idx);
4788 oid = SYSCTL_ADD_NODE(&vi->ctx,
4789 SYSCTL_CHILDREN(vi->ofld_txq_oid), OID_AUTO, name,
4790 CTLFLAG_RD | CTLFLAG_MPSAFE, NULL, "offload tx queue");
4791
4792 snprintf(name, sizeof(name), "%s ofld_txq%d",
4793 device_get_nameunit(vi->dev), idx);
4794 if (vi->nofldrxq > 0) {
4795 iqidx = vi->first_ofld_rxq + (idx % vi->nofldrxq);
4796 init_eq(sc, eq, EQ_OFLD, vi->qsize_txq, pi->tx_chan,
4797 &sc->sge.ofld_rxq[iqidx].iq, name);
4798 } else {
4799 iqidx = vi->first_rxq + (idx % vi->nrxq);
4800 init_eq(sc, eq, EQ_OFLD, vi->qsize_txq, pi->tx_chan,
4801 &sc->sge.rxq[iqidx].iq, name);
4802 }
4803
4804 rc = alloc_wrq(sc, vi, &ofld_txq->wrq, &vi->ctx, oid);
4805 if (rc != 0) {
4806 CH_ERR(vi, "failed to allocate ofld_txq%d: %d\n", idx,
4807 rc);
4808 sysctl_remove_oid(oid, 1, 1);
4809 return (rc);
4810 }
4811 MPASS(eq->flags & EQ_SW_ALLOCATED);
4812 /* Can't fail after this point. */
4813
4814 ofld_txq->tx_iscsi_pdus = counter_u64_alloc(M_WAITOK);
4815 ofld_txq->tx_iscsi_octets = counter_u64_alloc(M_WAITOK);
4816 ofld_txq->tx_iscsi_iso_wrs = counter_u64_alloc(M_WAITOK);
4817 ofld_txq->tx_toe_tls_records = counter_u64_alloc(M_WAITOK);
4818 ofld_txq->tx_toe_tls_octets = counter_u64_alloc(M_WAITOK);
4819 add_ofld_txq_sysctls(&vi->ctx, oid, ofld_txq);
4820 }
4821
4822 if (!(eq->flags & EQ_HW_ALLOCATED)) {
4823 rc = alloc_eq_hwq(sc, vi, eq);
4824 if (rc != 0) {
4825 CH_ERR(vi, "failed to create hw ofld_txq%d: %d\n", idx,
4826 rc);
4827 return (rc);
4828 }
4829 MPASS(eq->flags & EQ_HW_ALLOCATED);
4830 }
4831
4832 return (0);
4833}
4834
4835/*
4836 * Idempotent.
4837 */
4838static void
4839free_ofld_txq(struct vi_info *vi, struct sge_ofld_txq *ofld_txq)
4840{
4841 struct adapter *sc = vi->adapter;
4842 struct sge_eq *eq = &ofld_txq->wrq.eq;
4843
4844 if (eq->flags & EQ_HW_ALLOCATED) {
4845 MPASS(eq->flags & EQ_SW_ALLOCATED);
4846 free_eq_hwq(sc, NULL, eq);
4847 MPASS(!(eq->flags & EQ_HW_ALLOCATED));
4848 }
4849
4850 if (eq->flags & EQ_SW_ALLOCATED) {
4851 MPASS(!(eq->flags & EQ_HW_ALLOCATED));
4852 counter_u64_free(ofld_txq->tx_iscsi_pdus);
4853 counter_u64_free(ofld_txq->tx_iscsi_octets);
4854 counter_u64_free(ofld_txq->tx_iscsi_iso_wrs);
4855 counter_u64_free(ofld_txq->tx_toe_tls_records);
4856 counter_u64_free(ofld_txq->tx_toe_tls_octets);
4857 free_wrq(sc, &ofld_txq->wrq);
4858 MPASS(!(eq->flags & EQ_SW_ALLOCATED));
4859 bzero(ofld_txq, sizeof(*ofld_txq));
4860 }
4861}
4862
4863static void
4864add_ofld_txq_sysctls(struct sysctl_ctx_list *ctx, struct sysctl_oid *oid,
4865 struct sge_ofld_txq *ofld_txq)
4866{
4867 struct sysctl_oid_list *children;
4868
4869 if (ctx == NULL || oid == NULL)
4870 return;
4871
4872 children = SYSCTL_CHILDREN(oid);
4873 SYSCTL_ADD_COUNTER_U64(ctx, children, OID_AUTO, "tx_iscsi_pdus",
4874 CTLFLAG_RD, &ofld_txq->tx_iscsi_pdus,
4875 "# of iSCSI PDUs transmitted");
4876 SYSCTL_ADD_COUNTER_U64(ctx, children, OID_AUTO, "tx_iscsi_octets",
4877 CTLFLAG_RD, &ofld_txq->tx_iscsi_octets,
4878 "# of payload octets in transmitted iSCSI PDUs");
4879 SYSCTL_ADD_COUNTER_U64(ctx, children, OID_AUTO, "tx_iscsi_iso_wrs",
4880 CTLFLAG_RD, &ofld_txq->tx_iscsi_iso_wrs,
4881 "# of iSCSI segmentation offload work requests");
4882 SYSCTL_ADD_COUNTER_U64(ctx, children, OID_AUTO, "tx_toe_tls_records",
4883 CTLFLAG_RD, &ofld_txq->tx_toe_tls_records,
4884 "# of TOE TLS records transmitted");
4885 SYSCTL_ADD_COUNTER_U64(ctx, children, OID_AUTO, "tx_toe_tls_octets",
4886 CTLFLAG_RD, &ofld_txq->tx_toe_tls_octets,
4887 "# of payload octets in transmitted TOE TLS records");
4888}
4889#endif
4890
4891static void
4892oneseg_dma_callback(void *arg, bus_dma_segment_t *segs, int nseg, int error)
4893{
4894 bus_addr_t *ba = arg;
4895
4896 KASSERT(nseg == 1,
4897 ("%s meant for single segment mappings only.", __func__));
4898
4899 *ba = error ? 0 : segs->ds_addr;
4900}
4901
4902static inline void
4903ring_fl_db(struct adapter *sc, struct sge_fl *fl)
4904{
4905 uint32_t n, v;
4906
4907 n = IDXDIFF(fl->pidx >> 3, fl->dbidx, fl->sidx);
4908 MPASS(n > 0);
4909
4910 wmb();
4911 v = fl->dbval | V_PIDX(n);
4912 if (fl->udb)
4913 *fl->udb = htole32(v);
4914 else
4915 t4_write_reg(sc, sc->sge_kdoorbell_reg, v);
4916 IDXINCR(fl->dbidx, n, fl->sidx);
4917}
4918
4919/*
4920 * Fills up the freelist by allocating up to 'n' buffers. Buffers that are
4921 * recycled do not count towards this allocation budget.
4922 *
4923 * Returns non-zero to indicate that this freelist should be added to the list
4924 * of starving freelists.
4925 */
4926static int
4927refill_fl(struct adapter *sc, struct sge_fl *fl, int n)
4928{
4929 __be64 *d;
4930 struct fl_sdesc *sd;
4931 uintptr_t pa;
4932 caddr_t cl;
4933 struct rx_buf_info *rxb;
4934 struct cluster_metadata *clm;
4935 uint16_t max_pidx, zidx = fl->zidx;
4936 uint16_t hw_cidx = fl->hw_cidx; /* stable snapshot */
4937
4938 FL_LOCK_ASSERT_OWNED(fl);
4939
4940 /*
4941 * We always stop at the beginning of the hardware descriptor that's just
4942 * before the one with the hw cidx. This is to avoid hw pidx = hw cidx,
4943 * which would mean an empty freelist to the chip.
4944 */
4945 max_pidx = __predict_false(hw_cidx == 0) ? fl->sidx - 1 : hw_cidx - 1;
4946 if (fl->pidx == max_pidx * 8)
4947 return (0);
4948
4949 d = &fl->desc[fl->pidx];
4950 sd = &fl->sdesc[fl->pidx];
4951 rxb = &sc->sge.rx_buf_info[zidx];
4952
4953 while (n > 0) {
4954
4955 if (sd->cl != NULL) {
4956
4957 if (sd->nmbuf == 0) {
4958 /*
4959 * Fast recycle without involving any atomics on
4960 * the cluster's metadata (if the cluster has
4961 * metadata). This happens when all frames
4962 * received in the cluster were small enough to
4963 * fit within a single mbuf each.
4964 */
4965 fl->cl_fast_recycled++;
4966 goto recycled;
4967 }
4968
4969 /*
4970 * Cluster is guaranteed to have metadata. Clusters
4971 * without metadata always take the fast recycle path
4972 * when they're recycled.
4973 */
4974 clm = cl_metadata(sd);
4975 MPASS(clm != NULL);
4976
4977 if (atomic_fetchadd_int(&clm->refcount, -1) == 1) {
4978 fl->cl_recycled++;
4979 counter_u64_add(extfree_rels, 1);
4980 goto recycled;
4981 }
4982 sd->cl = NULL; /* gave up my reference */
4983 }
4984 MPASS(sd->cl == NULL);
4985 cl = uma_zalloc(rxb->zone, M_NOWAIT);
4986 if (__predict_false(cl == NULL)) {
4987 if (zidx != fl->safe_zidx) {
4988 zidx = fl->safe_zidx;
4989 rxb = &sc->sge.rx_buf_info[zidx];
4990 cl = uma_zalloc(rxb->zone, M_NOWAIT);
4991 }
4992 if (cl == NULL)
4993 break;
4994 }
4995 fl->cl_allocated++;
4996 n--;
4997
4998 pa = pmap_kextract((vm_offset_t)cl);
4999 sd->cl = cl;
5000 sd->zidx = zidx;
5001
5002 if (fl->flags & FL_BUF_PACKING) {
5003 *d = htobe64(pa | rxb->hwidx2);
5004 sd->moff = rxb->size2;
5005 } else {
5006 *d = htobe64(pa | rxb->hwidx1);
5007 sd->moff = 0;
5008 }
5009recycled:
5010 sd->nmbuf = 0;
5011 d++;
5012 sd++;
5013 if (__predict_false((++fl->pidx & 7) == 0)) {
5014 uint16_t pidx = fl->pidx >> 3;
5015
5016 if (__predict_false(pidx == fl->sidx)) {
5017 fl->pidx = 0;
5018 pidx = 0;
5019 sd = fl->sdesc;
5020 d = fl->desc;
5021 }
5022 if (n < 8 || pidx == max_pidx)
5023 break;
5024
5025 if (IDXDIFF(pidx, fl->dbidx, fl->sidx) >= 4)
5026 ring_fl_db(sc, fl);
5027 }
5028 }
5029
5030 if ((fl->pidx >> 3) != fl->dbidx)
5031 ring_fl_db(sc, fl);
5032
5033 return (FL_RUNNING_LOW(fl) && !(fl->flags & FL_STARVING));
5034}
5035
5036/*
5037 * Attempt to refill all starving freelists.
5038 */
5039static void
5040refill_sfl(void *arg)
5041{
5042 struct adapter *sc = arg;
5043 struct sge_fl *fl, *fl_temp;
5044
5045 mtx_assert(&sc->sfl_lock, MA_OWNED);
5046 TAILQ_FOREACH_SAFE(fl, &sc->sfl, link, fl_temp) {
5047 FL_LOCK(fl);
5048 refill_fl(sc, fl, 64);
5049 if (FL_NOT_RUNNING_LOW(fl) || fl->flags & FL_DOOMED) {
5050 TAILQ_REMOVE(&sc->sfl, fl, link);
5051 fl->flags &= ~FL_STARVING;
5052 }
5053 FL_UNLOCK(fl);
5054 }
5055
5056 if (!TAILQ_EMPTY(&sc->sfl))
5057 callout_schedule(&sc->sfl_callout, hz / 5);
5058}
5059
5060/*
5061 * Release the driver's reference on all buffers in the given freelist. Buffers
5062 * with kernel references cannot be freed and will prevent the driver from being
5063 * unloaded safely.
5064 */
5065void
5066free_fl_buffers(struct adapter *sc, struct sge_fl *fl)
5067{
5068 struct fl_sdesc *sd;
5069 struct cluster_metadata *clm;
5070 int i;
5071
5072 sd = fl->sdesc;
5073 for (i = 0; i < fl->sidx * 8; i++, sd++) {
5074 if (sd->cl == NULL)
5075 continue;
5076
5077 if (sd->nmbuf == 0)
5078 uma_zfree(sc->sge.rx_buf_info[sd->zidx].zone, sd->cl);
5079 else if (fl->flags & FL_BUF_PACKING) {
5080 clm = cl_metadata(sd);
5081 if (atomic_fetchadd_int(&clm->refcount, -1) == 1) {
5082 uma_zfree(sc->sge.rx_buf_info[sd->zidx].zone,
5083 sd->cl);
5084 counter_u64_add(extfree_rels, 1);
5085 }
5086 }
5087 sd->cl = NULL;
5088 }
5089
5090 if (fl->flags & FL_BUF_RESUME) {
5091 m_freem(fl->m0);
5092 fl->flags &= ~FL_BUF_RESUME;
5093 }
5094}
5095
5096static inline void
5097get_pkt_gl(struct mbuf *m, struct sglist *gl)
5098{
5099 int rc;
5100
5101 M_ASSERTPKTHDR(m);
5102
5103 sglist_reset(gl);
5104 rc = sglist_append_mbuf(gl, m);
5105 if (__predict_false(rc != 0)) {
5106 panic("%s: mbuf %p (%d segs) was vetted earlier but now fails "
5107 "with %d.", __func__, m, mbuf_nsegs(m), rc);
5108 }
5109
5110 KASSERT(gl->sg_nseg == mbuf_nsegs(m),
5111 ("%s: nsegs changed for mbuf %p from %d to %d", __func__, m,
5112 mbuf_nsegs(m), gl->sg_nseg));
5113#if 0 /* vm_wr not readily available here. */
5114 KASSERT(gl->sg_nseg > 0 && gl->sg_nseg <= max_nsegs_allowed(m, vm_wr),
5115 ("%s: %d segments, should have been 1 <= nsegs <= %d", __func__,
5116 gl->sg_nseg, max_nsegs_allowed(m, vm_wr)));
5117#endif
5118}
5119
5120/*
5121 * len16 for a txpkt WR with a GL. Includes the firmware work request header.
5122 */
5123static inline u_int
5124txpkt_len16(u_int nsegs, const u_int extra)
5125{
5126 u_int n;
5127
5128 MPASS(nsegs > 0);
5129
5130 nsegs--; /* first segment is part of ulptx_sgl */
5131 n = extra + sizeof(struct fw_eth_tx_pkt_wr) +
5132 sizeof(struct cpl_tx_pkt_core) +
5133 sizeof(struct ulptx_sgl) + 8 * ((3 * nsegs) / 2 + (nsegs & 1));
5134
5135 return (howmany(n, 16));
5136}
5137
5138/*
5139 * len16 for a txpkt_vm WR with a GL. Includes the firmware work
5140 * request header.
5141 */
5142static inline u_int
5143txpkt_vm_len16(u_int nsegs, const u_int extra)
5144{
5145 u_int n;
5146
5147 MPASS(nsegs > 0);
5148
5149 nsegs--; /* first segment is part of ulptx_sgl */
5150 n = extra + sizeof(struct fw_eth_tx_pkt_vm_wr) +
5151 sizeof(struct cpl_tx_pkt_core) +
5152 sizeof(struct ulptx_sgl) + 8 * ((3 * nsegs) / 2 + (nsegs & 1));
5153
5154 return (howmany(n, 16));
5155}
5156
5157static inline void
5158calculate_mbuf_len16(struct mbuf *m, bool vm_wr)
5159{
5160 const int lso = sizeof(struct cpl_tx_pkt_lso_core);
5161 const int tnl_lso = sizeof(struct cpl_tx_tnl_lso);
5162
5163 if (vm_wr) {
5164 if (needs_tso(m))
5165 set_mbuf_len16(m, txpkt_vm_len16(mbuf_nsegs(m), lso));
5166 else
5167 set_mbuf_len16(m, txpkt_vm_len16(mbuf_nsegs(m), 0));
5168 return;
5169 }
5170
5171 if (needs_tso(m)) {
5172 if (needs_vxlan_tso(m))
5173 set_mbuf_len16(m, txpkt_len16(mbuf_nsegs(m), tnl_lso));
5174 else
5175 set_mbuf_len16(m, txpkt_len16(mbuf_nsegs(m), lso));
5176 } else
5177 set_mbuf_len16(m, txpkt_len16(mbuf_nsegs(m), 0));
5178}
5179
5180/*
5181 * len16 for a txpkts type 0 WR with a GL. Does not include the firmware work
5182 * request header.
5183 */
5184static inline u_int
5185txpkts0_len16(u_int nsegs)
5186{
5187 u_int n;
5188
5189 MPASS(nsegs > 0);
5190
5191 nsegs--; /* first segment is part of ulptx_sgl */
5192 n = sizeof(struct ulp_txpkt) + sizeof(struct ulptx_idata) +
5193 sizeof(struct cpl_tx_pkt_core) + sizeof(struct ulptx_sgl) +
5194 8 * ((3 * nsegs) / 2 + (nsegs & 1));
5195
5196 return (howmany(n, 16));
5197}
5198
5199/*
5200 * len16 for a txpkts type 1 WR with a GL. Does not include the firmware work
5201 * request header.
5202 */
5203static inline u_int
5204txpkts1_len16(void)
5205{
5206 u_int n;
5207
5208 n = sizeof(struct cpl_tx_pkt_core) + sizeof(struct ulptx_sgl);
5209
5210 return (howmany(n, 16));
5211}
5212
5213static inline u_int
5214imm_payload(u_int ndesc)
5215{
5216 u_int n;
5217
5218 n = ndesc * EQ_ESIZE - sizeof(struct fw_eth_tx_pkt_wr) -
5219 sizeof(struct cpl_tx_pkt_core);
5220
5221 return (n);
5222}
5223
5224static inline uint64_t
5225csum_to_ctrl(struct adapter *sc, struct mbuf *m)
5226{
5227 uint64_t ctrl;
5228 int csum_type, l2hlen, l3hlen;
5229 int x, y;
5230 static const int csum_types[3][2] = {
5231 {TX_CSUM_TCPIP, TX_CSUM_TCPIP6},
5232 {TX_CSUM_UDPIP, TX_CSUM_UDPIP6},
5233 {TX_CSUM_IP, 0}
5234 };
5235
5236 M_ASSERTPKTHDR(m);
5237
5238 if (!needs_hwcsum(m))
5239 return (F_TXPKT_IPCSUM_DIS | F_TXPKT_L4CSUM_DIS);
5240
5241 MPASS(m->m_pkthdr.l2hlen >= ETHER_HDR_LEN);
5242 MPASS(m->m_pkthdr.l3hlen >= sizeof(struct ip));
5243
5244 if (needs_vxlan_csum(m)) {
5245 MPASS(m->m_pkthdr.l4hlen > 0);
5246 MPASS(m->m_pkthdr.l5hlen > 0);
5247 MPASS(m->m_pkthdr.inner_l2hlen >= ETHER_HDR_LEN);
5248 MPASS(m->m_pkthdr.inner_l3hlen >= sizeof(struct ip));
5249
5250 l2hlen = m->m_pkthdr.l2hlen + m->m_pkthdr.l3hlen +
5251 m->m_pkthdr.l4hlen + m->m_pkthdr.l5hlen +
5252 m->m_pkthdr.inner_l2hlen - ETHER_HDR_LEN;
5253 l3hlen = m->m_pkthdr.inner_l3hlen;
5254 } else {
5255 l2hlen = m->m_pkthdr.l2hlen - ETHER_HDR_LEN;
5256 l3hlen = m->m_pkthdr.l3hlen;
5257 }
5258
5259 ctrl = 0;
5260 if (!needs_l3_csum(m))
5261 ctrl |= F_TXPKT_IPCSUM_DIS;
5262
5263 if (m->m_pkthdr.csum_flags & (CSUM_IP_TCP | CSUM_INNER_IP_TCP |
5264 CSUM_IP6_TCP | CSUM_INNER_IP6_TCP))
5265 x = 0; /* TCP */
5266 else if (m->m_pkthdr.csum_flags & (CSUM_IP_UDP | CSUM_INNER_IP_UDP |
5267 CSUM_IP6_UDP | CSUM_INNER_IP6_UDP))
5268 x = 1; /* UDP */
5269 else
5270 x = 2;
5271
5272 if (m->m_pkthdr.csum_flags & (CSUM_IP | CSUM_IP_TCP | CSUM_IP_UDP |
5273 CSUM_INNER_IP | CSUM_INNER_IP_TCP | CSUM_INNER_IP_UDP))
5274 y = 0; /* IPv4 */
5275 else {
5276 MPASS(m->m_pkthdr.csum_flags & (CSUM_IP6_TCP | CSUM_IP6_UDP |
5277 CSUM_INNER_IP6_TCP | CSUM_INNER_IP6_UDP));
5278 y = 1; /* IPv6 */
5279 }
5280 /*
5281 * needs_hwcsum returned true earlier so there must be some kind of
5282 * checksum to calculate.
5283 */
5284 csum_type = csum_types[x][y];
5285 MPASS(csum_type != 0);
5286 if (csum_type == TX_CSUM_IP)
5287 ctrl |= F_TXPKT_L4CSUM_DIS;
5288 ctrl |= V_TXPKT_CSUM_TYPE(csum_type) | V_TXPKT_IPHDR_LEN(l3hlen);
5289 if (chip_id(sc) <= CHELSIO_T5)
5290 ctrl |= V_TXPKT_ETHHDR_LEN(l2hlen);
5291 else
5292 ctrl |= V_T6_TXPKT_ETHHDR_LEN(l2hlen);
5293
5294 return (ctrl);
5295}
5296
5297static inline void *
5298write_lso_cpl(void *cpl, struct mbuf *m0)
5299{
5300 struct cpl_tx_pkt_lso_core *lso;
5301 uint32_t ctrl;
5302
5303 KASSERT(m0->m_pkthdr.l2hlen > 0 && m0->m_pkthdr.l3hlen > 0 &&
5304 m0->m_pkthdr.l4hlen > 0,
5305 ("%s: mbuf %p needs TSO but missing header lengths",
5306 __func__, m0));
5307
5308 ctrl = V_LSO_OPCODE(CPL_TX_PKT_LSO) |
5309 F_LSO_FIRST_SLICE | F_LSO_LAST_SLICE |
5310 V_LSO_ETHHDR_LEN((m0->m_pkthdr.l2hlen - ETHER_HDR_LEN) >> 2) |
5311 V_LSO_IPHDR_LEN(m0->m_pkthdr.l3hlen >> 2) |
5312 V_LSO_TCPHDR_LEN(m0->m_pkthdr.l4hlen >> 2);
5313 if (m0->m_pkthdr.l3hlen == sizeof(struct ip6_hdr))
5314 ctrl |= F_LSO_IPV6;
5315
5316 lso = cpl;
5317 lso->lso_ctrl = htobe32(ctrl);
5318 lso->ipid_ofst = htobe16(0);
5319 lso->mss = htobe16(m0->m_pkthdr.tso_segsz);
5320 lso->seqno_offset = htobe32(0);
5321 lso->len = htobe32(m0->m_pkthdr.len);
5322
5323 return (lso + 1);
5324}
5325
5326static void *
5327write_tnl_lso_cpl(void *cpl, struct mbuf *m0)
5328{
5329 struct cpl_tx_tnl_lso *tnl_lso = cpl;
5330 uint32_t ctrl;
5331
5332 KASSERT(m0->m_pkthdr.inner_l2hlen > 0 &&
5333 m0->m_pkthdr.inner_l3hlen > 0 && m0->m_pkthdr.inner_l4hlen > 0 &&
5334 m0->m_pkthdr.inner_l5hlen > 0,
5335 ("%s: mbuf %p needs VXLAN_TSO but missing inner header lengths",
5336 __func__, m0));
5337 KASSERT(m0->m_pkthdr.l2hlen > 0 && m0->m_pkthdr.l3hlen > 0 &&
5338 m0->m_pkthdr.l4hlen > 0 && m0->m_pkthdr.l5hlen > 0,
5339 ("%s: mbuf %p needs VXLAN_TSO but missing outer header lengths",
5340 __func__, m0));
5341
5342 /* Outer headers. */
5343 ctrl = V_CPL_TX_TNL_LSO_OPCODE(CPL_TX_TNL_LSO) |
5344 F_CPL_TX_TNL_LSO_FIRST | F_CPL_TX_TNL_LSO_LAST |
5345 V_CPL_TX_TNL_LSO_ETHHDRLENOUT(
5346 (m0->m_pkthdr.l2hlen - ETHER_HDR_LEN) >> 2) |
5347 V_CPL_TX_TNL_LSO_IPHDRLENOUT(m0->m_pkthdr.l3hlen >> 2) |
5348 F_CPL_TX_TNL_LSO_IPLENSETOUT;
5349 if (m0->m_pkthdr.l3hlen == sizeof(struct ip6_hdr))
5350 ctrl |= F_CPL_TX_TNL_LSO_IPV6OUT;
5351 else {
5352 ctrl |= F_CPL_TX_TNL_LSO_IPHDRCHKOUT |
5353 F_CPL_TX_TNL_LSO_IPIDINCOUT;
5354 }
5355 tnl_lso->op_to_IpIdSplitOut = htobe32(ctrl);
5356 tnl_lso->IpIdOffsetOut = 0;
5357 tnl_lso->UdpLenSetOut_to_TnlHdrLen =
5358 htobe16(F_CPL_TX_TNL_LSO_UDPCHKCLROUT |
5359 F_CPL_TX_TNL_LSO_UDPLENSETOUT |
5360 V_CPL_TX_TNL_LSO_TNLHDRLEN(m0->m_pkthdr.l2hlen +
5361 m0->m_pkthdr.l3hlen + m0->m_pkthdr.l4hlen +
5362 m0->m_pkthdr.l5hlen) |
5363 V_CPL_TX_TNL_LSO_TNLTYPE(TX_TNL_TYPE_VXLAN));
5364 tnl_lso->r1 = 0;
5365
5366 /* Inner headers. */
5367 ctrl = V_CPL_TX_TNL_LSO_ETHHDRLEN(
5368 (m0->m_pkthdr.inner_l2hlen - ETHER_HDR_LEN) >> 2) |
5369 V_CPL_TX_TNL_LSO_IPHDRLEN(m0->m_pkthdr.inner_l3hlen >> 2) |
5370 V_CPL_TX_TNL_LSO_TCPHDRLEN(m0->m_pkthdr.inner_l4hlen >> 2);
5371 if (m0->m_pkthdr.inner_l3hlen == sizeof(struct ip6_hdr))
5372 ctrl |= F_CPL_TX_TNL_LSO_IPV6;
5373 tnl_lso->Flow_to_TcpHdrLen = htobe32(ctrl);
5374 tnl_lso->IpIdOffset = 0;
5375 tnl_lso->IpIdSplit_to_Mss =
5376 htobe16(V_CPL_TX_TNL_LSO_MSS(m0->m_pkthdr.tso_segsz));
5377 tnl_lso->TCPSeqOffset = 0;
5378 tnl_lso->EthLenOffset_Size =
5379 htobe32(V_CPL_TX_TNL_LSO_SIZE(m0->m_pkthdr.len));
5380
5381 return (tnl_lso + 1);
5382}
5383
5384#define VM_TX_L2HDR_LEN 16 /* ethmacdst to vlantci */
5385
5386/*
5387 * Write a VM txpkt WR for this packet to the hardware descriptors, update the
5388 * software descriptor, and advance the pidx. It is guaranteed that enough
5389 * descriptors are available.
5390 *
5391 * The return value is the # of hardware descriptors used.
5392 */
5393static u_int
5394write_txpkt_vm_wr(struct adapter *sc, struct sge_txq *txq, struct mbuf *m0)
5395{
5396 struct sge_eq *eq;
5397 struct fw_eth_tx_pkt_vm_wr *wr;
5398 struct tx_sdesc *txsd;
5399 struct cpl_tx_pkt_core *cpl;
5400 uint32_t ctrl; /* used in many unrelated places */
5401 uint64_t ctrl1;
5402 int len16, ndesc, pktlen;
5403 caddr_t dst;
5404
5405 TXQ_LOCK_ASSERT_OWNED(txq);
5406 M_ASSERTPKTHDR(m0);
5407
5408 len16 = mbuf_len16(m0);
5409 pktlen = m0->m_pkthdr.len;
5410 ctrl = sizeof(struct cpl_tx_pkt_core);
5411 if (needs_tso(m0))
5412 ctrl += sizeof(struct cpl_tx_pkt_lso_core);
5413 ndesc = tx_len16_to_desc(len16);
5414
5415 /* Firmware work request header */
5416 eq = &txq->eq;
5417 wr = (void *)&eq->desc[eq->pidx];
5418 wr->op_immdlen = htobe32(V_FW_WR_OP(FW_ETH_TX_PKT_VM_WR) |
5419 V_FW_ETH_TX_PKT_WR_IMMDLEN(ctrl));
5420
5421 ctrl = V_FW_WR_LEN16(len16);
5422 wr->equiq_to_len16 = htobe32(ctrl);
5423 wr->r3[0] = 0;
5424 wr->r3[1] = 0;
5425
5426 /*
5427 * Copy over ethmacdst, ethmacsrc, ethtype, and vlantci.
5428 * vlantci is ignored unless the ethtype is 0x8100, so it's
5429 * simpler to always copy it rather than making it
5430 * conditional. Also, it seems that we do not have to set
5431 * vlantci or fake the ethtype when doing VLAN tag insertion.
5432 */
5433 m_copydata(m0, 0, VM_TX_L2HDR_LEN, wr->ethmacdst);
5434
5435 if (needs_tso(m0)) {
5436 cpl = write_lso_cpl(wr + 1, m0);
5437 txq->tso_wrs++;
5438 } else
5439 cpl = (void *)(wr + 1);
5440
5441 /* Checksum offload */
5442 ctrl1 = csum_to_ctrl(sc, m0);
5443 if (ctrl1 != (F_TXPKT_IPCSUM_DIS | F_TXPKT_L4CSUM_DIS))
5444 txq->txcsum++; /* some hardware assistance provided */
5445
5446 /* VLAN tag insertion */
5447 if (needs_vlan_insertion(m0)) {
5448 ctrl1 |= F_TXPKT_VLAN_VLD |
5449 V_TXPKT_VLAN(m0->m_pkthdr.ether_vtag);
5450 txq->vlan_insertion++;
5451 }
5452
5453 /* CPL header */
5454 cpl->ctrl0 = txq->cpl_ctrl0;
5455 cpl->pack = 0;
5456 cpl->len = htobe16(pktlen);
5457 cpl->ctrl1 = htobe64(ctrl1);
5458
5459 /* SGL */
5460 dst = (void *)(cpl + 1);
5461
5462 /*
5463 * A packet using TSO will use up an entire descriptor for the
5464 * firmware work request header, LSO CPL, and TX_PKT_XT CPL.
5465 * If this descriptor is the last descriptor in the ring, wrap
5466 * around to the front of the ring explicitly for the start of
5467 * the sgl.
5468 */
5469 if (dst == (void *)&eq->desc[eq->sidx]) {
5470 dst = (void *)&eq->desc[0];
5471 write_gl_to_txd(txq, m0, &dst, 0);
5472 } else
5473 write_gl_to_txd(txq, m0, &dst, eq->sidx - ndesc < eq->pidx);
5474 txq->sgl_wrs++;
5475 txq->txpkt_wrs++;
5476
5477 txsd = &txq->sdesc[eq->pidx];
5478 txsd->m = m0;
5479 txsd->desc_used = ndesc;
5480
5481 return (ndesc);
5482}
5483
5484/*
5485 * Write a raw WR to the hardware descriptors, update the software
5486 * descriptor, and advance the pidx. It is guaranteed that enough
5487 * descriptors are available.
5488 *
5489 * The return value is the # of hardware descriptors used.
5490 */
5491static u_int
5492write_raw_wr(struct sge_txq *txq, void *wr, struct mbuf *m0, u_int available)
5493{
5494 struct sge_eq *eq = &txq->eq;
5495 struct tx_sdesc *txsd;
5496 struct mbuf *m;
5497 caddr_t dst;
5498 int len16, ndesc;
5499
5500 len16 = mbuf_len16(m0);
5501 ndesc = tx_len16_to_desc(len16);
5502 MPASS(ndesc <= available);
5503
5504 dst = wr;
5505 for (m = m0; m != NULL; m = m->m_next)
5506 copy_to_txd(eq, mtod(m, caddr_t), &dst, m->m_len);
5507
5508 txq->raw_wrs++;
5509
5510 txsd = &txq->sdesc[eq->pidx];
5511 txsd->m = m0;
5512 txsd->desc_used = ndesc;
5513
5514 return (ndesc);
5515}
5516
5517/*
5518 * Write a txpkt WR for this packet to the hardware descriptors, update the
5519 * software descriptor, and advance the pidx. It is guaranteed that enough
5520 * descriptors are available.
5521 *
5522 * The return value is the # of hardware descriptors used.
5523 */
5524static u_int
5525write_txpkt_wr(struct adapter *sc, struct sge_txq *txq, struct mbuf *m0,
5526 u_int available)
5527{
5528 struct sge_eq *eq;
5529 struct fw_eth_tx_pkt_wr *wr;
5530 struct tx_sdesc *txsd;
5531 struct cpl_tx_pkt_core *cpl;
5532 uint32_t ctrl; /* used in many unrelated places */
5533 uint64_t ctrl1;
5534 int len16, ndesc, pktlen, nsegs;
5535 caddr_t dst;
5536
5537 TXQ_LOCK_ASSERT_OWNED(txq);
5538 M_ASSERTPKTHDR(m0);
5539
5540 len16 = mbuf_len16(m0);
5541 nsegs = mbuf_nsegs(m0);
5542 pktlen = m0->m_pkthdr.len;
5543 ctrl = sizeof(struct cpl_tx_pkt_core);
5544 if (needs_tso(m0)) {
5545 if (needs_vxlan_tso(m0))
5546 ctrl += sizeof(struct cpl_tx_tnl_lso);
5547 else
5548 ctrl += sizeof(struct cpl_tx_pkt_lso_core);
5549 } else if (!(mbuf_cflags(m0) & MC_NOMAP) && pktlen <= imm_payload(2) &&
5550 available >= 2) {
5551 /* Immediate data. Recalculate len16 and set nsegs to 0. */
5552 ctrl += pktlen;
5553 len16 = howmany(sizeof(struct fw_eth_tx_pkt_wr) +
5554 sizeof(struct cpl_tx_pkt_core) + pktlen, 16);
5555 nsegs = 0;
5556 }
5557 ndesc = tx_len16_to_desc(len16);
5558 MPASS(ndesc <= available);
5559
5560 /* Firmware work request header */
5561 eq = &txq->eq;
5562 wr = (void *)&eq->desc[eq->pidx];
5563 wr->op_immdlen = htobe32(V_FW_WR_OP(FW_ETH_TX_PKT_WR) |
5564 V_FW_ETH_TX_PKT_WR_IMMDLEN(ctrl));
5565
5566 ctrl = V_FW_WR_LEN16(len16);
5567 wr->equiq_to_len16 = htobe32(ctrl);
5568 wr->r3 = 0;
5569
5570 if (needs_tso(m0)) {
5571 if (needs_vxlan_tso(m0)) {
5572 cpl = write_tnl_lso_cpl(wr + 1, m0);
5573 txq->vxlan_tso_wrs++;
5574 } else {
5575 cpl = write_lso_cpl(wr + 1, m0);
5576 txq->tso_wrs++;
5577 }
5578 } else
5579 cpl = (void *)(wr + 1);
5580
5581 /* Checksum offload */
5582 ctrl1 = csum_to_ctrl(sc, m0);
5583 if (ctrl1 != (F_TXPKT_IPCSUM_DIS | F_TXPKT_L4CSUM_DIS)) {
5584 /* some hardware assistance provided */
5585 if (needs_vxlan_csum(m0))
5586 txq->vxlan_txcsum++;
5587 else
5588 txq->txcsum++;
5589 }
5590
5591 /* VLAN tag insertion */
5592 if (needs_vlan_insertion(m0)) {
5593 ctrl1 |= F_TXPKT_VLAN_VLD |
5594 V_TXPKT_VLAN(m0->m_pkthdr.ether_vtag);
5595 txq->vlan_insertion++;
5596 }
5597
5598 /* CPL header */
5599 cpl->ctrl0 = txq->cpl_ctrl0;
5600 cpl->pack = 0;
5601 cpl->len = htobe16(pktlen);
5602 cpl->ctrl1 = htobe64(ctrl1);
5603
5604 /* SGL */
5605 dst = (void *)(cpl + 1);
5606 if (__predict_false((uintptr_t)dst == (uintptr_t)&eq->desc[eq->sidx]))
5607 dst = (caddr_t)&eq->desc[0];
5608 if (nsegs > 0) {
5609
5610 write_gl_to_txd(txq, m0, &dst, eq->sidx - ndesc < eq->pidx);
5611 txq->sgl_wrs++;
5612 } else {
5613 struct mbuf *m;
5614
5615 for (m = m0; m != NULL; m = m->m_next) {
5616 copy_to_txd(eq, mtod(m, caddr_t), &dst, m->m_len);
5617#ifdef INVARIANTS
5618 pktlen -= m->m_len;
5619#endif
5620 }
5621#ifdef INVARIANTS
5622 KASSERT(pktlen == 0, ("%s: %d bytes left.", __func__, pktlen));
5623#endif
5624 txq->imm_wrs++;
5625 }
5626
5627 txq->txpkt_wrs++;
5628
5629 txsd = &txq->sdesc[eq->pidx];
5630 txsd->m = m0;
5631 txsd->desc_used = ndesc;
5632
5633 return (ndesc);
5634}
5635
5636static inline bool
5637cmp_l2hdr(struct txpkts *txp, struct mbuf *m)
5638{
5639 int len;
5640
5641 MPASS(txp->npkt > 0);
5642 MPASS(m->m_len >= VM_TX_L2HDR_LEN);
5643
5644 if (txp->ethtype == be16toh(ETHERTYPE_VLAN))
5645 len = VM_TX_L2HDR_LEN;
5646 else
5647 len = sizeof(struct ether_header);
5648
5649 return (memcmp(m->m_data, &txp->ethmacdst[0], len) != 0);
5650}
5651
5652static inline void
5653save_l2hdr(struct txpkts *txp, struct mbuf *m)
5654{
5655 MPASS(m->m_len >= VM_TX_L2HDR_LEN);
5656
5657 memcpy(&txp->ethmacdst[0], mtod(m, const void *), VM_TX_L2HDR_LEN);
5658}
5659
5660static int
5661add_to_txpkts_vf(struct adapter *sc, struct sge_txq *txq, struct mbuf *m,
5662 int avail, bool *send)
5663{
5664 struct txpkts *txp = &txq->txp;
5665
5666 /* Cannot have TSO and coalesce at the same time. */
5667 if (cannot_use_txpkts(m)) {
5668cannot_coalesce:
5669 *send = txp->npkt > 0;
5670 return (EINVAL);
5671 }
5672
5673 /* VF allows coalescing of type 1 (1 GL) only */
5674 if (mbuf_nsegs(m) > 1)
5675 goto cannot_coalesce;
5676
5677 *send = false;
5678 if (txp->npkt > 0) {
5679 MPASS(tx_len16_to_desc(txp->len16) <= avail);
5680 MPASS(txp->npkt < txp->max_npkt);
5681 MPASS(txp->wr_type == 1); /* VF supports type 1 only */
5682
5683 if (tx_len16_to_desc(txp->len16 + txpkts1_len16()) > avail) {
5684retry_after_send:
5685 *send = true;
5686 return (EAGAIN);
5687 }
5688 if (m->m_pkthdr.len + txp->plen > 65535)
5689 goto retry_after_send;
5690 if (cmp_l2hdr(txp, m))
5691 goto retry_after_send;
5692
5693 txp->len16 += txpkts1_len16();
5694 txp->plen += m->m_pkthdr.len;
5695 txp->mb[txp->npkt++] = m;
5696 if (txp->npkt == txp->max_npkt)
5697 *send = true;
5698 } else {
5699 txp->len16 = howmany(sizeof(struct fw_eth_tx_pkts_vm_wr), 16) +
5700 txpkts1_len16();
5701 if (tx_len16_to_desc(txp->len16) > avail)
5702 goto cannot_coalesce;
5703 txp->npkt = 1;
5704 txp->wr_type = 1;
5705 txp->plen = m->m_pkthdr.len;
5706 txp->mb[0] = m;
5707 save_l2hdr(txp, m);
5708 }
5709 return (0);
5710}
5711
5712static int
5713add_to_txpkts_pf(struct adapter *sc, struct sge_txq *txq, struct mbuf *m,
5714 int avail, bool *send)
5715{
5716 struct txpkts *txp = &txq->txp;
5717 int nsegs;
5718
5719 MPASS(!(sc->flags & IS_VF));
5720
5721 /* Cannot have TSO and coalesce at the same time. */
5722 if (cannot_use_txpkts(m)) {
5723cannot_coalesce:
5724 *send = txp->npkt > 0;
5725 return (EINVAL);
5726 }
5727
5728 *send = false;
5729 nsegs = mbuf_nsegs(m);
5730 if (txp->npkt == 0) {
5731 if (m->m_pkthdr.len > 65535)
5732 goto cannot_coalesce;
5733 if (nsegs > 1) {
5734 txp->wr_type = 0;
5735 txp->len16 =
5736 howmany(sizeof(struct fw_eth_tx_pkts_wr), 16) +
5737 txpkts0_len16(nsegs);
5738 } else {
5739 txp->wr_type = 1;
5740 txp->len16 =
5741 howmany(sizeof(struct fw_eth_tx_pkts_wr), 16) +
5742 txpkts1_len16();
5743 }
5744 if (tx_len16_to_desc(txp->len16) > avail)
5745 goto cannot_coalesce;
5746 txp->npkt = 1;
5747 txp->plen = m->m_pkthdr.len;
5748 txp->mb[0] = m;
5749 } else {
5750 MPASS(tx_len16_to_desc(txp->len16) <= avail);
5751 MPASS(txp->npkt < txp->max_npkt);
5752
5753 if (m->m_pkthdr.len + txp->plen > 65535) {
5754retry_after_send:
5755 *send = true;
5756 return (EAGAIN);
5757 }
5758
5759 MPASS(txp->wr_type == 0 || txp->wr_type == 1);
5760 if (txp->wr_type == 0) {
5761 if (tx_len16_to_desc(txp->len16 +
5762 txpkts0_len16(nsegs)) > min(avail, SGE_MAX_WR_NDESC))
5763 goto retry_after_send;
5764 txp->len16 += txpkts0_len16(nsegs);
5765 } else {
5766 if (nsegs != 1)
5767 goto retry_after_send;
5768 if (tx_len16_to_desc(txp->len16 + txpkts1_len16()) >
5769 avail)
5770 goto retry_after_send;
5771 txp->len16 += txpkts1_len16();
5772 }
5773
5774 txp->plen += m->m_pkthdr.len;
5775 txp->mb[txp->npkt++] = m;
5776 if (txp->npkt == txp->max_npkt)
5777 *send = true;
5778 }
5779 return (0);
5780}
5781
5782/*
5783 * Write a txpkts WR for the packets in txp to the hardware descriptors, update
5784 * the software descriptor, and advance the pidx. It is guaranteed that enough
5785 * descriptors are available.
5786 *
5787 * The return value is the # of hardware descriptors used.
5788 */
5789static u_int
5790write_txpkts_wr(struct adapter *sc, struct sge_txq *txq)
5791{
5792 const struct txpkts *txp = &txq->txp;
5793 struct sge_eq *eq = &txq->eq;
5794 struct fw_eth_tx_pkts_wr *wr;
5795 struct tx_sdesc *txsd;
5796 struct cpl_tx_pkt_core *cpl;
5797 uint64_t ctrl1;
5798 int ndesc, i, checkwrap;
5799 struct mbuf *m, *last;
5800 void *flitp;
5801
5802 TXQ_LOCK_ASSERT_OWNED(txq);
5803 MPASS(txp->npkt > 0);
5804 MPASS(txp->len16 <= howmany(SGE_MAX_WR_LEN, 16));
5805
5806 wr = (void *)&eq->desc[eq->pidx];
5807 wr->op_pkd = htobe32(V_FW_WR_OP(FW_ETH_TX_PKTS_WR));
5808 wr->equiq_to_len16 = htobe32(V_FW_WR_LEN16(txp->len16));
5809 wr->plen = htobe16(txp->plen);
5810 wr->npkt = txp->npkt;
5811 wr->r3 = 0;
5812 wr->type = txp->wr_type;
5813 flitp = wr + 1;
5814
5815 /*
5816 * At this point we are 16B into a hardware descriptor. If checkwrap is
5817 * set then we know the WR is going to wrap around somewhere. We'll
5818 * check for that at appropriate points.
5819 */
5820 ndesc = tx_len16_to_desc(txp->len16);
5821 last = NULL;
5822 checkwrap = eq->sidx - ndesc < eq->pidx;
5823 for (i = 0; i < txp->npkt; i++) {
5824 m = txp->mb[i];
5825 if (txp->wr_type == 0) {
5826 struct ulp_txpkt *ulpmc;
5827 struct ulptx_idata *ulpsc;
5828
5829 /* ULP master command */
5830 ulpmc = flitp;
5831 ulpmc->cmd_dest = htobe32(V_ULPTX_CMD(ULP_TX_PKT) |
5832 V_ULP_TXPKT_DEST(0) | V_ULP_TXPKT_FID(eq->iqid));
5833 ulpmc->len = htobe32(txpkts0_len16(mbuf_nsegs(m)));
5834
5835 /* ULP subcommand */
5836 ulpsc = (void *)(ulpmc + 1);
5837 ulpsc->cmd_more = htobe32(V_ULPTX_CMD(ULP_TX_SC_IMM) |
5838 F_ULP_TX_SC_MORE);
5839 ulpsc->len = htobe32(sizeof(struct cpl_tx_pkt_core));
5840
5841 cpl = (void *)(ulpsc + 1);
5842 if (checkwrap &&
5843 (uintptr_t)cpl == (uintptr_t)&eq->desc[eq->sidx])
5844 cpl = (void *)&eq->desc[0];
5845 } else {
5846 cpl = flitp;
5847 }
5848
5849 /* Checksum offload */
5850 ctrl1 = csum_to_ctrl(sc, m);
5851 if (ctrl1 != (F_TXPKT_IPCSUM_DIS | F_TXPKT_L4CSUM_DIS)) {
5852 /* some hardware assistance provided */
5853 if (needs_vxlan_csum(m))
5854 txq->vxlan_txcsum++;
5855 else
5856 txq->txcsum++;
5857 }
5858
5859 /* VLAN tag insertion */
5860 if (needs_vlan_insertion(m)) {
5861 ctrl1 |= F_TXPKT_VLAN_VLD |
5862 V_TXPKT_VLAN(m->m_pkthdr.ether_vtag);
5863 txq->vlan_insertion++;
5864 }
5865
5866 /* CPL header */
5867 cpl->ctrl0 = txq->cpl_ctrl0;
5868 cpl->pack = 0;
5869 cpl->len = htobe16(m->m_pkthdr.len);
5870 cpl->ctrl1 = htobe64(ctrl1);
5871
5872 flitp = cpl + 1;
5873 if (checkwrap &&
5874 (uintptr_t)flitp == (uintptr_t)&eq->desc[eq->sidx])
5875 flitp = (void *)&eq->desc[0];
5876
5877 write_gl_to_txd(txq, m, (caddr_t *)(&flitp), checkwrap);
5878
5879 if (last != NULL)
5880 last->m_nextpkt = m;
5881 last = m;
5882 }
5883
5884 txq->sgl_wrs++;
5885 if (txp->wr_type == 0) {
5886 txq->txpkts0_pkts += txp->npkt;
5887 txq->txpkts0_wrs++;
5888 } else {
5889 txq->txpkts1_pkts += txp->npkt;
5890 txq->txpkts1_wrs++;
5891 }
5892
5893 txsd = &txq->sdesc[eq->pidx];
5894 txsd->m = txp->mb[0];
5895 txsd->desc_used = ndesc;
5896
5897 return (ndesc);
5898}
5899
5900static u_int
5901write_txpkts_vm_wr(struct adapter *sc, struct sge_txq *txq)
5902{
5903 const struct txpkts *txp = &txq->txp;
5904 struct sge_eq *eq = &txq->eq;
5905 struct fw_eth_tx_pkts_vm_wr *wr;
5906 struct tx_sdesc *txsd;
5907 struct cpl_tx_pkt_core *cpl;
5908 uint64_t ctrl1;
5909 int ndesc, i;
5910 struct mbuf *m, *last;
5911 void *flitp;
5912
5913 TXQ_LOCK_ASSERT_OWNED(txq);
5914 MPASS(txp->npkt > 0);
5915 MPASS(txp->wr_type == 1); /* VF supports type 1 only */
5916 MPASS(txp->mb[0] != NULL);
5917 MPASS(txp->len16 <= howmany(SGE_MAX_WR_LEN, 16));
5918
5919 wr = (void *)&eq->desc[eq->pidx];
5920 wr->op_pkd = htobe32(V_FW_WR_OP(FW_ETH_TX_PKTS_VM_WR));
5921 wr->equiq_to_len16 = htobe32(V_FW_WR_LEN16(txp->len16));
5922 wr->r3 = 0;
5923 wr->plen = htobe16(txp->plen);
5924 wr->npkt = txp->npkt;
5925 wr->r4 = 0;
5926 memcpy(&wr->ethmacdst[0], &txp->ethmacdst[0], 16);
5927 flitp = wr + 1;
5928
5929 /*
5930 * At this point we are 32B into a hardware descriptor. Each mbuf in
5931 * the WR will take 32B so we check for the end of the descriptor ring
5932 * before writing odd mbufs (mb[1], 3, 5, ..)
5933 */
5934 ndesc = tx_len16_to_desc(txp->len16);
5935 last = NULL;
5936 for (i = 0; i < txp->npkt; i++) {
5937 m = txp->mb[i];
5938 if (i & 1 && (uintptr_t)flitp == (uintptr_t)&eq->desc[eq->sidx])
5939 flitp = &eq->desc[0];
5940 cpl = flitp;
5941
5942 /* Checksum offload */
5943 ctrl1 = csum_to_ctrl(sc, m);
5944 if (ctrl1 != (F_TXPKT_IPCSUM_DIS | F_TXPKT_L4CSUM_DIS))
5945 txq->txcsum++; /* some hardware assistance provided */
5946
5947 /* VLAN tag insertion */
5948 if (needs_vlan_insertion(m)) {
5949 ctrl1 |= F_TXPKT_VLAN_VLD |
5950 V_TXPKT_VLAN(m->m_pkthdr.ether_vtag);
5951 txq->vlan_insertion++;
5952 }
5953
5954 /* CPL header */
5955 cpl->ctrl0 = txq->cpl_ctrl0;
5956 cpl->pack = 0;
5957 cpl->len = htobe16(m->m_pkthdr.len);
5958 cpl->ctrl1 = htobe64(ctrl1);
5959
5960 flitp = cpl + 1;
5961 MPASS(mbuf_nsegs(m) == 1);
5962 write_gl_to_txd(txq, m, (caddr_t *)(&flitp), 0);
5963
5964 if (last != NULL)
5965 last->m_nextpkt = m;
5966 last = m;
5967 }
5968
5969 txq->sgl_wrs++;
5970 txq->txpkts1_pkts += txp->npkt;
5971 txq->txpkts1_wrs++;
5972
5973 txsd = &txq->sdesc[eq->pidx];
5974 txsd->m = txp->mb[0];
5975 txsd->desc_used = ndesc;
5976
5977 return (ndesc);
5978}
5979
5980/*
5981 * If the SGL ends on an address that is not 16 byte aligned, this function will
5982 * add a 0 filled flit at the end.
5983 */
5984static void
5985write_gl_to_txd(struct sge_txq *txq, struct mbuf *m, caddr_t *to, int checkwrap)
5986{
5987 struct sge_eq *eq = &txq->eq;
5988 struct sglist *gl = txq->gl;
5989 struct sglist_seg *seg;
5990 __be64 *flitp, *wrap;
5991 struct ulptx_sgl *usgl;
5992 int i, nflits, nsegs;
5993
5994 KASSERT(((uintptr_t)(*to) & 0xf) == 0,
5995 ("%s: SGL must start at a 16 byte boundary: %p", __func__, *to));
5996 MPASS((uintptr_t)(*to) >= (uintptr_t)&eq->desc[0]);
5997 MPASS((uintptr_t)(*to) < (uintptr_t)&eq->desc[eq->sidx]);
5998
5999 get_pkt_gl(m, gl);
6000 nsegs = gl->sg_nseg;
6001 MPASS(nsegs > 0);
6002
6003 nflits = (3 * (nsegs - 1)) / 2 + ((nsegs - 1) & 1) + 2;
6004 flitp = (__be64 *)(*to);
6005 wrap = (__be64 *)(&eq->desc[eq->sidx]);
6006 seg = &gl->sg_segs[0];
6007 usgl = (void *)flitp;
6008
6009 /*
6010 * We start at a 16 byte boundary somewhere inside the tx descriptor
6011 * ring, so we're at least 16 bytes away from the status page. There is
6012 * no chance of a wrap around in the middle of usgl (which is 16 bytes).
6013 */
6014
6015 usgl->cmd_nsge = htobe32(V_ULPTX_CMD(ULP_TX_SC_DSGL) |
6016 V_ULPTX_NSGE(nsegs));
6017 usgl->len0 = htobe32(seg->ss_len);
6018 usgl->addr0 = htobe64(seg->ss_paddr);
6019 seg++;
6020
6021 if (checkwrap == 0 || (uintptr_t)(flitp + nflits) <= (uintptr_t)wrap) {
6022
6023 /* Won't wrap around at all */
6024
6025 for (i = 0; i < nsegs - 1; i++, seg++) {
6026 usgl->sge[i / 2].len[i & 1] = htobe32(seg->ss_len);
6027 usgl->sge[i / 2].addr[i & 1] = htobe64(seg->ss_paddr);
6028 }
6029 if (i & 1)
6030 usgl->sge[i / 2].len[1] = htobe32(0);
6031 flitp += nflits;
6032 } else {
6033
6034 /* Will wrap somewhere in the rest of the SGL */
6035
6036 /* 2 flits already written, write the rest flit by flit */
6037 flitp = (void *)(usgl + 1);
6038 for (i = 0; i < nflits - 2; i++) {
6039 if (flitp == wrap)
6040 flitp = (void *)eq->desc;
6041 *flitp++ = get_flit(seg, nsegs - 1, i);
6042 }
6043 }
6044
6045 if (nflits & 1) {
6046 MPASS(((uintptr_t)flitp) & 0xf);
6047 *flitp++ = 0;
6048 }
6049
6050 MPASS((((uintptr_t)flitp) & 0xf) == 0);
6051 if (__predict_false(flitp == wrap))
6052 *to = (void *)eq->desc;
6053 else
6054 *to = (void *)flitp;
6055}
6056
6057static inline void
6058copy_to_txd(struct sge_eq *eq, caddr_t from, caddr_t *to, int len)
6059{
6060
6061 MPASS((uintptr_t)(*to) >= (uintptr_t)&eq->desc[0]);
6062 MPASS((uintptr_t)(*to) < (uintptr_t)&eq->desc[eq->sidx]);
6063
6064 if (__predict_true((uintptr_t)(*to) + len <=
6065 (uintptr_t)&eq->desc[eq->sidx])) {
6066 bcopy(from, *to, len);
6067 (*to) += len;
6068 } else {
6069 int portion = (uintptr_t)&eq->desc[eq->sidx] - (uintptr_t)(*to);
6070
6071 bcopy(from, *to, portion);
6072 from += portion;
6073 portion = len - portion; /* remaining */
6074 bcopy(from, (void *)eq->desc, portion);
6075 (*to) = (caddr_t)eq->desc + portion;
6076 }
6077}
6078
6079static inline void
6080ring_eq_db(struct adapter *sc, struct sge_eq *eq, u_int n)
6081{
6082 u_int db;
6083
6084 MPASS(n > 0);
6085
6086 db = eq->doorbells;
6087 if (n > 1)
6088 clrbit(&db, DOORBELL_WCWR);
6089 wmb();
6090
6091 switch (ffs(db) - 1) {
6092 case DOORBELL_UDB:
6093 *eq->udb = htole32(V_QID(eq->udb_qid) | V_PIDX(n));
6094 break;
6095
6096 case DOORBELL_WCWR: {
6097 volatile uint64_t *dst, *src;
6098 int i;
6099
6100 /*
6101 * Queues whose 128B doorbell segment fits in the page do not
6102 * use relative qid (udb_qid is always 0). Only queues with
6103 * doorbell segments can do WCWR.
6104 */
6105 KASSERT(eq->udb_qid == 0 && n == 1,
6106 ("%s: inappropriate doorbell (0x%x, %d, %d) for eq %p",
6107 __func__, eq->doorbells, n, eq->dbidx, eq));
6108
6109 dst = (volatile void *)((uintptr_t)eq->udb + UDBS_WR_OFFSET -
6110 UDBS_DB_OFFSET);
6111 i = eq->dbidx;
6112 src = (void *)&eq->desc[i];
6113 while (src != (void *)&eq->desc[i + 1])
6114 *dst++ = *src++;
6115 wmb();
6116 break;
6117 }
6118
6119 case DOORBELL_UDBWC:
6120 *eq->udb = htole32(V_QID(eq->udb_qid) | V_PIDX(n));
6121 wmb();
6122 break;
6123
6124 case DOORBELL_KDB:
6125 t4_write_reg(sc, sc->sge_kdoorbell_reg,
6126 V_QID(eq->cntxt_id) | V_PIDX(n));
6127 break;
6128 }
6129
6130 IDXINCR(eq->dbidx, n, eq->sidx);
6131}
6132
6133static inline u_int
6134reclaimable_tx_desc(struct sge_eq *eq)
6135{
6136 uint16_t hw_cidx;
6137
6138 hw_cidx = read_hw_cidx(eq);
6139 return (IDXDIFF(hw_cidx, eq->cidx, eq->sidx));
6140}
6141
6142static inline u_int
6143total_available_tx_desc(struct sge_eq *eq)
6144{
6145 uint16_t hw_cidx, pidx;
6146
6147 hw_cidx = read_hw_cidx(eq);
6148 pidx = eq->pidx;
6149
6150 if (pidx == hw_cidx)
6151 return (eq->sidx - 1);
6152 else
6153 return (IDXDIFF(hw_cidx, pidx, eq->sidx) - 1);
6154}
6155
6156static inline uint16_t
6157read_hw_cidx(struct sge_eq *eq)
6158{
6159 struct sge_qstat *spg = (void *)&eq->desc[eq->sidx];
6160 uint16_t cidx = spg->cidx; /* stable snapshot */
6161
6162 return (be16toh(cidx));
6163}
6164
6165/*
6166 * Reclaim 'n' descriptors approximately.
6167 */
6168static u_int
6169reclaim_tx_descs(struct sge_txq *txq, u_int n)
6170{
6171 struct tx_sdesc *txsd;
6172 struct sge_eq *eq = &txq->eq;
6173 u_int can_reclaim, reclaimed;
6174
6175 TXQ_LOCK_ASSERT_OWNED(txq);
6176 MPASS(n > 0);
6177
6178 reclaimed = 0;
6179 can_reclaim = reclaimable_tx_desc(eq);
6180 while (can_reclaim && reclaimed < n) {
6181 int ndesc;
6182 struct mbuf *m, *nextpkt;
6183
6184 txsd = &txq->sdesc[eq->cidx];
6185 ndesc = txsd->desc_used;
6186
6187 /* Firmware doesn't return "partial" credits. */
6188 KASSERT(can_reclaim >= ndesc,
6189 ("%s: unexpected number of credits: %d, %d",
6190 __func__, can_reclaim, ndesc));
6191 KASSERT(ndesc != 0,
6192 ("%s: descriptor with no credits: cidx %d",
6193 __func__, eq->cidx));
6194
6195 for (m = txsd->m; m != NULL; m = nextpkt) {
6196 nextpkt = m->m_nextpkt;
6197 m->m_nextpkt = NULL;
6198 m_freem(m);
6199 }
6200 reclaimed += ndesc;
6201 can_reclaim -= ndesc;
6202 IDXINCR(eq->cidx, ndesc, eq->sidx);
6203 }
6204
6205 return (reclaimed);
6206}
6207
6208static void
6209tx_reclaim(void *arg, int n)
6210{
6211 struct sge_txq *txq = arg;
6212 struct sge_eq *eq = &txq->eq;
6213
6214 do {
6215 if (TXQ_TRYLOCK(txq) == 0)
6216 break;
6217 n = reclaim_tx_descs(txq, 32);
6218 if (eq->cidx == eq->pidx)
6219 eq->equeqidx = eq->pidx;
6220 TXQ_UNLOCK(txq);
6221 } while (n > 0);
6222}
6223
6224static __be64
6225get_flit(struct sglist_seg *segs, int nsegs, int idx)
6226{
6227 int i = (idx / 3) * 2;
6228
6229 switch (idx % 3) {
6230 case 0: {
6231 uint64_t rc;
6232
6233 rc = (uint64_t)segs[i].ss_len << 32;
6234 if (i + 1 < nsegs)
6235 rc |= (uint64_t)(segs[i + 1].ss_len);
6236
6237 return (htobe64(rc));
6238 }
6239 case 1:
6240 return (htobe64(segs[i].ss_paddr));
6241 case 2:
6242 return (htobe64(segs[i + 1].ss_paddr));
6243 }
6244
6245 return (0);
6246}
6247
6248static int
6249find_refill_source(struct adapter *sc, int maxp, bool packing)
6250{
6251 int i, zidx = -1;
6252 struct rx_buf_info *rxb = &sc->sge.rx_buf_info[0];
6253
6254 if (packing) {
6255 for (i = 0; i < SW_ZONE_SIZES; i++, rxb++) {
6256 if (rxb->hwidx2 == -1)
6257 continue;
6258 if (rxb->size1 < PAGE_SIZE &&
6259 rxb->size1 < largest_rx_cluster)
6260 continue;
6261 if (rxb->size1 > largest_rx_cluster)
6262 break;
6263 MPASS(rxb->size1 - rxb->size2 >= CL_METADATA_SIZE);
6264 if (rxb->size2 >= maxp)
6265 return (i);
6266 zidx = i;
6267 }
6268 } else {
6269 for (i = 0; i < SW_ZONE_SIZES; i++, rxb++) {
6270 if (rxb->hwidx1 == -1)
6271 continue;
6272 if (rxb->size1 > largest_rx_cluster)
6273 break;
6274 if (rxb->size1 >= maxp)
6275 return (i);
6276 zidx = i;
6277 }
6278 }
6279
6280 return (zidx);
6281}
6282
6283static void
6284add_fl_to_sfl(struct adapter *sc, struct sge_fl *fl)
6285{
6286 mtx_lock(&sc->sfl_lock);
6287 FL_LOCK(fl);
6288 if ((fl->flags & FL_DOOMED) == 0) {
6289 fl->flags |= FL_STARVING;
6290 TAILQ_INSERT_TAIL(&sc->sfl, fl, link);
6291 callout_reset(&sc->sfl_callout, hz / 5, refill_sfl, sc);
6292 }
6293 FL_UNLOCK(fl);
6294 mtx_unlock(&sc->sfl_lock);
6295}
6296
6297static void
6298handle_wrq_egr_update(struct adapter *sc, struct sge_eq *eq)
6299{
6300 struct sge_wrq *wrq = (void *)eq;
6301
6302 atomic_readandclear_int(&eq->equiq);
6303 taskqueue_enqueue(sc->tq[eq->tx_chan], &wrq->wrq_tx_task);
6304}
6305
6306static void
6307handle_eth_egr_update(struct adapter *sc, struct sge_eq *eq)
6308{
6309 struct sge_txq *txq = (void *)eq;
6310
6311 MPASS(eq->type == EQ_ETH);
6312
6313 atomic_readandclear_int(&eq->equiq);
6314 if (mp_ring_is_idle(txq->r))
6315 taskqueue_enqueue(sc->tq[eq->tx_chan], &txq->tx_reclaim_task);
6316 else
6317 mp_ring_check_drainage(txq->r, 64);
6318}
6319
6320static int
6321handle_sge_egr_update(struct sge_iq *iq, const struct rss_header *rss,
6322 struct mbuf *m)
6323{
6324 const struct cpl_sge_egr_update *cpl = (const void *)(rss + 1);
6325 unsigned int qid = G_EGR_QID(ntohl(cpl->opcode_qid));
6326 struct adapter *sc = iq->adapter;
6327 struct sge *s = &sc->sge;
6328 struct sge_eq *eq;
6329 static void (*h[])(struct adapter *, struct sge_eq *) = {NULL,
6330 &handle_wrq_egr_update, &handle_eth_egr_update,
6331 &handle_wrq_egr_update};
6332
6333 KASSERT(m == NULL, ("%s: payload with opcode %02x", __func__,
6334 rss->opcode));
6335
6336 eq = s->eqmap[qid - s->eq_start - s->eq_base];
6337 (*h[eq->type])(sc, eq);
6338
6339 return (0);
6340}
6341
6342/* handle_fw_msg works for both fw4_msg and fw6_msg because this is valid */
6343CTASSERT(offsetof(struct cpl_fw4_msg, data) == \
6344 offsetof(struct cpl_fw6_msg, data));
6345
6346static int
6347handle_fw_msg(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m)
6348{
6349 struct adapter *sc = iq->adapter;
6350 const struct cpl_fw6_msg *cpl = (const void *)(rss + 1);
6351
6352 KASSERT(m == NULL, ("%s: payload with opcode %02x", __func__,
6353 rss->opcode));
6354
6355 if (cpl->type == FW_TYPE_RSSCPL || cpl->type == FW6_TYPE_RSSCPL) {
6356 const struct rss_header *rss2;
6357
6358 rss2 = (const struct rss_header *)&cpl->data[0];
6359 return (t4_cpl_handler[rss2->opcode](iq, rss2, m));
6360 }
6361
6362 return (t4_fw_msg_handler[cpl->type](sc, &cpl->data[0]));
6363}
6364
6370static int
6371t4_handle_wrerr_rpl(struct adapter *adap, const __be64 *rpl)
6372{
6373 u8 opcode = *(const u8 *)rpl;
6374 const struct fw_error_cmd *e = (const void *)rpl;
6375 unsigned int i;
6376
6377 if (opcode != FW_ERROR_CMD) {
6378 log(LOG_ERR,
6379 "%s: Received WRERR_RPL message with opcode %#x\n",
6380 device_get_nameunit(adap->dev), opcode);
6381 return (EINVAL);
6382 }
6383 log(LOG_ERR, "%s: FW_ERROR (%s) ", device_get_nameunit(adap->dev),
6384 G_FW_ERROR_CMD_FATAL(be32toh(e->op_to_type)) ? "fatal" :
6385 "non-fatal");
6386 switch (G_FW_ERROR_CMD_TYPE(be32toh(e->op_to_type))) {
6387 case FW_ERROR_TYPE_EXCEPTION:
6388 log(LOG_ERR, "exception info:\n");
6389 for (i = 0; i < nitems(e->u.exception.info); i++)
6390 log(LOG_ERR, "%s%08x", i == 0 ? "\t" : " ",
6391 be32toh(e->u.exception.info[i]));
6392 log(LOG_ERR, "\n");
6393 break;
6394 case FW_ERROR_TYPE_HWMODULE:
6395 log(LOG_ERR, "HW module regaddr %08x regval %08x\n",
6396 be32toh(e->u.hwmodule.regaddr),
6397 be32toh(e->u.hwmodule.regval));
6398 break;
6399 case FW_ERROR_TYPE_WR:
6400 log(LOG_ERR, "WR cidx %d PF %d VF %d eqid %d hdr:\n",
6401 be16toh(e->u.wr.cidx),
6402 G_FW_ERROR_CMD_PFN(be16toh(e->u.wr.pfn_vfn)),
6403 G_FW_ERROR_CMD_VFN(be16toh(e->u.wr.pfn_vfn)),
6404 be32toh(e->u.wr.eqid));
6405 for (i = 0; i < nitems(e->u.wr.wrhdr); i++)
6406 log(LOG_ERR, "%s%02x", i == 0 ? "\t" : " ",
6407 e->u.wr.wrhdr[i]);
6408 log(LOG_ERR, "\n");
6409 break;
6410 case FW_ERROR_TYPE_ACL:
6411 log(LOG_ERR, "ACL cidx %d PF %d VF %d eqid %d %s",
6412 be16toh(e->u.acl.cidx),
6413 G_FW_ERROR_CMD_PFN(be16toh(e->u.acl.pfn_vfn)),
6414 G_FW_ERROR_CMD_VFN(be16toh(e->u.acl.pfn_vfn)),
6415 be32toh(e->u.acl.eqid),
6416 G_FW_ERROR_CMD_MV(be16toh(e->u.acl.mv_pkd)) ? "vlanid" :
6417 "MAC");
6418 for (i = 0; i < nitems(e->u.acl.val); i++)
6419 log(LOG_ERR, " %02x", e->u.acl.val[i]);
6420 log(LOG_ERR, "\n");
6421 break;
6422 default:
6423 log(LOG_ERR, "type %#x\n",
6424 G_FW_ERROR_CMD_TYPE(be32toh(e->op_to_type)));
6425 return (EINVAL);
6426 }
6427 return (0);
6428}
6429
6430static inline bool
6431bufidx_used(struct adapter *sc, int idx)
6432{
6433 struct rx_buf_info *rxb = &sc->sge.rx_buf_info[0];
6434 int i;
6435
6436 for (i = 0; i < SW_ZONE_SIZES; i++, rxb++) {
6437 if (rxb->size1 > largest_rx_cluster)
6438 continue;
6439 if (rxb->hwidx1 == idx || rxb->hwidx2 == idx)
6440 return (true);
6441 }
6442
6443 return (false);
6444}
6445
6446static int
6447sysctl_bufsizes(SYSCTL_HANDLER_ARGS)
6448{
6449 struct adapter *sc = arg1;
6450 struct sge_params *sp = &sc->params.sge;
6451 int i, rc;
6452 struct sbuf sb;
6453 char c;
6454
6455 sbuf_new(&sb, NULL, 128, SBUF_AUTOEXTEND);
6456 for (i = 0; i < SGE_FLBUF_SIZES; i++) {
6457 if (bufidx_used(sc, i))
6458 c = '*';
6459 else
6460 c = '\0';
6461
6462 sbuf_printf(&sb, "%u%c ", sp->sge_fl_buffer_size[i], c);
6463 }
6464 sbuf_trim(&sb);
6465 sbuf_finish(&sb);
6466 rc = sysctl_handle_string(oidp, sbuf_data(&sb), sbuf_len(&sb), req);
6467 sbuf_delete(&sb);
6468 return (rc);
6469}
6470
6471#ifdef RATELIMIT
6472#if defined(INET) || defined(INET6)
6473/*
6474 * len16 for a txpkt WR with a GL. Includes the firmware work request header.
6475 */
6476static inline u_int
6477txpkt_eo_len16(u_int nsegs, u_int immhdrs, u_int tso)
6478{
6479 u_int n;
6480
6481 MPASS(immhdrs > 0);
6482
6483 n = roundup2(sizeof(struct fw_eth_tx_eo_wr) +
6484 sizeof(struct cpl_tx_pkt_core) + immhdrs, 16);
6485 if (__predict_false(nsegs == 0))
6486 goto done;
6487
6488 nsegs--; /* first segment is part of ulptx_sgl */
6489 n += sizeof(struct ulptx_sgl) + 8 * ((3 * nsegs) / 2 + (nsegs & 1));
6490 if (tso)
6491 n += sizeof(struct cpl_tx_pkt_lso_core);
6492
6493done:
6494 return (howmany(n, 16));
6495}
6496#endif
6497
6498#define ETID_FLOWC_NPARAMS 6
6499#define ETID_FLOWC_LEN (roundup2((sizeof(struct fw_flowc_wr) + \
6500 ETID_FLOWC_NPARAMS * sizeof(struct fw_flowc_mnemval)), 16))
6501#define ETID_FLOWC_LEN16 (howmany(ETID_FLOWC_LEN, 16))
6502
6503static int
6504send_etid_flowc_wr(struct cxgbe_rate_tag *cst, struct port_info *pi,
6505 struct vi_info *vi)
6506{
6507 struct wrq_cookie cookie;
6508 u_int pfvf = pi->adapter->pf << S_FW_VIID_PFN;
6509 struct fw_flowc_wr *flowc;
6510
6511 mtx_assert(&cst->lock, MA_OWNED);
6512 MPASS((cst->flags & (EO_FLOWC_PENDING | EO_FLOWC_RPL_PENDING)) ==
6513 EO_FLOWC_PENDING);
6514
6515 flowc = start_wrq_wr(&cst->eo_txq->wrq, ETID_FLOWC_LEN16, &cookie);
6516 if (__predict_false(flowc == NULL))
6517 return (ENOMEM);
6518
6519 bzero(flowc, ETID_FLOWC_LEN);
6520 flowc->op_to_nparams = htobe32(V_FW_WR_OP(FW_FLOWC_WR) |
6521 V_FW_FLOWC_WR_NPARAMS(ETID_FLOWC_NPARAMS) | V_FW_WR_COMPL(0));
6522 flowc->flowid_len16 = htonl(V_FW_WR_LEN16(ETID_FLOWC_LEN16) |
6523 V_FW_WR_FLOWID(cst->etid));
6524 flowc->mnemval[0].mnemonic = FW_FLOWC_MNEM_PFNVFN;
6525 flowc->mnemval[0].val = htobe32(pfvf);
6526 flowc->mnemval[1].mnemonic = FW_FLOWC_MNEM_CH;
6527 flowc->mnemval[1].val = htobe32(pi->tx_chan);
6528 flowc->mnemval[2].mnemonic = FW_FLOWC_MNEM_PORT;
6529 flowc->mnemval[2].val = htobe32(pi->tx_chan);
6530 flowc->mnemval[3].mnemonic = FW_FLOWC_MNEM_IQID;
6531 flowc->mnemval[3].val = htobe32(cst->iqid);
6532 flowc->mnemval[4].mnemonic = FW_FLOWC_MNEM_EOSTATE;
6533 flowc->mnemval[4].val = htobe32(FW_FLOWC_MNEM_EOSTATE_ESTABLISHED);
6534 flowc->mnemval[5].mnemonic = FW_FLOWC_MNEM_SCHEDCLASS;
6535 flowc->mnemval[5].val = htobe32(cst->schedcl);
6536
6537 commit_wrq_wr(&cst->eo_txq->wrq, flowc, &cookie);
6538
6539 cst->flags &= ~EO_FLOWC_PENDING;
6540 cst->flags |= EO_FLOWC_RPL_PENDING;
6541 MPASS(cst->tx_credits >= ETID_FLOWC_LEN16); /* flowc is first WR. */
6542 cst->tx_credits -= ETID_FLOWC_LEN16;
6543
6544 return (0);
6545}
6546
6547#define ETID_FLUSH_LEN16 (howmany(sizeof (struct fw_flowc_wr), 16))
6548
6549void
6550send_etid_flush_wr(struct cxgbe_rate_tag *cst)
6551{
6552 struct fw_flowc_wr *flowc;
6553 struct wrq_cookie cookie;
6554
6555 mtx_assert(&cst->lock, MA_OWNED);
6556
6557 flowc = start_wrq_wr(&cst->eo_txq->wrq, ETID_FLUSH_LEN16, &cookie);
6558 if (__predict_false(flowc == NULL))
6559 CXGBE_UNIMPLEMENTED(__func__);
6560
6561 bzero(flowc, ETID_FLUSH_LEN16 * 16);
6562 flowc->op_to_nparams = htobe32(V_FW_WR_OP(FW_FLOWC_WR) |
6563 V_FW_FLOWC_WR_NPARAMS(0) | F_FW_WR_COMPL);
6564 flowc->flowid_len16 = htobe32(V_FW_WR_LEN16(ETID_FLUSH_LEN16) |
6565 V_FW_WR_FLOWID(cst->etid));
6566
6567 commit_wrq_wr(&cst->eo_txq->wrq, flowc, &cookie);
6568
6569 cst->flags |= EO_FLUSH_RPL_PENDING;
6570 MPASS(cst->tx_credits >= ETID_FLUSH_LEN16);
6571 cst->tx_credits -= ETID_FLUSH_LEN16;
6572 cst->ncompl++;
6573}
6574
6575static void
6576write_ethofld_wr(struct cxgbe_rate_tag *cst, struct fw_eth_tx_eo_wr *wr,
6577 struct mbuf *m0, int compl)
6578{
6579 struct cpl_tx_pkt_core *cpl;
6580 uint64_t ctrl1;
6581 uint32_t ctrl; /* used in many unrelated places */
6582 int len16, pktlen, nsegs, immhdrs;
6583 uintptr_t p;
6584 struct ulptx_sgl *usgl;
6585 struct sglist sg;
6586 struct sglist_seg segs[38]; /* XXX: find real limit. XXX: get off the stack */
6587
6588 mtx_assert(&cst->lock, MA_OWNED);
6589 M_ASSERTPKTHDR(m0);
6590 KASSERT(m0->m_pkthdr.l2hlen > 0 && m0->m_pkthdr.l3hlen > 0 &&
6591 m0->m_pkthdr.l4hlen > 0,
6592 ("%s: ethofld mbuf %p is missing header lengths", __func__, m0));
6593
6594 len16 = mbuf_eo_len16(m0);
6595 nsegs = mbuf_eo_nsegs(m0);
6596 pktlen = m0->m_pkthdr.len;
6597 ctrl = sizeof(struct cpl_tx_pkt_core);
6598 if (needs_tso(m0))
6599 ctrl += sizeof(struct cpl_tx_pkt_lso_core);
6600 immhdrs = m0->m_pkthdr.l2hlen + m0->m_pkthdr.l3hlen + m0->m_pkthdr.l4hlen;
6601 ctrl += immhdrs;
6602
6603 wr->op_immdlen = htobe32(V_FW_WR_OP(FW_ETH_TX_EO_WR) |
6604 V_FW_ETH_TX_EO_WR_IMMDLEN(ctrl) | V_FW_WR_COMPL(!!compl));
6605 wr->equiq_to_len16 = htobe32(V_FW_WR_LEN16(len16) |
6606 V_FW_WR_FLOWID(cst->etid));
6607 wr->r3 = 0;
6608 if (needs_outer_udp_csum(m0)) {
6609 wr->u.udpseg.type = FW_ETH_TX_EO_TYPE_UDPSEG;
6610 wr->u.udpseg.ethlen = m0->m_pkthdr.l2hlen;
6611 wr->u.udpseg.iplen = htobe16(m0->m_pkthdr.l3hlen);
6612 wr->u.udpseg.udplen = m0->m_pkthdr.l4hlen;
6613 wr->u.udpseg.rtplen = 0;
6614 wr->u.udpseg.r4 = 0;
6615 wr->u.udpseg.mss = htobe16(pktlen - immhdrs);
6616 wr->u.udpseg.schedpktsize = wr->u.udpseg.mss;
6617 wr->u.udpseg.plen = htobe32(pktlen - immhdrs);
6618 cpl = (void *)(wr + 1);
6619 } else {
6620 MPASS(needs_outer_tcp_csum(m0));
6621 wr->u.tcpseg.type = FW_ETH_TX_EO_TYPE_TCPSEG;
6622 wr->u.tcpseg.ethlen = m0->m_pkthdr.l2hlen;
6623 wr->u.tcpseg.iplen = htobe16(m0->m_pkthdr.l3hlen);
6624 wr->u.tcpseg.tcplen = m0->m_pkthdr.l4hlen;
6625 wr->u.tcpseg.tsclk_tsoff = mbuf_eo_tsclk_tsoff(m0);
6626 wr->u.tcpseg.r4 = 0;
6627 wr->u.tcpseg.r5 = 0;
6628 wr->u.tcpseg.plen = htobe32(pktlen - immhdrs);
6629
6630 if (needs_tso(m0)) {
6631 struct cpl_tx_pkt_lso_core *lso = (void *)(wr + 1);
6632
6633 wr->u.tcpseg.mss = htobe16(m0->m_pkthdr.tso_segsz);
6634
6635 ctrl = V_LSO_OPCODE(CPL_TX_PKT_LSO) |
6636 F_LSO_FIRST_SLICE | F_LSO_LAST_SLICE |
6637 V_LSO_ETHHDR_LEN((m0->m_pkthdr.l2hlen -
6638 ETHER_HDR_LEN) >> 2) |
6639 V_LSO_IPHDR_LEN(m0->m_pkthdr.l3hlen >> 2) |
6640 V_LSO_TCPHDR_LEN(m0->m_pkthdr.l4hlen >> 2);
6641 if (m0->m_pkthdr.l3hlen == sizeof(struct ip6_hdr))
6642 ctrl |= F_LSO_IPV6;
6643 lso->lso_ctrl = htobe32(ctrl);
6644 lso->ipid_ofst = htobe16(0);
6645 lso->mss = htobe16(m0->m_pkthdr.tso_segsz);
6646 lso->seqno_offset = htobe32(0);
6647 lso->len = htobe32(pktlen);
6648
6649 cpl = (void *)(lso + 1);
6650 } else {
6651 wr->u.tcpseg.mss = htobe16(0xffff);
6652 cpl = (void *)(wr + 1);
6653 }
6654 }
6655
6656 /* Checksum offload must be requested for ethofld. */
6657 MPASS(needs_outer_l4_csum(m0));
6658 ctrl1 = csum_to_ctrl(cst->adapter, m0);
6659
6660 /* VLAN tag insertion */
6661 if (needs_vlan_insertion(m0)) {
6662 ctrl1 |= F_TXPKT_VLAN_VLD |
6663 V_TXPKT_VLAN(m0->m_pkthdr.ether_vtag);
6664 }
6665
6666 /* CPL header */
6667 cpl->ctrl0 = cst->ctrl0;
6668 cpl->pack = 0;
6669 cpl->len = htobe16(pktlen);
6670 cpl->ctrl1 = htobe64(ctrl1);
6671
6672 /* Copy Ethernet, IP & TCP/UDP hdrs as immediate data */
6673 p = (uintptr_t)(cpl + 1);
6674 m_copydata(m0, 0, immhdrs, (void *)p);
6675
6676 /* SGL */
6677 if (nsegs > 0) {
6678 int i, pad;
6679
6680 /* zero-pad upto next 16Byte boundary, if not 16Byte aligned */
6681 p += immhdrs;
6682 pad = 16 - (immhdrs & 0xf);
6683 bzero((void *)p, pad);
6684
6685 usgl = (void *)(p + pad);
6686 usgl->cmd_nsge = htobe32(V_ULPTX_CMD(ULP_TX_SC_DSGL) |
6687 V_ULPTX_NSGE(nsegs));
6688
6689 sglist_init(&sg, nitems(segs), segs);
6690 for (; m0 != NULL; m0 = m0->m_next) {
6691 if (__predict_false(m0->m_len == 0))
6692 continue;
6693 if (immhdrs >= m0->m_len) {
6694 immhdrs -= m0->m_len;
6695 continue;
6696 }
6697 if (m0->m_flags & M_EXTPG)
6698 sglist_append_mbuf_epg(&sg, m0,
6699 mtod(m0, vm_offset_t), m0->m_len);
6700 else
6701 sglist_append(&sg, mtod(m0, char *) + immhdrs,
6702 m0->m_len - immhdrs);
6703 immhdrs = 0;
6704 }
6705 MPASS(sg.sg_nseg == nsegs);
6706
6707 /*
6708 * Zero pad last 8B in case the WR doesn't end on a 16B
6709 * boundary.
6710 */
6711 *(uint64_t *)((char *)wr + len16 * 16 - 8) = 0;
6712
6713 usgl->len0 = htobe32(segs[0].ss_len);
6714 usgl->addr0 = htobe64(segs[0].ss_paddr);
6715 for (i = 0; i < nsegs - 1; i++) {
6716 usgl->sge[i / 2].len[i & 1] = htobe32(segs[i + 1].ss_len);
6717 usgl->sge[i / 2].addr[i & 1] = htobe64(segs[i + 1].ss_paddr);
6718 }
6719 if (i & 1)
6720 usgl->sge[i / 2].len[1] = htobe32(0);
6721 }
6722
6723}
6724
6725static void
6726ethofld_tx(struct cxgbe_rate_tag *cst)
6727{
6728 struct mbuf *m;
6729 struct wrq_cookie cookie;
6730 int next_credits, compl;
6731 struct fw_eth_tx_eo_wr *wr;
6732
6733 mtx_assert(&cst->lock, MA_OWNED);
6734
6735 while ((m = mbufq_first(&cst->pending_tx)) != NULL) {
6736 M_ASSERTPKTHDR(m);
6737
6738 /* How many len16 credits do we need to send this mbuf. */
6739 next_credits = mbuf_eo_len16(m);
6740 MPASS(next_credits > 0);
6741 if (next_credits > cst->tx_credits) {
6742 /*
6743 * Tx will make progress eventually because there is at
6744 * least one outstanding fw4_ack that will return
6745 * credits and kick the tx.
6746 */
6747 MPASS(cst->ncompl > 0);
6748 return;
6749 }
6750 wr = start_wrq_wr(&cst->eo_txq->wrq, next_credits, &cookie);
6751 if (__predict_false(wr == NULL)) {
6752 /* XXX: wishful thinking, not a real assertion. */
6753 MPASS(cst->ncompl > 0);
6754 return;
6755 }
6756 cst->tx_credits -= next_credits;
6757 cst->tx_nocompl += next_credits;
6758 compl = cst->ncompl == 0 || cst->tx_nocompl >= cst->tx_total / 2;
6759 ETHER_BPF_MTAP(cst->com.ifp, m);
6760 write_ethofld_wr(cst, wr, m, compl);
6761 commit_wrq_wr(&cst->eo_txq->wrq, wr, &cookie);
6762 if (compl) {
6763 cst->ncompl++;
6764 cst->tx_nocompl = 0;
6765 }
6766 (void) mbufq_dequeue(&cst->pending_tx);
6767
6768 /*
6769 * Drop the mbuf's reference on the tag now rather
6770 * than waiting until m_freem(). This ensures that
6771 * cxgbe_rate_tag_free gets called when the inp drops
6772 * its reference on the tag and there are no more
6773 * mbufs in the pending_tx queue and can flush any
6774 * pending requests. Otherwise if the last mbuf
6775 * doesn't request a completion the etid will never be
6776 * released.
6777 */
6778 m->m_pkthdr.snd_tag = NULL;
6779 m->m_pkthdr.csum_flags &= ~CSUM_SND_TAG;
6780 m_snd_tag_rele(&cst->com);
6781
6782 mbufq_enqueue(&cst->pending_fwack, m);
6783 }
6784}
6785
6786int
6787ethofld_transmit(struct ifnet *ifp, struct mbuf *m0)
6788{
6789 struct cxgbe_rate_tag *cst;
6790 int rc;
6791
6792 MPASS(m0->m_nextpkt == NULL);
6793 MPASS(m0->m_pkthdr.csum_flags & CSUM_SND_TAG);
6794 MPASS(m0->m_pkthdr.snd_tag != NULL);
6795 cst = mst_to_crt(m0->m_pkthdr.snd_tag);
6796
6797 mtx_lock(&cst->lock);
6798 MPASS(cst->flags & EO_SND_TAG_REF);
6799
6800 if (__predict_false(cst->flags & EO_FLOWC_PENDING)) {
6801 struct vi_info *vi = ifp->if_softc;
6802 struct port_info *pi = vi->pi;
6803 struct adapter *sc = pi->adapter;
6804 const uint32_t rss_mask = vi->rss_size - 1;
6805 uint32_t rss_hash;
6806
6807 cst->eo_txq = &sc->sge.ofld_txq[vi->first_ofld_txq];
6808 if (M_HASHTYPE_ISHASH(m0))
6809 rss_hash = m0->m_pkthdr.flowid;
6810 else
6811 rss_hash = arc4random();
6812 /* We assume RSS hashing */
6813 cst->iqid = vi->rss[rss_hash & rss_mask];
6814 cst->eo_txq += rss_hash % vi->nofldtxq;
6815 rc = send_etid_flowc_wr(cst, pi, vi);
6816 if (rc != 0)
6817 goto done;
6818 }
6819
6820 if (__predict_false(cst->plen + m0->m_pkthdr.len > eo_max_backlog)) {
6821 rc = ENOBUFS;
6822 goto done;
6823 }
6824
6825 mbufq_enqueue(&cst->pending_tx, m0);
6826 cst->plen += m0->m_pkthdr.len;
6827
6828 /*
6829 * Hold an extra reference on the tag while generating work
6830 * requests to ensure that we don't try to free the tag during
6831 * ethofld_tx() in case we are sending the final mbuf after
6832 * the inp was freed.
6833 */
6834 m_snd_tag_ref(&cst->com);
6835 ethofld_tx(cst);
6836 mtx_unlock(&cst->lock);
6837 m_snd_tag_rele(&cst->com);
6838 return (0);
6839
6840done:
6841 mtx_unlock(&cst->lock);
6842 if (__predict_false(rc != 0))
6843 m_freem(m0);
6844 return (rc);
6845}
6846
6847static int
6848ethofld_fw4_ack(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m0)
6849{
6850 struct adapter *sc = iq->adapter;
6851 const struct cpl_fw4_ack *cpl = (const void *)(rss + 1);
6852 struct mbuf *m;
6853 u_int etid = G_CPL_FW4_ACK_FLOWID(be32toh(OPCODE_TID(cpl)));
6854 struct cxgbe_rate_tag *cst;
6855 uint8_t credits = cpl->credits;
6856
6857 cst = lookup_etid(sc, etid);
6858 mtx_lock(&cst->lock);
6859 if (__predict_false(cst->flags & EO_FLOWC_RPL_PENDING)) {
6860 MPASS(credits >= ETID_FLOWC_LEN16);
6861 credits -= ETID_FLOWC_LEN16;
6862 cst->flags &= ~EO_FLOWC_RPL_PENDING;
6863 }
6864
6865 KASSERT(cst->ncompl > 0,
6866 ("%s: etid %u (%p) wasn't expecting completion.",
6867 __func__, etid, cst));
6868 cst->ncompl--;
6869
6870 while (credits > 0) {
6871 m = mbufq_dequeue(&cst->pending_fwack);
6872 if (__predict_false(m == NULL)) {
6873 /*
6874 * The remaining credits are for the final flush that
6875 * was issued when the tag was freed by the kernel.
6876 */
6877 MPASS((cst->flags &
6878 (EO_FLUSH_RPL_PENDING | EO_SND_TAG_REF)) ==
6879 EO_FLUSH_RPL_PENDING);
6880 MPASS(credits == ETID_FLUSH_LEN16);
6881 MPASS(cst->tx_credits + cpl->credits == cst->tx_total);
6882 MPASS(cst->ncompl == 0);
6883
6884 cst->flags &= ~EO_FLUSH_RPL_PENDING;
6885 cst->tx_credits += cpl->credits;
6886 cxgbe_rate_tag_free_locked(cst);
6887 return (0); /* cst is gone. */
6888 }
6889 KASSERT(m != NULL,
6890 ("%s: too many credits (%u, %u)", __func__, cpl->credits,
6891 credits));
6892 KASSERT(credits >= mbuf_eo_len16(m),
6893 ("%s: too few credits (%u, %u, %u)", __func__,
6894 cpl->credits, credits, mbuf_eo_len16(m)));
6895 credits -= mbuf_eo_len16(m);
6896 cst->plen -= m->m_pkthdr.len;
6897 m_freem(m);
6898 }
6899
6900 cst->tx_credits += cpl->credits;
6901 MPASS(cst->tx_credits <= cst->tx_total);
6902
6903 if (cst->flags & EO_SND_TAG_REF) {
6904 /*
6905 * As with ethofld_transmit(), hold an extra reference
6906 * so that the tag is stable across ethold_tx().
6907 */
6908 m_snd_tag_ref(&cst->com);
6909 m = mbufq_first(&cst->pending_tx);
6910 if (m != NULL && cst->tx_credits >= mbuf_eo_len16(m))
6911 ethofld_tx(cst);
6912 mtx_unlock(&cst->lock);
6913 m_snd_tag_rele(&cst->com);
6914 } else {
6915 /*
6916 * There shouldn't be any pending packets if the tag
6917 * was freed by the kernel since any pending packet
6918 * should hold a reference to the tag.
6919 */
6920 MPASS(mbufq_first(&cst->pending_tx) == NULL);
6921 mtx_unlock(&cst->lock);
6922 }
6923
6924 return (0);
6925}
6926#endif
CPL_COOKIE_HASHFILTER
@ CPL_COOKIE_HASHFILTER
Definition: adapter.h:402
CPL_COOKIE_FILTER
@ CPL_COOKIE_FILTER
Definition: adapter.h:398
CPL_COOKIE_ETHOFLD
@ CPL_COOKIE_ETHOFLD
Definition: adapter.h:403
CPL_COOKIE_RESERVED
@ CPL_COOKIE_RESERVED
Definition: adapter.h:397
CPL_COOKIE_TOM
@ CPL_COOKIE_TOM
Definition: adapter.h:401
NUM_CPL_COOKIES
@ NUM_CPL_COOKIES
Definition: adapter.h:406
cntxt_id
u_int cntxt_id
Definition: adapter.h:10
CL_METADATA_SIZE
@ CL_METADATA_SIZE
Definition: adapter.h:116
TX_SGL_SEGS_VM
@ TX_SGL_SEGS_VM
Definition: adapter.h:121
TX_SGL_SEGS_TSO
@ TX_SGL_SEGS_TSO
Definition: adapter.h:120
SGE_MAX_WR_NDESC
@ SGE_MAX_WR_NDESC
Definition: adapter.h:118
SW_ZONE_SIZES
@ SW_ZONE_SIZES
Definition: adapter.h:114
IQ_ESIZE
@ IQ_ESIZE
Definition: adapter.h:98
CTRL_EQ_QSIZE
@ CTRL_EQ_QSIZE
Definition: adapter.h:108
TX_SGL_SEGS_VXLAN_TSO
@ TX_SGL_SEGS_VXLAN_TSO
Definition: adapter.h:124
TX_SGL_SEGS
@ TX_SGL_SEGS
Definition: adapter.h:119
EQ_ESIZE
@ EQ_ESIZE
Definition: adapter.h:105
TX_SGL_SEGS_VM_TSO
@ TX_SGL_SEGS_VM_TSO
Definition: adapter.h:122
FW_IQ_QSIZE
@ FW_IQ_QSIZE
Definition: adapter.h:101
fl
struct sge_fl fl
Definition: adapter.h:1
an_handler_t
int(* an_handler_t)(struct sge_iq *, const struct rsp_ctrl *)
Definition: adapter.h:413
alloc_wrqe
static struct wrqe * alloc_wrqe(int wr_len, struct sge_wrq *wrq)
Definition: adapter.h:1437
t4_read_reg
static uint32_t t4_read_reg(struct adapter *sc, uint32_t reg)
Definition: adapter.h:1104
eq
struct sge_eq eq
Definition: adapter.h:0
gl
struct sglist * gl
Definition: adapter.h:5
wrq
struct sge_wrq wrq
Definition: adapter.h:0
txp
struct txpkts txp
Definition: adapter.h:9
TXQ_TRYLOCK
#define TXQ_TRYLOCK(txq)
Definition: adapter.h:1048
cidx
uint16_t cidx
Definition: adapter.h:1
tag
void * tag
Definition: adapter.h:2
FL_LOCK
#define FL_LOCK(fl)
Definition: adapter.h:1030
lro
struct lro_ctrl lro
Definition: adapter.h:4
ba
bus_addr_t ba
Definition: adapter.h:19
pidx
uint16_t pidx
Definition: adapter.h:2
rxq
struct sge_rxq * rxq
Definition: adapter.h:3
FL_NOT_RUNNING_LOW
#define FL_NOT_RUNNING_LOW(fl)
Definition: adapter.h:534
fw_msg_handler_t
int(* fw_msg_handler_t)(struct adapter *, const __be64 *)
Definition: adapter.h:414
SYSCTL_ADD_UQUAD
#define SYSCTL_ADD_UQUAD
Definition: adapter.h:82
r
struct mp_ring * r
Definition: adapter.h:3
ifp
struct ifnet * ifp
Definition: adapter.h:2
t4_write_reg
static void t4_write_reg(struct adapter *sc, uint32_t reg, uint32_t val)
Definition: adapter.h:1112
IDXDIFF
#define IDXDIFF(head, tail, wrap)
Definition: adapter.h:1078
FL_UNLOCK
#define FL_UNLOCK(fl)
Definition: adapter.h:1032
STAILQ_HEAD
STAILQ_HEAD(, wrqe) wr_list
IQ_HW_ALLOCATED
@ IQ_HW_ALLOCATED
Definition: adapter.h:383
IQS_DISABLED
@ IQS_DISABLED
Definition: adapter.h:386
IQ_ADJ_CREDIT
@ IQ_ADJ_CREDIT
Definition: adapter.h:382
IQS_BUSY
@ IQS_BUSY
Definition: adapter.h:387
IQ_LRO_ENABLED
@ IQ_LRO_ENABLED
Definition: adapter.h:381
NM_BUSY
@ NM_BUSY
Definition: adapter.h:393
IQ_SW_ALLOCATED
@ IQ_SW_ALLOCATED
Definition: adapter.h:378
IQ_RX_TIMESTAMP
@ IQ_RX_TIMESTAMP
Definition: adapter.h:380
IQS_IDLE
@ IQS_IDLE
Definition: adapter.h:388
NM_ON
@ NM_ON
Definition: adapter.h:392
IQ_HAS_FL
@ IQ_HAS_FL
Definition: adapter.h:379
nm_rxq
struct sge_nm_rxq * nm_rxq
Definition: adapter.h:4
TXQ_LOCK_ASSERT_OWNED
#define TXQ_LOCK_ASSERT_OWNED(txq)
Definition: adapter.h:1050
t4_wrq_tx
static void t4_wrq_tx(struct adapter *sc, struct wrqe *wr)
Definition: adapter.h:1463
EQ_LOCK_ASSERT_OWNED
#define EQ_LOCK_ASSERT_OWNED(eq)
Definition: adapter.h:1044
ss_len
uint16_t ss_len
Definition: adapter.h:26
for_each_rxq
#define for_each_rxq(vi, iter, q)
Definition: adapter.h:1056
forwarding_intr_to_fwq
static int forwarding_intr_to_fwq(struct adapter *sc)
Definition: adapter.h:1088
EQ_UNLOCK
#define EQ_UNLOCK(eq)
Definition: adapter.h:1043
FL_LOCK_ASSERT_OWNED
#define FL_LOCK_ASSERT_OWNED(fl)
Definition: adapter.h:1033
EQ_LOCK
#define EQ_LOCK(eq)
Definition: adapter.h:1041
for_each_nm_rxq
#define for_each_nm_rxq(vi, iter, q)
Definition: adapter.h:1068
for_each_ofld_rxq
#define for_each_ofld_rxq(vi, iter, q)
Definition: adapter.h:1062
IDXINCR
#define IDXINCR(idx, incr, wrap)
Definition: adapter.h:1075
DOORBELL_KDB
@ DOORBELL_KDB
Definition: adapter.h:456
DOORBELL_UDBWC
@ DOORBELL_UDBWC
Definition: adapter.h:456
DOORBELL_UDB
@ DOORBELL_UDB
Definition: adapter.h:456
DOORBELL_WCWR
@ DOORBELL_WCWR
Definition: adapter.h:456
iq
struct sge_iq iq
Definition: adapter.h:0
sysctl_tc
int sysctl_tc(SYSCTL_HANDLER_ARGS)
Definition: t4_sched.c:592
iq_to_rxq
static struct sge_rxq * iq_to_rxq(struct sge_iq *iq)
Definition: adapter.h:666
udb
volatile uint32_t * udb
Definition: adapter.h:8
FL_DOOMED
@ FL_DOOMED
Definition: adapter.h:527
FL_STARVING
@ FL_STARVING
Definition: adapter.h:526
FL_BUF_PACKING
@ FL_BUF_PACKING
Definition: adapter.h:528
FL_BUF_RESUME
@ FL_BUF_RESUME
Definition: adapter.h:529
EQ_HW_ALLOCATED
@ EQ_HW_ALLOCATED
Definition: adapter.h:450
EQ_CTRL
@ EQ_CTRL
Definition: adapter.h:444
EQ_ETH
@ EQ_ETH
Definition: adapter.h:445
EQ_ENABLED
@ EQ_ENABLED
Definition: adapter.h:451
EQ_QFLUSH
@ EQ_QFLUSH
Definition: adapter.h:452
EQ_OFLD
@ EQ_OFLD
Definition: adapter.h:446
EQ_SW_ALLOCATED
@ EQ_SW_ALLOCATED
Definition: adapter.h:449
TXQ_UNLOCK
#define TXQ_UNLOCK(txq)
Definition: adapter.h:1049
for_each_ofld_txq
#define for_each_ofld_txq(vi, iter, q)
Definition: adapter.h:1059
sidx
uint16_t sidx
Definition: adapter.h:3
free_wrqe
static void free_wrqe(struct wrqe *wr)
Definition: adapter.h:1457
for_each_txq
#define for_each_txq(vi, iter, q)
Definition: adapter.h:1053
DF_DISABLE_TCB_CACHE
@ DF_DISABLE_TCB_CACHE
Definition: adapter.h:187
DF_VERBOSE_SLOWINTR
@ DF_VERBOSE_SLOWINTR
Definition: adapter.h:189
TX_USES_VM_WR
@ TX_USES_VM_WR
Definition: adapter.h:181
MASTER_PF
@ MASTER_PF
Definition: adapter.h:161
ADAP_FATAL_ERR
@ ADAP_FATAL_ERR
Definition: adapter.h:169
IS_VF
@ IS_VF
Definition: adapter.h:163
BUF_PACKING_OK
@ BUF_PACKING_OK
Definition: adapter.h:162
for_each_nm_txq
#define for_each_nm_txq(vi, iter, q)
Definition: adapter.h:1065
INTR_INTX
@ INTR_INTX
Definition: adapter.h:130
cpl_handler_t
int(* cpl_handler_t)(struct sge_iq *, const struct rss_header *, struct mbuf *)
Definition: adapter.h:411
iqidx
int iqidx
Definition: adapter.h:20
FL_RUNNING_LOW
#define FL_RUNNING_LOW(fl)
Definition: adapter.h:532
ADAPTER_LOCK_ASSERT_NOTOWNED
#define ADAPTER_LOCK_ASSERT_NOTOWNED(sc)
Definition: adapter.h:1018
CXGBE_UNIMPLEMENTED
#define CXGBE_UNIMPLEMENTED(s)
Definition: adapter.h:69
last_tx
uint64_t last_tx
Definition: adapter.h:8
tx_len16_to_desc
static int tx_len16_to_desc(int len16)
Definition: adapter.h:1490
t4_slow_intr_handler
bool t4_slow_intr_handler(struct adapter *adapter, bool verbose)
Definition: t4_hw.c:5295
CHELSIO_T5
#define CHELSIO_T5
Definition: common.h:415
is_ftid
static int is_ftid(const struct adapter *sc, u_int tid)
Definition: common.h:471
t4_iq_free
int t4_iq_free(struct adapter *adap, unsigned int mbox, unsigned int pf, unsigned int vf, unsigned int iqtype, unsigned int iqid, unsigned int fl0id, unsigned int fl1id)
Definition: t4_hw.c:8677
is_hashfilter
static int is_hashfilter(const struct adapter *adap)
Definition: common.h:502
t4_wr_mbox
static int t4_wr_mbox(struct adapter *adap, int mbox, const void *cmd, int size, void *rpl)
Definition: common.h:595
t4_set_reg_field
void t4_set_reg_field(struct adapter *adap, unsigned int addr, u32 mask, u32 val)
Definition: t4_hw.c:100
t4_ctrl_eq_free
int t4_ctrl_eq_free(struct adapter *adap, unsigned int mbox, unsigned int pf, unsigned int vf, unsigned int eqid)
Definition: t4_hw.c:8756
is_etid
static int is_etid(const struct adapter *sc, u_int tid)
Definition: common.h:485
t4_handle_fw_rpl
int t4_handle_fw_rpl(struct adapter *adap, const __be64 *rpl)
Definition: t4_hw.c:9052
CHELSIO_T6
#define CHELSIO_T6
Definition: common.h:416
t4_fatal_err
void t4_fatal_err(struct adapter *adapter, bool fw_error)
Definition: t4_main.c:3554
chip_id
static int chip_id(struct adapter *adap)
Definition: common.h:512
for_each_port
#define for_each_port(adapter, iter)
Definition: common.h:468
t4_set_params
int t4_set_params(struct adapter *adap, unsigned int mbox, unsigned int pf, unsigned int vf, unsigned int nparams, const u32 *params, const u32 *val)
Definition: t4_hw.c:7885
t4_tp_pio_write
void t4_tp_pio_write(struct adapter *adap, const u32 *buff, u32 nregs, u32 start_index, bool sleep_ok)
Definition: t4_hw.c:5857
is_t4
static int is_t4(struct adapter *adap)
Definition: common.h:522
is_hpftid
static int is_hpftid(const struct adapter *sc, u_int tid)
Definition: common.h:478
t4_eth_eq_free
int t4_eth_eq_free(struct adapter *adap, unsigned int mbox, unsigned int pf, unsigned int vf, unsigned int eqid)
Definition: t4_hw.c:8731
is_ktls
static int is_ktls(const struct adapter *adap)
Definition: common.h:507
t4_tp_pio_read
void t4_tp_pio_read(struct adapter *adap, u32 *buff, u32 nregs, u32 start_index, bool sleep_ok)
Definition: t4_hw.c:5840
us_to_core_ticks
static unsigned int us_to_core_ticks(const struct adapter *adap, unsigned int us)
Definition: common.h:547
t4_ofld_eq_free
int t4_ofld_eq_free(struct adapter *adap, unsigned int mbox, unsigned int pf, unsigned int vf, unsigned int eqid)
Definition: t4_hw.c:8781
EO_FLOWC_PENDING
@ EO_FLOWC_PENDING
Definition: offload.h:84
EO_SND_TAG_REF
@ EO_SND_TAG_REF
Definition: offload.h:86
EO_FLOWC_RPL_PENDING
@ EO_FLOWC_RPL_PENDING
Definition: offload.h:85
EO_FLUSH_RPL_PENDING
@ EO_FLUSH_RPL_PENDING
Definition: offload.h:87
mst_to_crt
static struct cxgbe_rate_tag * mst_to_crt(struct m_snd_tag *t)
Definition: offload.h:111
__be32
uint32_t __be32
Definition: osdep.h:69
ilog2
static int ilog2(long x)
Definition: osdep.h:136
__be64
uint64_t __be64
Definition: osdep.h:70
u8
uint8_t u8
Definition: osdep.h:59
CH_ERR
#define CH_ERR(adap, fmt,...)
Definition: osdep.h:45
u32
uint32_t u32
Definition: osdep.h:61
adapter_params::tp
struct tp_params tp
Definition: common.h:360
adapter_params::pci
struct pci_params pci
Definition: common.h:362
adapter_params::sge
struct sge_params sge
Definition: common.h:359
adapter_params::max_pkts_per_eth_tx_pkts_wr
unsigned int max_pkts_per_eth_tx_pkts_wr
Definition: common.h:410
adapter_params::vpd
struct vpd_params vpd
Definition: common.h:361
adapter::fwq_oid
struct sysctl_oid * fwq_oid
Definition: adapter.h:974
adapter::mbox
unsigned int mbox
Definition: adapter.h:883
adapter::cryptocaps
uint16_t cryptocaps
Definition: adapter.h:968
adapter::lro_timeout
int lro_timeout
Definition: adapter.h:903
adapter::sc_do_rxcopy
int sc_do_rxcopy
Definition: adapter.h:904
adapter::intr_type
int intr_type
Definition: adapter.h:888
adapter::pf
unsigned int pf
Definition: adapter.h:882
adapter::dmat
bus_dma_tag_t dmat
Definition: adapter.h:900
adapter::flags
int flags
Definition: adapter.h:940
adapter::swintr
int swintr
Definition: adapter.h:1009
adapter::udbs_base
volatile uint8_t * udbs_base
Definition: adapter.h:880
adapter::chip_params
const struct chip_params * chip_params
Definition: adapter.h:959
adapter::ctx
struct sysctl_ctx_list ctx
Definition: adapter.h:972
adapter::debug_flags
int debug_flags
Definition: adapter.h:941
adapter::error_flags
int error_flags
Definition: adapter.h:942
adapter::params
struct adapter_params params
Definition: adapter.h:958
adapter::tt
struct tom_tunables tt
Definition: adapter.h:922
adapter::intr_count
int intr_count
Definition: adapter.h:889
adapter::sge_gts_reg
int sge_gts_reg
Definition: adapter.h:897
adapter::vres
struct t4_virt_res vres
Definition: adapter.h:960
adapter::port
struct port_info * port[MAX_NPORTS]
Definition: adapter.h:912
adapter::ctrlq_oid
struct sysctl_oid * ctrlq_oid
Definition: adapter.h:973
adapter::sfl_lock
struct mtx sfl_lock
Definition: adapter.h:980
adapter::sge
struct sge sge
Definition: adapter.h:902
adapter::sfl_callout
struct callout sfl_callout
Definition: adapter.h:982
adapter::doorbells
uint8_t doorbells
Definition: adapter.h:936
adapter::tq
struct taskqueue * tq[MAX_NCHAN]
Definition: adapter.h:911
adapter::dev
device_t dev
Definition: adapter.h:866
adapter::sge_kdoorbell_reg
int sge_kdoorbell_reg
Definition: adapter.h:898
chip_params::sge_fl_db
u32 sge_fl_db
Definition: common.h:315
cluster_metadata
Definition: adapter.h:344
cluster_metadata::zone
uma_zone_t zone
Definition: adapter.h:345
cluster_metadata::refcount
u_int refcount
Definition: adapter.h:347
cluster_metadata::cl
caddr_t cl
Definition: adapter.h:346
cpl_act_open_rpl
Definition: t4_msg.h:894
cpl_act_open_rpl::atid_status
__be32 atid_status
Definition: t4_msg.h:897
cpl_fw4_ack
Definition: t4_msg.h:2757
cpl_fw4_ack::credits
u8 credits
Definition: t4_msg.h:2760
cpl_fw4_msg
Definition: t4_msg.h:2748
cpl_fw6_msg
Definition: t4_msg.h:2798
cpl_fw6_msg::data
__be64 data[4]
Definition: t4_msg.h:2804
cpl_fw6_msg::type
u8 type
Definition: t4_msg.h:2801
cpl_l2t_write_rpl
Definition: t4_msg.h:2331
cpl_rx_pkt
Definition: t4_msg.h:1938
cpl_rx_pkt::vlan_ex
__u8 vlan_ex
Definition: t4_msg.h:1949
cpl_rx_pkt::l2info
__be32 l2info
Definition: t4_msg.h:1957
cpl_rx_pkt::csum
__be16 csum
Definition: t4_msg.h:1954
cpl_rx_pkt::err_vec
__be16 err_vec
Definition: t4_msg.h:1959
cpl_rx_pkt::ip_frag
__u8 ip_frag
Definition: t4_msg.h:1948
cpl_rx_pkt::vlan
__be16 vlan
Definition: t4_msg.h:1955
cpl_rx_pkt::csum_calc
__u8 csum_calc
Definition: t4_msg.h:1951
cpl_set_tcb_rpl
Definition: t4_msg.h:987
cpl_set_tcb_rpl::cookie
__u8 cookie
Definition: t4_msg.h:991
cpl_sge_egr_update
Definition: t4_msg.h:2687
cpl_sge_egr_update::opcode_qid
RSS_HDR __be32 opcode_qid
Definition: t4_msg.h:2689
cpl_tx_pkt_core
Definition: t4_msg.h:1264
cpl_tx_pkt_core::pack
__be16 pack
Definition: t4_msg.h:1266
cpl_tx_pkt_core::len
__be16 len
Definition: t4_msg.h:1267
cpl_tx_pkt_core::ctrl1
__be64 ctrl1
Definition: t4_msg.h:1268
cpl_tx_pkt_core::ctrl0
__be32 ctrl0
Definition: t4_msg.h:1265
cpl_tx_pkt_lso_core
Definition: t4_msg.h:1409
cpl_tx_pkt_lso_core::ipid_ofst
__be16 ipid_ofst
Definition: t4_msg.h:1411
cpl_tx_pkt_lso_core::len
__be32 len
Definition: t4_msg.h:1414
cpl_tx_pkt_lso_core::mss
__be16 mss
Definition: t4_msg.h:1412
cpl_tx_pkt_lso_core::lso_ctrl
__be32 lso_ctrl
Definition: t4_msg.h:1410
cpl_tx_pkt_lso_core::seqno_offset
__be32 seqno_offset
Definition: t4_msg.h:1413
cpl_tx_tnl_lso
Definition: t4_msg.h:2985
cpl_tx_tnl_lso::IpIdOffsetOut
__be16 IpIdOffsetOut
Definition: t4_msg.h:2987
cpl_tx_tnl_lso::EthLenOffset_Size
__be32 EthLenOffset_Size
Definition: t4_msg.h:2994
cpl_tx_tnl_lso::TCPSeqOffset
__be32 TCPSeqOffset
Definition: t4_msg.h:2993
cpl_tx_tnl_lso::op_to_IpIdSplitOut
__be32 op_to_IpIdSplitOut
Definition: t4_msg.h:2986
cpl_tx_tnl_lso::IpIdSplit_to_Mss
__be16 IpIdSplit_to_Mss
Definition: t4_msg.h:2992
cpl_tx_tnl_lso::UdpLenSetOut_to_TnlHdrLen
__be16 UdpLenSetOut_to_TnlHdrLen
Definition: t4_msg.h:2988
cpl_tx_tnl_lso::r1
__be64 r1
Definition: t4_msg.h:2989
cpl_tx_tnl_lso::Flow_to_TcpHdrLen
__be32 Flow_to_TcpHdrLen
Definition: t4_msg.h:2990
cpl_tx_tnl_lso::IpIdOffset
__be16 IpIdOffset
Definition: t4_msg.h:2991
cxgbe_rate_tag
Definition: offload.h:90
cxgbe_rate_tag::plen
int plen
Definition: offload.h:98
cxgbe_rate_tag::lock
struct mtx lock
Definition: offload.h:94
cxgbe_rate_tag::iqid
uint16_t iqid
Definition: offload.h:101
cxgbe_rate_tag::tx_total
uint8_t tx_total
Definition: offload.h:104
cxgbe_rate_tag::com
struct m_snd_tag com
Definition: offload.h:91
cxgbe_rate_tag::schedcl
int8_t schedcl
Definition: offload.h:102
cxgbe_rate_tag::flags
u_int flags
Definition: offload.h:93
cxgbe_rate_tag::tx_credits
uint8_t tx_credits
Definition: offload.h:105
cxgbe_rate_tag::ncompl
uint8_t ncompl
Definition: offload.h:107
cxgbe_rate_tag::tx_nocompl
uint8_t tx_nocompl
Definition: offload.h:106
cxgbe_rate_tag::eo_txq
struct sge_ofld_txq * eo_txq
Definition: offload.h:99
cxgbe_rate_tag::adapter
struct adapter * adapter
Definition: offload.h:92
cxgbe_rate_tag::ctrl0
uint32_t ctrl0
Definition: offload.h:100
cxgbe_rate_tag::pending_fwack
struct mbufq pending_tx pending_fwack
Definition: offload.h:97
cxgbe_rate_tag::etid
int etid
Definition: offload.h:96
fl_sdesc
Definition: adapter.h:350
fl_sdesc::moff
int16_t moff
Definition: adapter.h:353
fl_sdesc::zidx
uint8_t zidx
Definition: adapter.h:354
fl_sdesc::cl
caddr_t cl
Definition: adapter.h:351
fl_sdesc::nmbuf
uint16_t nmbuf
Definition: adapter.h:352
fw_eq_ctrl_cmd
Definition: t4fw_interface.h:6138
fw_eq_ctrl_cmd::eqaddr
__be64 eqaddr
Definition: t4fw_interface.h:6145
fw_eq_ctrl_cmd::dcaen_to_eqsize
__be32 dcaen_to_eqsize
Definition: t4fw_interface.h:6144
fw_eq_ctrl_cmd::fetchszm_to_iqid
__be32 fetchszm_to_iqid
Definition: t4fw_interface.h:6143
fw_eq_ctrl_cmd::cmpliqid_eqid
__be32 cmpliqid_eqid
Definition: t4fw_interface.h:6141
fw_eq_ctrl_cmd::physeqid_pkd
__be32 physeqid_pkd
Definition: t4fw_interface.h:6142
fw_eq_ctrl_cmd::alloc_to_len16
__be32 alloc_to_len16
Definition: t4fw_interface.h:6140
fw_eq_ctrl_cmd::op_to_vfn
__be32 op_to_vfn
Definition: t4fw_interface.h:6139
fw_eq_eth_cmd
Definition: t4fw_interface.h:5922
fw_eq_eth_cmd::dcaen_to_eqsize
__be32 dcaen_to_eqsize
Definition: t4fw_interface.h:5928
fw_eq_eth_cmd::op_to_vfn
__be32 op_to_vfn
Definition: t4fw_interface.h:5923
fw_eq_eth_cmd::physeqid_pkd
__be32 physeqid_pkd
Definition: t4fw_interface.h:5926
fw_eq_eth_cmd::autoequiqe_to_viid
__be32 autoequiqe_to_viid
Definition: t4fw_interface.h:5930
fw_eq_eth_cmd::fetchszm_to_iqid
__be32 fetchszm_to_iqid
Definition: t4fw_interface.h:5927
fw_eq_eth_cmd::alloc_to_len16
__be32 alloc_to_len16
Definition: t4fw_interface.h:5924
fw_eq_eth_cmd::eqid_pkd
__be32 eqid_pkd
Definition: t4fw_interface.h:5925
fw_eq_eth_cmd::eqaddr
__be64 eqaddr
Definition: t4fw_interface.h:5929
fw_eq_ofld_cmd
Definition: t4fw_interface.h:6325
fw_error_cmd::fw_error::fw_error_acl::val
__u8 val[6]
Definition: t4fw_interface.h:9584
fw_error_cmd::fw_error::fw_error_acl::cidx
__be16 cidx
Definition: t4fw_interface.h:9580
fw_error_cmd::fw_error::fw_error_acl::eqid
__be32 eqid
Definition: t4fw_interface.h:9582
fw_error_cmd::fw_error::fw_error_acl::mv_pkd
__be16 mv_pkd
Definition: t4fw_interface.h:9583
fw_error_cmd::fw_error::fw_error_acl::pfn_vfn
__be16 pfn_vfn
Definition: t4fw_interface.h:9581
fw_error_cmd::fw_error::fw_error_exception::info
__be32 info[6]
Definition: t4fw_interface.h:9567
fw_error_cmd::fw_error::fw_error_hwmodule::regval
__be32 regval
Definition: t4fw_interface.h:9571
fw_error_cmd::fw_error::fw_error_hwmodule::regaddr
__be32 regaddr
Definition: t4fw_interface.h:9570
fw_error_cmd::fw_error::fw_error_wr::wrhdr
__u8 wrhdr[16]
Definition: t4fw_interface.h:9577
fw_error_cmd::fw_error::fw_error_wr::pfn_vfn
__be16 pfn_vfn
Definition: t4fw_interface.h:9575
fw_error_cmd::fw_error::fw_error_wr::cidx
__be16 cidx
Definition: t4fw_interface.h:9574
fw_error_cmd::fw_error::fw_error_wr::eqid
__be32 eqid
Definition: t4fw_interface.h:9576
fw_error_cmd
Definition: t4fw_interface.h:9562
fw_error_cmd::u
union fw_error_cmd::fw_error u
fw_error_cmd::op_to_type
__be32 op_to_type
Definition: t4fw_interface.h:9563
fw_eth_tx_eo_wr::fw_eth_tx_eo::fw_eth_tx_eo_tcpseg::r4
__be16 r4
Definition: t4fw_interface.h:822
fw_eth_tx_eo_wr::fw_eth_tx_eo::fw_eth_tx_eo_tcpseg::tsclk_tsoff
__u8 tsclk_tsoff
Definition: t4fw_interface.h:821
fw_eth_tx_eo_wr::fw_eth_tx_eo::fw_eth_tx_eo_tcpseg::plen
__be32 plen
Definition: t4fw_interface.h:825
fw_eth_tx_eo_wr::fw_eth_tx_eo::fw_eth_tx_eo_tcpseg::ethlen
__u8 ethlen
Definition: t4fw_interface.h:818
fw_eth_tx_eo_wr::fw_eth_tx_eo::fw_eth_tx_eo_tcpseg::r5
__be16 r5
Definition: t4fw_interface.h:824
fw_eth_tx_eo_wr::fw_eth_tx_eo::fw_eth_tx_eo_tcpseg::mss
__be16 mss
Definition: t4fw_interface.h:823
fw_eth_tx_eo_wr::fw_eth_tx_eo::fw_eth_tx_eo_tcpseg::iplen
__be16 iplen
Definition: t4fw_interface.h:819
fw_eth_tx_eo_wr::fw_eth_tx_eo::fw_eth_tx_eo_tcpseg::tcplen
__u8 tcplen
Definition: t4fw_interface.h:820
fw_eth_tx_eo_wr::fw_eth_tx_eo::fw_eth_tx_eo_tcpseg::type
__u8 type
Definition: t4fw_interface.h:817
fw_eth_tx_eo_wr::fw_eth_tx_eo::fw_eth_tx_eo_udpseg::iplen
__be16 iplen
Definition: t4fw_interface.h:808
fw_eth_tx_eo_wr::fw_eth_tx_eo::fw_eth_tx_eo_udpseg::ethlen
__u8 ethlen
Definition: t4fw_interface.h:807
fw_eth_tx_eo_wr::fw_eth_tx_eo::fw_eth_tx_eo_udpseg::schedpktsize
__be16 schedpktsize
Definition: t4fw_interface.h:813
fw_eth_tx_eo_wr::fw_eth_tx_eo::fw_eth_tx_eo_udpseg::type
__u8 type
Definition: t4fw_interface.h:806
fw_eth_tx_eo_wr::fw_eth_tx_eo::fw_eth_tx_eo_udpseg::r4
__be16 r4
Definition: t4fw_interface.h:811
fw_eth_tx_eo_wr::fw_eth_tx_eo::fw_eth_tx_eo_udpseg::mss
__be16 mss
Definition: t4fw_interface.h:812
fw_eth_tx_eo_wr::fw_eth_tx_eo::fw_eth_tx_eo_udpseg::rtplen
__u8 rtplen
Definition: t4fw_interface.h:810
fw_eth_tx_eo_wr::fw_eth_tx_eo::fw_eth_tx_eo_udpseg::plen
__be32 plen
Definition: t4fw_interface.h:814
fw_eth_tx_eo_wr::fw_eth_tx_eo::fw_eth_tx_eo_udpseg::udplen
__u8 udplen
Definition: t4fw_interface.h:809
fw_eth_tx_eo_wr
Definition: t4fw_interface.h:800
fw_eth_tx_eo_wr::equiq_to_len16
__be32 equiq_to_len16
Definition: t4fw_interface.h:802
fw_eth_tx_eo_wr::r3
__be64 r3
Definition: t4fw_interface.h:803
fw_eth_tx_eo_wr::op_immdlen
__be32 op_immdlen
Definition: t4fw_interface.h:801
fw_eth_tx_eo_wr::u
union fw_eth_tx_eo_wr::fw_eth_tx_eo u
fw_eth_tx_pkt_vm_wr
Definition: t4fw_interface.h:1170
fw_eth_tx_pkt_vm_wr::op_immdlen
__be32 op_immdlen
Definition: t4fw_interface.h:1171
fw_eth_tx_pkt_vm_wr::ethmacdst
__u8 ethmacdst[6]
Definition: t4fw_interface.h:1174
fw_eth_tx_pkt_vm_wr::equiq_to_len16
__be32 equiq_to_len16
Definition: t4fw_interface.h:1172
fw_eth_tx_pkt_vm_wr::r3
__be32 r3[2]
Definition: t4fw_interface.h:1173
fw_eth_tx_pkt_wr
Definition: t4fw_interface.h:701
fw_eth_tx_pkt_wr::equiq_to_len16
__be32 equiq_to_len16
Definition: t4fw_interface.h:703
fw_eth_tx_pkt_wr::op_immdlen
__be32 op_immdlen
Definition: t4fw_interface.h:702
fw_eth_tx_pkt_wr::r3
__be64 r3
Definition: t4fw_interface.h:704
fw_eth_tx_pkts_vm_wr
Definition: t4fw_interface.h:1180
fw_eth_tx_pkts_vm_wr::npkt
__u8 npkt
Definition: t4fw_interface.h:1185
fw_eth_tx_pkts_vm_wr::op_pkd
__be32 op_pkd
Definition: t4fw_interface.h:1181
fw_eth_tx_pkts_vm_wr::equiq_to_len16
__be32 equiq_to_len16
Definition: t4fw_interface.h:1182
fw_eth_tx_pkts_vm_wr::r3
__be32 r3
Definition: t4fw_interface.h:1183
fw_eth_tx_pkts_vm_wr::plen
__be16 plen
Definition: t4fw_interface.h:1184
fw_eth_tx_pkts_vm_wr::r4
__u8 r4
Definition: t4fw_interface.h:1186
fw_eth_tx_pkts_vm_wr::ethmacdst
__u8 ethmacdst[6]
Definition: t4fw_interface.h:1187
fw_eth_tx_pkts_wr
Definition: t4fw_interface.h:771
fw_eth_tx_pkts_wr::r3
__be32 r3
Definition: t4fw_interface.h:774
fw_eth_tx_pkts_wr::equiq_to_len16
__be32 equiq_to_len16
Definition: t4fw_interface.h:773
fw_eth_tx_pkts_wr::plen
__be16 plen
Definition: t4fw_interface.h:775
fw_eth_tx_pkts_wr::type
__u8 type
Definition: t4fw_interface.h:777
fw_eth_tx_pkts_wr::npkt
__u8 npkt
Definition: t4fw_interface.h:776
fw_eth_tx_pkts_wr::op_pkd
__be32 op_pkd
Definition: t4fw_interface.h:772
fw_flowc_mnemval::val
__be32 val
Definition: t4fw_interface.h:1045
fw_flowc_mnemval::mnemonic
__u8 mnemonic
Definition: t4fw_interface.h:1043
fw_flowc_wr
Definition: t4fw_interface.h:1048
fw_flowc_wr::mnemval
struct fw_flowc_mnemval mnemval[0]
Definition: t4fw_interface.h:1052
fw_flowc_wr::op_to_nparams
__be32 op_to_nparams
Definition: t4fw_interface.h:1049
fw_flowc_wr::flowid_len16
__be32 flowid_len16
Definition: t4fw_interface.h:1050
fw_iq_cmd
Definition: t4fw_interface.h:5213
fw_iq_cmd::alloc_to_len16
__be32 alloc_to_len16
Definition: t4fw_interface.h:5215
fw_iq_cmd::fl0addr
__be64 fl0addr
Definition: t4fw_interface.h:5227
fw_iq_cmd::fl0id
__be16 fl0id
Definition: t4fw_interface.h:5218
fw_iq_cmd::iqid
__be16 iqid
Definition: t4fw_interface.h:5217
fw_iq_cmd::op_to_vfn
__be32 op_to_vfn
Definition: t4fw_interface.h:5214
fw_iq_cmd::iqns_to_fl0congen
__be32 iqns_to_fl0congen
Definition: t4fw_interface.h:5224
fw_iq_cmd::iqsize
__be16 iqsize
Definition: t4fw_interface.h:5222
fw_iq_cmd::iqaddr
__be64 iqaddr
Definition: t4fw_interface.h:5223
fw_iq_cmd::fl0size
__be16 fl0size
Definition: t4fw_interface.h:5226
fw_iq_cmd::iqdroprss_to_iqesize
__be16 iqdroprss_to_iqesize
Definition: t4fw_interface.h:5221
fw_iq_cmd::fl0dcaen_to_fl0cidxfthresh
__be16 fl0dcaen_to_fl0cidxfthresh
Definition: t4fw_interface.h:5225
fw_iq_cmd::physiqid
__be16 physiqid
Definition: t4fw_interface.h:5216
fw_iq_cmd::type_to_iqandstindex
__be32 type_to_iqandstindex
Definition: t4fw_interface.h:5220
iq_desc
Definition: adapter.h:368
iq_desc::rss
struct rss_header rss
Definition: adapter.h:369
iq_desc::rsp
struct rsp_ctrl rsp
Definition: adapter.h:371
mp_ring
Definition: t4_mp_ring.h:42
mp_ring::cookie
void * cookie
Definition: t4_mp_ring.h:47
pci_params::mps
unsigned int mps
Definition: common.h:290
port_info
Definition: adapter.h:304
port_info::tx_chan
uint8_t tx_chan
Definition: adapter.h:325
port_info::adapter
struct adapter * adapter
Definition: adapter.h:306
port_info::vi
struct vi_info * vi
Definition: adapter.h:308
port_info::rx_e_chan_map
uint8_t rx_e_chan_map
Definition: adapter.h:327
rsp_ctrl::u
union rsp_ctrl::@29 u
rsp_ctrl::last_flit
__be64 last_flit
Definition: t4_hw.h:124
rsp_ctrl::pldbuflen_qid
__be32 pldbuflen_qid
Definition: t4_hw.h:121
rsp_ctrl::type_gen
u8 type_gen
Definition: t4_hw.h:123
rss_header
Definition: t4_msg.h:372
rss_header::ipv6
__u8 ipv6
Definition: t4_msg.h:383
rss_header::hash_type
__u8 hash_type
Definition: t4_msg.h:384
rss_header::opcode
__u8 opcode
Definition: t4_msg.h:373
rss_header::hash_val
__be32 hash_val
Definition: t4_msg.h:390
rx_buf_info
Definition: adapter.h:491
rx_buf_info::size1
uint16_t size1
Definition: adapter.h:493
rx_buf_info::zone
uma_zone_t zone
Definition: adapter.h:492
rx_buf_info::hwidx2
int8_t hwidx2
Definition: adapter.h:498
rx_buf_info::hwidx1
int8_t hwidx1
Definition: adapter.h:497
rx_buf_info::size2
uint16_t size2
Definition: adapter.h:495
rx_buf_info::type
uint8_t type
Definition: adapter.h:499
sge_eq
Definition: adapter.h:464
sge_eq::sidx
uint16_t sidx
Definition: adapter.h:476
sge_eq::doorbells
uint8_t doorbells
Definition: adapter.h:469
sge_eq::tx_chan
uint8_t tx_chan
Definition: adapter.h:470
sge_eq::equiq
volatile u_int equiq
Definition: adapter.h:482
sge_eq::udb
volatile uint32_t * udb
Definition: adapter.h:474
sge_eq::flags
unsigned int flags
Definition: adapter.h:465
sge_eq::cntxt_id
unsigned int cntxt_id
Definition: adapter.h:466
sge_eq::equeqidx
uint16_t equeqidx
Definition: adapter.h:479
sge_eq::iq
struct sge_iq * iq
Definition: adapter.h:483
sge_eq::pidx
uint16_t pidx
Definition: adapter.h:478
sge_eq::dbidx
uint16_t dbidx
Definition: adapter.h:480
sge_eq::desc_tag
bus_dma_tag_t desc_tag
Definition: adapter.h:485
sge_eq::type
uint8_t type
Definition: adapter.h:468
sge_eq::desc_map
bus_dmamap_t desc_map
Definition: adapter.h:486
sge_eq::udb_qid
u_int udb_qid
Definition: adapter.h:475
sge_eq::desc
struct tx_desc * desc
Definition: adapter.h:473
sge_eq::abs_id
unsigned int abs_id
Definition: adapter.h:467
sge_eq::iqid
uint16_t iqid
Definition: adapter.h:481
sge_eq::lockname
char lockname[16]
Definition: adapter.h:488
sge_eq::cidx
uint16_t cidx
Definition: adapter.h:477
sge_eq::eq_lock
struct mtx eq_lock
Definition: adapter.h:471
sge_eq::ba
bus_addr_t ba
Definition: adapter.h:487
sge_fl
Definition: adapter.h:537
sge_fl::hw_cidx
volatile uint16_t hw_cidx
Definition: adapter.h:550
sge_fl::sidx
uint16_t sidx
Definition: adapter.h:549
sge_fl::cidx
uint32_t cidx
Definition: adapter.h:553
sge_fl::cl_fast_recycled
uint64_t cl_fast_recycled
Definition: adapter.h:562
sge_fl::desc_map
bus_dmamap_t desc_map
Definition: adapter.h:573
sge_fl::remaining
u_int remaining
Definition: adapter.h:567
sge_fl::dbidx
uint16_t dbidx
Definition: adapter.h:548
sge_fl::rx_offset
u_int rx_offset
Definition: adapter.h:557
sge_fl::ba
bus_addr_t ba
Definition: adapter.h:575
sge_fl::flags
int flags
Definition: adapter.h:544
sge_fl::zidx
uint16_t zidx
Definition: adapter.h:541
sge_fl::sdesc
struct fl_sdesc * sdesc
Definition: adapter.h:540
sge_fl::buf_boundary
uint16_t buf_boundary
Definition: adapter.h:545
sge_fl::desc_tag
bus_dma_tag_t desc_tag
Definition: adapter.h:572
sge_fl::cl_recycled
uint64_t cl_recycled
Definition: adapter.h:561
sge_fl::lockname
char lockname[16]
Definition: adapter.h:574
sge_fl::cntxt_id
uint16_t cntxt_id
Definition: adapter.h:570
sge_fl::safe_zidx
uint16_t safe_zidx
Definition: adapter.h:542
sge_fl::qsize
uint16_t qsize
Definition: adapter.h:569
sge_fl::fl_lock
struct mtx fl_lock
Definition: adapter.h:538
sge_fl::dbval
uint32_t dbval
Definition: adapter.h:556
sge_fl::pnext
struct mbuf ** pnext
Definition: adapter.h:566
sge_fl::desc
__be64 * desc
Definition: adapter.h:539
sge_fl::pidx
uint32_t pidx
Definition: adapter.h:554
sge_fl::cl_allocated
uint64_t cl_allocated
Definition: adapter.h:560
sge_fl::lowat
uint16_t lowat
Definition: adapter.h:543
sge_fl::udb
volatile uint32_t * udb
Definition: adapter.h:558
sge_fl::m0
struct mbuf * m0
Definition: adapter.h:565
sge_iq
Definition: adapter.h:419
sge_iq::intr_idx
int16_t intr_idx
Definition: adapter.h:433
sge_iq::ba
bus_addr_t ba
Definition: adapter.h:439
sge_iq::intr_params
uint8_t intr_params
Definition: adapter.h:426
sge_iq::desc_tag
bus_dma_tag_t desc_tag
Definition: adapter.h:437
sge_iq::flags
uint32_t flags
Definition: adapter.h:420
sge_iq::adapter
struct adapter * adapter
Definition: adapter.h:422
sge_iq::desc_map
bus_dmamap_t desc_map
Definition: adapter.h:438
sge_iq::qsize
uint16_t qsize
Definition: adapter.h:428
sge_iq::desc
struct iq_desc * desc
Definition: adapter.h:423
sge_iq::gen
uint8_t gen
Definition: adapter.h:425
sge_iq::sidx
uint16_t sidx
Definition: adapter.h:429
sge_iq::state
volatile int state
Definition: adapter.h:421
sge_iq::abs_id
uint16_t abs_id
Definition: adapter.h:432
sge_iq::intr_pktc_idx
int8_t intr_pktc_idx
Definition: adapter.h:424
sge_iq::cntxt_id
uint16_t cntxt_id
Definition: adapter.h:431
sge_iq::cong
int8_t cong
Definition: adapter.h:427
sge_iq::cidx
uint16_t cidx
Definition: adapter.h:430
sge_nm_rxq
Definition: adapter.h:756
sge_nm_txq
Definition: adapter.h:797
sge_ofld_rxq
Definition: adapter.h:673
sge_ofld_rxq::rx_iscsi_ddp_setup_error
counter_u64_t rx_iscsi_ddp_setup_error
Definition: adapter.h:677
sge_ofld_rxq::rx_iscsi_fl_pdus
uint64_t rx_iscsi_fl_pdus
Definition: adapter.h:680
sge_ofld_rxq::rx_iscsi_ddp_pdus
uint64_t rx_iscsi_ddp_pdus
Definition: adapter.h:678
sge_ofld_rxq::rx_toe_tls_records
u_long rx_toe_tls_records
Definition: adapter.h:685
sge_ofld_rxq::rx_iscsi_fl_octets
uint64_t rx_iscsi_fl_octets
Definition: adapter.h:681
sge_ofld_rxq::fl
struct sge_fl fl
Definition: adapter.h:675
sge_ofld_rxq::rx_iscsi_data_digest_errors
uint64_t rx_iscsi_data_digest_errors
Definition: adapter.h:684
sge_ofld_rxq::rx_iscsi_ddp_octets
uint64_t rx_iscsi_ddp_octets
Definition: adapter.h:679
sge_ofld_rxq::iq
struct sge_iq iq
Definition: adapter.h:674
sge_ofld_rxq::rx_iscsi_header_digest_errors
uint64_t rx_iscsi_header_digest_errors
Definition: adapter.h:683
sge_ofld_rxq::rx_toe_tls_octets
u_long rx_toe_tls_octets
Definition: adapter.h:686
sge_ofld_rxq::rx_iscsi_padding_errors
uint64_t rx_iscsi_padding_errors
Definition: adapter.h:682
sge_ofld_rxq::rx_iscsi_ddp_setup_ok
counter_u64_t rx_iscsi_ddp_setup_ok
Definition: adapter.h:676
sge_ofld_txq
Definition: adapter.h:746
sge_ofld_txq::tx_iscsi_octets
counter_u64_t tx_iscsi_octets
Definition: adapter.h:749
sge_ofld_txq::tx_iscsi_iso_wrs
counter_u64_t tx_iscsi_iso_wrs
Definition: adapter.h:750
sge_ofld_txq::tx_toe_tls_octets
counter_u64_t tx_toe_tls_octets
Definition: adapter.h:752
sge_ofld_txq::tx_toe_tls_records
counter_u64_t tx_toe_tls_records
Definition: adapter.h:751
sge_ofld_txq::wrq
struct sge_wrq wrq
Definition: adapter.h:747
sge_ofld_txq::tx_iscsi_pdus
counter_u64_t tx_iscsi_pdus
Definition: adapter.h:748
sge_params
Definition: common.h:237
sge_params::pad_boundary
int pad_boundary
Definition: common.h:246
sge_params::fl_starve_threshold2
int fl_starve_threshold2
Definition: common.h:241
sge_params::sge_control
u32 sge_control
Definition: common.h:249
sge_params::fl_starve_threshold
int fl_starve_threshold
Definition: common.h:240
sge_params::page_shift
int page_shift
Definition: common.h:242
sge_params::eq_s_qpp
int eq_s_qpp
Definition: common.h:243
sge_params::pack_boundary
int pack_boundary
Definition: common.h:247
sge_params::spg_len
int spg_len
Definition: common.h:245
sge_params::fl_pktshift
int fl_pktshift
Definition: common.h:248
sge_params::sge_fl_buffer_size
u32 sge_fl_buffer_size[SGE_FLBUF_SIZES]
Definition: common.h:250
sge_qstat
Definition: t4_hw.h:102
sge_qstat::cidx
volatile __be16 cidx
Definition: t4_hw.h:104
sge_rxq
Definition: adapter.h:648
sge_rxq::rxcsum
uint64_t rxcsum
Definition: adapter.h:657
sge_rxq::vxlan_rxcsum
uint64_t vxlan_rxcsum
Definition: adapter.h:659
sge_rxq::iq
struct sge_iq iq
Definition: adapter.h:649
sge_rxq::lro
struct lro_ctrl lro
Definition: adapter.h:653
sge_rxq::fl
struct sge_fl fl
Definition: adapter.h:650
sge_rxq::ifp
struct ifnet * ifp
Definition: adapter.h:652
sge_rxq::vlan_extraction
uint64_t vlan_extraction
Definition: adapter.h:658
sge_txq
Definition: adapter.h:598
sge_txq::raw_wrs
uint64_t raw_wrs
Definition: adapter.h:624
sge_txq::r
struct mp_ring * r
Definition: adapter.h:602
sge_txq::kern_tls_waste
uint64_t kern_tls_waste
Definition: adapter.h:633
sge_txq::txpkts0_pkts
uint64_t txpkts0_pkts
Definition: adapter.h:621
sge_txq::kern_tls_fin
uint64_t kern_tls_fin
Definition: adapter.h:636
sge_txq::kern_tls_options
uint64_t kern_tls_options
Definition: adapter.h:634
sge_txq::vlan_insertion
uint64_t vlan_insertion
Definition: adapter.h:615
sge_txq::kern_tls_octets
uint64_t kern_tls_octets
Definition: adapter.h:632
sge_txq::imm_wrs
uint64_t imm_wrs
Definition: adapter.h:616
sge_txq::txcsum
uint64_t txcsum
Definition: adapter.h:613
sge_txq::txpkt_wrs
uint64_t txpkt_wrs
Definition: adapter.h:618
sge_txq::cpl_ctrl0
__be32 cpl_ctrl0
Definition: adapter.h:605
sge_txq::kern_tls_header
uint64_t kern_tls_header
Definition: adapter.h:635
sge_txq::kern_tls_full
uint64_t kern_tls_full
Definition: adapter.h:631
sge_txq::ifp
struct ifnet * ifp
Definition: adapter.h:601
sge_txq::tx_reclaim_task
struct task tx_reclaim_task
Definition: adapter.h:610
sge_txq::sdesc
struct tx_sdesc * sdesc
Definition: adapter.h:603
sge_txq::txp
struct txpkts txp
Definition: adapter.h:608
sge_txq::txpkts0_wrs
uint64_t txpkts0_wrs
Definition: adapter.h:619
sge_txq::txpkts1_wrs
uint64_t txpkts1_wrs
Definition: adapter.h:620
sge_txq::sgl_wrs
uint64_t sgl_wrs
Definition: adapter.h:617
sge_txq::kern_tls_partial
uint64_t kern_tls_partial
Definition: adapter.h:630
sge_txq::last_tx
uint64_t last_tx
Definition: adapter.h:607
sge_txq::kern_tls_records
uint64_t kern_tls_records
Definition: adapter.h:628
sge_txq::tso_wrs
uint64_t tso_wrs
Definition: adapter.h:614
sge_txq::tc_idx
int tc_idx
Definition: adapter.h:606
sge_txq::txpkts1_pkts
uint64_t txpkts1_pkts
Definition: adapter.h:622
sge_txq::kern_tls_gcm
uint64_t kern_tls_gcm
Definition: adapter.h:639
sge_txq::kern_tls_fin_short
uint64_t kern_tls_fin_short
Definition: adapter.h:637
sge_txq::eq
struct sge_eq eq
Definition: adapter.h:599
sge_txq::txpkts_flush
uint64_t txpkts_flush
Definition: adapter.h:623
sge_txq::kern_tls_cbc
uint64_t kern_tls_cbc
Definition: adapter.h:638
sge_txq::kern_tls_short
uint64_t kern_tls_short
Definition: adapter.h:629
sge_txq::vxlan_txcsum
uint64_t vxlan_txcsum
Definition: adapter.h:626
sge_txq::gl
struct sglist * gl
Definition: adapter.h:604
sge_txq::vxlan_tso_wrs
uint64_t vxlan_tso_wrs
Definition: adapter.h:625
sge_wrq
Definition: adapter.h:713
sge_wrq::tx_wrs_ss
uint64_t tx_wrs_ss
Definition: adapter.h:730
sge_wrq::nwr_pending
u_int nwr_pending
Definition: adapter.h:724
sge_wrq::ndesc_needed
u_int ndesc_needed
Definition: adapter.h:725
sge_wrq::ss_len
uint16_t ss_len
Definition: adapter.h:740
sge_wrq::adapter
struct adapter * adapter
Definition: adapter.h:716
sge_wrq::wrq_tx_task
struct task wrq_tx_task
Definition: adapter.h:717
sge_wrq::ss_pidx
uint16_t ss_pidx
Definition: adapter.h:739
sge_wrq::tx_wrs_direct
uint64_t tx_wrs_direct
Definition: adapter.h:729
sge_wrq::ss
uint8_t ss[SGE_MAX_WR_LEN]
Definition: adapter.h:741
sge_wrq::tx_wrs_copied
uint64_t tx_wrs_copied
Definition: adapter.h:731
sge_wrq::eq
struct sge_eq eq
Definition: adapter.h:714
sge
Definition: adapter.h:821
sge::rx_buf_info
struct rx_buf_info rx_buf_info[SW_ZONE_SIZES]
Definition: adapter.h:850
sge::ofld_rxq
struct sge_ofld_rxq * ofld_rxq
Definition: adapter.h:836
sge::iq_base
uint16_t iq_base
Definition: adapter.h:841
sge::fwq
struct sge_iq fwq
Definition: adapter.h:831
sge::txq
struct sge_txq * txq
Definition: adapter.h:833
sge::ctrlq
struct sge_wrq * ctrlq
Definition: adapter.h:832
sge::eq_start
int eq_start
Definition: adapter.h:842
sge::iqmap_sz
int iqmap_sz
Definition: adapter.h:844
sge::eqmap_sz
int eqmap_sz
Definition: adapter.h:845
sge::eq_base
int eq_base
Definition: adapter.h:843
sge::safe_zidx
int8_t safe_zidx
Definition: adapter.h:849
sge::rxq
struct sge_rxq * rxq
Definition: adapter.h:834
sge::eqmap
struct sge_eq ** eqmap
Definition: adapter.h:847
sge::iq_start
uint16_t iq_start
Definition: adapter.h:840
sge::ofld_txq
struct sge_ofld_txq * ofld_txq
Definition: adapter.h:835
sge::iqmap
struct sge_iq ** iqmap
Definition: adapter.h:846
t4_range::size
u_int size
Definition: offload.h:187
t4_virt_res::ddp
struct t4_range ddp
Definition: offload.h:191
tom_tunables::tls
int tls
Definition: offload.h:227
tp_params::max_rx_pdu
uint32_t max_rx_pdu
Definition: common.h:262
tp_params::rx_pkt_encap
bool rx_pkt_encap
Definition: common.h:264
tx_sdesc
Definition: adapter.h:361
tx_sdesc::desc_used
uint8_t desc_used
Definition: adapter.h:363
tx_sdesc::m
struct mbuf * m
Definition: adapter.h:362
txpkts::score
uint8_t score
Definition: adapter.h:584
txpkts::max_npkt
uint8_t max_npkt
Definition: adapter.h:585
txpkts::wr_type
uint8_t wr_type
Definition: adapter.h:581
txpkts::len16
uint8_t len16
Definition: adapter.h:583
txpkts::npkt
uint8_t npkt
Definition: adapter.h:582
txpkts::ethtype
__be16 ethtype
Definition: adapter.h:591
txpkts::ethmacdst
__u8 ethmacdst[6]
Definition: adapter.h:589
txpkts::mb
struct mbuf * mb[15]
Definition: adapter.h:594
txpkts::plen
uint16_t plen
Definition: adapter.h:586
ulp_txpkt
Definition: t4_msg.h:2945
ulp_txpkt::cmd_dest
__be32 cmd_dest
Definition: t4_msg.h:2946
ulp_txpkt::len
__be32 len
Definition: t4_msg.h:2947
ulptx_idata
Definition: t4_msg.h:2883
ulptx_idata::len
__be32 len
Definition: t4_msg.h:2885
ulptx_idata::cmd_more
__be32 cmd_more
Definition: t4_msg.h:2884
ulptx_sge_pair::addr
__be64 addr[2]
Definition: t4_msg.h:2857
ulptx_sge_pair::len
__be32 len[2]
Definition: t4_msg.h:2856
ulptx_sgl
Definition: t4_msg.h:2860
ulptx_sgl::sge
struct ulptx_sge_pair sge[]
Definition: t4_msg.h:2865
ulptx_sgl::len0
__be32 len0
Definition: t4_msg.h:2862
ulptx_sgl::cmd_nsge
__be32 cmd_nsge
Definition: t4_msg.h:2861
ulptx_sgl::addr0
__be64 addr0
Definition: t4_msg.h:2863
vi_info
Definition: adapter.h:198
vi_info::rss
uint16_t * rss
Definition: adapter.h:209
vi_info::viid
uint16_t viid
Definition: adapter.h:210
vi_info::dev
device_t dev
Definition: adapter.h:199
vi_info::tmr_idx
int tmr_idx
Definition: adapter.h:236
vi_info::nofldtxq
int nofldtxq
Definition: adapter.h:228
vi_info::nofldrxq
int nofldrxq
Definition: adapter.h:230
vi_info::first_rxq
int first_rxq
Definition: adapter.h:227
vi_info::adapter
struct adapter * adapter
Definition: adapter.h:201
vi_info::nnmrxq
int nnmrxq
Definition: adapter.h:234
vi_info::ctx
struct sysctl_ctx_list ctx
Definition: adapter.h:248
vi_info::ofld_rxq_oid
struct sysctl_oid * ofld_rxq_oid
Definition: adapter.h:253
vi_info::vfvld
uint8_t vfvld
Definition: adapter.h:213
vi_info::rxq_oid
struct sysctl_oid * rxq_oid
Definition: adapter.h:249
vi_info::first_ofld_rxq
int first_ofld_rxq
Definition: adapter.h:231
vi_info::qsize_rxq
int qsize_rxq
Definition: adapter.h:240
vi_info::first_intr
int first_intr
Definition: adapter.h:220
vi_info::ofld_tmr_idx
int ofld_tmr_idx
Definition: adapter.h:237
vi_info::first_ofld_txq
int first_ofld_txq
Definition: adapter.h:229
vi_info::ofld_pktc_idx
int ofld_pktc_idx
Definition: adapter.h:239
vi_info::first_nm_rxq
int first_nm_rxq
Definition: adapter.h:235
vi_info::nrxq
int nrxq
Definition: adapter.h:226
vi_info::ifp
struct ifnet * ifp
Definition: adapter.h:203
vi_info::rss_size
uint16_t rss_size
Definition: adapter.h:215
vi_info::vin
uint16_t vin
Definition: adapter.h:212
vi_info::txq_oid
struct sysctl_oid * txq_oid
Definition: adapter.h:250
vi_info::ofld_txq_oid
struct sysctl_oid * ofld_txq_oid
Definition: adapter.h:254
vi_info::flags
unsigned long flags
Definition: adapter.h:206
vi_info::pfil
struct pfil_head * pfil
Definition: adapter.h:204
vi_info::pi
struct port_info * pi
Definition: adapter.h:200
vi_info::pktc_idx
int pktc_idx
Definition: adapter.h:238
vi_info::qsize_txq
int qsize_txq
Definition: adapter.h:241
vpd_params::cclk
unsigned int cclk
Definition: common.h:279
wrq_cookie
Definition: adapter.h:703
wrq_cookie::pidx
int pidx
Definition: adapter.h:706
wrq_cookie::ndesc
int ndesc
Definition: adapter.h:705
wrqe
Definition: adapter.h:696
wrqe::wrq
struct sge_wrq * wrq
Definition: adapter.h:698
wrqe::wr_len
int wr_len
Definition: adapter.h:699
F_RSPD_NEWBUF
#define F_RSPD_NEWBUF
Definition: t4_hw.h:130
UDBS_DB_OFFSET
@ UDBS_DB_OFFSET
Definition: t4_hw.h:63
UDBS_WR_OFFSET
@ UDBS_WR_OFFSET
Definition: t4_hw.h:64
UDBS_SEG_SIZE
@ UDBS_SEG_SIZE
Definition: t4_hw.h:61
UDBS_SEG_SHIFT
@ UDBS_SEG_SHIFT
Definition: t4_hw.h:62
F_QINTR_CNT_EN
#define F_QINTR_CNT_EN
Definition: t4_hw.h:158
G_RSPD_TYPE
#define G_RSPD_TYPE(x)
Definition: t4_hw.h:153
SGE_MAX_WR_LEN
@ SGE_MAX_WR_LEN
Definition: t4_hw.h:93
SGE_NCOUNTERS
@ SGE_NCOUNTERS
Definition: t4_hw.h:96
SGE_FLBUF_SIZES
@ SGE_FLBUF_SIZES
Definition: t4_hw.h:99
SGE_NTIMERS
@ SGE_NTIMERS
Definition: t4_hw.h:95
V_QINTR_TIMER_IDX
#define V_QINTR_TIMER_IDX(x)
Definition: t4_hw.h:162
F_RSPD_GEN
#define F_RSPD_GEN
Definition: t4_hw.h:144
G_RSPD_LEN
#define G_RSPD_LEN(x)
Definition: t4_hw.h:135
t6_ktls_write_wr
int t6_ktls_write_wr(struct sge_txq *txq, void *dst, struct mbuf *m, u_int nsegs, u_int available)
Definition: t4_kern_tls.c:2135
t6_ktls_parse_pkt
int t6_ktls_parse_pkt(struct mbuf *m, int *nsegsp, int *len16p)
Definition: t4_kern_tls.c:2129
t4_l2t.h
F_SYNC_WR
#define F_SYNC_WR
Definition: t4_l2t.h:38
mp_ring_alloc
int mp_ring_alloc(struct mp_ring **pr, int size, void *cookie, ring_drain_t drain, ring_can_drain_t can_drain, struct malloc_type *mt, struct mtx *lck, int flags)
Definition: t4_mp_ring.c:261
mp_ring_check_drainage
void mp_ring_check_drainage(struct mp_ring *r, int budget)
Definition: t4_mp_ring.c:479
mp_ring_sysctls
void mp_ring_sysctls(struct mp_ring *r, struct sysctl_ctx_list *ctx, struct sysctl_oid_list *children)
Definition: t4_mp_ring.c:535
mp_ring_is_idle
bool mp_ring_is_idle(struct mp_ring *r)
Definition: t4_mp_ring.c:522
mp_ring_free
void mp_ring_free(struct mp_ring *r)
Definition: t4_mp_ring.c:312
t4_mp_ring.h
t4_msg.h
F_CPL_TX_TNL_LSO_FIRST
#define F_CPL_TX_TNL_LSO_FIRST
Definition: t4_msg.h:3009
CPL_TX_PKT_LSO
@ CPL_TX_PKT_LSO
Definition: t4_msg.h:153
CPL_L2T_WRITE_RPL
@ CPL_L2T_WRITE_RPL
Definition: t4_msg.h:67
NUM_CPL_CMDS
@ NUM_CPL_CMDS
Definition: t4_msg.h:156
CPL_FW4_MSG
@ CPL_FW4_MSG
Definition: t4_msg.h:144
CPL_FW6_MSG
@ CPL_FW6_MSG
Definition: t4_msg.h:150
CPL_FW4_ACK
@ CPL_FW4_ACK
Definition: t4_msg.h:146
CPL_TX_PKT_XT
@ CPL_TX_PKT_XT
Definition: t4_msg.h:154
CPL_ACT_OPEN_RPL
@ CPL_ACT_OPEN_RPL
Definition: t4_msg.h:69
CPL_TX_TNL_LSO
@ CPL_TX_TNL_LSO
Definition: t4_msg.h:152
CPL_SGE_EGR_UPDATE
@ CPL_SGE_EGR_UPDATE
Definition: t4_msg.h:127
CPL_RX_PKT
@ CPL_RX_PKT
Definition: t4_msg.h:92
CPL_ABORT_RPL_RSS
@ CPL_ABORT_RPL_RSS
Definition: t4_msg.h:77
CPL_SET_TCB_RPL
@ CPL_SET_TCB_RPL
Definition: t4_msg.h:91
V_LSO_OPCODE
#define V_LSO_OPCODE(x)
Definition: t4_msg.h:1474
V_ULP_TXPKT_FID
#define V_ULP_TXPKT_FID(x)
Definition: t4_msg.h:2972
F_CPL_TX_TNL_LSO_IPV6OUT
#define F_CPL_TX_TNL_LSO_IPV6OUT
Definition: t4_msg.h:3031
FW6_TYPE_CMD_RPL
@ FW6_TYPE_CMD_RPL
Definition: t4_msg.h:2809
FW6_TYPE_RSSCPL
@ FW6_TYPE_RSSCPL
Definition: t4_msg.h:2813
NUM_FW6_TYPES
@ NUM_FW6_TYPES
Definition: t4_msg.h:2816
FW6_TYPE_WRERR_RPL
@ FW6_TYPE_WRERR_RPL
Definition: t4_msg.h:2814
F_LSO_LAST_SLICE
#define F_LSO_LAST_SLICE
Definition: t4_msg.h:1466
G_T6_RX_TNL_TYPE
#define G_T6_RX_TNL_TYPE(x)
Definition: t4_msg.h:2154
V_CPL_TX_TNL_LSO_IPHDRLEN
#define V_CPL_TX_TNL_LSO_IPHDRLEN(x)
Definition: t4_msg.h:3125
F_CPL_TX_TNL_LSO_IPHDRCHKOUT
#define F_CPL_TX_TNL_LSO_IPHDRCHKOUT
Definition: t4_msg.h:3051
F_CPL_TX_TNL_LSO_LAST
#define F_CPL_TX_TNL_LSO_LAST
Definition: t4_msg.h:3016
F_ULP_TX_SC_MORE
#define F_ULP_TX_SC_MORE
Definition: t4_msg.h:2853
TX_CSUM_UDPIP6
@ TX_CSUM_UDPIP6
Definition: t4_msg.h:266
TX_CSUM_TCPIP
@ TX_CSUM_TCPIP
Definition: t4_msg.h:263
TX_CSUM_UDPIP
@ TX_CSUM_UDPIP
Definition: t4_msg.h:264
TX_CSUM_TCPIP6
@ TX_CSUM_TCPIP6
Definition: t4_msg.h:265
TX_CSUM_IP
@ TX_CSUM_IP
Definition: t4_msg.h:267
V_TXPKT_CSUM_TYPE
#define V_TXPKT_CSUM_TYPE(x)
Definition: t4_msg.h:1385
F_TXPKT_VLAN_VLD
#define F_TXPKT_VLAN_VLD
Definition: t4_msg.h:1395
G_EGR_QID
#define G_EGR_QID(x)
Definition: t4_msg.h:2703
G_COOKIE
#define G_COOKIE(x)
Definition: t4_msg.h:985
F_TXPKT_L4CSUM_DIS
#define F_TXPKT_L4CSUM_DIS
Definition: t4_msg.h:1407
F_RXF_IP
#define F_RXF_IP
Definition: t4_msg.h:2019
V_TXPKT_PF
#define V_TXPKT_PF(x)
Definition: t4_msg.h:1286
V_TXPKT_ETHHDR_LEN
#define V_TXPKT_ETHHDR_LEN(x)
Definition: t4_msg.h:1374
V_CPL_TX_TNL_LSO_SIZE
#define V_CPL_TX_TNL_LSO_SIZE(x)
Definition: t4_msg.h:3164
V_CPL_TX_TNL_LSO_TNLHDRLEN
#define V_CPL_TX_TNL_LSO_TNLHDRLEN(x)
Definition: t4_msg.h:3099
F_CPL_TX_TNL_LSO_IPLENSETOUT
#define F_CPL_TX_TNL_LSO_IPLENSETOUT
Definition: t4_msg.h:3058
V_TXPKT_VLAN
#define V_TXPKT_VLAN(x)
Definition: t4_msg.h:1390
V_CPL_TX_TNL_LSO_ETHHDRLEN
#define V_CPL_TX_TNL_LSO_ETHHDRLEN(x)
Definition: t4_msg.h:3119
V_ULP_TXPKT_DEST
#define V_ULP_TXPKT_DEST(x)
Definition: t4_msg.h:2968
RX_PKT_TNL_TYPE_VXLAN
#define RX_PKT_TNL_TYPE_VXLAN
Definition: t4_msg.h:2157
V_TXPKT_VF
#define V_TXPKT_VF(x)
Definition: t4_msg.h:1281
OPCODE_TID
#define OPCODE_TID(cmd)
Definition: t4_msg.h:327
TX_TNL_TYPE_VXLAN
@ TX_TNL_TYPE_VXLAN
Definition: t4_msg.h:2981
ULP_TX_PKT
@ ULP_TX_PKT
Definition: t4_msg.h:2831
F_CPL_TX_TNL_LSO_UDPCHKCLROUT
#define F_CPL_TX_TNL_LSO_UDPCHKCLROUT
Definition: t4_msg.h:3089
V_ULPTX_CMD
#define V_ULPTX_CMD(x)
Definition: t4_msg.h:2845
V_LSO_TCPHDR_LEN
#define V_LSO_TCPHDR_LEN(x)
Definition: t4_msg.h:1443
G_T6_RX_TNLHDR_LEN
#define G_T6_RX_TNLHDR_LEN(x)
Definition: t4_msg.h:2163
V_CPL_TX_TNL_LSO_TNLTYPE
#define V_CPL_TX_TNL_LSO_TNLTYPE(x)
Definition: t4_msg.h:3093
G_TID_COOKIE
#define G_TID_COOKIE(x)
Definition: t4_msg.h:342
V_LSO_ETHHDR_LEN
#define V_LSO_ETHHDR_LEN(x)
Definition: t4_msg.h:1453
V_CPL_TX_TNL_LSO_MSS
#define V_CPL_TX_TNL_LSO_MSS(x)
Definition: t4_msg.h:3151
F_LSO_FIRST_SLICE
#define F_LSO_FIRST_SLICE
Definition: t4_msg.h:1470
G_CPL_FW4_ACK_FLOWID
#define G_CPL_FW4_ACK_FLOWID(x)
Definition: t4_msg.h:2783
V_ULPTX_NSGE
#define V_ULPTX_NSGE(x)
Definition: t4_msg.h:2890
V_TXPKT_VF_VLD
#define V_TXPKT_VF_VLD(x)
Definition: t4_msg.h:1290
F_CPL_TX_TNL_LSO_IPV6
#define F_CPL_TX_TNL_LSO_IPV6
Definition: t4_msg.h:3115
V_TXPKT_OPCODE
#define V_TXPKT_OPCODE(x)
Definition: t4_msg.h:1339
V_CPL_TX_TNL_LSO_ETHHDRLENOUT
#define V_CPL_TX_TNL_LSO_ETHHDRLENOUT(x)
Definition: t4_msg.h:3035
FW_TYPE_RSSCPL
@ FW_TYPE_RSSCPL
Definition: t4_msg.h:2711
V_CPL_TX_TNL_LSO_IPHDRLENOUT
#define V_CPL_TX_TNL_LSO_IPHDRLENOUT(x)
Definition: t4_msg.h:3042
V_TXPKT_IPHDR_LEN
#define V_TXPKT_IPHDR_LEN(x)
Definition: t4_msg.h:1360
V_CPL_TX_TNL_LSO_TCPHDRLEN
#define V_CPL_TX_TNL_LSO_TCPHDRLEN(x)
Definition: t4_msg.h:3131
F_CPL_TX_TNL_LSO_UDPLENSETOUT
#define F_CPL_TX_TNL_LSO_UDPLENSETOUT
Definition: t4_msg.h:3081
V_TXPKT_INTF
#define V_TXPKT_INTF(x)
Definition: t4_msg.h:1306
V_LSO_IPHDR_LEN
#define V_LSO_IPHDR_LEN(x)
Definition: t4_msg.h:1448
F_LSO_IPV6
#define F_LSO_IPV6
Definition: t4_msg.h:1458
V_T6_TXPKT_ETHHDR_LEN
#define V_T6_TXPKT_ETHHDR_LEN(x)
Definition: t4_msg.h:1379
G_T6_COMPR_RXERR_VEC
#define G_T6_COMPR_RXERR_VEC(x)
Definition: t4_msg.h:2117
V_CPL_TX_TNL_LSO_OPCODE
#define V_CPL_TX_TNL_LSO_OPCODE(x)
Definition: t4_msg.h:3000
F_TXPKT_IPCSUM_DIS
#define F_TXPKT_IPCSUM_DIS
Definition: t4_msg.h:1403
F_RXF_IP6
#define F_RXF_IP6
Definition: t4_msg.h:2023
F_CPL_TX_TNL_LSO_IPIDINCOUT
#define F_CPL_TX_TNL_LSO_IPIDINCOUT
Definition: t4_msg.h:3065
ULP_TX_SC_DSGL
@ ULP_TX_SC_DSGL
Definition: t4_msg.h:2837
ULP_TX_SC_IMM
@ ULP_TX_SC_IMM
Definition: t4_msg.h:2836
G_AOPEN_ATID
#define G_AOPEN_ATID(x)
Definition: t4_msg.h:909
GET_TID
#define GET_TID(cmd)
Definition: t4_msg.h:330
t4_regs.h
V_HPZ2
#define V_HPZ2(x)
Definition: t4_regs.h:37108
V_TIMERVALUE0
#define V_TIMERVALUE0(x)
Definition: t4_regs.h:1705
M_WRTHRTHRESH
#define M_WRTHRTHRESH
Definition: t4_regs.h:25360
V_TIMERVALUE4
#define V_TIMERVALUE4(x)
Definition: t4_regs.h:1729
V_TIMERVALUE3
#define V_TIMERVALUE3(x)
Definition: t4_regs.h:1722
V_TIMERVALUE2
#define V_TIMERVALUE2(x)
Definition: t4_regs.h:1717
A_SGE_ITP_CONTROL
#define A_SGE_ITP_CONTROL
Definition: t4_regs.h:1680
M_RDTHRESHOLD
#define M_RDTHRESHOLD
Definition: t4_regs.h:25343
V_CIDXINC
#define V_CIDXINC(x)
Definition: t4_regs.h:564
V_TSCALE
#define V_TSCALE(x)
Definition: t4_regs.h:1698
M_TSCALE
#define M_TSCALE
Definition: t4_regs.h:1697
V_HOSTPAGESIZEPF3
#define V_HOSTPAGESIZEPF3(x)
Definition: t4_regs.h:666
V_THRESHOLD_0
#define V_THRESHOLD_0(x)
Definition: t4_regs.h:1483
A_SGE_CONTROL2
#define A_SGE_CONTROL2
Definition: t4_regs.h:2296
A_PL_PF_INT_CAUSE
#define A_PL_PF_INT_CAUSE
Definition: t4_regs.h:37871
V_HOSTPAGESIZEPF5
#define V_HOSTPAGESIZEPF5(x)
Definition: t4_regs.h:656
F_TDDPTAGTCB
#define F_TDDPTAGTCB
Definition: t4_regs.h:36864
F_PFSW
#define F_PFSW
Definition: t4_regs.h:37875
V_PIDX
#define V_PIDX(x)
Definition: t4_regs.h:528
M_PKTSHIFT
#define M_PKTSHIFT
Definition: t4_regs.h:619
V_HPZ1
#define V_HPZ1(x)
Definition: t4_regs.h:37113
V_HOSTPAGESIZEPF1
#define V_HOSTPAGESIZEPF1(x)
Definition: t4_regs.h:676
V_RDTHRESHOLD
#define V_RDTHRESHOLD(x)
Definition: t4_regs.h:25344
V_HPZ0
#define V_HPZ0(x)
Definition: t4_regs.h:37118
V_THRESHOLD_3
#define V_THRESHOLD_3(x)
Definition: t4_regs.h:1498
V_HPZ3
#define V_HPZ3(x)
Definition: t4_regs.h:37103
V_THRESHOLD_1
#define V_THRESHOLD_1(x)
Definition: t4_regs.h:1488
V_EGRSTATUSPAGESIZE
#define V_EGRSTATUSPAGESIZE(x)
Definition: t4_regs.h:603
M_TIMERVALUE0
#define M_TIMERVALUE0
Definition: t4_regs.h:1704
V_WRTHRTHRESH
#define V_WRTHRTHRESH(x)
Definition: t4_regs.h:25361
A_PCIE_PF_CLI
#define A_PCIE_PF_CLI
Definition: t4_regs.h:4070
A_SGE_FL_BUFFER_SIZE2
#define A_SGE_FL_BUFFER_SIZE2
Definition: t4_regs.h:1270
V_TIMERVALUE5
#define V_TIMERVALUE5(x)
Definition: t4_regs.h:1734
A_TP_PARA_REG5
#define A_TP_PARA_REG5
Definition: t4_regs.h:22545
A_SGE_INGRESS_RX_THRESHOLD
#define A_SGE_INGRESS_RX_THRESHOLD
Definition: t4_regs.h:1479
A_SGE_TIMER_VALUE_4_AND_5
#define A_SGE_TIMER_VALUE_4_AND_5
Definition: t4_regs.h:1725
V_INGRESSQID
#define V_INGRESSQID(x)
Definition: t4_regs.h:550
A_SGE_FL_BUFFER_SIZE0
#define A_SGE_FL_BUFFER_SIZE0
Definition: t4_regs.h:1251
V_INGPADBOUNDARY
#define V_INGPADBOUNDARY(x)
Definition: t4_regs.h:630
MYPF_REG
#define MYPF_REG(reg_addr)
Definition: t4_regs.h:39
F_ISCSITAGTCB
#define F_ISCSITAGTCB
Definition: t4_regs.h:36860
A_SGE_TIMER_VALUE_0_AND_1
#define A_SGE_TIMER_VALUE_0_AND_1
Definition: t4_regs.h:1701
F_RESETDDPOFFSET
#define F_RESETDDPOFFSET
Definition: t4_regs.h:22579
A_TP_CMM_CONFIG
#define A_TP_CMM_CONFIG
Definition: t4_regs.h:25335
A_ULP_RX_ISCSI_PSZ
#define A_ULP_RX_ISCSI_PSZ
Definition: t4_regs.h:37099
V_THRESHOLD_2
#define V_THRESHOLD_2(x)
Definition: t4_regs.h:1493
A_SGE_HOST_PAGE_SIZE
#define A_SGE_HOST_PAGE_SIZE
Definition: t4_regs.h:642
V_HOSTPAGESIZEPF0
#define V_HOSTPAGESIZEPF0(x)
Definition: t4_regs.h:681
V_INDICATESIZE
#define V_INDICATESIZE(x)
Definition: t4_regs.h:22549
F_EGRSTATUSPAGESIZE
#define F_EGRSTATUSPAGESIZE
Definition: t4_regs.h:604
M_INGPADBOUNDARY
#define M_INGPADBOUNDARY
Definition: t4_regs.h:629
M_INDICATESIZE
#define M_INDICATESIZE
Definition: t4_regs.h:22548
V_SEINTARM
#define V_SEINTARM(x)
Definition: t4_regs.h:559
V_QID
#define V_QID(x)
Definition: t4_regs.h:519
A_ULP_RX_TDDP_PSZ
#define A_ULP_RX_TDDP_PSZ
Definition: t4_regs.h:37142
V_TIMERVALUE1
#define V_TIMERVALUE1(x)
Definition: t4_regs.h:1710
V_HOSTPAGESIZEPF7
#define V_HOSTPAGESIZEPF7(x)
Definition: t4_regs.h:646
A_SGE_FL_BUFFER_SIZE15
#define A_SGE_FL_BUFFER_SIZE15
Definition: t4_regs.h:1361
V_HOSTPAGESIZEPF4
#define V_HOSTPAGESIZEPF4(x)
Definition: t4_regs.h:661
V_HOSTPAGESIZEPF6
#define V_HOSTPAGESIZEPF6(x)
Definition: t4_regs.h:651
V_INGPACKBOUNDARY
#define V_INGPACKBOUNDARY(x)
Definition: t4_regs.h:2312
V_PKTSHIFT
#define V_PKTSHIFT(x)
Definition: t4_regs.h:620
A_SGE_CONTROL
#define A_SGE_CONTROL
Definition: t4_regs.h:578
A_SGE_FL_BUFFER_SIZE1
#define A_SGE_FL_BUFFER_SIZE1
Definition: t4_regs.h:1263
V_HOSTPAGESIZEPF2
#define V_HOSTPAGESIZEPF2(x)
Definition: t4_regs.h:671
F_WRTHRTHRESHEN
#define F_WRTHRTHRESHEN
Definition: t4_regs.h:25357
A_ULP_RX_CTL
#define A_ULP_RX_CTL
Definition: t4_regs.h:36805
M_INGPACKBOUNDARY
#define M_INGPACKBOUNDARY
Definition: t4_regs.h:2311
A_SGE_TIMER_VALUE_2_AND_3
#define A_SGE_TIMER_VALUE_2_AND_3
Definition: t4_regs.h:1713
F_REARMDDPOFFSET
#define F_REARMDDPOFFSET
Definition: t4_regs.h:22575
F_RXPKTCPLMODE
#define F_RXPKTCPLMODE
Definition: t4_regs.h:600
t4_regs_values.h
X_FETCHBURSTMIN_64B_T6
#define X_FETCHBURSTMIN_64B_T6
Definition: t4_regs_values.h:152
X_FETCHBURSTMIN_64B
#define X_FETCHBURSTMIN_64B
Definition: t4_regs_values.h:147
X_RSPD_TYPE_INTR
#define X_RSPD_TYPE_INTR
Definition: t4_regs_values.h:206
X_INGPADBOUNDARY_SHIFT
#define X_INGPADBOUNDARY_SHIFT
Definition: t4_regs_values.h:97
X_RSPD_TYPE_FLBUF
#define X_RSPD_TYPE_FLBUF
Definition: t4_regs_values.h:204
X_HOSTFCMODE_NONE
#define X_HOSTFCMODE_NONE
Definition: t4_regs_values.h:161
X_HOSTFCMODE_STATUS_PAGE
#define X_HOSTFCMODE_STATUS_PAGE
Definition: t4_regs_values.h:163
X_CIDXFLUSHTHRESH_128
#define X_CIDXFLUSHTHRESH_128
Definition: t4_regs_values.h:176
X_FETCHBURSTMIN_128B
#define X_FETCHBURSTMIN_128B
Definition: t4_regs_values.h:148
X_FETCHBURSTMAX_256B
#define X_FETCHBURSTMAX_256B
Definition: t4_regs_values.h:158
X_UPDATEDELIVERY_INTERRUPT
#define X_UPDATEDELIVERY_INTERRUPT
Definition: t4_regs_values.h:189
X_TIMERREG_UPDATE_CIDX
#define X_TIMERREG_UPDATE_CIDX
Definition: t4_regs_values.h:137
X_FETCHBURSTMAX_512B
#define X_FETCHBURSTMAX_512B
Definition: t4_regs_values.h:159
X_RSPD_TYPE_CPL
#define X_RSPD_TYPE_CPL
Definition: t4_regs_values.h:205
X_T6_INGPADBOUNDARY_SHIFT
#define X_T6_INGPADBOUNDARY_SHIFT
Definition: t4_regs_values.h:87
tscale
static int tscale
Definition: t4_sge.c:190
add_fl_to_sfl
static void add_fl_to_sfl(struct adapter *, struct sge_fl *)
Definition: t4_sge.c:6284
t4_register_an_handler
void t4_register_an_handler(an_handler_t h)
Definition: t4_sge.c:357
init_iq
static void init_iq(struct sge_iq *, struct adapter *, int, int, int, int, int)
Definition: t4_sge.c:3367
lro_mbufs
static int lro_mbufs
Definition: t4_sge.c:204
pullups
static counter_u64_t pullups
Definition: t4_sge.c:208
set_tcb_rpl_handler
static int set_tcb_rpl_handler(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m)
Definition: t4_sge.c:399
MC_TLS
#define MC_TLS
Definition: t4_sge.c:93
alloc_txq
static int alloc_txq(struct vi_info *, struct sge_txq *, int)
Definition: t4_sge.c:4558
t4_register_shared_cpl_handler
void t4_register_shared_cpl_handler(int opcode, cpl_handler_t h, int cookie)
Definition: t4_sge.c:496
add_eq_sysctls
static void add_eq_sysctls(struct adapter *, struct sysctl_ctx_list *, struct sysctl_oid *, struct sge_eq *)
Definition: t4_sge.c:4386
needs_vxlan_tso
static bool needs_vxlan_tso(struct mbuf *m)
Definition: t4_sge.c:2440
skip_scatter_segment
static int skip_scatter_segment(struct adapter *sc, struct sge_fl *fl, int fr_offset, int remaining)
Definition: t4_sge.c:1846
abort_rpl_rss_handlers
cpl_handler_t abort_rpl_rss_handlers[NUM_CPL_COOKIES]
Definition: t4_sge.c:353
write_txpkts_vm_wr
static u_int write_txpkts_vm_wr(struct adapter *, struct sge_txq *)
Definition: t4_sge.c:5901
SYSCTL_INT
SYSCTL_INT(_hw_cxgbe, OID_AUTO, fl_pktshift, CTLFLAG_RDTUN, &fl_pktshift, 0, "payload DMA offset in rx buffer (bytes)")
cong_drop
static int cong_drop
Definition: t4_sge.c:128
add_rxq_sysctls
static void add_rxq_sysctls(struct sysctl_ctx_list *, struct sysctl_oid *, struct sge_rxq *)
Definition: t4_sge.c:4044
reclaim_tx_descs
static u_int reclaim_tx_descs(struct sge_txq *, u_int)
Definition: t4_sge.c:6169
eth_tx
static u_int eth_tx(struct mp_ring *r, u_int cidx, u_int pidx, bool *coalescing)
Definition: t4_sge.c:3155
t4_an_handler
an_handler_t t4_an_handler
Definition: t4_sge.c:347
txpkts1_len16
static u_int txpkts1_len16(void)
Definition: t4_sge.c:5204
handle_sge_egr_update
static int handle_sge_egr_update(struct sge_iq *, const struct rss_header *, struct mbuf *)
Definition: t4_sge.c:6321
handle_wrq_egr_update
static void handle_wrq_egr_update(struct adapter *sc, struct sge_eq *eq)
Definition: t4_sge.c:6298
record_eth_tx_time
static bool record_eth_tx_time(struct sge_txq *txq)
Definition: t4_sge.c:3135
t4_sge_modload
void t4_sge_modload(void)
Definition: t4_sge.c:533
alloc_rxq
static int alloc_rxq(struct vi_info *, struct sge_rxq *, int, int, int)
Definition: t4_sge.c:3942
MC_RAW_WR
#define MC_RAW_WR
Definition: t4_sge.c:92
write_lso_cpl
static void * write_lso_cpl(void *cpl, struct mbuf *m0)
Definition: t4_sge.c:5298
get_scatter_segment
static struct mbuf * get_scatter_segment(struct adapter *sc, struct sge_fl *fl, int fr_offset, int remaining)
Definition: t4_sge.c:1720
qsize_to_fthresh
static u_int qsize_to_fthresh(int qsize)
Definition: t4_sge.c:4199
t4_tx_coalesce
static int t4_tx_coalesce
Definition: t4_sge.c:216
VM_TX_L2HDR_LEN
#define VM_TX_L2HDR_LEN
Definition: t4_sge.c:5384
init_fl
static void init_fl(struct adapter *, struct sge_fl *, int, int, char *)
Definition: t4_sge.c:3394
act_open_rpl_handlers
cpl_handler_t act_open_rpl_handlers[NUM_CPL_COOKIES]
Definition: t4_sge.c:352
count_mbuf_nsegs
static int count_mbuf_nsegs(struct mbuf *m, int skip, uint8_t *cflags)
Definition: t4_sge.c:2611
free_iq_fl_hwq
static int free_iq_fl_hwq(struct adapter *, struct sge_iq *, struct sge_fl *)
Definition: t4_sge.c:3717
ring_fl_db
static void ring_fl_db(struct adapter *, struct sge_fl *)
Definition: t4_sge.c:4903
MC_NOMAP
#define MC_NOMAP
Definition: t4_sge.c:91
alloc_iq_fl
static int alloc_iq_fl(struct vi_info *, struct sge_iq *, struct sge_fl *, struct sysctl_ctx_list *, struct sysctl_oid *)
Definition: t4_sge.c:3487
lro_entries
static int lro_entries
Definition: t4_sge.c:197
add_to_txpkts_pf
static int add_to_txpkts_pf(struct adapter *, struct sge_txq *, struct mbuf *, int, bool *)
Definition: t4_sge.c:5713
t4_create_dma_tag
int t4_create_dma_tag(struct adapter *sc)
Definition: t4_sge.c:977
t4_intr
void t4_intr(void *arg)
Definition: t4_sge.c:1339
buffer_packing
static int buffer_packing
Definition: t4_sge.c:138
spg_len
static int spg_len
Definition: t4_sge.c:118
bufidx_used
static bool bufidx_used(struct adapter *sc, int idx)
Definition: t4_sge.c:6431
alloc_wr_mbuf
struct mbuf * alloc_wr_mbuf(int len, int how)
Definition: t4_sge.c:2386
handle_fw_msg
static int handle_fw_msg(struct sge_iq *, const struct rss_header *, struct mbuf *)
Definition: t4_sge.c:6347
max_nsegs_allowed
static int max_nsegs_allowed(struct mbuf *m, bool vm_wr)
Definition: t4_sge.c:2654
t4_tweak_chip_settings
void t4_tweak_chip_settings(struct adapter *sc)
Definition: t4_sge.c:712
free_wrq
static void free_wrq(struct adapter *, struct sge_wrq *)
Definition: t4_sge.c:4527
add_fl_sysctls
static void add_fl_sysctls(struct adapter *, struct sysctl_ctx_list *, struct sysctl_oid *, struct sge_fl *)
Definition: t4_sge.c:3756
commit_wrq_wr
void commit_wrq_wr(struct sge_wrq *wrq, void *w, struct wrq_cookie *cookie)
Definition: t4_sge.c:2994
free_fl_buffers
void free_fl_buffers(struct adapter *sc, struct sge_fl *fl)
Definition: t4_sge.c:5066
read_hw_cidx
static uint16_t read_hw_cidx(struct sge_eq *)
Definition: t4_sge.c:6157
move_to_next_rxbuf
static void move_to_next_rxbuf(struct sge_fl *fl)
Definition: t4_sge.c:1526
discard_tx
static int discard_tx(struct sge_eq *eq)
Definition: t4_sge.c:3089
free_eq
static void free_eq(struct adapter *, struct sge_eq *)
Definition: t4_sge.c:4374
calculate_mbuf_len16
static void calculate_mbuf_len16(struct mbuf *, bool)
Definition: t4_sge.c:5158
wr_can_update_eq
static int wr_can_update_eq(void *p)
Definition: t4_sge.c:3096
service_iq_fl
static int service_iq_fl(struct sge_iq *, int)
Definition: t4_sge.c:1543
mbuf_nsegs
static int mbuf_nsegs(struct mbuf *m)
Definition: t4_sge.c:2256
mbuf_len16
static int mbuf_len16(struct mbuf *m)
Definition: t4_sge.c:2291
t4_setup_vi_queues
int t4_setup_vi_queues(struct vi_info *vi)
Definition: t4_sge.c:1107
add_to_txpkts_vf
static int add_to_txpkts_vf(struct adapter *, struct sge_txq *, struct mbuf *, int, bool *)
Definition: t4_sge.c:5661
tx_reclaim
static void tx_reclaim(void *, int)
Definition: t4_sge.c:6209
eth_rx
static int eth_rx(struct adapter *, struct sge_rxq *, const struct iq_desc *, u_int)
Definition: t4_sge.c:1906
parse_pkt
int parse_pkt(struct mbuf **mp, bool vm_wr)
Definition: t4_sge.c:2682
needs_hwcsum
static bool needs_hwcsum(struct mbuf *m)
Definition: t4_sge.c:2406
can_resume_eth_tx
static u_int can_resume_eth_tx(struct mp_ring *r)
Definition: t4_sge.c:3073
free_iq_fl
static void free_iq_fl(struct adapter *, struct sge_iq *, struct sge_fl *)
Definition: t4_sge.c:3530
l2t_write_rpl_handlers
cpl_handler_t l2t_write_rpl_handlers[NUM_CPL_COOKIES]
Definition: t4_sge.c:351
free_eq_hwq
static int free_eq_hwq(struct adapter *, struct vi_info *, struct sge_eq *)
needs_vxlan_csum
static bool needs_vxlan_csum(struct mbuf *m)
Definition: t4_sge.c:2431
set_txupdate_flags
static void set_txupdate_flags(struct sge_txq *txq, u_int avail, struct fw_eth_tx_pkt_wr *wr)
Definition: t4_sge.c:3114
t4_verify_chip_settings
int t4_verify_chip_settings(struct adapter *sc)
Definition: t4_sge.c:920
t4_sge_modunload
void t4_sge_modunload(void)
Definition: t4_sge.c:613
init_eq
static void init_eq(struct adapter *, struct sge_eq *, int, int, uint8_t, struct sge_iq *, char *)
Definition: t4_sge.c:3420
get_segment_len
static int get_segment_len(struct adapter *sc, struct sge_fl *fl, int plen)
Definition: t4_sge.c:1891
t4_register_fw_msg_handler
void t4_register_fw_msg_handler(int type, fw_msg_handler_t h)
Definition: t4_sge.c:368
t4_sge_sysctls
void t4_sge_sysctls(struct adapter *sc, struct sysctl_ctx_list *ctx, struct sysctl_oid_list *children)
Definition: t4_sge.c:994
t4_cpl_handler
cpl_handler_t t4_cpl_handler[NUM_CPL_CMDS]
Definition: t4_sge.c:349
service_iq
static int service_iq(struct sge_iq *, int)
Definition: t4_sge.c:1384
refill_sfl
static void refill_sfl(void *)
Definition: t4_sge.c:5040
t4_init_shared_cpl_handlers
static void t4_init_shared_cpl_handlers(void)
Definition: t4_sge.c:485
free_rxq
static void free_rxq(struct vi_info *, struct sge_rxq *)
Definition: t4_sge.c:4024
reclaimable_tx_desc
static u_int reclaimable_tx_desc(struct sge_eq *)
Definition: t4_sge.c:6134
t4_intr_evt
void t4_intr_evt(void *arg)
Definition: t4_sge.c:1325
needs_outer_tcp_csum
static bool needs_outer_tcp_csum(struct mbuf *m)
Definition: t4_sge.c:2475
ring_eq_db
static void ring_eq_db(struct adapter *, struct sge_eq *, u_int)
Definition: t4_sge.c:6080
t4_tx_coalesce_gap
static int t4_tx_coalesce_gap
Definition: t4_sge.c:229
save_l2hdr
static void save_l2hdr(struct txpkts *txp, struct mbuf *m)
Definition: t4_sge.c:5653
write_txpkt_wr
static u_int write_txpkt_wr(struct adapter *, struct sge_txq *, struct mbuf *, u_int)
Definition: t4_sge.c:5525
__FBSDID
__FBSDID("$FreeBSD$")
cl_metadata
static struct cluster_metadata * cl_metadata(struct fl_sdesc *sd)
Definition: t4_sge.c:1697
free_ring
int free_ring(struct adapter *sc, bus_dma_tag_t tag, bus_dmamap_t map, bus_addr_t pa, void *va)
Definition: t4_sge.c:3467
total_available_tx_desc
static u_int total_available_tx_desc(struct sge_eq *)
Definition: t4_sge.c:6143
needs_vlan_insertion
static bool needs_vlan_insertion(struct mbuf *m)
Definition: t4_sge.c:2509
t4_teardown_vi_queues
int t4_teardown_vi_queues(struct vi_info *vi)
Definition: t4_sge.c:1215
count_mbuf_ext_pgs
static int count_mbuf_ext_pgs(struct mbuf *m, int skip, vm_paddr_t *nextaddr)
Definition: t4_sge.c:2545
alloc_ctrlq
static int alloc_ctrlq(struct adapter *, int)
Definition: t4_sge.c:3861
get_pkt_gl
static void get_pkt_gl(struct mbuf *, struct sglist *)
Definition: t4_sge.c:5097
free_ctrlq
static void free_ctrlq(struct adapter *, int)
Definition: t4_sge.c:3909
txpkt_len16
static u_int txpkt_len16(u_int, const u_int)
Definition: t4_sge.c:5124
write_raw_wr
static u_int write_raw_wr(struct sge_txq *, void *, struct mbuf *, u_int)
Definition: t4_sge.c:5492
set_mbuf_len16
static void set_mbuf_len16(struct mbuf *m, uint8_t len16)
Definition: t4_sge.c:2304
t4_handle_wrerr_rpl
static int t4_handle_wrerr_rpl(struct adapter *, const __be64 *)
Definition: t4_sge.c:6371
sysctl_bufsizes
static int sysctl_bufsizes(SYSCTL_HANDLER_ARGS)
Definition: t4_sge.c:6447
add_txq_sysctls
static void add_txq_sysctls(struct vi_info *, struct sysctl_ctx_list *, struct sysctl_oid *, struct sge_txq *)
Definition: t4_sge.c:4676
alloc_eq
static int alloc_eq(struct adapter *, struct sge_eq *, struct sysctl_ctx_list *, struct sysctl_oid *)
Definition: t4_sge.c:4351
fl_pktshift
static int fl_pktshift
Definition: t4_sge.c:99
fw4_ack_handler
static int fw4_ack_handler(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m)
Definition: t4_sge.c:468
needs_l3_csum
static bool needs_l3_csum(struct mbuf *m)
Definition: t4_sge.c:2464
get_fl_payload
static struct mbuf * get_fl_payload(struct adapter *, struct sge_fl *, uint32_t)
free_txq
static void free_txq(struct vi_info *, struct sge_txq *)
Definition: t4_sge.c:4653
safest_rx_cluster
static int safest_rx_cluster
Definition: t4_sge.c:163
CTASSERT
CTASSERT(offsetof(struct cpl_fw4_msg, data)==offsetof(struct cpl_fw6_msg, data))
wrq_tx_drain
static void wrq_tx_drain(void *, int)
Definition: t4_sge.c:2108
eth_eq_alloc
static int eth_eq_alloc(struct adapter *, struct vi_info *, struct sge_eq *)
Definition: t4_sge.c:4259
imm_payload
static u_int imm_payload(u_int ndesc)
Definition: t4_sge.c:5214
t4_setup_adapter_queues
int t4_setup_adapter_queues(struct adapter *sc)
Definition: t4_sge.c:1036
txpkts0_len16
static u_int txpkts0_len16(u_int)
Definition: t4_sge.c:5185
RX_COPY_THRESHOLD
#define RX_COPY_THRESHOLD
Definition: t4_sge.c:87
fl_pad
int fl_pad
Definition: t4_sge.c:109
write_gl_to_txd
static void write_gl_to_txd(struct sge_txq *, struct mbuf *, caddr_t *, int)
Definition: t4_sge.c:5985
skip_fl_payload
static void skip_fl_payload(struct adapter *sc, struct sge_fl *fl, int plen)
Definition: t4_sge.c:1877
tnl_cong
int tnl_cong(struct port_info *pi, int drop)
Definition: t4_sge.c:3927
add_wrq_sysctls
static void add_wrq_sysctls(struct sysctl_ctx_list *, struct sysctl_oid *, struct sge_wrq *)
Definition: t4_sge.c:4537
t4_destroy_dma_tag
int t4_destroy_dma_tag(struct adapter *sc)
Definition: t4_sge.c:1020
defrags
static counter_u64_t defrags
Definition: t4_sge.c:212
abort_rpl_rss_handler
static int abort_rpl_rss_handler(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m)
Definition: t4_sge.c:452
csum_to_ctrl
static uint64_t csum_to_ctrl(struct adapter *sc, struct mbuf *m)
Definition: t4_sge.c:5225
txerr_ratecheck
static struct timeval txerr_ratecheck
Definition: t4_sge.c:2673
write_txpkt_vm_wr
static u_int write_txpkt_vm_wr(struct adapter *, struct sge_txq *, struct mbuf *)
Definition: t4_sge.c:5394
t4_fw_msg_handler
fw_msg_handler_t t4_fw_msg_handler[NUM_FW6_TYPES]
Definition: t4_sge.c:348
refill_fl
static int refill_fl(struct adapter *, struct sge_fl *, int)
Definition: t4_sge.c:4927
write_tnl_lso_cpl
static void * write_tnl_lso_cpl(void *cpl, struct mbuf *m0)
Definition: t4_sge.c:5327
SYSCTL_COUNTER_U64
SYSCTL_COUNTER_U64(_hw_cxgbe, OID_AUTO, pullups, CTLFLAG_RD, &pullups, "Number of mbuf pullups performed")
txerr_interval
static const struct timeval txerr_interval
Definition: t4_sge.c:2674
write_txpkts_wr
static u_int write_txpkts_wr(struct adapter *, struct sge_txq *)
Definition: t4_sge.c:5790
hwsz_ok
static int hwsz_ok(struct adapter *sc, int hwsz)
Definition: t4_sge.c:842
ctrl_eq_alloc
static int ctrl_eq_alloc(struct adapter *, struct sge_eq *)
Definition: t4_sge.c:4213
fw4_ack_handlers
cpl_handler_t fw4_ack_handlers[NUM_CPL_COOKIES]
Definition: t4_sge.c:354
t4_tx_coalesce_pkts
static int t4_tx_coalesce_pkts
Definition: t4_sge.c:225
copy_to_txd
static void copy_to_txd(struct sge_eq *, caddr_t, caddr_t *, int)
Definition: t4_sge.c:6058
alloc_fwq
static int alloc_fwq(struct adapter *)
Definition: t4_sge.c:3800
t4_init_rx_buf_info
void t4_init_rx_buf_info(struct adapter *sc)
Definition: t4_sge.c:854
t4_sge_extfree_refs
uint64_t t4_sge_extfree_refs(void)
Definition: t4_sge.c:623
cmp_l2hdr
static bool cmp_l2hdr(struct txpkts *txp, struct mbuf *m)
Definition: t4_sge.c:5637
alloc_iq_fl_hwq
static int alloc_iq_fl_hwq(struct vi_info *, struct sge_iq *, struct sge_fl *)
Definition: t4_sge.c:3554
fl_pack
static int fl_pack
Definition: t4_sge.c:148
alloc_wrq
static int alloc_wrq(struct adapter *, struct vi_info *, struct sge_wrq *, struct sysctl_ctx_list *, struct sysctl_oid *)
Definition: t4_sge.c:4501
extfree_refs
static counter_u64_t extfree_refs
Definition: t4_sge.c:344
rxb_free
static void rxb_free(struct mbuf *m)
Definition: t4_sge.c:1704
cannot_use_txpkts
static bool cannot_use_txpkts(struct mbuf *m)
Definition: t4_sge.c:3081
t4_intr_err
void t4_intr_err(void *arg)
Definition: t4_sge.c:1301
free_fwq
static void free_fwq(struct adapter *)
Definition: t4_sge.c:3840
find_refill_source
static int find_refill_source(struct adapter *, int, bool)
Definition: t4_sge.c:6249
t4_intr_all
void t4_intr_all(void *arg)
Definition: t4_sge.c:1282
start_wrq_wr
void * start_wrq_wr(struct sge_wrq *wrq, int len16, struct wrq_cookie *cookie)
Definition: t4_sge.c:2940
max_rx_payload
static int max_rx_payload(struct adapter *sc, struct ifnet *ifp, const bool ofld)
Definition: t4_sge.c:1093
last_flit_to_ns
static uint64_t last_flit_to_ns(struct adapter *sc, uint64_t lf)
Definition: t4_sge.c:1515
set_tcb_rpl_handlers
cpl_handler_t set_tcb_rpl_handlers[NUM_CPL_COOKIES]
Definition: t4_sge.c:350
l2t_write_rpl_handler
static int l2t_write_rpl_handler(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m)
Definition: t4_sge.c:426
get_flit
static __be64 get_flit(struct sglist_seg *, int, int)
Definition: t4_sge.c:6225
extfree_rels
static counter_u64_t extfree_rels
Definition: t4_sge.c:345
setup_pad_and_pack_boundaries
static void setup_pad_and_pack_boundaries(struct adapter *sc)
Definition: t4_sge.c:637
needs_tso
static bool needs_tso(struct mbuf *m)
Definition: t4_sge.c:2420
mbuf_cflags
static int mbuf_cflags(struct mbuf *m)
Definition: t4_sge.c:2275
alloc_ring
int alloc_ring(struct adapter *sc, size_t len, bus_dma_tag_t *tag, bus_dmamap_t *map, bus_addr_t *pa, void **va)
Definition: t4_sge.c:3435
MAX_PACK_BOUNDARY
#define MAX_PACK_BOUNDARY
Definition: t4_sge.c:634
set_mbuf_cflags
static void set_mbuf_cflags(struct mbuf *m, uint8_t flags)
Definition: t4_sge.c:2283
set_mbuf_nsegs
static void set_mbuf_nsegs(struct mbuf *m, uint8_t nsegs)
Definition: t4_sge.c:2267
t4_update_fl_bufsize
void t4_update_fl_bufsize(struct ifnet *ifp)
Definition: t4_sge.c:2222
t4_register_cpl_handler
void t4_register_cpl_handler(int opcode, cpl_handler_t h)
Definition: t4_sge.c:387
txpkt_vm_len16
static u_int txpkt_vm_len16(u_int, const u_int)
Definition: t4_sge.c:5143
oneseg_dma_callback
static void oneseg_dma_callback(void *, bus_dma_segment_t *, int, int)
Definition: t4_sge.c:4892
act_open_rpl_handler
static int act_open_rpl_handler(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m)
Definition: t4_sge.c:439
drain_wrq_wr_list
static void drain_wrq_wr_list(struct adapter *, struct sge_wrq *)
Definition: t4_sge.c:2120
alloc_eq_hwq
static int alloc_eq_hwq(struct adapter *, struct vi_info *, struct sge_eq *)
Definition: t4_sge.c:4409
t4_teardown_adapter_queues
int t4_teardown_adapter_queues(struct adapter *sc)
Definition: t4_sge.c:1075
t4_wrq_tx_locked
void t4_wrq_tx_locked(struct adapter *sc, struct sge_wrq *wrq, struct wrqe *wr)
Definition: t4_sge.c:2197
largest_rx_cluster
static int largest_rx_cluster
Definition: t4_sge.c:155
add_iq_sysctls
static void add_iq_sysctls(struct sysctl_ctx_list *, struct sysctl_oid *, struct sge_iq *)
Definition: t4_sge.c:3734
handle_eth_egr_update
static void handle_eth_egr_update(struct adapter *sc, struct sge_eq *eq)
Definition: t4_sge.c:6307
V_FW_EQ_CTRL_CMD_CMPLIQID
#define V_FW_EQ_CTRL_CMD_CMPLIQID(x)
Definition: t4fw_interface.h:6197
V_FW_ETH_TX_EO_WR_TSOFF
#define V_FW_ETH_TX_EO_WR_TSOFF(x)
Definition: t4fw_interface.h:875
V_FW_EQ_CTRL_CMD_FBMIN
#define V_FW_EQ_CTRL_CMD_FBMIN(x)
Definition: t4fw_interface.h:6295
FW_ETH_TX_EO_TYPE_TCPSEG
@ FW_ETH_TX_EO_TYPE_TCPSEG
Definition: t4fw_interface.h:794
FW_ETH_TX_EO_TYPE_UDPSEG
@ FW_ETH_TX_EO_TYPE_UDPSEG
Definition: t4fw_interface.h:793
G_FW_ERROR_CMD_MV
#define G_FW_ERROR_CMD_MV(x)
Definition: t4fw_interface.h:9630
FW_IQ_TYPE_FL_INT_CAP
@ FW_IQ_TYPE_FL_INT_CAP
Definition: t4fw_interface.h:5202
V_FW_EQ_ETH_CMD_PFN
#define V_FW_EQ_ETH_CMD_PFN(x)
Definition: t4fw_interface.h:5937
G_FW_EQ_OFLD_CMD_EQID
#define G_FW_EQ_OFLD_CMD_EQID(x)
Definition: t4fw_interface.h:6385
V_FW_EQ_ETH_CMD_FBMIN
#define V_FW_EQ_ETH_CMD_FBMIN(x)
Definition: t4fw_interface.h:6076
V_FW_EQ_OFLD_CMD_HOSTFCMODE
#define V_FW_EQ_OFLD_CMD_HOSTFCMODE(x)
Definition: t4fw_interface.h:6431
F_FW_IQ_CMD_IQSTART
#define F_FW_IQ_CMD_IQSTART
Definition: t4fw_interface.h:5272
G_FW_WR_OP
#define G_FW_WR_OP(x)
Definition: t4fw_interface.h:170
V_FW_ETH_TX_PKT_WR_IMMDLEN
#define V_FW_ETH_TX_PKT_WR_IMMDLEN(x)
Definition: t4fw_interface.h:709
V_FW_EQ_ETH_CMD_PCIECHN
#define V_FW_EQ_ETH_CMD_PCIECHN(x)
Definition: t4fw_interface.h:6051
V_FW_EQ_CTRL_CMD_PCIECHN
#define V_FW_EQ_CTRL_CMD_PCIECHN(x)
Definition: t4fw_interface.h:6270
F_FW_IQ_CMD_FL0FETCHRO
#define F_FW_IQ_CMD_FL0FETCHRO
Definition: t4fw_interface.h:5506
S_FW_VIID_PFN
#define S_FW_VIID_PFN
Definition: t4fw_interface.h:6510
F_FW_EQ_ETH_CMD_AUTOEQUIQE
#define F_FW_EQ_ETH_CMD_AUTOEQUIQE
Definition: t4fw_interface.h:6110
F_FW_IQ_CMD_FL0DATARO
#define F_FW_IQ_CMD_FL0DATARO
Definition: t4fw_interface.h:5464
V_FW_EQ_ETH_CMD_VFN
#define V_FW_EQ_ETH_CMD_VFN(x)
Definition: t4fw_interface.h:5943
F_FW_EQ_OFLD_CMD_ALLOC
#define F_FW_EQ_OFLD_CMD_ALLOC
Definition: t4fw_interface.h:6352
F_FW_WR_EQUIQ
#define F_FW_WR_EQUIQ
Definition: t4fw_interface.h:215
V_FW_IQ_CMD_IQPCIECH
#define V_FW_IQ_CMD_IQPCIECH(x)
Definition: t4fw_interface.h:5342
G_FW_ERROR_CMD_FATAL
#define G_FW_ERROR_CMD_FATAL(x)
Definition: t4fw_interface.h:9593
FW_CAPS_CONFIG_TLSKEYS
@ FW_CAPS_CONFIG_TLSKEYS
Definition: t4fw_interface.h:4691
V_FW_EQ_OFLD_CMD_EQSIZE
#define V_FW_EQ_OFLD_CMD_EQSIZE(x)
Definition: t4fw_interface.h:6502
V_FW_EQ_CTRL_CMD_HOSTFCMODE
#define V_FW_EQ_CTRL_CMD_HOSTFCMODE(x)
Definition: t4fw_interface.h:6250
F_FW_WR_EQUEQ
#define F_FW_WR_EQUEQ
Definition: t4fw_interface.h:223
F_FW_IQ_CMD_FL0CONGEN
#define F_FW_IQ_CMD_FL0CONGEN
Definition: t4fw_interface.h:5540
V_FW_WR_OP
#define V_FW_WR_OP(x)
Definition: t4fw_interface.h:169
V_FW_EQ_OFLD_CMD_VFN
#define V_FW_EQ_OFLD_CMD_VFN(x)
Definition: t4fw_interface.h:6343
FW_IQ_CMD
@ FW_IQ_CMD
Definition: t4fw_interface.h:4130
FW_EQ_OFLD_CMD
@ FW_EQ_OFLD_CMD
Definition: t4fw_interface.h:4134
FW_ERROR_CMD
@ FW_ERROR_CMD
Definition: t4fw_interface.h:4167
FW_EQ_CTRL_CMD
@ FW_EQ_CTRL_CMD
Definition: t4fw_interface.h:4133
FW_EQ_ETH_CMD
@ FW_EQ_ETH_CMD
Definition: t4fw_interface.h:4132
F_FW_EQ_CTRL_CMD_ALLOC
#define F_FW_EQ_CTRL_CMD_ALLOC
Definition: t4fw_interface.h:6165
F_FW_EQ_CTRL_CMD_EQSTART
#define F_FW_EQ_CTRL_CMD_EQSTART
Definition: t4fw_interface.h:6186
V_FW_IQ_CMD_IQINTCNTTHRESH
#define V_FW_IQ_CMD_IQINTCNTTHRESH(x)
Definition: t4fw_interface.h:5361
F_FW_IQ_CMD_ALLOC
#define F_FW_IQ_CMD_ALLOC
Definition: t4fw_interface.h:5251
V_FW_WR_LEN16
#define V_FW_WR_LEN16(x)
Definition: t4fw_interface.h:236
V_FW_FLOWC_WR_NPARAMS
#define V_FW_FLOWC_WR_NPARAMS(x)
Definition: t4fw_interface.h:1058
V_FW_IQ_CMD_VIID
#define V_FW_IQ_CMD_VIID(x)
Definition: t4fw_interface.h:5296
V_FW_IQ_CMD_FL0HOSTFCMODE
#define V_FW_IQ_CMD_FL0HOSTFCMODE(x)
Definition: t4fw_interface.h:5510
FW_ERROR_TYPE_HWMODULE
@ FW_ERROR_TYPE_HWMODULE
Definition: t4fw_interface.h:9382
FW_ERROR_TYPE_WR
@ FW_ERROR_TYPE_WR
Definition: t4fw_interface.h:9383
FW_ERROR_TYPE_EXCEPTION
@ FW_ERROR_TYPE_EXCEPTION
Definition: t4fw_interface.h:9381
FW_ERROR_TYPE_ACL
@ FW_ERROR_TYPE_ACL
Definition: t4fw_interface.h:9384
V_FW_EQ_OFLD_CMD_PCIECHN
#define V_FW_EQ_OFLD_CMD_PCIECHN(x)
Definition: t4fw_interface.h:6451
V_FW_PARAMS_PARAM_YZ
#define V_FW_PARAMS_PARAM_YZ(x)
Definition: t4fw_interface.h:5035
G_FW_ERROR_CMD_TYPE
#define G_FW_ERROR_CMD_TYPE(x)
Definition: t4fw_interface.h:9600
F_FW_EQ_ETH_CMD_AUTOEQUEQE
#define F_FW_EQ_ETH_CMD_AUTOEQUEQE
Definition: t4fw_interface.h:6117
G_FW_ERROR_CMD_PFN
#define G_FW_ERROR_CMD_PFN(x)
Definition: t4fw_interface.h:9618
G_FW_ERROR_CMD_VFN
#define G_FW_ERROR_CMD_VFN(x)
Definition: t4fw_interface.h:9624
V_FW_WR_COMPL
#define V_FW_WR_COMPL(x)
Definition: t4fw_interface.h:195
V_FW_IQ_CMD_FL0CNGCHMAP
#define V_FW_IQ_CMD_FL0CNGCHMAP(x)
Definition: t4fw_interface.h:5427
V_FW_EQ_CTRL_CMD_FBMAX
#define V_FW_EQ_CTRL_CMD_FBMAX(x)
Definition: t4fw_interface.h:6301
F_FW_CMD_WRITE
#define F_FW_CMD_WRITE
Definition: t4fw_interface.h:4209
F_FW_IQ_CMD_FL0CONGCIF
#define F_FW_IQ_CMD_FL0CONGCIF
Definition: t4fw_interface.h:5471
F_FW_IQ_CMD_FL0PADEN
#define F_FW_IQ_CMD_FL0PADEN
Definition: t4fw_interface.h:5526
F_FW_IQ_CMD_IQASYNCH
#define F_FW_IQ_CMD_IQASYNCH
Definition: t4fw_interface.h:5292
F_FW_EQ_ETH_CMD_EQSTART
#define F_FW_EQ_ETH_CMD_EQSTART
Definition: t4fw_interface.h:5973
V_FW_EQ_ETH_CMD_HOSTFCMODE
#define V_FW_EQ_ETH_CMD_HOSTFCMODE(x)
Definition: t4fw_interface.h:6031
V_FW_IQ_CMD_IQANDSTINDEX
#define V_FW_IQ_CMD_IQANDSTINDEX(x)
Definition: t4fw_interface.h:5322
V_FW_IQ_CMD_PFN
#define V_FW_IQ_CMD_PFN(x)
Definition: t4fw_interface.h:5236
V_FW_EQ_ETH_CMD_EQSIZE
#define V_FW_EQ_ETH_CMD_EQSIZE(x)
Definition: t4fw_interface.h:6101
V_FW_EQ_OFLD_CMD_CIDXFTHRESH
#define V_FW_EQ_OFLD_CMD_CIDXFTHRESH(x)
Definition: t4fw_interface.h:6496
V_FW_EQ_CTRL_CMD_EQSIZE
#define V_FW_EQ_CTRL_CMD_EQSIZE(x)
Definition: t4fw_interface.h:6321
F_FW_IQ_CMD_FL0PACKEN
#define F_FW_IQ_CMD_FL0PACKEN
Definition: t4fw_interface.h:5533
FW_FLOWC_MNEM_EOSTATE
@ FW_FLOWC_MNEM_EOSTATE
Definition: t4fw_interface.h:1033
FW_FLOWC_MNEM_PFNVFN
@ FW_FLOWC_MNEM_PFNVFN
Definition: t4fw_interface.h:1023
FW_FLOWC_MNEM_CH
@ FW_FLOWC_MNEM_CH
Definition: t4fw_interface.h:1024
FW_FLOWC_MNEM_SCHEDCLASS
@ FW_FLOWC_MNEM_SCHEDCLASS
Definition: t4fw_interface.h:1034
FW_FLOWC_MNEM_IQID
@ FW_FLOWC_MNEM_IQID
Definition: t4fw_interface.h:1026
FW_FLOWC_MNEM_PORT
@ FW_FLOWC_MNEM_PORT
Definition: t4fw_interface.h:1025
V_FW_IQ_CMD_TYPE
#define V_FW_IQ_CMD_TYPE(x)
Definition: t4fw_interface.h:5283
F_FW_EQ_CTRL_CMD_FETCHRO
#define F_FW_EQ_CTRL_CMD_FETCHRO
Definition: t4fw_interface.h:6246
V_FW_ETH_TX_EO_WR_IMMDLEN
#define V_FW_ETH_TX_EO_WR_IMMDLEN(x)
Definition: t4fw_interface.h:863
V_FW_PARAMS_MNEM
#define V_FW_PARAMS_MNEM(x)
Definition: t4fw_interface.h:5005
F_FW_IQ_CMD_IQANDST
#define F_FW_IQ_CMD_IQANDST
Definition: t4fw_interface.h:5305
G_FW_EQ_CTRL_CMD_EQID
#define G_FW_EQ_CTRL_CMD_EQID(x)
Definition: t4fw_interface.h:6204
V_FW_EQ_CTRL_CMD_IQID
#define V_FW_EQ_CTRL_CMD_IQID(x)
Definition: t4fw_interface.h:6276
V_FW_ETH_TX_EO_WR_TSCLK
#define V_FW_ETH_TX_EO_WR_TSCLK(x)
Definition: t4fw_interface.h:869
V_FW_WR_FLOWID
#define V_FW_WR_FLOWID(x)
Definition: t4fw_interface.h:229
V_FW_CMD_OP
#define V_FW_CMD_OP(x)
Definition: t4fw_interface.h:4190
V_FW_EQ_ETH_CMD_FBMAX
#define V_FW_EQ_ETH_CMD_FBMAX(x)
Definition: t4fw_interface.h:6082
V_FW_IQ_CMD_IQESIZE
#define V_FW_IQ_CMD_IQESIZE(x)
Definition: t4fw_interface.h:5381
V_FW_EQ_OFLD_CMD_FBMIN
#define V_FW_EQ_OFLD_CMD_FBMIN(x)
Definition: t4fw_interface.h:6476
FW_FLOWC_MNEM_EOSTATE_ESTABLISHED
@ FW_FLOWC_MNEM_EOSTATE_ESTABLISHED
Definition: t4fw_interface.h:1013
FW_ULPTX_WR
@ FW_ULPTX_WR
Definition: t4fw_interface.h:100
FW_ETH_TX_PKTS_WR
@ FW_ETH_TX_PKTS_WR
Definition: t4fw_interface.h:104
FW_ETH_TX_PKTS2_WR
@ FW_ETH_TX_PKTS2_WR
Definition: t4fw_interface.h:105
FW_ETH_TX_PKTS_VM_WR
@ FW_ETH_TX_PKTS_VM_WR
Definition: t4fw_interface.h:113
FW_ETH_TX_PKT_WR
@ FW_ETH_TX_PKT_WR
Definition: t4fw_interface.h:102
FW_FLOWC_WR
@ FW_FLOWC_WR
Definition: t4fw_interface.h:109
FW_ETH_TX_EO_WR
@ FW_ETH_TX_EO_WR
Definition: t4fw_interface.h:106
FW_ETH_TX_PKT_VM_WR
@ FW_ETH_TX_PKT_VM_WR
Definition: t4fw_interface.h:112
V_FW_EQ_ETH_CMD_VIID
#define V_FW_EQ_ETH_CMD_VIID(x)
Definition: t4fw_interface.h:6121
F_FW_IQ_CMD_IQGTSMODE
#define F_FW_IQ_CMD_IQGTSMODE
Definition: t4fw_interface.h:5338
F_FW_IQ_CMD_IQFLINTCONGEN
#define F_FW_IQ_CMD_IQFLINTCONGEN
Definition: t4fw_interface.h:5410
F_FW_EQ_OFLD_CMD_EQSTART
#define F_FW_EQ_OFLD_CMD_EQSTART
Definition: t4fw_interface.h:6373
V_FW_EQ_OFLD_CMD_FBMAX
#define V_FW_EQ_OFLD_CMD_FBMAX(x)
Definition: t4fw_interface.h:6482
F_FW_EQ_ETH_CMD_FETCHRO
#define F_FW_EQ_ETH_CMD_FETCHRO
Definition: t4fw_interface.h:6027
F_FW_CMD_REQUEST
#define F_FW_CMD_REQUEST
Definition: t4fw_interface.h:4197
V_FW_EQ_CTRL_CMD_VFN
#define V_FW_EQ_CTRL_CMD_VFN(x)
Definition: t4fw_interface.h:6156
F_FW_CMD_EXEC
#define F_FW_CMD_EXEC
Definition: t4fw_interface.h:4215
FW_PARAMS_MNEM_DMAQ
@ FW_PARAMS_MNEM_DMAQ
Definition: t4fw_interface.h:4763
V_FW_EQ_CTRL_CMD_CIDXFTHRESH
#define V_FW_EQ_CTRL_CMD_CIDXFTHRESH(x)
Definition: t4fw_interface.h:6315
V_FW_PARAMS_PARAM_X
#define V_FW_PARAMS_PARAM_X(x)
Definition: t4fw_interface.h:5011
FW_PARAMS_PARAM_DMAQ_CONM_CTXT
@ FW_PARAMS_PARAM_DMAQ_CONM_CTXT
Definition: t4fw_interface.h:4985
F_FW_EQ_OFLD_CMD_FETCHRO
#define F_FW_EQ_OFLD_CMD_FETCHRO
Definition: t4fw_interface.h:6427
F_FW_EQ_ETH_CMD_ALLOC
#define F_FW_EQ_ETH_CMD_ALLOC
Definition: t4fw_interface.h:5952
G_FW_EQ_ETH_CMD_EQID
#define G_FW_EQ_ETH_CMD_EQID(x)
Definition: t4fw_interface.h:5985
G_FW_EQ_OFLD_CMD_PHYSEQID
#define G_FW_EQ_OFLD_CMD_PHYSEQID(x)
Definition: t4fw_interface.h:6391
G_FW_EQ_ETH_CMD_PHYSEQID
#define G_FW_EQ_ETH_CMD_PHYSEQID(x)
Definition: t4fw_interface.h:5991
V_FW_EQ_OFLD_CMD_IQID
#define V_FW_EQ_OFLD_CMD_IQID(x)
Definition: t4fw_interface.h:6457
V_FW_EQ_ETH_CMD_IQID
#define V_FW_EQ_ETH_CMD_IQID(x)
Definition: t4fw_interface.h:6057
V_FW_IQ_CMD_VFN
#define V_FW_IQ_CMD_VFN(x)
Definition: t4fw_interface.h:5242
V_FW_EQ_CTRL_CMD_PFN
#define V_FW_EQ_CTRL_CMD_PFN(x)
Definition: t4fw_interface.h:6150
V_FW_IQ_CMD_FL0FBMAX
#define V_FW_IQ_CMD_FL0FBMAX(x)
Definition: t4fw_interface.h:5563
V_FW_IQ_CMD_FL0FBMIN
#define V_FW_IQ_CMD_FL0FBMIN(x)
Definition: t4fw_interface.h:5557
G_FW_EQ_CTRL_CMD_PHYSEQID
#define G_FW_EQ_CTRL_CMD_PHYSEQID(x)
Definition: t4fw_interface.h:6210
V_FW_IQ_CMD_IQANUD
#define V_FW_IQ_CMD_IQANUD(x)
Definition: t4fw_interface.h:5316
FW_LEN16
#define FW_LEN16(fw_struct)
Definition: t4fw_interface.h:4232
V_FW_EQ_OFLD_CMD_PFN
#define V_FW_EQ_OFLD_CMD_PFN(x)
Definition: t4fw_interface.h:6337
F_FW_WR_COMPL
#define F_FW_WR_COMPL
Definition: t4fw_interface.h:198
fw_error_cmd::fw_error::exception
struct fw_error_cmd::fw_error::fw_error_exception exception
fw_error_cmd::fw_error::wr
struct fw_error_cmd::fw_error::fw_error_wr wr
fw_error_cmd::fw_error::hwmodule
struct fw_error_cmd::fw_error::fw_error_hwmodule hwmodule
fw_error_cmd::fw_error::acl
struct fw_error_cmd::fw_error::fw_error_acl acl
fw_eth_tx_eo_wr::fw_eth_tx_eo::udpseg
struct fw_eth_tx_eo_wr::fw_eth_tx_eo::fw_eth_tx_eo_udpseg udpseg
fw_eth_tx_eo_wr::fw_eth_tx_eo::tcpseg
struct fw_eth_tx_eo_wr::fw_eth_tx_eo::fw_eth_tx_eo_tcpseg tcpseg