39#include <sys/kernel.h>
41#include <sys/module.h>
42#include <sys/protosw.h>
43#include <sys/refcount.h>
44#include <sys/domain.h>
45#include <sys/fnv_hash.h>
46#include <sys/socket.h>
47#include <sys/socketvar.h>
48#include <sys/sysctl.h>
49#include <net/ethernet.h>
51#include <net/if_types.h>
52#include <net/if_vlan_var.h>
54#include <net/route/nhop.h>
55#include <netinet/in.h>
56#include <netinet/in_fib.h>
57#include <netinet/in_pcb.h>
58#include <netinet/ip.h>
59#include <netinet/ip6.h>
60#include <netinet6/in6_fib.h>
61#include <netinet6/scope6_var.h>
62#include <netinet/tcp_timer.h>
64#include <netinet/tcp_fsm.h>
65#include <netinet/tcp_var.h>
66#include <netinet/toecore.h>
67#include <netinet/cc/cc.h>
91static void send_abort_rpl_synqe(
struct toedev *,
struct synq_entry *,
int);
97 u_int stid, n, f, mask;
108 (
"%s: stid region (%u, %u) not properly aligned. n = %u",
111 mtx_lock(&t->stid_lock);
113 mtx_unlock(&t->stid_lock);
126 TAILQ_INSERT_HEAD(&t->
stids, sr, link);
131 TAILQ_FOREACH(s, &t->
stids, link) {
134 if (s->
free >= n + f) {
137 TAILQ_INSERT_AFTER(&t->
stids, s, sr, link);
142 if (__predict_false(stid != t->
nstids)) {
143 panic(
"%s: stids TAILQ (%p) corrupt."
144 " At %d instead of %d at the end of the queue.",
148 mtx_unlock(&t->stid_lock);
157 mtx_unlock(&t->stid_lock);
159 KASSERT(((stid + t->
stid_base) & mask) == 0,
160 (
"%s: EDOOFUS.", __func__));
179 KASSERT(sr->
used > 0, (
"%s: nonsense free (%d)", __func__, sr->
used));
181 mtx_lock(&t->stid_lock);
182 s = TAILQ_PREV(sr, stid_head, link);
188 (
"%s: stids_in_use (%u) < stids being freed (%u)", __func__,
191 TAILQ_REMOVE(&t->
stids, sr, link);
192 mtx_unlock(&t->stid_lock);
200 INP_WLOCK_ASSERT(
inp);
202 lctx = malloc(
sizeof(
struct listen_ctx), M_CXGBE, M_NOWAIT | M_ZERO);
206 lctx->
stid = alloc_stid(sc, lctx,
inp->inp_vflag & INP_IPV6);
207 if (lctx->
stid < 0) {
212 if (
inp->inp_vflag & INP_IPV6 &&
213 !IN6_ARE_ADDR_EQUAL(&in6addr_any, &
inp->in6p_laddr)) {
215 if (lctx->
ce == NULL) {
226 lctx->
vnet =
inp->inp_socket->so_vnet;
236 struct inpcb *inp = lctx->
inp;
238 INP_WLOCK_ASSERT(inp);
240 (
"%s: refcount %d", __func__, lctx->
refcount));
241 KASSERT(lctx->
stid >= 0, (
"%s: bad stid %d.", __func__, lctx->
stid));
243 CTR4(
KTR_CXGBE,
"%s: stid %u, lctx %p, inp %p",
244 __func__, lctx->
stid, lctx, lctx->
inp);
251 return (in_pcbrele_wlocked(inp));
261static inline uint32_t
262listen_hashfn(
void *key, u_long mask)
265 return (fnv_32_buf(&key,
sizeof(key), FNV1_32_INIT) & mask);
278 LIST_INSERT_HEAD(&td->listen_hash[bucket], lctx, link);
287listen_hash_find(
struct adapter *sc,
struct inpcb *
inp)
294 LIST_FOREACH(lctx, &td->listen_hash[bucket], link) {
307listen_hash_del(
struct adapter *sc,
struct inpcb *
inp)
314 LIST_FOREACH_SAFE(lctx, &td->listen_hash[bucket], link, l) {
316 LIST_REMOVE(lctx, link);
334 struct inpcb *inp = lctx->
inp;
337 INP_WLOCK_ASSERT(inp);
338 if (refcount_release(&lctx->
refcount))
339 inp_freed = free_lctx(sc, lctx);
341 return (inp_freed ? NULL : inp);
347 struct mbuf *m = synqe->
syn;
348 struct ifnet *
ifp = m->m_pkthdr.rcvif;
355 const int nparams = 6;
356 const int flowclen =
sizeof(*flowc) + nparams *
sizeof(
struct fw_flowc_mnemval);
359 INP_WLOCK_ASSERT(synqe->
lctx->
inp);
368 panic(
"%s: allocation failure.", __func__);
371 memset(flowc, 0, wr->
wr_len);
394send_abort_rpl_synqe(
struct toedev *tod,
struct synq_entry *synqe,
397 struct adapter *sc = tod->tod_softc;
401 INP_WLOCK_ASSERT(synqe->
lctx->
inp);
403 CTR5(
KTR_CXGBE,
"%s: synqe %p (0x%x), tid %d%s",
404 __func__, synqe, synqe->
flags, synqe->
tid,
406 " (abort already in progress)" :
"");
412 send_flowc_wr_synqe(sc, synqe);
418 panic(
"%s: allocation failure.", __func__);
424 req->
cmd = rst_status;
434 struct inpcb *inp = lctx->
inp;
438 log(LOG_ERR,
"%s: allocation failure", __func__);
447 req->
local_ip = inp->inp_laddr.s_addr;
462 struct inpcb *inp = lctx->
inp;
466 log(LOG_ERR,
"%s: allocation failure", __func__);
475 req->
local_ip_hi = *(uint64_t *)&inp->in6p_laddr.s6_addr[0];
476 req->
local_ip_lo = *(uint64_t *)&inp->in6p_laddr.s6_addr[8];
496 panic(
"%s: allocation failure.", __func__);
504 req->
rsvd = htobe16(0);
519 struct adapter *sc = tod->tod_softc;
522 struct inpcb *inp = tp->t_inpcb;
527 INP_WLOCK_ASSERT(inp);
531 EVL_MAKETAG(0xfff, 0, 0), inp);
533 if (!settings.offload)
537 if (inp->inp_vflag & INP_IPV6 && IN6_IS_ADDR_LOOPBACK(&inp->in6p_laddr))
539 if (!(inp->inp_vflag & INP_IPV6) &&
540 IN_LOOPBACK(ntohl(inp->inp_laddr.s_addr)))
547 log(LOG_ERR,
"%s: listen request ignored, %s is busy",
548 __func__, device_get_nameunit(sc->
dev));
552 KASSERT(uld_active(sc,
ULD_TOM),
553 (
"%s: TOM not initialized", __func__));
569 vi->
ifp->if_capenable & IFCAP_TOE)
576 if (listen_hash_find(sc, inp) != NULL)
579 lctx = alloc_lctx(sc, inp, vi);
582 "%s: listen request ignored, %s couldn't allocate lctx\n",
583 __func__, device_get_nameunit(sc->
dev));
586 listen_hash_add(sc, lctx);
588 CTR6(
KTR_CXGBE,
"%s: stid %u (%s), lctx %p, inp %p vflag 0x%x",
589 __func__, lctx->
stid, tcpstates[tp->t_state], lctx, inp,
592 if (inp->inp_vflag & INP_IPV6)
593 rc = create_server6(sc, lctx);
595 rc = create_server(sc, lctx);
597 log(LOG_ERR,
"%s: %s failed to create hw listener: %d.\n",
598 __func__, device_get_nameunit(sc->
dev), rc);
599 (void) listen_hash_del(sc, inp);
600 inp = release_lctx(sc, lctx);
602 KASSERT(inp != NULL, (
"%s: inp freed", __func__));
617 struct adapter *sc = tod->tod_softc;
618 struct inpcb *inp = tp->t_inpcb;
620 INP_WLOCK_ASSERT(inp);
622 lctx = listen_hash_del(sc, inp);
626 CTR4(
KTR_CXGBE,
"%s: stid %u, lctx %p, flags %x", __func__, lctx->
stid,
634 return (EINPROGRESS);
637 destroy_server(sc, lctx);
647 MPASS(
flags == M_WAITOK ||
flags == M_NOWAIT);
649 synqe = malloc(
sizeof(*synqe), M_CXGBE,
flags);
650 if (__predict_true(synqe != NULL)) {
652 refcount_init(&synqe->
refcnt, 1);
665 refcount_acquire(&synqe->
refcnt);
668static inline struct inpcb *
674 MPASS(synqe->
lctx != NULL);
678 INP_WLOCK_ASSERT(inp);
680 if (refcount_release(&synqe->
refcnt)) {
681 inp = release_lctx(sc, synqe->
lctx);
683 free(synqe, M_CXGBE);
700 struct adapter *sc = tod->tod_softc;
702 struct inpcb *inp = synqe->
lctx->
inp;
708 inp = release_synqe(sc, synqe);
720 struct ip *ip = mtod(m,
struct ip *);
723 if (ip->ip_v == IPVERSION)
724 th = (
void *)(ip + 1);
726 th = (
void *)((
struct ip6_hdr *)ip + 1);
727 bzero(&to,
sizeof(to));
728 tcp_dooptions(&to, (
void *)(th + 1),
729 (th->th_off << 2) -
sizeof(*th), TO_SYN);
732 synqe->
iss = be32toh(th->th_seq);
733 synqe->
irs = be32toh(th->th_ack) - 1;
734 synqe->
ts = to.to_tsval;
750 struct inpcb *inp = lctx->
inp;
756 (
"%s: unexpected opcode 0x%x", __func__, opcode));
757 KASSERT(m == NULL, (
"%s: wasn't expecting payload", __func__));
758 KASSERT(lctx->
stid == stid, (
"%s: lctx stid mismatch", __func__));
762 CTR4(
KTR_CXGBE,
"%s: stid %d, status %u, flags 0x%x",
763 __func__, stid, status, lctx->
flags);
765 lctx->
flags &= ~LCTX_RPL_PENDING;
768 log(LOG_ERR,
"listener (stid %u) failed: %d\n", stid, status);
775 if (inp->inp_flags & INP_DROPPED) {
776 KASSERT(listen_hash_del(sc, inp) == NULL,
777 (
"%s: inp %p still in listen hash", __func__, inp));
781 if (inp->inp_flags & INP_DROPPED && status !=
CPL_ERR_NONE) {
782 if (release_lctx(sc, lctx) != NULL)
792 if (inp->inp_flags & INP_DROPPED) {
793 destroy_server(sc, lctx);
804 listen_hash_del(sc, inp);
805 if (release_lctx(sc, lctx) != NULL)
825 struct inpcb *inp = lctx->
inp;
831 (
"%s: unexpected opcode 0x%x", __func__, opcode));
832 KASSERT(m == NULL, (
"%s: wasn't expecting payload", __func__));
833 KASSERT(lctx->
stid == stid, (
"%s: lctx stid mismatch", __func__));
835 CTR3(
KTR_CXGBE,
"%s: stid %u, status %u", __func__, stid, status);
838 log(LOG_ERR,
"%s: failed (%u) to close listener for stid %u\n",
839 __func__, status, stid);
844 inp = release_lctx(sc, lctx);
855 struct inpcb *inp = lctx->
inp;
859 INP_WLOCK_ASSERT(inp);
860 ntids = inp->inp_vflag & INP_IPV6 ? 2 : 1;
865 inp = release_synqe(sc, synqe);
876 done_with_synqe(sc, synqe);
885 unsigned int tid =
GET_TID(cpl);
888 struct inpcb *inp = lctx->
inp;
895 (
"%s: unexpected opcode 0x%x", __func__, opcode));
896 KASSERT(m == NULL, (
"%s: wasn't expecting payload", __func__));
897 KASSERT(synqe->
tid == tid, (
"%s: toep tid mismatch", __func__));
899 CTR6(
KTR_CXGBE,
"%s: tid %u, synqe %p (0x%x), lctx %p, status %d",
910 send_flowc_wr_synqe(sc, synqe);
922 done_with_synqe(sc, synqe);
935 unsigned int tid =
GET_TID(cpl);
938 struct inpcb *inp = lctx->
inp;
944 (
"%s: unexpected opcode 0x%x", __func__, opcode));
945 KASSERT(m == NULL, (
"%s: wasn't expecting payload", __func__));
946 KASSERT(synqe->
tid == tid, (
"%s: toep tid mismatch", __func__));
948 CTR6(
KTR_CXGBE,
"%s: tid %u, synqe %p (0x%x), lctx %p, status %d",
953 (
"%s: wasn't expecting abort reply for synqe %p (0x%x)",
954 __func__, synqe, synqe->
flags));
956 done_with_synqe(sc, synqe);
965 struct adapter *sc = tod->tod_softc;
967 struct inpcb *inp = sotoinpcb(so);
971 INP_WLOCK_ASSERT(
inp);
973 (
"%s: %p not a synq_entry?", __func__, arg));
974 MPASS(toep->
tid == synqe->
tid);
981 inp->inp_flowtype = (
inp->inp_vflag & INP_IPV6) ?
982 M_HASHTYPE_RSS_TCP_IPV6 : M_HASHTYPE_RSS_TCP_IPV4;
987t4opt_to_tcpopt(
const struct tcp_options *t4opt,
struct tcpopt *to)
989 bzero(to,
sizeof(*to));
992 to->to_flags |= TOF_MSS;
993 to->to_mss = be16toh(t4opt->
mss);
996 if (t4opt->
wsf > 0 && t4opt->
wsf < 15) {
997 to->to_flags |= TOF_SCALE;
998 to->to_wscale = t4opt->
wsf;
1002 to->to_flags |= TOF_TS;
1005 to->to_flags |= TOF_SACKPERM;
1011 u_int hlen = be32toh(cpl->
hdr_len);
1016 return (
G_ETH_HDR_LEN(hlen) >
sizeof(
struct ether_vlan_header));
1020pass_accept_req_to_protohdrs(
struct adapter *sc,
const struct mbuf *m,
1021 struct in_conninfo *inc,
struct tcphdr *th, uint8_t *iptos)
1024 const struct ether_header *eh;
1025 unsigned int hlen = be32toh(cpl->
hdr_len);
1027 const struct tcphdr *tcp;
1029 eh = (
const void *)(cpl + 1);
1040 if (((
struct ip *)l3hdr)->ip_v == IPVERSION) {
1041 const struct ip *ip = (
const void *)l3hdr;
1042 *iptos = ip->ip_tos;
1046 if (((
struct ip *)l3hdr)->ip_v == (IPV6_VERSION >> 4)) {
1047 const struct ip6_hdr *ip6 = (
const void *)l3hdr;
1048 *iptos = (ntohl(ip6->ip6_flow) >> 20) & 0xff;
1054 bzero(inc,
sizeof(*inc));
1055 inc->inc_fport = tcp->th_sport;
1056 inc->inc_lport = tcp->th_dport;
1057 if (((
struct ip *)l3hdr)->ip_v == IPVERSION) {
1058 const struct ip *ip = (
const void *)l3hdr;
1060 inc->inc_faddr = ip->ip_src;
1061 inc->inc_laddr = ip->ip_dst;
1063 const struct ip6_hdr *ip6 = (
const void *)l3hdr;
1065 inc->inc_flags |= INC_ISIPV6;
1066 inc->inc6_faddr = ip6->ip6_src;
1067 inc->inc6_laddr = ip6->ip6_dst;
1072 bcopy(tcp, th,
sizeof(*th));
1073 tcp_fields_to_host(th);
1078get_l2te_for_nexthop(
struct port_info *pi,
struct ifnet *
ifp,
1079 struct in_conninfo *inc)
1082 struct sockaddr_in6 sin6;
1083 struct sockaddr *dst = (
void *)&sin6;
1084 struct nhop_object *nh;
1086 if (inc->inc_flags & INC_ISIPV6) {
1087 bzero(dst,
sizeof(
struct sockaddr_in6));
1088 dst->sa_len =
sizeof(
struct sockaddr_in6);
1089 dst->sa_family = AF_INET6;
1091 if (IN6_IS_ADDR_LINKLOCAL(&inc->inc6_laddr)) {
1097 nh = fib6_lookup(RT_DEFAULT_FIB, &inc->inc6_faddr, 0, NHR_NONE, 0);
1100 if (nh->nh_ifp !=
ifp)
1102 if (nh->nh_flags & NHF_GATEWAY)
1103 ((
struct sockaddr_in6 *)dst)->sin6_addr = nh->gw6_sa.sin6_addr;
1105 ((
struct sockaddr_in6 *)dst)->sin6_addr = inc->inc6_faddr;
1107 dst->sa_len =
sizeof(
struct sockaddr_in);
1108 dst->sa_family = AF_INET;
1110 nh = fib4_lookup(RT_DEFAULT_FIB, inc->inc_faddr, 0, NHR_NONE, 0);
1113 if (nh->nh_ifp !=
ifp)
1115 if (nh->nh_flags & NHF_GATEWAY)
1116 if (nh->gw_sa.sa_family == AF_INET)
1117 ((
struct sockaddr_in *)dst)->sin_addr = nh->gw4_sa.sin_addr;
1119 *((
struct sockaddr_in6 *)dst) = nh->gw6_sa;
1121 ((
struct sockaddr_in *)dst)->sin_addr = inc->inc_faddr;
1130 uint32_t opt2,
int tid)
1148 rpl5->
iss = htobe32(synqe->
iss);
1156#define REJECT_PASS_ACCEPT_REQ(tunnel) do { \
1161 reject_reason = __LINE__; \
1185 unsigned int tid =
GET_TID(cpl);
1189 struct in_conninfo inc;
1194 struct ifnet *hw_ifp, *
ifp;
1197 int reject_reason, v, ntids;
1198 uint16_t vid, l2info;
1199 struct epoch_tracker et;
1207 (
"%s: unexpected opcode 0x%x", __func__, opcode));
1208 KASSERT(lctx->
stid == stid, (
"%s: lctx stid mismatch", __func__));
1210 CTR4(
KTR_CXGBE,
"%s: stid %u, tid %u, lctx %p", __func__, stid, tid,
1218 l2info = be16toh(cpl->
l2info);
1220 hw_ifp = pi->
vi[0].
ifp;
1221 m->m_pkthdr.rcvif = hw_ifp;
1223 CURVNET_SET(lctx->
vnet);
1229 if (encapsulated_syn(sc, cpl)) {
1230 REJECT_PASS_ACCEPT_REQ(
true);
1238 REJECT_PASS_ACCEPT_REQ(
true);
1244 REJECT_PASS_ACCEPT_REQ(
true);
1247 m->m_pkthdr.rcvif = hw_ifp;
1248 tod = TOEDEV(hw_ifp);
1255 REJECT_PASS_ACCEPT_REQ(
true);
1264 vid = EVL_VLANOFTAG(be16toh(cpl->
vlan));
1265 if (vid != 0xfff && vid != 0) {
1266 ifp = VLAN_DEVAT(hw_ifp, vid);
1268 REJECT_PASS_ACCEPT_REQ(
true);
1276 if (lctx->
vnet !=
ifp->if_vnet)
1277 REJECT_PASS_ACCEPT_REQ(
true);
1279 pass_accept_req_to_protohdrs(sc, m, &inc, &th, &iptos);
1280 if (inc.inc_flags & INC_ISIPV6) {
1283 if ((
ifp->if_capenable & IFCAP_TOE6) == 0)
1284 REJECT_PASS_ACCEPT_REQ(
true);
1290 NET_EPOCH_ENTER(et);
1291 if (!in6_ifhasaddr(
ifp, &inc.inc6_laddr)) {
1293 REJECT_PASS_ACCEPT_REQ(
true);
1300 if ((
ifp->if_capenable & IFCAP_TOE4) == 0)
1301 REJECT_PASS_ACCEPT_REQ(
true);
1307 NET_EPOCH_ENTER(et);
1308 if (!in_ifhasaddr(
ifp, inc.inc_laddr)) {
1310 REJECT_PASS_ACCEPT_REQ(
true);
1316 e = get_l2te_for_nexthop(pi,
ifp, &inc);
1319 REJECT_PASS_ACCEPT_REQ(
true);
1323 if (toe_4tuple_check(&inc, &th,
ifp) != 0) {
1325 REJECT_PASS_ACCEPT_REQ(
false);
1332 if (__predict_false(inp->inp_flags & INP_DROPPED)) {
1335 REJECT_PASS_ACCEPT_REQ(
false);
1337 so = inp->inp_socket;
1340 EVL_MAKETAG(0xfff, 0, 0), inp);
1342 if (!settings.offload) {
1345 REJECT_PASS_ACCEPT_REQ(
true);
1348 synqe = alloc_synqe(sc, lctx, M_NOWAIT);
1349 if (synqe == NULL) {
1352 REJECT_PASS_ACCEPT_REQ(
true);
1365 t4opt_to_tcpopt(&cpl->
tcpopt, &to);
1366 toe_syncache_add(&inc, &to, &th, inp, tod, synqe, iptos);
1380 if (send_synack(sc, synqe, opt0, opt2, tid) != 0) {
1385 REJECT_PASS_ACCEPT_REQ(
true);
1389 "%s: stid %u, tid %u, synqe %p, opt0 %#016lx, opt2 %#08x",
1390 __func__, stid, tid, synqe, be64toh(opt0), be32toh(opt2));
1393 REJECT_PASS_ACCEPT_REQ(
false);
1401 CTR4(
KTR_CXGBE,
"%s: stid %u, tid %u, REJECT (%d)", __func__, stid, tid,
1410 inp = release_synqe(sc, synqe);
1420 m_adj(m,
sizeof(*cpl));
1421 m->m_pkthdr.csum_flags |= (CSUM_IP_CHECKED | CSUM_IP_VALID |
1422 CSUM_DATA_VALID | CSUM_PSEUDO_HDR);
1423 m->m_pkthdr.csum_data = 0xffff;
1424 hw_ifp->if_input(hw_ifp, m);
1427 return (reject_reason);
1433 struct tcphdr *th,
struct tcpopt *to)
1435 uint16_t tcp_opt = be16toh(cpl->
tcp_opt);
1439 pass_accept_req_to_protohdrs(sc, synqe->
syn, inc, th, &iptos);
1442 th->th_flags = TH_ACK;
1443 th->th_ack = synqe->
iss + 1;
1444 th->th_seq = be32toh(cpl->
rcv_isn);
1445 bzero(to,
sizeof(*to));
1447 to->to_flags |= TOF_TS;
1448 to->to_tsecr = synqe->
ts;
1460#if defined(KTR) || defined(INVARIANTS)
1463 unsigned int tid =
GET_TID(cpl);
1466 struct inpcb *inp = lctx->
inp, *new_inp;
1470 struct in_conninfo inc;
1472 struct epoch_tracker et;
1479 (
"%s: unexpected opcode 0x%x", __func__, opcode));
1480 KASSERT(m == NULL, (
"%s: wasn't expecting payload", __func__));
1481 KASSERT(lctx->
stid == stid, (
"%s: lctx stid mismatch", __func__));
1483 (
"%s: tid %u (ctx %p) not a synqe", __func__, tid, synqe));
1485 CURVNET_SET(lctx->
vnet);
1486 NET_EPOCH_ENTER(et);
1490 "%s: stid %u, tid %u, synqe %p (0x%x), inp_flags 0x%x",
1491 __func__, stid, tid, synqe, synqe->
flags, inp->inp_flags);
1493 ifp = synqe->
syn->m_pkthdr.rcvif;
1496 (
"%s: vi %p, sc %p mismatch", __func__, vi, sc));
1498 if (__predict_false(inp->inp_flags & INP_DROPPED)) {
1508 (
"%s: CPL arrived on unexpected rxq. %d %d", __func__,
1521 MPASS(be32toh(cpl->
snd_isn) - 1 == synqe->
iss);
1522 MPASS(be32toh(cpl->
rcv_isn) - 1 == synqe->
irs);
1527 synqe_to_protohdrs(sc, synqe, cpl, &inc, &th, &to);
1528 if (inc.inc_flags & INC_ISIPV6) {
1529 if (lctx->
ce == NULL) {
1531 if (toep->
ce == NULL) {
1537 toep->
ce = lctx->
ce;
1540 so = inp->inp_socket;
1541 KASSERT(so != NULL, (
"%s: socket is NULL", __func__));
1543 rstreason = toe_syncache_expand(&inc, &to, &th, &so);
1544 if (rstreason < 0) {
1551 }
else if (rstreason == 0 || so == NULL) {
1557 new_inp = sotoinpcb(so);
1558 INP_WLOCK_ASSERT(new_inp);
1559 MPASS(so->so_vnet == lctx->
vnet);
1569 tcp_timer_activate(intotcpcb(new_inp), TT_KEEP, 0);
1573 INP_WUNLOCK(new_inp);
1576 inp = release_synqe(sc, synqe);
static struct wrqe * alloc_wrqe(int wr_len, struct sge_wrq *wrq)
#define ADAPTER_UNLOCK(sc)
#define for_each_vi(_pi, _iter, _vi)
void t4_register_cpl_handler(int, cpl_handler_t)
static void * wrtod(struct wrqe *wr)
static void t4_wrq_tx(struct adapter *sc, struct wrqe *wr)
static struct sge_ofld_rxq * iq_to_ofld_rxq(struct sge_iq *iq)
void release_tid(struct adapter *, int, struct sge_wrq *)
CTASSERT(sizeof(struct iq_desc)==IQ_ESIZE)
static int chip_id(struct adapter *adap)
#define for_each_port(adapter, iter)
static int is_t4(struct adapter *adap)
#define INIT_TP_WR(w, tid)
#define INIT_TP_WR_MIT_CPL(w, cpl, tid)
struct rwlock policy_lock
struct port_info * port[MAX_NPORTS]
struct tcp_options tcpopt
struct fw_flowc_mnemval mnemval[0]
struct sge_ofld_rxq * ofld_rxq
struct stid_region stid_region
struct sge_ofld_rxq * ofld_rxq
struct sge_ofld_txq * ofld_txq
struct conn_params params
volatile int ok_to_respond
struct listen_ctx ** stid_tab
struct conn_params params
struct mtx lctx_hash_lock
void t4_release_clip_entry(struct adapter *sc, struct clip_entry *ce)
void t4_hold_clip_entry(struct adapter *sc, struct clip_entry *ce)
struct clip_entry * t4_get_clip_entry(struct adapter *sc, struct in6_addr *in6, bool add)
static void t4_l2t_release(struct l2t_entry *e)
#define G_T6_IP_HDR_LEN(x)
#define G_T6_ETH_HDR_LEN(x)
#define G_PASS_OPEN_TID(x)
#define V_SYN_RSS_QUEUE(x)
#define MK_OPCODE_TID(opcode, tid)
#define G_TCPOPT_TSTAMP(x)
void t4_uninit_listen_cpl_handlers(void)
void * lookup_tid(struct adapter *, int)
void t4_syncache_removed(struct toedev *, void *)
int do_abort_rpl_synqe(struct sge_iq *, const struct rss_header *, struct mbuf *)
void remove_tid(struct adapter *, int, int)
int init_toepcb(struct vi_info *, struct toepcb *)
void insert_tid(struct adapter *, int, void *, int)
int t4_listen_start(struct toedev *, struct tcpcb *)
void make_established(struct toepcb *, uint32_t, uint32_t, uint16_t)
__be64 calc_options0(struct vi_info *, struct conn_params *)
int t4_syncache_respond(struct toedev *, void *, struct mbuf *)
void free_toepcb(struct toepcb *)
int t4_listen_stop(struct toedev *, struct tcpcb *)
void t4_init_listen_cpl_handlers(void)
void t4_offload_socket(struct toedev *, void *, struct socket *)
void init_conn_params(struct vi_info *, struct offload_settings *, struct in_conninfo *, struct socket *, const struct tcp_options *, int16_t, struct conn_params *cp)
void offload_socket(struct socket *, struct toepcb *)
void synack_failure_cleanup(struct adapter *, int)
int do_abort_req_synqe(struct sge_iq *, const struct rss_header *, struct mbuf *)
__be32 calc_options2(struct vi_info *, struct conn_params *)
void t4_syncache_added(struct toedev *, void *)
struct toepcb * alloc_toepcb(struct vi_info *, int)
void send_abort_rpl(struct adapter *, struct sge_ofld_txq *, int, int)
void update_tid(struct adapter *, int, void *)
const struct offload_settings * lookup_offload_policy(struct adapter *, int, struct mbuf *, uint16_t, struct inpcb *)
struct l2t_entry * t4_l2t_get(struct port_info *, struct ifnet *, struct sockaddr *)
static int t4_l2t_send(struct adapter *sc, struct wrqe *wr, struct l2t_entry *e)
#define V_FW_FLOWC_WR_NPARAMS(x)
#define V_FW_WR_FLOWID(x)