35#include "opt_kern_tls.h"
36#include "opt_ratelimit.h"
41#include <sys/kernel.h>
44#include <sys/limits.h>
45#include <sys/module.h>
46#include <sys/protosw.h>
47#include <sys/domain.h>
48#include <sys/refcount.h>
49#include <sys/rmlock.h>
50#include <sys/socket.h>
51#include <sys/socketvar.h>
52#include <sys/sysctl.h>
53#include <sys/taskqueue.h>
55#include <net/if_var.h>
56#include <net/if_types.h>
57#include <net/if_vlan_var.h>
58#include <netinet/in.h>
59#include <netinet/in_pcb.h>
60#include <netinet/in_var.h>
61#include <netinet/ip.h>
62#include <netinet/ip6.h>
63#include <netinet6/scope6_var.h>
65#include <netinet/tcp_fsm.h>
66#include <netinet/tcp_seq.h>
67#include <netinet/tcp_timer.h>
68#include <netinet/tcp_var.h>
69#include <netinet/toecore.h>
70#include <netinet/cc/cc.h>
83static struct protosw *tcp_protosw;
84static struct protosw toe_protosw;
85static struct pr_usrreqs toe_usrreqs;
87static struct protosw *tcp6_protosw;
88static struct protosw toe6_protosw;
89static struct pr_usrreqs toe6_usrreqs;
92static int t4_tom_mod_load(
void);
93static int t4_tom_mod_unload(
void);
97static int t4_tom_activate(
struct adapter *);
98static int t4_tom_deactivate(
struct adapter *);
100static struct uld_info tom_uld_info = {
102 .activate = t4_tom_activate,
103 .deactivate = t4_tom_deactivate,
106static void release_offload_resources(
struct toepcb *);
107static int alloc_tid_tabs(
struct tid_info *);
108static void free_tid_tabs(
struct tid_info *);
110static void reclaim_wr_resources(
void *,
int);
139 toep = malloc(len, M_CXGBE, M_ZERO | flags);
143 refcount_init(&toep->refcount, 1);
149 mbufq_init(&toep->ulp_pduq, INT_MAX);
150 mbufq_init(&toep->ulp_pdu_reclaimq, INT_MAX);
151 toep->txsd_total = txsd_total;
152 toep->txsd_avail = txsd_total;
175 CH_ERR(vi,
"tid %d cannot be bound to traffic class %d "
176 "because it is not configured (its state is %d)\n",
209 if (refcount_release(&toep->
refcount) == 0)
213 (
"%s: attached to an inpcb", __func__));
215 (
"%s: CPL pending", __func__));
232 struct inpcb *inp = sotoinpcb(so);
233 struct tcpcb *tp = intotcpcb(inp);
236 INP_WLOCK_ASSERT(inp);
241 sb->sb_flags |= SB_NOCOALESCE;
245 sb->sb_flags |= SB_NOCOALESCE;
246 if (inp->inp_vflag & INP_IPV6)
247 so->so_proto = &toe6_protosw;
249 so->so_proto = &toe_protosw;
255 tp->t_flags |= TF_TOE;
264 TAILQ_INSERT_HEAD(&td->toep_list, toep, link);
272 so->so_proto = tcp6_protosw;
274 so->so_proto = tcp_protosw;
281 struct inpcb *inp = sotoinpcb(so);
282 struct tcpcb *tp = intotcpcb(inp);
283 struct toepcb *toep = tp->t_toe;
287 INP_WLOCK_ASSERT(inp);
291 sb->sb_flags &= ~SB_NOCOALESCE;
295 sb->sb_flags &= ~SB_NOCOALESCE;
301 tp->t_flags &= ~TF_TOE;
304 toep->
flags &= ~TPF_ATTACHED;
305 if (in_pcbrele_wlocked(inp))
306 panic(
"%s: inp freed.", __func__);
309 TAILQ_REMOVE(&td->toep_list, toep, link);
314release_offload_resources(
struct toepcb *toep)
321 (
"%s: %p has CPL pending.", __func__, toep));
323 (
"%s: %p is still attached.", __func__, toep));
325 CTR5(
KTR_CXGBE,
"%s: toep %p (tid %d, l2te %p, ce %p)",
326 __func__, toep, tid, toep->
l2te, toep->
ce);
333 MPASS(mbufq_len(&toep->
ulp_pduq) == 0);
339 MPASS(TAILQ_EMPTY(&toep->aiotx_jobq));
356 TAILQ_REMOVE(&td->toep_list, toep, link);
371t4_pcb_detach(
struct toedev *tod __unused,
struct tcpcb *tp)
373#if defined(KTR) || defined(INVARIANTS)
374 struct inpcb *inp = tp->t_inpcb;
376 struct toepcb *toep = tp->t_toe;
378 INP_WLOCK_ASSERT(
inp);
380 KASSERT(toep != NULL, (
"%s: toep is NULL", __func__));
382 (
"%s: not attached", __func__));
385 if (tp->t_state == TCPS_SYN_SENT) {
386 CTR6(
KTR_CXGBE,
"%s: atid %d, toep %p (0x%x), inp %p (0x%x)",
391 "t4_pcb_detach: tid %d (%s), toep %p (0x%x), inp %p (0x%x)",
392 toep->
tid, tcpstates[tp->t_state], toep, toep->
flags,
inp,
402 tp->t_flags &= ~TF_TOE;
403 toep->
flags &= ~TPF_ATTACHED;
406 release_offload_resources(toep);
413t4_ctloutput(
struct toedev *tod,
struct tcpcb *tp,
int dir,
int name)
415 struct adapter *sc = tod->tod_softc;
416 struct toepcb *toep = tp->t_toe;
421 CTR4(
KTR_CXGBE,
"%s: tp %p, dir %u, name %u", __func__, tp, dir, name);
425 if (tp->t_state != TCPS_ESTABLISHED)
427 toep->
params.
nagle = tp->t_flags & TF_NODELAY ? 0 : 1;
436static inline uint64_t
437get_tcb_tflags(
const uint64_t *tcb)
440 return ((be64toh(tcb[14]) << 32) | (be64toh(tcb[15]) >> 32));
443static inline uint32_t
444get_tcb_field(
const uint64_t *tcb, u_int word, uint32_t mask, u_int shift)
446#define LAST_WORD ((TCB_SIZE / 4) - 1)
451 MPASS(word <= LAST_WORD);
454 flit_idx = (LAST_WORD - word) / 2;
457 t1 = be64toh(tcb[flit_idx]) >> shift;
459 if (fls(mask) > 64 - shift) {
465 t2 = be64toh(tcb[flit_idx - 1]) << (64 - shift);
467 return ((t2 | t1) & mask);
470#define GET_TCB_FIELD(tcb, F) \
471 get_tcb_field(tcb, W_TCB_##F, M_TCB_##F, S_TCB_##F)
482 MPASS(tid < sc->tids.ntids);
486 if (__predict_false(cpl == NULL))
488 bzero(cpl,
sizeof(*cpl));
500alloc_tcb_histent(
struct adapter *sc, u_int tid,
int flags)
504 MPASS(flags == M_NOWAIT || flags == M_WAITOK);
506 te = malloc(
sizeof(*te), M_CXGBE, M_ZERO | flags);
509 mtx_init(&te->
te_lock,
"TCB entry", NULL, MTX_DEF);
535 MPASS(tid < sc->tids.ntids);
540 rw_wlock(&td->tcb_history_lock);
545 te = alloc_tcb_histent(sc, tid, M_NOWAIT);
551 rc = send_get_tcb(sc, tid);
560 rw_wunlock(&td->tcb_history_lock);
570 rw_assert(&td->tcb_history_lock, RA_WLOCKED);
571 mtx_assert(&te->
te_lock, MA_OWNED);
575 free_tcb_histent(te);
576 rw_wunlock(&td->tcb_history_lock);
580lookup_tcb_histent(
struct adapter *sc, u_int tid,
bool addrem)
585 MPASS(tid < sc->tids.ntids);
591 rw_wlock(&td->tcb_history_lock);
593 rw_rlock(&td->tcb_history_lock);
600 rw_wunlock(&td->tcb_history_lock);
602 rw_runlock(&td->tcb_history_lock);
613 mtx_assert(&te->
te_lock, MA_OWNED);
615 rw_assert(&td->tcb_history_lock, RA_RLOCKED);
616 rw_runlock(&td->tcb_history_lock);
620request_tcb(
void *arg)
624 mtx_assert(&te->
te_lock, MA_OWNED);
635update_tcb_histent(
struct tcb_histent *te,
const uint64_t *tcb)
638 uint64_t tflags = get_tcb_tflags(tcb);
641 if (GET_TCB_FIELD(tcb, SND_MAX_RAW) != GET_TCB_FIELD(tcb, SND_UNA_RAW)) {
642 if (GET_TCB_FIELD(tcb, T_RXTSHIFT) != 0)
644 if (GET_TCB_FIELD(tcb, T_DUPACKS) != 0)
650 if (GET_TCB_FIELD(tcb, SND_MAX_RAW) != 0) {
655 snd_wnd = GET_TCB_FIELD(tcb, RCV_ADV);
657 snd_wnd <<= GET_TCB_FIELD(tcb, RCV_SCALE);
658 if (GET_TCB_FIELD(tcb, SND_CWND) < snd_wnd)
693 const uint64_t *tcb = (
const uint64_t *)(
const void *)(cpl + 1);
695 const u_int tid =
GET_TID(cpl);
698 remove = GET_TCB_FIELD(tcb, T_STATE) == TCPS_CLOSED;
699 te = lookup_tcb_histent(sc, tid, remove);
702 device_printf(sc->
dev,
"tcb %u: flags 0x%016jx, state %u, "
703 "srtt %u, sscale %u, rscale %u, cookie 0x%x\n", tid,
704 (uintmax_t)get_tcb_tflags(tcb), GET_TCB_FIELD(tcb, T_STATE),
705 GET_TCB_FIELD(tcb, T_SRTT), GET_TCB_FIELD(tcb, SND_SCALE),
706 GET_TCB_FIELD(tcb, RCV_SCALE), cpl->
cookie);
713 remove_tcb_histent(te);
715 update_tcb_histent(te, tcb);
716 callout_reset(&te->
te_callout, hz / 10, request_tcb, te);
717 release_tcb_histent(te);
725fill_tcp_info_from_tcb(
struct adapter *sc, uint64_t *tcb,
struct tcp_info *ti)
729 ti->tcpi_state = GET_TCB_FIELD(tcb, T_STATE);
731 v = GET_TCB_FIELD(tcb, T_SRTT);
734 v = GET_TCB_FIELD(tcb, T_RTTVAR);
737 ti->tcpi_snd_ssthresh = GET_TCB_FIELD(tcb, SND_SSTHRESH);
738 ti->tcpi_snd_cwnd = GET_TCB_FIELD(tcb, SND_CWND);
739 ti->tcpi_rcv_nxt = GET_TCB_FIELD(tcb, RCV_NXT);
741 v = GET_TCB_FIELD(tcb, TX_MAX);
742 ti->tcpi_snd_nxt = v - GET_TCB_FIELD(tcb, SND_NXT_RAW);
745 ti->tcpi_rcv_wscale = GET_TCB_FIELD(tcb, SND_SCALE);
746 ti->tcpi_rcv_space = GET_TCB_FIELD(tcb, RCV_WND);
749 ti->tcpi_snd_wscale = GET_TCB_FIELD(tcb, RCV_SCALE);
750 ti->tcpi_snd_wnd = GET_TCB_FIELD(tcb, RCV_ADV);
752 ti->tcpi_snd_wnd <<= ti->tcpi_snd_wscale;
754 ti->tcpi_snd_wscale = 0;
763 fill_tcp_info_from_tcb(sc, te->
te_tcb, ti);
771read_tcb_using_memwin(
struct adapter *sc, u_int tid, uint64_t *buf)
777 MPASS(tid < sc->tids.ntids);
785 for (i = 0, j =
TCB_SIZE - 16; i < j; i += 16, j -= 16) {
786 for (k = 0; k < 16; k++) {
788 tcb[i + k] = tcb[j + k];
795fill_tcp_info(
struct adapter *sc, u_int tid,
struct tcp_info *ti)
797 uint64_t tcb[
TCB_SIZE /
sizeof(uint64_t)];
800 ti->tcpi_toe_tid = tid;
801 te = lookup_tcb_histent(sc, tid,
false);
803 fill_tcp_info_from_history(sc, te, ti);
804 release_tcb_histent(te);
809 read_tcb_using_memwin(sc, tid, tcb);
810 fill_tcp_info_from_tcb(sc, tcb, ti);
819t4_tcp_info(
struct toedev *tod,
struct tcpcb *tp,
struct tcp_info *ti)
821 struct adapter *sc = tod->tod_softc;
822 struct toepcb *toep = tp->t_toe;
824 INP_WLOCK_ASSERT(tp->t_inpcb);
827 fill_tcp_info(sc, toep->
tid, ti);
832t4_alloc_tls_session(
struct toedev *tod,
struct tcpcb *tp,
833 struct ktls_session *
tls,
int direction)
835 struct toepcb *toep = tp->t_toe;
837 INP_WLOCK_ASSERT(tp->t_inpcb);
845#define LEN__SET_TCB_FIELD_ULP (sizeof(struct ulp_txpkt) + \
846 sizeof(struct ulptx_idata) + sizeof(struct cpl_set_tcb_field_core))
850 uint64_t val, uint32_t
tid)
860 ulpsc->
len = htobe32(
sizeof(*req));
872 ulpsc->
len = htobe32(0);
885 const int flowclen16 = howmany(flowclen, 16);
888 CH_ERR(sc,
"%s: tid %u out of tx credits (%d, %d).\n", __func__,
894 if (__predict_false(flowc == NULL)) {
895 CH_ERR(sc,
"ENOMEM in %s for tid %u.\n", __func__, toep->
tid);
916t4_pmtu_update(
struct toedev *tod,
struct tcpcb *tp, tcp_seq seq,
int mtu)
922 struct inpcb *inp = tp->t_inpcb;
923 struct toepcb *toep = tp->t_toe;
927 INP_WLOCK_ASSERT(inp);
933 CTR6(
KTR_CXGBE,
"%s: tid %d, seq 0x%08x, mtu %u, mtu_idx %u (%d)",
938 (SEQ_LT(seq, tp->snd_una) || SEQ_GEQ(seq, tp->snd_max))) {
940 "%s: tid %d, seq 0x%08x not in range [0x%08x, 0x%08x).",
941 __func__, toep->
tid, seq, tp->snd_una, tp->snd_max);
946 for (idx = 0; idx <
NMTUS - 1 && mtus[idx + 1] <= mtu; idx++)
958 CH_ERR(sc,
"failed to change mtu_idx of tid %d (%u -> %u).\n",
973 if (inp->inp_inc.inc_flags & INC_ISIPV6)
974 tp->t_maxseg -=
sizeof(
struct ip6_hdr) + sizeof(struct tcphdr);
976 tp->t_maxseg -=
sizeof(
struct ip) + sizeof(struct tcphdr);
978 if (tp->t_flags & TF_RCVD_TSTMP)
982 send_mss_flowc_wr(sc, toep);
986 struct in_conninfo inc = {0};
988 inc.inc_fibnum = inp->inp_inc.inc_fibnum;
989 if (inp->inp_inc.inc_flags & INC_ISIPV6) {
990 inc.inc_flags |= INC_ISIPV6;
991 inc.inc6_faddr = inp->inp_inc.inc6_faddr;
993 inc.inc_faddr = inp->inp_inc.inc_faddr;
995 tcp_hc_updatemtu(&inc, mtu);
998 CTR6(
KTR_CXGBE,
"%s: tid %d, mtu_idx %u (%u), t_maxseg %u, emss %u",
1010 struct inpcb *inp = toep->
inp;
1013 KASSERT(inp != NULL, (
"%s: inp is NULL", __func__));
1014 INP_WLOCK_ASSERT(inp);
1016 (
"%s: CPL not pending already?", __func__));
1018 CTR6(
KTR_CXGBE,
"%s: tid %d, toep %p (0x%x), inp %p (0x%x)",
1019 __func__, toep->
tid, toep, toep->
flags, inp, inp->inp_flags);
1032 release_offload_resources(toep);
1034 if (!in_pcbrele_wlocked(inp))
1038 struct mtx *lock = mtx_pool_find(mtxpool_sleep, toep);
1090find_best_mtu_idx(
struct adapter *sc,
struct in_conninfo *inc,
1098 mss = s->
mss > 0 ? s->
mss : tcp_mssopt(inc);
1099 if (inc->inc_flags & INC_ISIPV6)
1100 mtu = mss +
sizeof(
struct ip6_hdr) + sizeof(struct tcphdr);
1102 mtu = mss +
sizeof(
struct ip) + sizeof(struct tcphdr);
1104 for (i = 0; i <
NMTUS - 1 && mtus[i + 1] <= mtu; i++)
1118 SOCKBUF_LOCK_ASSERT(&so->so_rcv);
1120 wnd = sbspace(&so->so_rcv);
1131 unsigned long space = sb_max;
1136 while (wscale < TCP_MAX_WINSHIFT && (TCP_MAXWIN << wscale) < space)
1173 return (htobe64(opt0));
1196 MPASS(cp->
sack == 0 || cp->
sack == 1);
1205 MPASS(cp->
ecn == 0 || cp->
ecn == 1);
1223#ifdef USE_DDP_RX_FLOW_CONTROL
1228 return (htobe32(opt2));
1236 uint64_t ntuple = 0;
1258 return (htobe32((uint32_t)ntuple));
1264is_tls_sock(
struct socket *so,
struct adapter *sc)
1266 struct inpcb *inp = sotoinpcb(so);
1288 struct in_conninfo *inc,
struct socket *so,
1294 struct inpcb *inp = sotoinpcb(so);
1295 struct tcpcb *tp = intotcpcb(inp);
1306 struct cc_algo *cc = CC_ALGO(tp);
1308 if (strcasecmp(cc->name,
"reno") == 0)
1310 else if (strcasecmp(cc->name,
"tahoe") == 0)
1312 if (strcasecmp(cc->name,
"newreno") == 0)
1314 if (strcasecmp(cc->name,
"highspeed") == 0)
1335 cp->
nagle = tp->t_flags & TF_NODELAY ? 0 : 1;
1338 if (V_tcp_always_keepalive || so_options_get(so) & SO_KEEPALIVE)
1354 (s->
tls > 0 || (s->
tls < 0 && is_tls_sock(so, sc))))
1356 else if (s->
ddp > 0 ||
1357 (s->
ddp < 0 && sc->
tt.
ddp && (so_options_get(so) & SO_NO_DDP) == 0))
1377 cp->
mtu_idx = find_best_mtu_idx(sc, inc, s);
1393 if (SOLISTENING(so)) {
1395 MPASS(tcpopt != NULL);
1399 (s->
tstamp > 0 || (s->
tstamp < 0 && V_tcp_do_rfc1323)))
1406 (s->
sack > 0 || (s->
sack < 0 && V_tcp_do_sack)))
1412 if (tcpopt->
wsf > 0 && tcpopt->
wsf < 15 && V_tcp_do_rfc1323)
1419 (s->
ecn > 0 || (s->
ecn < 0 && V_tcp_do_ecn)))
1429 else if (so->sol_sbsnd_flags & SB_AUTOSIZE &&
1430 V_tcp_do_autosndbuf)
1433 cp->
sndbuf = so->sol_sbsnd_hiwat;
1439 (s->
tstamp < 0 && (tp->t_flags & TF_REQ_TSTMP)))
1446 (s->
sack < 0 && (tp->t_flags & TF_SACK_PERMIT)))
1452 if (tp->t_flags & TF_REQ_SCALE)
1458 if (s->
ecn > 0 || (s->
ecn < 0 && V_tcp_do_ecn == 1))
1463 SOCKBUF_LOCK(&so->so_rcv);
1465 SOCKBUF_UNLOCK(&so->so_rcv);
1471 SOCKBUF_LOCK(&so->so_snd);
1472 if (so->so_snd.sb_flags & SB_AUTOSIZE &&
1473 V_tcp_do_autosndbuf)
1476 cp->
sndbuf = so->so_snd.sb_hiwat;
1477 SOCKBUF_UNLOCK(&so->so_snd);
1497alloc_tid_tab(
struct tid_info *t,
int flags)
1500 MPASS(t->
ntids > 0);
1517 (
"%s: %d tids still in use.", __func__, t->
tids_in_use));
1524alloc_stid_tab(
struct tid_info *t,
int flags)
1534 mtx_init(&t->stid_lock,
"stid lock", NULL, MTX_DEF);
1536 TAILQ_INIT(&t->
stids);
1547 (
"%s: %d tids still in use.", __func__, t->
stids_in_use));
1549 if (mtx_initialized(&t->stid_lock))
1550 mtx_destroy(&t->stid_lock);
1568 rc = alloc_tid_tab(t, M_NOWAIT);
1572 rc = alloc_stid_tab(t, M_NOWAIT);
1588 rw_init(&td->tcb_history_lock,
"TCB history");
1590 M_CXGBE, M_ZERO | M_NOWAIT);
1607 if (rw_initialized(&td->tcb_history_lock))
1608 rw_destroy(&td->tcb_history_lock);
1617 KASSERT(TAILQ_EMPTY(&td->toep_list),
1618 (
"%s: TOE PCB list is not empty.", __func__));
1620 (
"%s: lctx hash table is not empty.", __func__));
1625 hashdestroy(td->listen_hash, M_CXGBE, td->
listen_mask);
1634 free_tcb_history(sc, td);
1635 free_tid_tabs(&sc->
tids);
1640prepare_pkt(
int open_type, uint16_t vtag,
struct inpcb *inp,
int *pktlen,
1647 max(
sizeof(
struct ether_header),
sizeof(
struct ether_vlan_header)) +
1648 max(
sizeof(
struct ip),
sizeof(
struct ip6_hdr)) +
1649 sizeof(
struct tcphdr);
1653 pkt = malloc(maxlen, M_CXGBE, M_ZERO | M_NOWAIT);
1657 ipv6 = inp->inp_vflag & INP_IPV6;
1660 if (EVL_VLANOFTAG(vtag) == 0xfff) {
1661 struct ether_header *eh = (
void *)pkt;
1664 eh->ether_type = htons(ETHERTYPE_IPV6);
1666 eh->ether_type = htons(ETHERTYPE_IP);
1670 struct ether_vlan_header *evh = (
void *)pkt;
1672 evh->evl_encap_proto = htons(ETHERTYPE_VLAN);
1673 evh->evl_tag = htons(vtag);
1675 evh->evl_proto = htons(ETHERTYPE_IPV6);
1677 evh->evl_proto = htons(ETHERTYPE_IP);
1679 len +=
sizeof(*evh);
1683 struct ip6_hdr *ip6 = (
void *)&pkt[len];
1685 ip6->ip6_vfc = IPV6_VERSION;
1686 ip6->ip6_plen = htons(
sizeof(
struct tcphdr));
1687 ip6->ip6_nxt = IPPROTO_TCP;
1689 ip6->ip6_src = inp->in6p_laddr;
1690 ip6->ip6_dst = inp->in6p_faddr;
1692 ip6->ip6_src = inp->in6p_laddr;
1693 ip6->ip6_dst = ip6->ip6_src;
1696 len +=
sizeof(*ip6);
1698 struct ip *ip = (
void *)&pkt[len];
1700 ip->ip_v = IPVERSION;
1701 ip->ip_hl =
sizeof(*ip) >> 2;
1702 ip->ip_tos = inp->inp_ip_tos;
1703 ip->ip_len = htons(
sizeof(
struct ip) +
sizeof(
struct tcphdr));
1704 ip->ip_ttl = inp->inp_ip_ttl;
1705 ip->ip_p = IPPROTO_TCP;
1707 ip->ip_src = inp->inp_laddr;
1708 ip->ip_dst = inp->inp_faddr;
1710 ip->ip_src = inp->inp_laddr;
1711 ip->ip_dst = ip->ip_src;
1717 th = (
void *)&pkt[len];
1719 th->th_sport = inp->inp_lport;
1720 th->th_dport = inp->inp_fport;
1722 th->th_sport = inp->inp_lport;
1723 th->th_dport = th->th_sport;
1727 *pktlen = *buflen = len;
1733 uint16_t vtag,
struct inpcb *inp)
1738 int i, matched, pktlen, buflen;
1771 return (&disallow_offloading_settings);
1773 return (&allow_offloading_settings);
1776 switch (open_type) {
1779 pkt = prepare_pkt(open_type, vtag, inp, &pktlen, &buflen);
1783 pkt = mtod(m,
char *);
1791 return (&disallow_offloading_settings);
1794 if (pkt == NULL || pktlen == 0 || buflen == 0)
1795 return (&disallow_offloading_settings);
1799 for (i = 0; i < op->
nrules; i++,
r++) {
1800 if (
r->open_type != open_type &&
1804 matched = bpf_filter(
r->bpf_prog.bf_insns, pkt, pktlen, buflen);
1812 return (matched ? &
r->settings : &disallow_offloading_settings);
1816reclaim_wr_resources(
void *arg,
int count)
1821 u_int opcode, atid, tid;
1826 STAILQ_SWAP(&td->unsent_wr_list, &twr_list,
wrqe);
1829 while ((wr = STAILQ_FIRST(&twr_list)) != NULL) {
1830 STAILQ_REMOVE_HEAD(&twr_list, link);
1839 CTR2(
KTR_CXGBE,
"%s: atid %u ", __func__, atid);
1845 CTR2(
KTR_CXGBE,
"%s: tid %u ", __func__, tid);
1850 log(LOG_ERR,
"%s: leaked work request %p, wr_len %d, "
1851 "opcode %x\n", __func__, wr, wr->
wr_len, opcode);
1862t4_tom_activate(
struct adapter *sc)
1872 td = malloc(
sizeof(*td), M_CXGBE, M_ZERO | M_NOWAIT);
1878 TAILQ_INIT(&td->toep_list);
1886 mtx_init(&td->
unsent_wr_lock,
"Unsent WR list lock", NULL, MTX_DEF);
1887 STAILQ_INIT(&td->unsent_wr_list);
1891 rc = alloc_tid_tabs(&sc->
tids);
1902 alloc_tcb_history(sc, td);
1907 tod->tod_softc = sc;
1915 tod->tod_pcb_detach = t4_pcb_detach;
1921 tod->tod_ctloutput = t4_ctloutput;
1922 tod->tod_tcp_info = t4_tcp_info;
1924 tod->tod_alloc_tls_session = t4_alloc_tls_session;
1926 tod->tod_pmtu_update = t4_pmtu_update;
1930 TOEDEV(vi->
ifp) = &td->
tod;
1939 free_tom_data(sc, td);
1944t4_tom_deactivate(
struct adapter *sc)
1961 if (!TAILQ_EMPTY(&td->toep_list))
1972 if (!STAILQ_EMPTY(&td->unsent_wr_list))
1978 free_tom_data(sc, td);
1986t4_aio_queue_tom(
struct socket *so,
struct kaiocb *job)
1988 struct tcpcb *tp = so_sototcpcb(so);
1989 struct toepcb *toep = tp->t_toe;
1996 if (SOLISTENING(so))
2001 if (error != EOPNOTSUPP)
2009t4_tom_mod_load(
void)
2022 tcp_protosw = pffindproto(PF_INET, IPPROTO_TCP, SOCK_STREAM);
2023 if (tcp_protosw == NULL)
2024 return (ENOPROTOOPT);
2025 bcopy(tcp_protosw, &toe_protosw,
sizeof(toe_protosw));
2026 bcopy(tcp_protosw->pr_usrreqs, &toe_usrreqs,
sizeof(toe_usrreqs));
2027 toe_usrreqs.pru_aio_queue = t4_aio_queue_tom;
2028 toe_protosw.pr_usrreqs = &toe_usrreqs;
2030 tcp6_protosw = pffindproto(PF_INET6, IPPROTO_TCP, SOCK_STREAM);
2031 if (tcp6_protosw == NULL)
2032 return (ENOPROTOOPT);
2033 bcopy(tcp6_protosw, &toe6_protosw,
sizeof(toe6_protosw));
2034 bcopy(tcp6_protosw->pr_usrreqs, &toe6_usrreqs,
sizeof(toe6_usrreqs));
2035 toe6_usrreqs.pru_aio_queue = t4_aio_queue_tom;
2036 toe6_protosw.pr_usrreqs = &toe6_usrreqs;
2038 return (t4_register_uld(&tom_uld_info));
2042tom_uninit(
struct adapter *sc,
void *arg __unused)
2049 t4_deactivate_uld(sc,
ULD_TOM);
2055t4_tom_mod_unload(
void)
2059 if (t4_unregister_uld(&tom_uld_info) == EBUSY)
2083 rc = t4_tom_mod_load();
2087 rc = t4_tom_mod_unload();
2094 printf(
"t4_tom: compiled without TCP_OFFLOAD support.\n");
int begin_synchronized_op(struct adapter *, struct vi_info *, int, char *)
#define ADAPTER_UNLOCK(sc)
static uint32_t t4_read_reg(struct adapter *sc, uint32_t reg)
void t4_release_cl_rl(struct adapter *, int, int)
#define for_each_vi(_pi, _iter, _vi)
static int read_via_memwin(struct adapter *sc, int idx, uint32_t addr, uint32_t *val, int len)
void t4_register_cpl_handler(int, cpl_handler_t)
static void * wrtod(struct wrqe *wr)
void commit_wrq_wr(struct sge_wrq *, void *, struct wrq_cookie *)
STAILQ_HEAD(, wrqe) wr_list
void t4_register_shared_cpl_handler(int, cpl_handler_t, int)
void t4_iterate(void(*)(struct adapter *, void *), void *)
#define ASSERT_SYNCHRONIZED_OP(sc)
void release_tid(struct adapter *, int, struct sge_wrq *)
void end_synchronized_op(struct adapter *, int)
void * start_wrq_wr(struct sge_wrq *, int, struct wrq_cookie *)
void t4_set_reg_field(struct adapter *adap, unsigned int addr, u32 mask, u32 val)
static u_int tcp_ticks_to_us(const struct adapter *adap, u_int ticks)
static int chip_id(struct adapter *adap)
#define for_each_port(adapter, iter)
static int is_t4(struct adapter *adap)
static int port_top_speed(const struct port_info *pi)
#define INIT_TP_WR(w, tid)
#define INIT_ULPTX_WRH(w, wrlen, atomic, tid)
#define CH_ERR(adap, fmt,...)
unsigned short mtus[NMTUS]
unsigned int ofldq_wr_cred
struct t4_offload_policy * policy
struct adapter_params params
struct rwlock policy_lock
struct port_info * port[MAX_NPORTS]
struct fw_flowc_mnemval mnemval[0]
struct tx_sched_params * sched_params
struct sge_ofld_rxq * ofld_rxq
struct sge_ofld_txq * ofld_txq
struct offload_rule * rule
struct callout te_callout
struct adapter * te_adapter
uint64_t te_tcb[TCB_SIZE/sizeof(uint64_t)]
struct listen_ctx ** stid_tab
struct conn_params params
struct sge_ofld_rxq * ofld_rxq
struct mbufq ulp_pdu_reclaimq
struct sge_ofld_txq * ofld_txq
struct ofld_tx_sdesc txsd[]
struct mtx unsent_wr_lock
struct tcb_histent ** tcb_history
struct task reclaim_wr_resources
struct mtx toep_list_lock
struct mtx lctx_hash_lock
int update_hc_on_pmtu_change
int cop_managed_offloading
unsigned short tx_modq[MAX_NCHAN]
struct tx_cl_rl_params cl_rl[]
void t4_release_clip_entry(struct adapter *sc, struct clip_entry *ce)
#define LEN__SET_TCB_FIELD_ULP
static void * mk_set_tcb_field_ulp(struct ulp_txpkt *ulpmc, uint64_t word, uint64_t mask, uint64_t val, uint32_t tid, uint32_t qid)
static void t4_l2t_release(struct l2t_entry *e)
@ CPL_ERR_KEEPALV_NEG_ADVICE
@ CPL_ERR_PERSIST_NEG_ADVICE
#define F_CONG_CNTRL_VALID
#define F_RSS_QUEUE_VALID
#define V_ULP_TXPKT_DEST(x)
#define V_FILTER_TUPLE(x)
#define CPL_L2T_VLAN_NONE
#define V_RX_FC_DISABLE(x)
#define F_RX_COALESCE_VALID
#define MK_OPCODE_TID(opcode, tid)
#define A_TP_CMM_TCB_BASE
#define A_ULP_RX_TDDP_PSZ
#define G_DUPACKTHRESH(x)
#define A_ULP_RX_TDDP_TAGMASK
#define V_FT_VNID_ID_VF(x)
#define V_FT_VNID_ID_PF(x)
#define V_FT_VNID_ID_VLD(x)
#define V_TCB_TIMESTAMP(x)
#define V_TF_CCTRL_ECN(x)
#define V_TF_CCTRL_ECE(x)
#define V_TF_RECV_SCALE(x)
#define V_TCB_T_MAXSEG(x)
#define V_TF_CCTRL_RFR(x)
#define V_TF_CCTRL_CWR(x)
static int t4_tom_modevent(module_t mod, int cmd, void *arg)
MODULE_DEPEND(t4_tom, toecore, 1, 1, 1)
DECLARE_MODULE(t4_tom, t4_tom_moddata, SI_SUB_EXEC, SI_ORDER_ANY)
MODULE_VERSION(t4_tom, 1)
static moduledata_t t4_tom_moddata
void t4_free_ppod_region(struct ppod_region *)
int t4_send_fin(struct toedev *, struct tcpcb *)
void t4_uninit_listen_cpl_handlers(void)
void tls_detach(struct toepcb *)
void * lookup_tid(struct adapter *, int)
static int ulp_mode(struct toepcb *toep)
void t4_ddp_mod_load(void)
#define TS_SND_BACKLOGGED
void t4_syncache_removed(struct toedev *, void *)
void tls_uninit_toep(struct toepcb *)
int tls_alloc_ktls(struct toepcb *, struct ktls_session *, int)
void t4_init_cpl_io_handlers(void)
void ddp_uninit_toep(struct toepcb *)
int t4_tod_output(struct toedev *, struct tcpcb *)
void act_open_failure_cleanup(struct adapter *, u_int, u_int)
uint64_t select_ntuple(struct vi_info *, struct l2t_entry *)
void t4_ddp_mod_unload(void)
struct toepcb * hold_toepcb(struct toepcb *)
void remove_tid(struct adapter *, int, int)
void t4_set_tcb_field(struct adapter *, struct sge_wrq *, struct toepcb *, uint16_t, uint64_t, uint64_t, int, int)
int init_toepcb(struct vi_info *, struct toepcb *)
void insert_tid(struct adapter *, int, void *, int)
int t4_listen_start(struct toedev *, struct tcpcb *)
int add_tid_to_history(struct adapter *, u_int)
void t4_tls_mod_load(void)
__be64 calc_options0(struct vi_info *, struct conn_params *)
int t4_syncache_respond(struct toedev *, void *, struct mbuf *)
void free_toepcb(struct toepcb *)
void ddp_init_toep(struct toepcb *)
int t4_listen_stop(struct toedev *, struct tcpcb *)
void t4_tls_mod_unload(void)
void t4_init_listen_cpl_handlers(void)
int t4_send_rst(struct toedev *, struct tcpcb *)
void t4_offload_socket(struct toedev *, void *, struct socket *)
void init_conn_params(struct vi_info *, struct offload_settings *, struct in_conninfo *, struct socket *, const struct tcp_options *, int16_t, struct conn_params *cp)
int t4_connect(struct toedev *, struct socket *, struct nhop_object *, struct sockaddr *)
int t4_aio_queue_ddp(struct socket *, struct kaiocb *)
int select_rcv_wscale(void)
void t4_init_connect_cpl_handlers(void)
void t4_uninit_connect_cpl_handlers(void)
void t4_uninit_cpl_io_handlers(void)
void release_ddp_resources(struct toepcb *toep)
void offload_socket(struct socket *, struct toepcb *)
u_long select_rcv_wnd(struct socket *)
void synack_failure_cleanup(struct adapter *, int)
void final_cpl_received(struct toepcb *)
void undo_offload_socket(struct socket *)
void ddp_assert_empty(struct toepcb *)
void aiotx_init_toep(struct toepcb *)
__be32 calc_options2(struct vi_info *, struct conn_params *)
void t4_syncache_added(struct toedev *, void *)
bool can_tls_offload(struct adapter *)
static struct adapter * td_adapter(struct tom_data *td)
int t4_aio_queue_aiotx(struct socket *, struct kaiocb *)
struct toepcb * alloc_toepcb(struct vi_info *, int)
void tls_init_toep(struct toepcb *)
int t4_init_ppod_region(struct ppod_region *, struct t4_range *, u_int, const char *)
void restore_so_proto(struct socket *, bool)
void t4_rcvd(struct toedev *, struct tcpcb *)
void update_tid(struct adapter *, int, void *)
const struct offload_settings * lookup_offload_policy(struct adapter *, int, struct mbuf *, uint16_t, struct inpcb *)
int do_l2t_write_rpl2(struct sge_iq *, const struct rss_header *, struct mbuf *)
void t4_l2_update(struct toedev *, struct ifnet *, struct sockaddr *, uint8_t *, uint16_t)
#define V_FW_FLOWC_WR_NPARAMS(x)
#define V_FW_WR_FLOWID(x)