45#include <sys/capsicum.h>
46#include <sys/condvar.h>
49#include <sys/kernel.h>
50#include <sys/kthread.h>
55#include <sys/module.h>
56#include <sys/protosw.h>
57#include <sys/socket.h>
58#include <sys/socketvar.h>
59#include <sys/sysctl.h>
63#include <machine/bus.h>
65#include <vm/vm_page.h>
67#include <netinet/in.h>
68#include <netinet/in_pcb.h>
69#include <netinet/tcp.h>
70#include <netinet/tcp_var.h>
71#include <netinet/toecore.h>
73#include <dev/iscsi/icl.h>
74#include <dev/iscsi/iscsi_proto.h>
75#include <icl_conn_if.h>
77#include <cam/scsi/scsi_all.h>
78#include <cam/scsi/scsi_da.h>
79#include <cam/ctl/ctl_io.h>
80#include <cam/ctl/ctl.h>
81#include <cam/ctl/ctl_backend.h>
82#include <cam/ctl/ctl_error.h>
83#include <cam/ctl/ctl_frontend.h>
84#include <cam/ctl/ctl_debug.h>
85#include <cam/ctl/ctl_ha.h>
86#include <cam/ctl/ctl_ioctl.h>
89#include <cam/cam_ccb.h>
90#include <cam/cam_xpt.h>
91#include <cam/cam_debug.h>
92#include <cam/cam_sim.h>
93#include <cam/cam_xpt_sim.h>
94#include <cam/cam_xpt_periph.h>
95#include <cam/cam_periph.h>
96#include <cam/cam_compat.h>
97#include <cam/scsi/scsi_message.h>
108#define TT_HASH(icc, tt) (G_PPOD_TAG(tt) & (icc)->cmp_hash_mask)
110struct cxgbei_ddp_state {
117SYSCTL_NODE(_kern_icl, OID_AUTO, cxgbei, CTLFLAG_RD | CTLFLAG_MPSAFE, 0,
118 "Chelsio iSCSI offload");
119static int first_burst_length = 8192;
120SYSCTL_INT(_kern_icl_cxgbei, OID_AUTO, first_burst_length, CTLFLAG_RWTUN,
121 &first_burst_length, 0,
"First burst length");
122static int max_burst_length = 2 * 1024 * 1024;
123SYSCTL_INT(_kern_icl_cxgbei, OID_AUTO, max_burst_length, CTLFLAG_RWTUN,
124 &max_burst_length, 0,
"Maximum burst length");
125static int sendspace = 1048576;
126SYSCTL_INT(_kern_icl_cxgbei, OID_AUTO, sendspace, CTLFLAG_RWTUN,
127 &sendspace, 0,
"Default send socket buffer size");
128static int recvspace = 1048576;
129SYSCTL_INT(_kern_icl_cxgbei, OID_AUTO, recvspace, CTLFLAG_RWTUN,
130 &recvspace, 0,
"Default receive socket buffer size");
132static volatile u_int icl_cxgbei_ncons;
134static icl_conn_new_pdu_t icl_cxgbei_conn_new_pdu;
135static icl_conn_pdu_data_segment_length_t
136 icl_cxgbei_conn_pdu_data_segment_length;
137static icl_conn_pdu_append_bio_t icl_cxgbei_conn_pdu_append_bio;
138static icl_conn_pdu_append_data_t icl_cxgbei_conn_pdu_append_data;
139static icl_conn_pdu_get_bio_t icl_cxgbei_conn_pdu_get_bio;
140static icl_conn_pdu_get_data_t icl_cxgbei_conn_pdu_get_data;
141static icl_conn_pdu_queue_t icl_cxgbei_conn_pdu_queue;
142static icl_conn_pdu_queue_cb_t icl_cxgbei_conn_pdu_queue_cb;
143static icl_conn_handoff_t icl_cxgbei_conn_handoff;
144static icl_conn_free_t icl_cxgbei_conn_free;
145static icl_conn_close_t icl_cxgbei_conn_close;
146static icl_conn_task_setup_t icl_cxgbei_conn_task_setup;
147static icl_conn_task_done_t icl_cxgbei_conn_task_done;
148static icl_conn_transfer_setup_t icl_cxgbei_conn_transfer_setup;
149static icl_conn_transfer_done_t icl_cxgbei_conn_transfer_done;
151static kobj_method_t icl_cxgbei_methods[] = {
152 KOBJMETHOD(icl_conn_new_pdu, icl_cxgbei_conn_new_pdu),
154 KOBJMETHOD(icl_conn_pdu_data_segment_length,
155 icl_cxgbei_conn_pdu_data_segment_length),
156 KOBJMETHOD(icl_conn_pdu_append_bio, icl_cxgbei_conn_pdu_append_bio),
157 KOBJMETHOD(icl_conn_pdu_append_data, icl_cxgbei_conn_pdu_append_data),
158 KOBJMETHOD(icl_conn_pdu_get_bio, icl_cxgbei_conn_pdu_get_bio),
159 KOBJMETHOD(icl_conn_pdu_get_data, icl_cxgbei_conn_pdu_get_data),
160 KOBJMETHOD(icl_conn_pdu_queue, icl_cxgbei_conn_pdu_queue),
161 KOBJMETHOD(icl_conn_pdu_queue_cb, icl_cxgbei_conn_pdu_queue_cb),
162 KOBJMETHOD(icl_conn_handoff, icl_cxgbei_conn_handoff),
163 KOBJMETHOD(icl_conn_free, icl_cxgbei_conn_free),
164 KOBJMETHOD(icl_conn_close, icl_cxgbei_conn_close),
165 KOBJMETHOD(icl_conn_task_setup, icl_cxgbei_conn_task_setup),
166 KOBJMETHOD(icl_conn_task_done, icl_cxgbei_conn_task_done),
167 KOBJMETHOD(icl_conn_transfer_setup, icl_cxgbei_conn_transfer_setup),
168 KOBJMETHOD(icl_conn_transfer_done, icl_cxgbei_conn_transfer_done),
172DEFINE_CLASS(icl_cxgbei, icl_cxgbei_methods,
sizeof(
struct icl_cxgbei_conn));
179 KASSERT(icp->
ref_cnt != 0, (
"freeing deleted PDU"));
181 MPASS(ic ==
ip->ip_conn);
183 m_freem(
ip->ip_ahs_mbuf);
184 m_freem(
ip->ip_data_mbuf);
185 m_freem(
ip->ip_bhs_mbuf);
187 KASSERT(ic != NULL || icp->
ref_cnt == 1,
188 (
"orphaned PDU has oustanding references"));
190 if (atomic_fetchadd_int(&icp->
ref_cnt, -1) != 1)
195 if (__predict_true(ic != NULL))
196 refcount_release(&ic->ic_outstanding_pdus);
201icl_cxgbei_pdu_call_cb(
struct icl_pdu *
ip)
210 if (__predict_true(
ip->ip_conn != NULL))
211 refcount_release(&
ip->ip_conn->ic_outstanding_pdus);
217icl_cxgbei_pdu_done(
struct icl_pdu *
ip,
int error)
224 m_freem(
ip->ip_ahs_mbuf);
225 ip->ip_ahs_mbuf = NULL;
226 m_freem(
ip->ip_data_mbuf);
227 ip->ip_data_mbuf = NULL;
228 m_freem(
ip->ip_bhs_mbuf);
229 ip->ip_bhs_mbuf = NULL;
235 if (atomic_fetchadd_int(&icp->
ref_cnt, -1) == 1)
236 icl_cxgbei_pdu_call_cb(
ip);
238 __assert_unreachable();
242icl_cxgbei_mbuf_done(
struct mbuf *mb)
252 icl_cxgbei_pdu_call_cb(&icp->
ip);
262 icp = malloc(
sizeof(*icp), M_CXGBEI, flags | M_ZERO);
263 if (__predict_false(icp == NULL))
270 m = m_gethdr(flags, MT_DATA);
271 if (__predict_false(m == NULL)) {
277 ip->ip_bhs = mtod(m,
struct iscsi_bhs *);
278 memset(ip->ip_bhs, 0,
sizeof(*ip->ip_bhs));
279 m->m_len =
sizeof(
struct iscsi_bhs);
280 m->m_pkthdr.len = m->m_len;
291 refcount_acquire(&ic->ic_outstanding_pdus);
298static struct icl_pdu *
299icl_cxgbei_conn_new_pdu(
struct icl_conn *ic,
int flags)
304 if (__predict_false(ip == NULL))
312icl_pdu_data_segment_length(
const struct icl_pdu *request)
316 len += request->ip_bhs->bhs_data_segment_len[0];
318 len += request->ip_bhs->bhs_data_segment_len[1];
320 len += request->ip_bhs->bhs_data_segment_len[2];
326icl_cxgbei_conn_pdu_data_segment_length(
struct icl_conn *ic,
327 const struct icl_pdu *request)
330 return (icl_pdu_data_segment_length(request));
336 struct icl_pdu *ip = &icp->
ip;
337 uint8_t ulp_submode, padding;
338 struct mbuf *m, *last;
339 struct iscsi_bhs *bhs;
345 m = ip->ip_data_mbuf;
354 padding = roundup2(ip->ip_data_len, 4) - ip->ip_data_len;
356 MPASS(padding <= M_TRAILINGSPACE(last));
357 bzero(mtod(last, uint8_t *) + last->m_len, padding);
358 last->m_len += padding;
361 MPASS(ip->ip_data_len == 0);
362 ulp_submode &= ~ULP_CRC_DATA;
370 MPASS(m->m_pkthdr.len ==
sizeof(
struct iscsi_bhs));
371 MPASS(m->m_len ==
sizeof(
struct iscsi_bhs));
374 data_len = ip->ip_data_len;
375 if (data_len > icc->
ic.ic_max_send_data_segment_length) {
376 struct iscsi_bhs_data_in *bhsdi;
379 KASSERT(padding == 0, (
"%s: ISO with padding %d for icp %p",
380 __func__, padding, icp));
381 switch (bhs->bhs_opcode) {
382 case ISCSI_BHS_OPCODE_SCSI_DATA_OUT:
385 case ISCSI_BHS_OPCODE_SCSI_DATA_IN:
389 panic(
"invalid opcode %#x for ISO", bhs->bhs_opcode);
391 data_len = icc->
ic.ic_max_send_data_segment_length;
392 bhsdi = (
struct iscsi_bhs_data_in *)bhs;
393 if (bhsdi->bhsdi_flags & BHSDI_FLAGS_F) {
399 bhsdi->bhsdi_flags &= ~BHSDI_FLAGS_F;
406 bhs->bhs_data_segment_len[2] = data_len;
407 bhs->bhs_data_segment_len[1] = data_len >> 8;
408 bhs->bhs_data_segment_len[0] = data_len >> 16;
413 m->m_pkthdr.len += ip->ip_data_len + padding;
414 m->m_next = ip->ip_data_mbuf;
416 ip->ip_bhs_mbuf = NULL;
417 ip->ip_data_mbuf = NULL;
424 if (atomic_fetchadd_int(&icp->
ref_cnt, -1) == 1)
425 icl_cxgbei_pdu_call_cb(ip);
431icl_cxgbei_tx_main(
void *arg)
433 struct epoch_tracker et;
435 struct icl_conn *ic = &icc->
ic;
437 struct socket *so = ic->ic_socket;
438 struct inpcb *inp = sotoinpcb(so);
442 STAILQ_HEAD(, icl_pdu) tx_pdus = STAILQ_HEAD_INITIALIZER(tx_pdus);
444 mbufq_init(&mq, INT_MAX);
447 while (__predict_true(!ic->ic_disconnecting)) {
448 while (STAILQ_EMPTY(&icc->sent_pdus)) {
450 mtx_sleep(&icc->
tx_active, ic->ic_lock, 0,
"-", 0);
451 if (__predict_false(ic->ic_disconnecting))
456 STAILQ_SWAP(&icc->sent_pdus, &tx_pdus, icl_pdu);
459 while ((ip = STAILQ_FIRST(&tx_pdus)) != NULL) {
460 STAILQ_REMOVE_HEAD(&tx_pdus, ip_next);
464 MPASS((m->m_pkthdr.len & 3) == 0);
466 mbufq_enqueue(&mq, m);
470 if (__predict_false(ic->ic_disconnecting) ||
471 __predict_false(ic->ic_socket == NULL)) {
476 CURVNET_SET(toep->
vnet);
481 if (__predict_false(inp->inp_flags & (INP_DROPPED |
502icl_cxgbei_rx_main(
void *arg)
505 struct icl_conn *ic = &icc->
ic;
508 STAILQ_HEAD(, icl_pdu) rx_pdus = STAILQ_HEAD_INITIALIZER(rx_pdus);
511 sb = &ic->ic_socket->so_rcv;
513 while (__predict_true(!ic->ic_disconnecting)) {
514 while (STAILQ_EMPTY(&icc->rcvd_pdus)) {
516 mtx_sleep(&icc->
rx_active, SOCKBUF_MTX(sb), 0,
"-", 0);
517 if (__predict_false(ic->ic_disconnecting))
522 if (__predict_false(sbused(sb)) != 0) {
531 cantrcvmore = (sb->sb_state & SBS_CANTRCVMORE) != 0;
532 MPASS(STAILQ_EMPTY(&rx_pdus));
533 STAILQ_SWAP(&icc->rcvd_pdus, &rx_pdus, icl_pdu);
537 while ((ip = STAILQ_FIRST(&rx_pdus)) != NULL) {
538 STAILQ_REMOVE_HEAD(&rx_pdus, ip_next);
540 icl_cxgbei_pdu_done(ip, ENOTCONN);
555 mtx_sleep(&icc->
rx_active, SOCKBUF_MTX(sb), 0,
"-", 0);
562cxgbei_free_mext_pg(
struct mbuf *m)
574 icp = m->m_ext.ext_arg1;
575 if (atomic_fetchadd_int(&icp->
ref_cnt, -1) == 1)
576 icl_cxgbei_pdu_call_cb(&icp->
ip);
580cxgbei_getm(
size_t len,
int flags)
582 struct mbuf *m, *m0, *m_tail;
587 while (len >= MJUM16BYTES) {
588 m = m_getjcl(M_NOWAIT, MT_DATA, 0, MJUM16BYTES);
589 if (__predict_false(m == NULL)) {
590 if ((flags & M_WAITOK) != 0) {
607 m = m_getm2(NULL, len, flags, MT_DATA, 0);
608 if (__predict_false(m == NULL)) {
622icl_cxgbei_conn_pdu_append_bio(
struct icl_conn *ic,
struct icl_pdu *ip,
623 struct bio *bp,
size_t offset,
size_t len,
int flags)
626 struct mbuf *m, *m_tail;
628 size_t page_offset, todo, mtodo;
633 MPASS(ic == ip->ip_conn);
634 KASSERT(len > 0, (
"%s: len is %jd", __func__, (intmax_t)len));
636 m_tail = ip->ip_data_mbuf;
638 for (; m_tail->m_next != NULL; m_tail = m_tail->m_next)
641 MPASS(bp->bio_flags & BIO_UNMAPPED);
642 if (offset < PAGE_SIZE - bp->bio_ma_offset) {
643 page_offset = bp->bio_ma_offset + offset;
646 offset -= PAGE_SIZE - bp->bio_ma_offset;
647 for (i = 1; offset >= PAGE_SIZE; i++)
649 page_offset = offset;
652 if (flags & ICL_NOCOPY) {
656 m = mb_alloc_ext_pgs(flags & ~ICL_NOCOPY,
657 cxgbei_free_mext_pg);
658 if (__predict_false(m == NULL))
660 atomic_add_int(&icp->
ref_cnt, 1);
661 m->m_ext.ext_arg1 = icp;
662 m->m_epg_1st_off = page_offset;
665 todo = MIN(len, PAGE_SIZE - page_offset);
667 m->m_epg_pa[m->m_epg_npgs] =
668 VM_PAGE_TO_PHYS(bp->bio_ma[i]);
670 m->m_epg_last_len = todo;
672 m->m_ext.ext_size += PAGE_SIZE;
673 MBUF_EXT_PGS_ASSERT_SANITY(m);
675 if (m->m_epg_npgs == MBUF_PEXT_MAX_PGS) {
679 ip->ip_data_mbuf = m;
681 ip->ip_data_len += m->m_len;
694 ip->ip_data_mbuf = m;
695 ip->ip_data_len += m->m_len;
700 m = cxgbei_getm(len, flags);
701 if (__predict_false(m == NULL))
704 if (ip->ip_data_mbuf == NULL) {
705 ip->ip_data_mbuf = m;
706 ip->ip_data_len = len;
709 ip->ip_data_len += len;
713 todo = MIN(len, PAGE_SIZE - page_offset);
715 mapped = pmap_map_io_transient(bp->bio_ma + i, &vaddr, 1,
719 mtodo = min(todo, M_SIZE(m) - m->m_len);
720 memcpy(mtod(m,
char *) + m->m_len, (
char *)vaddr +
723 if (m->m_len == M_SIZE(m))
725 page_offset += mtodo;
729 if (__predict_false(mapped))
730 pmap_unmap_io_transient(bp->bio_ma + 1, &vaddr, 1,
738 MPASS(ip->ip_data_len <= max(ic->ic_max_send_data_segment_length,
745icl_cxgbei_conn_pdu_append_data(
struct icl_conn *ic,
struct icl_pdu *ip,
746 const void *addr,
size_t len,
int flags)
749 struct mbuf *m, *m_tail;
753 MPASS(ic == ip->ip_conn);
754 KASSERT(len > 0, (
"%s: len is %jd", __func__, (intmax_t)len));
756 m_tail = ip->ip_data_mbuf;
758 for (; m_tail->m_next != NULL; m_tail = m_tail->m_next)
761 if (flags & ICL_NOCOPY) {
762 m = m_get(flags & ~ICL_NOCOPY, MT_DATA);
764 ICL_WARN(
"failed to allocate mbuf");
768 m->m_flags |= M_RDONLY;
769 m_extaddref(m, __DECONST(
char *, addr), len, &icp->
ref_cnt,
770 icl_cxgbei_mbuf_done, icp, NULL);
772 if (ip->ip_data_mbuf == NULL) {
773 ip->ip_data_mbuf = m;
774 ip->ip_data_len = len;
777 m_tail = m_tail->m_next;
778 ip->ip_data_len += len;
784 m = cxgbei_getm(len, flags);
785 if (__predict_false(m == NULL))
788 if (ip->ip_data_mbuf == NULL) {
789 ip->ip_data_mbuf = m;
790 ip->ip_data_len = len;
793 ip->ip_data_len += len;
795 src = (
const char *)addr;
796 for (; m != NULL; m = m->m_next) {
797 m->m_len = min(len, M_SIZE(m));
798 memcpy(mtod(m,
void *), src, m->m_len);
804 MPASS(ip->ip_data_len <= max(ic->ic_max_send_data_segment_length,
811icl_cxgbei_conn_pdu_get_bio(
struct icl_conn *ic,
struct icl_pdu *ip,
812 size_t pdu_off,
struct bio *bp,
size_t bio_off,
size_t len)
816 size_t page_offset, todo;
823 MPASS(bp->bio_flags & BIO_UNMAPPED);
824 if (bio_off < PAGE_SIZE - bp->bio_ma_offset) {
825 page_offset = bp->bio_ma_offset + bio_off;
828 bio_off -= PAGE_SIZE - bp->bio_ma_offset;
829 for (i = 1; bio_off >= PAGE_SIZE; i++)
830 bio_off -= PAGE_SIZE;
831 page_offset = bio_off;
835 todo = MIN(len, PAGE_SIZE - page_offset);
837 mapped = pmap_map_io_transient(bp->bio_ma + i, &vaddr, 1,
839 m_copydata(
ip->ip_data_mbuf, pdu_off, todo, (
char *)vaddr +
841 if (__predict_false(mapped))
842 pmap_unmap_io_transient(bp->bio_ma + 1, &vaddr, 1,
853icl_cxgbei_conn_pdu_get_data(
struct icl_conn *ic,
struct icl_pdu *
ip,
854 size_t off,
void *addr,
size_t len)
860 m_copydata(
ip->ip_data_mbuf, off, len, addr);
864icl_cxgbei_conn_pdu_queue(
struct icl_conn *ic,
struct icl_pdu *
ip)
866 icl_cxgbei_conn_pdu_queue_cb(ic,
ip, NULL);
870icl_cxgbei_conn_pdu_queue_cb(
struct icl_conn *ic,
struct icl_pdu *
ip,
875 struct socket *so = ic->ic_socket;
877 MPASS(ic == ip->ip_conn);
878 MPASS(ip->ip_bhs_mbuf != NULL);
880 MPASS(ip->ip_ahs_mbuf == NULL && ip->ip_ahs_len == 0);
882 ICL_CONN_LOCK_ASSERT(ic);
887 if (ic->ic_disconnecting || so == NULL || !sowriteable(so)) {
888 icl_cxgbei_pdu_done(ip, ENOTCONN);
892 STAILQ_INSERT_TAIL(&icc->sent_pdus, ip, ip_next);
899static struct icl_conn *
900icl_cxgbei_new_conn(
const char *name,
struct mtx *lock)
905 refcount_acquire(&icl_cxgbei_ncons);
907 icc = (
struct icl_cxgbei_conn *)kobj_create(&icl_cxgbei_class, M_CXGBE,
910 STAILQ_INIT(&icc->rcvd_pdus);
911 STAILQ_INIT(&icc->sent_pdus);
914 mtx_init(&icc->
cmp_lock,
"cxgbei_cmp", NULL, MTX_DEF);
920 refcount_init(&
ic->ic_outstanding_pdus, 0);
923 ic->ic_offload =
"cxgbei";
924 ic->ic_unmapped =
true;
926 CTR2(
KTR_CXGBE,
"%s: icc %p", __func__, icc);
932icl_cxgbei_conn_free(
struct icl_conn *
ic)
938 CTR2(
KTR_CXGBE,
"%s: icc %p", __func__, icc);
942 kobj_delete((
struct kobj *)icc, M_CXGBE);
943 refcount_release(&icl_cxgbei_ncons);
947icl_cxgbei_setsockopt(
struct icl_conn *
ic,
struct socket *so,
int sspace,
951 int error, one = 1,
ss, rs;
953 ss = max(sendspace, sspace);
954 rs = max(recvspace, rspace);
956 error = soreserve(so,
ss, rs);
959 SOCKBUF_LOCK(&so->so_snd);
960 so->so_snd.sb_flags |= SB_AUTOSIZE;
961 SOCKBUF_UNLOCK(&so->so_snd);
962 SOCKBUF_LOCK(&so->so_rcv);
963 so->so_rcv.sb_flags |= SB_AUTOSIZE;
964 SOCKBUF_UNLOCK(&so->so_rcv);
969 bzero(&opt,
sizeof(opt));
970 opt.sopt_dir = SOPT_SET;
971 opt.sopt_level = IPPROTO_TCP;
972 opt.sopt_name = TCP_NODELAY;
974 opt.sopt_valsize =
sizeof(one);
975 error = sosetopt(so, &opt);
985struct find_ofld_adapter_rr {
991find_offload_adapter(
struct adapter *sc,
void *arg)
993 struct find_ofld_adapter_rr *fa = arg;
994 struct socket *so = fa->so;
1000 MPASS(so->so_proto->pr_protocol == IPPROTO_TCP);
1008 inp = sotoinpcb(so);
1010 if ((inp->inp_flags & (INP_DROPPED | INP_TIMEWAIT)) == 0) {
1011 tp = intotcpcb(inp);
1012 if (tp->t_flags & TF_TOE && tp->tod == &td->
tod)
1033send_iscsi_flowc_wr(
struct adapter *sc,
struct toepcb *toep,
int maxlen)
1037 const u_int nparams = 1;
1046 panic(
"%s: allocation failure.", __func__);
1049 memset(flowc, 0, wr->
wr_len);
1062 (
"%s: not enough credits (%d)", __func__, toep->
tx_credits));
1072set_ulp_mode_iscsi(
struct adapter *sc,
struct toepcb *toep, u_int ulp_submode)
1076 CTR3(
KTR_CXGBE,
"%s: tid %u, ULP_MODE_ISCSI, submode=%#x",
1077 __func__, toep->
tid, ulp_submode);
1096icl_cxgbei_conn_handoff(
struct icl_conn *ic,
int fd)
1099 struct find_ofld_adapter_rr fa;
1105 cap_rights_t rights;
1106 u_int max_rx_pdu_len, max_tx_pdu_len;
1107 int error, max_iso_pdus;
1110 ICL_CONN_LOCK_ASSERT_NOT(ic);
1115 error = fget(curthread, fd,
1116 cap_rights_init_one(&rights, CAP_SOCK_CLIENT), &fp);
1119 if (fp->f_type != DTYPE_SOCKET) {
1120 fdrop(fp, curthread);
1124 if (so->so_type != SOCK_STREAM ||
1125 so->so_proto->pr_protocol != IPPROTO_TCP) {
1126 fdrop(fp, curthread);
1131 if (ic->ic_socket != NULL) {
1132 ICL_CONN_UNLOCK(ic);
1133 fdrop(fp, curthread);
1136 ic->ic_disconnecting =
false;
1138 fp->f_ops = &badfileops;
1140 fdrop(fp, curthread);
1141 ICL_CONN_UNLOCK(ic);
1147 if (fa.sc == NULL) {
1153 max_rx_pdu_len = ISCSI_BHS_SIZE + ic->ic_max_recv_data_segment_length;
1154 max_tx_pdu_len = ISCSI_BHS_SIZE + ic->ic_max_send_data_segment_length;
1155 if (ic->ic_header_crc32c) {
1156 max_rx_pdu_len += ISCSI_HEADER_DIGEST_SIZE;
1157 max_tx_pdu_len += ISCSI_HEADER_DIGEST_SIZE;
1159 if (ic->ic_data_crc32c) {
1160 max_rx_pdu_len += ISCSI_DATA_DIGEST_SIZE;
1161 max_tx_pdu_len += ISCSI_DATA_DIGEST_SIZE;
1164 inp = sotoinpcb(so);
1166 tp = intotcpcb(
inp);
1167 if (
inp->inp_flags & (INP_DROPPED | INP_TIMEWAIT)) {
1176 MPASS(tp->t_flags & TF_TOE);
1177 MPASS(tp->tod != NULL);
1178 MPASS(tp->t_toe != NULL);
1191 if (ic->ic_header_crc32c)
1193 if (ic->ic_data_crc32c)
1197 !is_memfree(icc->
sc)) {
1199 ic->ic_hw_isomax = max_iso_pdus *
1200 ic->ic_max_send_data_segment_length;
1207 send_iscsi_flowc_wr(icc->
sc, toep,
1208 roundup(max_iso_pdus * max_tx_pdu_len, tp->t_maxseg));
1212 error = kthread_add(icl_cxgbei_tx_main, icc, NULL, &icc->
tx_thread, 0,
1213 0,
"%stx (cxgbei)", ic->ic_name);
1217 error = kthread_add(icl_cxgbei_rx_main, icc, NULL, &icc->
rx_thread, 0,
1218 0,
"%srx (cxgbei)", ic->ic_name);
1222 error = icl_cxgbei_setsockopt(ic, so, max_tx_pdu_len, max_rx_pdu_len);
1225 icl_cxgbei_conn_close(ic);
1230icl_cxgbei_conn_close(
struct icl_conn *ic)
1240 ICL_CONN_LOCK_ASSERT_NOT(ic);
1244 if (ic->ic_disconnecting || so == NULL) {
1245 CTR4(
KTR_CXGBE,
"%s: icc %p (disconnecting = %d), so %p",
1246 __func__, icc, ic->ic_disconnecting, so);
1247 ICL_CONN_UNLOCK(ic);
1250 ic->ic_disconnecting =
true;
1253 KASSERT(ic->ic_outstanding_pdus == 0,
1254 (
"destroying session with %d outstanding PDUs",
1255 ic->ic_outstanding_pdus));
1258 CTR3(
KTR_CXGBE,
"%s: tid %d, icc %p", __func__, toep ? toep->
tid : -1,
1267 mtx_sleep(icc->
tx_thread, ic->ic_lock, 0,
"conclo", 0);
1271 while (!STAILQ_EMPTY(&icc->sent_pdus)) {
1272 ip = STAILQ_FIRST(&icc->sent_pdus);
1273 STAILQ_REMOVE_HEAD(&icc->sent_pdus, ip_next);
1274 icl_cxgbei_pdu_done(ip, ENOTCONN);
1276 ICL_CONN_UNLOCK(ic);
1278 inp = sotoinpcb(so);
1289 mtx_sleep(icc->
rx_thread, SOCKBUF_MTX(sb), 0,
"conclo", 0);
1295 while (!STAILQ_EMPTY(&icc->rcvd_pdus)) {
1296 ip = STAILQ_FIRST(&icc->rcvd_pdus);
1297 STAILQ_REMOVE_HEAD(&icc->rcvd_pdus, ip_next);
1298 icl_cxgbei_pdu_done(ip, ENOTCONN);
1315 if (toep->
inp != NULL) {
1324 ic->ic_socket = NULL;
1325 ICL_CONN_UNLOCK(ic);
1342 struct mtx *lock = mtx_pool_find(mtxpool_sleep, toep);
1346 mtx_sleep(toep, lock, PSOCK,
"conclo2", 0);
1364 LIST_FOREACH(cmp2, &icc->
cmp_table[TT_HASH(icc,
tt)], link) {
1365 KASSERT(cmp2->
tt !=
tt, (
"%s: duplicate cmp", __func__));
1368 LIST_INSERT_HEAD(&icc->
cmp_table[TT_HASH(icc,
tt)], cmp, link);
1378 LIST_FOREACH(cmp, &icc->
cmp_table[TT_HASH(icc,
tt)], link) {
1396 LIST_FOREACH(cmp2, &icc->
cmp_table[TT_HASH(icc, cmp->
tt)], link) {
1400 panic(
"%s: could not find cmp", __func__);
1403 LIST_REMOVE(cmp, link);
1408icl_cxgbei_conn_task_setup(
struct icl_conn *ic,
struct icl_pdu *ip,
1409 struct ccb_scsiio *csio, uint32_t *ittp,
void **arg)
1416 struct cxgbei_ddp_state *ddp;
1423 ICL_CONN_LOCK_ASSERT(ic);
1427 MPASS(*arg == NULL);
1429 if ((csio->ccb_h.flags & CAM_DIR_MASK) != CAM_DIR_IN ||
1430 csio->dxfer_len < ci->
ddp_threshold || ic->ic_disconnecting ||
1431 ic->ic_socket == NULL) {
1440 *ittp = htobe32(itt);
1441 MPASS(*arg == NULL);
1452 ddp = malloc(
sizeof(*ddp), M_CXGBEI, M_NOWAIT | M_ZERO);
1459 mbufq_init(&mq, INT_MAX);
1460 switch (csio->ccb_h.flags & CAM_DATA_MASK) {
1463 (
struct bio *)csio->data_ptr, prsv);
1465 free(ddp, M_CXGBEI);
1470 (
struct bio *)csio->data_ptr, &mq);
1471 if (__predict_false(rc != 0)) {
1474 free(ddp, M_CXGBEI);
1478 case CAM_DATA_VADDR:
1480 csio->dxfer_len, prsv);
1482 free(ddp, M_CXGBEI);
1487 (vm_offset_t)csio->data_ptr, csio->dxfer_len, &mq);
1488 if (__predict_false(rc != 0)) {
1491 free(ddp, M_CXGBEI);
1496 free(ddp, M_CXGBEI);
1505 inp = sotoinpcb(ic->ic_socket);
1507 if ((inp->inp_flags & (INP_DROPPED | INP_TIMEWAIT)) != 0) {
1511 free(ddp, M_CXGBEI);
1514 mbufq_concat(&toep->
ulp_pduq, &mq);
1517 ddp->cmp.last_datasn = -1;
1518 cxgbei_insert_cmp(icc, &ddp->cmp, prsv->
prsv_tag);
1526icl_cxgbei_conn_task_done(
struct icl_conn *ic,
void *arg)
1530 struct cxgbei_ddp_state *ddp = arg;
1532 cxgbei_rm_cmp(
ic_to_icc(ic), &ddp->cmp);
1534 free(ddp, M_CXGBEI);
1539ddp_sgl_check(
struct ctl_sg_entry *sg,
int entries,
int xferlen)
1546 if (((vm_offset_t)sg[--entries].addr & 3U) != 0)
1550 total_len += sg[entries].len;
1553 while (--entries >= 0) {
1554 if (((vm_offset_t)sg[entries].addr & PAGE_MASK) != 0 ||
1555 (sg[entries].len % PAGE_SIZE) != 0)
1558 total_len += sg[entries].len;
1562 MPASS(total_len == xferlen);
1566#define io_to_ddp_state(io) ((io)->io_hdr.ctl_private[CTL_PRIV_FRONTEND2].ptr)
1569icl_cxgbei_conn_transfer_setup(
struct icl_conn *ic,
struct icl_pdu *ip,
1570 union ctl_io *io, uint32_t *tttp,
void **arg)
1574 struct ctl_scsiio *ctsio = &io->scsiio;
1578 struct cxgbei_ddp_state *ddp;
1580 struct ctl_sg_entry *sgl, sg_entry;
1583 int sg_entries = ctsio->kern_sg_entries;
1585 int xferlen, rc = 0, alias;
1589 MPASS(*arg == NULL);
1591 if (ctsio->ext_data_filled == 0) {
1597 MPASS(ic ==
ip->ip_conn);
1598 MPASS(
ip->ip_bhs_mbuf != NULL);
1600 first_burst = icl_pdu_data_segment_length(
ip);
1611 xferlen = ctsio->kern_data_len;
1612 if (xferlen < first_burst ||
1613 xferlen - first_burst < ci->ddp_threshold) {
1622 *tttp = htobe32(ttt);
1623 MPASS(io_to_ddp_state(io) == NULL);
1630 if (sg_entries == 0) {
1633 sgl->addr = (
void *)ctsio->kern_data_ptr;
1636 sgl = (
void *)ctsio->kern_data_ptr;
1638 if (!ddp_sgl_check(sgl, sg_entries, xferlen))
1645 MPASS(io_to_ddp_state(io) == NULL);
1646 ddp = malloc(
sizeof(*ddp), M_CXGBEI, M_NOWAIT | M_ZERO);
1655 free(ddp, M_CXGBEI);
1659 mbufq_init(&mq, INT_MAX);
1662 if (__predict_false(rc != 0)) {
1665 free(ddp, M_CXGBEI);
1674 if (ic->ic_disconnecting || ic->ic_socket == NULL) {
1675 ICL_CONN_UNLOCK(ic);
1678 free(ddp, M_CXGBEI);
1679 return (ECONNRESET);
1681 inp = sotoinpcb(ic->ic_socket);
1683 ICL_CONN_UNLOCK(ic);
1684 if ((inp->inp_flags & (INP_DROPPED | INP_TIMEWAIT)) != 0) {
1688 free(ddp, M_CXGBEI);
1689 return (ECONNRESET);
1691 mbufq_concat(&toep->
ulp_pduq, &mq);
1694 ddp->cmp.next_buffer_offset = ctsio->kern_rel_offset +
1696 ddp->cmp.last_datasn = -1;
1697 cxgbei_insert_cmp(icc, &ddp->cmp, prsv->
prsv_tag);
1699 io_to_ddp_state(io) = ddp;
1709 ddp = io_to_ddp_state(ctsio);
1719 ddp->cmp.last_datasn = -1;
1720 cxgbei_insert_cmp(icc, &ddp->cmp, prsv->
prsv_tag);
1728icl_cxgbei_conn_transfer_done(
struct icl_conn *ic,
void *arg)
1730 struct ctl_scsiio *ctsio = arg;
1732 if (ctsio != NULL) {
1733 struct cxgbei_ddp_state *ddp;
1735 ddp = io_to_ddp_state(ctsio);
1738 cxgbei_rm_cmp(
ic_to_icc(ic), &ddp->cmp);
1739 if (ctsio->kern_data_len == ctsio->ext_data_filled ||
1740 ic->ic_disconnecting) {
1742 free(ddp, M_CXGBEI);
1743 io_to_ddp_state(ctsio) = NULL;
1749cxgbei_limits(
struct adapter *sc,
void *arg)
1751 struct icl_drv_limits *idl = arg;
1764 if (idl->idl_max_recv_data_segment_length > max_dsl)
1765 idl->idl_max_recv_data_segment_length = max_dsl;
1768 if (idl->idl_max_send_data_segment_length > max_dsl)
1769 idl->idl_max_send_data_segment_length = max_dsl;
1776icl_cxgbei_limits(
struct icl_drv_limits *idl)
1780 idl->idl_max_recv_data_segment_length = (1 << 24) - 1;
1781 idl->idl_max_send_data_segment_length = (1 << 24) - 1;
1784 idl->idl_max_burst_length = max_burst_length;
1785 idl->idl_first_burst_length = first_burst_length;
1797 refcount_init(&icl_cxgbei_ncons, 0);
1799 rc = icl_register(
"cxgbei",
false, -100, icl_cxgbei_limits,
1800 icl_cxgbei_new_conn);
1809 if (icl_cxgbei_ncons != 0)
1812 icl_unregister(
"cxgbei",
false);
int begin_synchronized_op(struct adapter *, struct vi_info *, int, char *)
static struct wrqe * alloc_wrqe(int wr_len, struct sge_wrq *wrq)
static uint32_t t4_read_reg(struct adapter *sc, uint32_t reg)
uint8_t ss[SGE_MAX_WR_LEN]
static void * wrtod(struct wrqe *wr)
STAILQ_HEAD(, wrqe) wr_list
static void t4_wrq_tx(struct adapter *sc, struct wrqe *wr)
void t4_iterate(void(*)(struct adapter *, void *), void *)
void end_synchronized_op(struct adapter *, int)
static int chip_id(struct adapter *adap)
static int is_t5(struct adapter *adap)
static struct icl_cxgbei_pdu * ip_to_icp(struct icl_pdu *ip)
int icl_cxgbei_mod_load(void)
static struct icl_cxgbei_conn * ic_to_icc(struct icl_conn *ic)
struct icl_pdu * icl_cxgbei_new_pdu(int)
void icl_cxgbei_conn_pdu_free(struct icl_conn *, struct icl_pdu *)
#define CXGBEI_CONN_SIGNATURE
int icl_cxgbei_mod_unload(void)
#define CXGBEI_MAX_ISO_PAYLOAD
struct cxgbei_cmp * cxgbei_find_cmp(struct icl_cxgbei_conn *, uint32_t)
void parse_pdus(struct icl_cxgbei_conn *, struct sockbuf *)
void icl_cxgbei_new_pdu_set_conn(struct icl_pdu *, struct icl_conn *)
struct fw_flowc_mnemval mnemval[0]
struct thread * rx_thread
unsigned long cmp_hash_mask
struct cxgbei_cmp_head * cmp_table
struct thread * tx_thread
counter_u64_t rx_iscsi_ddp_setup_error
counter_u64_t rx_iscsi_ddp_setup_ok
struct conn_params params
struct sge_ofld_rxq * ofld_rxq
struct sge_ofld_txq * ofld_txq
struct ofld_tx_sdesc txsd[]
MALLOC_DEFINE(M_CXGBE, "cxgbe", "Chelsio T4/T5 Ethernet driver and services")
#define A_MA_TARGET_MEM_ENABLE
#define F_EXT_MEM1_ENABLE
SYSCTL_INT(_hw_cxgbe, OID_AUTO, fl_pktshift, CTLFLAG_RDTUN, &fl_pktshift, 0, "payload DMA offset in rx buffer (bytes)")
#define V_TCB_ULP_TYPE(x)
#define V_TF_RX_FLOW_CONTROL_DISABLE(x)
static void set_mbuf_iscsi_iso_flags(struct mbuf *m, uint8_t flags)
static int ulp_mode(struct toepcb *toep)
int t4_alloc_page_pods_for_sgl(struct ppod_region *, struct ctl_sg_entry *, int, struct ppod_reservation *)
int t4_alloc_page_pods_for_bio(struct ppod_region *, struct bio *, struct ppod_reservation *)
static void set_mbuf_iscsi_iso(struct mbuf *m, bool iso)
static void set_mbuf_iscsi_iso_mss(struct mbuf *m, uint16_t mss)
struct toepcb * hold_toepcb(struct toepcb *)
void t4_set_tcb_field(struct adapter *, struct sge_wrq *, struct toepcb *, uint16_t, uint64_t, uint64_t, int, int)
void t4_free_page_pods(struct ppod_reservation *)
void free_toepcb(struct toepcb *)
int t4_alloc_page_pods_for_buf(struct ppod_region *, vm_offset_t, int, struct ppod_reservation *)
static void set_mbuf_ulp_submode(struct mbuf *m, uint8_t ulp_submode)
int t4_write_page_pods_for_sgl(struct adapter *, struct toepcb *, struct ppod_reservation *, struct ctl_sg_entry *, int, int, struct mbufq *)
int t4_write_page_pods_for_bio(struct adapter *, struct toepcb *, struct ppod_reservation *, struct bio *, struct mbufq *)
int t4_write_page_pods_for_buf(struct adapter *, struct toepcb *, struct ppod_reservation *, vm_offset_t, int, struct mbufq *)
void t4_push_pdus(struct adapter *, struct toepcb *, int)
#define V_FW_FLOWC_WR_NPARAMS(x)
@ FW_FLOWC_MNEM_TXDATAPLEN_MAX
#define V_FW_WR_FLOWID(x)