39#include <sys/kernel.h>
41#include <sys/module.h>
46#include <sys/gsb_crc32.h>
47#include <sys/kthread.h>
49#include <sys/socket.h>
50#include <sys/socketvar.h>
54#include <sys/condvar.h>
57#include <netinet/in.h>
58#include <netinet/in_pcb.h>
59#include <netinet/toecore.h>
60#include <netinet/tcp_var.h>
61#include <netinet/tcp_fsm.h>
63#include <cam/scsi/scsi_all.h>
64#include <cam/scsi/scsi_da.h>
65#include <cam/ctl/ctl_io.h>
66#include <cam/ctl/ctl.h>
67#include <cam/ctl/ctl_backend.h>
68#include <cam/ctl/ctl_error.h>
69#include <cam/ctl/ctl_frontend.h>
70#include <cam/ctl/ctl_debug.h>
71#include <cam/ctl/ctl_ha.h>
72#include <cam/ctl/ctl_ioctl.h>
74#include <dev/iscsi/icl.h>
75#include <dev/iscsi/iscsi_proto.h>
76#include <dev/iscsi/iscsi_ioctl.h>
77#include <dev/iscsi/iscsi.h>
78#include <cam/ctl/ctl_frontend_iscsi.h>
81#include <cam/cam_ccb.h>
82#include <cam/cam_xpt.h>
83#include <cam/cam_debug.h>
84#include <cam/cam_sim.h>
85#include <cam/cam_xpt_sim.h>
86#include <cam/cam_xpt_periph.h>
87#include <cam/cam_periph.h>
88#include <cam/cam_compat.h>
89#include <cam/scsi/scsi_message.h>
98read_pdu_limits(
struct adapter *sc, uint32_t *max_tx_data_len,
101 uint32_t tx_len, rx_len,
r, v;
112 rx_len = min(rx_len, v);
113 tx_len = min(tx_len, v);
119 rx_len -= ISCSI_BHS_SIZE + ISCSI_HEADER_DIGEST_SIZE +
120 ISCSI_DATA_DIGEST_SIZE;
121 tx_len -= ISCSI_BHS_SIZE + ISCSI_HEADER_DIGEST_SIZE +
122 ISCSI_DATA_DIGEST_SIZE;
133 tx_len = min(tx_len, 15360);
135 rx_len = rounddown2(rx_len, 512);
136 tx_len = rounddown2(tx_len, 512);
139 *max_tx_data_len = tx_len;
140 *max_rx_data_len = rx_len;
151 struct sysctl_oid *oid;
152 struct sysctl_oid_list *children;
164 device_printf(sc->
dev,
165 "%s: failed to initialize the iSCSI page pod region: %u.\n",
178 device_printf(sc->
dev,
179 "tagmask 0x%08x does not match computed mask 0x%08x.\n",
r,
187 sysctl_ctx_init(&ci->
ctx);
188 oid = device_get_sysctl_tree(sc->
dev);
189 children = SYSCTL_CHILDREN(oid);
191 oid = SYSCTL_ADD_NODE(&ci->
ctx, children, OID_AUTO,
"iscsi",
192 CTLFLAG_RD | CTLFLAG_MPSAFE, NULL,
"iSCSI ULP settings");
193 children = SYSCTL_CHILDREN(oid);
196 SYSCTL_ADD_UINT(&ci->
ctx, children, OID_AUTO,
"ddp_threshold",
197 CTLFLAG_RW, &ci->
ddp_threshold, 0,
"Rx zero copy threshold");
199 SYSCTL_ADD_UINT(&ci->
ctx, children, OID_AUTO,
"max_rx_data_len",
201 "Maximum receive data segment length");
202 SYSCTL_ADD_UINT(&ci->
ctx, children, OID_AUTO,
"max_tx_data_len",
204 "Maximum transmit data segment length");
219 uint16_t len = be16toh(cpl->
len);
222 MPASS(m->m_pkthdr.len == len +
sizeof(*cpl));
227 m_copydata(m,
sizeof(*cpl), ISCSI_BHS_SIZE, (caddr_t)
ip->ip_bhs);
234 MPASS(toep->
ulpcb2 == NULL);
238 CTR5(
KTR_CXGBE,
"%s: tid %u, cpl->len %u, pdu_len_ddp 0x%04x, icp %p",
239 __func__, tid, len, len_ddp, icp);
257 MPASS(m->m_pkthdr.len == be16toh(cpl->
len) +
sizeof(*cpl));
271 MPASS(icp->
ip.ip_data_mbuf == NULL);
272 MPASS(icp->
ip.ip_data_len == m->m_pkthdr.len -
sizeof(*cpl));
276 m_adj(m,
sizeof(*cpl));
279 icp->
ip.ip_data_mbuf = m;
288 MPASS(toep->
ulpcb2 == NULL);
293 CTR4(
KTR_CXGBE,
"%s: tid %u, cpl->len %u, icp %p", __func__, tid,
294 be16toh(cpl->
len), icp);
301mbuf_crc32c_helper(
void *arg,
void *data, u_int len)
303 uint32_t *digestp = arg;
305 *digestp = calculate_crc32c(*digestp, data, len);
309static struct icl_pdu *
311 struct sockbuf *sb, u_int total_len)
315 struct iscsi_bhs bhs;
318 u_int ahs_len, data_len, header_len, pdu_len;
319 uint32_t calc_digest, wire_digest;
322 uio.uio_segflg = UIO_SYSSPACE;
323 uio.uio_rw = UIO_READ;
324 uio.uio_td = curthread;
326 header_len =
sizeof(
struct iscsi_bhs);
327 if (icc->
ic.ic_header_crc32c)
328 header_len += ISCSI_HEADER_DIGEST_SIZE;
330 if (total_len < header_len) {
331 ICL_WARN(
"truncated pre-offload PDU with len %u", total_len);
335 iov[0].iov_base = &bhs;
336 iov[0].iov_len =
sizeof(bhs);
337 iov[1].iov_base = &wire_digest;
338 iov[1].iov_len =
sizeof(wire_digest);
342 uio.uio_resid = header_len;
343 error = soreceive(so, NULL, &uio, NULL, NULL, NULL);
345 ICL_WARN(
"failed to read BHS from pre-offload PDU: %d", error);
349 ahs_len = bhs.bhs_total_ahs_len * 4;
350 data_len = bhs.bhs_data_segment_len[0] << 16 |
351 bhs.bhs_data_segment_len[1] << 8 |
352 bhs.bhs_data_segment_len[2];
353 pdu_len = header_len + ahs_len + roundup2(data_len, 4);
354 if (icc->
ic.ic_data_crc32c && data_len != 0)
355 pdu_len += ISCSI_DATA_DIGEST_SIZE;
357 if (total_len < pdu_len) {
358 ICL_WARN(
"truncated pre-offload PDU len %u vs %u", total_len,
364 ICL_WARN(
"received pre-offload PDU with AHS");
368 if (icc->
ic.ic_header_crc32c) {
369 calc_digest = calculate_crc32c(0xffffffff, (caddr_t)&bhs,
371 calc_digest ^= 0xffffffff;
372 if (calc_digest != wire_digest) {
373 ICL_WARN(
"received pre-offload PDU 0x%02x with "
374 "invalid header digest (0x%x vs 0x%x)",
375 bhs.bhs_opcode, wire_digest, calc_digest);
384 uio.uio_resid = roundup2(data_len, 4);
385 if (icc->
ic.ic_data_crc32c)
386 uio.uio_resid += ISCSI_DATA_DIGEST_SIZE;
388 error = soreceive(so, NULL, &uio, &m, NULL, NULL);
390 ICL_WARN(
"failed to read data payload from "
391 "pre-offload PDU: %d", error);
395 if (icc->
ic.ic_data_crc32c) {
396 m_copydata(m, roundup2(data_len, 4),
397 sizeof(wire_digest), (caddr_t)&wire_digest);
399 calc_digest = 0xffffffff;
400 m_apply(m, 0, roundup2(data_len, 4), mbuf_crc32c_helper,
402 calc_digest ^= 0xffffffff;
403 if (calc_digest != wire_digest) {
404 ICL_WARN(
"received pre-offload PDU 0x%02x "
405 "with invalid data digest (0x%x vs 0x%x)",
406 bhs.bhs_opcode, wire_digest, calc_digest);
417 ip->ip_data_len = data_len;
418 ip->ip_data_mbuf = m;
425 struct icl_conn *ic = &icc->
ic;
426 struct socket *so = ic->ic_socket;
428 struct icl_pdu *ip, *lastip;
431 SOCKBUF_LOCK_ASSERT(sb);
433 CTR3(
KTR_CXGBE,
"%s: tid %u, %u bytes in so_rcv", __func__, toep->
tid,
437 while (sbused(sb) != 0 && (sb->sb_state & SBS_CANTRCVMORE) == 0) {
438 total_len = sbused(sb);
441 ip = parse_pdu(so, toep, icc, sb, total_len);
450 STAILQ_INSERT_HEAD(&icc->rcvd_pdus, ip, ip_next);
452 STAILQ_INSERT_AFTER(&icc->rcvd_pdus, lastip, ip,
467 struct inpcb *inp = toep->
inp;
476 struct epoch_tracker et;
485 pdu_len = be16toh(cpl->
len);
486 val = be32toh(cpl->
ddpvld);
490 "%s: tid %u, cpl->len %u, ddpvld 0x%08x, icp_flags 0x%08x",
491 __func__, tid, pdu_len, val, icp->
icp_flags);
497 ICL_WARN(
"received PDU 0x%02x with invalid padding",
498 ip->ip_bhs->bhs_opcode);
502 ICL_WARN(
"received PDU 0x%02x with invalid header digest",
503 ip->ip_bhs->bhs_opcode);
507 ICL_WARN(
"received PDU 0x%02x with invalid data digest",
508 ip->ip_bhs->bhs_opcode);
511 if (val &
F_DDP_PDU && ip->ip_data_mbuf == NULL) {
513 MPASS(ip->ip_data_len > 0);
520 if (__predict_false(inp->inp_flags & (INP_DROPPED | INP_TIMEWAIT))) {
521 CTR4(
KTR_CXGBE,
"%s: tid %u, rx (%d bytes), inp_flags 0x%x",
522 __func__, tid, pdu_len, inp->inp_flags);
535 tp->rcv_nxt = icp->
icp_seq + pdu_len;
536 tp->t_rcvtime = ticks;
543 so = inp->inp_socket;
548 if (__predict_false(icc == NULL || sb->sb_state & SBS_CANTRCVMORE)) {
550 "%s: tid %u, excess rx (%d bytes), icc %p, sb_state 0x%x",
551 __func__, tid, pdu_len, icc, sb->sb_state);
555 CURVNET_SET(so->so_vnet);
558 tp = tcp_drop(tp, ECONNRESET);
583 STAILQ_INSERT_TAIL(&icc->rcvd_pdus, ip, ip_next);
599 struct epoch_tracker et;
607 struct inpcb *inp = toep->
inp;
609 uint16_t len = be16toh(cpl->
len);
610 u_int data_digest_len;
617 struct iscsi_bhs_data_out *bhsdo;
618 u_int val = be32toh(cpl->
ddpvld);
619 u_int npdus, pdu_len;
620 uint32_t prev_seg_len;
623 MPASS(m->m_pkthdr.len == len +
sizeof(*cpl));
642 "%s: tid %u, cpl->len %u, ddpvld 0x%08x, icp %p",
643 __func__, tid, pdu_len, val, icp);
647 m_copydata(m,
sizeof(*cpl), ISCSI_BHS_SIZE, (caddr_t)ip->ip_bhs);
648 bhsdo = (
struct iscsi_bhs_data_out *)ip->ip_bhs;
649 ip->ip_data_len = bhsdo->bhsdo_data_segment_len[0] << 16 |
650 bhsdo->bhsdo_data_segment_len[1] << 8 |
651 bhsdo->bhsdo_data_segment_len[2];
657 ICL_WARN(
"received PDU 0x%02x with invalid padding",
658 ip->ip_bhs->bhs_opcode);
662 ICL_WARN(
"received PDU 0x%02x with invalid header digest",
663 ip->ip_bhs->bhs_opcode);
667 ICL_WARN(
"received PDU 0x%02x with invalid data digest",
668 ip->ip_bhs->bhs_opcode);
673 if (__predict_false(inp->inp_flags & (INP_DROPPED | INP_TIMEWAIT))) {
674 CTR4(
KTR_CXGBE,
"%s: tid %u, rx (%d bytes), inp_flags 0x%x",
675 __func__, tid, pdu_len, inp->inp_flags);
690 if (__predict_false(icc == NULL)) {
691 CTR4(
KTR_CXGBE,
"%s: tid %u, excess rx (%d bytes), icc %p",
692 __func__, tid, pdu_len, icc);
698 tp->rcv_nxt = icp->
icp_seq + pdu_len;
699 tp->t_rcvtime = ticks;
723 ISCSI_DATA_DIGEST_SIZE : 0;
724 MPASS(roundup2(ip->ip_data_len, 4) == pdu_len - len - data_digest_len);
727 if (val &
F_DDP_PDU && ip->ip_data_mbuf == NULL) {
729 MPASS(ip->ip_data_len > 0);
731 bhsdo = (
struct iscsi_bhs_data_out *)ip->ip_bhs;
733 switch (ip->ip_bhs->bhs_opcode & ~ISCSI_BHS_OPCODE_IMMEDIATE) {
734 case ISCSI_BHS_OPCODE_SCSI_DATA_IN:
736 be32toh(bhsdo->bhsdo_initiator_task_tag));
738 case ISCSI_BHS_OPCODE_SCSI_DATA_OUT:
740 be32toh(bhsdo->bhsdo_target_transfer_tag));
743 __assert_unreachable();
752 prev_seg_len = be32toh(bhsdo->bhsdo_buffer_offset) -
755 if (prev_seg_len != 0) {
756 uint32_t orig_datasn;
763 ip->ip_data_len += prev_seg_len;
764 bhsdo->bhsdo_data_segment_len[2] = ip->ip_data_len;
765 bhsdo->bhsdo_data_segment_len[1] = ip->ip_data_len >> 8;
766 bhsdo->bhsdo_data_segment_len[0] = ip->ip_data_len >> 16;
767 bhsdo->bhsdo_buffer_offset =
770 orig_datasn = htobe32(bhsdo->bhsdo_datasn);
772 bhsdo->bhsdo_datasn = htobe32(cmp->
last_datasn + 1);
774 ip->ip_additional_pdus = npdus - 1;
776 MPASS(htobe32(bhsdo->bhsdo_datasn) ==
787 MPASS(ip->ip_data_len == ip->ip_data_mbuf->m_pkthdr.len);
790 tp->rcv_nxt = icp->
icp_seq + pdu_len;
791 tp->t_rcvtime = ticks;
798 so = inp->inp_socket;
801 if (__predict_false(sb->sb_state & SBS_CANTRCVMORE)) {
803 "%s: tid %u, excess rx (%d bytes), icc %p, sb_state 0x%x",
804 __func__, tid, pdu_len, icc, sb->sb_state);
808 CURVNET_SET(so->so_vnet);
811 tp = tcp_drop(tp, ECONNRESET);
826 STAILQ_INSERT_TAIL(&icc->rcvd_pdus, ip, ip_next);
841cxgbei_activate(
struct adapter *sc)
849 KASSERT(0, (
"%s: iSCSI offload already enabled on adapter %p",
855 device_printf(sc->
dev,
856 "not iSCSI offload capable, or capability disabled.\n");
861 ci = malloc(
sizeof(*ci), M_CXGBE, M_ZERO | M_WAITOK);
865 rc = cxgbei_init(sc, ci);
877cxgbei_deactivate(
struct adapter *sc)
884 sysctl_ctx_free(&ci->
ctx);
894cxgbei_activate_all(
struct adapter *sc,
void *arg __unused)
908cxgbei_deactivate_all(
struct adapter *sc,
void *arg __unused)
920static struct uld_info cxgbei_uld_info = {
922 .activate = cxgbei_activate,
923 .deactivate = cxgbei_deactivate,
936 rc = t4_register_uld(&cxgbei_uld_info);
946cxgbei_mod_unload(
void)
951 if (t4_unregister_uld(&cxgbei_uld_info) == EBUSY)
971 rc = cxgbei_mod_load();
979 rc = cxgbei_mod_unload();
986 printf(
"cxgbei: compiled without TCP_OFFLOAD support.\n");
int begin_synchronized_op(struct adapter *, struct vi_info *, int, char *)
static uint32_t t4_read_reg(struct adapter *sc, uint32_t reg)
void t4_register_cpl_handler(int, cpl_handler_t)
void t4_iterate(void(*)(struct adapter *, void *), void *)
#define ASSERT_SYNCHRONIZED_OP(sc)
void end_synchronized_op(struct adapter *, int)
#define CXGBE_UNIMPLEMENTED(s)
void t4_set_reg_field(struct adapter *adap, unsigned int addr, u32 mask, u32 val)
static int chip_id(struct adapter *adap)
static moduledata_t cxgbei_mod
MODULE_VERSION(cxgbei, 1)
DECLARE_MODULE(cxgbei, cxgbei_mod, SI_SUB_EXEC, SI_ORDER_ANY)
static int cxgbei_modevent(module_t mod, int cmd, void *arg)
MODULE_DEPEND(cxgbei, t4_tom, 1, 1, 1)
static struct icl_cxgbei_pdu * ip_to_icp(struct icl_pdu *ip)
int icl_cxgbei_mod_load(void)
struct icl_pdu * icl_cxgbei_new_pdu(int)
void icl_cxgbei_conn_pdu_free(struct icl_conn *, struct icl_pdu *)
#define CXGBEI_CONN_SIGNATURE
int icl_cxgbei_mod_unload(void)
struct cxgbei_cmp * cxgbei_find_cmp(struct icl_cxgbei_conn *, uint32_t)
void parse_pdus(struct icl_cxgbei_conn *, struct sockbuf *)
void icl_cxgbei_new_pdu_set_conn(struct icl_pdu *, struct icl_conn *)
uint32_t next_buffer_offset
struct sysctl_ctx_list ctx
uint64_t rx_iscsi_fl_pdus
uint64_t rx_iscsi_ddp_pdus
uint64_t rx_iscsi_fl_octets
uint64_t rx_iscsi_data_digest_errors
uint64_t rx_iscsi_ddp_octets
uint64_t rx_iscsi_header_digest_errors
uint64_t rx_iscsi_padding_errors
struct sge_ofld_rxq * ofld_rxq
#define F_DDP_PADDING_ERR
#define G_ISCSI_PDU_LEN(x)
#define F_DDP_DATACRC_ERR
#define G_PMMAXXFERLEN1(x)
#define G_PMMAXXFERLEN0(x)
#define A_ULP_RX_ISCSI_PSZ
#define A_TP_PMM_TX_PAGE_SIZE
#define A_ULP_RX_ISCSI_TAGMASK
#define A_TP_PMM_RX_PAGE_SIZE
#define V_ISCSITAGMASK(x)
void t4_free_ppod_region(struct ppod_region *)
void * lookup_tid(struct adapter *, int)
int t4_init_ppod_region(struct ppod_region *, struct t4_range *, u_int, const char *)