41#include <sys/malloc.h>
42#include <sys/socket.h>
43#include <sys/socketvar.h>
44#include <sys/sockio.h>
45#include <sys/taskqueue.h>
46#include <netinet/in.h>
48#include <net/route/nhop.h>
50#include <netinet/in_systm.h>
51#include <netinet/in_pcb.h>
52#include <netinet6/in6_pcb.h>
53#include <netinet/ip.h>
54#include <netinet/in_fib.h>
55#include <netinet6/in6_fib.h>
56#include <netinet6/scope6_var.h>
57#include <netinet/ip_var.h>
58#include <netinet/tcp_var.h>
59#include <netinet/tcp.h>
60#include <netinet/tcpip.h>
62#include <netinet/toecore.h>
67#include <linux/types.h>
71#define TOEPCB(so) ((struct toepcb *)(so_sototcpcb((so))->t_toe))
74#include <linux/module.h>
75#include <linux/workqueue.h>
76#include <linux/if_vlan.h>
77#include <net/netevent.h>
78#include <rdma/rdma_cm.h>
80static spinlock_t req_lock;
82static struct work_struct c4iw_task;
83static struct workqueue_struct *c4iw_taskq;
85static spinlock_t err_cqe_lock;
87static DEFINE_MUTEX(listen_port_mutex);
89static void process_req(
struct work_struct *ctx);
90static void start_ep_timer(
struct c4iw_ep *ep);
91static int stop_ep_timer(
struct c4iw_ep *ep);
92static int set_tcpinfo(
struct c4iw_ep *ep);
93static void process_timeout(
struct c4iw_ep *ep);
94static void process_err_cqes(
void);
95static void *alloc_ep(
int size, gfp_t flags);
96static void close_socket(
struct socket *so);
97static int send_mpa_req(
struct c4iw_ep *ep);
98static int send_mpa_reject(
struct c4iw_ep *ep,
const void *pdata,
u8 plen);
99static int send_mpa_reply(
struct c4iw_ep *ep,
const void *pdata,
u8 plen);
100static void close_complete_upcall(
struct c4iw_ep *ep,
int status);
101static int send_abort(
struct c4iw_ep *ep);
102static void peer_close_upcall(
struct c4iw_ep *ep);
103static void peer_abort_upcall(
struct c4iw_ep *ep);
104static void connect_reply_upcall(
struct c4iw_ep *ep,
int status);
105static int connect_request_upcall(
struct c4iw_ep *ep);
106static void established_upcall(
struct c4iw_ep *ep);
107static int process_mpa_reply(
struct c4iw_ep *ep);
108static int process_mpa_request(
struct c4iw_ep *ep);
109static void process_peer_close(
struct c4iw_ep *ep);
110static void process_conn_error(
struct c4iw_ep *ep);
111static void process_close_complete(
struct c4iw_ep *ep);
112static void ep_timeout(
unsigned long arg);
113static void setiwsockopt(
struct socket *so);
114static void init_iwarp_socket(
struct socket *so,
void *arg);
115static void uninit_iwarp_socket(
struct socket *so);
116static void process_data(
struct c4iw_ep *ep);
117static void process_connected(
struct c4iw_ep *ep);
118static int c4iw_so_upcall(
struct socket *so,
void *arg,
int waitflag);
119static void process_socket_event(
struct c4iw_ep *ep);
120static void release_ep_resources(
struct c4iw_ep *ep);
121static int process_terminate(
struct c4iw_ep *ep);
124static int add_ep_to_req_list(
struct c4iw_ep *ep,
int ep_events);
125static struct listen_port_info *
129find_real_listen_ep(
struct c4iw_listen_ep *master_lep,
struct socket *so);
130static int get_ifnet_from_raddr(
struct sockaddr_storage *raddr,
133 struct socket *new_so);
134#define START_EP_TIMER(ep) \
136 CTR3(KTR_IW_CXGBE, "start_ep_timer (%s:%d) ep %p", \
137 __func__, __LINE__, (ep)); \
138 start_ep_timer(ep); \
141#define STOP_EP_TIMER(ep) \
143 CTR3(KTR_IW_CXGBE, "stop_ep_timer (%s:%d) ep %p", \
144 __func__, __LINE__, (ep)); \
148#define GET_LOCAL_ADDR(pladdr, so) \
150 struct sockaddr_storage *__a = NULL; \
151 struct inpcb *__inp = sotoinpcb(so); \
152 KASSERT(__inp != NULL, \
153 ("GET_LOCAL_ADDR(%s):so:%p, inp = NULL", __func__, so)); \
154 if (__inp->inp_vflag & INP_IPV4) \
155 in_getsockaddr(so, (struct sockaddr **)&__a); \
157 in6_getsockaddr(so, (struct sockaddr **)&__a); \
159 free(__a, M_SONAME); \
162#define GET_REMOTE_ADDR(praddr, so) \
164 struct sockaddr_storage *__a = NULL; \
165 struct inpcb *__inp = sotoinpcb(so); \
166 KASSERT(__inp != NULL, \
167 ("GET_REMOTE_ADDR(%s):so:%p, inp = NULL", __func__, so)); \
168 if (__inp->inp_vflag & INP_IPV4) \
169 in_getpeeraddr(so, (struct sockaddr **)&__a); \
171 in6_getpeeraddr(so, (struct sockaddr **)&__a); \
173 free(__a, M_SONAME); \
176static char *states[] = {
205static void deref_qp(
struct c4iw_ep *ep)
212static void ref_qp(
struct c4iw_ep *ep)
219struct listen_port_info {
221 struct list_head list;
222 struct list_head lep_list;
266static struct listen_port_info *
270 struct listen_port_info *
port_info = NULL;
273 port = (laddr->ss_family == AF_INET) ?
274 ((
struct sockaddr_in *)laddr)->sin_port :
275 ((
struct sockaddr_in6 *)laddr)->sin6_port;
277 mutex_lock(&listen_port_mutex);
279 list_for_each_entry(
port_info, &listen_port_list, list)
287 list_add_tail(&
port_info->list, &listen_port_list);
293 mutex_unlock(&listen_port_mutex);
301 struct listen_port_info *
port_info = NULL;
305 port = (laddr->ss_family == AF_INET) ?
306 ((
struct sockaddr_in *)laddr)->sin_port :
307 ((
struct sockaddr_in6 *)laddr)->sin6_port;
309 mutex_lock(&listen_port_mutex);
312 list_for_each_entry(
port_info, &listen_port_list, list) {
328 mutex_unlock(&listen_port_mutex);
336find_real_listen_ep(
struct c4iw_listen_ep *master_lep,
struct socket *so)
340 struct ifnet *
ifp = NULL, *hw_ifp = NULL;
341 struct listen_port_info *
port_info = NULL;
342 int i = 0, found_portinfo = 0, found_lep = 0;
350 ifp = TOEPCB(so)->l2te->ifp;
351 if (
ifp->if_type == IFT_L2VLAN) {
352 hw_ifp = VLAN_TRUNKDEV(
ifp);
353 if (hw_ifp == NULL) {
355 "vlan ifnet %p, sock %p, master_lep %p",
356 __func__,
ifp, so, master_lep);
364 ((
struct sockaddr_in *)&master_lep->
com.
local_addr)->sin_port :
365 ((
struct sockaddr_in6 *)&master_lep->
com.
local_addr)->sin6_port;
368 mutex_lock(&listen_port_mutex);
369 list_for_each_entry(
port_info, &listen_port_list, list)
381 list_for_each_entry(lep, &
port_info->lep_list, listen_ep_list) {
384 if (hw_ifp == adap->
port[i]->
vi[0].
ifp) {
391 mutex_unlock(&listen_port_mutex);
392 return found_lep ? lep : (NULL);
395static void process_timeout(
struct c4iw_ep *ep)
400 CTR4(
KTR_IW_CXGBE,
"%s ep :%p, tid:%u, state %d", __func__,
405 connect_reply_upcall(ep, -ETIMEDOUT);
419 close_complete_upcall(ep, -ETIMEDOUT);
431 CTR4(
KTR_IW_CXGBE,
"%s unexpected state ep %p tid %u state %u"
441struct cqe_list_entry {
442 struct list_head entry;
448process_err_cqes(
void)
451 struct cqe_list_entry *cle;
453 spin_lock_irqsave(&err_cqe_lock, flag);
454 while (!list_empty(&err_cqe_list)) {
455 struct list_head *tmp;
456 tmp = err_cqe_list.next;
458 tmp->next = tmp->prev = NULL;
459 spin_unlock_irqrestore(&err_cqe_lock, flag);
460 cle = list_entry(tmp,
struct cqe_list_entry, entry);
463 spin_lock_irqsave(&err_cqe_lock, flag);
465 spin_unlock_irqrestore(&err_cqe_lock, flag);
471process_req(
struct work_struct *ctx)
478 spin_lock_irqsave(&req_lock, flag);
479 while (!TAILQ_EMPTY(&req_list)) {
480 epc = TAILQ_FIRST(&req_list);
481 TAILQ_REMOVE(&req_list, epc, entry);
482 epc->entry.tqe_prev = NULL;
485 spin_unlock_irqrestore(&req_lock, flag);
486 mutex_lock(&epc->
mutex);
487 CTR5(
KTR_IW_CXGBE,
"%s: so %p, ep %p, ep_state %s events 0x%x",
490 process_terminate((
struct c4iw_ep *)epc);
492 process_timeout((
struct c4iw_ep *)epc);
494 process_socket_event((
struct c4iw_ep *)epc);
495 mutex_unlock(&epc->
mutex);
498 spin_lock_irqsave(&req_lock, flag);
500 spin_unlock_irqrestore(&req_lock, flag);
511 struct socket *so = ep->
com.
so;
512 struct inpcb *inp = sotoinpcb(so);
519 if ((tp->t_flags & TF_TOE) == 0) {
521 log(LOG_ERR,
"%s: connection not offloaded (so %p, ep %p)\n",
536get_ifnet_from_raddr(
struct sockaddr_storage *raddr,
struct ifnet **
ifp)
539 struct nhop_object *nh;
541 if (raddr->ss_family == AF_INET) {
542 struct sockaddr_in *raddr4 = (
struct sockaddr_in *)raddr;
544 nh = fib4_lookup(RT_DEFAULT_FIB, raddr4->sin_addr, 0,
547 struct sockaddr_in6 *raddr6 = (
struct sockaddr_in6 *)raddr;
548 struct in6_addr addr6;
551 memset(&addr6, 0,
sizeof(addr6));
552 in6_splitscope((
struct in6_addr *)&raddr6->sin6_addr,
554 nh = fib6_lookup(RT_DEFAULT_FIB, &addr6, scopeid,
567close_socket(
struct socket *so)
569 uninit_iwarp_socket(so);
574process_peer_close(
struct c4iw_ep *ep)
580 CTR4(
KTR_IW_CXGBE,
"%s:ppcB ep %p so %p state %s", __func__, ep,
593 connect_reply_upcall(ep, -ECONNABORTED);
597 close_socket(ep->
com.
so);
598 deref_cm_id(&ep->
com);
628 peer_close_upcall(ep);
645 CTR2(
KTR_IW_CXGBE,
"%s:ppc8 %p MORIBUND DEAD", __func__,
653 close_socket(ep->
com.
so);
654 close_complete_upcall(ep, 0);
667 panic(
"%s: ep %p state %d", __func__, ep,
688process_conn_error(
struct c4iw_ep *ep)
695 CTR5(
KTR_IW_CXGBE,
"%s:pceB ep %p so %p so->so_error %u state %s",
696 __func__, ep, ep->
com.
so, ep->
com.
so->so_error,
708 connect_reply_upcall(ep, -ECONNRESET);
733 "%s - qp <- error failed!\n",
736 peer_abort_upcall(ep);
744 __func__, ep->
com.
so->so_error);
748 panic(
"%s: ep %p state %d", __func__, ep,
state);
753 close_socket(ep->
com.
so);
762process_close_complete(
struct c4iw_ep *ep)
767 CTR4(
KTR_IW_CXGBE,
"%s:pccB ep %p so %p state %s", __func__, ep,
782 CTR2(
KTR_IW_CXGBE,
"%s:pcc1 %p MORIBUND DEAD", __func__,
797 close_socket(ep->
com.
so);
798 close_complete_upcall(ep, 0);
804 CTR2(
KTR_IW_CXGBE,
"%s:pcc5 %p ABORTING", __func__, ep);
813 panic(
"%s:pcc6 %p unknown ep state", __func__, ep);
820 release_ep_resources(ep);
827setiwsockopt(
struct socket *so)
833 sopt.sopt_dir = SOPT_SET;
834 sopt.sopt_level = IPPROTO_TCP;
835 sopt.sopt_name = TCP_NODELAY;
836 sopt.sopt_val = (caddr_t)&on;
837 sopt.sopt_valsize =
sizeof on;
839 rc = -sosetopt(so, &sopt);
841 log(LOG_ERR,
"%s: can't set TCP_NODELAY on so %p (%d)\n",
847init_iwarp_socket(
struct socket *so,
void *arg)
849 if (SOLISTENING(so)) {
851 solisten_upcall_set(so, c4iw_so_upcall, arg);
852 so->so_state |= SS_NBIO;
855 SOCKBUF_LOCK(&so->so_rcv);
856 soupcall_set(so, SO_RCV, c4iw_so_upcall, arg);
857 so->so_state |= SS_NBIO;
858 SOCKBUF_UNLOCK(&so->so_rcv);
863uninit_iwarp_socket(
struct socket *so)
865 if (SOLISTENING(so)) {
867 solisten_upcall_set(so, NULL, NULL);
870 SOCKBUF_LOCK(&so->so_rcv);
871 soupcall_clear(so, SO_RCV);
872 SOCKBUF_UNLOCK(&so->so_rcv);
877process_data(
struct c4iw_ep *ep)
883 CTR5(
KTR_IW_CXGBE,
"%s: so %p, ep %p, state %s, sbused %d", __func__,
888 disconnect = process_mpa_reply(ep);
891 disconnect = process_mpa_request(ep);
897 MPASS(ep->
com.
qp != NULL);
901 if (ret != -EINPROGRESS)
905 log(LOG_ERR,
"%s: Unexpected streaming data. ep %p, "
906 "state %d, so %p, so_state 0x%x, sbused %u\n",
908 ep->
com.
so->so_state, sbused(&ep->
com.
so->so_rcv));
917process_connected(
struct c4iw_ep *ep)
919 struct socket *so = ep->
com.
so;
921 if ((so->so_state & SS_ISCONNECTED) && !so->so_error) {
922 if (send_mpa_req(ep))
925 connect_reply_upcall(ep, -so->so_error);
936static inline int c4iw_zero_addr(
struct sockaddr *addr)
938 struct in6_addr *ip6;
940 if (addr->sa_family == AF_INET)
942 ntohl(((
struct sockaddr_in *)addr)->sin_addr.s_addr));
944 ip6 = &((
struct sockaddr_in6 *) addr)->sin6_addr;
945 return (ip6->s6_addr32[0] | ip6->s6_addr32[1] |
946 ip6->s6_addr32[2] | ip6->s6_addr32[3]) == 0;
950static inline int c4iw_loopback_addr(
struct sockaddr *addr)
952 if (addr->sa_family == AF_INET)
954 ntohl(((
struct sockaddr_in *) addr)->sin_addr.s_addr));
956 return IN6_IS_ADDR_LOOPBACK(
957 &((
struct sockaddr_in6 *) addr)->sin6_addr);
960static inline int c4iw_any_addr(
struct sockaddr *addr)
962 return c4iw_zero_addr(addr) || c4iw_loopback_addr(addr);
966process_newconn(
struct c4iw_listen_ep *master_lep,
struct socket *new_so)
970 struct sockaddr_in *remote = NULL;
973 MPASS(new_so != NULL);
975 if (c4iw_any_addr((
struct sockaddr *)&master_lep->
com.
local_addr)) {
980 real_lep = find_real_listen_ep(master_lep, new_so);
981 if (real_lep == NULL) {
982 CTR2(
KTR_IW_CXGBE,
"%s: Could not find the real listen "
983 "ep for sock: %p", __func__, new_so);
984 log(LOG_ERR,
"%s: Could not find the real listen ep for "
985 "sock: %p\n", __func__, new_so);
993 real_lep = master_lep;
995 new_ep = alloc_ep(
sizeof(*new_ep), GFP_KERNEL);
997 CTR6(
KTR_IW_CXGBE,
"%s: master_lep %p, real_lep: %p, new ep %p, "
998 "listening so %p, new so %p", __func__, master_lep, real_lep,
999 new_ep, master_lep->
com.
so, new_so);
1002 new_ep->
com.
so = new_so;
1010 init_timer(&new_ep->
timer);
1013 setiwsockopt(new_so);
1014 ret = soaccept(new_so, (
struct sockaddr **)&remote);
1017 "%s:listen sock:%p, new sock:%p, ret:%d",
1018 __func__, master_lep->
com.
so, new_so, ret);
1020 free(remote, M_SONAME);
1026 free(remote, M_SONAME);
1028 START_EP_TIMER(new_ep);
1037 init_iwarp_socket(new_so, &new_ep->
com);
1039 ret = process_mpa_request(new_ep);
1051add_ep_to_req_list(
struct c4iw_ep *ep,
int new_ep_event)
1055 spin_lock_irqsave(&req_lock, flag);
1056 if (ep && ep->
com.
so) {
1058 if (!ep->
com.entry.tqe_prev) {
1060 TAILQ_INSERT_TAIL(&req_list, &ep->
com, entry);
1061 queue_work(c4iw_taskq, &c4iw_task);
1064 spin_unlock_irqrestore(&req_lock, flag);
1070c4iw_so_upcall(
struct socket *so,
void *arg,
int waitflag)
1075 "%s: so %p, so_state 0x%x, ep %p, ep_state %s, tqe_prev %p",
1076 __func__, so, so->so_state, ep, states[ep->
com.
state],
1077 ep->
com.entry.tqe_prev);
1079 MPASS(ep->
com.
so == so);
1097 unsigned int tid =
GET_TID(cpl);
1102 INP_WLOCK(toep->
inp);
1103 so = inp_inpcbtosocket(toep->
inp);
1104 ep = so->so_rcv.sb_upcallarg;
1105 INP_WUNLOCK(toep->
inp);
1107 CTR3(
KTR_IW_CXGBE,
"%s: so %p, ep %p", __func__, so, ep);
1114process_socket_event(
struct c4iw_ep *ep)
1117 struct socket *so = ep->
com.
so;
1120 CTR3(
KTR_IW_CXGBE,
"%s: Pending socket event discarded "
1121 "ep %p ep_state %s", __func__, ep, states[state]);
1125 CTR6(
KTR_IW_CXGBE,
"process_socket_event: so %p, so_state 0x%x, "
1126 "so_err %d, sb_state 0x%x, ep %p, ep_state %s", so, so->so_state,
1127 so->so_error, so->so_rcv.sb_state, ep, states[state]);
1130 process_connected(ep);
1136 struct socket *listen_so = so, *new_so = NULL;
1139 SOLISTEN_LOCK(listen_so);
1141 error = solisten_dequeue(listen_so, &new_so,
1145 "error %d", __func__, lep, listen_so,
1149 process_newconn(lep, new_so);
1154 SOLISTEN_LOCK(listen_so);
1155 }
while (listen_so->sol_qlen);
1156 SOLISTEN_UNLOCK(listen_so);
1163 process_conn_error(ep);
1168 if ((so->so_rcv.sb_state & SBS_CANTRCVMORE) && state <=
CLOSING) {
1169 process_peer_close(ep);
1174 if (!(so->so_state & SS_ISDISCONNECTED))
1179 if (so->so_state & SS_ISDISCONNECTED) {
1180 process_close_complete(ep);
1185 if (sbused(&ep->
com.
so->so_rcv)) {
1198 log(LOG_ERR,
"%s: Unprocessed socket event so %p, "
1199 "so_state 0x%x, so_err %d, sb_state 0x%x, ep %p, ep_state %s\n",
1200 __func__, so, so->so_state, so->so_error, so->so_rcv.sb_state,
1205SYSCTL_NODE(_hw, OID_AUTO, iw_cxgbe, CTLFLAG_RD | CTLFLAG_MPSAFE, 0,
1206 "iw_cxgbe driver parameters");
1208static int dack_mode = 0;
1209SYSCTL_INT(_hw_iw_cxgbe, OID_AUTO, dack_mode, CTLFLAG_RWTUN, &dack_mode, 0,
1210 "Delayed ack mode (default = 0)");
1214 "Per-connection max ORD/IRD (default = 8)");
1216static int enable_tcp_timestamps;
1217SYSCTL_INT(_hw_iw_cxgbe, OID_AUTO, enable_tcp_timestamps, CTLFLAG_RWTUN, &enable_tcp_timestamps, 0,
1218 "Enable tcp timestamps (default = 0)");
1220static int enable_tcp_sack;
1221SYSCTL_INT(_hw_iw_cxgbe, OID_AUTO, enable_tcp_sack, CTLFLAG_RWTUN, &enable_tcp_sack, 0,
1222 "Enable tcp SACK (default = 0)");
1224static int enable_tcp_window_scaling = 1;
1225SYSCTL_INT(_hw_iw_cxgbe, OID_AUTO, enable_tcp_window_scaling, CTLFLAG_RWTUN, &enable_tcp_window_scaling, 0,
1226 "Enable tcp window scaling (default = 1)");
1230 "Enable debug logging (default = 0)");
1232static int peer2peer = 1;
1233SYSCTL_INT(_hw_iw_cxgbe, OID_AUTO, peer2peer, CTLFLAG_RWTUN, &peer2peer, 0,
1234 "Support peer2peer ULPs (default = 1)");
1237SYSCTL_INT(_hw_iw_cxgbe, OID_AUTO, p2p_type, CTLFLAG_RWTUN, &p2p_type, 0,
1238 "RDMAP opcode to use for the RTR message: 1 = RDMA_READ 0 = RDMA_WRITE (default 1)");
1240static int ep_timeout_secs = 60;
1241SYSCTL_INT(_hw_iw_cxgbe, OID_AUTO, ep_timeout_secs, CTLFLAG_RWTUN, &ep_timeout_secs, 0,
1242 "CM Endpoint operation timeout in seconds (default = 60)");
1244static int mpa_rev = 1;
1245SYSCTL_INT(_hw_iw_cxgbe, OID_AUTO, mpa_rev, CTLFLAG_RWTUN, &mpa_rev, 0,
1246 "MPA Revision, 0 supports amso1100, 1 is RFC5044 spec compliant, 2 is IETF MPA Peer Connect Draft compliant (default = 1)");
1248static int markers_enabled;
1249SYSCTL_INT(_hw_iw_cxgbe, OID_AUTO, markers_enabled, CTLFLAG_RWTUN, &markers_enabled, 0,
1250 "Enable MPA MARKERS (default(0) = disabled)");
1252static int crc_enabled = 1;
1253SYSCTL_INT(_hw_iw_cxgbe, OID_AUTO, crc_enabled, CTLFLAG_RWTUN, &crc_enabled, 0,
1254 "Enable MPA CRC (default(1) = enabled)");
1256static int rcv_win = 256 * 1024;
1257SYSCTL_INT(_hw_iw_cxgbe, OID_AUTO, rcv_win, CTLFLAG_RWTUN, &rcv_win, 0,
1258 "TCP receive window in bytes (default = 256KB)");
1260static int snd_win = 128 * 1024;
1261SYSCTL_INT(_hw_iw_cxgbe, OID_AUTO, snd_win, CTLFLAG_RWTUN, &snd_win, 0,
1262 "TCP send window in bytes (default = 128KB)");
1266 "Use DSGL for PBL/FastReg (default=1)");
1270 "inline vs dsgl threshold (default=128)");
1272static int reuseaddr = 0;
1273SYSCTL_INT(_hw_iw_cxgbe, OID_AUTO, reuseaddr, CTLFLAG_RWTUN, &reuseaddr, 0,
1274 "Enable SO_REUSEADDR & SO_REUSEPORT socket options on all iWARP client connections(default = 0)");
1277start_ep_timer(
struct c4iw_ep *ep)
1280 if (timer_pending(&ep->
timer)) {
1281 CTR2(
KTR_IW_CXGBE,
"%s: ep %p, already started", __func__, ep);
1282 printk(KERN_ERR
"%s timer already started! ep %p\n", __func__,
1288 ep->
timer.expires = jiffies + ep_timeout_secs * HZ;
1289 ep->
timer.data = (
unsigned long)ep;
1290 ep->
timer.function = ep_timeout;
1291 add_timer(&ep->
timer);
1295stop_ep_timer(
struct c4iw_ep *ep)
1298 del_timer_sync(&ep->
timer);
1307alloc_ep(
int size, gfp_t gfp)
1311 epc = kzalloc(size, gfp);
1315 kref_init(&epc->
kref);
1316 mutex_init(&epc->
mutex);
1325#if defined(KTR) || defined(INVARIANTS)
1330#if defined(KTR) || defined(INVARIANTS)
1333 KASSERT(!epc->entry.tqe_prev, (
"%s epc %p still on req list",
1337 CTR4(
KTR_IW_CXGBE,
"%s: ep %p, history 0x%lx, flags 0x%lx",
1342static void release_ep_resources(
struct c4iw_ep *ep)
1351send_mpa_req(
struct c4iw_ep *ep)
1357 char mpa_rev_to_use = mpa_rev;
1362 mpalen =
sizeof(*mpa) + ep->
plen;
1363 if (mpa_rev_to_use == 2)
1366 mpa = malloc(mpalen, M_CXGBE, M_NOWAIT);
1374 memset(mpa, 0, mpalen);
1382 if (mpa_rev_to_use == 1) {
1387 if (mpa_rev_to_use == 2) {
1390 mpa_v2_params.ird = htons((
u16)ep->
ird);
1391 mpa_v2_params.ord = htons((
u16)ep->
ord);
1397 mpa_v2_params.ord |=
1400 mpa_v2_params.ord |=
1421 m = m_getm(NULL, mpalen, M_NOWAIT, MT_DATA);
1429 m_copyback(m, 0, mpalen, (
void *)mpa);
1432 err = -sosend(ep->
com.
so, NULL, NULL, m, NULL, MSG_DONTWAIT,
1443 CTR3(
KTR_IW_CXGBE,
"%s:smrE %p, error: %d", __func__, ep, err);
1446 connect_reply_upcall(ep, err);
1447 CTR3(
KTR_IW_CXGBE,
"%s:smrE %p, error: %d", __func__, ep, err);
1451static int send_mpa_reject(
struct c4iw_ep *ep,
const void *pdata,
u8 plen)
1462 mpalen =
sizeof(*mpa) + plen;
1471 mpa = malloc(mpalen, M_CXGBE, M_NOWAIT);
1475 memset(mpa, 0, mpalen);
1486 mpa_v2_params.ird = htons(((
u16)ep->
ird) |
1489 mpa_v2_params.ord = htons(((
u16)ep->
ord) | (peer2peer ?
1501 CTR5(
KTR_IW_CXGBE,
"%s:smrej3 %p %d %d %d", __func__, ep,
1502 mpa_v2_params.ird, mpa_v2_params.ord, ep->
plen);
1507 m = m_getm(NULL, mpalen, M_NOWAIT, MT_DATA);
1512 m_copyback(m, 0, mpalen, (
void *)mpa);
1515 err = -sosend(ep->
com.
so, NULL, NULL, m, NULL, MSG_DONTWAIT, ep->
com.
thread);
1522static int send_mpa_reply(
struct c4iw_ep *ep,
const void *pdata,
u8 plen)
1532 mpalen =
sizeof(*mpa) + plen;
1541 mpa = malloc(mpalen, M_CXGBE, M_NOWAIT);
1545 memset(mpa, 0,
sizeof(*mpa));
1557 mpa_v2_params.ird = htons((
u16)ep->
ird);
1558 mpa_v2_params.ord = htons((
u16)ep->
ord);
1559 CTR5(
KTR_IW_CXGBE,
"%s:smrep3 %p %d %d %d", __func__, ep,
1569 mpa_v2_params.ord |=
1572 __func__, ep, p2p_type, mpa_v2_params.ird,
1577 mpa_v2_params.ord |=
1580 __func__, ep, p2p_type, mpa_v2_params.ird,
1595 m = m_getm(NULL, mpalen, M_NOWAIT, MT_DATA);
1600 m_copyback(m, 0, mpalen, (
void *)mpa);
1606 err = -sosend(ep->
com.
so, NULL, NULL, m, NULL, MSG_DONTWAIT,
1608 CTR3(
KTR_IW_CXGBE,
"%s:smrepE %p %d", __func__, ep, err);
1614static void close_complete_upcall(
struct c4iw_ep *ep,
int status)
1616 struct iw_cm_event event;
1619 memset(&event, 0,
sizeof(event));
1620 event.event = IW_CM_EVENT_CLOSE;
1621 event.status = status;
1627 deref_cm_id(&ep->
com);
1636 struct socket *so = ep->
com.
so;
1637 struct sockopt sopt;
1641 CTR5(
KTR_IW_CXGBE,
"%s ep %p so %p state %s tid %d", __func__, ep, so,
1648 sopt.sopt_dir = SOPT_SET;
1649 sopt.sopt_level = SOL_SOCKET;
1650 sopt.sopt_name = SO_LINGER;
1651 sopt.sopt_val = (caddr_t)&l;
1652 sopt.sopt_valsize =
sizeof l;
1653 sopt.sopt_td = NULL;
1654 rc = -sosetopt(so, &sopt);
1656 log(LOG_ERR,
"%s: sosetopt(%p, linger = 0) failed with %d.\n",
1660 uninit_iwarp_socket(so);
1671 release_ep_resources(ep);
1677static void peer_close_upcall(
struct c4iw_ep *ep)
1679 struct iw_cm_event event;
1682 memset(&event, 0,
sizeof(event));
1683 event.event = IW_CM_EVENT_DISCONNECT;
1694static void peer_abort_upcall(
struct c4iw_ep *ep)
1696 struct iw_cm_event event;
1699 memset(&event, 0,
sizeof(event));
1700 event.event = IW_CM_EVENT_CLOSE;
1701 event.status = -ECONNRESET;
1707 deref_cm_id(&ep->
com);
1713static void connect_reply_upcall(
struct c4iw_ep *ep,
int status)
1715 struct iw_cm_event event;
1717 CTR3(
KTR_IW_CXGBE,
"%s:cruB %p, status: %d", __func__, ep, status);
1718 memset(&event, 0,
sizeof(event));
1719 event.event = IW_CM_EVENT_CONNECT_REPLY;
1720 event.status = ((status == -ECONNABORTED) || (status == -EPIPE)) ?
1721 -ECONNRESET : status;
1725 if ((status == 0) || (status == -ECONNREFUSED)) {
1731 event.ord = ep->
ird;
1732 event.ird = ep->
ord;
1733 event.private_data_len = ep->
plen -
1735 event.private_data = ep->
mpa_pkt +
1744 event.private_data_len = ep->
plen;
1745 event.private_data = ep->
mpa_pkt +
1757 if(status == -ECONNABORTED) {
1759 CTR3(
KTR_IW_CXGBE,
"%s:cruE %p %d", __func__, ep, status);
1765 CTR3(
KTR_IW_CXGBE,
"%s:cru4 %p %d", __func__, ep, status);
1766 deref_cm_id(&ep->
com);
1772static int connect_request_upcall(
struct c4iw_ep *ep)
1774 struct iw_cm_event event;
1777 CTR3(
KTR_IW_CXGBE,
"%s: ep %p, mpa_v1 %d", __func__, ep,
1780 memset(&event, 0,
sizeof(event));
1781 event.event = IW_CM_EVENT_CONNECT_REQUEST;
1784 event.provider_data = ep;
1789 event.ird = ep->
ird;
1790 event.private_data_len = ep->
plen -
1799 event.private_data_len = ep->
plen;
1807 CTR3(
KTR_IW_CXGBE,
"%s: ep %p, Failure while notifying event to"
1808 " IWCM, err:%d", __func__, ep, ret);
1821static void established_upcall(
struct c4iw_ep *ep)
1823 struct iw_cm_event event;
1826 memset(&event, 0,
sizeof(event));
1827 event.event = IW_CM_EVENT_ESTABLISHED;
1828 event.ird = ep->
ord;
1829 event.ord = ep->
ird;
1841#define RELAXED_IRD_NEGOTIATION 1
1855static int process_mpa_reply(
struct c4iw_ep *ep)
1860 u16 resp_ird, resp_ord;
1861 u8 rtr_mismatch = 0, insuff_ird = 0;
1865 struct mbuf *top, *m;
1866 int flags = MSG_DONTWAIT;
1877 if (STOP_EP_TIMER(ep))
1880 uio.uio_resid = 1000000;
1882 err = soreceive(ep->
com.
so, NULL, &uio, &top, NULL, &flags);
1886 if (err == EWOULDBLOCK) {
1897 if (ep->
com.
so->so_rcv.sb_mb) {
1900 printf(
"%s data after soreceive called! so %p sb_mb %p top %p\n",
1901 __func__, ep->
com.
so, ep->
com.
so->so_rcv.sb_mb, top);
1918 goto err_stop_timer;
1946 printk(KERN_ERR
MOD "%s MPA version mismatch. Local = %d, "
1947 " Received = %d\n", __func__, mpa_rev, mpa->
revision);
1949 goto err_stop_timer;
1956 goto err_stop_timer;
1968 goto err_stop_timer;
1979 goto err_stop_timer;
1997 err = -ECONNREFUSED;
1998 goto err_stop_timer;
2024 resp_ird = ntohs(mpa_v2_params->
ird) &
2026 resp_ord = ntohs(mpa_v2_params->
ord) &
2034 if (ep->
ird < resp_ord) {
2035 if (RELAXED_IRD_NEGOTIATION && resp_ord <=
2040 }
else if (ep->
ird > resp_ord) {
2043 if (ep->
ord > resp_ird) {
2044 if (RELAXED_IRD_NEGOTIATION)
2055 if (ntohs(mpa_v2_params->
ird) &
2059 if (ntohs(mpa_v2_params->
ord) &
2066 else if (ntohs(mpa_v2_params->
ord) &
2091 if (set_tcpinfo(ep)) {
2094 printf(
"%s set_tcpinfo error\n", __func__);
2099 CTR6(
KTR_IW_CXGBE,
"%s - crc_enabled = %d, recv_marker_enabled = %d, "
2100 "xmit_marker_enabled = %d, version = %d p2p_type = %d", __func__,
2147 printk(KERN_ERR
"%s: RTR mismatch, sending TERM\n", __func__);
2168 printk(KERN_ERR
"%s: Insufficient IRD, sending TERM\n",
2186 connect_reply_upcall(ep, err);
2204process_mpa_request(
struct c4iw_ep *ep)
2209 int flags = MSG_DONTWAIT;
2215 CTR3(
KTR_IW_CXGBE,
"%s: ep %p, state %s", __func__, ep, states[state]);
2226 uio.uio_segflg = UIO_SYSSPACE;
2227 uio.uio_rw = UIO_READ;
2230 rc = soreceive(ep->
com.
so, NULL, &uio, NULL, NULL, &flags);
2234 goto err_stop_timer;
2236 KASSERT(uio.uio_offset > 0, (
"%s: sorecieve on so %p read no data",
2237 __func__, ep->
com.
so));
2256 log(LOG_ERR,
"%s: MPA version mismatch. Local = %d,"
2257 " Received = %d\n", __func__, mpa_rev, mpa->
revision);
2258 goto err_stop_timer;
2262 goto err_stop_timer;
2269 goto err_stop_timer;
2275 goto err_stop_timer;
2304 ep->
ird = ntohs(mpa_v2_params->
ird) &
2308 ep->
ord = ntohs(mpa_v2_params->
ord) &
2313 __func__, ep->
ird, ep->
ord);
2316 if (ntohs(mpa_v2_params->
ord) &
2320 else if (ntohs(mpa_v2_params->
ord) &
2326 }
else if (mpa->
revision == 1 && peer2peer)
2329 if (set_tcpinfo(ep))
2330 goto err_stop_timer;
2332 CTR5(
KTR_IW_CXGBE,
"%s: crc_enabled = %d, recv_marker_enabled = %d, "
2333 "xmit_marker_enabled = %d, version = %d", __func__,
2342 if (connect_request_upcall(ep))
2386 abort = send_mpa_reject(ep, pdata, pdata_len);
2396 CTR3(
KTR_IW_CXGBE,
"%s:crc4 %p, err: %d", __func__, ep, err);
2400int c4iw_accept_cr(
struct iw_cm_id *cm_id,
struct iw_cm_conn_param *conn_param)
2437 if (conn_param->ord >
ep->
ird) {
2438 if (RELAXED_IRD_NEGOTIATION) {
2439 conn_param->ord =
ep->
ird;
2441 ep->
ird = conn_param->ird;
2442 ep->
ord = conn_param->ord;
2443 send_mpa_reject(
ep, conn_param->private_data,
2444 conn_param->private_data_len);
2449 if (conn_param->ird <
ep->
ord) {
2450 if (RELAXED_IRD_NEGOTIATION &&
2452 conn_param->ird =
ep->
ord;
2459 ep->
ird = conn_param->ird;
2460 ep->
ord = conn_param->ord;
2463 if (peer2peer &&
ep->
ird == 0)
2472 CTR4(
KTR_IW_CXGBE,
"%s %d ird %d ord %d", __func__, __LINE__,
2476 ref_cm_id(&
ep->
com);
2498 goto err_defef_cm_id;
2501 err = send_mpa_reply(
ep, conn_param->private_data,
2502 conn_param->private_data_len);
2505 goto err_defef_cm_id;
2509 established_upcall(
ep);
2515 deref_cm_id(&
ep->
com);
2528c4iw_sock_create(
struct sockaddr_storage *laddr,
struct socket **so)
2532 struct socket *sock = NULL;
2533 struct sockopt sopt;
2535 ret = sock_create_kern(laddr->ss_family,
2536 SOCK_STREAM, IPPROTO_TCP, &sock);
2538 CTR2(
KTR_IW_CXGBE,
"%s:Failed to create TCP socket. err %d",
2544 bzero(&sopt,
sizeof(
struct sockopt));
2545 sopt.sopt_dir = SOPT_SET;
2546 sopt.sopt_level = SOL_SOCKET;
2547 sopt.sopt_name = SO_REUSEADDR;
2549 sopt.sopt_val = &on;
2550 sopt.sopt_valsize =
sizeof(on);
2551 ret = -sosetopt(sock, &sopt);
2553 log(LOG_ERR,
"%s: sosetopt(%p, SO_REUSEADDR) "
2554 "failed with %d.\n", __func__, sock, ret);
2556 bzero(&sopt,
sizeof(
struct sockopt));
2557 sopt.sopt_dir = SOPT_SET;
2558 sopt.sopt_level = SOL_SOCKET;
2559 sopt.sopt_name = SO_REUSEPORT;
2561 sopt.sopt_val = &on;
2562 sopt.sopt_valsize =
sizeof(on);
2563 ret = -sosetopt(sock, &sopt);
2565 log(LOG_ERR,
"%s: sosetopt(%p, SO_REUSEPORT) "
2566 "failed with %d.\n", __func__, sock, ret);
2570 ret = -sobind(sock, (
struct sockaddr *)laddr, curthread);
2578 size = laddr->ss_family == AF_INET6 ?
2579 sizeof(
struct sockaddr_in6) : sizeof(struct sockaddr_in);
2580 ret = sock_getname(sock, (struct sockaddr *)laddr, &size, 0);
2592int c4iw_connect(
struct iw_cm_id *cm_id,
struct iw_cm_conn_param *conn_param)
2597 struct ifnet *nh_ifp;
2598 struct epoch_tracker et;
2600 struct rdma_cm_id *rdma_id = (
struct rdma_cm_id*)cm_id->context;
2601 struct vnet *vnet = rdma_id->route.addr.dev_addr.net;
2614 ep = alloc_ep(
sizeof(*ep), GFP_KERNEL);
2615 cm_id->provider_data = ep;
2617 init_timer(&ep->
timer);
2618 ep->
plen = conn_param->private_data_len;
2624 conn_param->private_data, ep->
plen);
2626 ep->
ird = conn_param->ird;
2627 ep->
ord = conn_param->ord;
2629 if (peer2peer && ep->
ord == 0) {
2637 ref_cm_id(&ep->
com);
2649 NET_EPOCH_ENTER(et);
2651 err = get_ifnet_from_raddr(&cm_id->remote_addr, &nh_ifp);
2658 printk(KERN_ERR
MOD "%s - cannot find route.\n", __func__);
2663 if (!(nh_ifp->if_capenable & IFCAP_TOE) ||
2664 TOEDEV(nh_ifp) == NULL) {
2673 err = c4iw_sock_create(&cm_id->local_addr, &ep->
com.
so);
2677 setiwsockopt(ep->
com.
so);
2678 init_iwarp_socket(ep->
com.
so, &ep->
com);
2687 uninit_iwarp_socket(ep->
com.
so);
2689 sock_release(ep->
com.
so);
2691 deref_cm_id(&ep->
com);
2707 struct listen_port_info *
port_info = NULL;
2710 CTR3(
KTR_IW_CXGBE,
"%s: cm_id %p, backlog %s", __func__, cm_id,
2713 CTR2(
KTR_IW_CXGBE,
"%s: cm_id %p, fatal error", __func__,
2717 lep = alloc_ep(
sizeof(*lep), GFP_KERNEL);
2719 ref_cm_id(&lep->
com);
2724 cm_id->provider_data = lep;
2738 if (c4iw_any_addr((
struct sockaddr *)&lep->
com.
local_addr)) {
2755 rc = c4iw_sock_create(&cm_id->local_addr, &lep->
com.
so);
2757 CTR2(
KTR_IW_CXGBE,
"%s:Failed to create socket. err %d",
2764 CTR3(
KTR_IW_CXGBE,
"%s:Failed to listen on sock:%p. err %d",
2765 __func__, lep->
com.
so, rc);
2768 init_iwarp_socket(lep->
com.
so, &lep->
com);
2773 sock_release(lep->
com.
so);
2776 rem_ep_from_listenlist(lep);
2777 deref_cm_id(&lep->
com);
2788 CTR3(
KTR_IW_CXGBE,
"%s: cm_id %p, state %s", __func__, cm_id,
2792 if (c4iw_any_addr((
struct sockaddr *)&lep->
com.
local_addr)) {
2794 if (!rem_ep_from_listenlist(lep))
2795 close_socket(lep->
com.
so);
2797 close_socket(lep->
com.
so);
2798 deref_cm_id(&lep->
com);
2825 CTR3(
KTR_IW_CXGBE,
"%s:ced1 fatal error %p %s", __func__, ep,
2831 close_complete_upcall(ep, -ECONNRESET);
2876 "%s ignoring disconnect ep %p state %u", __func__,
2894 close_complete_upcall(ep, -ECONNRESET);
2904 CURVNET_SET(ep->
com.
so->so_vnet);
2905 ret = sodisconnect(ep->
com.
so);
2912 close_complete_upcall(ep, -ECONNRESET);
2933#ifdef C4IW_EP_REDIRECT
2934int c4iw_ep_redirect(
void *ctx,
struct dst_entry *old,
struct dst_entry *
new,
2942 PDBG(
"%s ep %p redirect to dst %p l2t %p\n", __func__, ep,
new,
2945 cxgb4_l2t_release(ep->
l2t);
2955static void ep_timeout(
unsigned long arg)
2973 uint64_t val = be64toh(*rpl);
2977 ret = (int)((val >> 8) & 0xff);
2986static int fw6_cqe_handler(
struct adapter *sc,
const __be64 *rpl)
2988 struct cqe_list_entry *cle;
2991 cle = malloc(
sizeof(*cle), M_CXGBE, M_NOWAIT);
2993 cle->err_cqe = *(
const struct t4_cqe *)(&rpl[0]);
2995 spin_lock_irqsave(&err_cqe_lock, flag);
2996 list_add_tail(&cle->entry, &err_cqe_list);
2997 queue_work(c4iw_taskq, &c4iw_task);
2998 spin_unlock_irqrestore(&err_cqe_lock, flag);
3004process_terminate(
struct c4iw_ep *ep)
3010 if (ep && ep->
com.
qp) {
3012 printk(KERN_WARNING
MOD "TERM received tid %u qpid %u\n",
3018 printk(KERN_WARNING
MOD "TERM received tid %u no ep/qp\n",
3033 TAILQ_INIT(&req_list);
3034 spin_lock_init(&req_lock);
3035 INIT_LIST_HEAD(&err_cqe_list);
3036 spin_lock_init(&err_cqe_lock);
3038 INIT_WORK(&c4iw_task, process_req);
3040 c4iw_taskq = create_singlethread_workqueue(
"iw_cxgbe");
3049 WARN_ON(!TAILQ_EMPTY(&req_list));
3050 WARN_ON(!list_empty(&err_cqe_list));
3051 flush_workqueue(c4iw_taskq);
3052 destroy_workqueue(c4iw_taskq);
void t4_register_an_handler(an_handler_t)
void t4_register_cpl_handler(int, cpl_handler_t)
TAILQ_HEAD(wrq_incomplete_wrs, wrq_cookie) incomplete_wrs
void t4_register_fw_msg_handler(int, fw_msg_handler_t)
#define for_each_port(adapter, iter)
#define min_t(type, _a, _b)
LIST_HEAD(cxgbei_cmp_head, cxgbei_cmp)
#define C4IW_EVENT_SOCKET
int c4iw_create_listen(struct iw_cm_id *cm_id, int backlog)
static void c4iw_wake_up(struct c4iw_wr_wait *wr_waitp, int ret)
int c4iw_reject_cr(struct iw_cm_id *cm_id, const void *pdata, u8 pdata_len)
#define MPA_V2_RDMA_WRITE_RTR
static struct c4iw_listen_ep * to_listen_ep(struct iw_cm_id *cm_id)
int c4iw_modify_qp(struct c4iw_dev *rhp, struct c4iw_qp *qhp, enum c4iw_qp_attr_mask mask, struct c4iw_qp_attributes *attrs, int internal)
static struct c4iw_ep * to_ep(struct iw_cm_id *cm_id)
static int cur_max_read_depth(struct c4iw_dev *dev)
static struct c4iw_qp * get_qhp(struct c4iw_dev *rhp, u32 qpid)
int c4iw_ev_handler(struct sge_iq *, const struct rsp_ctrl *)
#define MPA_V2_PEER2PEER_MODEL
#define MPA_MAX_PRIVATE_DATA
#define MPA_ENHANCED_RDMA_CONN
#define MPA_V2_IRD_ORD_MASK
int __init c4iw_cm_init(void)
int __c4iw_ep_disconnect(struct c4iw_ep *ep, int abrupt, gfp_t gfp)
int c4iw_connect(struct iw_cm_id *cm_id, struct iw_cm_conn_param *conn_param)
#define MPA_V2_RDMA_READ_RTR
static struct c4iw_dev * to_c4iw_dev(struct ib_device *ibdev)
static int c4iw_fatal_error(struct c4iw_rdev *rdev)
void c4iw_qp_rem_ref(struct ib_qp *qp)
int c4iw_destroy_listen(struct iw_cm_id *cm_id)
#define C4IW_EVENT_TIMEOUT
@ C4IW_QP_STATE_TERMINATE
void c4iw_ev_dispatch(struct c4iw_dev *dev, struct t4_cqe *err_cqe)
@ C4IW_QP_ATTR_NEXT_STATE
@ C4IW_QP_ATTR_LLP_STREAM_HANDLE
static void c4iw_init_wr_wait(struct c4iw_wr_wait *wr_waitp)
#define PDBG(fmt, args...)
void __exit c4iw_cm_term(void)
void c4iw_qp_add_ref(struct ib_qp *qp)
int c4iw_accept_cr(struct iw_cm_id *cm_id, struct iw_cm_conn_param *conn_param)
int c4iw_ep_redirect(void *ctx, struct dst_entry *old, struct dst_entry *new, struct l2t_entry *l2t)
int c4iw_ep_disconnect(struct c4iw_ep *ep, int abrupt, gfp_t gfp)
void _c4iw_free_ep(struct kref *kref)
#define container_of(p, s, f)
unsigned int max_ordird_qp
struct adapter_params params
struct port_info * port[MAX_NPORTS]
struct sockaddr_storage local_addr
struct c4iw_wr_wait wr_wait
struct sockaddr_storage remote_addr
struct c4iw_mpa_attributes mpa_attr
u8 mpa_pkt[sizeof(struct mpa_message)+MPA_MAX_PRIVATE_DATA]
struct c4iw_listen_ep * parent_ep
struct c4iw_ep_common com
struct c4iw_ep_common com
struct list_head listen_ep_list
struct c4iw_mpa_attributes mpa_attr
struct c4iw_ep * llp_stream_handle
SYSCTL_INT(_hw_cxgbe, OID_AUTO, fl_pktshift, CTLFLAG_RDTUN, &fl_pktshift, 0, "payload DMA offset in rx buffer (bytes)")
void * lookup_tid(struct adapter *, int)
@ FW_RI_INIT_P2PTYPE_READ_REQ
@ FW_RI_INIT_P2PTYPE_DISABLED
@ FW_RI_INIT_P2PTYPE_RDMA_WRITE