39#include <sys/kernel.h>
41#include <sys/module.h>
42#include <sys/protosw.h>
43#include <sys/domain.h>
44#include <sys/socket.h>
45#include <sys/socketvar.h>
46#include <sys/sysctl.h>
47#include <net/ethernet.h>
49#include <net/if_types.h>
50#include <net/if_vlan_var.h>
52#include <net/route/nhop.h>
53#include <netinet/in.h>
54#include <netinet/in_pcb.h>
55#include <netinet/ip.h>
57#include <netinet/tcp_fsm.h>
58#include <netinet/tcp_var.h>
59#include <netinet/toecore.h>
60#include <netinet/cc/cc.h>
82 struct inpcb *inp = toep->
inp;
84 KASSERT(m == NULL, (
"%s: wasn't expecting payload", __func__));
85 KASSERT(toep->
tid == atid, (
"%s: toep tid/atid mismatch", __func__));
87 CTR3(
KTR_CXGBE,
"%s: atid %u, tid %u", __func__, atid, tid);
90 CURVNET_SET(toep->
vnet);
93 insert_tid(sc, tid, toep, inp->inp_vflag & INP_IPV6 ? 2 : 1);
94 if (inp->inp_flags & INP_DROPPED) {
105 inp->inp_flowtype = M_HASHTYPE_OPAQUE;
106 inp->inp_flowid = tid;
118 struct inpcb *inp = toep->
inp;
119 struct toedev *tod = &toep->
td->
tod;
120 struct epoch_tracker et;
125 CURVNET_SET(toep->
vnet);
126 if (status != EAGAIN)
129 toe_connect_failed(tod, inp, status);
131 if (status != EAGAIN)
150 KASSERT(m == NULL, (
"%s: wasn't expecting payload", __func__));
151 KASSERT(toep->
tid == atid, (
"%s: toep tid/atid mismatch", __func__));
153 CTR3(
KTR_CXGBE,
"%s: atid %u, status %u ", __func__, atid, status);
186#define DONT_OFFLOAD_ACTIVE_OPEN(x) do { \
192#define DONT_OFFLOAD_ACTIVE_OPEN(x) do { \
199act_open_cpl_size(
struct adapter *sc,
int isipv6)
202 static const int sz_table[3][2] = {
220 return (sz_table[idx][!!isipv6]);
234t4_connect(
struct toedev *tod,
struct socket *so,
struct nhop_object *nh,
235 struct sockaddr *nam)
237 struct adapter *sc = tod->tod_softc;
238 struct toepcb *toep = NULL;
239 struct wrqe *wr = NULL;
240 struct ifnet *rt_ifp = nh->nh_ifp;
242 int qid_atid, rc, isipv6;
243 struct inpcb *inp = sotoinpcb(so);
244 struct tcpcb *tp = intotcpcb(inp);
249 struct epoch_tracker et;
250 uint16_t vid = 0xfff, pcp = 0;
252 INP_WLOCK_ASSERT(inp);
253 KASSERT(nam->sa_family == AF_INET || nam->sa_family == AF_INET6,
254 (
"%s: dest addr %p has family %u", __func__, nam, nam->sa_family));
256 if (rt_ifp->if_type == IFT_ETHER)
257 vi = rt_ifp->if_softc;
258 else if (rt_ifp->if_type == IFT_L2VLAN) {
259 struct ifnet *
ifp = VLAN_TRUNKDEV(rt_ifp);
262 VLAN_TAG(rt_ifp, &vid);
263 VLAN_PCP(rt_ifp, &pcp);
264 }
else if (rt_ifp->if_type == IFT_IEEE8023ADLAG)
265 DONT_OFFLOAD_ACTIVE_OPEN(ENOSYS);
267 DONT_OFFLOAD_ACTIVE_OPEN(ENOTSUP);
269 DONT_OFFLOAD_ACTIVE_OPEN(ENOTSUP);
273 EVL_MAKETAG(vid, pcp, 0), inp);
275 if (!settings.offload)
276 DONT_OFFLOAD_ACTIVE_OPEN(EPERM);
280 DONT_OFFLOAD_ACTIVE_OPEN(ENOMEM);
284 DONT_OFFLOAD_ACTIVE_OPEN(ENOMEM);
287 nh->nh_flags & NHF_GATEWAY ? &nh->gw_sa : nam);
288 if (toep->
l2te == NULL)
289 DONT_OFFLOAD_ACTIVE_OPEN(ENOMEM);
291 toep->
vnet = so->so_vnet;
296 isipv6 = nam->sa_family == AF_INET6;
299 DONT_OFFLOAD_ACTIVE_OPEN(ENOMEM);
309 if ((inp->inp_vflag & INP_IPV6) == 0)
310 DONT_OFFLOAD_ACTIVE_OPEN(ENOTSUP);
313 if (toep->
ce == NULL)
314 DONT_OFFLOAD_ACTIVE_OPEN(ENOENT);
323 cpl5->
iss = htobe32(tp->iss);
329 cpl6->
iss = htobe32(tp->iss);
336 cpl->
local_ip_hi = *(uint64_t *)&inp->in6p_laddr.s6_addr[0];
337 cpl->
local_ip_lo = *(uint64_t *)&inp->in6p_laddr.s6_addr[8];
339 cpl->
peer_ip_hi = *(uint64_t *)&inp->in6p_faddr.s6_addr[0];
340 cpl->
peer_ip_lo = *(uint64_t *)&inp->in6p_faddr.s6_addr[8];
345 "%s: atid %u, toep %p, inp %p, opt0 %#016lx, opt2 %#08x",
346 __func__, toep->
tid, toep, inp, be64toh(cpl->
opt0),
360 cpl5->
iss = htobe32(tp->iss);
366 cpl6->
iss = htobe32(tp->iss);
378 "%s: atid %u, toep %p, inp %p, opt0 %#016lx, opt2 %#08x",
379 __func__, toep->
tid, toep, inp, be64toh(cpl->
opt0),
397 CTR3(
KTR_CXGBE,
"%s: not offloading (%d), rc %d", __func__, reason, rc);
static struct wrqe * alloc_wrqe(int wr_len, struct sge_wrq *wrq)
void free_atid(struct adapter *, int)
void * lookup_atid(struct adapter *, int)
int alloc_atid(struct adapter *, void *)
void t4_register_cpl_handler(int, cpl_handler_t)
static void * wrtod(struct wrqe *wr)
void t4_register_shared_cpl_handler(int, cpl_handler_t, int)
void release_tid(struct adapter *, int, struct sge_wrq *)
static void free_wrqe(struct wrqe *wr)
static int chip_id(struct adapter *adap)
#define INIT_TP_WR(w, tid)
struct rwlock policy_lock
struct conn_params params
struct sge_ofld_rxq * ofld_rxq
void t4_release_clip_entry(struct adapter *sc, struct clip_entry *ce)
struct clip_entry * t4_get_clip_entry(struct adapter *sc, struct in6_addr *in6, bool add)
static void t4_l2t_release(struct l2t_entry *e)
#define G_AOPEN_STATUS(x)
static int act_open_has_tid(int status)
#define MK_OPCODE_TID(opcode, tid)
static int act_open_rpl_status_to_errno(int status)
void act_open_failure_cleanup(struct adapter *, u_int, u_int)
uint64_t select_ntuple(struct vi_info *, struct l2t_entry *)
int init_toepcb(struct vi_info *, struct toepcb *)
void insert_tid(struct adapter *, int, void *, int)
void make_established(struct toepcb *, uint32_t, uint32_t, uint16_t)
__be64 calc_options0(struct vi_info *, struct conn_params *)
void free_toepcb(struct toepcb *)
void init_conn_params(struct vi_info *, struct offload_settings *, struct in_conninfo *, struct socket *, const struct tcp_options *, int16_t, struct conn_params *cp)
int t4_connect(struct toedev *, struct socket *, struct nhop_object *, struct sockaddr *)
void send_flowc_wr(struct toepcb *, struct tcpcb *)
void t4_init_connect_cpl_handlers(void)
void t4_uninit_connect_cpl_handlers(void)
void offload_socket(struct socket *, struct toepcb *)
void final_cpl_received(struct toepcb *)
void undo_offload_socket(struct socket *)
__be32 calc_options2(struct vi_info *, struct conn_params *)
struct toepcb * alloc_toepcb(struct vi_info *, int)
void send_reset(struct adapter *, struct toepcb *, uint32_t)
const struct offload_settings * lookup_offload_policy(struct adapter *, int, struct mbuf *, uint16_t, struct inpcb *)
struct l2t_entry * t4_l2t_get(struct port_info *, struct ifnet *, struct sockaddr *)
static int t4_l2t_send(struct adapter *sc, struct wrqe *wr, struct l2t_entry *e)