40#define LISTEN_HASH_SIZE 32
46#define MIN_RCV_WND (24 * 1024U)
52#define MAX_RCV_WND ((1U << 27) - 1)
54#define DDP_RSVD_WIN (16 * 1024U)
55#define SB_DDP_INDICATE SB_IN_TOE
57#define USE_DDP_RX_FLOW_CONTROL
59#define PPOD_SZ(n) ((n) * sizeof(struct pagepod))
60#define PPOD_SIZE (PPOD_SZ(1))
159#define PS_PPODS_WRITTEN 0x0001
194 struct clip_entry *
ce;
232#define DDP_LOCK(toep) mtx_lock(&(toep)->ddp.lock)
233#define DDP_UNLOCK(toep) mtx_unlock(&(toep)->ddp.lock)
234#define DDP_ASSERT_LOCKED(toep) mtx_assert(&(toep)->ddp.lock, MA_OWNED)
257#define LCTX_RPL_PENDING 1
269 struct clip_entry *
ce;
273#define TE_RPL_PENDING 1
277#define TS_RTO (1 << 0)
278#define TS_DUPACKS (1 << 1)
279#define TS_FASTREXMT (1 << 2)
280#define TS_SND_BACKLOGGED (1 << 3)
281#define TS_CWND_LIMITED (1 << 4)
282#define TS_ECN_ECE (1 << 5)
283#define TS_ECN_CWR (1 << 6)
284#define TS_RESERVED (1 << 7)
311 struct rwlock tcb_history_lock
__aligned(CACHE_LINE_SIZE);
332 return (td->
tod.tod_softc);
340 m->m_pkthdr.PH_per.eight[6] = raw;
348 return (m->m_pkthdr.PH_per.eight[6]);
356 m->m_pkthdr.PH_per.eight[0] = ulp_submode;
364 return (m->m_pkthdr.PH_per.eight[0]);
372 m->m_pkthdr.PH_per.eight[1] = iso;
380 return (m->m_pkthdr.PH_per.eight[1]);
384#define CXGBE_ISO_TYPE(flags) ((flags) & 0x3)
385#define CXGBE_ISO_F 0x4
392 m->m_pkthdr.PH_per.eight[2] =
flags;
400 return (m->m_pkthdr.PH_per.eight[2]);
408 m->m_pkthdr.PH_per.sixteen[2] = mss;
411static inline uint16_t
416 return (m->m_pkthdr.PH_per.sixteen[2]);
435 struct in_conninfo *,
struct socket *,
const struct tcp_options *, int16_t,
446int t4_connect(
struct toedev *,
struct socket *,
struct nhop_object *,
483 uint16_t, uint64_t, uint64_t,
int,
int);
505 struct ppod_reservation *,
struct ctl_sg_entry *,
int,
int,
struct mbufq *);
508 struct mbuf **,
struct mbuf **,
int *);
521 struct mbuf *, uint16_t,
struct inpcb *);
TAILQ_HEAD(, pageset) cached_pagesets
TAILQ_HEAD(, kaiocb) aiojobq
struct sge_ofld_rxq * ofld_rxq
LIST_ENTRY(listen_ctx) link
struct ppod_reservation prsv
TAILQ_ENTRY(pageset) link
struct ppod_region * prsv_pr
struct conn_params params
volatile int ok_to_respond
struct callout te_callout
struct adapter * te_adapter
uint64_t te_tcb[TCB_SIZE/sizeof(uint64_t)]
struct conn_params params
struct sge_ofld_rxq * ofld_rxq
TAILQ_HEAD(, kaiocb) aiotx_jobq
struct mbufq ulp_pdu_reclaimq
struct sge_ofld_txq * ofld_txq
struct ofld_tx_sdesc txsd[]
struct mtx unsent_wr_lock
struct tcb_histent ** tcb_history
STAILQ_HEAD(, wrqe) unsent_wr_list
struct task reclaim_wr_resources
LIST_HEAD(, listen_ctx) *listen_hash
TAILQ_HEAD(, toepcb) toep_list
struct mtx toep_list_lock
struct rwlock tcb_history_lock __aligned(CACHE_LINE_SIZE)
struct mtx lctx_hash_lock
static void set_mbuf_iscsi_iso_flags(struct mbuf *m, uint8_t flags)
void t4_free_ppod_region(struct ppod_region *)
int t4_send_fin(struct toedev *, struct tcpcb *)
void t4_uninit_listen_cpl_handlers(void)
void insert_ddp_data(struct toepcb *, uint32_t)
void tls_detach(struct toepcb *)
int t4_soreceive_ddp(struct socket *, struct sockaddr **, struct uio *, struct mbuf **, struct mbuf **, int *)
void * lookup_tid(struct adapter *, int)
static int ulp_mode(struct toepcb *toep)
void t4_ddp_mod_load(void)
int t4_write_page_pods_for_ps(struct adapter *, struct sge_wrq *, int, struct pageset *)
void t4_syncache_removed(struct toedev *, void *)
void tls_uninit_toep(struct toepcb *)
int tls_alloc_ktls(struct toepcb *, struct ktls_session *, int)
int t4_alloc_page_pods_for_sgl(struct ppod_region *, struct ctl_sg_entry *, int, struct ppod_reservation *)
int t4_alloc_page_pods_for_bio(struct ppod_region *, struct bio *, struct ppod_reservation *)
void t4_init_cpl_io_handlers(void)
void ddp_uninit_toep(struct toepcb *)
static bool mbuf_iscsi_iso(struct mbuf *m)
int t4_tod_output(struct toedev *, struct tcpcb *)
void act_open_failure_cleanup(struct adapter *, u_int, u_int)
void tls_stop_handshake_timer(struct toepcb *)
int do_abort_rpl_synqe(struct sge_iq *, const struct rss_header *, struct mbuf *)
void handle_ddp_indicate(struct toepcb *)
void t4_push_frames(struct adapter *, struct toepcb *, int)
static void set_mbuf_iscsi_iso(struct mbuf *m, bool iso)
void t4_rcvd_locked(struct toedev *, struct tcpcb *)
uint64_t select_ntuple(struct vi_info *, struct l2t_entry *)
static void set_mbuf_iscsi_iso_mss(struct mbuf *m, uint16_t mss)
void t4_ddp_mod_unload(void)
struct toepcb * hold_toepcb(struct toepcb *)
void remove_tid(struct adapter *, int, int)
void t4_set_tcb_field(struct adapter *, struct sge_wrq *, struct toepcb *, uint16_t, uint64_t, uint64_t, int, int)
void t4_free_page_pods(struct ppod_reservation *)
int tls_tx_key(struct toepcb *)
int init_toepcb(struct vi_info *, struct toepcb *)
void insert_tid(struct adapter *, int, void *, int)
static struct tom_data * tod_td(struct toedev *tod)
int t4_listen_start(struct toedev *, struct tcpcb *)
int add_tid_to_history(struct adapter *, u_int)
void t4_tls_mod_load(void)
void make_established(struct toepcb *, uint32_t, uint32_t, uint16_t)
__be64 calc_options0(struct vi_info *, struct conn_params *)
int t4_syncache_respond(struct toedev *, void *, struct mbuf *)
void free_toepcb(struct toepcb *)
int t4_alloc_page_pods_for_buf(struct ppod_region *, vm_offset_t, int, struct ppod_reservation *)
void ddp_queue_toep(struct toepcb *)
void ddp_init_toep(struct toepcb *)
int t4_listen_stop(struct toedev *, struct tcpcb *)
void t4_push_ktls(struct adapter *, struct toepcb *, int)
int send_rx_credits(struct adapter *, struct toepcb *, int)
void t4_tls_mod_unload(void)
void send_rx_modulate(struct adapter *, struct toepcb *)
void t4_init_listen_cpl_handlers(void)
void do_rx_data_tls(const struct cpl_rx_data *, struct toepcb *, struct mbuf *)
int t4_alloc_page_pods_for_ps(struct ppod_region *, struct pageset *)
int t4_send_rst(struct toedev *, struct tcpcb *)
static void set_mbuf_ulp_submode(struct mbuf *m, uint8_t ulp_submode)
int t4_write_page_pods_for_sgl(struct adapter *, struct toepcb *, struct ppod_reservation *, struct ctl_sg_entry *, int, int, struct mbufq *)
void t4_offload_socket(struct toedev *, void *, struct socket *)
void init_conn_params(struct vi_info *, struct offload_settings *, struct in_conninfo *, struct socket *, const struct tcp_options *, int16_t, struct conn_params *cp)
static uint8_t mbuf_ulp_submode(struct mbuf *m)
int t4_connect(struct toedev *, struct socket *, struct nhop_object *, struct sockaddr *)
int t4_aio_queue_ddp(struct socket *, struct kaiocb *)
int select_rcv_wscale(void)
static bool mbuf_raw_wr(struct mbuf *m)
void tls_establish(struct toepcb *)
void send_flowc_wr(struct toepcb *, struct tcpcb *)
static uint16_t mbuf_iscsi_iso_mss(struct mbuf *m)
void t4_init_connect_cpl_handlers(void)
void t4_uninit_connect_cpl_handlers(void)
void t4_uninit_cpl_io_handlers(void)
void release_ddp_resources(struct toepcb *toep)
void offload_socket(struct socket *, struct toepcb *)
int t4_write_page_pods_for_bio(struct adapter *, struct toepcb *, struct ppod_reservation *, struct bio *, struct mbufq *)
u_long select_rcv_wnd(struct socket *)
void synack_failure_cleanup(struct adapter *, int)
void final_cpl_received(struct toepcb *)
TAILQ_HEAD(pagesetq, pageset)
void undo_offload_socket(struct socket *)
void ddp_assert_empty(struct toepcb *)
int t4_close_conn(struct adapter *, struct toepcb *)
int do_abort_req_synqe(struct sge_iq *, const struct rss_header *, struct mbuf *)
void aiotx_init_toep(struct toepcb *)
void handle_ddp_close(struct toepcb *, struct tcpcb *, uint32_t)
static void set_mbuf_raw_wr(struct mbuf *m, bool raw)
__be32 calc_options2(struct vi_info *, struct conn_params *)
void t4_syncache_added(struct toedev *, void *)
bool can_tls_offload(struct adapter *)
static uint8_t mbuf_iscsi_iso_flags(struct mbuf *m)
static struct adapter * td_adapter(struct tom_data *td)
int t4_aio_queue_aiotx(struct socket *, struct kaiocb *)
struct toepcb * alloc_toepcb(struct vi_info *, int)
void tls_init_toep(struct toepcb *)
int t4_init_ppod_region(struct ppod_region *, struct t4_range *, u_int, const char *)
int t4_write_page_pods_for_buf(struct adapter *, struct toepcb *, struct ppod_reservation *, vm_offset_t, int, struct mbufq *)
void t4_push_pdus(struct adapter *, struct toepcb *, int)
void send_reset(struct adapter *, struct toepcb *, uint32_t)
void restore_so_proto(struct socket *, bool)
void send_abort_rpl(struct adapter *, struct sge_ofld_txq *, int, int)
void t4_rcvd(struct toedev *, struct tcpcb *)
void update_tid(struct adapter *, int, void *)
const struct offload_settings * lookup_offload_policy(struct adapter *, int, struct mbuf *, uint16_t, struct inpcb *)