33#ifndef __T4_ADAPTER_H__
34#define __T4_ADAPTER_H__
36#include <sys/kernel.h>
38#include <sys/counter.h>
42#include <sys/malloc.h>
43#include <sys/rwlock.h>
48#include <dev/pci/pcivar.h>
49#include <dev/pci/pcireg.h>
50#include <machine/bus.h>
51#include <sys/socket.h>
52#include <sys/sysctl.h>
53#include <sys/taskqueue.h>
54#include <net/ethernet.h>
56#include <net/if_var.h>
57#include <net/if_media.h>
59#include <netinet/in.h>
60#include <netinet/tcp_lro.h>
67#define KTR_CXGBE KTR_SPARE3
69#define CXGBE_UNIMPLEMENTED(s) \
70 panic("%s (%s, line %d) not implemented yet.", s, __FILE__, __LINE__)
76#define CXGBE_LIST_HEAD(name, type) \
78 struct type *lh_first; \
81#ifndef SYSCTL_ADD_UQUAD
82#define SYSCTL_ADD_UQUAD SYSCTL_ADD_QUAD
83#define sysctl_handle_64 sysctl_handle_quad
84#define CTLTYPE_U64 CTLTYPE_QUAD
111#if MJUMPAGESIZE != MCLBYTES
192#define IS_DOOMED(vi) ((vi)->flags & DOOMED)
193#define SET_DOOMED(vi) do {(vi)->flags |= DOOMED;} while (0)
194#define IS_BUSY(sc) ((sc)->flags & CXGBE_BUSY)
195#define SET_BUSY(sc) do {(sc)->flags |= CXGBE_BUSY;} while (0)
196#define CLR_BUSY(sc) do {(sc)->flags &= ~CXGBE_BUSY;} while (0)
248 struct sysctl_ctx_list
ctx;
339 struct sysctl_ctx_list
ctx;
342#define IS_MAIN_VI(vi) ((vi) == &((vi)->pi->vi[0]))
367#define IQ_PAD (IQ_ESIZE - sizeof(struct rsp_ctrl) - sizeof(struct rss_header))
532#define FL_RUNNING_LOW(fl) \
533 (IDXDIFF(fl->dbidx * 8, fl->cidx, fl->sidx * 8) <= fl->lowat)
534#define FL_NOT_RUNNING_LOW(fl) \
535 (IDXDIFF(fl->dbidx * 8, fl->cidx, fl->sidx * 8) >= 2 * fl->lowat)
755#define INVALID_NM_RXQ_CNTXT_ID ((uint16_t)(-1))
796#define INVALID_NM_TXQ_CNTXT_ID ((u_int)(-1))
875 bus_space_handle_t
bh;
972 struct sysctl_ctx_list
ctx;
1015#define ADAPTER_LOCK(sc) mtx_lock(&(sc)->sc_lock)
1016#define ADAPTER_UNLOCK(sc) mtx_unlock(&(sc)->sc_lock)
1017#define ADAPTER_LOCK_ASSERT_OWNED(sc) mtx_assert(&(sc)->sc_lock, MA_OWNED)
1018#define ADAPTER_LOCK_ASSERT_NOTOWNED(sc) mtx_assert(&(sc)->sc_lock, MA_NOTOWNED)
1020#define ASSERT_SYNCHRONIZED_OP(sc) \
1021 KASSERT(IS_BUSY(sc) && \
1022 (mtx_owned(&(sc)->sc_lock) || sc->last_op_thr == curthread), \
1023 ("%s: operation not synchronized.", __func__))
1025#define PORT_LOCK(pi) mtx_lock(&(pi)->pi_lock)
1026#define PORT_UNLOCK(pi) mtx_unlock(&(pi)->pi_lock)
1027#define PORT_LOCK_ASSERT_OWNED(pi) mtx_assert(&(pi)->pi_lock, MA_OWNED)
1028#define PORT_LOCK_ASSERT_NOTOWNED(pi) mtx_assert(&(pi)->pi_lock, MA_NOTOWNED)
1030#define FL_LOCK(fl) mtx_lock(&(fl)->fl_lock)
1031#define FL_TRYLOCK(fl) mtx_trylock(&(fl)->fl_lock)
1032#define FL_UNLOCK(fl) mtx_unlock(&(fl)->fl_lock)
1033#define FL_LOCK_ASSERT_OWNED(fl) mtx_assert(&(fl)->fl_lock, MA_OWNED)
1034#define FL_LOCK_ASSERT_NOTOWNED(fl) mtx_assert(&(fl)->fl_lock, MA_NOTOWNED)
1036#define RXQ_FL_LOCK(rxq) FL_LOCK(&(rxq)->fl)
1037#define RXQ_FL_UNLOCK(rxq) FL_UNLOCK(&(rxq)->fl)
1038#define RXQ_FL_LOCK_ASSERT_OWNED(rxq) FL_LOCK_ASSERT_OWNED(&(rxq)->fl)
1039#define RXQ_FL_LOCK_ASSERT_NOTOWNED(rxq) FL_LOCK_ASSERT_NOTOWNED(&(rxq)->fl)
1041#define EQ_LOCK(eq) mtx_lock(&(eq)->eq_lock)
1042#define EQ_TRYLOCK(eq) mtx_trylock(&(eq)->eq_lock)
1043#define EQ_UNLOCK(eq) mtx_unlock(&(eq)->eq_lock)
1044#define EQ_LOCK_ASSERT_OWNED(eq) mtx_assert(&(eq)->eq_lock, MA_OWNED)
1045#define EQ_LOCK_ASSERT_NOTOWNED(eq) mtx_assert(&(eq)->eq_lock, MA_NOTOWNED)
1047#define TXQ_LOCK(txq) EQ_LOCK(&(txq)->eq)
1048#define TXQ_TRYLOCK(txq) EQ_TRYLOCK(&(txq)->eq)
1049#define TXQ_UNLOCK(txq) EQ_UNLOCK(&(txq)->eq)
1050#define TXQ_LOCK_ASSERT_OWNED(txq) EQ_LOCK_ASSERT_OWNED(&(txq)->eq)
1051#define TXQ_LOCK_ASSERT_NOTOWNED(txq) EQ_LOCK_ASSERT_NOTOWNED(&(txq)->eq)
1053#define for_each_txq(vi, iter, q) \
1054 for (q = &vi->adapter->sge.txq[vi->first_txq], iter = 0; \
1055 iter < vi->ntxq; ++iter, ++q)
1056#define for_each_rxq(vi, iter, q) \
1057 for (q = &vi->adapter->sge.rxq[vi->first_rxq], iter = 0; \
1058 iter < vi->nrxq; ++iter, ++q)
1059#define for_each_ofld_txq(vi, iter, q) \
1060 for (q = &vi->adapter->sge.ofld_txq[vi->first_ofld_txq], iter = 0; \
1061 iter < vi->nofldtxq; ++iter, ++q)
1062#define for_each_ofld_rxq(vi, iter, q) \
1063 for (q = &vi->adapter->sge.ofld_rxq[vi->first_ofld_rxq], iter = 0; \
1064 iter < vi->nofldrxq; ++iter, ++q)
1065#define for_each_nm_txq(vi, iter, q) \
1066 for (q = &vi->adapter->sge.nm_txq[vi->first_nm_txq], iter = 0; \
1067 iter < vi->nnmtxq; ++iter, ++q)
1068#define for_each_nm_rxq(vi, iter, q) \
1069 for (q = &vi->adapter->sge.nm_rxq[vi->first_nm_rxq], iter = 0; \
1070 iter < vi->nnmrxq; ++iter, ++q)
1071#define for_each_vi(_pi, _iter, _vi) \
1072 for ((_vi) = (_pi)->vi, (_iter) = 0; (_iter) < (_pi)->nvi; \
1075#define IDXINCR(idx, incr, wrap) do { \
1076 idx = wrap - idx > incr ? idx + incr : incr - (wrap - idx); \
1078#define IDXDIFF(head, tail, wrap) \
1079 ((head) >= (tail) ? (head) - (tail) : (wrap) - (tail) + (head))
1082#define T4_EXTRA_INTR 2
1085#define T4VF_EXTRA_INTR 1
1100 return (__predict_false(off_limits != 0));
1103static inline uint32_t
1108 return bus_space_read_4(sc->
bt, sc->
bh, reg);
1116 bus_space_write_4(sc->
bt, sc->
bh, reg, val);
1119static inline uint64_t
1125 return bus_space_read_8(sc->
bt, sc->
bh, reg);
1127 return (uint64_t)bus_space_read_4(sc->
bt, sc->
bh, reg) +
1128 ((uint64_t)bus_space_read_4(sc->
bt, sc->
bh, reg + 4) << 32);
1139 bus_space_write_8(sc->
bt, sc->
bh, reg, val);
1141 bus_space_write_4(sc->
bt, sc->
bh, reg, val);
1142 bus_space_write_4(sc->
bt, sc->
bh, reg + 4, val>> 32);
1151 *val = pci_read_config(sc->
dev, reg, 1);
1159 pci_write_config(sc->
dev, reg, val, 1);
1168 *val = pci_read_config(sc->
dev, reg, 2);
1176 pci_write_config(sc->
dev, reg, val, 2);
1184 *val = pci_read_config(sc->
dev, reg, 4);
1192 pci_write_config(sc->
dev, reg, val, 4);
1199 return (sc->
port[idx]);
1206 bcopy(hw_addr, pi->
vi[0].
hw_addr, ETHER_ADDR_LEN);
1230 const char *msg,
const __be64 *
const p,
const bool err)
1236 log(err ? LOG_ERR : LOG_DEBUG,
1237 "%s: mbox %u %s %016llx %016llx %016llx %016llx "
1238 "%016llx %016llx %016llx %016llx\n",
1239 device_get_nameunit(sc->
dev), mbox, msg,
1245 log(err ? LOG_ERR : LOG_DEBUG,
1246 "%s: mbox %u %s %016llx %016llx %016llx %016llx "
1247 "%016llx %016llx %016llx %016llx\n",
1248 device_get_nameunit(sc->
dev), mbox, msg,
1302 struct m_snd_tag **);
1305int t6_ktls_try(
struct ifnet *,
struct socket *,
struct ktls_session *);
1325u_int t4_tls_key_info_size(
const struct ktls_session *);
1326int t4_tls_proto_ver(
const struct ktls_session *);
1327int t4_tls_cipher_mode(
const struct ktls_session *);
1328int t4_tls_auth_mode(
const struct ktls_session *);
1329int t4_tls_hmac_ctrl(
const struct ktls_session *);
1330void t4_tls_key_ctx(
const struct ktls_session *,
int,
struct tls_keyctx *);
1331int t4_alloc_tls_keyid(
struct adapter *);
1332void t4_free_tls_keyid(
struct adapter *,
int);
1333void t4_write_tlskey_wr(
const struct ktls_session *,
int,
int,
int,
int,
1340void cxgbe_nm_attach(
struct vi_info *);
1341void cxgbe_nm_detach(
struct vi_info *);
1358 struct sysctl_oid_list *);
1361 bus_addr_t *,
void **);
1372void t4_nm_intr(
void *);
1373void t4_vi_intr(
void *);
1389int ethofld_transmit(
struct ifnet *,
struct mbuf *);
1414void t4_init_etid_table(
struct adapter *);
1415void t4_free_etid_table(
struct adapter *);
1417int cxgbe_rate_tag_alloc(
struct ifnet *,
union if_snd_tag_alloc_params *,
1418 struct m_snd_tag **);
1420void cxgbe_ratelimit_query(
struct ifnet *,
struct if_ratelimit_query_results *);
1436static inline struct wrqe *
1439 int len = offsetof(
struct wrqe, wr) +
wr_len;
1442 wr = malloc(len, M_CXGBE, M_NOWAIT);
1443 if (__predict_false(wr == NULL))
1453 return (&wr->wr[0]);
1482 const uint32_t *val,
int len)
1485 return (
rw_via_memwin(sc, idx, addr, (
void *)(uintptr_t)val, len, 1));
1493 return (howmany(len16,
EQ_ESIZE / 16));
void vi_sysctls(struct vi_info *)
void t4_add_adapter(struct adapter *)
int tnl_cong(struct port_info *, int)
int begin_synchronized_op(struct adapter *, struct vi_info *, int, char *)
static void t4_os_pci_read_cfg4(struct adapter *sc, int reg, uint32_t *val)
unsigned int t4_qsize_txq
void t4_tracer_modunload(void)
int t4_os_pci_restore_state(struct adapter *)
int(* an_handler_t)(struct sge_iq *, const struct rsp_ctrl *)
static struct wrqe * alloc_wrqe(int wr_len, struct sge_wrq *wrq)
void t4_sge_sysctls(struct adapter *, struct sysctl_ctx_list *, struct sysctl_oid_list *)
void t4_update_tx_sched(struct adapter *)
int t4_filter_rpl(struct sge_iq *, const struct rss_header *, struct mbuf *)
int t4_hashfilter_ao_rpl(struct sge_iq *, const struct rss_header *, struct mbuf *)
void t4_init_gmac_hash(const char *, int, char *)
int t4_setup_intr_handlers(struct adapter *)
int parse_pkt(struct mbuf **, bool)
int t4_trace_pkt(struct sge_iq *, const struct rss_header *, struct mbuf *)
void cxgbe_media_status(struct ifnet *, struct ifmediareq *)
int vi_init(struct vi_info *)
static uint32_t t4_read_reg(struct adapter *sc, uint32_t reg)
int t4_init_tx_sched(struct adapter *)
void t4_sge_modload(void)
int t4_reserve_cl_rl_kbps(struct adapter *, int, u_int, int *)
void t4_register_an_handler(an_handler_t)
int t4_get_tracer(struct adapter *, struct t4_tracer *)
void t4_release_cl_rl(struct adapter *, int, int)
static void CH_DUMP_MBOX(struct adapter *sc, int mbox, const int reg, const char *msg, const __be64 *const p, const bool err)
int set_filter_mask(struct adapter *, uint32_t)
void t4_update_fl_bufsize(struct ifnet *)
void t4_os_portmod_changed(struct port_info *)
void t4_tweak_chip_settings(struct adapter *)
int t4_del_hashfilter_rpl(struct sge_iq *, const struct rss_header *, struct mbuf *)
uint8_t ss[SGE_MAX_WR_LEN]
int del_filter(struct adapter *, struct t4_filter *)
int t5_trace_pkt(struct sge_iq *, const struct rss_header *, struct mbuf *)
static struct port_info * adap2pinfo(struct adapter *sc, int idx)
void free_atid(struct adapter *, int)
int t4_setup_adapter_queues(struct adapter *)
struct mbuf * alloc_wr_mbuf(int, int)
void t4_tracer_port_detach(struct adapter *)
unsigned int t4_qsize_rxq
void * lookup_atid(struct adapter *, int)
void free_fl_buffers(struct adapter *, struct sge_fl *)
int alloc_atid(struct adapter *, void *)
int sysctl_tc_params(SYSCTL_HANDLER_ARGS)
int update_mac_settings(struct ifnet *, int)
@ CS_HW_UPDATE_IN_PROGRESS
static int read_via_memwin(struct adapter *sc, int idx, uint32_t addr, uint32_t *val, int len)
static void t4_write_reg64(struct adapter *sc, uint32_t reg, uint64_t val)
int t4_hashfilter_tcb_rpl(struct sge_iq *, const struct rss_header *, struct mbuf *)
int(* fw_msg_handler_t)(struct adapter *, const __be64 *)
void t4_register_cpl_handler(int, cpl_handler_t)
void doom_vi(struct adapter *, struct vi_info *)
static void t4_write_reg(struct adapter *sc, uint32_t reg, uint32_t val)
static void * wrtod(struct wrqe *wr)
void commit_wrq_wr(struct sge_wrq *, void *, struct wrq_cookie *)
int rw_via_memwin(struct adapter *, int, uint32_t, uint32_t *, int, int)
int t4_create_dma_tag(struct adapter *)
device_method_t cxgbe_methods[]
void t4_init_devnames(struct adapter *)
int t4_set_tracer(struct adapter *, struct t4_tracer *)
static int tx_resume_threshold(struct sge_eq *eq)
void t4_wrq_tx_locked(struct adapter *, struct sge_wrq *, struct wrqe *)
int t4_set_sched_queue(struct adapter *, struct t4_sched_queue *)
void t4_register_shared_cpl_handler(int, cpl_handler_t, int)
static void t4_wrq_tx(struct adapter *sc, struct wrqe *wr)
void t4_sge_modunload(void)
static int t4_use_ldst(struct adapter *sc)
int t4_free_tx_sched(struct adapter *)
static void t4_os_pci_read_cfg2(struct adapter *sc, int reg, uint16_t *val)
void t4_init_hmac_digest(const struct auth_hash *, u_int, const char *, int, char *)
int t4_setup_vi_queues(struct vi_info *)
int t4_teardown_adapter_queues(struct adapter *)
static struct sge_ofld_rxq * iq_to_ofld_rxq(struct sge_iq *iq)
static void t4_os_set_hw_addr(struct port_info *pi, uint8_t hw_addr[])
int set_filter_mode(struct adapter *, uint32_t)
void t4_iterate(void(*)(struct adapter *, void *), void *)
static bool hw_off_limits(struct adapter *sc)
int t4_detach_common(device_t)
int t4_map_bar_2(struct adapter *)
static int forwarding_intr_to_fwq(struct adapter *sc)
void t4_aes_getdeckey(void *, const void *, unsigned int)
int t4_map_bars_0_and_4(struct adapter *)
static void t4_os_pci_write_cfg1(struct adapter *sc, int reg, uint8_t val)
int t4_destroy_dma_tag(struct adapter *)
int t4_os_pci_save_state(struct adapter *)
int get_filter_mode(struct adapter *, uint32_t *)
static void t4_os_pci_read_cfg1(struct adapter *sc, int reg, uint8_t *val)
int get_filter(struct adapter *, struct t4_filter *)
int cxgbe_media_change(struct ifnet *)
void t4_tracer_modload(void)
static int write_via_memwin(struct adapter *sc, int idx, uint32_t addr, const uint32_t *val, int len)
int sysctl_tc(SYSCTL_HANDLER_ARGS)
static struct sge_rxq * iq_to_rxq(struct sge_iq *iq)
int t4_os_find_pci_capability(struct adapter *, int)
void release_tid(struct adapter *, int, struct sge_wrq *)
void end_synchronized_op(struct adapter *, int)
CTASSERT(sizeof(struct iq_desc)==IQ_ESIZE)
static void t4_os_pci_write_cfg4(struct adapter *sc, int reg, uint32_t val)
static void t4_os_pci_write_cfg2(struct adapter *sc, int reg, uint16_t val)
static void free_wrqe(struct wrqe *wr)
void t4_sysctls(struct adapter *)
uint64_t t4_sge_extfree_refs(void)
int t4_set_sched_class(struct adapter *, struct t4_sched_params *)
void t4_os_link_changed(struct port_info *)
int adapter_init(struct adapter *)
void free_hftid_hash(struct tid_info *)
void * start_wrq_wr(struct sge_wrq *, int, struct wrq_cookie *)
int set_filter(struct adapter *, struct t4_filter *)
int(* cpl_handler_t)(struct sge_iq *, const struct rss_header *, struct mbuf *)
void t4_copy_partial_hash(int, union authctx *, void *)
void t4_init_rx_buf_info(struct adapter *)
int t4_verify_chip_settings(struct adapter *)
int free_ring(struct adapter *, bus_dma_tag_t, bus_dmamap_t, bus_addr_t, void *)
static int tx_len16_to_desc(int len16)
int alloc_ring(struct adapter *, size_t, bus_dma_tag_t *, bus_dmamap_t *, bus_addr_t *, void **)
void t4_register_fw_msg_handler(int, fw_msg_handler_t)
int t4_teardown_vi_queues(struct vi_info *)
void t4_os_cim_err(struct adapter *)
static uint64_t t4_read_reg64(struct adapter *sc, uint32_t reg)
struct sge_nm_rxq * nm_rxq
struct sysctl_oid * fwq_oid
const void * reset_thread
const struct devnames * names
uint8_t chan_map[MAX_NCHAN]
struct adapter::irq __aligned(CACHE_LINE_SIZE) *irq
struct t4_offload_policy * policy
volatile uint8_t * udbs_base
const struct chip_params * chip_params
struct sysctl_ctx_list ctx
SLIST_ENTRY(adapter) link
struct resource * regs_res
struct resource * udbs_res
TAILQ_HEAD(, clip_entry) clip_pending
struct adapter_params params
struct rwlock policy_lock
struct resource * msix_res
struct port_info * port[MAX_NPORTS]
struct task fatal_error_task
struct sysctl_oid * ctrlq_oid
CXGBE_LIST_HEAD(, clip_entry) *clip_table
struct callout sfl_callout
struct timeout_task clip_task
struct taskqueue * tq[MAX_NCHAN]
const char * vi_ifnet_name
const char * pf03_drv_name
const char * vf_nexus_name
const char * vf_ifnet_name
struct rwlock mw_lock __aligned(CACHE_LINE_SIZE)
struct sysctl_ctx_list ctx
struct link_config link_cfg
struct tx_sched_params * sched_params
volatile uint16_t hw_cidx
uint64_t cl_fast_recycled
STAILQ_ENTRY(sge_iq) link
__be64 *fl_desc __aligned(CACHE_LINE_SIZE)
bus_dma_tag_t fl_desc_tag
uint32_t fl_cidx __aligned(CACHE_LINE_SIZE)
volatile int nm_state __aligned(CACHE_LINE_SIZE)
bus_dma_tag_t iq_desc_tag
counter_u64_t rx_iscsi_ddp_setup_error
uint64_t rx_iscsi_fl_pdus
uint64_t rx_iscsi_ddp_pdus
u_long rx_toe_tls_records
uint64_t rx_iscsi_fl_octets
uint64_t rx_iscsi_data_digest_errors
uint64_t rx_iscsi_ddp_octets
uint64_t rx_iscsi_header_digest_errors
uint64_t rx_iscsi_padding_errors
counter_u64_t rx_iscsi_ddp_setup_ok
counter_u64_t tx_iscsi_octets
counter_u64_t tx_iscsi_iso_wrs
counter_u64_t tx_toe_tls_octets
counter_u64_t tx_toe_tls_records
counter_u64_t tx_iscsi_pdus
uint64_t kern_tls_options
uint8_t ss[SGE_MAX_WR_LEN] __aligned(16)
struct task tx_reclaim_task
uint64_t kern_tls_partial
uint64_t kern_tls_records
uint64_t kern_tls_fin_short
STAILQ_HEAD(, wrqe) wr_list
uint8_t ss[SGE_MAX_WR_LEN]
TAILQ_HEAD(wrq_incomplete_wrs, wrq_cookie) incomplete_wrs
struct sge_ofld_rxq * ofld_rxq
struct sge_nm_rxq * nm_rxq
struct sge_nm_txq * nm_txq
struct sge_ofld_txq * ofld_txq
enum fw_sched_params_rate ratemode
enum fw_sched_params_rate ratemode
enum fw_sched_params_unit rateunit
enum fw_sched_params_mode mode
struct tx_ch_rl_params ch_rl
struct tx_cl_rl_params cl_rl[]
struct sysctl_ctx_list ctx
struct sysctl_oid * ofld_rxq_oid
struct sysctl_oid * rxq_oid
struct fw_vi_stats_vf stats
struct timeval last_refreshed
struct sysctl_oid * nm_rxq_oid
uint8_t hw_addr[ETHER_ADDR_LEN]
struct sysctl_oid * txq_oid
struct sysctl_oid * ofld_txq_oid
struct sysctl_oid * nm_txq_oid
TAILQ_ENTRY(wrq_cookie) link
int t6_ktls_write_wr(struct sge_txq *txq, void *dst, struct mbuf *m, u_int nsegs, u_int available)
int cxgbe_tls_tag_alloc(struct ifnet *ifp, union if_snd_tag_alloc_params *params, struct m_snd_tag **pt)
void t6_ktls_modunload(void)
int t6_ktls_parse_pkt(struct mbuf *m, int *nsegsp, int *len16p)
void t6_ktls_modload(void)