43#define SGE_PF_KDOORBELL 0x0
44#define QID_MASK 0xffff8000U
46#define QID(x) ((x) << QID_SHIFT)
47#define DBPRIO 0x00004000U
48#define PIDX_MASK 0x00003fffU
50#define PIDX(x) ((x) << PIDX_SHIFT)
53#define INGRESSQID_MASK 0xffff0000U
54#define INGRESSQID_SHIFT 16
55#define INGRESSQID(x) ((x) << INGRESSQID_SHIFT)
56#define TIMERREG_MASK 0x0000e000U
57#define TIMERREG_SHIFT 13
58#define TIMERREG(x) ((x) << TIMERREG_SHIFT)
59#define SEINTARM_MASK 0x00001000U
60#define SEINTARM_SHIFT 12
61#define SEINTARM(x) ((x) << SEINTARM_SHIFT)
62#define CIDXINC_MASK 0x00000fffU
63#define CIDXINC_SHIFT 0
64#define CIDXINC(x) ((x) << CIDXINC_SHIFT)
66#define T4_MAX_NUM_PD 65536
67#define T4_MAX_MR_SIZE (~0ULL)
68#define T4_PAGESIZE_MASK 0xffffffff000
69#define T4_STAG_UNSET 0xffffffff
71#define A_PCIE_MA_SYNC 0x30b4
87#define T4_EQ_ENTRY_SIZE 64
89#define T4_SQ_NUM_SLOTS 5
90#define T4_SQ_NUM_BYTES (T4_EQ_ENTRY_SIZE * T4_SQ_NUM_SLOTS)
91#define T4_MAX_SEND_SGE ((T4_SQ_NUM_BYTES - sizeof(struct fw_ri_send_wr) - \
92 sizeof(struct fw_ri_isgl)) / sizeof(struct fw_ri_sge))
93#define T4_MAX_SEND_INLINE ((T4_SQ_NUM_BYTES - sizeof(struct fw_ri_send_wr) - \
94 sizeof(struct fw_ri_immd)))
95#define T4_MAX_WRITE_INLINE ((T4_SQ_NUM_BYTES - \
96 sizeof(struct fw_ri_rdma_write_wr) - \
97 sizeof(struct fw_ri_immd)))
98#define T4_MAX_WRITE_SGE ((T4_SQ_NUM_BYTES - \
99 sizeof(struct fw_ri_rdma_write_wr) - \
100 sizeof(struct fw_ri_isgl)) / sizeof(struct fw_ri_sge))
101#define T4_MAX_FR_IMMD ((T4_SQ_NUM_BYTES - sizeof(struct fw_ri_fr_nsmr_wr) - \
102 sizeof(struct fw_ri_immd)) & ~31UL)
103#define T4_MAX_FR_IMMD_DEPTH (T4_MAX_FR_IMMD / sizeof(u64))
104#define T4_MAX_FR_DSGL 1024
105#define T4_MAX_FR_DSGL_DEPTH (T4_MAX_FR_DSGL / sizeof(u64))
106#define T4_MAX_FR_FW_DSGL 4096
107#define T4_MAX_FR_FW_DSGL_DEPTH (T4_MAX_FR_FW_DSGL / sizeof(u64))
109#define T4_RQ_NUM_SLOTS 2
110#define T4_RQ_NUM_BYTES (T4_EQ_ENTRY_SIZE * T4_RQ_NUM_SLOTS)
111#define T4_MAX_RECV_SGE 4
146#define T4_ERR_SUCCESS 0x0
147#define T4_ERR_STAG 0x1
150#define T4_ERR_PDID 0x2
151#define T4_ERR_QPID 0x3
152#define T4_ERR_ACCESS 0x4
153#define T4_ERR_WRAP 0x5
154#define T4_ERR_BOUND 0x6
155#define T4_ERR_INVALIDATE_SHARED_MR 0x7
157#define T4_ERR_INVALIDATE_MR_WITH_MW_BOUND 0x8
159#define T4_ERR_ECC 0x9
160#define T4_ERR_ECC_PSTAG 0xA
163#define T4_ERR_PBL_ADDR_BOUND 0xB
165#define T4_ERR_SWFLUSH 0xC
166#define T4_ERR_CRC 0x10
167#define T4_ERR_MARKER 0x11
168#define T4_ERR_PDU_LEN_ERR 0x12
169#define T4_ERR_OUT_OF_RQE 0x13
170#define T4_ERR_DDP_VERSION 0x14
171#define T4_ERR_RDMA_VERSION 0x15
172#define T4_ERR_OPCODE 0x16
173#define T4_ERR_DDP_QUEUE_NUM 0x17
174#define T4_ERR_MSN 0x18
175#define T4_ERR_TBIT 0x19
176#define T4_ERR_MO 0x1A
178#define T4_ERR_MSN_GAP 0x1B
179#define T4_ERR_MSN_RANGE 0x1C
180#define T4_ERR_IRD_OVERFLOW 0x1D
181#define T4_ERR_RQE_ADDR_BOUND 0x1E
183#define T4_ERR_INTERNAL_ERR 0x1F
214#define M_CQE_QPID 0xFFFFF
215#define G_CQE_QPID(x) ((((x) >> S_CQE_QPID)) & M_CQE_QPID)
216#define V_CQE_QPID(x) ((x)<<S_CQE_QPID)
218#define S_CQE_SWCQE 11
219#define M_CQE_SWCQE 0x1
220#define G_CQE_SWCQE(x) ((((x) >> S_CQE_SWCQE)) & M_CQE_SWCQE)
221#define V_CQE_SWCQE(x) ((x)<<S_CQE_SWCQE)
223#define S_CQE_STATUS 5
224#define M_CQE_STATUS 0x1F
225#define G_CQE_STATUS(x) ((((x) >> S_CQE_STATUS)) & M_CQE_STATUS)
226#define V_CQE_STATUS(x) ((x)<<S_CQE_STATUS)
229#define M_CQE_TYPE 0x1
230#define G_CQE_TYPE(x) ((((x) >> S_CQE_TYPE)) & M_CQE_TYPE)
231#define V_CQE_TYPE(x) ((x)<<S_CQE_TYPE)
233#define S_CQE_OPCODE 0
234#define M_CQE_OPCODE 0xF
235#define G_CQE_OPCODE(x) ((((x) >> S_CQE_OPCODE)) & M_CQE_OPCODE)
236#define V_CQE_OPCODE(x) ((x)<<S_CQE_OPCODE)
238#define SW_CQE(x) (G_CQE_SWCQE(be32_to_cpu((x)->header)))
239#define CQE_QPID(x) (G_CQE_QPID(be32_to_cpu((x)->header)))
240#define CQE_TYPE(x) (G_CQE_TYPE(be32_to_cpu((x)->header)))
241#define SQ_TYPE(x) (CQE_TYPE((x)))
242#define RQ_TYPE(x) (!CQE_TYPE((x)))
243#define CQE_STATUS(x) (G_CQE_STATUS(be32_to_cpu((x)->header)))
244#define CQE_OPCODE(x) (G_CQE_OPCODE(be32_to_cpu((x)->header)))
246#define CQE_SEND_OPCODE(x)(\
247 (G_CQE_OPCODE(be32_to_cpu((x)->header)) == FW_RI_SEND) || \
248 (G_CQE_OPCODE(be32_to_cpu((x)->header)) == FW_RI_SEND_WITH_SE) || \
249 (G_CQE_OPCODE(be32_to_cpu((x)->header)) == FW_RI_SEND_WITH_INV) || \
250 (G_CQE_OPCODE(be32_to_cpu((x)->header)) == FW_RI_SEND_WITH_SE_INV))
252#define CQE_LEN(x) (be32_to_cpu((x)->len))
255#define CQE_WRID_STAG(x) (be32_to_cpu((x)->u.rcqe.stag))
256#define CQE_WRID_MSN(x) (be32_to_cpu((x)->u.rcqe.msn))
259#define CQE_WRID_SQ_IDX(x) ((x)->u.scqe.cidx)
260#define CQE_WRID_FR_STAG(x) (be32_to_cpu((x)->u.scqe.stag))
263#define CQE_WRID_HI(x) ((x)->u.gen.wrid_hi)
264#define CQE_WRID_LOW(x) ((x)->u.gen.wrid_low)
265#define CQE_DRAIN_COOKIE(x) (x)->u.drain_cookie;
268#define S_CQE_GENBIT 63
269#define M_CQE_GENBIT 0x1
270#define G_CQE_GENBIT(x) (((x) >> S_CQE_GENBIT) & M_CQE_GENBIT)
271#define V_CQE_GENBIT(x) ((x)<<S_CQE_GENBIT)
273#define S_CQE_OVFBIT 62
274#define M_CQE_OVFBIT 0x1
275#define G_CQE_OVFBIT(x) ((((x) >> S_CQE_OVFBIT)) & M_CQE_OVFBIT)
277#define S_CQE_IQTYPE 60
278#define M_CQE_IQTYPE 0x3
279#define G_CQE_IQTYPE(x) ((((x) >> S_CQE_IQTYPE)) & M_CQE_IQTYPE)
281#define M_CQE_TS 0x0fffffffffffffffULL
282#define G_CQE_TS(x) ((x) & M_CQE_TS)
284#define CQE_OVFBIT(x) ((unsigned)G_CQE_OVFBIT(be64_to_cpu((x)->bits_type_ts)))
285#define CQE_GENBIT(x) ((unsigned)G_CQE_GENBIT(be64_to_cpu((x)->bits_type_ts)))
286#define CQE_TS(x) (G_CQE_TS(be64_to_cpu((x)->bits_type_ts)))
303#if defined(__i386__) || defined(__x86_64__) || defined(CONFIG_PPC64)
304 return pgprot_writecombine(prot);
306 return pgprot_noncached(prot);
489 if (wc && inc == 1 && wq->
sq.
bar2_qid == 0 && wqe) {
514 if (wc && inc == 1 && wq->
rq.
bar2_qid == 0 && wqe) {
653 prev_cidx = cq->
size - 1;
655 prev_cidx = cq->
cidx - 1;
660 printk(KERN_ERR
MOD "cq overflow cqid %u\n", cq->
cqid);
#define DIV_ROUND_UP(x, y)
DEFINE_DMA_UNMAP_ADDR(mapping)
struct t4_cqe::@74::@76 scqe
struct t4_cqe::@74::@75 rcqe
struct t4_cqe::@74::@77 gen
DEFINE_DMA_UNMAP_ADDR(mapping)
struct t4_swsqe * oldest_read
DEFINE_DMA_UNMAP_ADDR(mapping)
static void t4_ring_rq_db(struct t4_wq *wq, u16 inc, union t4_recv_wr *wqe, u8 wc)
static void write_gts(struct t4_cq *cq, u32 val)
static void pio_copy(u64 __iomem *dst, u64 *src)
static void t4_swcq_produce(struct t4_cq *cq)
static int t4_rq_empty(struct t4_wq *wq)
static void t4_hwcq_consume(struct t4_cq *cq)
static int t4_wq_in_error(struct t4_wq *wq)
static u16 t4_sq_host_wq_pidx(struct t4_wq *wq)
static pgprot_t t4_pgprot_wc(pgprot_t prot)
static int t4_clear_cq_armed(struct t4_cq *cq)
static void init_wr_hdr(union t4_wr *wqe, u16 wrid, enum fw_wr_opcodes opcode, u8 flags, u8 len16)
static int t4_next_cqe(struct t4_cq *cq, struct t4_cqe **cqe)
static void t4_ring_sq_db(struct t4_wq *wq, u16 inc, union t4_wr *wqe, u8 wc)
static void t4_set_wq_in_error(struct t4_wq *wq)
static int t4_rq_full(struct t4_wq *wq)
static u32 t4_sq_avail(struct t4_wq *wq)
static void t4_sq_produce(struct t4_wq *wq, u8 len16)
static int t4_cq_notempty(struct t4_cq *cq)
static void t4_rq_consume(struct t4_wq *wq)
static void t4_sq_consume(struct t4_wq *wq)
static int t4_rqes_posted(struct t4_wq *wq)
static void t4_set_cq_in_error(struct t4_cq *cq)
static void t4_swcq_consume(struct t4_cq *cq)
static u32 t4_rq_avail(struct t4_wq *wq)
static u16 t4_rq_host_wq_pidx(struct t4_wq *wq)
static int t4_sq_empty(struct t4_wq *wq)
static int t4_arm_cq(struct t4_cq *cq, int se)
static int t4_next_hw_cqe(struct t4_cq *cq, struct t4_cqe **cqe)
static u16 t4_sq_wq_size(struct t4_wq *wq)
static int t4_sq_full(struct t4_wq *wq)
static struct t4_cqe * t4_next_sw_cqe(struct t4_cq *cq)
static int t4_valid_cqe(struct t4_cq *cq, struct t4_cqe *cqe)
static int t4_cq_in_error(struct t4_cq *cq)
static u16 t4_rq_wq_size(struct t4_wq *wq)
static int t4_sq_onchip(struct t4_sq *sq)
static void t4_rq_produce(struct t4_wq *wq, u8 len16)
#define SGE_UDB_WCDOORBELL
#define SGE_UDB_KDOORBELL
struct fw_ri_recv_wr recv
struct t4_status_page status
__be64 flits[T4_EQ_ENTRY_SIZE/sizeof(__be64) *T4_RQ_NUM_SLOTS]
struct fw_ri_bind_mw_wr bind
struct fw_ri_fr_nsmr_wr fr
struct t4_status_page status
struct fw_ri_inv_lstag_wr inv
struct fw_ri_fr_nsmr_tpte_wr fr_tpte
__be64 flits[T4_EQ_ENTRY_SIZE/sizeof(__be64) *T4_SQ_NUM_SLOTS]
struct fw_ri_rdma_write_wr write
struct fw_ri_rdma_read_wr read
struct fw_ri_send_wr send