42#include <sys/kernel.h>
47#include <sys/rwlock.h>
48#include <sys/socket.h>
76 res_wr->
cookie = (
unsigned long) &wr_wait;
89 dma_free_coherent(rhp->
ibdev.dma_device,
91 dma_unmap_addr(cq, mapping));
105 int user = (uctx != &rdev->
uctx);
109 u64 cq_bar2_qoffset = 0;
130 dma_unmap_addr_set(cq, mapping, cq->
dma_addr);
141 memset(res_wr, 0,
wr_len);
147 res_wr->
cookie = (
unsigned long) &wr_wait;
171 CTR2(
KTR_IW_CXGBE,
"%s wait_event wr_wait %p", __func__, &wr_wait);
195 dma_unmap_addr(cq, mapping));
204static void insert_recv_cqe(
struct t4_wq *wq,
struct t4_cq *cq)
208 CTR5(
KTR_IW_CXGBE,
"%s wq %p cq %p sw_cidx %u sw_pidx %u", __func__, wq,
210 memset(&cqe, 0,
sizeof(cqe));
227 CTR5(
KTR_IW_CXGBE,
"%s wq %p cq %p rq.in_use %u skip count %u",
228 __func__, wq, cq, wq->
rq.
in_use, count);
230 insert_recv_cqe(wq, cq);
236static void insert_sq_cqe(
struct t4_wq *wq,
struct t4_cq *cq,
241 CTR5(
KTR_IW_CXGBE,
"%s wq %p cq %p sw_cidx %u sw_pidx %u", __func__, wq,
243 memset(&cqe, 0,
sizeof(cqe));
255static void advance_oldest_read(
struct t4_wq *wq);
274 insert_sq_cqe(wq, cq, swsqe);
277 advance_oldest_read(wq);
289static void flush_completed_wrs(
struct t4_wq *wq,
struct t4_cq *cq)
312 "%s moving cqe into swcq sq idx %u cq idx %u\n",
326static void create_read_req_cqe(
struct t4_wq *wq,
struct t4_cqe *hw_cqe,
338static void advance_oldest_read(
struct t4_wq *wq)
345 while (rptr != wq->
sq.
pidx) {
350 if (++rptr == wq->
sq.
size)
363 struct t4_cqe *hw_cqe, *swcqe, read_cqe;
407 advance_oldest_read(&qhp->
wq);
415 create_read_req_cqe(&qhp->
wq, hw_cqe, &read_cqe);
417 advance_oldest_read(&qhp->
wq);
425 swsqe->
cqe = *hw_cqe;
427 flush_completed_wrs(&qhp->
wq, &chp->
cq);
462 CTR2(
KTR_IW_CXGBE,
"%s count zero %d", __func__, *count);
469 if (++ptr == cq->
size)
472 CTR3(
KTR_IW_CXGBE,
"%s cq %p count %d", __func__, cq, *count);
492 u8 *cqe_flushed,
u64 *cookie,
u32 *credit)
495 struct t4_cqe *hw_cqe, read_cqe;
504 "%s CQE OVF %u qpid 0x%0x genbit %u type %u status 0x%0x", __func__,
508 "%s opcode 0x%0x len 0x%0x wrid_hi_stag 0x%x wrid_low_msn 0x%x",
580 advance_oldest_read(wq);
589 create_read_req_cqe(wq, hw_cqe, &read_cqe);
591 advance_oldest_read(wq);
639 "%s out of order completion going in sw_sq at idx %u",
642 swsqe->
cqe = *hw_cqe;
667 if (idx < wq->sq.cidx)
691 flush_completed_wrs(wq, cq);
695 CTR4(
KTR_IW_CXGBE,
"%s cq %p cqid 0x%x skip sw cqe cidx %u",
699 CTR4(
KTR_IW_CXGBE,
"%s cq %p cqid 0x%x skip hw cqe cidx %u",
715static int c4iw_poll_cq_one(
struct c4iw_cq *chp,
struct ib_wc *wc)
718 struct t4_cqe cqe = {0, 0}, *rd_cqe;
734 spin_lock(&qhp->
lock);
737 ret = poll_cq(wq, &(chp->
cq), &cqe, &cqe_flushed, &cookie, &credit);
746 CTR5(
KTR_IW_CXGBE,
"%s qpid 0x%x type %d opcode %d status 0x%x",
749 CTR5(
KTR_IW_CXGBE,
"%s len %u wrid hi 0x%x lo 0x%x cookie 0x%llx",
751 (
unsigned long long)cookie);
758 wc->opcode = IB_WC_RECV;
762 wc->wc_flags |= IB_WC_WITH_INVALIDATE;
768 wc->opcode = IB_WC_RDMA_WRITE;
771 wc->opcode = IB_WC_RDMA_READ;
776 wc->opcode = IB_WC_SEND;
777 wc->wc_flags |= IB_WC_WITH_INVALIDATE;
781 wc->opcode = IB_WC_SEND;
784 wc->opcode = IB_WC_LOCAL_INV;
787 wc->opcode = IB_WC_REG_MR;
795 wc->opcode = IB_WC_SEND;
798 printf(
"Unexpected opcode %d "
799 "in the CQE received for QPID = 0x%0x\n",
807 wc->status = IB_WC_WR_FLUSH_ERR;
812 wc->status = IB_WC_SUCCESS;
815 wc->status = IB_WC_LOC_ACCESS_ERR;
818 wc->status = IB_WC_LOC_PROT_ERR;
822 wc->status = IB_WC_LOC_ACCESS_ERR;
825 wc->status = IB_WC_GENERAL_ERR;
828 wc->status = IB_WC_LOC_LEN_ERR;
832 wc->status = IB_WC_MW_BIND_ERR;
848 wc->status = IB_WC_FATAL_ERR;
851 wc->status = IB_WC_WR_FLUSH_ERR;
854 printf(
"Unexpected cqe_status 0x%x for QPID = 0x%0x\n",
856 wc->status = IB_WC_FATAL_ERR;
861 spin_unlock(&qhp->
lock);
865int c4iw_poll_cq(
struct ib_cq *ibcq,
int num_entries,
struct ib_wc *wc)
874 spin_lock_irqsave(&chp->
lock, flags);
875 for (npolled = 0; npolled < num_entries; ++npolled) {
877 err = c4iw_poll_cq_one(chp, wc + npolled);
878 }
while (err == -EAGAIN);
882 spin_unlock_irqrestore(&chp->
lock, flags);
883 return !err || err == -ENODATA ? npolled : err;
896 wait_event(chp->
wait, !atomic_read(&chp->
refcnt));
898 ucontext = rdma_udata_to_drv_context(udata,
struct c4iw_ucontext,
904int c4iw_create_cq(
struct ib_cq *ibcq,
const struct ib_cq_init_attr *attr,
905 struct ib_udata *udata)
907 struct ib_device *ibdev = ibcq->device;
908 int entries = attr->cqe;
909 int vector = attr->comp_vector;
915 size_t memsize, hwentries;
918 CTR3(
KTR_IW_CXGBE,
"%s ib_dev %p entries %d", __func__, ibdev, entries);
924 ucontext = rdma_udata_to_drv_context(udata,
struct c4iw_ucontext,
936 entries = roundup(entries, 16);
950 memsize = hwentries *
sizeof *chp->
cq.
queue;
956 memsize = roundup(memsize, PAGE_SIZE);
961 ret = create_cq(&rhp->
rdev, &chp->
cq,
968 chp->
ibcq.cqe = entries - 2;
969 spin_lock_init(&chp->
lock);
971 atomic_set(&chp->
refcnt, 1);
972 init_waitqueue_head(&chp->
wait);
979 mm = kmalloc(
sizeof *mm, GFP_KERNEL);
982 mm2 = kmalloc(
sizeof *mm2, GFP_KERNEL);
986 memset(&uresp, 0,
sizeof(uresp));
988 uresp.cqid = chp->
cq.
cqid;
989 uresp.size = chp->
cq.
size;
992 uresp.key = ucontext->
key;
993 ucontext->
key += PAGE_SIZE;
994 uresp.gts_key = ucontext->
key;
995 ucontext->
key += PAGE_SIZE;
997 ret = ib_copy_to_udata(udata, &uresp,
998 sizeof(uresp) -
sizeof(uresp.reserved));
1002 mm->
key = uresp.key;
1007 mm2->
key = uresp.gts_key;
1009 mm2->
len = PAGE_SIZE;
1013 "%s cqid 0x%0x chp %p size %u memsize %zu, dma_addr 0x%0llx",
1030int c4iw_resize_cq(
struct ib_cq *cq,
int cqe,
struct ib_udata *udata)
1035int c4iw_arm_cq(
struct ib_cq *ibcq,
enum ib_cq_notify_flags flags)
1042 spin_lock_irqsave(&chp->
lock, flag);
1044 (flags & IB_CQ_SOLICITED_MASK) == IB_CQ_SOLICITED);
1045 if (flags & IB_CQ_REPORT_MISSED_EVENTS)
1047 spin_unlock_irqrestore(&chp->
lock, flag);
static struct wrqe * alloc_wrqe(int wr_len, struct sge_wrq *wrq)
static void * wrtod(struct wrqe *wr)
static void t4_wrq_tx(struct adapter *sc, struct wrqe *wr)
int t4_bar2_sge_qregs(struct adapter *adapter, unsigned int qid, enum t4_bar2_qtype qtype, int user, u64 *pbar2_qoffset, unsigned int *pbar2_qid)
static int c4iw_wait_for_reply(struct c4iw_rdev *rdev, struct c4iw_wr_wait *wr_waitp, u32 hwtid, u32 qpid, struct socket *so, const char *func)
int c4iw_arm_cq(struct ib_cq *ibcq, enum ib_cq_notify_flags flags)
void c4iw_count_rcqes(struct t4_cq *cq, struct t4_wq *wq, int *count)
int c4iw_flush_rq(struct t4_wq *wq, struct t4_cq *cq, int count)
static void insert_mmap(struct c4iw_ucontext *ucontext, struct c4iw_mm_entry *mm)
static struct c4iw_qp * get_qhp(struct c4iw_dev *rhp, u32 qpid)
void c4iw_flush_hw_cq(struct c4iw_cq *cq)
int c4iw_poll_cq(struct ib_cq *ibcq, int num_entries, struct ib_wc *wc)
static struct c4iw_cq * to_c4iw_cq(struct ib_cq *ibcq)
int c4iw_resize_cq(struct ib_cq *cq, int cqe, struct ib_udata *udata)
static struct c4iw_dev * to_c4iw_dev(struct ib_device *ibdev)
static int insert_handle(struct c4iw_dev *rhp, struct idr *idr, void *handle, u32 id)
#define C4IW_DRAIN_OPCODE
void c4iw_invalidate_mr(struct c4iw_dev *rhp, u32 rkey)
static void remove_handle(struct c4iw_dev *rhp, struct idr *idr, u32 id)
void c4iw_put_cqid(struct c4iw_rdev *rdev, u32 qid, struct c4iw_dev_ucontext *uctx)
int c4iw_create_cq(struct ib_cq *ibcq, const struct ib_cq_init_attr *attr, struct ib_udata *udata)
int c4iw_flush_sq(struct c4iw_qp *qhp)
static void c4iw_init_wr_wait(struct c4iw_wr_wait *wr_waitp)
u32 c4iw_get_cqid(struct c4iw_rdev *rdev, struct c4iw_dev_ucontext *uctx)
static struct c4iw_dev * rdev_to_c4iw_dev(struct c4iw_rdev *rdev)
void c4iw_destroy_cq(struct ib_cq *ib_cq, struct ib_udata *udata)
#define DIV_ROUND_UP(x, y)
spinlock_t comp_handler_lock
struct c4iw_hw_queue hw_queue
struct c4iw_dev_ucontext uctx
struct c4iw_dev_ucontext uctx
struct ib_ucontext ibucontext
struct sge_ofld_rxq * ofld_rxq
struct t4_cqe::@74::@76 scqe
struct t4_swsqe * oldest_read
#define CQE_DRAIN_COOKIE(x)
static void t4_swcq_produce(struct t4_cq *cq)
static int t4_rq_empty(struct t4_wq *wq)
static void t4_hwcq_consume(struct t4_cq *cq)
static int t4_wq_in_error(struct t4_wq *wq)
#define T4_ERR_IRD_OVERFLOW
static int t4_next_cqe(struct t4_cq *cq, struct t4_cqe **cqe)
#define T4_ERR_OUT_OF_RQE
static void t4_set_wq_in_error(struct t4_wq *wq)
#define T4_ERR_PDU_LEN_ERR
#define T4_ERR_DDP_VERSION
static int t4_cq_notempty(struct t4_cq *cq)
#define T4_ERR_INTERNAL_ERR
#define T4_ERR_DDP_QUEUE_NUM
static void t4_rq_consume(struct t4_wq *wq)
static void t4_sq_consume(struct t4_wq *wq)
#define CQE_SEND_OPCODE(x)
static void t4_swcq_consume(struct t4_cq *cq)
#define CQE_WRID_FR_STAG(x)
#define T4_ERR_INVALIDATE_SHARED_MR
static int t4_arm_cq(struct t4_cq *cq, int se)
#define T4_ERR_INVALIDATE_MR_WITH_MW_BOUND
static int t4_next_hw_cqe(struct t4_cq *cq, struct t4_cqe **cqe)
#define CQE_WRID_SQ_IDX(x)
#define T4_ERR_RDMA_VERSION
#define F_FW_RI_RES_WR_IQO
#define V_FW_RI_RES_WR_IQANUS(x)
#define V_FW_RI_RES_WR_IQANUD(x)
#define V_FW_RI_RES_WR_NRES(x)
#define V_FW_RI_RES_WR_IQANDSTINDEX(x)
#define F_FW_RI_RES_WR_IQANDST
#define F_FW_RI_RES_WR_IQDROPRSS
#define V_FW_RI_RES_WR_IQINTCNTTHRESH(x)
#define V_FW_RI_RES_WR_IQESIZE(x)
#define V_FW_RI_RES_WR_IQPCIECH(x)