40#include <linux/types.h>
41#include <linux/kref.h>
42#include <rdma/ib_umem.h>
43#include <asm/atomic.h>
48#define T4_ULPTX_MIN_IO 32
49#define C4IW_MAX_INLINE_SIZE 96
50#define T4_ULPTX_MAX_DMA 1024
53mr_exceeds_hw_limits(
struct c4iw_dev *dev,
u64 length)
56 return (
is_t5(dev->
rdev.
adap) && length >= 8*1024*1024*1024ULL);
75 wr_len = roundup(
sizeof *ulpmc +
sizeof *sgl, 16);
86 ulpmc->wr.wr_lo = wait ? (
u64)(
unsigned long)&wr_wait : 0;
110_c4iw_write_mem_inline(
struct c4iw_rdev *rdev,
u32 addr,
u32 len,
void *data)
115 u8 wr_len, *to_dp, *from_dp;
116 int copy_len, num_wqe, i, ret = 0;
126 CTR3(
KTR_IW_CXGBE,
"%s addr 0x%x len %u", __func__, addr, len);
129 for (i = 0; i < num_wqe; i++) {
131 copy_len = min(len, C4IW_MAX_INLINE_SIZE);
132 wr_len = roundup(
sizeof *ulpmc +
sizeof *ulpsc +
133 roundup(copy_len, T4_ULPTX_MIN_IO), 16);
143 if (i == (num_wqe-1)) {
164 to_dp = (
u8 *)(ulpsc + 1);
165 from_dp = (
u8 *)data + i * C4IW_MAX_INLINE_SIZE;
167 memcpy(to_dp, from_dp, copy_len);
169 memset(to_dp, 0, copy_len);
170 if (copy_len % T4_ULPTX_MIN_IO)
171 memset(to_dp + copy_len, 0, T4_ULPTX_MIN_IO -
172 (copy_len % T4_ULPTX_MIN_IO));
174 len -= C4IW_MAX_INLINE_SIZE;
191 daddr = dma_map_single(rhp->
ibdev.dma_device, data, len, DMA_TO_DEVICE);
192 if (dma_mapping_error(rhp->
ibdev.dma_device, daddr))
197 if (remain < T4_ULPTX_MAX_DMA) {
198 if (remain & ~T4_ULPTX_MIN_IO)
199 dmalen = remain & ~(T4_ULPTX_MIN_IO-1);
203 dmalen = T4_ULPTX_MAX_DMA;
205 ret = _c4iw_write_mem_dma_aligned(
rdev, addr, dmalen,
206 (
void *)daddr, !remain);
210 data = (
u8 *)data + dmalen;
211 daddr = daddr + dmalen;
214 ret = _c4iw_write_mem_inline(
rdev, addr, remain, data);
216 dma_unmap_single(rhp->
ibdev.dma_device, save, len, DMA_TO_DEVICE);
230 if (_c4iw_write_mem_dma(
rdev, addr, len, data)) {
231 log(LOG_ERR,
"%s: dma map "
232 "failure (non fatal)\n", __func__);
233 return _c4iw_write_mem_inline(
rdev, addr, len,
238 return _c4iw_write_mem_inline(
rdev, addr, len, data);
240 return _c4iw_write_mem_inline(
rdev, addr, len, data);
253 int bind_enabled,
u32 zbva,
u64 to,
264 stag_state = stag_state > 0;
265 stag_idx = (*stag) >> 8;
280 *stag = (stag_idx << 8) | (atomic_inc_return(&key) & 0xff);
283 "%s stag_state 0x%0x type 0x%0x pdid 0x%0x, stag_idx 0x%x",
284 __func__, stag_state, type, pdid, stag_idx);
288 memset(&tpt, 0,
sizeof(tpt));
309 err = write_adapter_mem(rdev, stag_idx +
313 if (reset_tpt_entry) {
323 u32 pbl_addr,
u32 pbl_size)
327 CTR4(
KTR_IW_CXGBE,
"%s *pdb_addr 0x%x, pbl_base 0x%x, pbl_size %d",
330 err = write_adapter_mem(rdev, pbl_addr >> 5, pbl_size << 3, pbl);
337 return write_tpt_entry(rdev, 1, &stag, 0, 0, 0, 0, 0, 0, 0UL, 0, 0,
341static int allocate_window(
struct c4iw_rdev *rdev,
u32 * stag,
u32 pdid)
344 return write_tpt_entry(rdev, 0, stag, 0, pdid,
FW_RI_STAG_MW, 0, 0, 0,
348static int deallocate_window(
struct c4iw_rdev *rdev,
u32 stag)
350 return write_tpt_entry(rdev, 1, &stag, 0, 0, 0, 0, 0, 0, 0UL, 0, 0, 0,
355 u32 pbl_size,
u32 pbl_addr)
358 return write_tpt_entry(rdev, 0, stag, 0, pdid,
FW_RI_STAG_NSMR, 0, 0, 0,
359 0UL, 0, 0, pbl_size, pbl_addr);
362static int finish_mem_reg(
struct c4iw_mr *mhp,
u32 stag)
369 mhp->
ibmr.rkey = mhp->
ibmr.lkey = stag;
370 CTR3(
KTR_IW_CXGBE,
"%s mmid 0x%x mhp %p", __func__, mmid, mhp);
375 struct c4iw_mr *mhp,
int shift)
380 ret = write_tpt_entry(&rhp->
rdev, 0, &stag, 1, mhp->
attr.
pdid,
388 ret = finish_mem_reg(mhp, stag);
395static int alloc_pbl(
struct c4iw_mr *mhp,
int npages)
420 mhp = kzalloc(
sizeof(*mhp), GFP_KERNEL);
422 return ERR_PTR(-ENOMEM);
434 ret = write_tpt_entry(&
rhp->
rdev, 0, &stag, 1, php->
pdid,
440 ret = finish_mem_reg(mhp, stag);
453 u64 virt,
int acc,
struct ib_udata *udata)
459 struct scatterlist *sg;
467 return ERR_PTR(-EINVAL);
469 if ((length + start) < start)
470 return ERR_PTR(-EINVAL);
475 if (mr_exceeds_hw_limits(
rhp, length))
476 return ERR_PTR(-EINVAL);
478 mhp = kzalloc(
sizeof(*mhp), GFP_KERNEL);
480 return ERR_PTR(-ENOMEM);
484 mhp->
umem = ib_umem_get(pd->uobject->context, start, length, acc, 0);
485 if (IS_ERR(mhp->
umem)) {
486 err = PTR_ERR(mhp->
umem);
491 shift = ffs(mhp->
umem->page_size) - 1;
494 err = alloc_pbl(mhp, n);
498 pages = (
__be64 *) __get_free_page(GFP_KERNEL);
505 for_each_sg(mhp->
umem->sg_head.sgl, sg, mhp->
umem->nmap, entry) {
506 len = sg_dma_len(sg) >> shift;
507 for (k = 0; k < len; ++k) {
509 mhp->
umem->page_size * k);
510 if (i == PAGE_SIZE /
sizeof *pages) {
511 err = write_pbl(&mhp->
rhp->
rdev,
524 err = write_pbl(&mhp->
rhp->
rdev, pages,
528 free_page((
unsigned long) pages);
539 err = register_mem(
rhp, php, mhp, shift);
550 ib_umem_release(mhp->
umem);
555struct ib_mw *
c4iw_alloc_mw(
struct ib_pd *pd,
enum ib_mw_type type,
556 struct ib_udata *udata)
565 if (type != IB_MW_TYPE_1)
566 return ERR_PTR(-EINVAL);
570 mhp = kzalloc(
sizeof(*mhp), GFP_KERNEL);
572 return ERR_PTR(-ENOMEM);
573 ret = allocate_window(&
rhp->
rdev, &stag, php->
pdid);
583 mhp->
ibmw.rkey = stag;
587 return ERR_PTR(-ENOMEM);
589 CTR4(
KTR_IW_CXGBE,
"%s mmid 0x%x mhp %p stag 0x%x", __func__, mmid, mhp,
602 mmid = (mw->rkey) >> 8;
606 CTR4(
KTR_IW_CXGBE,
"%s ib_mw %p mmid 0x%x ptr %p", __func__, mw, mmid,
612 enum ib_mr_type mr_type,
613 u32 max_num_sg,
struct ib_udata *udata)
621 int length = roundup(max_num_sg *
sizeof(
u64), 32);
626 if (mr_type != IB_MR_TYPE_MEM_REG ||
628 return ERR_PTR(-EINVAL);
630 mhp = kzalloc(
sizeof(*mhp), GFP_KERNEL);
636 mhp->
mpl = dma_alloc_coherent(
rhp->
ibdev.dma_device,
637 length, &mhp->
mpl_addr, GFP_KERNEL);
645 ret = alloc_pbl(mhp, max_num_sg);
658 mhp->
ibmr.rkey = mhp->
ibmr.lkey = stag;
664 PDBG(
"%s mmid 0x%x mhp %p stag 0x%x\n", __func__, mmid, mhp, stag);
673 dma_free_coherent(
rhp->
ibdev.dma_device,
680static int c4iw_set_page(
struct ib_mr *
ibmr,
u64 addr)
693 int sg_nents,
unsigned int *sg_offset)
699 return ib_sg_to_pages(
ibmr, sg, sg_nents, sg_offset, c4iw_set_page);
703int c4iw_dereg_mr(
struct ib_mr *ib_mr,
struct ib_udata *udata)
721 kfree((
void *) (
unsigned long) mhp->
kva);
723 ib_umem_release(mhp->
umem);
724 CTR3(
KTR_IW_CXGBE,
"%s mmid 0x%x ptr %p", __func__, mmid, mhp);
734 spin_lock_irqsave(&
rhp->
lock, flags);
738 spin_unlock_irqrestore(&
rhp->
lock, flags);
static struct wrqe * alloc_wrqe(int wr_len, struct sge_wrq *wrq)
static void * wrtod(struct wrqe *wr)
static void t4_wrq_tx(struct adapter *sc, struct wrqe *wr)
static int is_t5(struct adapter *adap)
struct ib_mr * c4iw_get_dma_mr(struct ib_pd *pd, int acc)
int c4iw_dealloc_mw(struct ib_mw *mw)
static int c4iw_wait_for_reply(struct c4iw_rdev *rdev, struct c4iw_wr_wait *wr_waitp, u32 hwtid, u32 qpid, struct socket *so, const char *func)
struct ib_mr * c4iw_alloc_mr(struct ib_pd *pd, enum ib_mr_type mr_type, u32 max_num_sg, struct ib_udata *udata)
static u32 c4iw_ib_to_tpt_access(int a)
static struct c4iw_mr * get_mhp(struct c4iw_dev *rhp, u32 mmid)
struct ib_mr * c4iw_reg_user_mr(struct ib_pd *pd, u64 start, u64 length, u64 virt, int acc, struct ib_udata *udata)
int c4iw_dereg_mr(struct ib_mr *ib_mr, struct ib_udata *udata)
u32 c4iw_pblpool_alloc(struct c4iw_rdev *rdev, int size)
#define PBL_OFF(rdev_p, a)
static struct c4iw_mw * to_c4iw_mw(struct ib_mw *ibmw)
u32 c4iw_get_resource(struct c4iw_id_table *id_table)
static struct c4iw_mr * to_c4iw_mr(struct ib_mr *ibmr)
static int c4iw_fatal_error(struct c4iw_rdev *rdev)
static int insert_handle(struct c4iw_dev *rhp, struct idr *idr, void *handle, u32 id)
void c4iw_invalidate_mr(struct c4iw_dev *rhp, u32 rkey)
static void remove_handle(struct c4iw_dev *rhp, struct idr *idr, u32 id)
void c4iw_pblpool_free(struct c4iw_rdev *rdev, u32 addr, int size)
#define C4IW_MAX_PAGE_SIZE
static int t4_max_fr_depth(struct c4iw_rdev *rdev, bool use_dsgl)
struct ib_mw * c4iw_alloc_mw(struct ib_pd *pd, enum ib_mw_type type, struct ib_udata *udata)
static struct c4iw_pd * to_c4iw_pd(struct ib_pd *ibpd)
static void c4iw_init_wr_wait(struct c4iw_wr_wait *wr_waitp)
#define PDBG(fmt, args...)
int c4iw_map_mr_sg(struct ib_mr *ibmr, struct scatterlist *sg, int sg_nents, unsigned int *sg_offset)
static struct c4iw_dev * rdev_to_c4iw_dev(struct c4iw_rdev *rdev)
void c4iw_put_resource(struct c4iw_id_table *id_table, u32 entry)
#define INIT_ULPTX_WR(w, wrlen, atomic, tid)
#define DIV_ROUND_UP(x, y)
struct adapter_params params
struct tpt_attributes attr
struct tpt_attributes attr
struct c4iw_resource resource
struct c4iw_id_table tpt_table
struct sge_ofld_rxq * ofld_rxq
enum fw_ri_mem_perms perms
#define V_ULP_MEMIO_ADDR(x)
#define V_T5_ULP_MEMIO_FID(x)
#define V_T5_ULP_MEMIO_ORDER(x)
#define V_ULP_MEMIO_DATA_LEN(x)
#define F_T5_ULP_MEMIO_IMM
#define V_FW_RI_TPTE_PBLADDR(x)
#define V_FW_RI_TPTE_STAGKEY(x)
#define V_FW_RI_TPTE_STAGTYPE(x)
#define V_FW_RI_TPTE_PDID(x)
#define V_FW_RI_TPTE_PS(x)
#define V_FW_RI_TPTE_PERM(x)
#define F_FW_RI_TPTE_MWBINDEN
#define M_FW_RI_TPTE_STAGKEY
#define V_FW_RI_TPTE_ADDRTYPE(x)
#define V_FW_RI_TPTE_STAGSTATE(x)
#define F_FW_RI_TPTE_VALID