diff options
Diffstat (limited to 'drivers/infiniband/hw/bnxt_re/qplib_fp.c')
| -rw-r--r-- | drivers/infiniband/hw/bnxt_re/qplib_fp.c | 193 |
1 files changed, 106 insertions, 87 deletions
diff --git a/drivers/infiniband/hw/bnxt_re/qplib_fp.c b/drivers/infiniband/hw/bnxt_re/qplib_fp.c index b98b054148cd..71c34d5b0ac0 100644 --- a/drivers/infiniband/hw/bnxt_re/qplib_fp.c +++ b/drivers/infiniband/hw/bnxt_re/qplib_fp.c @@ -44,6 +44,7 @@ #include <linux/slab.h> #include <linux/pci.h> #include <linux/prefetch.h> +#include <linux/if_ether.h> #include "roce_hsi.h" @@ -244,6 +245,7 @@ static void bnxt_qplib_service_nq(unsigned long data) u16 type; int budget = nq->budget; uintptr_t q_handle; + bool gen_p5 = bnxt_qplib_is_chip_gen_p5(nq->res->cctx); /* Service the NQ until empty */ raw_cons = hwq->cons; @@ -290,7 +292,7 @@ static void bnxt_qplib_service_nq(unsigned long data) q_handle |= (u64)le32_to_cpu(nqsrqe->srq_handle_high) << 32; bnxt_qplib_arm_srq((struct bnxt_qplib_srq *)q_handle, - DBR_DBR_TYPE_SRQ_ARMENA); + DBC_DBC_TYPE_SRQ_ARMENA); if (!nq->srqn_handler(nq, (struct bnxt_qplib_srq *)q_handle, nqsrqe->event)) @@ -312,7 +314,9 @@ static void bnxt_qplib_service_nq(unsigned long data) } if (hwq->cons != raw_cons) { hwq->cons = raw_cons; - NQ_DB_REARM(nq->bar_reg_iomem, hwq->cons, hwq->max_elements); + bnxt_qplib_ring_nq_db_rearm(nq->bar_reg_iomem, hwq->cons, + hwq->max_elements, nq->ring_id, + gen_p5); } } @@ -336,9 +340,11 @@ static irqreturn_t bnxt_qplib_nq_irq(int irq, void *dev_instance) void bnxt_qplib_nq_stop_irq(struct bnxt_qplib_nq *nq, bool kill) { + bool gen_p5 = bnxt_qplib_is_chip_gen_p5(nq->res->cctx); tasklet_disable(&nq->worker); /* Mask h/w interrupt */ - NQ_DB(nq->bar_reg_iomem, nq->hwq.cons, nq->hwq.max_elements); + bnxt_qplib_ring_nq_db(nq->bar_reg_iomem, nq->hwq.cons, + nq->hwq.max_elements, nq->ring_id, gen_p5); /* Sync with last running IRQ handler */ synchronize_irq(nq->vector); if (kill) @@ -373,6 +379,7 @@ void bnxt_qplib_disable_nq(struct bnxt_qplib_nq *nq) int bnxt_qplib_nq_start_irq(struct bnxt_qplib_nq *nq, int nq_indx, int msix_vector, bool need_init) { + bool gen_p5 = bnxt_qplib_is_chip_gen_p5(nq->res->cctx); int rc; if (nq->requested) @@ -399,7 +406,8 @@ int bnxt_qplib_nq_start_irq(struct bnxt_qplib_nq *nq, int nq_indx, nq->vector, nq_indx); } nq->requested = true; - NQ_DB_REARM(nq->bar_reg_iomem, nq->hwq.cons, nq->hwq.max_elements); + bnxt_qplib_ring_nq_db_rearm(nq->bar_reg_iomem, nq->hwq.cons, + nq->hwq.max_elements, nq->ring_id, gen_p5); return rc; } @@ -433,7 +441,8 @@ int bnxt_qplib_enable_nq(struct pci_dev *pdev, struct bnxt_qplib_nq *nq, rc = -ENOMEM; goto fail; } - nq->bar_reg_iomem = ioremap_nocache(nq_base + nq->bar_reg_off, 4); + /* Unconditionally map 8 bytes to support 57500 series */ + nq->bar_reg_iomem = ioremap_nocache(nq_base + nq->bar_reg_off, 8); if (!nq->bar_reg_iomem) { rc = -ENOMEM; goto fail; @@ -462,15 +471,17 @@ void bnxt_qplib_free_nq(struct bnxt_qplib_nq *nq) int bnxt_qplib_alloc_nq(struct pci_dev *pdev, struct bnxt_qplib_nq *nq) { + u8 hwq_type; + nq->pdev = pdev; if (!nq->hwq.max_elements || nq->hwq.max_elements > BNXT_QPLIB_NQE_MAX_CNT) nq->hwq.max_elements = BNXT_QPLIB_NQE_MAX_CNT; - + hwq_type = bnxt_qplib_get_hwq_type(nq->res); if (bnxt_qplib_alloc_init_hwq(nq->pdev, &nq->hwq, NULL, 0, &nq->hwq.max_elements, BNXT_QPLIB_MAX_NQE_ENTRY_SIZE, 0, - PAGE_SIZE, HWQ_TYPE_L2_CMPL)) + PAGE_SIZE, hwq_type)) return -ENOMEM; nq->budget = 8; @@ -481,21 +492,19 @@ int bnxt_qplib_alloc_nq(struct pci_dev *pdev, struct bnxt_qplib_nq *nq) static void bnxt_qplib_arm_srq(struct bnxt_qplib_srq *srq, u32 arm_type) { struct bnxt_qplib_hwq *srq_hwq = &srq->hwq; - struct dbr_dbr db_msg = { 0 }; void __iomem *db; - u32 sw_prod = 0; + u32 sw_prod; + u64 val = 0; /* Ring DB */ - sw_prod = (arm_type == DBR_DBR_TYPE_SRQ_ARM) ? srq->threshold : - HWQ_CMP(srq_hwq->prod, srq_hwq); - db_msg.index = cpu_to_le32((sw_prod << DBR_DBR_INDEX_SFT) & - DBR_DBR_INDEX_MASK); - db_msg.type_xid = cpu_to_le32(((srq->id << DBR_DBR_XID_SFT) & - DBR_DBR_XID_MASK) | arm_type); - db = (arm_type == DBR_DBR_TYPE_SRQ_ARMENA) ? - srq->dbr_base : srq->dpi->dbr; - wmb(); /* barrier before db ring */ - __iowrite64_copy(db, &db_msg, sizeof(db_msg) / sizeof(u64)); + sw_prod = (arm_type == DBC_DBC_TYPE_SRQ_ARM) ? + srq->threshold : HWQ_CMP(srq_hwq->prod, srq_hwq); + db = (arm_type == DBC_DBC_TYPE_SRQ_ARMENA) ? srq->dbr_base : + srq->dpi->dbr; + val = ((srq->id << DBC_DBC_XID_SFT) & DBC_DBC_XID_MASK) | arm_type; + val <<= 32; + val |= (sw_prod << DBC_DBC_INDEX_SFT) & DBC_DBC_INDEX_MASK; + writeq(val, db); } int bnxt_qplib_destroy_srq(struct bnxt_qplib_res *res, @@ -590,7 +599,7 @@ int bnxt_qplib_create_srq(struct bnxt_qplib_res *res, srq->id = le32_to_cpu(resp.xid); srq->dbr_base = res->dpi_tbl.dbr_bar_reg_iomem; if (srq->threshold) - bnxt_qplib_arm_srq(srq, DBR_DBR_TYPE_SRQ_ARMENA); + bnxt_qplib_arm_srq(srq, DBC_DBC_TYPE_SRQ_ARMENA); srq->arm_req = false; return 0; @@ -614,7 +623,7 @@ int bnxt_qplib_modify_srq(struct bnxt_qplib_res *res, srq_hwq->max_elements - sw_cons + sw_prod; if (count > srq->threshold) { srq->arm_req = false; - bnxt_qplib_arm_srq(srq, DBR_DBR_TYPE_SRQ_ARM); + bnxt_qplib_arm_srq(srq, DBC_DBC_TYPE_SRQ_ARM); } else { /* Deferred arming */ srq->arm_req = true; @@ -702,10 +711,10 @@ int bnxt_qplib_post_srq_recv(struct bnxt_qplib_srq *srq, srq_hwq->max_elements - sw_cons + sw_prod; spin_unlock(&srq_hwq->lock); /* Ring DB */ - bnxt_qplib_arm_srq(srq, DBR_DBR_TYPE_SRQ); + bnxt_qplib_arm_srq(srq, DBC_DBC_TYPE_SRQ); if (srq->arm_req == true && count > srq->threshold) { srq->arm_req = false; - bnxt_qplib_arm_srq(srq, DBR_DBR_TYPE_SRQ_ARM); + bnxt_qplib_arm_srq(srq, DBC_DBC_TYPE_SRQ_ARM); } done: return rc; @@ -853,18 +862,19 @@ exit: int bnxt_qplib_create_qp(struct bnxt_qplib_res *res, struct bnxt_qplib_qp *qp) { struct bnxt_qplib_rcfw *rcfw = res->rcfw; - struct sq_send *hw_sq_send_hdr, **hw_sq_send_ptr; - struct cmdq_create_qp req; - struct creq_create_qp_resp resp; - struct bnxt_qplib_pbl *pbl; - struct sq_psn_search **psn_search_ptr; unsigned long int psn_search, poff = 0; + struct sq_psn_search **psn_search_ptr; struct bnxt_qplib_q *sq = &qp->sq; struct bnxt_qplib_q *rq = &qp->rq; + int i, rc, req_size, psn_sz = 0; + struct sq_send **hw_sq_send_ptr; + struct creq_create_qp_resp resp; struct bnxt_qplib_hwq *xrrq; - int i, rc, req_size, psn_sz; u16 cmd_flags = 0, max_ssge; - u32 sw_prod, qp_flags = 0; + struct cmdq_create_qp req; + struct bnxt_qplib_pbl *pbl; + u32 qp_flags = 0; + u16 max_rsge; RCFW_CMD_PREP(req, CREATE_QP, cmd_flags); @@ -874,8 +884,11 @@ int bnxt_qplib_create_qp(struct bnxt_qplib_res *res, struct bnxt_qplib_qp *qp) req.qp_handle = cpu_to_le64(qp->qp_handle); /* SQ */ - psn_sz = (qp->type == CMDQ_CREATE_QP_TYPE_RC) ? - sizeof(struct sq_psn_search) : 0; + if (qp->type == CMDQ_CREATE_QP_TYPE_RC) { + psn_sz = bnxt_qplib_is_chip_gen_p5(res->cctx) ? + sizeof(struct sq_psn_search_ext) : + sizeof(struct sq_psn_search); + } sq->hwq.max_elements = sq->max_wqe; rc = bnxt_qplib_alloc_init_hwq(res->pdev, &sq->hwq, sq->sglist, sq->nmap, &sq->hwq.max_elements, @@ -905,10 +918,16 @@ int bnxt_qplib_create_qp(struct bnxt_qplib_res *res, struct bnxt_qplib_qp *qp) poff = (psn_search & ~PAGE_MASK) / BNXT_QPLIB_MAX_PSNE_ENTRY_SIZE; } - for (i = 0; i < sq->hwq.max_elements; i++) + for (i = 0; i < sq->hwq.max_elements; i++) { sq->swq[i].psn_search = &psn_search_ptr[get_psne_pg(i + poff)] [get_psne_idx(i + poff)]; + /*psns_ext will be used only for P5 chips. */ + sq->swq[i].psn_ext = + (struct sq_psn_search_ext *) + &psn_search_ptr[get_psne_pg(i + poff)] + [get_psne_idx(i + poff)]; + } } pbl = &sq->hwq.pbl[PBL_LVL_0]; req.sq_pbl = cpu_to_le64(pbl->pg_map_arr[0]); @@ -929,14 +948,6 @@ int bnxt_qplib_create_qp(struct bnxt_qplib_res *res, struct bnxt_qplib_qp *qp) CMDQ_CREATE_QP_SQ_PG_SIZE_PG_1G : CMDQ_CREATE_QP_SQ_PG_SIZE_PG_4K); - /* initialize all SQ WQEs to LOCAL_INVALID (sq prep for hw fetch) */ - hw_sq_send_ptr = (struct sq_send **)sq->hwq.pbl_ptr; - for (sw_prod = 0; sw_prod < sq->hwq.max_elements; sw_prod++) { - hw_sq_send_hdr = &hw_sq_send_ptr[get_sqe_pg(sw_prod)] - [get_sqe_idx(sw_prod)]; - hw_sq_send_hdr->wqe_type = SQ_BASE_WQE_TYPE_LOCAL_INVALID; - } - if (qp->scq) req.scq_cid = cpu_to_le32(qp->scq->id); @@ -1007,8 +1018,9 @@ int bnxt_qplib_create_qp(struct bnxt_qplib_res *res, struct bnxt_qplib_qp *qp) req.sq_fwo_sq_sge = cpu_to_le16( ((max_ssge & CMDQ_CREATE_QP_SQ_SGE_MASK) << CMDQ_CREATE_QP_SQ_SGE_SFT) | 0); + max_rsge = bnxt_qplib_is_chip_gen_p5(res->cctx) ? 6 : rq->max_sge; req.rq_fwo_rq_sge = cpu_to_le16( - ((rq->max_sge & CMDQ_CREATE_QP_RQ_SGE_MASK) + ((max_rsge & CMDQ_CREATE_QP_RQ_SGE_MASK) << CMDQ_CREATE_QP_RQ_SGE_SFT) | 0); /* ORRQ and IRRQ */ if (psn_sz) { @@ -1053,6 +1065,7 @@ int bnxt_qplib_create_qp(struct bnxt_qplib_res *res, struct bnxt_qplib_qp *qp) qp->id = le32_to_cpu(resp.xid); qp->cur_qp_state = CMDQ_MODIFY_QP_NEW_STATE_RESET; + qp->cctx = res->cctx; INIT_LIST_HEAD(&qp->sq_flush); INIT_LIST_HEAD(&qp->rq_flush); rcfw->qp_tbl[qp->id].qp_id = qp->id; @@ -1494,19 +1507,16 @@ void *bnxt_qplib_get_qp1_rq_buf(struct bnxt_qplib_qp *qp, void bnxt_qplib_post_send_db(struct bnxt_qplib_qp *qp) { struct bnxt_qplib_q *sq = &qp->sq; - struct dbr_dbr db_msg = { 0 }; u32 sw_prod; + u64 val = 0; + val = (((qp->id << DBC_DBC_XID_SFT) & DBC_DBC_XID_MASK) | + DBC_DBC_TYPE_SQ); + val <<= 32; sw_prod = HWQ_CMP(sq->hwq.prod, &sq->hwq); - - db_msg.index = cpu_to_le32((sw_prod << DBR_DBR_INDEX_SFT) & - DBR_DBR_INDEX_MASK); - db_msg.type_xid = - cpu_to_le32(((qp->id << DBR_DBR_XID_SFT) & DBR_DBR_XID_MASK) | - DBR_DBR_TYPE_SQ); + val |= (sw_prod << DBC_DBC_INDEX_SFT) & DBC_DBC_INDEX_MASK; /* Flush all the WQE writes to HW */ - wmb(); - __iowrite64_copy(qp->dpi->dbr, &db_msg, sizeof(db_msg) / sizeof(u64)); + writeq(val, qp->dpi->dbr); } int bnxt_qplib_post_send(struct bnxt_qplib_qp *qp, @@ -1617,7 +1627,8 @@ int bnxt_qplib_post_send(struct bnxt_qplib_qp *qp, ((offsetof(typeof(*sqe), data) + 15) >> 4); sqe->inv_key_or_imm_data = cpu_to_le32( wqe->send.inv_key); - if (qp->type == CMDQ_CREATE_QP_TYPE_UD) { + if (qp->type == CMDQ_CREATE_QP_TYPE_UD || + qp->type == CMDQ_CREATE_QP_TYPE_GSI) { sqe->q_key = cpu_to_le32(wqe->send.q_key); sqe->dst_qp = cpu_to_le32( wqe->send.dst_qp & SQ_SEND_DST_QP_MASK); @@ -1741,14 +1752,26 @@ int bnxt_qplib_post_send(struct bnxt_qplib_qp *qp, } swq->next_psn = sq->psn & BTH_PSN_MASK; if (swq->psn_search) { - swq->psn_search->opcode_start_psn = cpu_to_le32( - ((swq->start_psn << SQ_PSN_SEARCH_START_PSN_SFT) & - SQ_PSN_SEARCH_START_PSN_MASK) | - ((wqe->type << SQ_PSN_SEARCH_OPCODE_SFT) & - SQ_PSN_SEARCH_OPCODE_MASK)); - swq->psn_search->flags_next_psn = cpu_to_le32( - ((swq->next_psn << SQ_PSN_SEARCH_NEXT_PSN_SFT) & - SQ_PSN_SEARCH_NEXT_PSN_MASK)); + u32 opcd_spsn; + u32 flg_npsn; + + opcd_spsn = ((swq->start_psn << SQ_PSN_SEARCH_START_PSN_SFT) & + SQ_PSN_SEARCH_START_PSN_MASK); + opcd_spsn |= ((wqe->type << SQ_PSN_SEARCH_OPCODE_SFT) & + SQ_PSN_SEARCH_OPCODE_MASK); + flg_npsn = ((swq->next_psn << SQ_PSN_SEARCH_NEXT_PSN_SFT) & + SQ_PSN_SEARCH_NEXT_PSN_MASK); + if (bnxt_qplib_is_chip_gen_p5(qp->cctx)) { + swq->psn_ext->opcode_start_psn = + cpu_to_le32(opcd_spsn); + swq->psn_ext->flags_next_psn = + cpu_to_le32(flg_npsn); + } else { + swq->psn_search->opcode_start_psn = + cpu_to_le32(opcd_spsn); + swq->psn_search->flags_next_psn = + cpu_to_le32(flg_npsn); + } } queue_err: if (sch_handler) { @@ -1785,19 +1808,16 @@ done: void bnxt_qplib_post_recv_db(struct bnxt_qplib_qp *qp) { struct bnxt_qplib_q *rq = &qp->rq; - struct dbr_dbr db_msg = { 0 }; u32 sw_prod; + u64 val = 0; + val = (((qp->id << DBC_DBC_XID_SFT) & DBC_DBC_XID_MASK) | + DBC_DBC_TYPE_RQ); + val <<= 32; sw_prod = HWQ_CMP(rq->hwq.prod, &rq->hwq); - db_msg.index = cpu_to_le32((sw_prod << DBR_DBR_INDEX_SFT) & - DBR_DBR_INDEX_MASK); - db_msg.type_xid = - cpu_to_le32(((qp->id << DBR_DBR_XID_SFT) & DBR_DBR_XID_MASK) | - DBR_DBR_TYPE_RQ); - + val |= (sw_prod << DBC_DBC_INDEX_SFT) & DBC_DBC_INDEX_MASK; /* Flush the writes to HW Rx WQE before the ringing Rx DB */ - wmb(); - __iowrite64_copy(qp->dpi->dbr, &db_msg, sizeof(db_msg) / sizeof(u64)); + writeq(val, qp->dpi->dbr); } int bnxt_qplib_post_recv(struct bnxt_qplib_qp *qp, @@ -1881,32 +1901,28 @@ done: /* Spinlock must be held */ static void bnxt_qplib_arm_cq_enable(struct bnxt_qplib_cq *cq) { - struct dbr_dbr db_msg = { 0 }; + u64 val = 0; - db_msg.type_xid = - cpu_to_le32(((cq->id << DBR_DBR_XID_SFT) & DBR_DBR_XID_MASK) | - DBR_DBR_TYPE_CQ_ARMENA); + val = ((cq->id << DBC_DBC_XID_SFT) & DBC_DBC_XID_MASK) | + DBC_DBC_TYPE_CQ_ARMENA; + val <<= 32; /* Flush memory writes before enabling the CQ */ - wmb(); - __iowrite64_copy(cq->dbr_base, &db_msg, sizeof(db_msg) / sizeof(u64)); + writeq(val, cq->dbr_base); } static void bnxt_qplib_arm_cq(struct bnxt_qplib_cq *cq, u32 arm_type) { struct bnxt_qplib_hwq *cq_hwq = &cq->hwq; - struct dbr_dbr db_msg = { 0 }; u32 sw_cons; + u64 val = 0; /* Ring DB */ + val = ((cq->id << DBC_DBC_XID_SFT) & DBC_DBC_XID_MASK) | arm_type; + val <<= 32; sw_cons = HWQ_CMP(cq_hwq->cons, cq_hwq); - db_msg.index = cpu_to_le32((sw_cons << DBR_DBR_INDEX_SFT) & - DBR_DBR_INDEX_MASK); - db_msg.type_xid = - cpu_to_le32(((cq->id << DBR_DBR_XID_SFT) & DBR_DBR_XID_MASK) | - arm_type); + val |= (sw_cons << DBC_DBC_INDEX_SFT) & DBC_DBC_INDEX_MASK; /* flush memory writes before arming the CQ */ - wmb(); - __iowrite64_copy(cq->dpi->dbr, &db_msg, sizeof(db_msg) / sizeof(u64)); + writeq(val, cq->dpi->dbr); } int bnxt_qplib_create_cq(struct bnxt_qplib_res *res, struct bnxt_qplib_cq *cq) @@ -2053,6 +2069,7 @@ static int __flush_rq(struct bnxt_qplib_q *rq, struct bnxt_qplib_qp *qp, opcode = CQ_BASE_CQE_TYPE_RES_RC; break; case CMDQ_CREATE_QP_TYPE_UD: + case CMDQ_CREATE_QP_TYPE_GSI: opcode = CQ_BASE_CQE_TYPE_RES_UD; break; } @@ -2125,7 +2142,7 @@ static int do_wa9060(struct bnxt_qplib_qp *qp, struct bnxt_qplib_cq *cq, sq->send_phantom = true; /* TODO: Only ARM if the previous SQE is ARMALL */ - bnxt_qplib_arm_cq(cq, DBR_DBR_TYPE_CQ_ARMALL); + bnxt_qplib_arm_cq(cq, DBC_DBC_TYPE_CQ_ARMALL); rc = -EAGAIN; goto out; @@ -2410,12 +2427,14 @@ static int bnxt_qplib_cq_process_res_ud(struct bnxt_qplib_cq *cq, } cqe = *pcqe; cqe->opcode = hwcqe->cqe_type_toggle & CQ_BASE_CQE_TYPE_MASK; - cqe->length = le32_to_cpu(hwcqe->length); + cqe->length = (u32)le16_to_cpu(hwcqe->length); + cqe->cfa_meta = le16_to_cpu(hwcqe->cfa_metadata); cqe->invrkey = le32_to_cpu(hwcqe->imm_data); cqe->flags = le16_to_cpu(hwcqe->flags); cqe->status = hwcqe->status; cqe->qp_handle = (u64)(unsigned long)qp; - memcpy(cqe->smac, hwcqe->src_mac, 6); + /*FIXME: Endianness fix needed for smace */ + memcpy(cqe->smac, hwcqe->src_mac, ETH_ALEN); wr_id_idx = le32_to_cpu(hwcqe->src_qp_high_srq_or_rq_wr_id) & CQ_RES_UD_SRQ_OR_RQ_WR_ID_MASK; cqe->src_qp = le16_to_cpu(hwcqe->src_qp_low) | @@ -2794,7 +2813,7 @@ int bnxt_qplib_poll_cq(struct bnxt_qplib_cq *cq, struct bnxt_qplib_cqe *cqe, } if (cq->hwq.cons != raw_cons) { cq->hwq.cons = raw_cons; - bnxt_qplib_arm_cq(cq, DBR_DBR_TYPE_CQ); + bnxt_qplib_arm_cq(cq, DBC_DBC_TYPE_CQ); } exit: return num_cqes - budget; |