626 lines
22 KiB
Diff
626 lines
22 KiB
Diff
From 9f440ac471dce7fa44f0ecb6df4ca2ac0509d105 Mon Sep 17 00:00:00 2001
|
|
From: =?UTF-8?q?=E6=9D=8E=E5=AF=8C=E8=89=B3?= <li.fuyan@zte.com.cn>
|
|
Date: Fri, 28 Mar 2025 15:30:41 +0800
|
|
Subject: [PATCH] libzrdma:Add sq/rq flush cqe and log optimization
|
|
MIME-Version: 1.0
|
|
Content-Type: text/plain; charset=UTF-8
|
|
Content-Transfer-Encoding: 8bit
|
|
|
|
Signed-off-by: 李富艳 <li.fuyan@zte.com.cn>
|
|
---
|
|
providers/zrdma/main.h | 10 +-
|
|
providers/zrdma/zxdh_hw.c | 253 ++++++++++++++++++++++-------------
|
|
providers/zrdma/zxdh_verbs.c | 72 +++++-----
|
|
3 files changed, 206 insertions(+), 129 deletions(-)
|
|
|
|
diff --git a/providers/zrdma/main.h b/providers/zrdma/main.h
|
|
index e28c77b..7c78fed 100644
|
|
--- a/providers/zrdma/main.h
|
|
+++ b/providers/zrdma/main.h
|
|
@@ -38,15 +38,17 @@ enum {
|
|
ZXDH_DBG_SRQ = 1 << 2,
|
|
};
|
|
extern uint32_t zxdh_debug_mask;
|
|
-#define zxdh_dbg(ctx, mask, format, arg...) \
|
|
+#define zxdh_dbg(mask, format, arg...) \
|
|
do { \
|
|
if (mask & zxdh_debug_mask) { \
|
|
- int zxdh_dbg_tmp = errno; \
|
|
- verbs_debug(ctx, format, ##arg); \
|
|
- errno = zxdh_dbg_tmp; \
|
|
+ int tmp = errno; \
|
|
+ fprintf(stdout, "%s:%d: " format, __func__, __LINE__, \
|
|
+ ##arg); \
|
|
+ errno = tmp; \
|
|
} \
|
|
} while (0)
|
|
|
|
+
|
|
struct zxdh_udevice {
|
|
struct verbs_device ibv_dev;
|
|
};
|
|
diff --git a/providers/zrdma/zxdh_hw.c b/providers/zrdma/zxdh_hw.c
|
|
index 99489dc..fb8f016 100644
|
|
--- a/providers/zrdma/zxdh_hw.c
|
|
+++ b/providers/zrdma/zxdh_hw.c
|
|
@@ -1785,19 +1785,26 @@ static inline void build_comp_status(__u32 cq_type,
|
|
}
|
|
return;
|
|
}
|
|
- if (info->major_err == ZXDH_RETRY_ACK_MAJOR_ERR &&
|
|
- info->minor_err == ZXDH_RETRY_ACK_MINOR_ERR) {
|
|
- info->comp_status = ZXDH_COMPL_STATUS_RETRY_ACK_ERR;
|
|
+
|
|
+ switch (info->major_err) {
|
|
+ case ZXDH_RETRY_ACK_MAJOR_ERR:
|
|
+ if (info->minor_err == ZXDH_RETRY_ACK_MINOR_ERR) {
|
|
+ info->comp_status = ZXDH_COMPL_STATUS_RETRY_ACK_ERR;
|
|
+ return;
|
|
+ }
|
|
+ if (info->minor_err == ZXDH_TX_WINDOW_QUERY_ITEM_MINOR_ERR) {
|
|
+ info->comp_status =
|
|
+ ZXDH_COMPL_STATUS_TX_WINDOW_QUERY_ITEM_ERR;
|
|
+ return;
|
|
+ }
|
|
+ break;
|
|
+ case ZXDH_FLUSH_MAJOR_ERR:
|
|
+ info->comp_status = ZXDH_COMPL_STATUS_FLUSHED;
|
|
return;
|
|
- }
|
|
- if (info->major_err == ZXDH_RETRY_ACK_MAJOR_ERR &&
|
|
- info->minor_err == ZXDH_TX_WINDOW_QUERY_ITEM_MINOR_ERR) {
|
|
- info->comp_status = ZXDH_COMPL_STATUS_TX_WINDOW_QUERY_ITEM_ERR;
|
|
+ default:
|
|
+ info->comp_status = ZXDH_COMPL_STATUS_UNKNOWN;
|
|
return;
|
|
}
|
|
- info->comp_status = (info->major_err == ZXDH_FLUSH_MAJOR_ERR) ?
|
|
- ZXDH_COMPL_STATUS_FLUSHED :
|
|
- ZXDH_COMPL_STATUS_UNKNOWN;
|
|
}
|
|
|
|
__le64 *get_current_cqe(struct zxdh_cq *cq)
|
|
@@ -1837,9 +1844,9 @@ static inline void zxdh_get_cq_poll_info(struct zxdh_qp *qp,
|
|
}
|
|
}
|
|
|
|
-static void update_cq_poll_info(struct zxdh_qp *qp,
|
|
- struct zxdh_cq_poll_info *info, __u32 wqe_idx,
|
|
- __u64 qword0)
|
|
+static enum zxdh_status_code update_cq_poll_info(struct zxdh_qp *qp,
|
|
+ struct zxdh_cq_poll_info *info,
|
|
+ __u32 wqe_idx, __u64 qword0)
|
|
{
|
|
info->wr_id = qp->sq_wrtrk_array[wqe_idx].wrid;
|
|
if (!info->comp_status)
|
|
@@ -1847,6 +1854,7 @@ static void update_cq_poll_info(struct zxdh_qp *qp,
|
|
info->op_type = (__u8)FIELD_GET(ZXDHCQ_OP, qword0);
|
|
ZXDH_RING_SET_TAIL(qp->sq_ring,
|
|
wqe_idx + qp->sq_wrtrk_array[wqe_idx].quanta);
|
|
+ return ZXDH_SUCCESS;
|
|
}
|
|
|
|
static enum zxdh_status_code
|
|
@@ -1862,9 +1870,9 @@ process_tx_window_query_item_err(struct zxdh_qp *qp,
|
|
ib_qp = &iwuqp->vqp.qp;
|
|
ret = zxdh_query_qpc(ib_qp, &qpc);
|
|
if (ret) {
|
|
- verbs_err(verbs_get_ctx(ib_qp->context),
|
|
- "process tx window query item query qpc failed:%d\n",
|
|
- ret);
|
|
+ zxdh_dbg(ZXDH_DBG_QP,
|
|
+ "process tx window query item query qpc failed:%d\n",
|
|
+ ret);
|
|
return ZXDH_ERR_RETRY_ACK_ERR;
|
|
}
|
|
if (qpc.tx_last_ack_psn != qp->qp_last_ack_qsn)
|
|
@@ -1876,9 +1884,9 @@ process_tx_window_query_item_err(struct zxdh_qp *qp,
|
|
|
|
ret = zxdh_reset_qp(ib_qp, ZXDH_RESET_RETRY_TX_ITEM_FLAG);
|
|
if (ret) {
|
|
- verbs_err(verbs_get_ctx(ib_qp->context),
|
|
- "process tx window query item reset qp failed:%d\n",
|
|
- ret);
|
|
+ zxdh_dbg(ZXDH_DBG_QP,
|
|
+ "process tx window query item reset qp failed:%d\n",
|
|
+ ret);
|
|
return ZXDH_ERR_RETRY_ACK_ERR;
|
|
}
|
|
qp->qp_reset_cnt++;
|
|
@@ -1899,8 +1907,8 @@ process_retry_ack_err(struct zxdh_qp *qp, struct zxdh_cq_poll_info *info)
|
|
ib_qp = &iwuqp->vqp.qp;
|
|
ret = zxdh_query_qpc(ib_qp, &qpc);
|
|
if (ret) {
|
|
- verbs_err(verbs_get_ctx(ib_qp->context),
|
|
- "process retry ack query qpc failed:%d\n", ret);
|
|
+ zxdh_dbg(ZXDH_DBG_QP, "process retry ack query qpc failed:%d\n",
|
|
+ ret);
|
|
return ZXDH_ERR_RETRY_ACK_ERR;
|
|
}
|
|
if (!(qpc.retry_cqe_sq_opcode >= ZXDH_RETRY_CQE_SQ_OPCODE_ERR &&
|
|
@@ -1926,14 +1934,122 @@ process_retry_ack_err(struct zxdh_qp *qp, struct zxdh_cq_poll_info *info)
|
|
ZXDH_RETRY_CQE_SQ_OPCODE |
|
|
ZXDH_TX_READ_RETRY_FLAG_SET);
|
|
if (ret) {
|
|
- verbs_err(verbs_get_ctx(ib_qp->context),
|
|
- "process retry ack modify qpc failed:%d\n", ret);
|
|
+ zxdh_dbg(ZXDH_DBG_QP,
|
|
+ "process retry ack modify qpc failed:%d\n", ret);
|
|
return ZXDH_ERR_RETRY_ACK_ERR;
|
|
}
|
|
qp->cqe_retry_cnt++;
|
|
return ZXDH_ERR_RETRY_ACK_NOT_EXCEED_ERR;
|
|
}
|
|
|
|
+static enum zxdh_status_code
|
|
+zxdh_flush_sq_comp_info(struct zxdh_qp *qp, struct zxdh_cq_poll_info *info,
|
|
+ bool *move_cq_head)
|
|
+{
|
|
+ if (!ZXDH_RING_MORE_WORK(qp->sq_ring)) {
|
|
+ ZXDH_RING_INIT(qp->sq_ring, qp->sq_ring.size)
|
|
+ return ZXDH_ERR_Q_EMPTY;
|
|
+ }
|
|
+ do {
|
|
+ __le64 *sw_wqe;
|
|
+ __u64 wqe_qword;
|
|
+ __u64 wqe_idx;
|
|
+ wqe_idx = qp->sq_ring.tail;
|
|
+ sw_wqe = qp->sq_base[wqe_idx].elem;
|
|
+ get_64bit_val(sw_wqe, 0, &wqe_qword);
|
|
+ info->op_type = (__u8)FIELD_GET(ZXDHQPSQ_OPCODE, wqe_qword);
|
|
+ ZXDH_RING_SET_TAIL(qp->sq_ring,
|
|
+ wqe_idx +
|
|
+ qp->sq_wrtrk_array[wqe_idx].quanta);
|
|
+
|
|
+ if (info->op_type != ZXDH_OP_TYPE_NOP) {
|
|
+ info->wr_id = qp->sq_wrtrk_array[wqe_idx].wrid;
|
|
+ break;
|
|
+ }
|
|
+ } while (1);
|
|
+ qp->sq_flush_seen = true;
|
|
+ if (!ZXDH_RING_MORE_WORK(qp->sq_ring)) {
|
|
+ qp->sq_flush_complete = true;
|
|
+ ZXDH_RING_INIT(qp->sq_ring, qp->sq_ring.size)
|
|
+ } else
|
|
+ *move_cq_head = false;
|
|
+ return ZXDH_SUCCESS;
|
|
+}
|
|
+
|
|
+static enum zxdh_status_code zxdh_sq_comp_info(struct zxdh_qp *qp,
|
|
+ struct zxdh_cq_poll_info *info,
|
|
+ __u32 wqe_idx, __u64 qword0,
|
|
+ bool *move_cq_head)
|
|
+{
|
|
+ enum zxdh_status_code status_code;
|
|
+ switch (info->comp_status) {
|
|
+ case ZXDH_COMPL_STATUS_SUCCESS:
|
|
+ case ZXDH_COMPL_STATUS_UNKNOWN:
|
|
+ break;
|
|
+ case ZXDH_COMPL_STATUS_RETRY_ACK_ERR:
|
|
+ if (qp->qp_type == ZXDH_QP_TYPE_ROCE_RC) {
|
|
+ status_code = process_retry_ack_err(qp, info);
|
|
+ return (status_code == ZXDH_ERR_RETRY_ACK_ERR) ?
|
|
+ update_cq_poll_info(qp, info, wqe_idx,
|
|
+ qword0) :
|
|
+ status_code;
|
|
+ }
|
|
+ break;
|
|
+ case ZXDH_COMPL_STATUS_TX_WINDOW_QUERY_ITEM_ERR:
|
|
+ if (qp->qp_type == ZXDH_QP_TYPE_ROCE_RC) {
|
|
+ status_code =
|
|
+ process_tx_window_query_item_err(qp, info);
|
|
+ return (status_code == ZXDH_ERR_RETRY_ACK_ERR) ?
|
|
+ update_cq_poll_info(qp, info, wqe_idx,
|
|
+ qword0) :
|
|
+ status_code;
|
|
+ }
|
|
+ break;
|
|
+ case ZXDH_COMPL_STATUS_FLUSHED:
|
|
+ return zxdh_flush_sq_comp_info(qp, info, move_cq_head);
|
|
+ default:
|
|
+ break;
|
|
+ }
|
|
+ return update_cq_poll_info(qp, info, wqe_idx, qword0);
|
|
+}
|
|
+
|
|
+static enum zxdh_status_code zxdh_rq_comp_info(struct zxdh_qp *qp,
|
|
+ struct zxdh_cq_poll_info *info,
|
|
+ __u32 wqe_idx, __u64 qword2,
|
|
+ __u64 qword3, bool *move_cq_head)
|
|
+{
|
|
+ struct zxdh_uqp *iwuqp = NULL;
|
|
+ struct zxdh_usrq *iwusrq = NULL;
|
|
+ struct zxdh_srq *srq = NULL;
|
|
+ if (qp->is_srq) {
|
|
+ iwuqp = container_of(qp, struct zxdh_uqp, qp);
|
|
+ iwusrq = iwuqp->srq;
|
|
+ srq = &iwusrq->srq;
|
|
+ zxdh_free_srq_wqe(srq, wqe_idx);
|
|
+ info->wr_id = srq->srq_wrid_array[wqe_idx];
|
|
+ zxdh_get_cq_poll_info(qp, info, qword2, qword3);
|
|
+ } else {
|
|
+ if (unlikely(info->comp_status == ZXDH_COMPL_STATUS_FLUSHED ||
|
|
+ info->comp_status == ZXDH_COMPL_STATUS_UNKNOWN)) {
|
|
+ if (!ZXDH_RING_MORE_WORK(qp->rq_ring)) {
|
|
+ return ZXDH_ERR_Q_EMPTY;
|
|
+ }
|
|
+ wqe_idx = qp->rq_ring.tail;
|
|
+ }
|
|
+ info->wr_id = qp->rq_wrid_array[wqe_idx];
|
|
+ zxdh_get_cq_poll_info(qp, info, qword2, qword3);
|
|
+ ZXDH_RING_SET_TAIL(qp->rq_ring, wqe_idx + 1);
|
|
+ if (info->comp_status == ZXDH_COMPL_STATUS_FLUSHED) {
|
|
+ qp->rq_flush_seen = true;
|
|
+ if (!ZXDH_RING_MORE_WORK(qp->rq_ring))
|
|
+ qp->rq_flush_complete = true;
|
|
+ else
|
|
+ *move_cq_head = false;
|
|
+ }
|
|
+ }
|
|
+ return ZXDH_SUCCESS;
|
|
+}
|
|
+
|
|
/**
|
|
* zxdh_cq_poll_cmpl - get cq completion info
|
|
* @cq: hw cq
|
|
@@ -1942,7 +2058,6 @@ process_retry_ack_err(struct zxdh_qp *qp, struct zxdh_cq_poll_info *info)
|
|
enum zxdh_status_code zxdh_cq_poll_cmpl(struct zxdh_cq *cq,
|
|
struct zxdh_cq_poll_info *info)
|
|
{
|
|
- enum zxdh_status_code status_code;
|
|
__u64 comp_ctx, qword0, qword2, qword3;
|
|
__le64 *cqe;
|
|
struct zxdh_qp *qp;
|
|
@@ -1951,9 +2066,6 @@ enum zxdh_status_code zxdh_cq_poll_cmpl(struct zxdh_cq *cq,
|
|
int ret_code;
|
|
bool move_cq_head = true;
|
|
__u8 polarity;
|
|
- struct zxdh_usrq *iwusrq = NULL;
|
|
- struct zxdh_srq *srq = NULL;
|
|
- struct zxdh_uqp *iwuqp;
|
|
|
|
cqe = get_current_cqe(cq);
|
|
|
|
@@ -1973,7 +2085,7 @@ enum zxdh_status_code zxdh_cq_poll_cmpl(struct zxdh_cq *cq,
|
|
ret_code = ZXDH_ERR_Q_DESTROYED;
|
|
goto exit;
|
|
}
|
|
- iwuqp = container_of(qp, struct zxdh_uqp, qp);
|
|
+
|
|
info->qp_handle = (zxdh_qp_handle)(unsigned long)qp;
|
|
q_type = (__u8)FIELD_GET(ZXDH_CQ_SQ, qword0);
|
|
info->solicited_event = (bool)FIELD_GET(ZXDHCQ_SOEVENT, qword0);
|
|
@@ -1993,74 +2105,19 @@ enum zxdh_status_code zxdh_cq_poll_cmpl(struct zxdh_cq *cq,
|
|
|
|
info->qp_id = (__u32)FIELD_GET(ZXDHCQ_QPID, qword2);
|
|
info->imm_valid = false;
|
|
-
|
|
- info->qp_handle = (zxdh_qp_handle)(unsigned long)qp;
|
|
switch (q_type) {
|
|
- case ZXDH_CQE_QTYPE_RQ:
|
|
- if (qp->is_srq) {
|
|
- iwusrq = iwuqp->srq;
|
|
- srq = &iwusrq->srq;
|
|
- zxdh_free_srq_wqe(srq, wqe_idx);
|
|
- info->wr_id = srq->srq_wrid_array[wqe_idx];
|
|
- zxdh_get_cq_poll_info(qp, info, qword2, qword3);
|
|
- } else {
|
|
- if (unlikely(info->comp_status ==
|
|
- ZXDH_COMPL_STATUS_FLUSHED ||
|
|
- info->comp_status ==
|
|
- ZXDH_COMPL_STATUS_UNKNOWN)) {
|
|
- if (!ZXDH_RING_MORE_WORK(qp->rq_ring)) {
|
|
- ret_code = ZXDH_ERR_Q_EMPTY;
|
|
- goto exit;
|
|
- }
|
|
- wqe_idx = qp->rq_ring.tail;
|
|
- }
|
|
- info->wr_id = qp->rq_wrid_array[wqe_idx];
|
|
- zxdh_get_cq_poll_info(qp, info, qword2, qword3);
|
|
- ZXDH_RING_SET_TAIL(qp->rq_ring, wqe_idx + 1);
|
|
- if (info->comp_status == ZXDH_COMPL_STATUS_FLUSHED) {
|
|
- qp->rq_flush_seen = true;
|
|
- if (!ZXDH_RING_MORE_WORK(qp->rq_ring))
|
|
- qp->rq_flush_complete = true;
|
|
- else
|
|
- move_cq_head = false;
|
|
- }
|
|
- pring = &qp->rq_ring;
|
|
- }
|
|
- ret_code = ZXDH_SUCCESS;
|
|
- break;
|
|
case ZXDH_CQE_QTYPE_SQ:
|
|
- if (info->comp_status == ZXDH_COMPL_STATUS_RETRY_ACK_ERR &&
|
|
- qp->qp_type == ZXDH_QP_TYPE_ROCE_RC) {
|
|
- status_code = process_retry_ack_err(qp, info);
|
|
- if (status_code == ZXDH_ERR_RETRY_ACK_ERR) {
|
|
- update_cq_poll_info(qp, info, wqe_idx, qword0);
|
|
- ret_code = ZXDH_SUCCESS;
|
|
- } else {
|
|
- ret_code = status_code;
|
|
- }
|
|
- } else if (info->comp_status ==
|
|
- ZXDH_COMPL_STATUS_TX_WINDOW_QUERY_ITEM_ERR &&
|
|
- qp->qp_type == ZXDH_QP_TYPE_ROCE_RC) {
|
|
- status_code =
|
|
- process_tx_window_query_item_err(qp, info);
|
|
- if (status_code == ZXDH_ERR_RETRY_ACK_ERR) {
|
|
- update_cq_poll_info(qp, info, wqe_idx, qword0);
|
|
- ret_code = ZXDH_SUCCESS;
|
|
- } else {
|
|
- ret_code = status_code;
|
|
- }
|
|
- } else if (info->comp_status == ZXDH_COMPL_STATUS_FLUSHED) {
|
|
- info->wr_id = qp->sq_wrtrk_array[wqe_idx].wrid;
|
|
- ZXDH_RING_INIT(qp->sq_ring, qp->sq_ring.size);
|
|
- ret_code = ZXDH_SUCCESS;
|
|
- } else {
|
|
- update_cq_poll_info(qp, info, wqe_idx, qword0);
|
|
- ret_code = ZXDH_SUCCESS;
|
|
- }
|
|
+ ret_code = zxdh_sq_comp_info(qp, info, wqe_idx, qword0,
|
|
+ &move_cq_head);
|
|
+ pring = &qp->sq_ring;
|
|
+ break;
|
|
+ case ZXDH_CQE_QTYPE_RQ:
|
|
+ ret_code = zxdh_rq_comp_info(qp, info, wqe_idx, qword2, qword3,
|
|
+ &move_cq_head);
|
|
+ pring = &qp->rq_ring;
|
|
break;
|
|
default:
|
|
- zxdh_dbg(verbs_get_ctx(iwuqp->vqp.qp.context), ZXDH_DBG_CQ,
|
|
- "zxdh get cqe type unknow!\n");
|
|
+ zxdh_dbg(ZXDH_DBG_CQ, "zxdh get cqe type unknow!\n");
|
|
ret_code = ZXDH_ERR_Q_DESTROYED;
|
|
break;
|
|
}
|
|
@@ -2538,6 +2595,16 @@ enum zxdh_status_code zxdh_srq_init(struct zxdh_srq *srq,
|
|
ZXDH_RING_INIT(srq->srq_list_ring, srq->srq_list_size);
|
|
srq->srq_ring.tail = srq->srq_size - 1;
|
|
srq->srq_list_polarity = 1;
|
|
+ zxdh_dbg(ZXDH_DBG_SRQ, "%s srq_wqe_size_multiplier:%d srqshift:%d\n",
|
|
+ __func__, srq->srq_wqe_size_multiplier, srqshift);
|
|
+ zxdh_dbg(
|
|
+ ZXDH_DBG_SRQ,
|
|
+ "%s srq->srq_id:%d srq_base:0x%p srq_list_base:0x%p srq_db_base:0x%p\n",
|
|
+ __func__, srq->srq_id, srq->srq_base, srq->srq_list_base,
|
|
+ srq->srq_db_base);
|
|
+ zxdh_dbg(ZXDH_DBG_SRQ,
|
|
+ "%s srq->srq_id:%d srq_ring_size:%d srq->srq_list_size:%d\n",
|
|
+ __func__, srq->srq_id, srq_ring_size, srq->srq_list_size);
|
|
return 0;
|
|
}
|
|
|
|
@@ -2558,4 +2625,6 @@ void zxdh_free_srq_wqe(struct zxdh_srq *srq, int wqe_index)
|
|
set_64bit_val(wqe, 0, hdr);
|
|
|
|
pthread_spin_unlock(&iwusrq->lock);
|
|
+ zxdh_dbg(ZXDH_DBG_SRQ, "%s srq->srq_id:%d wqe_index:%d\n", __func__,
|
|
+ srq->srq_id, wqe_index);
|
|
}
|
|
diff --git a/providers/zrdma/zxdh_verbs.c b/providers/zrdma/zxdh_verbs.c
|
|
index f67f8c7..9cf1240 100644
|
|
--- a/providers/zrdma/zxdh_verbs.c
|
|
+++ b/providers/zrdma/zxdh_verbs.c
|
|
@@ -1485,13 +1485,12 @@ static struct ibv_qp *create_qp(struct ibv_context *ibv_ctx,
|
|
}
|
|
|
|
if (attr->cap.max_inline_data > dev_attrs->max_hw_inline) {
|
|
- zxdh_dbg(&iwvctx->ibv_ctx, ZXDH_DBG_QP,
|
|
- "max_inline_data over max_hw_inline\n");
|
|
+ zxdh_dbg(ZXDH_DBG_QP, "max_inline_data over max_hw_inline\n");
|
|
attr->cap.max_inline_data = dev_attrs->max_hw_inline;
|
|
}
|
|
|
|
- zxdh_get_sq_wqe_shift(attr->cap.max_send_sge, attr->cap.max_inline_data,
|
|
- &sqshift);
|
|
+ zxdh_get_sq_wqe_shift(attr->cap.max_send_sge,
|
|
+ attr->cap.max_inline_data, &sqshift);
|
|
status = zxdh_get_sqdepth(dev_attrs, attr->cap.max_send_wr, sqshift,
|
|
&sqdepth);
|
|
if (status) {
|
|
@@ -2661,9 +2660,8 @@ static void zxdh_srq_wqe_init(struct zxdh_usrq *iwusrq)
|
|
__u64 hdr;
|
|
|
|
srq = &iwusrq->srq;
|
|
- zxdh_dbg(verbs_get_ctx(iwusrq->ibv_srq.context), ZXDH_DBG_SRQ,
|
|
- "%s head:%d tail:%d\n", __func__, srq->srq_ring.head,
|
|
- srq->srq_ring.tail);
|
|
+ zxdh_dbg(ZXDH_DBG_SRQ, "%s head:%d tail:%d\n", __func__,
|
|
+ srq->srq_ring.head, srq->srq_ring.tail);
|
|
for (i = srq->srq_ring.head; i < srq->srq_ring.tail; i++) {
|
|
wqe = zxdh_get_srq_wqe(srq, i);
|
|
|
|
@@ -2707,7 +2705,7 @@ static size_t zxdh_get_total_srq_size(struct zxdh_usrq *iwusrq, int srqdepth,
|
|
total_srq_queue_size + total_srq_list_size + total_srq_db_size;
|
|
iwusrq->total_buf_size = total_srq_size;
|
|
zxdh_dbg(
|
|
- verbs_get_ctx(iwusrq->ibv_srq.context), ZXDH_DBG_SRQ,
|
|
+ ZXDH_DBG_SRQ,
|
|
"%s total_srq_queue_size:%ld total_srq_list_size:%ld total_srq_db_size:%ld srqdepth:%d\n",
|
|
__func__, total_srq_queue_size, total_srq_list_size,
|
|
total_srq_db_size, srqdepth);
|
|
@@ -2730,7 +2728,7 @@ static int zxdh_alloc_srq_buf(struct zxdh_usrq *iwusrq,
|
|
(__le64 *)&info->srq_list_base[iwusrq->list_buf_size /
|
|
(sizeof(__u16))];
|
|
*(__le64 *)info->srq_db_base = ZXDH_SRQ_DB_INIT_VALUE;
|
|
- zxdh_dbg(verbs_get_ctx(iwusrq->ibv_srq.context), ZXDH_DBG_SRQ,
|
|
+ zxdh_dbg(ZXDH_DBG_SRQ,
|
|
"%s srq_base:0x%p srq_list_base:0x%p srq_db_base:0x%p\n",
|
|
__func__, info->srq_base, info->srq_list_base,
|
|
info->srq_db_base);
|
|
@@ -2782,7 +2780,7 @@ static int create_srq(struct ibv_pd *pd, struct zxdh_usrq *iwusrq,
|
|
info->srq_size = resp.actual_srq_size;
|
|
info->srq_list_size = resp.actual_srq_list_size;
|
|
zxdh_dbg(
|
|
- verbs_get_ctx(iwusrq->ibv_srq.context), ZXDH_DBG_SRQ,
|
|
+ ZXDH_DBG_SRQ,
|
|
"%s info->srq_id:%d info->srq_size:%d info->srq_list_size:%d\n",
|
|
__func__, info->srq_id, info->srq_size, info->srq_list_size);
|
|
|
|
@@ -2814,19 +2812,21 @@ static int zxdh_vmapped_srq(struct zxdh_usrq *iwusrq, struct ibv_pd *pd,
|
|
ret = zxdh_alloc_srq_buf(iwusrq, info, total_srq_size);
|
|
if (ret)
|
|
return -ENOMEM;
|
|
- zxdh_dbg(verbs_get_ctx(iwusrq->ibv_srq.context), ZXDH_DBG_SRQ,
|
|
- "%s srq_pages:%ld srq_list_pages:%ld\n", __func__, srq_pages,
|
|
- srq_list_pages);
|
|
+ zxdh_dbg(ZXDH_DBG_SRQ, "%s srq_pages:%ld srq_list_pages:%ld\n",
|
|
+ __func__, srq_pages, srq_list_pages);
|
|
|
|
ret = zxdh_reg_srq_mr(pd, info, total_srq_size, srq_pages,
|
|
srq_list_pages, iwusrq);
|
|
+ zxdh_dbg(ZXDH_DBG_SRQ, "%s %d ret:%d\n", __func__, __LINE__, ret);
|
|
if (ret) {
|
|
errno = ret;
|
|
goto err_dereg_srq_mr;
|
|
}
|
|
ret = create_srq(pd, iwusrq, attr, info);
|
|
+ zxdh_dbg(ZXDH_DBG_SRQ, "%s %d ret:%d\n", __func__, __LINE__, ret);
|
|
if (ret)
|
|
goto err_srq;
|
|
+
|
|
return 0;
|
|
err_srq:
|
|
ibv_cmd_dereg_mr(&iwusrq->vmr);
|
|
@@ -2907,8 +2907,7 @@ struct ibv_srq *zxdh_ucreate_srq(struct ibv_pd *pd,
|
|
dev_attrs = &iwvctx->dev_attrs;
|
|
|
|
if ((zxdh_check_srq_init_attr(srq_init_attr, dev_attrs)) != 0) {
|
|
- verbs_err(&iwvctx->ibv_ctx,
|
|
- "zxdh_check_srq_init_attr failed\n");
|
|
+ zxdh_dbg(ZXDH_DBG_SRQ, "zxdh_check_srq_init_attr failed\n");
|
|
errno = EINVAL;
|
|
return NULL;
|
|
}
|
|
@@ -2922,12 +2921,12 @@ struct ibv_srq *zxdh_ucreate_srq(struct ibv_pd *pd,
|
|
srq_init_attr->attr.max_wr, srqshift,
|
|
&srqdepth);
|
|
zxdh_dbg(
|
|
- &iwvctx->ibv_ctx, ZXDH_DBG_SRQ,
|
|
+ ZXDH_DBG_SRQ,
|
|
"%s %d status:%d srqshift:%d srqdepth:%d dev_attrs->max_hw_srq_quanta:%d srq_init_attr->attr.max_wr:%d\n",
|
|
__func__, __LINE__, status, srqshift, srqdepth,
|
|
dev_attrs->max_hw_srq_quanta, srq_init_attr->attr.max_wr);
|
|
if (status != 0) {
|
|
- verbs_err(&iwvctx->ibv_ctx, "zxdh_get_srqdepth failed\n");
|
|
+ zxdh_dbg(ZXDH_DBG_SRQ, "zxdh_get_srqdepth failed\n");
|
|
errno = EINVAL;
|
|
return NULL;
|
|
}
|
|
@@ -2940,19 +2939,21 @@ struct ibv_srq *zxdh_ucreate_srq(struct ibv_pd *pd,
|
|
|
|
if (zxdh_init_iwusrq(iwusrq, srq_init_attr, srqdepth, srqshift, &info,
|
|
dev_attrs)) {
|
|
- verbs_err(&iwvctx->ibv_ctx, "calloc srq_wrid_array failed\n");
|
|
+ zxdh_dbg(ZXDH_DBG_SRQ, "calloc srq_wrid_array failed\n");
|
|
goto err_srq_wrid_array;
|
|
}
|
|
status = zxdh_vmapped_srq(iwusrq, pd, srq_init_attr, srqdepth, &info);
|
|
+ zxdh_dbg(ZXDH_DBG_SRQ, "%s %d status:%d\n", __func__, __LINE__, status);
|
|
if (status) {
|
|
- verbs_err(&iwvctx->ibv_ctx, "zxdh_vmapped_srq failed\n");
|
|
+ zxdh_dbg(ZXDH_DBG_SRQ, "zxdh_vmapped_srq failed\n");
|
|
errno = status;
|
|
goto err_vmapped_srq;
|
|
}
|
|
|
|
status = zxdh_srq_init(&iwusrq->srq, &info);
|
|
+ zxdh_dbg(ZXDH_DBG_SRQ, "%s %d status:%d\n", __func__, __LINE__, status);
|
|
if (status) {
|
|
- verbs_err(&iwvctx->ibv_ctx, "zxdh_srq_init failed\n");
|
|
+ zxdh_dbg(ZXDH_DBG_SRQ, "zxdh_srq_init failed\n");
|
|
errno = EINVAL;
|
|
goto err_free_srq_init;
|
|
}
|
|
@@ -2960,9 +2961,8 @@ struct ibv_srq *zxdh_ucreate_srq(struct ibv_pd *pd,
|
|
|
|
srq_init_attr->attr.max_wr = (srqdepth - ZXDH_SRQ_RSVD) >> srqshift;
|
|
|
|
- zxdh_dbg(&iwvctx->ibv_ctx, ZXDH_DBG_SRQ,
|
|
- "iwusrq->srq_id:%d info.srq_size:%d\n", iwusrq->srq_id,
|
|
- info.srq_size);
|
|
+ zxdh_dbg(ZXDH_DBG_SRQ, "%s iwusrq->srq_id:%d info.srq_size:%d\n",
|
|
+ __func__, iwusrq->srq_id, info.srq_size);
|
|
return &iwusrq->ibv_srq;
|
|
|
|
err_free_srq_init:
|
|
@@ -2976,6 +2976,7 @@ err_srq_wrid_array:
|
|
errno = EINVAL;
|
|
err_free_srq:
|
|
free(iwusrq);
|
|
+ zxdh_dbg(ZXDH_DBG_SRQ, "%s %d\n", __func__, __LINE__);
|
|
return NULL;
|
|
}
|
|
|
|
@@ -2996,8 +2997,8 @@ int zxdh_udestroy_srq(struct ibv_srq *srq)
|
|
ret = zxdh_destroy_vmapped_srq(iwusrq);
|
|
if (ret)
|
|
goto err;
|
|
- zxdh_dbg(verbs_get_ctx(iwusrq->ibv_srq.context), ZXDH_DBG_SRQ,
|
|
- "iwusrq->srq_id:%d\n", iwusrq->srq_id);
|
|
+ zxdh_dbg(ZXDH_DBG_SRQ, "%s iwusrq->srq_id:%d\n", __func__,
|
|
+ iwusrq->srq_id);
|
|
zxdh_free_hw_buf(iwusrq->srq.srq_base, iwusrq->total_buf_size);
|
|
free(iwusrq->srq.srq_wrid_array);
|
|
free(iwusrq);
|
|
@@ -3024,9 +3025,8 @@ int zxdh_umodify_srq(struct ibv_srq *srq, struct ibv_srq_attr *srq_attr,
|
|
sizeof(cmd));
|
|
if (ret == 0)
|
|
iwusrq->srq_limit = srq_attr->srq_limit;
|
|
- zxdh_dbg(verbs_get_ctx(iwusrq->ibv_srq.context), ZXDH_DBG_SRQ,
|
|
- "iwusrq->srq_id:%d srq_attr->srq_limit:%d\n", iwusrq->srq_id,
|
|
- srq_attr->srq_limit);
|
|
+ zxdh_dbg(ZXDH_DBG_SRQ, "%s iwusrq->srq_id:%d srq_attr->srq_limit:%d\n",
|
|
+ __func__, iwusrq->srq_id, srq_attr->srq_limit);
|
|
return ret;
|
|
}
|
|
|
|
@@ -3090,6 +3090,13 @@ static void zxdh_fill_srq_wqe(struct zxdh_usrq *iwusrq, struct zxdh_srq *srq,
|
|
|
|
udma_to_device_barrier(); /* make sure WQE is populated before valid bit is set */
|
|
set_64bit_val(wqe_64, 0, hdr);
|
|
+
|
|
+ zxdh_dbg(ZXDH_DBG_SRQ, "%s wqe_64[0]:0x%llx\n", __func__, wqe_64[0]);
|
|
+ zxdh_dbg(ZXDH_DBG_SRQ, "%s wqe_64[1]:0x%llx\n", __func__, wqe_64[1]);
|
|
+ zxdh_dbg(ZXDH_DBG_SRQ, "%s wqe_64[2]:0x%llx\n", __func__, wqe_64[2]);
|
|
+ zxdh_dbg(ZXDH_DBG_SRQ, "%s wqe_64[3]:0x%llx\n", __func__, wqe_64[3]);
|
|
+ zxdh_dbg(ZXDH_DBG_SRQ, "%s wqe_64[4]:0x%llx\n", __func__, wqe_64[4]);
|
|
+ zxdh_dbg(ZXDH_DBG_SRQ, "%s wqe_64[5]:0x%llx\n", __func__, wqe_64[5]);
|
|
}
|
|
|
|
static void zxdh_get_wqe_index(struct zxdh_srq *srq, __le16 *wqe_16, __u16 *buf,
|
|
@@ -3112,6 +3119,7 @@ static void zxdh_update_srq_db_base(struct zxdh_usrq *iwusrq, __u16 idx)
|
|
|
|
udma_to_device_barrier(); /* make sure WQE is populated before valid bit is set */
|
|
set_64bit_val(iwusrq->srq.srq_db_base, 0, hdr);
|
|
+ zxdh_dbg(ZXDH_DBG_SRQ, "%s srq_db_base(hdr):0x%llx\n", __func__, hdr);
|
|
}
|
|
|
|
/**
|
|
@@ -3140,8 +3148,7 @@ int zxdh_upost_srq_recv(struct ibv_srq *srq, struct ibv_recv_wr *recv_wr,
|
|
buf_size = iwusrq->max_wr * sizeof(__u16);
|
|
buf = malloc(buf_size);
|
|
if (buf == NULL) {
|
|
- verbs_err(verbs_get_ctx(iwusrq->ibv_srq.context),
|
|
- "malloc buf_size failed\n");
|
|
+ zxdh_dbg(ZXDH_DBG_SRQ, "malloc buf_size failed\n");
|
|
err = -ENOMEM;
|
|
goto out;
|
|
}
|
|
@@ -3161,9 +3168,8 @@ int zxdh_upost_srq_recv(struct ibv_srq *srq, struct ibv_recv_wr *recv_wr,
|
|
zxdh_fill_srq_wqe(iwusrq, hw_srq, wqe_64, recv_wr);
|
|
}
|
|
|
|
- zxdh_dbg(verbs_get_ctx(iwusrq->ibv_srq.context), ZXDH_DBG_SRQ,
|
|
- "nreq:%d err:%d iwusrq->srq_id:%d\n", nreq, err,
|
|
- iwusrq->srq_id);
|
|
+ zxdh_dbg(ZXDH_DBG_SRQ, "%s nreq:%d err:%d iwusrq->srq_id:%d\n",
|
|
+ __func__, nreq, err, iwusrq->srq_id);
|
|
|
|
if (err == 0) {
|
|
zxdh_get_wqe_index(hw_srq, wqe_16, buf, nreq, &idx);
|
|
--
|
|
2.27.0
|
|
|