packages/rdma-core
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity
rdma-core/0049-libzrdma-Add-interface-aligned-with-kernel.patch
Xinghai Cen 5a10c8bdbf fix some libzrdma bugs and add some optimization 
Signed-off-by: 李富艳 <li.fuyan@zte.com.cn>
2025-04-23 10:30:13 +08:00

305 lines
11 KiB
Diff
Raw Blame History

From da8370c2360deb73af7a211bec2be76b025cb5d3 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=E6=9D=8E=E5=AF=8C=E8=89=B3?= <li.fuyan@zte.com.cn>
Date: Fri, 28 Mar 2025 11:36:50 +0800
Subject: [PATCH] libzrdma:Add interface aligned with kernel
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
Signed-off-by: 李富艳 <li.fuyan@zte.com.cn>
---
kernel-headers/rdma/zxdh-abi.h | 23 ++++++++-------
providers/zrdma/main.c | 34 +++++++++++++----------
providers/zrdma/zxdh_defs.h | 8 +++---
providers/zrdma/zxdh_devids.h | 9 ++++++
providers/zrdma/zxdh_hw.c | 9 ++++--
providers/zrdma/zxdh_verbs.h | 51 ++++++----------------------------
6 files changed, 60 insertions(+), 74 deletions(-)
diff --git a/kernel-headers/rdma/zxdh-abi.h b/kernel-headers/rdma/zxdh-abi.h
index 665f874..59c0160 100644
--- a/kernel-headers/rdma/zxdh-abi.h
+++ b/kernel-headers/rdma/zxdh-abi.h
@@ -6,10 +6,9 @@
#include <linux/types.h>
-/* zxdh must support legacy GEN_1 i40iw kernel
- * and user-space whose last ABI ver is 5
- */
+/* user-space whose last ABI ver is 5 */
#define ZXDH_ABI_VER 5
+#define ZXDH_CONTEXT_VER_V1 5
enum zxdh_memreg_type {
ZXDH_MEMREG_TYPE_MEM = 0,
@@ -35,7 +34,7 @@ struct zxdh_alloc_ucontext_resp {
__u32 wq_size; /* size of the WQs (SQ+RQ) in the mmaped area */
__u8 kernel_ver;
__u8 db_addr_type;
- __u8 rsvd[2];
+ __u16 rdma_tool_flags;
__aligned_u64 feature_flags;
__aligned_u64 sq_db_mmap_key;
__aligned_u64 cq_db_mmap_key;
@@ -51,8 +50,8 @@ struct zxdh_alloc_ucontext_resp {
__u32 min_hw_cq_size;
__u32 max_hw_cq_size;
__u16 max_hw_sq_chunk;
- __u8 hw_rev;
- __u8 rsvd2;
+ __u8 rsvd;
+ __u8 chip_rev;
};
struct zxdh_alloc_pd_resp {
@@ -82,13 +81,13 @@ struct zxdh_create_srq_req {
};
struct zxdh_mem_reg_req {
- __u16 reg_type; /* enum zxdh_memreg_type */
- __u16 cq_pages;
- __u16 rq_pages;
- __u16 sq_pages;
- __u16 srq_pages;
+ __u32 reg_type; /* enum zxdh_memreg_type */
+ __u32 cq_pages;
+ __u32 rq_pages;
+ __u32 sq_pages;
+ __u32 srq_pages;
__u16 srq_list_pages;
- __u8 rsvd[4];
+ __u8 rsvd[2];
};
struct zxdh_reg_mr_resp {
diff --git a/providers/zrdma/main.c b/providers/zrdma/main.c
index e25a1a2..4626a21 100644
--- a/providers/zrdma/main.c
+++ b/providers/zrdma/main.c
@@ -22,6 +22,12 @@ static const struct verbs_match_ent hca_table[] = {
ZXDH_HCA(PCI_VENDOR_ID_ZXDH_EVB, ZXDH_DEV_ID_ADAPTIVE_EVB_VF),
ZXDH_HCA(PCI_VENDOR_ID_ZXDH_E312, ZXDH_DEV_ID_ADAPTIVE_E312_PF),
ZXDH_HCA(PCI_VENDOR_ID_ZXDH_E312, ZXDH_DEV_ID_ADAPTIVE_E312_VF),
+ ZXDH_HCA(PCI_VENDOR_ID_ZXDH_E310, ZXDH_DEV_ID_ADAPTIVE_E310_PF),
+ ZXDH_HCA(PCI_VENDOR_ID_ZXDH_E310, ZXDH_DEV_ID_ADAPTIVE_E310_VF),
+ ZXDH_HCA(PCI_VENDOR_ID_ZXDH_E310_RDMA, ZXDH_DEV_ID_ADAPTIVE_E310_RDMA_PF),
+ ZXDH_HCA(PCI_VENDOR_ID_ZXDH_E310_RDMA, ZXDH_DEV_ID_ADAPTIVE_E310_RDMA_VF),
+ ZXDH_HCA(PCI_VENDOR_ID_ZXDH_E316, ZXDH_DEV_ID_ADAPTIVE_E316_PF),
+ ZXDH_HCA(PCI_VENDOR_ID_ZXDH_E316, ZXDH_DEV_ID_ADAPTIVE_E316_VF),
ZXDH_HCA(PCI_VENDOR_ID_ZXDH_X512, ZXDH_DEV_ID_ADAPTIVE_X512_PF),
ZXDH_HCA(PCI_VENDOR_ID_ZXDH_X512, ZXDH_DEV_ID_ADAPTIVE_X512_VF),
{}
@@ -100,7 +106,6 @@ static struct verbs_context *zxdh_ualloc_context(struct ibv_device *ibdev,
struct zxdh_get_context cmd;
struct zxdh_get_context_resp resp = {};
__u64 sq_db_mmap_key, cq_db_mmap_key;
- __u8 user_ver = ZXDH_ABI_VER;
iwvctx = verbs_init_and_alloc_context(ibdev, cmd_fd, iwvctx, ibv_ctx,
RDMA_DRIVER_ZXDH);
@@ -109,22 +114,16 @@ static struct verbs_context *zxdh_ualloc_context(struct ibv_device *ibdev,
zxdh_set_debug_mask();
iwvctx->zxdh_write_imm_split_switch = zxdh_get_write_imm_split_switch();
- cmd.userspace_ver = user_ver;
+
+ cmd.userspace_ver = ZXDH_CONTEXT_VER_V1;
if (ibv_cmd_get_context(&iwvctx->ibv_ctx,
(struct ibv_get_context *)&cmd, sizeof(cmd),
- &resp.ibv_resp, sizeof(resp))) {
- cmd.userspace_ver = 4;
- if (ibv_cmd_get_context(
- &iwvctx->ibv_ctx, (struct ibv_get_context *)&cmd,
- sizeof(cmd), &resp.ibv_resp, sizeof(resp)))
- goto err_free;
- user_ver = cmd.userspace_ver;
- }
+ &resp.ibv_resp, sizeof(resp)))
+ goto err_free;
verbs_set_ops(&iwvctx->ibv_ctx, &zxdh_uctx_ops);
iwvctx->dev_attrs.feature_flags = resp.feature_flags;
- iwvctx->dev_attrs.hw_rev = resp.hw_rev;
iwvctx->dev_attrs.max_hw_wq_frags = resp.max_hw_wq_frags;
iwvctx->dev_attrs.max_hw_read_sges = resp.max_hw_read_sges;
iwvctx->dev_attrs.max_hw_inline = resp.max_hw_inline;
@@ -135,11 +134,20 @@ static struct verbs_context *zxdh_ualloc_context(struct ibv_device *ibdev,
iwvctx->dev_attrs.max_hw_sq_chunk = resp.max_hw_sq_chunk;
iwvctx->dev_attrs.max_hw_cq_size = resp.max_hw_cq_size;
iwvctx->dev_attrs.min_hw_cq_size = resp.min_hw_cq_size;
- iwvctx->abi_ver = user_ver;
+ iwvctx->abi_ver = ZXDH_ABI_VER;
+ iwvctx->dev_attrs.chip_rev = resp.chip_rev;
+ iwvctx->dev_attrs.rdma_tool_flags = resp.rdma_tool_flags;
sq_db_mmap_key = resp.sq_db_mmap_key;
cq_db_mmap_key = resp.cq_db_mmap_key;
+ iwvctx->dev_attrs.db_addr_type = resp.db_addr_type;
+ iwvctx->dev_attrs.sq_db_pa = resp.sq_db_pa;
+ iwvctx->dev_attrs.cq_db_pa = resp.cq_db_pa;
+
+ if (iwvctx->dev_attrs.db_addr_type != ZXDH_DB_ADDR_BAR)
+ goto err_free;
+
iwvctx->sq_db = zxdh_mmap(cmd_fd, sq_db_mmap_key);
if (iwvctx->sq_db == MAP_FAILED)
goto err_free;
@@ -160,10 +168,8 @@ static struct verbs_context *zxdh_ualloc_context(struct ibv_device *ibdev,
iwvctx->iwupd = container_of(ibv_pd, struct zxdh_upd, ibv_pd);
add_private_ops(iwvctx);
return &iwvctx->ibv_ctx;
-
err_free:
free(iwvctx);
-
return NULL;
}
diff --git a/providers/zrdma/zxdh_defs.h b/providers/zrdma/zxdh_defs.h
index 3863fb9..8772e7b 100644
--- a/providers/zrdma/zxdh_defs.h
+++ b/providers/zrdma/zxdh_defs.h
@@ -389,8 +389,8 @@ static inline void db_wr32(__u32 val, __u32 *wqe_word)
*wqe_word = val;
}
-#define read_wqe_need_split(pre_cal_psn, next_psn) \
- (((pre_cal_psn < next_psn) && (pre_cal_psn != 0)) || \
- ((next_psn <= 0x7FFFFF) && (pre_cal_psn > 0x800000)))
-
+#define read_wqe_need_split(pre_cal_psn, next_psn, chip_rev) \
+ (!(chip_rev == 2) && \
+ (((pre_cal_psn < next_psn) && (pre_cal_psn != 0)) || \
+ ((next_psn <= 0x7FFFFF) && (pre_cal_psn > 0x800000))))
#endif /* ZXDH_DEFS_H */
diff --git a/providers/zrdma/zxdh_devids.h b/providers/zrdma/zxdh_devids.h
index ac23124..3430f5f 100644
--- a/providers/zrdma/zxdh_devids.h
+++ b/providers/zrdma/zxdh_devids.h
@@ -6,12 +6,21 @@
/* ZXDH VENDOR ID */
#define PCI_VENDOR_ID_ZXDH_EVB 0x16c3
#define PCI_VENDOR_ID_ZXDH_E312 0x1cf2
+#define PCI_VENDOR_ID_ZXDH_E310 0x1cf2
+#define PCI_VENDOR_ID_ZXDH_E310_RDMA 0x1cf2
+#define PCI_VENDOR_ID_ZXDH_E316 0x1cf2
#define PCI_VENDOR_ID_ZXDH_X512 0x1cf2
/* ZXDH Devices ID */
#define ZXDH_DEV_ID_ADAPTIVE_EVB_PF 0x8040 /* ZXDH EVB PF DEVICE ID*/
#define ZXDH_DEV_ID_ADAPTIVE_EVB_VF 0x8041 /* ZXDH EVB VF DEVICE ID*/
#define ZXDH_DEV_ID_ADAPTIVE_E312_PF 0x8049 /* ZXDH E312 PF DEVICE ID*/
#define ZXDH_DEV_ID_ADAPTIVE_E312_VF 0x8060 /* ZXDH E312 VF DEVICE ID*/
+#define ZXDH_DEV_ID_ADAPTIVE_E310_PF 0x8061 /* ZXDH E310 PF DEVICE ID*/
+#define ZXDH_DEV_ID_ADAPTIVE_E310_VF 0x8062 /* ZXDH E310 VF DEVICE ID*/
+#define ZXDH_DEV_ID_ADAPTIVE_E310_RDMA_PF 0x8084 /* ZXDH E310_RDMA PF DEVICE ID*/
+#define ZXDH_DEV_ID_ADAPTIVE_E310_RDMA_VF 0x8085 /* ZXDH E310_RDMA VF DEVICE ID*/
+#define ZXDH_DEV_ID_ADAPTIVE_E316_PF 0x807e /* ZXDH E316 PF DEVICE ID*/
+#define ZXDH_DEV_ID_ADAPTIVE_E316_VF 0x807f /* ZXDH E316 VF DEVICE ID*/
#define ZXDH_DEV_ID_ADAPTIVE_X512_PF 0x806B /* ZXDH X512 PF DEVICE ID*/
#define ZXDH_DEV_ID_ADAPTIVE_X512_VF 0x806C /* ZXDH X512 VF DEVICE ID*/
#endif /* ZXDH_DEVIDS_H */
diff --git a/providers/zrdma/zxdh_hw.c b/providers/zrdma/zxdh_hw.c
index 073b198..99489dc 100644
--- a/providers/zrdma/zxdh_hw.c
+++ b/providers/zrdma/zxdh_hw.c
@@ -703,8 +703,12 @@ enum zxdh_status_code zxdh_rdma_read(struct zxdh_qp *qp,
struct zxdh_post_sq_info split_part2_info = { 0 };
struct zxdh_rdma_read *op_info;
enum zxdh_status_code ret_code;
+ struct zxdh_uqp *iwuqp;
+ struct zxdh_uvcontext *iwvctx;
__u32 i, total_size = 0, pre_cal_psn = 0;
-
+ iwuqp = container_of(qp, struct zxdh_uqp, qp);
+ iwvctx = container_of(iwuqp->vqp.qp.context, struct zxdh_uvcontext,
+ ibv_ctx.context);
op_info = &info->op.rdma_read;
if (qp->max_sq_frag_cnt < op_info->num_lo_sges)
return ZXDH_ERR_INVALID_FRAG_COUNT;
@@ -720,7 +724,8 @@ enum zxdh_status_code zxdh_rdma_read(struct zxdh_qp *qp,
op_info->rem_addr.len = total_size;
pre_cal_psn = qp->next_psn;
qp_tx_psn_add(&pre_cal_psn, total_size, qp->mtu);
- if (read_wqe_need_split(pre_cal_psn, qp->next_psn)) {
+ if (read_wqe_need_split(pre_cal_psn, qp->next_psn,
+ iwvctx->dev_attrs.chip_rev)) {
split_two_part_info(qp, info, qp->next_psn, pre_cal_psn,
&split_part1_info, &split_part2_info);
ret_code = zxdh_post_rdma_read(qp, &split_part1_info, post_sq,
diff --git a/providers/zrdma/zxdh_verbs.h b/providers/zrdma/zxdh_verbs.h
index 40aa7bb..1a26cf4 100644
--- a/providers/zrdma/zxdh_verbs.h
+++ b/providers/zrdma/zxdh_verbs.h
@@ -71,47 +71,13 @@
#define ZXDH_WQEALLOC_WQE_DESC_INDEX GENMASK(31, 20)
-enum zxdh_device_caps_const {
- ZXDH_WQE_SIZE = 4,
- ZXDH_SRQE_SIZE = 2,
- ZXDH_CQP_WQE_SIZE = 8,
- ZXDH_CQE_SIZE = 8,
- ZXDH_EXTENDED_CQE_SIZE = 8,
- ZXDH_AEQE_SIZE = 2,
- ZXDH_CEQE_SIZE = 1,
- ZXDH_CQP_CTX_SIZE = 8,
- ZXDH_SHADOW_AREA_SIZE = 8,
- ZXDH_GATHER_STATS_BUF_SIZE = 1024,
- ZXDH_MIN_IW_QP_ID = 0,
- ZXDH_QUERY_FPM_BUF_SIZE = 176,
- ZXDH_COMMIT_FPM_BUF_SIZE = 176,
- ZXDH_MAX_IW_QP_ID = 262143,
- ZXDH_MIN_CEQID = 0,
- ZXDH_MAX_CEQID = 1023,
- ZXDH_CEQ_MAX_COUNT = ZXDH_MAX_CEQID + 1,
- ZXDH_MIN_CQID = 0,
- ZXDH_MAX_CQID = 524287,
- ZXDH_MIN_AEQ_ENTRIES = 1,
- ZXDH_MAX_AEQ_ENTRIES = 524287,
- ZXDH_MIN_CEQ_ENTRIES = 1,
- ZXDH_MAX_CEQ_ENTRIES = 262143,
- ZXDH_MIN_CQ_SIZE = 1,
- ZXDH_MAX_CQ_SIZE = 1048575,
- ZXDH_DB_ID_ZERO = 0,
- ZXDH_MAX_WQ_FRAGMENT_COUNT = 13,
- ZXDH_MAX_SGE_RD = 13,
- ZXDH_MAX_OUTBOUND_MSG_SIZE = 2147483647,
- ZXDH_MAX_INBOUND_MSG_SIZE = 2147483647,
- ZXDH_MAX_PUSH_PAGE_COUNT = 1024,
- ZXDH_MAX_PE_ENA_VF_COUNT = 32,
- ZXDH_MAX_VF_FPM_ID = 47,
- ZXDH_MAX_SQ_PAYLOAD_SIZE = 2147483648,
- ZXDH_MAX_INLINE_DATA_SIZE = 217,
- ZXDH_MAX_WQ_ENTRIES = 32768,
- ZXDH_Q2_BUF_SIZE = 256,
- ZXDH_QP_CTX_SIZE = 256,
- ZXDH_MAX_PDS = 262144,
-};
+#define ZXDH_SRQE_SIZE 2
+#define ZXDH_CQE_SIZE 8
+#define ZXDH_EXTENDED_CQE_SIZE 8
+#define ZXDH_MAX_INLINE_DATA_SIZE 217
+#define ZXDH_MAX_SQ_PAYLOAD_SIZE 2147483648
+#define ZXDH_MIN_CQ_SIZE 1
+#define ZXDH_MAX_CQ_SIZE 2097152
enum zxdh_addressing_type {
ZXDH_ADDR_TYPE_ZERO_BASED = 0,
@@ -394,8 +360,9 @@ struct zxdh_dev_attrs {
__u32 max_hw_cq_size;
__u16 max_hw_sq_chunk;
__u32 max_hw_srq_wr;
- __u8 hw_rev;
__u8 db_addr_type;
+ __u8 chip_rev;
+ __u16 rdma_tool_flags;
};
struct zxdh_hw_attrs {
--
2.27.0
Reference in New Issue View Git Blame Copy Permalink