From 08b80f6450477832b1a194f18fbed60367da46de Mon Sep 17 00:00:00 2001
From: Chengchang Tang <tangchengchang@huawei.com>
Date: Mon, 10 May 2021 17:13:49 +0800
Subject: [PATCH 25/25] libhns: Add direct verbs support to config DCA

driver inclusion
category: feature
bugzilla: https://gitee.com/src-openeuler/rdma-core/issues/I9C2AQ

------------------------------------------------------------------

Add two direct verbs to config DCA:
1. hnsdv_open_device() is used to config DCA memory pool.
2. hnsdv_create_qp() is used to create a DCA QP.

Signed-off-by: Chengchang Tang <tangchengchang@huawei.com>
Reviewed-by: Yangyang Li <liyangyang20@huawei.com>
---
 debian/control                   |  2 +-
 providers/hns/hns_roce_u.c       | 80 ++++++++++++++++++++++++++++----
 providers/hns/hns_roce_u.h       |  4 +-
 providers/hns/hns_roce_u_buf.c   |  3 ++
 providers/hns/hns_roce_u_verbs.c | 39 ++++++++++++++--
 providers/hns/hnsdv.h            | 29 +++++++++++-
 providers/hns/libhns.map         |  1 +
 7 files changed, 140 insertions(+), 18 deletions(-)

diff --git a/debian/control b/debian/control
index 160824f..2a55372 100644
--- a/debian/control
+++ b/debian/control
@@ -87,7 +87,7 @@ Description: User space provider drivers for libibverbs
   - efa: Amazon Elastic Fabric Adapter
   - erdma: Alibaba Elastic RDMA (iWarp) Adapter
   - hfi1verbs: Intel Omni-Path HFI
-  - hns: HiSilicon Hip06 SoC
+  - hns: HiSilicon Hip08+ SoC
   - ipathverbs: QLogic InfiniPath HCAs
   - irdma: Intel Ethernet Connection RDMA
   - mana: Microsoft Azure Network Adapter
diff --git a/providers/hns/hns_roce_u.c b/providers/hns/hns_roce_u.c
index 56ff201..93a0312 100644
--- a/providers/hns/hns_roce_u.c
+++ b/providers/hns/hns_roce_u.c
@@ -132,8 +132,55 @@ static int mmap_dca(struct hns_roce_context *ctx, int cmd_fd,
 	return 0;
 }
 
+struct ibv_context *hnsdv_open_device(struct ibv_device *device,
+				      struct hnsdv_context_attr *attr)
+{
+	if (!is_hns_dev(device)) {
+		errno = EOPNOTSUPP;
+		return NULL;
+	}
+
+	return verbs_open_device(device, attr);
+}
+
+static void set_dca_pool_param(struct hns_roce_context *ctx,
+			       struct hnsdv_context_attr *attr, int page_size)
+{
+	struct hns_roce_dca_ctx *dca_ctx = &ctx->dca_ctx;
+
+	if (attr->comp_mask & HNSDV_CONTEXT_MASK_DCA_UNIT_SIZE)
+		dca_ctx->unit_size = align(attr->dca_unit_size, page_size);
+	else
+		dca_ctx->unit_size = page_size * HNS_DCA_DEFAULT_UNIT_PAGES;
+
+	/* The memory pool cannot be expanded, only init the DCA context. */
+	if (dca_ctx->unit_size == 0)
+		return;
+
+	/* If not set, the memory pool can be expanded unlimitedly. */
+	if (attr->comp_mask & HNSDV_CONTEXT_MASK_DCA_MAX_SIZE)
+		dca_ctx->max_size = DIV_ROUND_UP(attr->dca_max_size,
+						 dca_ctx->unit_size) *
+					dca_ctx->unit_size;
+	else
+		dca_ctx->max_size = HNS_DCA_MAX_MEM_SIZE;
+
+	/* If not set, the memory pool cannot be shrunk. */
+	if (attr->comp_mask & HNSDV_CONTEXT_MASK_DCA_MIN_SIZE)
+		dca_ctx->min_size = DIV_ROUND_UP(attr->dca_min_size,
+						 dca_ctx->unit_size) *
+					dca_ctx->unit_size;
+	else
+		dca_ctx->min_size = HNS_DCA_MAX_MEM_SIZE;
+
+	verbs_debug(&ctx->ibv_ctx,
+		    "Support DCA, unit %u, max %lu, min %lu Bytes.\n",
+		    dca_ctx->unit_size, dca_ctx->max_size, dca_ctx->min_size);
+}
+
 static int init_dca_context(struct hns_roce_context *ctx, int cmd_fd,
 			    struct hns_roce_alloc_ucontext_resp *resp,
+			    struct hnsdv_context_attr *attr,
 			    int page_size)
 {
 	struct hns_roce_dca_ctx *dca_ctx = &ctx->dca_ctx;
@@ -145,14 +192,18 @@ static int init_dca_context(struct hns_roce_context *ctx, int cmd_fd,
 	if (!(ctx->config & HNS_ROCE_UCTX_RSP_DCA_FLAGS))
 		return 0;
 
+	dca_ctx->unit_size = 0;
+	dca_ctx->mem_cnt = 0;
+
 	list_head_init(&dca_ctx->mem_list);
 	ret = pthread_spin_init(&dca_ctx->lock, PTHREAD_PROCESS_PRIVATE);
 	if (ret)
 		return ret;
 
-	dca_ctx->unit_size = page_size * HNS_DCA_DEFAULT_UNIT_PAGES;
-	dca_ctx->max_size = HNS_DCA_MAX_MEM_SIZE;
-	dca_ctx->mem_cnt = 0;
+	if (!attr || !(attr->flags & HNSDV_CONTEXT_FLAGS_DCA))
+		return 0;
+
+	set_dca_pool_param(ctx, attr, page_size);
 
 	if (mmap_key) {
 		const unsigned int bits_per_qp = 2 * HNS_DCA_BITS_PER_STATUS;
@@ -253,18 +304,28 @@ static int set_context_attr(struct hns_roce_device *hr_dev,
 	return 0;
 }
 
-static void ucontext_set_cmd(struct hns_roce_alloc_ucontext *cmd, int page_size)
+static void ucontext_set_cmd(struct hns_roce_alloc_ucontext *cmd,
+			     struct hnsdv_context_attr *attr)
 {
 	cmd->config |= HNS_ROCE_EXSGE_FLAGS | HNS_ROCE_RQ_INLINE_FLAGS |
-		       HNS_ROCE_CQE_INLINE_FLAGS | HNS_ROCE_UCTX_CONFIG_DCA;
-	cmd->comp = HNS_ROCE_ALLOC_UCTX_COMP_DCA_MAX_QPS;
-	cmd->dca_max_qps = page_size * 8 / 2 * HNS_DCA_BITS_PER_STATUS;
+		       HNS_ROCE_CQE_INLINE_FLAGS;
+
+	if (!attr || !(attr->flags & HNSDV_CONTEXT_FLAGS_DCA))
+		return;
+
+	cmd->config |= HNS_ROCE_UCTX_CONFIG_DCA;
+
+	if (attr->comp_mask & HNSDV_CONTEXT_MASK_DCA_PRIME_QPS) {
+		cmd->comp |= HNS_ROCE_ALLOC_UCTX_COMP_DCA_MAX_QPS;
+		cmd->dca_max_qps = attr->dca_prime_qps;
+	}
 }
 
 static struct verbs_context *hns_roce_alloc_context(struct ibv_device *ibdev,
 						    int cmd_fd,
 						    void *private_data)
 {
+	struct hnsdv_context_attr *ctx_attr = private_data;
 	struct hns_roce_device *hr_dev = to_hr_dev(ibdev);
 	struct hns_roce_alloc_ucontext_resp resp = {};
 	struct hns_roce_alloc_ucontext cmd = {};
@@ -275,7 +336,7 @@ static struct verbs_context *hns_roce_alloc_context(struct ibv_device *ibdev,
 	if (!context)
 		return NULL;
 
-	ucontext_set_cmd(&cmd, hr_dev->page_size);
+	ucontext_set_cmd(&cmd, ctx_attr);
 	if (ibv_cmd_get_context(&context->ibv_ctx, &cmd.ibv_cmd, sizeof(cmd),
 				&resp.ibv_resp, sizeof(resp)))
 		goto err_free;
@@ -288,7 +349,8 @@ static struct verbs_context *hns_roce_alloc_context(struct ibv_device *ibdev,
 	if (context->uar == MAP_FAILED)
 		goto err_free;
 
-	if (init_dca_context(context, cmd_fd, &resp, hr_dev->page_size))
+	if (init_dca_context(context, cmd_fd,
+			     &resp, ctx_attr, hr_dev->page_size))
 		goto err_free;
 
 	if (init_reset_context(context, cmd_fd, &resp, hr_dev->page_size))
diff --git a/providers/hns/hns_roce_u.h b/providers/hns/hns_roce_u.h
index 5bddb00..691bf61 100644
--- a/providers/hns/hns_roce_u.h
+++ b/providers/hns/hns_roce_u.h
@@ -584,6 +584,8 @@ static inline void clear_bit_unlock(atomic_bitmap_t *p, uint32_t nr)
 	atomic_fetch_and(p, ~HNS_ROCE_BIT_MASK(nr));
 }
 
+bool is_hns_dev(struct ibv_device *device);
+
 int hns_roce_u_query_device(struct ibv_context *context,
 			    const struct ibv_query_device_ex_input *input,
 			    struct ibv_device_attr_ex *attr, size_t attr_size);
@@ -672,8 +674,6 @@ void hns_roce_cleanup_dca_mem(struct hns_roce_context *ctx);
 
 void hns_roce_init_qp_indices(struct hns_roce_qp *qp);
 
-bool is_hns_dev(struct ibv_device *device);
-
 extern const struct hns_roce_u_hw hns_roce_u_hw_v2;
 
 #endif /* _HNS_ROCE_U_H */
diff --git a/providers/hns/hns_roce_u_buf.c b/providers/hns/hns_roce_u_buf.c
index 08c0fbc..780683e 100644
--- a/providers/hns/hns_roce_u_buf.c
+++ b/providers/hns/hns_roce_u_buf.c
@@ -56,6 +56,9 @@ int hns_roce_alloc_buf(struct hns_roce_buf *buf, unsigned int size,
 
 void hns_roce_free_buf(struct hns_roce_buf *buf)
 {
+	if (!buf->buf)
+		return;
+
 	ibv_dofork_range(buf->buf, buf->length);
 
 	munmap(buf->buf, buf->length);
diff --git a/providers/hns/hns_roce_u_verbs.c b/providers/hns/hns_roce_u_verbs.c
index 248d862..8964d53 100644
--- a/providers/hns/hns_roce_u_verbs.c
+++ b/providers/hns/hns_roce_u_verbs.c
@@ -1072,6 +1072,15 @@ enum {
 				  IBV_QP_INIT_ATTR_SEND_OPS_FLAGS,
 };
 
+enum {
+        SEND_OPS_FLAG_MASK =
+                IBV_QP_EX_WITH_RDMA_WRITE | IBV_QP_EX_WITH_RDMA_WRITE_WITH_IMM |
+                IBV_QP_EX_WITH_SEND | IBV_QP_EX_WITH_SEND_WITH_IMM |
+                IBV_QP_EX_WITH_RDMA_READ | IBV_QP_EX_WITH_ATOMIC_CMP_AND_SWP |
+                IBV_QP_EX_WITH_ATOMIC_FETCH_AND_ADD | IBV_QP_EX_WITH_LOCAL_INV |
+                IBV_QP_EX_WITH_SEND_WITH_INV,
+};
+
 static int check_qp_create_mask(struct hns_roce_context *ctx,
 				struct ibv_qp_init_attr_ex *attr)
 {
@@ -1080,6 +1089,10 @@ static int check_qp_create_mask(struct hns_roce_context *ctx,
 	if (!check_comp_mask(attr->comp_mask, CREATE_QP_SUP_COMP_MASK))
 		return EOPNOTSUPP;
 
+	if (attr->comp_mask & IBV_QP_INIT_ATTR_SEND_OPS_FLAGS &&
+	    !check_comp_mask(attr->send_ops_flags, SEND_OPS_FLAG_MASK))
+		return -EOPNOTSUPP;
+
 	switch (attr->qp_type) {
 	case IBV_QPT_UD:
 		if (hr_dev->hw_version == HNS_ROCE_HW_VER2)
@@ -1311,9 +1324,21 @@ static int calc_qp_buff_size(struct hns_roce_device *hr_dev,
 	return 0;
 }
 
-static inline bool check_qp_support_dca(bool pool_en, enum ibv_qp_type qp_type)
+static inline bool check_qp_support_dca(struct hns_roce_dca_ctx *dca_ctx,
+					struct ibv_qp_init_attr_ex *attr,
+					struct hnsdv_qp_init_attr *hns_attr)
 {
-	if (pool_en && (qp_type == IBV_QPT_RC || qp_type == IBV_QPT_XRC_SEND))
+	/* DCA pool disable */
+	if (!dca_ctx->unit_size)
+		return false;
+
+	/* Unsupport type */
+	if (attr->qp_type != IBV_QPT_RC && attr->qp_type != IBV_QPT_XRC_SEND)
+		return false;
+
+	if (hns_attr &&
+	    (hns_attr->comp_mask & HNSDV_QP_INIT_ATTR_MASK_QP_CREATE_FLAGS) &&
+	    (hns_attr->create_flags & HNSDV_QP_CREATE_ENABLE_DCA_MODE))
 		return true;
 
 	return false;
@@ -1331,6 +1356,7 @@ static void qp_free_wqe(struct hns_roce_qp *qp)
 }
 
 static int qp_alloc_wqe(struct ibv_qp_init_attr_ex *attr,
+			struct hnsdv_qp_init_attr *hns_attr,
 			struct hns_roce_qp *qp, struct hns_roce_context *ctx)
 {
 	struct hns_roce_device *hr_dev = to_hr_dev(ctx->ibv_ctx.context.device);
@@ -1354,7 +1380,8 @@ static int qp_alloc_wqe(struct ibv_qp_init_attr_ex *attr,
 			goto err_alloc;
 	}
 
-	if (check_qp_support_dca(ctx->dca_ctx.max_size != 0, attr->qp_type)) {
+	if (check_qp_support_dca(&ctx->dca_ctx, attr, hns_attr) &&
+	    ctx->dca_ctx.max_size > 0) {
 		/* when DCA is enabled, use a buffer list to store page addr */
 		qp->buf.buf = NULL;
 		qp->dca_wqe.max_cnt = hr_hw_page_count(qp->buf_size);
@@ -1362,6 +1389,7 @@ static int qp_alloc_wqe(struct ibv_qp_init_attr_ex *attr,
 		qp->dca_wqe.bufs = calloc(qp->dca_wqe.max_cnt, sizeof(void *));
 		if (!qp->dca_wqe.bufs)
 			goto err_alloc;
+		verbs_debug(&ctx->ibv_ctx, "alloc DCA buf.\n");
 	} else {
 		if (hns_roce_alloc_buf(&qp->buf, qp->buf_size,
 				       HNS_HW_PAGE_SIZE))
@@ -1651,12 +1679,13 @@ void hns_roce_free_qp_buf(struct hns_roce_qp *qp, struct hns_roce_context *ctx)
 }
 
 static int hns_roce_alloc_qp_buf(struct ibv_qp_init_attr_ex *attr,
+				 struct hnsdv_qp_init_attr *hns_attr,
 				 struct hns_roce_qp *qp,
 				 struct hns_roce_context *ctx)
 {
 	int ret;
 
-	ret = qp_alloc_wqe(attr, qp, ctx);
+	ret = qp_alloc_wqe(attr, hns_attr, qp, ctx);
 	if (ret)
 		return ret;
 
@@ -1731,7 +1760,7 @@ static struct ibv_qp *create_qp(struct ibv_context *ibv_ctx,
 	if (ret)
 		goto err_spinlock;
 
-	ret = hns_roce_alloc_qp_buf(attr, qp, context);
+	ret = hns_roce_alloc_qp_buf(attr, hns_attr, qp, context);
 	if (ret)
 		goto err_buf;
 
diff --git a/providers/hns/hnsdv.h b/providers/hns/hnsdv.h
index 451b26e..68bf001 100644
--- a/providers/hns/hnsdv.h
+++ b/providers/hns/hnsdv.h
@@ -22,17 +22,42 @@ enum hnsdv_qp_congest_ctrl_type {
 	HNSDV_QP_CREATE_ENABLE_DIP = 1 << 3,
 };
 
+enum hnsdv_qp_create_flags {
+	HNSDV_QP_CREATE_ENABLE_DCA_MODE = 1 << 0,
+};
+
+enum hnsdv_context_comp_mask {
+	HNSDV_CONTEXT_MASK_DCA_PRIME_QPS = 1 << 0,
+	HNSDV_CONTEXT_MASK_DCA_UNIT_SIZE = 1 << 1,
+	HNSDV_CONTEXT_MASK_DCA_MAX_SIZE = 1 << 2,
+	HNSDV_CONTEXT_MASK_DCA_MIN_SIZE = 1 << 3,
+};
+
 enum hnsdv_qp_init_attr_mask {
+	HNSDV_QP_INIT_ATTR_MASK_QP_CREATE_FLAGS	= 1 << 0,
 	HNSDV_QP_INIT_ATTR_MASK_QP_CONGEST_TYPE	= 1 << 1,
 };
 
+struct hnsdv_context_attr {
+	uint64_t flags; /* Use enum hnsdv_context_attr_flags */
+	uint64_t comp_mask; /* Use enum hnsdv_context_comp_mask */
+	uint32_t dca_prime_qps;
+	uint32_t dca_unit_size;
+	uint64_t dca_max_size;
+	uint64_t dca_min_size;
+};
+
 struct hnsdv_qp_init_attr {
 	uint64_t comp_mask; /* Use enum hnsdv_qp_init_attr_mask */
-	uint32_t create_flags;
+	uint32_t create_flags; /* Use enum hnsdv_qp_create_flags */
 	uint8_t congest_type; /* Use enum hnsdv_qp_congest_ctrl_type */
 	uint8_t reserved[3];
 };
 
+enum hnsdv_context_attr_flags {
+	HNSDV_CONTEXT_FLAGS_DCA = 1 << 0,
+};
+
 enum hnsdv_query_context_comp_mask {
 	HNSDV_CONTEXT_MASK_CONGEST_TYPE = 1 << 0,
 };
@@ -50,6 +75,8 @@ int hnsdv_query_device(struct ibv_context *ctx_in,
 struct ibv_qp *hnsdv_create_qp(struct ibv_context *context,
 			       struct ibv_qp_init_attr_ex *qp_attr,
 			       struct hnsdv_qp_init_attr *hns_qp_attr);
+struct ibv_context *hnsdv_open_device(struct ibv_device *device,
+				      struct hnsdv_context_attr *attr);
 
 #ifdef __cplusplus
 }
diff --git a/providers/hns/libhns.map b/providers/hns/libhns.map
index e9bf417..a955346 100644
--- a/providers/hns/libhns.map
+++ b/providers/hns/libhns.map
@@ -5,5 +5,6 @@ HNS_1.0 {
 		hnsdv_is_supported;
 		hnsdv_create_qp;
 		hnsdv_query_device;
+		hnsdv_open_device;
 	local: *;
 };
-- 
2.33.0

