From 7623e3ac166fbe81455a60970bc1c6937ac58374 Mon Sep 17 00:00:00 2001 From: JangShui Yang <1175135535@qq.com> Date: Tue, 19 Nov 2024 11:51:09 +0800 Subject: [PATCH] libwd: update the source code --- ...-support-for-the-DAE-algorithm-layer.patch | 2146 +++++++++++++++++ 0018-drv-dae-support-hashagg-algorithm.patch | 1771 ++++++++++++++ ...d_xxx_init2_-instead-of-wd_xxx_init2.patch | 235 ++ ...x-sm4-CE-driver-initialization-error.patch | 90 + ...-queue-configuration-parameter-error.patch | 169 ++ 0022-acc-uadk-fix-double-free-error.patch | 29 + ...adk-fix-the-null-pointer-check-error.patch | 119 + 0024-uadk-add-null-pointer-check.patch | 172 ++ ...e-main-function-to-testsuit.c-and-rm.patch | 1060 ++++++++ ...k-fix-for-sec-software-compute-check.patch | 42 + 0027-uadk-fix-some-code-bugs.patch | 85 + ...her-add-length-check-of-CBC_CTS-mode.patch | 75 + 0029-uadk-fix-for-cipher-update-iv.patch | 121 + 0030-uadk-v1-fix-for-cipher-update-iv.patch | 143 ++ ...ce-the-print-rating-of-specific-logs.patch | 126 + ...eplace-wd_spinlock-to-pthread_spin_l.patch | 288 +++ ...heck-queue-status-before-sending-doo.patch | 194 ++ ...k-hardware-status-when-BD-not-proces.patch | 39 + ...ueue-status-before-sending-doorbells.patch | 215 ++ ...ace-wd_spinlock-to-pthread_spin_lock.patch | 289 +++ 0037-uadk-fix-for-hashagg-include-files.patch | 55 + 0038-uadk-fix-code-compatibility-issue.patch | 57 + 0039-Update-wd_design.md.patch | 25 + libwd.spec | 28 +- 24 files changed, 7572 insertions(+), 1 deletion(-) create mode 100644 0017-uadk-support-for-the-DAE-algorithm-layer.patch create mode 100644 0018-drv-dae-support-hashagg-algorithm.patch create mode 100644 0019-uadk_tool-use-wd_xxx_init2_-instead-of-wd_xxx_init2.patch create mode 100644 0020-uadk-isa-ce-fix-sm4-CE-driver-initialization-error.patch create mode 100644 0021-acc-uadk-fix-queue-configuration-parameter-error.patch create mode 100644 0022-acc-uadk-fix-double-free-error.patch create mode 100644 0023-uadk-fix-the-null-pointer-check-error.patch create mode 100644 0024-uadk-add-null-pointer-check.patch create mode 100644 0025-uadk-test-move-main-function-to-testsuit.c-and-rm.patch create mode 100644 0026-uadk-fix-for-sec-software-compute-check.patch create mode 100644 0027-uadk-fix-some-code-bugs.patch create mode 100644 0028-uadk-cipher-add-length-check-of-CBC_CTS-mode.patch create mode 100644 0029-uadk-fix-for-cipher-update-iv.patch create mode 100644 0030-uadk-v1-fix-for-cipher-update-iv.patch create mode 100644 0031-uadk-reduce-the-print-rating-of-specific-logs.patch create mode 100644 0032-Revert-uadk-v1-replace-wd_spinlock-to-pthread_spin_l.patch create mode 100644 0033-Revert-uadk-v1-check-queue-status-before-sending-doo.patch create mode 100644 0034-uadk-v1-not-check-hardware-status-when-BD-not-proces.patch create mode 100644 0035-uadk-v1-check-queue-status-before-sending-doorbells.patch create mode 100644 0036-uadk-v1-replace-wd_spinlock-to-pthread_spin_lock.patch create mode 100644 0037-uadk-fix-for-hashagg-include-files.patch create mode 100644 0038-uadk-fix-code-compatibility-issue.patch create mode 100644 0039-Update-wd_design.md.patch diff --git a/0017-uadk-support-for-the-DAE-algorithm-layer.patch b/0017-uadk-support-for-the-DAE-algorithm-layer.patch new file mode 100644 index 0000000..c8c0b88 --- /dev/null +++ b/0017-uadk-support-for-the-DAE-algorithm-layer.patch @@ -0,0 +1,2146 @@ +From 8b5f8566984a839036ea04511ae805c28d3c9424 Mon Sep 17 00:00:00 2001 +From: Wenkai Lin +Date: Tue, 20 Aug 2024 09:41:23 +0800 +Subject: [PATCH 17/39] uadk: support for the DAE algorithm layer + +Data analysis engine (DAE) uses dedicated acceleration hardware +to perform operations on SQL statements, currently, the sum and +count operations are supported. + +Signed-off-by: Wenkai Lin +Signed-off-by: Qi Tao +--- + Makefile.am | 15 +- + include/drv/wd_agg_drv.h | 56 ++ + include/wd_agg.h | 233 ++++++ + include/wd_alg.h | 4 + + include/wd_dae.h | 80 ++ + include/wd_util.h | 1 + + libwd_dae.map | 21 + + wd_agg.c | 1572 ++++++++++++++++++++++++++++++++++++++ + wd_util.c | 2 + + 9 files changed, 1982 insertions(+), 2 deletions(-) + create mode 100644 include/drv/wd_agg_drv.h + create mode 100644 include/wd_agg.h + create mode 100644 include/wd_dae.h + create mode 100644 libwd_dae.map + create mode 100644 wd_agg.c + +diff --git a/Makefile.am b/Makefile.am +index 5742f28..52af71b 100644 +--- a/Makefile.am ++++ b/Makefile.am +@@ -36,12 +36,12 @@ pkginclude_HEADERS = include/wd.h include/wd_cipher.h include/wd_aead.h \ + include/wd_comp.h include/wd_dh.h include/wd_digest.h \ + include/wd_rsa.h include/uacce.h include/wd_alg_common.h \ + include/wd_ecc.h include/wd_sched.h include/wd_alg.h \ +- include/wd_zlibwrapper.h ++ include/wd_zlibwrapper.h include/wd_dae.h include/wd_agg.h + + nobase_pkginclude_HEADERS = v1/wd.h v1/wd_cipher.h v1/wd_aead.h v1/uacce.h v1/wd_dh.h \ + v1/wd_digest.h v1/wd_rsa.h v1/wd_bmm.h + +-lib_LTLIBRARIES=libwd.la libwd_comp.la libwd_crypto.la ++lib_LTLIBRARIES=libwd.la libwd_comp.la libwd_crypto.la libwd_dae.la + + uadk_driversdir=$(libdir)/uadk + uadk_drivers_LTLIBRARIES=libhisi_sec.la libhisi_hpre.la libhisi_zip.la \ +@@ -66,6 +66,9 @@ libwd_la_SOURCES=wd.c wd_mempool.c wd.h wd_alg.c wd_alg.h \ + v1/drv/hisi_sec_udrv.c v1/drv/hisi_sec_udrv.h \ + v1/drv/hisi_rng_udrv.c v1/drv/hisi_rng_udrv.h + ++libwd_dae_la_SOURCES=wd_dae.h wd_agg.h wd_agg_drv.h wd_agg.c \ ++ wd_util.c wd_util.h wd_sched.c wd_sched.h wd.c wd.h ++ + libwd_comp_la_SOURCES=wd_comp.c wd_comp.h wd_comp_drv.h wd_util.c wd_util.h \ + wd_sched.c wd_sched.h wd.c wd.h wd_zlibwrapper.c + +@@ -112,6 +115,9 @@ libhisi_zip_la_LIBADD = -ldl + libwd_crypto_la_LIBADD = $(libwd_la_OBJECTS) -ldl -lnuma + libwd_crypto_la_DEPENDENCIES = libwd.la + ++libwd_dae_la_LIBADD = $(libwd_la_OBJECTS) -ldl -lnuma ++libwd_dae_la_DEPENDENCIES = libwd.la ++ + libhisi_sec_la_LIBADD = $(libwd_la_OBJECTS) $(libwd_crypto_la_OBJECTS) + libhisi_sec_la_DEPENDENCIES = libwd.la libwd_crypto.la + +@@ -127,6 +133,7 @@ libisa_sve_la_DEPENDENCIES = libwd.la libwd_crypto.la + else + UADK_WD_SYMBOL= -Wl,--version-script,$(top_srcdir)/libwd.map + UADK_CRYPTO_SYMBOL= -Wl,--version-script,$(top_srcdir)/libwd_crypto.map ++UADK_DAE_SYMBOL= -Wl,--version-script,$(top_srcdir)/libwd_dae.map + UADK_COMP_SYMBOL= -Wl,--version-script,$(top_srcdir)/libwd_comp.map + UADK_V1_SYMBOL= -Wl,--version-script,$(top_srcdir)/v1/libwd.map + +@@ -141,6 +148,10 @@ libwd_crypto_la_LIBADD= -lwd -ldl -lnuma + libwd_crypto_la_LDFLAGS=$(UADK_VERSION) $(UADK_CRYPTO_SYMBOL) -lpthread + libwd_crypto_la_DEPENDENCIES= libwd.la + ++libwd_dae_la_LIBADD= -lwd -ldl -lnuma ++libwd_dae_la_LDFLAGS=$(UADK_VERSION) $(UADK_DAE_SYMBOL) ++libwd_dae_la_DEPENDENCIES= libwd.la ++ + libhisi_zip_la_LIBADD= -lwd -ldl -lwd_comp + libhisi_zip_la_LDFLAGS=$(UADK_VERSION) + libhisi_zip_la_DEPENDENCIES= libwd.la libwd_comp.la +diff --git a/include/drv/wd_agg_drv.h b/include/drv/wd_agg_drv.h +new file mode 100644 +index 0000000..cf99a70 +--- /dev/null ++++ b/include/drv/wd_agg_drv.h +@@ -0,0 +1,56 @@ ++/* SPDX-License-Identifier: Apache-2.0 */ ++/* ++ * Copyright 2024 Huawei Technologies Co.,Ltd. All rights reserved. ++ */ ++ ++#ifndef __WD_AGG_DRV_H ++#define __WD_AGG_DRV_H ++ ++#include ++#include "wd_agg.h" ++ ++#ifdef __cplusplus ++extern "C" { ++#endif ++ ++enum wd_agg_strm_pos { ++ WD_AGG_STREAM_INPUT, ++ WD_AGG_STREAM_OUTPUT, ++ WD_AGG_REHASH_INPUT, ++ WD_AGG_REHASH_OUTPUT, ++}; ++ ++struct wd_agg_msg { ++ __u32 tag; ++ __u32 key_cols_num; ++ __u32 agg_cols_num; ++ __u32 result; ++ __u32 in_row_count; ++ __u32 out_row_count; ++ __u32 row_count; ++ enum wd_agg_strm_pos pos; ++ enum wd_dae_data_type count_all_data_type; ++ bool output_done; ++ bool is_count_all; ++ struct wd_agg_req req; ++ struct wd_dae_charset charset_info; ++ struct wd_dae_hash_table hash_table; ++ struct wd_key_col_info *key_cols_info; ++ struct wd_agg_col_info *agg_cols_info; ++ void *priv; ++}; ++ ++struct wd_agg_ops { ++ int (*get_row_size)(void *priv); ++ int (*sess_init)(struct wd_agg_sess_setup *setup, void **priv); ++ void (*sess_uninit)(void *priv); ++ int (*hash_table_init)(struct wd_dae_hash_table *hash_table, void *priv); ++}; ++ ++struct wd_agg_msg *wd_agg_get_msg(__u32 idx, __u32 tag); ++ ++#ifdef __cplusplus ++} ++#endif ++ ++#endif /* __WD_AGG_DRV_H */ +diff --git a/include/wd_agg.h b/include/wd_agg.h +new file mode 100644 +index 0000000..e37023b +--- /dev/null ++++ b/include/wd_agg.h +@@ -0,0 +1,233 @@ ++/* SPDX-License-Identifier: Apache-2.0 */ ++/* ++ * Copyright 2024 Huawei Technologies Co.,Ltd. All rights reserved. ++ */ ++ ++#ifndef __WD_AGG_H ++#define __WD_AGG_H ++ ++#include ++#include ++#include "wd_util.h" ++#include "wd_dae.h" ++ ++#ifdef __cplusplus ++extern "C" { ++#endif ++ ++/** ++ * wd_agg_alg - Aggregation operation type. ++ */ ++enum wd_agg_alg { ++ WD_AGG_SUM, ++ WD_AGG_COUNT, ++ WD_AGG_ALG_TYPE_MAX, ++}; ++ ++/** ++ * wd_agg_task_error_type - Aggregation task error type. ++ */ ++enum wd_agg_task_error_type { ++ WD_AGG_TASK_DONE, ++ WD_AGG_IN_EPARA, ++ WD_AGG_NEED_REHASH, ++ WD_AGG_SUM_OVERFLOW, ++ WD_AGG_INVALID_HASH_TABLE, ++ WD_AGG_INVALID_VARCHAR, ++ WD_AGG_PARSE_ERROR, ++ WD_AGG_BUS_ERROR, ++}; ++ ++/** ++ * wd_key_col_info - Key column information. ++ * @col_data_info: For CHAR, it is size of data, at least 1B. ++ * For VARCHAR, it is size of data in hash table, 0 means the max size. ++ * For DECIMAL, it is precision of data, high 8 bit: decimal part precision, ++ * low 8 bit: the whole data precision. ++ * @input_data_type: Key column data type. ++ */ ++struct wd_key_col_info { ++ __u16 col_data_info; ++ enum wd_dae_data_type input_data_type; ++}; ++ ++/** ++ * wd_agg_col_info - Agg column information. ++ * @col_alg_num: Number of aggregation operations for this column. ++ * @col_data_info: For CHAR, it is size of data, at least 1B. ++ * For VARCHAR, it is size of data in hash table, 0 means the max size. ++ * For DECIMAL, it is precision of data, high 8 bit: decimal part precision, ++ * low 8 bit: the whole data precision. ++ * @input_data_type: Agg column data type. ++ * @output_data_types: Output agg column data type. ++ * @output_col_algs: Output agg column operation type, the sequence must be ++ * the same as that of output_data_types. ++ */ ++struct wd_agg_col_info { ++ __u32 col_alg_num; ++ __u16 col_data_info; ++ enum wd_dae_data_type input_data_type; ++ enum wd_dae_data_type output_data_types[WD_AGG_ALG_TYPE_MAX]; ++ enum wd_agg_alg output_col_algs[WD_AGG_ALG_TYPE_MAX]; ++}; ++ ++/** ++ * wd_agg_sess_setup - Agg session setup information. ++ * @key_cols_num: Number of key columns. ++ * @key_cols_info: Information of key columns. ++ * @agg_cols_num: Number of agg columns. ++ * @agg_cols_info: Information of agg columns. ++ * @is_count_all: Whether to perform the count(*) operation. ++ * @count_all_data_types: Output agg column data type. ++ * @charset_info: Charset information ++ * @sched_param: Parameters of the scheduling policy, ++ * usually allocated according to struct sched_params. ++ */ ++struct wd_agg_sess_setup { ++ __u32 key_cols_num; ++ struct wd_key_col_info *key_cols_info; ++ __u32 agg_cols_num; ++ struct wd_agg_col_info *agg_cols_info; ++ bool is_count_all; ++ enum wd_dae_data_type count_all_data_type; ++ struct wd_dae_charset charset_info; ++ void *sched_param; ++}; ++ ++struct wd_agg_req; ++typedef void *wd_alg_agg_cb_t(struct wd_agg_req *req, void *cb_param); ++ ++/** ++ * wd_agg_req - Aggregation operation request. ++ * @key_cols: Address of key columns. ++ * @out_key_cols: Address of output key columns. ++ * @agg_cols: Address of agg columns. ++ * @out_agg_cols: Address of output agg columns. If count(*) exist, ++ * count(*) output address must be the last column. ++ * @key_cols_num: Number of key columns. ++ * @out_key_cols_num: Number of output key columns. ++ * @agg_cols_num: Number of agg columns. ++ * @out_agg_cols_num: Number of output agg columns. ++ * @in_row_count: Row count of input column. ++ * @out_row_count: Expected row count of output column. ++ * @real_in_row_count: Row count of input data that has been processed. ++ * @real_out_row_count: Real row count of output column. ++ * @cb: Callback function. ++ * @cb_param: Parameters of the callback function. ++ * @sum_overflow_cols: If sum result is overflow, the value will be true. ++ * If the pointer is null, only the state is set to WD_AGG_SUM_OVERFLOW. ++ * @state: Error information written back by the hardware. ++ * @output_done: If all data in hash table has been output. ++ * @priv: Private data from user(reserved). ++ */ ++struct wd_agg_req { ++ struct wd_dae_col_addr *key_cols; ++ struct wd_dae_col_addr *out_key_cols; ++ struct wd_dae_col_addr *agg_cols; ++ struct wd_dae_col_addr *out_agg_cols; ++ __u32 key_cols_num; ++ __u32 out_key_cols_num; ++ __u32 agg_cols_num; ++ __u32 out_agg_cols_num; ++ __u32 in_row_count; ++ __u32 out_row_count; ++ __u32 real_in_row_count; ++ __u32 real_out_row_count; ++ wd_alg_agg_cb_t *cb; ++ void *cb_param; ++ __u8 *sum_overflow_cols; ++ enum wd_agg_task_error_type state; ++ bool output_done; ++ void *priv; ++}; ++ ++/** ++ * wd_agg_init() - A simplify interface to initializate uadk ++ * encryption and decryption. This interface keeps most functions of ++ * wd_agg_init(). Users just need to descripe the deployment of ++ * business scenarios. Then the initialization will request appropriate ++ * resources to support the business scenarios. ++ * To make the initializate simpler, ctx_params support set NULL. ++ * And then the function will set them as driver's default. ++ * ++ * @alg: The algorithm users want to use. ++ * @sched_type: The scheduling type users want to use. ++ * @task_type: Task types, including soft computing, hardware and hybrid computing. ++ * @ctx_params: The ctxs resources users want to use. Include per operation ++ * type ctx numbers and business process run numa. ++ * ++ * Return 0 if succeed and others if fail. ++ */ ++int wd_agg_init(char *alg, __u32 sched_type, int task_type, struct wd_ctx_params *ctx_params); ++ ++/** ++ * wd_agg_uninit() - Uninitialise ctx configuration and scheduler. ++ */ ++void wd_agg_uninit(void); ++ ++/** ++ * wd_agg_alloc_sess() - Allocate a wd agg session ++ * @setup: Parameters to setup this session. ++ * ++ * Return 0 if fail and others if succeed. ++ */ ++handle_t wd_agg_alloc_sess(struct wd_agg_sess_setup *setup); ++ ++/** ++ * wd_agg_free_sess() - Free the wd agg session ++ * @sess: The session need to be freed. ++ */ ++void wd_agg_free_sess(handle_t h_sess); ++ ++/** ++ * wd_agg_set_hash_table() - Set hash table to the wd agg session ++ * @sess, Session to be initialized. ++ * @info, Hash table information to set. ++ * ++ * Return 0 if succeed and others if fail. ++ */ ++int wd_agg_set_hash_table(handle_t h_sess, struct wd_dae_hash_table *info); ++ ++/** ++ * wd_agg_add_input_sync()/wd_agg_get_output_sync() - Input or output agg operation ++ * @sess: Wd agg session ++ * @req: Operational data. ++ * ++ * Return 0 if succeed and others if fail. ++ */ ++int wd_agg_add_input_sync(handle_t h_sess, struct wd_agg_req *req); ++int wd_agg_get_output_sync(handle_t h_sess, struct wd_agg_req *req); ++int wd_agg_add_input_async(handle_t h_sess, struct wd_agg_req *req); ++int wd_agg_get_output_async(handle_t h_sess, struct wd_agg_req *req); ++ ++/** ++ * wd_agg_rehash_sync - Rehash operation, only the synchronous mode is supported. ++ * @sess: Wd agg session ++ * @req: Operational data. ++ * ++ * Return 0 if succeed and others if fail. ++ */ ++int wd_agg_rehash_sync(handle_t h_sess, struct wd_agg_req *req); ++ ++/** ++ * wd_agg_poll() - Poll finished request. ++ * This function will call poll_policy function which is registered to wd_agg ++ * by user. ++ * ++ * Return 0 if succeed and others if fail. ++ */ ++int wd_agg_poll(__u32 expt, __u32 *count); ++ ++/** ++ * wd_agg_get_table_rowsize - Get the hash table's row size. ++ * @h_sess: Wd agg session handler. ++ * ++ * Return negative value if fail and others if succeed. ++ */ ++int wd_agg_get_table_rowsize(handle_t h_sess); ++ ++#ifdef __cplusplus ++} ++#endif ++ ++#endif /* __WD_AGG_H */ +diff --git a/include/wd_alg.h b/include/wd_alg.h +index 1735896..aba855d 100644 +--- a/include/wd_alg.h ++++ b/include/wd_alg.h +@@ -95,6 +95,7 @@ enum alg_dev_type { + * result of the task packets from the hardware device. + * @get_usage: callback interface used to obtain the + * utilization rate of devices. ++ * @get_extend_ops: callback interface to get private operation of drivers. + */ + struct wd_alg_driver { + const char *drv_name; +@@ -111,6 +112,7 @@ struct wd_alg_driver { + int (*send)(struct wd_alg_driver *drv, handle_t ctx, void *drv_msg); + int (*recv)(struct wd_alg_driver *drv, handle_t ctx, void *drv_msg); + int (*get_usage)(void *param); ++ int (*get_extend_ops)(void *ops); + }; + + inline int wd_alg_driver_init(struct wd_alg_driver *drv, void *conf) +@@ -201,10 +203,12 @@ struct wd_alg_list *wd_get_alg_head(void); + void hisi_sec2_probe(void); + void hisi_hpre_probe(void); + void hisi_zip_probe(void); ++void hisi_dae_probe(void); + + void hisi_sec2_remove(void); + void hisi_hpre_remove(void); + void hisi_zip_remove(void); ++void hisi_dae_remove(void); + + #endif + +diff --git a/include/wd_dae.h b/include/wd_dae.h +new file mode 100644 +index 0000000..5f22d90 +--- /dev/null ++++ b/include/wd_dae.h +@@ -0,0 +1,80 @@ ++/* SPDX-License-Identifier: Apache-2.0 */ ++/* ++ * Copyright 2024 Huawei Technologies Co.,Ltd. All rights reserved. ++ */ ++ ++#ifndef __WD_DAE_H ++#define __WD_DAE_H ++ ++#include ++#include ++#include ++#include "wd.h" ++ ++#ifdef __cplusplus ++extern "C" { ++#endif ++ ++/** ++ * wd_dae_data_type - Data type of DAE ++ */ ++enum wd_dae_data_type { ++ WD_DAE_DATE, ++ WD_DAE_INT, ++ WD_DAE_LONG, ++ WD_DAE_SHORT_DECIMAL, ++ WD_DAE_LONG_DECIMAL, ++ WD_DAE_CHAR, ++ WD_DAE_VARCHAR, ++ WD_DAE_DATA_TYPE_MAX, ++}; ++ ++/** ++ * wd_dae_charset - Charset information of DAE ++ */ ++struct wd_dae_charset { ++ bool binary_format; ++ bool space; ++ bool subwoofer; ++}; ++ ++/** ++ * wd_dae_col_addr - Column information of DAE. ++ * @empty: 0 indicates that the data is valid, 1 indicate invalid. ++ * @value: Indicates the value of the data. ++ * @offset: Indicates the length of the string data, only for VARCHAR. ++ * @empty_size: The value is equal to row_count * sizeof(__u8). ++ * @value_size: The value is equal to row_count * sizeof(data_type). ++ * @offset_size: The value is equal to (row_count + 1) * sizeof(__u32). ++ */ ++struct wd_dae_col_addr { ++ __u8 *empty; ++ void *value; ++ __u32 *offset; ++ __u64 empty_size; ++ __u64 value_size; ++ __u64 offset_size; ++}; ++ ++/** ++ * wd_dae_hash_table - Hash table information of DAE. ++ * @std_table: Address of standard hash table. ++ * @ext_table: Address of external hash table. ++ * @std_table_row_num: Row number of standard hash table. ++ * @ext_table_row_num: Row number of external hash table. ++ * @table_row_size: Row size of hash table, user should get it ++ * from wd_agg_get_table_rowsize. ++ */ ++struct wd_dae_hash_table { ++ void *std_table; ++ void *ext_table; ++ __u32 std_table_row_num; ++ __u32 ext_table_row_num; ++ __u32 table_row_size; ++}; ++ ++#ifdef __cplusplus ++} ++#endif ++ ++#endif /* __WD_DAE_H */ +diff --git a/include/wd_util.h b/include/wd_util.h +index 1c4af0b..dc1e41f 100644 +--- a/include/wd_util.h ++++ b/include/wd_util.h +@@ -41,6 +41,7 @@ enum wd_type { + WD_RSA_TYPE, + WD_DH_TYPE, + WD_ECC_TYPE, ++ WD_AGG_TYPE, + WD_TYPE_MAX, + }; + +diff --git a/libwd_dae.map b/libwd_dae.map +new file mode 100644 +index 0000000..4c51b85 +--- /dev/null ++++ b/libwd_dae.map +@@ -0,0 +1,21 @@ ++UADK_CRYPTO_2.0 { ++global: ++ wd_agg_alloc_sess; ++ wd_agg_free_sess; ++ wd_agg_get_table_rowsize; ++ wd_agg_set_hash_table; ++ wd_agg_init; ++ wd_agg_uninit; ++ wd_agg_add_input_sync; ++ wd_agg_add_input_async; ++ wd_agg_get_output_sync; ++ wd_agg_get_output_async; ++ wd_agg_rehash_sync; ++ wd_agg_get_msg; ++ wd_agg_poll; ++ ++ wd_sched_rr_instance; ++ wd_sched_rr_alloc; ++ wd_sched_rr_release; ++local: *; ++}; +diff --git a/wd_agg.c b/wd_agg.c +new file mode 100644 +index 0000000..e493bb8 +--- /dev/null ++++ b/wd_agg.c +@@ -0,0 +1,1572 @@ ++/* SPDX-License-Identifier: Apache-2.0 */ ++/* ++ * Copyright 2024 Huawei Technologies Co.,Ltd. All rights reserved. ++ */ ++ ++#include ++#include ++#include ++#include ++#include "include/drv/wd_agg_drv.h" ++#include "wd_agg.h" ++ ++#define DECIMAL_PRECISION_OFFSET 8 ++#define DAE_INT_SIZE 4 ++#define DAE_LONG_SIZE 8 ++#define DAE_LONG_DECIMAL_SIZE 16 ++ ++/* Sum of the max row number of standard and external hash table */ ++#define MAX_HASH_TABLE_ROW_NUM 0x1FFFFFFFE ++ ++enum wd_agg_sess_state { ++ WD_AGG_SESS_UNINIT, /* Uninit session */ ++ WD_AGG_SESS_INIT, /* Hash table has been set */ ++ WD_AGG_SESS_INPUT, /* Input stage has started */ ++ WD_AGG_SESS_RESET, /* Hash table has been reset */ ++ WD_AGG_SESS_REHASH, /* Rehash stage has started */ ++ WD_AGG_SESS_OUTPUT, /* Output stage has started */ ++}; ++ ++struct wd_agg_setting { ++ enum wd_status status; ++ struct wd_ctx_config_internal config; ++ struct wd_sched sched; ++ struct wd_async_msg_pool pool; ++ struct wd_alg_driver *driver; ++ void *priv; ++ void *dlhandle; ++ void *dlh_list; ++} wd_agg_setting; ++ ++struct wd_agg_sess_key_conf { ++ __u32 cols_num; ++ __u64 *data_size; ++ struct wd_key_col_info *cols_info; ++}; ++ ++struct wd_agg_sess_agg_conf { ++ __u32 cols_num; ++ __u32 out_cols_num; ++ __u64 *data_size; ++ __u64 *out_data_size; ++ struct wd_agg_col_info *cols_info; ++ bool is_count_all; ++ enum wd_dae_data_type count_all_data_type; ++}; ++ ++struct wd_agg_sess { ++ char *alg_name; ++ wd_dev_mask_t *dev_mask; ++ struct wd_alg_agg *drv; ++ void *priv; ++ void *sched_key; ++ enum wd_agg_sess_state state; ++ struct wd_agg_ops ops; ++ struct wd_agg_sess_key_conf key_conf; ++ struct wd_agg_sess_agg_conf agg_conf; ++ struct wd_dae_charset charset_info; ++ struct wd_dae_hash_table hash_table; ++ struct wd_dae_hash_table rehash_table; ++}; ++ ++static char *wd_agg_alg_name = "hashagg"; ++static struct wd_init_attrs wd_agg_init_attrs; ++static int wd_agg_poll_ctx(__u32 idx, __u32 expt, __u32 *count); ++ ++static void wd_agg_close_driver(void) ++{ ++#ifndef WD_STATIC_DRV ++ wd_dlclose_drv(wd_agg_setting.dlh_list); ++#else ++ wd_release_drv(wd_agg_setting.driver); ++ hisi_dae_remove(); ++#endif ++} ++ ++static int wd_agg_open_driver(void) ++{ ++#ifndef WD_STATIC_DRV ++ /* ++ * Driver lib file path could set by env param. ++ * then open tham by wd_dlopen_drv() ++ * use NULL means dynamic query path ++ */ ++ wd_agg_setting.dlh_list = wd_dlopen_drv(NULL); ++ if (!wd_agg_setting.dlh_list) { ++ WD_ERR("fail to open driver lib files.\n"); ++ return -WD_EINVAL; ++ } ++#else ++ hisi_dae_probe(); ++#endif ++ return WD_SUCCESS; ++} ++ ++static bool wd_agg_alg_check(const char *alg_name) ++{ ++ if (!strcmp(alg_name, wd_agg_alg_name)) ++ return true; ++ return false; ++} ++ ++static int check_count_out_data_type(enum wd_dae_data_type type) ++{ ++ switch (type) { ++ case WD_DAE_INT: ++ case WD_DAE_LONG: ++ case WD_DAE_SHORT_DECIMAL: ++ case WD_DAE_LONG_DECIMAL: ++ break; ++ case WD_DAE_DATE: ++ case WD_DAE_CHAR: ++ case WD_DAE_VARCHAR: ++ default: ++ return -WD_EINVAL; ++ } ++ ++ return WD_SUCCESS; ++} ++ ++static int check_col_data_info(enum wd_dae_data_type type, __u16 col_data_info) ++{ ++ __u8 all_precision, decimal_precision; ++ ++ switch (type) { ++ case WD_DAE_DATE: ++ case WD_DAE_INT: ++ case WD_DAE_LONG: ++ case WD_DAE_VARCHAR: ++ break; ++ case WD_DAE_SHORT_DECIMAL: ++ case WD_DAE_LONG_DECIMAL: ++ /* High 8 bit: decimal part precision, low 8 bit: the whole data precision */ ++ all_precision = col_data_info; ++ decimal_precision = col_data_info >> DECIMAL_PRECISION_OFFSET; ++ if (!all_precision || decimal_precision > all_precision) { ++ WD_ERR("failed to check agg data precision, all: %u, decimal: %u!\n", ++ all_precision, decimal_precision); ++ return -WD_EINVAL; ++ } ++ break; ++ case WD_DAE_CHAR: ++ if (!col_data_info) { ++ WD_ERR("invalid: agg char length is zero!\n"); ++ return -WD_EINVAL; ++ } ++ break; ++ default: ++ WD_ERR("invalid: agg data type is %d!\n", type); ++ return -WD_EINVAL; ++ } ++ ++ return WD_SUCCESS; ++} ++ ++static int get_col_data_type_size(enum wd_dae_data_type type, __u16 col_data_info, ++ __u64 *col, __u32 idx) ++{ ++ switch (type) { ++ case WD_DAE_DATE: ++ case WD_DAE_INT: ++ col[idx] = DAE_INT_SIZE; ++ break; ++ case WD_DAE_LONG: ++ case WD_DAE_SHORT_DECIMAL: ++ col[idx] = DAE_LONG_SIZE; ++ break; ++ case WD_DAE_LONG_DECIMAL: ++ col[idx] = DAE_LONG_DECIMAL_SIZE; ++ break; ++ case WD_DAE_CHAR: ++ col[idx] = col_data_info; ++ break; ++ case WD_DAE_VARCHAR: ++ col[idx] = 0; ++ break; ++ default: ++ return -WD_EINVAL; ++ } ++ return WD_SUCCESS; ++} ++ ++static int check_key_cols_info(struct wd_agg_sess_setup *setup) ++{ ++ struct wd_key_col_info *info = setup->key_cols_info; ++ __u32 i; ++ int ret; ++ ++ for (i = 0; i < setup->key_cols_num; i++) { ++ ret = check_col_data_info(info[i].input_data_type, info[i].col_data_info); ++ if (ret) { ++ WD_ERR("failed to check agg key col data info! col idx: %u\n", i); ++ return ret; ++ } ++ } ++ ++ return WD_SUCCESS; ++} ++ ++static int check_agg_cols_info(struct wd_agg_sess_setup *setup, __u32 *out_agg_cols_num) ++{ ++ struct wd_agg_col_info *info = setup->agg_cols_info; ++ __u32 alg_cnt[WD_AGG_ALG_TYPE_MAX]; ++ enum wd_agg_alg alg; ++ __u32 i, j, k; ++ int ret; ++ ++ /* When there is only a count(*) task, it returns */ ++ if (!info) ++ return 0; ++ ++ for (i = 0, k = 0; i < setup->agg_cols_num; i++) { ++ ret = check_col_data_info(info[i].input_data_type, info[i].col_data_info); ++ if (ret) { ++ WD_ERR("failed to check agg col data info! col idx: %u\n", i); ++ return ret; ++ } ++ ++ if (!info[i].col_alg_num || info[i].col_alg_num > WD_AGG_ALG_TYPE_MAX) { ++ WD_ERR("failed to check agg col_alg_num: %u! col idx: %u\n", ++ info[i].col_alg_num, i); ++ return -WD_EINVAL; ++ } ++ ++ memset(alg_cnt, 0, sizeof(alg_cnt)); ++ for (j = 0; j < info[i].col_alg_num; j++, k++) { ++ if (info[i].output_data_types[j] >= WD_DAE_DATA_TYPE_MAX) { ++ WD_ERR("failed to check agg col output data type! col idx: %u\n", ++ i); ++ return -WD_EINVAL; ++ } ++ alg = info[i].output_col_algs[j]; ++ if (alg >= WD_AGG_ALG_TYPE_MAX || alg_cnt[alg]) { ++ WD_ERR("invalid agg output col alg type: %d, col idx: %u\n", ++ alg, i); ++ return -WD_EINVAL; ++ } ++ alg_cnt[alg] += 1; ++ } ++ } ++ ++ *out_agg_cols_num += k; ++ ++ return WD_SUCCESS; ++} ++ ++static int wd_agg_check_sess_params(struct wd_agg_sess_setup *setup, __u32 *out_agg_cols_num) ++{ ++ if (!setup) { ++ WD_ERR("invalid: agg sess setup is NULL!\n"); ++ return -WD_EINVAL; ++ } ++ ++ if (!setup->key_cols_num || !setup->key_cols_info) { ++ WD_ERR("invalid: agg key cols is NULL, num: %u\n", setup->key_cols_num); ++ return -WD_EINVAL; ++ } ++ ++ if (!setup->is_count_all) { ++ if (!setup->agg_cols_num || !setup->agg_cols_info) { ++ WD_ERR("invalid: agg input cols is NULL, num: %u\n", setup->agg_cols_num); ++ return -WD_EINVAL; ++ } ++ } else { ++ if (setup->agg_cols_num && !setup->agg_cols_info) { ++ WD_ERR("invalid: agg cols info address is NULL!\n"); ++ return -WD_EINVAL; ++ } ++ if (check_count_out_data_type(setup->count_all_data_type)) { ++ WD_ERR("invalid: agg count all output data type: %u\n", ++ setup->count_all_data_type); ++ return -WD_EINVAL; ++ } ++ *out_agg_cols_num = 1; ++ } ++ ++ if (check_key_cols_info(setup)) { ++ WD_ERR("failed to check agg setup key cols info!\n"); ++ return -WD_EINVAL; ++ } ++ ++ if (check_agg_cols_info(setup, out_agg_cols_num)) { ++ WD_ERR("failed to check agg setup agg cols info!\n"); ++ return -WD_EINVAL; ++ } ++ ++ return WD_SUCCESS; ++} ++ ++static int fill_agg_session(struct wd_agg_sess *sess, struct wd_agg_sess_setup *setup) ++{ ++ __u64 key_size, agg_size, key_data_size, agg_data_size, out_agg_data_size; ++ struct wd_key_col_info *key = setup->key_cols_info; ++ struct wd_agg_col_info *agg = setup->agg_cols_info; ++ __u32 i, j, k; ++ ++ key_size = setup->key_cols_num * sizeof(struct wd_key_col_info); ++ agg_size = setup->agg_cols_num * sizeof(struct wd_agg_col_info); ++ sess->key_conf.cols_info = malloc(key_size); ++ if (!sess->key_conf.cols_info) ++ return -WD_ENOMEM; ++ sess->agg_conf.cols_info = malloc(agg_size); ++ if (!sess->agg_conf.cols_info) ++ goto out_key; ++ ++ memcpy(sess->key_conf.cols_info, key, key_size); ++ memcpy(sess->agg_conf.cols_info, agg, agg_size); ++ ++ key_data_size = setup->key_cols_num * sizeof(__u64); ++ agg_data_size = setup->agg_cols_num * sizeof(__u64); ++ out_agg_data_size = sess->agg_conf.out_cols_num * sizeof(__u64); ++ sess->key_conf.data_size = malloc(key_data_size + agg_data_size + out_agg_data_size); ++ if (!sess->key_conf.data_size) ++ goto out_agg; ++ ++ for (i = 0; i < setup->key_cols_num; i++) ++ (void)get_col_data_type_size(key[i].input_data_type, key[i].col_data_info, ++ sess->key_conf.data_size, i); ++ ++ sess->agg_conf.data_size = sess->key_conf.data_size + setup->key_cols_num; ++ for (i = 0; i < setup->agg_cols_num; i++) ++ (void)get_col_data_type_size(agg[i].input_data_type, agg[i].col_data_info, ++ sess->agg_conf.data_size, i); ++ ++ sess->agg_conf.out_data_size = sess->agg_conf.data_size + setup->agg_cols_num; ++ for (i = 0, k = 0; i < setup->agg_cols_num; i++) ++ for (j = 0; j < agg[i].col_alg_num; j++, k++) ++ (void)get_col_data_type_size(agg[i].output_data_types[j], ++ agg[i].col_data_info, ++ sess->agg_conf.out_data_size, k); ++ ++ sess->key_conf.cols_num = setup->key_cols_num; ++ sess->agg_conf.cols_num = setup->agg_cols_num; ++ sess->agg_conf.is_count_all = setup->is_count_all; ++ sess->agg_conf.count_all_data_type = setup->count_all_data_type; ++ __atomic_store_n(&sess->state, WD_AGG_SESS_UNINIT, __ATOMIC_RELEASE); ++ ++ return WD_SUCCESS; ++out_agg: ++ free(sess->agg_conf.cols_info); ++out_key: ++ free(sess->key_conf.cols_info); ++ return -WD_ENOMEM; ++} ++ ++static int wd_agg_init_sess_priv(struct wd_agg_sess *sess, struct wd_agg_sess_setup *setup) ++{ ++ int ret; ++ ++ if (sess->ops.sess_init) { ++ if (!sess->ops.sess_uninit) { ++ WD_ERR("failed to get session uninit ops!\n"); ++ return -WD_EINVAL; ++ } ++ ret = sess->ops.sess_init(setup, &sess->priv); ++ if (ret) { ++ WD_ERR("failed to init session priv!\n"); ++ return ret; ++ } ++ } ++ ++ if (sess->ops.get_row_size) { ++ ret = sess->ops.get_row_size(sess->priv); ++ if (ret <= 0) { ++ if (sess->ops.sess_uninit) ++ sess->ops.sess_uninit(sess->priv); ++ WD_ERR("failed to get hash table row size: %d!\n", ret); ++ return ret; ++ } ++ sess->hash_table.table_row_size = ret; ++ } ++ ++ return WD_SUCCESS; ++} ++ ++handle_t wd_agg_alloc_sess(struct wd_agg_sess_setup *setup) ++{ ++ __u32 out_agg_cols_num = 0; ++ struct wd_agg_sess *sess; ++ int ret; ++ ++ ret = wd_agg_check_sess_params(setup, &out_agg_cols_num); ++ if (ret) ++ return (handle_t)0; ++ ++ sess = malloc(sizeof(struct wd_agg_sess)); ++ if (!sess) { ++ WD_ERR("failed to alloc agg session memory!\n"); ++ return (handle_t)0; ++ } ++ memset(sess, 0, sizeof(struct wd_agg_sess)); ++ sess->agg_conf.out_cols_num = out_agg_cols_num; ++ ++ sess->alg_name = wd_agg_alg_name; ++ ret = wd_drv_alg_support(sess->alg_name, wd_agg_setting.driver); ++ if (!ret) { ++ WD_ERR("failed to support agg algorithm: %s!\n", sess->alg_name); ++ goto err_sess; ++ } ++ ++ /* Some simple scheduler don't need scheduling parameters */ ++ sess->sched_key = (void *)wd_agg_setting.sched.sched_init( ++ wd_agg_setting.sched.h_sched_ctx, setup->sched_param); ++ if (WD_IS_ERR(sess->sched_key)) { ++ WD_ERR("failed to init agg session schedule key!\n"); ++ goto err_sess; ++ } ++ ++ ret = wd_agg_setting.driver->get_extend_ops(&sess->ops); ++ if (ret) { ++ WD_ERR("failed to get agg extend ops!\n"); ++ goto err_sess; ++ } ++ ++ ret = wd_agg_init_sess_priv(sess, setup); ++ if (ret) ++ goto err_sess; ++ ++ ret = fill_agg_session(sess, setup); ++ if (ret) { ++ WD_ERR("failed to fill agg session!\n"); ++ goto uninit_priv; ++ } ++ ++ return (handle_t)sess; ++ ++uninit_priv: ++ if (sess->ops.sess_uninit) ++ sess->ops.sess_uninit(sess->priv); ++err_sess: ++ if (sess->sched_key) ++ free(sess->sched_key); ++ free(sess); ++ return (handle_t)0; ++} ++ ++void wd_agg_free_sess(handle_t h_sess) ++{ ++ struct wd_agg_sess *sess = (struct wd_agg_sess *)h_sess; ++ ++ if (unlikely(!sess)) { ++ WD_ERR("invalid: agg input sess is NULL!\n"); ++ return; ++ } ++ ++ free(sess->key_conf.cols_info); ++ free(sess->agg_conf.cols_info); ++ free(sess->key_conf.data_size); ++ ++ if (sess->ops.sess_uninit) ++ sess->ops.sess_uninit(sess->priv); ++ if (sess->sched_key) ++ free(sess->sched_key); ++ ++ free(sess); ++} ++ ++int wd_agg_get_table_rowsize(handle_t h_sess) ++{ ++ struct wd_agg_sess *sess = (struct wd_agg_sess *)h_sess; ++ ++ if (unlikely(!sess)) { ++ WD_ERR("invalid: agg input sess is NULL!\n"); ++ return -WD_EINVAL; ++ } ++ ++ if (unlikely(!sess->hash_table.table_row_size)) { ++ WD_ERR("invalid: agg sess hash table row size is 0!\n"); ++ return -WD_EINVAL; ++ } ++ ++ return sess->hash_table.table_row_size; ++} ++ ++static int wd_agg_check_sess_state(struct wd_agg_sess *sess, enum wd_agg_sess_state *expected) ++{ ++ enum wd_agg_sess_state next; ++ int ret; ++ ++ if (sess->hash_table.std_table) { ++ *expected = WD_AGG_SESS_INPUT; ++ next = WD_AGG_SESS_RESET; ++ } else { ++ *expected = WD_AGG_SESS_UNINIT; ++ next = WD_AGG_SESS_INIT; ++ } ++ ++ ret = __atomic_compare_exchange_n(&sess->state, expected, next, ++ true, __ATOMIC_RELAXED, __ATOMIC_RELAXED); ++ if (!ret) { ++ WD_ERR("invalid: agg sess state is %d!\n", *expected); ++ return -WD_EINVAL; ++ } ++ ++ return WD_SUCCESS; ++} ++ ++int wd_agg_set_hash_table(handle_t h_sess, struct wd_dae_hash_table *info) ++{ ++ struct wd_agg_sess *sess = (struct wd_agg_sess *)h_sess; ++ struct wd_dae_hash_table *hash_table, *rehash_table; ++ enum wd_agg_sess_state expected; ++ int ret; ++ ++ if (!sess || !info) { ++ WD_ERR("invalid: agg sess or hash table is NULL!\n"); ++ return -WD_EINVAL; ++ } ++ ++ ret = wd_agg_check_sess_state(sess, &expected); ++ if (ret) ++ return ret; ++ ++ if (info->table_row_size != sess->hash_table.table_row_size) { ++ WD_ERR("invalid: agg hash table row size is not equal, expt: %u, real: %u!\n", ++ sess->hash_table.table_row_size, info->table_row_size); ++ ret = -WD_EINVAL; ++ goto out; ++ } ++ ++ if (!info->std_table) { ++ WD_ERR("invalid: agg standard hash table is NULL!\n"); ++ ret = -WD_EINVAL; ++ goto out; ++ } ++ ++ if (info->std_table_row_num < sess->hash_table.std_table_row_num) { ++ WD_ERR("invalid: agg standard hash table is too small, expt: %u, real: %u!\n", ++ sess->hash_table.std_table_row_num, info->std_table_row_num); ++ ret = -WD_EINVAL; ++ goto out; ++ } ++ ++ if (!info->ext_table_row_num || !info->ext_table) ++ WD_INFO("info: agg extern hash table is NULL!\n"); ++ ++ hash_table = &sess->hash_table; ++ rehash_table = &sess->rehash_table; ++ ++ memcpy(rehash_table, hash_table, sizeof(struct wd_dae_hash_table)); ++ memcpy(hash_table, info, sizeof(struct wd_dae_hash_table)); ++ ++ if (sess->ops.hash_table_init) { ++ ret = sess->ops.hash_table_init(hash_table, sess->priv); ++ if (ret) { ++ memcpy(hash_table, rehash_table, sizeof(struct wd_dae_hash_table)); ++ memset(rehash_table, 0, sizeof(struct wd_dae_hash_table)); ++ goto out; ++ } ++ } ++ ++ return WD_SUCCESS; ++ ++out: ++ __atomic_store_n(&sess->state, expected, __ATOMIC_RELEASE); ++ return ret; ++} ++ ++static void wd_agg_clear_status(void) ++{ ++ wd_alg_clear_init(&wd_agg_setting.status); ++} ++ ++static int wd_agg_alg_init(struct wd_ctx_config *config, struct wd_sched *sched) ++{ ++ int ret; ++ ++ ret = wd_set_epoll_en("WD_AGG_EPOLL_EN", &wd_agg_setting.config.epoll_en); ++ if (ret < 0) ++ return ret; ++ ++ ret = wd_init_ctx_config(&wd_agg_setting.config, config); ++ if (ret < 0) ++ return ret; ++ ++ ret = wd_init_sched(&wd_agg_setting.sched, sched); ++ if (ret < 0) ++ goto out_clear_ctx_config; ++ ++ /* Allocate async pool for every ctx */ ++ ret = wd_init_async_request_pool(&wd_agg_setting.pool, config, WD_POOL_MAX_ENTRIES, ++ sizeof(struct wd_agg_msg)); ++ if (ret < 0) ++ goto out_clear_sched; ++ ++ ret = wd_alg_init_driver(&wd_agg_setting.config, wd_agg_setting.driver); ++ if (ret) ++ goto out_clear_pool; ++ ++ return WD_SUCCESS; ++ ++out_clear_pool: ++ wd_uninit_async_request_pool(&wd_agg_setting.pool); ++out_clear_sched: ++ wd_clear_sched(&wd_agg_setting.sched); ++out_clear_ctx_config: ++ wd_clear_ctx_config(&wd_agg_setting.config); ++ return ret; ++} ++ ++static int wd_agg_alg_uninit(void) ++{ ++ void *priv = wd_agg_setting.priv; ++ ++ if (!priv) ++ return -WD_EINVAL; ++ ++ /* Uninit async request pool */ ++ wd_uninit_async_request_pool(&wd_agg_setting.pool); ++ ++ /* Unset config, sched, driver */ ++ wd_clear_sched(&wd_agg_setting.sched); ++ ++ wd_alg_uninit_driver(&wd_agg_setting.config, wd_agg_setting.driver); ++ ++ return WD_SUCCESS; ++} ++ ++int wd_agg_init(char *alg, __u32 sched_type, int task_type, struct wd_ctx_params *ctx_params) ++{ ++ struct wd_ctx_params agg_ctx_params = {0}; ++ struct wd_ctx_nums agg_ctx_num = {0}; ++ int ret = -WD_EINVAL; ++ int state; ++ bool flag; ++ ++ pthread_atfork(NULL, NULL, wd_agg_clear_status); ++ ++ state = wd_alg_try_init(&wd_agg_setting.status); ++ if (state) ++ return state; ++ ++ if (!alg || sched_type >= SCHED_POLICY_BUTT || ++ task_type < 0 || task_type >= TASK_MAX_TYPE) { ++ WD_ERR("invalid: agg init input param is wrong!\n"); ++ goto out_uninit; ++ } ++ ++ flag = wd_agg_alg_check(alg); ++ if (!flag) { ++ WD_ERR("invalid: agg: %s unsupported!\n", alg); ++ goto out_uninit; ++ } ++ ++ state = wd_agg_open_driver(); ++ if (state) ++ goto out_uninit; ++ ++ while (ret != 0) { ++ memset(&wd_agg_setting.config, 0, sizeof(struct wd_ctx_config_internal)); ++ ++ /* Get alg driver and dev name */ ++ wd_agg_setting.driver = wd_alg_drv_bind(task_type, alg); ++ if (!wd_agg_setting.driver) { ++ WD_ERR("failed to bind %s driver.\n", alg); ++ goto out_dlopen; ++ } ++ ++ agg_ctx_params.ctx_set_num = &agg_ctx_num; ++ ret = wd_ctx_param_init(&agg_ctx_params, ctx_params, wd_agg_setting.driver, ++ WD_AGG_TYPE, 1); ++ if (ret) { ++ if (ret == -WD_EAGAIN) { ++ wd_disable_drv(wd_agg_setting.driver); ++ wd_alg_drv_unbind(wd_agg_setting.driver); ++ continue; ++ } ++ goto out_driver; ++ } ++ ++ wd_agg_init_attrs.alg = alg; ++ wd_agg_init_attrs.sched_type = sched_type; ++ wd_agg_init_attrs.driver = wd_agg_setting.driver; ++ wd_agg_init_attrs.ctx_params = &agg_ctx_params; ++ wd_agg_init_attrs.alg_init = wd_agg_alg_init; ++ wd_agg_init_attrs.alg_poll_ctx = wd_agg_poll_ctx; ++ ret = wd_alg_attrs_init(&wd_agg_init_attrs); ++ if (ret) { ++ if (ret == -WD_ENODEV) { ++ wd_disable_drv(wd_agg_setting.driver); ++ wd_alg_drv_unbind(wd_agg_setting.driver); ++ wd_ctx_param_uninit(&agg_ctx_params); ++ continue; ++ } ++ WD_ERR("fail to init alg attrs.\n"); ++ goto out_params_uninit; ++ } ++ } ++ ++ wd_alg_set_init(&wd_agg_setting.status); ++ wd_ctx_param_uninit(&agg_ctx_params); ++ ++ return WD_SUCCESS; ++ ++out_params_uninit: ++ wd_ctx_param_uninit(&agg_ctx_params); ++out_driver: ++ wd_alg_drv_unbind(wd_agg_setting.driver); ++out_dlopen: ++ wd_agg_close_driver(); ++out_uninit: ++ wd_alg_clear_init(&wd_agg_setting.status); ++ return ret; ++} ++ ++void wd_agg_uninit(void) ++{ ++ int ret; ++ ++ ret = wd_agg_alg_uninit(); ++ if (ret) ++ return; ++ ++ wd_alg_attrs_uninit(&wd_agg_init_attrs); ++ wd_alg_drv_unbind(wd_agg_setting.driver); ++ wd_agg_close_driver(); ++ wd_agg_setting.dlh_list = NULL; ++ wd_alg_clear_init(&wd_agg_setting.status); ++} ++ ++static void fill_request_msg_input(struct wd_agg_msg *msg, struct wd_agg_req *req, ++ struct wd_agg_sess *sess, bool is_rehash) ++{ ++ memcpy(&msg->req, req, sizeof(struct wd_agg_req)); ++ ++ msg->key_cols_num = sess->key_conf.cols_num; ++ msg->agg_cols_num = sess->agg_conf.cols_num; ++ memcpy(&msg->hash_table, &sess->hash_table, sizeof(struct wd_dae_hash_table)); ++ msg->row_count = req->in_row_count; ++ msg->priv = sess->priv; ++ if (!is_rehash) { ++ msg->pos = WD_AGG_STREAM_INPUT; ++ msg->agg_cols_info = sess->agg_conf.cols_info; ++ msg->key_cols_info = sess->key_conf.cols_info; ++ msg->is_count_all = sess->agg_conf.is_count_all; ++ msg->count_all_data_type = sess->agg_conf.count_all_data_type; ++ } else { ++ msg->pos = WD_AGG_REHASH_INPUT; ++ } ++} ++ ++static void fill_request_msg_output(struct wd_agg_msg *msg, struct wd_agg_req *req, ++ struct wd_agg_sess *sess, bool is_rehash) ++{ ++ memcpy(&msg->req, req, sizeof(struct wd_agg_req)); ++ ++ msg->key_cols_num = sess->key_conf.cols_num; ++ msg->agg_cols_num = sess->agg_conf.cols_num; ++ msg->priv = sess->priv; ++ if (!is_rehash) { ++ msg->pos = WD_AGG_STREAM_OUTPUT; ++ msg->is_count_all = sess->agg_conf.is_count_all; ++ msg->count_all_data_type = sess->agg_conf.count_all_data_type; ++ memcpy(&msg->hash_table, &sess->hash_table, sizeof(struct wd_dae_hash_table)); ++ } else { ++ msg->pos = WD_AGG_REHASH_OUTPUT; ++ memcpy(&msg->hash_table, &sess->rehash_table, sizeof(struct wd_dae_hash_table)); ++ } ++ msg->key_cols_info = sess->key_conf.cols_info; ++ msg->agg_cols_info = sess->agg_conf.cols_info; ++ msg->row_count = req->out_row_count; ++} ++ ++static int wd_agg_check_common_params(struct wd_agg_sess *sess, struct wd_agg_req *req, __u8 mode) ++{ ++ if (unlikely(!sess)) { ++ WD_ERR("invalid: agg session is NULL!\n"); ++ return -WD_EINVAL; ++ } ++ ++ if (unlikely(!sess->key_conf.data_size || !sess->agg_conf.data_size || ++ !sess->agg_conf.out_data_size)) { ++ WD_ERR("invalid: agg session data size is NULL!\n"); ++ return -WD_EINVAL; ++ } ++ ++ if (unlikely(!req)) { ++ WD_ERR("invalid: agg input req is NULL!\n"); ++ return -WD_EINVAL; ++ } ++ ++ if (unlikely(mode == CTX_MODE_ASYNC && !req->cb)) { ++ WD_ERR("invalid: agg req cb is NULL!\n"); ++ return -WD_EINVAL; ++ } ++ ++ return WD_SUCCESS; ++} ++ ++static int check_out_col_addr(struct wd_dae_col_addr *col, __u32 row_count, ++ enum wd_dae_data_type type, __u64 data_size) ++{ ++ if (unlikely(!col->empty || col->empty_size < row_count * sizeof(col->empty[0]))) { ++ WD_ERR("failed to check agg empty col, size: %llu!\n", col->empty_size); ++ return -WD_EINVAL; ++ } ++ if (unlikely(!col->value)) { ++ WD_ERR("invalid: agg value col addr is NULL!\n"); ++ return -WD_EINVAL; ++ } ++ /* Only VARCHAR type use offset col to indicate the length of value col */ ++ if (type == WD_DAE_VARCHAR) { ++ /* Offset col row count should be 1 more than row_count */ ++ if (unlikely(!col->offset || ++ col->offset_size < (row_count + 1) * sizeof(col->offset[0]))) { ++ WD_ERR("failed to check agg offset col, size: %llu!\n", ++ col->offset_size); ++ return -WD_EINVAL; ++ } ++ } else { ++ if (unlikely(col->value_size < row_count * data_size)) { ++ WD_ERR("failed to check agg value col size: %llu!\n", col->value_size); ++ return -WD_EINVAL; ++ } ++ } ++ ++ return WD_SUCCESS; ++} ++ ++static int check_in_col_addr(struct wd_dae_col_addr *col, __u32 row_count, ++ enum wd_dae_data_type type, __u64 data_size) ++{ ++ __u32 offset_len; ++ ++ if (unlikely(!col->empty || col->empty_size != row_count * sizeof(col->empty[0]))) { ++ WD_ERR("failed to check agg empty col addr, size: %llu!\n", col->empty_size); ++ return -WD_EINVAL; ++ } ++ ++ if (unlikely(!col->value)) { ++ WD_ERR("invalid: agg value col addr is NULL!\n"); ++ return -WD_EINVAL; ++ } ++ /* Only VARCHAR type use offset col to indicate the length of value col */ ++ if (type == WD_DAE_VARCHAR) { ++ /* Offset col row count should be 1 more than row_count */ ++ offset_len = row_count + 1; ++ if (unlikely(!col->offset || ++ col->offset_size != offset_len * sizeof(col->offset[0]))) { ++ WD_ERR("failed to check agg offset col addr, size: %llu!\n", ++ col->offset_size); ++ return -WD_EINVAL; ++ } ++ if (unlikely(col->offset[offset_len - 1] < col->offset[0] || ++ col->offset[offset_len - 1] - col->offset[0] != col->value_size)) { ++ WD_ERR("failed to check agg varchar value col size: %llu!\n", ++ col->value_size); ++ return -WD_EINVAL; ++ } ++ } else { ++ if (unlikely(col->value_size != row_count * data_size)) { ++ WD_ERR("failed to check agg value col size: %llu!\n", col->value_size); ++ return -WD_EINVAL; ++ } ++ } ++ ++ return WD_SUCCESS; ++} ++ ++static int check_key_col_addr(struct wd_dae_col_addr *cols, __u32 cols_num, ++ struct wd_agg_sess *sess, __u32 row_count, bool is_input) ++{ ++ int (*func)(struct wd_dae_col_addr *col, __u32 row_count, ++ enum wd_dae_data_type type, __u64 data_size); ++ __u32 i; ++ int ret; ++ ++ if (sess->key_conf.cols_num != cols_num) { ++ WD_ERR("agg req key cols num is wrong!\n"); ++ return -WD_EINVAL; ++ } ++ ++ if (is_input) ++ func = check_in_col_addr; ++ else ++ func = check_out_col_addr; ++ ++ for (i = 0; i < cols_num; i++) { ++ ret = func(cols + i, row_count, sess->key_conf.cols_info[i].input_data_type, ++ sess->key_conf.data_size[i]); ++ if (unlikely(ret)) { ++ WD_ERR("failed to check agg req key col! col idx: %u\n", i); ++ return ret; ++ } ++ } ++ ++ return WD_SUCCESS; ++} ++ ++static int check_agg_col_addr(struct wd_dae_col_addr *cols, __u32 cols_num, ++ struct wd_agg_sess *sess, __u32 row_count) ++{ ++ __u32 i; ++ int ret; ++ ++ /* When there is only a count(*) task, it returns */ ++ if (!cols) ++ return 0; ++ ++ if (sess->agg_conf.cols_num != cols_num) { ++ WD_ERR("agg req input agg cols num is wrong!\n"); ++ return -WD_EINVAL; ++ } ++ ++ for (i = 0; i < cols_num; i++) { ++ ret = check_in_col_addr(cols + i, row_count, ++ sess->agg_conf.cols_info[i].input_data_type, ++ sess->agg_conf.data_size[i]); ++ if (unlikely(ret)) { ++ WD_ERR("failed to check agg req input agg col! col idx: %u\n", i); ++ return ret; ++ } ++ } ++ ++ return WD_SUCCESS; ++} ++ ++static int check_out_agg_col_addr(struct wd_dae_col_addr *cols, __u32 cols_num, ++ struct wd_agg_sess *sess, __u32 row_count) ++{ ++ __u32 i, j, k; ++ int ret; ++ ++ if (sess->agg_conf.out_cols_num != cols_num) { ++ WD_ERR("agg req output agg cols num is wrong!\n"); ++ return -WD_EINVAL; ++ } ++ ++ for (i = 0, k = 0; i < sess->agg_conf.cols_num; i++) { ++ for (j = 0; j < sess->agg_conf.cols_info[i].col_alg_num; j++, k++) { ++ ret = check_out_col_addr(cols + k, row_count, ++ sess->agg_conf.cols_info[i].output_data_types[j], ++ sess->agg_conf.out_data_size[k]); ++ if (unlikely(ret)) { ++ WD_ERR("failed to check agg req output agg col! col idx: %u\n", i); ++ return ret; ++ } ++ } ++ } ++ return WD_SUCCESS; ++} ++ ++static int wd_agg_check_input_req(struct wd_agg_sess *sess, struct wd_agg_req *req) ++{ ++ int ret; ++ ++ if (unlikely(req->key_cols_num != sess->key_conf.cols_num)) { ++ WD_ERR("invalid: agg req key_cols_num is not equal!\n"); ++ return -WD_EINVAL; ++ } ++ ++ if (unlikely(req->agg_cols_num != sess->agg_conf.cols_num)) { ++ WD_ERR("invalid: agg req agg_cols_num is not equal!\n"); ++ return -WD_EINVAL; ++ } ++ ++ if (unlikely(!req->key_cols)) { ++ WD_ERR("invalid: agg req key_cols is NULL!\n"); ++ return -WD_EINVAL; ++ } ++ ++ if (unlikely(req->agg_cols_num && !req->agg_cols)) { ++ WD_ERR("invalid: agg req agg_cols is NULL!\n"); ++ return -WD_EINVAL; ++ } ++ ++ if (!req->in_row_count) { ++ WD_ERR("agg req input row count is zero!\n"); ++ return -WD_EINVAL; ++ } ++ ++ ret = check_key_col_addr(req->key_cols, req->key_cols_num, sess, req->in_row_count, true); ++ if (unlikely(ret)) { ++ WD_ERR("failed to check agg req key cols addr!\n"); ++ return -WD_EINVAL; ++ } ++ ++ ret = check_agg_col_addr(req->agg_cols, req->agg_cols_num, sess, req->in_row_count); ++ if (unlikely(ret)) { ++ WD_ERR("failed to check agg req agg cols addr!\n"); ++ return -WD_EINVAL; ++ } ++ ++ return WD_SUCCESS; ++} ++ ++static int wd_agg_check_input_params(struct wd_agg_sess *sess, struct wd_agg_req *req, __u8 mode) ++{ ++ int ret; ++ ++ ret = wd_agg_check_common_params(sess, req, mode); ++ if (ret) ++ return ret; ++ ++ return wd_agg_check_input_req(sess, req); ++} ++ ++static int wd_agg_check_output_req(struct wd_agg_sess *sess, struct wd_agg_req *req) ++{ ++ int ret; ++ ++ if (unlikely(req->out_key_cols_num != sess->key_conf.cols_num)) { ++ WD_ERR("invalid: agg req out_key_cols_num is not equal!\n"); ++ return -WD_EINVAL; ++ } ++ ++ if (unlikely(req->out_agg_cols_num != sess->agg_conf.out_cols_num)) { ++ WD_ERR("invalid: agg req out_agg_cols_num is not equal!\n"); ++ return -WD_EINVAL; ++ } ++ ++ if (unlikely(!req->out_key_cols)) { ++ WD_ERR("invalid: agg req out_key_cols is NULL!\n"); ++ return -WD_EINVAL; ++ } ++ ++ if (unlikely(!req->out_agg_cols)) { ++ WD_ERR("invalid: agg req out_agg_cols is NULL!\n"); ++ return -WD_EINVAL; ++ } ++ ++ if (!req->out_row_count) { ++ WD_ERR("agg req output row count is zero!\n"); ++ return -WD_EINVAL; ++ } ++ ++ ret = check_key_col_addr(req->out_key_cols, req->out_key_cols_num, sess, ++ req->out_row_count, false); ++ if (unlikely(ret)) { ++ WD_ERR("failed to check agg req out key cols addr!\n"); ++ return -WD_EINVAL; ++ } ++ ++ ret = check_out_agg_col_addr(req->out_agg_cols, req->out_agg_cols_num, sess, ++ req->out_row_count); ++ if (unlikely(ret)) { ++ WD_ERR("failed to check agg req out agg cols addr!\n"); ++ return -WD_EINVAL; ++ } ++ ++ return WD_SUCCESS; ++} ++ ++static int wd_agg_check_output_params(struct wd_agg_sess *sess, struct wd_agg_req *req, __u8 mode) ++{ ++ int ret; ++ ++ ret = wd_agg_check_common_params(sess, req, mode); ++ if (ret) ++ return ret; ++ ++ return wd_agg_check_output_req(sess, req); ++} ++ ++static int wd_agg_check_rehash_params(struct wd_agg_sess *sess, struct wd_agg_req *req) ++{ ++ int ret; ++ ++ ret = wd_agg_check_common_params(sess, req, CTX_MODE_SYNC); ++ if (ret) ++ return ret; ++ ++ ret = wd_agg_check_output_req(sess, req); ++ if (ret) ++ WD_ERR("failed to check agg output req for rehash!\n"); ++ ++ return ret; ++} ++ ++static int wd_agg_sync_job(struct wd_agg_sess *sess, struct wd_agg_req *req, ++ struct wd_agg_msg *msg) ++{ ++ struct wd_ctx_config_internal *config = &wd_agg_setting.config; ++ struct wd_msg_handle msg_handle; ++ struct wd_ctx_internal *ctx; ++ __u32 idx; ++ int ret; ++ ++ idx = wd_agg_setting.sched.pick_next_ctx(wd_agg_setting.sched.h_sched_ctx, ++ sess->sched_key, CTX_MODE_SYNC); ++ ret = wd_check_ctx(config, CTX_MODE_SYNC, idx); ++ if (unlikely(ret)) ++ return ret; ++ ++ wd_dfx_msg_cnt(config, WD_CTX_CNT_NUM, idx); ++ ctx = config->ctxs + idx; ++ ++ msg_handle.send = wd_agg_setting.driver->send; ++ msg_handle.recv = wd_agg_setting.driver->recv; ++ ++ pthread_spin_lock(&ctx->lock); ++ ret = wd_handle_msg_sync(wd_agg_setting.driver, &msg_handle, ctx->ctx, ++ msg, NULL, config->epoll_en); ++ pthread_spin_unlock(&ctx->lock); ++ ++ return ret; ++} ++ ++static int wd_agg_input_try_init(struct wd_agg_sess *sess, enum wd_agg_sess_state *expected) ++{ ++ (void)__atomic_compare_exchange_n(&sess->state, expected, WD_AGG_SESS_INPUT, ++ true, __ATOMIC_RELAXED, __ATOMIC_RELAXED); ++ switch (*expected) { ++ case WD_AGG_SESS_INIT: ++ case WD_AGG_SESS_INPUT: ++ break; ++ default: ++ WD_ERR("invalid: agg input sess state is %d!\n", *expected); ++ return -WD_EINVAL; ++ } ++ ++ return WD_SUCCESS; ++} ++ ++static int wd_agg_check_msg_result(__u32 result) ++{ ++ switch (result) { ++ case WD_AGG_TASK_DONE: ++ case WD_AGG_SUM_OVERFLOW: ++ return 0; ++ case WD_AGG_IN_EPARA: ++ case WD_AGG_NEED_REHASH: ++ case WD_AGG_INVALID_HASH_TABLE: ++ case WD_AGG_INVALID_VARCHAR: ++ case WD_AGG_PARSE_ERROR: ++ case WD_AGG_BUS_ERROR: ++ WD_ERR("failed to check agg message state: %u!\n", result); ++ return -WD_EIO; ++ default: ++ return -WD_EINVAL; ++ } ++} ++ ++int wd_agg_add_input_sync(handle_t h_sess, struct wd_agg_req *req) ++{ ++ struct wd_agg_sess *sess = (struct wd_agg_sess *)h_sess; ++ enum wd_agg_sess_state expected = WD_AGG_SESS_INIT; ++ struct wd_agg_msg msg; ++ int ret; ++ ++ ret = wd_agg_check_input_params(sess, req, CTX_MODE_SYNC); ++ if (unlikely(ret)) { ++ WD_ERR("failed to check agg input params!\n"); ++ return ret; ++ } ++ ++ ret = wd_agg_input_try_init(sess, &expected); ++ if (unlikely(ret)) ++ return ret; ++ ++ memset(&msg, 0, sizeof(struct wd_agg_msg)); ++ fill_request_msg_input(&msg, req, sess, false); ++ req->state = 0; ++ ++ ret = wd_agg_sync_job(sess, req, &msg); ++ if (unlikely(ret)) { ++ if (expected == WD_AGG_SESS_INIT) ++ __atomic_store_n(&sess->state, expected, __ATOMIC_RELEASE); ++ WD_ERR("failed to do agg add input sync job!\n"); ++ return ret; ++ } ++ ++ req->state = msg.result; ++ req->real_in_row_count = msg.in_row_count; ++ ++ return WD_SUCCESS; ++} ++ ++static int wd_agg_async_job(struct wd_agg_sess *sess, struct wd_agg_req *req, bool is_input) ++{ ++ struct wd_ctx_config_internal *config = &wd_agg_setting.config; ++ struct wd_ctx_internal *ctx; ++ __u32 idx; ++ int msg_id, ret; ++ struct wd_agg_msg *msg; ++ ++ idx = wd_agg_setting.sched.pick_next_ctx(wd_agg_setting.sched.h_sched_ctx, ++ sess->sched_key, CTX_MODE_ASYNC); ++ ret = wd_check_ctx(config, CTX_MODE_ASYNC, idx); ++ if (unlikely(ret)) ++ return ret; ++ ++ ctx = config->ctxs + idx; ++ msg_id = wd_get_msg_from_pool(&wd_agg_setting.pool, idx, (void **)&msg); ++ if (unlikely(msg_id < 0)) { ++ WD_ERR("busy, failed to get agg msg from pool!\n"); ++ return -WD_EBUSY; ++ } ++ ++ if (is_input) ++ fill_request_msg_input(msg, req, sess, false); ++ else ++ fill_request_msg_output(msg, req, sess, false); ++ msg->tag = msg_id; ++ ret = wd_alg_driver_send(wd_agg_setting.driver, ctx->ctx, msg); ++ if (unlikely(ret < 0)) { ++ if (ret != -WD_EBUSY) ++ WD_ERR("wd agg async send err!\n"); ++ ++ goto fail_with_msg; ++ } ++ ++ wd_dfx_msg_cnt(config, WD_CTX_CNT_NUM, idx); ++ ++ return WD_SUCCESS; ++ ++fail_with_msg: ++ wd_put_msg_to_pool(&wd_agg_setting.pool, idx, msg->tag); ++ return ret; ++} ++ ++int wd_agg_add_input_async(handle_t h_sess, struct wd_agg_req *req) ++{ ++ struct wd_agg_sess *sess = (struct wd_agg_sess *)h_sess; ++ enum wd_agg_sess_state expected = WD_AGG_SESS_INIT; ++ int ret; ++ ++ ret = wd_agg_check_input_params(sess, req, CTX_MODE_ASYNC); ++ if (unlikely(ret)) { ++ WD_ERR("failed to check agg async input params!\n"); ++ return ret; ++ } ++ ++ ret = wd_agg_input_try_init(sess, &expected); ++ if (unlikely(ret)) ++ return ret; ++ ++ ret = wd_agg_async_job(sess, req, true); ++ if (unlikely(ret)) { ++ if (expected == WD_AGG_SESS_INIT) ++ __atomic_store_n(&sess->state, expected, __ATOMIC_RELEASE); ++ WD_ERR("failed to do agg add input async job!\n"); ++ } ++ ++ return ret; ++} ++ ++static int wd_agg_output_try_init(struct wd_agg_sess *sess, enum wd_agg_sess_state *expected) ++{ ++ (void)__atomic_compare_exchange_n(&sess->state, expected, WD_AGG_SESS_OUTPUT, ++ true, __ATOMIC_RELAXED, __ATOMIC_RELAXED); ++ switch (*expected) { ++ case WD_AGG_SESS_OUTPUT: ++ case WD_AGG_SESS_INPUT: ++ break; ++ default: ++ WD_ERR("invalid: agg output sess state is %d!\n", *expected); ++ return -WD_EINVAL; ++ } ++ return WD_SUCCESS; ++} ++ ++int wd_agg_get_output_sync(handle_t h_sess, struct wd_agg_req *req) ++{ ++ struct wd_agg_sess *sess = (struct wd_agg_sess *)h_sess; ++ enum wd_agg_sess_state expected = WD_AGG_SESS_INPUT; ++ struct wd_agg_msg msg; ++ int ret; ++ ++ ret = wd_agg_check_output_params(sess, req, CTX_MODE_SYNC); ++ if (unlikely(ret)) { ++ WD_ERR("failed to check agg output params!\n"); ++ return ret; ++ } ++ ++ ret = wd_agg_output_try_init(sess, &expected); ++ if (unlikely(ret)) ++ return ret; ++ ++ memset(&msg, 0, sizeof(struct wd_agg_msg)); ++ fill_request_msg_output(&msg, req, sess, false); ++ req->state = 0; ++ ++ ret = wd_agg_sync_job(sess, req, &msg); ++ if (unlikely(ret)) { ++ if (expected == WD_AGG_SESS_INPUT) ++ __atomic_store_n(&sess->state, expected, __ATOMIC_RELEASE); ++ WD_ERR("failed to do agg get output sync job!\n"); ++ return ret; ++ } ++ ++ req->state = msg.result; ++ req->real_out_row_count = msg.out_row_count; ++ req->output_done = msg.output_done; ++ ++ return WD_SUCCESS; ++} ++ ++int wd_agg_get_output_async(handle_t h_sess, struct wd_agg_req *req) ++{ ++ struct wd_agg_sess *sess = (struct wd_agg_sess *)h_sess; ++ enum wd_agg_sess_state expected = WD_AGG_SESS_INPUT; ++ int ret; ++ ++ ret = wd_agg_check_output_params(sess, req, CTX_MODE_ASYNC); ++ if (unlikely(ret)) { ++ WD_ERR("failed to check agg async output params!\n"); ++ return ret; ++ } ++ ++ ret = wd_agg_output_try_init(sess, &expected); ++ if (unlikely(ret)) ++ return ret; ++ ++ ret = wd_agg_async_job(sess, req, false); ++ if (unlikely(ret)) { ++ if (expected == WD_AGG_SESS_INPUT) ++ __atomic_store_n(&sess->state, expected, __ATOMIC_RELEASE); ++ WD_ERR("failed to do agg get output async job!\n"); ++ } ++ ++ return ret; ++} ++ ++static int set_col_size_inner(struct wd_dae_col_addr *col, struct wd_dae_col_addr *expt, ++ __u32 row_count, __u64 data_size, enum wd_dae_data_type type) ++{ ++ col->empty_size = expt->empty_size; ++ if (type != WD_DAE_VARCHAR) { ++ col->value_size = row_count * data_size; ++ return WD_SUCCESS; ++ } ++ ++ if (unlikely(!col->offset || col->offset[row_count] < col->offset[0])) { ++ WD_ERR("invalid: hashagg offset col param is wrong!\n"); ++ return -WD_EINVAL; ++ } ++ ++ col->offset_size = expt->offset_size; ++ col->value_size = col->offset[row_count] - col->offset[0]; ++ ++ return WD_SUCCESS; ++} ++ ++static int wd_agg_set_keycol_size(struct wd_agg_sess *sess, struct wd_dae_col_addr *key, ++ struct wd_dae_col_addr *expt, __u32 row_count) ++{ ++ __u32 i; ++ int ret; ++ ++ for (i = 0; i < sess->key_conf.cols_num; i++) { ++ ret = set_col_size_inner(key + i, expt, row_count, sess->key_conf.data_size[i], ++ sess->key_conf.cols_info[i].input_data_type); ++ if (unlikely(ret)) ++ return ret; ++ } ++ ++ return WD_SUCCESS; ++} ++ ++static int wd_agg_set_aggcol_size(struct wd_agg_sess *sess, struct wd_dae_col_addr *agg, ++ struct wd_dae_col_addr *expt, __u32 row_count) ++{ ++ __u64 data_size = 0; ++ __u32 i, j, k; ++ int ret; ++ ++ for (i = 0, k = 0; i < sess->agg_conf.cols_num; i++) { ++ for (j = 0; j < sess->agg_conf.cols_info[i].col_alg_num; j++, k++) { ++ ret = set_col_size_inner(agg + k, expt, row_count, ++ sess->agg_conf.out_data_size[k], ++ sess->agg_conf.cols_info[i].output_data_types[j]); ++ if (unlikely(ret)) ++ return ret; ++ } ++ } ++ ++ if (sess->agg_conf.is_count_all) { ++ (void)get_col_data_type_size(sess->agg_conf.count_all_data_type, 0, &data_size, 0); ++ ret = set_col_size_inner(agg + k, expt, row_count, data_size, ++ sess->agg_conf.count_all_data_type); ++ if (unlikely(ret)) ++ return ret; ++ } ++ ++ return WD_SUCCESS; ++} ++ ++static int wd_agg_set_col_size(struct wd_agg_sess *sess, struct wd_agg_req *req, ++ __u32 row_count) ++{ ++ struct wd_dae_col_addr expt = {0}; ++ int ret; ++ ++ expt.empty_size = sizeof(__u8) * row_count; ++ expt.offset_size = sizeof(__u32) * (row_count + 1); ++ ++ ret = wd_agg_set_keycol_size(sess, req->key_cols, &expt, row_count); ++ if (unlikely(ret)) ++ return ret; ++ ++ ret = wd_agg_set_aggcol_size(sess, req->agg_cols, &expt, row_count); ++ if (unlikely(ret)) ++ return ret; ++ ++ return WD_SUCCESS; ++} ++ ++static int wd_agg_rehash_sync_inner(struct wd_agg_sess *sess, struct wd_agg_req *req) ++{ ++ struct wd_agg_msg msg = {0}; ++ bool output_done; ++ int ret; ++ ++ fill_request_msg_output(&msg, req, sess, true); ++ req->state = 0; ++ ++ ret = wd_agg_sync_job(sess, req, &msg); ++ if (unlikely(ret)) ++ return ret; ++ ++ ret = wd_agg_check_msg_result(msg.result); ++ if (unlikely(ret)) ++ return ret; ++ ++ req->real_out_row_count = msg.out_row_count; ++ output_done = msg.output_done; ++ if (!msg.out_row_count) { ++ req->output_done = true; ++ return WD_SUCCESS; ++ } ++ ++ req->key_cols = req->out_key_cols; ++ req->agg_cols = req->out_agg_cols; ++ req->key_cols_num = req->out_key_cols_num; ++ req->agg_cols_num = req->out_agg_cols_num; ++ wd_agg_set_col_size(sess, req, req->real_out_row_count); ++ req->in_row_count = req->real_out_row_count; ++ ++ memset(&msg, 0, sizeof(struct wd_agg_msg)); ++ fill_request_msg_input(&msg, req, sess, true); ++ ++ ret = wd_agg_sync_job(sess, req, &msg); ++ if (unlikely(ret)) ++ return ret; ++ ++ ret = wd_agg_check_msg_result(msg.result); ++ if (unlikely(ret)) ++ return ret; ++ ++ req->state = msg.result; ++ req->output_done = output_done; ++ ++ return WD_SUCCESS; ++} ++ ++static int wd_agg_rehash_try_init(struct wd_agg_sess *sess, enum wd_agg_sess_state *expected) ++{ ++ int ret; ++ ++ ret = __atomic_compare_exchange_n(&sess->state, expected, WD_AGG_SESS_REHASH, ++ true, __ATOMIC_RELAXED, __ATOMIC_RELAXED); ++ if (!ret) { ++ WD_ERR("invalid: agg rehash sess state is %d!\n", *expected); ++ return -WD_EINVAL; ++ } ++ ++ return WD_SUCCESS; ++} ++ ++int wd_agg_rehash_sync(handle_t h_sess, struct wd_agg_req *req) ++{ ++ struct wd_agg_sess *sess = (struct wd_agg_sess *)h_sess; ++ enum wd_agg_sess_state expected = WD_AGG_SESS_RESET; ++ struct wd_agg_req src_req; ++ __u64 cnt = 0; ++ __u64 max_cnt; ++ int ret; ++ ++ ret = wd_agg_check_rehash_params(sess, req); ++ if (unlikely(ret)) { ++ WD_ERR("failed to check agg rehash params!\n"); ++ return ret; ++ } ++ ++ ret = wd_agg_rehash_try_init(sess, &expected); ++ if (unlikely(ret)) ++ return ret; ++ ++ memcpy(&src_req, req, sizeof(struct wd_agg_req)); ++ max_cnt = MAX_HASH_TABLE_ROW_NUM / req->out_row_count; ++ while (cnt < max_cnt) { ++ ret = wd_agg_rehash_sync_inner(sess, &src_req); ++ if (ret) { ++ __atomic_store_n(&sess->state, WD_AGG_SESS_RESET, __ATOMIC_RELEASE); ++ WD_ERR("failed to do agg rehash task!\n"); ++ return ret; ++ } ++ if (src_req.output_done) ++ break; ++ cnt++; ++ } ++ ++ __atomic_store_n(&sess->state, WD_AGG_SESS_INPUT, __ATOMIC_RELEASE); ++ return WD_SUCCESS; ++} ++ ++struct wd_agg_msg *wd_agg_get_msg(__u32 idx, __u32 tag) ++{ ++ return wd_find_msg_in_pool(&wd_agg_setting.pool, idx, tag); ++} ++ ++static int wd_agg_poll_ctx(__u32 idx, __u32 expt, __u32 *count) ++{ ++ struct wd_ctx_config_internal *config = &wd_agg_setting.config; ++ struct wd_agg_msg resp_msg, *msg; ++ struct wd_ctx_internal *ctx; ++ struct wd_agg_req *req; ++ __u64 recv_count = 0; ++ __u32 tmp = expt; ++ int ret; ++ ++ *count = 0; ++ ++ ret = wd_check_ctx(config, CTX_MODE_ASYNC, idx); ++ if (unlikely(ret)) ++ return ret; ++ ++ ctx = config->ctxs + idx; ++ ++ do { ++ ret = wd_alg_driver_recv(wd_agg_setting.driver, ctx->ctx, &resp_msg); ++ if (ret == -WD_EAGAIN) { ++ return ret; ++ } else if (unlikely(ret < 0)) { ++ WD_ERR("wd agg recv hw err!\n"); ++ return ret; ++ } ++ recv_count++; ++ msg = wd_find_msg_in_pool(&wd_agg_setting.pool, idx, resp_msg.tag); ++ if (unlikely(!msg)) { ++ WD_ERR("failed to get agg msg from pool!\n"); ++ return -WD_EINVAL; ++ } ++ ++ msg->tag = resp_msg.tag; ++ msg->req.state = resp_msg.result; ++ msg->req.real_in_row_count = resp_msg.in_row_count; ++ msg->req.real_out_row_count = resp_msg.out_row_count; ++ msg->req.output_done = resp_msg.output_done; ++ req = &msg->req; ++ ++ req->cb(req, req->cb_param); ++ /* Free msg cache to msg_pool */ ++ wd_put_msg_to_pool(&wd_agg_setting.pool, idx, resp_msg.tag); ++ *count = recv_count; ++ } while (--tmp); ++ ++ return ret; ++} ++ ++int wd_agg_poll(__u32 expt, __u32 *count) ++{ ++ handle_t h_ctx = wd_agg_setting.sched.h_sched_ctx; ++ struct wd_sched *sched = &wd_agg_setting.sched; ++ ++ if (unlikely(!expt || !count)) { ++ WD_ERR("invalid: agg poll input param is NULL!\n"); ++ return -WD_EINVAL; ++ } ++ ++ return sched->poll_policy(h_ctx, expt, count); ++} +diff --git a/wd_util.c b/wd_util.c +index 8d362d7..8adfd4c 100644 +--- a/wd_util.c ++++ b/wd_util.c +@@ -62,6 +62,7 @@ static const char *wd_env_name[WD_TYPE_MAX] = { + "WD_RSA_CTX_NUM", + "WD_DH_CTX_NUM", + "WD_ECC_CTX_NUM", ++ "WD_AGG_CTX_NUM", + }; + + struct async_task { +@@ -105,6 +106,7 @@ static struct acc_alg_item alg_options[] = { + {"gzip", "gzip"}, + {"deflate", "deflate"}, + {"lz77_zstd", "lz77_zstd"}, ++ {"hashagg", "hashagg"}, + + {"rsa", "rsa"}, + {"dh", "dh"}, +-- +2.25.1 + diff --git a/0018-drv-dae-support-hashagg-algorithm.patch b/0018-drv-dae-support-hashagg-algorithm.patch new file mode 100644 index 0000000..449519f --- /dev/null +++ b/0018-drv-dae-support-hashagg-algorithm.patch @@ -0,0 +1,1771 @@ +From d458dabb3fa63e07c5b0a0a9e0d140525ef4783a Mon Sep 17 00:00:00 2001 +From: Weili Qian +Date: Tue, 20 Aug 2024 09:43:51 +0800 +Subject: [PATCH 18/39] drv/dae: support hashagg algorithm + +UADK supports hardware acceleration for the hashagg algorithm. +Currently, the sum and count operations are supported, +which can be used in data analysis scenarios. + +Signed-off-by: Weili Qian +Signed-off-by: Qi Tao +--- + Makefile.am | 13 +- + drv/hisi_dae.c | 1686 ++++++++++++++++++++++++++++++++++++++++++++ + drv/hisi_qm_udrv.h | 2 + + 3 files changed, 1700 insertions(+), 1 deletion(-) + create mode 100644 drv/hisi_dae.c + +diff --git a/Makefile.am b/Makefile.am +index 52af71b..ba3308a 100644 +--- a/Makefile.am ++++ b/Makefile.am +@@ -45,7 +45,7 @@ lib_LTLIBRARIES=libwd.la libwd_comp.la libwd_crypto.la libwd_dae.la + + uadk_driversdir=$(libdir)/uadk + uadk_drivers_LTLIBRARIES=libhisi_sec.la libhisi_hpre.la libhisi_zip.la \ +- libisa_ce.la libisa_sve.la ++ libisa_ce.la libisa_sve.la libhisi_dae.la + + libwd_la_SOURCES=wd.c wd_mempool.c wd.h wd_alg.c wd_alg.h \ + v1/wd.c v1/wd.h v1/wd_adapter.c v1/wd_adapter.h \ +@@ -101,6 +101,9 @@ libisa_sve_la_SOURCES=drv/hash_mb/hash_mb.c wd_digest_drv.h drv/hash_mb/hash_mb. + drv/hash_mb/md5_sve_common.S drv/hash_mb/md5_mb_asimd_x1.S \ + drv/hash_mb/md5_mb_asimd_x4.S drv/hash_mb/md5_mb_sve.S + ++libhisi_dae_la_SOURCES=drv/hisi_dae.c drv/hisi_qm_udrv.c \ ++ hisi_qm_udrv.h ++ + if WD_STATIC_DRV + AM_CFLAGS += -DWD_STATIC_DRV -fPIC + AM_CFLAGS += -DWD_NO_LOG +@@ -130,6 +133,9 @@ libisa_ce_la_DEPENDENCIES = libwd.la libwd_crypto.la + libisa_sve_la_LIBADD = $(libwd_la_OBJECTS) $(libwd_crypto_la_OBJECTS) + libisa_sve_la_DEPENDENCIES = libwd.la libwd_crypto.la + ++libhisi_dae_la_LIBADD = $(libwd_la_OBJECTS) $(libwd_dae_la_OBJECTS) ++libhisi_dae_la_DEPENDENCIES = libwd.la libwd_dae.la ++ + else + UADK_WD_SYMBOL= -Wl,--version-script,$(top_srcdir)/libwd.map + UADK_CRYPTO_SYMBOL= -Wl,--version-script,$(top_srcdir)/libwd_crypto.map +@@ -171,6 +177,11 @@ libisa_ce_la_DEPENDENCIES= libwd.la libwd_crypto.la + libisa_sve_la_LIBADD= -lwd -lwd_crypto + libisa_sve_la_LDFLAGS=$(UADK_VERSION) + libisa_sve_la_DEPENDENCIES= libwd.la libwd_crypto.la ++ ++libhisi_dae_la_LIBADD= -lwd -lwd_dae ++libhisi_dae_la_LDFLAGS=$(UADK_VERSION) ++libhisi_dae_la_DEPENDENCIES= libwd.la libwd_dae.la ++ + endif # WD_STATIC_DRV + + pkgconfigdir = $(libdir)/pkgconfig +diff --git a/drv/hisi_dae.c b/drv/hisi_dae.c +new file mode 100644 +index 0000000..09d2387 +--- /dev/null ++++ b/drv/hisi_dae.c +@@ -0,0 +1,1686 @@ ++// SPDX-License-Identifier: Apache-2.0 ++/* Copyright 2024 Huawei Technologies Co.,Ltd. All rights reserved. */ ++ ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include "hisi_qm_udrv.h" ++#include "../include/drv/wd_agg_drv.h" ++ ++#define DAE_HASH_AGG_TYPE 2 ++#define DAE_EXT_SQE_SIZE 128 ++#define DAE_CTX_Q_NUM_DEF 1 ++ ++/* will remove in next version */ ++#define DAE_HASHAGG_SUM 0x1 ++#define DAE_HASHAGG_COUNT 0x2 ++#define DAE_HASH_COUNT_ALL 0x1 ++ ++/* column information */ ++#define DAE_MAX_KEY_COLS 9 ++#define DAE_MAX_INPUT_COLS 9 ++#define DAE_MAX_OUTPUT_COLS 9 ++#define DAE_MAX_CHAR_COL_NUM 5 ++#define DAE_MAX_CHAR_SIZE 32 ++#define DAE_MAX_VCHAR_SIZE 30 ++#define DAE_MAX_8B_COLS_NUM 8 ++#define DAE_MAX_16B_COLS_NUM 8 ++#define DAE_MAX_ADDR_NUM 32 ++#define DAE_MIN_ROW_SIZE 11 ++#define DAE_MAX_ROW_SIZE 256 ++#define DAE_VCHAR_OFFSET_SIZE 2 ++#define DAE_COL_BIT_NUM 4 ++#define DAE_AGG_START_COL 16 ++#define DAE_HASHAGG_MAX_ROW_NUN 50000 ++ ++/* align size */ ++#define DAE_CHAR_ALIGN_SIZE 4 ++#define DAE_TABLE_ALIGN_SIZE 128 ++#define DAE_ADDR_ALIGN_SIZE 128 ++ ++/* decimal infomartion */ ++#define DAE_DECIMAL_PRECISION_OFFSET 8 ++#define DAE_DECIMAL128_MAX_PRECISION 38 ++#define DAE_DECIMAL64_MAX_PRECISION 18 ++ ++/* hash table */ ++#define HASH_EXT_TABLE_INVALID_OFFSET 5 ++#define HASH_EXT_TABLE_VALID 0x80 ++#define HASH_TABLE_HEAD_TAIL_SIZE 8 ++#define HASH_TABLE_EMPTY_SIZE 4 ++#define HASH_TABLE_WITDH_POWER 2 ++#define HASH_TABLE_MIN_WIDTH 10 ++#define HASH_TABLE_MAX_WIDTH 43 ++#define HASH_TABLE_OFFSET_3ROW 3 ++#define HASH_TABLE_OFFSET_1ROW 1 ++ ++#define __ALIGN_MASK(x, mask) (((x) + (mask)) & ~(mask)) ++#define ALIGN(x, a) __ALIGN_MASK(x, (typeof(x))(a)-1) ++#define PTR_ALIGN(p, a) ((typeof(p))ALIGN((uintptr_t)(p), (a))) ++ ++#define BIT(nr) (1UL << (nr)) ++#define BITS_PER_LONG (__SIZEOF_LONG__ * 8) ++#define GENMASK(h, l) \ ++ (((~0UL) << (l)) & (~0UL >> (BITS_PER_LONG - 1 - (h)))) ++ ++/* DAE hardware protocol data */ ++enum dae_stage { ++ DAE_HASH_AGGREGATE = 0x0, ++ DAE_HASHAGG_OUTPUT = 0x7, ++}; ++ ++enum dae_op_type { ++ DAE_COUNT = 0x1, ++ DAE_SUM = 0x5, ++}; ++ ++enum dae_done_flag { ++ DAE_HW_TASK_NOT_PROCESS = 0x0, ++ DAE_HW_TASK_DONE = 0x1, ++ DAE_HW_TASK_ERR = 0x2, ++}; ++ ++enum dae_error_type { ++ DAE_TASK_SUCCESS = 0x0, ++ DAE_TASK_BD_ERROR_MIN = 0x1, ++ DAE_TASK_BD_ERROR_MAX = 0x7f, ++ DAE_HASH_TABLE_NEED_REHASH = 0x82, ++ DAE_HASH_TABLE_INVALID = 0x83, ++ DAE_HASHAGG_VCHAR_OVERFLOW = 0x84, ++ DAE_HASHAGG_RESULT_OVERFLOW = 0x85, ++ DAE_HASHAGG_BUS_ERROR = 0x86, ++ DAE_HASHAGG_VCHAR_LEN_ERROR = 0x87, ++}; ++ ++enum dae_data_type { ++ DAE_SINT32 = 0x0, ++ DAE_SINT64 = 0x2, ++ DAE_DECIMAL64 = 0x9, ++ DAE_DECIMAL128 = 0xA, ++ DAE_CHAR = 0xC, ++ DAE_VCHAR = 0xD, ++}; ++ ++enum dae_date_type_size { ++ SINT32_SIZE = 4, ++ SINT64_SIZE = 8, ++ DECIMAL128_SIZE = 16, ++ DEFAULT_VCHAR_SIZE = 30, ++}; ++ ++enum dae_table_row_size { ++ ROW_SIZE32 = 32, ++ ROW_SIZE64 = 64, ++ ROW_SIZE128 = 128, ++ ROW_SIZE256 = 256, ++ ROW_SIZE512 = 512, ++}; ++ ++enum dae_sum_optype { ++ DECIMAL64_TO_DECIMAL64 = 0x2, ++ DECIMAL64_TO_DECIMAL128 = 0x3, ++}; ++ ++struct dae_sqe { ++ __u32 bd_type : 6; ++ __u32 resv1 : 2; ++ __u32 task_type : 6; ++ __u32 resv2 : 2; ++ __u32 task_type_ext : 6; ++ __u32 resv3 : 9; ++ __u32 bd_invlid : 1; ++ __u16 table_row_size; ++ __u16 resv4; ++ __u32 resv5; ++ __u32 low_tag; ++ __u32 hi_tag; ++ __u32 row_num; ++ __u32 resv6; ++ __u32 src_table_width : 6; ++ __u32 dst_table_width : 6; ++ __u32 resv7 : 4; ++ __u32 counta_vld : 1; ++ __u32 resv8 : 15; ++ /* ++ * high 4bits: compare mode if data type is char/vchar, ++ * out type if operation is sum. ++ * low 4bits: input value type. ++ */ ++ __u8 key_data_type[16]; ++ __u8 agg_data_type[16]; ++ __u32 resv9[8]; ++ __u32 key_col_bitmap; ++ __u32 agg_col_bitmap; ++ __u64 addr_list; ++ __u32 done_flag : 3; ++ __u32 output_end : 1; ++ __u32 ext_err_type : 12; ++ __u32 err_type : 8; ++ __u32 wtype : 8; ++ __u32 out_raw_num; ++ __u32 vchar_err_offset; ++ __u16 sum_overflow_cols; ++ __u16 resv10; ++}; ++ ++struct dae_ext_sqe { ++ /* ++ * If date type is char/vchar, data info fill data type size ++ * If data type is decimal64/decimal128, data info fill data precision ++ */ ++ __u16 key_data_info[16]; ++ __u16 agg_data_info[16]; ++ /* Aggregated output from input agg col index */ ++ __u64 out_from_in_idx; ++ /* Aggregated output from input agg col operation, sum or count */ ++ __u64 out_optype; ++ __u32 resv[12]; ++}; ++ ++struct dae_col_addr { ++ __u64 empty_addr; ++ __u64 empty_size; ++ __u64 value_addr; ++ __u64 value_size; ++}; ++ ++struct dae_table_addr { ++ __u64 std_table_addr; ++ __u64 std_table_size; ++ __u64 ext_table_addr; ++ __u64 ext_table_size; ++}; ++ ++struct dae_addr_list { ++ __u64 ext_sqe_addr; ++ __u64 ext_sqe_size; ++ struct dae_table_addr src_table; ++ struct dae_table_addr dst_table; ++ __u64 resv_addr[6]; ++ struct dae_col_addr input_addr[32]; ++ struct dae_col_addr output_addr[32]; ++}; ++ ++struct dae_extend_addr { ++ struct dae_ext_sqe *ext_sqe; ++ struct dae_addr_list *addr_list; ++ __u8 *addr_status; ++ __u16 addr_num; ++ __u16 tail; ++}; ++ ++static enum dae_data_type hw_data_type_order[] = { ++ DAE_VCHAR, DAE_CHAR, DAE_DECIMAL128, ++ DAE_DECIMAL64, DAE_SINT64, DAE_SINT32, ++}; ++ ++struct hw_agg_data { ++ enum dae_data_type hw_type; ++ __u32 optype; ++ __u32 usr_col_idx; ++ __u16 data_info; ++ __u8 sum_outtype; ++}; ++ ++struct hashagg_output_src { ++ /* Aggregated output from input agg col index. */ ++ __u32 out_from_in_idx; ++ /* Aggregated output from input agg col operation, sum or count. */ ++ __u32 out_optype; ++}; ++ ++struct hashagg_col_data { ++ struct hw_agg_data key_data[DAE_MAX_KEY_COLS]; ++ struct hw_agg_data input_data[DAE_MAX_INPUT_COLS]; ++ struct hw_agg_data output_data[DAE_MAX_OUTPUT_COLS]; ++ struct hashagg_output_src normal_output[DAE_MAX_OUTPUT_COLS]; ++ struct hashagg_output_src rehash_output[DAE_MAX_OUTPUT_COLS]; ++ ++ __u32 key_num; ++ __u32 input_num; ++ __u32 output_num; ++ bool is_count_all; ++}; ++ ++struct hash_table_data { ++ void *std_table; ++ void *ext_table; ++ __u64 std_table_size; ++ __u64 ext_table_size; ++ __u32 table_width; ++}; ++ ++struct hashagg_ctx { ++ struct hashagg_col_data cols_data; ++ struct hash_table_data table_data; ++ struct hash_table_data rehash_table; ++ pthread_spinlock_t lock; ++ __u32 row_size; ++ __u16 sum_overflow_cols; ++}; ++ ++struct hisi_dae_ctx { ++ struct wd_ctx_config_internal config; ++}; ++ ++static int get_free_ext_addr(struct dae_extend_addr *ext_addr) ++{ ++ __u16 addr_num = ext_addr->addr_num; ++ __u16 idx = ext_addr->tail; ++ __u16 cnt = 0; ++ ++ while (__atomic_test_and_set(&ext_addr->addr_status[idx], __ATOMIC_ACQUIRE)) { ++ idx = (idx + 1) % addr_num; ++ cnt++; ++ if (cnt == addr_num) ++ return -WD_EBUSY; ++ } ++ ++ ext_addr->tail = (idx + 1) % addr_num; ++ ++ return idx; ++} ++ ++static void put_ext_addr(struct dae_extend_addr *ext_addr, int idx) ++{ ++ __atomic_clear(&ext_addr->addr_status[idx], __ATOMIC_RELEASE); ++} ++ ++static void fill_hashagg_task_type(struct wd_agg_msg *msg, struct dae_sqe *sqe) ++{ ++ switch (msg->pos) { ++ case WD_AGG_REHASH_INPUT: ++ case WD_AGG_STREAM_INPUT: ++ sqe->task_type_ext = DAE_HASH_AGGREGATE; ++ break; ++ case WD_AGG_STREAM_OUTPUT: ++ case WD_AGG_REHASH_OUTPUT: ++ sqe->task_type_ext = DAE_HASHAGG_OUTPUT; ++ break; ++ } ++} ++ ++static void fill_hashagg_output_order(struct dae_sqe *sqe, struct dae_ext_sqe *ext_sqe, ++ struct wd_agg_msg *msg) ++{ ++ struct hashagg_ctx *agg_ctx = msg->priv; ++ struct hashagg_col_data *cols_data = &agg_ctx->cols_data; ++ struct hashagg_output_src *output_src = cols_data->normal_output; ++ __u32 out_cols_num = cols_data->output_num; ++ __u32 offset = 0; ++ __u32 i; ++ ++ if (msg->pos == WD_AGG_REHASH_INPUT) ++ output_src = cols_data->rehash_output; ++ ++ if (cols_data->is_count_all && msg->pos != WD_AGG_REHASH_INPUT) { ++ sqe->counta_vld = DAE_HASH_COUNT_ALL; ++ out_cols_num--; ++ } ++ ++ for (i = 0; i < out_cols_num; i++) { ++ ext_sqe->out_from_in_idx |= (__u64)output_src[i].out_from_in_idx << offset; ++ ext_sqe->out_optype |= (__u64)output_src[i].out_optype << offset; ++ offset += DAE_COL_BIT_NUM; ++ } ++} ++ ++static void fill_hashagg_table_data(struct dae_sqe *sqe, struct dae_addr_list *addr_list, ++ struct wd_agg_msg *msg) ++{ ++ struct hashagg_ctx *agg_ctx = (struct hashagg_ctx *)msg->priv; ++ struct hash_table_data *table_data = NULL; ++ struct dae_table_addr *hw_table = NULL; ++ ++ switch (msg->pos) { ++ case WD_AGG_STREAM_INPUT: ++ case WD_AGG_REHASH_INPUT: ++ hw_table = &addr_list->dst_table; ++ table_data = &agg_ctx->table_data; ++ break; ++ case WD_AGG_STREAM_OUTPUT: ++ hw_table = &addr_list->src_table; ++ table_data = &agg_ctx->table_data; ++ break; ++ case WD_AGG_REHASH_OUTPUT: ++ hw_table = &addr_list->src_table; ++ table_data = &agg_ctx->rehash_table; ++ break; ++ } ++ ++ sqe->table_row_size = agg_ctx->row_size; ++ sqe->src_table_width = table_data->table_width; ++ sqe->dst_table_width = table_data->table_width; ++ ++ hw_table->std_table_addr = (__u64)(uintptr_t)table_data->std_table; ++ hw_table->std_table_size = table_data->std_table_size; ++ hw_table->ext_table_addr = (__u64)(uintptr_t)table_data->ext_table; ++ hw_table->ext_table_size = table_data->ext_table_size; ++} ++ ++static void fill_hashagg_key_data(struct dae_sqe *sqe, struct dae_ext_sqe *ext_sqe, ++ struct dae_addr_list *addr_list, struct wd_agg_msg *msg) ++{ ++ struct hashagg_ctx *agg_ctx = msg->priv; ++ struct hw_agg_data *key_data = agg_ctx->cols_data.key_data; ++ struct wd_dae_col_addr *usr_key_addr; ++ struct dae_col_addr *hw_key_addr; ++ __u32 j = DAE_MAX_ADDR_NUM - 1; ++ __u16 usr_col_idx; ++ __u32 i; ++ ++ sqe->key_col_bitmap = GENMASK(msg->key_cols_num - 1, 0); ++ ++ if (msg->pos == WD_AGG_STREAM_INPUT || msg->pos == WD_AGG_REHASH_INPUT) { ++ usr_key_addr = msg->req.key_cols; ++ hw_key_addr = addr_list->input_addr; ++ } else { ++ usr_key_addr = msg->req.out_key_cols; ++ hw_key_addr = addr_list->output_addr; ++ } ++ ++ for (i = 0; i < msg->key_cols_num; i++) { ++ sqe->key_data_type[i] = key_data[i].hw_type; ++ ext_sqe->key_data_info[i] = key_data[i].data_info; ++ usr_col_idx = key_data[i].usr_col_idx; ++ hw_key_addr[i].empty_addr = (__u64)(uintptr_t)usr_key_addr[usr_col_idx].empty; ++ hw_key_addr[i].empty_size = usr_key_addr[usr_col_idx].empty_size; ++ ++ hw_key_addr[i].value_addr = (__u64)(uintptr_t)usr_key_addr[usr_col_idx].value; ++ hw_key_addr[i].value_size = usr_key_addr[usr_col_idx].value_size; ++ /* offset is fill in final value addr */ ++ if (key_data[i].hw_type == DAE_VCHAR) { ++ hw_key_addr[j].value_addr = ++ (__u64)(uintptr_t)usr_key_addr[usr_col_idx].offset; ++ hw_key_addr[j].value_size = usr_key_addr[usr_col_idx].offset_size; ++ j--; ++ /* ++ * Since vchar size may be 0, which causes the hardware abnormal. ++ * Therefore, the size is set to 1 according to hardware protocol. ++ */ ++ if (!hw_key_addr[i].value_size) ++ hw_key_addr[i].value_size = 1; ++ } ++ } ++} ++ ++static void fill_hashagg_normal_info(struct dae_sqe *sqe, struct dae_ext_sqe *ext_sqe, ++ struct hashagg_col_data *cols_data, __u32 agg_cols_num) ++{ ++ struct hw_agg_data *agg_data = cols_data->input_data; ++ __u32 i; ++ ++ for (i = 0; i < agg_cols_num; i++) { ++ sqe->agg_data_type[i] = agg_data[i].hw_type; ++ sqe->agg_data_type[i] |= agg_data[i].sum_outtype << DAE_COL_BIT_NUM; ++ ext_sqe->agg_data_info[i] = agg_data[i].data_info; ++ } ++ ++ sqe->agg_col_bitmap = GENMASK(agg_cols_num + DAE_AGG_START_COL - 1, DAE_AGG_START_COL); ++} ++ ++static void fill_hashagg_rehash_info(struct dae_sqe *sqe, struct dae_ext_sqe *ext_sqe, ++ struct hw_agg_data *agg_data, __u32 agg_cols_num) ++{ ++ __u32 i; ++ ++ for (i = 0; i < agg_cols_num; i++) { ++ sqe->agg_data_type[i] = agg_data[i].hw_type; ++ sqe->agg_data_type[i] |= agg_data[i].sum_outtype << DAE_COL_BIT_NUM; ++ ext_sqe->agg_data_info[i] = agg_data[i].data_info; ++ } ++ ++ sqe->agg_col_bitmap = GENMASK(agg_cols_num + DAE_AGG_START_COL - 1, DAE_AGG_START_COL); ++} ++ ++static void fill_hashagg_input_data(struct dae_sqe *sqe, struct dae_ext_sqe *ext_sqe, ++ struct dae_addr_list *addr_list, struct wd_agg_msg *msg) ++{ ++ struct hashagg_ctx *agg_ctx = msg->priv; ++ struct hashagg_col_data *cols_data = &agg_ctx->cols_data; ++ struct wd_dae_col_addr *usr_agg_addr; ++ struct dae_col_addr *hw_agg_addr; ++ struct hw_agg_data *agg_data; ++ __u32 agg_col_num = 0; ++ __u32 i, usr_col_idx; ++ ++ switch (msg->pos) { ++ case WD_AGG_STREAM_INPUT: ++ agg_data = cols_data->input_data; ++ hw_agg_addr = &addr_list->input_addr[DAE_AGG_START_COL]; ++ usr_agg_addr = msg->req.agg_cols; ++ agg_col_num = msg->agg_cols_num; ++ fill_hashagg_normal_info(sqe, ext_sqe, cols_data, agg_col_num); ++ break; ++ case WD_AGG_REHASH_INPUT: ++ agg_data = cols_data->output_data; ++ hw_agg_addr = &addr_list->input_addr[DAE_AGG_START_COL]; ++ usr_agg_addr = msg->req.agg_cols; ++ agg_col_num = cols_data->output_num; ++ fill_hashagg_rehash_info(sqe, ext_sqe, agg_data, agg_col_num); ++ break; ++ case WD_AGG_STREAM_OUTPUT: ++ case WD_AGG_REHASH_OUTPUT: ++ agg_data = cols_data->output_data; ++ hw_agg_addr = &addr_list->output_addr[DAE_AGG_START_COL]; ++ usr_agg_addr = msg->req.out_agg_cols; ++ agg_col_num = cols_data->output_num; ++ fill_hashagg_normal_info(sqe, ext_sqe, cols_data, cols_data->input_num); ++ break; ++ } ++ ++ for (i = 0; i < agg_col_num; i++) { ++ usr_col_idx = agg_data[i].usr_col_idx; ++ hw_agg_addr[i].empty_addr = (__u64)(uintptr_t)usr_agg_addr[usr_col_idx].empty; ++ hw_agg_addr[i].empty_size = usr_agg_addr[usr_col_idx].empty_size; ++ hw_agg_addr[i].value_addr = (__u64)(uintptr_t)usr_agg_addr[usr_col_idx].value; ++ /* ++ * If only the count is performed on this agg column, set data type to SINT64 ++ * and change the value of size. ++ */ ++ if (msg->pos == WD_AGG_STREAM_INPUT && agg_data[i].optype == DAE_HASHAGG_COUNT) ++ hw_agg_addr[i].value_size = (__u64)msg->row_count * SINT64_SIZE; ++ else ++ hw_agg_addr[i].value_size = usr_agg_addr[usr_col_idx].value_size; ++ } ++} ++ ++static void fill_hashagg_ext_addr(struct dae_sqe *sqe, struct dae_ext_sqe *ext_sqe, ++ struct dae_addr_list *addr_list) ++{ ++ memset(ext_sqe, 0, DAE_EXT_SQE_SIZE); ++ memset(addr_list, 0, sizeof(struct dae_addr_list)); ++ sqe->addr_list = (__u64)(uintptr_t)addr_list; ++ addr_list->ext_sqe_addr = (__u64)(uintptr_t)ext_sqe; ++ addr_list->ext_sqe_size = DAE_EXT_SQE_SIZE; ++} ++ ++static int check_hashagg_param(struct wd_agg_msg *msg) ++{ ++ if (!msg) { ++ WD_ERR("invalid: input hashagg msg is NULL!\n"); ++ return -WD_EINVAL; ++ } ++ ++ if ((msg->pos == WD_AGG_STREAM_INPUT || msg->pos == WD_AGG_REHASH_INPUT) && ++ msg->row_count > DAE_HASHAGG_MAX_ROW_NUN) { ++ WD_ERR("invalid: input hashagg row count %u is more than %d!\n", ++ msg->row_count, DAE_HASHAGG_MAX_ROW_NUN); ++ return -WD_EINVAL; ++ } ++ ++ return WD_SUCCESS; ++} ++ ++static int hashagg_send(struct wd_alg_driver *drv, handle_t ctx, void *hashagg_msg) ++{ ++ handle_t h_qp = (handle_t)wd_ctx_get_priv(ctx); ++ struct hisi_qp *qp = (struct hisi_qp *)h_qp; ++ struct dae_extend_addr *ext_addr = qp->priv; ++ struct wd_agg_msg *msg = hashagg_msg; ++ struct dae_addr_list *addr_list; ++ struct dae_ext_sqe *ext_sqe; ++ struct dae_sqe sqe = {0}; ++ __u16 send_cnt = 0; ++ int ret, idx; ++ ++ ret = check_hashagg_param(msg); ++ if (ret) ++ return ret; ++ ++ fill_hashagg_task_type(msg, &sqe); ++ sqe.row_num = msg->row_count; ++ ++ idx = get_free_ext_addr(ext_addr); ++ if (idx < 0) ++ return -WD_EBUSY; ++ addr_list = &ext_addr->addr_list[idx]; ++ ext_sqe = &ext_addr->ext_sqe[idx]; ++ ++ fill_hashagg_ext_addr(&sqe, ext_sqe, addr_list); ++ fill_hashagg_table_data(&sqe, addr_list, msg); ++ fill_hashagg_key_data(&sqe, ext_sqe, addr_list, msg); ++ fill_hashagg_input_data(&sqe, ext_sqe, addr_list, msg); ++ fill_hashagg_output_order(&sqe, ext_sqe, msg); ++ ++ hisi_set_msg_id(h_qp, &msg->tag); ++ sqe.low_tag = msg->tag; ++ sqe.hi_tag = idx; ++ ++ ret = hisi_qm_send(h_qp, &sqe, 1, &send_cnt); ++ if (unlikely(ret)) { ++ if (ret != -WD_EBUSY) ++ WD_ERR("failed to send to hardware, ret = %d!\n", ret); ++ put_ext_addr(ext_addr, idx); ++ return ret; ++ } ++ ++ return WD_SUCCESS; ++} ++ ++static void fill_sum_overflow_cols(struct dae_sqe *sqe, struct wd_agg_msg *msg, ++ struct hashagg_ctx *agg_ctx) ++{ ++ __u32 i, output_num, usr_col_idx; ++ struct hw_agg_data *agg_data; ++ ++ pthread_spin_lock(&agg_ctx->lock); ++ agg_ctx->sum_overflow_cols |= sqe->sum_overflow_cols; ++ if (!agg_ctx->sum_overflow_cols) { ++ pthread_spin_unlock(&agg_ctx->lock); ++ return; ++ } ++ pthread_spin_unlock(&agg_ctx->lock); ++ ++ if (msg->result == WD_AGG_TASK_DONE) ++ msg->result = WD_AGG_SUM_OVERFLOW; ++ ++ if (!msg->req.sum_overflow_cols) ++ return; ++ ++ agg_data = agg_ctx->cols_data.output_data; ++ output_num = agg_ctx->cols_data.output_num; ++ for (i = 0; i < output_num; i++) { ++ usr_col_idx = agg_data[i].usr_col_idx; ++ if (agg_ctx->sum_overflow_cols & BIT(i)) ++ msg->req.sum_overflow_cols[usr_col_idx] = 1; ++ else ++ msg->req.sum_overflow_cols[usr_col_idx] = 0; ++ } ++} ++ ++static void fill_hashagg_msg_task_done(struct dae_sqe *sqe, struct wd_agg_msg *msg, ++ struct wd_agg_msg *temp_msg, struct hashagg_ctx *agg_ctx) ++{ ++ if (sqe->task_type_ext == DAE_HASHAGG_OUTPUT) { ++ msg->out_row_count = sqe->out_raw_num; ++ msg->output_done = sqe->output_end; ++ } else { ++ msg->in_row_count = temp_msg->row_count; ++ } ++} ++ ++static void fill_hashagg_msg_task_err(struct dae_sqe *sqe, struct wd_agg_msg *msg, ++ struct wd_agg_msg *temp_msg, struct hashagg_ctx *agg_ctx) ++{ ++ switch (sqe->err_type) { ++ case DAE_TASK_BD_ERROR_MIN ... DAE_TASK_BD_ERROR_MAX: ++ WD_ERR("failed to do hashagg task, bd error! etype=0x%x!\n", sqe->err_type); ++ msg->result = WD_AGG_PARSE_ERROR; ++ break; ++ case DAE_HASH_TABLE_NEED_REHASH: ++ msg->result = WD_AGG_NEED_REHASH; ++ break; ++ case DAE_HASH_TABLE_INVALID: ++ msg->result = WD_AGG_INVALID_HASH_TABLE; ++ break; ++ case DAE_HASHAGG_VCHAR_OVERFLOW: ++ WD_ERR("failed to do hashagg task, vchar size overflow! consumed row num: %u!\n", ++ sqe->vchar_err_offset); ++ msg->result = WD_AGG_INVALID_VARCHAR; ++ msg->in_row_count = sqe->vchar_err_offset; ++ break; ++ case DAE_HASHAGG_RESULT_OVERFLOW: ++ msg->in_row_count = temp_msg->row_count; ++ msg->result = WD_AGG_SUM_OVERFLOW; ++ break; ++ case DAE_HASHAGG_BUS_ERROR: ++ WD_ERR("failed to do hashagg task, bus error! etype %u!\n", sqe->err_type); ++ msg->result = WD_AGG_BUS_ERROR; ++ break; ++ case DAE_HASHAGG_VCHAR_LEN_ERROR: ++ WD_ERR("failed to do hashagg task, vchar col size error!\n"); ++ msg->result = WD_AGG_PARSE_ERROR; ++ break; ++ default: ++ WD_ERR("failed to do hashagg task! done_flag=0x%x, etype=0x%x, ext_type = 0x%x!\n", ++ sqe->done_flag, sqe->err_type, sqe->ext_err_type); ++ msg->result = WD_AGG_PARSE_ERROR; ++ break; ++ } ++ ++ if (sqe->task_type_ext == DAE_HASHAGG_OUTPUT) { ++ msg->out_row_count = sqe->out_raw_num; ++ msg->output_done = sqe->output_end; ++ } ++} ++ ++static int hashagg_recv(struct wd_alg_driver *drv, handle_t ctx, void *hashagg_msg) ++{ ++ handle_t h_qp = (handle_t)wd_ctx_get_priv(ctx); ++ struct hisi_qp *qp = (struct hisi_qp *)h_qp; ++ struct dae_extend_addr *ext_addr = qp->priv; ++ struct wd_agg_msg *msg = hashagg_msg; ++ struct wd_agg_msg *temp_msg = msg; ++ struct hashagg_ctx *agg_ctx; ++ struct dae_sqe sqe = {0}; ++ __u16 recv_cnt = 0; ++ int ret; ++ ++ ret = hisi_qm_recv(h_qp, &sqe, 1, &recv_cnt); ++ if (ret) ++ return ret; ++ ++ ret = hisi_check_bd_id(h_qp, msg->tag, sqe.low_tag); ++ if (ret) ++ goto out; ++ ++ msg->tag = sqe.low_tag; ++ if (qp->q_info.qp_mode == CTX_MODE_ASYNC) { ++ temp_msg = wd_agg_get_msg(qp->q_info.idx, msg->tag); ++ if (!temp_msg) { ++ msg->result = WD_AGG_IN_EPARA; ++ WD_ERR("failed to get send msg! idx = %u, tag = %u.\n", ++ qp->q_info.idx, msg->tag); ++ ret = -WD_EINVAL; ++ goto out; ++ } ++ } ++ ++ agg_ctx = (struct hashagg_ctx *)temp_msg->priv; ++ msg->result = WD_AGG_TASK_DONE; ++ msg->in_row_count = 0; ++ ++ if (likely(sqe.done_flag == DAE_HW_TASK_DONE)) { ++ fill_hashagg_msg_task_done(&sqe, msg, temp_msg, agg_ctx); ++ } else if (sqe.done_flag == DAE_HW_TASK_ERR) { ++ fill_hashagg_msg_task_err(&sqe, msg, temp_msg, agg_ctx); ++ } else { ++ msg->result = WD_AGG_PARSE_ERROR; ++ WD_ERR("failed to do hashagg task, hardware does not process the task!\n"); ++ } ++ ++ fill_sum_overflow_cols(&sqe, temp_msg, agg_ctx); ++ ++out: ++ put_ext_addr(ext_addr, sqe.hi_tag); ++ return ret; ++} ++ ++static int key_vchar_num_size_check(struct wd_key_col_info *key_info, __u32 cols_num) ++{ ++ __u32 i, count = 0; ++ ++ for (i = 0; i < cols_num; i++) { ++ switch (key_info[i].input_data_type) { ++ case WD_DAE_CHAR: ++ if (key_info[i].col_data_info > DAE_MAX_CHAR_SIZE) { ++ WD_ERR("invalid: key %u char size %u is more than support %d!\n", ++ i, key_info[i].col_data_info, DAE_MAX_CHAR_SIZE); ++ return -WD_EINVAL; ++ } ++ count++; ++ break; ++ case WD_DAE_VARCHAR: ++ if (key_info[i].col_data_info > DAE_MAX_VCHAR_SIZE) { ++ WD_ERR("invalid: key %u vchar size %u is more than support %d!\n", ++ i, key_info[i].col_data_info, DAE_MAX_VCHAR_SIZE); ++ return -WD_EINVAL; ++ } ++ count++; ++ break; ++ default: ++ break; ++ } ++ } ++ ++ if (count > DAE_MAX_CHAR_COL_NUM) { ++ WD_ERR("invalid: key char and vchar col num %u is more than device support %d!\n", ++ count, DAE_MAX_CHAR_COL_NUM); ++ return -WD_EINVAL; ++ } ++ ++ return WD_SUCCESS; ++} ++ ++static int agg_get_output_num(enum wd_dae_data_type type, ++ __u32 *size8, __u32 *size16) ++{ ++ switch (type) { ++ case WD_DAE_LONG: ++ case WD_DAE_SHORT_DECIMAL: ++ (*size8)++; ++ break; ++ case WD_DAE_LONG_DECIMAL: ++ (*size16)++; ++ break; ++ default: ++ WD_ERR("invalid: output data type %d not support!\n", type); ++ return -WD_EINVAL; ++ } ++ ++ return WD_SUCCESS; ++} ++ ++static int agg_output_num_check(struct wd_agg_col_info *agg_cols, __u32 cols_num, bool is_count_all) ++{ ++ __u32 size8 = 0, size16 = 0; ++ __u32 i, j, count_num; ++ int ret; ++ ++ for (i = 0; i < cols_num; i++) { ++ for (j = 0; j < agg_cols[i].col_alg_num; j++) { ++ ret = agg_get_output_num(agg_cols[i].output_data_types[j], ++ &size8, &size16); ++ if (ret) ++ return ret; ++ } ++ } ++ ++ if (is_count_all) ++ size8++; ++ ++ if (size8 > DAE_MAX_8B_COLS_NUM || size16 > DAE_MAX_16B_COLS_NUM) { ++ WD_ERR("invalid: output col num 8B-16B %u-%u is more than support %d-%d !\n", ++ size8, size16, DAE_MAX_8B_COLS_NUM, DAE_MAX_16B_COLS_NUM); ++ return -WD_EINVAL; ++ } ++ count_num = size8 + size16; ++ if (count_num > DAE_MAX_OUTPUT_COLS) { ++ WD_ERR("invalid: agg output cols num %u is more than device support %d!\n", ++ count_num, DAE_MAX_OUTPUT_COLS); ++ return -WD_EINVAL; ++ } ++ ++ return WD_SUCCESS; ++} ++ ++static int hashagg_init_param_check(struct wd_agg_sess_setup *setup) ++{ ++ int ret; ++ ++ if (setup->agg_cols_num > DAE_MAX_INPUT_COLS) { ++ WD_ERR("invalid: agg input cols num %u is more than device support %d!\n", ++ setup->agg_cols_num, DAE_MAX_INPUT_COLS); ++ return -WD_EINVAL; ++ } ++ ++ if (setup->key_cols_num > DAE_MAX_KEY_COLS) { ++ WD_ERR("invalid: key cols num %u is more than device support %d!\n", ++ setup->key_cols_num, DAE_MAX_KEY_COLS); ++ return -WD_EINVAL; ++ } ++ ++ if (setup->is_count_all && setup->count_all_data_type != WD_DAE_LONG) { ++ WD_ERR("invalid: count all output data type %d error, only support long!\n", ++ setup->count_all_data_type); ++ return -WD_EINVAL; ++ } ++ ++ ret = key_vchar_num_size_check(setup->key_cols_info, setup->key_cols_num); ++ if (ret) ++ return -WD_EINVAL; ++ ++ return agg_output_num_check(setup->agg_cols_info, setup->agg_cols_num, setup->is_count_all); ++} ++ ++static __u32 hashagg_get_data_type_size(enum dae_data_type type, __u16 data_info) ++{ ++ switch (type) { ++ case DAE_SINT32: ++ return SINT32_SIZE; ++ case DAE_SINT64: ++ case DAE_DECIMAL64: ++ return SINT64_SIZE; ++ case DAE_DECIMAL128: ++ return DECIMAL128_SIZE; ++ case DAE_CHAR: ++ return ALIGN(data_info, DAE_CHAR_ALIGN_SIZE); ++ case DAE_VCHAR: ++ return data_info; ++ } ++ ++ return 0; ++} ++ ++static int transfer_key_col_info(struct wd_key_col_info *key_cols, ++ struct hw_agg_data *key_data, __u32 col_num) ++{ ++ __u32 i; ++ ++ for (i = 0; i < col_num; i++) { ++ switch (key_cols[i].input_data_type) { ++ case WD_DAE_VARCHAR: ++ if (!key_cols[i].col_data_info) ++ key_data[i].data_info = ALIGN(DAE_MAX_VCHAR_SIZE + ++ DAE_VCHAR_OFFSET_SIZE, DAE_CHAR_ALIGN_SIZE); ++ else ++ key_data[i].data_info = ALIGN(key_cols[i].col_data_info + ++ DAE_VCHAR_OFFSET_SIZE, DAE_CHAR_ALIGN_SIZE); ++ key_data[i].hw_type = DAE_VCHAR; ++ break; ++ case WD_DAE_CHAR: ++ key_data[i].data_info = key_cols[i].col_data_info; ++ key_data[i].hw_type = DAE_CHAR; ++ break; ++ case WD_DAE_LONG_DECIMAL: ++ key_data[i].hw_type = DAE_DECIMAL128; ++ break; ++ case WD_DAE_SHORT_DECIMAL: ++ key_data[i].hw_type = DAE_DECIMAL64; ++ break; ++ case WD_DAE_LONG: ++ key_data[i].hw_type = DAE_SINT64; ++ break; ++ case WD_DAE_INT: ++ case WD_DAE_DATE: ++ key_data[i].hw_type = DAE_SINT32; ++ break; ++ default: ++ WD_ERR("invalid: unsupport key col %u data type %d!\n", ++ i, key_cols[i].input_data_type); ++ return -WD_EINVAL; ++ } ++ } ++ ++ return WD_SUCCESS; ++} ++ ++static int transfer_key_to_hw_type(struct hashagg_col_data *cols_data, ++ struct wd_agg_sess_setup *setup) ++{ ++ struct wd_key_col_info *key_cols = setup->key_cols_info; ++ struct hw_agg_data *hw_key_data = cols_data->key_data; ++ struct hw_agg_data tmp_key_data[DAE_MAX_KEY_COLS] = {0}; ++ __u32 type_num = ARRAY_SIZE(hw_data_type_order); ++ __u32 cols_num = setup->key_cols_num; ++ __u32 i, j, k = 0; ++ int ret; ++ ++ ret = transfer_key_col_info(key_cols, tmp_key_data, cols_num); ++ if (ret) ++ return ret; ++ ++ for (i = 0; i < type_num; i++) { ++ for (j = 0; j < cols_num; j++) { ++ if (hw_data_type_order[i] != tmp_key_data[j].hw_type) ++ continue; ++ hw_key_data[k].usr_col_idx = j; ++ hw_key_data[k].hw_type = tmp_key_data[j].hw_type; ++ hw_key_data[k++].data_info = tmp_key_data[j].data_info; ++ } ++ } ++ ++ cols_data->key_num = cols_num; ++ ++ return WD_SUCCESS; ++} ++ ++static int hashagg_decimal_precision_check(__u16 data_info, bool longdecimal) ++{ ++ __u8 all_precision; ++ ++ /* ++ * low 8bits: overall precision ++ * high 8bits: precision of the decimal part ++ */ ++ all_precision = data_info; ++ if (longdecimal) { ++ if (all_precision > DAE_DECIMAL128_MAX_PRECISION) { ++ WD_ERR("invalid: longdecimal precision %u is more than support %d!\n", ++ all_precision, DAE_DECIMAL128_MAX_PRECISION); ++ return -WD_EINVAL; ++ } ++ return WD_SUCCESS; ++ } ++ ++ if (all_precision > DAE_DECIMAL64_MAX_PRECISION) { ++ WD_ERR("invalid: shortdecimal precision %u is more than support %d!\n", ++ all_precision, DAE_DECIMAL64_MAX_PRECISION); ++ return -WD_EINVAL; ++ } ++ ++ return WD_SUCCESS; ++} ++ ++static int hashagg_check_sum_info(struct wd_agg_col_info *agg_col, ++ struct hw_agg_data *user_input_data, ++ struct hw_agg_data *user_output_data, __u32 index) ++{ ++ int ret; ++ ++ switch (agg_col->input_data_type) { ++ case WD_DAE_LONG: ++ if (agg_col->output_data_types[index] != WD_DAE_LONG) { ++ WD_ERR("invalid: long type do sum output data type %d error!\n", ++ agg_col->output_data_types[index]); ++ return -WD_EINVAL; ++ } ++ user_input_data->hw_type = DAE_SINT64; ++ user_output_data->hw_type = DAE_SINT64; ++ break; ++ case WD_DAE_SHORT_DECIMAL: ++ if (agg_col->output_data_types[index] == WD_DAE_SHORT_DECIMAL) { ++ ret = hashagg_decimal_precision_check(agg_col->col_data_info, false); ++ if (ret) ++ return ret; ++ user_input_data->sum_outtype = DECIMAL64_TO_DECIMAL64; ++ user_output_data->hw_type = DAE_DECIMAL64; ++ /* For rehash, rehash will do sum */ ++ user_output_data->sum_outtype = DECIMAL64_TO_DECIMAL64; ++ } else if (agg_col->output_data_types[index] == WD_DAE_LONG_DECIMAL) { ++ ret = hashagg_decimal_precision_check(agg_col->col_data_info, true); ++ if (ret) ++ return ret; ++ user_input_data->sum_outtype = DECIMAL64_TO_DECIMAL128; ++ user_output_data->hw_type = DAE_DECIMAL128; ++ } else { ++ WD_ERR("invalid: short decimal do sum output data type %d error!\n", ++ agg_col->output_data_types[index]); ++ return -WD_EINVAL; ++ } ++ user_input_data->hw_type = DAE_DECIMAL64; ++ break; ++ case WD_DAE_LONG_DECIMAL: ++ if (agg_col->output_data_types[index] != WD_DAE_LONG_DECIMAL) { ++ WD_ERR("invalid: long decimal do sum output data type %d error!\n", ++ agg_col->output_data_types[index]); ++ return -WD_EINVAL; ++ } ++ ret = hashagg_decimal_precision_check(agg_col->col_data_info, true); ++ if (ret) ++ return ret; ++ user_input_data->hw_type = DAE_DECIMAL128; ++ user_output_data->hw_type = DAE_DECIMAL128; ++ break; ++ default: ++ WD_ERR("invalid: device not support col data type %d do sum!\n", ++ agg_col->input_data_type); ++ return -WD_EINVAL; ++ } ++ ++ return WD_SUCCESS; ++} ++ ++static int hashagg_check_count_info(enum wd_dae_data_type input_type, ++ enum wd_dae_data_type output_type) ++{ ++ if (input_type > WD_DAE_VARCHAR) { ++ WD_ERR("invalid: device not support agg col data type %d do count!\n", input_type); ++ return -WD_EINVAL; ++ } ++ ++ if (output_type != WD_DAE_LONG) { ++ WD_ERR("invalid: input data type %d do count output data type %d error!\n", ++ input_type, output_type); ++ return -WD_EINVAL; ++ } ++ ++ return WD_SUCCESS; ++} ++ ++static int hashagg_check_input_data(struct wd_agg_col_info *agg_col, ++ struct hw_agg_data *user_input_data, ++ struct hw_agg_data *user_output_data, __u32 index) ++{ ++ int ret; ++ ++ switch (agg_col->output_col_algs[index]) { ++ case WD_AGG_SUM: ++ ret = hashagg_check_sum_info(agg_col, user_input_data, user_output_data, index); ++ if (ret) ++ return ret; ++ user_input_data->optype |= DAE_HASHAGG_SUM; ++ user_output_data->optype = DAE_SUM; ++ break; ++ case WD_AGG_COUNT: ++ ret = hashagg_check_count_info(agg_col->input_data_type, ++ agg_col->output_data_types[index]); ++ if (ret) ++ return ret; ++ user_input_data->optype |= DAE_HASHAGG_COUNT; ++ user_output_data->hw_type = DAE_SINT64; ++ user_output_data->optype = DAE_COUNT; ++ break; ++ default: ++ WD_ERR("invalid: device not support alg %d!\n", agg_col->output_col_algs[index]); ++ return -WD_EINVAL; ++ } ++ user_output_data->data_info = agg_col->col_data_info; ++ ++ return WD_SUCCESS; ++} ++ ++static int transfer_input_col_info(struct wd_agg_col_info *agg_cols, ++ struct hw_agg_data *user_input_data, ++ struct hw_agg_data *user_output_data, ++ __u32 cols_num, __u32 *output_num) ++{ ++ __u32 i, j, k = 0; ++ int ret; ++ ++ for (i = 0; i < cols_num; i++) { ++ for (j = 0; j < agg_cols[i].col_alg_num; j++) { ++ ret = hashagg_check_input_data(&agg_cols[i], &user_input_data[i], ++ &user_output_data[k], j); ++ if (ret) ++ return ret; ++ user_output_data[k++].usr_col_idx = i; ++ } ++ user_input_data[i].data_info = agg_cols[i].col_data_info; ++ /* ++ * If agg col only do count, change the type to DAE_SINT64 and ++ * send it to the hardware. ++ */ ++ if (user_input_data[i].optype == DAE_HASHAGG_COUNT) ++ user_input_data[i].hw_type = DAE_SINT64; ++ } ++ ++ *output_num = k; ++ ++ return WD_SUCCESS; ++} ++ ++static void hashagg_swap_out_index(struct hw_agg_data *user_output_data, ++ __u32 cols_num, __u32 *usr_idx, ++ __u32 old_idx, __u32 new_idx) ++{ ++ __u32 i; ++ ++ for (i = 0; i < cols_num; i++) { ++ if (usr_idx[i] == old_idx) ++ user_output_data[i].usr_col_idx = new_idx; ++ } ++} ++ ++static void transfer_input_to_hw_order(struct hashagg_col_data *cols_data, ++ struct hw_agg_data *user_input_data, ++ struct hw_agg_data *user_output_data) ++{ ++ struct hw_agg_data *input_data = cols_data->input_data; ++ __u32 type_num = ARRAY_SIZE(hw_data_type_order); ++ __u32 cols_num = cols_data->input_num; ++ __u32 usr_col_idx[DAE_MAX_INPUT_COLS]; ++ __u32 i, j, k = 0; ++ ++ /* ++ * Since device need input col idx, so save usr input idx, ++ * then update to send to hw idx. ++ */ ++ for (i = 0; i < cols_data->output_num; i++) ++ usr_col_idx[i] = user_output_data[i].usr_col_idx; ++ ++ for (i = 0; i < type_num; i++) { ++ for (j = 0; j < cols_num; j++) { ++ if (hw_data_type_order[i] != user_input_data[j].hw_type) ++ continue; ++ /* ++ * Since aggregated output need input hardware agg index, ++ * so fill the final idx. ++ */ ++ hashagg_swap_out_index(user_output_data, cols_data->output_num, ++ usr_col_idx, j, k); ++ input_data[k].usr_col_idx = j; ++ input_data[k].sum_outtype = user_input_data[j].sum_outtype; ++ input_data[k].data_info = user_input_data[j].data_info; ++ input_data[k].hw_type = user_input_data[j].hw_type; ++ input_data[k].optype = user_input_data[j].optype; ++ k++; ++ } ++ } ++} ++ ++static void transfer_output_to_hw_order(struct hashagg_col_data *cols_data, ++ struct hw_agg_data *usr_output_data, ++ bool is_count_all) ++{ ++ struct hashagg_output_src *normal_output = cols_data->normal_output; ++ struct hashagg_output_src *rehash_output = cols_data->rehash_output; ++ struct hw_agg_data *output_data = cols_data->output_data; ++ struct hw_agg_data *input_data = cols_data->input_data; ++ __u32 type_num = ARRAY_SIZE(hw_data_type_order); ++ __u32 cols_num = cols_data->output_num; ++ __u32 i, j, k = 0; ++ ++ if (is_count_all) { ++ /* Count all output will fill in the last column */ ++ output_data[cols_num].hw_type = DAE_SINT64; ++ output_data[cols_num].usr_col_idx = cols_num; ++ rehash_output[cols_num].out_optype = DAE_SUM; ++ rehash_output[cols_num].out_from_in_idx = cols_num; ++ cols_data->output_num++; ++ cols_data->is_count_all = true; ++ } ++ ++ for (i = 0; i < type_num; i++) { ++ for (j = 0; j < cols_num; j++) { ++ if (hw_data_type_order[i] != usr_output_data[j].hw_type) ++ continue; ++ /* ++ * Rehash only performs the sum operation. The number of output columns ++ * is the same as the number of input columns. The order does not ++ * need to be adjusted. ++ */ ++ rehash_output[k].out_from_in_idx = k; ++ rehash_output[k].out_optype = DAE_SUM; ++ ++ normal_output[k].out_from_in_idx = usr_output_data[j].usr_col_idx; ++ normal_output[k].out_optype = usr_output_data[j].optype; ++ output_data[k].hw_type = usr_output_data[j].hw_type; ++ output_data[k].usr_col_idx = j; ++ output_data[k].data_info = ++ input_data[usr_output_data[j].usr_col_idx].data_info; ++ k++; ++ } ++ } ++} ++ ++static int transfer_data_to_hw_type(struct hashagg_col_data *cols_data, ++ struct wd_agg_sess_setup *setup) ++{ ++ struct hw_agg_data user_input_data[DAE_MAX_INPUT_COLS] = {0}; ++ struct hw_agg_data user_output_data[DAE_MAX_OUTPUT_COLS] = {0}; ++ struct wd_agg_col_info *agg_cols = setup->agg_cols_info; ++ int ret; ++ ++ ret = transfer_input_col_info(agg_cols, user_input_data, user_output_data, ++ setup->agg_cols_num, &cols_data->output_num); ++ if (ret) ++ return ret; ++ ++ cols_data->input_num = setup->agg_cols_num; ++ ++ transfer_input_to_hw_order(cols_data, user_input_data, user_output_data); ++ transfer_output_to_hw_order(cols_data, user_output_data, setup->is_count_all); ++ ++ return WD_SUCCESS; ++} ++ ++static int hashagg_get_table_rowsize(struct hashagg_col_data *cols_data) ++{ ++ struct hw_agg_data *output_col = cols_data->output_data; ++ struct hw_agg_data *key_data = cols_data->key_data; ++ __u32 output_num = cols_data->output_num; ++ __u32 key_num = cols_data->key_num; ++ __u32 row_count_size = 0; ++ __u32 i; ++ ++ for (i = 0; i < key_num; i++) ++ row_count_size += hashagg_get_data_type_size(key_data[i].hw_type, ++ key_data[i].data_info); ++ ++ for (i = 0; i < output_num; i++) ++ row_count_size += hashagg_get_data_type_size(output_col[i].hw_type, ++ output_col[i].data_info); ++ ++ row_count_size += HASH_TABLE_EMPTY_SIZE; ++ if (row_count_size < DAE_MIN_ROW_SIZE || row_count_size > DAE_MAX_ROW_SIZE) { ++ WD_ERR("invalid: device not support hash table row size %u!\n", row_count_size); ++ return -WD_EINVAL; ++ } ++ ++ row_count_size += HASH_TABLE_HEAD_TAIL_SIZE; ++ if (row_count_size <= ROW_SIZE32) ++ return ROW_SIZE32; ++ ++ if (row_count_size <= ROW_SIZE64) ++ return ROW_SIZE64; ++ ++ if (row_count_size <= ROW_SIZE128) ++ return ROW_SIZE128; ++ ++ if (row_count_size <= ROW_SIZE256) ++ return ROW_SIZE256; ++ ++ return ROW_SIZE512; ++} ++ ++static int hashagg_fill_agg_ctx(struct hashagg_ctx *agg_ctx, struct wd_agg_sess_setup *setup) ++{ ++ struct hashagg_col_data *cols_data = &agg_ctx->cols_data; ++ int ret; ++ ++ ret = transfer_key_to_hw_type(cols_data, setup); ++ if (ret) ++ return ret; ++ ++ ret = transfer_data_to_hw_type(cols_data, setup); ++ if (ret) ++ return ret; ++ ++ ret = hashagg_get_table_rowsize(cols_data); ++ if (ret < 0) ++ return -WD_EINVAL; ++ agg_ctx->row_size = ret; ++ ++ agg_ctx->sum_overflow_cols = 0; ++ ++ return WD_SUCCESS; ++} ++ ++static void hashagg_sess_priv_uninit(void *priv) ++{ ++ struct hashagg_ctx *agg_ctx = priv; ++ ++ if (!agg_ctx) { ++ WD_ERR("invalid: dae sess uninit priv is NULL!\n"); ++ return; ++ } ++ ++ pthread_spin_destroy(&agg_ctx->lock); ++ free(agg_ctx); ++} ++ ++static int hashagg_sess_priv_init(struct wd_agg_sess_setup *setup, void **priv) ++{ ++ struct hashagg_ctx *agg_ctx; ++ int ret; ++ ++ if (!setup || !priv) { ++ WD_ERR("invalid: dae sess priv is NULL!\n"); ++ return -WD_EINVAL; ++ } ++ ++ ret = hashagg_init_param_check(setup); ++ if (ret) ++ return -WD_EINVAL; ++ ++ agg_ctx = calloc(1, sizeof(struct hashagg_ctx)); ++ if (!agg_ctx) ++ return -WD_ENOMEM; ++ ++ ret = hashagg_fill_agg_ctx(agg_ctx, setup); ++ if (ret) ++ goto free_agg_ctx; ++ ++ ret = pthread_spin_init(&agg_ctx->lock, PTHREAD_PROCESS_SHARED); ++ if (ret) ++ goto free_agg_ctx; ++ ++ *priv = agg_ctx; ++ ++ return WD_SUCCESS; ++ ++free_agg_ctx: ++ free(agg_ctx); ++ return ret; ++} ++ ++static void dae_uninit_qp_priv(handle_t h_qp) ++{ ++ struct hisi_qp *qp = (struct hisi_qp *)h_qp; ++ struct dae_extend_addr *ext_addr = (struct dae_extend_addr *)qp->priv; ++ ++ free(ext_addr->addr_list); ++ free(ext_addr->addr_status); ++ free(ext_addr->ext_sqe); ++ free(ext_addr); ++ qp->priv = NULL; ++} ++ ++static int dae_init_qp_priv(handle_t h_qp) ++{ ++ struct hisi_qp *qp = (struct hisi_qp *)h_qp; ++ __u16 sq_depth = qp->q_info.sq_depth; ++ struct dae_extend_addr *ext_addr; ++ int ret = -WD_ENOMEM; ++ ++ ext_addr = calloc(1, sizeof(struct dae_extend_addr)); ++ if (!ext_addr) ++ return ret; ++ ++ ext_addr->ext_sqe = aligned_alloc(DAE_ADDR_ALIGN_SIZE, DAE_EXT_SQE_SIZE * sq_depth); ++ if (!ext_addr->ext_sqe) ++ goto free_ext_addr; ++ ++ ext_addr->addr_status = calloc(1, sizeof(__u8) * sq_depth); ++ if (!ext_addr->addr_status) ++ goto free_ext_sqe; ++ ++ ext_addr->addr_list = aligned_alloc(DAE_ADDR_ALIGN_SIZE, ++ sizeof(struct dae_addr_list) * sq_depth); ++ if (!ext_addr->addr_list) ++ goto free_addr_status; ++ ++ ext_addr->addr_num = sq_depth; ++ qp->priv = ext_addr; ++ ++ return WD_SUCCESS; ++ ++free_addr_status: ++ free(ext_addr->addr_status); ++free_ext_sqe: ++ free(ext_addr->ext_sqe); ++free_ext_addr: ++ free(ext_addr); ++ ++ return ret; ++} ++ ++static int dae_get_row_size(void *param) ++{ ++ struct hashagg_ctx *agg_ctx = param; ++ ++ if (!agg_ctx) ++ return -WD_EINVAL; ++ ++ return agg_ctx->row_size; ++} ++ ++static __u32 dae_ext_table_rownum(void **ext_table, struct wd_dae_hash_table *hash_table, ++ __u32 row_size) ++{ ++ __u64 tlb_size, tmp_size, row_num; ++ void *tmp_table; ++ ++ /* ++ * The first row of the extended hash table stores the hash table information, ++ * and the second row stores the aggregated data. The 128-bytes aligned address ++ * in the second row provides the optimal performance. ++ */ ++ tmp_table = PTR_ALIGN(hash_table->ext_table, DAE_TABLE_ALIGN_SIZE); ++ tlb_size = (__u64)hash_table->table_row_size * hash_table->ext_table_row_num; ++ tmp_size = (__u64)(uintptr_t)tmp_table - (__u64)(uintptr_t)hash_table->ext_table; ++ if (tmp_size >= tlb_size) ++ return 0; ++ ++ row_num = (tlb_size - tmp_size) / row_size; ++ if (row_size == ROW_SIZE32) { ++ if (tmp_size >= row_size) { ++ tmp_table = (__u8 *)tmp_table - row_size; ++ row_num += 1; ++ } else { ++ /* ++ * When row size is 32 bytes, the first 96 bytes are not used. ++ * Ensure that the address of the second row is 128 bytes aligned. ++ */ ++ if (row_num > HASH_TABLE_OFFSET_3ROW) { ++ tmp_table = (__u8 *)tmp_table + HASH_TABLE_OFFSET_3ROW * row_size; ++ row_num -= HASH_TABLE_OFFSET_3ROW; ++ } else { ++ return 0; ++ } ++ } ++ } else if (row_size == ROW_SIZE64) { ++ if (tmp_size >= row_size) { ++ tmp_table = (__u8 *)tmp_table - row_size; ++ row_num += 1; ++ } else { ++ /* ++ * When row size is 64 bytes, the first 64 bytes are not used. ++ * Ensure that the address of the second row is 128 bytes aligned. ++ */ ++ if (row_num > HASH_TABLE_OFFSET_1ROW) { ++ tmp_table = (__u8 *)tmp_table + HASH_TABLE_OFFSET_1ROW * row_size; ++ row_num -= HASH_TABLE_OFFSET_1ROW; ++ } else { ++ return 0; ++ } ++ } ++ } ++ ++ *ext_table = tmp_table; ++ ++ return row_num; ++} ++ ++static int dae_ext_table_init(struct hashagg_ctx *agg_ctx, ++ struct wd_dae_hash_table *hash_table, bool is_rehash) ++{ ++ struct hash_table_data *hw_table = &agg_ctx->table_data; ++ __u64 ext_size = hw_table->ext_table_size; ++ __u32 row_size = agg_ctx->row_size; ++ __u64 tlb_size, row_num; ++ void *ext_table; ++ __u8 *ext_valid; ++ __u64 *ext_row; ++ ++ row_num = dae_ext_table_rownum(&ext_table, hash_table, row_size); ++ if (row_num <= 1) { ++ WD_ERR("invalid: after aligned, extend table row num is less than device need!\n"); ++ return -WD_EINVAL; ++ } ++ ++ tlb_size = row_num * row_size; ++ if (is_rehash && tlb_size <= ext_size) { ++ WD_ERR("invalid: rehash extend table size %llu is not longer than current %llu!\n", ++ tlb_size, ext_size); ++ return -WD_EINVAL; ++ } ++ ++ /* ++ * If table has been initialized, save the previous data ++ * before replacing the new table. ++ */ ++ if (is_rehash) ++ memcpy(&agg_ctx->rehash_table, hw_table, sizeof(struct hash_table_data)); ++ ++ /* Initialize the extend table value. */ ++ memset(ext_table, 0, tlb_size); ++ ext_valid = (__u8 *)ext_table + HASH_EXT_TABLE_INVALID_OFFSET; ++ *ext_valid = HASH_EXT_TABLE_VALID; ++ ext_row = (__u64 *)ext_table + 1; ++ *ext_row = row_num - 1; ++ ++ hw_table->ext_table = ext_table; ++ hw_table->ext_table_size = tlb_size; ++ ++ return WD_SUCCESS; ++} ++ ++static int dae_std_table_init(struct hash_table_data *hw_table, ++ struct wd_dae_hash_table *hash_table, __u32 row_size) ++{ ++ __u64 tlb_size, row_num, tmp_size; ++ ++ /* ++ * Hash table address must be 128-bytes aligned, and the number ++ * of rows in a standard hash table must be a power of 2. ++ */ ++ hw_table->std_table = PTR_ALIGN(hash_table->std_table, DAE_TABLE_ALIGN_SIZE); ++ tlb_size = (__u64)hash_table->table_row_size * hash_table->std_table_row_num; ++ tmp_size = (__u64)(uintptr_t)hw_table->std_table - (__u64)(uintptr_t)hash_table->std_table; ++ if (tmp_size >= tlb_size) { ++ WD_ERR("invalid: after aligned, standard table size is less than 0!\n"); ++ return -WD_EINVAL; ++ } ++ ++ row_num = (tlb_size - tmp_size) / row_size; ++ if (!row_num) { ++ WD_ERR("invalid: standard table row num is 0!\n"); ++ return -WD_EINVAL; ++ } ++ ++ hw_table->table_width = log2(row_num); ++ if (hw_table->table_width < HASH_TABLE_MIN_WIDTH || ++ hw_table->table_width > HASH_TABLE_MAX_WIDTH) { ++ WD_ERR("invalid: standard table width %u is out of device support range %d~%d!\n", ++ hw_table->table_width, HASH_TABLE_MIN_WIDTH, HASH_TABLE_MAX_WIDTH); ++ return -WD_EINVAL; ++ } ++ ++ row_num = pow(HASH_TABLE_WITDH_POWER, hw_table->table_width); ++ hw_table->std_table_size = row_num * row_size; ++ memset(hw_table->std_table, 0, hw_table->std_table_size); ++ ++ return WD_SUCCESS; ++} ++ ++static int dae_hash_table_init(struct wd_dae_hash_table *hash_table, void *priv) ++{ ++ struct hashagg_ctx *agg_ctx = priv; ++ struct hash_table_data *hw_table; ++ bool is_rehash = false; ++ int ret; ++ ++ if (!agg_ctx || !hash_table) ++ return -WD_EINVAL; ++ ++ if (!agg_ctx->row_size || agg_ctx->row_size > hash_table->table_row_size) { ++ WD_ERR("invalid: row size %u is error, device need %u!\n", ++ hash_table->table_row_size, agg_ctx->row_size); ++ return -WD_EINVAL; ++ } ++ ++ /* hash_std_table is checked by caller */ ++ if (!hash_table->ext_table || !hash_table->ext_table_row_num) { ++ WD_ERR("invalid: hash extend table is null!\n"); ++ return -WD_EINVAL; ++ } ++ ++ hw_table = &agg_ctx->table_data; ++ if (hw_table->std_table_size) ++ is_rehash = true; ++ ++ ret = dae_ext_table_init(agg_ctx, hash_table, is_rehash); ++ if (ret) ++ return ret; ++ ++ ret = dae_std_table_init(hw_table, hash_table, agg_ctx->row_size); ++ if (ret) ++ goto update_table; ++ ++ return WD_SUCCESS; ++ ++update_table: ++ if (is_rehash) ++ memcpy(hw_table, &agg_ctx->rehash_table, sizeof(struct hash_table_data)); ++ else ++ memset(hw_table, 0, sizeof(struct hash_table_data)); ++ return ret; ++} ++ ++static int dae_init(struct wd_alg_driver *drv, void *conf) ++{ ++ struct wd_ctx_config_internal *config = conf; ++ struct hisi_qm_priv qm_priv; ++ struct hisi_dae_ctx *priv; ++ handle_t h_qp = 0; ++ handle_t h_ctx; ++ __u32 i, j; ++ int ret; ++ ++ if (!config || !config->ctx_num) { ++ WD_ERR("invalid: dae init config is null or ctx num is 0!\n"); ++ return -WD_EINVAL; ++ } ++ ++ priv = malloc(sizeof(struct hisi_dae_ctx)); ++ if (!priv) ++ return -WD_ENOMEM; ++ ++ qm_priv.op_type = DAE_HASH_AGG_TYPE; ++ qm_priv.sqe_size = sizeof(struct dae_sqe); ++ /* Allocate qp for each context */ ++ for (i = 0; i < config->ctx_num; i++) { ++ h_ctx = config->ctxs[i].ctx; ++ qm_priv.qp_mode = config->ctxs[i].ctx_mode; ++ /* Setting the epoll en to 0 for ASYNC ctx */ ++ qm_priv.epoll_en = (qm_priv.qp_mode == CTX_MODE_SYNC) ? ++ config->epoll_en : 0; ++ qm_priv.idx = i; ++ h_qp = hisi_qm_alloc_qp(&qm_priv, h_ctx); ++ if (!h_qp) { ++ ret = -WD_ENOMEM; ++ goto out; ++ } ++ config->ctxs[i].sqn = qm_priv.sqn; ++ ret = dae_init_qp_priv(h_qp); ++ if (ret) ++ goto free_h_qp; ++ } ++ memcpy(&priv->config, config, sizeof(struct wd_ctx_config_internal)); ++ drv->priv = priv; ++ ++ return WD_SUCCESS; ++ ++free_h_qp: ++ hisi_qm_free_qp(h_qp); ++out: ++ for (j = 0; j < i; j++) { ++ h_qp = (handle_t)wd_ctx_get_priv(config->ctxs[j].ctx); ++ dae_uninit_qp_priv(h_qp); ++ hisi_qm_free_qp(h_qp); ++ } ++ free(priv); ++ return ret; ++} ++ ++static void dae_exit(struct wd_alg_driver *drv) ++{ ++ struct hisi_dae_ctx *priv = (struct hisi_dae_ctx *)drv->priv; ++ struct wd_ctx_config_internal *config; ++ handle_t h_qp; ++ __u32 i; ++ ++ if (!priv) ++ return; ++ ++ config = &priv->config; ++ for (i = 0; i < config->ctx_num; i++) { ++ h_qp = (handle_t)wd_ctx_get_priv(config->ctxs[i].ctx); ++ dae_uninit_qp_priv(h_qp); ++ hisi_qm_free_qp(h_qp); ++ } ++ ++ free(priv); ++ drv->priv = NULL; ++} ++ ++static int dae_get_usage(void *param) ++{ ++ return 0; ++} ++ ++static int dae_get_extend_ops(void *ops) ++{ ++ struct wd_agg_ops *agg_ops = (struct wd_agg_ops *)ops; ++ ++ if (!agg_ops) ++ return -WD_EINVAL; ++ ++ agg_ops->get_row_size = dae_get_row_size; ++ agg_ops->hash_table_init = dae_hash_table_init; ++ agg_ops->sess_init = hashagg_sess_priv_init; ++ agg_ops->sess_uninit = hashagg_sess_priv_uninit; ++ ++ return WD_SUCCESS; ++} ++ ++static struct wd_alg_driver hashagg_driver = { ++ .drv_name = "hisi_zip", ++ .alg_name = "hashagg", ++ .calc_type = UADK_ALG_HW, ++ .priority = 100, ++ .queue_num = DAE_CTX_Q_NUM_DEF, ++ .op_type_num = 1, ++ .fallback = 0, ++ .init = dae_init, ++ .exit = dae_exit, ++ .send = hashagg_send, ++ .recv = hashagg_recv, ++ .get_usage = dae_get_usage, ++ .get_extend_ops = dae_get_extend_ops, ++}; ++ ++#ifdef WD_STATIC_DRV ++void hisi_dae_probe(void) ++#else ++static void __attribute__((constructor)) hisi_dae_probe(void) ++#endif ++{ ++ int ret; ++ ++ WD_INFO("Info: register DAE alg drivers!\n"); ++ ++ ret = wd_alg_driver_register(&hashagg_driver); ++ if (ret && ret != -WD_ENODEV) ++ WD_ERR("failed to register DAE hashagg driver!\n"); ++} ++ ++#ifdef WD_STATIC_DRV ++void hisi_dae_remove(void) ++#else ++static void __attribute__((destructor)) hisi_dae_remove(void) ++#endif ++{ ++ WD_INFO("Info: unregister DAE alg drivers!\n"); ++ ++ wd_alg_driver_unregister(&hashagg_driver); ++} +diff --git a/drv/hisi_qm_udrv.h b/drv/hisi_qm_udrv.h +index b94e8e2..b02e8e7 100644 +--- a/drv/hisi_qm_udrv.h ++++ b/drv/hisi_qm_udrv.h +@@ -86,6 +86,8 @@ struct hisi_qp { + struct hisi_qm_queue_info q_info; + handle_t h_sgl_pool; + handle_t h_ctx; ++ /* Private area for driver use, point to queue specifial data */ ++ void *priv; + }; + + /* Capabilities */ +-- +2.25.1 + diff --git a/0019-uadk_tool-use-wd_xxx_init2_-instead-of-wd_xxx_init2.patch b/0019-uadk_tool-use-wd_xxx_init2_-instead-of-wd_xxx_init2.patch new file mode 100644 index 0000000..378522a --- /dev/null +++ b/0019-uadk_tool-use-wd_xxx_init2_-instead-of-wd_xxx_init2.patch @@ -0,0 +1,235 @@ +From c0c3359090a8e6fcaf59c606dccee8e554c89ff2 Mon Sep 17 00:00:00 2001 +From: Zhangfei Gao +Date: Wed, 28 Aug 2024 09:42:39 +0000 +Subject: [PATCH 19/39] uadk_tool: use wd_xxx_init2_ instead of wd_xxx_init2 + +wd_xxx_init2 does not take para cparam, which take ctx number. +So default ctx number is used with poor performance. + +numactl --cpubind=0 --membind=0 \ +uadk_tool benchmark --alg sha512 --mode sva --opt 0 --async --pktlen 8192 \ +--seconds 20 --multi 1 --thread 8 --ctxnum 8 --init2 +sha512 8192Bytes 3982625.60KiB/s 497.8Kops 1292.90% + +after fix: +numactl --cpubind=0 --membind=0 \ +uadk_tool benchmark --alg sha512 --mode sva --opt 0 --async --pktlen 8192 \ +--seconds 20 --multi 1 --thread 8 --ctxnum 8 --init2 +sha512 8192Bytes 11276124.40KiB/s 1409.5Kops 1026.70% + +Signed-off-by: Zhangfei Gao +--- + uadk_tool/benchmark/hpre_uadk_benchmark.c | 36 +++++++++++++++++++++-- + uadk_tool/benchmark/sec_uadk_benchmark.c | 35 ++++++++++++++++++++-- + uadk_tool/benchmark/zip_uadk_benchmark.c | 36 +++++++++++++++++++++-- + 3 files changed, 99 insertions(+), 8 deletions(-) + +diff --git a/uadk_tool/benchmark/hpre_uadk_benchmark.c b/uadk_tool/benchmark/hpre_uadk_benchmark.c +index 5dd6a39..bc0687d 100644 +--- a/uadk_tool/benchmark/hpre_uadk_benchmark.c ++++ b/uadk_tool/benchmark/hpre_uadk_benchmark.c +@@ -610,7 +610,10 @@ static void uninit_hpre_ctx_config2(int subtype) + + static int init_hpre_ctx_config2(struct acc_option *options) + { ++ struct wd_ctx_params cparams = {0}; ++ struct wd_ctx_nums *ctx_set_num; + int subtype = options->subtype; ++ int mode = options->syncmode; + char alg_name[MAX_ALG_NAME]; + int ret; + +@@ -620,22 +623,49 @@ static int init_hpre_ctx_config2(struct acc_option *options) + return -EINVAL; + } + ++ ctx_set_num = calloc(1, sizeof(*ctx_set_num)); ++ if (!ctx_set_num) { ++ WD_ERR("failed to alloc ctx_set_size!\n"); ++ return -WD_ENOMEM; ++ } ++ ++ cparams.op_type_num = 1; ++ cparams.ctx_set_num = ctx_set_num; ++ cparams.bmp = numa_allocate_nodemask(); ++ if (!cparams.bmp) { ++ WD_ERR("failed to create nodemask!\n"); ++ ret = -WD_ENOMEM; ++ goto out_freectx; ++ } ++ ++ numa_bitmask_setall(cparams.bmp); ++ ++ if (mode == CTX_MODE_SYNC) ++ ctx_set_num->sync_ctx_num = g_ctxnum; ++ else ++ ctx_set_num->async_ctx_num = g_ctxnum; ++ + /* init2 */ + switch (subtype) { + case RSA_TYPE: +- return wd_rsa_init2(alg_name, SCHED_POLICY_RR, TASK_HW); ++ return wd_rsa_init2_(alg_name, SCHED_POLICY_RR, TASK_HW, &cparams); + case DH_TYPE: +- return wd_dh_init2(alg_name, SCHED_POLICY_RR, TASK_HW); ++ return wd_dh_init2_(alg_name, SCHED_POLICY_RR, TASK_HW, &cparams); + case ECDH_TYPE: + case ECDSA_TYPE: + case SM2_TYPE: + case X25519_TYPE: + case X448_TYPE: +- return wd_ecc_init2(alg_name, SCHED_POLICY_RR, TASK_HW); ++ return wd_ecc_init2_(alg_name, SCHED_POLICY_RR, TASK_HW, &cparams); + default: + HPRE_TST_PRT("failed to parse alg subtype on uninit2!\n"); + return -EINVAL; + } ++ ++out_freectx: ++ free(ctx_set_num); ++ ++ return ret; + } + + /*-------------------------------uadk benchmark main code-------------------------------------*/ +diff --git a/uadk_tool/benchmark/sec_uadk_benchmark.c b/uadk_tool/benchmark/sec_uadk_benchmark.c +index 41b7416..411d0c4 100644 +--- a/uadk_tool/benchmark/sec_uadk_benchmark.c ++++ b/uadk_tool/benchmark/sec_uadk_benchmark.c +@@ -795,7 +795,10 @@ static void uninit_ctx_config2(int subtype) + + static int init_ctx_config2(struct acc_option *options) + { ++ struct wd_ctx_params cparams = {0}; ++ struct wd_ctx_nums *ctx_set_num; + int subtype = options->subtype; ++ int mode = options->syncmode; + char alg_name[MAX_ALG_NAME]; + int ret; + +@@ -805,10 +808,32 @@ static int init_ctx_config2(struct acc_option *options) + return -EINVAL; + } + ++ ctx_set_num = calloc(1, sizeof(*ctx_set_num)); ++ if (!ctx_set_num) { ++ WD_ERR("failed to alloc ctx_set_size!\n"); ++ return -WD_ENOMEM; ++ } ++ ++ cparams.op_type_num = 1; ++ cparams.ctx_set_num = ctx_set_num; ++ cparams.bmp = numa_allocate_nodemask(); ++ if (!cparams.bmp) { ++ WD_ERR("failed to create nodemask!\n"); ++ ret = -WD_ENOMEM; ++ goto out_freectx; ++ } ++ ++ numa_bitmask_setall(cparams.bmp); ++ ++ if (mode == CTX_MODE_SYNC) ++ ctx_set_num->sync_ctx_num = g_ctxnum; ++ else ++ ctx_set_num->async_ctx_num = g_ctxnum; ++ + /* init */ + switch(subtype) { + case CIPHER_TYPE: +- ret = wd_cipher_init2(alg_name, SCHED_POLICY_RR, TASK_HW); ++ ret = wd_cipher_init2_(alg_name, SCHED_POLICY_RR, TASK_HW, &cparams); + if (ret) + SEC_TST_PRT("failed to do cipher init2!\n"); + break; +@@ -818,12 +843,12 @@ static int init_ctx_config2(struct acc_option *options) + SEC_TST_PRT("failed to do cipher intruction init2!\n"); + break; + case AEAD_TYPE: +- ret = wd_aead_init2(alg_name, SCHED_POLICY_RR, TASK_HW); ++ ret = wd_aead_init2_(alg_name, SCHED_POLICY_RR, TASK_HW, &cparams); + if (ret) + SEC_TST_PRT("failed to do aead init2!\n"); + break; + case DIGEST_TYPE: +- ret = wd_digest_init2(alg_name, options->sched_type, options->task_type); ++ ret = wd_digest_init2_(alg_name, options->sched_type, options->task_type, &cparams); + if (ret) + SEC_TST_PRT("failed to do digest init2!\n"); + break; +@@ -833,7 +858,11 @@ static int init_ctx_config2(struct acc_option *options) + return ret; + } + ++out_freectx: ++ free(ctx_set_num); ++ + return ret; ++ + } + + static void get_aead_data(u8 *addr, u32 size) +diff --git a/uadk_tool/benchmark/zip_uadk_benchmark.c b/uadk_tool/benchmark/zip_uadk_benchmark.c +index 22aa916..2133297 100644 +--- a/uadk_tool/benchmark/zip_uadk_benchmark.c ++++ b/uadk_tool/benchmark/zip_uadk_benchmark.c +@@ -299,8 +299,11 @@ static void uninit_ctx_config2(void) + + static int init_ctx_config2(struct acc_option *options) + { ++ struct wd_ctx_params cparams = {0}; ++ struct wd_ctx_nums *ctx_set_num; ++ int mode = options->syncmode; + char alg_name[MAX_ALG_NAME]; +- int ret = 0; ++ int ret; + + ret = get_alg_name(options->algtype, alg_name); + if (ret) { +@@ -308,14 +311,43 @@ static int init_ctx_config2(struct acc_option *options) + return -EINVAL; + } + ++ ctx_set_num = calloc(WD_DIR_MAX, sizeof(*ctx_set_num)); ++ if (!ctx_set_num) { ++ WD_ERR("failed to alloc ctx_set_size!\n"); ++ return -WD_ENOMEM; ++ } ++ ++ cparams.op_type_num = WD_DIR_MAX; ++ cparams.ctx_set_num = ctx_set_num; ++ cparams.bmp = numa_allocate_nodemask(); ++ if (!cparams.bmp) { ++ WD_ERR("failed to create nodemask!\n"); ++ ret = -WD_ENOMEM; ++ goto out_freectx; ++ } ++ ++ numa_bitmask_setall(cparams.bmp); ++ ++ for (int i = 0; i < WD_DIR_MAX; i++) { ++ if (mode == CTX_MODE_SYNC) ++ ctx_set_num[i].sync_ctx_num = g_ctxnum; ++ else ++ ctx_set_num[i].async_ctx_num = g_ctxnum; ++ } ++ + /* init */ +- ret = wd_comp_init2(alg_name, SCHED_POLICY_RR, TASK_HW); ++ ret = wd_comp_init2_(alg_name, SCHED_POLICY_RR, TASK_HW, &cparams); + if (ret) { + ZIP_TST_PRT("failed to do comp init2!\n"); + return ret; + } + + return 0; ++ ++out_freectx: ++ free(ctx_set_num); ++ ++ return ret; + } + + static int specified_device_request_ctx(struct acc_option *options) +-- +2.25.1 + diff --git a/0020-uadk-isa-ce-fix-sm4-CE-driver-initialization-error.patch b/0020-uadk-isa-ce-fix-sm4-CE-driver-initialization-error.patch new file mode 100644 index 0000000..563ab26 --- /dev/null +++ b/0020-uadk-isa-ce-fix-sm4-CE-driver-initialization-error.patch @@ -0,0 +1,90 @@ +From bdcebcd37ec5745e19d2e72447ce56edbeabaac9 Mon Sep 17 00:00:00 2001 +From: Qi Tao +Date: Thu, 5 Sep 2024 19:25:16 +0800 +Subject: [PATCH 20/39] uadk/isa-ce: fix sm4 CE driver initialization error. + +CE driver of sm4 should alloc and free private ctx by himself. +The cleancode issue of sm3 CE driver is also solved. + +Signed-off-by: Qi Tao +--- + drv/isa_ce_sm3.c | 15 ++++++--------- + drv/isa_ce_sm4.c | 18 +++++++++++++++--- + 2 files changed, 21 insertions(+), 12 deletions(-) + +diff --git a/drv/isa_ce_sm3.c b/drv/isa_ce_sm3.c +index 59f3940..99cd640 100644 +--- a/drv/isa_ce_sm3.c ++++ b/drv/isa_ce_sm3.c +@@ -373,22 +373,19 @@ static int sm3_ce_drv_recv(struct wd_alg_driver *drv, handle_t ctx, void *digest + static int sm3_ce_drv_init(struct wd_alg_driver *drv, void *conf) + { + struct wd_ctx_config_internal *config = (struct wd_ctx_config_internal *)conf; +- struct sm3_ce_drv_ctx *sctx = (struct sm3_ce_drv_ctx *)drv->priv; ++ struct sm3_ce_drv_ctx *priv; + + /* Fallback init is NULL */ + if (!drv || !conf) + return 0; + +- config->epoll_en = 0; +- +- /* return if already inited */ +- if (sctx) +- return WD_SUCCESS; +- sctx = malloc(sizeof(struct sm3_ce_drv_ctx)); +- if (!sctx) ++ priv = malloc(sizeof(struct sm3_ce_drv_ctx)); ++ if (!priv) + return -WD_EINVAL; + +- memcpy(&sctx->config, config, sizeof(struct wd_ctx_config_internal)); ++ config->epoll_en = 0; ++ memcpy(&priv->config, config, sizeof(struct wd_ctx_config_internal)); ++ drv->priv = priv; + + return WD_SUCCESS; + } +diff --git a/drv/isa_ce_sm4.c b/drv/isa_ce_sm4.c +index e937893..3404465 100644 +--- a/drv/isa_ce_sm4.c ++++ b/drv/isa_ce_sm4.c +@@ -34,20 +34,32 @@ + static int isa_ce_init(struct wd_alg_driver *drv, void *conf) + { + struct wd_ctx_config_internal *config = conf; +- struct sm4_ce_drv_ctx *sctx = drv->priv; ++ struct sm4_ce_drv_ctx *priv; + + /* Fallback init is NULL */ + if (!drv || !conf) + return 0; + ++ priv = malloc(sizeof(struct sm4_ce_drv_ctx)); ++ if (!priv) ++ return -WD_EINVAL; ++ + config->epoll_en = 0; +- memcpy(&sctx->config, config, sizeof(struct wd_ctx_config_internal)); ++ memcpy(&priv->config, config, sizeof(struct wd_ctx_config_internal)); ++ drv->priv = priv; + +- return 0; ++ return WD_SUCCESS; + } + + static void isa_ce_exit(struct wd_alg_driver *drv) + { ++ struct sm4_ce_drv_ctx *sctx = (struct sm4_ce_drv_ctx *)drv->priv; ++ ++ if (!sctx) ++ return; ++ ++ free(sctx); ++ drv->priv = NULL; + } + + /* increment upper 96 bits of 128-bit counter by 1 */ +-- +2.25.1 + diff --git a/0021-acc-uadk-fix-queue-configuration-parameter-error.patch b/0021-acc-uadk-fix-queue-configuration-parameter-error.patch new file mode 100644 index 0000000..27594d8 --- /dev/null +++ b/0021-acc-uadk-fix-queue-configuration-parameter-error.patch @@ -0,0 +1,169 @@ +From 05b1b0b349a88f5d8db91f485e2d19a10a1aefdc Mon Sep 17 00:00:00 2001 +From: Qi Tao +Date: Wed, 18 Sep 2024 09:59:10 +0800 +Subject: [PATCH 21/39] acc/uadk: fix queue configuration parameter error + +When executing an asynchronous task, the scheduler may +incorrectly specify a synchronous queue. +This will cause an error in the queue configuration +parameters and result in a segment error. +In wd_get_msg_from_pool(), msg_num == 0 indicates that +the asynchronous task scheduler is executed but the +synchronous queue is specified.Therefore, +when msg_num == 0 is detected in this function, +the program needs to return an exception immediately +to avoid errors. + +Signed-off-by: Qi Tao +--- + wd_aead.c | 2 +- + wd_agg.c | 4 ++-- + wd_cipher.c | 4 ++-- + wd_comp.c | 2 +- + wd_dh.c | 6 ++++-- + wd_digest.c | 4 ++-- + wd_ecc.c | 6 ++++-- + wd_rsa.c | 6 ++++-- + wd_util.c | 4 ++++ + 9 files changed, 24 insertions(+), 14 deletions(-) + +diff --git a/wd_aead.c b/wd_aead.c +index 65949f7..608f0e6 100644 +--- a/wd_aead.c ++++ b/wd_aead.c +@@ -805,7 +805,7 @@ int wd_do_aead_async(handle_t h_sess, struct wd_aead_req *req) + idx, (void **)&msg); + if (unlikely(msg_id < 0)) { + WD_ERR("failed to get msg from pool!\n"); +- return -WD_EBUSY; ++ return msg_id; + } + + fill_request_msg(msg, req, sess); +diff --git a/wd_agg.c b/wd_agg.c +index e493bb8..b536efe 100644 +--- a/wd_agg.c ++++ b/wd_agg.c +@@ -1192,8 +1192,8 @@ static int wd_agg_async_job(struct wd_agg_sess *sess, struct wd_agg_req *req, bo + ctx = config->ctxs + idx; + msg_id = wd_get_msg_from_pool(&wd_agg_setting.pool, idx, (void **)&msg); + if (unlikely(msg_id < 0)) { +- WD_ERR("busy, failed to get agg msg from pool!\n"); +- return -WD_EBUSY; ++ WD_ERR("failed to get agg msg from pool!\n"); ++ return msg_id; + } + + if (is_input) +diff --git a/wd_cipher.c b/wd_cipher.c +index 646aa89..f6b035a 100644 +--- a/wd_cipher.c ++++ b/wd_cipher.c +@@ -741,8 +741,8 @@ int wd_do_cipher_async(handle_t h_sess, struct wd_cipher_req *req) + msg_id = wd_get_msg_from_pool(&wd_cipher_setting.pool, idx, + (void **)&msg); + if (unlikely(msg_id < 0)) { +- WD_ERR("busy, failed to get msg from pool!\n"); +- return -WD_EBUSY; ++ WD_ERR("failed to get msg from pool!\n"); ++ return msg_id; + } + + fill_request_msg(msg, req, sess); +diff --git a/wd_comp.c b/wd_comp.c +index 34ddbcf..0fa5f92 100644 +--- a/wd_comp.c ++++ b/wd_comp.c +@@ -860,7 +860,7 @@ int wd_do_comp_async(handle_t h_sess, struct wd_comp_req *req) + tag = wd_get_msg_from_pool(&wd_comp_setting.pool, idx, (void **)&msg); + if (unlikely(tag < 0)) { + WD_ERR("failed to get msg from pool!\n"); +- return -WD_EBUSY; ++ return tag; + } + fill_comp_msg(sess, msg, req); + msg->tag = tag; +diff --git a/wd_dh.c b/wd_dh.c +index cdcba14..043c3be 100644 +--- a/wd_dh.c ++++ b/wd_dh.c +@@ -422,8 +422,10 @@ int wd_do_dh_async(handle_t sess, struct wd_dh_req *req) + ctx = config->ctxs + idx; + + mid = wd_get_msg_from_pool(&wd_dh_setting.pool, idx, (void **)&msg); +- if (mid < 0) +- return -WD_EBUSY; ++ if (unlikely(mid < 0)) { ++ WD_ERR("failed to get msg from pool!\n"); ++ return mid; ++ } + + ret = fill_dh_msg(msg, req, (struct wd_dh_sess *)sess); + if (ret) +diff --git a/wd_digest.c b/wd_digest.c +index 943fd8c..58f621a 100644 +--- a/wd_digest.c ++++ b/wd_digest.c +@@ -706,8 +706,8 @@ int wd_do_digest_async(handle_t h_sess, struct wd_digest_req *req) + msg_id = wd_get_msg_from_pool(&wd_digest_setting.pool, idx, + (void **)&msg); + if (unlikely(msg_id < 0)) { +- WD_ERR("busy, failed to get msg from pool!\n"); +- return -WD_EBUSY; ++ WD_ERR("failed to get msg from pool!\n"); ++ return msg_id; + } + + fill_request_msg(msg, req, dsess); +diff --git a/wd_ecc.c b/wd_ecc.c +index 7c0c77a..b1712c5 100644 +--- a/wd_ecc.c ++++ b/wd_ecc.c +@@ -2268,8 +2268,10 @@ int wd_do_ecc_async(handle_t sess, struct wd_ecc_req *req) + ctx = config->ctxs + idx; + + mid = wd_get_msg_from_pool(&wd_ecc_setting.pool, idx, (void **)&msg); +- if (mid < 0) +- return -WD_EBUSY; ++ if (unlikely(mid < 0)) { ++ WD_ERR("failed to get msg from pool!\n"); ++ return mid; ++ } + + ret = fill_ecc_msg(msg, req, (struct wd_ecc_sess *)sess); + if (ret) +diff --git a/wd_rsa.c b/wd_rsa.c +index b858491..366e766 100644 +--- a/wd_rsa.c ++++ b/wd_rsa.c +@@ -483,8 +483,10 @@ int wd_do_rsa_async(handle_t sess, struct wd_rsa_req *req) + ctx = config->ctxs + idx; + + mid = wd_get_msg_from_pool(&wd_rsa_setting.pool, idx, (void **)&msg); +- if (mid < 0) +- return -WD_EBUSY; ++ if (unlikely(mid < 0)) { ++ WD_ERR("failed to get msg from pool!\n"); ++ return mid; ++ } + + ret = fill_rsa_msg(msg, req, (struct wd_rsa_sess *)sess); + if (ret) +diff --git a/wd_util.c b/wd_util.c +index 8adfd4c..d58e8b8 100644 +--- a/wd_util.c ++++ b/wd_util.c +@@ -442,6 +442,10 @@ int wd_get_msg_from_pool(struct wd_async_msg_pool *pool, + __u32 cnt = 0; + __u32 idx = p->tail; + ++ /* Scheduler set a sync ctx */ ++ if (!msg_num) ++ return -WD_EINVAL; ++ + while (__atomic_test_and_set(&p->used[idx], __ATOMIC_ACQUIRE)) { + idx = (idx + 1) % msg_num; + cnt++; +-- +2.25.1 + diff --git a/0022-acc-uadk-fix-double-free-error.patch b/0022-acc-uadk-fix-double-free-error.patch new file mode 100644 index 0000000..ac081b5 --- /dev/null +++ b/0022-acc-uadk-fix-double-free-error.patch @@ -0,0 +1,29 @@ +From 5d709269eb00c7b7edd95da5eb467c9aad72263d Mon Sep 17 00:00:00 2001 +From: Qi Tao +Date: Wed, 18 Sep 2024 10:02:04 +0800 +Subject: [PATCH 22/39] acc/uadk: fix double-free error + +Fix double-free error. + +Signed-off-by: Qi Tao +--- + wd.c | 3 --- + 1 file changed, 3 deletions(-) + +diff --git a/wd.c b/wd.c +index abadcf9..6d01158 100644 +--- a/wd.c ++++ b/wd.c +@@ -938,9 +938,6 @@ void wd_release_alg_cap(struct wd_capability *head) + cap_pnext = cap_pnext->next; + free(cap_node); + } +- +- if (head) +- free(head); + } + + struct wd_capability *wd_get_alg_cap(void) +-- +2.25.1 + diff --git a/0023-uadk-fix-the-null-pointer-check-error.patch b/0023-uadk-fix-the-null-pointer-check-error.patch new file mode 100644 index 0000000..e170ab1 --- /dev/null +++ b/0023-uadk-fix-the-null-pointer-check-error.patch @@ -0,0 +1,119 @@ +From e002fec5535a971db0c95a27290e61e3ae844a4c Mon Sep 17 00:00:00 2001 +From: Qi Tao +Date: Fri, 20 Sep 2024 15:13:53 +0800 +Subject: [PATCH 23/39] uadk: fix the null pointer check error + +After the pointer is used, the null pointer check is performed, +which is incorrect. + +Signed-off-by: Qi Tao +--- + drv/hisi_comp.c | 3 ++- + drv/hisi_hpre.c | 3 ++- + v1/drv/hisi_zip_udrv.c | 3 ++- + wd_zlibwrapper.c | 8 ++++---- + 4 files changed, 10 insertions(+), 7 deletions(-) + +diff --git a/drv/hisi_comp.c b/drv/hisi_comp.c +index 2fa5eff..4c8e18b 100644 +--- a/drv/hisi_comp.c ++++ b/drv/hisi_comp.c +@@ -836,7 +836,7 @@ out: + static void hisi_zip_exit(struct wd_alg_driver *drv) + { + struct hisi_zip_ctx *priv = (struct hisi_zip_ctx *)drv->priv; +- struct wd_ctx_config_internal *config = &priv->config; ++ struct wd_ctx_config_internal *config; + handle_t h_qp; + __u32 i; + +@@ -845,6 +845,7 @@ static void hisi_zip_exit(struct wd_alg_driver *drv) + return; + } + ++ config = &priv->config; + for (i = 0; i < config->ctx_num; i++) { + h_qp = (handle_t)wd_ctx_get_priv(config->ctxs[i].ctx); + hisi_qm_free_qp(h_qp); +diff --git a/drv/hisi_hpre.c b/drv/hisi_hpre.c +index 68a11ae..37bb5ee 100644 +--- a/drv/hisi_hpre.c ++++ b/drv/hisi_hpre.c +@@ -584,7 +584,7 @@ static int hpre_ecc_init(struct wd_alg_driver *drv, void *conf) + static void hpre_exit(struct wd_alg_driver *drv) + { + struct hisi_hpre_ctx *priv = (struct hisi_hpre_ctx *)drv->priv; +- struct wd_ctx_config_internal *config = &priv->config; ++ struct wd_ctx_config_internal *config; + handle_t h_qp; + __u32 i; + +@@ -593,6 +593,7 @@ static void hpre_exit(struct wd_alg_driver *drv) + return; + } + ++ config = &priv->config; + for (i = 0; i < config->ctx_num; i++) { + h_qp = (handle_t)wd_ctx_get_priv(config->ctxs[i].ctx); + hisi_qm_free_qp(h_qp); +diff --git a/v1/drv/hisi_zip_udrv.c b/v1/drv/hisi_zip_udrv.c +index cc55ef5..01d76a3 100644 +--- a/v1/drv/hisi_zip_udrv.c ++++ b/v1/drv/hisi_zip_udrv.c +@@ -254,7 +254,6 @@ int qm_parse_zip_sqe(void *hw_msg, const struct qm_queue_info *info, + __u16 i, __u16 usr) + { + struct wcrypto_comp_msg *recv_msg = info->req_cache[i]; +- struct wcrypto_comp_tag *tag = (void *)(uintptr_t)recv_msg->udata; + struct hisi_zip_sqe *sqe = hw_msg; + __u16 ctx_st = sqe->ctx_dw0 & HZ_CTX_ST_MASK; + __u16 lstblk = sqe->dw3 & HZ_LSTBLK_MASK; +@@ -262,12 +261,14 @@ int qm_parse_zip_sqe(void *hw_msg, const struct qm_queue_info *info, + __u32 type = sqe->dw9 & HZ_REQ_TYPE_MASK; + uintptr_t phy_in, phy_out, phy_ctxbuf; + struct wd_queue *q = info->q; ++ struct wcrypto_comp_tag *tag; + + if (unlikely(!recv_msg)) { + WD_ERR("info->req_cache is null at index:%hu\n", i); + return 0; + } + ++ tag = (void *)(uintptr_t)recv_msg->udata; + if (usr && sqe->tag != usr) + return 0; + +diff --git a/wd_zlibwrapper.c b/wd_zlibwrapper.c +index 953837a..b3b315f 100644 +--- a/wd_zlibwrapper.c ++++ b/wd_zlibwrapper.c +@@ -220,9 +220,6 @@ static int wd_zlib_do_request(z_streamp strm, int flush, enum wd_comp_op_type ty + __u32 dst_len = strm->avail_out; + int ret; + +- if (unlikely(!strm)) +- return Z_STREAM_ERROR; +- + if (unlikely(flush != Z_SYNC_FLUSH && flush != Z_FINISH)) { + WD_ERR("invalid: flush is %d!\n", flush); + return Z_STREAM_ERROR; +@@ -267,12 +264,15 @@ int wd_deflate_init(z_streamp strm, int level, int windowbits) + + int wd_deflate(z_streamp strm, int flush) + { ++ if (unlikely(!strm)) ++ return Z_STREAM_ERROR; ++ + return wd_zlib_do_request(strm, flush, WD_DIR_COMPRESS); + } + + int wd_deflate_reset(z_streamp strm) + { +- if (!strm) ++ if (unlikely(!strm)) + return Z_STREAM_ERROR; + + wd_comp_reset_sess((handle_t)strm->reserved); +-- +2.25.1 + diff --git a/0024-uadk-add-null-pointer-check.patch b/0024-uadk-add-null-pointer-check.patch new file mode 100644 index 0000000..dcc1a41 --- /dev/null +++ b/0024-uadk-add-null-pointer-check.patch @@ -0,0 +1,172 @@ +From 3765bba6fceaa59f2afdd32e4061bbde07797c57 Mon Sep 17 00:00:00 2001 +From: Qi Tao +Date: Thu, 10 Oct 2024 11:36:44 +0800 +Subject: [PATCH 24/39] uadk: add null pointer check + +Add null pointer check to avoid segment errors. + +Signed-off-by: Qi Tao +--- + drv/hash_mb/hash_mb.c | 6 +++--- + drv/hisi_comp.c | 8 +++----- + drv/hisi_dae.c | 6 +++--- + drv/hisi_hpre.c | 8 +++----- + drv/hisi_sec.c | 8 +++----- + drv/isa_ce_sm3.c | 6 +++--- + drv/isa_ce_sm4.c | 6 +++--- + 7 files changed, 21 insertions(+), 27 deletions(-) + +diff --git a/drv/hash_mb/hash_mb.c b/drv/hash_mb/hash_mb.c +index e4a9564..9ad36b8 100644 +--- a/drv/hash_mb/hash_mb.c ++++ b/drv/hash_mb/hash_mb.c +@@ -217,11 +217,11 @@ static int hash_mb_init(struct wd_alg_driver *drv, void *conf) + + static void hash_mb_exit(struct wd_alg_driver *drv) + { +- struct hash_mb_ctx *priv = (struct hash_mb_ctx *)drv->priv; +- +- if (!priv) ++ if(!drv || !drv->priv) + return; + ++ struct hash_mb_ctx *priv = (struct hash_mb_ctx *)drv->priv; ++ + hash_mb_queue_uninit(&priv->config, priv->config.ctx_num); + free(priv); + drv->priv = NULL; +diff --git a/drv/hisi_comp.c b/drv/hisi_comp.c +index 4c8e18b..c556998 100644 +--- a/drv/hisi_comp.c ++++ b/drv/hisi_comp.c +@@ -835,16 +835,14 @@ out: + + static void hisi_zip_exit(struct wd_alg_driver *drv) + { ++ if(!drv || !drv->priv) ++ return; ++ + struct hisi_zip_ctx *priv = (struct hisi_zip_ctx *)drv->priv; + struct wd_ctx_config_internal *config; + handle_t h_qp; + __u32 i; + +- if (!priv) { +- /* return if already exit */ +- return; +- } +- + config = &priv->config; + for (i = 0; i < config->ctx_num; i++) { + h_qp = (handle_t)wd_ctx_get_priv(config->ctxs[i].ctx); +diff --git a/drv/hisi_dae.c b/drv/hisi_dae.c +index 09d2387..864d21b 100644 +--- a/drv/hisi_dae.c ++++ b/drv/hisi_dae.c +@@ -1604,14 +1604,14 @@ out: + + static void dae_exit(struct wd_alg_driver *drv) + { ++ if(!drv || !drv->priv) ++ return; ++ + struct hisi_dae_ctx *priv = (struct hisi_dae_ctx *)drv->priv; + struct wd_ctx_config_internal *config; + handle_t h_qp; + __u32 i; + +- if (!priv) +- return; +- + config = &priv->config; + for (i = 0; i < config->ctx_num; i++) { + h_qp = (handle_t)wd_ctx_get_priv(config->ctxs[i].ctx); +diff --git a/drv/hisi_hpre.c b/drv/hisi_hpre.c +index 37bb5ee..313cdcc 100644 +--- a/drv/hisi_hpre.c ++++ b/drv/hisi_hpre.c +@@ -583,16 +583,14 @@ static int hpre_ecc_init(struct wd_alg_driver *drv, void *conf) + + static void hpre_exit(struct wd_alg_driver *drv) + { ++ if(!drv || !drv->priv) ++ return; ++ + struct hisi_hpre_ctx *priv = (struct hisi_hpre_ctx *)drv->priv; + struct wd_ctx_config_internal *config; + handle_t h_qp; + __u32 i; + +- if (!priv) { +- /* return if already exit */ +- return; +- } +- + config = &priv->config; + for (i = 0; i < config->ctx_num; i++) { + h_qp = (handle_t)wd_ctx_get_priv(config->ctxs[i].ctx); +diff --git a/drv/hisi_sec.c b/drv/hisi_sec.c +index 0a1bcc1..6377586 100644 +--- a/drv/hisi_sec.c ++++ b/drv/hisi_sec.c +@@ -3102,16 +3102,14 @@ out: + + static void hisi_sec_exit(struct wd_alg_driver *drv) + { ++ if(!drv || !drv->priv) ++ return; ++ + struct hisi_sec_ctx *priv = (struct hisi_sec_ctx *)drv->priv; + struct wd_ctx_config_internal *config; + handle_t h_qp; + __u32 i; + +- if (!priv) { +- /* return if already exit */ +- return; +- } +- + config = &priv->config; + for (i = 0; i < config->ctx_num; i++) { + h_qp = (handle_t)wd_ctx_get_priv(config->ctxs[i].ctx); +diff --git a/drv/isa_ce_sm3.c b/drv/isa_ce_sm3.c +index 99cd640..54c2a9e 100644 +--- a/drv/isa_ce_sm3.c ++++ b/drv/isa_ce_sm3.c +@@ -392,11 +392,11 @@ static int sm3_ce_drv_init(struct wd_alg_driver *drv, void *conf) + + static void sm3_ce_drv_exit(struct wd_alg_driver *drv) + { +- struct sm3_ce_drv_ctx *sctx = (struct sm3_ce_drv_ctx *)drv->priv; +- +- if (!sctx) ++ if(!drv || !drv->priv) + return; + ++ struct sm3_ce_drv_ctx *sctx = (struct sm3_ce_drv_ctx *)drv->priv; ++ + free(sctx); + drv->priv = NULL; + } +diff --git a/drv/isa_ce_sm4.c b/drv/isa_ce_sm4.c +index 3404465..5e448fa 100644 +--- a/drv/isa_ce_sm4.c ++++ b/drv/isa_ce_sm4.c +@@ -53,11 +53,11 @@ static int isa_ce_init(struct wd_alg_driver *drv, void *conf) + + static void isa_ce_exit(struct wd_alg_driver *drv) + { +- struct sm4_ce_drv_ctx *sctx = (struct sm4_ce_drv_ctx *)drv->priv; +- +- if (!sctx) ++ if(!drv || !drv->priv) + return; + ++ struct sm4_ce_drv_ctx *sctx = (struct sm4_ce_drv_ctx *)drv->priv; ++ + free(sctx); + drv->priv = NULL; + } +-- +2.25.1 + diff --git a/0025-uadk-test-move-main-function-to-testsuit.c-and-rm.patch b/0025-uadk-test-move-main-function-to-testsuit.c-and-rm.patch new file mode 100644 index 0000000..eb97128 --- /dev/null +++ b/0025-uadk-test-move-main-function-to-testsuit.c-and-rm.patch @@ -0,0 +1,1060 @@ +From 731198ed2bfd57f44898f42be180e57477425143 Mon Sep 17 00:00:00 2001 +From: Hao Fang +Date: Thu, 10 Oct 2024 14:15:51 +0800 +Subject: [PATCH 25/39] uadk/test: move main function to testsuit.c and rm + +test_sva_perf.c + +test_suit.c should be the main function file, +and these two files have the duplicate code. + +Signed-off-by: Hao Fang +Signed-off-by: Qi Tao +--- + test/hisi_zip_test/Makefile.am | 2 +- + test/hisi_zip_test/test_sva_perf.c | 524 ----------------------------- + test/hisi_zip_test/testsuit.c | 474 +++++++++++++++++++++++++- + 3 files changed, 474 insertions(+), 526 deletions(-) + delete mode 100644 test/hisi_zip_test/test_sva_perf.c + +diff --git a/test/hisi_zip_test/Makefile.am b/test/hisi_zip_test/Makefile.am +index f671582..8b464f5 100644 +--- a/test/hisi_zip_test/Makefile.am ++++ b/test/hisi_zip_test/Makefile.am +@@ -3,7 +3,7 @@ AUTOMAKE_OPTIONS = subdir-objects + + bin_PROGRAMS=zip_sva_perf + +-zip_sva_perf_SOURCES=test_sva_perf.c test_lib.c testsuit.c ++zip_sva_perf_SOURCES=testsuit.c test_lib.c + + if WD_STATIC_DRV + zip_sva_perf_LDADD=../../.libs/libwd.a ../../.libs/libwd_comp.a \ +diff --git a/test/hisi_zip_test/test_sva_perf.c b/test/hisi_zip_test/test_sva_perf.c +deleted file mode 100644 +index 98670a5..0000000 +--- a/test/hisi_zip_test/test_sva_perf.c ++++ /dev/null +@@ -1,524 +0,0 @@ +-// SPDX-License-Identifier: Apache-2.0 +-/* +- * Copyright 2020-2021 Huawei Technologies Co.,Ltd. All rights reserved. +- * Copyright 2020-2021 Linaro ltd. +- */ +- +-/* +- * Test performance of the SVA API +- */ +-#include /* For __NR_perf_event_open */ +-#include +-#include +-#include +-#include +-#include +-#include +-#include +-#include +-#include +-#include +-#include +- +-#include +- +-#include "test_lib.h" +-#include "wd_sched.h" +- +-enum hizip_stats_variable { +- ST_SETUP_TIME, +- ST_RUN_TIME, +- ST_CPU_TIME, +- +- /* CPU usage */ +- ST_USER_TIME, +- ST_SYSTEM_TIME, +- +- /* Faults */ +- ST_MINFLT, +- ST_MAJFLT, +- +- /* Context switches */ +- ST_INVCTX, +- ST_VCTX, +- +- /* Signals */ +- ST_SIGNALS, +- +- /* Aggregated */ +- ST_SPEED, +- ST_TOTAL_SPEED, +- ST_CPU_IDLE, +- ST_FAULTS, +- ST_IOPF, +- +- ST_COMPRESSION_RATIO, +- +- NUM_STATS +-}; +- +-struct hizip_stats { +- double v[NUM_STATS]; +-}; +- +-int perf_event_open(struct perf_event_attr *attr, +- pid_t pid, int cpu, int group_fd, +- unsigned long flags) +-{ +- return syscall(__NR_perf_event_open, attr, pid, cpu, group_fd, flags); +-} +- +-unsigned long long perf_event_put(int *perf_fds, int nr_fds); +- +-int perf_event_get(const char *event_name, int **perf_fds, int *nr_fds) +-{ +- int ret; +- int cpu; +- FILE *fd; +- int nr_cpus; +- unsigned int event_id; +- char event_id_file[256]; +- struct perf_event_attr event = { +- .type = PERF_TYPE_TRACEPOINT, +- .size = sizeof(event), +- .disabled = true, +- }; +- +- *perf_fds = NULL; +- *nr_fds = 0; +- +- nr_cpus = sysconf(_SC_NPROCESSORS_ONLN); +- if (nr_cpus <= 0) { +- WD_ERR("invalid number of CPUs\n"); +- return nr_cpus; +- } +- +- ret = snprintf(event_id_file, sizeof(event_id_file), +- "/sys/kernel/debug/tracing/events/%s/id", event_name); +- if (ret >= sizeof(event_id_file)) { +- WD_ERR("event_id buffer overflow\n"); +- return -EOVERFLOW; +- } +- fd = fopen(event_id_file, "r"); +- if (fd == NULL) { +- ret = -errno; +- WD_ERR("Couldn't open file %s\n", event_id_file); +- return ret; +- } +- +- if (fscanf(fd, "%d", &event_id) != 1) { +- WD_ERR("Couldn't parse file %s\n", event_id_file); +- return -EINVAL; +- } +- fclose(fd); +- event.config = event_id; +- +- *perf_fds = calloc(nr_cpus, sizeof(int)); +- if (!*perf_fds) +- return -ENOMEM; +- *nr_fds = nr_cpus; +- +- /* +- * An event is bound to either a CPU or a PID. If we want both, we need +- * to open the event on all CPUs. Note that we can't use a perf group +- * since they have to be on the same CPU. +- */ +- for (cpu = 0; cpu < nr_cpus; cpu++) { +- int fd = perf_event_open(&event, -1, cpu, -1, 0); +- +- if (fd < 0) { +- WD_ERR("Couldn't get perf event %s on CPU%d: %d\n", +- event_name, cpu, errno); +- perf_event_put(*perf_fds, cpu); +- return fd; +- } +- +- ioctl(fd, PERF_EVENT_IOC_RESET, 0); +- ioctl(fd, PERF_EVENT_IOC_ENABLE, 0); +- (*perf_fds)[cpu] = fd; +- } +- +- return 0; +-} +- +-/* +- * Closes the perf fd and return the sample count. If it wasn't open, return 0. +- */ +-unsigned long long perf_event_put(int *perf_fds, int nr_fds) +-{ +- int ret; +- int cpu; +- uint64_t count, total = 0; +- +- if (!perf_fds) +- return 0; +- +- for (cpu = 0; cpu < nr_fds; cpu++) { +- int fd = perf_fds[cpu]; +- +- if (fd <= 0) { +- WD_ERR("Invalid perf fd %d\n", cpu); +- continue; +- } +- +- ioctl(fd, PERF_EVENT_IOC_DISABLE, 0); +- +- ret = read(fd, &count, sizeof(count)); +- if (ret < sizeof(count)) +- WD_ERR("Couldn't read perf event for CPU%d\n", cpu); +- +- total += count; +- close(fd); +- +- } +- +- free(perf_fds); +- return total; +-} +- +-static void set_thp(struct test_options *opts) +-{ +- char *p; +- char s[14]; +- FILE *file; +- +- file = fopen("/sys/kernel/mm/transparent_hugepage/enabled", "r"); +- if (!file) +- goto out_err; +- p = fgets(s, 14, file); +- fclose(file); +- if (!p) +- goto out_err; +- +- if (strcmp(s, "never") == 0) { +- printf("Cannot test THP with enable=never\n"); +- return; +- } +- +- file = fopen("/sys/kernel/mm/transparent_hugepage/defrag", "r"); +- if (!file) +- goto out_err; +- p = fgets(s, 14, file); +- fclose(file); +- if (!p) +- goto out_err; +- +- if (strcmp(s, "defer") == 0 || strcmp(s, "never") == 0) { +- printf("Cannot test THP with defrag=%s\n", s); +- return; +- } +- +- return; +-out_err: +- printf("THP unsupported?\n"); +-} +- +-void stat_setup(struct hizip_test_info *info) +-{ +- clock_gettime(CLOCK_MONOTONIC_RAW, &info->tv.setup_time); +- clock_gettime(CLOCK_PROCESS_CPUTIME_ID, &info->tv.setup_cputime); +- getrusage(RUSAGE_SELF, &info->tv.setup_rusage); +-} +- +-void stat_start(struct hizip_test_info *info) +-{ +- clock_gettime(CLOCK_MONOTONIC_RAW, &info->tv.start_time); +- clock_gettime(CLOCK_PROCESS_CPUTIME_ID, &info->tv.start_cputime); +- getrusage(RUSAGE_SELF, &info->tv.start_rusage); +-} +- +-void stat_end(struct hizip_test_info *info) +-{ +- struct test_options *opts = info->opts; +- struct hizip_stats *stats = info->stats; +- double v; +- size_t total_out; +- unsigned long total_len; +- +- total_out = __atomic_load_n(&info->total_out, __ATOMIC_ACQUIRE); +- clock_gettime(CLOCK_MONOTONIC_RAW, &info->tv.end_time); +- clock_gettime(CLOCK_PROCESS_CPUTIME_ID, &info->tv.end_cputime); +- getrusage(RUSAGE_SELF, &info->tv.end_rusage); +- +- stats->v[ST_SETUP_TIME] = (info->tv.start_time.tv_sec - +- info->tv.setup_time.tv_sec) * 1000000000 + +- info->tv.start_time.tv_nsec - +- info->tv.setup_time.tv_nsec; +- stats->v[ST_RUN_TIME] = (info->tv.end_time.tv_sec - +- info->tv.start_time.tv_sec) * 1000000000 + +- info->tv.end_time.tv_nsec - +- info->tv.start_time.tv_nsec; +- +- stats->v[ST_CPU_TIME] = (info->tv.end_cputime.tv_sec - +- info->tv.setup_cputime.tv_sec) * 1000000000 + +- info->tv.end_cputime.tv_nsec - +- info->tv.setup_cputime.tv_nsec; +- stats->v[ST_USER_TIME] = (info->tv.end_rusage.ru_utime.tv_sec - +- info->tv.setup_rusage.ru_utime.tv_sec) * +- 1000000 + +- info->tv.end_rusage.ru_utime.tv_usec - +- info->tv.setup_rusage.ru_utime.tv_usec; +- stats->v[ST_SYSTEM_TIME] = (info->tv.end_rusage.ru_stime.tv_sec - +- info->tv.setup_rusage.ru_stime.tv_sec) * +- 1000000 + +- info->tv.end_rusage.ru_stime.tv_usec - +- info->tv.setup_rusage.ru_stime.tv_usec; +- +- stats->v[ST_MINFLT] = info->tv.end_rusage.ru_minflt - +- info->tv.setup_rusage.ru_minflt; +- stats->v[ST_MAJFLT] = info->tv.end_rusage.ru_majflt - +- info->tv.setup_rusage.ru_majflt; +- +- stats->v[ST_VCTX] = info->tv.end_rusage.ru_nvcsw - +- info->tv.setup_rusage.ru_nvcsw; +- stats->v[ST_INVCTX] = info->tv.end_rusage.ru_nivcsw - +- info->tv.setup_rusage.ru_nivcsw; +- +- stats->v[ST_SIGNALS] = info->tv.end_rusage.ru_nsignals - +- info->tv.setup_rusage.ru_nsignals; +- +- /* check last loop is enough, same as below hizip_verify_output */ +- stats->v[ST_COMPRESSION_RATIO] = (double)opts->total_len / +- total_out * 100; +- +- total_len = opts->total_len * opts->compact_run_num; +- /* ST_RUN_TIME records nanoseconds */ +- stats->v[ST_SPEED] = (total_len * opts->thread_num * 1000) / +- (1.024 * 1.024 * stats->v[ST_RUN_TIME]); +- +- stats->v[ST_TOTAL_SPEED] = (total_len * opts->thread_num * 1000) / +- ((stats->v[ST_RUN_TIME] + +- stats->v[ST_SETUP_TIME]) * 1.024 * 1.024); +- +- v = stats->v[ST_RUN_TIME] + stats->v[ST_SETUP_TIME]; +- stats->v[ST_CPU_IDLE] = (v - stats->v[ST_CPU_TIME]) / v * 100; +- stats->v[ST_FAULTS] = stats->v[ST_MAJFLT] + stats->v[ST_MINFLT]; +-} +- +-static void handle_sigbus(int sig) +-{ +- printf("SIGBUS!\n"); +- _exit(0); +-} +- +-int main(int argc, char **argv) +-{ +- struct test_options opts = { +- .alg_type = WD_GZIP, +- .op_type = WD_DIR_COMPRESS, +- .q_num = 1, +- .run_num = 1, +- .compact_run_num = 1, +- .thread_num = 1, +- .sync_mode = 0, +- .block_size = 512000, +- .total_len = opts.block_size * 10, +- .verify = false, +- .verbose = false, +- .is_decomp = false, +- .is_stream = false, +- .is_file = false, +- .display_stats = STATS_PRETTY, +- .children = 0, +- .faults = 0, +- .data_fmt = 0, +- }; +- struct option long_options[] = { +- {"self", no_argument, 0, 0 }, +- {"in", required_argument, 0, 0 }, +- {"out", required_argument, 0, 0 }, +- {"ilist", required_argument, 0, 0 }, +- {"olist", required_argument, 0, 0 }, +- {"env", no_argument, 0, 0 }, +- {0, 0, 0, 0 }, +- }; +- int show_help = 0; +- int opt, option_idx; +- int self = 0; +- +- opts.fd_in = -1; +- opts.fd_out = -1; +- opts.fd_ilist = -1; +- opts.fd_olist = -1; +- opts.alg_type = WD_COMP_ALG_MAX; +- while ((opt = getopt_long(argc, argv, COMMON_OPTSTRING "f:o:w:k:r:", +- long_options, &option_idx)) != -1) { +- switch (opt) { +- case 0: +- switch (option_idx) { +- case 0: /* self */ +- self = 1; +- break; +- case 1: /* in */ +- if (optarg) { +- opts.fd_in = open(optarg, O_RDONLY); +- if (opts.fd_in < 0) { +- printf("Fail to open %s\n", +- optarg); +- show_help = 1; +- } else +- opts.is_file = true; +- } else { +- printf("Input file is missing!\n"); +- show_help = 1; +- } +- if (lseek(opts.fd_in, 0, SEEK_SET) < 0) { +- printf("Fail on lseek()!\n"); +- show_help = 1; +- } +- break; +- case 2: /* out */ +- if (optarg) { +- opts.fd_out = open(optarg, +- O_CREAT | O_WRONLY, +- S_IWUSR | S_IRGRP | +- S_IROTH); +- if (opts.fd_out < 0) { +- printf("Fail to open %s\n", +- optarg); +- show_help = 1; +- } else +- opts.is_file = true; +- } else { +- printf("Output file is missing!\n"); +- show_help = 1; +- } +- if (lseek(opts.fd_out, 0, SEEK_SET) < 0) { +- printf("Fail on lseek()!\n"); +- show_help = 1; +- } +- break; +- case 3: /* ilist */ +- if (!optarg) { +- printf("IN list file is missing!\n"); +- show_help = 1; +- break; +- } +- opts.fd_ilist = open(optarg, O_RDONLY); +- if (opts.fd_ilist < 0) { +- printf("Fail to open %s\n", optarg); +- show_help = 1; +- break; +- } +- opts.is_file = true; +- if (lseek(opts.fd_ilist, 0, SEEK_SET) < 0) { +- printf("Fail on lseek()!\n"); +- show_help = 1; +- break; +- } +- break; +- case 4: /* olist */ +- if (!optarg) { +- printf("OUT list file is missing!\n"); +- show_help = 1; +- break; +- } +- opts.fd_olist = open(optarg, +- O_CREAT | O_WRONLY, +- S_IWUSR | S_IRGRP | +- S_IROTH); +- if (opts.fd_olist < 0) { +- printf("Fail to open %s\n", optarg); +- show_help = 1; +- break; +- } +- opts.is_file = true; +- if (lseek(opts.fd_olist, 0, SEEK_SET) < 0) { +- printf("Fail on lseek()!\n"); +- show_help = 1; +- break; +- } +- break; +- case 5: /* env */ +- opts.use_env = true; +- break; +- default: +- show_help = 1; +- break; +- } +- break; +- case 'f': +- if (strcmp(optarg, "none") == 0) { +- opts.display_stats = STATS_NONE; +- } else if (strcmp(optarg, "csv") == 0) { +- opts.display_stats = STATS_CSV; +- } else if (strcmp(optarg, "pretty") == 0) { +- opts.display_stats = STATS_PRETTY; +- } else { +- SYS_ERR_COND(1, "invalid argument to -f: '%s'\n", optarg); +- break; +- } +- break; +- case 'o': +- switch (optarg[0]) { +- case 'p': +- opts.option |= PERFORMANCE; +- break; +- case 't': +- opts.option |= TEST_THP; +- set_thp(&opts); +- break; +- default: +- SYS_ERR_COND(1, "invalid argument to -o: '%s'\n", optarg); +- break; +- } +- break; +- case 'c': +- opts.option |= TEST_ZLIB; +- break; +- case 'r': +- opts.children = strtol(optarg, NULL, 0); +- if (opts.children < 0) +- show_help = 1; +- break; +- case 'k': +- switch (optarg[0]) { +- case 'b': +- opts.faults |= INJECT_SIG_BIND; +- break; +- case 't': +- opts.faults |= INJECT_TLB_FAULT; +- break; +- case 'w': +- opts.faults |= INJECT_SIG_WORK; +- break; +- default: +- SYS_ERR_COND(1, "invalid argument to -k: '%s'\n", optarg); +- break; +- } +- break; +- default: +- show_help = parse_common_option(opt, optarg, &opts); +- break; +- } +- } +- +- signal(SIGBUS, handle_sigbus); +- +- if (!show_help) { +- if (self) +- return run_self_test(&opts); +- return run_cmd(&opts); +- } +- +- hizip_test_adjust_len(&opts); +- +- SYS_ERR_COND(show_help || optind > argc, +- COMMON_HELP +- " -f output format for the statistics\n" +- " 'none' do not output statistics\n" +- " 'pretty' human readable format\n" +- " 'csv' raw, machine readable\n" +- " -o options\n" +- " 'perf' prefaults the output pages\n" +- " 'thp' try to enable transparent huge pages\n" +- " 'zlib' use zlib instead of the device\n" +- " -r number of children to create\n" +- " -k kill thread\n" +- " 'bind' kills the process after bind\n" +- " 'tlb' tries to access an unmapped buffer\n" +- " 'work' kills the process while the queue is working\n", +- argv[0] +- ); +- return 0; +-} +diff --git a/test/hisi_zip_test/testsuit.c b/test/hisi_zip_test/testsuit.c +index ac1a5f6..e1b157a 100644 +--- a/test/hisi_zip_test/testsuit.c ++++ b/test/hisi_zip_test/testsuit.c +@@ -3,12 +3,22 @@ + * Copyright 2020-2021 Huawei Technologies Co.,Ltd. All rights reserved. + * Copyright 2020-2021 Linaro ltd. + */ +- + #include ++#include /* For __NR_perf_event_open */ ++#include ++#include ++#include ++#include + #include ++#include ++#include ++#include + #include + #include ++#include ++ + #include "test_lib.h" ++#include "wd_sched.h" + + #define POLL_STRING_LEN 128 + +@@ -1370,3 +1380,465 @@ int run_cmd(struct test_options *opts) + ret = run_one_cmd(opts); + return ret; + } ++ ++int perf_event_open(struct perf_event_attr *attr, ++ pid_t pid, int cpu, int group_fd, ++ unsigned long flags) ++{ ++ return syscall(__NR_perf_event_open, attr, pid, cpu, group_fd, flags); ++} ++ ++unsigned long long perf_event_put(int *perf_fds, int nr_fds); ++ ++int perf_event_get(const char *event_name, int **perf_fds, int *nr_fds) ++{ ++ int ret; ++ int cpu; ++ FILE *fd; ++ int nr_cpus; ++ unsigned int event_id; ++ char event_id_file[256]; ++ struct perf_event_attr event = { ++ .type = PERF_TYPE_TRACEPOINT, ++ .size = sizeof(event), ++ .disabled = true, ++ }; ++ ++ *perf_fds = NULL; ++ *nr_fds = 0; ++ ++ nr_cpus = sysconf(_SC_NPROCESSORS_ONLN); ++ if (nr_cpus <= 0) { ++ WD_ERR("invalid number of CPUs\n"); ++ return nr_cpus; ++ } ++ ++ ret = snprintf(event_id_file, sizeof(event_id_file), ++ "/sys/kernel/debug/tracing/events/%s/id", event_name); ++ if (ret >= sizeof(event_id_file)) { ++ WD_ERR("event_id buffer overflow\n"); ++ return -EOVERFLOW; ++ } ++ fd = fopen(event_id_file, "r"); ++ if (fd == NULL) { ++ ret = -errno; ++ WD_ERR("Couldn't open file %s\n", event_id_file); ++ return ret; ++ } ++ ++ if (fscanf(fd, "%d", &event_id) != 1) { ++ WD_ERR("Couldn't parse file %s\n", event_id_file); ++ return -EINVAL; ++ } ++ fclose(fd); ++ event.config = event_id; ++ ++ *perf_fds = calloc(nr_cpus, sizeof(int)); ++ if (!*perf_fds) ++ return -ENOMEM; ++ *nr_fds = nr_cpus; ++ ++ /* ++ * An event is bound to either a CPU or a PID. If we want both, we need ++ * to open the event on all CPUs. Note that we can't use a perf group ++ * since they have to be on the same CPU. ++ */ ++ for (cpu = 0; cpu < nr_cpus; cpu++) { ++ int fd = perf_event_open(&event, -1, cpu, -1, 0); ++ ++ if (fd < 0) { ++ WD_ERR("Couldn't get perf event %s on CPU%d: %d\n", ++ event_name, cpu, errno); ++ perf_event_put(*perf_fds, cpu); ++ return fd; ++ } ++ ++ ioctl(fd, PERF_EVENT_IOC_RESET, 0); ++ ioctl(fd, PERF_EVENT_IOC_ENABLE, 0); ++ (*perf_fds)[cpu] = fd; ++ } ++ ++ return 0; ++} ++ ++/* ++ * Closes the perf fd and return the sample count. If it wasn't open, return 0. ++ */ ++unsigned long long perf_event_put(int *perf_fds, int nr_fds) ++{ ++ int ret; ++ int cpu; ++ uint64_t count, total = 0; ++ ++ if (!perf_fds) ++ return 0; ++ ++ for (cpu = 0; cpu < nr_fds; cpu++) { ++ int fd = perf_fds[cpu]; ++ ++ if (fd <= 0) { ++ WD_ERR("Invalid perf fd %d\n", cpu); ++ continue; ++ } ++ ++ ioctl(fd, PERF_EVENT_IOC_DISABLE, 0); ++ ++ ret = read(fd, &count, sizeof(count)); ++ if (ret < sizeof(count)) ++ WD_ERR("Couldn't read perf event for CPU%d\n", cpu); ++ ++ total += count; ++ close(fd); ++ ++ } ++ ++ free(perf_fds); ++ return total; ++} ++ ++static void set_thp(struct test_options *opts) ++{ ++ char *p; ++ char s[14]; ++ FILE *file; ++ ++ file = fopen("/sys/kernel/mm/transparent_hugepage/enabled", "r"); ++ if (!file) ++ goto out_err; ++ p = fgets(s, 14, file); ++ fclose(file); ++ if (!p) ++ goto out_err; ++ ++ if (strcmp(s, "never") == 0) { ++ printf("Cannot test THP with enable=never\n"); ++ return; ++ } ++ ++ file = fopen("/sys/kernel/mm/transparent_hugepage/defrag", "r"); ++ if (!file) ++ goto out_err; ++ p = fgets(s, 14, file); ++ fclose(file); ++ if (!p) ++ goto out_err; ++ ++ if (strcmp(s, "defer") == 0 || strcmp(s, "never") == 0) { ++ printf("Cannot test THP with defrag=%s\n", s); ++ return; ++ } ++ ++ return; ++out_err: ++ printf("THP unsupported?\n"); ++} ++ ++void stat_setup(struct hizip_test_info *info) ++{ ++ clock_gettime(CLOCK_MONOTONIC_RAW, &info->tv.setup_time); ++ clock_gettime(CLOCK_PROCESS_CPUTIME_ID, &info->tv.setup_cputime); ++ getrusage(RUSAGE_SELF, &info->tv.setup_rusage); ++} ++ ++void stat_start(struct hizip_test_info *info) ++{ ++ clock_gettime(CLOCK_MONOTONIC_RAW, &info->tv.start_time); ++ clock_gettime(CLOCK_PROCESS_CPUTIME_ID, &info->tv.start_cputime); ++ getrusage(RUSAGE_SELF, &info->tv.start_rusage); ++} ++ ++void stat_end(struct hizip_test_info *info) ++{ ++ struct test_options *opts = info->opts; ++ struct hizip_stats *stats = info->stats; ++ double v; ++ size_t total_out; ++ unsigned long total_len; ++ ++ total_out = __atomic_load_n(&info->total_out, __ATOMIC_ACQUIRE); ++ clock_gettime(CLOCK_MONOTONIC_RAW, &info->tv.end_time); ++ clock_gettime(CLOCK_PROCESS_CPUTIME_ID, &info->tv.end_cputime); ++ getrusage(RUSAGE_SELF, &info->tv.end_rusage); ++ ++ stats->v[ST_SETUP_TIME] = (info->tv.start_time.tv_sec - ++ info->tv.setup_time.tv_sec) * 1000000000 + ++ info->tv.start_time.tv_nsec - ++ info->tv.setup_time.tv_nsec; ++ stats->v[ST_RUN_TIME] = (info->tv.end_time.tv_sec - ++ info->tv.start_time.tv_sec) * 1000000000 + ++ info->tv.end_time.tv_nsec - ++ info->tv.start_time.tv_nsec; ++ ++ stats->v[ST_CPU_TIME] = (info->tv.end_cputime.tv_sec - ++ info->tv.setup_cputime.tv_sec) * 1000000000 + ++ info->tv.end_cputime.tv_nsec - ++ info->tv.setup_cputime.tv_nsec; ++ stats->v[ST_USER_TIME] = (info->tv.end_rusage.ru_utime.tv_sec - ++ info->tv.setup_rusage.ru_utime.tv_sec) * ++ 1000000 + ++ info->tv.end_rusage.ru_utime.tv_usec - ++ info->tv.setup_rusage.ru_utime.tv_usec; ++ stats->v[ST_SYSTEM_TIME] = (info->tv.end_rusage.ru_stime.tv_sec - ++ info->tv.setup_rusage.ru_stime.tv_sec) * ++ 1000000 + ++ info->tv.end_rusage.ru_stime.tv_usec - ++ info->tv.setup_rusage.ru_stime.tv_usec; ++ ++ stats->v[ST_MINFLT] = info->tv.end_rusage.ru_minflt - ++ info->tv.setup_rusage.ru_minflt; ++ stats->v[ST_MAJFLT] = info->tv.end_rusage.ru_majflt - ++ info->tv.setup_rusage.ru_majflt; ++ ++ stats->v[ST_VCTX] = info->tv.end_rusage.ru_nvcsw - ++ info->tv.setup_rusage.ru_nvcsw; ++ stats->v[ST_INVCTX] = info->tv.end_rusage.ru_nivcsw - ++ info->tv.setup_rusage.ru_nivcsw; ++ ++ stats->v[ST_SIGNALS] = info->tv.end_rusage.ru_nsignals - ++ info->tv.setup_rusage.ru_nsignals; ++ ++ /* check last loop is enough, same as below hizip_verify_output */ ++ stats->v[ST_COMPRESSION_RATIO] = (double)opts->total_len / ++ total_out * 100; ++ ++ total_len = opts->total_len * opts->compact_run_num; ++ /* ST_RUN_TIME records nanoseconds */ ++ stats->v[ST_SPEED] = (total_len * opts->thread_num * 1000) / ++ (1.024 * 1.024 * stats->v[ST_RUN_TIME]); ++ ++ stats->v[ST_TOTAL_SPEED] = (total_len * opts->thread_num * 1000) / ++ ((stats->v[ST_RUN_TIME] + ++ stats->v[ST_SETUP_TIME]) * 1.024 * 1.024); ++ ++ v = stats->v[ST_RUN_TIME] + stats->v[ST_SETUP_TIME]; ++ stats->v[ST_CPU_IDLE] = (v - stats->v[ST_CPU_TIME]) / v * 100; ++ stats->v[ST_FAULTS] = stats->v[ST_MAJFLT] + stats->v[ST_MINFLT]; ++} ++ ++static void handle_sigbus(int sig) ++{ ++ printf("SIGBUS!\n"); ++ _exit(0); ++} ++ ++int main(int argc, char **argv) ++{ ++ struct test_options opts = { ++ .alg_type = WD_GZIP, ++ .op_type = WD_DIR_COMPRESS, ++ .q_num = 1, ++ .run_num = 1, ++ .compact_run_num = 1, ++ .thread_num = 1, ++ .sync_mode = 0, ++ .block_size = 512000, ++ .total_len = opts.block_size * 10, ++ .verify = false, ++ .verbose = false, ++ .is_decomp = false, ++ .is_stream = false, ++ .is_file = false, ++ .display_stats = STATS_PRETTY, ++ .children = 0, ++ .faults = 0, ++ .data_fmt = 0, ++ }; ++ struct option long_options[] = { ++ {"self", no_argument, 0, 0 }, ++ {"in", required_argument, 0, 0 }, ++ {"out", required_argument, 0, 0 }, ++ {"ilist", required_argument, 0, 0 }, ++ {"olist", required_argument, 0, 0 }, ++ {"env", no_argument, 0, 0 }, ++ {0, 0, 0, 0 }, ++ }; ++ int show_help = 0; ++ int opt, option_idx; ++ int self = 0; ++ ++ opts.fd_in = -1; ++ opts.fd_out = -1; ++ opts.fd_ilist = -1; ++ opts.fd_olist = -1; ++ opts.alg_type = WD_COMP_ALG_MAX; ++ while ((opt = getopt_long(argc, argv, COMMON_OPTSTRING "f:o:w:k:r:", ++ long_options, &option_idx)) != -1) { ++ switch (opt) { ++ case 0: ++ switch (option_idx) { ++ case 0: /* self */ ++ self = 1; ++ break; ++ case 1: /* in */ ++ if (optarg) { ++ opts.fd_in = open(optarg, O_RDONLY); ++ if (opts.fd_in < 0) { ++ printf("Fail to open %s\n", ++ optarg); ++ show_help = 1; ++ } else ++ opts.is_file = true; ++ } else { ++ printf("Input file is missing!\n"); ++ show_help = 1; ++ } ++ if (lseek(opts.fd_in, 0, SEEK_SET) < 0) { ++ printf("Fail on lseek()!\n"); ++ show_help = 1; ++ } ++ break; ++ case 2: /* out */ ++ if (optarg) { ++ opts.fd_out = open(optarg, ++ O_CREAT | O_WRONLY, ++ S_IWUSR | S_IRGRP | ++ S_IROTH); ++ if (opts.fd_out < 0) { ++ printf("Fail to open %s\n", ++ optarg); ++ show_help = 1; ++ } else ++ opts.is_file = true; ++ } else { ++ printf("Output file is missing!\n"); ++ show_help = 1; ++ } ++ if (lseek(opts.fd_out, 0, SEEK_SET) < 0) { ++ printf("Fail on lseek()!\n"); ++ show_help = 1; ++ } ++ break; ++ case 3: /* ilist */ ++ if (!optarg) { ++ printf("IN list file is missing!\n"); ++ show_help = 1; ++ break; ++ } ++ opts.fd_ilist = open(optarg, O_RDONLY); ++ if (opts.fd_ilist < 0) { ++ printf("Fail to open %s\n", optarg); ++ show_help = 1; ++ break; ++ } ++ opts.is_file = true; ++ if (lseek(opts.fd_ilist, 0, SEEK_SET) < 0) { ++ printf("Fail on lseek()!\n"); ++ show_help = 1; ++ break; ++ } ++ break; ++ case 4: /* olist */ ++ if (!optarg) { ++ printf("OUT list file is missing!\n"); ++ show_help = 1; ++ break; ++ } ++ opts.fd_olist = open(optarg, ++ O_CREAT | O_WRONLY, ++ S_IWUSR | S_IRGRP | ++ S_IROTH); ++ if (opts.fd_olist < 0) { ++ printf("Fail to open %s\n", optarg); ++ show_help = 1; ++ break; ++ } ++ opts.is_file = true; ++ if (lseek(opts.fd_olist, 0, SEEK_SET) < 0) { ++ printf("Fail on lseek()!\n"); ++ show_help = 1; ++ break; ++ } ++ break; ++ case 5: /* env */ ++ opts.use_env = true; ++ break; ++ default: ++ show_help = 1; ++ break; ++ } ++ break; ++ case 'f': ++ if (strcmp(optarg, "none") == 0) { ++ opts.display_stats = STATS_NONE; ++ } else if (strcmp(optarg, "csv") == 0) { ++ opts.display_stats = STATS_CSV; ++ } else if (strcmp(optarg, "pretty") == 0) { ++ opts.display_stats = STATS_PRETTY; ++ } else { ++ SYS_ERR_COND(1, "invalid argument to -f: '%s'\n", optarg); ++ break; ++ } ++ break; ++ case 'o': ++ switch (optarg[0]) { ++ case 'p': ++ opts.option |= PERFORMANCE; ++ break; ++ case 't': ++ opts.option |= TEST_THP; ++ set_thp(&opts); ++ break; ++ default: ++ SYS_ERR_COND(1, "invalid argument to -o: '%s'\n", optarg); ++ break; ++ } ++ break; ++ case 'c': ++ opts.option |= TEST_ZLIB; ++ break; ++ case 'r': ++ opts.children = strtol(optarg, NULL, 0); ++ if (opts.children < 0) ++ show_help = 1; ++ break; ++ case 'k': ++ switch (optarg[0]) { ++ case 'b': ++ opts.faults |= INJECT_SIG_BIND; ++ break; ++ case 't': ++ opts.faults |= INJECT_TLB_FAULT; ++ break; ++ case 'w': ++ opts.faults |= INJECT_SIG_WORK; ++ break; ++ default: ++ SYS_ERR_COND(1, "invalid argument to -k: '%s'\n", optarg); ++ break; ++ } ++ break; ++ default: ++ show_help = parse_common_option(opt, optarg, &opts); ++ break; ++ } ++ } ++ ++ signal(SIGBUS, handle_sigbus); ++ ++ if (!show_help) { ++ if (self) ++ return run_self_test(&opts); ++ return run_cmd(&opts); ++ } ++ ++ hizip_test_adjust_len(&opts); ++ ++ SYS_ERR_COND(show_help || optind > argc, ++ COMMON_HELP ++ " -f output format for the statistics\n" ++ " 'none' do not output statistics\n" ++ " 'pretty' human readable format\n" ++ " 'csv' raw, machine readable\n" ++ " -o options\n" ++ " 'perf' prefaults the output pages\n" ++ " 'thp' try to enable transparent huge pages\n" ++ " 'zlib' use zlib instead of the device\n" ++ " -r number of children to create\n" ++ " -k kill thread\n" ++ " 'bind' kills the process after bind\n" ++ " 'tlb' tries to access an unmapped buffer\n" ++ " 'work' kills the process while the queue is working\n", ++ argv[0] ++ ); ++ return 0; ++} +\ No newline at end of file +-- +2.25.1 + diff --git a/0026-uadk-fix-for-sec-software-compute-check.patch b/0026-uadk-fix-for-sec-software-compute-check.patch new file mode 100644 index 0000000..1cb1908 --- /dev/null +++ b/0026-uadk-fix-for-sec-software-compute-check.patch @@ -0,0 +1,42 @@ +From b6790254696fd1d67a798d9d2523cf3ec03b6e3d Mon Sep 17 00:00:00 2001 +From: Wenkai Lin +Date: Thu, 10 Oct 2024 15:54:18 +0800 +Subject: [PATCH 26/39] uadk: fix for sec software compute check + +For aead stream mode, the final message is not sent to hardware when +the qm V3 version is used or when the length of the message is 0, +so software compute check should return true to stop qm_recv process. + +Signed-off-by: Wenkai Lin +Signed-off-by: Qi Tao +--- + drv/hisi_sec.c | 13 +++++++++++-- + 1 file changed, 11 insertions(+), 2 deletions(-) + +diff --git a/drv/hisi_sec.c b/drv/hisi_sec.c +index 6377586..9d69831 100644 +--- a/drv/hisi_sec.c ++++ b/drv/hisi_sec.c +@@ -2647,8 +2647,17 @@ static void parse_aead_bd2(struct hisi_qp *qp, struct hisi_sec_sqe *sqe, + static bool soft_compute_check(struct hisi_qp *qp, struct wd_aead_msg *msg) + { + /* Asynchronous mode does not use the sent message, so ignores it */ +- if (msg->cmode == WD_CIPHER_GCM) +- return (msg->msg_state == AEAD_MSG_END) && qp->q_info.qp_mode == CTX_MODE_SYNC; ++ if (qp->q_info.qp_mode == CTX_MODE_ASYNC) ++ return false; ++ /* ++ * For aead gcm stream mode, due to some hardware limitations, ++ * the final message was not sent to hardware if the qm is ++ * not higher than v3 version or the input length of the ++ * message is 0, the software calculation has been executed. ++ */ ++ if (msg->msg_state == AEAD_MSG_END && msg->cmode == WD_CIPHER_GCM && ++ (qp->q_info.hw_type <= HISI_QM_API_VER3_BASE || !msg->in_bytes)) ++ return true; + + return false; + } +-- +2.25.1 + diff --git a/0027-uadk-fix-some-code-bugs.patch b/0027-uadk-fix-some-code-bugs.patch new file mode 100644 index 0000000..7fc8bc6 --- /dev/null +++ b/0027-uadk-fix-some-code-bugs.patch @@ -0,0 +1,85 @@ +From e18ed1501180603036224a139a977634de4ddcf4 Mon Sep 17 00:00:00 2001 +From: Longfang Liu +Date: Thu, 10 Oct 2024 15:56:04 +0800 +Subject: [PATCH 27/39] uadk: fix some code bugs + +There are some previously undiscovered issues in these code files. +These issues include: +1. Unchecked function return value +2. Unverified external input parameters +3. Illegal assignment operation + +Therefore, it needs to be fixed. + +Signed-off-by: Longfang Liu +Signed-off-by: Qi Tao +--- + drv/hisi_comp.c | 2 ++ + wd_ecc.c | 6 +++--- + wd_util.c | 11 +++++++++-- + 3 files changed, 14 insertions(+), 5 deletions(-) + +diff --git a/drv/hisi_comp.c b/drv/hisi_comp.c +index c556998..b4c216f 100644 +--- a/drv/hisi_comp.c ++++ b/drv/hisi_comp.c +@@ -510,6 +510,8 @@ static int fill_buf_lz77_zstd_sgl(handle_t h_qp, struct hisi_zip_sqe *sqe, + fill_buf_type_sgl(sqe); + + seq_start = get_seq_start_list(req); ++ if (unlikely(!seq_start)) ++ return -WD_EINVAL; + + data->literals_start = req->list_dst; + data->sequences_start = seq_start; +diff --git a/wd_ecc.c b/wd_ecc.c +index b1712c5..292338a 100644 +--- a/wd_ecc.c ++++ b/wd_ecc.c +@@ -1504,7 +1504,7 @@ static int fill_ecc_msg(struct wd_ecc_msg *msg, struct wd_ecc_req *req, + void *key = NULL; + + memcpy(&msg->req, req, sizeof(msg->req)); +- msg->hash = sess->setup.hash; ++ memcpy(&msg->hash, &sess->setup.hash, sizeof(msg->hash)); + msg->key_bytes = sess->key_size; + msg->curve_id = sess->setup.cv.cfg.id; + msg->result = WD_EINVAL; +@@ -1655,10 +1655,10 @@ static int set_sign_in_param(struct wd_ecc_sign_in *sin, + + static int generate_random(struct wd_ecc_sess *sess, struct wd_dtb *k) + { +- struct wd_rand_mt rand_t = sess->setup.rand; ++ struct wd_rand_mt *rand_t = &sess->setup.rand; + int ret; + +- ret = rand_t.cb(k->data, k->dsize, rand_t.usr); ++ ret = rand_t->cb(k->data, k->dsize, rand_t->usr); + if (ret) + WD_ERR("failed to do rand cb, ret = %d!\n", ret); + +diff --git a/wd_util.c b/wd_util.c +index d58e8b8..e908dcb 100644 +--- a/wd_util.c ++++ b/wd_util.c +@@ -421,8 +421,15 @@ void wd_uninit_async_request_pool(struct wd_async_msg_pool *pool) + void *wd_find_msg_in_pool(struct wd_async_msg_pool *pool, + int ctx_idx, __u32 tag) + { +- struct msg_pool *p = &pool->pools[ctx_idx]; +- __u32 msg_num = p->msg_num; ++ struct msg_pool *p; ++ __u32 msg_num; ++ ++ if ((__u32)ctx_idx > pool->pool_num) { ++ WD_ERR("invalid: message ctx id index is %d!\n", ctx_idx); ++ return NULL; ++ } ++ p = &pool->pools[ctx_idx]; ++ msg_num = p->msg_num; + + /* tag value start from 1 */ + if (tag == 0 || tag > msg_num) { +-- +2.25.1 + diff --git a/0028-uadk-cipher-add-length-check-of-CBC_CTS-mode.patch b/0028-uadk-cipher-add-length-check-of-CBC_CTS-mode.patch new file mode 100644 index 0000000..d707c9b --- /dev/null +++ b/0028-uadk-cipher-add-length-check-of-CBC_CTS-mode.patch @@ -0,0 +1,75 @@ +From 11e9545abb4e861f2cc589dfa76f20564033c995 Mon Sep 17 00:00:00 2001 +From: Yang Shen +Date: Fri, 11 Oct 2024 11:27:54 +0800 +Subject: [PATCH 28/39] uadk/cipher - add length check of CBC_CTS mode + +Add length check of CBC_CTS mode. + +Signed-off-by: Yang Shen +Signed-off-by: Qi Tao +--- + wd_cipher.c | 30 ++++++++++++++++++++++-------- + 1 file changed, 22 insertions(+), 8 deletions(-) + +diff --git a/wd_cipher.c b/wd_cipher.c +index f6b035a..ec6fb15 100644 +--- a/wd_cipher.c ++++ b/wd_cipher.c +@@ -588,9 +588,10 @@ static int cipher_iv_len_check(struct wd_cipher_req *req, + return ret; + } + +-static int cipher_len_check(handle_t h_sess, struct wd_cipher_req *req) ++static int cipher_in_len_check(handle_t h_sess, struct wd_cipher_req *req) + { + struct wd_cipher_sess *sess = (struct wd_cipher_sess *)h_sess; ++ int ret = 0; + + if (!req->in_bytes) { + WD_ERR("invalid: cipher input length is zero!\n"); +@@ -600,14 +601,27 @@ static int cipher_len_check(handle_t h_sess, struct wd_cipher_req *req) + if (sess->alg != WD_CIPHER_AES && sess->alg != WD_CIPHER_SM4) + return 0; + +- if ((req->in_bytes & (AES_BLOCK_SIZE - 1)) && +- (sess->mode == WD_CIPHER_CBC || sess->mode == WD_CIPHER_ECB)) { +- WD_ERR("failed to check input bytes of AES or SM4, size = %u\n", +- req->in_bytes); +- return -WD_EINVAL; ++ switch (sess->mode) { ++ case WD_CIPHER_ECB: ++ case WD_CIPHER_CBC: ++ if (req->in_bytes & (AES_BLOCK_SIZE - 1)) ++ ret = -WD_EINVAL; ++ break; ++ case WD_CIPHER_CBC_CS1: ++ case WD_CIPHER_CBC_CS2: ++ case WD_CIPHER_CBC_CS3: ++ if (req->in_bytes < AES_BLOCK_SIZE) ++ ret = -WD_EINVAL; ++ break; ++ default: ++ break; + } + +- return 0; ++ if (ret) ++ WD_ERR("invalid: %s input bytes is %u!\n", ++ wd_cipher_alg_name[sess->alg][sess->mode], req->in_bytes); ++ ++ return ret; + } + + static int wd_cipher_check_params(handle_t h_sess, +@@ -632,7 +646,7 @@ static int wd_cipher_check_params(handle_t h_sess, + return -WD_EINVAL; + } + +- ret = cipher_len_check(h_sess, req); ++ ret = cipher_in_len_check(h_sess, req); + if (unlikely(ret)) + return ret; + +-- +2.25.1 + diff --git a/0029-uadk-fix-for-cipher-update-iv.patch b/0029-uadk-fix-for-cipher-update-iv.patch new file mode 100644 index 0000000..9e5b0d7 --- /dev/null +++ b/0029-uadk-fix-for-cipher-update-iv.patch @@ -0,0 +1,121 @@ +From d99d0e2683c6507304f72905b773623968c0962a Mon Sep 17 00:00:00 2001 +From: Wenkai Lin +Date: Fri, 11 Oct 2024 11:29:45 +0800 +Subject: [PATCH 29/39] uadk: fix for cipher update iv + +The cipher iv is incorrectly updated, as a result, the uadk +fails to encrypt data. The CFB, OFB, and CTR mode are modified +according to the algorithm protocol. + +Signed-off-by: Wenkai Lin +Signed-off-by: Qi Tao +--- + drv/hisi_sec.c | 40 ++++++++++++++++++++++++++++------------ + wd_cipher.c | 6 ++++++ + 2 files changed, 34 insertions(+), 12 deletions(-) + +diff --git a/drv/hisi_sec.c b/drv/hisi_sec.c +index 9d69831..bdde39f 100644 +--- a/drv/hisi_sec.c ++++ b/drv/hisi_sec.c +@@ -733,11 +733,14 @@ static void ctr_iv_inc(__u8 *counter, __u32 len) + + static void update_iv(struct wd_cipher_msg *msg) + { ++ __u8 i; ++ + switch (msg->mode) { + case WD_CIPHER_CBC: + case WD_CIPHER_CBC_CS1: + case WD_CIPHER_CBC_CS2: + case WD_CIPHER_CBC_CS3: ++ case WD_CIPHER_CFB: + if (msg->op_type == WD_CIPHER_ENCRYPTION && + msg->out_bytes >= msg->iv_bytes) + memcpy(msg->iv, msg->out + msg->out_bytes - +@@ -748,14 +751,15 @@ static void update_iv(struct wd_cipher_msg *msg) + msg->iv_bytes, msg->iv_bytes); + break; + case WD_CIPHER_OFB: +- case WD_CIPHER_CFB: +- if (msg->out_bytes >= msg->iv_bytes) +- memcpy(msg->iv, msg->out + msg->out_bytes - +- msg->iv_bytes, msg->iv_bytes); ++ if (msg->in_bytes < msg->iv_bytes) ++ break; ++ /* The iv_bytes has been checked and it is not greater than AES_BLOCK_SIZE. */ ++ for (i = 0; i < msg->iv_bytes; i++) ++ msg->iv[i] = *((__u8 *)msg->in + msg->in_bytes - msg->iv_bytes + i) ^ ++ *((__u8 *)msg->out + msg->out_bytes - msg->iv_bytes + i); + break; + case WD_CIPHER_CTR: +- ctr_iv_inc(msg->iv, msg->iv_bytes >> +- CTR_MODE_LEN_SHIFT); ++ ctr_iv_inc(msg->iv, msg->in_bytes >> CTR_MODE_LEN_SHIFT); + break; + default: + break; +@@ -764,8 +768,16 @@ static void update_iv(struct wd_cipher_msg *msg) + + static void update_iv_sgl(struct wd_cipher_msg *msg) + { ++ __u8 out[AES_BLOCK_SIZE] = {0}; ++ __u8 in[AES_BLOCK_SIZE] = {0}; ++ __u8 i; ++ + switch (msg->mode) { + case WD_CIPHER_CBC: ++ case WD_CIPHER_CBC_CS1: ++ case WD_CIPHER_CBC_CS2: ++ case WD_CIPHER_CBC_CS3: ++ case WD_CIPHER_CFB: + if (msg->op_type == WD_CIPHER_ENCRYPTION && + msg->out_bytes >= msg->iv_bytes) + hisi_qm_sgl_copy(msg->iv, msg->out, +@@ -780,16 +792,20 @@ static void update_iv_sgl(struct wd_cipher_msg *msg) + + break; + case WD_CIPHER_OFB: +- case WD_CIPHER_CFB: +- if (msg->out_bytes >= msg->iv_bytes) +- hisi_qm_sgl_copy(msg->iv, msg->out, ++ /* The iv_bytes has been checked and it is not greater than AES_BLOCK_SIZE. */ ++ if (msg->in_bytes >= msg->iv_bytes) { ++ hisi_qm_sgl_copy(in, msg->in, ++ msg->in_bytes - msg->iv_bytes, ++ msg->iv_bytes, COPY_SGL_TO_PBUFF); ++ hisi_qm_sgl_copy(out, msg->out, + msg->out_bytes - msg->iv_bytes, + msg->iv_bytes, COPY_SGL_TO_PBUFF); +- ++ for (i = 0; i < msg->iv_bytes; i++) ++ msg->iv[i] = *(in + i) ^ *(out + i); ++ } + break; + case WD_CIPHER_CTR: +- ctr_iv_inc(msg->iv, msg->iv_bytes >> +- CTR_MODE_LEN_SHIFT); ++ ctr_iv_inc(msg->iv, msg->in_bytes >> CTR_MODE_LEN_SHIFT); + break; + default: + break; +diff --git a/wd_cipher.c b/wd_cipher.c +index ec6fb15..0e5de25 100644 +--- a/wd_cipher.c ++++ b/wd_cipher.c +@@ -646,6 +646,12 @@ static int wd_cipher_check_params(handle_t h_sess, + return -WD_EINVAL; + } + ++ if (unlikely(req->in_bytes != req->out_bytes)) { ++ WD_ERR("cipher set out_bytes is error, size = %u\n", ++ req->out_bytes); ++ return -WD_EINVAL; ++ } ++ + ret = cipher_in_len_check(h_sess, req); + if (unlikely(ret)) + return ret; +-- +2.25.1 + diff --git a/0030-uadk-v1-fix-for-cipher-update-iv.patch b/0030-uadk-v1-fix-for-cipher-update-iv.patch new file mode 100644 index 0000000..d99a09c --- /dev/null +++ b/0030-uadk-v1-fix-for-cipher-update-iv.patch @@ -0,0 +1,143 @@ +From 4a66322e7be478c48b2ebcb2104fec114d37cd7b Mon Sep 17 00:00:00 2001 +From: Wenkai Lin +Date: Fri, 11 Oct 2024 11:31:23 +0800 +Subject: [PATCH 30/39] uadk/v1: fix for cipher update iv + +The cipher iv is incorrectly updated, as a result, the uadk +fails to encrypt data. The CFB, OFB mode are modified +according to the algorithm protocol. BD2 does not support +the ofb mode, it works with cipher_ofb_data_handle, so it +still uses the old way to update iv. + +Signed-off-by: Wenkai Lin +Signed-off-by: Qi Tao +--- + v1/drv/hisi_sec_udrv.c | 80 +++++++++++++++++++++++++++++++++--------- + 1 file changed, 63 insertions(+), 17 deletions(-) + +diff --git a/v1/drv/hisi_sec_udrv.c b/v1/drv/hisi_sec_udrv.c +index ac6df5f..2c7ca20 100644 +--- a/v1/drv/hisi_sec_udrv.c ++++ b/v1/drv/hisi_sec_udrv.c +@@ -253,30 +253,76 @@ static void update_iv_from_res(__u8 *dst, __u8 *src, size_t offset, __u16 bytes, + } + } + +-static void update_iv(struct wcrypto_cipher_msg *msg) ++static int update_iv_ofb(struct wcrypto_cipher_msg *msg, size_t offset) + { ++ __u8 out[AES_BLOCK_SIZE] = {0}; ++ __u8 in[AES_BLOCK_SIZE] = {0}; ++ __u8 iv[AES_BLOCK_SIZE] = {0}; ++ int ret; ++ __u8 i; ++ ++ /* The iv_bytes is not greater than AES_BLOCK_SIZE. */ ++ if (msg->data_fmt == WD_SGL_BUF) { ++ ret = wd_sgl_cp_to_pbuf((struct wd_sgl *)msg->out, offset, ++ (void *)out, msg->iv_bytes); ++ if (unlikely(ret)) ++ return ret; ++ ret = wd_sgl_cp_to_pbuf((struct wd_sgl *)msg->in, offset, ++ (void *)in, msg->iv_bytes); ++ if (unlikely(ret)) ++ return ret; ++ for (i = 0; i < msg->iv_bytes; i++) ++ iv[i] = in[i] ^ out[i]; ++ ++ return wd_sgl_cp_from_pbuf((struct wd_sgl *)msg->iv, 0, ++ (void *)iv, msg->iv_bytes); ++ } ++ ++ for (i = 0; i < msg->iv_bytes; i++) ++ msg->iv[i] = *((__u8 *)msg->in + offset + i) ^ ++ *((__u8 *)msg->out + offset + i); ++ ++ return 0; ++} ++ ++static void update_iv(struct wcrypto_cipher_msg *msg, int bd_type) ++{ ++ size_t offset; ++ int ret; ++ + switch (msg->mode) { + case WCRYPTO_CIPHER_CBC: + case WCRYPTO_CIPHER_CBC_CS1: + case WCRYPTO_CIPHER_CBC_CS2: + case WCRYPTO_CIPHER_CBC_CS3: +- if (msg->op_type == WCRYPTO_CIPHER_ENCRYPTION && +- msg->out_bytes >= msg->iv_bytes) +- update_iv_from_res(msg->iv, msg->out, +- msg->out_bytes - msg->iv_bytes, ++ case WCRYPTO_CIPHER_CFB: ++ /* The out_bytes is equal to the in_bytes. */ ++ if (msg->out_bytes < msg->iv_bytes) ++ break; ++ ++ offset = msg->out_bytes - msg->iv_bytes; ++ if (msg->op_type == WCRYPTO_CIPHER_ENCRYPTION) ++ update_iv_from_res(msg->iv, msg->out, offset, + msg->iv_bytes, msg->data_fmt); +- if (msg->op_type == WCRYPTO_CIPHER_DECRYPTION && +- msg->in_bytes >= msg->iv_bytes) +- update_iv_from_res(msg->iv, msg->in, +- msg->in_bytes - msg->iv_bytes, ++ else ++ update_iv_from_res(msg->iv, msg->in, offset, + msg->iv_bytes, msg->data_fmt); + break; + case WCRYPTO_CIPHER_OFB: +- case WCRYPTO_CIPHER_CFB: +- if (msg->out_bytes >= msg->iv_bytes) +- update_iv_from_res(msg->iv, msg->out, +- msg->out_bytes - msg->iv_bytes, ++ /* The out_bytes is equal to the in_bytes. */ ++ if (msg->out_bytes < msg->iv_bytes) ++ break; ++ ++ offset = msg->out_bytes - msg->iv_bytes; ++ /* BD2 does not support the ofb mode, it works with cipher_ofb_data_handle. */ ++ if (bd_type == BD_TYPE2) { ++ update_iv_from_res(msg->iv, msg->out, offset, + msg->iv_bytes, msg->data_fmt); ++ } else if (bd_type == BD_TYPE3) { ++ ret = update_iv_ofb(msg, offset); ++ if (unlikely(ret)) ++ WD_ERR("failed to update ofb iv!\n"); ++ } + break; + case WCRYPTO_CIPHER_CTR: + ctr_iv_inc(msg->iv, msg->in_bytes >> CTR_MODE_LEN_SHIFT, +@@ -680,8 +726,8 @@ static int cipher_param_check(struct wcrypto_cipher_msg *msg) + int ret; + + if (unlikely(msg->in_bytes > MAX_CIPHER_LENGTH || +- !msg->in_bytes)) { +- WD_ERR("input cipher len is too large!\n"); ++ !msg->in_bytes || msg->in_bytes != msg->out_bytes)) { ++ WD_ERR("failed to check input cipher len(%u)!\n", msg->in_bytes); + return -WD_EINVAL; + } + +@@ -1849,7 +1895,7 @@ static void parse_cipher_bd2(struct wd_queue *q, struct hisi_sec_sqe *sqe, + sqe->type2.c_ivin_addr_l, sqe->type2.c_ivin_addr_h, + cipher_msg->data_fmt); + +- update_iv(cipher_msg); ++ update_iv(cipher_msg, BD_TYPE2); + + if (cipher_msg->mode == WCRYPTO_CIPHER_OFB) + cipher_ofb_data_handle(cipher_msg); +@@ -1883,7 +1929,7 @@ static void parse_cipher_bd3(struct wd_queue *q, struct hisi_sec_bd3_sqe *sqe, + sqe->ipsec_scene.c_ivin_addr_h, + cipher_msg->data_fmt); + +- update_iv(cipher_msg); ++ update_iv(cipher_msg, BD_TYPE3); + } + + /* +-- +2.25.1 + diff --git a/0031-uadk-reduce-the-print-rating-of-specific-logs.patch b/0031-uadk-reduce-the-print-rating-of-specific-logs.patch new file mode 100644 index 0000000..1a8dbde --- /dev/null +++ b/0031-uadk-reduce-the-print-rating-of-specific-logs.patch @@ -0,0 +1,126 @@ +From c103303271a64066c38ae6ada5d68a789e39949a Mon Sep 17 00:00:00 2001 +From: Chenghai Huang +Date: Sat, 12 Oct 2024 11:02:15 +0800 +Subject: [PATCH 31/39] uadk - reduce the print rating of specific logs + +Reduce the frequency of printing error logs when the status +is 0xe for avoid a large amount of printed information. + +Signed-off-by: Chenghai Huang +Signed-off-by: Qi Tao +--- + drv/hisi_comp.c | 11 +++++++++-- + v1/drv/hisi_zip_udrv.c | 40 ++++++++++++++++++++++++++++++++++++---- + 2 files changed, 45 insertions(+), 6 deletions(-) + +diff --git a/drv/hisi_comp.c b/drv/hisi_comp.c +index b4c216f..07d8754 100644 +--- a/drv/hisi_comp.c ++++ b/drv/hisi_comp.c +@@ -29,6 +29,9 @@ + + #define ZSTD_MAX_SIZE (1 << 17) + ++/* Error status 0xe indicates that dest_avail_out insufficient */ ++#define ERR_DSTLEN_OUT 0xe ++ + #define swab32(x) \ + ((((x) & 0x000000ff) << 24) | \ + (((x) & 0x0000ff00) << 8) | \ +@@ -1039,8 +1042,12 @@ static int parse_zip_sqe(struct hisi_qp *qp, struct hisi_zip_sqe *sqe, + + if (unlikely(status != 0 && status != HZ_NEGACOMPRESS && + status != HZ_CRC_ERR && status != HZ_DECOMP_END)) { +- WD_ERR("bad request(ctx_st = 0x%x, status = 0x%x, algorithm type = %u)!\n", +- ctx_st, status, type); ++ if (status == ERR_DSTLEN_OUT) ++ WD_DEBUG("bad request(ctx_st=0x%x, status=0x%x, algorithm type=%u)!\n", ++ ctx_st, status, type); ++ else ++ WD_ERR("bad request(ctx_st=0x%x, status=0x%x, algorithm type=%u)!\n", ++ ctx_st, status, type); + recv_msg->req.status = WD_IN_EPARA; + } + +diff --git a/v1/drv/hisi_zip_udrv.c b/v1/drv/hisi_zip_udrv.c +index 01d76a3..8497ebc 100644 +--- a/v1/drv/hisi_zip_udrv.c ++++ b/v1/drv/hisi_zip_udrv.c +@@ -26,6 +26,7 @@ + #include + #include + #include ++#include + #include + #include "v1/wd_util.h" + #include "v1/wd_comp.h" +@@ -57,6 +58,10 @@ + #define ZSTD_FREQ_DATA_SIZE 784 + #define REPCODE_SIZE 12 + ++/* Error status 0xe indicates that dest_avail_out insufficient */ ++#define ERR_DSTLEN_OUT 0xe ++#define PRINT_TIME_INTERVAL 21600 ++ + #define CTX_PRIV1_OFFSET 4 + #define CTX_PRIV2_OFFSET 8 + #define CTX_REPCODE1_OFFSET 12 +@@ -95,6 +100,35 @@ struct zip_fill_sqe_ops { + void (*fill_sqe_hw_info)(void *ssqe, struct wcrypto_comp_msg *msg); + }; + ++static unsigned int g_err_print_enable = 1; ++ ++static void zip_err_print_alarm_end(int sig) ++{ ++ if (sig == SIGALRM) { ++ g_err_print_enable = 1; ++ alarm(0); ++ } ++} ++ ++static void zip_err_print_time_start(void) ++{ ++ g_err_print_enable = 0; ++ signal(SIGALRM, zip_err_print_alarm_end); ++ alarm(PRINT_TIME_INTERVAL); ++} ++ ++static void zip_err_bd_print(__u16 ctx_st, __u32 status, __u32 type) ++{ ++ if (status != ERR_DSTLEN_OUT) { ++ WD_ERR("bad status(ctx_st=0x%x, s=0x%x, t=%u)\n", ++ ctx_st, status, type); ++ } else if (g_err_print_enable == 1) { ++ WD_ERR("bad status(ctx_st=0x%x, s=0x%x, t=%u)\n", ++ ctx_st, status, type); ++ zip_err_print_time_start(); ++ } ++} ++ + static int fill_zip_comp_alg_v1(struct hisi_zip_sqe *sqe, + struct wcrypto_comp_msg *msg) + { +@@ -274,8 +308,7 @@ int qm_parse_zip_sqe(void *hw_msg, const struct qm_queue_info *info, + + if (status != 0 && status != HW_NEGACOMPRESS && + status != HW_CRC_ERR && status != HW_DECOMP_END) { +- WD_ERR("bad status(ctx_st=0x%x, s=0x%x, t=%u)\n", +- ctx_st, status, type); ++ zip_err_bd_print(ctx_st, status, type); + recv_msg->status = WD_IN_EPARA; + } else { + recv_msg->status = 0; +@@ -707,8 +740,7 @@ int qm_parse_zip_sqe_v3(void *hw_msg, const struct qm_queue_info *info, + return 0; + + if (status != 0 && status != HW_NEGACOMPRESS && status != HW_DECOMP_END) { +- WD_ERR("bad status(ctx_st=0x%x, s=0x%x, t=%u)\n", +- ctx_st, status, type); ++ zip_err_bd_print(ctx_st, status, type); + recv_msg->status = WD_IN_EPARA; + } else { + recv_msg->status = 0; +-- +2.25.1 + diff --git a/0032-Revert-uadk-v1-replace-wd_spinlock-to-pthread_spin_l.patch b/0032-Revert-uadk-v1-replace-wd_spinlock-to-pthread_spin_l.patch new file mode 100644 index 0000000..5c75a94 --- /dev/null +++ b/0032-Revert-uadk-v1-replace-wd_spinlock-to-pthread_spin_l.patch @@ -0,0 +1,288 @@ +From ff746e0ed02fdd0f547b4e62d89377efba7f9302 Mon Sep 17 00:00:00 2001 +From: Weili Qian +Date: Sat, 12 Oct 2024 11:03:18 +0800 +Subject: [PATCH 32/39] Revert "uadk/v1: replace wd_spinlock to + pthread_spin_lock" + +This reverts commit f7c2a7a3e5116dc0ce4af539070f2ed93bb18af8. + +Signed-off-by: Weili Qian +Signed-off-by: Qi Tao +--- + v1/drv/hisi_qm_udrv.c | 51 +++++++++++++----------------------------- + v1/drv/hisi_qm_udrv.h | 4 ++-- + v1/drv/hisi_rng_udrv.c | 25 ++++++--------------- + v1/drv/hisi_rng_udrv.h | 2 +- + 4 files changed, 26 insertions(+), 56 deletions(-) + +diff --git a/v1/drv/hisi_qm_udrv.c b/v1/drv/hisi_qm_udrv.c +index 1d4f1d8..175a5c4 100644 +--- a/v1/drv/hisi_qm_udrv.c ++++ b/v1/drv/hisi_qm_udrv.c +@@ -20,7 +20,6 @@ + #include + #include + #include +-#include + #include + #include + #include +@@ -459,11 +458,6 @@ static int qm_init_queue_info(struct wd_queue *q) + struct hisi_qp_ctx qp_ctx = {0}; + int ret; + +- if (!info->sqe_size) { +- WD_ERR("invalid: sqe size is 0!\n"); +- return -WD_EINVAL; +- } +- + info->sq_tail_index = 0; + info->cq_head_index = 0; + info->cqc_phase = 1; +@@ -508,6 +502,11 @@ static int qm_set_queue_info(struct wd_queue *q) + ret = qm_set_queue_regions(q); + if (ret) + return -WD_EINVAL; ++ if (!info->sqe_size) { ++ WD_ERR("sqe size =%d err!\n", info->sqe_size); ++ ret = -WD_EINVAL; ++ goto err_with_regions; ++ } + info->cq_base = (void *)((uintptr_t)info->sq_base + + info->sqe_size * info->sq_depth); + +@@ -535,24 +534,8 @@ static int qm_set_queue_info(struct wd_queue *q) + goto err_with_regions; + } + +- ret = pthread_spin_init(&info->sd_lock, PTHREAD_PROCESS_PRIVATE); +- if (ret) { +- WD_ERR("failed to init qinfo sd_lock!\n"); +- goto free_cache; +- } +- +- ret = pthread_spin_init(&info->rc_lock, PTHREAD_PROCESS_PRIVATE); +- if (ret) { +- WD_ERR("failed to init qinfo rc_lock!\n"); +- goto uninit_lock; +- } +- + return 0; + +-uninit_lock: +- pthread_spin_destroy(&info->sd_lock); +-free_cache: +- free(info->req_cache); + err_with_regions: + qm_unset_queue_regions(q); + return ret; +@@ -593,10 +576,8 @@ void qm_uninit_queue(struct wd_queue *q) + struct q_info *qinfo = q->qinfo; + struct qm_queue_info *info = qinfo->priv; + +- pthread_spin_destroy(&info->rc_lock); +- pthread_spin_destroy(&info->sd_lock); +- free(info->req_cache); + qm_unset_queue_regions(q); ++ free(info->req_cache); + free(qinfo->priv); + qinfo->priv = NULL; + } +@@ -624,10 +605,10 @@ int qm_send(struct wd_queue *q, void **req, __u32 num) + int ret; + __u32 i; + +- pthread_spin_lock(&info->sd_lock); ++ wd_spinlock(&info->sd_lock); + if (unlikely((__u32)__atomic_load_n(&info->used, __ATOMIC_RELAXED) > + info->sq_depth - num - 1)) { +- pthread_spin_unlock(&info->sd_lock); ++ wd_unspinlock(&info->sd_lock); + WD_ERR("queue is full!\n"); + return -WD_EBUSY; + } +@@ -636,7 +617,7 @@ int qm_send(struct wd_queue *q, void **req, __u32 num) + ret = info->sqe_fill[qinfo->atype](req[i], qinfo->priv, + info->sq_tail_index); + if (unlikely(ret != WD_SUCCESS)) { +- pthread_spin_unlock(&info->sd_lock); ++ wd_unspinlock(&info->sd_lock); + WD_ERR("sqe fill error, ret %d!\n", ret); + return -WD_EINVAL; + } +@@ -648,7 +629,7 @@ int qm_send(struct wd_queue *q, void **req, __u32 num) + } + + ret = qm_tx_update(info, num); +- pthread_spin_unlock(&info->sd_lock); ++ wd_unspinlock(&info->sd_lock); + + return ret; + } +@@ -681,9 +662,9 @@ static int check_ds_rx_base(struct qm_queue_info *info, + return 0; + + if (before) { +- pthread_spin_lock(&info->rc_lock); ++ wd_spinlock(&info->rc_lock); + qm_rx_from_cache(info, resp, num); +- pthread_spin_unlock(&info->rc_lock); ++ wd_unspinlock(&info->rc_lock); + WD_ERR("wd queue hw error happened before qm receive!\n"); + } else { + WD_ERR("wd queue hw error happened after qm receive!\n"); +@@ -724,7 +705,7 @@ int qm_recv(struct wd_queue *q, void **resp, __u32 num) + if (unlikely(ret)) + return ret; + +- pthread_spin_lock(&info->rc_lock); ++ wd_spinlock(&info->rc_lock); + for (i = 0; i < num; i++) { + cqe = info->cq_base + info->cq_head_index * sizeof(struct cqe); + if (info->cqc_phase != CQE_PHASE(cqe)) +@@ -733,7 +714,7 @@ int qm_recv(struct wd_queue *q, void **resp, __u32 num) + mb(); /* make sure the data is all in memory before read */ + sq_head = CQE_SQ_HEAD_INDEX(cqe); + if (unlikely(sq_head >= info->sq_depth)) { +- pthread_spin_unlock(&info->rc_lock); ++ wd_unspinlock(&info->rc_lock); + WD_ERR("CQE_SQ_HEAD_INDEX(%u) error\n", sq_head); + return -WD_EIO; + } +@@ -745,7 +726,7 @@ int qm_recv(struct wd_queue *q, void **resp, __u32 num) + if (!ret) { + break; + } else if (ret < 0) { +- pthread_spin_unlock(&info->rc_lock); ++ wd_unspinlock(&info->rc_lock); + WD_ERR("recv sqe error %u\n", sq_head); + return ret; + } +@@ -766,7 +747,7 @@ int qm_recv(struct wd_queue *q, void **resp, __u32 num) + ret = i; + } + +- pthread_spin_unlock(&info->rc_lock); ++ wd_unspinlock(&info->rc_lock); + + return ret; + } +diff --git a/v1/drv/hisi_qm_udrv.h b/v1/drv/hisi_qm_udrv.h +index 06ac66a..4d54cf6 100644 +--- a/v1/drv/hisi_qm_udrv.h ++++ b/v1/drv/hisi_qm_udrv.h +@@ -166,8 +166,8 @@ struct qm_queue_info { + qm_sqe_parse sqe_parse[WCRYPTO_MAX_ALG]; + hisi_qm_sqe_fill_priv sqe_fill_priv; + hisi_qm_sqe_parse_priv sqe_parse_priv; +- pthread_spinlock_t sd_lock; +- pthread_spinlock_t rc_lock; ++ struct wd_lock sd_lock; ++ struct wd_lock rc_lock; + struct wd_queue *q; + int (*sgl_info)(struct hw_sgl_info *info); + int (*sgl_init)(void *pool, struct wd_sgl *sgl); +diff --git a/v1/drv/hisi_rng_udrv.c b/v1/drv/hisi_rng_udrv.c +index 605ef27..86a20cb 100644 +--- a/v1/drv/hisi_rng_udrv.c ++++ b/v1/drv/hisi_rng_udrv.c +@@ -17,7 +17,6 @@ + #include + #include + #include +-#include + #include + #include + #include +@@ -35,7 +34,6 @@ int rng_init_queue(struct wd_queue *q) + { + struct q_info *qinfo = q->qinfo; + struct rng_queue_info *info; +- int ret; + + info = calloc(1, sizeof(*info)); + if (!info) { +@@ -43,20 +41,12 @@ int rng_init_queue(struct wd_queue *q) + return -ENOMEM; + } + +- ret = pthread_spin_init(&info->lock, PTHREAD_PROCESS_PRIVATE); +- if (ret) { +- free(info); +- WD_ERR("failed to init rng qinfo lock!\n"); +- return ret; +- } +- + qinfo->priv = info; + info->mmio_base = wd_drv_mmap_qfr(q, WD_UACCE_QFRT_MMIO, 0); + if (info->mmio_base == MAP_FAILED) { + info->mmio_base = NULL; ++ free(qinfo->priv); + qinfo->priv = NULL; +- pthread_spin_destroy(&info->lock); +- free(info); + WD_ERR("mmap trng mmio fail\n"); + return -ENOMEM; + } +@@ -73,7 +63,6 @@ void rng_uninit_queue(struct wd_queue *q) + + free(qinfo->priv); + qinfo->priv = NULL; +- pthread_spin_destroy(&info->lock); + } + + int rng_send(struct wd_queue *q, void **req, __u32 num) +@@ -81,14 +70,14 @@ int rng_send(struct wd_queue *q, void **req, __u32 num) + struct q_info *qinfo = q->qinfo; + struct rng_queue_info *info = qinfo->priv; + +- pthread_spin_lock(&info->lock); ++ wd_spinlock(&info->lock); + if (!info->req_cache[info->send_idx]) { + info->req_cache[info->send_idx] = req[0]; + info->send_idx++; +- pthread_spin_unlock(&info->lock); ++ wd_unspinlock(&info->lock); + return 0; + } +- pthread_spin_unlock(&info->lock); ++ wd_unspinlock(&info->lock); + + WD_ERR("queue is full!\n"); + return -WD_EBUSY; +@@ -139,16 +128,16 @@ int rng_recv(struct wd_queue *q, void **resp, __u32 num) + struct wcrypto_cb_tag *tag; + __u32 currsize = 0; + +- pthread_spin_lock(&info->lock); ++ wd_spinlock(&info->lock); + msg = info->req_cache[info->recv_idx]; + if (!msg) { +- pthread_spin_unlock(&info->lock); ++ wd_unspinlock(&info->lock); + return 0; + } + + info->req_cache[info->recv_idx] = NULL; + info->recv_idx++; +- pthread_spin_unlock(&info->lock); ++ wd_unspinlock(&info->lock); + + tag = (void *)(uintptr_t)msg->usr_tag; + if (usr && tag->ctx_id != usr) +diff --git a/v1/drv/hisi_rng_udrv.h b/v1/drv/hisi_rng_udrv.h +index 3efa10e..56814a4 100644 +--- a/v1/drv/hisi_rng_udrv.h ++++ b/v1/drv/hisi_rng_udrv.h +@@ -29,7 +29,7 @@ struct rng_queue_info { + void *req_cache[TRNG_Q_DEPTH]; + __u8 send_idx; + __u8 recv_idx; +- pthread_spinlock_t lock; ++ struct wd_lock lock; + }; + + int rng_init_queue(struct wd_queue *q); +-- +2.25.1 + diff --git a/0033-Revert-uadk-v1-check-queue-status-before-sending-doo.patch b/0033-Revert-uadk-v1-check-queue-status-before-sending-doo.patch new file mode 100644 index 0000000..2b9b986 --- /dev/null +++ b/0033-Revert-uadk-v1-check-queue-status-before-sending-doo.patch @@ -0,0 +1,194 @@ +From 304509683bf91f0457e63d56fa3b497b66926df0 Mon Sep 17 00:00:00 2001 +From: Weili Qian +Date: Sat, 12 Oct 2024 11:07:33 +0800 +Subject: [PATCH 33/39] Revert "uadk/v1: check queue status before sending + doorbells" + +This reverts commit afc1082d85c6cdac7d3eaaac8130ffb5fa39b138. + +Signed-off-by: Weili Qian +Signed-off-by: Qi Tao +--- + v1/drv/hisi_hpre_udrv.c | 10 +++---- + v1/drv/hisi_qm_udrv.c | 60 ++++++++++++++++------------------------- + v1/drv/hisi_qm_udrv.h | 4 +-- + v1/wd_util.h | 8 +++++- + 4 files changed, 36 insertions(+), 46 deletions(-) + +diff --git a/v1/drv/hisi_hpre_udrv.c b/v1/drv/hisi_hpre_udrv.c +index 05518ab..5832e9a 100644 +--- a/v1/drv/hisi_hpre_udrv.c ++++ b/v1/drv/hisi_hpre_udrv.c +@@ -1930,10 +1930,10 @@ static int fill_sm2_enc_sqe(void *msg, struct qm_queue_info *info, __u16 idx) + goto fail_fill_sqe; + } + ++ /* make sure the request is all in memory before doorbell */ ++ mb(); + info->sq_tail_index = i; +- ret = qm_tx_update(info, 1); +- if (unlikely(ret)) +- goto fail_fill_sqe; ++ qm_tx_update(info, 1); + + return ret; + +@@ -2083,9 +2083,7 @@ static int parse_first_sqe(void *hw_msg, struct qm_queue_info *info, __u16 idx, + WD_ERR("first BD error = %u\n", msg->result); + + info->cq_head_index = i; +- ret = qm_rx_update(info, 1); +- if (unlikely(ret)) +- return ret; ++ qm_rx_update(info, 1); + + return 1; + } +diff --git a/v1/drv/hisi_qm_udrv.c b/v1/drv/hisi_qm_udrv.c +index 175a5c4..ea4b199 100644 +--- a/v1/drv/hisi_qm_udrv.c ++++ b/v1/drv/hisi_qm_udrv.c +@@ -582,20 +582,10 @@ void qm_uninit_queue(struct wd_queue *q) + qinfo->priv = NULL; + } + +-int qm_tx_update(struct qm_queue_info *info, __u32 num) ++void qm_tx_update(struct qm_queue_info *info, __u32 num) + { +- if (unlikely(wd_reg_read(info->ds_tx_base) == 1)) { +- WD_ERR("wd queue hw error happened before qm send!\n"); +- return -WD_HW_EACCESS; +- } +- +- /* make sure the request is all in memory before doorbell */ +- mb(); +- + info->db(info, DOORBELL_CMD_SQ, info->sq_tail_index, 0); + __atomic_add_fetch(&info->used, num, __ATOMIC_RELAXED); +- +- return WD_SUCCESS; + } + + int qm_send(struct wd_queue *q, void **req, __u32 num) +@@ -605,6 +595,11 @@ int qm_send(struct wd_queue *q, void **req, __u32 num) + int ret; + __u32 i; + ++ if (unlikely(wd_reg_read(info->ds_tx_base) == 1)) { ++ WD_ERR("wd queue hw error happened before qm send!\n"); ++ return -WD_HW_EACCESS; ++ } ++ + wd_spinlock(&info->sd_lock); + if (unlikely((__u32)__atomic_load_n(&info->used, __ATOMIC_RELAXED) > + info->sq_depth - num - 1)) { +@@ -628,10 +623,19 @@ int qm_send(struct wd_queue *q, void **req, __u32 num) + info->sq_tail_index++; + } + +- ret = qm_tx_update(info, num); ++ /* make sure the request is all in memory before doorbell */ ++ mb(); ++ qm_tx_update(info, num); + wd_unspinlock(&info->sd_lock); + +- return ret; ++ return WD_SUCCESS; ++} ++ ++void qm_rx_update(struct qm_queue_info *info, __u32 num) ++{ ++ /* set c_flag to enable interrupt when use poll */ ++ info->db(info, DOORBELL_CMD_CQ, info->cq_head_index, info->is_poll); ++ __atomic_sub_fetch(&info->used, num, __ATOMIC_RELAXED); + } + + void qm_rx_from_cache(struct qm_queue_info *info, void **resp, __u32 num) +@@ -673,24 +677,6 @@ static int check_ds_rx_base(struct qm_queue_info *info, + return -WD_HW_EACCESS; + } + +-int qm_rx_update(struct qm_queue_info *info, __u32 num) +-{ +- int ret; +- +- ret = check_ds_rx_base(info, NULL, 0, 0); +- if (unlikely(ret)) +- return ret; +- +- /* make sure queue status check is complete. */ +- rmb(); +- +- /* set c_flag to enable interrupt when use poll */ +- info->db(info, DOORBELL_CMD_CQ, info->cq_head_index, info->is_poll); +- __atomic_sub_fetch(&info->used, num, __ATOMIC_RELAXED); +- +- return WD_SUCCESS; +-} +- + int qm_recv(struct wd_queue *q, void **resp, __u32 num) + { + struct q_info *qinfo = q->qinfo; +@@ -741,15 +727,15 @@ int qm_recv(struct wd_queue *q, void **resp, __u32 num) + } + } + +- if (i) { +- ret = qm_rx_update(info, i); +- if (!ret) +- ret = i; +- } ++ if (i) ++ qm_rx_update(info, i); + + wd_unspinlock(&info->rc_lock); ++ ret = check_ds_rx_base(info, resp, num, 0); ++ if (unlikely(ret)) ++ return ret; + +- return ret; ++ return i; + } + + static int hw_type_check(struct wd_queue *q, const char *hw_type) +diff --git a/v1/drv/hisi_qm_udrv.h b/v1/drv/hisi_qm_udrv.h +index 4d54cf6..542d20d 100644 +--- a/v1/drv/hisi_qm_udrv.h ++++ b/v1/drv/hisi_qm_udrv.h +@@ -191,8 +191,8 @@ int qm_init_hwsgl_mem(struct wd_queue *q, void *pool, struct wd_sgl *sgl); + int qm_uninit_hwsgl_mem(struct wd_queue *q, void *pool, struct wd_sgl *sgl); + int qm_merge_hwsgl(struct wd_queue *q, void *pool, + struct wd_sgl *dst_sgl, struct wd_sgl *src_sgl); +-int qm_tx_update(struct qm_queue_info *info, __u32 num); +-int qm_rx_update(struct qm_queue_info *info, __u32 num); ++void qm_tx_update(struct qm_queue_info *info, __u32 num); ++void qm_rx_update(struct qm_queue_info *info, __u32 num); + void qm_rx_from_cache(struct qm_queue_info *info, void **resp, __u32 num); + + #define HISI_QM_API_VER_BASE "hisi_qm_v1" +diff --git a/v1/wd_util.h b/v1/wd_util.h +index 21137d2..9180fc1 100644 +--- a/v1/wd_util.h ++++ b/v1/wd_util.h +@@ -383,11 +383,17 @@ struct wcrypto_ecc_out { + static inline void wd_reg_write(void *reg_addr, uint32_t value) + { + *((uint32_t *)reg_addr) = value; ++ wmb(); /* load fence */ + } + + static inline uint32_t wd_reg_read(void *reg_addr) + { +- return *((uint32_t *)reg_addr); ++ uint32_t temp; ++ ++ temp = *((uint32_t *)reg_addr); ++ rmb(); /* load fence */ ++ ++ return temp; + } + + void wd_spinlock(struct wd_lock *lock); +-- +2.25.1 + diff --git a/0034-uadk-v1-not-check-hardware-status-when-BD-not-proces.patch b/0034-uadk-v1-not-check-hardware-status-when-BD-not-proces.patch new file mode 100644 index 0000000..988df4b --- /dev/null +++ b/0034-uadk-v1-not-check-hardware-status-when-BD-not-proces.patch @@ -0,0 +1,39 @@ +From a2904158df65b9af7a85ff0af94cad81df164231 Mon Sep 17 00:00:00 2001 +From: Weili Qian +Date: Sat, 12 Oct 2024 11:11:27 +0800 +Subject: [PATCH 34/39] uadk/v1: not check hardware status when BD not + processed + +If the CQE status is not updated during packet receiving, +the hardware has not processed the packet. In this case, +0 can be returned without checking the hardware status. +If a fault occurs, the hardware status will be checked +when the user attempts to receive packets next time. + +Signed-off-by: Weili Qian +Signed-off-by: Qi Tao +--- + v1/drv/hisi_qm_udrv.c | 6 +++++- + 1 file changed, 5 insertions(+), 1 deletion(-) + +diff --git a/v1/drv/hisi_qm_udrv.c b/v1/drv/hisi_qm_udrv.c +index ea4b199..731492b 100644 +--- a/v1/drv/hisi_qm_udrv.c ++++ b/v1/drv/hisi_qm_udrv.c +@@ -727,8 +727,12 @@ int qm_recv(struct wd_queue *q, void **resp, __u32 num) + } + } + +- if (i) ++ if (i) { + qm_rx_update(info, i); ++ } else { ++ wd_unspinlock(&info->rc_lock); ++ return 0; ++ } + + wd_unspinlock(&info->rc_lock); + ret = check_ds_rx_base(info, resp, num, 0); +-- +2.25.1 + diff --git a/0035-uadk-v1-check-queue-status-before-sending-doorbells.patch b/0035-uadk-v1-check-queue-status-before-sending-doorbells.patch new file mode 100644 index 0000000..89edf16 --- /dev/null +++ b/0035-uadk-v1-check-queue-status-before-sending-doorbells.patch @@ -0,0 +1,215 @@ +From 62f8f743652212ed6e56c9d81b6cb9ae23b2be92 Mon Sep 17 00:00:00 2001 +From: Weili Qian +Date: Sat, 12 Oct 2024 11:12:43 +0800 +Subject: [PATCH 35/39] uadk/v1: check queue status before sending doorbells + +When the device needs to be reset, the queue status is +set to disable before resetting. The user process checks +the queue status before sending the doorbell. If the queue +is disable, the user process returns failure. + +Currently, the task execution order in user mode is as follows: +1. check the queue status. +2. fill in or parse the BD. +3. send the doorbell to the hardware. + +To reduce the possibility of sending doorbells during reset, +the task execution order is modified as follows: +1. fill in or parse the BD. +2. check the queue status. +3. send the doorbell to the hardware. + +In addition, a rmb() is added before the doorbell is +sent to ensure that the queue status check is complete. + +rmb() and wmb() can be replaced by mb() in hisi_qm_send(). +Therefore, the barrier on the wd_reg_read() and wd_reg_write() +can be deleted. + +Signed-off-by: Weili Qian +Signed-off-by: Qi Tao +--- + v1/drv/hisi_hpre_udrv.c | 10 ++++--- + v1/drv/hisi_qm_udrv.c | 60 ++++++++++++++++++++++++----------------- + v1/drv/hisi_qm_udrv.h | 4 +-- + v1/wd_util.h | 8 +----- + 4 files changed, 44 insertions(+), 38 deletions(-) + +diff --git a/v1/drv/hisi_hpre_udrv.c b/v1/drv/hisi_hpre_udrv.c +index 5832e9a..05518ab 100644 +--- a/v1/drv/hisi_hpre_udrv.c ++++ b/v1/drv/hisi_hpre_udrv.c +@@ -1930,10 +1930,10 @@ static int fill_sm2_enc_sqe(void *msg, struct qm_queue_info *info, __u16 idx) + goto fail_fill_sqe; + } + +- /* make sure the request is all in memory before doorbell */ +- mb(); + info->sq_tail_index = i; +- qm_tx_update(info, 1); ++ ret = qm_tx_update(info, 1); ++ if (unlikely(ret)) ++ goto fail_fill_sqe; + + return ret; + +@@ -2083,7 +2083,9 @@ static int parse_first_sqe(void *hw_msg, struct qm_queue_info *info, __u16 idx, + WD_ERR("first BD error = %u\n", msg->result); + + info->cq_head_index = i; +- qm_rx_update(info, 1); ++ ret = qm_rx_update(info, 1); ++ if (unlikely(ret)) ++ return ret; + + return 1; + } +diff --git a/v1/drv/hisi_qm_udrv.c b/v1/drv/hisi_qm_udrv.c +index 731492b..175a5c4 100644 +--- a/v1/drv/hisi_qm_udrv.c ++++ b/v1/drv/hisi_qm_udrv.c +@@ -582,10 +582,20 @@ void qm_uninit_queue(struct wd_queue *q) + qinfo->priv = NULL; + } + +-void qm_tx_update(struct qm_queue_info *info, __u32 num) ++int qm_tx_update(struct qm_queue_info *info, __u32 num) + { ++ if (unlikely(wd_reg_read(info->ds_tx_base) == 1)) { ++ WD_ERR("wd queue hw error happened before qm send!\n"); ++ return -WD_HW_EACCESS; ++ } ++ ++ /* make sure the request is all in memory before doorbell */ ++ mb(); ++ + info->db(info, DOORBELL_CMD_SQ, info->sq_tail_index, 0); + __atomic_add_fetch(&info->used, num, __ATOMIC_RELAXED); ++ ++ return WD_SUCCESS; + } + + int qm_send(struct wd_queue *q, void **req, __u32 num) +@@ -595,11 +605,6 @@ int qm_send(struct wd_queue *q, void **req, __u32 num) + int ret; + __u32 i; + +- if (unlikely(wd_reg_read(info->ds_tx_base) == 1)) { +- WD_ERR("wd queue hw error happened before qm send!\n"); +- return -WD_HW_EACCESS; +- } +- + wd_spinlock(&info->sd_lock); + if (unlikely((__u32)__atomic_load_n(&info->used, __ATOMIC_RELAXED) > + info->sq_depth - num - 1)) { +@@ -623,19 +628,10 @@ int qm_send(struct wd_queue *q, void **req, __u32 num) + info->sq_tail_index++; + } + +- /* make sure the request is all in memory before doorbell */ +- mb(); +- qm_tx_update(info, num); ++ ret = qm_tx_update(info, num); + wd_unspinlock(&info->sd_lock); + +- return WD_SUCCESS; +-} +- +-void qm_rx_update(struct qm_queue_info *info, __u32 num) +-{ +- /* set c_flag to enable interrupt when use poll */ +- info->db(info, DOORBELL_CMD_CQ, info->cq_head_index, info->is_poll); +- __atomic_sub_fetch(&info->used, num, __ATOMIC_RELAXED); ++ return ret; + } + + void qm_rx_from_cache(struct qm_queue_info *info, void **resp, __u32 num) +@@ -677,6 +673,24 @@ static int check_ds_rx_base(struct qm_queue_info *info, + return -WD_HW_EACCESS; + } + ++int qm_rx_update(struct qm_queue_info *info, __u32 num) ++{ ++ int ret; ++ ++ ret = check_ds_rx_base(info, NULL, 0, 0); ++ if (unlikely(ret)) ++ return ret; ++ ++ /* make sure queue status check is complete. */ ++ rmb(); ++ ++ /* set c_flag to enable interrupt when use poll */ ++ info->db(info, DOORBELL_CMD_CQ, info->cq_head_index, info->is_poll); ++ __atomic_sub_fetch(&info->used, num, __ATOMIC_RELAXED); ++ ++ return WD_SUCCESS; ++} ++ + int qm_recv(struct wd_queue *q, void **resp, __u32 num) + { + struct q_info *qinfo = q->qinfo; +@@ -728,18 +742,14 @@ int qm_recv(struct wd_queue *q, void **resp, __u32 num) + } + + if (i) { +- qm_rx_update(info, i); +- } else { +- wd_unspinlock(&info->rc_lock); +- return 0; ++ ret = qm_rx_update(info, i); ++ if (!ret) ++ ret = i; + } + + wd_unspinlock(&info->rc_lock); +- ret = check_ds_rx_base(info, resp, num, 0); +- if (unlikely(ret)) +- return ret; + +- return i; ++ return ret; + } + + static int hw_type_check(struct wd_queue *q, const char *hw_type) +diff --git a/v1/drv/hisi_qm_udrv.h b/v1/drv/hisi_qm_udrv.h +index 542d20d..4d54cf6 100644 +--- a/v1/drv/hisi_qm_udrv.h ++++ b/v1/drv/hisi_qm_udrv.h +@@ -191,8 +191,8 @@ int qm_init_hwsgl_mem(struct wd_queue *q, void *pool, struct wd_sgl *sgl); + int qm_uninit_hwsgl_mem(struct wd_queue *q, void *pool, struct wd_sgl *sgl); + int qm_merge_hwsgl(struct wd_queue *q, void *pool, + struct wd_sgl *dst_sgl, struct wd_sgl *src_sgl); +-void qm_tx_update(struct qm_queue_info *info, __u32 num); +-void qm_rx_update(struct qm_queue_info *info, __u32 num); ++int qm_tx_update(struct qm_queue_info *info, __u32 num); ++int qm_rx_update(struct qm_queue_info *info, __u32 num); + void qm_rx_from_cache(struct qm_queue_info *info, void **resp, __u32 num); + + #define HISI_QM_API_VER_BASE "hisi_qm_v1" +diff --git a/v1/wd_util.h b/v1/wd_util.h +index 9180fc1..21137d2 100644 +--- a/v1/wd_util.h ++++ b/v1/wd_util.h +@@ -383,17 +383,11 @@ struct wcrypto_ecc_out { + static inline void wd_reg_write(void *reg_addr, uint32_t value) + { + *((uint32_t *)reg_addr) = value; +- wmb(); /* load fence */ + } + + static inline uint32_t wd_reg_read(void *reg_addr) + { +- uint32_t temp; +- +- temp = *((uint32_t *)reg_addr); +- rmb(); /* load fence */ +- +- return temp; ++ return *((uint32_t *)reg_addr); + } + + void wd_spinlock(struct wd_lock *lock); +-- +2.25.1 + diff --git a/0036-uadk-v1-replace-wd_spinlock-to-pthread_spin_lock.patch b/0036-uadk-v1-replace-wd_spinlock-to-pthread_spin_lock.patch new file mode 100644 index 0000000..7234884 --- /dev/null +++ b/0036-uadk-v1-replace-wd_spinlock-to-pthread_spin_lock.patch @@ -0,0 +1,289 @@ +From 2ee2190cf494361a79fae68bd35a4c280026414e Mon Sep 17 00:00:00 2001 +From: Wenkai Lin +Date: Sat, 12 Oct 2024 11:15:08 +0800 +Subject: [PATCH 36/39] uadk/v1: replace wd_spinlock to pthread_spin_lock + +Due to memory differences, using wd_spinlock may +cause synchronization problems, it is better to use +the standard pthread spin lock of glibc. + +Signed-off-by: Wenkai Lin +Signed-off-by: Qi Tao +--- + v1/drv/hisi_qm_udrv.c | 51 +++++++++++++++++++++++++++++------------- + v1/drv/hisi_qm_udrv.h | 4 ++-- + v1/drv/hisi_rng_udrv.c | 25 +++++++++++++++------ + v1/drv/hisi_rng_udrv.h | 2 +- + 4 files changed, 56 insertions(+), 26 deletions(-) + +diff --git a/v1/drv/hisi_qm_udrv.c b/v1/drv/hisi_qm_udrv.c +index 175a5c4..1d4f1d8 100644 +--- a/v1/drv/hisi_qm_udrv.c ++++ b/v1/drv/hisi_qm_udrv.c +@@ -20,6 +20,7 @@ + #include + #include + #include ++#include + #include + #include + #include +@@ -458,6 +459,11 @@ static int qm_init_queue_info(struct wd_queue *q) + struct hisi_qp_ctx qp_ctx = {0}; + int ret; + ++ if (!info->sqe_size) { ++ WD_ERR("invalid: sqe size is 0!\n"); ++ return -WD_EINVAL; ++ } ++ + info->sq_tail_index = 0; + info->cq_head_index = 0; + info->cqc_phase = 1; +@@ -502,11 +508,6 @@ static int qm_set_queue_info(struct wd_queue *q) + ret = qm_set_queue_regions(q); + if (ret) + return -WD_EINVAL; +- if (!info->sqe_size) { +- WD_ERR("sqe size =%d err!\n", info->sqe_size); +- ret = -WD_EINVAL; +- goto err_with_regions; +- } + info->cq_base = (void *)((uintptr_t)info->sq_base + + info->sqe_size * info->sq_depth); + +@@ -534,8 +535,24 @@ static int qm_set_queue_info(struct wd_queue *q) + goto err_with_regions; + } + ++ ret = pthread_spin_init(&info->sd_lock, PTHREAD_PROCESS_PRIVATE); ++ if (ret) { ++ WD_ERR("failed to init qinfo sd_lock!\n"); ++ goto free_cache; ++ } ++ ++ ret = pthread_spin_init(&info->rc_lock, PTHREAD_PROCESS_PRIVATE); ++ if (ret) { ++ WD_ERR("failed to init qinfo rc_lock!\n"); ++ goto uninit_lock; ++ } ++ + return 0; + ++uninit_lock: ++ pthread_spin_destroy(&info->sd_lock); ++free_cache: ++ free(info->req_cache); + err_with_regions: + qm_unset_queue_regions(q); + return ret; +@@ -576,8 +593,10 @@ void qm_uninit_queue(struct wd_queue *q) + struct q_info *qinfo = q->qinfo; + struct qm_queue_info *info = qinfo->priv; + +- qm_unset_queue_regions(q); ++ pthread_spin_destroy(&info->rc_lock); ++ pthread_spin_destroy(&info->sd_lock); + free(info->req_cache); ++ qm_unset_queue_regions(q); + free(qinfo->priv); + qinfo->priv = NULL; + } +@@ -605,10 +624,10 @@ int qm_send(struct wd_queue *q, void **req, __u32 num) + int ret; + __u32 i; + +- wd_spinlock(&info->sd_lock); ++ pthread_spin_lock(&info->sd_lock); + if (unlikely((__u32)__atomic_load_n(&info->used, __ATOMIC_RELAXED) > + info->sq_depth - num - 1)) { +- wd_unspinlock(&info->sd_lock); ++ pthread_spin_unlock(&info->sd_lock); + WD_ERR("queue is full!\n"); + return -WD_EBUSY; + } +@@ -617,7 +636,7 @@ int qm_send(struct wd_queue *q, void **req, __u32 num) + ret = info->sqe_fill[qinfo->atype](req[i], qinfo->priv, + info->sq_tail_index); + if (unlikely(ret != WD_SUCCESS)) { +- wd_unspinlock(&info->sd_lock); ++ pthread_spin_unlock(&info->sd_lock); + WD_ERR("sqe fill error, ret %d!\n", ret); + return -WD_EINVAL; + } +@@ -629,7 +648,7 @@ int qm_send(struct wd_queue *q, void **req, __u32 num) + } + + ret = qm_tx_update(info, num); +- wd_unspinlock(&info->sd_lock); ++ pthread_spin_unlock(&info->sd_lock); + + return ret; + } +@@ -662,9 +681,9 @@ static int check_ds_rx_base(struct qm_queue_info *info, + return 0; + + if (before) { +- wd_spinlock(&info->rc_lock); ++ pthread_spin_lock(&info->rc_lock); + qm_rx_from_cache(info, resp, num); +- wd_unspinlock(&info->rc_lock); ++ pthread_spin_unlock(&info->rc_lock); + WD_ERR("wd queue hw error happened before qm receive!\n"); + } else { + WD_ERR("wd queue hw error happened after qm receive!\n"); +@@ -705,7 +724,7 @@ int qm_recv(struct wd_queue *q, void **resp, __u32 num) + if (unlikely(ret)) + return ret; + +- wd_spinlock(&info->rc_lock); ++ pthread_spin_lock(&info->rc_lock); + for (i = 0; i < num; i++) { + cqe = info->cq_base + info->cq_head_index * sizeof(struct cqe); + if (info->cqc_phase != CQE_PHASE(cqe)) +@@ -714,7 +733,7 @@ int qm_recv(struct wd_queue *q, void **resp, __u32 num) + mb(); /* make sure the data is all in memory before read */ + sq_head = CQE_SQ_HEAD_INDEX(cqe); + if (unlikely(sq_head >= info->sq_depth)) { +- wd_unspinlock(&info->rc_lock); ++ pthread_spin_unlock(&info->rc_lock); + WD_ERR("CQE_SQ_HEAD_INDEX(%u) error\n", sq_head); + return -WD_EIO; + } +@@ -726,7 +745,7 @@ int qm_recv(struct wd_queue *q, void **resp, __u32 num) + if (!ret) { + break; + } else if (ret < 0) { +- wd_unspinlock(&info->rc_lock); ++ pthread_spin_unlock(&info->rc_lock); + WD_ERR("recv sqe error %u\n", sq_head); + return ret; + } +@@ -747,7 +766,7 @@ int qm_recv(struct wd_queue *q, void **resp, __u32 num) + ret = i; + } + +- wd_unspinlock(&info->rc_lock); ++ pthread_spin_unlock(&info->rc_lock); + + return ret; + } +diff --git a/v1/drv/hisi_qm_udrv.h b/v1/drv/hisi_qm_udrv.h +index 4d54cf6..06ac66a 100644 +--- a/v1/drv/hisi_qm_udrv.h ++++ b/v1/drv/hisi_qm_udrv.h +@@ -166,8 +166,8 @@ struct qm_queue_info { + qm_sqe_parse sqe_parse[WCRYPTO_MAX_ALG]; + hisi_qm_sqe_fill_priv sqe_fill_priv; + hisi_qm_sqe_parse_priv sqe_parse_priv; +- struct wd_lock sd_lock; +- struct wd_lock rc_lock; ++ pthread_spinlock_t sd_lock; ++ pthread_spinlock_t rc_lock; + struct wd_queue *q; + int (*sgl_info)(struct hw_sgl_info *info); + int (*sgl_init)(void *pool, struct wd_sgl *sgl); +diff --git a/v1/drv/hisi_rng_udrv.c b/v1/drv/hisi_rng_udrv.c +index 86a20cb..605ef27 100644 +--- a/v1/drv/hisi_rng_udrv.c ++++ b/v1/drv/hisi_rng_udrv.c +@@ -17,6 +17,7 @@ + #include + #include + #include ++#include + #include + #include + #include +@@ -34,6 +35,7 @@ int rng_init_queue(struct wd_queue *q) + { + struct q_info *qinfo = q->qinfo; + struct rng_queue_info *info; ++ int ret; + + info = calloc(1, sizeof(*info)); + if (!info) { +@@ -41,12 +43,20 @@ int rng_init_queue(struct wd_queue *q) + return -ENOMEM; + } + ++ ret = pthread_spin_init(&info->lock, PTHREAD_PROCESS_PRIVATE); ++ if (ret) { ++ free(info); ++ WD_ERR("failed to init rng qinfo lock!\n"); ++ return ret; ++ } ++ + qinfo->priv = info; + info->mmio_base = wd_drv_mmap_qfr(q, WD_UACCE_QFRT_MMIO, 0); + if (info->mmio_base == MAP_FAILED) { + info->mmio_base = NULL; +- free(qinfo->priv); + qinfo->priv = NULL; ++ pthread_spin_destroy(&info->lock); ++ free(info); + WD_ERR("mmap trng mmio fail\n"); + return -ENOMEM; + } +@@ -63,6 +73,7 @@ void rng_uninit_queue(struct wd_queue *q) + + free(qinfo->priv); + qinfo->priv = NULL; ++ pthread_spin_destroy(&info->lock); + } + + int rng_send(struct wd_queue *q, void **req, __u32 num) +@@ -70,14 +81,14 @@ int rng_send(struct wd_queue *q, void **req, __u32 num) + struct q_info *qinfo = q->qinfo; + struct rng_queue_info *info = qinfo->priv; + +- wd_spinlock(&info->lock); ++ pthread_spin_lock(&info->lock); + if (!info->req_cache[info->send_idx]) { + info->req_cache[info->send_idx] = req[0]; + info->send_idx++; +- wd_unspinlock(&info->lock); ++ pthread_spin_unlock(&info->lock); + return 0; + } +- wd_unspinlock(&info->lock); ++ pthread_spin_unlock(&info->lock); + + WD_ERR("queue is full!\n"); + return -WD_EBUSY; +@@ -128,16 +139,16 @@ int rng_recv(struct wd_queue *q, void **resp, __u32 num) + struct wcrypto_cb_tag *tag; + __u32 currsize = 0; + +- wd_spinlock(&info->lock); ++ pthread_spin_lock(&info->lock); + msg = info->req_cache[info->recv_idx]; + if (!msg) { +- wd_unspinlock(&info->lock); ++ pthread_spin_unlock(&info->lock); + return 0; + } + + info->req_cache[info->recv_idx] = NULL; + info->recv_idx++; +- wd_unspinlock(&info->lock); ++ pthread_spin_unlock(&info->lock); + + tag = (void *)(uintptr_t)msg->usr_tag; + if (usr && tag->ctx_id != usr) +diff --git a/v1/drv/hisi_rng_udrv.h b/v1/drv/hisi_rng_udrv.h +index 56814a4..3efa10e 100644 +--- a/v1/drv/hisi_rng_udrv.h ++++ b/v1/drv/hisi_rng_udrv.h +@@ -29,7 +29,7 @@ struct rng_queue_info { + void *req_cache[TRNG_Q_DEPTH]; + __u8 send_idx; + __u8 recv_idx; +- struct wd_lock lock; ++ pthread_spinlock_t lock; + }; + + int rng_init_queue(struct wd_queue *q); +-- +2.25.1 + diff --git a/0037-uadk-fix-for-hashagg-include-files.patch b/0037-uadk-fix-for-hashagg-include-files.patch new file mode 100644 index 0000000..c3e2544 --- /dev/null +++ b/0037-uadk-fix-for-hashagg-include-files.patch @@ -0,0 +1,55 @@ +From 50c8227aa41910090035c64b3a76035cab9089fa Mon Sep 17 00:00:00 2001 +From: Qi Tao +Date: Fri, 1 Nov 2024 16:17:59 +0800 +Subject: [PATCH 37/39] uadk: fix for hashagg include files + +A compilation error occurred because the wd_util.h +is not an externally visible header file, so use +wd.h and wd_alg_common.h here. + +Signed-off-by: Qi Tao +--- + include/drv/wd_agg_drv.h | 1 + + include/wd_agg.h | 1 - + include/wd_dae.h | 1 + + 3 files changed, 2 insertions(+), 1 deletion(-) + +diff --git a/include/drv/wd_agg_drv.h b/include/drv/wd_agg_drv.h +index cf99a70..3d26eeb 100644 +--- a/include/drv/wd_agg_drv.h ++++ b/include/drv/wd_agg_drv.h +@@ -8,6 +8,7 @@ + + #include + #include "wd_agg.h" ++#include "wd_util.h" + + #ifdef __cplusplus + extern "C" { +diff --git a/include/wd_agg.h b/include/wd_agg.h +index e37023b..aac3d40 100644 +--- a/include/wd_agg.h ++++ b/include/wd_agg.h +@@ -8,7 +8,6 @@ + + #include + #include +-#include "wd_util.h" + #include "wd_dae.h" + + #ifdef __cplusplus +diff --git a/include/wd_dae.h b/include/wd_dae.h +index 5f22d90..aa9f966 100644 +--- a/include/wd_dae.h ++++ b/include/wd_dae.h +@@ -9,6 +9,7 @@ + #include + #include + #include ++#include "wd_alg_common.h" + #include "wd.h" + + #ifdef __cplusplus +-- +2.25.1 + diff --git a/0038-uadk-fix-code-compatibility-issue.patch b/0038-uadk-fix-code-compatibility-issue.patch new file mode 100644 index 0000000..33e89e5 --- /dev/null +++ b/0038-uadk-fix-code-compatibility-issue.patch @@ -0,0 +1,57 @@ +From 710eef70b45b48e040022181fd28cc65883a8fb1 Mon Sep 17 00:00:00 2001 +From: Qi Tao +Date: Fri, 1 Nov 2024 16:19:31 +0800 +Subject: [PATCH 38/39] uadk: fix code compatibility issue + +The code check "req->in_bytes != req->out_bytes" is incompatible +with the software of other versions. Therefore, the check is deleted +and the size check of out_bytes and iv_bytes is added to the driver. + +Signed-off-by: Qi Tao +--- + drv/hisi_sec.c | 4 ++-- + wd_cipher.c | 6 ------ + 2 files changed, 2 insertions(+), 8 deletions(-) + +diff --git a/drv/hisi_sec.c b/drv/hisi_sec.c +index bdde39f..10db124 100644 +--- a/drv/hisi_sec.c ++++ b/drv/hisi_sec.c +@@ -751,7 +751,7 @@ static void update_iv(struct wd_cipher_msg *msg) + msg->iv_bytes, msg->iv_bytes); + break; + case WD_CIPHER_OFB: +- if (msg->in_bytes < msg->iv_bytes) ++ if (msg->in_bytes < msg->iv_bytes || msg->out_bytes < msg->iv_bytes) + break; + /* The iv_bytes has been checked and it is not greater than AES_BLOCK_SIZE. */ + for (i = 0; i < msg->iv_bytes; i++) +@@ -793,7 +793,7 @@ static void update_iv_sgl(struct wd_cipher_msg *msg) + break; + case WD_CIPHER_OFB: + /* The iv_bytes has been checked and it is not greater than AES_BLOCK_SIZE. */ +- if (msg->in_bytes >= msg->iv_bytes) { ++ if (msg->in_bytes >= msg->iv_bytes && msg->out_bytes >= msg->iv_bytes) { + hisi_qm_sgl_copy(in, msg->in, + msg->in_bytes - msg->iv_bytes, + msg->iv_bytes, COPY_SGL_TO_PBUFF); +diff --git a/wd_cipher.c b/wd_cipher.c +index 0e5de25..ec6fb15 100644 +--- a/wd_cipher.c ++++ b/wd_cipher.c +@@ -646,12 +646,6 @@ static int wd_cipher_check_params(handle_t h_sess, + return -WD_EINVAL; + } + +- if (unlikely(req->in_bytes != req->out_bytes)) { +- WD_ERR("cipher set out_bytes is error, size = %u\n", +- req->out_bytes); +- return -WD_EINVAL; +- } +- + ret = cipher_in_len_check(h_sess, req); + if (unlikely(ret)) + return ret; +-- +2.25.1 + diff --git a/0039-Update-wd_design.md.patch b/0039-Update-wd_design.md.patch new file mode 100644 index 0000000..6bfa5ce --- /dev/null +++ b/0039-Update-wd_design.md.patch @@ -0,0 +1,25 @@ +From 90fb6f227427f568e34337309075ed7a3f71bab9 Mon Sep 17 00:00:00 2001 +From: Liulongfang <67134878+Liulongfang@users.noreply.github.com> +Date: Wed, 13 Nov 2024 11:51:59 +0800 +Subject: [PATCH 39/39] Update wd_design.md + +The current UADK framework has been changed from coupling HiSilicon hardware accelerator to being able to adapt to hardware acceleration and software acceleration. Thus realizing the driver dynamic loading function +--- + docs/wd_design.md | 1 + + 1 file changed, 1 insertion(+) + +diff --git a/docs/wd_design.md b/docs/wd_design.md +index 3e5297e..ab39fb1 100644 +--- a/docs/wd_design.md ++++ b/docs/wd_design.md +@@ -82,6 +82,7 @@ + | | | sample_sched is moved from user space into UADK | + | | | framework. | + | 1.4 | |1) Update *wd_alg_init* reentrancy. | ++| 1.5 | |1) Added driver dynamic loading function. | + + + ## Terminology +-- +2.25.1 + diff --git a/libwd.spec b/libwd.spec index a8b573e..4e435dd 100644 --- a/libwd.spec +++ b/libwd.spec @@ -2,7 +2,7 @@ Name: libwd Summary: User Space Accelerator Development Kit Version: 2.7.0 -Release: 1 +Release: 2 License: Apache-2.0 Source: %{name}-%{version}.tar.gz @@ -33,6 +33,29 @@ Patch0013: 0013-uadk-v1-update-the-symbol-table-for-libraries.patch Patch0014: 0014-uadk-modify-address-check.patch Patch0015: 0015-uadk-check-calloc-return-value.patch Patch0016: 0016-drv-hisi-sec-modify-minor-errors-in-hisi_sec.c.patch +Patch0017: 0017-uadk-support-for-the-DAE-algorithm-layer.patch +Patch0018: 0018-drv-dae-support-hashagg-algorithm.patch +Patch0019: 0019-uadk_tool-use-wd_xxx_init2_-instead-of-wd_xxx_init2.patch +Patch0020: 0020-uadk-isa-ce-fix-sm4-CE-driver-initialization-error.patch +Patch0021: 0021-acc-uadk-fix-queue-configuration-parameter-error.patch +Patch0022: 0022-acc-uadk-fix-double-free-error.patch +Patch0023: 0023-uadk-fix-the-null-pointer-check-error.patch +Patch0024: 0024-uadk-add-null-pointer-check.patch +Patch0025: 0025-uadk-test-move-main-function-to-testsuit.c-and-rm.patch +Patch0026: 0026-uadk-fix-for-sec-software-compute-check.patch +Patch0027: 0027-uadk-fix-some-code-bugs.patch +Patch0028: 0028-uadk-cipher-add-length-check-of-CBC_CTS-mode.patch +Patch0029: 0029-uadk-fix-for-cipher-update-iv.patch +Patch0030: 0030-uadk-v1-fix-for-cipher-update-iv.patch +Patch0031: 0031-uadk-reduce-the-print-rating-of-specific-logs.patch +Patch0032: 0032-Revert-uadk-v1-replace-wd_spinlock-to-pthread_spin_l.patch +Patch0033: 0033-Revert-uadk-v1-check-queue-status-before-sending-doo.patch +Patch0034: 0034-uadk-v1-not-check-hardware-status-when-BD-not-proces.patch +Patch0035: 0035-uadk-v1-check-queue-status-before-sending-doorbells.patch +Patch0036: 0036-uadk-v1-replace-wd_spinlock-to-pthread_spin_lock.patch +Patch0037: 0037-uadk-fix-for-hashagg-include-files.patch +Patch0038: 0038-uadk-fix-code-compatibility-issue.patch +Patch0039: 0039-Update-wd_design.md.patch %description This package contains the User Space Accelerator Library @@ -111,6 +134,9 @@ rm -rf ${RPM_BUILD_ROOT} /sbin/ldconfig %changelog +* Tue Nov 19 2024 JiangShui Yang 2.7.0-2 +- libwd: update the source code + * Fri Aug 23 2024 JiangShui Yang 2.7.0-1 - libwd: update the source code to 2.7.0