!735 combine ten similar submissions into one(0125-fix-clang-build-error.patch) and Upgrade from upstream

From: @wu-jichao123 
Reviewed-by: @taotao-sauce 
Signed-off-by: @taotao-sauce

0028-use-supervisor-to-notify-sandbox-exit-event.patch

@@ -0,0 +1,254 @@
+From 835185f7c4739993c2ca26d737bb0a45277ad932 Mon Sep 17 00:00:00 2001
+From: jikai <jikai11@huawei.com>
+Date: Wed, 20 Mar 2024 15:48:42 +0800
+Subject: [PATCH 028/149] use supervisor to notify sandbox exit event
+
+Signed-off-by: jikai <jikai11@huawei.com>
+---
+ src/daemon/modules/api/container_api.h        |  2 +-
+ .../modules/container/restore/restore.c       |  6 +++-
+ .../modules/container/supervisor/supervisor.c | 15 +++++++-
+ .../modules/service/service_container.c       |  9 +++--
+ .../controller/shim/shim_controller.cc        | 35 ++-----------------
+ src/daemon/sandbox/sandbox_ops.cc             | 23 ++++++++++++
+ src/daemon/sandbox/sandbox_ops.h              |  2 ++
+ 7 files changed, 53 insertions(+), 39 deletions(-)
+
+diff --git a/src/daemon/modules/api/container_api.h b/src/daemon/modules/api/container_api.h
+index 4602d244..43d66d64 100644
+--- a/src/daemon/modules/api/container_api.h
++++ b/src/daemon/modules/api/container_api.h
+@@ -270,7 +270,7 @@ bool container_is_valid_state_string(const char *state);
+ void container_update_health_monitor(const char *container_id);
+ 
+ extern int container_supervisor_add_exit_monitor(int fd, const pid_ppid_info_t *pid_info, const char *name,
+-                                                 const char *runtime);
++                                                 const char *runtime, bool sandbox_container);
+ 
+ extern char *container_exit_fifo_create(const char *cont_state_path);
+ 
+diff --git a/src/daemon/modules/container/restore/restore.c b/src/daemon/modules/container/restore/restore.c
+index 2669ea22..76868e28 100644
+--- a/src/daemon/modules/container/restore/restore.c
++++ b/src/daemon/modules/container/restore/restore.c
+@@ -57,6 +57,7 @@ static int restore_supervisor(const container_t *cont)
+     char *statepath = cont->state_path;
+     char *runtime = cont->runtime;
+     pid_ppid_info_t pid_info = { 0 };
++    bool sandbox_container = false;
+ 
+     nret = snprintf(container_state, sizeof(container_state), "%s/%s", statepath, id);
+     if (nret < 0 || (size_t)nret >= sizeof(container_state)) {
+@@ -90,8 +91,11 @@ static int restore_supervisor(const container_t *cont)
+     pid_info.ppid = cont->state->state->p_pid;
+     pid_info.start_time = cont->state->state->start_time;
+     pid_info.pstart_time = cont->state->state->p_start_time;
++#ifdef ENABLE_CRI_API_V1
++    sandbox_container = is_sandbox_container(cont->common_config->sandbox_info);
++#endif
+ 
+-    if (container_supervisor_add_exit_monitor(exit_fifo_fd, &pid_info, id, runtime)) {
++    if (container_supervisor_add_exit_monitor(exit_fifo_fd, &pid_info, id, runtime, sandbox_container)) {
+         ERROR("Failed to add exit monitor to supervisor");
+         ret = -1;
+         goto out;
+diff --git a/src/daemon/modules/container/supervisor/supervisor.c b/src/daemon/modules/container/supervisor/supervisor.c
+index 1f9a043c..63289283 100644
+--- a/src/daemon/modules/container/supervisor/supervisor.c
++++ b/src/daemon/modules/container/supervisor/supervisor.c
+@@ -38,6 +38,9 @@
+ #include "container_api.h"
+ #include "event_type.h"
+ #include "utils_file.h"
++#ifdef ENABLE_CRI_API_V1
++#include "sandbox_ops.h"
++#endif
+ 
+ pthread_mutex_t g_supervisor_lock = PTHREAD_MUTEX_INITIALIZER;
+ struct epoll_descr g_supervisor_descr;
+@@ -47,6 +50,7 @@ struct supervisor_handler_data {
+     int exit_code;
+     char *name;
+     char *runtime;
++    bool is_sandbox_container;
+     pid_ppid_info_t pid_info;
+ };
+ 
+@@ -211,6 +215,14 @@ retry:
+ 
+     (void)isulad_monitor_send_container_event(name, STOPPED, (int)pid, data->exit_code, NULL, NULL);
+ 
++#ifdef ENABLE_CRI_API_V1
++    if (data->is_sandbox_container) {
++        if (sandbox_on_sandbox_exit(name, data->exit_code) < 0) {
++            ERROR("Failed to handle sandbox %s exit", name);
++        }
++    }
++#endif
++
+     supervisor_handler_data_free(data);
+ 
+     DAEMON_CLEAR_ERRMSG();
+@@ -259,7 +271,7 @@ static int supervisor_exit_cb(int fd, uint32_t events, void *cbdata, struct epol
+ 
+ /* supervisor add exit monitor */
+ int container_supervisor_add_exit_monitor(int fd, const pid_ppid_info_t *pid_info, const char *name,
+-                                          const char *runtime)
++                                          const char *runtime, bool sandbox_container)
+ {
+     int ret = 0;
+     struct supervisor_handler_data *data = NULL;
+@@ -285,6 +297,7 @@ int container_supervisor_add_exit_monitor(int fd, const pid_ppid_info_t *pid_inf
+     data->fd = fd;
+     data->name = util_strdup_s(name);
+     data->runtime = util_strdup_s(runtime);
++    data->is_sandbox_container = sandbox_container;
+     data->pid_info.pid = pid_info->pid;
+     data->pid_info.start_time = pid_info->start_time;
+     data->pid_info.ppid = pid_info->ppid;
+diff --git a/src/daemon/modules/service/service_container.c b/src/daemon/modules/service/service_container.c
+index a3606a82..7b34cc7f 100644
+--- a/src/daemon/modules/service/service_container.c
++++ b/src/daemon/modules/service/service_container.c
+@@ -275,13 +275,14 @@ static void clean_resources_on_failure(const container_t *cont, const char *engi
+     return;
+ }
+ 
+-static int do_post_start_on_success(const char *id, const char *runtime, const char *pidfile, int exit_fifo_fd,
++static int do_post_start_on_success(const char *id, const char *runtime, bool sandbox_container,
++                                    const char *pidfile, int exit_fifo_fd,
+                                     const pid_ppid_info_t *pid_info)
+ {
+     int ret = 0;
+ 
+     // exit_fifo_fd was closed in container_supervisor_add_exit_monitor
+-    if (container_supervisor_add_exit_monitor(exit_fifo_fd, pid_info, id, runtime)) {
++    if (container_supervisor_add_exit_monitor(exit_fifo_fd, pid_info, id, runtime, sandbox_container)) {
+         ERROR("Failed to add exit monitor to supervisor");
+         ret = -1;
+     }
+@@ -749,6 +750,7 @@ static int do_start_container(container_t *cont, const char *console_fifos[], bo
+     oci_runtime_spec *oci_spec = NULL;
+     rt_create_params_t create_params = { 0 };
+     rt_start_params_t start_params = { 0 };
++    bool sandbox_container;
+ 
+     nret = snprintf(bundle, sizeof(bundle), "%s/%s", cont->root_path, id);
+     if (nret < 0 || (size_t)nret >= sizeof(bundle)) {
+@@ -897,6 +899,7 @@ static int do_start_container(container_t *cont, const char *console_fifos[], bo
+     if (cont->common_config->sandbox_info != NULL) {
+         create_params.task_addr = cont->common_config->sandbox_info->task_address;
+     }
++    sandbox_container = is_sandbox_container(cont->common_config->sandbox_info);
+ #endif
+ 
+     if (runtime_create(id, runtime, &create_params) != 0) {
+@@ -921,7 +924,7 @@ static int do_start_container(container_t *cont, const char *console_fifos[], bo
+ 
+     ret = runtime_start(id, runtime, &start_params, pid_info);
+     if (ret == 0) {
+-        if (do_post_start_on_success(id, runtime, pidfile, exit_fifo_fd, pid_info) != 0) {
++        if (do_post_start_on_success(id, runtime, sandbox_container, pidfile, exit_fifo_fd, pid_info) != 0) {
+             ERROR("Failed to do post start on runtime start success");
+             ret = -1;
+             goto clean_resources;
+diff --git a/src/daemon/sandbox/controller/shim/shim_controller.cc b/src/daemon/sandbox/controller/shim/shim_controller.cc
+index 39fcf8ea..593fade9 100644
+--- a/src/daemon/sandbox/controller/shim/shim_controller.cc
++++ b/src/daemon/sandbox/controller/shim/shim_controller.cc
+@@ -397,39 +397,8 @@ bool ShimController::Stop(const std::string &sandboxId, uint32_t timeoutSecs, Er
+ 
+ bool ShimController::Wait(std::shared_ptr<SandboxStatusCallback> cb, const std::string &sandboxId, Errors &error)
+ {
+-    std::thread([this, cb, sandboxId]() {
+-        if (m_cb == nullptr || m_cb->container.wait == nullptr) {
+-            ERROR("Unimplemented callback");
+-            return;
+-        }
+-
+-        auto requestWrapper = makeUniquePtrCStructWrapper<container_wait_request>(free_container_wait_request);
+-        if (requestWrapper == nullptr) {
+-            ERROR("Out of memory");
+-            return;
+-        }
+-        auto request = requestWrapper->get();
+-        request->id = isula_strdup_s(sandboxId.c_str());
+-        request->condition = WAIT_CONDITION_STOPPED;
+-        container_wait_response *response { nullptr };
+-
+-        int ret = m_cb->container.wait(request, &response);
+-        auto responseWrapper = makeUniquePtrCStructWrapper<container_wait_response>(response, free_container_wait_response);
+-
+-        if (ret != 0) {
+-            std::string msg = (response != nullptr && response->errmsg != nullptr) ? response->errmsg : "internal";
+-            ERROR("Failed to wait sandbox %s: %s", sandboxId.c_str(), msg.c_str());
+-            return;
+-        }
+-
+-        ControllerExitInfo info;
+-        auto currentTime = std::chrono::high_resolution_clock::now();
+-        auto duration = currentTime.time_since_epoch();
+-        info.exitedAt = std::chrono::duration_cast<std::chrono::nanoseconds>(duration).count();
+-        info.exitStatus = response->exit_code;
+-        cb->OnSandboxExit(info);
+-    }).detach();
+-
++    // ShimController will use sandbox_on_exit callback of supervisor in lower container level
++    // to notify the sandbox exit event
+     return true;
+ }
+ 
+diff --git a/src/daemon/sandbox/sandbox_ops.cc b/src/daemon/sandbox/sandbox_ops.cc
+index 005063c0..b7fb40bf 100644
+--- a/src/daemon/sandbox/sandbox_ops.cc
++++ b/src/daemon/sandbox/sandbox_ops.cc
+@@ -18,6 +18,7 @@
+ #include <isula_libutils/log.h>
+ 
+ #include "controller_manager.h"
++#include "sandbox_manager.h"
+ #include "namespace.h"
+ #include "utils.h"
+ 
+@@ -175,3 +176,25 @@ int sandbox_purge_exec(const container_config_v2_common_config *config, const ch
+ {
+     return do_sandbox_purge(config, exec_id);
+ }
++
++int sandbox_on_sandbox_exit(const char *sandbox_id, int exit_code)
++{
++    if (nullptr == sandbox_id) {
++        ERROR("Invalid parameter: sandbox_id");
++        return -1;
++    }
++
++    auto sandbox = sandbox::SandboxManager::GetInstance()->GetSandbox(sandbox_id);
++    if (nullptr == sandbox) {
++        ERROR("Sandbox %s not found", sandbox_id);
++        return -1;
++    }
++
++    sandbox::ControllerExitInfo info;
++    auto currentTime = std::chrono::high_resolution_clock::now();
++    auto duration = currentTime.time_since_epoch();
++    info.exitedAt = std::chrono::duration_cast<std::chrono::nanoseconds>(duration).count();
++    info.exitStatus = exit_code;
++    sandbox->OnSandboxExit(info);
++    return 0;
++}
+diff --git a/src/daemon/sandbox/sandbox_ops.h b/src/daemon/sandbox/sandbox_ops.h
+index bef884fb..8189efd6 100644
+--- a/src/daemon/sandbox/sandbox_ops.h
++++ b/src/daemon/sandbox/sandbox_ops.h
+@@ -36,6 +36,8 @@ int sandbox_purge_container(const container_config_v2_common_config *config);
+ 
+ int sandbox_purge_exec(const container_config_v2_common_config *config, const char *exec_id);
+ 
++int sandbox_on_sandbox_exit(const char *sandbox_id, int exit_code);
++
+ #ifdef __cplusplus
+ }
+ #endif
+-- 
+2.25.1
+

0030-adaptor-unit-test-for-cgroup-module.patch

@@ -0,0 +1,351 @@
+From 59e7ea0f16e83e0bdbc39bdc41d1ade8d3db885e Mon Sep 17 00:00:00 2001
+From: zhongtao <zhongtao17@huawei.com>
+Date: Thu, 22 Feb 2024 09:52:30 +0800
+Subject: [PATCH 030/149] adaptor unit test for cgroup module
+
+Signed-off-by: zhongtao <zhongtao17@huawei.com>
+---
+ src/daemon/modules/image/CMakeLists.txt              |  6 ++++--
+ test/cgroup/cpu/CMakeLists.txt                       |  7 +++++--
+ test/cgroup/cpu/cgroup_cpu_ut.cc                     | 12 ++++++++++--
+ test/image/oci/oci_config_merge/CMakeLists.txt       |  7 +++++--
+ test/image/oci/registry/CMakeLists.txt               |  7 +++++--
+ test/network/network_ns/CMakeLists.txt               |  7 +++++--
+ test/runtime/isula/CMakeLists.txt                    |  7 +++++--
+ test/runtime/lcr/CMakeLists.txt                      |  7 +++++--
+ .../execute/execution_extend/CMakeLists.txt          |  6 +++++-
+ test/specs/specs/CMakeLists.txt                      |  7 +++++--
+ test/specs/specs_extend/CMakeLists.txt               |  7 +++++--
+ test/specs/verify/CMakeLists.txt                     |  7 +++++--
+ test/volume/CMakeLists.txt                           |  7 +++++--
+ 13 files changed, 69 insertions(+), 25 deletions(-)
+
+diff --git a/src/daemon/modules/image/CMakeLists.txt b/src/daemon/modules/image/CMakeLists.txt
+index f8bc5840..d8b78ce1 100644
+--- a/src/daemon/modules/image/CMakeLists.txt
++++ b/src/daemon/modules/image/CMakeLists.txt
+@@ -68,8 +68,9 @@ set(LIB_ISULAD_IMG_SRCS
+     ${CMAKE_SOURCE_DIR}/src/utils/buffer/buffer.c
+     ${CMAKE_SOURCE_DIR}/src/daemon/common/err_msg.c
+     ${CMAKE_SOURCE_DIR}/src/daemon/common/sysinfo.c
+-    ${CMAKE_SOURCE_DIR}/src/daemon/common/cgroup.c
+-    ${CMAKE_SOURCE_DIR}/src/daemon/common/cgroup_v1.c
++    ${CMAKE_SOURCE_DIR}/src/daemon/common/cgroup/cgroup.c
++    ${CMAKE_SOURCE_DIR}/src/daemon/common/cgroup/cgroup_v1.c
++    ${CMAKE_SOURCE_DIR}/src/daemon/common/cgroup/cgroup_v2.c
+     ${CMAKE_SOURCE_DIR}/src/utils/tar/util_gzip.c
+     ${CMAKE_SOURCE_DIR}/src/daemon/config/isulad_config.c
+     ${CMAKE_SOURCE_DIR}/src/daemon/config/daemon_arguments.c
+@@ -102,6 +103,7 @@ target_include_directories(${LIB_ISULAD_IMG} PUBLIC
+     ${CMAKE_SOURCE_DIR}/src/utils/tar
+     ${CMAKE_SOURCE_DIR}/src/daemon/config
+     ${CMAKE_SOURCE_DIR}/src/daemon/common
++    ${CMAKE_SOURCE_DIR}/src/daemon/common/cgroup
+     ${CMAKE_SOURCE_DIR}/src/daemon/modules/spec/
+     ${CMAKE_SOURCE_DIR}/src/utils/cutils
+     ${CMAKE_SOURCE_DIR}/src/utils/cutils/map
+diff --git a/test/cgroup/cpu/CMakeLists.txt b/test/cgroup/cpu/CMakeLists.txt
+index 32fe0a23..30bfc417 100644
+--- a/test/cgroup/cpu/CMakeLists.txt
++++ b/test/cgroup/cpu/CMakeLists.txt
+@@ -6,8 +6,10 @@ add_executable(${EXE}
+     ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/utils/cutils/path.c
+     ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/daemon/common/err_msg.c
+     ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/daemon/common/sysinfo.c
+-    ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/daemon/common/cgroup.c
+-    ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/daemon/common/cgroup_v1.c
++    ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/daemon/common/cgroup/cgroup.c
++    ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/daemon/common/cgroup/cgroup_v1.c
++    ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/daemon/common/cgroup/cgroup_v2.c
++    ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/daemon/common/cgroup/cgroup_common.c
+     ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/cmd/command_parser.c
+     ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/daemon/config/daemon_arguments.c
+     ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/daemon/config/isulad_config.c
+@@ -20,6 +22,7 @@ target_include_directories(${EXE} PUBLIC
+     ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/common
+     ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/daemon/config
+     ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/daemon/common
++    ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/daemon/common/cgroup
+     ${CMAKE_BINARY_DIR}/conf
+     ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/config
+     ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/cmd
+diff --git a/test/cgroup/cpu/cgroup_cpu_ut.cc b/test/cgroup/cpu/cgroup_cpu_ut.cc
+index eaee90c0..6e6e04f4 100644
+--- a/test/cgroup/cpu/cgroup_cpu_ut.cc
++++ b/test/cgroup/cpu/cgroup_cpu_ut.cc
+@@ -69,13 +69,21 @@ TEST(CgroupCpuUnitTest, test_common_find_cgroup_mnt_and_root)
+ {
+     char *mnt = NULL;
+     char *root = NULL;
+-    ASSERT_EQ(common_find_cgroup_mnt_and_root(nullptr, &mnt, &root), -1);
++
++    int ret = cgroup_ops_init();
++    ASSERT_EQ(ret, 0);
++
++    ASSERT_EQ(common_get_cgroup_mnt_and_root_path(nullptr, &mnt, &root), -1);
+ }
+ 
+ TEST(CgroupCpuUnitTest, test_sysinfo_cgroup_controller_cpurt_mnt_path)
+ {
+     MOCK_SET(util_common_calloc_s, nullptr);
+     ASSERT_EQ(get_sys_info(true), nullptr);
+-    ASSERT_EQ(sysinfo_cgroup_controller_cpurt_mnt_path(), nullptr);
++    
++    int ret = cgroup_ops_init();
++    ASSERT_EQ(ret, 0);
++
++    ASSERT_EQ(sysinfo_get_cpurt_mnt_path(), nullptr);
+     MOCK_CLEAR(util_common_calloc_s);
+ }
+diff --git a/test/image/oci/oci_config_merge/CMakeLists.txt b/test/image/oci/oci_config_merge/CMakeLists.txt
+index 90809080..d13ec738 100644
+--- a/test/image/oci/oci_config_merge/CMakeLists.txt
++++ b/test/image/oci/oci_config_merge/CMakeLists.txt
+@@ -25,8 +25,10 @@ add_executable(${EXE}
+     ${CMAKE_CURRENT_SOURCE_DIR}/../../../../src/daemon/common/err_msg.c
+     ${CMAKE_CURRENT_SOURCE_DIR}/../../../../src/daemon/config/isulad_config.c
+     ${CMAKE_CURRENT_SOURCE_DIR}/../../../../src/daemon/common/sysinfo.c
+-    ${CMAKE_CURRENT_SOURCE_DIR}/../../../../src/daemon/common/cgroup.c
+-    ${CMAKE_CURRENT_SOURCE_DIR}/../../../../src/daemon/common/cgroup_v1.c
++    ${CMAKE_CURRENT_SOURCE_DIR}/../../../../src/daemon/common/cgroup/cgroup.c
++    ${CMAKE_CURRENT_SOURCE_DIR}/../../../../src/daemon/common/cgroup/cgroup_v1.c
++    ${CMAKE_CURRENT_SOURCE_DIR}/../../../../src/daemon/common/cgroup/cgroup_v2.c
++    ${CMAKE_CURRENT_SOURCE_DIR}/../../../../src/daemon/common/cgroup/cgroup_common.c
+     ${CMAKE_CURRENT_SOURCE_DIR}/../../../../src/cmd/command_parser.c
+     ${CMAKE_CURRENT_SOURCE_DIR}/../../../../src/daemon/config/daemon_arguments.c
+     ${CMAKE_CURRENT_SOURCE_DIR}/../../../../test/image/oci/oci_ut_common.cc
+@@ -55,6 +57,7 @@ target_include_directories(${EXE} PUBLIC
+     ${CMAKE_CURRENT_SOURCE_DIR}/../../../../src/utils/cutils/map
+     ${CMAKE_CURRENT_SOURCE_DIR}/../../../../src/daemon/config
+     ${CMAKE_CURRENT_SOURCE_DIR}/../../../../src/daemon/common
++    ${CMAKE_CURRENT_SOURCE_DIR}/../../../../src/daemon/common/cgroup
+     ${CMAKE_CURRENT_SOURCE_DIR}/../../../../src/daemon/modules/volume
+     ${CMAKE_CURRENT_SOURCE_DIR}/../../../../src/daemon/modules/api
+     ${CMAKE_CURRENT_SOURCE_DIR}/../../../../src/daemon/modules/runtime/engines
+diff --git a/test/image/oci/registry/CMakeLists.txt b/test/image/oci/registry/CMakeLists.txt
+index 77a7907e..5b5bc3f5 100644
+--- a/test/image/oci/registry/CMakeLists.txt
++++ b/test/image/oci/registry/CMakeLists.txt
+@@ -26,8 +26,10 @@ add_executable(${EXE}
+     ${CMAKE_CURRENT_SOURCE_DIR}/../../../../src/daemon/modules/image/oci/storage/image_store/image_type.c
+     ${CMAKE_CURRENT_SOURCE_DIR}/../../../../src/daemon/modules/image/oci/registry_type.c
+     ${CMAKE_CURRENT_SOURCE_DIR}/../../../../src/daemon/common/sysinfo.c
+-    ${CMAKE_CURRENT_SOURCE_DIR}/../../../../src/daemon/common/cgroup.c
+-    ${CMAKE_CURRENT_SOURCE_DIR}/../../../../src/daemon/common/cgroup_v1.c
++    ${CMAKE_CURRENT_SOURCE_DIR}/../../../../src/daemon/common/cgroup/cgroup.c
++    ${CMAKE_CURRENT_SOURCE_DIR}/../../../../src/daemon/common/cgroup/cgroup_v1.c
++    ${CMAKE_CURRENT_SOURCE_DIR}/../../../../src/daemon/common/cgroup/cgroup_v2.c
++    ${CMAKE_CURRENT_SOURCE_DIR}/../../../../src/daemon/common/cgroup/cgroup_common.c
+     ${CMAKE_CURRENT_SOURCE_DIR}/../../../../src/daemon/modules/image/oci/storage/image_store/image_store.c
+     ${CMAKE_CURRENT_SOURCE_DIR}/../../../../src/daemon/modules/image/oci/storage/remote_layer_support/ro_symlink_maintain.c
+     ${CMAKE_CURRENT_SOURCE_DIR}/../../../../src/daemon/modules/image/oci/registry/registry.c
+@@ -58,6 +60,7 @@ target_include_directories(${EXE} PUBLIC
+     ${CMAKE_CURRENT_SOURCE_DIR}/../../../../src/daemon/modules/api
+     ${CMAKE_CURRENT_SOURCE_DIR}/../../../../src/daemon/config
+     ${CMAKE_CURRENT_SOURCE_DIR}/../../../../src/daemon/common
++    ${CMAKE_CURRENT_SOURCE_DIR}/../../../../src/daemon/common/cgroup
+     ${CMAKE_CURRENT_SOURCE_DIR}/../../../../src/daemon
+     ${CMAKE_CURRENT_SOURCE_DIR}/../../../../src/daemon/modules/image/oci/storage
+     ${CMAKE_CURRENT_SOURCE_DIR}/../../../../src/daemon/modules/image/oci/storage/image_store
+diff --git a/test/network/network_ns/CMakeLists.txt b/test/network/network_ns/CMakeLists.txt
+index 50520427..71b8039d 100644
+--- a/test/network/network_ns/CMakeLists.txt
++++ b/test/network/network_ns/CMakeLists.txt
+@@ -30,8 +30,10 @@ add_executable(${EXE}
+     ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/daemon/config/daemon_arguments.c
+     ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/daemon/common/err_msg.c
+     ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/daemon/common/sysinfo.c
+-    ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/daemon/common/cgroup.c
+-    ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/daemon/common/cgroup_v1.c
++    ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/daemon/common/cgroup/cgroup.c
++    ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/daemon/common/cgroup/cgroup_v1.c
++    ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/daemon/common/cgroup/cgroup_v2.c
++    ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/daemon/common/cgroup/cgroup_common.c
+     ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/cmd/command_parser.c
+     ${CMAKE_CURRENT_SOURCE_DIR}/../../../test/image/oci/oci_ut_common.cc
+     ${CMAKE_CURRENT_SOURCE_DIR}/../../../test/mocks/image_mock.cc
+@@ -55,6 +57,7 @@ target_include_directories(${EXE} PUBLIC
+     ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/cmd/options
+     ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/daemon/config
+     ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/daemon/common
++    ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/daemon/common/cgroup
+     ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/daemon/modules/api
+     ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/daemon/modules/volume
+     ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/daemon/modules/runtime
+diff --git a/test/runtime/isula/CMakeLists.txt b/test/runtime/isula/CMakeLists.txt
+index cc1178b8..c1f0a5cc 100644
+--- a/test/runtime/isula/CMakeLists.txt
++++ b/test/runtime/isula/CMakeLists.txt
+@@ -21,8 +21,10 @@ add_executable(${EXE}
+     ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/utils/cutils/mainloop.c
+     ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/daemon/common/err_msg.c
+     ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/daemon/common/sysinfo.c
+-    ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/daemon/common/cgroup.c
+-    ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/daemon/common/cgroup_v1.c
++    ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/daemon/common/cgroup/cgroup.c
++    ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/daemon/common/cgroup/cgroup_v1.c
++    ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/daemon/common/cgroup/cgroup_v2.c
++    ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/daemon/common/cgroup/cgroup_common.c
+     ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/cmd/command_parser.c
+     ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/daemon/config/daemon_arguments.c
+     ${CMAKE_CURRENT_SOURCE_DIR}/../../../test/image/oci/oci_ut_common.cc
+@@ -51,6 +53,7 @@ target_include_directories(${EXE} PUBLIC
+     ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/cmd
+     ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/daemon/config
+     ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/daemon/common
++    ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/daemon/common/cgroup
+     ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/daemon/modules/container
+     ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/daemon/modules/container/restart_manager
+     ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/daemon/modules/container/health_check
+diff --git a/test/runtime/lcr/CMakeLists.txt b/test/runtime/lcr/CMakeLists.txt
+index 424a6101..c3b93d67 100644
+--- a/test/runtime/lcr/CMakeLists.txt
++++ b/test/runtime/lcr/CMakeLists.txt
+@@ -18,8 +18,10 @@ add_executable(${EXE}
+     ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/utils/cutils/map/rb_tree.c
+     ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/daemon/common/err_msg.c
+     ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/daemon/common/sysinfo.c
+-    ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/daemon/common/cgroup.c
+-    ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/daemon/common/cgroup_v1.c
++    ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/daemon/common/cgroup/cgroup.c
++    ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/daemon/common/cgroup/cgroup_v1.c
++    ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/daemon/common/cgroup/cgroup_v2.c
++    ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/daemon/common/cgroup/cgroup_common.c
+     ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/cmd/command_parser.c
+     ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/daemon/config/daemon_arguments.c
+     ${CMAKE_CURRENT_SOURCE_DIR}/../../../test/image/oci/oci_ut_common.cc
+@@ -39,6 +41,7 @@ target_include_directories(${EXE} PUBLIC
+     ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/utils/cutils/map
+     ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/daemon/config
+     ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/daemon/common
++    ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/daemon/common/cgroup
+     ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/daemon/modules/api
+     ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/daemon/modules/container
+     ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/daemon/modules/container/restart_manager
+diff --git a/test/services/execution/execute/execution_extend/CMakeLists.txt b/test/services/execution/execute/execution_extend/CMakeLists.txt
+index 68e0f443..f0875fd7 100644
+--- a/test/services/execution/execute/execution_extend/CMakeLists.txt
++++ b/test/services/execution/execute/execution_extend/CMakeLists.txt
+@@ -21,7 +21,10 @@ add_executable(${EXE}
+     ${CMAKE_CURRENT_SOURCE_DIR}/../../../../../src/utils/cutils/mainloop.c
+     ${CMAKE_CURRENT_SOURCE_DIR}/../../../../../src/utils/cutils/filters.c
+     ${CMAKE_CURRENT_SOURCE_DIR}/../../../../../src/daemon/common/err_msg.c
+-    ${CMAKE_CURRENT_SOURCE_DIR}/../../../../../src/daemon/common/cgroup.c
++    ${CMAKE_CURRENT_SOURCE_DIR}/../../../../../src/daemon/common/cgroup/cgroup.c
++    ${CMAKE_CURRENT_SOURCE_DIR}/../../../../../src/daemon/common/cgroup/cgroup_v1.c
++    ${CMAKE_CURRENT_SOURCE_DIR}/../../../../../src/daemon/common/cgroup/cgroup_v2.c
++    ${CMAKE_CURRENT_SOURCE_DIR}/../../../../../src/daemon/common/cgroup/cgroup_common.c
+     ${CMAKE_CURRENT_SOURCE_DIR}/../../../../../src/daemon/modules/events_sender/event_sender.c
+     ${CMAKE_CURRENT_SOURCE_DIR}/../../../../../src/utils/console/console.c
+     ${CMAKE_CURRENT_SOURCE_DIR}/../../../../../src/utils/cutils/utils.c
+@@ -59,6 +62,7 @@ target_include_directories(${EXE} PUBLIC
+     ${CMAKE_CURRENT_SOURCE_DIR}/../../../../../src/utils/console
+     ${CMAKE_CURRENT_SOURCE_DIR}/../../../../../src/daemon/config
+     ${CMAKE_CURRENT_SOURCE_DIR}/../../../../../src/daemon/common
++    ${CMAKE_CURRENT_SOURCE_DIR}/../../../../../src/daemon/common/cgroup
+     ${CMAKE_CURRENT_SOURCE_DIR}/../../../../../src/utils/cutils
+     ${CMAKE_CURRENT_SOURCE_DIR}/../../../../../src/daemon/modules/api
+     ${CMAKE_CURRENT_SOURCE_DIR}/../../../../../src/daemon/modules/image
+diff --git a/test/specs/specs/CMakeLists.txt b/test/specs/specs/CMakeLists.txt
+index 508123fa..45f688f9 100644
+--- a/test/specs/specs/CMakeLists.txt
++++ b/test/specs/specs/CMakeLists.txt
+@@ -28,8 +28,10 @@ add_executable(${EXE}
+     ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/daemon/modules/spec/specs_security.c
+     ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/daemon/common/err_msg.c
+     ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/daemon/common/sysinfo.c
+-    ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/daemon/common/cgroup.c
+-    ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/daemon/common/cgroup_v1.c
++    ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/daemon/common/cgroup/cgroup.c
++    ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/daemon/common/cgroup/cgroup_v1.c
++    ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/daemon/common/cgroup/cgroup_v2.c
++    ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/daemon/common/cgroup/cgroup_common.c
+     ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/cmd/command_parser.c
+     ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/daemon/config/daemon_arguments.c
+     ${CMAKE_CURRENT_SOURCE_DIR}/../../../test/image/oci/oci_ut_common.cc
+@@ -54,6 +56,7 @@ target_include_directories(${EXE} PUBLIC
+     ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/cmd/isulad
+     ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/daemon/config
+     ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/daemon/common
++    ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/daemon/common/cgroup
+     ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/daemon/modules/api
+     ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/daemon/modules/volume
+     ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/daemon/modules/runtime
+diff --git a/test/specs/specs_extend/CMakeLists.txt b/test/specs/specs_extend/CMakeLists.txt
+index bf4b378e..1b737089 100644
+--- a/test/specs/specs_extend/CMakeLists.txt
++++ b/test/specs/specs_extend/CMakeLists.txt
+@@ -28,8 +28,10 @@ add_executable(${EXE}
+     ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/daemon/modules/spec/specs_security.c
+     ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/daemon/common/err_msg.c
+     ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/daemon/common/sysinfo.c
+-    ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/daemon/common/cgroup.c
+-    ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/daemon/common/cgroup_v1.c
++    ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/daemon/common/cgroup/cgroup.c
++    ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/daemon/common/cgroup/cgroup_v1.c
++    ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/daemon/common/cgroup/cgroup_v2.c
++    ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/daemon/common/cgroup/cgroup_common.c
+     ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/cmd/command_parser.c
+     ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/daemon/config/daemon_arguments.c
+     ${CMAKE_CURRENT_SOURCE_DIR}/../../../test/image/oci/oci_ut_common.cc
+@@ -54,6 +56,7 @@ target_include_directories(${EXE} PUBLIC
+     ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/cmd/isulad
+     ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/daemon/config
+     ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/daemon/common
++    ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/daemon/common/cgroup
+     ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/daemon/modules/volume
+     ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/daemon/modules/api
+     ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/daemon/modules/container
+diff --git a/test/specs/verify/CMakeLists.txt b/test/specs/verify/CMakeLists.txt
+index 0e60a39e..b0602127 100644
+--- a/test/specs/verify/CMakeLists.txt
++++ b/test/specs/verify/CMakeLists.txt
+@@ -23,8 +23,10 @@ add_executable(${EXE}
+     ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/utils/cutils/map/rb_tree.c
+     ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/daemon/common/err_msg.c
+     ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/daemon/common/sysinfo.c
+-    ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/daemon/common/cgroup.c
+-    ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/daemon/common/cgroup_v1.c
++    ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/daemon/common/cgroup/cgroup.c
++    ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/daemon/common/cgroup/cgroup_v1.c
++    ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/daemon/common/cgroup/cgroup_v2.c
++    ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/daemon/common/cgroup/cgroup_common.c
+     ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/daemon/modules/spec/verify.c
+     ${CMAKE_CURRENT_SOURCE_DIR}/../../../test/image/oci/oci_ut_common.cc
+     ${CMAKE_CURRENT_SOURCE_DIR}/../../../test/mocks/containers_store_mock.cc
+@@ -50,6 +52,7 @@ target_include_directories(${EXE} PUBLIC
+     ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/cmd/isulad
+     ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/daemon/config
+     ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/daemon/common
++    ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/daemon/common/cgroup
+     ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/daemon/modules/api
+     ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/daemon/modules/volume
+     ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/daemon/modules/runtime
+diff --git a/test/volume/CMakeLists.txt b/test/volume/CMakeLists.txt
+index cc309352..27d07330 100644
+--- a/test/volume/CMakeLists.txt
++++ b/test/volume/CMakeLists.txt
+@@ -20,8 +20,10 @@ add_executable(${EXE}
+     ${CMAKE_CURRENT_SOURCE_DIR}/../../src/utils/sha256/sha256.c
+     ${CMAKE_CURRENT_SOURCE_DIR}/../../src/daemon/common/err_msg.c
+     ${CMAKE_CURRENT_SOURCE_DIR}/../../src/daemon/common/sysinfo.c
+-    ${CMAKE_CURRENT_SOURCE_DIR}/../../src/daemon/common/cgroup.c
+-    ${CMAKE_CURRENT_SOURCE_DIR}/../../src/daemon/common/cgroup_v1.c
++    ${CMAKE_CURRENT_SOURCE_DIR}/../../src/daemon/common/cgroup/cgroup.c
++    ${CMAKE_CURRENT_SOURCE_DIR}/../../src/daemon/common/cgroup/cgroup_v1.c
++    ${CMAKE_CURRENT_SOURCE_DIR}/../../src/daemon/common/cgroup/cgroup_v2.c
++    ${CMAKE_CURRENT_SOURCE_DIR}/../../src/daemon/common/cgroup/cgroup_common.c
+     ${CMAKE_CURRENT_SOURCE_DIR}/../../src/cmd/command_parser.c
+     volume_ut.cc)
+ 
+@@ -34,6 +36,7 @@ target_include_directories(${EXE} PUBLIC
+     ${CMAKE_CURRENT_SOURCE_DIR}/../../src/utils/http
+     ${CMAKE_CURRENT_SOURCE_DIR}/../../src/daemon/modules/api
+     ${CMAKE_CURRENT_SOURCE_DIR}/../../src/daemon/common
++    ${CMAKE_CURRENT_SOURCE_DIR}/../../src/daemon/common/cgroup
+     ${CMAKE_CURRENT_SOURCE_DIR}/../../src/daemon/modules/volume
+     ${CMAKE_BINARY_DIR}/conf
+     ${CMAKE_CURRENT_SOURCE_DIR}/../../src/utils/sha256
+-- 
+2.25.1
+

0031-cgroup-v2-does-not-support-isulad-setting-cpu_rt-opt.patch

@@ -0,0 +1,71 @@
+From 8e11a1eea62cb8061f1613379ff83bd9a721fa50 Mon Sep 17 00:00:00 2001
+From: zhongtao <zhongtao17@huawei.com>
+Date: Wed, 31 Jan 2024 18:10:46 +0800
+Subject: [PATCH 031/149] cgroup v2 does not support isulad setting cpu_rt
+ options
+
+Signed-off-by: zhongtao <zhongtao17@huawei.com>
+---
+ src/cmd/isulad/isulad_commands.c | 32 ++++++++++++++++++++++++++++++++
+ 1 file changed, 32 insertions(+)
+
+diff --git a/src/cmd/isulad/isulad_commands.c b/src/cmd/isulad/isulad_commands.c
+index 5fb55689..619e36d1 100644
+--- a/src/cmd/isulad/isulad_commands.c
++++ b/src/cmd/isulad/isulad_commands.c
+@@ -34,6 +34,7 @@
+ #include "utils_verify.h"
+ #include "opt_ulimit.h"
+ #include "opt_log.h"
++#include "sysinfo.h"
+ 
+ const char isulad_desc[] = "GLOBAL OPTIONS:";
+ const char isulad_usage[] = "[global options]";
+@@ -411,6 +412,33 @@ out:
+     return ret;
+ }
+ 
++static int check_args_cpu_rt(const struct service_arguments *args)
++{
++    int ret = 0;
++    __isula_auto_sysinfo_t sysinfo_t *sysinfo = NULL;
++
++    sysinfo = get_sys_info(true);
++    if (sysinfo == NULL) {
++        COMMAND_ERROR("Failed to get system info");
++        ERROR("Failed to get system info");
++        return -1;
++    }
++
++    if (!(sysinfo->cgcpuinfo.cpu_rt_period) && args->json_confs->cpu_rt_period != 0) {
++        COMMAND_ERROR("Invalid --cpu-rt-period: Your kernel does not support cgroup rt period");
++        ERROR("Invalid --cpu-rt-period: Your kernel does not support cgroup rt period");
++        return -1;
++    }
++
++    if (!(sysinfo->cgcpuinfo.cpu_rt_runtime) && args->json_confs->cpu_rt_runtime != 0) {
++        COMMAND_ERROR("Invalid --cpu-rt-runtime: Your kernel does not support cgroup rt runtime");
++        ERROR("Invalid --cpu-rt-runtime: Your kernel does not support cgroup rt runtime");
++        return -1;
++    }
++
++    return ret;
++}
++
+ int check_args(struct service_arguments *args)
+ {
+     int ret = 0;
+@@ -471,6 +499,10 @@ int check_args(struct service_arguments *args)
+         goto out;
+     }
+ 
++    if (check_args_cpu_rt(args) != 0) {
++        ret = -1;
++        goto out;
++    }
+ out:
+     return ret;
+ }
+-- 
+2.25.1
+

0032-add-test-that-isulad-cannot-set-cpu_rt-parameters-wh.patch

@@ -0,0 +1,30 @@
+From 1ab0f4608fb749b50aa6f8d8188db23aa8a6e1ac Mon Sep 17 00:00:00 2001
+From: zhongtao <zhongtao17@huawei.com>
+Date: Thu, 1 Feb 2024 10:48:45 +0800
+Subject: [PATCH 032/149] add test that isulad cannot set cpu_rt parameters
+ when adding cgroup v2
+
+Signed-off-by: zhongtao <zhongtao17@huawei.com>
+---
+ CI/test_cases/container_cases/cpu_rt.sh | 5 ++++-
+ 1 file changed, 4 insertions(+), 1 deletion(-)
+
+diff --git a/CI/test_cases/container_cases/cpu_rt.sh b/CI/test_cases/container_cases/cpu_rt.sh
+index bdc43a5e..23d3baed 100755
+--- a/CI/test_cases/container_cases/cpu_rt.sh
++++ b/CI/test_cases/container_cases/cpu_rt.sh
+@@ -106,7 +106,10 @@ function test_kernel_without_cpurt()
+ 
+     msg_info "${test} starting..."
+ 
+-    start_isulad_without_valgrind --cpu-rt-period 1000000 --cpu-rt-runtime 950000
++    isulad --cpu-rt-period 1000000 --cpu-rt-runtime 950000 2>&1 | grep 'Your kernel does not support cgroup rt period'
++    [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - kernel does not support cpu-rt, but start isulad with cpu-rt success" && ((ret++))
++
++    start_isulad_without_valgrind
+ 
+     isula pull ${image}
+     [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - failed to pull image: ${image}" && return ${FAILURE}
+-- 
+2.25.1
+

0033-fix-sandbox-container-bool-value-uninitialized.patch

@@ -0,0 +1,26 @@
+From f62df3dedbbe11bb56e6da7dd610c573fd3ed828 Mon Sep 17 00:00:00 2001
+From: jikai <jikai11@huawei.com>
+Date: Mon, 25 Mar 2024 10:01:56 +0800
+Subject: [PATCH 033/149] fix sandbox container bool value uninitialized
+
+Signed-off-by: jikai <jikai11@huawei.com>
+---
+ src/daemon/modules/service/service_container.c | 2 +-
+ 1 file changed, 1 insertion(+), 1 deletion(-)
+
+diff --git a/src/daemon/modules/service/service_container.c b/src/daemon/modules/service/service_container.c
+index 7b34cc7f..a8090d5a 100644
+--- a/src/daemon/modules/service/service_container.c
++++ b/src/daemon/modules/service/service_container.c
+@@ -750,7 +750,7 @@ static int do_start_container(container_t *cont, const char *console_fifos[], bo
+     oci_runtime_spec *oci_spec = NULL;
+     rt_create_params_t create_params = { 0 };
+     rt_start_params_t start_params = { 0 };
+-    bool sandbox_container;
++    bool sandbox_container = false;
+ 
+     nret = snprintf(bundle, sizeof(bundle), "%s/%s", cont->root_path, id);
+     if (nret < 0 || (size_t)nret >= sizeof(bundle)) {
+-- 
+2.25.1
+

0034-bugfix-for-cpurt.sh.patch

@@ -0,0 +1,47 @@
+From 411483ad9b2a0c50190f9b56779d41889c895014 Mon Sep 17 00:00:00 2001
+From: zhongtao <zhongtao17@huawei.com>
+Date: Wed, 27 Mar 2024 10:29:11 +0800
+Subject: [PATCH 034/149] bugfix for cpurt.sh
+
+Signed-off-by: zhongtao <zhongtao17@huawei.com>
+---
+ CI/test_cases/container_cases/cpu_rt.sh | 9 ++++-----
+ 1 file changed, 4 insertions(+), 5 deletions(-)
+
+diff --git a/CI/test_cases/container_cases/cpu_rt.sh b/CI/test_cases/container_cases/cpu_rt.sh
+index 23d3baed..64dcd81f 100755
+--- a/CI/test_cases/container_cases/cpu_rt.sh
++++ b/CI/test_cases/container_cases/cpu_rt.sh
+@@ -109,7 +109,7 @@ function test_kernel_without_cpurt()
+     isulad --cpu-rt-period 1000000 --cpu-rt-runtime 950000 2>&1 | grep 'Your kernel does not support cgroup rt period'
+     [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - kernel does not support cpu-rt, but start isulad with cpu-rt success" && ((ret++))
+ 
+-    start_isulad_without_valgrind
++    start_isulad_with_valgrind
+ 
+     isula pull ${image}
+     [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - failed to pull image: ${image}" && return ${FAILURE}
+@@ -194,6 +194,9 @@ function do_test()
+         test_cpurt_isulad_abnormal $runtime || ((ret++))
+         test_isula_update_normal $runtime || ((ret++))
+         test_isula_update_abnormal $runtime || ((ret++))
++        stop_isulad_without_valgrind
++        # set cpu-rt to the initial state
++        start_isulad_with_valgrind --cpu-rt-period 1000000 --cpu-rt-runtime 0
+     else
+         test_kernel_without_cpurt $runtime || ((ans++))
+     fi
+@@ -211,10 +214,6 @@ do
+ 
+     do_test $element || ((ans++))
+ 
+-    stop_isulad_without_valgrind
+-    # set cpu-rt to the initial state
+-    start_isulad_with_valgrind --cpu-rt-period 1000000 --cpu-rt-runtime 0
+-
+     isula rm -f $(isula ps -aq)
+ done
+ 
+-- 
+2.25.1
+

0035-monitor-cgroup-oom-killed-event-and-update-to-cri-of.patch

@@ -0,0 +1,868 @@
+From 947cf87a87ec49409ae509e5142b8134454d1547 Mon Sep 17 00:00:00 2001
+From: jikai <jikai11@huawei.com>
+Date: Thu, 28 Mar 2024 12:51:09 +0000
+Subject: [PATCH 035/149] monitor cgroup oom killed event and update to cri of
+ container
+
+Signed-off-by: jikai <jikai11@huawei.com>
+---
+ src/daemon/common/cgroup/cgroup.c             |  91 +++++++++-
+ src/daemon/common/cgroup/cgroup.h             |   5 +
+ src/daemon/common/cgroup/cgroup_common.h      |  13 ++
+ src/daemon/common/cgroup/cgroup_v1.c          | 160 ++++++++++++++++++
+ src/daemon/common/cgroup/cgroup_v2.c          | 138 ++++++++++++++-
+ .../v1/v1_cri_container_manager_service.cc    |   3 +
+ src/daemon/modules/api/container_api.h        |   5 +-
+ .../container/container_events_handler.c      |  12 +-
+ .../modules/container/container_state.c       |  15 ++
+ .../modules/container/restore/restore.c       |  10 +-
+ .../modules/container/supervisor/supervisor.c |  54 +++++-
+ src/daemon/modules/events/collector.c         |   7 +-
+ .../modules/service/service_container.c       |  11 +-
+ 13 files changed, 498 insertions(+), 26 deletions(-)
+
+diff --git a/src/daemon/common/cgroup/cgroup.c b/src/daemon/common/cgroup/cgroup.c
+index 837b514a..d3f1445a 100644
+--- a/src/daemon/common/cgroup/cgroup.c
++++ b/src/daemon/common/cgroup/cgroup.c
+@@ -133,4 +133,93 @@ char *common_get_own_cgroup_path(const char *subsystem)
+     }
+ 
+     return g_cgroup_ops.get_own_cgroup_path(subsystem);
+-}
+\ No newline at end of file
++}
++
++char *common_convert_cgroup_path(const char *cgroup_path)
++{
++    char *token = NULL;
++    char result[PATH_MAX + 1] = {0};
++    __isula_auto_array_t char **arr = NULL;
++
++    if (cgroup_path == NULL) {
++        ERROR("Invalid NULL cgroup path");
++        return NULL;
++    }
++
++    // for cgroup fs cgroup path, return directly
++    if (!util_has_suffix(cgroup_path, ".slice")) {
++        return util_strdup_s(cgroup_path);
++    }
++
++    // for systemd cgroup, cgroup_path should have the form slice:prefix:id,
++    // convert it to a true path, such as from test-a.slice:isulad:id
++    // to test.slice/test-a.slice/isulad-id.scope
++    arr = util_string_split_n(cgroup_path, ':', 3);
++    if (arr == NULL || util_array_len((const char **)arr) != 3) {
++        ERROR("Invalid systemd cgroup parent");
++        return NULL;
++    }
++
++    token = strchr(arr[0], '-');
++    while (token != NULL) {
++        *token = '\0';
++        if (strlen(arr[0]) > PATH_MAX || strlen(result) + 1 + strlen(".slice") >
++            PATH_MAX - strlen(arr[0])) {
++            ERROR("Invalid systemd cgroup parent: exceeds max length of path");
++            *token = '-';
++            return NULL;
++        }
++        if (result[0] != '\0') {
++            strcat(result, "/");
++        }
++        strcat(result, arr[0]);
++        strcat(result, ".slice");
++        *token = '-';
++        token = strchr(token + 1, '-');
++    }
++
++    // Add /arr[0]/arr[1]-arr[2].scope, 3 include two slashes and one dash
++    if (strlen(cgroup_path) > PATH_MAX || strlen(result) + 3 + strlen(".scope") >
++        PATH_MAX - strlen(arr[0] - strlen(arr[1]) - strlen(arr[2]))) {
++        ERROR("Invalid systemd cgroup parent: exceeds max length of path");
++        return NULL;
++    }
++
++    (void)strcat(result, "/");
++    (void)strcat(result, arr[0]);
++    (void)strcat(result, "/");
++    (void)strcat(result, arr[1]);
++    (void)strcat(result, "-");
++    (void)strcat(result, arr[2]);
++    (void)strcat(result, ".scope");
++
++    return util_strdup_s(result);
++}
++
++cgroup_oom_handler_info_t *common_get_cgroup_oom_handler(int fd, const char *name, const char *cgroup_path, const char *exit_fifo)
++{
++    if (g_cgroup_ops.get_cgroup_oom_handler == NULL) {
++        ERROR("Unimplmented get_cgroup_oom_handler op");
++        return NULL;
++    }
++
++    return g_cgroup_ops.get_cgroup_oom_handler(fd, name, cgroup_path, exit_fifo);
++}
++
++void common_free_cgroup_oom_handler_info(cgroup_oom_handler_info_t *info)
++{
++    if (info == NULL) {
++        return;
++    }
++
++    if (info->oom_event_fd >= 0) {
++        close(info->oom_event_fd);
++    }
++    if (info->cgroup_file_fd >= 0) {
++        close(info->cgroup_file_fd);
++    }
++
++    free(info->name);
++    free(info->cgroup_memory_event_path);
++    free(info);
++}
+diff --git a/src/daemon/common/cgroup/cgroup.h b/src/daemon/common/cgroup/cgroup.h
+index 1efc3ca6..8c76d99d 100644
+--- a/src/daemon/common/cgroup/cgroup.h
++++ b/src/daemon/common/cgroup/cgroup.h
+@@ -41,6 +41,11 @@ int common_get_cgroup_mnt_and_root_path(const char *subsystem, char **mountpoint
+ char *common_get_init_cgroup_path(const char *subsystem);
+ char *common_get_own_cgroup_path(const char *subsystem);
+ 
++char *common_convert_cgroup_path(const char *cgroup_path);
++
++cgroup_oom_handler_info_t *common_get_cgroup_oom_handler(int fd, const char *name, const char *cgroup_path, const char *exit_fifo);
++void common_free_cgroup_oom_handler_info(cgroup_oom_handler_info_t *info);
++
+ #ifdef __cplusplus
+ }
+ #endif
+diff --git a/src/daemon/common/cgroup/cgroup_common.h b/src/daemon/common/cgroup/cgroup_common.h
+index 2a0935cb..e3912bf0 100644
+--- a/src/daemon/common/cgroup/cgroup_common.h
++++ b/src/daemon/common/cgroup/cgroup_common.h
+@@ -116,6 +116,17 @@ typedef struct {
+     cgroup_pids_metrics_t cgpids_metrics;
+ } cgroup_metrics_t;
+ 
++#define CGROUP_OOM_HANDLE_CONTINUE false
++#define CGROUP_OOM_HANDLE_CLOSE true
++
++typedef struct _cgroup_oom_handler_info_t {
++    int oom_event_fd;
++    int cgroup_file_fd;
++    char *name;
++    char *cgroup_memory_event_path;
++    bool (*oom_event_handler)(int, void *);
++} cgroup_oom_handler_info_t;
++
+ typedef struct {
+     int (*get_cgroup_version)(void);
+     int (*get_cgroup_info)(cgroup_mem_info_t *meminfo, cgroup_cpu_info_t *cpuinfo,
+@@ -128,6 +139,8 @@ typedef struct {
+ 
+     char *(*get_init_cgroup_path)(const char *subsystem);
+     char *(*get_own_cgroup_path)(const char *subsystem);
++
++    cgroup_oom_handler_info_t *(*get_cgroup_oom_handler)(int fd, const char *name, const char *cgroup_path, const char *exit_fifo);
+ } cgroup_ops;
+ 
+ #ifdef __cplusplus
+diff --git a/src/daemon/common/cgroup/cgroup_v1.c b/src/daemon/common/cgroup/cgroup_v1.c
+index 51cf7512..41f3110a 100644
+--- a/src/daemon/common/cgroup/cgroup_v1.c
++++ b/src/daemon/common/cgroup/cgroup_v1.c
+@@ -12,14 +12,20 @@
+  * Create: 2023-03-29
+  * Description: provide cgroup v1 functions
+  ******************************************************************************/
++#ifndef _GNU_SOURCE
++#define _GNU_SOURCE
++#endif
++
+ #include "cgroup.h"
+ 
+ #include <stdio.h>
+ #include <stdlib.h>
++#include <sys/eventfd.h>
+ 
+ #include "utils.h"
+ #include "sysinfo.h"
+ #include "err_msg.h"
++#include "events_sender_api.h"
+ 
+ #define CGROUP_HUGETLB_LIMIT "hugetlb.%s.limit_in_bytes"
+ #define CGROUP_MOUNT_PATH_PREFIX "/sys/fs/cgroup/"
+@@ -1045,6 +1051,159 @@ static char *common_get_cgroup_path(const char *path, const char *subsystem)
+     return res;
+ }
+ 
++static bool oom_cb_cgroup_v1(int fd, void *cbdata)
++{
++    cgroup_oom_handler_info_t *info = (cgroup_oom_handler_info_t *)cbdata;
++    /* Try to read cgroup.event_control and known if the cgroup was removed
++     * if the cgroup was removed and only one event received,
++     * we know that it is a cgroup removal event rather than an oom event
++     */
++    bool cgroup_removed = false;
++    if (info == NULL) {
++        ERROR("Invalide callback data");
++        return CGROUP_OOM_HANDLE_CLOSE;
++    }
++
++    if (access(info->cgroup_memory_event_path, F_OK) < 0) {
++        DEBUG("Cgroup event path was removed");
++        cgroup_removed = true;
++    }
++
++    uint64_t event_count;
++    ssize_t num_read = util_read_nointr(fd, &event_count, sizeof(uint64_t));
++    if (num_read < 0) {
++        ERROR("Failed to read oom event from eventfd");
++        return CGROUP_OOM_HANDLE_CLOSE;
++    }
++
++    if (num_read == 0) {
++        return CGROUP_OOM_HANDLE_CLOSE;
++    }
++
++    if (num_read != sizeof(uint64_t)) {
++        ERROR("Failed to read full oom event from eventfd");
++        return CGROUP_OOM_HANDLE_CLOSE;
++    }
++
++    if (event_count == 0) {
++        ERROR("Unexpected event count when reading for oom event");
++        return CGROUP_OOM_HANDLE_CLOSE;
++    }
++
++    if (event_count == 1 && cgroup_removed) {
++        return CGROUP_OOM_HANDLE_CLOSE;
++    }
++
++    INFO("OOM event detected");
++    (void)isulad_monitor_send_container_event(info->name, OOM, -1, 0, NULL, NULL);
++
++    return CGROUP_OOM_HANDLE_CLOSE;
++}
++
++static char *get_memory_cgroup_path_v1(const char *cgroup_path)
++{
++    int nret = 0;
++    __isula_auto_free char *converted_cgroup_path = NULL;
++    __isula_auto_free char *mnt = NULL;
++    __isula_auto_free char *root = NULL;
++    char fpath[PATH_MAX] = { 0 };
++
++    converted_cgroup_path = common_convert_cgroup_path(cgroup_path);
++    if (converted_cgroup_path == NULL) {
++        ERROR("Failed to transfer cgroup path");
++        return NULL;
++    }
++
++    nret = get_cgroup_mnt_and_root_path_v1("memory", &mnt, &root);
++    if (nret != 0 || mnt == NULL || root == NULL) {
++        ERROR("Can not find cgroup mnt and root path for subsystem 'memory'");
++        return NULL;
++    }
++
++    // When iSulad is run inside docker, the root is based of the host cgroup.
++    // Replace root to "/"
++    if (strncmp(root, "/docker/", strlen("/docker/")) == 0) {
++        root[1] = '\0';
++    }
++
++    nret = snprintf(fpath, sizeof(fpath), "%s/%s", mnt, root);
++    if (nret < 0 || (size_t)nret >= sizeof(fpath)) {
++        ERROR("Failed to print string");
++        return NULL;
++    }
++
++    return util_path_join(fpath, converted_cgroup_path);
++}
++
++static cgroup_oom_handler_info_t *get_cgroup_oom_handler_v1(int fd, const char *name, const char *cgroup_path, const char *exit_fifo)
++{
++    __isula_auto_free char *memory_cgroup_path = NULL;
++    __isula_auto_free char *memory_cgroup_oom_control_path = NULL;
++    __isula_auto_free char *data = NULL;
++    __isula_auto_close int cgroup_event_control_fd = -1;
++    if (name == NULL || cgroup_path == NULL || exit_fifo == NULL) {
++        ERROR("Invalid arguments");
++        return NULL;
++    }
++
++    cgroup_oom_handler_info_t *info = util_common_calloc_s(sizeof(cgroup_oom_handler_info_t));
++    if (info == NULL) {
++        ERROR("Out of memory");
++        return NULL;
++    }
++    info->name = util_strdup_s(name);
++    info->cgroup_file_fd =  -1;
++    info->oom_event_fd = -1;
++    info->oom_event_handler = oom_cb_cgroup_v1;
++
++    memory_cgroup_path = get_memory_cgroup_path_v1(cgroup_path);
++    if (memory_cgroup_path == NULL) {
++        ERROR("Failed to get memory cgroup path");
++        goto cleanup;
++    }
++
++    info->cgroup_memory_event_path = util_path_join(memory_cgroup_path, "cgroup.event_control");
++    if (info->cgroup_memory_event_path == NULL) {
++        ERROR("Failed to join memory cgroup file path");
++        goto cleanup;
++    }
++
++    cgroup_event_control_fd = util_open(info->cgroup_memory_event_path, O_WRONLY | O_CLOEXEC, 0);
++    if (cgroup_event_control_fd < 0) {
++        ERROR("Failed to open %s", info->cgroup_memory_event_path);
++        goto cleanup;
++    }
++
++    memory_cgroup_oom_control_path = util_path_join(memory_cgroup_path, "memory.oom_control");
++    if (memory_cgroup_oom_control_path == NULL) {
++        ERROR("Failed to join memory cgroup file path");
++        goto cleanup;
++    }
++
++    info->cgroup_file_fd = util_open(memory_cgroup_oom_control_path, O_RDONLY | O_CLOEXEC, 0);
++    if (info->cgroup_file_fd < 0) {
++        ERROR("Failed to open %s", memory_cgroup_oom_control_path);
++        goto cleanup;
++    }
++
++    info->oom_event_fd = eventfd(0, EFD_CLOEXEC);
++    if (info->oom_event_fd < 0) {
++        ERROR("Failed to create oom eventfd");
++        goto cleanup;
++    }
++
++    if (asprintf(&data, "%d %d", info->oom_event_fd, info->cgroup_file_fd) < 0 ||
++        util_write_nointr(cgroup_event_control_fd, data, strlen(data)) < 0) {
++        ERROR("Failed to write to cgroup.event_control");
++        goto cleanup;
++    }
++
++    return info;
++cleanup:
++    common_free_cgroup_oom_handler_info(info);
++    return NULL;
++}
++
+ char *get_init_cgroup_path_v1(const char *subsystem)
+ {
+     return common_get_cgroup_path("/proc/1/cgroup", subsystem);
+@@ -1071,5 +1230,6 @@ int cgroup_v1_ops_init(cgroup_ops *ops)
+     ops->get_cgroup_mnt_and_root_path = get_cgroup_mnt_and_root_path_v1;
+     ops->get_init_cgroup_path = get_init_cgroup_path_v1;
+     ops->get_own_cgroup_path = get_own_cgroup_v1;
++    ops->get_cgroup_oom_handler = get_cgroup_oom_handler_v1;
+     return 0;
+ }
+\ No newline at end of file
+diff --git a/src/daemon/common/cgroup/cgroup_v2.c b/src/daemon/common/cgroup/cgroup_v2.c
+index 65cf90d8..a36258f0 100644
+--- a/src/daemon/common/cgroup/cgroup_v2.c
++++ b/src/daemon/common/cgroup/cgroup_v2.c
+@@ -17,12 +17,14 @@
+ #include <stdio.h>
+ #include <stdlib.h>
+ #include <sys/stat.h>
++#include <sys/inotify.h>
+ 
+ #include <isula_libutils/auto_cleanup.h>
+ 
+ #include "utils.h"
+ #include "path.h"
+ #include "sysinfo.h"
++#include "events_sender_api.h"
+ 
+ // Cgroup V2 Item Definition
+ #define CGROUP2_CPU_WEIGHT "cpu.weight"
+@@ -408,10 +410,143 @@ static int get_cgroup_metrics_v2(const char *cgroup_path, cgroup_metrics_t *cgro
+ 
+ static int get_cgroup_mnt_and_root_v2(const char *subsystem, char **mountpoint, char **root)
+ {
+-    *mountpoint = util_strdup_s(CGROUP_ISULAD_PATH);
++    if (mountpoint != NULL) {
++        *mountpoint = util_strdup_s(CGROUP_ISULAD_PATH);
++    }
+     return 0;
+ }
+ 
++static bool oom_cb_cgroup_v2(int fd, void *cbdata)
++{
++    const size_t events_size = sizeof(struct inotify_event) + NAME_MAX + 1;
++    char events[events_size];
++    cgroup_oom_handler_info_t *info = (cgroup_oom_handler_info_t *)cbdata;
++
++    if (info == NULL) {
++        ERROR("Invalid callback data");
++        return CGROUP_OOM_HANDLE_CLOSE;
++    }
++
++    ssize_t num_read = util_read_nointr(fd, &events, events_size);
++    if (num_read < 0) {
++        ERROR("Failed to read oom event from eventfd in v2");
++        return CGROUP_OOM_HANDLE_CLOSE;
++    }
++
++    if (((struct inotify_event *)events)->mask & ( IN_DELETE | IN_DELETE_SELF)) {
++        return CGROUP_OOM_HANDLE_CLOSE;
++    }
++
++    __isula_auto_file FILE *fp = fopen(info->cgroup_memory_event_path, "re");
++    if (fp == NULL) {
++        ERROR("Failed to open cgroups file: %s", info->cgroup_memory_event_path);
++        return CGROUP_OOM_HANDLE_CLOSE;
++    }
++
++    __isula_auto_free char *line = NULL;
++    size_t len = 0;
++    ssize_t read;
++    while ((read = getline(&line, &len, fp)) != -1) {
++        int count;
++        const char *oom_str = "oom ";
++        const char *oom_kill_str = "oom_kill ";
++        const int oom_len = strlen(oom_str), oom_kill_len = strlen(oom_kill_str);
++
++        if (read >= oom_kill_len + 2 && memcmp(line, oom_kill_str, oom_kill_len) == 0) {
++            len = oom_kill_len;
++        } else if (read >= oom_len + 2 && memcmp(line, oom_str, oom_len) == 0) {
++            len = oom_len;
++        } else {
++            continue;
++        }
++
++        // to make use of util_safe_int, it requires it ends with '\0'
++        line[strcspn(line, "\n")] = '\0';
++        if (util_safe_int(&line[len], &count) < 0) {
++            ERROR("Failed to parse: %s", &line[len]);
++            continue;
++        }
++
++        if (count == 0) {
++            continue;
++        }
++
++        INFO("OOM event detected in cgroup v2");
++        (void)isulad_monitor_send_container_event(info->name, OOM, -1, 0, NULL, NULL);
++
++        return CGROUP_OOM_HANDLE_CLOSE;
++    }
++
++    return CGROUP_OOM_HANDLE_CONTINUE;
++}
++
++static char *get_real_cgroup_path_v2(const char *cgroup_path)
++{
++    __isula_auto_free char *converted_cgroup_path = NULL;
++    converted_cgroup_path = common_convert_cgroup_path(cgroup_path);
++    if (converted_cgroup_path == NULL) {
++        ERROR("Failed to convert cgroup path");
++        return NULL;
++    }
++
++    return util_path_join(CGROUP_MOUNTPOINT, converted_cgroup_path);
++}
++
++cgroup_oom_handler_info_t *get_cgroup_oom_handler_v2(int fd, const char *name, const char *cgroup_path, const char *exit_fifo)
++{
++    __isula_auto_free char *real_cgroup_path = NULL;
++    if (name == NULL || cgroup_path == NULL || exit_fifo == NULL) {
++        ERROR("Invalid arguments");
++        return NULL;
++    }
++
++    cgroup_oom_handler_info_t *info = util_common_calloc_s(sizeof(cgroup_oom_handler_info_t));
++    if (info == NULL) {
++        ERROR("Out of memory");
++        return NULL;
++    }
++
++    info->name = util_strdup_s(name);
++    info->oom_event_fd = -1;
++    info->cgroup_file_fd = -1;
++    info->oom_event_handler = oom_cb_cgroup_v2;
++
++    real_cgroup_path = get_real_cgroup_path_v2(cgroup_path);
++    if (real_cgroup_path == NULL) {
++        ERROR("Failed to transfer cgroup path: %s", cgroup_path);
++        goto cleanup;
++    }
++
++    info->cgroup_memory_event_path = util_path_join(real_cgroup_path, "memory.events");
++    if (info->cgroup_memory_event_path == NULL) {
++        ERROR("Failed to join path");
++        goto cleanup;
++    }
++
++    if ((info->oom_event_fd = inotify_init()) < 0) {
++        ERROR("Failed to init inotify fd");
++        goto cleanup;
++    }
++
++    if (inotify_add_watch(info->oom_event_fd, info->cgroup_memory_event_path, IN_MODIFY) < 0) {
++        ERROR("Failed to watch inotify fd for %s", info->cgroup_memory_event_path);
++        goto cleanup;
++    }
++
++    // watch exit fifo for container exit, so we can close the inotify fd
++    // because inotify cannot watch cgroup file delete event
++    if (inotify_add_watch(info->oom_event_fd, exit_fifo, IN_DELETE | IN_DELETE_SELF) < 0) {
++        ERROR("Failed to watch inotify fd for %s", exit_fifo);
++        goto cleanup;
++    }
++
++    return info;
++
++cleanup:
++    common_free_cgroup_oom_handler_info(info);
++    return NULL;
++}
++
+ int get_cgroup_version_v2()
+ {
+     return CGROUP_VERSION_2;
+@@ -426,5 +561,6 @@ int cgroup_v2_ops_init(cgroup_ops *ops)
+     ops->get_cgroup_info = get_cgroup_info_v2;
+     ops->get_cgroup_metrics = get_cgroup_metrics_v2;
+     ops->get_cgroup_mnt_and_root_path = get_cgroup_mnt_and_root_v2;
++    ops->get_cgroup_oom_handler = get_cgroup_oom_handler_v2;
+     return 0;
+ }
+\ No newline at end of file
+diff --git a/src/daemon/entry/cri/v1/v1_cri_container_manager_service.cc b/src/daemon/entry/cri/v1/v1_cri_container_manager_service.cc
+index 47a33c2c..cac5c0ba 100644
+--- a/src/daemon/entry/cri/v1/v1_cri_container_manager_service.cc
++++ b/src/daemon/entry/cri/v1/v1_cri_container_manager_service.cc
+@@ -1055,6 +1055,9 @@ void ContainerManagerService::UpdateBaseStatusFromInspect(
+         } else { // Case 3
+             state = runtime::v1::CONTAINER_CREATED;
+         }
++        if (inspect->state->oom_killed) {
++            reason = "OOMKilled";
++        }
+         if (inspect->state->error != nullptr) {
+             message = inspect->state->error;
+         }
+diff --git a/src/daemon/modules/api/container_api.h b/src/daemon/modules/api/container_api.h
+index 43d66d64..830fd696 100644
+--- a/src/daemon/modules/api/container_api.h
++++ b/src/daemon/modules/api/container_api.h
+@@ -221,6 +221,8 @@ void container_state_set_restarting(container_state_t *s, int exit_code);
+ void container_state_set_paused(container_state_t *s);
+ void container_state_reset_paused(container_state_t *s);
+ 
++void container_state_set_oom_killed(container_state_t *s);
++
+ void container_state_set_dead(container_state_t *s);
+ 
+ void container_state_increase_restart_count(container_state_t *s);
+@@ -269,8 +271,7 @@ bool container_is_valid_state_string(const char *state);
+ 
+ void container_update_health_monitor(const char *container_id);
+ 
+-extern int container_supervisor_add_exit_monitor(int fd, const pid_ppid_info_t *pid_info, const char *name,
+-                                                 const char *runtime, bool sandbox_container);
++extern int container_supervisor_add_exit_monitor(int fd, const char *exit_fifo, const pid_ppid_info_t *pid_info, const container_t *cont);
+ 
+ extern char *container_exit_fifo_create(const char *cont_state_path);
+ 
+diff --git a/src/daemon/modules/container/container_events_handler.c b/src/daemon/modules/container/container_events_handler.c
+index b84f1ad5..109a628c 100644
+--- a/src/daemon/modules/container/container_events_handler.c
++++ b/src/daemon/modules/container/container_events_handler.c
+@@ -114,7 +114,7 @@ static int container_state_changed(container_t *cont, const struct isulad_events
+     bool has_been_manually_stopped = false;
+ 
+     /* only handle Exit event */
+-    if (events->type != EVENTS_TYPE_STOPPED1) {
++    if (events->type != EVENTS_TYPE_STOPPED1 && events->type != EVENTS_TYPE_OOM) {
+         return 0;
+     }
+ 
+@@ -187,6 +187,16 @@ static int container_state_changed(container_t *cont, const struct isulad_events
+             }
+ 
+             break;
++
++        case EVENTS_TYPE_OOM: {
++            container_lock(cont);
++            container_state_set_oom_killed(cont->state);
++            if (container_state_to_disk(cont)) {
++                WARN("Failed to save container \"%s\" to disk", id);
++            }
++            container_unlock(cont);
++            break;
++        }
+         default:
+             /* ignore garbage */
+             break;
+diff --git a/src/daemon/modules/container/container_state.c b/src/daemon/modules/container/container_state.c
+index f31959fa..452a2b26 100644
+--- a/src/daemon/modules/container/container_state.c
++++ b/src/daemon/modules/container/container_state.c
+@@ -154,6 +154,7 @@ void container_state_set_running(container_state_t *s, const pid_ppid_info_t *pi
+         state->paused = false;
+     }
+     state->exit_code = 0;
++    state->oom_killed = false;
+ 
+     if (pid_info != NULL) {
+         state->pid = pid_info->pid;
+@@ -222,6 +223,19 @@ void container_state_set_paused(container_state_t *s)
+     container_state_unlock(s);
+ }
+ 
++void container_state_set_oom_killed(container_state_t *s)
++{
++    if (s == NULL || s->state == NULL) {
++        return;
++    }
++
++    container_state_lock(s);
++
++    s->state->oom_killed = true;
++
++    container_state_unlock(s);
++}
++
+ /* state reset paused */
+ void container_state_reset_paused(container_state_t *s)
+ {
+@@ -573,6 +587,7 @@ container_inspect_state *container_state_to_inspect_state(container_state_t *s)
+     state->running = s->state->running;
+     state->paused = s->state->paused;
+     state->restarting = s->state->restarting;
++    state->oom_killed = s->state->oom_killed;
+     state->pid = s->state->pid;
+ 
+     state->exit_code = s->state->exit_code;
+diff --git a/src/daemon/modules/container/restore/restore.c b/src/daemon/modules/container/restore/restore.c
+index 76868e28..52f68d21 100644
+--- a/src/daemon/modules/container/restore/restore.c
++++ b/src/daemon/modules/container/restore/restore.c
+@@ -24,6 +24,7 @@
+ #include <isula_libutils/container_config_v2.h>
+ #include <isula_libutils/host_config.h>
+ #include <isula_libutils/log.h>
++#include <isula_libutils/auto_cleanup.h>
+ 
+ #include "isulad_config.h"
+ 
+@@ -44,6 +45,8 @@
+ #include "utils_file.h"
+ #include "utils_timestamp.h"
+ #include "id_name_manager.h"
++#include "cgroup.h"
++#include "specs_api.h"
+ 
+ /* restore supervisor */
+ static int restore_supervisor(const container_t *cont)
+@@ -55,9 +58,7 @@ static int restore_supervisor(const container_t *cont)
+     char *exit_fifo = NULL;
+     char *id = cont->common_config->id;
+     char *statepath = cont->state_path;
+-    char *runtime = cont->runtime;
+     pid_ppid_info_t pid_info = { 0 };
+-    bool sandbox_container = false;
+ 
+     nret = snprintf(container_state, sizeof(container_state), "%s/%s", statepath, id);
+     if (nret < 0 || (size_t)nret >= sizeof(container_state)) {
+@@ -91,11 +92,8 @@ static int restore_supervisor(const container_t *cont)
+     pid_info.ppid = cont->state->state->p_pid;
+     pid_info.start_time = cont->state->state->start_time;
+     pid_info.pstart_time = cont->state->state->p_start_time;
+-#ifdef ENABLE_CRI_API_V1
+-    sandbox_container = is_sandbox_container(cont->common_config->sandbox_info);
+-#endif
+ 
+-    if (container_supervisor_add_exit_monitor(exit_fifo_fd, &pid_info, id, runtime, sandbox_container)) {
++    if (container_supervisor_add_exit_monitor(exit_fifo_fd, exit_fifo, &pid_info, cont)) {
+         ERROR("Failed to add exit monitor to supervisor");
+         ret = -1;
+         goto out;
+diff --git a/src/daemon/modules/container/supervisor/supervisor.c b/src/daemon/modules/container/supervisor/supervisor.c
+index 63289283..1b7da383 100644
+--- a/src/daemon/modules/container/supervisor/supervisor.c
++++ b/src/daemon/modules/container/supervisor/supervisor.c
+@@ -41,6 +41,8 @@
+ #ifdef ENABLE_CRI_API_V1
+ #include "sandbox_ops.h"
+ #endif
++#include "cgroup.h"
++#include "specs_api.h"
+ 
+ pthread_mutex_t g_supervisor_lock = PTHREAD_MUTEX_INITIALIZER;
+ struct epoll_descr g_supervisor_descr;
+@@ -269,24 +271,52 @@ static int supervisor_exit_cb(int fd, uint32_t events, void *cbdata, struct epol
+     return EPOLL_LOOP_HANDLE_CONTINUE;
+ }
+ 
++static int oom_handle_cb(int fd, uint32_t events, void *cbdata, struct epoll_descr *descr)
++{
++    cgroup_oom_handler_info_t *oom_handler_info = (cgroup_oom_handler_info_t *)cbdata;
++    bool close_oom_handler = CGROUP_OOM_HANDLE_CLOSE;
++    // supervisor only handle one oom event, so we remove the handler directly
++    if (oom_handler_info != NULL && oom_handler_info->oom_event_handler != NULL) {
++        close_oom_handler = oom_handler_info->oom_event_handler(fd, oom_handler_info);
++    }
++
++    if (close_oom_handler == CGROUP_OOM_HANDLE_CLOSE) {
++        supervisor_handler_lock();
++        epoll_loop_del_handler(&g_supervisor_descr, fd);
++        supervisor_handler_unlock();
++
++        common_free_cgroup_oom_handler_info(oom_handler_info);
++    }
++
++    return EPOLL_LOOP_HANDLE_CONTINUE;
++}
++
+ /* supervisor add exit monitor */
+-int container_supervisor_add_exit_monitor(int fd, const pid_ppid_info_t *pid_info, const char *name,
+-                                          const char *runtime, bool sandbox_container)
++int container_supervisor_add_exit_monitor(int fd, const char *exit_fifo, const pid_ppid_info_t *pid_info, const container_t *cont)
+ {
+     int ret = 0;
+     struct supervisor_handler_data *data = NULL;
++    cgroup_oom_handler_info_t *oom_handler_info = NULL;
++    __isula_auto_free char *cgroup_path = NULL;
+ 
+     if (fd < 0) {
+         ERROR("Invalid exit fifo fd");
+         return -1;
+     }
+ 
+-    if (pid_info == NULL || name == NULL || runtime == NULL) {
++    if (pid_info == NULL || cont == NULL || cont->common_config == NULL) {
+         ERROR("Invalid input arguments");
+         close(fd);
+         return -1;
+     }
+ 
++    cgroup_path = merge_container_cgroups_path(cont->common_config->id, cont->hostconfig);
++    if (cgroup_path == NULL) {
++        ERROR("Failed to get cgroup path");
++        close(fd);
++        return -1;
++    }
++
+     data = util_common_calloc_s(sizeof(struct supervisor_handler_data));
+     if (data == NULL) {
+         ERROR("Memory out");
+@@ -295,15 +325,26 @@ int container_supervisor_add_exit_monitor(int fd, const pid_ppid_info_t *pid_inf
+     }
+ 
+     data->fd = fd;
+-    data->name = util_strdup_s(name);
+-    data->runtime = util_strdup_s(runtime);
+-    data->is_sandbox_container = sandbox_container;
++    data->name = util_strdup_s(cont->common_config->id);
++    data->runtime = util_strdup_s(cont->runtime);
++#ifdef ENABLE_CRI_API_V1
++    data->is_sandbox_container = is_sandbox_container(cont->common_config->sandbox_info);
++#endif
+     data->pid_info.pid = pid_info->pid;
+     data->pid_info.start_time = pid_info->start_time;
+     data->pid_info.ppid = pid_info->ppid;
+     data->pid_info.pstart_time = pid_info->pstart_time;
++    oom_handler_info = common_get_cgroup_oom_handler(fd, cont->common_config->id, cgroup_path, exit_fifo);
+ 
+     supervisor_handler_lock();
++    if (oom_handler_info != NULL) {
++        ret = epoll_loop_add_handler(&g_supervisor_descr, oom_handler_info->oom_event_fd, oom_handle_cb, oom_handler_info);
++        if (ret != 0) {
++            ERROR("Failed to add handler for oom event");
++            goto err;
++        }
++    }
++
+     ret = epoll_loop_add_handler(&g_supervisor_descr, fd, supervisor_exit_cb, data);
+     if (ret != 0) {
+         ERROR("Failed to add handler for exit fifo");
+@@ -314,6 +355,7 @@ int container_supervisor_add_exit_monitor(int fd, const pid_ppid_info_t *pid_inf
+ 
+ err:
+     supervisor_handler_data_free(data);
++    common_free_cgroup_oom_handler_info(oom_handler_info);
+ out:
+     supervisor_handler_unlock();
+     return ret;
+diff --git a/src/daemon/modules/events/collector.c b/src/daemon/modules/events/collector.c
+index fb4a7fea..af688742 100644
+--- a/src/daemon/modules/events/collector.c
++++ b/src/daemon/modules/events/collector.c
+@@ -133,6 +133,9 @@ static container_events_type_t lcrsta2Evetype(int value)
+         case THAWED:
+             et = EVENTS_TYPE_THAWED;
+             break;
++        case OOM:
++            et = EVENTS_TYPE_OOM;
++            break;
+         default:
+             et = EVENTS_TYPE_EXIT;
+             break;
+@@ -822,8 +825,8 @@ static int post_event_to_events_hander(const struct isulad_events_format *events
+         return -1;
+     }
+ 
+-    /* only post STOPPED event to events_hander */
+-    if (events->type != EVENTS_TYPE_STOPPED1) {
++    /* only post STOPPED event and OOM event to events_hander */
++    if (events->type != EVENTS_TYPE_STOPPED1 && events->type != EVENTS_TYPE_OOM) {
+         return 0;
+     }
+ 
+diff --git a/src/daemon/modules/service/service_container.c b/src/daemon/modules/service/service_container.c
+index a8090d5a..eb7ce4f4 100644
+--- a/src/daemon/modules/service/service_container.c
++++ b/src/daemon/modules/service/service_container.c
+@@ -275,14 +275,13 @@ static void clean_resources_on_failure(const container_t *cont, const char *engi
+     return;
+ }
+ 
+-static int do_post_start_on_success(const char *id, const char *runtime, bool sandbox_container,
+-                                    const char *pidfile, int exit_fifo_fd,
+-                                    const pid_ppid_info_t *pid_info)
++static int do_post_start_on_success(container_t *cont, int exit_fifo_fd,
++                                    const char *exit_fifo, const pid_ppid_info_t *pid_info)
+ {
+     int ret = 0;
+ 
+     // exit_fifo_fd was closed in container_supervisor_add_exit_monitor
+-    if (container_supervisor_add_exit_monitor(exit_fifo_fd, pid_info, id, runtime, sandbox_container)) {
++    if (container_supervisor_add_exit_monitor(exit_fifo_fd, exit_fifo, pid_info, cont)) {
+         ERROR("Failed to add exit monitor to supervisor");
+         ret = -1;
+     }
+@@ -750,7 +749,6 @@ static int do_start_container(container_t *cont, const char *console_fifos[], bo
+     oci_runtime_spec *oci_spec = NULL;
+     rt_create_params_t create_params = { 0 };
+     rt_start_params_t start_params = { 0 };
+-    bool sandbox_container = false;
+ 
+     nret = snprintf(bundle, sizeof(bundle), "%s/%s", cont->root_path, id);
+     if (nret < 0 || (size_t)nret >= sizeof(bundle)) {
+@@ -899,7 +897,6 @@ static int do_start_container(container_t *cont, const char *console_fifos[], bo
+     if (cont->common_config->sandbox_info != NULL) {
+         create_params.task_addr = cont->common_config->sandbox_info->task_address;
+     }
+-    sandbox_container = is_sandbox_container(cont->common_config->sandbox_info);
+ #endif
+ 
+     if (runtime_create(id, runtime, &create_params) != 0) {
+@@ -924,7 +921,7 @@ static int do_start_container(container_t *cont, const char *console_fifos[], bo
+ 
+     ret = runtime_start(id, runtime, &start_params, pid_info);
+     if (ret == 0) {
+-        if (do_post_start_on_success(id, runtime, sandbox_container, pidfile, exit_fifo_fd, pid_info) != 0) {
++        if (do_post_start_on_success(cont, exit_fifo_fd, exit_fifo, pid_info) != 0) {
+             ERROR("Failed to do post start on runtime start success");
+             ret = -1;
+             goto clean_resources;
+-- 
+2.25.1
+

0036-add-ci-cases-for-oomkilled-monitor.patch

@@ -0,0 +1,279 @@
+From 0111a575f829b946068dcb11286f0d84363cfc3d Mon Sep 17 00:00:00 2001
+From: jikai <jikai11@huawei.com>
+Date: Thu, 28 Mar 2024 12:51:53 +0000
+Subject: [PATCH 036/149] add ci cases for oomkilled monitor
+
+Signed-off-by: jikai <jikai11@huawei.com>
+---
+ CI/test_cases/container_cases/inspect.sh      | 14 ++++++++
+ test/cgroup/cpu/CMakeLists.txt                |  2 ++
+ .../image/oci/oci_config_merge/CMakeLists.txt |  1 +
+ test/image/oci/registry/CMakeLists.txt        |  1 +
+ test/mocks/sender_mock.cc                     | 34 +++++++++++++++++++
+ test/mocks/sender_mock.h                      | 31 +++++++++++++++++
+ test/network/network_ns/CMakeLists.txt        |  1 +
+ test/runtime/isula/CMakeLists.txt             |  1 +
+ test/runtime/lcr/CMakeLists.txt               |  1 +
+ test/specs/specs/CMakeLists.txt               |  1 +
+ test/specs/specs_extend/CMakeLists.txt        |  1 +
+ test/specs/verify/CMakeLists.txt              |  1 +
+ test/volume/CMakeLists.txt                    |  3 +-
+ 13 files changed, 91 insertions(+), 1 deletion(-)
+ create mode 100644 test/mocks/sender_mock.cc
+ create mode 100644 test/mocks/sender_mock.h
+
+diff --git a/CI/test_cases/container_cases/inspect.sh b/CI/test_cases/container_cases/inspect.sh
+index cde9ea1f..b4f4a785 100755
+--- a/CI/test_cases/container_cases/inspect.sh
++++ b/CI/test_cases/container_cases/inspect.sh
+@@ -27,6 +27,7 @@ function test_inspect_spec()
+ {
+     local ret=0
+     local image="busybox"
++    local ubuntu_image="ubuntu"
+     local test="container inspect test => (${FUNCNAME[@]})"
+ 
+     msg_info "${test} starting..."
+@@ -37,6 +38,12 @@ function test_inspect_spec()
+     isula images | grep busybox
+     [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - missing list image: ${image}" && ((ret++))
+ 
++    isula pull ${ubuntu_image}
++    [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - failed to pull image: ${ubuntu_image}" && return ${FAILURE}
++
++    isula images | grep ubuntu
++    [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - missing list image: ${ubuntu_image}" && ((ret++))
++
+     containername=test_inspect
+ 
+     isula create --name $containername --ipc host --pid host --uts host --restart=on-failure:10 --hook-spec ${test_data_path}/test-hookspec.json --cpu-shares 100 --memory 5MB --memory-reservation 4MB --cpu-period 1000000 --cpu-quota 200000  --cpuset-cpus 1 --cpuset-mems 0 --kernel-memory 50M --pids-limit=10000 --volume /home:/root --env a=1 $image /bin/sh ls
+@@ -139,6 +146,13 @@ function test_inspect_spec()
+ 
+     isula rm -f $containername
+ 
++    isula run -it -m 4m --name $containername $ubuntu_image perl -e 'for ($i = 0; $i < 100000000; $i++) { $a .= " " x 1024 }'
++
++    isula inspect -f "{{json .State.OOMKilled}} {{.Name}}" $containername 2>&1 | sed -n '1p' | grep "true"
++    [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - failed to check container with image: ${ubuntu_image}" && ((ret++))
++
++    isula rm -f $containername
++
+     msg_info "${test} finished with return ${ret}..."
+     return ${ret}
+ }
+diff --git a/test/cgroup/cpu/CMakeLists.txt b/test/cgroup/cpu/CMakeLists.txt
+index 30bfc417..9c3cfa12 100644
+--- a/test/cgroup/cpu/CMakeLists.txt
++++ b/test/cgroup/cpu/CMakeLists.txt
+@@ -13,6 +13,7 @@ add_executable(${EXE}
+     ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/cmd/command_parser.c
+     ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/daemon/config/daemon_arguments.c
+     ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/daemon/config/isulad_config.c
++    ${CMAKE_CURRENT_SOURCE_DIR}/../../../test/mocks/sender_mock.cc
+     cgroup_cpu_ut.cc)
+ 
+ target_include_directories(${EXE} PUBLIC
+@@ -23,6 +24,7 @@ target_include_directories(${EXE} PUBLIC
+     ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/daemon/config
+     ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/daemon/common
+     ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/daemon/common/cgroup
++    ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/daemon/modules/api
+     ${CMAKE_BINARY_DIR}/conf
+     ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/config
+     ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/cmd
+diff --git a/test/image/oci/oci_config_merge/CMakeLists.txt b/test/image/oci/oci_config_merge/CMakeLists.txt
+index d13ec738..ffd3999d 100644
+--- a/test/image/oci/oci_config_merge/CMakeLists.txt
++++ b/test/image/oci/oci_config_merge/CMakeLists.txt
+@@ -35,6 +35,7 @@ add_executable(${EXE}
+     ${CMAKE_CURRENT_SOURCE_DIR}/../../../../test/mocks/containers_store_mock.cc
+     ${CMAKE_CURRENT_SOURCE_DIR}/../../../../test/mocks/namespace_mock.cc
+     ${CMAKE_CURRENT_SOURCE_DIR}/../../../../test/mocks/container_unix_mock.cc
++    ${CMAKE_CURRENT_SOURCE_DIR}/../../../../test/mocks/sender_mock.cc
+     ${CMAKE_CURRENT_SOURCE_DIR}/../../../../src/daemon/modules/spec/parse_volume.c
+     ${CMAKE_CURRENT_SOURCE_DIR}/../../../../src/daemon/modules/spec/specs.c
+     ${CMAKE_CURRENT_SOURCE_DIR}/../../../../src/daemon/modules/spec/parse_volume.c
+diff --git a/test/image/oci/registry/CMakeLists.txt b/test/image/oci/registry/CMakeLists.txt
+index 5b5bc3f5..6166c2d0 100644
+--- a/test/image/oci/registry/CMakeLists.txt
++++ b/test/image/oci/registry/CMakeLists.txt
+@@ -44,6 +44,7 @@ add_executable(${EXE}
+     ${CMAKE_CURRENT_SOURCE_DIR}/../../../mocks/storage_mock.cc
+     ${CMAKE_CURRENT_SOURCE_DIR}/../../../mocks/oci_image_mock.cc
+     ${CMAKE_CURRENT_SOURCE_DIR}/../../../mocks/http_mock.cc
++    ${CMAKE_CURRENT_SOURCE_DIR}/../../../mocks/sender_mock.cc
+     registry_ut.cc)
+ 
+ target_include_directories(${EXE} PUBLIC
+diff --git a/test/mocks/sender_mock.cc b/test/mocks/sender_mock.cc
+new file mode 100644
+index 00000000..26028d7f
+--- /dev/null
++++ b/test/mocks/sender_mock.cc
+@@ -0,0 +1,34 @@
++/******************************************************************************
++ * Copyright (c) Huawei Technologies Co., Ltd. 2020. All rights reserved.
++ * iSulad licensed under the Mulan PSL v2.
++ * You can use this software according to the terms and conditions of the Mulan PSL v2.
++ * You may obtain a copy of Mulan PSL v2 at:
++ *     http://license.coscl.org.cn/MulanPSL2
++ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR
++ * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR
++ * PURPOSE.
++ * See the Mulan PSL v2 for more details.
++ * Author: jikai
++ * Create: 2024-03-29
++ * Description: provide collector mock
++ ******************************************************************************/
++
++#include "sender_mock.h"
++
++namespace {
++MockEventSender *g_sender_mock = nullptr;
++}
++
++void MockEventSender_SetMock(MockEventSender *mock)
++{
++    g_sender_mock = mock;
++}
++
++int isulad_monitor_send_container_event(const char *name, runtime_state_t state, int pid, int exit_code,
++                                        const char *args, const char *extra_annations)
++{
++    if (g_sender_mock != nullptr) {
++        return g_sender_mock->IsuladMonitorEventSendContainerEvent(name, state, pid, exit_code, args, extra_annations);
++    }
++    return 0;
++}
+diff --git a/test/mocks/sender_mock.h b/test/mocks/sender_mock.h
+new file mode 100644
+index 00000000..f4fe75f0
+--- /dev/null
++++ b/test/mocks/sender_mock.h
+@@ -0,0 +1,31 @@
++/******************************************************************************
++ * Copyright (c) Huawei Technologies Co., Ltd. 2020. All rights reserved.
++ * iSulad licensed under the Mulan PSL v2.
++ * You can use this software according to the terms and conditions of the Mulan PSL v2.
++ * You may obtain a copy of Mulan PSL v2 at:
++ *     http://license.coscl.org.cn/MulanPSL2
++ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR
++ * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR
++ * PURPOSE.
++ * See the Mulan PSL v2 for more details.
++ * Author: jikai
++ * Create: 2024-03-30
++ * Description: provide sender mock
++ ******************************************************************************/
++
++#ifndef _ISULAD_TEST_MOCKS_SENDER_MOCK_H
++#define _ISULAD_TEST_MOCKS_SENDER_MOCK_H
++
++#include <gmock/gmock.h>
++#include "events_sender_api.h"
++
++class MockEventSender {
++public:
++    MOCK_METHOD6(IsuladMonitorEventSendContainerEvent, int(const char *name, runtime_state_t state, int pid, int exit_code,
++                                                       const char *args, const char *extra_annations));
++};
++
++void MockEventSender_SetMock(MockEventSender *mock);
++
++#endif
++
+diff --git a/test/network/network_ns/CMakeLists.txt b/test/network/network_ns/CMakeLists.txt
+index 71b8039d..6f3f36a0 100644
+--- a/test/network/network_ns/CMakeLists.txt
++++ b/test/network/network_ns/CMakeLists.txt
+@@ -43,6 +43,7 @@ add_executable(${EXE}
+     ${CMAKE_CURRENT_SOURCE_DIR}/../../../test/mocks/selinux_label_mock.cc
+     ${CMAKE_CURRENT_SOURCE_DIR}/../../../test/mocks/isulad_config_mock.cc
+     ${CMAKE_CURRENT_SOURCE_DIR}/../../../test/mocks/storage_mock.cc
++    ${CMAKE_CURRENT_SOURCE_DIR}/../../../test/mocks/sender_mock.cc
+ network_ns_ut.cc)
+ 
+ target_include_directories(${EXE} PUBLIC
+diff --git a/test/runtime/isula/CMakeLists.txt b/test/runtime/isula/CMakeLists.txt
+index c1f0a5cc..15636623 100644
+--- a/test/runtime/isula/CMakeLists.txt
++++ b/test/runtime/isula/CMakeLists.txt
+@@ -31,6 +31,7 @@ add_executable(${EXE}
+     ${CMAKE_CURRENT_SOURCE_DIR}/../../../test/mocks/engine_mock.cc
+     ${CMAKE_CURRENT_SOURCE_DIR}/../../../test/mocks/isulad_config_mock.cc
+     ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/daemon/modules/runtime/isula/isula_rt_ops.c
++    ${CMAKE_CURRENT_SOURCE_DIR}/../../../test/mocks/sender_mock.cc
+     isula_rt_ops_ut.cc)
+ 
+ target_include_directories(${EXE} PUBLIC
+diff --git a/test/runtime/lcr/CMakeLists.txt b/test/runtime/lcr/CMakeLists.txt
+index c3b93d67..5b2ed11a 100644
+--- a/test/runtime/lcr/CMakeLists.txt
++++ b/test/runtime/lcr/CMakeLists.txt
+@@ -29,6 +29,7 @@ add_executable(${EXE}
+     ${CMAKE_CURRENT_SOURCE_DIR}/../../../test/mocks/namespace_mock.cc
+     ${CMAKE_CURRENT_SOURCE_DIR}/../../../test/mocks/container_unix_mock.cc
+     ${CMAKE_CURRENT_SOURCE_DIR}/../../../test/mocks/engine_mock.cc
++    ${CMAKE_CURRENT_SOURCE_DIR}/../../../test/mocks/sender_mock.cc
+     ${CMAKE_CURRENT_SOURCE_DIR}/../../../test/mocks/isulad_config_mock.cc
+     ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/daemon/modules/runtime/engines/lcr/lcr_rt_ops.c
+     lcr_rt_ops_ut.cc)
+diff --git a/test/specs/specs/CMakeLists.txt b/test/specs/specs/CMakeLists.txt
+index 45f688f9..12c11f51 100644
+--- a/test/specs/specs/CMakeLists.txt
++++ b/test/specs/specs/CMakeLists.txt
+@@ -43,6 +43,7 @@ add_executable(${EXE}
+     ${CMAKE_CURRENT_SOURCE_DIR}/../../../test/mocks/isulad_config_mock.cc
+     ${CMAKE_CURRENT_SOURCE_DIR}/../../../test/mocks/storage_mock.cc
+     ${CMAKE_CURRENT_SOURCE_DIR}/../../../test/mocks/image_mock.cc
++    ${CMAKE_CURRENT_SOURCE_DIR}/../../../test/mocks/sender_mock.cc
+     specs_ut.cc)
+ 
+ target_include_directories(${EXE} PUBLIC
+diff --git a/test/specs/specs_extend/CMakeLists.txt b/test/specs/specs_extend/CMakeLists.txt
+index 1b737089..2fd37e1c 100644
+--- a/test/specs/specs_extend/CMakeLists.txt
++++ b/test/specs/specs_extend/CMakeLists.txt
+@@ -43,6 +43,7 @@ add_executable(${EXE}
+     ${CMAKE_CURRENT_SOURCE_DIR}/../../../test/mocks/isulad_config_mock.cc
+     ${CMAKE_CURRENT_SOURCE_DIR}/../../../test/mocks/storage_mock.cc
+     ${CMAKE_CURRENT_SOURCE_DIR}/../../../test/mocks/image_mock.cc
++    ${CMAKE_CURRENT_SOURCE_DIR}/../../../test/mocks/sender_mock.cc
+     specs_extend_ut.cc)
+ 
+ target_include_directories(${EXE} PUBLIC
+diff --git a/test/specs/verify/CMakeLists.txt b/test/specs/verify/CMakeLists.txt
+index b0602127..7f000cd1 100644
+--- a/test/specs/verify/CMakeLists.txt
++++ b/test/specs/verify/CMakeLists.txt
+@@ -38,6 +38,7 @@ add_executable(${EXE}
+     ${CMAKE_CURRENT_SOURCE_DIR}/../../../test/mocks/storage_mock.cc
+     ${CMAKE_CURRENT_SOURCE_DIR}/../../../test/mocks/image_mock.cc
+     ${CMAKE_CURRENT_SOURCE_DIR}/../../../test/mocks/storage_mock.cc
++    ${CMAKE_CURRENT_SOURCE_DIR}/../../../test/mocks/sender_mock.cc
+     verify_ut.cc)
+ 
+ target_include_directories(${EXE} PUBLIC
+diff --git a/test/volume/CMakeLists.txt b/test/volume/CMakeLists.txt
+index 27d07330..1f9dac03 100644
+--- a/test/volume/CMakeLists.txt
++++ b/test/volume/CMakeLists.txt
+@@ -24,6 +24,7 @@ add_executable(${EXE}
+     ${CMAKE_CURRENT_SOURCE_DIR}/../../src/daemon/common/cgroup/cgroup_v1.c
+     ${CMAKE_CURRENT_SOURCE_DIR}/../../src/daemon/common/cgroup/cgroup_v2.c
+     ${CMAKE_CURRENT_SOURCE_DIR}/../../src/daemon/common/cgroup/cgroup_common.c
++    ${CMAKE_CURRENT_SOURCE_DIR}/../../test/mocks/sender_mock.cc
+     ${CMAKE_CURRENT_SOURCE_DIR}/../../src/cmd/command_parser.c
+     volume_ut.cc)
+ 
+@@ -43,6 +44,6 @@ target_include_directories(${EXE} PUBLIC
+     ${CMAKE_CURRENT_SOURCE_DIR}/../../src/utils/console
+     )
+ 
+-target_link_libraries(${EXE} ${GTEST_BOTH_LIBRARIES} ${CMAKE_THREAD_LIBS_INIT} ${ISULA_LIBUTILS_LIBRARY} -lcrypto -lyajl -lz)
++target_link_libraries(${EXE} ${GTEST_BOTH_LIBRARIES} ${GMOCK_LIBRARY} ${GMOCK_MAIN_LIBRARY} ${CMAKE_THREAD_LIBS_INIT} ${ISULA_LIBUTILS_LIBRARY} -lcrypto -lyajl -lz)
+ add_test(NAME ${EXE} COMMAND ${EXE} --gtest_output=xml:${EXE}-Results.xml)
+ set_tests_properties(${EXE} PROPERTIES TIMEOUT 120)
+-- 
+2.25.1
+

0038-add-modify-for-cgroup-v2-ci-test.patch

@@ -0,0 +1,608 @@
+From 33b26f27dd897574d73ce8654620a13edbeb947e Mon Sep 17 00:00:00 2001
+From: zhongtao <zhongtao17@huawei.com>
+Date: Tue, 2 Apr 2024 02:31:58 +1400
+Subject: [PATCH 038/149] add modify for cgroup v2 ci test
+
+Signed-off-by: zhongtao <zhongtao17@huawei.com>
+---
+ CI/test_cases/manual_cases/cgroupv2.sh | 276 +++++++++++++++----------
+ 1 file changed, 165 insertions(+), 111 deletions(-)
+
+diff --git a/CI/test_cases/manual_cases/cgroupv2.sh b/CI/test_cases/manual_cases/cgroupv2.sh
+index f8982f08..8e431688 100755
+--- a/CI/test_cases/manual_cases/cgroupv2.sh
++++ b/CI/test_cases/manual_cases/cgroupv2.sh
+@@ -31,59 +31,59 @@ function test_cgroup2_cpu()
+ 
+   if [[ -f /sys/fs/cgroup/isulad/cpu.weight ]];then
+     # min value
+-    isula run -ti --rm --cpu-shares 2 busybox cat /sys/fs/cgroup/cpu.weight | grep ^1$'\r'
++    isula run --runtime $1 -ti --rm --cpu-shares 2 busybox cat /sys/fs/cgroup/cpu.weight | grep ^1$'\r'
+     [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - cgroup2 cpu.weight min value failed" && ((ret++))
+ 
+     # max value
+-    isula run -ti --rm --cpu-shares 262144 busybox cat /sys/fs/cgroup/cpu.weight | grep ^10000$'\r'
++    isula run --runtime $1 -ti --rm --cpu-shares 262144 busybox cat /sys/fs/cgroup/cpu.weight | grep ^10000$'\r'
+     [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - cgroup2 cpu.weight max value failed" && ((ret++))
+ 
+     # invalid value
+-    isula run -ti --rm --cpu-shares -1 busybox echo hello
++    isula run --runtime $1 -ti --rm --cpu-shares -1 busybox echo hello
+     [[ $? -eq 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - cgroup2 cpu.weight -1 failed" && ((ret++))
+ 
+     # default value
+-    isula run -ti --rm --cpu-shares 0 busybox cat /sys/fs/cgroup/cpu.weight | grep ^100$'\r'
++    isula run --runtime $1 -ti --rm --cpu-shares 0 busybox cat /sys/fs/cgroup/cpu.weight | grep ^100$'\r'
+     [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - cgroup2 cpu.weight default value failed" && ((ret++))
+   fi
+ 
+   if [[ -f /sys/fs/cgroup/isulad/cpu.max ]];then
+     # normal value
+-    isula run -ti --rm --cpu-quota 50000 --cpu-period 12345 busybox cat /sys/fs/cgroup/cpu.max | grep ^"50000 12345"$'\r'
++    isula run --runtime $1 -ti --rm --cpu-quota 50000 --cpu-period 12345 busybox cat /sys/fs/cgroup/cpu.max | grep ^"50000 12345"$'\r'
+     [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - cgroup2 cpu.max normal value failed" && ((ret++))
+ 
+     # invalid min period
+-    isula run -ti --rm --cpu-quota 50000 --cpu-period 999 busybox echo hello
++    isula run --runtime $1 -ti --rm --cpu-quota 50000 --cpu-period 999 busybox echo hello
+     [[ $? -eq 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - cgroup2 cpu.max invalid min period failed" && ((ret++))
+ 
+     # invalid max period
+-    isula run -ti --rm --cpu-quota 50000 --cpu-period 1000001 busybox echo hello
++    isula run --runtime $1 -ti --rm --cpu-quota 50000 --cpu-period 1000001 busybox echo hello
+     [[ $? -eq 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - cgroup2 cpu.max invalid max period failed" && ((ret++))
+ 
+     # invalid quota
+-    isula run -ti --rm --cpu-quota 999 --cpu-period 1000000 busybox echo hello
++    isula run --runtime $1 -ti --rm --cpu-quota 999 --cpu-period 1000000 busybox echo hello
+     [[ $? -eq 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - cgroup2 cpu.max invalid quota failed" && ((ret++))
+ 
+     # default 0 quota
+-    isula run -ti --rm --cpu-quota 0 --cpu-period 1000000 busybox cat /sys/fs/cgroup/cpu.max | grep ^"max 1000000"$'\r'
++    isula run --runtime $1 -ti --rm --cpu-quota 0 --cpu-period 1000000 busybox cat /sys/fs/cgroup/cpu.max | grep ^"max 1000000"$'\r'
+     [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - cgroup2 cpu.max default 0 quota failed" && ((ret++))
+ 
+     # default -1 quota
+-    isula run -ti --rm --cpu-quota -1 --cpu-period 1000000 busybox cat /sys/fs/cgroup/cpu.max | grep ^"max 1000000"$'\r'
++    isula run --runtime $1 -ti --rm --cpu-quota -1 --cpu-period 1000000 busybox cat /sys/fs/cgroup/cpu.max | grep ^"max 1000000"$'\r'
+     [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - cgroup2 cpu.max default -1 quota failed" && ((ret++))
+ 
+     # cpus 1
+-    isula run -ti --rm --cpus 1 busybox cat /sys/fs/cgroup/cpu.max | grep ^"100000 100000"$'\r'
++    isula run --runtime $1 -ti --rm --cpus 1 busybox cat /sys/fs/cgroup/cpu.max | grep ^"100000 100000"$'\r'
+     [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - cgroup2 cpu.max cpus 1 failed" && ((ret++))
+ 
+     # cpus 0
+-    isula run -ti --rm --cpus 0 busybox cat /sys/fs/cgroup/cpu.max | grep ^"max 100000"$'\r'
++    isula run --runtime $1 -ti --rm --cpus 0 busybox cat /sys/fs/cgroup/cpu.max | grep ^"max 100000"$'\r'
+     [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - cgroup2 cpu.max cpus 0 failed" && ((ret++))
+   fi
+ 
+   if [[ -f /sys/fs/cgroup/isulad/cpuset.cpus.effective ]];then
+     # normal value
+-    isula run -tid -n cpuset --cpuset-cpus 0 --cpuset-mems 0 busybox sh
++    isula run --runtime $1 -tid -n cpuset --cpuset-cpus 0 --cpuset-mems 0 busybox sh
+     [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - cgroup2 cpuset run container failed" && ((ret++))
+ 
+     isula exec -ti cpuset cat /sys/fs/cgroup/cpuset.cpus | grep ^0$'\r'
+@@ -96,19 +96,19 @@ function test_cgroup2_cpu()
+     [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - cgroup2 cpuset remove container failed" && ((ret++))
+ 
+     # invalid cpus -1 value
+-    isula run -tid -n cpuset --cpuset-cpus -1 busybox sh
++    isula run --runtime $1 -tid -n cpuset --cpuset-cpus -1 busybox sh
+     [[ $? -eq 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - cgroup2 cpuset cpus invalid -1 failed" && ((ret++))
+ 
+     # invalid cpus 100000 value
+-    isula run -tid -n cpuset --cpuset-cpus 100000 busybox sh
++    isula run --runtime $1 -tid -n cpuset --cpuset-cpus 100000 busybox sh
+     [[ $? -eq 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - cgroup2 cpuset cpus invalid 100000 failed" && ((ret++))
+ 
+     # invalid mems -1 value
+-    isula run -tid -n cpuset --cpuset-mems -1 busybox sh
++    isula run --runtime $1 -tid -n cpuset --cpuset-mems -1 busybox sh
+     [[ $? -eq 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - cgroup2 cpuset mems invalid -1 failed" && ((ret++))
+ 
+     # invalid mems 100000 value
+-    isula run -tid -n cpuset --cpuset-mems 100000 busybox sh
++    isula run --runtime $1 -tid -n cpuset --cpuset-mems 100000 busybox sh
+     [[ $? -eq 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - cgroup2 cpuset mems invalid 100000 failed" && ((ret++))
+   fi
+ 
+@@ -121,33 +121,38 @@ function test_cgroup2_io()
+ 
+   if [[ -f "/sys/fs/cgroup/isulad/io.bfq.weight" ]];then
+     # min value
+-    isula run -ti --rm --blkio-weight 10 busybox cat "/sys/fs/cgroup/io.bfq.weight" | grep 1$'\r'
+-    [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - cgroup2 io.bfq.weight min value failed" && ((ret++))
++    if [ $1 == "lcr" ]; then
++      isula run --runtime $1 -ti --rm --blkio-weight 10 busybox cat "/sys/fs/cgroup/io.bfq.weight" | grep 1$'\r'
++      [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - cgroup2 io.bfq.weight min value failed" && ((ret++))
++    else
++      isula run --runtime $1 -ti --rm --blkio-weight 10 busybox cat "/sys/fs/cgroup/io.bfq.weight" | grep 10$'\r'
++      [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - cgroup2 io.bfq.weight min value failed" && ((ret++))
++    fi
+ 
+     # max value
+-    isula run -ti --rm --blkio-weight 1000 busybox cat "/sys/fs/cgroup/io.bfq.weight" | grep 1000$'\r'
++    isula run --runtime $1 -ti --rm --blkio-weight 1000 busybox cat "/sys/fs/cgroup/io.bfq.weight" | grep 1000$'\r'
+     [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - cgroup2 io.bfq.weight max value failed" && ((ret++))
+ 
+     # default value
+-    isula run -ti --rm --blkio-weight 0 busybox cat "/sys/fs/cgroup/io.bfq.weight" | grep 100$'\r'
++    isula run --runtime $1 -ti --rm --blkio-weight 0 busybox cat "/sys/fs/cgroup/io.bfq.weight" | grep 100$'\r'
+     [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - cgroup2 io.bfq.weight default value failed" && ((ret++))
+ 
+     # invalid value
+-    isula run -ti --rm --blkio-weight -1 busybox echo hello
++    isula run --runtime $1 -ti --rm --blkio-weight -1 busybox echo hello
+     [[ $? -eq 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - cgroup2 io.bfq.weight -1 failed" && ((ret++))
+   fi
+ 
+   if [[ -f "/sys/fs/cgroup/isulad/io.bfq.weight_device" ]];then
+     # min value
+-    isula run -ti --rm --blkio-weight-device /dev/null:10 busybox cat "/sys/fs/cgroup/io.bfq.weight_device" | grep ^"1:3 10"$'\r'
++    isula run --runtime $1 -ti --rm --blkio-weight-device /dev/null:10 busybox cat "/sys/fs/cgroup/io.bfq.weight_device" | grep ^"1:3 10"$'\r'
+     [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - cgroup2 io.bfq.weight_device max value failed" && ((ret++))
+ 
+     # max value
+-    isula run -ti --rm --blkio-weight-device /dev/null:1000 busybox cat "/sys/fs/cgroup/io.bfq.weight_device" | grep ^"1:3 10000"$'\r'
++    isula run --runtime $1 -ti --rm --blkio-weight-device /dev/null:1000 busybox cat "/sys/fs/cgroup/io.bfq.weight_device" | grep ^"1:3 10000"$'\r'
+     [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - cgroup2 io.bfq.weight_device max value failed" && ((ret++))
+ 
+     # disable weight device
+-    isula run -tid -n weight_device --rm --blkio-weight-device /dev/null:0 busybox sh
++    isula run --runtime $1 -tid -n weight_device --rm --blkio-weight-device /dev/null:0 busybox sh
+     [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - cgroup2 io.bfq.weight_device failed" && ((ret++))
+ 
+     isula exec -ti weight_device cat "/sys/fs/cgroup/io.bfq.weight_device" | grep "1:3"
+@@ -159,33 +164,43 @@ function test_cgroup2_io()
+ 
+   if [[ -f "/sys/fs/cgroup/isulad/io.weight" ]];then
+     # min value
+-    isula run -ti --rm --blkio-weight 10 busybox cat "/sys/fs/cgroup/io.weight" | grep ^"default 1"$'\r'
+-    [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - cgroup2 io.weight min value failed" && ((ret++))
++    if [ $1 == "lcr" ]; then
++      isula run --runtime $1 -ti --rm --blkio-weight 10 busybox cat "/sys/fs/cgroup/io.weight" | grep ^"default 1"$'\r'
++      [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - cgroup2 io.weight min value failed" && ((ret++))
++    else
++      isula run --runtime $1 -ti --rm --blkio-weight 10 busybox cat "/sys/fs/cgroup/io.bfq.weight" | grep ^"default 10"$'\r'
++      [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - cgroup2 io.weight min value failed" && ((ret++))
++    fi
+ 
+     # max value
+-    isula run -ti --rm --blkio-weight 1000 busybox cat "/sys/fs/cgroup/io.weight" | grep ^"default 10000"$'\r'
+-    [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - cgroup2 io.weight max value failed" && ((ret++))
++    if [ $1 == "lcr" ]; then
++      isula run --runtime $1 -ti --rm --blkio-weight 1000 busybox cat "/sys/fs/cgroup/io.weight" | grep ^"default 10000"$'\r'
++      [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - cgroup2 io.weight max value failed" && ((ret++))
++    else
++        isula run --runtime $1 -ti --rm --blkio-weight 1000 busybox cat "/sys/fs/cgroup/io.bfq.weight" | grep ^"default 1000"$'\r'
++        [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - cgroup2 io.weight max value failed" && ((ret++))
++    fi
+ 
+     # default value
+-    isula run -ti --rm --blkio-weight 0 busybox cat "/sys/fs/cgroup/io.weight" | grep ^"default 100"$'\r'
++    isula run --runtime $1 -ti --rm --blkio-weight 0 busybox cat "/sys/fs/cgroup/io.weight" | grep ^"default 100"$'\r'
+     [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - cgroup2 io.weight default value failed" && ((ret++))
+ 
+     # invalid value
+-    isula run -ti --rm --blkio-weight -1 busybox echo hello
++    isula run --runtime $1 -ti --rm --blkio-weight -1 busybox echo hello
+     [[ $? -eq 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - cgroup2 io.weight -1 failed" && ((ret++))
+   fi
+ 
+   if [[ -f "/sys/fs/cgroup/isulad/io.weight_device" ]];then
+     # min value
+-    isula run -ti --rm --blkio-weight-device /dev/null:10 busybox cat "/sys/fs/cgroup/io.weight_device" | grep ^"1:3 10"$'\r'
++    isula run --runtime $1 -ti --rm --blkio-weight-device /dev/null:10 busybox cat "/sys/fs/cgroup/io.weight_device" | grep ^"1:3 10"$'\r'
+     [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - cgroup2 io.weight max value failed" && ((ret++))
+ 
+     # max value
+-    isula run -ti --rm --blkio-weight-device /dev/null:1000 busybox cat "/sys/fs/cgroup/io.weight_device" | grep ^"1:3 10000"$'\r'
++    isula run --runtime $1 -ti --rm --blkio-weight-device /dev/null:1000 busybox cat "/sys/fs/cgroup/io.weight_device" | grep ^"1:3 10000"$'\r'
+     [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - cgroup2 io.weight max value failed" && ((ret++))
+ 
+     # disable weight device
+-    isula run -tid -n weight_device --rm --blkio-weight-device /dev/null:0 busybox sh
++    isula run --runtime $1 -tid -n weight_device --rm --blkio-weight-device /dev/null:0 busybox sh
+     [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - cgroup2 io.weight failed" && ((ret++))
+ 
+     isula exec -ti weight_device cat "/sys/fs/cgroup/io.weight_device" | grep ^"1:3"$'\r'
+@@ -197,16 +212,22 @@ function test_cgroup2_io()
+ 
+   if [[ -f /sys/fs/cgroup/isulad/io.max ]];then
+     # normal value
+-    isula run -ti --rm --device-read-bps /dev/null:1g --device-read-iops /dev/null:1000 --device-write-bps /dev/null:2g --device-write-iops /dev/null:2000 busybox cat /sys/fs/cgroup/io.max | grep ^"1:3 rbps=1073741824 wbps=2147483648 riops=1000 wiops=2000"$'\r'
++    isula run --runtime $1 -ti --rm --device-read-bps /dev/null:1g --device-read-iops /dev/null:1000 --device-write-bps /dev/null:2g --device-write-iops /dev/null:2000 busybox cat /sys/fs/cgroup/io.max | grep ^"1:3 rbps=1073741824 wbps=2147483648 riops=1000 wiops=2000"$'\r'
+     [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - cgroup2 io.max failed" && ((ret++))
+ 
+     # invalid
+-    isula run -ti --rm --device-read-bps /dev/null:-1 busybox echo hello
++    isula run --runtime $1 -ti --rm --device-read-bps /dev/null:-1 busybox echo hello
+     [[ $? -eq 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - cgroup2 io.max -1 failed" && ((ret++))
+ 
+-    # 0 is no limit
+-    isula run -ti --rm --device-read-bps /dev/null:0 --device-read-iops /dev/null:0 --device-write-bps /dev/null:0 --device-write-iops /dev/null:0 busybox echo hello
+-    [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - cgroup2 io.max 0 failed" && ((ret++))
++    if [ $1 == "lcr" ]; then
++      # 0 is no limit
++      isula run --runtime $1 -ti --rm --device-read-bps /dev/null:0 --device-read-iops /dev/null:0 --device-write-bps /dev/null:0 --device-write-iops /dev/null:0 busybox echo hello
++      [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - cgroup2 io.max 0 failed" && ((ret++))
++    else
++      # 0 is limit
++      isula run --runtime $1 -ti --rm --device-read-bps /dev/null:0 --device-read-iops /dev/null:0 --device-write-bps /dev/null:0 --device-write-iops /dev/null:0 busybox echo hello
++      [[ $? -eq 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - cgroup2 io.max 0 success" && ((ret++))
++    fi
+   fi
+ 
+   return ${ret}
+@@ -218,51 +239,51 @@ function test_cgroup2_memory()
+ 
+   if [[ -f /sys/fs/cgroup/isulad/memory.max ]];then
+     # normal value
+-    isula run -ti --rm -m 10m busybox cat /sys/fs/cgroup/memory.max | grep ^10485760$'\r'
++    isula run --runtime $1 -ti --rm -m 10m busybox cat /sys/fs/cgroup/memory.max | grep ^10485760$'\r'
+     [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - cgroup2 memory.max run container failed" && ((ret++))
+ 
+     # 0 is max
+-    isula run -ti --rm -m 0 busybox cat /sys/fs/cgroup/memory.max | grep ^max$'\r'
++    isula run --runtime $1 -ti --rm -m 0 busybox cat /sys/fs/cgroup/memory.max | grep ^max$'\r'
+     [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - cgroup2 memory.max 0 failed" && ((ret++))
+ 
+     # invalid
+-    isula run -ti --rm -m -1 busybox echo hello
++    isula run --runtime $1 -ti --rm -m -1 busybox echo hello
+     [[ $? -eq 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - cgroup2 memory.max -1 failed" && ((ret++))
+   fi
+ 
+   if [[ -f /sys/fs/cgroup/isulad/memory.low ]];then
+     # normal value
+-    isula run -ti --rm --memory-reservation 10m busybox cat /sys/fs/cgroup/memory.low | grep ^10485760$'\r'
++    isula run --runtime $1 -ti --rm --memory-reservation 10m busybox cat /sys/fs/cgroup/memory.low | grep ^10485760$'\r'
+     [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - cgroup2 memory.low normal value failed" && ((ret++))
+ 
+     # -1 is invalid
+-    isula run -ti --rm --memory-reservation -1 busybox echo hello
++    isula run --runtime $1 -ti --rm --memory-reservation -1 busybox echo hello
+     [[ $? -eq 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - cgroup2 memory.low invalid failed" && ((ret++))
+ 
+     # 0
+-    isula run -ti --rm --memory-reservation 0 busybox cat /sys/fs/cgroup/memory.low | grep ^0$'\r'
++    isula run --runtime $1 -ti --rm --memory-reservation 0 busybox cat /sys/fs/cgroup/memory.low | grep ^0$'\r'
+     [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - cgroup2 memory.low 0 failed" && ((ret++))
+   fi
+ 
+   if [[ -f /sys/fs/cgroup/isulad/memory.swap.max ]];then
+     # normal value
+-    isula run -ti --rm --memory 10m --memory-swap 20m busybox cat /sys/fs/cgroup/memory.swap.max | grep ^10485760$'\r'
++    isula run --runtime $1 -ti --rm --memory 10m --memory-swap 20m busybox cat /sys/fs/cgroup/memory.swap.max | grep ^10485760$'\r'
+     [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - cgroup2 memory.swap.max normal value failed" && ((ret++))
+ 
+     # invalid
+-    isula run -ti --rm --memory 10m --memory-swap 5m busybox echo hello
++    isula run --runtime $1 -ti --rm --memory 10m --memory-swap 5m busybox echo hello
+     [[ $? -eq 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - cgroup2 memory.swap.max invalid failed" && ((ret++))
+ 
+     # 0 is the same as memory
+-    isula run -ti --rm --memory 10m --memory-swap 0 busybox cat /sys/fs/cgroup/memory.swap.max | grep ^10485760$'\r'
++    isula run --runtime $1 -ti --rm --memory 10m --memory-swap 0 busybox cat /sys/fs/cgroup/memory.swap.max | grep ^10485760$'\r'
+     [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - cgroup2 memory.swap.max 0 failed" && ((ret++))
+ 
+     # -1 is max
+-    isula run -ti --rm --memory 10m --memory-swap -1 busybox cat /sys/fs/cgroup/memory.swap.max | grep ^max$'\r'
++    isula run --runtime $1 -ti --rm --memory 10m --memory-swap -1 busybox cat /sys/fs/cgroup/memory.swap.max | grep ^max$'\r'
+     [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - cgroup2 memory.swap.max -1 failed" && ((ret++))
+ 
+     # disable swap
+-    isula run -ti --rm --memory 10m --memory-swap 10m busybox cat /sys/fs/cgroup/memory.swap.max | grep ^0$'\r'
++    isula run --runtime $1 -ti --rm --memory 100m --memory-swap 100m busybox cat /sys/fs/cgroup/memory.swap.max | grep ^0$'\r'
+     [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - cgroup2 memory.swap.max disable swap failed" && ((ret++))
+   fi
+ 
+@@ -275,15 +296,15 @@ function test_cgroup2_pids()
+ 
+   if [[ -f /sys/fs/cgroup/isulad/pids.max ]];then
+     # normal value
+-    isula run -ti --rm --pids-limit 123456 busybox cat /sys/fs/cgroup/pids.max | grep ^123456$'\r'
++    isula run --runtime $1 -ti --rm --pids-limit 123456 busybox cat /sys/fs/cgroup/pids.max | grep ^123456$'\r'
+     [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - cgroup2 pids.max run container failed" && ((ret++))
+ 
+     # -1 is max
+-    isula run -ti --rm --pids-limit -1 busybox cat /sys/fs/cgroup/pids.max | grep ^max$'\r'
++    isula run --runtime $1 -ti --rm --pids-limit -1 busybox cat /sys/fs/cgroup/pids.max | grep ^max$'\r'
+     [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - cgroup2 pids.max run container failed" && ((ret++))
+ 
+     # 0 is max
+-    isula run -ti --rm --pids-limit 0 busybox cat /sys/fs/cgroup/pids.max | grep ^max$'\r'
++    isula run --runtime $1 -ti --rm --pids-limit 0 busybox cat /sys/fs/cgroup/pids.max | grep ^max$'\r'
+     [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - cgroup2 pids.max run container failed" && ((ret++))
+   fi
+ 
+@@ -295,7 +316,7 @@ function test_cgroup2_hugetlb()
+   local ret=0
+ 
+   if [[ -f /sys/fs/cgroup/isulad/hugetlb.2MB.max ]];then
+-    isula run -ti --rm --hugetlb-limit 2M:32M busybox cat /sys/fs/cgroup/hugetlb.2MB.max | grep ^33554432$'\r'
++    isula run --runtime $1 -ti --rm --hugetlb-limit 2M:32M busybox cat /sys/fs/cgroup/hugetlb.2MB.max | grep ^33554432$'\r'
+     [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - cgroup2 hugetlb.2M.max run container failed" && ((ret++))
+   fi
+ 
+@@ -307,7 +328,7 @@ function test_cgroup2_freeze()
+   local ret=0
+ 
+   if [[ -f /sys/fs/cgroup/isulad/cgroup.freeze ]];then
+-    isula run -tid -n freeze busybox sh
++    isula run --runtime $1 -tid -n freeze busybox sh
+     [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - cgroup2 freeze run container failed" && ((ret++))
+ 
+     isula pause freeze
+@@ -335,15 +356,15 @@ function test_cgroup2_files()
+ 
+   if [[ -f /sys/fs/cgroup/isulad/files.limit ]];then
+     # normal value
+-    isula run -ti --rm --files-limit 123 busybox cat /sys/fs/cgroup/files.limit | grep ^123$'\r'
++    isula run --runtime $1 -ti --rm --files-limit 123 busybox cat /sys/fs/cgroup/files.limit | grep ^123$'\r'
+     [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - cgroup2 files.limit run container failed" && ((ret++))
+ 
+     # -1 is max
+-    isula run -ti --rm --files-limit -1 busybox cat /sys/fs/cgroup/files.limit | grep ^max$'\r'
++    isula run --runtime $1 -ti --rm --files-limit -1 busybox cat /sys/fs/cgroup/files.limit | grep ^max$'\r'
+     [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - cgroup2 files.limit run container failed" && ((ret++))
+ 
+     # 0 is max
+-    isula run -ti --rm --files-limit 0 busybox cat /sys/fs/cgroup/files.limit | grep ^max$'\r'
++    isula run --runtime $1 -ti --rm --files-limit 0 busybox cat /sys/fs/cgroup/files.limit | grep ^max$'\r'
+     [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - cgroup2 files.limit run container failed" && ((ret++))
+   fi
+ 
+@@ -405,8 +426,13 @@ function test_cgroup2_cpu_update()
+     isula update --cpu-quota 0 --cpu-period 1000000 $cgroup2_update
+     [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - cgroup2 update cpu.max 0 quota failed" && ((ret++))
+ 
+-    isula exec -ti $cgroup2_update cat /sys/fs/cgroup/cpu.max | grep ^"max 1000000"$'\r'
+-    [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - cgroup2 update cpu.max 0 quota value not right" && ((ret++))
++    if [ $1 == "lcr" ]; then
++      isula exec -ti $cgroup2_update cat /sys/fs/cgroup/cpu.max | grep ^"max 1000000"$'\r'
++      [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - cgroup2 update cpu.max 0 quota value not right" && ((ret++))
++    else
++      isula exec -ti $cgroup2_update cat /sys/fs/cgroup/cpu.max | grep ^"50000 1000000"$'\r'
++      [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - cgroup2 update cpu.max 0 quota value not right" && ((ret++))
++    fi
+ 
+     # default -1 quota
+     isula update --cpu-quota -1 --cpu-period 1000000 $cgroup2_update
+@@ -416,7 +442,7 @@ function test_cgroup2_cpu_update()
+     [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - cgroup2 update cpu.max -1 quota value not right" && ((ret++))
+ 
+     # cpus 1
+-    isula run -tid -n cpu_update busybox sh
++    isula run --runtime $1 -tid -n cpu_update busybox sh
+     [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - cgroup2 run cpu_update failed" && ((ret++))
+ 
+     isula update --cpus 1 cpu_update
+@@ -476,8 +502,13 @@ function test_cgroup2_io_update()
+     isula update --blkio-weight 10 $cgroup2_update
+     [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - cgroup2 update io.bfq.weight min value failed" && ((ret++))
+ 
+-    isula exec -ti $cgroup2_update cat "/sys/fs/cgroup/io.bfq.weight" | grep 1$'\r'
+-    [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - cgroup2 update io.bfq.weight min value not right" && ((ret++))
++    if [ $1 == "lcr" ]; then
++      isula exec -ti $cgroup2_update cat "/sys/fs/cgroup/io.bfq.weight" | grep 1$'\r'
++      [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - cgroup2 update io.bfq.weight min value not right" && ((ret++))
++    else
++      isula exec -ti $cgroup2_update cat "/sys/fs/cgroup/io.bfq.weight" | grep 10$'\r'
++      [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - cgroup2 update io.bfq.weight min value not right" && ((ret++))
++    fi
+ 
+     # max value
+     isula update --blkio-weight 1000 $cgroup2_update
+@@ -503,22 +534,38 @@ function test_cgroup2_io_update()
+     isula update --blkio-weight 10 $cgroup2_update
+     [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - cgroup2 update io.weight min value failed" && ((ret++))
+ 
+-    isula exec -ti $cgroup2_update cat "/sys/fs/cgroup/io.weight" | grep ^"default 1"$'\r'
+-    [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - cgroup2 update io.weight min value not right" && ((ret++))
++    if [ $1 == "lcr" ]; then
++      isula exec -ti $cgroup2_update cat "/sys/fs/cgroup/io.weight" | grep ^"default 1"$'\r'
++      [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - cgroup2 update io.weight min value not right" && ((ret++))
++    else
++      isula exec -ti $cgroup2_update cat "/sys/fs/cgroup/io.bfq.weight" | grep ^"default 10"$'\r'
++      [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - cgroup2 update io.weight min value not right" && ((ret++))
++    fi
+ 
+     # max value
+     isula update --blkio-weight 1000 $cgroup2_update
+     [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - cgroup2 update io.weight max value failed" && ((ret++))
+ 
+-    isula exec -ti $cgroup2_update cat "/sys/fs/cgroup/io.weight" | grep ^"default 10000"$'\r'
+-    [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - cgroup2 update io.weight max value not right" && ((ret++))
++    if [ $1 == "lcr" ]; then
++      isula exec -ti $cgroup2_update cat "/sys/fs/cgroup/io.weight" | grep ^"default 10000"$'\r'
++      [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - cgroup2 update io.weight max value not right" && ((ret++))
++    else
++      isula exec -ti $cgroup2_update cat "/sys/fs/cgroup/io.bfq.weight" | grep ^"default 1000"$'\r'
++      [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - cgroup2 update io.weight max value not right" && ((ret++))
++    fi
+ 
+     # 0 means value not change
+     isula update --blkio-weight 0 $cgroup2_update
+     [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - cgroup2 update io.weight 0 failed" && ((ret++))
+ 
+-    isula exec -ti $cgroup2_update cat "/sys/fs/cgroup/io.weight" | grep ^"default 10000"$'\r'
+-    [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - cgroup2 update io.weight 0 not right" && ((ret++))
++
++    if [ $1 == "lcr" ]; then
++      isula exec -ti $cgroup2_update cat "/sys/fs/cgroup/io.weight" | grep ^"default 10000"$'\r'
++      [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - cgroup2 update io.weight max value not right" && ((ret++))
++    else
++      isula exec -ti $cgroup2_update cat "/sys/fs/cgroup/io.bfq.weight" | grep ^"default 1000"$'\r'
++      [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - cgroup2 update io.weight max value not right" && ((ret++))
++    fi
+ 
+     # invalid value
+     isula update --blkio-weight -1 $cgroup2_update echo hello
+@@ -591,12 +638,13 @@ function test_cgroup2_memory_update()
+     isula exec -ti $cgroup2_update cat /sys/fs/cgroup/memory.swap.max | grep ^10485760$'\r'
+     [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - cgroup2 update memory.swap.max 0 value not right" && ((ret++))
+ 
+-    # -1 is max
+-    isula update --memory 10m --memory-swap -1 $cgroup2_update
+-    [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - cgroup2 update memory.swap.max -1 failed" && ((ret++))
+-
+-    isula exec -ti $cgroup2_update cat /sys/fs/cgroup/memory.swap.max | grep ^max$'\r'
+-    [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - cgroup2 update memory.swap.max -1 value not right" && ((ret++))
++    if [ $1 == "lcr" ]; then
++      # -1 is max
++      isula update --memory 10m --memory-swap -1 $cgroup2_update
++      [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - cgroup2 update memory.swap.max -1 failed" && ((ret++))
++      isula exec -ti $cgroup2_update cat /sys/fs/cgroup/memory.swap.max | grep ^max$'\r'
++      [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - cgroup2 update memory.swap.max -1 value not right" && ((ret++))
++    fi
+ 
+     # disable swap
+     isula update --memory 10m --memory-swap 10m $cgroup2_update
+@@ -613,16 +661,16 @@ function test_cgroup2_unsupported()
+ {
+   local ret=0
+ 
+-  isula run -ti --rm --cpu-rt-period 1000000 --cpu-rt-runtime 1000000 busybox echo hello
++  isula run --runtime $1 -ti --rm --cpu-rt-period 1000000 --cpu-rt-runtime 1000000 busybox echo hello
+   [[ $? -eq 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - cgroup2 --cpu-rt-period and --cpu-rt-runtime should failed" && ((ret++))
+ 
+-  isula run -ti --rm --kernel-memory 100m busybox echo hello
++  isula run --runtime $1 -ti --rm --kernel-memory 100m busybox echo hello
+   [[ $? -eq 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - cgroup2 --kernel-memory should failed" && ((ret++))
+ 
+-  isula run -ti --rm --memory-swappiness 50 busybox echo hello
++  isula run --runtime $1 -ti --rm --memory-swappiness 50 busybox echo hello
+   [[ $? -eq 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - cgroup2 --memory-swappiness should failed" && ((ret++))
+ 
+-  isula run -ti --rm --oom-kill-disable busybox echo hello
++  isula run --runtime $1 -ti --rm --oom-kill-disable busybox echo hello
+   [[ $? -eq 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - cgroup2 --oom-kill-disable should failed" && ((ret++))
+ 
+   isula update --cpu-rt-period 1000000 --cpu-rt-runtime 1000000 $cgroup2_update
+@@ -641,7 +689,7 @@ function test_cgroup2_parent()
+   rmdir /sys/fs/cgroup/isulad
+   rmdir /sys/fs/cgroup/abc
+ 
+-  id=`isula run -tid --cgroup-parent /abc -m 10m busybox sh`
++  id=`isula run --runtime $1 -tid --cgroup-parent /abc -m 10m busybox sh`
+   cat /sys/fs/cgroup/abc/$id/memory.max | grep ^10485760$
+   [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - cgroup2 --cgroup-parent cannot work" && ((ret++))
+ 
+@@ -657,39 +705,39 @@ function test_cgroup2_device()
+   mknod_num=$(echo $dev_num | sed 's/:/ /g')
+ 
+   # read only
+-  isula run -ti --rm --device=$dev_name:/dev/sdx:r busybox sh -c 'echo q | fdisk /dev/sdx | grep "read only"'
++  isula run --runtime $1 -ti --rm --device=$dev_name:/dev/sdx:r busybox sh -c 'echo q | fdisk /dev/sdx | grep "read only"'
+   [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - cgroup2 --device r failed" && ((ret++))
+ 
+-  isula run -ti --rm --device=$dev_name:/dev/sdx:rm busybox sh -c 'echo q | fdisk /dev/sdx | grep "read only"'
++  isula run --runtime $1 -ti --rm --device=$dev_name:/dev/sdx:rm busybox sh -c 'echo q | fdisk /dev/sdx | grep "read only"'
+   [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - cgroup2 --device rm failed" && ((ret++))
+ 
+-  isula run -ti --rm --device-cgroup-rule="b $dev_num r" busybox sh -c "mknod /dev/sdx b $mknod_num && echo q | fdisk /dev/sdx | grep 'read only'"
++  isula run --runtime $1 -ti --rm --device-cgroup-rule="b $dev_num r" busybox sh -c "mknod /dev/sdx b $mknod_num && echo q | fdisk /dev/sdx | grep 'read only'"
+   [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - cgroup2 --device r failed" && ((ret++))
+ 
+-  isula run -ti --rm --device-cgroup-rule="b $dev_num rm" busybox sh -c "mknod /dev/sdx b $mknod_num && echo q | fdisk /dev/sdx | grep 'read only'"
++  isula run --runtime $1 -ti --rm --device-cgroup-rule="b $dev_num rm" busybox sh -c "mknod /dev/sdx b $mknod_num && echo q | fdisk /dev/sdx | grep 'read only'"
+   [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - cgroup2 --device rm failed" && ((ret++))
+ 
+   # can't read
+-  isula run -ti --rm --device=$dev_name:/dev/sdx:w busybox sh -c 'echo q | fdisk /dev/sdx 2>&1 | grep "t open"'
++  isula run --runtime $1 -ti --rm --device=$dev_name:/dev/sdx:w busybox sh -c 'echo q | fdisk /dev/sdx 2>&1 | grep "t open"'
+   [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - cgroup2 --device w failed" && ((ret++))
+ 
+-  isula run -ti --rm --device=$dev_name:/dev/sdx:wm busybox sh -c 'echo q | fdisk /dev/sdx 2>&1 | grep "t open"'
++  isula run --runtime $1 -ti --rm --device=$dev_name:/dev/sdx:wm busybox sh -c 'echo q | fdisk /dev/sdx 2>&1 | grep "t open"'
+   [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - cgroup2 --device wm failed" && ((ret++))
+ 
+-  isula run -ti --rm --device-cgroup-rule="b $dev_num w" busybox sh -c "mknod /dev/sdx b $mknod_num && echo q | fdisk /dev/sdx 2>&1 | grep 't open'"
++  isula run --runtime $1 -ti --rm --device-cgroup-rule="b $dev_num w" busybox sh -c "mknod /dev/sdx b $mknod_num && echo q | fdisk /dev/sdx 2>&1 | grep 't open'"
+   [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - cgroup2 --device w failed" && ((ret++))
+ 
+-  isula run -ti --rm --device-cgroup-rule="b $dev_num wm" busybox sh -c "mknod /dev/sdx b $mknod_num && echo q | fdisk /dev/sdx 2>&1 | grep 't open'"
++  isula run --runtime $1 -ti --rm --device-cgroup-rule="b $dev_num wm" busybox sh -c "mknod /dev/sdx b $mknod_num && echo q | fdisk /dev/sdx 2>&1 | grep 't open'"
+   [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - cgroup2 --device wm failed" && ((ret++))
+ 
+   # can't read write
+-  isula run -ti --rm --device=$dev_name:/dev/sdx:m busybox sh -c 'echo q | fdisk /dev/sdx 2>&1 | grep "t open"'
++  isula run --runtime $1 -ti --rm --device=$dev_name:/dev/sdx:m busybox sh -c 'echo q | fdisk /dev/sdx 2>&1 | grep "t open"'
+   [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - cgroup2 --device m" && ((ret++))
+ 
+-  isula run -ti --rm --device-cgroup-rule="b $dev_num m" busybox sh -c "mknod /dev/sdx b $mknod_num && echo q | fdisk /dev/sdx 2>&1 | grep 't open'"
++  isula run --runtime $1 -ti --rm --device-cgroup-rule="b $dev_num m" busybox sh -c "mknod /dev/sdx b $mknod_num && echo q | fdisk /dev/sdx 2>&1 | grep 't open'"
+   [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - cgroup2 --device wm failed" && ((ret++))
+ 
+-  isula run -ti --rm --device-cgroup-rule="b *:* m" busybox sh -c "mknod /dev/sdx b $mknod_num && echo q | fdisk /dev/sdx 2>&1 | grep 't open'"
++  isula run --runtime $1 -ti --rm --device-cgroup-rule="b *:* m" busybox sh -c "mknod /dev/sdx b $mknod_num && echo q | fdisk /dev/sdx 2>&1 | grep 't open'"
+   [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - cgroup2 --device wm failed" && ((ret++))
+ 
+   return ${ret}
+@@ -723,7 +771,7 @@ function prepare_test_cgroupv2()
+ 
+   isula rm -f `isula ps -a -q`
+ 
+-  isula run -tid -n $cgroup2_update busybox sh
++  isula run --runtime $1 -tid -n $cgroup2_update busybox sh
+   [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - cgroup2 run container failed" && ((ret++))
+ 
+   return ${ret}
+@@ -740,25 +788,31 @@ declare -i ans=0
+ msg_info "${test} starting..."
+ [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - start isulad failed" && ((ret++))
+ 
+-prepare_test_cgroupv2 || ((ans++))
+-if [ "$cgroupv2" == "1" ];then
+-  test_cgroup2_cpu || ((ans++))
+-  test_cgroup2_io || ((ans++))
+-  test_cgroup2_memory || ((ans++))
+-  test_cgroup2_pids || ((ans++))
+-  test_cgroup2_hugetlb || ((ans++))
+-  test_cgroup2_freeze || ((ans++))
+-  test_cgroup2_files || ((ans++))
+-  test_cgroup2_cpu_update || ((ans++))
+-  test_cgroup2_io_update || ((ans++))
+-  test_cgroup2_memory_update || ((ans++))
+-  test_cgroup2_unsupported || ((ans++))
+-  test_cgroup2_parent || ((ans++))
+-  test_cgroup2_device || ((ans++))
+-else
+-  msg_info "${test} not cgroup v2 enviorment, ignore test..."
+-fi
+-post_test_cgroupv2
++for element in ${RUNTIME_LIST[@]};
++do
++  prepare_test_cgroupv2 $element || ((ans++))
++  if [ "$cgroupv2" == "1" ];then
++    local test="cgroup v2 test => (${element})"
++    msg_info "${test} starting..."
++    test_cgroup2_cpu $element || ((ans++))
++    test_cgroup2_io $element || ((ans++))
++    test_cgroup2_memory $element || ((ans++))
++    test_cgroup2_pids $element || ((ans++))
++    test_cgroup2_hugetlb $element || ((ans++))
++    test_cgroup2_freeze $element || ((ans++))
++    test_cgroup2_files $element || ((ans++))
++    test_cgroup2_cpu_update $element || ((ans++))
++    test_cgroup2_io_update $element || ((ans++))
++    test_cgroup2_memory_update $element || ((ans++))
++    test_cgroup2_unsupported $element || ((ans++))
++    test_cgroup2_parent $element || ((ans++))
++    test_cgroup2_device $element || ((ans++))
++    msg_info "${test} finished with return ${ans}..."
++  else
++    msg_info "${test} not cgroup v2 enviorment, ignore test..."
++  fi
++  post_test_cgroupv2 $element
++done
+ 
+ msg_info "${test} finished with return ${ans}..."
+ 
+-- 
+2.25.1
+

0039-fix-run-ubuntu-container-bug-in-inspect.sh.patch

@@ -0,0 +1,27 @@
+From 8e1fe0302bf1a871f66a296e456811e878b1fa3b Mon Sep 17 00:00:00 2001
+From: jikai <jikai11@huawei.com>
+Date: Tue, 2 Apr 2024 10:06:18 +0800
+Subject: [PATCH 039/149] fix run ubuntu container bug in inspect.sh
+
+Signed-off-by: jikai <jikai11@huawei.com>
+---
+ CI/test_cases/container_cases/inspect.sh | 3 ++-
+ 1 file changed, 2 insertions(+), 1 deletion(-)
+
+diff --git a/CI/test_cases/container_cases/inspect.sh b/CI/test_cases/container_cases/inspect.sh
+index b4f4a785..86aed3d8 100755
+--- a/CI/test_cases/container_cases/inspect.sh
++++ b/CI/test_cases/container_cases/inspect.sh
+@@ -146,7 +146,8 @@ function test_inspect_spec()
+ 
+     isula rm -f $containername
+ 
+-    isula run -it -m 4m --name $containername $ubuntu_image perl -e 'for ($i = 0; $i < 100000000; $i++) { $a .= " " x 1024 }'
++    # use more than 10m memory limit, otherwise it might fail to run
++    isula run -it -m 10m --name $containername $ubuntu_image perl -e 'for ($i = 0; $i < 100000000; $i++) { $a .= " " x 1024 }'
+ 
+     isula inspect -f "{{json .State.OOMKilled}} {{.Name}}" $containername 2>&1 | sed -n '1p' | grep "true"
+     [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - failed to check container with image: ${ubuntu_image}" && ((ret++))
+-- 
+2.25.1
+

0041-fix-cpurt-init-bug-for-systemd-cgroup.patch

@@ -0,0 +1,74 @@
+From fe11b34a3c2843ea2198b310160b182d63aeb63b Mon Sep 17 00:00:00 2001
+From: jikai <jikai11@huawei.com>
+Date: Tue, 2 Apr 2024 11:22:09 +0800
+Subject: [PATCH 041/149] fix cpurt init bug for systemd-cgroup
+
+Signed-off-by: jikai <jikai11@huawei.com>
+---
+ src/daemon/common/cgroup/cgroup.c            | 13 +++++++------
+ src/daemon/executor/container_cb/execution.c | 13 +++++++------
+ 2 files changed, 14 insertions(+), 12 deletions(-)
+
+diff --git a/src/daemon/common/cgroup/cgroup.c b/src/daemon/common/cgroup/cgroup.c
+index d3f1445a..007dbb70 100644
+--- a/src/daemon/common/cgroup/cgroup.c
++++ b/src/daemon/common/cgroup/cgroup.c
+@@ -146,17 +146,18 @@ char *common_convert_cgroup_path(const char *cgroup_path)
+         return NULL;
+     }
+ 
+-    // for cgroup fs cgroup path, return directly
+-    if (!util_has_suffix(cgroup_path, ".slice")) {
+-        return util_strdup_s(cgroup_path);
+-    }
+-
+     // for systemd cgroup, cgroup_path should have the form slice:prefix:id,
+     // convert it to a true path, such as from test-a.slice:isulad:id
+     // to test.slice/test-a.slice/isulad-id.scope
+     arr = util_string_split_n(cgroup_path, ':', 3);
+     if (arr == NULL || util_array_len((const char **)arr) != 3) {
+-        ERROR("Invalid systemd cgroup parent");
++        // not a systemd cgroup, return cgroup path directly
++        return util_strdup_s(cgroup_path);
++    }
++
++    // for cgroup fs cgroup path, return directly
++    if (!util_has_suffix(arr[0], ".slice")) {
++        ERROR("Invalid systemd cgroup path: %s", cgroup_path);
+         return NULL;
+     }
+ 
+diff --git a/src/daemon/executor/container_cb/execution.c b/src/daemon/executor/container_cb/execution.c
+index 88c6b354..4bf3621d 100644
+--- a/src/daemon/executor/container_cb/execution.c
++++ b/src/daemon/executor/container_cb/execution.c
+@@ -435,11 +435,12 @@ static int cpurt_controller_init(const char *id, const host_config *host_spec)
+     }
+ 
+     if (conf_get_systemd_cgroup()) {
+-        // currently it is the same as docker, yet it is unclear that
+-        // if systemd cgroup is used and cgroup parent is set to a slice rather than system.slice
+-        // should iSulad set cpu.rt_runtime_us and cpu.rt_period_us for the parent path?
+-        // in fact, even if system.slice is used,
+-        // cpu.rt_runtime_us and cpu.rt_period_us might still needed to be set manually
++        __isula_auto_free char *converted_cgroup = common_convert_cgroup_path(cgroups_path);
++        if (converted_cgroup == NULL) {
++            ERROR("Failed to convert cgroup path");
++            return -1;
++        }
++
+         __isula_auto_free char *init_cgroup = common_get_init_cgroup_path("cpu");
+         if (init_cgroup == NULL) {
+             ERROR("Failed to get init cgroup");
+@@ -451,7 +452,7 @@ static int cpurt_controller_init(const char *id, const host_config *host_spec)
+             ERROR("Failed to get own cgroup");
+             return -1;
+         }
+-        char *new_cgroups_path = util_path_join(init_cgroup, cgroups_path);
++        char *new_cgroups_path = util_path_join(init_cgroup, converted_cgroup);
+         if (new_cgroups_path == NULL) {
+             ERROR("Failed to join path");
+             return -1;
+-- 
+2.25.1
+

0042-fix-message-queue-concurrent-bug.patch

@@ -0,0 +1,41 @@
+From f90a145d9d29682295aebf2bcd30865ee5f6491f Mon Sep 17 00:00:00 2001
+From: jikai <jikai11@huawei.com>
+Date: Tue, 2 Apr 2024 07:53:54 +0000
+Subject: [PATCH 042/149] fix message queue concurrent bug
+
+Signed-off-by: jikai <jikai11@huawei.com>
+---
+ src/daemon/mailbox/message_queue.c | 7 ++++---
+ 1 file changed, 4 insertions(+), 3 deletions(-)
+
+diff --git a/src/daemon/mailbox/message_queue.c b/src/daemon/mailbox/message_queue.c
+index 7fe044f2..7e53301e 100644
+--- a/src/daemon/mailbox/message_queue.c
++++ b/src/daemon/mailbox/message_queue.c
+@@ -106,11 +106,12 @@ message_queue *message_queue_create(void (*release)(void *))
+         return NULL;
+     }
+ 
+-    bq = blocking_queue_create(BLOCKING_QUEUE_NO_TIMEOUT, release);
+-    if (bq == NULL) {
++    mq->messages = blocking_queue_create(BLOCKING_QUEUE_NO_TIMEOUT, release);
++    if (mq->messages == NULL) {
+         ERROR("Failed to create events queue");
+         return NULL;
+     }
++    bq = mq->messages;
+ 
+     mq->subscribers = map_new(MAP_PTR_INT, MAP_DEFAULT_CMP_FUNC, message_queue_subscriber_free);
+     if (mq->subscribers == NULL) {
+@@ -131,7 +132,7 @@ message_queue *message_queue_create(void (*release)(void *))
+         return NULL;
+     }
+ 
+-    mq->messages = isula_transfer_ptr(bq);
++    bq = NULL;
+     return isula_transfer_ptr(mq);
+ }
+ 
+-- 
+2.25.1
+

0043-specify-runtime-as-runc-for-oom-test-CI.patch

@@ -0,0 +1,26 @@
+From 7af700c4021ef9961aaac37ffa5767bd4f3dd184 Mon Sep 17 00:00:00 2001
+From: jikai <jikai11@huawei.com>
+Date: Tue, 2 Apr 2024 08:00:37 +0000
+Subject: [PATCH 043/149] specify runtime as runc for oom test CI
+
+Signed-off-by: jikai <jikai11@huawei.com>
+---
+ CI/test_cases/container_cases/inspect.sh | 2 +-
+ 1 file changed, 1 insertion(+), 1 deletion(-)
+
+diff --git a/CI/test_cases/container_cases/inspect.sh b/CI/test_cases/container_cases/inspect.sh
+index 86aed3d8..5d976281 100755
+--- a/CI/test_cases/container_cases/inspect.sh
++++ b/CI/test_cases/container_cases/inspect.sh
+@@ -147,7 +147,7 @@ function test_inspect_spec()
+     isula rm -f $containername
+ 
+     # use more than 10m memory limit, otherwise it might fail to run
+-    isula run -it -m 10m --name $containername $ubuntu_image perl -e 'for ($i = 0; $i < 100000000; $i++) { $a .= " " x 1024 }'
++    isula run -it -m 10m --runtime runc --name $containername $ubuntu_image perl -e 'for ($i = 0; $i < 100000000; $i++) { $a .= " " x 1024 }'
+ 
+     isula inspect -f "{{json .State.OOMKilled}} {{.Name}}" $containername 2>&1 | sed -n '1p' | grep "true"
+     [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - failed to check container with image: ${ubuntu_image}" && ((ret++))
+-- 
+2.25.1
+

0044-set-oomkilled-in-cri.patch

@@ -0,0 +1,27 @@
+From 5393ce7d02bb73ce4760edefa959dfb4846f1958 Mon Sep 17 00:00:00 2001
+From: jikai <jikai11@huawei.com>
+Date: Tue, 2 Apr 2024 11:19:06 +0000
+Subject: [PATCH 044/149] set oomkilled in cri
+
+Signed-off-by: jikai <jikai11@huawei.com>
+---
+ src/daemon/common/cri/v1/v1_cri_helpers.cc | 3 +++
+ 1 file changed, 3 insertions(+)
+
+diff --git a/src/daemon/common/cri/v1/v1_cri_helpers.cc b/src/daemon/common/cri/v1/v1_cri_helpers.cc
+index a3488894..ea5c8bb5 100644
+--- a/src/daemon/common/cri/v1/v1_cri_helpers.cc
++++ b/src/daemon/common/cri/v1/v1_cri_helpers.cc
+@@ -506,6 +506,9 @@ void UpdateBaseStatusFromInspect(
+         } else { // Case 3
+             state = runtime::v1::CONTAINER_CREATED;
+         }
++        if (inspect->state->oom_killed == true) {
++            reason = "OOMKilled";
++        }
+         if (inspect->state->error != nullptr) {
+             message = inspect->state->error;
+         }
+-- 
+2.25.1
+

0046-oom-monitor-in-manual-cases.patch

@@ -0,0 +1,120 @@
+From aa77c85ea6879698663d4ef9e01bb63a0db1e57d Mon Sep 17 00:00:00 2001
+From: jikai <jikai11@huawei.com>
+Date: Wed, 3 Apr 2024 09:34:39 +0000
+Subject: [PATCH 046/149] oom monitor in manual cases
+
+Signed-off-by: jikai <jikai11@huawei.com>
+---
+ CI/test_cases/container_cases/inspect.sh  | 15 ------
+ CI/test_cases/manual_cases/oom_monitor.sh | 59 +++++++++++++++++++++++
+ 2 files changed, 59 insertions(+), 15 deletions(-)
+ create mode 100755 CI/test_cases/manual_cases/oom_monitor.sh
+
+diff --git a/CI/test_cases/container_cases/inspect.sh b/CI/test_cases/container_cases/inspect.sh
+index 5d976281..cde9ea1f 100755
+--- a/CI/test_cases/container_cases/inspect.sh
++++ b/CI/test_cases/container_cases/inspect.sh
+@@ -27,7 +27,6 @@ function test_inspect_spec()
+ {
+     local ret=0
+     local image="busybox"
+-    local ubuntu_image="ubuntu"
+     local test="container inspect test => (${FUNCNAME[@]})"
+ 
+     msg_info "${test} starting..."
+@@ -38,12 +37,6 @@ function test_inspect_spec()
+     isula images | grep busybox
+     [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - missing list image: ${image}" && ((ret++))
+ 
+-    isula pull ${ubuntu_image}
+-    [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - failed to pull image: ${ubuntu_image}" && return ${FAILURE}
+-
+-    isula images | grep ubuntu
+-    [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - missing list image: ${ubuntu_image}" && ((ret++))
+-
+     containername=test_inspect
+ 
+     isula create --name $containername --ipc host --pid host --uts host --restart=on-failure:10 --hook-spec ${test_data_path}/test-hookspec.json --cpu-shares 100 --memory 5MB --memory-reservation 4MB --cpu-period 1000000 --cpu-quota 200000  --cpuset-cpus 1 --cpuset-mems 0 --kernel-memory 50M --pids-limit=10000 --volume /home:/root --env a=1 $image /bin/sh ls
+@@ -146,14 +139,6 @@ function test_inspect_spec()
+ 
+     isula rm -f $containername
+ 
+-    # use more than 10m memory limit, otherwise it might fail to run
+-    isula run -it -m 10m --runtime runc --name $containername $ubuntu_image perl -e 'for ($i = 0; $i < 100000000; $i++) { $a .= " " x 1024 }'
+-
+-    isula inspect -f "{{json .State.OOMKilled}} {{.Name}}" $containername 2>&1 | sed -n '1p' | grep "true"
+-    [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - failed to check container with image: ${ubuntu_image}" && ((ret++))
+-
+-    isula rm -f $containername
+-
+     msg_info "${test} finished with return ${ret}..."
+     return ${ret}
+ }
+diff --git a/CI/test_cases/manual_cases/oom_monitor.sh b/CI/test_cases/manual_cases/oom_monitor.sh
+new file mode 100755
+index 00000000..a1c2503d
+--- /dev/null
++++ b/CI/test_cases/manual_cases/oom_monitor.sh
+@@ -0,0 +1,59 @@
++#!/bin/bash
++#
++# attributes: isulad oom monitor
++# concurrent: NA
++# spend time: 6
++
++#######################################################################
++##- Copyright (c) Huawei Technologies Co., Ltd. 2020. All rights reserved.
++# - iSulad licensed under the Mulan PSL v2.
++# - You can use this software according to the terms and conditions of the Mulan PSL v2.
++# - You may obtain a copy of Mulan PSL v2 at:
++# -     http://license.coscl.org.cn/MulanPSL2
++# - THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR
++# - IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR
++# - PURPOSE.
++# - See the Mulan PSL v2 for more details.
++##- @Description:CI
++##- @Author: jikai
++##- @Create: 2024-04-03
++#######################################################################
++
++declare -r curr_path=$(dirname $(readlink -f "$0"))
++source ../helpers.sh
++test_data_path=$(realpath $curr_path/test_data)
++
++function test_oom_monitor()
++{
++    local ret=0
++    local ubuntu_image="ubuntu"
++    local test="container oom monitor test => (${FUNCNAME[@]})"
++    containername="oommonitor"
++
++    msg_info "${test} starting..."
++
++    isula pull ${ubuntu_image}
++    [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - failed to pull image: ${ubuntu_image}" && return ${FAILURE}
++
++    isula images | grep ubuntu
++    [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - missing list image: ${ubuntu_image}" && ((ret++))
++
++    # use more than 10m memory limit, otherwise it might fail to run
++    # iSulad monitor cgroup file for oom event, however oom triggers cgroup files delete
++    # if cgroup files were deleted before oom event was handled for iSulad we might failed to detect oom event
++    isula run -it -m 10m --runtime runc --name $containername $ubuntu_image perl -e 'for ($i = 0; $i < 100000000; $i++) { $a .= " " x 1024 }'
++
++    isula inspect -f "{{json .State.OOMKilled}} {{.Name}}" $containername 2>&1 | sed -n '1p' | grep "true"
++    [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - failed to check container with image: ${ubuntu_image}" && ((ret++))
++
++    isula rm -f $containername
++
++    msg_info "${test} finished with return ${ret}..."
++    return ${ret}
++}
++
++declare -i ans=0
++
++test_oom_monitor || ((ans++))
++
++show_result ${ans} "${curr_path}/${0}"
+-- 
+2.25.1
+

0047-add-usage-restrictions-for-CRI-1.29-update.patch

@@ -0,0 +1,27 @@
+From 9f066a405a95299c182ef7356b6518a9457af298 Mon Sep 17 00:00:00 2001
+From: jikai <jikai11@huawei.com>
+Date: Mon, 8 Apr 2024 02:52:11 +0000
+Subject: [PATCH 047/149] add usage restrictions for CRI 1.29 update
+
+Signed-off-by: jikai <jikai11@huawei.com>
+---
+ docs/design/detailed/CRI/CRI_1.29_update_design.md | 6 +++++-
+ 1 file changed, 5 insertions(+), 1 deletion(-)
+
+diff --git a/docs/design/detailed/CRI/CRI_1.29_update_design.md b/docs/design/detailed/CRI/CRI_1.29_update_design.md
+index 0a0c860b..0c66db48 100644
+--- a/docs/design/detailed/CRI/CRI_1.29_update_design.md
++++ b/docs/design/detailed/CRI/CRI_1.29_update_design.md
+@@ -234,4 +234,8 @@ enum ContainerEventType {
+ 
+ ### 使用限制
+ 
+-以上特性仅保证容器运行时设置为runc时支持。
++1. 以上新增特性，iSulad仅提供容器运行时设置为runc时的支持。
++2. 由于cgroup oom会同时触发容器cgroup路径删除，若iSulad对oom事件处理发生在
++cgroup路径删除之后，iSulad则无法成功捕捉容器oom事件，
++可能导致ContainerStatus中reason字段设置不正确。
++3. iSulad不支持交叉使用不同的cgroup驱动管理容器，启动容器后iSulad的cgroup驱动配置不应该发生变化。
+-- 
+2.25.1
+

0049-distinguish-between-runtime-and-runtime_cmd-in-isula.patch

@@ -0,0 +1,145 @@
+From 491baece02522128720b3bd992a76dc5148aa7b2 Mon Sep 17 00:00:00 2001
+From: zhongtao <zhongtao17@huawei.com>
+Date: Mon, 8 Apr 2024 11:37:13 +0800
+Subject: [PATCH 049/149] distinguish between runtime and runtime_cmd in
+ isulad-shim
+
+Signed-off-by: zhongtao <zhongtao17@huawei.com>
+---
+ src/cmd/isulad-shim/process.c                 | 20 +++++++++----------
+ src/cmd/isulad-shim/process.h                 |  4 ++--
+ .../modules/runtime/isula/isula_rt_ops.c      |  2 ++
+ 3 files changed, 14 insertions(+), 12 deletions(-)
+
+diff --git a/src/cmd/isulad-shim/process.c b/src/cmd/isulad-shim/process.c
+index 8a4ca175..6b5f8f7f 100644
+--- a/src/cmd/isulad-shim/process.c
++++ b/src/cmd/isulad-shim/process.c
+@@ -1131,7 +1131,7 @@ static int init_root_path(process_t *p)
+         return SHIM_ERR;
+     }
+ 
+-    if (buffer->nappend(buffer, PATH_MAX, "%s/%s", state_path, p->runtime) < 0) {
++    if (buffer->nappend(buffer, PATH_MAX, "%s/%s", state_path, p->state->runtime) < 0) {
+         ERROR("Failed to append state_path\n");
+         isula_buffer_free(buffer);
+         return SHIM_ERR;
+@@ -1146,7 +1146,7 @@ static int init_root_path(process_t *p)
+     return SHIM_OK;
+ }
+ 
+-process_t *new_process(char *id, char *bundle, char *runtime)
++process_t *new_process(char *id, char *bundle, char *runtime_cmd)
+ {
+     shim_client_process_state *p_state;
+     process_t *p = NULL;
+@@ -1174,7 +1174,7 @@ process_t *new_process(char *id, char *bundle, char *runtime)
+ 
+     p->id = id;
+     p->bundle = bundle;
+-    p->runtime = runtime;
++    p->runtime_cmd = runtime_cmd;
+     p->state = p_state;
+     p->console_sock_path = NULL;
+     p->exit_fd = -1;
+@@ -1247,7 +1247,7 @@ static void set_common_params(process_t *p, const char *params[], int *index, co
+ {
+     int j;
+ 
+-    params[(*index)++]  = p->runtime;
++    params[(*index)++]  = p->runtime_cmd;
+     for (j = 0; j < p->state->runtime_args_len; j++) {
+         params[(*index)++]  = p->state->runtime_args[j];
+     }
+@@ -1261,7 +1261,7 @@ static void set_common_params(process_t *p, const char *params[], int *index, co
+ 
+     // In addition to kata, other commonly used oci runtimes (runc, crun, youki, gvisor)
+     // need to set the --root option
+-    if (strcasecmp(p->runtime, "kata-runtime") != 0) {
++    if (strcasecmp(p->state->runtime, "kata-runtime") != 0) {
+         params[(*index)++] = "--root";
+         params[(*index)++] = p->root_path;
+     }
+@@ -1347,7 +1347,7 @@ static void process_kill_all(process_t *p)
+     params[i++] = p->id;
+     params[i++] = "SIGKILL";
+ 
+-    (void)cmd_combined_output(p->runtime, params, output, &output_len);
++    (void)cmd_combined_output(p->runtime_cmd, params, output, &output_len);
+ 
+     return;
+ }
+@@ -1375,7 +1375,7 @@ static void process_delete(process_t *p)
+     params[i++] = "--force";
+     params[i++] = p->id;
+ 
+-    (void)cmd_combined_output(p->runtime, params, output, &output_len);
++    (void)cmd_combined_output(p->runtime_cmd, params, output, &output_len);
+ 
+     return;
+ }
+@@ -1444,8 +1444,8 @@ static void exec_runtime_process(process_t *p, int exec_fd)
+ 
+     const char *params[MAX_RUNTIME_ARGS] = { 0 };
+     get_runtime_cmd(p, log_path, pid_path, process_desc, params);
+-    execvp(p->runtime, (char * const *)params);
+-    (void)dprintf(exec_fd, "run process: %s error: %s", p->runtime, strerror(errno));
++    execvp(p->runtime_cmd, (char * const *)params);
++    (void)dprintf(exec_fd, "run process: %s error: %s", p->runtime_cmd, strerror(errno));
+     _exit(EXIT_FAILURE);
+ }
+ 
+@@ -1586,7 +1586,7 @@ static int waitpid_with_timeout(int ctr_pid,  int *status, const uint64_t timeou
+ static int wait_container_process_with_timeout(process_t *p, const uint64_t timeout, int *status)
+ {
+     // currently, kata runtime does not support setting timeout during exec
+-    if (strcasecmp(p->runtime, "kata-runtime") != 0 && timeout > 0) {
++    if (strcasecmp(p->state->runtime, "kata-runtime") != 0 && timeout > 0) {
+         return waitpid_with_timeout(p->ctr_pid, status, timeout);
+     }
+ 
+diff --git a/src/cmd/isulad-shim/process.h b/src/cmd/isulad-shim/process.h
+index 32ba7366..05fd87b0 100644
+--- a/src/cmd/isulad-shim/process.h
++++ b/src/cmd/isulad-shim/process.h
+@@ -44,7 +44,7 @@ typedef struct {
+ typedef struct process {
+     char *id;
+     char *bundle;
+-    char *runtime;
++    char *runtime_cmd;
+     char *console_sock_path; // pty socket path
+     char *workdir;
+     char *root_path;
+@@ -70,7 +70,7 @@ typedef struct {
+     int status;
+ } process_exit_t;
+ 
+-process_t* new_process(char *id, char *bundle, char *runtime);
++process_t* new_process(char *id, char *bundle, char *runtime_cmd);
+ 
+ int prepare_attach_socket(process_t *p);
+ 
+diff --git a/src/daemon/modules/runtime/isula/isula_rt_ops.c b/src/daemon/modules/runtime/isula/isula_rt_ops.c
+index b9aba3e3..bc3c36c8 100644
+--- a/src/daemon/modules/runtime/isula/isula_rt_ops.c
++++ b/src/daemon/modules/runtime/isula/isula_rt_ops.c
+@@ -1154,6 +1154,7 @@ int rt_isula_create(const char *id, const char *runtime, const rt_create_params_
+     p.isulad_stdin = (char *)params->stdin;
+     p.isulad_stdout = (char *)params->stdout;
+     p.isulad_stderr = (char *)params->stderr;
++    p.runtime = (char *)runtime;
+     p.runtime_args = (char **)runtime_args;
+     p.runtime_args_len = runtime_args_len;
+     p.attach_socket = attach_socket;
+@@ -1409,6 +1410,7 @@ static int preparation_exec(const char *id, const char *runtime, const char *wor
+     p.isulad_stdout = (char *)params->console_fifos[1];
+     p.isulad_stderr = (char *)params->console_fifos[2];
+     p.resize_fifo = resize_fifo_dir;
++    p.runtime = (char *)runtime;
+     p.runtime_args = (char **)runtime_args;
+     p.runtime_args_len = runtime_args_len;
+     copy_process(&p, process);
+-- 
+2.25.1
+

0050-Use-user-defined-shm-for-CRI-request.patch

@@ -0,0 +1,171 @@
+From 162123bdec0f45f7b2001b2b0b83705cc6b9b1b1 Mon Sep 17 00:00:00 2001
+From: xuxuepeng <xuxuepeng1@huawei.com>
+Date: Mon, 8 Apr 2024 20:53:57 +0800
+Subject: [PATCH 050/149] Use user defined shm for CRI request
+
+Signed-off-by: xuxuepeng <xuxuepeng1@huawei.com>
+---
+ src/daemon/modules/spec/specs_mount.c | 128 +++++++++++++++++---------
+ 1 file changed, 85 insertions(+), 43 deletions(-)
+
+diff --git a/src/daemon/modules/spec/specs_mount.c b/src/daemon/modules/spec/specs_mount.c
+index 20bf5378..6903ae40 100644
+--- a/src/daemon/modules/spec/specs_mount.c
++++ b/src/daemon/modules/spec/specs_mount.c
+@@ -2799,33 +2799,31 @@ out_free:
+     return ret;
+ }
+ 
+-#define SHM_MOUNT_POINT "/dev/shm"
+-static int set_shm_path(host_config *host_spec, container_config_v2_common_config *v2_spec)
++static inline int set_sharable_ipc_mode(host_config *host_spec, container_config_v2_common_config *v2_spec)
+ {
+-    int ret = 0;
+-    container_t *cont = NULL;
+-    char *tmp_cid = NULL;
+-    char *right_path = NULL;
+-
+-    // ignore shm of system container
+-    if (host_spec->system_container) {
++    free(v2_spec->shm_path);
++#ifdef ENABLE_CRI_API_V1
++    // In the case of sandbox API is used, the shm path has already been created in CRI,
++    // so we need to use the sandbox's shm path
++    if (is_sandbox_container(v2_spec->sandbox_info)) {
++        v2_spec->shm_path = util_strdup_s(v2_spec->sandbox_info->shm_path);
+         return 0;
+     }
+-    // setup shareable dirs
+-    if (is_shareable_ipc(host_spec->ipc_mode)) {
+-        // has mount for /dev/shm
+-        if (has_mount_shm(host_spec, v2_spec)) {
+-            return 0;
+-        }
++#endif
++    v2_spec->shm_path = get_prepare_share_shm_path(host_spec->runtime, v2_spec->id);
++    if (v2_spec->shm_path == NULL) {
++        ERROR("Failed to get prepare share shm path");
++        return -1;
++    }
+ 
+-        v2_spec->shm_path = get_prepare_share_shm_path(host_spec->runtime, v2_spec->id);
+-        if (v2_spec->shm_path == NULL) {
+-            ERROR("Failed to get prepare share shm path");
+-            return -1;
+-        }
++    return 0;
++}
+ 
+-        return 0;
+-    }
++static inline int set_connected_container_shm_path(host_config *host_spec, container_config_v2_common_config *v2_spec)
++{
++    container_t *cont = NULL;
++    char *tmp_cid = NULL;
++    char *right_path = NULL;
+ 
+ #ifdef ENABLE_CRI_API_V1
+     // Sandbox API is used and the connected container is actually a sandbox
+@@ -2833,34 +2831,78 @@ static int set_shm_path(host_config *host_spec, container_config_v2_common_confi
+     if (namespace_is_sandbox(host_spec->ipc_mode, v2_spec->sandbox_info)) {
+         free(v2_spec->shm_path);
+         v2_spec->shm_path = util_strdup_s(v2_spec->sandbox_info->shm_path);
+-        goto out;
++        return 0;
+     }
+ #endif
+ 
+-    if (namespace_is_container(host_spec->ipc_mode)) {
+-        tmp_cid = namespace_get_connected_container(host_spec->ipc_mode);
+-        cont = containers_store_get(tmp_cid);
+-        if (cont == NULL) {
+-            ERROR("Invalid share path: %s", host_spec->ipc_mode);
+-            ret = -1;
+-            goto out;
+-        }
+-        right_path = util_strdup_s(cont->common_config->shm_path);
+-        container_unref(cont);
+-    } else if (namespace_is_host(host_spec->ipc_mode)) {
+-        if (!util_file_exists(SHM_MOUNT_POINT)) {
+-            ERROR("/dev/shm is not mounted, but must be for --ipc=host");
+-            ret = -1;
+-            goto out;
+-        }
+-        right_path = util_strdup_s(SHM_MOUNT_POINT);
++    tmp_cid = namespace_get_connected_container(host_spec->ipc_mode);
++    cont = containers_store_get(tmp_cid);
++    if (cont == NULL) {
++        ERROR("Invalid share path: %s", host_spec->ipc_mode);
++        return -1;
+     }
++    right_path = util_strdup_s(cont->common_config->shm_path);
++    container_unref(cont);
+ 
+     free(v2_spec->shm_path);
+     v2_spec->shm_path = right_path;
+-out:
+-    free(tmp_cid);
+-    return ret;
++
++    return 0;
++}
++
++#define SHM_MOUNT_POINT "/dev/shm"
++static inline int set_host_ipc_shm_path(container_config_v2_common_config *v2_spec)
++{
++    if (!util_file_exists(SHM_MOUNT_POINT)) {
++        ERROR("/dev/shm is not mounted, but must be for --ipc=host");
++        return -1;
++    }
++    free(v2_spec->shm_path);
++    v2_spec->shm_path = util_strdup_s(SHM_MOUNT_POINT);
++    return 0;
++}
++
++/**
++ * There are 4 cases for setting shm path:
++ * 1. The user defined /dev/shm in mounts, which takes the first priority
++ * 2. If sharable is set in ipc mode (or by default ipc_mode is null), the container provides shm path,
++ *    in the case of sandbox API is used, the sandbox module has already provided shm path
++ * 3. Use the connected container's shm path if ipc_mode is set to container:<cid>, 
++ *    if connected containerd is a sandbox, use the sandbox's shm path
++ * 4. Use /dev/shm if ipc_mode is set to host
++ */
++static int set_shm_path(host_config *host_spec, container_config_v2_common_config *v2_spec)
++{
++    // ignore shm of system container
++    if (host_spec->system_container) {
++        return 0;
++    }
++
++    // case 1: Defined in mounts already
++    if (has_mount_shm(host_spec, v2_spec)) {
++        return 0;
++    }
++
++    // case 2: Container has its own IPC namespace
++    if (is_shareable_ipc(host_spec->ipc_mode)) {
++        return set_sharable_ipc_mode(host_spec, v2_spec);
++    }
++
++    // case 3: Connected container
++    if (namespace_is_container(host_spec->ipc_mode)) {
++        return set_connected_container_shm_path(host_spec, v2_spec);
++    }
++
++    // case 4: Host IPC namespace
++    if (namespace_is_host(host_spec->ipc_mode)) {
++        return set_host_ipc_shm_path(v2_spec);
++    }
++
++    // Otherwise, the case is unknown, nothing is set
++    free(v2_spec->shm_path);
++    v2_spec->shm_path = NULL;
++
++    return 0;
+ }
+ 
+ int destination_compare(const void *p1, const void *p2)
+-- 
+2.25.1
+

0051-Fix-memory-leak-in-set_connected_container_shm_path.patch

@@ -0,0 +1,26 @@
+From 1052a7b67a35b3504045729e1408d4ace8bf50ca Mon Sep 17 00:00:00 2001
+From: xuxuepeng <xuxuepeng1@huawei.com>
+Date: Tue, 9 Apr 2024 06:35:03 +0800
+Subject: [PATCH 051/149] Fix memory leak in set_connected_container_shm_path
+
+Signed-off-by: xuxuepeng <xuxuepeng1@huawei.com>
+---
+ src/daemon/modules/spec/specs_mount.c | 2 +-
+ 1 file changed, 1 insertion(+), 1 deletion(-)
+
+diff --git a/src/daemon/modules/spec/specs_mount.c b/src/daemon/modules/spec/specs_mount.c
+index 6903ae40..50ee9a85 100644
+--- a/src/daemon/modules/spec/specs_mount.c
++++ b/src/daemon/modules/spec/specs_mount.c
+@@ -2822,7 +2822,7 @@ static inline int set_sharable_ipc_mode(host_config *host_spec, container_config
+ static inline int set_connected_container_shm_path(host_config *host_spec, container_config_v2_common_config *v2_spec)
+ {
+     container_t *cont = NULL;
+-    char *tmp_cid = NULL;
++    __isula_auto_free char *tmp_cid = NULL;
+     char *right_path = NULL;
+ 
+ #ifdef ENABLE_CRI_API_V1
+-- 
+2.25.1
+

0052-init-enable_pod_events-as-false.patch

@@ -0,0 +1,62 @@
+From cfbb9f5ea40b3b654d7b6f9ad861877e97ed24be Mon Sep 17 00:00:00 2001
+From: jikai <jikai11@huawei.com>
+Date: Thu, 11 Apr 2024 02:04:47 +0000
+Subject: [PATCH 052/149] init enable_pod_events as false
+
+Signed-off-by: jikai <jikai11@huawei.com>
+---
+ src/daemon/entry/connect/grpc/cri/cri_service.cc               | 3 +--
+ src/daemon/entry/connect/grpc/cri/cri_service.h                | 1 -
+ .../connect/grpc/cri/v1/cri_v1_runtime_runtime_service.cc      | 2 +-
+ 3 files changed, 2 insertions(+), 4 deletions(-)
+
+diff --git a/src/daemon/entry/connect/grpc/cri/cri_service.cc b/src/daemon/entry/connect/grpc/cri/cri_service.cc
+index d10a60b5..80bcfef0 100644
+--- a/src/daemon/entry/connect/grpc/cri/cri_service.cc
++++ b/src/daemon/entry/connect/grpc/cri/cri_service.cc
+@@ -89,9 +89,8 @@ int CRIService::Init(const isulad_daemon_configs *config)
+ 
+ #ifdef ENABLE_CRI_API_V1
+     m_enableCRIV1 = config->enable_cri_v1;
+-    m_enablePodEvents = config->enable_pod_events;
+     if (m_enableCRIV1) {
+-        m_runtimeV1RuntimeService.Init(m_podSandboxImage, m_pluginManager, m_enablePodEvents, err);
++        m_runtimeV1RuntimeService.Init(m_podSandboxImage, m_pluginManager, config->enable_pod_events, err);
+         if (err.NotEmpty()) {
+             ERROR("Init CRI v1 runtime service failed: %s", err.GetCMessage());
+             return -1;
+diff --git a/src/daemon/entry/connect/grpc/cri/cri_service.h b/src/daemon/entry/connect/grpc/cri/cri_service.h
+index 041c7c63..77b2eb72 100644
+--- a/src/daemon/entry/connect/grpc/cri/cri_service.h
++++ b/src/daemon/entry/connect/grpc/cri/cri_service.h
+@@ -56,7 +56,6 @@ private:
+     std::string m_podSandboxImage;
+     std::shared_ptr<Network::PluginManager> m_pluginManager;
+     bool m_enableCRIV1;
+-    bool m_enablePodEvents;
+ };
+ 
+ }
+diff --git a/src/daemon/entry/connect/grpc/cri/v1/cri_v1_runtime_runtime_service.cc b/src/daemon/entry/connect/grpc/cri/v1/cri_v1_runtime_runtime_service.cc
+index bc5ab591..e2591ce0 100644
+--- a/src/daemon/entry/connect/grpc/cri/v1/cri_v1_runtime_runtime_service.cc
++++ b/src/daemon/entry/connect/grpc/cri/v1/cri_v1_runtime_runtime_service.cc
+@@ -62,6 +62,7 @@ void RuntimeV1RuntimeServiceImpl::Init(std::string &podSandboxImage,
+         return;
+     }
+ 
++    m_enablePodEvents = false;
+     if (enablePodEvents) {
+         if (mailbox_register_topic_handler(MAILBOX_TOPIC_CRI_CONTAINER, cri_container_topic_handler,
+                                            this, cri_container_topic_release, true) != 0) {
+@@ -72,7 +73,6 @@ void RuntimeV1RuntimeServiceImpl::Init(std::string &podSandboxImage,
+         m_enablePodEvents = enablePodEvents;
+     }
+ 
+-
+     m_rService = std::unique_ptr<CRIV1::CRIRuntimeService>(new CRIRuntimeServiceImpl(podSandboxImage, cb, networkPlugin, m_enablePodEvents));
+ }
+ 
+-- 
+2.25.1
+

0053-remove-container-root-path-in-rt_lcr_rm-if-lcr-runti.patch

@@ -0,0 +1,33 @@
+From 0cb96d50302f9f3ad1c17e0bb650ac37db4d5206 Mon Sep 17 00:00:00 2001
+From: jikai <jikai11@huawei.com>
+Date: Wed, 10 Apr 2024 08:41:46 +0000
+Subject: [PATCH 053/149] remove container root path in rt_lcr_rm if lcr
+ runtime missing
+
+Signed-off-by: jikai <jikai11@huawei.com>
+---
+ src/daemon/modules/runtime/engines/lcr/lcr_rt_ops.c | 8 +++++++-
+ 1 file changed, 7 insertions(+), 1 deletion(-)
+
+diff --git a/src/daemon/modules/runtime/engines/lcr/lcr_rt_ops.c b/src/daemon/modules/runtime/engines/lcr/lcr_rt_ops.c
+index 6b862958..978da079 100644
+--- a/src/daemon/modules/runtime/engines/lcr/lcr_rt_ops.c
++++ b/src/daemon/modules/runtime/engines/lcr/lcr_rt_ops.c
+@@ -238,7 +238,13 @@ int rt_lcr_rm(const char *name, const char *runtime, const rt_rm_params_t *param
+ 
+     engine_ops = engines_get_handler(runtime);
+     if (engine_ops == NULL || engine_ops->engine_delete_op == NULL) {
+-        ERROR("Failed to get engine delete operations");
++        // if engine_ops is NULL, container root path may have been corrupted, try to remove by daemon
++        // If user runs container with lcr but remove lcr runtime after, there might be resources remaining
++        ERROR("Failed to get engine delete operations, container %s root path may have been corrupted, try to remove by daemon", name);
++        if (remove_container_rootpath(name, params->rootpath) == 0) {
++            ret = 0;
++            goto out;
++        }
+         ret = -1;
+         goto out;
+     }
+-- 
+2.25.1
+

0054-ensure-sandbox-can-be-removed-if-sandbox-container-r.patch

@@ -0,0 +1,58 @@
+From 21afb41e02886df0b5251889cc443f28b7da274f Mon Sep 17 00:00:00 2001
+From: jikai <jikai11@huawei.com>
+Date: Thu, 11 Apr 2024 01:21:34 +0000
+Subject: [PATCH 054/149] ensure sandbox can be removed if sandbox container
+ removed
+
+Signed-off-by: jikai <jikai11@huawei.com>
+---
+ .../sandbox/controller/shim/shim_controller.cc   | 16 ++++++++++++----
+ src/daemon/sandbox/sandbox.cc                    |  3 ++-
+ 2 files changed, 14 insertions(+), 5 deletions(-)
+
+diff --git a/src/daemon/sandbox/controller/shim/shim_controller.cc b/src/daemon/sandbox/controller/shim/shim_controller.cc
+index 593fade9..4da637c7 100644
+--- a/src/daemon/sandbox/controller/shim/shim_controller.cc
++++ b/src/daemon/sandbox/controller/shim/shim_controller.cc
+@@ -517,12 +517,20 @@ bool ShimController::Shutdown(const std::string &sandboxId, Errors &error)
+     container_delete_response *response {nullptr};
+     int ret = m_cb->container.remove(request, &response);
+     auto responseWrapper = makeUniquePtrCStructWrapper<container_delete_response>(response, free_container_delete_response);
++    if (ret == 0) {
++        return true;
++    }
+ 
+-    if (ret != 0) {
+-        std::string msg = (response != nullptr && response->errmsg != nullptr) ? response->errmsg : "internal";
+-        ERROR("Failed to remove sandbox %s: %s", sandboxId.c_str(), msg.c_str());
+-        error.SetError(msg);
++    std::string errMsg = "internal";
++    if (response != nullptr && response->errmsg != nullptr) {
++        if (strstr(response->errmsg, "No such container") != nullptr) {
++            ERROR("Container for sandbox %s not found", sandboxId.c_str());
++            return true;
++        }
++        errMsg = response->errmsg;
+     }
++    ERROR("Failed to remove sandbox %s: %s", sandboxId.c_str(), errMsg.c_str());
++    error.SetError(errMsg);
+     return error.Empty();
+ }
+ 
+diff --git a/src/daemon/sandbox/sandbox.cc b/src/daemon/sandbox/sandbox.cc
+index c70116c1..bae5b8db 100644
+--- a/src/daemon/sandbox/sandbox.cc
++++ b/src/daemon/sandbox/sandbox.cc
+@@ -757,7 +757,8 @@ auto Sandbox::Remove(Errors &error) -> bool
+ 
+     WriteGuard<RWMutex> lock(m_mutex);
+ 
+-    if (!DoStop(DEFAULT_STOP_TIMEOUT, error)) {
++    // Only stop the sandbox when it is running
++    if (IsReady() && !DoStop(DEFAULT_STOP_TIMEOUT, error)) {
+         ERROR("Failed to stop Sandbox before removing, id='%s'", m_id.c_str());
+         return false;
+     }
+-- 
+2.25.1
+

0055-bugfix-for-shim-timeout-exit-error-log-changes.patch

@@ -0,0 +1,175 @@
+From 35ffb77f568124e6e7c8fd7b3d021878b92c13f7 Mon Sep 17 00:00:00 2001
+From: zhongtao <zhongtao17@huawei.com>
+Date: Tue, 9 Apr 2024 20:04:33 +0800
+Subject: [PATCH 055/149] bugfix for shim timeout exit error log changes
+
+Signed-off-by: zhongtao <zhongtao17@huawei.com>
+---
+ .../modules/runtime/isula/isula_rt_ops.c      | 55 ++++++++++++-------
+ 1 file changed, 34 insertions(+), 21 deletions(-)
+
+diff --git a/src/daemon/modules/runtime/isula/isula_rt_ops.c b/src/daemon/modules/runtime/isula/isula_rt_ops.c
+index bc3c36c8..1875cf5b 100644
+--- a/src/daemon/modules/runtime/isula/isula_rt_ops.c
++++ b/src/daemon/modules/runtime/isula/isula_rt_ops.c
+@@ -861,6 +861,8 @@ static int shim_create(shim_create_args *args)
+     pid_t pid = 0;
+     int shim_stderr_pipe[2] = { -1, -1 };
+     int shim_stdout_pipe[2] = { -1, -1 };
++    // used to accept exec error msg
++    int exec_err_pipe[2] = {-1, -1};
+     int num = 0;
+     int ret = 0;
+     char exec_buff[BUFSIZ + 1] = { 0 };
+@@ -904,6 +906,11 @@ static int shim_create(shim_create_args *args)
+         return -1;
+     }
+ 
++    if (pipe2(exec_err_pipe, O_CLOEXEC) != 0) {
++        ERROR("Failed to create pipe for exec err");
++        return -1;
++    }
++
+     pid = fork();
+     if (pid < 0) {
+         SYSERROR("Failed fork for shim parent");
+@@ -911,30 +918,32 @@ static int shim_create(shim_create_args *args)
+         close(shim_stderr_pipe[1]);
+         close(shim_stdout_pipe[0]);
+         close(shim_stdout_pipe[1]);
++        close(exec_err_pipe[0]);
++        close(exec_err_pipe[1]);
+         return -1;
+     }
+ 
+     if (pid == (pid_t)0) {
+         if (chdir(args->workdir) < 0) {
+-            (void)dprintf(shim_stderr_pipe[1], "%s: failed chdir to %s", args->id, args->workdir);
++            (void)dprintf(exec_err_pipe[1], "%s: failed chdir to %s", args->id, args->workdir);
+             exit(EXIT_FAILURE);
+         }
+ 
+         //prevent the child process from having the same standard streams as the parent process
+         if (isula_null_stdfds() != 0) {
+-            (void)dprintf(shim_stderr_pipe[1], "failed to set std console to /dev/null");
++            (void)dprintf(exec_err_pipe[1], "failed to set std console to /dev/null");
+             exit(EXIT_FAILURE);           
+         }
+ 
+         if (args->fg) {
+             // child process, dup2 shim_stdout_pipe[1] to STDOUT, get container process exit_code in STDOUT
+             if (dup2(shim_stdout_pipe[1], STDOUT_FILENO) < 0) {
+-                (void)dprintf(shim_stderr_pipe[1], "Dup stdout fd error: %s", strerror(errno));
++                (void)dprintf(exec_err_pipe[1], "Dup stdout fd error: %s", strerror(errno));
+                 exit(EXIT_FAILURE);
+             }
+             // child process, dup2 shim_stderr_pipe[1] to STDERR, get isulad-shim errmsg in STDERR
+             if (dup2(shim_stderr_pipe[1], STDERR_FILENO) < 0) {
+-                (void)dprintf(shim_stderr_pipe[1], "Dup stderr fd error: %s", strerror(errno));
++                (void)dprintf(exec_err_pipe[1], "Dup stderr fd error: %s", strerror(errno));
+                 exit(EXIT_FAILURE);
+             }
+             goto realexec;
+@@ -942,18 +951,18 @@ static int shim_create(shim_create_args *args)
+ 
+         // clear NOTIFY_SOCKET from the env to adapt runc create
+         if (unsetenv("NOTIFY_SOCKET") != 0) {
+-            (void)dprintf(shim_stderr_pipe[1], "%s: unset env NOTIFY_SOCKET failed %s", args->id, strerror(errno));
++            (void)dprintf(exec_err_pipe[1], "%s: unset env NOTIFY_SOCKET failed %s", args->id, strerror(errno));
+             exit(EXIT_FAILURE);
+         }
+ 
+         pid = fork();
+         if (pid < 0) {
+-            (void)dprintf(shim_stderr_pipe[1], "%s: fork shim-process failed %s", args->id, strerror(errno));
++            (void)dprintf(exec_err_pipe[1], "%s: fork shim-process failed %s", args->id, strerror(errno));
+             _exit(EXIT_FAILURE);
+         }
+         if (pid != 0) {
+             if (file_write_int(fpid, pid) != 0) {
+-                (void)dprintf(shim_stderr_pipe[1], "%s: write %s with %d failed", args->id, fpid, pid);
++                (void)dprintf(exec_err_pipe[1], "%s: write %s with %d failed", args->id, fpid, pid);
+             }
+             _exit(EXIT_SUCCESS);
+         }
+@@ -962,35 +971,38 @@ realexec:
+         /* real shim process. */
+         close(shim_stderr_pipe[0]);
+         close(shim_stdout_pipe[0]);
++        close(exec_err_pipe[0]);
+ 
+         if (setsid() < 0) {
+-            (void)dprintf(shim_stderr_pipe[1], "%s: failed setsid for process %d", args->id, getpid());
++            (void)dprintf(exec_err_pipe[1], "%s: failed setsid for process %d", args->id, getpid());
+             exit(EXIT_FAILURE);
+         }
+ 
+         if (util_check_inherited(true, shim_stderr_pipe[1]) != 0) {
+-            (void)dprintf(shim_stderr_pipe[1], "close inherited fds failed");
++            (void)dprintf(exec_err_pipe[1], "close inherited fds failed");
+             exit(EXIT_FAILURE);
+         }
+ 
+         if (setenv(SHIIM_LOG_PATH_ENV, engine_log_path, 1) != 0) {
+-            (void)dprintf(shim_stderr_pipe[1], "%s: failed to set SHIIM_LOG_PATH_ENV for process %d", args->id, getpid());
++            (void)dprintf(exec_err_pipe[1], "%s: failed to set SHIIM_LOG_PATH_ENV for process %d", args->id, getpid());
+             exit(EXIT_FAILURE);
+         }
+ 
+         if (setenv(SHIIM_LOG_LEVEL_ENV, log_level, 1) != 0) {
+-            (void)dprintf(shim_stderr_pipe[1], "%s: failed to set SHIIM_LOG_LEVEL_ENV env for process %d", args->id, getpid());
++            (void)dprintf(exec_err_pipe[1], "%s: failed to set SHIIM_LOG_LEVEL_ENV env for process %d", args->id, getpid());
+             exit(EXIT_FAILURE);
+         }
+ 
+         execvp(SHIM_BINARY, (char * const *)params);
+-        (void)dprintf(shim_stderr_pipe[1], "run process: %s failed: %s", SHIM_BINARY, strerror(errno));
++        (void)dprintf(exec_err_pipe[1], "run process: %s failed: %s", SHIM_BINARY, strerror(errno));
+         exit(EXIT_FAILURE);
+     }
+ 
+     close(shim_stderr_pipe[1]);
+     close(shim_stdout_pipe[1]);
+-    num = util_read_nointr(shim_stderr_pipe[0], exec_buff, sizeof(exec_buff) - 1);
++    close(exec_err_pipe[1]);
++    num = util_read_nointr(exec_err_pipe[0], exec_buff, sizeof(exec_buff) - 1);
++    close(exec_err_pipe[0]);
+ 
+     status = util_wait_for_pid_status(pid);
+     if (status < 0) {
+@@ -1035,8 +1047,10 @@ realexec:
+ out:
+     close(shim_stdout_pipe[0]);
+     if (ret != 0) {
+-        show_runtime_errlog(args->workdir);
+         show_shim_errlog(shim_stderr_pipe[0]);
++        // Since users are more concerned about runtime error information, 
++        // the runtime log will overwrite the shim log if it exists.
++        show_runtime_errlog(args->workdir);
+         if (args->timeout != NULL) {
+             kill(pid, SIGKILL); /* can kill other process? */
+         }
+@@ -1491,14 +1505,13 @@ int rt_isula_exec(const char *id, const char *runtime, const rt_exec_params_t *p
+     args.exit_code = exit_code;
+     args.timeout = timeout;
+     ret = shim_create(&args);
+-    if (args.shim_exit_code == SHIM_EXIT_TIMEOUT) {
+-        ret = -1;
+-        isulad_set_error_message("Exec container error;exec timeout");
+-        ERROR("isulad-shim %d exit for execing timeout", pid);
+-        goto errlog_out;
+-    }
+     if (ret != 0) {
+-        ERROR("%s: failed create shim process for exec %s", id, exec_id);
++        if (args.shim_exit_code == SHIM_EXIT_TIMEOUT) {
++            isulad_set_error_message("Exec container error;exec timeout");
++            ERROR("isulad-shim %d exit for execing timeout", pid);
++        } else {
++            ERROR("%s: failed create shim process for exec %s", id, exec_id);
++        }
+         goto errlog_out;
+     }
+ 
+-- 
+2.25.1
+

0056-bugfix-for-the-pre-created-pipe-was-not-closed-when-.patch

@@ -0,0 +1,36 @@
+From e8ba4368f4be369f99d7da6fc04dcbe173985cd0 Mon Sep 17 00:00:00 2001
+From: zhongtao <zhongtao17@huawei.com>
+Date: Fri, 12 Apr 2024 14:42:05 +0800
+Subject: [PATCH 056/149] bugfix for the pre-created pipe was not closed when
+ the pipe creation failed
+
+Signed-off-by: zhongtao <zhongtao17@huawei.com>
+---
+ src/daemon/modules/runtime/isula/isula_rt_ops.c | 6 ++++++
+ 1 file changed, 6 insertions(+)
+
+diff --git a/src/daemon/modules/runtime/isula/isula_rt_ops.c b/src/daemon/modules/runtime/isula/isula_rt_ops.c
+index 1875cf5b..47a14b1d 100644
+--- a/src/daemon/modules/runtime/isula/isula_rt_ops.c
++++ b/src/daemon/modules/runtime/isula/isula_rt_ops.c
+@@ -903,11 +903,17 @@ static int shim_create(shim_create_args *args)
+ 
+     if (pipe2(shim_stdout_pipe, O_CLOEXEC) != 0) {
+         ERROR("Failed to create pipe for shim stdout");
++        close(shim_stderr_pipe[0]);
++        close(shim_stderr_pipe[1]);
+         return -1;
+     }
+ 
+     if (pipe2(exec_err_pipe, O_CLOEXEC) != 0) {
+         ERROR("Failed to create pipe for exec err");
++        close(shim_stderr_pipe[0]);
++        close(shim_stderr_pipe[1]);
++        close(shim_stdout_pipe[0]);
++        close(shim_stdout_pipe[1]);
+         return -1;
+     }
+ 
+-- 
+2.25.1
+

0057-add-debug-msg-info-in-image_load.sh.patch

@@ -0,0 +1,26 @@
+From c3f7cf2a54188e5fc890a8d23b95254ac69cfa52 Mon Sep 17 00:00:00 2001
+From: zhongtao <zhongtao17@huawei.com>
+Date: Fri, 12 Apr 2024 15:23:07 +0800
+Subject: [PATCH 057/149] add debug msg info in image_load.sh
+
+Signed-off-by: zhongtao <zhongtao17@huawei.com>
+---
+ CI/test_cases/image_cases/image_load.sh | 2 ++
+ 1 file changed, 2 insertions(+)
+
+diff --git a/CI/test_cases/image_cases/image_load.sh b/CI/test_cases/image_cases/image_load.sh
+index a2cada5f..d50b3203 100755
+--- a/CI/test_cases/image_cases/image_load.sh
++++ b/CI/test_cases/image_cases/image_load.sh
+@@ -103,6 +103,8 @@ function test_concurrent_load()
+       [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - fail to do isulad load $i" && ((ret++))
+   done
+ 
++  tail -n 50 /var/lib/isulad/isulad.log
++
+   ubuntu_id=`isula inspect -f '{{.image.id}}' ubuntu`
+   [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - fail to inspect image: ubuntu" && ((ret++))
+ 
+-- 
+2.25.1
+

0058-empty-pointer-check-in-lcr_rt_ops.patch

@@ -0,0 +1,214 @@
+From ea1bc00c894a3717ea375f5ff40c3eb05447ae17 Mon Sep 17 00:00:00 2001
+From: jikai <jikai11@huawei.com>
+Date: Sat, 13 Apr 2024 14:07:33 +0000
+Subject: [PATCH 058/149] empty pointer check in lcr_rt_ops
+
+Signed-off-by: jikai <jikai11@huawei.com>
+---
+ .../modules/runtime/engines/lcr/lcr_rt_ops.c  | 84 ++++++++++++++++++-
+ 1 file changed, 81 insertions(+), 3 deletions(-)
+
+diff --git a/src/daemon/modules/runtime/engines/lcr/lcr_rt_ops.c b/src/daemon/modules/runtime/engines/lcr/lcr_rt_ops.c
+index 978da079..a89d0375 100644
+--- a/src/daemon/modules/runtime/engines/lcr/lcr_rt_ops.c
++++ b/src/daemon/modules/runtime/engines/lcr/lcr_rt_ops.c
+@@ -53,6 +53,11 @@ int rt_lcr_create(const char *name, const char *runtime, const rt_create_params_
+     char *runtime_root = NULL;
+     struct engine_operation *engine_ops = NULL;
+ 
++    if (name == NULL || runtime == NULL || params == NULL) {
++        ERROR("Nullptr arguments not allowed");
++        return -1;
++    }
++
+     if (conf_get_systemd_cgroup()) {
+         ERROR("Systemd cgroup not supported for lcr runtime");
+         isulad_set_error_message("Systemd cgroup not supported for lcr runtime");
+@@ -129,6 +134,11 @@ int rt_lcr_start(const char *name, const char *runtime, const rt_start_params_t
+     struct engine_operation *engine_ops = NULL;
+     engine_start_request_t request = { 0 };
+ 
++    if (name == NULL || runtime == NULL || params == NULL || pid_info == NULL) {
++        ERROR("Nullptr arguments not allowed");
++        return -1;
++    }
++
+     engine_ops = engines_get_handler(runtime);
+     if (engine_ops == NULL || engine_ops->engine_start_op == NULL) {
+         ERROR("Failed to get engine start operations");
+@@ -183,6 +193,11 @@ int rt_lcr_clean_resource(const char *name, const char *runtime, const rt_clean_
+     int ret = 0;
+     struct engine_operation *engine_ops = NULL;
+ 
++    if (name == NULL || runtime == NULL || params == NULL) {
++        ERROR("Nullptr arguments not allowed");
++        return -1;
++    }
++
+     engine_ops = engines_get_handler(runtime);
+     if (engine_ops == NULL || engine_ops->engine_clean_op == NULL) {
+         ERROR("Failed to get engine clean operations");
+@@ -236,6 +251,15 @@ int rt_lcr_rm(const char *name, const char *runtime, const rt_rm_params_t *param
+     int ret = 0;
+     struct engine_operation *engine_ops = NULL;
+ 
++    if (name == NULL || runtime == NULL || params == NULL) {
++        ERROR("Nullptr arguments not allowed");
++        return -1;
++    }
++    if (params->rootpath == NULL) {
++        ERROR("Missing root path");
++        return -1;
++    }
++
+     engine_ops = engines_get_handler(runtime);
+     if (engine_ops == NULL || engine_ops->engine_delete_op == NULL) {
+         // if engine_ops is NULL, container root path may have been corrupted, try to remove by daemon
+@@ -284,6 +308,11 @@ int rt_lcr_status(const char *name, const char *runtime, const rt_status_params_
+     int nret = 0;
+     struct engine_operation *engine_ops = NULL;
+ 
++    if (name == NULL || runtime == NULL || params == NULL || status == NULL) {
++        ERROR("Nullptr arguments not allowed");
++        return -1;
++    }
++
+     engine_ops = engines_get_handler(runtime);
+     if (engine_ops == NULL || engine_ops->engine_get_container_status_op == NULL) {
+         ERROR("Failed to get engine status operations");
+@@ -322,6 +351,11 @@ int rt_lcr_resources_stats(const char *name, const char *runtime, const rt_stats
+     int nret = 0;
+     struct engine_operation *engine_ops = NULL;
+ 
++    if (name == NULL || runtime == NULL || params == NULL || rs_stats == NULL) {
++        ERROR("Nullptr arguments not allowed");
++        return -1;
++    }
++
+     engine_ops = engines_get_handler(runtime);
+     if (engine_ops == NULL || engine_ops->engine_get_container_resources_stats_op == NULL) {
+         ERROR("Failed to get engine stats operations");
+@@ -451,6 +485,11 @@ int rt_lcr_exec(const char *id, const char *runtime, const rt_exec_params_t *par
+     char *user = NULL;
+     char *add_gids = NULL;
+ 
++    if (id == NULL || runtime == NULL || params == NULL || exit_code == NULL) {
++        ERROR("Nullptr arguments not allowed");
++        return -1;
++    }
++
+     engine_ops = engines_get_handler(runtime);
+     if (engine_ops == NULL || engine_ops->engine_exec_op == NULL) {
+         DEBUG("Failed to get engine exec operations");
+@@ -519,6 +558,11 @@ int rt_lcr_pause(const char *name, const char *runtime, const rt_pause_params_t
+     int ret = 0;
+     struct engine_operation *engine_ops = NULL;
+ 
++    if (name == NULL || runtime == NULL || params == NULL) {
++        ERROR("Nullptr arguments not allowed");
++        return -1;
++    }
++
+     engine_ops = engines_get_handler(runtime);
+     if (engine_ops == NULL || engine_ops->engine_pause_op == NULL) {
+         DEBUG("Failed to get engine pause operations");
+@@ -549,6 +593,11 @@ int rt_lcr_resume(const char *name, const char *runtime, const rt_resume_params_
+     int ret = 0;
+     struct engine_operation *engine_ops = NULL;
+ 
++    if (name == NULL || runtime == NULL || params == NULL) {
++        ERROR("Nullptr arguments not allowed");
++        return -1;
++    }
++
+     engine_ops = engines_get_handler(runtime);
+     if (engine_ops == NULL || engine_ops->engine_resume_op == NULL) {
+         DEBUG("Failed to get engine resume operations");
+@@ -579,6 +628,11 @@ int rt_lcr_attach(const char *name, const char *runtime, const rt_attach_params_
+     int ret = 0;
+     struct engine_operation *engine_ops = NULL;
+ 
++    if (name == NULL || runtime == NULL || params == NULL) {
++        ERROR("Null argument");
++        return -1;
++    }
++
+     engine_ops = engines_get_handler(runtime);
+     if (engine_ops == NULL || engine_ops->engine_console_op == NULL) {
+         DEBUG("Failed to get engine attach operations");
+@@ -641,6 +695,11 @@ int rt_lcr_update(const char *id, const char *runtime, const rt_update_params_t
+     struct engine_operation *engine_ops = NULL;
+     struct engine_cgroup_resources cr = { 0 };
+ 
++    if (id == NULL || runtime == NULL || params == NULL) {
++        ERROR("Nullptr arguments not allowed");
++        return -1;
++    }
++
+     engine_ops = engines_get_handler(runtime);
+     if (engine_ops == NULL || engine_ops->engine_update_op == NULL) {
+         DEBUG("Failed to get engine update operations");
+@@ -673,10 +732,9 @@ int rt_lcr_listpids(const char *name, const char *runtime, const rt_listpids_par
+     int ret = 0;
+     struct engine_operation *engine_ops = NULL;
+ 
+-    if (out == NULL) {
++    if (name == NULL || runtime == NULL || params == NULL || out == NULL) {
+         ERROR("Invalid arguments");
+-        ret = -1;
+-        goto out;
++        return -1;
+     }
+ 
+     engine_ops = engines_get_handler(runtime);
+@@ -709,6 +767,11 @@ int rt_lcr_resize(const char *id, const char *runtime, const rt_resize_params_t
+     int ret = 0;
+     struct engine_operation *engine_ops = NULL;
+ 
++    if (id == NULL || runtime == NULL || params == NULL) {
++        ERROR("Invalid arguments");
++        return -1;
++    }
++
+     engine_ops = engines_get_handler(runtime);
+     if (engine_ops == NULL || engine_ops->engine_resize_op == NULL) {
+         DEBUG("Failed to get engine resume operations");
+@@ -740,6 +803,11 @@ int rt_lcr_exec_resize(const char *id, const char *runtime, const rt_exec_resize
+     int ret = 0;
+     struct engine_operation *engine_ops = NULL;
+ 
++    if (id == NULL || runtime == NULL || params == NULL) {
++        ERROR("Nullptr arguments not allowed");
++        return -1;
++    }
++
+     engine_ops = engines_get_handler(runtime);
+     if (engine_ops == NULL || engine_ops->engine_resize_op == NULL) {
+         DEBUG("Failed to get engine resume operations");
+@@ -767,6 +835,11 @@ out:
+ 
+ int rt_lcr_kill(const char *id, const char *runtime, const rt_kill_params_t *params)
+ {
++    if (id == NULL || runtime == NULL || params == NULL || params->pid < 0) {
++        ERROR("Invalid arguments not allowed");
++        return -1;
++    }
++
+     if (util_process_alive(params->pid, params->start_time) == false) {
+         if (params->signal == params->stop_signal || params->signal == SIGKILL) {
+             WARN("Process %d is not alive", params->pid);
+@@ -798,6 +871,11 @@ int rt_lcr_rebuild_config(const char *name, const char *runtime, const rt_rebuil
+     oci_runtime_spec *oci_spec = NULL;
+     __isula_auto_free parser_error err = NULL;
+ 
++    if (name == NULL || runtime == NULL || params == NULL) {
++        ERROR("Invalid arguments not allowed");
++        return -1;
++    }
++
+     engine_ops = engines_get_handler(runtime);
+     if (engine_ops == NULL || engine_ops->engine_create_op == NULL) {
+         ERROR("Failed to get engine rebuild config operations");
+-- 
+2.25.1
+

0059-modify-some-grpc-status-codes-of-cri-in-case-of-erro.patch

@@ -0,0 +1,315 @@
+From 628f4ceb329e16991ed33d3a460bcf8f5542ba99 Mon Sep 17 00:00:00 2001
+From: jikai <jikai11@huawei.com>
+Date: Mon, 8 Apr 2024 09:30:04 +0000
+Subject: [PATCH 059/149] modify some grpc status codes of cri in case of error
+
+Signed-off-by: jikai <jikai11@huawei.com>
+---
+ .../cri/v1/cri_v1_runtime_runtime_service.cc  | 41 +++++++++++++------
+ .../cri/v1/cri_v1_runtime_runtime_service.h   |  3 ++
+ .../v1alpha/cri_runtime_runtime_service.cc    | 41 +++++++++++++------
+ .../cri/v1alpha/cri_runtime_runtime_service.h |  2 +
+ 4 files changed, 63 insertions(+), 24 deletions(-)
+
+diff --git a/src/daemon/entry/connect/grpc/cri/v1/cri_v1_runtime_runtime_service.cc b/src/daemon/entry/connect/grpc/cri/v1/cri_v1_runtime_runtime_service.cc
+index e2591ce0..fb5aad3c 100644
+--- a/src/daemon/entry/connect/grpc/cri/v1/cri_v1_runtime_runtime_service.cc
++++ b/src/daemon/entry/connect/grpc/cri/v1/cri_v1_runtime_runtime_service.cc
+@@ -50,6 +50,23 @@ static void cri_container_topic_release(void *arg)
+     delete resp;
+ }
+ 
++grpc::Status RuntimeV1RuntimeServiceImpl::ToGRPCStatus(Errors &error)
++{
++    if (error.Empty()) {
++        return grpc::Status::OK;
++    }
++    if (error.GetMessage().find("Failed to find") != std::string::npos) {
++        return grpc::Status(grpc::StatusCode::NOT_FOUND, error.GetMessage());
++    }
++
++    // Attach exceeded timeout for lxc and Exec container error;exec timeout for runc
++    if (error.GetMessage().find("Attach exceeded timeout") != std::string::npos
++        || error.GetMessage().find("Exec container error;exec timeout") != std::string::npos) {
++        return grpc::Status(grpc::StatusCode::DEADLINE_EXCEEDED, error.GetMessage());
++    }
++    return grpc::Status(grpc::StatusCode::UNKNOWN, error.GetMessage());
++}
++
+ void RuntimeV1RuntimeServiceImpl::Init(std::string &podSandboxImage,
+                                        std::shared_ptr<Network::PluginManager> networkPlugin,
+                                        bool enablePodEvents, Errors &err)
+@@ -167,7 +184,7 @@ grpc::Status RuntimeV1RuntimeServiceImpl::CreateContainer(grpc::ServerContext *c
+         m_rService->CreateContainer(request->pod_sandbox_id(), request->config(), request->sandbox_config(), error);
+     if (!error.Empty() || responseID.empty()) {
+         ERROR("Object: CRI, Type: Failed to create container");
+-        return grpc::Status(grpc::StatusCode::UNKNOWN, error.GetMessage());
++        return ToGRPCStatus(error);
+     }
+     reply->set_container_id(responseID);
+ 
+@@ -192,7 +209,7 @@ grpc::Status RuntimeV1RuntimeServiceImpl::StartContainer(grpc::ServerContext *co
+     m_rService->StartContainer(request->container_id(), error);
+     if (!error.Empty()) {
+         ERROR("Object: CRI, Type: Failed to start container %s", request->container_id().c_str());
+-        return grpc::Status(grpc::StatusCode::UNKNOWN, error.GetMessage());
++        return ToGRPCStatus(error);
+     }
+ 
+     EVENT("Event: {Object: CRI, Type: Started Container: %s}", request->container_id().c_str());
+@@ -216,7 +233,7 @@ grpc::Status RuntimeV1RuntimeServiceImpl::StopContainer(grpc::ServerContext *con
+     m_rService->StopContainer(request->container_id(), (int64_t)request->timeout(), error);
+     if (!error.Empty()) {
+         ERROR("Object: CRI, Type: Failed to stop container %s", request->container_id().c_str());
+-        return grpc::Status(grpc::StatusCode::UNKNOWN, error.GetMessage());
++        return ToGRPCStatus(error);
+     }
+ 
+     EVENT("Event: {Object: CRI, Type: Stopped Container: %s}", request->container_id().c_str());
+@@ -240,7 +257,7 @@ grpc::Status RuntimeV1RuntimeServiceImpl::RemoveContainer(grpc::ServerContext *c
+     m_rService->RemoveContainer(request->container_id(), error);
+     if (!error.Empty()) {
+         ERROR("Object: CRI, Type: Failed to remove container %s", request->container_id().c_str());
+-        return grpc::Status(grpc::StatusCode::UNKNOWN, error.GetMessage());
++        return ToGRPCStatus(error);
+     }
+ 
+     EVENT("Event: {Object: CRI, Type: Removed Container: %s}", request->container_id().c_str());
+@@ -359,7 +376,7 @@ grpc::Status RuntimeV1RuntimeServiceImpl::ContainerStatus(grpc::ServerContext *c
+         m_rService->ContainerStatus(request->container_id(), error);
+     if (!error.Empty() || !contStatus) {
+         ERROR("Object: CRI, Type: Failed to get container status %s", request->container_id().c_str());
+-        return grpc::Status(grpc::StatusCode::UNKNOWN, error.GetMessage());
++        return ToGRPCStatus(error);
+     }
+     *(reply->mutable_status()) = *contStatus;
+ 
+@@ -384,7 +401,7 @@ grpc::Status RuntimeV1RuntimeServiceImpl::ExecSync(grpc::ServerContext *context,
+     m_rService->ExecSync(request->container_id(), request->cmd(), request->timeout(), reply, error);
+     if (!error.Empty()) {
+         ERROR("Object: CRI, Type: Failed to sync exec container: %s", request->container_id().c_str());
+-        return grpc::Status(grpc::StatusCode::UNKNOWN, error.GetMessage());
++        return ToGRPCStatus(error);
+     }
+ 
+     WARN("Event: {Object: CRI, Type: sync execed Container: %s}", request->container_id().c_str());
+@@ -437,7 +454,7 @@ grpc::Status RuntimeV1RuntimeServiceImpl::StopPodSandbox(grpc::ServerContext *co
+     if (!error.Empty()) {
+         ERROR("Object: CRI, Type: Failed to stop pod:%s due to %s", request->pod_sandbox_id().c_str(),
+               error.GetMessage().c_str());
+-        return grpc::Status(grpc::StatusCode::UNKNOWN, error.GetMessage());
++        return ToGRPCStatus(error);
+     }
+ 
+     EVENT("Event: {Object: CRI, Type: Stopped Pod: %s}", request->pod_sandbox_id().c_str());
+@@ -462,7 +479,7 @@ grpc::Status RuntimeV1RuntimeServiceImpl::RemovePodSandbox(grpc::ServerContext *
+     if (!error.Empty()) {
+         ERROR("Object: CRI, Type: Failed to remove pod:%s due to %s", request->pod_sandbox_id().c_str(),
+               error.GetMessage().c_str());
+-        return grpc::Status(grpc::StatusCode::UNKNOWN, error.GetMessage());
++        return ToGRPCStatus(error);
+     }
+ 
+     EVENT("Event: {Object: CRI, Type: Removed Pod: %s}", request->pod_sandbox_id().c_str());
+@@ -487,7 +504,7 @@ grpc::Status RuntimeV1RuntimeServiceImpl::PodSandboxStatus(grpc::ServerContext *
+     if (!error.Empty()) {
+         ERROR("Object: CRI, Type: Failed to status pod:%s due to %s", request->pod_sandbox_id().c_str(),
+               error.GetMessage().c_str());
+-        return grpc::Status(grpc::StatusCode::UNKNOWN, error.GetMessage());
++        return ToGRPCStatus(error);
+     }
+ 
+     INFO("Event: {Object: CRI, Type: Statused Pod: %s}", request->pod_sandbox_id().c_str());
+@@ -608,7 +625,7 @@ RuntimeV1RuntimeServiceImpl::UpdateContainerResources(grpc::ServerContext *conte
+     if (error.NotEmpty()) {
+         ERROR("Object: CRI, Type: Failed to update container:%s due to %s", request->container_id().c_str(),
+               error.GetMessage().c_str());
+-        return grpc::Status(grpc::StatusCode::UNKNOWN, error.GetMessage());
++        return ToGRPCStatus(error);
+     }
+ 
+     WARN("Event: {Object: CRI, Type: Updated container resources: %s}", request->container_id().c_str());
+@@ -633,7 +650,7 @@ grpc::Status RuntimeV1RuntimeServiceImpl::Exec(grpc::ServerContext *context,
+     if (!error.Empty()) {
+         ERROR("Object: CRI, Type: Failed to exec container:%s due to %s", request->container_id().c_str(),
+               error.GetMessage().c_str());
+-        return grpc::Status(grpc::StatusCode::UNKNOWN, error.GetMessage());
++        return ToGRPCStatus(error);
+     }
+ 
+     EVENT("Event: {Object: CRI, Type: execed Container: %s}", request->container_id().c_str());
+@@ -658,7 +675,7 @@ grpc::Status RuntimeV1RuntimeServiceImpl::Attach(grpc::ServerContext *context,
+     if (!error.Empty()) {
+         ERROR("Object: CRI, Type: Failed to attach container:%s due to %s", request->container_id().c_str(),
+               error.GetMessage().c_str());
+-        return grpc::Status(grpc::StatusCode::UNKNOWN, error.GetMessage());
++        return ToGRPCStatus(error);
+     }
+ 
+     EVENT("Event: {Object: CRI, Type: attched Container: %s}", request->container_id().c_str());
+diff --git a/src/daemon/entry/connect/grpc/cri/v1/cri_v1_runtime_runtime_service.h b/src/daemon/entry/connect/grpc/cri/v1/cri_v1_runtime_runtime_service.h
+index 842d1811..1cf375a4 100644
+--- a/src/daemon/entry/connect/grpc/cri/v1/cri_v1_runtime_runtime_service.h
++++ b/src/daemon/entry/connect/grpc/cri/v1/cri_v1_runtime_runtime_service.h
+@@ -114,6 +114,9 @@ public:
+                                     grpc::ServerWriter<runtime::v1::ContainerEventResponse> *writer) override;
+ 
+ private:
++
++    grpc::Status ToGRPCStatus(Errors &error);
++
+     std::unique_ptr<CRIV1::CRIRuntimeService> m_rService;
+     bool m_enablePodEvents;
+ };
+diff --git a/src/daemon/entry/connect/grpc/cri/v1alpha/cri_runtime_runtime_service.cc b/src/daemon/entry/connect/grpc/cri/v1alpha/cri_runtime_runtime_service.cc
+index 5e85702c..1c83f4ca 100644
+--- a/src/daemon/entry/connect/grpc/cri/v1alpha/cri_runtime_runtime_service.cc
++++ b/src/daemon/entry/connect/grpc/cri/v1alpha/cri_runtime_runtime_service.cc
+@@ -23,6 +23,23 @@
+ 
+ using namespace CRI;
+ 
++grpc::Status RuntimeRuntimeServiceImpl::ToGRPCStatus(Errors &error)
++{
++    if (error.Empty()) {
++        return grpc::Status::OK;
++    }
++    if (error.GetMessage().find("Failed to find") != std::string::npos) {
++        return grpc::Status(grpc::StatusCode::NOT_FOUND, error.GetMessage());
++    }
++
++    // Attach exceeded timeout for lxc and Exec container error;exec timeout for runc
++    if (error.GetMessage().find("Attach exceeded timeout") != std::string::npos
++        || error.GetMessage().find("Exec container error;exec timeout") != std::string::npos) {
++        return grpc::Status(grpc::StatusCode::DEADLINE_EXCEEDED, error.GetMessage());
++    }
++    return grpc::Status(grpc::StatusCode::UNKNOWN, error.GetMessage());
++}
++
+ void RuntimeRuntimeServiceImpl::Init(std::string &podSandboxImage,
+                                      std::shared_ptr<Network::PluginManager> networkPlugin, Errors &err)
+ {
+@@ -80,7 +97,7 @@ grpc::Status RuntimeRuntimeServiceImpl::CreateContainer(grpc::ServerContext *con
+         m_rService->CreateContainer(request->pod_sandbox_id(), request->config(), request->sandbox_config(), error);
+     if (!error.Empty() || responseID.empty()) {
+         ERROR("Object: CRI, Type: Failed to create container");
+-        return grpc::Status(grpc::StatusCode::UNKNOWN, error.GetMessage());
++        return ToGRPCStatus(error);
+     }
+     reply->set_container_id(responseID);
+ 
+@@ -105,7 +122,7 @@ grpc::Status RuntimeRuntimeServiceImpl::StartContainer(grpc::ServerContext *cont
+     m_rService->StartContainer(request->container_id(), error);
+     if (!error.Empty()) {
+         ERROR("Object: CRI, Type: Failed to start container %s", request->container_id().c_str());
+-        return grpc::Status(grpc::StatusCode::UNKNOWN, error.GetMessage());
++        return ToGRPCStatus(error);
+     }
+ 
+     EVENT("Event: {Object: CRI, Type: Started Container: %s}", request->container_id().c_str());
+@@ -129,7 +146,7 @@ grpc::Status RuntimeRuntimeServiceImpl::StopContainer(grpc::ServerContext *conte
+     m_rService->StopContainer(request->container_id(), (int64_t)request->timeout(), error);
+     if (!error.Empty()) {
+         ERROR("Object: CRI, Type: Failed to stop container %s", request->container_id().c_str());
+-        return grpc::Status(grpc::StatusCode::UNKNOWN, error.GetMessage());
++        return ToGRPCStatus(error);
+     }
+ 
+     EVENT("Event: {Object: CRI, Type: Stopped Container: %s}", request->container_id().c_str());
+@@ -153,7 +170,7 @@ grpc::Status RuntimeRuntimeServiceImpl::RemoveContainer(grpc::ServerContext *con
+     m_rService->RemoveContainer(request->container_id(), error);
+     if (!error.Empty()) {
+         ERROR("Object: CRI, Type: Failed to remove container %s", request->container_id().c_str());
+-        return grpc::Status(grpc::StatusCode::UNKNOWN, error.GetMessage());
++        return ToGRPCStatus(error);
+     }
+ 
+     EVENT("Event: {Object: CRI, Type: Removed Container: %s}", request->container_id().c_str());
+@@ -272,7 +289,7 @@ grpc::Status RuntimeRuntimeServiceImpl::ContainerStatus(grpc::ServerContext *con
+         m_rService->ContainerStatus(request->container_id(), error);
+     if (!error.Empty() || !contStatus) {
+         ERROR("Object: CRI, Type: Failed to get container status %s", request->container_id().c_str());
+-        return grpc::Status(grpc::StatusCode::UNKNOWN, error.GetMessage());
++        return ToGRPCStatus(error);
+     }
+     *(reply->mutable_status()) = *contStatus;
+ 
+@@ -297,7 +314,7 @@ grpc::Status RuntimeRuntimeServiceImpl::ExecSync(grpc::ServerContext *context,
+     m_rService->ExecSync(request->container_id(), request->cmd(), request->timeout(), reply, error);
+     if (!error.Empty()) {
+         ERROR("Object: CRI, Type: Failed to sync exec container: %s", request->container_id().c_str());
+-        return grpc::Status(grpc::StatusCode::UNKNOWN, error.GetMessage());
++        return ToGRPCStatus(error);
+     }
+ 
+     WARN("Event: {Object: CRI, Type: sync execed Container: %s}", request->container_id().c_str());
+@@ -351,7 +368,7 @@ grpc::Status RuntimeRuntimeServiceImpl::StopPodSandbox(grpc::ServerContext *cont
+     if (!error.Empty()) {
+         ERROR("Object: CRI, Type: Failed to stop pod:%s due to %s", request->pod_sandbox_id().c_str(),
+               error.GetMessage().c_str());
+-        return grpc::Status(grpc::StatusCode::UNKNOWN, error.GetMessage());
++        return ToGRPCStatus(error);
+     }
+ 
+     EVENT("Event: {Object: CRI, Type: Stopped Pod: %s}", request->pod_sandbox_id().c_str());
+@@ -376,7 +393,7 @@ grpc::Status RuntimeRuntimeServiceImpl::RemovePodSandbox(grpc::ServerContext *co
+     if (!error.Empty()) {
+         ERROR("Object: CRI, Type: Failed to remove pod:%s due to %s", request->pod_sandbox_id().c_str(),
+               error.GetMessage().c_str());
+-        return grpc::Status(grpc::StatusCode::UNKNOWN, error.GetMessage());
++        return ToGRPCStatus(error);
+     }
+ 
+     EVENT("Event: {Object: CRI, Type: Removed Pod: %s}", request->pod_sandbox_id().c_str());
+@@ -402,7 +419,7 @@ grpc::Status RuntimeRuntimeServiceImpl::PodSandboxStatus(grpc::ServerContext *co
+     if (!error.Empty() || !podStatus) {
+         ERROR("Object: CRI, Type: Failed to status pod:%s due to %s", request->pod_sandbox_id().c_str(),
+               error.GetMessage().c_str());
+-        return grpc::Status(grpc::StatusCode::UNKNOWN, error.GetMessage());
++        return ToGRPCStatus(error);
+     }
+     *(reply->mutable_status()) = *podStatus;
+ 
+@@ -523,7 +540,7 @@ RuntimeRuntimeServiceImpl::UpdateContainerResources(grpc::ServerContext *context
+     if (error.NotEmpty()) {
+         ERROR("Object: CRI, Type: Failed to update container:%s due to %s", request->container_id().c_str(),
+               error.GetMessage().c_str());
+-        return grpc::Status(grpc::StatusCode::UNKNOWN, error.GetMessage());
++        return ToGRPCStatus(error);
+     }
+ 
+     WARN("Event: {Object: CRI, Type: Updated container resources: %s}", request->container_id().c_str());
+@@ -548,7 +565,7 @@ grpc::Status RuntimeRuntimeServiceImpl::Exec(grpc::ServerContext *context,
+     if (!error.Empty()) {
+         ERROR("Object: CRI, Type: Failed to exec container:%s due to %s", request->container_id().c_str(),
+               error.GetMessage().c_str());
+-        return grpc::Status(grpc::StatusCode::UNKNOWN, error.GetMessage());
++        return ToGRPCStatus(error);
+     }
+ 
+     EVENT("Event: {Object: CRI, Type: execed Container: %s}", request->container_id().c_str());
+@@ -573,7 +590,7 @@ grpc::Status RuntimeRuntimeServiceImpl::Attach(grpc::ServerContext *context,
+     if (!error.Empty()) {
+         ERROR("Object: CRI, Type: Failed to attach container:%s due to %s", request->container_id().c_str(),
+               error.GetMessage().c_str());
+-        return grpc::Status(grpc::StatusCode::UNKNOWN, error.GetMessage());
++        return ToGRPCStatus(error);
+     }
+ 
+     EVENT("Event: {Object: CRI, Type: attched Container: %s}", request->container_id().c_str());
+diff --git a/src/daemon/entry/connect/grpc/cri/v1alpha/cri_runtime_runtime_service.h b/src/daemon/entry/connect/grpc/cri/v1alpha/cri_runtime_runtime_service.h
+index e0f75897..210e67cc 100644
+--- a/src/daemon/entry/connect/grpc/cri/v1alpha/cri_runtime_runtime_service.h
++++ b/src/daemon/entry/connect/grpc/cri/v1alpha/cri_runtime_runtime_service.h
+@@ -103,6 +103,8 @@ public:
+                         runtime::v1alpha2::StatusResponse *reply) override;
+ 
+ private:
++    grpc::Status ToGRPCStatus(Errors &err);
++
+     std::unique_ptr<CRI::CRIRuntimeService> m_rService;
+ };
+ 
+-- 
+2.25.1
+

0060-cdi-return-int-instead-of-error-string.patch

@@ -0,0 +1,363 @@
+From e7b94411b174c8445d9bdc84ec6c94b5d4343470 Mon Sep 17 00:00:00 2001
+From: liuxu <liuxu156@huawei.com>
+Date: Mon, 15 Apr 2024 15:38:57 +0800
+Subject: [PATCH 060/149] cdi:return int instead of error string
+
+---
+ src/daemon/modules/api/cdi_operate_api.h         |  7 ++++---
+ .../device/cdi/behavior/cdi_container_edits.c    | 12 ++++++------
+ .../device/cdi/behavior/cdi_container_edits.h    |  6 +++---
+ .../modules/device/cdi/behavior/cdi_device.c     |  2 +-
+ .../modules/device/cdi/behavior/cdi_device.h     |  2 +-
+ .../modules/device/cdi/behavior/cdi_spec.c       |  2 +-
+ .../modules/device/cdi/behavior/cdi_spec.h       |  2 +-
+ .../modules/device/cdi/behavior/cdi_spec_dirs.c  |  4 ++--
+ .../modules/device/cdi/behavior/cdi_spec_dirs.h  |  6 +++---
+ .../device/cdi/behavior/parser/cdi_parser.c      | 16 ++++++++--------
+ .../device/cdi/behavior/parser/cdi_parser.h      |  8 ++++----
+ src/daemon/modules/device/cdi/cdi_annotations.c  |  5 +++--
+ src/daemon/modules/device/cdi/cdi_annotations.h  |  3 ++-
+ src/daemon/modules/device/cdi/cdi_cache.c        | 14 +++++++-------
+ src/daemon/modules/device/cdi/cdi_cache.h        |  8 ++++----
+ src/daemon/modules/device/cdi_operate.c          | 13 +++++++------
+ 16 files changed, 57 insertions(+), 53 deletions(-)
+
+diff --git a/src/daemon/modules/api/cdi_operate_api.h b/src/daemon/modules/api/cdi_operate_api.h
+index 4f4c339e..49820ed7 100644
+--- a/src/daemon/modules/api/cdi_operate_api.h
++++ b/src/daemon/modules/api/cdi_operate_api.h
+@@ -26,11 +26,12 @@ extern "C" {
+ 
+ int cdi_operate_registry_init(char **specs_dirs, size_t specs_dirs_len);
+ 
+-char *cdi_operate_refresh(void);
++int cdi_operate_refresh(void);
+ 
+-string_array *cdi_operate_inject_devices(oci_runtime_spec *spec, string_array *devices, char **error);
++int cdi_operate_inject_devices(oci_runtime_spec *spec, string_array *devices);
+ 
+-char *cdi_operate_parse_annotations(json_map_string_string *annotations, string_array **keys, string_array **devices);
++int cdi_operate_parse_annotations(json_map_string_string *annotations, string_array **keys,
++                                  string_array **devices, char **error);
+ 
+ #ifdef __cplusplus
+ }
+diff --git a/src/daemon/modules/device/cdi/behavior/cdi_container_edits.c b/src/daemon/modules/device/cdi/behavior/cdi_container_edits.c
+index ce7b16db..590118b1 100644
+--- a/src/daemon/modules/device/cdi/behavior/cdi_container_edits.c
++++ b/src/daemon/modules/device/cdi/behavior/cdi_container_edits.c
+@@ -27,19 +27,19 @@
+ // POSTSTOP_HOOK is the name of the OCI "poststop" hook.
+ #define POSTSTOP_HOOK "poststop"
+ 
+-char *cdi_container_edits_apply(cdi_container_edits *e, oci_runtime_spec *spec)
++int cdi_container_edits_apply(cdi_container_edits *e, oci_runtime_spec *spec)
+ {
+-    return NULL;
++    return 0;
+ }
+ 
+-char *cdi_container_edits_validate(cdi_container_edits *e)
++int cdi_container_edits_validate(cdi_container_edits *e, char **error)
+ {
+-    return NULL;
++    return 0;
+ }
+ 
+-cdi_container_edits *cdi_container_edits_append(cdi_container_edits *e, cdi_container_edits *o)
++int cdi_container_edits_append(cdi_container_edits *e, cdi_container_edits *o)
+ {
+-    return NULL;
++    return 0;
+ }
+ 
+ bool cdi_container_edits_is_empty(cdi_container_edits *e)
+diff --git a/src/daemon/modules/device/cdi/behavior/cdi_container_edits.h b/src/daemon/modules/device/cdi/behavior/cdi_container_edits.h
+index 7b16d2bc..ea921e37 100644
+--- a/src/daemon/modules/device/cdi/behavior/cdi_container_edits.h
++++ b/src/daemon/modules/device/cdi/behavior/cdi_container_edits.h
+@@ -27,9 +27,9 @@
+ extern "C" {
+ #endif
+ 
+-char *cdi_container_edits_apply(cdi_container_edits *e, oci_runtime_spec *spec);
+-char *cdi_container_edits_validate(cdi_container_edits *e);
+-cdi_container_edits *cdi_container_edits_append(cdi_container_edits *e, cdi_container_edits *o);
++int cdi_container_edits_apply(cdi_container_edits *e, oci_runtime_spec *spec);
++int cdi_container_edits_validate(cdi_container_edits *e, char **error);
++int cdi_container_edits_append(cdi_container_edits *e, cdi_container_edits *o);
+ bool cdi_container_edits_is_empty(cdi_container_edits *e);
+ 
+ #ifdef __cplusplus
+diff --git a/src/daemon/modules/device/cdi/behavior/cdi_device.c b/src/daemon/modules/device/cdi/behavior/cdi_device.c
+index 9904e9ee..0fef8f42 100644
+--- a/src/daemon/modules/device/cdi/behavior/cdi_device.c
++++ b/src/daemon/modules/device/cdi/behavior/cdi_device.c
+@@ -34,7 +34,7 @@ char *cdi_device_get_qualified_name(struct cdi_cache_device *d)
+     return NULL;
+ }
+ 
+-cdi_container_edits *cdi_device_edits(struct cdi_cache_device *d)
++cdi_container_edits *cdi_device_get_edits(struct cdi_cache_device *d)
+ {
+     return NULL;
+ }
+diff --git a/src/daemon/modules/device/cdi/behavior/cdi_device.h b/src/daemon/modules/device/cdi/behavior/cdi_device.h
+index 3f460152..5d63a576 100644
+--- a/src/daemon/modules/device/cdi/behavior/cdi_device.h
++++ b/src/daemon/modules/device/cdi/behavior/cdi_device.h
+@@ -37,7 +37,7 @@ void free_cdi_cache_device(struct cdi_cache_device *d);
+ struct cdi_cache_device *cdi_device_new_device(struct cdi_cache_spec *spec, cdi_device *d, char **error);
+ struct cdi_cache_spec *cdi_device_get_spec(struct cdi_cache_device *d);
+ char *cdi_device_get_qualified_name(struct cdi_cache_device *d);
+-cdi_container_edits *cdi_device_edits(struct cdi_cache_device *d);
++cdi_container_edits *cdi_device_get_edits(struct cdi_cache_device *d);
+ 
+ #ifdef __cplusplus
+ }
+diff --git a/src/daemon/modules/device/cdi/behavior/cdi_spec.c b/src/daemon/modules/device/cdi/behavior/cdi_spec.c
+index 38fc9e38..f79b5a44 100644
+--- a/src/daemon/modules/device/cdi/behavior/cdi_spec.c
++++ b/src/daemon/modules/device/cdi/behavior/cdi_spec.c
+@@ -54,7 +54,7 @@ int cdi_spec_get_priority(struct cdi_cache_spec *s)
+     return 0;
+ }
+ 
+-cdi_container_edits *cdi_spec_edits(struct cdi_cache_spec *s)
++cdi_container_edits *cdi_spec_get_edits(struct cdi_cache_spec *s)
+ {
+     return NULL;
+ }
+diff --git a/src/daemon/modules/device/cdi/behavior/cdi_spec.h b/src/daemon/modules/device/cdi/behavior/cdi_spec.h
+index bd4fc9d1..87248041 100644
+--- a/src/daemon/modules/device/cdi/behavior/cdi_spec.h
++++ b/src/daemon/modules/device/cdi/behavior/cdi_spec.h
+@@ -47,7 +47,7 @@ const char *cdi_spec_get_class(struct cdi_cache_spec *s);
+ struct cdi_cache_device *cdi_spec_get_cache_device(struct cdi_cache_spec *s, const char *name);
+ const char *cdi_spec_get_path(struct cdi_cache_spec *s);
+ int cdi_spec_get_priority(struct cdi_cache_spec *s);
+-cdi_container_edits *cdi_spec_edits(struct cdi_cache_spec *s);
++cdi_container_edits *cdi_spec_get_edits(struct cdi_cache_spec *s);
+ 
+ #ifdef __cplusplus
+ }
+diff --git a/src/daemon/modules/device/cdi/behavior/cdi_spec_dirs.c b/src/daemon/modules/device/cdi/behavior/cdi_spec_dirs.c
+index 5df4c937..e340abc0 100644
+--- a/src/daemon/modules/device/cdi/behavior/cdi_spec_dirs.c
++++ b/src/daemon/modules/device/cdi/behavior/cdi_spec_dirs.c
+@@ -23,7 +23,7 @@ string_array g_default_spec_dirs = {
+     .cap = DEFAULT_SPEC_DIRS_LEN,
+ };
+  
+-char *cdi_scan_spec_dirs(string_array *dirs, struct cdi_scan_fn_maps *scan_fn_maps, cdi_scan_spec_func scan_fn)
++int cdi_scan_spec_dirs(string_array *dirs, struct cdi_scan_fn_maps *scan_fn_maps, cdi_scan_spec_func scan_fn)
+ {
+-    return NULL;
++    return 0;
+ }
+diff --git a/src/daemon/modules/device/cdi/behavior/cdi_spec_dirs.h b/src/daemon/modules/device/cdi/behavior/cdi_spec_dirs.h
+index bd00e318..73d8c0f5 100644
+--- a/src/daemon/modules/device/cdi/behavior/cdi_spec_dirs.h
++++ b/src/daemon/modules/device/cdi/behavior/cdi_spec_dirs.h
+@@ -35,10 +35,10 @@ struct cdi_scan_fn_maps {
+     map_t *spec_errors;
+     string_array *result;
+ };
+-typedef char *(*cdi_scan_spec_func)(struct cdi_scan_fn_maps *scan_fn_maps, const char *path, int priority,
+-                                    struct cdi_cache_spec *spec, char **error);
++typedef void(*cdi_scan_spec_func)(struct cdi_scan_fn_maps *scan_fn_maps, const char *path, int priority,
++                                    struct cdi_cache_spec *spec, char *error);
+ 
+-char *cdi_scan_spec_dirs(string_array *dirs, struct cdi_scan_fn_maps *scan_fn_maps, cdi_scan_spec_func scan_fn);
++int cdi_scan_spec_dirs(string_array *dirs, struct cdi_scan_fn_maps *scan_fn_maps, cdi_scan_spec_func scan_fn);
+ 
+ #ifdef __cplusplus
+ }
+diff --git a/src/daemon/modules/device/cdi/behavior/parser/cdi_parser.c b/src/daemon/modules/device/cdi/behavior/parser/cdi_parser.c
+index 45048f9a..14293c72 100644
+--- a/src/daemon/modules/device/cdi/behavior/parser/cdi_parser.c
++++ b/src/daemon/modules/device/cdi/behavior/parser/cdi_parser.c
+@@ -24,9 +24,9 @@ bool cdi_parser_is_qualified_name(const char *device)
+     return true;
+ }
+ 
+-char *cdi_parser_parse_qualified_name(const char *device, char **vendor, char **class, char **name)
++int cdi_parser_parse_qualified_name(const char *device, char **vendor, char **class, char **name)
+ {
+-    return NULL;
++    return 0;
+ }
+ 
+ int cdi_parser_parse_device(const char *device, char **vendor, char **class, char **name)
+@@ -39,17 +39,17 @@ int cdi_parser_parse_qualifier(const char *kind, char **vendor, char **class)
+     return 0;
+ }
+ 
+-char *cdi_parser_validate_vendor_name(const char *vendor)
++int cdi_parser_validate_vendor_name(const char *vendor, char **error)
+ {
+-    return NULL;
++    return 0;
+ }
+ 
+-char *cdi_parser_validate_class_name(const char *class)
++int cdi_parser_validate_class_name(const char *class, char **error)
+ {
+-    return NULL;
++    return 0;
+ }
+ 
+-char *cdi_parser_validate_device_name(const char *name)
++int cdi_parser_validate_device_name(const char *name, char **error)
+ {
+-    return NULL;
++    return 0;
+ }
+diff --git a/src/daemon/modules/device/cdi/behavior/parser/cdi_parser.h b/src/daemon/modules/device/cdi/behavior/parser/cdi_parser.h
+index d9c057ea..467641a1 100644
+--- a/src/daemon/modules/device/cdi/behavior/parser/cdi_parser.h
++++ b/src/daemon/modules/device/cdi/behavior/parser/cdi_parser.h
+@@ -24,12 +24,12 @@ extern "C" {
+ 
+ char *cdi_parser_qualified_name(const char *vendor, const char *class, const char *name);
+ bool cdi_parser_is_qualified_name(const char *device);
+-char *cdi_parser_parse_qualified_name(const char *device, char **vendor, char **class, char **name);
++int cdi_parser_parse_qualified_name(const char *device, char **vendor, char **class, char **name);
+ int cdi_parser_parse_device(const char *device, char **vendor, char **class, char **name);
+ int cdi_parser_parse_qualifier(const char *kind, char **vendor, char **class);
+-char *cdi_parser_validate_vendor_name(const char *vendor);
+-char *cdi_parser_validate_class_name(const char *class);
+-char *cdi_parser_validate_device_name(const char *name);
++int cdi_parser_validate_vendor_name(const char *vendor, char **error);
++int cdi_parser_validate_class_name(const char *class, char **error);
++int cdi_parser_validate_device_name(const char *name, char **error);
+ 
+ #ifdef __cplusplus
+ }
+diff --git a/src/daemon/modules/device/cdi/cdi_annotations.c b/src/daemon/modules/device/cdi/cdi_annotations.c
+index 3cb9be84..cfe6e099 100644
+--- a/src/daemon/modules/device/cdi/cdi_annotations.c
++++ b/src/daemon/modules/device/cdi/cdi_annotations.c
+@@ -25,7 +25,8 @@
+ 
+ #define CDI_ANNOTATIONS_PREFIX "cdi.k8s.io/"
+ 
+-char *cdi_parse_annotations(json_map_string_string *annotations, string_array **keys, string_array **devices)
++int cdi_parse_annotations(json_map_string_string *annotations, string_array **keys,
++                          string_array **devices, char **error)
+ {
+-    return NULL;
++    return 0;
+ }
+diff --git a/src/daemon/modules/device/cdi/cdi_annotations.h b/src/daemon/modules/device/cdi/cdi_annotations.h
+index 52355099..49930963 100644
+--- a/src/daemon/modules/device/cdi/cdi_annotations.h
++++ b/src/daemon/modules/device/cdi/cdi_annotations.h
+@@ -23,7 +23,8 @@
+ extern "C" {
+ #endif
+ 
+-char *cdi_parse_annotations(json_map_string_string *annotations, string_array **keys, string_array **devices);
++int cdi_parse_annotations(json_map_string_string *annotations, string_array **keys,
++                          string_array **devices, char **error);
+ 
+ #ifdef __cplusplus
+ }
+diff --git a/src/daemon/modules/device/cdi/cdi_cache.c b/src/daemon/modules/device/cdi/cdi_cache.c
+index 9c54acbf..cfc23a1c 100644
+--- a/src/daemon/modules/device/cdi/cdi_cache.c
++++ b/src/daemon/modules/device/cdi/cdi_cache.c
+@@ -19,24 +19,24 @@ void free_cdi_cache(struct cdi_cache *c)
+     (void)c;
+ }
+ 
+-struct cdi_cache *cdi_new_cache(string_array *spec_dirs, char **error)
++struct cdi_cache *cdi_new_cache(string_array *spec_dirs)
+ {
+     return NULL;
+ }
+ 
+-static string_array *cdi_inject_devices(struct cdi_cache *c, oci_runtime_spec *oci_spec, string_array *devices, char **error)
++static int cdi_inject_devices(struct cdi_cache *c, oci_runtime_spec *oci_spec, string_array *devices)
+ {
+-    return NULL;
++    return 0;
+ }
+ 
+-static char *cdi_configure(struct cdi_cache *c, string_array *spec_dirs)
++static int cdi_configure(struct cdi_cache *c, string_array *spec_dirs)
+ {
+-    return NULL;
++    return 0;
+ }
+ 
+-static char *cdi_refresh(struct cdi_cache *c)
++static int cdi_refresh(struct cdi_cache *c)
+ {
+-    return NULL;
++    return 0;
+ }
+ 
+ static map_t *cdi_get_errors(struct cdi_cache *c)
+diff --git a/src/daemon/modules/device/cdi/cdi_cache.h b/src/daemon/modules/device/cdi/cdi_cache.h
+index 92fb64af..34c27471 100644
+--- a/src/daemon/modules/device/cdi/cdi_cache.h
++++ b/src/daemon/modules/device/cdi/cdi_cache.h
+@@ -33,12 +33,12 @@ struct cdi_cache;
+ struct cdi_cache_ops {
+     // injecting CDI devices into an OCI Spec.
+     // Resolver
+-    string_array *(*inject_devices)(struct cdi_cache *c, oci_runtime_spec *spec, string_array *devices, char **error);
++    int (*inject_devices)(struct cdi_cache *c, oci_runtime_spec *spec, string_array *devices);
+     
+     // refreshing the cache of CDI Specs and devices.
+     // Refresher
+-    char *(*configure)(struct cdi_cache *c, string_array *spec_dirs);
+-    char *(*refresh)(struct cdi_cache *c);
++    int (*configure)(struct cdi_cache *c, string_array *spec_dirs);
++    int (*refresh)(struct cdi_cache *c);
+     map_t *(*get_errors)(struct cdi_cache *c);
+     string_array *(*get_spec_directories)(struct cdi_cache *c);
+     map_t *(*get_spec_dir_errors)(struct cdi_cache *c);
+@@ -65,7 +65,7 @@ struct cdi_cache {
+ 
+ void free_cdi_cache(struct cdi_cache *c);
+  
+-struct cdi_cache *cdi_new_cache(string_array *spec_dirs, char **error);
++struct cdi_cache *cdi_new_cache(string_array *spec_dirs);
+ struct cdi_cache_ops *cdi_get_cache_ops(void);
+ 
+ #ifdef __cplusplus
+diff --git a/src/daemon/modules/device/cdi_operate.c b/src/daemon/modules/device/cdi_operate.c
+index c7aa77d8..c5187ab1 100644
+--- a/src/daemon/modules/device/cdi_operate.c
++++ b/src/daemon/modules/device/cdi_operate.c
+@@ -19,17 +19,18 @@ int cdi_operate_registry_init(char **specs_dirs, size_t specs_dirs_len)
+     return 0;
+ }
+ 
+-char *cdi_operate_refresh(void)
++int cdi_operate_refresh(void)
+ {
+-    return NULL;
++    return 0;
+ }
+ 
+-string_array *cdi_operate_inject_devices(oci_runtime_spec *spec, string_array *devices, char **error)
++int cdi_operate_inject_devices(oci_runtime_spec *spec, string_array *devices)
+ {
+-    return NULL;
++    return 0;
+ }
+ 
+-char *cdi_operate_parse_annotations(json_map_string_string *annotations, string_array **keys, string_array **devices)
++int cdi_operate_parse_annotations(json_map_string_string *annotations, string_array **keys,
++                                  string_array **devices, char **error)
+ {
+-    return NULL;
++    return 0;
+ }
+\ No newline at end of file
+-- 
+2.25.1
+

0061-cdi-support-modules-operate-registry-annotations.patch

@@ -0,0 +1,516 @@
+From 4527dc8b6f7ab438742a8f7403b24420f646236d Mon Sep 17 00:00:00 2001
+From: liuxu <liuxu156@huawei.com>
+Date: Mon, 8 Apr 2024 11:57:16 +0800
+Subject: [PATCH 061/149] cdi:support modules operate/registry/annotations
+
+---
+ .../modules/device/cdi/cdi_annotations.c      |  72 +++++++
+ src/daemon/modules/device/cdi/cdi_registry.c  |  14 +-
+ src/daemon/modules/device/cdi_operate.c       |  53 +++++-
+ src/utils/cutils/utils.c                      |  11 ++
+ src/utils/cutils/utils.h                      |   2 +
+ src/utils/cutils/utils_array.c                | 175 +++++++++++++++++-
+ src/utils/cutils/utils_array.h                |  31 ++++
+ 7 files changed, 350 insertions(+), 8 deletions(-)
+
+diff --git a/src/daemon/modules/device/cdi/cdi_annotations.c b/src/daemon/modules/device/cdi/cdi_annotations.c
+index cfe6e099..020816d7 100644
+--- a/src/daemon/modules/device/cdi/cdi_annotations.c
++++ b/src/daemon/modules/device/cdi/cdi_annotations.c
+@@ -21,12 +21,84 @@
+ 
+ #include "error.h"
+ #include "utils.h"
++#include "utils_array.h"
+ #include "cdi_parser.h"
+ 
+ #define CDI_ANNOTATIONS_PREFIX "cdi.k8s.io/"
+ 
++static int parse_devices(string_array *devices, const char *value, char **error)
++{
++    __isula_auto_array_t char **parts = NULL;
++    char **pos;
++
++    parts = util_string_split(value, ',');
++    if (parts == NULL) {
++        ERROR("Invalid CDI device value %s", value);
++        format_errorf(error, "Invalid CDI device value %s", value);
++        return -1;
++    }
++    for (pos = parts; pos != NULL && *pos != NULL; pos++) {
++        if (!cdi_parser_is_qualified_name(*pos)) {
++            ERROR("Invalid CDI device name %s", *pos);
++            format_errorf(error, "Invalid CDI device name %s", *pos);
++            return -1;
++        }
++        if (util_append_string_array(devices, *pos) != 0) {
++            ERROR("Out of memory");
++            *error = util_strdup_s("Out of memory");
++            return -1;
++        }
++    }
++
++    return 0;
++}
++
+ int cdi_parse_annotations(json_map_string_string *annotations, string_array **keys,
+                           string_array **devices, char **error)
+ {
++    char *key = NULL;
++    char *value = NULL;
++    size_t i;
++    __isula_auto_string_array_t string_array *keys_array = NULL;
++    __isula_auto_string_array_t string_array *devices_array = NULL;
++
++    if (annotations == NULL || keys == NULL || devices == NULL || error == NULL) {
++        ERROR("Invalid argument");
++        return -1;
++    }
++
++    keys_array = util_common_calloc_s(sizeof(*keys_array));
++    if (keys_array == NULL) {
++        ERROR("Out of memory");
++        *error = util_strdup_s("Out of memory");
++        return -1;
++    }
++    devices_array = util_common_calloc_s(sizeof(*devices_array));
++    if (devices_array == NULL) {
++        ERROR("Out of memory");
++        *error = util_strdup_s("Out of memory");
++        return -1;
++    }
++
++    for (i = 0; i < annotations->len; i++) {
++        key = annotations->keys[i];
++        value = annotations->values[i];
++        if (!util_has_prefix(key, CDI_ANNOTATIONS_PREFIX)) {
++            continue;
++        }
++        if (parse_devices(devices_array, value, error) != 0) {
++            return -1;
++        }
++        if (util_append_string_array(keys_array, key) != 0) {
++            ERROR("Out of memory");
++            *error = util_strdup_s("Out of memory");
++            return -1;
++        }
++    }
++
++    *keys = keys_array;
++    keys_array = NULL;
++    *devices = devices_array;
++    devices_array = NULL;
+     return 0;
+ }
+diff --git a/src/daemon/modules/device/cdi/cdi_registry.c b/src/daemon/modules/device/cdi/cdi_registry.c
+index 68767a5f..be381132 100644
+--- a/src/daemon/modules/device/cdi/cdi_registry.c
++++ b/src/daemon/modules/device/cdi/cdi_registry.c
+@@ -14,12 +14,24 @@
+  ******************************************************************************/
+ #include "cdi_registry.h"
+ 
++#include <util_atomic.h>
++#include <isula_libutils/auto_cleanup.h>
++
++static struct cdi_registry g_cdi_reg = { 0 };
++
+ int cdi_registry_init(string_array *spec_dirs)
+ {
++    // isulad will use default dirs when spec_dirs == NULL
++    g_cdi_reg.cdi_cache = cdi_new_cache(spec_dirs);
++    if (g_cdi_reg.cdi_cache == NULL) {
++        ERROR("Failed to init registry");
++        return -1;
++    }
++    g_cdi_reg.ops = cdi_get_cache_ops();
+     return 0;
+ }
+ 
+ struct cdi_registry *cdi_get_registry(void)
+ {
+-    return NULL;
++    return &g_cdi_reg;
+ }
+diff --git a/src/daemon/modules/device/cdi_operate.c b/src/daemon/modules/device/cdi_operate.c
+index c5187ab1..f99bb7e4 100644
+--- a/src/daemon/modules/device/cdi_operate.c
++++ b/src/daemon/modules/device/cdi_operate.c
+@@ -14,23 +14,66 @@
+  ******************************************************************************/
+ #include "cdi_operate_api.h"
+ 
++#include <isula_libutils/log.h>
++
++#include "utils.h"
++#include "error.h"
++#include "cdi_registry.h"
++#include "cdi_annotations.h"
++#include "cdi_spec_dirs.h"
++
+ int cdi_operate_registry_init(char **specs_dirs, size_t specs_dirs_len)
+ {
+-    return 0;
++    string_array spec_dirs_array = {
++        .items = specs_dirs,
++        .len = specs_dirs_len,
++        .cap = specs_dirs_len,
++    };
++    
++    return cdi_registry_init(&spec_dirs_array);
+ }
+ 
+ int cdi_operate_refresh(void)
+ {
+-    return 0;
++    struct cdi_registry *registry = cdi_get_registry();
++    if (registry == NULL || registry->ops == NULL || registry->ops->refresh == NULL) {
++        ERROR("Failed to get registry");
++        return -1;
++    }
++    
++    return registry->ops->refresh(registry->cdi_cache);
+ }
+ 
+ int cdi_operate_inject_devices(oci_runtime_spec *spec, string_array *devices)
+ {
+-    return 0;
++    struct cdi_registry *registry = NULL;
++
++    if (spec == NULL || devices == NULL) {
++        ERROR("Invalid params");
++        return -1;
++    }
++    
++    registry = cdi_get_registry();
++    if (registry == NULL || registry->ops == NULL || registry->ops->inject_devices == NULL) {
++        ERROR("Failed to get registry");
++        return -1;
++    }
++    
++    return registry->ops->inject_devices(registry->cdi_cache, spec, devices);
+ }
+ 
+ int cdi_operate_parse_annotations(json_map_string_string *annotations, string_array **keys,
+                                   string_array **devices, char **error)
+ {
+-    return 0;
+-}
+\ No newline at end of file
++    if (error == NULL) {
++        ERROR("Invalid argument");
++        return -1;
++    }
++    if (annotations == NULL || keys == NULL || devices == NULL) {
++        ERROR("Invalid params");
++        *error = util_strdup_s("Invalid params");
++        return -1;
++    }
++
++    return cdi_parse_annotations(annotations, keys, devices, error);
++}
+diff --git a/src/utils/cutils/utils.c b/src/utils/cutils/utils.c
+index 8da2cc60..9a33f935 100644
+--- a/src/utils/cutils/utils.c
++++ b/src/utils/cutils/utils.c
+@@ -59,6 +59,17 @@ int malloc_trim(size_t pad)
+ }
+ #endif
+ 
++void util_swap_ptr(void **p1, void **p2)
++{
++    void *tmp;
++    if (p1 == NULL || p2 == NULL) {
++        return;
++    }
++    tmp = *p1;
++    *p1 = *p2;
++    *p2 = tmp;
++}
++
+ int util_mem_realloc(void **newptr, size_t newsize, void *oldptr, size_t oldsize)
+ {
+     void *tmp = NULL;
+diff --git a/src/utils/cutils/utils.h b/src/utils/cutils/utils.h
+index 3acf0698..3671272a 100644
+--- a/src/utils/cutils/utils.h
++++ b/src/utils/cutils/utils.h
+@@ -320,6 +320,8 @@ struct signame {
+         }                         \
+     } while (0)
+ 
++void util_swap_ptr(void **p1, void **p2);
++
+ int util_mem_realloc(void **newptr, size_t newsize, void *oldptr, size_t oldsize);
+ 
+ int util_check_inherited(bool closeall, int fd_to_ignore);
+diff --git a/src/utils/cutils/utils_array.c b/src/utils/cutils/utils_array.c
+index 25f19b8b..72294005 100644
+--- a/src/utils/cutils/utils_array.c
++++ b/src/utils/cutils/utils_array.c
+@@ -86,6 +86,27 @@ void util_free_sensitive_array(char **array)
+     free(array);
+ }
+ 
++char **util_copy_array_by_len(char **array, size_t len)
++{
++    char **new_array = NULL;
++    size_t i;
++ 
++    if (array == NULL || len == 0) {
++        return NULL;
++    }
++ 
++    new_array = util_smart_calloc_s(sizeof(char *), len);
++    if (new_array == NULL) {
++        ERROR("Out of memory");
++        return NULL;
++    }
++ 
++    for (i = 0; i < len; i++) {
++        new_array[i] = util_strdup_s(array[i]);
++    }
++    return new_array;
++}
++
+ int util_array_append(char ***array, const char *element)
+ {
+     size_t len;
+@@ -166,7 +187,7 @@ bool util_array_contain(const char **array, const char *element)
+     return false;
+ }
+ 
+-static size_t get_string_array_scale_size(size_t old_size)
++static size_t get_array_scale_size(size_t old_size)
+ {
+ #define DOUBLE_THRESHOLD 1024
+     const size_t max_threshold = MAX_MEMORY_SIZE / sizeof(char *);
+@@ -188,7 +209,7 @@ static size_t get_string_array_scale_size(size_t old_size)
+ 
+ static bool do_expand_array(string_array *array)
+ {
+-    size_t new_size = get_string_array_scale_size(array->cap);
++    size_t new_size = get_array_scale_size(array->cap);
+     char **new_items = NULL;
+ 
+     // array capability sure less than MAX_MEMORY_SIZE
+@@ -237,6 +258,29 @@ out:
+     return 0;
+ }
+ 
++string_array *util_copy_string_array(string_array *sarr)
++{
++    string_array *ptr = NULL;
++    size_t i;
++ 
++    if (sarr == NULL) {
++        ERROR("Invalid string array");
++        return NULL;
++    }
++ 
++    ptr = util_string_array_new(sarr->cap);
++    if (ptr == NULL) {
++        ERROR("Out of memory");
++        return NULL;
++    }
++    for (i = 0; i < sarr->len; i++) {
++        ptr->items[i] = util_strdup_s(sarr->items[i]);
++        ptr->len += 1;
++    }
++ 
++    return ptr;
++}
++
+ bool util_string_array_contain(const string_array *sarr, const char *elem)
+ {
+     size_t i;
+@@ -339,3 +383,130 @@ int util_common_array_append_pointer(void ***array, void *element)
+ 
+     return 0;
+ }
++
++void *util_clone_ptr(void *item)
++{
++    return item;
++}
++
++common_array *util_common_array_new(size_t len, free_common_array_item_cb free_item_cb,
++                                    clone_common_array_item_cb clone_item_cb)
++{
++    common_array *ptr = NULL;
++
++    if (len == 0 || free_item_cb == NULL || clone_item_cb == NULL) {
++        return NULL;
++    }
++
++    ptr = (common_array *)util_common_calloc_s(sizeof(common_array));
++    if (ptr == NULL) {
++        ERROR("Out of memory");
++        return NULL;
++    }
++
++    ptr->items = (void **)util_smart_calloc_s(sizeof(void *), len);
++    if (ptr->items == NULL) {
++        ERROR("Out of memory");
++        free(ptr);
++        return NULL;
++    }
++
++    ptr->len = 0;
++    ptr->cap = len;
++    ptr->free_item_cb = free_item_cb;
++    ptr->clone_item_cb = clone_item_cb;
++
++    return ptr;
++}
++
++void util_free_common_array(common_array *ptr)
++{
++    size_t i;
++
++    if (ptr == NULL || ptr->free_item_cb == NULL) {
++        return;
++    }
++
++    for (i = 0; i < ptr->len; i++) {
++        ptr->free_item_cb(ptr->items[i]);
++        ptr->items[i] = NULL;
++    }
++    free(ptr->items);
++    ptr->items = NULL;
++    ptr->len = 0;
++    ptr->cap = 0;
++    ptr->free_item_cb = NULL;
++    ptr->clone_item_cb = NULL;
++
++    free(ptr);
++}
++
++static bool do_expand_common_array(common_array *array)
++{
++    size_t new_size = get_array_scale_size(array->cap);
++    void **new_items = NULL;
++
++    // array capability sure less than MAX_MEMORY_SIZE
++    // so we need to check Overflow:
++    if (new_size == array->cap) {
++        ERROR("Too large common array, overflow memory");
++        return false;
++    }
++
++    // new_size * sizeof(*new_items) and list->len * sizeof(*list->items)
++    if (util_mem_realloc((void **)&new_items, new_size * sizeof(void *), (void *)array->items,
++                         array->len * sizeof(void *)) != 0) {
++        ERROR("Out of memory");
++        return false;
++    }
++    array->items = new_items;
++    array->cap = new_size;
++
++    return true;
++}
++
++int util_append_common_array(common_array *arr, void *val)
++{
++    if (arr == NULL || arr->clone_item_cb == NULL) {
++        ERROR("Invalid common array");
++        return -1;
++    }
++
++    if (val == NULL) {
++        DEBUG("Empty new item, just ignore it");
++        return 0;
++    }
++
++    if (arr->len < arr->cap) {
++        goto out;
++    }
++
++    // expand common array
++    if (!do_expand_common_array(arr)) {
++        return -1;
++    }
++
++out:
++    arr->items[arr->len] = arr->clone_item_cb(val);
++    arr->len += 1;
++    return 0;
++}
++
++int util_merge_common_array(common_array *dest_arr, common_array *src_arr)
++{
++    size_t i;
++
++    if (dest_arr == NULL || dest_arr->clone_item_cb == NULL ||
++        src_arr == NULL || src_arr->clone_item_cb == NULL) {
++        ERROR("Invalid common array");
++        return -1;
++    }
++
++    for (i = 0; i < src_arr->len; i++) {
++        if (util_append_common_array(dest_arr, src_arr->items[i]) != 0) {
++            ERROR("Failed to append element");
++            return -1;
++        }
++    }
++    return 0;
++}
+diff --git a/src/utils/cutils/utils_array.h b/src/utils/cutils/utils_array.h
+index 1c084595..0c4fd217 100644
+--- a/src/utils/cutils/utils_array.h
++++ b/src/utils/cutils/utils_array.h
+@@ -30,6 +30,8 @@ void util_free_array_by_len(char **array, size_t len);
+ 
+ void util_free_array(char **array);
+ 
++char **util_copy_array_by_len(char **array, size_t len);
++
+ int util_grow_array(char ***orig_array, size_t *orig_capacity, size_t size,
+                     size_t increment);
+ 
+@@ -52,6 +54,8 @@ void util_free_string_array(string_array *ptr);
+ 
+ int util_append_string_array(string_array *sarr, const char *val);
+ 
++string_array *util_copy_string_array(string_array *sarr);
++
+ bool util_string_array_contain(const string_array *sarr, const char *elem);
+ 
+ void util_free_sensitive_array(char **array);
+@@ -63,6 +67,33 @@ define_auto_cleanup_callback(util_free_array, char *);
+ // define auto free macro for char *
+ #define __isula_auto_array_t auto_cleanup_tag(util_free_array)
+ 
++define_auto_cleanup_callback(util_free_string_array, string_array);
++#define __isula_auto_string_array_t auto_cleanup_tag(util_free_string_array)
++
++typedef void (*free_common_array_item_cb)(void *item);
++typedef void *(*clone_common_array_item_cb)(void *item);
++typedef struct common_array_t {
++    void **items;
++    size_t len;
++    size_t cap;
++    free_common_array_item_cb free_item_cb;
++    clone_common_array_item_cb clone_item_cb;
++} common_array;
++
++void *util_clone_ptr(void *item);
++
++common_array *util_common_array_new(size_t len, free_common_array_item_cb free_item_cb,
++                                    clone_common_array_item_cb clone_item_cb);
++
++void util_free_common_array(common_array *ptr);
++
++int util_append_common_array(common_array *arr, void *val);
++
++int util_merge_common_array(common_array *dest_arr, common_array *src_arr);
++
++define_auto_cleanup_callback(util_free_common_array, common_array);
++#define __isula_auto_common_array_t auto_cleanup_tag(util_free_common_array)
++
+ #ifdef __cplusplus
+ }
+ #endif
+-- 
+2.25.1
+

0062-do-not-umount-shmpath-for-sandbox-container.patch

@@ -0,0 +1,30 @@
+From 8e35525073b52b5d161984015de641bd21570380 Mon Sep 17 00:00:00 2001
+From: jikai <jikai11@huawei.com>
+Date: Tue, 16 Apr 2024 10:32:50 +0000
+Subject: [PATCH 062/149] do not umount shmpath for sandbox container
+
+Signed-off-by: jikai <jikai11@huawei.com>
+---
+ src/daemon/modules/service/service_container.c | 6 ++++++
+ 1 file changed, 6 insertions(+)
+
+diff --git a/src/daemon/modules/service/service_container.c b/src/daemon/modules/service/service_container.c
+index eb7ce4f4..a2322309 100644
+--- a/src/daemon/modules/service/service_container.c
++++ b/src/daemon/modules/service/service_container.c
+@@ -1718,6 +1718,12 @@ void umount_share_shm(container_t *cont)
+         return;
+     }
+     if (cont->hostconfig->ipc_mode == NULL || namespace_is_shareable(cont->hostconfig->ipc_mode)) {
++#ifdef ENABLE_CRI_API_V1
++        // For sandbox in cri v1, the shm path is created and umounted in CRI
++        if (is_sandbox_container(cont->common_config->sandbox_info)) {
++            return;
++        }
++#endif
+         if (cont->common_config == NULL || cont->common_config->shm_path == NULL) {
+             return;
+         }
+-- 
+2.25.1
+

0063-remove-default-systemd-cgroup-and-enable-cri-v1-valu.patch

@@ -0,0 +1,27 @@
+From c092597565e5f24b29ecd83b4b371a19a9c2db0d Mon Sep 17 00:00:00 2001
+From: jikai <jikai11@huawei.com>
+Date: Wed, 17 Apr 2024 01:52:45 +0000
+Subject: [PATCH 063/149] remove default systemd-cgroup and enable-cri-v1 value
+ in daemon.json
+
+Signed-off-by: jikai <jikai11@huawei.com>
+---
+ src/contrib/config/daemon.json | 4 +---
+ 1 file changed, 1 insertion(+), 3 deletions(-)
+
+diff --git a/src/contrib/config/daemon.json b/src/contrib/config/daemon.json
+index 69362c26..711dda94 100644
+--- a/src/contrib/config/daemon.json
++++ b/src/contrib/config/daemon.json
+@@ -35,7 +35,5 @@
+     "insecure-skip-verify-enforce": false,
+     "cri-runtimes": {
+         "kata": "io.containerd.kata.v2"
+-    },
+-    "enable-cri-v1": false,
+-    "systemd-cgroup": false
++    }
+ }
+-- 
+2.25.1
+

0064-cdi-support-module-cache.patch

@@ -0,0 +1,862 @@
+From f87f0fddaec4b3aea72f3b9c91f2dc91b89683ce Mon Sep 17 00:00:00 2001
+From: liuxu <liuxu156@huawei.com>
+Date: Tue, 16 Apr 2024 11:44:22 +0800
+Subject: [PATCH 064/149] cdi:support module cache
+
+---
+ .../device/cdi/behavior/cdi_spec_dirs.h       |   3 +-
+ src/daemon/modules/device/cdi/cdi_cache.c     | 760 +++++++++++++++++-
+ src/daemon/modules/device/cdi/cdi_cache.h     |   9 +-
+ 3 files changed, 747 insertions(+), 25 deletions(-)
+
+diff --git a/src/daemon/modules/device/cdi/behavior/cdi_spec_dirs.h b/src/daemon/modules/device/cdi/behavior/cdi_spec_dirs.h
+index 73d8c0f5..eedcabad 100644
+--- a/src/daemon/modules/device/cdi/behavior/cdi_spec_dirs.h
++++ b/src/daemon/modules/device/cdi/behavior/cdi_spec_dirs.h
+@@ -32,8 +32,7 @@ struct cdi_scan_fn_maps {
+     map_t *specs;
+     map_t *devices;
+     map_t *conflicts;
+-    map_t *spec_errors;
+-    string_array *result;
++    bool *refresh_error_flag;
+ };
+ typedef void(*cdi_scan_spec_func)(struct cdi_scan_fn_maps *scan_fn_maps, const char *path, int priority,
+                                     struct cdi_cache_spec *spec, char *error);
+diff --git a/src/daemon/modules/device/cdi/cdi_cache.c b/src/daemon/modules/device/cdi/cdi_cache.c
+index cfc23a1c..37767855 100644
+--- a/src/daemon/modules/device/cdi/cdi_cache.c
++++ b/src/daemon/modules/device/cdi/cdi_cache.c
+@@ -14,56 +14,784 @@
+  ******************************************************************************/
+ #include "cdi_cache.h"
+ 
++#include <stdlib.h>
++#include <pthread.h>
++#include <sys/inotify.h>
++#include <sys/prctl.h>
++#include <isula_libutils/log.h>
++#include <isula_libutils/auto_cleanup.h>
++#include <isula_libutils/utils_array.h>
++
++#include "utils.h"
++#include "utils_file.h"
++#include "path.h"
++#include "error.h"
++#include "cdi_device.h"
++#include "cdi_spec.h"
++#include "cdi_spec_dirs.h"
++#include "cdi_container_edits.h"
++
++// cache
++static int cdi_set_spec_dirs(struct cdi_cache *c, string_array *spec_dirs);
++static int configure(struct cdi_cache *c, string_array *spec_dirs);
++static int refresh(struct cdi_cache *c);
++static bool refresh_if_required(struct cdi_cache *c, bool force, int *ret);
++
++// watch
++static void free_cdi_watch(struct cdi_watch *watch);
++static void watch_setup(struct cdi_watch *watch, string_array *dirs);
++static void watch_start(struct cdi_cache *c);
++static void watch_stop(struct cdi_watch *w);
++static void *watch_thread_func(void *arg);
++static bool watch_update(struct cdi_watch *w, const char *removed, int wd);
++
++static int cdi_set_spec_dirs(struct cdi_cache *c, string_array *spec_dirs)
++{
++    __isula_auto_string_array_t string_array *new_spec_dirs = NULL;
++    char clean_path[PATH_MAX] = { 0 };
++    size_t i;
++
++    if (c == NULL || spec_dirs == NULL) {
++        ERROR("Invalid argument");
++        return -1;
++    }
++    if (spec_dirs->len == 0) {
++        return 0;
++    }
++
++    new_spec_dirs = util_string_array_new(spec_dirs->len);
++    if (new_spec_dirs == NULL) {
++        ERROR("Out of memory");
++        return -1;
++    }
++    for (i = 0; i < spec_dirs->len; i++) {
++        if (util_clean_path(spec_dirs->items[i], clean_path, sizeof(clean_path)) == NULL) {
++            ERROR("Failed to get clean path %s", spec_dirs->items[i]);
++            return -1;
++        }
++        if (util_append_string_array(new_spec_dirs, clean_path) != 0) {
++            ERROR("Out of memory");
++            return -1;
++        }
++    }
++    util_free_string_array(c->spec_dirs);
++    c->spec_dirs = new_spec_dirs;
++    new_spec_dirs = NULL;
++
++    return 0;
++}
++
+ void free_cdi_cache(struct cdi_cache *c)
+ {
+-    (void)c;
++    if (c == NULL) {
++        return;
++    }
++
++    util_free_string_array(c->spec_dirs);
++    c->spec_dirs = NULL;
++    map_free(c->specs);
++    c->specs = NULL;
++    map_free(c->devices);
++    c->devices = NULL;
++    free_cdi_watch(c->watch);
++    c->watch = NULL;
++
++    free(c);
+ }
+ 
+ struct cdi_cache *cdi_new_cache(string_array *spec_dirs)
+ {
++    struct cdi_cache *c = NULL;
++    int ret = 0;
++
++    c = util_common_calloc_s(sizeof(*c));
++    if (c == NULL) {
++        ERROR("Out of memory");
++        return NULL;
++    }
++    c->refresh_error_flag = false;
++    c->auto_refresh = true;
++    c->watch = util_common_calloc_s(sizeof(struct cdi_watch));
++    if (c->watch == NULL) {
++        ERROR("Out of memory");
++        goto free_out;
++    }
++    c->watch->watcher_fd = -1;
++
++    if (cdi_set_spec_dirs(c, &g_default_spec_dirs) != 0) {
++        ERROR("Failed to set spec dirs by default");
++        goto free_out;
++    }
++
++    (void)pthread_mutex_lock(&c->mutex);
++    ret = configure(c, spec_dirs);
++    (void)pthread_mutex_unlock(&c->mutex);
++    if (ret != 0) {
++        ERROR("Failed to configure");
++        goto free_out;
++    }
++
++    return c;
++
++free_out:
++    free_cdi_cache(c);
+     return NULL;
+ }
+ 
+-static int cdi_inject_devices(struct cdi_cache *c, oci_runtime_spec *oci_spec, string_array *devices)
++static int cdi_configure(struct cdi_cache *c, string_array *spec_dirs)
+ {
+-    return 0;
++    int ret = 0;
++
++    if (c == NULL) {
++        ERROR("Invalid arguments");
++        return -1;
++    }
++
++    (void)pthread_mutex_lock(&c->mutex);
++    ret = configure(c, spec_dirs);
++    (void)pthread_mutex_unlock(&c->mutex);
++
++    return ret;
+ }
+ 
+-static int cdi_configure(struct cdi_cache *c, string_array *spec_dirs)
++static int configure(struct cdi_cache *c, string_array *spec_dirs)
+ {
++    int ret = 0;
++
++    if (spec_dirs != NULL) {
++        ret = cdi_set_spec_dirs(c, spec_dirs);
++        if (ret != 0) {
++            ERROR("Failed to apply cache spec dirs");
++            return -1;
++        }
++    }
++
++    watch_stop(c->watch);
++    if (c->auto_refresh) {
++        watch_setup(c->watch, c->spec_dirs);
++        watch_start(c);
++    }
++    (void)refresh(c);
+     return 0;
+ }
+ 
+ static int cdi_refresh(struct cdi_cache *c)
+ {
+-    return 0;
++    bool refreshed;
++    int ret = 0;
++    
++    if (c == NULL) {
++        ERROR("Invalid arguments");
++        return -1;
++    }
++
++    (void)pthread_mutex_lock(&c->mutex);
++    refreshed = refresh_if_required(c, !c->auto_refresh, &ret);
++    if (refreshed) {
++        goto unlock_out;
++    }
++
++    ret = c->refresh_error_flag ? -1 : 0;
++unlock_out:
++    (void)pthread_mutex_unlock(&c->mutex);
++    return ret;
+ }
+ 
+-static map_t *cdi_get_errors(struct cdi_cache *c)
++static void map_cdi_cache_specs_kvfree(void *key, void *value)
+ {
+-    return NULL;
++    free(key);
++    util_free_common_array((common_array *)value);
+ }
+ 
+-static string_array *cdi_get_spec_directories(struct cdi_cache *c)
++static void map_cdi_cache_device_kvfree(void *key, void *value)
+ {
+-    return NULL;
++    free(key);
++    free_cdi_cache_device((struct cdi_cache_device *)value);
+ }
+ 
+-static map_t *cdi_get_spec_dir_errors(struct cdi_cache *c)
++static void set_refresh_error_flag(bool *refresh_error_flag, const char *error, const char *path)
+ {
+-    return NULL;
++    *refresh_error_flag = true;
++    ERROR("Cdi refresh error: %s, spec %s", error, path);
++}
++
++static bool resolve_conflict(struct cdi_scan_fn_maps *scan_fn_maps, const char *name,
++                             struct cdi_cache_device *dev, struct cdi_cache_device *old)
++{
++    map_t *conflicts = scan_fn_maps->conflicts;
++    bool *refresh_error_flag = scan_fn_maps->refresh_error_flag;
++    struct cdi_cache_spec *dev_spec = NULL;
++    struct cdi_cache_spec *old_spec = NULL;
++    int dev_prio;
++    int old_prio;
++    bool val = true;
++    const char *dev_path = NULL;
++    const char *old_path = NULL;
++
++    dev_spec = cdi_device_get_spec(dev);
++    old_spec = cdi_device_get_spec(old);
++    dev_prio = cdi_spec_get_priority(dev_spec);
++    old_prio = cdi_spec_get_priority(old_spec);
++    if (dev_prio > old_prio) {
++        return false;
++    } else if (dev_prio == old_prio) {
++        dev_path = cdi_spec_get_path(dev_spec);
++        old_path = cdi_spec_get_path(old_spec);
++        *refresh_error_flag = true;
++        ERROR("Conflicting device %s (specs %s, %s)", name, dev_path, old_path);
++        if (!map_replace(conflicts, (void *)name, (void *)&val)) {
++            ERROR("Failed to insert bool to conflicts by name %s", name);
++            return true;
++        }
++    } else {
++        // do nothing
++    }
++
++    return true;
++}
++
++static void refresh_scan_spec_func(struct cdi_scan_fn_maps *scan_fn_maps, const char *path, 
++                                    int priority, struct cdi_cache_spec *spec, char *error)
++{
++    map_t *specs = scan_fn_maps->specs;
++    map_t *devices = scan_fn_maps->devices;
++    bool *refresh_error_flag = scan_fn_maps->refresh_error_flag;
++    char clean_path[PATH_MAX] = { 0 };
++    __isula_auto_free char *tmp_error = NULL;
++    const char *vendor = NULL;
++    __isula_auto_common_array_t common_array *spec_array = NULL;
++    map_itor *itor = NULL;
++    __isula_auto_free char *qualified = NULL;
++    struct cdi_cache_device *dev = NULL;
++    struct cdi_cache_device *other = NULL;
++
++    if (util_clean_path(path, clean_path, sizeof(clean_path)) == NULL) {
++        ERROR("Failed to get clean path %s", path);
++        format_errorf(&tmp_error, "Failed to get clean path %s", path);
++        return;
++    }
++    if (error != NULL) {
++        ERROR("Failed to load CDI Spec %s", error);
++        format_errorf(&tmp_error, "Failed to load CDI Spec %s", error);
++        goto error_out;
++    }
++
++    vendor = cdi_spec_get_vendor(spec);
++    spec_array = map_search(specs, (void *)vendor);
++    if (spec_array == NULL) {
++        spec_array = util_common_array_new(1, (free_common_array_item_cb)free_cdi_cache_spec, util_clone_ptr);
++        if (spec_array == NULL) {
++            ERROR("Out of memory");
++            tmp_error = util_strdup_s("Out of memory");
++            goto error_out;
++        }
++        if (!map_insert(specs, (void *)vendor, spec_array)) {
++            ERROR("Failed to insert spec array to specs");
++            tmp_error = util_strdup_s("Failed to insert spec array to specs");
++            goto error_out;
++        }
++    }
++    if (util_append_common_array(spec_array, spec) != 0) {
++        ERROR("Failed to append spec");
++        tmp_error = util_strdup_s("Failed to append spec");
++        goto error_out;
++    }
++    spec_array = NULL;
++
++    itor = map_itor_new(spec->devices);
++    if (itor == NULL) {
++        ERROR("Out of memory, create new map itor failed");
++        tmp_error = util_strdup_s("Out of memory, create new map itor failed");
++        goto error_out;
++    }
++    for (; map_itor_valid(itor); map_itor_next(itor)) {
++        dev = map_itor_value(itor);
++        qualified = cdi_device_get_qualified_name(dev);
++        other = map_search(devices, (void *)qualified);
++        if (other != NULL) {
++            if (resolve_conflict(scan_fn_maps, qualified, dev, other)) {
++                continue;
++            }
++        }
++        if (!map_replace(devices, (void *)qualified, dev)) {
++            ERROR("Failed to insert device to devices by name %s", qualified);
++            format_errorf(&tmp_error, "Failed to insert device to devices by name %s", qualified);
++            goto error_out;
++        }
++        free(qualified);
++        qualified = NULL;
++    }
++    goto out;
++
++error_out:
++    set_refresh_error_flag(refresh_error_flag, tmp_error, path);
++out:
++    map_itor_free(itor);
++    return;
++}
++
++static int refresh(struct cdi_cache *c)
++{
++    int ret = 0;
++    __isula_auto_free char *error = NULL;
++    map_t *specs = NULL;
++    map_t *devices = NULL;
++    map_t *conflicts = NULL;
++    struct cdi_scan_fn_maps scan_fn_maps = { 0 };
++    map_itor *itor = NULL;
++    char *conflict = NULL;
++
++    c->refresh_error_flag = false;
++    specs = map_new(MAP_STR_PTR, MAP_DEFAULT_CMP_FUNC, map_cdi_cache_specs_kvfree);
++    if (specs == NULL) {
++        ERROR("Out of memory");
++        ret = -1;
++        goto free_out;
++    }
++    devices = map_new(MAP_STR_PTR, MAP_DEFAULT_CMP_FUNC, map_cdi_cache_device_kvfree);
++    if (devices == NULL) {
++        ERROR("Out of memory");
++        ret = -1;
++        goto free_out;
++    }
++    conflicts = map_new(MAP_STR_BOOL, MAP_DEFAULT_CMP_FUNC, MAP_DEFAULT_FREE_FUNC);
++    if (conflicts == NULL) {
++        ERROR("Out of memory");
++        ret = -1;
++        goto free_out;
++    }
++
++    scan_fn_maps.specs = specs;
++    scan_fn_maps.devices = devices;
++    scan_fn_maps.conflicts = conflicts;
++    scan_fn_maps.refresh_error_flag = &c->refresh_error_flag;
++    // ignore error when scan spec dirs
++    (void)cdi_scan_spec_dirs(c->spec_dirs, &scan_fn_maps, refresh_scan_spec_func);
++
++    itor = map_itor_new(conflicts);
++    if (itor == NULL) {
++        ERROR("Out of memory, create new map itor failed");
++        ret = -1;
++        goto free_out;
++    }
++    for (; map_itor_valid(itor); map_itor_next(itor)) {
++        conflict = map_itor_key(itor);
++        if ((map_search(devices, conflict) != NULL) &&
++            !map_remove(devices, conflict)) {
++            ERROR("Failed to remove conflict device from devices");
++            ret = -1;
++            goto free_out;
++        }
++    }
++
++    util_swap_ptr((void **)&c->specs, (void **)&specs);
++    util_swap_ptr((void **)&c->devices, (void **)&devices);
++
++    ret = c->refresh_error_flag ? -1 : 0;
++    
++free_out:
++    map_itor_free(itor);
++    map_free(specs);
++    map_free(devices);
++    map_free(conflicts);
++    return ret;
++}
++
++static bool refresh_if_required(struct cdi_cache *c, bool force, int *ret)
++{
++    if (force || (c->auto_refresh && watch_update(c->watch, NULL, -1))) {
++        *ret = refresh(c);
++        return true;
++    }
++    return false;
++}
++
++static void map_spec_ptr_kvfree(void *key, void *value)
++{
++    // do not need free spec*
++    (void)key;
++    free(value);
++}
++
++static int cdi_inject_devices(struct cdi_cache *c, oci_runtime_spec *oci_spec, string_array *devices)
++{
++    int ret = 0;
++    __isula_auto_string_array_t string_array *unresolved = NULL;
++    cdi_container_edits *edits = NULL;
++    map_t *specs = NULL;
++    size_t i;
++    const char *device = NULL;
++    struct cdi_cache_device *d = NULL;
++    int tmp_val = 0;
++    __isula_auto_free char *unresolved_str = NULL;
++
++    if (c == NULL || devices == NULL) {
++        ERROR("Can't inject devices");
++        return -1;
++    }
++    if (oci_spec == NULL) {
++        ERROR("Can't inject devices, nil OCI Spec");
++        return -1;
++    }
++
++    unresolved = util_common_calloc_s(sizeof(*unresolved));
++    if (unresolved == NULL) {
++        ERROR("Out of memory");
++        return -1;
++    }
++    specs = map_new(MAP_PTR_INT, MAP_DEFAULT_CMP_FUNC, map_spec_ptr_kvfree);
++    if (specs == NULL) {
++        ERROR("Out of memory");
++        ret = -1;
++        goto out;
++    }
++    edits = util_common_calloc_s(sizeof(*edits));
++    if (edits == NULL) {
++        ERROR("Out of memory");
++        ret = -1;
++        goto out;
++    }
++
++    (void)pthread_mutex_lock(&c->mutex);
++
++    (void)refresh_if_required(c, false, &ret);
++
++    for(i = 0; i < devices->len; i++) {
++        device = devices->items[i];
++        d = map_search(c->devices, (void *)device);
++        if (d == NULL) {
++            if (util_append_string_array(unresolved, device) != 0) {
++                ERROR("Out of memory");
++                ret = -1;
++                goto out;
++            }
++            continue;
++        }
++        if (map_search(specs, (void *)cdi_device_get_spec(d)) == NULL) {
++            if (!map_insert(specs, (void *)cdi_device_get_spec(d), (void *)&tmp_val)) {
++                ERROR("Failed to insert spec ptr to specs when find device %s", device);
++                ret = -1;
++                goto out;
++            }
++            if (cdi_container_edits_append(edits, cdi_spec_get_edits(cdi_device_get_spec(d))) != 0) {
++                ERROR("Failed to append edits when find device %s", device);
++                ret = -1;
++                goto out;
++            }
++        }
++        if (cdi_container_edits_append(edits, cdi_device_get_edits(d)) != 0) {
++            ERROR("Failed to append edits when find device %s", device);
++            ret = -1;
++            goto out;
++        }
++    }
++
++    if (unresolved->len != 0) {
++        unresolved_str = util_string_join(", ", (const char **)unresolved->items, unresolved->len);
++        ERROR("Unresolvable CDI devices %s", unresolved_str);
++        ret = -1;
++        goto out;
++    }
++
++    ret = cdi_container_edits_apply(edits, oci_spec);
++    if (ret != 0) {
++        ERROR("Failed to apply edits when inject devices");
++        ret = -1;
++    }
++
++out:
++    (void)pthread_mutex_unlock(&c->mutex);
++    map_free(specs);
++    free_cdi_container_edits(edits);
++    return ret;
+ }
+ 
+ static struct cdi_cache_ops g_cdi_cache_ops = {
+     .inject_devices = cdi_inject_devices,
+     .configure = cdi_configure,
+-    .refresh = cdi_refresh,
+-    .get_errors = cdi_get_errors,
+-    .get_spec_directories = cdi_get_spec_directories,
+-    .get_spec_dir_errors = cdi_get_spec_dir_errors
++    .refresh = cdi_refresh
+ };
+ 
+ struct cdi_cache_ops *cdi_get_cache_ops(void)
+ {
+     return &g_cdi_cache_ops;
+-}
+\ No newline at end of file
++}
++
++static void free_cdi_watch(struct cdi_watch *w)
++{
++    if (w == NULL) {
++        return;
++    }
++
++    watch_stop(w);
++    free(w);
++}
++
++static int init_tracked(struct cdi_watch *w, string_array *dirs)
++{
++    size_t i;
++    bool tmp_value = false;
++
++    w->tracked = map_new(MAP_STR_BOOL, MAP_DEFAULT_CMP_FUNC, MAP_DEFAULT_FREE_FUNC);
++    if (w->tracked == NULL) {
++        ERROR("Out of memory");
++        return -1;
++    }
++    for(i = 0; i < dirs->len; i++) {
++        if (!map_replace(w->tracked, (void *)dirs->items[i], (void *)&tmp_value)) {
++            ERROR("Failed to insert tracked by dir %s", dirs->items[i]);
++            goto error_out;
++        }
++    }
++    w->wd_dirs = map_new(MAP_INT_STR, MAP_DEFAULT_CMP_FUNC, MAP_DEFAULT_FREE_FUNC);
++    if (w->wd_dirs == NULL) {
++        ERROR("Out of memory");
++        goto error_out;
++    }
++
++    return 0;
++
++error_out:
++    map_free(w->tracked);
++    w->tracked = NULL;
++    return -1;
++}
++
++static void watch_setup(struct cdi_watch *w, string_array *dirs)
++{
++    __isula_auto_free char *error = NULL;
++
++    if (w == NULL || dirs == NULL || dirs->len == 0) {
++        ERROR("Invalid param");
++        return;
++    }
++
++    if (init_tracked(w, dirs) != 0) {
++        ERROR("Failed to initialize tracked");
++        return;
++    }
++
++    w->watcher_fd = inotify_init();
++    if (w->watcher_fd < 0) {
++        ERROR("Failed to initialize inotify fd");
++        map_free(w->tracked);
++        w->tracked = NULL;
++        map_free(w->wd_dirs);
++        w->wd_dirs = NULL;
++        return;
++    }
++
++    (void)watch_update(w, NULL, -1);
++}
++
++static void watch_start(struct cdi_cache *c)
++{
++    pthread_t thread = 0;
++    int ret = 0;
++
++    ret = pthread_create(&thread, NULL, watch_thread_func, c);
++    if (ret != 0) {
++        ERROR("Cdi watch thread create failed");
++        return;
++    }
++}
++
++static void watch_stop(struct cdi_watch *w)
++{
++    if (w == NULL) {
++        return;
++    }
++
++    if (w->watcher_fd >= 0) {
++        close(w->watcher_fd);
++        w->watcher_fd = -1;
++    }
++    map_free(w->tracked);
++    w->tracked = NULL;
++    map_free(w->wd_dirs);
++    w->wd_dirs = NULL;
++}
++
++// wait_events wait until inotify
++static int wait_events(int watcher_fd)
++{
++    fd_set rfds;
++    FD_ZERO(&rfds);
++    FD_SET(watcher_fd, &rfds);
++    return select(FD_SETSIZE, &rfds, NULL, NULL, NULL);
++}
++
++#define CDI_WATCH_EVENTS    (IN_MOVED_TO | IN_MOVED_FROM | IN_DELETE | IN_MODIFY | IN_MOVE_SELF | IN_DELETE_SELF)
++
++static int process_cdi_events(int watcher_fd, struct cdi_cache *c)
++{
++    ssize_t events_length = 0;
++    ssize_t events_index = 0;
++    struct inotify_event *cdi_event = NULL;
++    char buffer[MAXLINE] __attribute__((aligned(__alignof__(struct inotify_event)))) = { 0 };
++    int update_cnt = 0;
++    __isula_auto_free char *event_dir = NULL;
++
++    events_length = util_read_nointr(watcher_fd, buffer, sizeof(buffer));
++    if (events_length <= 0) {
++        ERROR("Failed to wait events");
++        return -1;
++    }
++
++    (void)pthread_mutex_lock(&c->mutex);
++    
++    while (events_index < events_length) {
++        cdi_event = (struct inotify_event *)(&buffer[events_index]);
++        ssize_t event_size = (ssize_t)(cdi_event->len) + (ssize_t)offsetof(struct inotify_event, name);
++        if (event_size == 0 || event_size > (events_length - events_index)) {
++            break;
++        }
++        events_index += event_size;
++
++        /*  
++         *  file: 
++         *      Rename:  mask == IN_MOVED_TO | IN_MOVED_FROM
++         *      Remove:  mask == IN_MOVED_FROM || mask == IN_DELETE
++         *      Write:   mask == IN_MODIFY
++         *  dir:
++         *      Remove: mask == IN_MOVE_SELF || mask == IN_DELETE_SELF
++         */
++        if ((cdi_event->mask & CDI_WATCH_EVENTS) == 0) {
++            continue;
++        }
++        DEBUG("Cdi spec file %s is changed", cdi_event->name);
++        if (cdi_event->mask == IN_MODIFY) {
++            if (!util_has_suffix(cdi_event->name, ".json")) {
++                WARN("Invalid spec %s ext", cdi_event->name);
++                continue;
++            }
++        }
++        event_dir = util_strdup_s(map_search(c->watch->wd_dirs, &(cdi_event->wd)));
++        if (!(cdi_event->mask == IN_DELETE_SELF || cdi_event->mask == IN_MOVE_SELF)) {
++            free(event_dir);
++            event_dir = NULL;
++        }
++        watch_update(c->watch, event_dir, cdi_event->wd);
++        update_cnt++;
++    }
++    if (update_cnt > 0) {
++        (void)refresh(c);
++    }
++
++    (void)pthread_mutex_unlock(&c->mutex);
++    return 0;
++}
++
++// Watch Spec directory changes, triggering a refresh if necessary.
++static void *watch_thread_func(void *arg)
++{
++    struct cdi_cache *c = (struct cdi_cache *)arg;
++    int errcode = 0;
++    int watcher_fd = -1;
++
++    errcode = pthread_detach(pthread_self());
++    if (errcode != 0) {
++        errno = errcode;
++        SYSERROR("Detach thread failed");
++        return NULL;
++    }
++
++    prctl(PR_SET_NAME, "cdi-watcher");
++
++    watcher_fd = c->watch->watcher_fd;
++    if (watcher_fd < 0) {
++        ERROR("Invalid inotify fd");
++        return NULL;
++    }
++
++    for (;;) {
++        if (wait_events(watcher_fd) < 0) {
++            ERROR("Failed to wait events");
++            break;
++        }
++        if (process_cdi_events(watcher_fd, c) != 0) {
++            break;
++        }
++    }
++    return NULL;
++}
++
++static void update_remove_watch_dir(struct cdi_watch *w, const char *dir, int wd)
++{
++    bool tmp_value = false;
++    if (wd >= 0) {
++        (void)inotify_rm_watch(w->watcher_fd, wd);
++        if ((map_search(w->wd_dirs, &wd) != NULL) &&
++            !map_remove(w->wd_dirs, &wd)) {
++            ERROR("Failed to remove watch fd of %s", dir);
++        }
++    }
++    if (!map_replace(w->tracked, (void *)dir, (void *)&tmp_value)) {
++        ERROR("Failed to insert tracked by dir %s", dir);
++    }
++}
++
++static void update_add_watch_dir(struct cdi_watch *w, const char *dir, bool *update)
++{
++    int wd = -1;
++    bool tmp_value = true;
++    __isula_auto_free char *error = NULL;
++
++    wd = inotify_add_watch(w->watcher_fd, dir, CDI_WATCH_EVENTS);
++    if (wd < 0) {
++        if (errno == ENOENT) {
++            SYSINFO("Watch device dir %s", dir);
++        } else {
++            SYSERROR("Failed to watch device dir %s", dir);
++        }
++        return;
++    } else {
++        DEBUG("Watching %s for device disovery", dir);
++        tmp_value = true;
++        if (!map_replace(w->tracked, (void *)dir, (void *)&tmp_value)) {
++            ERROR("Failed to insert tracked by dir %s", dir);
++            goto error_out;
++        }
++        if (!map_replace(w->wd_dirs, (void *)&wd, (void *)dir)) {
++            ERROR("Failed to insert dir %s by wd", dir);
++            goto error_out;
++        }
++        *update = true;
++    }
++    return;
++
++error_out:
++    update_remove_watch_dir(w, dir, wd);
++}
++
++static bool watch_update(struct cdi_watch *w, const char *removed, int wd)
++{
++    const char *dir = NULL;
++    bool *ok = NULL;
++    bool update = false;
++    map_itor *itor = NULL;
++    __isula_auto_free char *error = NULL;
++
++    itor = map_itor_new(w->tracked);
++    if (itor == NULL) {
++        ERROR("Out of memory, create new map itor failed");
++        return false;
++    }
++    for (; map_itor_valid(itor); map_itor_next(itor)) {
++        dir = map_itor_key(itor);
++        ok = map_itor_value(itor);
++        if (ok == NULL || *ok) {
++            continue;
++        }
++        update_add_watch_dir(w, dir, &update);
++    }
++
++    if (removed != NULL) {
++        update_remove_watch_dir(w, removed, wd);
++        WARN("Directory removed: %s", removed);
++        update = true;
++    }
++
++    map_itor_free(itor);
++    return update;
++}
+diff --git a/src/daemon/modules/device/cdi/cdi_cache.h b/src/daemon/modules/device/cdi/cdi_cache.h
+index 34c27471..da315de2 100644
+--- a/src/daemon/modules/device/cdi/cdi_cache.h
++++ b/src/daemon/modules/device/cdi/cdi_cache.h
+@@ -39,9 +39,6 @@ struct cdi_cache_ops {
+     // Refresher
+     int (*configure)(struct cdi_cache *c, string_array *spec_dirs);
+     int (*refresh)(struct cdi_cache *c);
+-    map_t *(*get_errors)(struct cdi_cache *c);
+-    string_array *(*get_spec_directories)(struct cdi_cache *c);
+-    map_t *(*get_spec_dir_errors)(struct cdi_cache *c);
+ };
+ 
+ struct cdi_watch {
+@@ -54,11 +51,9 @@ struct cdi_watch {
+ struct cdi_cache {
+     pthread_mutex_t mutex;
+     string_array *spec_dirs; // cdi-spec-dirs will scan for CDI Spec files
+-    map_t *specs;        // MAP_STR_PTR     specs[vendor] = cdi_cache_spec**
++    map_t *specs;        // MAP_STR_PTR     specs[vendor] = common_array of cdi_cache_spec*
+     map_t *devices;      // MAP_STR_PTR     devices[cdi_device.name] = cdi_cache_device*
+-    map_t *errors;       // MAP_STR_PTR     errors[cdi_cache_spec.path] = string_array *errors
+-    map_t *dir_errors;   // MAP_STR_STR     dir_errors[spec_dirs[i]] = error
+-
++    bool refresh_error_flag;
+     bool auto_refresh; 
+     struct cdi_watch *watch;
+ };
+-- 
+2.25.1
+

0065-change-default-subscribe-timeout-to-5min.patch

@@ -0,0 +1,76 @@
+From 5c89c23f5e0de06a17a9263114430674221a1ee0 Mon Sep 17 00:00:00 2001
+From: jikai <jikai11@huawei.com>
+Date: Wed, 17 Apr 2024 06:59:08 +0000
+Subject: [PATCH 065/149] change default subscribe timeout to 5min
+
+Signed-off-by: jikai <jikai11@huawei.com>
+---
+ src/daemon/mailbox/message_queue.c |  4 ++--
+ src/utils/cutils/blocking_queue.c  | 13 +++++++------
+ src/utils/cutils/blocking_queue.h  |  2 +-
+ 3 files changed, 10 insertions(+), 9 deletions(-)
+
+diff --git a/src/daemon/mailbox/message_queue.c b/src/daemon/mailbox/message_queue.c
+index 7e53301e..699ea0bc 100644
+--- a/src/daemon/mailbox/message_queue.c
++++ b/src/daemon/mailbox/message_queue.c
+@@ -20,8 +20,8 @@
+ 
+ #include "utils.h"
+ 
+-// default set subscriber timeout to 1000ms, maybe could be configured later
+-const int64_t subscribe_timeout = 1000;
++// default set subscriber timeout to 300s, maybe could be configured later
++const int64_t subscribe_timeout = 300;
+ 
+ static void message_queue_subscriber_free(void *key, void *val)
+ {
+diff --git a/src/utils/cutils/blocking_queue.c b/src/utils/cutils/blocking_queue.c
+index 7c9c5f50..9bdb2ca3 100644
+--- a/src/utils/cutils/blocking_queue.c
++++ b/src/utils/cutils/blocking_queue.c
+@@ -55,12 +55,11 @@ blocking_queue *blocking_queue_create(int64_t timeout, void (*release)(void *))
+     queue->release = release;
+ 
+     if (timeout >= 0) {
+-        queue->timeout.tv_sec = timeout / (Time_Second / Time_Milli);
+-        queue->timeout.tv_nsec = (timeout % (Time_Second / Time_Milli) ) * Time_Milli;
++        queue->timeout = timeout;
+     } else {
+-        queue->timeout.tv_sec = -1;
++        queue->timeout = -1;
+     }
+-    
++
+     return isula_transfer_ptr(queue);
+ }
+ 
+@@ -112,8 +111,10 @@ int blocking_queue_pop(blocking_queue *queue, void **data) {
+     lock = &queue->lock;
+ 
+     while (queue->head->next == NULL) {
+-        if (queue->timeout.tv_sec >= 0) {
+-            int ret = pthread_cond_timedwait(&queue->not_empty, &queue->lock, &queue->timeout);
++        if (queue->timeout >= 0) {
++            struct timespec timeout = { 0 };
++            timeout.tv_sec = queue->timeout + time(NULL);
++            int ret = pthread_cond_timedwait(&queue->not_empty, &queue->lock, &timeout);
+             if (ret != 0) {
+                 if (ret != ETIMEDOUT) {
+                     ERROR("Failed to wait cond");
+diff --git a/src/utils/cutils/blocking_queue.h b/src/utils/cutils/blocking_queue.h
+index 1c52a9d3..257779c3 100644
+--- a/src/utils/cutils/blocking_queue.h
++++ b/src/utils/cutils/blocking_queue.h
+@@ -37,7 +37,7 @@ typedef struct blocking_queue {
+     blocking_node *head;
+     blocking_node *tail;
+     pthread_mutex_t lock;
+-    struct timespec timeout;
++    int64_t timeout;
+     pthread_cond_t not_empty;
+     void (*release)(void *);
+ } blocking_queue;
+-- 
+2.25.1
+

0068-cdi-invoke-cdi-operate-when-init-isulad-and-create-c.patch

@@ -0,0 +1,371 @@
+From 0674bfac4dd1ab812432334c779ab718dc54bc8b Mon Sep 17 00:00:00 2001
+From: liuxu <liuxu156@huawei.com>
+Date: Thu, 11 Apr 2024 11:02:19 +0800
+Subject: [PATCH 068/149] cdi:invoke cdi operate when init isulad and create
+ container
+
+---
+ src/cmd/isulad/main.c                         | 11 +++
+ src/daemon/common/cri/v1/v1_cri_helpers.cc    | 79 +++++++++++++++++++
+ src/daemon/common/cri/v1/v1_cri_helpers.h     |  3 +
+ src/daemon/config/daemon_arguments.c          |  4 +
+ src/daemon/config/isulad_config.c             |  8 ++
+ .../v1/v1_cri_container_manager_service.cc    |  8 ++
+ .../executor/container_cb/execution_create.c  |  9 +++
+ .../modules/service/service_container.c       | 10 +++
+ src/daemon/modules/spec/specs_mount.c         | 43 +++++++++-
+ src/daemon/modules/spec/specs_mount.h         |  4 +
+ src/daemon/modules/spec/verify.c              |  2 +-
+ 11 files changed, 179 insertions(+), 2 deletions(-)
+
+diff --git a/src/cmd/isulad/main.c b/src/cmd/isulad/main.c
+index 9fa87bdb..3e2249d7 100644
+--- a/src/cmd/isulad/main.c
++++ b/src/cmd/isulad/main.c
+@@ -83,6 +83,9 @@
+ #endif
+ #include "id_name_manager.h"
+ #include "cgroup.h"
++#ifdef ENABLE_CDI
++#include "cdi_operate_api.h"
++#endif /* ENABLE_CDI */
+ 
+ sem_t g_daemon_shutdown_sem;
+ sem_t g_daemon_wait_shutdown_sem;
+@@ -1400,6 +1403,14 @@ static int isulad_server_init_common()
+     }
+ #endif
+ 
++#ifdef ENABLE_CDI
++    if (args->json_confs->enable_cdi &&
++        cdi_operate_registry_init(args->json_confs->cdi_spec_dirs, args->json_confs->cdi_spec_dirs_len) != 0) {
++        ERROR("Failed to init CDI module");
++        goto out;
++    }
++#endif /* ENABLE_CDI */
++
+     if (spec_module_init() != 0) {
+         ERROR("Failed to init spec module");
+         goto out;
+diff --git a/src/daemon/common/cri/v1/v1_cri_helpers.cc b/src/daemon/common/cri/v1/v1_cri_helpers.cc
+index ea5c8bb5..520d23d4 100644
+--- a/src/daemon/common/cri/v1/v1_cri_helpers.cc
++++ b/src/daemon/common/cri/v1/v1_cri_helpers.cc
+@@ -22,6 +22,7 @@
+ 
+ #include <isula_libutils/log.h>
+ #include <isula_libutils/parse_common.h>
++#include <isula_libutils/auto_cleanup.h>
+ 
+ #include "v1_cri_security_context.h"
+ #include "cri_helpers.h"
+@@ -33,6 +34,9 @@
+ #include "isulad_config.h"
+ #include "sha256.h"
+ #include "v1_naming.h"
++#ifdef ENABLE_CDI
++#include "cdi_operate_api.h"
++#endif /* ENABLE_CDI */
+ 
+ namespace CRIHelpersV1 {
+ 
+@@ -666,4 +670,79 @@ std::unique_ptr<runtime::v1::ContainerStatus> GetContainerStatus(service_executo
+     return contStatus;
+ }
+ 
++#ifdef ENABLE_CDI
++static int InsertCDIDevices(std::unordered_set<std::string> &fromCRI, const std::string &devName,
++                            string_array *requested, Errors &err)
++{
++    if (fromCRI.find(devName) == fromCRI.end()) {
++        fromCRI.insert(devName);
++        if (util_append_string_array(requested, devName.c_str()) != 0) {
++            ERROR("Out of memory");
++            err.Errorf("Out of memory");
++            return -1;
++        }
++        DEBUG("Appended device: %s", devName.c_str());
++    } else {
++        INFO("Skipping duplicate CDI device %s", devName.c_str());
++    }
++    return 0;
++}
++ 
++void GenerateCDIRequestedDevices(const runtime::v1::ContainerConfig &config, host_config *hostconfig, Errors &err)
++{
++    std::unordered_set<std::string> fromCRI;
++    __isula_auto_string_array_t string_array *requested = nullptr;
++    __isula_auto_string_array_t string_array *keys = nullptr;
++    __isula_auto_string_array_t string_array *devices = nullptr;
++    json_map_string_string *annotations = nullptr;
++    __isula_auto_free char *error = nullptr;
++ 
++    if (hostconfig == nullptr) {
++        ERROR("Invalid input arguments");
++        err.Errorf("Invalid input arguments");
++        return;
++    }
++    
++    if (config.cdi_devices().empty() && config.annotations().empty()) {
++        return;
++    }
++    requested = (string_array *)util_common_calloc_s(sizeof(*requested));
++    if (requested == nullptr) {
++        ERROR("Out of memory");
++        err.Errorf("Out of memory");
++        return;
++    }
++    if (!config.cdi_devices().empty()) {
++        for (int i = 0; i < config.cdi_devices().size(); i++) {
++            if (InsertCDIDevices(fromCRI, config.cdi_devices(i).name(), requested, err) != 0) {
++                goto free_out;
++            }
++        }
++    }
++    if (!config.annotations().empty()) {
++        annotations = CRIHelpers::MakeAnnotations(config.annotations(), err);
++        if (err.NotEmpty()) {
++            goto free_out;
++        }
++        if (cdi_operate_parse_annotations(annotations, &keys, &devices, &error) != 0) {
++            ERROR("Failed to parse CDI annotations: %s", error);
++            err.Errorf("Failed to parse CDI annotations: %s", error);
++            goto free_out;
++        }
++        for (size_t i = 0; i < devices->len; i++) {
++            if (InsertCDIDevices(fromCRI, std::string(devices->items[i]), requested, err) != 0) {
++                goto free_out;
++            }
++        }
++    }
++    hostconfig->cdi_requested_devices = requested->items;
++    requested->items = nullptr;
++    hostconfig->cdi_requested_devices_len = requested->len;
++    requested->len = 0;
++ 
++free_out:
++    free_json_map_string_string(annotations);
++}
++#endif /* ENABLE_CDI */
++
+ } // v1 namespace CRIHelpers
+diff --git a/src/daemon/common/cri/v1/v1_cri_helpers.h b/src/daemon/common/cri/v1/v1_cri_helpers.h
+index 1578c428..22cffd0d 100644
+--- a/src/daemon/common/cri/v1/v1_cri_helpers.h
++++ b/src/daemon/common/cri/v1/v1_cri_helpers.h
+@@ -79,6 +79,9 @@ std::string CRISandboxerConvert(const std::string &runtime);
+ 
+ void ApplySandboxSecurityContextToHostConfig(const runtime::v1::LinuxSandboxSecurityContext &context, host_config *hc,
+                                              Errors &error);
++#ifdef ENABLE_CDI
++void GenerateCDIRequestedDevices(const runtime::v1::ContainerConfig &config, host_config *hostconfig, Errors &err);
++#endif /* ENABLE_CDI */
+ 
+ auto GetContainerStatus(service_executor_t *m_cb, const std::string &containerID, Errors &error)
+ -> std::unique_ptr<runtime::v1::ContainerStatus>;
+diff --git a/src/daemon/config/daemon_arguments.c b/src/daemon/config/daemon_arguments.c
+index 0ae6268a..ef15934a 100644
+--- a/src/daemon/config/daemon_arguments.c
++++ b/src/daemon/config/daemon_arguments.c
+@@ -173,6 +173,10 @@ int service_arguments_init(struct service_arguments *args)
+         goto free_out;
+     }
+ 
++#ifdef ENABLE_CDI
++    args->json_confs->enable_cdi = false;
++#endif /* ENABLE_CDI */
++
+     ret = 0;
+ 
+ free_out:
+diff --git a/src/daemon/config/isulad_config.c b/src/daemon/config/isulad_config.c
+index 778ff921..695a0d95 100644
+--- a/src/daemon/config/isulad_config.c
++++ b/src/daemon/config/isulad_config.c
+@@ -1830,6 +1830,14 @@ int merge_json_confs_into_global(struct service_arguments *args)
+     args->json_confs->metrics_port = tmp_json_confs->metrics_port;
+ #endif
+ 
++#ifdef ENABLE_CDI
++    args->json_confs->enable_cdi = tmp_json_confs->enable_cdi;
++    args->json_confs->cdi_spec_dirs = tmp_json_confs->cdi_spec_dirs;
++    tmp_json_confs->cdi_spec_dirs = NULL;
++    args->json_confs->cdi_spec_dirs_len = tmp_json_confs->cdi_spec_dirs_len;
++    tmp_json_confs->cdi_spec_dirs_len = 0;
++#endif /* ENABLE_CDI */
++ 
+ out:
+     free(err);
+     free_isulad_daemon_configs(tmp_json_confs);
+diff --git a/src/daemon/entry/cri/v1/v1_cri_container_manager_service.cc b/src/daemon/entry/cri/v1/v1_cri_container_manager_service.cc
+index e86dafae..1097c32c 100644
+--- a/src/daemon/entry/cri/v1/v1_cri_container_manager_service.cc
++++ b/src/daemon/entry/cri/v1/v1_cri_container_manager_service.cc
+@@ -199,6 +199,14 @@ auto ContainerManagerService::GenerateCreateContainerHostConfig(
+         }
+     }
+ 
++#ifdef ENABLE_CDI
++    CRIHelpersV1::GenerateCDIRequestedDevices(containerConfig, hostconfig, error);
++    if (error.NotEmpty()) {
++        ERROR("Failed to generate CDI requested devices");
++        goto cleanup;
++    }
++#endif /* ENABLE_CDI */
++
+     return hostconfig;
+ 
+ cleanup:
+diff --git a/src/daemon/executor/container_cb/execution_create.c b/src/daemon/executor/container_cb/execution_create.c
+index a9102226..785b4e27 100644
+--- a/src/daemon/executor/container_cb/execution_create.c
++++ b/src/daemon/executor/container_cb/execution_create.c
+@@ -63,6 +63,7 @@
+ #include "runtime_api.h"
+ #include "id_name_manager.h"
+ #include "mailbox.h"
++#include "specs_mount.h"
+ 
+ #ifdef ENABLE_CRI_API_V1
+ static bool validate_sandbox_info(const container_sandbox_info *sandbox)
+@@ -512,6 +513,14 @@ static oci_runtime_spec *generate_oci_config(host_config *host_spec, const char
+         goto error_out;
+     }
+ 
++#ifdef ENABLE_CDI
++    ret = inject_CDI_devcies_for_oci_spec(oci_spec, host_spec);
++    if (ret != 0) {
++        ERROR("Failed to inject CDI devices");
++        goto error_out;
++    }
++#endif /* ENABLE_CDI */
++
+     return oci_spec;
+ 
+ error_out:
+diff --git a/src/daemon/modules/service/service_container.c b/src/daemon/modules/service/service_container.c
+index eb7ce4f4..b19a134a 100644
+--- a/src/daemon/modules/service/service_container.c
++++ b/src/daemon/modules/service/service_container.c
+@@ -2003,6 +2003,16 @@ static defs_process *make_exec_process_spec(const container_config *container_sp
+         }
+ 
+         spec->no_new_privileges = oci_spec->process->no_new_privileges;
++
++#ifdef ENABLE_CDI
++        // extend step: merge env from oci_spec which comes from injected devices
++        ret = defs_process_add_multiple_env(spec, (const char **)oci_spec->process->env,
++            oci_spec->process->env_len);
++        if (ret != 0) {
++            ERROR("Failed to dup oci env for exec process spec");
++            goto err_out;
++        }
++#endif /* ENABLE_CDI */
+     }
+ 
+     // for oci runtime:
+diff --git a/src/daemon/modules/spec/specs_mount.c b/src/daemon/modules/spec/specs_mount.c
+index 50ee9a85..12bd261b 100644
+--- a/src/daemon/modules/spec/specs_mount.c
++++ b/src/daemon/modules/spec/specs_mount.c
+@@ -28,6 +28,7 @@
+ #include <isula_libutils/container_config_v2.h>
+ #include <isula_libutils/json_common.h>
+ #include <isula_libutils/oci_runtime_config_linux.h>
++#include <isula_libutils/auto_cleanup.h>
+ #include <limits.h>
+ #include <stdint.h>
+ 
+@@ -54,6 +55,9 @@
+ #include "volume_api.h"
+ #include "parse_volume.h"
+ #include "specs_api.h"
++#ifdef ENABLE_CDI
++#include "cdi_operate_api.h"
++#endif /* ENABLE_CDI */
+ 
+ enum update_rw {
+     update_rw_untouch,
+@@ -3582,6 +3586,15 @@ int update_devcies_for_oci_spec(oci_runtime_spec *oci_spec, host_config *hostcon
+         oci_spec->linux->resources->devices_len += 1;
+     }
+ 
++    // extend step: inject CDI devcies
++#ifdef ENABLE_CDI
++    ret = inject_CDI_devcies_for_oci_spec(oci_spec, hostconfig);
++    if (ret != 0) {
++        ERROR("Failed to inject CDI devices");
++        return -1;
++    }
++#endif /* ENABLE_CDI */
++
+     // Step8: do update devices and cgroup device rules at here
+     if (hostconfig->privileged) {
+         // Step8.1: for priviledged container, we should merge all devices under /dev
+@@ -3592,4 +3605,32 @@ int update_devcies_for_oci_spec(oci_runtime_spec *oci_spec, host_config *hostcon
+     }
+ 
+     return ret;
+-}
+\ No newline at end of file
++}
++
++#ifdef ENABLE_CDI
++int inject_CDI_devcies_for_oci_spec(oci_runtime_spec *oci_spec, host_config *hostconfig)
++{
++    int ret = 0;
++    string_array devices_array = { 0 };
++    __isula_auto_free char *error = NULL;
++    
++    if (oci_spec == NULL || hostconfig == NULL) {
++        ERROR("Invalid params");
++        return -1;
++    }
++    if (hostconfig->cdi_requested_devices == NULL) {
++        return 0;
++    }
++    devices_array.items = hostconfig->cdi_requested_devices;
++    devices_array.len = hostconfig->cdi_requested_devices_len;
++    devices_array.cap = hostconfig->cdi_requested_devices_len;
++    if (cdi_operate_refresh() != 0) {
++        WARN("CDI registry has errors, please check past logs");
++    }
++    if (cdi_operate_inject_devices(oci_spec, &devices_array) != 0) {
++        ERROR("Failed to inject CDI devices");
++        ret = -1;
++    }
++    return ret;
++}
++#endif /* ENABLE_CDI */
+\ No newline at end of file
+diff --git a/src/daemon/modules/spec/specs_mount.h b/src/daemon/modules/spec/specs_mount.h
+index b742ca35..1406c557 100644
+--- a/src/daemon/modules/spec/specs_mount.h
++++ b/src/daemon/modules/spec/specs_mount.h
+@@ -49,6 +49,10 @@ int setup_ipc_dirs(host_config *host_spec, container_config_v2_common_config *v2
+ 
+ int update_devcies_for_oci_spec(oci_runtime_spec *oci_spec, host_config *hostconfig);
+ 
++#ifdef ENABLE_CDI
++int inject_CDI_devcies_for_oci_spec(oci_runtime_spec *oci_spec, host_config *hostconfig);
++#endif /* ENABLE_CDI */
++
+ #ifdef __cplusplus
+ }
+ #endif
+diff --git a/src/daemon/modules/spec/verify.c b/src/daemon/modules/spec/verify.c
+index af790d6e..617b7f23 100644
+--- a/src/daemon/modules/spec/verify.c
++++ b/src/daemon/modules/spec/verify.c
+@@ -1518,7 +1518,7 @@ static int verify_custom_mount(defs_mount **mounts, size_t len)
+ 
+     for (i = 0; i < len; ++i) {
+         iter = *(mounts + i);
+-        if (iter == NULL || strcmp(iter->type, MOUNT_TYPE_BIND)) {
++        if (iter == NULL || iter->type == NULL || strcmp(iter->type, MOUNT_TYPE_BIND)) {
+             continue;
+         }
+ 
+-- 
+2.25.1
+

0069-bugfix-fix-cni_operate_ut-ut.patch

@@ -0,0 +1,24 @@
+From a1f75fd089309d0f8620195ce7e517294be2c410 Mon Sep 17 00:00:00 2001
+From: liuxu <liuxu156@huawei.com>
+Date: Fri, 19 Apr 2024 18:37:05 +0800
+Subject: [PATCH 069/149] bugfix:fix cni_operate_ut ut
+
+---
+ test/network/cni_operate/CMakeLists.txt | 1 +
+ 1 file changed, 1 insertion(+)
+
+diff --git a/test/network/cni_operate/CMakeLists.txt b/test/network/cni_operate/CMakeLists.txt
+index 5b4d7c7d..752e5199 100644
+--- a/test/network/cni_operate/CMakeLists.txt
++++ b/test/network/cni_operate/CMakeLists.txt
+@@ -14,6 +14,7 @@ add_executable(${EXE}
+     ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/utils/cutils/utils_string.c
+     ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/utils/cutils/utils_array.c
+     ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/utils/cutils/utils_regex.c
++    ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/utils/cutils/utils_version.c
+     ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/utils/cutils/namespace.c
+     ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/utils/sha256/sha256.c
+     ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/utils/cutils/path.c
+-- 
+2.25.1
+

0070-isolate-sandboxer-code-by-using-macro.patch

@@ -0,0 +1,143 @@
+From c1d445e178cd610f8a6d9156012c6c7922eed9c5 Mon Sep 17 00:00:00 2001
+From: xuxuepeng <xuxuepeng1@huawei.com>
+Date: Sat, 20 Apr 2024 11:24:18 +0800
+Subject: [PATCH 070/149] isolate sandboxer code by using macro
+
+Signed-off-by: xuxuepeng <xuxuepeng1@huawei.com>
+---
+ cmake/options.cmake                                 | 2 +-
+ src/daemon/common/cri/v1/v1_cri_helpers.cc          | 7 +++++++
+ src/daemon/config/isulad_config.c                   | 2 ++
+ src/daemon/sandbox/controller/CMakeLists.txt        | 2 +-
+ src/daemon/sandbox/controller/controller_manager.cc | 6 ++++++
+ src/daemon/sandbox/controller/controller_manager.h  | 2 ++
+ 6 files changed, 19 insertions(+), 2 deletions(-)
+
+diff --git a/cmake/options.cmake b/cmake/options.cmake
+index c1eac472..a15b8194 100644
+--- a/cmake/options.cmake
++++ b/cmake/options.cmake
+@@ -51,7 +51,7 @@ if (ENABLE_CDI STREQUAL "ON")
+     endif()
+ endif()
+ 
+-option(ENABLE_SANDBOXER "Enable sandbox API" ON)
++option(ENABLE_SANDBOXER "Enable sandbox API" OFF)
+ if (ENABLE_SANDBOXER STREQUAL "ON")
+     add_definitions(-DENABLE_SANDBOXER)
+     set(ENABLE_SANDBOXER 1)
+diff --git a/src/daemon/common/cri/v1/v1_cri_helpers.cc b/src/daemon/common/cri/v1/v1_cri_helpers.cc
+index 520d23d4..1f797ad7 100644
+--- a/src/daemon/common/cri/v1/v1_cri_helpers.cc
++++ b/src/daemon/common/cri/v1/v1_cri_helpers.cc
+@@ -391,6 +391,7 @@ void GetContainerSandboxID(const std::string &containerID, std::string &realCont
+     realContainerID = info->id;
+ }
+ 
++#ifdef ENABLE_SANDBOXER
+ std::string CRISandboxerConvert(const std::string &runtime)
+ {
+     std::string sandboxer;
+@@ -429,6 +430,12 @@ out:
+     (void)isulad_server_conf_unlock();
+     return sandboxer;
+ }
++#else
++std::string CRISandboxerConvert(const std::string &runtime)
++{
++    return DEFAULT_SANDBOXER_NAME;
++}
++#endif
+ 
+ void ApplySandboxSecurityContextToHostConfig(const runtime::v1::LinuxSandboxSecurityContext &context, host_config *hc,
+                                              Errors &error)
+diff --git a/src/daemon/config/isulad_config.c b/src/daemon/config/isulad_config.c
+index 695a0d95..617db7a2 100644
+--- a/src/daemon/config/isulad_config.c
++++ b/src/daemon/config/isulad_config.c
+@@ -1757,8 +1757,10 @@ int merge_json_confs_into_global(struct service_arguments *args)
+     args->json_confs->runtimes = tmp_json_confs->runtimes;
+     tmp_json_confs->runtimes = NULL;
+ #ifdef ENABLE_CRI_API_V1
++#ifdef ENABLE_SANDBOXER
+     args->json_confs->cri_sandboxers = tmp_json_confs->cri_sandboxers;
+     tmp_json_confs->cri_sandboxers = NULL;
++#endif
+     args->json_confs->enable_cri_v1 = tmp_json_confs->enable_cri_v1;
+     args->json_confs->enable_pod_events = tmp_json_confs->enable_pod_events;
+ #endif
+diff --git a/src/daemon/sandbox/controller/CMakeLists.txt b/src/daemon/sandbox/controller/CMakeLists.txt
+index f846657a..8764c05b 100644
+--- a/src/daemon/sandbox/controller/CMakeLists.txt
++++ b/src/daemon/sandbox/controller/CMakeLists.txt
+@@ -9,7 +9,7 @@ set(local_sandbox_controller_top_incs
+     ${CMAKE_CURRENT_SOURCE_DIR}
+     )
+ 
+-if (ENABLE_SANDBOXER)
++if (ENABLE_CRI_API_V1 AND ENABLE_SANDBOXER)
+     add_subdirectory(sandboxer)
+     list (APPEND local_sandbox_controller_top_srcs
+         ${CONTROLLER_SANDBOXER_SRCS}
+diff --git a/src/daemon/sandbox/controller/controller_manager.cc b/src/daemon/sandbox/controller/controller_manager.cc
+index 21c6f5fe..91c98d26 100644
+--- a/src/daemon/sandbox/controller/controller_manager.cc
++++ b/src/daemon/sandbox/controller/controller_manager.cc
+@@ -20,7 +20,9 @@
+ #include <isula_libutils/defs.h>
+ 
+ #include "shim_controller.h"
++#ifdef ENABLE_SANDBOXER
+ #include "sandboxer_controller.h"
++#endif
+ #include "isulad_config.h"
+ #include "daemon_arguments.h"
+ 
+@@ -44,10 +46,12 @@ bool ControllerManager::Init(Errors &error)
+         return false;
+     }
+ 
++#ifdef ENABLE_SANDBOXER
+     // Initialize sandboxer controller
+     if (!RegisterAllSandboxerControllers(error)) {
+         return false;
+     }
++#endif
+     return true;
+ }
+ 
+@@ -75,6 +79,7 @@ auto ControllerManager::RegisterShimController(Errors &error) -> bool
+     return true;
+ }
+ 
++#ifdef ENABLE_SANDBOXER
+ auto ControllerManager::RegisterAllSandboxerControllers(Errors &error) -> bool
+ {
+     std::map<std::string, std::string> config;
+@@ -160,6 +165,7 @@ auto ControllerManager::RegisterSandboxerController(const std::string &sandboxer
+     INFO("Sandboxer controller initialized successfully, sandboxer: %s", sandboxer.c_str());
+     return true;
+ }
++#endif
+ 
+ auto ControllerManager::GetController(const std::string &name) -> std::shared_ptr<Controller>
+ {
+diff --git a/src/daemon/sandbox/controller/controller_manager.h b/src/daemon/sandbox/controller/controller_manager.h
+index 28b52c2f..3fd547cf 100644
+--- a/src/daemon/sandbox/controller/controller_manager.h
++++ b/src/daemon/sandbox/controller/controller_manager.h
+@@ -31,9 +31,11 @@ public:
+     auto GetController(const std::string &name) -> std::shared_ptr<Controller>;
+ private:
+     auto RegisterShimController(Errors &error) -> bool;
++#ifdef ENABLE_SANDBOXER
+     auto RegisterAllSandboxerControllers(Errors &error) -> bool;
+     auto LoadSandboxerControllersConfig(std::map<std::string, std::string> &config) -> bool;
+     auto RegisterSandboxerController(const std::string &sandboxer, const std::string &address, Errors &error) -> bool;
++#endif
+ 
+ protected:
+     std::map<std::string, std::shared_ptr<Controller>> m_controllers;
+-- 
+2.25.1
+

0071-Remove-sandboxer-ut-if-sandboxer-is-not-enabled.patch

@@ -0,0 +1,28 @@
+From 7c0c79e8ad4680f97651dd52721344961c803c15 Mon Sep 17 00:00:00 2001
+From: xuxuepeng <xuxuepeng1@huawei.com>
+Date: Sat, 20 Apr 2024 11:48:56 +0800
+Subject: [PATCH 071/149] Remove sandboxer ut if sandboxer is not enabled
+
+Signed-off-by: xuxuepeng <xuxuepeng1@huawei.com>
+---
+ test/sandbox/CMakeLists.txt | 8 +++++---
+ 1 file changed, 5 insertions(+), 3 deletions(-)
+
+diff --git a/test/sandbox/CMakeLists.txt b/test/sandbox/CMakeLists.txt
+index 68397477..38d7ccb9 100644
+--- a/test/sandbox/CMakeLists.txt
++++ b/test/sandbox/CMakeLists.txt
+@@ -1,5 +1,7 @@
+ project(iSulad_UT)
+ 
+-add_subdirectory(controller)
+-add_subdirectory(sandbox_manager)
+-add_subdirectory(sandbox)
++if (ENABLE_SANDBOXER)
++    add_subdirectory(controller)
++    add_subdirectory(sandbox_manager)
++    add_subdirectory(sandbox)
++endif()
+-- 
+2.25.1
+

0072-cdi-design-doc.patch

@@ -0,0 +1,391 @@
+From 72246a9e83ada3af7560817534013a93f40d5bae Mon Sep 17 00:00:00 2001
+From: liuxu <liuxu156@huawei.com>
+Date: Sat, 20 Apr 2024 11:46:11 +0800
+Subject: [PATCH 072/149] cdi:design doc
+
+Signed-off-by: liuxu <liuxu156@huawei.com>
+---
+ docs/design/README.md                     |   3 +
+ docs/design/README_zh.md                  |   3 +
+ docs/design/detailed/CDI/cdi_design_zh.md | 341 ++++++++++++++++++++++
+ 3 files changed, 347 insertions(+)
+ create mode 100644 docs/design/detailed/CDI/cdi_design_zh.md
+
+diff --git a/docs/design/README.md b/docs/design/README.md
+index cf29c0a1..d2a3702d 100644
+--- a/docs/design/README.md
++++ b/docs/design/README.md
+@@ -18,6 +18,9 @@ This section contains some design documents for users who want to learn more abo
+ 
+ - You can see how the CRI startup process is refactored in [cri_cni_refactor](./detailed/CRI/cri_cni_refactor.md).
+ 
++## CDI
++- You can see how the CDI is refactored in [cdi_design](./detailed/CDI/cdi_design_zh.md.md)。
++
+ ## Events
+ 
+ - You can see how the events modules are designed in [events_design](./detailed/Events/events_design.md).
+diff --git a/docs/design/README_zh.md b/docs/design/README_zh.md
+index 3382bfbe..c6172b6f 100644
+--- a/docs/design/README_zh.md
++++ b/docs/design/README_zh.md
+@@ -24,6 +24,9 @@
+ 
+ - 查看 CRI的启动程序的重构文档： [cri_cni_refactor](./detailed/CRI/cri_cni_refactor_zh.md) 。
+ 
++## CDI
++- 查看 CDI 的设计文档： [cdi_design](./detailed/CDI/cdi_design_zh.md.md)。
++
+ ## Events
+ 
+ - 查看 events 模块的设计文档： [events_design](./detailed/Events/events_design_zh.md) 。
+diff --git a/docs/design/detailed/CDI/cdi_design_zh.md b/docs/design/detailed/CDI/cdi_design_zh.md
+new file mode 100644
+index 00000000..15c88f93
+--- /dev/null
++++ b/docs/design/detailed/CDI/cdi_design_zh.md
+@@ -0,0 +1,341 @@
++| Author | liuxu                 |
++| ------ | --------------------- |
++| Date   | 2024-02-27            |
++| Email  | liuxu156@huawei.com   |
++
++# 背景介绍
++## What is CDI?
++CDI（容器设备接口）是容器运行时的一种规范，用于支持第三方设备。
++
++它引入了device作为资源的抽象概念。device由完全限定的名称唯一指定，该名称由vendor ID、一个device class和在每个vendor ID-device class对中唯一的name构成。
++```
++vendor.com/class=unique_name
++```
++vendor ID和device class （上例中的vendor.com/class）的组合称为device kind.
++
++CDI只关心使容器能够感知设备。CDI明确地忽略了诸如资源管理之类的领域（并期望由编排器处理）。由于这个原因，CDI规范实现起来很简单，并且为运行时和编排器提供了极大的灵活性。
++
++注：CDI模型基于容器网络接口（CNI）模型和规范。
++
++## Why is CDI needed?
++在Linux上，使容器具有设备感知能力过去只需在该容器中暴露一个设备节点。但是，随着设备和软件变得越来越复杂，供应商希望执行更多的操作，例如：
++- 向容器公开设备可能需要公开多个设备节点、从运行时命名空间挂载文件或隐藏procfs条目。
++- 执行容器和设备之间的兼容性检查（例如：检查容器是否可以在指定设备上运行）。
++- 执行特定于运行时的操作（例如：虚拟机与基于Linux容器的运行时）。
++- 执行特定于设备的操作（例如：清理GPU的内存或重新配置FPGA）。
++
++在缺乏第三方设备标准的情况下，供应商通常不得不为不同的运行时编写和维护多个插件，甚至直接在运行时中贡献特定于供应商的代码。此外，运行时不统一地暴露插件系统（甚至根本不暴露插件系统），导致在更高级别的抽象（例如Kubernetes设备插件）中重复功能。
++
++## How does CDI work?
++要使CDI正常工作，需要完成以下操作：
++- JSON格式的CDI文件应位于CDI规范目录中，它的作用是更新OCI spec。默认目录为/etc/cdi和/var/run/cdi
++- 应使用CRI的annotations与CDI_devices（本次特性支持）将唯一的设备名称传递给运行时
++- 容器运行时应该能够通过设备名称找到CDI文件在内存中对应的缓存信息，并使用缓存的内容更新容器配置。
++
++## How to configure CDI?
++### iSulad
++daemon.json中开启cri-v1和cdi配置：
++
++当 cdi-spec-dirs 不指定时，默认为"/etc/cdi", "/var/run/cdi"
++```json
++"enable-cri-v1": true,
++"cdi-spec-dirs": ["/etc/cdi", "/var/run/cdi"],  # 指定CDI规范所在目录
++"enable-cdi": true    # 打开CDI特性
++```
++
++在CRI创建容器的参数中使用CDI，以下两种方式均可：
++1. annotations中指定设备
++```json
++{
++    ... ...
++    "annotations": [
++        ... ...
++        // key值格式要求含有cdi.k8s.io作为前缀，后面跟随pluginName
++        {"cdi.k8s.io/test": "vendor.com/device=myDevice"},
++        ... ...
++    ]
++    ... ...
++}
++```
++2. CDI_Devices中指定设备
++```json
++{
++    ... ...
++    "CDI_Devices": [
++        ... ...
++        {"Name": "vendor.com/device=myDevice"},
++        ... ...
++    ]
++    ... ...
++}
++```
++
++# 方案目标
++## 概述
++容器设备接口（Container Device Interface，简称CDI）描述了容器运行时创建能够与第三方设备交互的容器的机制。
++
++对于第三方设备，与这些设备进行交互通常需要容器运行时公开多个设备节点。例如，第三方设备可能需要加载内核模块、装载主机库、暴露/屏蔽特定procfs路径。
++
++容器设备接口描述了一种允许第三方供应商执行这些操作的机制，从而不需要更改容器运行时。
++
++使用的机制是一个JSON文件（类似于容器网络接口(CNI）)，它允许供应商描述容器运行时应该对容器的OCI规范执行的操作。
++
++CDI 支持以下两个流程：
++
++A.设备安装
++1. 用户在机器上安装第三方设备驱动程序（和第三方设备/被测试的设备）。
++2. 设备驱动程序安装软件会在一个已知路径（/etc/cdi/vendor.json）上写入一个 JSON 文件。
++
++B.容器运行时
++1. 用户在创建容器时，CRI中指定设备名称
++2. 容器运行时会读取 JSON 文件。
++3. 容器运行时会验证 JSON 文件中是否描述了设备。
++4. 容器运行时会根据 JSON 文件中的指令转换 OCI 规范，并插入OCI Spec中
++# 总体设计
++**iSulad支持CDI功能目前仅支持CRI方式调用，CLI方式暂不支持**
++## 整体结构
++
++```mermaid
++flowchart TD
++classDef unFinish fill:#c19,stroke:#216,stroke-width:2px,color:#fff,stroke-dasharray: 5 5;
++subgraph isulad
++    subgraph CDIOperate
++        RegistryOps
++        AnnotationsOps
++        ......
++    end
++
++    subgraph CDI
++        D[ContainerEdits]
++        E[Device]
++        F[Registry]
++        G[Cache]
++        H[Spec]
++    end
++    CDIOperate --> CDI
++
++    OA[isula module] --> |call| CDIOperate
++    OB[CRI module] --> |call| CDIOperate
++    CDIOperate:::unFinish
++    CDI:::unFinish
++end
++    PA[CLI] --> OA
++    PB[CRI] --> OB
++```
++
++- CDIOperate封装了CDI模块，对外提供更合理的CDI功能的相关接口。
++- CDI负责实现CDI Specs的读取、校验、解析、devices注入OCI Spec等具体功能。
++## 时序设计
++在isulad启动后，以创建一个容器为例。
++
++图中isulad启动后，拉起一个新的线程isulad-cdi_watcher，负责监控cdi-spec-dirs，当cdi-spec-dirs中的cdi Spec文件发生修改、删除等动作时，重新扫描cdi-spec-dirs中的cdi Spec文件。
++
++```mermaid
++sequenceDiagram
++	participant CRI
++	participant isulad
++	participant cdi_watcher
++	isulad ->> cdi_watcher: init cdi
++    par 
++        loop
++            cdi_watcher ->> cdi_watcher:wait for inotify event
++            cdi_watcher ->> cdi_watcher:refresh cdi cache
++        end
++    and
++        opt CRI
++            CRI ->> isulad: create container
++            isulad ->> isulad:parse cdi annotations/CDI_devices
++            isulad ->> isulad:generates an OCI specification with cdi info
++        end
++    end
++```
++
++# 接口描述
++## 3.1 结构体和常量说明
++
++```c
++// 实现CDI Spec规范中的json字段，参考 https://github1s.com/containerd/containerd/blob/main/vendor/tags.cncf.io/container-device-interface/specs-go/config.go#L9 
++typedef struct {} cdi_spec;  
++typedef struct {} cdi_spec_device;  
++typedef struct {} cdi_spec_container_edits;  
++typedef struct {} cdi_spec_device_node;  
++typedef struct {} cdi_spec_mount;  
++typedef struct {} cdi_spec_hook;
++
++struct cdi_cache_device {
++    const cdi_device *raw_device;
++    const struct cdi_cache_spec *cache_spec;
++};
++
++struct cdi_cache_spec {
++    cdi_spec *raw_spec;
++    char *vendor;
++    char *class;
++    char *path;
++    int priority;
++    map_t *devices;  // MAP_STR_PTR  devices[cdi_device.name] = cdi_cache_device*
++};
++
++struct cdi_cache_ops {
++    // injecting CDI devices into an OCI Spec.
++    // Resolver
++    int (*inject_devices)(struct cdi_cache *c, oci_runtime_spec *spec, string_array *devices);
++    
++    // refreshing the cache of CDI Specs and devices.
++    // Refresher
++    int (*configure)(struct cdi_cache *c, string_array *spec_dirs);
++    int (*refresh)(struct cdi_cache *c);
++};
++
++struct cdi_watch {
++    int watcher_fd; // inotify fd
++    map_t *tracked; // MAP_STR_BOOL     tracked[spec_dirs[i]] = bool
++    map_t *wd_dirs; // MAP_INT_STR      wd_dirs[wd] = spec_dirs[i]
++};
++
++// Cache stores CDI Specs loaded from Spec directories.
++struct cdi_cache {
++    pthread_mutex_t mutex;
++    string_array *spec_dirs; // cdi-spec-dirs will scan for CDI Spec files
++    map_t *specs;        // MAP_STR_PTR     specs[vendor] = common_array of cdi_cache_spec*
++    // This map holding the reference to cdi device, the devices will not released when the map is freed.
++    map_t *devices;      // MAP_STR_PTR     devices[cdi_device.name] = cdi_cache_device*
++    bool refresh_error_flag;
++    bool auto_refresh; 
++    struct cdi_watch *watch;
++};
++
++struct cdi_registry {
++    struct cdi_cache *cdi_cache;
++    struct cdi_cache_ops *ops;
++};
++```
++涉及修改现有的结构体：
++```c
++// 用于从CRI向executor传递devices数据
++typedef struct {
++    ... ...
++    char **cdi_requested_devices;
++    size_t cdi_requested_devices_len;
++} host_config;
++
++// isulad的daemon.json增加cdi相关的基本配置
++typedef struct {
++    ... ...
++    char **cdi_spec_dirs;
++    size_t cdi_spec_dirs_len;
++    bool enable_cdi;
++} isulad_daemon_configs;
++```
++
++## 3.2 接口说明
++CDIOperate的设计目标：使得CDI规范有关的内容不过多的对外暴露，降低CDI模块和外部的耦合。
++### RegistryOps
++```c
++int cdi_operate_registry_init(char **specs_dirs, size_t specs_dirs_len);
++
++int cdi_operate_refresh(void);
++
++int cdi_operate_inject_devices(oci_runtime_spec *spec, string_array *devices);
++
++int cdi_operate_parse_annotations(json_map_string_string *annotations, string_array **keys,
++                                  string_array **devices, char **error);
++```
++
++# 详细设计
++## daemon.json
++daemon.json中增加cdi配置：
++```json
++"cdi-spec-dirs": ["/etc/cdi", "/var/run/cdi"],
++"enable-cdi": true
++```
++## CDIOperate被调用点
++### CreateContainer
++创建新的Container时需要将devices插入OCI Spec。
++本次开发暂不涉及isula支持CDI。
++
++为什么已经有isulad-cdi_watcher线程了，在create的时候还需要refresh?
++1. isulad-cdi_watcher在触发inotify event后，如果为Rename、Remove、Write事件，会直接执行refresh，将重新扫描cdi-spec-dirs中的CDI Specs到内存。
++2. 而在create container时，cdi_refresh先检查tracked，tracked标记了是否所有目录都已被跟踪，如果不是才会触发refresh。tracked在cache生成时初始化为未跟踪。
++
++```mermaid
++flowchart TD
++O((begin))
++A(ContainerManagerService::CreateContainer <br> CRI创建容器)
++B(ContainerManagerService::CDIParseDevices <br> 解析CDI有关的字段)
++C(container_cb::container_create_cb <br> 创建容器的前置准备)
++D(spec::merge_all_specs <br> 准备生成OCI Spec)
++E(spec::spec_inject_cdi_devices <br> 解析需要的设备名)
++F(CDIOperate::cdi_refresh <br> 刷新cdi的缓存)
++G(CDIOperate::cdi_inject_devices <br> 将容器需要的设备的相关信息插入OCI Spec)
++Z((create end))
++O --> A --> B --> C --> D --> E --> F --> G --> Z
++```
++
++ContainerManagerService::CDIParseDevices 需要支持两种CDI devices解析方式：
++1. CDIOperate::cdi_parse_annotations 解析ContainerConfig.annotations() 
++2. 解析ContainerConfig.CDI_devices()
++解析后放入container_create_request
++### main
++isulad 启动时，读取所有CDI Specs，初始化cache
++```mermaid
++flowchart TD
++O((begin))
++A(main)
++B(isulad_server_init_common <br> isulad服务端初始化)
++C(cdi::cdi_registry_init <br> 读取现有的CDI Specs <br> 并拉起新的线程监控CDI Specs的变化)
++Z(end)
++O --> A --> B --> C --> Z
++```
++
++## CDI 模块设计
++```mermaid
++flowchart TD
++    subgraph CDIoperate
++        operate
++    end
++
++    subgraph CDI
++        annotations
++        registry
++        cache
++        subgraph behavior
++            spec-dirs
++            spec
++            device
++            container-edits
++            version
++            parser
++        end
++    end
++
++    operate --> registry
++    operate --> annotations
++    registry --> cache
++    cache --> spec-dirs
++    cache --> device
++    cache --> spec
++    cache --> container-edits
++    device <--> spec
++    device --> parser
++    device --> container-edits
++    device --> parser
++    spec --> container-edits
++    spec --> version
++    version --> parser
++```
++
++### new cache
++```mermaid
++flowchart TD
++O((begin))
++A(ContainerManagerService::CreateContainer <br> CRI创建容器)
++B(ContainerManagerService::CDIParseDevices <br> 解析CDI有关的字段)
++C(container_cb::container_create_cb <br> 创建容器的前置准备)
++D(spec::merge_all_specs <br> 准备生成OCI Spec)
++E(spec::spec_inject_cdi_devices <br> 解析需要的设备名)
++F(CDIOperate::cdi_refresh <br> 刷新cdi的缓存)
++G(CDIOperate::cdi_inject_devices <br> 将容器需要的设备的相关信息插入OCI Spec)
++Z((create end))
++O --> A --> B --> C --> D --> E --> F --> G --> Z
++```
+-- 
+2.25.1
+

0073-bugfix-cdi-version-check.patch

@@ -0,0 +1,52 @@
+From 4d8a89e0a3700253db044f6641d91ab10ad6ce10 Mon Sep 17 00:00:00 2001
+From: liuxu <liuxu156@huawei.com>
+Date: Sat, 20 Apr 2024 19:36:55 +0800
+Subject: [PATCH 073/149] bugfix: cdi version check
+
+Signed-off-by: liuxu <liuxu156@huawei.com>
+---
+ src/daemon/modules/device/cdi/behavior/cdi_spec.c    | 2 +-
+ src/daemon/modules/device/cdi/behavior/cdi_version.c | 6 +++---
+ 2 files changed, 4 insertions(+), 4 deletions(-)
+
+diff --git a/src/daemon/modules/device/cdi/behavior/cdi_spec.c b/src/daemon/modules/device/cdi/behavior/cdi_spec.c
+index 235b1863..8783debc 100644
+--- a/src/daemon/modules/device/cdi/behavior/cdi_spec.c
++++ b/src/daemon/modules/device/cdi/behavior/cdi_spec.c
+@@ -199,7 +199,7 @@ static int cdi_spec_init(struct cdi_cache_spec *s)
+         return -1;
+     }
+     if (version_result) {
+-        ERROR("The spec version must be at least v%s", min_version);
++        ERROR("The %s spec version must be at least v%s", s->path, min_version);
+         return -1;
+     }
+ 
+diff --git a/src/daemon/modules/device/cdi/behavior/cdi_version.c b/src/daemon/modules/device/cdi/behavior/cdi_version.c
+index 882a965e..550f3107 100644
+--- a/src/daemon/modules/device/cdi/behavior/cdi_version.c
++++ b/src/daemon/modules/device/cdi/behavior/cdi_version.c
+@@ -138,9 +138,9 @@ static struct required_version_map g_valid_spec_versions[VALID_SPEC_VERSIONS_LEN
+     {CDI_V010, NULL},
+     {CDI_V020, NULL},
+     {CDI_V030, NULL},
+-    {CDI_V040, requires_v060},
++    {CDI_V040, requires_v040},
+     {CDI_V050, requires_v050},
+-    {CDI_V060, requires_v040}
++    {CDI_V060, requires_v060}
+ };
+ 
+ const char *cdi_minimum_required_version(cdi_spec *spec)
+@@ -166,7 +166,7 @@ const char *cdi_minimum_required_version(cdi_spec *spec)
+                 min_version = g_valid_spec_versions[i].version;
+             }
+         }
+-        if (strcmp(min_version, CDI_CURRENT_VERSION)) {
++        if (strcmp(min_version, CDI_CURRENT_VERSION) == 0) {
+             break;
+         }
+     }
+-- 
+2.25.1
+

0074-bugfix-of-background-execution-exec-error-command.patch

@@ -0,0 +1,33 @@
+From 2c86e55d98b0d62c534ff5810c1eb1d327d6425a Mon Sep 17 00:00:00 2001
+From: zhongtao <zhongtao17@huawei.com>
+Date: Tue, 23 Apr 2024 17:44:00 +1400
+Subject: [PATCH 074/149] bugfix of background execution exec error command
+
+Signed-off-by: zhongtao <zhongtao17@huawei.com>
+---
+ src/daemon/modules/runtime/isula/isula_rt_ops.c | 9 +++++++++
+ 1 file changed, 9 insertions(+)
+
+diff --git a/src/daemon/modules/runtime/isula/isula_rt_ops.c b/src/daemon/modules/runtime/isula/isula_rt_ops.c
+index 47a14b1d..854752ea 100644
+--- a/src/daemon/modules/runtime/isula/isula_rt_ops.c
++++ b/src/daemon/modules/runtime/isula/isula_rt_ops.c
+@@ -1099,6 +1099,15 @@ static int get_container_process_pid(const char *workdir)
+                 util_usleep_nointerupt(100000);
+                 continue;
+             }
++            // If isulad does not read the container process pid, but isulad-shim reads the pid,
++            // and the container process exits, isulad-shim exits accordingly.
++            // At this time, exec should return true, because the container process has been created successfully
++            // and exec is successful, just because The process executes too fast causing isulad to not be read correctly
++            file_read_int(fname, &pid);
++            if (pid != 0) {
++                DEBUG("Process exit and isulad-shim exit");
++                return pid;
++            }
+             ERROR("failed read pid from dead shim %s", workdir);
+             return -1;
+         }
+-- 
+2.25.1
+

0075-bugfix-for-setting-cpu-rt-to-a-negative-value-when-e.patch

@@ -0,0 +1,35 @@
+From bd18051dade97d4f75346cb67beea551a38ca13e Mon Sep 17 00:00:00 2001
+From: zhongtao <zhongtao17@huawei.com>
+Date: Tue, 23 Apr 2024 20:52:04 +0800
+Subject: [PATCH 075/149] bugfix for setting cpu-rt to a negative value when
+ env not supports cpu-rt
+
+Signed-off-by: zhongtao <zhongtao17@huawei.com>
+---
+ src/daemon/modules/spec/verify.c | 4 ++--
+ 1 file changed, 2 insertions(+), 2 deletions(-)
+
+diff --git a/src/daemon/modules/spec/verify.c b/src/daemon/modules/spec/verify.c
+index 617b7f23..57e16ef9 100644
+--- a/src/daemon/modules/spec/verify.c
++++ b/src/daemon/modules/spec/verify.c
+@@ -381,14 +381,14 @@ static int verify_cpu_realtime(const sysinfo_t *sysinfo, int64_t realtime_period
+ {
+     int ret = 0;
+ 
+-    if (realtime_period > 0 && !(sysinfo->cgcpuinfo.cpu_rt_period)) {
++    if (realtime_period != 0 && !(sysinfo->cgcpuinfo.cpu_rt_period)) {
+         ERROR("Invalid --cpu-rt-period: Your kernel does not support cgroup rt period");
+         isulad_set_error_message("Invalid --cpu-rt-period: Your kernel does not support cgroup rt period");
+         ret = -1;
+         goto out;
+     }
+ 
+-    if (realtime_runtime > 0 && !(sysinfo->cgcpuinfo.cpu_rt_runtime)) {
++    if (realtime_runtime != 0 && !(sysinfo->cgcpuinfo.cpu_rt_runtime)) {
+         ERROR("Invalid --cpu-rt-runtime: Your kernel does not support cgroup rt runtime");
+         isulad_set_error_message("Invalid --cpu-rt-period: Your kernel does not support cgroup rt runtime");
+         ret = -1;
+-- 
+2.25.1
+

0076-cdi-add-UT.patch

@@ -0,0 +1,741 @@
+From 0cd088174c94c56ee86506dab9a6a33f6e8fdaa4 Mon Sep 17 00:00:00 2001
+From: liuxu <liuxu156@huawei.com>
+Date: Thu, 25 Apr 2024 10:52:20 +0800
+Subject: [PATCH 076/149] cdi:add UT
+
+Signed-off-by: liuxu <liuxu156@huawei.com>
+---
+ CI/make-and-install.sh                        |   4 +-
+ test/cutils/CMakeLists.txt                    |   1 +
+ test/cutils/utils_array/utils_array_ut.cc     | 128 ++++++++++
+ test/cutils/utils_utils/utils_utils_ut.cc     |  12 +
+ test/cutils/utils_version/CMakeLists.txt      |  17 ++
+ test/cutils/utils_version/utils_version_ut.cc |  71 ++++++
+ .../image/oci/oci_config_merge/CMakeLists.txt |   1 +
+ test/mocks/cdi_operate_api_mock.cc            |  58 +++++
+ test/mocks/cdi_operate_api_mock.h             |  35 +++
+ test/sandbox/controller/shim/CMakeLists.txt   |   1 +
+ test/specs/specs/CMakeLists.txt               |   1 +
+ test/specs/specs/specs_ut.cc                  | 241 ++++++++++++++++++
+ test/specs/specs_extend/CMakeLists.txt        |   1 +
+ 13 files changed, 569 insertions(+), 2 deletions(-)
+ create mode 100644 test/cutils/utils_version/CMakeLists.txt
+ create mode 100644 test/cutils/utils_version/utils_version_ut.cc
+ create mode 100644 test/mocks/cdi_operate_api_mock.cc
+ create mode 100644 test/mocks/cdi_operate_api_mock.h
+
+diff --git a/CI/make-and-install.sh b/CI/make-and-install.sh
+index 9bb984cd..9d4c5533 100755
+--- a/CI/make-and-install.sh
++++ b/CI/make-and-install.sh
+@@ -72,7 +72,7 @@ cd $ISULAD_COPY_PATH
+ sed -i 's/fd == STDIN_FILENO || fd == STDOUT_FILENO || fd == STDERR_FILENO/fd == 0 || fd == 1 || fd == 2 || fd >= 1000/g' ./src/utils/cutils/utils.c
+ rm -rf build
+ mkdir build && cd build
+-cmake -DCMAKE_BUILD_TYPE=Debug -DENABLE_UT=ON -DENABLE_CRI_API_V1=ON -DENABLE_SHIM_V2=ON -DENABLE_METRICS=ON ..
++cmake -DCMAKE_BUILD_TYPE=Debug -DENABLE_UT=ON -DENABLE_CRI_API_V1=ON -DENABLE_CDI=ON -DENABLE_SHIM_V2=ON -DENABLE_METRICS=ON ..
+ make -j $(nproc)
+ make install
+ ctest -E "driver_devmapper_ut" -T memcheck --output-on-failure
+@@ -103,7 +103,7 @@ rm -rf build
+ mkdir build
+ cd build
+ if [[ ${enable_gcov} -ne 0 ]]; then
+-  cmake -DLIB_INSTALL_DIR=${builddir}/lib -DCMAKE_INSTALL_PREFIX=${builddir} -DCMAKE_INSTALL_SYSCONFDIR=${builddir}/etc -DCMAKE_BUILD_TYPE=Debug -DGCOV=ON -DENABLE_EMBEDDED=ON -DENABLE_COVERAGE=ON -DENABLE_CRI_API_V1=ON -DENABLE_UT=ON -DENABLE_METRICS=ON -DENABLE_REMOTE_LAYER_STORE=ON -DENABLE_GRPC_REMOTE_CONNECT=ON ..
++  cmake -DLIB_INSTALL_DIR=${builddir}/lib -DCMAKE_INSTALL_PREFIX=${builddir} -DCMAKE_INSTALL_SYSCONFDIR=${builddir}/etc -DCMAKE_BUILD_TYPE=Debug -DGCOV=ON -DENABLE_EMBEDDED=ON -DENABLE_COVERAGE=ON -DENABLE_CRI_API_V1=ON -DENABLE_CDI=ON -DENABLE_UT=ON -DENABLE_METRICS=ON -DENABLE_REMOTE_LAYER_STORE=ON -DENABLE_GRPC_REMOTE_CONNECT=ON ..
+ else
+   cmake -DLIB_INSTALL_DIR=${builddir}/lib -DCMAKE_INSTALL_PREFIX=${builddir} -DCMAKE_INSTALL_SYSCONFDIR=${builddir}/etc -DENABLE_EMBEDDED=ON -DENABLE_METRICS=ON -DENABLE_REMOTE_LAYER_STORE=ON -DENABLE_CRI_API_V1=ON -DENABLE_GRPC_REMOTE_CONNECT=ON ..
+ fi
+diff --git a/test/cutils/CMakeLists.txt b/test/cutils/CMakeLists.txt
+index 9e681cc9..bd9def02 100644
+--- a/test/cutils/CMakeLists.txt
++++ b/test/cutils/CMakeLists.txt
+@@ -34,4 +34,5 @@ add_subdirectory(utils_utils)
+ add_subdirectory(utils_verify)
+ add_subdirectory(utils_network)
+ add_subdirectory(utils_transform)
++add_subdirectory(utils_version)
+ add_subdirectory(map)
+diff --git a/test/cutils/utils_array/utils_array_ut.cc b/test/cutils/utils_array/utils_array_ut.cc
+index 7bd13c25..6c6e76fa 100644
+--- a/test/cutils/utils_array/utils_array_ut.cc
++++ b/test/cutils/utils_array/utils_array_ut.cc
+@@ -50,6 +50,33 @@ TEST(utils_array, test_util_free_array)
+     util_free_array(array);
+ }
+ 
++TEST(utils_array, test_util_copy_array_by_len)
++{
++    char **array = nullptr;
++    char **array_copy = nullptr;
++    size_t len = 3;
++
++    array = (char **)util_common_calloc_s(4 * sizeof(char *));
++    ASSERT_NE(array, nullptr);
++    array[0] = util_strdup_s("test1");
++    array[1] = util_strdup_s("test2");
++    array[2] = util_strdup_s("test3");
++
++    array_copy = util_copy_array_by_len(array, len);
++    ASSERT_NE(array_copy, nullptr);
++    for (size_t i = 0; i < len; i++) {
++        ASSERT_EQ(strcmp(array_copy[i], array[i]), 0);
++        free(array[i]);
++        free(array_copy[i]);
++    }
++
++    ASSERT_EQ(util_copy_array_by_len(array, 0), nullptr);
++    ASSERT_EQ(util_copy_array_by_len(nullptr, len), nullptr);
++
++    free(array);
++    free(array_copy);
++}
++
+ TEST(utils_array, test_util_grow_array)
+ {
+     char **array = nullptr;
+@@ -229,6 +256,34 @@ TEST(utils_array, test_util_append_string_array)
+     sarray = nullptr;
+ }
+ 
++TEST(utils_array, test_util_copy_string_array)
++{
++    __isula_auto_string_array_t string_array *sarray_copy = nullptr;
++    __isula_auto_string_array_t string_array *sarray = (string_array *)util_common_calloc_s(sizeof(string_array));
++    ASSERT_NE(sarray, nullptr);
++    int ret;
++
++    ret = util_append_string_array(sarray, "1234567890");
++    ASSERT_EQ(ret, 0);
++    ret = util_append_string_array(sarray, "abc");
++    ASSERT_EQ(ret, 0);
++    ret = util_append_string_array(sarray, "bcd");
++    ASSERT_EQ(ret, 0);
++    ASSERT_EQ(sarray->len, 3);
++
++    sarray_copy = util_copy_string_array(sarray);
++    ASSERT_NE(sarray_copy, nullptr);
++    ASSERT_EQ(sarray_copy->len, sarray->len);
++    for (size_t i = 0; i < sarray_copy->len; i++) {
++        ASSERT_EQ(strcmp(sarray_copy->items[i], sarray->items[i]), 0);
++    }
++
++    ASSERT_EQ(util_copy_string_array(nullptr), nullptr);
++    sarray->cap = 0;
++    ASSERT_EQ(util_copy_string_array(sarray), nullptr);
++    sarray->cap = sarray->len;
++}
++
+ TEST(utils_array, test_util_string_array_contain)
+ {
+     string_array *sarray = (string_array *)util_common_calloc_s(sizeof(string_array));
+@@ -299,3 +354,76 @@ TEST(utils_array, test_util_common_array_append_pointer)
+     delete element1;
+     delete element2;
+ }
++
++static void common_array_free_mock(void *ptr)
++{
++    (void)ptr;
++    return;
++}
++
++TEST(utils_array, test_util_append_common_array)
++{
++    __isula_auto_common_array_t common_array *carray = nullptr;
++    int ret;
++    int value1 = 1;
++    int value2 = 2;
++    int value3 = 3;
++
++    carray = util_common_array_new(1, common_array_free_mock, util_clone_ptr);
++    ASSERT_NE(carray, nullptr);
++
++    ret = util_append_common_array(carray, &value1);
++    ASSERT_EQ(ret, 0);
++    ASSERT_EQ(carray->items[0], &value1);
++    ASSERT_EQ(carray->len, 1);
++
++    ret = util_append_common_array(carray, &value2);
++    ASSERT_EQ(ret, 0);
++    ret = util_append_common_array(carray, &value3);
++    ASSERT_EQ(ret, 0);
++    ASSERT_EQ(carray->items[1], &value2);
++    ASSERT_EQ(carray->items[2], &value3);
++    ASSERT_EQ(carray->len, 3);
++
++    carray->clone_item_cb = nullptr;
++    ASSERT_EQ(util_append_common_array(carray, &value1), -1);
++    carray->clone_item_cb = util_clone_ptr;
++    ASSERT_EQ(util_append_common_array(carray, nullptr), 0);
++}
++
++TEST(utils_array, test_util_merge_common_array)
++{
++    __isula_auto_common_array_t common_array *carray1 = nullptr;
++    __isula_auto_common_array_t common_array *carray2 = nullptr;
++    int ret;
++    int value1 = 1;
++    int value2 = 2;
++
++    carray1 = util_common_array_new(1, common_array_free_mock, util_clone_ptr);
++    ASSERT_NE(carray1, nullptr);
++    carray2 = util_common_array_new(1, common_array_free_mock, util_clone_ptr);
++    ASSERT_NE(carray2, nullptr);
++
++    ret = util_append_common_array(carray1, &value1);
++    ASSERT_EQ(ret, 0);
++    ASSERT_EQ(carray1->items[0], &value1);
++    ASSERT_EQ(carray1->len, 1);
++    ret = util_append_common_array(carray2, &value2);
++    ASSERT_EQ(ret, 0);
++    ASSERT_EQ(carray2->items[0], &value2);
++    ASSERT_EQ(carray2->len, 1);
++
++    ret = util_merge_common_array(carray1, carray2);
++    ASSERT_EQ(ret, 0);
++    ASSERT_EQ(carray1->items[1], &value2);
++    ASSERT_EQ(carray1->len, 2);
++
++    ASSERT_EQ(util_merge_common_array(nullptr, carray2), -1);
++    ASSERT_EQ(util_merge_common_array(carray1, nullptr), -1);
++    carray1->clone_item_cb = nullptr;
++    ASSERT_EQ(util_merge_common_array(carray1, carray2), -1);
++    carray1->clone_item_cb = util_clone_ptr;
++    carray2->clone_item_cb = nullptr;
++    ASSERT_EQ(util_merge_common_array(carray1, carray2), -1);
++    carray2->clone_item_cb = util_clone_ptr;
++}
+\ No newline at end of file
+diff --git a/test/cutils/utils_utils/utils_utils_ut.cc b/test/cutils/utils_utils/utils_utils_ut.cc
+index 0720d1b1..a61e5a21 100644
+--- a/test/cutils/utils_utils/utils_utils_ut.cc
++++ b/test/cutils/utils_utils/utils_utils_ut.cc
+@@ -54,6 +54,18 @@ static int status_to_exit_code(int status)
+     return exit_code;
+ }
+ 
++TEST(utils_utils, test_util_swap_ptr)
++{
++    int val1 = 1;
++    int val2 = 2;
++    int *ptr1 = &val1;
++    int *ptr2 = &val2;
++
++    util_swap_ptr((void **)&ptr1, (void **)&ptr2);
++    ASSERT_EQ(*ptr1, val2);
++    ASSERT_EQ(*ptr2, val1);
++}
++
+ TEST(utils_utils, test_util_mem_realloc)
+ {
+     char *old = nullptr;
+diff --git a/test/cutils/utils_version/CMakeLists.txt b/test/cutils/utils_version/CMakeLists.txt
+new file mode 100644
+index 00000000..1ada8e93
+--- /dev/null
++++ b/test/cutils/utils_version/CMakeLists.txt
+@@ -0,0 +1,17 @@
++project(iSulad_UT)
++
++SET(EXE utils_version_ut)
++
++add_executable(${EXE}
++    utils_version_ut.cc)
++
++target_include_directories(${EXE} PUBLIC
++    ${GTEST_INCLUDE_DIR}
++    ${CMAKE_CURRENT_SOURCE_DIR}/../../include
++    ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/common
++    ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/utils/cutils
++    )
++
++target_link_libraries(${EXE} ${GTEST_BOTH_LIBRARIES} ${CMAKE_THREAD_LIBS_INIT} ${ISULA_LIBUTILS_LIBRARY} libutils_ut -lcrypto -lyajl -lz)
++add_test(NAME ${EXE} COMMAND ${EXE} --gtest_output=xml:${EXE}-Results.xml)
++set_tests_properties(${EXE} PROPERTIES TIMEOUT 120)
+diff --git a/test/cutils/utils_version/utils_version_ut.cc b/test/cutils/utils_version/utils_version_ut.cc
+new file mode 100644
+index 00000000..d1fc0932
+--- /dev/null
++++ b/test/cutils/utils_version/utils_version_ut.cc
+@@ -0,0 +1,71 @@
++/******************************************************************************
++ * Copyright (c) Huawei Technologies Co., Ltd. 2024. All rights reserved.
++ * iSulad licensed under the Mulan PSL v2.
++ * You can use this software according to the terms and conditions of the Mulan PSL v2.
++ * You may obtain a copy of Mulan PSL v2 at:
++ *     http://license.coscl.org.cn/MulanPSL2
++ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR
++ * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR
++ * PURPOSE.
++ * See the Mulan PSL v2 for more details.
++ * Author: liuxu
++ * Create: 2024-04-25
++ * Description: utils version unit test
++ *******************************************************************************/
++
++#include <stdlib.h>
++#include <stdio.h>
++#include <climits>
++#include <gtest/gtest.h>
++#include "mock.h"
++#include "utils_version.h"
++#include "utils.h"
++
++TEST(utils_version, test_util_version_compare)
++{
++    const char *version1 = "1.1.1";
++    const char *version2 = "1.1.2";
++    int diff_value = 0;
++
++    ASSERT_EQ(util_version_compare(version1, version2, &diff_value), 0);
++    ASSERT_TRUE(diff_value < 0);
++    ASSERT_EQ(util_version_compare(version1, version1, &diff_value), 0);
++    ASSERT_TRUE(diff_value == 0);
++    ASSERT_EQ(util_version_compare(version2, version1, &diff_value), 0);
++    ASSERT_TRUE(diff_value > 0);
++
++    ASSERT_EQ(util_version_compare(version1, nullptr, &diff_value), -1);
++    ASSERT_EQ(util_version_compare(nullptr, version2, &diff_value), -1);
++    ASSERT_EQ(util_version_compare(version1, version2, nullptr), -1);
++    ASSERT_EQ(util_version_compare("1.1.1.1", version2, nullptr), -1);
++    ASSERT_EQ(util_version_compare(version1, "a.b.1.1", nullptr), -1);
++}
++
++TEST(utils_version, test_util_version_greater_than)
++{
++    const char *version1 = "0.6.0";
++    const char *version2 = "1.0.0";
++    bool result = true;
++
++    ASSERT_EQ(util_version_greater_than(version1, version2, &result), 0);
++    ASSERT_FALSE(result);
++    ASSERT_EQ(util_version_greater_than(version1, version1, &result), 0);
++    ASSERT_FALSE(result);
++    ASSERT_EQ(util_version_greater_than(version2, version1, &result), 0);
++    ASSERT_TRUE(result);
++}
++
++TEST(utils_version, test_util_version_greater_than_or_equal_to)
++{
++    const char *version1 = "0.6.0";
++    const char *version2 = "1.0.0";
++    bool result = true;
++
++    ASSERT_EQ(util_version_greater_than_or_equal_to(version1, version2, &result), 0);
++    ASSERT_FALSE(result);
++    ASSERT_EQ(util_version_greater_than_or_equal_to(version1, version1, &result), 0);
++    ASSERT_TRUE(result);
++    ASSERT_EQ(util_version_greater_than_or_equal_to(version2, version1, &result), 0);
++    ASSERT_TRUE(result);
++}
++
+diff --git a/test/image/oci/oci_config_merge/CMakeLists.txt b/test/image/oci/oci_config_merge/CMakeLists.txt
+index ffd3999d..38ade4ec 100644
+--- a/test/image/oci/oci_config_merge/CMakeLists.txt
++++ b/test/image/oci/oci_config_merge/CMakeLists.txt
+@@ -47,6 +47,7 @@ add_executable(${EXE}
+     ${CMAKE_CURRENT_SOURCE_DIR}/../../../../test/mocks/selinux_label_mock.cc
+     ${CMAKE_CURRENT_SOURCE_DIR}/../../../../test/mocks/storage_mock.cc
+     ${CMAKE_CURRENT_SOURCE_DIR}/../../../../test/mocks/image_mock.cc
++    ${CMAKE_CURRENT_SOURCE_DIR}/../../../../test/mocks/cdi_operate_api_mock.cc
+     oci_config_merge_ut.cc)
+ 
+ target_include_directories(${EXE} PUBLIC
+diff --git a/test/mocks/cdi_operate_api_mock.cc b/test/mocks/cdi_operate_api_mock.cc
+new file mode 100644
+index 00000000..d8f9f9d4
+--- /dev/null
++++ b/test/mocks/cdi_operate_api_mock.cc
+@@ -0,0 +1,58 @@
++/******************************************************************************
++ * Copyright (c) Huawei Technologies Co., Ltd. 2023. All rights reserved.
++ * iSulad licensed under the Mulan PSL v2.
++ * You can use this software according to the terms and conditions of the Mulan PSL v2.
++ * You may obtain a copy of Mulan PSL v2 at:
++ *     http://license.coscl.org.cn/MulanPSL2
++ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR
++ * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR
++ * PURPOSE.
++ * See the Mulan PSL v2 for more details.
++ * Author: jikai
++ * Create: 2023-10-20
++ * Description: provide image api mock
++ ******************************************************************************/
++
++#include "cdi_operate_api_mock.h"
++
++namespace {
++std::shared_ptr<MockCdiOperateApi> g_cdi_operate_api_mock = nullptr;
++}
++
++void MockCdiOperateApi_SetMock(std::shared_ptr<MockCdiOperateApi> mock)
++{
++    g_cdi_operate_api_mock = mock;
++}
++
++int cdi_operate_registry_init(char **specs_dirs, size_t specs_dirs_len)
++{
++    if (g_cdi_operate_api_mock != nullptr) {
++        return g_cdi_operate_api_mock->CdiOperateRegistryInit(specs_dirs, specs_dirs_len);
++    }
++    return 0;
++}
++
++int cdi_operate_refresh(void)
++{
++    if (g_cdi_operate_api_mock != nullptr) {
++        return g_cdi_operate_api_mock->CdiOperateRefresh();
++    }
++    return 0;
++}
++
++int cdi_operate_inject_devices(oci_runtime_spec *spec, string_array *devices)
++{
++    if (g_cdi_operate_api_mock != nullptr) {
++        return g_cdi_operate_api_mock->CdiOperateInjectDevices(spec, devices);
++    }
++    return 0;
++}
++
++int cdi_operate_parse_annotations(json_map_string_string *annotations, string_array **keys,
++                                  string_array **devices, char **error)
++{
++    if (g_cdi_operate_api_mock != nullptr) {
++        return g_cdi_operate_api_mock->CdiOperateParseAnnotations(annotations, keys, devices, error);
++    }
++    return 0;
++}
+\ No newline at end of file
+diff --git a/test/mocks/cdi_operate_api_mock.h b/test/mocks/cdi_operate_api_mock.h
+new file mode 100644
+index 00000000..c118ee7a
+--- /dev/null
++++ b/test/mocks/cdi_operate_api_mock.h
+@@ -0,0 +1,35 @@
++/******************************************************************************
++ * Copyright (c) Huawei Technologies Co., Ltd. 2023. All rights reserved.
++ * iSulad licensed under the Mulan PSL v2.
++ * You can use this software according to the terms and conditions of the Mulan PSL v2.
++ * You may obtain a copy of Mulan PSL v2 at:
++ *     http://license.coscl.org.cn/MulanPSL2
++ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR
++ * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR
++ * PURPOSE.
++ * See the Mulan PSL v2 for more details.
++ * Author: liuxu
++ * Create: 2024-04-24
++ * Description: provide cdi api mock
++ ******************************************************************************/
++
++#ifndef ISULAD_TEST_MOCKS_CDI_OPERATE_API_MOCK_H
++#define ISULAD_TEST_MOCKS_CDI_OPERATE_API_MOCK_H
++
++#include <gmock/gmock.h>
++#include <memory>
++
++#include "cdi_operate_api.h"
++
++class MockCdiOperateApi {
++public:
++    MOCK_METHOD2(CdiOperateRegistryInit, int(char **specs_dirs, size_t specs_dirs_len));
++    MOCK_METHOD0(CdiOperateRefresh, int(void));
++    MOCK_METHOD2(CdiOperateInjectDevices, int(oci_runtime_spec *spec, string_array *devices));
++    MOCK_METHOD4(CdiOperateParseAnnotations, int(json_map_string_string *annotations, string_array **keys,
++                                                 string_array **devices, char **error));
++};
++
++void MockCdiOperateApi_SetMock(std::shared_ptr<MockCdiOperateApi> mock);
++
++#endif
+diff --git a/test/sandbox/controller/shim/CMakeLists.txt b/test/sandbox/controller/shim/CMakeLists.txt
+index 26a66e51..d18d1861 100644
+--- a/test/sandbox/controller/shim/CMakeLists.txt
++++ b/test/sandbox/controller/shim/CMakeLists.txt
+@@ -21,6 +21,7 @@ add_executable(${EXE}
+     ${CMAKE_CURRENT_SOURCE_DIR}/../../../../test/mocks/callback_mock.cc
+     ${CMAKE_CURRENT_SOURCE_DIR}/../../../../test/mocks/image_api_mock.cc
+     ${CMAKE_CURRENT_SOURCE_DIR}/../../../../test/mocks/service_container_api_mock.cc
++    ${CMAKE_CURRENT_SOURCE_DIR}/../../../../test/mocks/cdi_operate_api_mock.cc
+     ${CMAKE_CURRENT_SOURCE_DIR}/../../../../test/sandbox/controller/controller_common.cc
+     ${CMAKE_CURRENT_SOURCE_DIR}/../../../../test/sandbox/controller/shim/shim_controller_ut.cc
+ )
+diff --git a/test/specs/specs/CMakeLists.txt b/test/specs/specs/CMakeLists.txt
+index 12c11f51..892d44d7 100644
+--- a/test/specs/specs/CMakeLists.txt
++++ b/test/specs/specs/CMakeLists.txt
+@@ -44,6 +44,7 @@ add_executable(${EXE}
+     ${CMAKE_CURRENT_SOURCE_DIR}/../../../test/mocks/storage_mock.cc
+     ${CMAKE_CURRENT_SOURCE_DIR}/../../../test/mocks/image_mock.cc
+     ${CMAKE_CURRENT_SOURCE_DIR}/../../../test/mocks/sender_mock.cc
++    ${CMAKE_CURRENT_SOURCE_DIR}/../../../test/mocks/cdi_operate_api_mock.cc
+     specs_ut.cc)
+ 
+ target_include_directories(${EXE} PUBLIC
+diff --git a/test/specs/specs/specs_ut.cc b/test/specs/specs/specs_ut.cc
+index 6c42216d..47836e5b 100644
+--- a/test/specs/specs/specs_ut.cc
++++ b/test/specs/specs/specs_ut.cc
+@@ -564,6 +564,247 @@ TEST_F(SpecsUnitTest, test_update_devcies_for_oci_spec)
+     free(err);
+ }
+ 
++#ifdef ENABLE_CDI
++TEST_F(SpecsUnitTest, test_defs_process_add_multiple_env)
++{
++    size_t env_len = 2;
++    char **envs = (char **)util_common_calloc_s(sizeof(char *) * env_len);
++    ASSERT_NE(envs, nullptr);
++    defs_process *dp = (defs_process *)util_common_calloc_s(sizeof(defs_process));
++    ASSERT_NE(dp, nullptr);
++    dp->env_len = 1;
++    dp->env = (char **)util_common_calloc_s(sizeof(char *) * dp->env_len);
++    ASSERT_NE(dp->env, nullptr);
++    
++    envs[0] = util_strdup_s("key0=value0");
++    envs[1] = util_strdup_s("key1=value1");
++    dp->env[0] = util_strdup_s("key0=value0_old");
++    ASSERT_EQ(defs_process_add_multiple_env(dp, (const char **)envs, env_len), 0);
++    ASSERT_EQ(dp->env_len, 2);
++    ASSERT_EQ(strcmp(dp->env[0], envs[0]), 0);
++    ASSERT_EQ(strcmp(dp->env[1], envs[1]), 0);
++
++    ASSERT_EQ(defs_process_add_multiple_env(dp, nullptr, env_len), 0);
++    ASSERT_EQ(defs_process_add_multiple_env(dp, (const char **)envs, 0), 0);
++    ASSERT_EQ(defs_process_add_multiple_env(nullptr, (const char **)envs, env_len), -1);
++
++    free(envs[0]);
++    envs[0] = util_strdup_s("=value0");
++    ASSERT_EQ(defs_process_add_multiple_env(dp, (const char **)envs, env_len), -1);
++    free(envs[0]);
++    envs[0] = util_strdup_s("key0=");
++    ASSERT_EQ(defs_process_add_multiple_env(dp, (const char **)envs, env_len), -1);
++    free(envs[0]);
++    envs[0] = util_strdup_s("key0xxxx");
++    ASSERT_EQ(defs_process_add_multiple_env(dp, (const char **)envs, env_len), -1);
++
++    free(dp->env[0]);
++    dp->env[0] = util_strdup_s("=value0");
++    ASSERT_EQ(defs_process_add_multiple_env(dp, (const char **)envs, env_len), -1);
++    free(dp->env[0]);
++    dp->env[0] = util_strdup_s("key0=");
++    ASSERT_EQ(defs_process_add_multiple_env(dp, (const char **)envs, env_len), -1);
++    free(dp->env[0]);
++    dp->env[0] = util_strdup_s("key0xxxx");
++    ASSERT_EQ(defs_process_add_multiple_env(dp, (const char **)envs, env_len), -1);
++
++    free_defs_process(dp);
++    free(envs[0]);
++    free(envs[1]);
++    free(envs);
++}
++
++TEST_F(SpecsUnitTest, test_spec_add_multiple_process_env)
++{
++    size_t env_len = 2;
++    char **envs = (char **)util_common_calloc_s(sizeof(char *) * env_len);
++    ASSERT_NE(envs, nullptr);
++    oci_runtime_spec *oci_spec = (oci_runtime_spec *)util_common_calloc_s(sizeof(oci_runtime_spec));
++    ASSERT_NE(oci_spec, nullptr);
++    oci_spec->process = (defs_process *)util_common_calloc_s(sizeof(defs_process));
++    ASSERT_NE(oci_spec->process, nullptr);
++    oci_spec->process->env_len = 1;
++    oci_spec->process->env = (char **)util_common_calloc_s(sizeof(char *) * oci_spec->process->env_len);
++    ASSERT_NE(oci_spec->process->env, nullptr);
++    
++    envs[0] = util_strdup_s("key0=value0");
++    envs[1] = util_strdup_s("key1=value1");
++    oci_spec->process->env[0] = util_strdup_s("key0=value0_old");
++    ASSERT_EQ(spec_add_multiple_process_env(oci_spec, (const char **)envs, env_len), 0);
++    ASSERT_EQ(oci_spec->process->env_len, 2);
++    ASSERT_EQ(strcmp(oci_spec->process->env[0], envs[0]), 0);
++    ASSERT_EQ(strcmp(oci_spec->process->env[1], envs[1]), 0);
++
++    ASSERT_EQ(spec_add_multiple_process_env(oci_spec, nullptr, env_len), 0);
++    ASSERT_EQ(spec_add_multiple_process_env(oci_spec, (const char **)envs, 0), 0);
++    ASSERT_EQ(spec_add_multiple_process_env(nullptr, (const char **)envs, env_len), -1);
++
++    free(envs[0]);
++    envs[0] = util_strdup_s("=value0");
++    ASSERT_EQ(spec_add_multiple_process_env(oci_spec, (const char **)envs, env_len), -1);
++    free(envs[0]);
++    envs[0] = util_strdup_s("key0=");
++    ASSERT_EQ(spec_add_multiple_process_env(oci_spec, (const char **)envs, env_len), -1);
++    free(envs[0]);
++    envs[0] = util_strdup_s("key0xxxx");
++    ASSERT_EQ(spec_add_multiple_process_env(oci_spec, (const char **)envs, env_len), -1);
++
++    free(oci_spec->process->env[0]);
++    oci_spec->process->env[0] = util_strdup_s("=value0");
++    ASSERT_EQ(spec_add_multiple_process_env(oci_spec, (const char **)envs, env_len), -1);
++    free(oci_spec->process->env[0]);
++    oci_spec->process->env[0] = util_strdup_s("key0=");
++    ASSERT_EQ(spec_add_multiple_process_env(oci_spec, (const char **)envs, env_len), -1);
++    free(oci_spec->process->env[0]);
++    oci_spec->process->env[0] = util_strdup_s("key0xxxx");
++    ASSERT_EQ(spec_add_multiple_process_env(oci_spec, (const char **)envs, env_len), -1);
++
++    free_oci_runtime_spec(oci_spec);
++    free(envs[0]);
++    free(envs[1]);
++    free(envs);
++}
++
++TEST_F(SpecsUnitTest, test_spec_add_device)
++{
++    defs_device *device = (defs_device *)util_common_calloc_s(sizeof(defs_device));
++    ASSERT_NE(device, nullptr);
++    oci_runtime_spec *oci_spec = (oci_runtime_spec *)util_common_calloc_s(sizeof(oci_runtime_spec));
++    ASSERT_NE(oci_spec, nullptr);
++    oci_spec->linux = (oci_runtime_config_linux *)util_common_calloc_s(sizeof(oci_runtime_config_linux));
++    ASSERT_NE(oci_spec->linux, nullptr);
++    oci_spec->linux->devices_len = 1;
++    oci_spec->linux->devices = (defs_device **)util_common_calloc_s(sizeof(defs_device *) * oci_spec->linux->devices_len);
++    ASSERT_NE(oci_spec->linux->devices, nullptr);
++
++    device->path = util_strdup_s("/device/path");
++    oci_spec->linux->devices[0] = (defs_device *)util_common_calloc_s(sizeof(defs_device));
++    ASSERT_NE(oci_spec->linux->devices[0], nullptr);
++    oci_spec->linux->devices[0]->path = util_strdup_s("/device/path");
++    ASSERT_EQ(spec_add_device(oci_spec, device), 0);
++    ASSERT_EQ(oci_spec->linux->devices[0], device);
++
++    oci_spec->linux->devices[0] = nullptr;
++    oci_spec->linux->devices_len = 0;
++    ASSERT_EQ(spec_add_device(oci_spec, device), 0);
++    ASSERT_EQ(oci_spec->linux->devices_len, 1);
++    ASSERT_EQ(oci_spec->linux->devices[0], device);
++
++    ASSERT_EQ(spec_add_device(oci_spec, nullptr), -1);
++    ASSERT_EQ(spec_add_device(nullptr, device), -1);
++
++    free_oci_runtime_spec(oci_spec);
++}
++
++TEST_F(SpecsUnitTest, test_spec_add_linux_resources_device)
++{
++    oci_runtime_spec *oci_spec = (oci_runtime_spec *)util_common_calloc_s(sizeof(oci_runtime_spec));
++    ASSERT_NE(oci_spec, nullptr);
++    oci_spec->linux = (oci_runtime_config_linux *)util_common_calloc_s(sizeof(oci_runtime_config_linux));
++    ASSERT_NE(oci_spec->linux, nullptr);
++    oci_spec->linux->resources = (defs_resources *)util_common_calloc_s(sizeof(defs_resources));
++    ASSERT_NE(oci_spec->linux->resources, nullptr);
++    oci_spec->linux->resources->devices_len = 1;
++    oci_spec->linux->resources->devices = (defs_device_cgroup **)util_common_calloc_s(sizeof(defs_device_cgroup *) * oci_spec->linux->resources->devices_len);
++    ASSERT_NE(oci_spec->linux->resources->devices, nullptr);
++
++    oci_spec->linux->resources->devices[0] = (defs_device_cgroup *)util_common_calloc_s(sizeof(defs_device_cgroup));
++    ASSERT_NE(oci_spec->linux->resources->devices[0], nullptr);
++    ASSERT_EQ(spec_add_linux_resources_device(oci_spec, true, "bind", 10, 9, "rwm"), 0);
++    ASSERT_EQ(oci_spec->linux->resources->devices_len, 2);
++    ASSERT_EQ(oci_spec->linux->resources->devices[1]->allow, true);
++    ASSERT_EQ(strcmp(oci_spec->linux->resources->devices[1]->type, "bind"), 0);
++    ASSERT_EQ(oci_spec->linux->resources->devices[1]->major, 10);
++    ASSERT_EQ(oci_spec->linux->resources->devices[1]->minor, 9);
++    ASSERT_EQ(strcmp(oci_spec->linux->resources->devices[1]->access, "rwm"), 0);
++
++    ASSERT_EQ(spec_add_linux_resources_device(nullptr, true, "bind", 10, 9, "rwm"), -1);
++
++    free_oci_runtime_spec(oci_spec);
++}
++
++TEST_F(SpecsUnitTest, test_spec_remove_mount)
++{
++    oci_runtime_spec *oci_spec = (oci_runtime_spec *)util_common_calloc_s(sizeof(oci_runtime_spec));
++    ASSERT_NE(oci_spec, nullptr);
++    oci_spec->mounts_len = 2;
++    oci_spec->mounts = (defs_mount **)util_common_calloc_s(sizeof(defs_mount *) * oci_spec->mounts_len);
++    ASSERT_NE(oci_spec->mounts, nullptr);
++
++    oci_spec->mounts[0] = (defs_mount *)util_common_calloc_s(sizeof(defs_mount));
++    ASSERT_NE(oci_spec->mounts[0], nullptr);
++    oci_spec->mounts[1] = (defs_mount *)util_common_calloc_s(sizeof(defs_mount));
++    ASSERT_NE(oci_spec->mounts[1], nullptr);
++    oci_spec->mounts[0]->destination = util_strdup_s("/mount/path/0");
++    oci_spec->mounts[1]->destination = util_strdup_s("/mount/path/1");
++    spec_remove_mount(oci_spec, oci_spec->mounts[0]->destination);
++    ASSERT_EQ(oci_spec->mounts_len, 1);
++    ASSERT_EQ(strcmp(oci_spec->mounts[0]->destination, "/mount/path/1"), 0);
++
++    free_oci_runtime_spec(oci_spec);
++}
++
++TEST_F(SpecsUnitTest, test_spec_add_mount)
++{
++    defs_mount *mnt = (defs_mount *)util_common_calloc_s(sizeof(defs_mount));
++    ASSERT_NE(mnt, nullptr);
++    oci_runtime_spec *oci_spec = (oci_runtime_spec *)util_common_calloc_s(sizeof(oci_runtime_spec));
++    ASSERT_NE(oci_spec, nullptr);
++    oci_spec->mounts_len = 1;
++    oci_spec->mounts = (defs_mount **)util_common_calloc_s(sizeof(defs_mount *) * oci_spec->mounts_len );
++    ASSERT_NE(oci_spec->mounts, nullptr);
++
++    oci_spec->mounts[0] = (defs_mount *)util_common_calloc_s(sizeof(defs_mount));
++    ASSERT_NE(oci_spec->mounts[0], nullptr);
++    oci_spec->mounts[0]->destination = util_strdup_s("/mount/path/0");
++    ASSERT_EQ(spec_add_mount(oci_spec, mnt), 0);
++    ASSERT_EQ(oci_spec->mounts_len, 2);
++    ASSERT_EQ(oci_spec->mounts[1], mnt);
++
++    ASSERT_EQ(spec_add_mount(nullptr, mnt), -1);
++    ASSERT_EQ(spec_add_mount(oci_spec, nullptr), -1);
++
++    free_oci_runtime_spec(oci_spec);
++}
++
++#define TEST_SPEC_ADD_HOOKS_ITEM_DEF(hooktype)                                                                                  \
++    void test_spec_add_##hooktype##_hook(void)                                                                                  \
++    {                                                                                                                           \
++        defs_hook *hook = (defs_hook *)util_common_calloc_s(sizeof(defs_hook));                                                 \
++        ASSERT_NE(hook, nullptr);                                                                                               \
++        oci_runtime_spec *oci_spec = (oci_runtime_spec *)util_common_calloc_s(sizeof(oci_runtime_spec));                        \
++        ASSERT_NE(oci_spec, nullptr);                                                                                           \
++        oci_spec->hooks = (oci_runtime_spec_hooks *)util_common_calloc_s(sizeof(oci_runtime_spec_hooks));                        \
++        ASSERT_NE(oci_spec->hooks, nullptr);                                                                                           \
++        oci_spec->hooks->hooktype##_len = 1;                                                                                    \
++        oci_spec->hooks->hooktype = (defs_hook **)util_common_calloc_s(sizeof(defs_hook *) * oci_spec->hooks->hooktype##_len);  \
++        ASSERT_NE(oci_spec->hooks->hooktype, nullptr);                                                                          \
++                                                                                                                                \
++        oci_spec->hooks->hooktype[0] = (defs_hook *)util_common_calloc_s(sizeof(defs_hook));                                    \
++        ASSERT_NE(oci_spec->hooks->hooktype[0], nullptr);                                                                       \
++        ASSERT_EQ(spec_add_##hooktype##_hook(oci_spec, hook), 0);                                                               \
++        ASSERT_EQ(oci_spec->hooks->hooktype##_len, 2);                                                                          \
++        ASSERT_EQ(oci_spec->hooks->hooktype[1], hook);                                                                          \
++                                                                                                                                \
++        ASSERT_EQ(spec_add_##hooktype##_hook(nullptr, hook), -1);                                                               \
++        ASSERT_EQ(spec_add_##hooktype##_hook(oci_spec, nullptr), -1);                                                           \
++                                                                                                                                \
++        free_oci_runtime_spec(oci_spec);                                                                                        \
++    }
++
++TEST_SPEC_ADD_HOOKS_ITEM_DEF(prestart)
++TEST_SPEC_ADD_HOOKS_ITEM_DEF(poststart)
++TEST_SPEC_ADD_HOOKS_ITEM_DEF(poststop)
++
++TEST_F(SpecsUnitTest, test_spec_add_hook)
++{
++    test_spec_add_prestart_hook();
++    test_spec_add_poststart_hook();
++    test_spec_add_poststop_hook();
++}
++
++#endif /* ENABLE_CDI */
++
+ /********************************* UT for merge caps *******************************************/
+ struct capabilities_lens {
+     size_t bounding_len;
+diff --git a/test/specs/specs_extend/CMakeLists.txt b/test/specs/specs_extend/CMakeLists.txt
+index 2fd37e1c..bd4d2dd6 100644
+--- a/test/specs/specs_extend/CMakeLists.txt
++++ b/test/specs/specs_extend/CMakeLists.txt
+@@ -44,6 +44,7 @@ add_executable(${EXE}
+     ${CMAKE_CURRENT_SOURCE_DIR}/../../../test/mocks/storage_mock.cc
+     ${CMAKE_CURRENT_SOURCE_DIR}/../../../test/mocks/image_mock.cc
+     ${CMAKE_CURRENT_SOURCE_DIR}/../../../test/mocks/sender_mock.cc
++    ${CMAKE_CURRENT_SOURCE_DIR}/../../../test/mocks/cdi_operate_api_mock.cc
+     specs_extend_ut.cc)
+ 
+ target_include_directories(${EXE} PUBLIC
+-- 
+2.25.1
+

0077-remove-extra-s-in-CreateContainerLogSymlink.patch

@@ -0,0 +1,26 @@
+From cade2ae3b53848bbedb3f89ff45333f3871e878a Mon Sep 17 00:00:00 2001
+From: jikai <jikai11@huawei.com>
+Date: Thu, 25 Apr 2024 12:46:10 +0000
+Subject: [PATCH 077/149] remove extra %s in CreateContainerLogSymlink
+
+Signed-off-by: jikai <jikai11@huawei.com>
+---
+ src/daemon/common/cri/cri_helpers.cc | 2 +-
+ 1 file changed, 1 insertion(+), 1 deletion(-)
+
+diff --git a/src/daemon/common/cri/cri_helpers.cc b/src/daemon/common/cri/cri_helpers.cc
+index 5189ac00..d7ec9f36 100644
+--- a/src/daemon/common/cri/cri_helpers.cc
++++ b/src/daemon/common/cri/cri_helpers.cc
+@@ -552,7 +552,7 @@ void CreateContainerLogSymlink(const std::string &containerID, Errors &error)
+     if (symlink(realPath.c_str(), path.c_str()) != 0) {
+         SYSERROR("failed to create symbolic link %s to the container log file %s for container %s", path.c_str(), realPath.c_str(),
+                  containerID.c_str());
+-        error.Errorf("failed to create symbolic link %s to the container log file %s for container %s: %s", path.c_str(),
++        error.Errorf("failed to create symbolic link %s to the container log file %s for container %s", path.c_str(),
+                      realPath.c_str(), containerID.c_str());
+     }
+ }
+-- 
+2.25.1
+

0078-allow-env-variable-has-an-empty-value.patch

@@ -0,0 +1,69 @@
+From f0212d54afc695b2039f09456b10c47f8edaf2de Mon Sep 17 00:00:00 2001
+From: jikai <jikai11@huawei.com>
+Date: Sat, 27 Apr 2024 14:32:19 +0800
+Subject: [PATCH 078/149] allow env variable has an empty value
+
+Signed-off-by: jikai <jikai11@huawei.com>
+---
+ src/daemon/modules/spec/specs_extend.c | 10 ++++++----
+ src/utils/cutils/utils_string.c        |  3 +++
+ 2 files changed, 9 insertions(+), 4 deletions(-)
+
+diff --git a/src/daemon/modules/spec/specs_extend.c b/src/daemon/modules/spec/specs_extend.c
+index 199cba54..8cad2cbe 100644
+--- a/src/daemon/modules/spec/specs_extend.c
++++ b/src/daemon/modules/spec/specs_extend.c
+@@ -195,6 +195,7 @@ static int generate_env_map_from_file(FILE *fp, json_map_string_string *env_map)
+     char *pline = NULL;
+     size_t length = 0;
+     char *saveptr = NULL;
++    char empty_str[1] = {'\0'};
+ 
+     while (getline(&pline, &length, fp) != -1) {
+         util_trim_newline(pline);
+@@ -204,7 +205,9 @@ static int generate_env_map_from_file(FILE *fp, json_map_string_string *env_map)
+         }
+         key = strtok_r(pline, "=", &saveptr);
+         value = strtok_r(NULL, "=", &saveptr);
+-        if (key != NULL && value != NULL) {
++        // value of an env varible is allowed to be empty
++        value = value ? value : empty_str;
++        if (key != NULL) {
+             key = util_trim_space(key);
+             value = util_trim_space(value);
+             if ((size_t)(MAX_BUFFER_SIZE - 1) - strlen(key) < strlen(value)) {
+@@ -291,15 +294,14 @@ static int check_env_need_append(const oci_runtime_spec *oci_spec, const char *e
+ {
+     size_t i = 0;
+     char *key = NULL;
+-    char *value = NULL;
+     char *saveptr = NULL;
+ 
+     for (i = 0; i < oci_spec->process->env_len; i++) {
+         char *tmp_env = NULL;
+         tmp_env = util_strdup_s(oci_spec->process->env[i]);
+         key = strtok_r(tmp_env, "=", &saveptr);
+-        value = strtok_r(NULL, "=", &saveptr);
+-        if (key == NULL || value == NULL) {
++        // value of an env varible is allowed to be empty
++        if (key == NULL) {
+             ERROR("Bad env format");
+             free(tmp_env);
+             tmp_env = NULL;
+diff --git a/src/utils/cutils/utils_string.c b/src/utils/cutils/utils_string.c
+index 64afb570..11a65f19 100644
+--- a/src/utils/cutils/utils_string.c
++++ b/src/utils/cutils/utils_string.c
+@@ -534,6 +534,9 @@ static char *util_left_trim_space(char *str)
+ {
+     char *begin = str;
+     char *tmp = str;
++    if (strlen(str) == 0) {
++        return str;
++    }
+     while (isspace(*begin)) {
+         begin++;
+     }
+-- 
+2.25.1
+

0079-Fix-Failed-to-execute-image-pull-on-name-tag-digest-.patch

@@ -0,0 +1,72 @@
+From 1d90c5a855a267bb156d53b6a43bc451a993efd5 Mon Sep 17 00:00:00 2001
+From: huj13k4n9 <huj13k4n9@qq.com>
+Date: Mon, 29 Apr 2024 12:52:17 +0800
+Subject: [PATCH 079/149] Fix 'Failed to execute image pull' on
+ 'name:tag@digest' type image name
+
+---
+ src/daemon/modules/image/oci/utils_images.c | 6 ++++++
+ src/utils/cutils/utils_verify.c             | 3 ++-
+ src/utils/cutils/utils_verify.h             | 3 ++-
+ 3 files changed, 10 insertions(+), 2 deletions(-)
+
+diff --git a/src/daemon/modules/image/oci/utils_images.c b/src/daemon/modules/image/oci/utils_images.c
+index d94388bd..b4e2a1a8 100644
+--- a/src/daemon/modules/image/oci/utils_images.c
++++ b/src/daemon/modules/image/oci/utils_images.c
+@@ -207,6 +207,7 @@ int oci_split_image_name(const char *image_name, char **host, char **name, char
+     char *tag_digest_pos = NULL;
+     char *name_pos = NULL;
+     char *tmp_image_name = NULL;
++    char *name_end_pos = NULL;
+ 
+     if (!util_valid_image_name(image_name)) {
+         ERROR("Invalid full image name %s", image_name);
+@@ -234,6 +235,11 @@ int oci_split_image_name(const char *image_name, char **host, char **name, char
+         *name_pos = '\0';
+         name_pos++;
+         if (name != NULL) {
++            // Need to check if image name contains tag
++            name_end_pos = strchr(name_pos, ':');
++            if (name_end_pos != NULL) {
++                *name_end_pos = '\0';
++            }
+             *name = util_strdup_s(name_pos);
+         }
+         if (host != NULL) {
+diff --git a/src/utils/cutils/utils_verify.c b/src/utils/cutils/utils_verify.c
+index cd636fff..474e28f0 100644
+--- a/src/utils/cutils/utils_verify.c
++++ b/src/utils/cutils/utils_verify.c
+@@ -319,6 +319,7 @@ bool util_valid_image_name(const char *name)
+         }
+     }
+ 
++    // In name check phase, image name with both tag and digest is also allowed
+     if (util_reg_match(__NamePattern, copy)) {
+         goto cleanup;
+     }
+@@ -767,4 +768,4 @@ bool util_valid_search_name(const char *name)
+ 
+     return true;
+ }
+-#endif
+\ No newline at end of file
++#endif
+diff --git a/src/utils/cutils/utils_verify.h b/src/utils/cutils/utils_verify.h
+index bafd2a82..fc59f6c0 100644
+--- a/src/utils/cutils/utils_verify.h
++++ b/src/utils/cutils/utils_verify.h
+@@ -32,7 +32,8 @@ extern "C" {
+ #define __NamePattern                                                                 \
+     "^(([a-zA-Z0-9]|[a-zA-Z0-9][a-zA-Z0-9-]*[a-zA-Z0-9])"                             \
+     "((\\.([a-zA-Z0-9]|[a-zA-Z0-9][a-zA-Z0-9-]*[a-zA-Z0-9]))+)?(:[0-9]+)?/)?[a-z0-9]" \
+-    "+((([._]|__|[-]*)[a-z0-9]+)+)?((/[a-z0-9]+((([._]|__|[-]*)[a-z0-9]+)+)?)+)?$"
++    "+((([._]|__|[-]*)[a-z0-9]+)+)?((/[a-z0-9]+((([._]|__|[-]*)[a-z0-9]+)+)?)+)?"     \
++    "(:([A-Za-z_0-9][A-Za-z_0-9.-]{0,127}))?$"
+ 
+ #define __DIGESTPattern "@[a-z0-9]+:[a-z0-9]{32,}"
+ 
+-- 
+2.25.1
+

0080-bugfix-for-hostname-env-set-only-once.patch

@@ -0,0 +1,148 @@
+From 8ff32819d84f59085c4c541b00f9671db55d0fd1 Mon Sep 17 00:00:00 2001
+From: jikai <jikai11@huawei.com>
+Date: Mon, 29 Apr 2024 09:14:53 +0800
+Subject: [PATCH 080/149] bugfix for hostname env: set only once
+
+Signed-off-by: jikai <jikai11@huawei.com>
+---
+ src/daemon/modules/spec/specs.c        | 11 +++++-
+ src/daemon/modules/spec/specs_extend.c | 52 +++++++++++++++++---------
+ src/daemon/modules/spec/specs_extend.h |  2 +
+ 3 files changed, 46 insertions(+), 19 deletions(-)
+
+diff --git a/src/daemon/modules/spec/specs.c b/src/daemon/modules/spec/specs.c
+index 77ca70f9..65a860d4 100644
+--- a/src/daemon/modules/spec/specs.c
++++ b/src/daemon/modules/spec/specs.c
+@@ -1863,14 +1863,21 @@ static int merge_process_conf(oci_runtime_spec *oci_spec, const host_config *hos
+         goto out;
+     }
+ 
+-    /* environment variables */
++    /* 1. merge env from container_spec: --env or --env-file */
+     ret = merge_env(oci_spec, (const char **)container_spec->env, container_spec->env_len);
+     if (ret != 0) {
+         ERROR("Failed to merge environment variables");
+         goto out;
+     }
+ 
+-    /* env target file */
++    /* 2. merge default env hostname, only if hostname not set before */
++    ret = merge_hostname_env(oci_spec);
++    if (ret != 0) {
++        ERROR("Failed to merge hostname env");
++        goto out;
++    }
++
++    /* 3. persist env from --env-target-file, only if the env not set before, system container only */
+     ret = merge_env_target_file(oci_spec, host_spec->env_target_file);
+     if (ret != 0) {
+         ERROR("Failed to merge env target file");
+diff --git a/src/daemon/modules/spec/specs_extend.c b/src/daemon/modules/spec/specs_extend.c
+index 8cad2cbe..4c154281 100644
+--- a/src/daemon/modules/spec/specs_extend.c
++++ b/src/daemon/modules/spec/specs_extend.c
+@@ -420,34 +420,23 @@ out:
+ int merge_env(oci_runtime_spec *oci_spec, const char **env, size_t env_len)
+ {
+     int ret = 0;
+-    int nret = 0;
+     size_t new_size = 0;
+     size_t old_size = 0;
+     size_t i;
+     char **temp = NULL;
+-    // 10 is lenght of "HOSTNAME=" and '\0'
+-    char host_name_env[MAX_HOST_NAME_LEN + 10] = { 0 };
+-
+-    nret = snprintf(host_name_env, sizeof(host_name_env), "HOSTNAME=%s", oci_spec->hostname);
+-    if (nret < 0 || (size_t)nret >= sizeof(host_name_env)) {
+-        ret = -1;
+-        ERROR("Sprint failed");
+-        goto out;
+-    }
+ 
+     ret = make_sure_oci_spec_process(oci_spec);
+     if (ret < 0) {
+         goto out;
+     }
+ 
+-    if (env_len > LIST_ENV_SIZE_MAX - oci_spec->process->env_len - 1) {
++    if (env_len > LIST_ENV_SIZE_MAX - oci_spec->process->env_len) {
+         ERROR("The length of envionment variables is too long, the limit is %lld", LIST_ENV_SIZE_MAX);
+         isulad_set_error_message("The length of envionment variables is too long, the limit is %d", LIST_ENV_SIZE_MAX);
+         ret = -1;
+         goto out;
+     }
+-    // add 1 for hostname env
+-    new_size = (oci_spec->process->env_len + env_len + 1) * sizeof(char *);
++    new_size = (oci_spec->process->env_len + env_len) * sizeof(char *);
+     old_size = oci_spec->process->env_len * sizeof(char *);
+     ret = util_mem_realloc((void **)&temp, new_size, oci_spec->process->env, old_size);
+     if (ret != 0) {
+@@ -458,10 +447,6 @@ int merge_env(oci_runtime_spec *oci_spec, const char **env, size_t env_len)
+ 
+     oci_spec->process->env = temp;
+ 
+-    // append hostname env into default oci spec env list
+-    oci_spec->process->env[oci_spec->process->env_len] = util_strdup_s(host_name_env);
+-    oci_spec->process->env_len++;
+-
+     for (i = 0; i < env_len && env != NULL; i++) {
+         oci_spec->process->env[oci_spec->process->env_len] = util_strdup_s(env[i]);
+         oci_spec->process->env_len++;
+@@ -470,6 +455,39 @@ out:
+     return ret;
+ }
+ 
++int merge_hostname_env(oci_runtime_spec *oci_spec)
++{
++    int nret = 0;
++    bool is_append = true;
++    // 10 is lenght of "HOSTNAME=" and '\0'
++    char host_name_env[MAX_HOST_NAME_LEN + 10] = { 0 };
++    const char *envs[1] = {host_name_env};
++
++    if (make_sure_oci_spec_process(oci_spec) < 0) {
++        return -1;
++    }
++
++    if (check_env_need_append(oci_spec, "HOSTNAME", &is_append) < 0) {
++        return -1;
++    }
++
++    if (!is_append) {
++        return 0;
++    }
++
++    nret = snprintf(host_name_env, sizeof(host_name_env), "HOSTNAME=%s", oci_spec->hostname);
++    if (nret < 0 || (size_t)nret >= sizeof(host_name_env)) {
++        ERROR("Sprint failed");
++        return -1;
++    }
++
++    if (merge_env(oci_spec, (const char **)envs, 1) < 0) {
++        return -1;
++    }
++
++    return 0;
++}
++
+ char *oci_container_get_env(const oci_runtime_spec *oci_spec, const char *key)
+ {
+     const defs_process *op = NULL;
+diff --git a/src/daemon/modules/spec/specs_extend.h b/src/daemon/modules/spec/specs_extend.h
+index d70f5bec..15ec6b2f 100644
+--- a/src/daemon/modules/spec/specs_extend.h
++++ b/src/daemon/modules/spec/specs_extend.h
+@@ -50,6 +50,8 @@ int make_userns_remap(oci_runtime_spec *container, const char *user_remap);
+ 
+ int merge_env(oci_runtime_spec *oci_spec, const char **env, size_t env_len);
+ 
++int merge_hostname_env(oci_runtime_spec *oci_spec);
++
+ int merge_env_target_file(oci_runtime_spec *oci_spec, const char *env_target_file);
+ 
+ char *oci_container_get_env(const oci_runtime_spec *oci_spec, const char *key);
+-- 
+2.25.1
+

0081-set-the-sandbox-status-to-not-ready-under-abnormal-c.patch

@@ -0,0 +1,86 @@
+From 934d289aa535bbb87bfe484c4de34275b968fb87 Mon Sep 17 00:00:00 2001
+From: zhongtao <zhongtao17@huawei.com>
+Date: Wed, 8 May 2024 11:40:40 +0800
+Subject: [PATCH 081/149] set the sandbox status to not ready under abnormal
+ circumstances
+
+Signed-off-by: zhongtao <zhongtao17@huawei.com>
+---
+ src/daemon/sandbox/sandbox.cc | 34 +++++++++++++++++++++++++---------
+ src/daemon/sandbox/sandbox.h  |  1 +
+ 2 files changed, 26 insertions(+), 9 deletions(-)
+
+diff --git a/src/daemon/sandbox/sandbox.cc b/src/daemon/sandbox/sandbox.cc
+index bae5b8db..279bf628 100644
+--- a/src/daemon/sandbox/sandbox.cc
++++ b/src/daemon/sandbox/sandbox.cc
+@@ -371,6 +371,8 @@ void Sandbox::DoUpdateStatus(std::unique_ptr<ControllerSandboxStatus> status, Er
+     m_state.exitedAt = status->exitedAt;
+     if (status->state == std::string(SANDBOX_READY_STATE_STR)) {
+         m_state.status = SANDBOX_STATUS_RUNNING;
++    } else {
++        m_state.status = SANDBOX_STATUS_STOPPED;
+     }
+ }
+ 
+@@ -459,6 +461,24 @@ auto Sandbox::Save(Errors &error) -> bool
+     return true;
+ }
+ 
++bool Sandbox::DoStatusUpdateAndWaitInLoad(const std::string &sandboxID, Errors &error)
++{
++    if (!UpdateStatus(error)) {
++        ERROR("Failed to update status of Sandbox, id='%s'", sandboxID.c_str());
++        return false;
++    }
++
++    // Regardless of whether the sandbox is ready,
++    // Wait() is required to call to monitor whether the kuasar sandbox is ready or exits.
++    // TODO: distinguish the meaning of Wait() return value in different states of sandbox
++    if (!m_controller->Wait(shared_from_this(), sandboxID, error)) {
++        ERROR("Failed to restore wait callback");
++        return false;
++    }
++
++    return true;
++}
++
+ auto Sandbox::Load(Errors &error) -> bool
+ {
+     if (!LoadState(error)) {
+@@ -478,15 +498,11 @@ auto Sandbox::Load(Errors &error) -> bool
+ 
+     LoadNetworkSetting();
+ 
+-    if (!UpdateStatus(error)) {
+-        ERROR("Failed to update status of Sandbox, id='%s'", m_id.c_str());
+-        return false;
+-    }
+-
+-    // TODO: distinguish the meaning of Wait() return value in different states of sandbox
+-    if (!m_controller->Wait(shared_from_this(), m_id, error)) {
+-        ERROR("Failed to restore wait callback");
+-        return false;
++    // When the sandbox status acquisition fails or wait fails, the sandbox status is set to not ready,
++    // and the user decides whether to delete the sandbox.
++    if (!DoStatusUpdateAndWaitInLoad(m_id, error)) {
++        WriteGuard<RWMutex> lock(m_stateMutex);
++        m_state.status = SANDBOX_STATUS_STOPPED;
+     }
+ 
+     return true;
+diff --git a/src/daemon/sandbox/sandbox.h b/src/daemon/sandbox/sandbox.h
+index 20a8e338..42fbee2a 100644
+--- a/src/daemon/sandbox/sandbox.h
++++ b/src/daemon/sandbox/sandbox.h
+@@ -156,6 +156,7 @@ private:
+     auto SetupSandboxFiles(Errors &error) -> bool;
+     void DoUpdateStatus(std::unique_ptr<ControllerSandboxStatus> status, Errors &error);
+     void DoUpdateExitedStatus(const ControllerExitInfo &exitInfo);
++    bool DoStatusUpdateAndWaitInLoad(const std::string &sandboxID, Errors &error);
+ 
+     auto GetMetadataJsonPath() ->  std::string;
+     auto GetStatePath() -> std::string;
+-- 
+2.25.1
+

0082-fix-shim-controller-set-incorrect-sandbox-status-sta.patch

@@ -0,0 +1,61 @@
+From 1d51e3e9f14199854cc2d586651c5809345aee18 Mon Sep 17 00:00:00 2001
+From: zhongtao <zhongtao17@huawei.com>
+Date: Wed, 8 May 2024 14:48:47 +0800
+Subject: [PATCH 082/149] fix shim controller set incorrect sandbox status
+ state
+
+Signed-off-by: jikai <jikai11@huawei.com>
+---
+ src/daemon/sandbox/controller/controller.h            | 3 +++
+ src/daemon/sandbox/controller/shim/shim_controller.cc | 6 ++++--
+ src/daemon/sandbox/sandbox.cc                         | 3 ---
+ 3 files changed, 7 insertions(+), 5 deletions(-)
+
+diff --git a/src/daemon/sandbox/controller/controller.h b/src/daemon/sandbox/controller/controller.h
+index f479a0ac..9ad45855 100644
+--- a/src/daemon/sandbox/controller/controller.h
++++ b/src/daemon/sandbox/controller/controller.h
+@@ -27,6 +27,9 @@
+ 
+ namespace sandbox {
+ 
++#define SANDBOX_READY_STATE_STR "SANDBOX_READY"
++#define SANDBOX_NOTREADY_STATE_STR "SANDBOX_NOTREADY"
++
+ struct ControllerMountInfo {
+     std::string source;
+     std::string destination;
+diff --git a/src/daemon/sandbox/controller/shim/shim_controller.cc b/src/daemon/sandbox/controller/shim/shim_controller.cc
+index 4da637c7..ce09c076 100644
+--- a/src/daemon/sandbox/controller/shim/shim_controller.cc
++++ b/src/daemon/sandbox/controller/shim/shim_controller.cc
+@@ -446,8 +446,10 @@ void ShimController::InspectResponseToSandboxStatus(container_inspect *inspect,
+     sandboxStatus.id = inspect->id;
+     if (inspect->state != nullptr) {
+         sandboxStatus.pid = inspect->state->pid;
+-        if (inspect->state->status != nullptr) {
+-            sandboxStatus.state = std::string(inspect->state->status);
++        if (inspect->state->running) {
++            sandboxStatus.state = std::string(SANDBOX_READY_STATE_STR);
++        } else {
++            sandboxStatus.state = std::string(SANDBOX_NOTREADY_STATE_STR);
+         }
+     }
+ 
+diff --git a/src/daemon/sandbox/sandbox.cc b/src/daemon/sandbox/sandbox.cc
+index 279bf628..d44abb99 100644
+--- a/src/daemon/sandbox/sandbox.cc
++++ b/src/daemon/sandbox/sandbox.cc
+@@ -39,9 +39,6 @@
+ #include "utils_timestamp.h"
+ #include "mailbox.h"
+ 
+-#define SANDBOX_READY_STATE_STR "SANDBOX_READY"
+-#define SANDBOX_NOTREADY_STATE_STR "SANDBOX_NOTREADY"
+-
+ namespace sandbox {
+ 
+ const std::string SHM_MOUNT_POINT = "/dev/shm";
+-- 
+2.25.1
+

0083-fix-bug-for-invalid-env-write.patch

@@ -0,0 +1,158 @@
+From fb48f036fece9d64c4cfc19c52091afad5f42fd9 Mon Sep 17 00:00:00 2001
+From: jikai <jikai11@huawei.com>
+Date: Sat, 11 May 2024 03:46:02 +0000
+Subject: [PATCH 083/149] fix bug for invalid env write
+
+Signed-off-by: jikai <jikai11@huawei.com>
+---
+ src/daemon/modules/spec/specs_extend.c | 57 +++++++++-----------------
+ src/utils/cutils/utils_verify.c        | 25 +++++++++++
+ src/utils/cutils/utils_verify.h        |  2 +
+ 3 files changed, 46 insertions(+), 38 deletions(-)
+
+diff --git a/src/daemon/modules/spec/specs_extend.c b/src/daemon/modules/spec/specs_extend.c
+index 4c154281..f4208405 100644
+--- a/src/daemon/modules/spec/specs_extend.c
++++ b/src/daemon/modules/spec/specs_extend.c
+@@ -190,41 +190,33 @@ int make_userns_remap(oci_runtime_spec *container, const char *user_remap)
+ static int generate_env_map_from_file(FILE *fp, json_map_string_string *env_map)
+ {
+     int ret = 0;
+-    char *key = NULL;
+-    char *value = NULL;
+-    char *pline = NULL;
++    __isula_auto_free char *pline = NULL;
+     size_t length = 0;
+-    char *saveptr = NULL;
+-    char empty_str[1] = {'\0'};
+ 
+     while (getline(&pline, &length, fp) != -1) {
++        __isula_auto_free char *key = NULL;
++        __isula_auto_free char *value = NULL;
+         util_trim_newline(pline);
+         pline = util_trim_space(pline);
+         if (pline == NULL || pline[0] == '#') {
+             continue;
+         }
+-        key = strtok_r(pline, "=", &saveptr);
+-        value = strtok_r(NULL, "=", &saveptr);
+-        // value of an env varible is allowed to be empty
+-        value = value ? value : empty_str;
+-        if (key != NULL) {
+-            key = util_trim_space(key);
+-            value = util_trim_space(value);
+-            if ((size_t)(MAX_BUFFER_SIZE - 1) - strlen(key) < strlen(value)) {
+-                ERROR("env length exceed %d bytes", MAX_BUFFER_SIZE);
+-                ret = -1;
+-                goto out;
+-            }
+-            ret = append_json_map_string_string(env_map, key, value);
+-            if (ret < 0) {
+-                ERROR("append env to map failed");
+-                goto out;
+-            }
++        if (util_valid_split_env(pline, &key, &value) < 0) {
++            // ignore invalid env
++            continue;
++        }
++        if ((size_t)(MAX_BUFFER_SIZE - 1) - strlen(key) < strlen(value)) {
++            ERROR("env length exceed %d bytes", MAX_BUFFER_SIZE);
++            return -1;
++        }
++        ret = append_json_map_string_string(env_map, key, value);
++        if (ret < 0) {
++            ERROR("append env to map failed");
++            return -1;
+         }
+     }
+-out:
+-    free(pline);
+-    return ret;
++
++    return 0;
+ }
+ 
+ static json_map_string_string *parse_env_target_file(const char *env_path)
+@@ -293,28 +285,17 @@ static int do_append_env(char ***env, size_t *env_len, const char *key, const ch
+ static int check_env_need_append(const oci_runtime_spec *oci_spec, const char *env_key, bool *is_append)
+ {
+     size_t i = 0;
+-    char *key = NULL;
+-    char *saveptr = NULL;
+ 
+     for (i = 0; i < oci_spec->process->env_len; i++) {
+-        char *tmp_env = NULL;
+-        tmp_env = util_strdup_s(oci_spec->process->env[i]);
+-        key = strtok_r(tmp_env, "=", &saveptr);
+-        // value of an env varible is allowed to be empty
+-        if (key == NULL) {
++        __isula_auto_free char *key = NULL;
++        if (util_valid_split_env(oci_spec->process->env[i], &key, NULL) < 0) {
+             ERROR("Bad env format");
+-            free(tmp_env);
+-            tmp_env = NULL;
+             return -1;
+         }
+         if (strcmp(key, env_key) == 0) {
+             *is_append = false;
+-            free(tmp_env);
+-            tmp_env = NULL;
+             return 0;
+         }
+-        free(tmp_env);
+-        tmp_env = NULL;
+     }
+     return 0;
+ }
+diff --git a/src/utils/cutils/utils_verify.c b/src/utils/cutils/utils_verify.c
+index 474e28f0..6f1da12c 100644
+--- a/src/utils/cutils/utils_verify.c
++++ b/src/utils/cutils/utils_verify.c
+@@ -651,6 +651,31 @@ bool util_valid_device_cgroup_rule(const char *value)
+     return util_reg_match(patten, value) == 0;
+ }
+ 
++int util_valid_split_env(const char *env, char **key, char **value)
++{
++    __isula_auto_array_t char **arr = NULL;
++
++    arr = util_string_split_n(env, '=', 2);
++    if (arr == NULL) {
++        ERROR("Failed to split env string");
++        return -1;
++    }
++
++    if (strlen(arr[0]) == 0) {
++        ERROR("Invalid environment variable: %s", env);
++        return -1;
++    }
++
++    if (key != NULL) {
++        *key = util_strdup_s(arr[0]);
++    }
++    if (value != NULL) {
++        *value = util_strdup_s(util_array_len((const char **)arr) > 1 ? arr[1] : "");
++    }
++
++    return 0;
++}
++
+ int util_valid_env(const char *env, char **dst)
+ {
+     int ret = 0;
+diff --git a/src/utils/cutils/utils_verify.h b/src/utils/cutils/utils_verify.h
+index fc59f6c0..58b22b85 100644
+--- a/src/utils/cutils/utils_verify.h
++++ b/src/utils/cutils/utils_verify.h
+@@ -119,6 +119,8 @@ bool util_valid_positive_interger(const char *value);
+ 
+ bool util_valid_device_cgroup_rule(const char *value);
+ 
++int util_valid_split_env(const char *env, char **key, char **value);
++
+ int util_valid_env(const char *env, char **dst);
+ 
+ bool util_valid_sysctl(const char *sysctl_key);
+-- 
+2.25.1
+

0084-trim-key-value-for-env.patch

@@ -0,0 +1,26 @@
+From de9ed770a254c8f67ac228f56fe461e1c834609c Mon Sep 17 00:00:00 2001
+From: jikai <jikai11@huawei.com>
+Date: Sat, 11 May 2024 07:51:35 +0000
+Subject: [PATCH 084/149] trim key/value for env
+
+Signed-off-by: jikai <jikai11@huawei.com>
+---
+ src/daemon/modules/spec/specs_extend.c | 2 ++
+ 1 file changed, 2 insertions(+)
+
+diff --git a/src/daemon/modules/spec/specs_extend.c b/src/daemon/modules/spec/specs_extend.c
+index f4208405..926aaf3c 100644
+--- a/src/daemon/modules/spec/specs_extend.c
++++ b/src/daemon/modules/spec/specs_extend.c
+@@ -205,6 +205,8 @@ static int generate_env_map_from_file(FILE *fp, json_map_string_string *env_map)
+             // ignore invalid env
+             continue;
+         }
++        key = util_trim_space(key);
++        value = util_trim_space(value);
+         if ((size_t)(MAX_BUFFER_SIZE - 1) - strlen(key) < strlen(value)) {
+             ERROR("env length exceed %d bytes", MAX_BUFFER_SIZE);
+             return -1;
+-- 
+2.25.1
+

0085-cdi-allow-env-variable-has-an-empty-value.patch

@@ -0,0 +1,135 @@
+From 9208d73274da0bd18c0d77cdf59ead3dc8e06021 Mon Sep 17 00:00:00 2001
+From: liuxu <liuxu156@huawei.com>
+Date: Fri, 10 May 2024 18:12:49 +0800
+Subject: [PATCH 085/149] cdi:allow env variable has an empty value
+
+Signed-off-by: liuxu <liuxu156@huawei.com>
+---
+ src/daemon/modules/spec/specs.c | 28 ++++++----------------------
+ test/specs/specs/specs_ut.cc    | 16 ++++++++--------
+ 2 files changed, 14 insertions(+), 30 deletions(-)
+
+diff --git a/src/daemon/modules/spec/specs.c b/src/daemon/modules/spec/specs.c
+index 65a860d4..e779c22e 100644
+--- a/src/daemon/modules/spec/specs.c
++++ b/src/daemon/modules/spec/specs.c
+@@ -2607,17 +2607,11 @@ int spec_module_init(void)
+ static int add_env(defs_process *dp, const char *env, const char *key)
+ {
+     size_t i;
+-    char *oci_key = NULL;
+-    char *oci_value = NULL;
+-    char *saveptr = NULL;
+-    __isula_auto_free char *tmp_env = NULL;
+  
+     for (i = 0; i < dp->env_len; i++) {
+-        tmp_env = util_strdup_s(dp->env[i]);
+-        oci_key = strtok_r(tmp_env, "=", &saveptr);
+-        oci_value = strtok_r(NULL, "=", &saveptr);
+-        if (oci_key == NULL || oci_value == NULL) {
+-            ERROR("Bad env format");
++        __isula_auto_free char *oci_key = NULL;
++        if (util_valid_split_env(dp->env[i], &oci_key, NULL) < 0) {
++            ERROR("Bad env format, %s", dp->env[i]);
+             return -1;
+         }
+         if (strcmp(key, oci_key) == 0) {
+@@ -2625,8 +2619,6 @@ static int add_env(defs_process *dp, const char *env, const char *key)
+             dp->env[i] = util_strdup_s(env);
+             return 0;
+         }
+-        free(tmp_env);
+-        tmp_env = NULL;
+     }
+     if (util_mem_realloc((void **)&dp->env, (dp->env_len + 1) * sizeof(char *),
+                          (void *)dp->env, dp->env_len * sizeof(char *)) != 0) {
+@@ -2641,10 +2633,6 @@ static int add_env(defs_process *dp, const char *env, const char *key)
+ int defs_process_add_multiple_env(defs_process *dp, const char **envs, size_t env_len)
+ {
+     size_t i;
+-    char *key = NULL;
+-    char *value = NULL;
+-    char *saveptr = NULL;
+-    __isula_auto_free char *tmp_env = NULL;
+ 
+     if (envs == NULL || env_len == 0) {
+         DEBUG("empty envs");
+@@ -2656,18 +2644,14 @@ int defs_process_add_multiple_env(defs_process *dp, const char **envs, size_t en
+     }
+ 
+     for (i = 0; i < env_len; i++) {
+-        tmp_env = util_strdup_s(envs[i]);
+-        key = strtok_r(tmp_env, "=", &saveptr);
+-        value = strtok_r(NULL, "=", &saveptr);
+-        if (key == NULL || value == NULL) {
+-            ERROR("Bad env format: %s", tmp_env);
++        __isula_auto_free char *key = NULL;
++        if (util_valid_split_env(envs[i], &key, NULL) < 0) {
++            ERROR("Bad env format: %s", envs[i]);
+             return -1;
+         }
+         if (add_env(dp, envs[i], key) != 0) {
+             return -1;
+         }
+-        free(tmp_env);
+-        tmp_env = NULL;
+     }
+ 
+     return 0;
+diff --git a/test/specs/specs/specs_ut.cc b/test/specs/specs/specs_ut.cc
+index 47836e5b..3f108f0f 100644
+--- a/test/specs/specs/specs_ut.cc
++++ b/test/specs/specs/specs_ut.cc
+@@ -593,20 +593,20 @@ TEST_F(SpecsUnitTest, test_defs_process_add_multiple_env)
+     ASSERT_EQ(defs_process_add_multiple_env(dp, (const char **)envs, env_len), -1);
+     free(envs[0]);
+     envs[0] = util_strdup_s("key0=");
+-    ASSERT_EQ(defs_process_add_multiple_env(dp, (const char **)envs, env_len), -1);
++    ASSERT_EQ(defs_process_add_multiple_env(dp, (const char **)envs, env_len), 0);
+     free(envs[0]);
+     envs[0] = util_strdup_s("key0xxxx");
+-    ASSERT_EQ(defs_process_add_multiple_env(dp, (const char **)envs, env_len), -1);
++    ASSERT_EQ(defs_process_add_multiple_env(dp, (const char **)envs, env_len), 0);
+ 
+     free(dp->env[0]);
+     dp->env[0] = util_strdup_s("=value0");
+     ASSERT_EQ(defs_process_add_multiple_env(dp, (const char **)envs, env_len), -1);
+     free(dp->env[0]);
+     dp->env[0] = util_strdup_s("key0=");
+-    ASSERT_EQ(defs_process_add_multiple_env(dp, (const char **)envs, env_len), -1);
++    ASSERT_EQ(defs_process_add_multiple_env(dp, (const char **)envs, env_len), 0);
+     free(dp->env[0]);
+     dp->env[0] = util_strdup_s("key0xxxx");
+-    ASSERT_EQ(defs_process_add_multiple_env(dp, (const char **)envs, env_len), -1);
++    ASSERT_EQ(defs_process_add_multiple_env(dp, (const char **)envs, env_len), 0);
+ 
+     free_defs_process(dp);
+     free(envs[0]);
+@@ -644,20 +644,20 @@ TEST_F(SpecsUnitTest, test_spec_add_multiple_process_env)
+     ASSERT_EQ(spec_add_multiple_process_env(oci_spec, (const char **)envs, env_len), -1);
+     free(envs[0]);
+     envs[0] = util_strdup_s("key0=");
+-    ASSERT_EQ(spec_add_multiple_process_env(oci_spec, (const char **)envs, env_len), -1);
++    ASSERT_EQ(spec_add_multiple_process_env(oci_spec, (const char **)envs, env_len), 0);
+     free(envs[0]);
+     envs[0] = util_strdup_s("key0xxxx");
+-    ASSERT_EQ(spec_add_multiple_process_env(oci_spec, (const char **)envs, env_len), -1);
++    ASSERT_EQ(spec_add_multiple_process_env(oci_spec, (const char **)envs, env_len), 0);
+ 
+     free(oci_spec->process->env[0]);
+     oci_spec->process->env[0] = util_strdup_s("=value0");
+     ASSERT_EQ(spec_add_multiple_process_env(oci_spec, (const char **)envs, env_len), -1);
+     free(oci_spec->process->env[0]);
+     oci_spec->process->env[0] = util_strdup_s("key0=");
+-    ASSERT_EQ(spec_add_multiple_process_env(oci_spec, (const char **)envs, env_len), -1);
++    ASSERT_EQ(spec_add_multiple_process_env(oci_spec, (const char **)envs, env_len), 0);
+     free(oci_spec->process->env[0]);
+     oci_spec->process->env[0] = util_strdup_s("key0xxxx");
+-    ASSERT_EQ(spec_add_multiple_process_env(oci_spec, (const char **)envs, env_len), -1);
++    ASSERT_EQ(spec_add_multiple_process_env(oci_spec, (const char **)envs, env_len), 0);
+ 
+     free_oci_runtime_spec(oci_spec);
+     free(envs[0]);
+-- 
+2.25.1
+

0086-cdi-test-case-and-gateway.patch

@@ -0,0 +1,359 @@
+From 3fb6de60ecf10278a676a41f8d0c334d1f90d303 Mon Sep 17 00:00:00 2001
+From: liuxu <liuxu156@huawei.com>
+Date: Wed, 17 Apr 2024 18:02:48 +0800
+Subject: [PATCH 086/149] cdi:test case and gateway
+
+Signed-off-by: liuxu <liuxu156@huawei.com>
+---
+ CI/make-and-install.sh                        |   2 +-
+ CI/test_cases/container_cases/cdi_test.sh     | 205 ++++++++++++++++++
+ .../criconfigs/container-config-cdi.json      |  50 +++++
+ .../test_data/cdi_devices.json                |  53 +++++
+ 4 files changed, 309 insertions(+), 1 deletion(-)
+ create mode 100755 CI/test_cases/container_cases/cdi_test.sh
+ create mode 100644 CI/test_cases/container_cases/criconfigs/container-config-cdi.json
+ create mode 100644 CI/test_cases/container_cases/test_data/cdi_devices.json
+
+diff --git a/CI/make-and-install.sh b/CI/make-and-install.sh
+index 9d4c5533..61281965 100755
+--- a/CI/make-and-install.sh
++++ b/CI/make-and-install.sh
+@@ -105,7 +105,7 @@ cd build
+ if [[ ${enable_gcov} -ne 0 ]]; then
+   cmake -DLIB_INSTALL_DIR=${builddir}/lib -DCMAKE_INSTALL_PREFIX=${builddir} -DCMAKE_INSTALL_SYSCONFDIR=${builddir}/etc -DCMAKE_BUILD_TYPE=Debug -DGCOV=ON -DENABLE_EMBEDDED=ON -DENABLE_COVERAGE=ON -DENABLE_CRI_API_V1=ON -DENABLE_CDI=ON -DENABLE_UT=ON -DENABLE_METRICS=ON -DENABLE_REMOTE_LAYER_STORE=ON -DENABLE_GRPC_REMOTE_CONNECT=ON ..
+ else
+-  cmake -DLIB_INSTALL_DIR=${builddir}/lib -DCMAKE_INSTALL_PREFIX=${builddir} -DCMAKE_INSTALL_SYSCONFDIR=${builddir}/etc -DENABLE_EMBEDDED=ON -DENABLE_METRICS=ON -DENABLE_REMOTE_LAYER_STORE=ON -DENABLE_CRI_API_V1=ON -DENABLE_GRPC_REMOTE_CONNECT=ON ..
++  cmake -DLIB_INSTALL_DIR=${builddir}/lib -DCMAKE_INSTALL_PREFIX=${builddir} -DCMAKE_INSTALL_SYSCONFDIR=${builddir}/etc -DENABLE_EMBEDDED=ON -DENABLE_METRICS=ON -DENABLE_REMOTE_LAYER_STORE=ON -DENABLE_CRI_API_V1=ON -DENABLE_CDI=ON -DENABLE_GRPC_REMOTE_CONNECT=ON ..
+ fi
+ make -j $(nproc)
+ make install
+diff --git a/CI/test_cases/container_cases/cdi_test.sh b/CI/test_cases/container_cases/cdi_test.sh
+new file mode 100755
+index 00000000..dd7b1177
+--- /dev/null
++++ b/CI/test_cases/container_cases/cdi_test.sh
+@@ -0,0 +1,205 @@
++#!/bin/bash
++#
++# attributes: isulad cdi
++# concurrent: NA
++# spend time: 41
++
++#######################################################################
++##- Copyright (c) Huawei Technologies Co., Ltd. 2024. All rights reserved.
++# - iSulad licensed under the Mulan PSL v2.
++# - You can use this software according to the terms and conditions of the Mulan PSL v2.
++# - You may obtain a copy of Mulan PSL v2 at:
++# -     http://license.coscl.org.cn/MulanPSL2
++# - THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR
++# - IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR
++# - PURPOSE.
++# - See the Mulan PSL v2 for more details.
++##- @Description:CI
++##- @Author: liuxu
++##- @Create: 2024-04-16
++#######################################################################
++
++source ../helpers.sh
++curr_path=$(dirname $(readlink -f "$0"))
++data_path=$(realpath $curr_path/criconfigs)
++pause_img_path=$(realpath $curr_path/test_data)
++cdi_static_dir="/etc/cdi"
++
++function do_pre()
++{
++    cp /etc/isulad/daemon.json /etc/isulad/daemon.bak
++    sed -i "s#\"pod-sandbox-image\": \"\"#\"pod-sandbox-image\": \"mirrorgooglecontainers/pause-amd64:3.0\"#g" /etc/isulad/daemon.json
++    sed -i "/\"cni-conf-dir\": \".*\"/a\ \ \ \ \"enable-cri-v1\": true," /etc/isulad/daemon.json
++    sed -i "/\"cni-conf-dir\": \".*\"/a\ \ \ \ \"enable-cdi\": true," /etc/isulad/daemon.json
++
++    check_valgrind_log
++    start_isulad_without_valgrind
++
++    isula load -i ${pause_img_path}/pause.tar
++    [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - failed to load pause" && return ${FAILURE}
++
++    isula pull busybox
++    [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - failed to pull busybox" && return ${FAILURE}
++
++    crictl images | grep "mirrorgooglecontainers/pause-amd64"
++    [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - Failed to find mirrorgooglecontainers/pause-amd64 image" && return ${FAILURE}
++
++    return 0
++}
++
++function do_post()
++{
++    cp -f /etc/isulad/daemon.bak /etc/isulad/daemon.json
++    check_valgrind_log
++    start_isulad_without_valgrind
++}
++
++function verify_injected_vendor0() {
++    # check env
++    output=$(crictl exec --sync "$1" sh -c 'echo $VENDOR0')
++    [[ "$output" != "injected" ]] && msg_err "${FUNCNAME[0]}:${LINENO} - env check failed" && return ${FAILURE}
++
++    # check hooks
++    cat /tmp/cdi_hook_test.log | grep "prestart"
++    [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - hook check failed" && return ${FAILURE}
++
++    # check mounts
++    output=$(crictl exec --sync "$1" sh -c 'stat -c %a /tmp/cdi_mounts_test')
++    [[ "$output" != "755" ]] && msg_err "${FUNCNAME[0]}:${LINENO} - mount check failed" && return ${FAILURE}
++
++    return 0
++}
++
++function verify_injected_loop8() {
++    # check env
++    output=$(crictl exec --sync "$1" sh -c 'echo $LOOP8')
++    [[ "$output" != "CDI8" ]] && msg_err "${FUNCNAME[0]}:${LINENO} - env check failed" && return ${FAILURE}
++
++    # check device nodes
++    output=$(crictl exec --sync "$1" sh -c 'stat -c %a /dev/loop8')
++    [[ "$output" != "640" ]] && msg_err "${FUNCNAME[0]}:${LINENO} - device nodes check failed" && return ${FAILURE}
++    output=$(crictl exec --sync "$1" sh -c 'stat -c %t.%T /dev/loop8')
++    [[ "$output" != "7.8" ]] && msg_err "${FUNCNAME[0]}:${LINENO} - device nodes check failed" && return ${FAILURE}
++    output=$(crictl exec --sync "$1" sh -c 'stat -c %t.%T /dev/loop8c')
++    [[ "$output" != "7.b" ]] && msg_err "${FUNCNAME[0]}:${LINENO} - device nodes check failed" && return ${FAILURE}
++
++    # check mounts
++    output=$(crictl exec --sync "$1" sh -c 'stat -c %a /tmp/cdi_mounts_test_loop8')
++    [[ "$output" != "755" ]] && msg_err "${FUNCNAME[0]}:${LINENO} - mount check failed" && return ${FAILURE}
++
++    return 0
++}
++
++function verify_injected_loop9() {
++    # check env
++    output=$(crictl exec --sync "$1" sh -c 'echo $LOOP9')
++    [[ "$output" != "present" ]] && msg_err "${FUNCNAME[0]}:${LINENO} - env check failed" && return ${FAILURE}
++
++    # check device nodes
++    output=$(crictl exec --sync "$1" sh -c 'stat -c %a /dev/loop9')
++    [[ "$output" != "644" ]] && msg_err "${FUNCNAME[0]}:${LINENO} - device nodes check failed" && return ${FAILURE}
++    output=$(crictl exec --sync "$1" sh -c 'stat -c %t.%T /dev/loop9')
++    [[ "$output" != "7.9" ]] && msg_err "${FUNCNAME[0]}:${LINENO} - device nodes check failed" && return ${FAILURE}
++
++    return 0
++}
++
++function check_full_cdi()
++{
++    verify_injected_vendor0 $1
++    [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - verify_injected_vendor0 failed" && return ${FAILURE}
++    
++    verify_injected_loop8 $1
++    [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - verify_injected_loop8 failed" && return ${FAILURE}
++    
++    verify_injected_loop9 $1
++    [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - verify_injected_loop9 failed" && return ${FAILURE}
++
++    return 0
++}
++
++function do_test_help()
++{
++    msg_info "cdi test starting..."
++
++    isula rm -f `isula ps -a -q`
++
++    sid=`crictl runp ${data_path}/$1`
++    [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - Failed to run sandbox" && return ${FAILURE}
++
++    cid=`crictl create $sid ${data_path}/$2 ${data_path}/$1`
++    [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - create container failed" && return ${FAILURE}
++
++    crictl start $cid
++    [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - start container failed" && return ${FAILURE}
++
++    crictl stats
++    [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - stats container failed" && return ${FAILURE}
++
++    check_full_cdi $cid
++    [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - check cdi failed" && return ${FAILURE}
++
++    crictl stop $cid
++    [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - stop container failed" && return ${FAILURE}
++
++    crictl rm $cid
++    [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - rm container failed" && return ${FAILURE}
++
++    crictl stopp $sid
++    [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - stop sandbox failed" && return ${FAILURE}
++
++    crictl rmp $sid
++    [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - rm sandbox failed" && return ${FAILURE}
++
++    return 0
++}
++
++function do_test_full_cdi()
++{
++    if [ ! -d "$cdi_static_dir" ]; then
++        mkdir -p "$cdi_static_dir"
++    fi
++    cp -f ${pause_img_path}/cdi_devices.json ${cdi_static_dir}/cdi_devices.json
++
++    mkdir -p /tmp/cdi_mounts_test
++    cat  > /tmp/cdi_mounts_test_loop8  << EOF
++origin data
++EOF
++    chmod  755  /tmp/cdi_mounts_test_loop8
++    mkdir -p /tmp/cdi_mounts_test_loop9
++    
++    mknod /dev/loop8 b 7 8
++    mknod /dev/loop9 b 7 9
++    mknod /dev/loop8c c 7 11
++
++    cat  > /tmp/cdi_printargs.sh  << EOF
++#!/bin/bash
++echo "\$(date +'%Y-%m-%d %H:%M:%S') Input parameter: \$1 \$2" >> /tmp/cdi_hook_test.log
++EOF
++    chmod  755  /tmp/cdi_printargs.sh
++
++    do_test_help "sandbox-config.json" "container-config-cdi.json" || ((ans++))
++
++    rm -f /tmp/cdi_printargs.sh
++    rm -f /tmp/cdi_hook_test.log
++    rm -f /dev/loop8
++    rm -f /dev/loop9
++    rm -f /dev/loop8c
++
++    rm -f ${cdi_static_dir}/cdi_devices.json
++    rm -f /tmp/cdi_printargs
++    rmdir /tmp/cdi_mounts_test
++    rm -f /tmp/cdi_mounts_test_loop8
++    rmdir /tmp/cdi_mounts_test_loop9
++    rm -f /tmp/cdi_printargs.sh
++
++    return 0
++}
++
++declare -i ans=0
++
++do_pre || ((ans++))
++do_test_full_cdi || ((ans++))
++do_post
++
++show_result ${ans} "${curr_path}/${0}"
+diff --git a/CI/test_cases/container_cases/criconfigs/container-config-cdi.json b/CI/test_cases/container_cases/criconfigs/container-config-cdi.json
+new file mode 100644
+index 00000000..b9805c8a
+--- /dev/null
++++ b/CI/test_cases/container_cases/criconfigs/container-config-cdi.json
+@@ -0,0 +1,50 @@
++{
++    "metadata": {
++        "name": "haozi"
++    },
++    "image":{
++        "image": "busybox:latest"
++    },
++    "command": [
++        "/bin/sh",
++        "-c",
++        "i=0; while true; do echo \"$i: $(date)\"; i=$((i+1)); sleep 10; done"
++    ],
++    "labels": {
++        "filter_label_key": "filter_label_val"
++    },
++    "annotations": {
++        "extension.network.kubernetes.io/cni/instancename": "pod_instance_name",
++        "cdi.k8s.io/test": "vendor0.com/device=loop8,vendor0.com/device=loop9"
++    },
++    "CDI_Devices":[
++        { "Name": "vendor0.com/device=loop8" },
++        { "Name": "vendor0.com/device=loop9" } 
++    ],
++    "mounts" : [
++        {
++            "container_path": "/tmp/contpath",
++            "host_path": "/tmp/hostpath",
++            "readonly": true,
++            "selinux_relabel": true,
++            "propagation": 0
++        },
++        {
++            "container_path": "/tmp/contpath2",
++            "host_path": "/tmp/hostpath2",
++            "readonly": false,
++            "selinux_relabel": false,
++            "propagation": 0
++        }
++    ],
++    "linux": {
++	    "security_context": {
++		    "namespace_options": {
++		        "host_network": true,
++		        "host_pid": false,
++		        "host_ipc": false
++		    }
++	    }
++    },
++    "log_path": "cri_test.log"
++}
+diff --git a/CI/test_cases/container_cases/test_data/cdi_devices.json b/CI/test_cases/container_cases/test_data/cdi_devices.json
+new file mode 100644
+index 00000000..f7fe65d4
+--- /dev/null
++++ b/CI/test_cases/container_cases/test_data/cdi_devices.json
+@@ -0,0 +1,53 @@
++{
++  "cdiVersion": "0.6.0",
++  "kind": "vendor0.com/device",
++  "annotations":{
++      "cdi_annotation":"cdi_annotation_value"
++  },
++  "devices": [
++    {
++      "name": "loop8",
++      "annotations":{
++          "loop8_annotation":"loop8_annotation_value"
++      },
++      "containerEdits": {
++          "env": [
++              "LOOP8=CDI8",
++              "PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/dev"
++          ],
++          "deviceNodes": [
++              {"path": "/dev/loop8", "type": "b", "fileMode": 416},
++              {"path": "/dev/loop8c", "type": "c"}
++          ],
++          "mounts": [
++              {"hostPath": "/tmp/cdi_mounts_test_loop8", "containerPath": "/tmp/cdi_mounts_test_loop8", "options": ["ro","nosuid","nodev","bind"]},
++              {"hostPath": "tmpfs", "containerPath": "/tmp/data", "type": "tmpfs", "options": ["nosuid","strictatime","mode=755","size=65536k"]}
++          ]
++      }
++    },{
++      "name": "loop9",
++      "containerEdits": {
++          "env":["LOOP9=present"],
++          "deviceNodes": [{
++                  "path": "/dev/loop9", "type": "b", "major": 7, "minor": 9, "fileMode": 420
++              }
++          ]
++      }
++    }
++  ],
++  "containerEdits": {
++    "env": [
++      "VENDOR0=injected",
++      "BAR=BARVALUE1"
++    ],
++    "hooks": [
++      {"hookName": "prestart", "path": "/tmp/cdi_printargs.sh", "args":["prestart0", "prestart1"], "env":["prestartenv=value"]},
++      {"hookName": "prestart", "path": "/tmp/cdi_printargs.sh", "args":["prestart0", "prestart1"], "env":["prestartenv=value"]},
++      {"hookName": "poststart", "path": "/tmp/cdi_printargs.sh", "args":["poststart0", "poststart1"], "env":["poststartenv=value"]},
++      {"hookName": "poststop", "path": "/tmp/cdi_printargs.sh", "args":["poststop0", "poststop1"], "env":["poststopenv=value"]}
++    ],
++    "mounts": [
++      {"hostPath": "/tmp/cdi_mounts_test", "containerPath": "/tmp/cdi_mounts_test", "options": ["ro","nosuid","nodev","bind"]}
++    ]
++  }
++}
+\ No newline at end of file
+-- 
+2.25.1
+

0088-testcase-close-cdi-testcase.patch

@@ -0,0 +1,29 @@
+From eba353bef72bf62cd47f1b03a9fbd4c621ad479e Mon Sep 17 00:00:00 2001
+From: liuxu <liuxu156@huawei.com>
+Date: Thu, 16 May 2024 18:00:01 +0800
+Subject: [PATCH 088/149] testcase:close cdi testcase
+
+Signed-off-by: liuxu <liuxu156@huawei.com>
+---
+ CI/test_cases/container_cases/cdi_test.sh | 6 +++---
+ 1 file changed, 3 insertions(+), 3 deletions(-)
+
+diff --git a/CI/test_cases/container_cases/cdi_test.sh b/CI/test_cases/container_cases/cdi_test.sh
+index dd7b1177..f9fd4567 100755
+--- a/CI/test_cases/container_cases/cdi_test.sh
++++ b/CI/test_cases/container_cases/cdi_test.sh
+@@ -198,8 +198,8 @@ EOF
+ 
+ declare -i ans=0
+ 
+-do_pre || ((ans++))
+-do_test_full_cdi || ((ans++))
+-do_post
++# do_pre || ((ans++))
++# do_test_full_cdi || ((ans++))
++# do_post
+ 
+ show_result ${ans} "${curr_path}/${0}"
+-- 
+2.25.1
+

0089-docs-update-cni-doc.patch

@@ -0,0 +1,68 @@
+From 7fc8578097b9f8254962dc4fb277492b3251e5cb Mon Sep 17 00:00:00 2001
+From: liuxu <liuxu156@huawei.com>
+Date: Thu, 16 May 2024 17:56:08 +0800
+Subject: [PATCH 089/149] docs:update cni doc
+
+Signed-off-by: liuxu <liuxu156@huawei.com>
+---
+ README.md                | 2 +-
+ README_zh.md             | 2 +-
+ docs/design/README.md    | 2 ++
+ docs/design/README_zh.md | 2 ++
+ 4 files changed, 6 insertions(+), 2 deletions(-)
+
+diff --git a/README.md b/README.md
+index 694ddbc2..7ddc62ab 100644
+--- a/README.md
++++ b/README.md
+@@ -228,7 +228,7 @@ Using [ptcr](https://gitee.com/openeuler/ptcr) as a performance test tool , it s
+ The standard specification versions that `iSulad` is compatible with are as follows:
+ 
+ - Compatible with OCI 1.0.0.
+-- Compatible with CNI 0.3.0 and above.
++- Compatible with CNI 0.3.0 - 1.0.0, iSulad supports CNI 1.0.0 from 2.1.4 version.
+ - Compatible with lcr 2.1.x and above.
+ 
+ ## Kubernetes Support
+diff --git a/README_zh.md b/README_zh.md
+index 5db28f3a..45ac66ac 100755
+--- a/README_zh.md
++++ b/README_zh.md
+@@ -224,7 +224,7 @@ $ sudo isula rm test
+ `iSulad` 能够兼容的标准规范版本如下：
+ 
+ - 兼容 1.0.0 版本的OCI
+-- 兼容 0.3.0 版本以上的CNI
++- 兼容 0.3.0-1.0.0 版本的CNI，iSulad从2.1.4版本后支持 CNI 1.0.0版本
+ - 兼容 2.1.x 版本以上的lcr
+ 
+ ## Kubernetes Support
+diff --git a/docs/design/README.md b/docs/design/README.md
+index d2a3702d..c171cb20 100644
+--- a/docs/design/README.md
++++ b/docs/design/README.md
+@@ -43,6 +43,8 @@ This section contains some design documents for users who want to learn more abo
+ 
+ - You can see how the cni operator modules are designed in [cni_operator_design](./detailed/Network/cni_operator_design.md).
+ 
++- You can see how the cni operator modules update to CNI v1.0.0 in [cni_1.0.0_change](./detailed/Network/cni_1.0.0_change.md)。
++
+ - You can see how the CRI adapter modules are designed in [CRI_adapter_design](./detailed/Network/CRI_adapter_design.md).
+ 
+ - You can see how the native network adapter modules are designed in [native_network_adapter_design](./detailed/Network/native_network_adapter_design.md).
+diff --git a/docs/design/README_zh.md b/docs/design/README_zh.md
+index c6172b6f..0f4cf13e 100644
+--- a/docs/design/README_zh.md
++++ b/docs/design/README_zh.md
+@@ -49,6 +49,8 @@
+ 
+ - 查看 cni operator 模块的设计文档： [cni_operator_design](./detailed/Network/cni_operator_design_zh.md) 。
+ 
++- 查看 cni operator 模块升级到CNI v1.0.0的设计文档： [cni_1.0.0_change](./detailed/Network/cni_1.0.0_change.md) 。
++
+ - 查看 CRI adapter 模块的设计文档： [CRI_adapter_design](./detailed/Network/CRI_adapter_design_zh.md) 。
+ 
+ - 查看 native network adapter 模块的设计文档： [native_network_adapter_design](./detailed/Network/native_network_adapter_design_zh.md) 。
+-- 
+2.25.1
+

0090-modify-the-user-error-log-to-be-the-same-as-before.patch

@@ -0,0 +1,100 @@
+From 3b0f34c7cd55686cf18f65efbdc0be8a84f13e3e Mon Sep 17 00:00:00 2001
+From: zhongtao <zhongtao17@huawei.com>
+Date: Mon, 20 May 2024 17:54:04 +1400
+Subject: [PATCH 090/149] modify the user error log to be the same as before
+
+Signed-off-by: zhongtao <zhongtao17@huawei.com>
+---
+ src/daemon/common/id_name_manager.c           |  4 +--
+ .../executor/container_cb/execution_create.c  | 26 +++++++++++--------
+ .../container_cb/execution_information.c      |  4 ++-
+ 3 files changed, 20 insertions(+), 14 deletions(-)
+
+diff --git a/src/daemon/common/id_name_manager.c b/src/daemon/common/id_name_manager.c
+index 263a584d..f64094b9 100644
+--- a/src/daemon/common/id_name_manager.c
++++ b/src/daemon/common/id_name_manager.c
+@@ -242,7 +242,7 @@ static bool try_add_name(const char *name)
+     }
+ 
+     if (!util_valid_container_name(name)) {
+-        ERROR("Failed to add invalid name: %s", name);
++        ERROR("Invalid container name (%s), only [a-zA-Z0-9][a-zA-Z0-9_.-]+$ are allowed.", name);
+         return false;
+     }
+ 
+@@ -262,7 +262,7 @@ static bool try_remove_name(const char *name)
+     }
+ 
+     if (!util_valid_container_name(name)) {
+-        ERROR("Failed to remove invalid name: %s", name);
++        ERROR("Invalid container name (%s), only [a-zA-Z0-9][a-zA-Z0-9_.-]+$ are allowed.", name);
+         return false;
+     }
+ 
+diff --git a/src/daemon/executor/container_cb/execution_create.c b/src/daemon/executor/container_cb/execution_create.c
+index 785b4e27..041089dd 100644
+--- a/src/daemon/executor/container_cb/execution_create.c
++++ b/src/daemon/executor/container_cb/execution_create.c
+@@ -761,8 +761,17 @@ static int maintain_container_id(const container_create_request *request, char *
+ #endif
+ 
+     if (!nret) {
+-        ERROR("Failed to add entry to id name manager with new id and name");
+-        isulad_set_error_message("Failed to add entry to id name manager with new id and name");
++        __isula_auto_free char *used_id = NULL;
++        used_id = container_name_index_get(name);
++        if(used_id != NULL) {
++            ERROR("Name %s is in use by container %s", name, used_id);
++            isulad_set_error_message("Conflict. The name \"%s\" is already in use by container %s. "
++                                    "You have to remove (or rename) that container to be able to reuse that name.",
++                                    name, used_id);
++        } else {
++            ERROR("Failed to add entry to id name manager with new id and name");
++            isulad_set_error_message("Failed to add entry to id name manager with new id and name");
++        }
+         ret = -1;
+         goto out;
+     }
+@@ -775,19 +784,14 @@ static int maintain_container_id(const container_create_request *request, char *
+         goto out;
+     }
+ 
+-    char *used_id = NULL;
+-    used_id = container_name_index_get(name);
+-    ERROR("Name %s is in use by container %s", name, used_id);
+-    isulad_set_error_message("Conflict. The name \"%s\" is already in use by container %s. "
+-                             "You have to remove (or rename) that container to be able to reuse that name.",
+-                             name, used_id);
+-    free(used_id);
+-    used_id = NULL;
+-    ret = -1;
+     if (!skip_id_name_manage && !id_name_manager_remove_entry(id, name)) {
+         WARN("Failed to remove %s and %s from id name manager", id, name);
+     }
+ 
++    ERROR("Failed to add %s to container name index", name);
++    isulad_set_error_message("Failed to add %s to container name index", name);
++    ret = -1;
++
+ out:
+     *out_id = id;
+     *out_name = name;
+diff --git a/src/daemon/executor/container_cb/execution_information.c b/src/daemon/executor/container_cb/execution_information.c
+index c02cc830..58924257 100644
+--- a/src/daemon/executor/container_cb/execution_information.c
++++ b/src/daemon/executor/container_cb/execution_information.c
+@@ -1149,7 +1149,9 @@ static int container_rename(container_t *cont, const char *new_name)
+ 
+     if (!id_name_manager_rename(new_name, old_name)) {
+         ERROR("Failed to rename %s to %s in id-name manager", old_name, new_name);
+-        isulad_set_error_message("Failed to rename %s to %s in id-name manager", old_name, new_name);
++        isulad_set_error_message("Conflict. The name \"%s\" is already in use by container %s. "
++                                 "You have to remove (or rename) that container to be able to reuse that name.",
++                                 new_name, new_name);
+         ret = -1;
+         goto out;
+     }
+-- 
+2.25.1
+

0091-add-enable-cri-v1-in-k8s-integration.patch

@@ -0,0 +1,54 @@
+From 1f69ffe589f7225a1db83377e276ddbab963bd16 Mon Sep 17 00:00:00 2001
+From: jikai <jikai11@huawei.com>
+Date: Tue, 21 May 2024 01:13:08 +0000
+Subject: [PATCH 091/149] add enable cri v1 in k8s integration
+
+Signed-off-by: jikai <jikai11@huawei.com>
+---
+ docs/manual/k8s_integration.md    | 9 +++++++++
+ docs/manual/k8s_integration_zh.md | 9 +++++++++
+ 2 files changed, 18 insertions(+)
+
+diff --git a/docs/manual/k8s_integration.md b/docs/manual/k8s_integration.md
+index 8fcd0a54..14de0ef4 100644
+--- a/docs/manual/k8s_integration.md
++++ b/docs/manual/k8s_integration.md
+@@ -20,6 +20,15 @@
+ 
+    if `hosts` is not configured, the default endpoint is `unix:///var/run/isulad.sock`.
+ 
++   `iSulad` supports both `CRI V1alpha2` and `CRI V1`, and uses `CRI V1alph2` by default.
++   If `CRI V1` is required, it can be configured in `/etc/isulad/daemon.json` to enable `CRI V1`:
++
++   ```json
++       "enable-cri-v1": true,
++   ```
++
++   If `iSulad` is compiled from source codes, `-D ENABLE_CRI_API_V1=ON` option is required in cmake.
++
+ 2. Restart `isulad`:
+ 
+    ```bash
+diff --git a/docs/manual/k8s_integration_zh.md b/docs/manual/k8s_integration_zh.md
+index 6dda1e4d..26ba6cc4 100644
+--- a/docs/manual/k8s_integration_zh.md
++++ b/docs/manual/k8s_integration_zh.md
+@@ -20,6 +20,15 @@
+ 
+    如果`hosts`没有配置，默认的`endpoint`为``unix:///var/run/isulad.sock``
+ 
++   `iSulad`同时支持`CRI V1alpha2`和`CRI V1`两种`CRI`接口，默认使用`CRI V1alph2`，若需使用`CRI V1`，
++   需要在`/etc/isulad/daemon.json`对`iSulad`进行相关配置，配置方式为：
++
++   ```json
++       "enable-cri-v1": true,
++   ```
++
++   若使用源码编译`iSulad`，还需在编译时增加cmake编译选项`-D ENABLE_CRI_API_V1=ON`。
++
+ 2. 重启`isulad`
+ 
+    ```bash
+-- 
+2.25.1
+

0092-isolate-oom-monitor-codes.patch

@@ -0,0 +1,317 @@
+From d97656a8b99f4fa95a9c15abfbac777a94b84d55 Mon Sep 17 00:00:00 2001
+From: jikai <jikai11@huawei.com>
+Date: Mon, 20 May 2024 08:48:00 +0000
+Subject: [PATCH 092/149] isolate oom monitor codes
+
+Signed-off-by: jikai <jikai11@huawei.com>
+---
+ cmake/options.cmake                                |  7 +++++++
+ src/daemon/common/cgroup/cgroup.c                  |  2 ++
+ src/daemon/common/cgroup/cgroup.h                  |  2 ++
+ src/daemon/common/cgroup/cgroup_common.h           |  4 ++++
+ src/daemon/common/cgroup/cgroup_v1.c               |  8 ++++++++
+ src/daemon/common/cgroup/cgroup_v2.c               |  8 ++++++++
+ src/daemon/common/cri/v1/v1_cri_helpers.cc         |  2 ++
+ src/daemon/modules/container/container_state.c     |  2 ++
+ .../modules/container/supervisor/supervisor.c      | 14 ++++++++++++++
+ 9 files changed, 49 insertions(+)
+
+diff --git a/cmake/options.cmake b/cmake/options.cmake
+index a15b8194..5b17f631 100644
+--- a/cmake/options.cmake
++++ b/cmake/options.cmake
+@@ -58,6 +58,13 @@ if (ENABLE_SANDBOXER STREQUAL "ON")
+     message("${Green}--  Enable sandbox API${ColourReset}")
+ endif()
+ 
++option(ENABLE_OOM_MONITOR "Enable oom monitor" ON)
++IF (ENABLE_OOM_MONITOR STREQUAL "ON")
++    add_definitions(-DENABLE_OOM_MONITOR)
++    set(ENABLE_OOM_MONITOR 1)
++    message("${Green}--  Enable oom monitor${ColourReset}")
++endif()
++
+ option(ENABLE_SYSTEMD_NOTIFY "Enable systemd notify" ON)
+ if (ENABLE_SYSTEMD_NOTIFY STREQUAL "ON")
+     add_definitions(-DSYSTEMD_NOTIFY)
+diff --git a/src/daemon/common/cgroup/cgroup.c b/src/daemon/common/cgroup/cgroup.c
+index 71bf9801..77fafdae 100644
+--- a/src/daemon/common/cgroup/cgroup.c
++++ b/src/daemon/common/cgroup/cgroup.c
+@@ -197,6 +197,7 @@ char *common_convert_cgroup_path(const char *cgroup_path)
+     return util_strdup_s(result);
+ }
+ 
++#ifdef ENABLE_OOM_MONITOR
+ cgroup_oom_handler_info_t *common_get_cgroup_oom_handler(int fd, const char *name, const char *cgroup_path,
+                                                          const char *exit_fifo)
+ {
+@@ -225,3 +226,4 @@ void common_free_cgroup_oom_handler_info(cgroup_oom_handler_info_t *info)
+     free(info->cgroup_memory_event_path);
+     free(info);
+ }
++#endif
+diff --git a/src/daemon/common/cgroup/cgroup.h b/src/daemon/common/cgroup/cgroup.h
+index 0bbb70a0..1ebbfa98 100644
+--- a/src/daemon/common/cgroup/cgroup.h
++++ b/src/daemon/common/cgroup/cgroup.h
+@@ -43,9 +43,11 @@ char *common_get_own_cgroup_path(const char *subsystem);
+ 
+ char *common_convert_cgroup_path(const char *cgroup_path);
+ 
++#ifdef ENABLE_OOM_MONITOR
+ cgroup_oom_handler_info_t *common_get_cgroup_oom_handler(int fd, const char *name, const char *cgroup_path,
+                                                          const char *exit_fifo);
+ void common_free_cgroup_oom_handler_info(cgroup_oom_handler_info_t *info);
++#endif
+ 
+ #ifdef __cplusplus
+ }
+diff --git a/src/daemon/common/cgroup/cgroup_common.h b/src/daemon/common/cgroup/cgroup_common.h
+index 46a7de50..01fc669c 100644
+--- a/src/daemon/common/cgroup/cgroup_common.h
++++ b/src/daemon/common/cgroup/cgroup_common.h
+@@ -116,6 +116,7 @@ typedef struct {
+     cgroup_pids_metrics_t cgpids_metrics;
+ } cgroup_metrics_t;
+ 
++#ifdef ENABLE_OOM_MONITOR
+ #define CGROUP_OOM_HANDLE_CONTINUE false
+ #define CGROUP_OOM_HANDLE_CLOSE true
+ 
+@@ -126,6 +127,7 @@ typedef struct _cgroup_oom_handler_info_t {
+     char *cgroup_memory_event_path;
+     bool (*oom_event_handler)(int, void *);
+ } cgroup_oom_handler_info_t;
++#endif
+ 
+ typedef struct {
+     int (*get_cgroup_version)(void);
+@@ -140,8 +142,10 @@ typedef struct {
+     char *(*get_init_cgroup_path)(const char *subsystem);
+     char *(*get_own_cgroup_path)(const char *subsystem);
+ 
++#ifdef ENABLE_OOM_MONITOR
+     cgroup_oom_handler_info_t *(*get_cgroup_oom_handler)(int fd, const char *name, const char *cgroup_path,
+                                                          const char *exit_fifo);
++#endif
+ } cgroup_ops;
+ 
+ #ifdef __cplusplus
+diff --git a/src/daemon/common/cgroup/cgroup_v1.c b/src/daemon/common/cgroup/cgroup_v1.c
+index 45b1d096..018336ea 100644
+--- a/src/daemon/common/cgroup/cgroup_v1.c
++++ b/src/daemon/common/cgroup/cgroup_v1.c
+@@ -20,12 +20,16 @@
+ 
+ #include <stdio.h>
+ #include <stdlib.h>
++#ifdef ENABLE_OOM_MONITOR
+ #include <sys/eventfd.h>
++#endif
+ 
+ #include "utils.h"
+ #include "sysinfo.h"
+ #include "err_msg.h"
++#ifdef ENABLE_OOM_MONITOR
+ #include "events_sender_api.h"
++#endif
+ 
+ #define CGROUP_HUGETLB_LIMIT "hugetlb.%s.limit_in_bytes"
+ #define CGROUP_MOUNT_PATH_PREFIX "/sys/fs/cgroup/"
+@@ -1052,6 +1056,7 @@ static char *common_get_cgroup_path(const char *path, const char *subsystem)
+     return res;
+ }
+ 
++#ifdef ENABLE_OOM_MONITOR
+ static bool oom_cb_cgroup_v1(int fd, void *cbdata)
+ {
+     cgroup_oom_handler_info_t *info = (cgroup_oom_handler_info_t *)cbdata;
+@@ -1205,6 +1210,7 @@ cleanup:
+     common_free_cgroup_oom_handler_info(info);
+     return NULL;
+ }
++#endif
+ 
+ char *get_init_cgroup_path_v1(const char *subsystem)
+ {
+@@ -1232,6 +1238,8 @@ int cgroup_v1_ops_init(cgroup_ops *ops)
+     ops->get_cgroup_mnt_and_root_path = get_cgroup_mnt_and_root_path_v1;
+     ops->get_init_cgroup_path = get_init_cgroup_path_v1;
+     ops->get_own_cgroup_path = get_own_cgroup_v1;
++#ifdef ENABLE_OOM_MONITOR
+     ops->get_cgroup_oom_handler = get_cgroup_oom_handler_v1;
++#endif
+     return 0;
+ }
+\ No newline at end of file
+diff --git a/src/daemon/common/cgroup/cgroup_v2.c b/src/daemon/common/cgroup/cgroup_v2.c
+index 76754dc1..ce72e6c4 100644
+--- a/src/daemon/common/cgroup/cgroup_v2.c
++++ b/src/daemon/common/cgroup/cgroup_v2.c
+@@ -17,14 +17,18 @@
+ #include <stdio.h>
+ #include <stdlib.h>
+ #include <sys/stat.h>
++#ifdef ENABLE_OOM_MONITOR
+ #include <sys/inotify.h>
++#endif
+ 
+ #include <isula_libutils/auto_cleanup.h>
+ 
+ #include "utils.h"
+ #include "path.h"
+ #include "sysinfo.h"
++#ifdef ENABLE_OOM_MONITOR
+ #include "events_sender_api.h"
++#endif
+ 
+ // Cgroup V2 Item Definition
+ #define CGROUP2_CPU_WEIGHT "cpu.weight"
+@@ -416,6 +420,7 @@ static int get_cgroup_mnt_and_root_v2(const char *subsystem, char **mountpoint,
+     return 0;
+ }
+ 
++#ifdef ENABLE_OOM_MONITOR
+ static bool oom_cb_cgroup_v2(int fd, void *cbdata)
+ {
+     const size_t events_size = sizeof(struct inotify_event) + NAME_MAX + 1;
+@@ -547,6 +552,7 @@ cleanup:
+     common_free_cgroup_oom_handler_info(info);
+     return NULL;
+ }
++#endif
+ 
+ int get_cgroup_version_v2()
+ {
+@@ -562,6 +568,8 @@ int cgroup_v2_ops_init(cgroup_ops *ops)
+     ops->get_cgroup_info = get_cgroup_info_v2;
+     ops->get_cgroup_metrics = get_cgroup_metrics_v2;
+     ops->get_cgroup_mnt_and_root_path = get_cgroup_mnt_and_root_v2;
++#ifdef ENABLE_OOM_MONITOR
+     ops->get_cgroup_oom_handler = get_cgroup_oom_handler_v2;
++#endif
+     return 0;
+ }
+\ No newline at end of file
+diff --git a/src/daemon/common/cri/v1/v1_cri_helpers.cc b/src/daemon/common/cri/v1/v1_cri_helpers.cc
+index 478dd105..31b6b137 100644
+--- a/src/daemon/common/cri/v1/v1_cri_helpers.cc
++++ b/src/daemon/common/cri/v1/v1_cri_helpers.cc
+@@ -517,9 +517,11 @@ void UpdateBaseStatusFromInspect(
+         } else { // Case 3
+             state = runtime::v1::CONTAINER_CREATED;
+         }
++#ifdef ENABLE_OOM_MONITOR
+         if (inspect->state->oom_killed == true) {
+             reason = "OOMKilled";
+         }
++#endif
+         if (inspect->state->error != nullptr) {
+             message = inspect->state->error;
+         }
+diff --git a/src/daemon/modules/container/container_state.c b/src/daemon/modules/container/container_state.c
+index 452a2b26..f8ad0537 100644
+--- a/src/daemon/modules/container/container_state.c
++++ b/src/daemon/modules/container/container_state.c
+@@ -587,7 +587,9 @@ container_inspect_state *container_state_to_inspect_state(container_state_t *s)
+     state->running = s->state->running;
+     state->paused = s->state->paused;
+     state->restarting = s->state->restarting;
++#ifdef ENABLE_OOM_MONITOR
+     state->oom_killed = s->state->oom_killed;
++#endif
+     state->pid = s->state->pid;
+ 
+     state->exit_code = s->state->exit_code;
+diff --git a/src/daemon/modules/container/supervisor/supervisor.c b/src/daemon/modules/container/supervisor/supervisor.c
+index 39d9fdb8..294783eb 100644
+--- a/src/daemon/modules/container/supervisor/supervisor.c
++++ b/src/daemon/modules/container/supervisor/supervisor.c
+@@ -42,8 +42,10 @@
+ #ifdef ENABLE_CRI_API_V1
+ #include "sandbox_ops.h"
+ #endif
++#ifdef ENABLE_OOM_MONITOR
+ #include "cgroup.h"
+ #include "specs_api.h"
++#endif
+ 
+ pthread_mutex_t g_supervisor_lock = PTHREAD_MUTEX_INITIALIZER;
+ struct epoll_descr g_supervisor_descr;
+@@ -286,6 +288,7 @@ static int supervisor_exit_cb(int fd, uint32_t events, void *cbdata, struct epol
+     return EPOLL_LOOP_HANDLE_CONTINUE;
+ }
+ 
++#ifdef ENABLE_OOM_MONITOR
+ static int oom_handle_cb(int fd, uint32_t events, void *cbdata, struct epoll_descr *descr)
+ {
+     cgroup_oom_handler_info_t *oom_handler_info = (cgroup_oom_handler_info_t *)cbdata;
+@@ -305,6 +308,7 @@ static int oom_handle_cb(int fd, uint32_t events, void *cbdata, struct epoll_des
+ 
+     return EPOLL_LOOP_HANDLE_CONTINUE;
+ }
++#endif
+ 
+ /* supervisor add exit monitor */
+ int container_supervisor_add_exit_monitor(int fd, const char *exit_fifo, const pid_ppid_info_t *pid_info,
+@@ -312,8 +316,10 @@ int container_supervisor_add_exit_monitor(int fd, const char *exit_fifo, const p
+ {
+     int ret = 0;
+     struct supervisor_handler_data *data = NULL;
++#ifdef ENABLE_OOM_MONITOR
+     cgroup_oom_handler_info_t *oom_handler_info = NULL;
+     __isula_auto_free char *cgroup_path = NULL;
++#endif
+ 
+     if (fd < 0) {
+         ERROR("Invalid exit fifo fd");
+@@ -326,12 +332,14 @@ int container_supervisor_add_exit_monitor(int fd, const char *exit_fifo, const p
+         return -1;
+     }
+ 
++#ifdef ENABLE_OOM_MONITOR
+     cgroup_path = merge_container_cgroups_path(cont->common_config->id, cont->hostconfig);
+     if (cgroup_path == NULL) {
+         ERROR("Failed to get cgroup path");
+         close(fd);
+         return -1;
+     }
++#endif
+ 
+     data = util_common_calloc_s(sizeof(struct supervisor_handler_data));
+     if (data == NULL) {
+@@ -353,9 +361,12 @@ int container_supervisor_add_exit_monitor(int fd, const char *exit_fifo, const p
+     data->pid_info.start_time = pid_info->start_time;
+     data->pid_info.ppid = pid_info->ppid;
+     data->pid_info.pstart_time = pid_info->pstart_time;
++#ifdef ENABLE_OOM_MONITOR
+     oom_handler_info = common_get_cgroup_oom_handler(fd, cont->common_config->id, cgroup_path, exit_fifo);
++#endif
+ 
+     supervisor_handler_lock();
++#ifdef ENABLE_OOM_MONITOR
+     if (oom_handler_info != NULL) {
+         ret = epoll_loop_add_handler(&g_supervisor_descr, oom_handler_info->oom_event_fd, oom_handle_cb, oom_handler_info);
+         if (ret != 0) {
+@@ -363,6 +374,7 @@ int container_supervisor_add_exit_monitor(int fd, const char *exit_fifo, const p
+             goto err;
+         }
+     }
++#endif
+ 
+     ret = epoll_loop_add_handler(&g_supervisor_descr, fd, supervisor_exit_cb, data);
+     if (ret != 0) {
+@@ -374,7 +386,9 @@ int container_supervisor_add_exit_monitor(int fd, const char *exit_fifo, const p
+ 
+ err:
+     supervisor_handler_data_free(data);
++#ifdef ENABLE_OOM_MONITOR
+     common_free_cgroup_oom_handler_info(oom_handler_info);
++#endif
+ out:
+     supervisor_handler_unlock();
+     return ret;
+-- 
+2.25.1
+

0093-change-fork-process-exit-mode.patch

@@ -0,0 +1,64 @@
+From 0ff5a421e31096fbd10cf00c45a3849297023391 Mon Sep 17 00:00:00 2001
+From: zhongtao <zhongtao17@huawei.com>
+Date: Thu, 23 May 2024 01:09:41 +1400
+Subject: [PATCH 093/149] change fork process exit mode
+
+---
+ src/utils/tar/util_archive.c | 16 ++++++++--------
+ 1 file changed, 8 insertions(+), 8 deletions(-)
+
+diff --git a/src/utils/tar/util_archive.c b/src/utils/tar/util_archive.c
+index 52b51162..204dab83 100644
+--- a/src/utils/tar/util_archive.c
++++ b/src/utils/tar/util_archive.c
+@@ -897,9 +897,9 @@ int archive_unpack(const struct io_read_wrapper *content, const char *dstdir, co
+ 
+ child_out:
+         if (ret != 0) {
+-            exit(EXIT_FAILURE);
++            _exit(EXIT_FAILURE);
+         }
+-        exit(EXIT_SUCCESS);
++        _exit(EXIT_SUCCESS);
+     }
+     close(pipe_stderr[1]);
+     pipe_stderr[1] = -1;
+@@ -1342,9 +1342,9 @@ int archive_chroot_tar(const char *path, const char *file, const char *root_dir,
+ child_out:
+ 
+         if (ret != 0) {
+-            exit(EXIT_FAILURE);
++            _exit(EXIT_FAILURE);
+         } else {
+-            exit(EXIT_SUCCESS);
++            _exit(EXIT_SUCCESS);
+         }
+     }
+     close(pipe_for_read[1]);
+@@ -1577,9 +1577,9 @@ int archive_chroot_untar_stream(const struct io_read_wrapper *context, const cha
+ 
+ child_out:
+         if (ret != 0) {
+-            exit(EXIT_FAILURE);
++            _exit(EXIT_FAILURE);
+         }
+-        exit(EXIT_SUCCESS);
++        _exit(EXIT_SUCCESS);
+     }
+ 
+     close(pipe_stderr[1]);
+@@ -1727,9 +1727,9 @@ child_out:
+         free(tar_base_name);
+ 
+         if (ret != 0) {
+-            exit(EXIT_FAILURE);
++            _exit(EXIT_FAILURE);
+         } else {
+-            exit(EXIT_SUCCESS);
++            _exit(EXIT_SUCCESS);
+         }
+     }
+ 
+-- 
+2.25.1
+

0094-fix-error-log-for-verify_cpu_realtime.patch

@@ -0,0 +1,26 @@
+From 1671a136d1b7d209c453a8ad2b1bf062a3afbe09 Mon Sep 17 00:00:00 2001
+From: zhongtao <zhongtao17@huawei.com>
+Date: Tue, 28 May 2024 17:23:45 +1400
+Subject: [PATCH 094/149] fix error log for verify_cpu_realtime
+
+Signed-off-by: zhongtao <zhongtao17@huawei.com>
+---
+ src/daemon/modules/spec/verify.c | 2 +-
+ 1 file changed, 1 insertion(+), 1 deletion(-)
+
+diff --git a/src/daemon/modules/spec/verify.c b/src/daemon/modules/spec/verify.c
+index 57e16ef9..1ce76c1d 100644
+--- a/src/daemon/modules/spec/verify.c
++++ b/src/daemon/modules/spec/verify.c
+@@ -390,7 +390,7 @@ static int verify_cpu_realtime(const sysinfo_t *sysinfo, int64_t realtime_period
+ 
+     if (realtime_runtime != 0 && !(sysinfo->cgcpuinfo.cpu_rt_runtime)) {
+         ERROR("Invalid --cpu-rt-runtime: Your kernel does not support cgroup rt runtime");
+-        isulad_set_error_message("Invalid --cpu-rt-period: Your kernel does not support cgroup rt runtime");
++        isulad_set_error_message("Invalid --cpu-rt-runtime: Your kernel does not support cgroup rt runtime");
+         ret = -1;
+         goto out;
+     }
+-- 
+2.25.1
+

0095-bugfix-change-max-network-name-len.patch

@@ -0,0 +1,54 @@
+From e2a7e6bfb0f0e97e5e1543fac7a5e0807fadaba0 Mon Sep 17 00:00:00 2001
+From: liuxu <liuxu156@huawei.com>
+Date: Thu, 30 May 2024 16:50:56 +0800
+Subject: [PATCH 095/149] bugfix: change max network name len
+
+Signed-off-by: liuxu <liuxu156@huawei.com>
+---
+ docs/design/detailed/Network/native_network_adapter_design.md   | 2 +-
+ .../design/detailed/Network/native_network_adapter_design_zh.md | 2 +-
+ src/utils/cutils/utils.h                                        | 2 +-
+ 3 files changed, 3 insertions(+), 3 deletions(-)
+
+diff --git a/docs/design/detailed/Network/native_network_adapter_design.md b/docs/design/detailed/Network/native_network_adapter_design.md
+index b58989eb..1010bd73 100644
+--- a/docs/design/detailed/Network/native_network_adapter_design.md
++++ b/docs/design/detailed/Network/native_network_adapter_design.md
+@@ -204,7 +204,7 @@ Client:
+ 1. Parse the parameters passed in by the user.
+ 2. Verify the incoming parameters, including:
+    - Only one network is allowed to be created at a time, that is, at most one name can be specified.
+-   - If name is specified, check whether the length of name exceeds MAX_NETWORK_NAME_LEN(128).
++   - If name is specified, check whether the length of name exceeds MAX_NETWORK_NAME_LEN(255).
+ 3. Send the request to the server
+ 
+ Server:
+diff --git a/docs/design/detailed/Network/native_network_adapter_design_zh.md b/docs/design/detailed/Network/native_network_adapter_design_zh.md
+index 30860f3d..62c4b6ef 100644
+--- a/docs/design/detailed/Network/native_network_adapter_design_zh.md
++++ b/docs/design/detailed/Network/native_network_adapter_design_zh.md
+@@ -204,7 +204,7 @@ int native_network_add_container_list(const char *network_name, const char *cont
+ 1. 解析用户传入的参数
+ 2. 对传入的参数进行校验，包括
+    - 每次只允许创建一个网络， 即最多指定一个name
+-   - 若指定name，检查name长度是否超过MAX_NETWORK_NAME_LEN(128)
++   - 若指定name，检查name长度是否超过MAX_NETWORK_NAME_LEN(255)
+ 3. 发送请求到服务端
+ 
+ 服务端：
+diff --git a/src/utils/cutils/utils.h b/src/utils/cutils/utils.h
+index 3671272a..4417a165 100644
+--- a/src/utils/cutils/utils.h
++++ b/src/utils/cutils/utils.h
+@@ -99,7 +99,7 @@ int malloc_trim(size_t pad);
+ #define MAX_IMAGE_REF_LEN 384
+ #define MAX_CONTAINER_NAME_LEN 1024
+ #define MAX_RUNTIME_NAME_LEN 32
+-#define MAX_NETWORK_NAME_LEN 128
++#define MAX_NETWORK_NAME_LEN 255
+ 
+ #define LOGIN_USERNAME_LEN 255
+ #define LOGIN_PASSWORD_LEN 255
+-- 
+2.25.1
+

0096-del-useless-info.patch

@@ -0,0 +1,26 @@
+From b36cfa4325f43b3fa1468ba360b3d51f6ef1c3ca Mon Sep 17 00:00:00 2001
+From: liuxu <liuxu156@huawei.com>
+Date: Thu, 30 May 2024 17:14:48 +0800
+Subject: [PATCH 096/149] del useless info
+
+Signed-off-by: liuxu <liuxu156@huawei.com>
+---
+ release_notes | 2 +-
+ 1 file changed, 1 insertion(+), 1 deletion(-)
+
+diff --git a/release_notes b/release_notes
+index 2d564c7e..fdcd3690 100644
+--- a/release_notes
++++ b/release_notes
+@@ -72,7 +72,7 @@
+ 
+     dev stats:
+       - 357 files changed, 7886 insertions(+), 2849 deletions(-)
+-      - contributors: zhongtao, jikai, haozi007, jake, liuxu, xuxuepeng, zhangxiaoyu, sailorvii, chen524, dreamloy, l00804245, yangjiaqi
++      - contributors: zhongtao, jikai, haozi007, jake, liuxu, xuxuepeng, zhangxiaoyu, sailorvii, chen524, dreamloy, yangjiaqi
+ 
+ 2023-11-07 xuepengxu release 2.1.4
+     - !2238 modify the default value of EANBLE_IMAGE_LIBARAY to off * modify the default value of EANBLE_IMAGE_LIBARAY to off
+-- 
+2.25.1
+

0097-code-improve.patch

@@ -0,0 +1,83 @@
+From 2aa26649f20ae5992ace7bb8cb62a5ee9c3f7a81 Mon Sep 17 00:00:00 2001
+From: zhongtao <zhongtao17@huawei.com>
+Date: Thu, 30 May 2024 21:30:43 +1400
+Subject: [PATCH 097/149] code improve
+
+Signed-off-by: zhongtao <zhongtao17@huawei.com>
+---
+ src/cmd/isula/volume/prune.c                             | 2 +-
+ src/cmd/options/opt_log.c                                | 2 +-
+ src/daemon/executor/container_cb/execution_extend.c      | 3 ++-
+ src/daemon/executor/container_cb/execution_information.c | 2 +-
+ src/daemon/modules/image/oci/registry_type.c             | 2 +-
+ 5 files changed, 6 insertions(+), 5 deletions(-)
+
+diff --git a/src/cmd/isula/volume/prune.c b/src/cmd/isula/volume/prune.c
+index c8d632ed..3b5bfbf1 100644
+--- a/src/cmd/isula/volume/prune.c
++++ b/src/cmd/isula/volume/prune.c
+@@ -79,7 +79,7 @@ out:
+ 
+ int cmd_volume_prune_main(int argc, const char **argv)
+ {
+-    int i = 0;
++    size_t i = 0;
+     struct isula_libutils_log_config lconf = { 0 };
+     int exit_code = 1;
+     command_t cmd;
+diff --git a/src/cmd/options/opt_log.c b/src/cmd/options/opt_log.c
+index b1abcfaf..8ffb9966 100644
+--- a/src/cmd/options/opt_log.c
++++ b/src/cmd/options/opt_log.c
+@@ -67,7 +67,7 @@ static int log_opt_syslog_facility(const char *key, const char *value, char **pa
+                                       "authpriv", "ftp",    "local0", "local1", "local2",
+                                       "local3",   "local4", "local5", "local6", "local7"
+                                     };
+-    int i;
++    size_t i;
+     size_t f_len = sizeof(facility_values) / sizeof(const char *);
+ 
+     for (i = 0; i < f_len; i++) {
+diff --git a/src/daemon/executor/container_cb/execution_extend.c b/src/daemon/executor/container_cb/execution_extend.c
+index 52401633..ed072848 100644
+--- a/src/daemon/executor/container_cb/execution_extend.c
++++ b/src/daemon/executor/container_cb/execution_extend.c
+@@ -990,7 +990,8 @@ out:
+ 
+ static int update_container_unified(const char *id, const host_config *hostconfig, host_config *chostconfig)
+ {
+-    int i, cgroup_version;
++    int cgroup_version;
++    size_t i;
+ 
+     if (hostconfig->unified == NULL || hostconfig->unified->len == 0) {
+         return 0;
+diff --git a/src/daemon/executor/container_cb/execution_information.c b/src/daemon/executor/container_cb/execution_information.c
+index 58924257..4aee3aef 100644
+--- a/src/daemon/executor/container_cb/execution_information.c
++++ b/src/daemon/executor/container_cb/execution_information.c
+@@ -244,7 +244,7 @@ static int get_proxy_env(char **proxy, const char *type)
+     }
+     *col_pos = '\0';
+     nret = snprintf(*proxy, proxy_len, "%s:%s%s", tmp_proxy, mask_str, at_pos);
+-    if (nret < 0 || nret >= proxy_len) {
++    if (nret < 0 || (size_t)nret >= proxy_len) {
+         ret = -1;
+         free(*proxy);
+         *proxy = NULL;
+diff --git a/src/daemon/modules/image/oci/registry_type.c b/src/daemon/modules/image/oci/registry_type.c
+index 6c9ff747..7a2c25ed 100644
+--- a/src/daemon/modules/image/oci/registry_type.c
++++ b/src/daemon/modules/image/oci/registry_type.c
+@@ -59,7 +59,7 @@ void free_layer_blob(layer_blob *layer)
+ 
+ void free_pull_desc(pull_descriptor *desc)
+ {
+-    int i = 0;
++    size_t i = 0;
+ 
+     if (desc == NULL) {
+         return;
+-- 
+2.25.1
+

0098-cdi-add-debug-info.patch

@@ -0,0 +1,33 @@
+From 5a5f4879246783932ab620b2461a7cd832ddc1f0 Mon Sep 17 00:00:00 2001
+From: liuxu <liuxu156@huawei.com>
+Date: Fri, 31 May 2024 16:38:26 +0800
+Subject: [PATCH 098/149] cdi:add debug info
+
+Signed-off-by: liuxu <liuxu156@huawei.com>
+---
+ src/daemon/modules/device/cdi/cdi_cache.c | 2 ++
+ 1 file changed, 2 insertions(+)
+
+diff --git a/src/daemon/modules/device/cdi/cdi_cache.c b/src/daemon/modules/device/cdi/cdi_cache.c
+index e9a9b804..cd7158dd 100644
+--- a/src/daemon/modules/device/cdi/cdi_cache.c
++++ b/src/daemon/modules/device/cdi/cdi_cache.c
+@@ -306,6 +306,7 @@ static void refresh_scan_spec_func(struct cdi_scan_fn_maps *scan_fn_maps, const
+             ERROR("Failed to insert device to devices by name %s", qualified);
+             goto error_out;
+         }
++        DEBUG("Add device %s into memory", qualified);
+         free(qualified);
+         qualified = NULL;
+     }
+@@ -445,6 +446,7 @@ static int cdi_inject_devices(struct cdi_cache *c, oci_runtime_spec *oci_spec, s
+ 
+     for (i = 0; i < devices->len; i++) {
+         device = devices->items[i];
++        DEBUG("Search cdi devices %s.", device);
+         d = map_search(c->devices, (void *)device);
+         if (d == NULL) {
+             if (util_append_string_array(unresolved, device) != 0) {
+-- 
+2.25.1
+

0099-bugfix-cni-network-name-UT.patch

@@ -0,0 +1,28 @@
+From 4a98535064319a9df3143d9c4b397f44fbbb56c5 Mon Sep 17 00:00:00 2001
+From: liuxu <liuxu156@huawei.com>
+Date: Mon, 3 Jun 2024 16:11:06 +0800
+Subject: [PATCH 099/149] bugfix:cni network name UT
+
+Signed-off-by: liuxu <liuxu156@huawei.com>
+---
+ test/cutils/utils_network/utils_network_ut.cc | 4 ++--
+ 1 file changed, 2 insertions(+), 2 deletions(-)
+
+diff --git a/test/cutils/utils_network/utils_network_ut.cc b/test/cutils/utils_network/utils_network_ut.cc
+index 33eb6eb5..68f6f011 100644
+--- a/test/cutils/utils_network/utils_network_ut.cc
++++ b/test/cutils/utils_network/utils_network_ut.cc
+@@ -344,8 +344,8 @@ TEST(utils_network, test_net_contain_ip)
+ TEST(utils_network, test_validate_network_name)
+ {
+     ASSERT_EQ(util_validate_network_name(nullptr), false);
+-    ASSERT_EQ(util_validate_network_name(std::string(128, 'a').c_str()), true);
+-    ASSERT_EQ(util_validate_network_name(std::string(129, 'a').c_str()), false);
++    ASSERT_EQ(util_validate_network_name(std::string(255, 'a').c_str()), true);
++    ASSERT_EQ(util_validate_network_name(std::string(256, 'a').c_str()), false);
+     ASSERT_EQ(util_validate_network_name(std::string(".abce").c_str()), false);
+ }
+ 
+-- 
+2.25.1
+

0100-bugfix-malloc-right-type-size.patch

@@ -0,0 +1,40 @@
+From 7bf26415fcba090e281324ba92f7d7e6487b94fc Mon Sep 17 00:00:00 2001
+From: liuxu <liuxu156@huawei.com>
+Date: Tue, 4 Jun 2024 16:30:19 +0800
+Subject: [PATCH 100/149] bugfix:malloc right type size
+
+Signed-off-by: liuxu <liuxu156@huawei.com>
+---
+ src/client/connect/rest/rest_volumes_client.c | 2 +-
+ src/daemon/modules/service/service_network.c  | 2 +-
+ 2 files changed, 2 insertions(+), 2 deletions(-)
+
+diff --git a/src/client/connect/rest/rest_volumes_client.c b/src/client/connect/rest/rest_volumes_client.c
+index 88254e6c..2327c28e 100644
+--- a/src/client/connect/rest/rest_volumes_client.c
++++ b/src/client/connect/rest/rest_volumes_client.c
+@@ -251,7 +251,7 @@ static int prune_request_to_rest(const struct isula_prune_volume_request *reques
+     parser_error err = NULL;
+     int ret = 0;
+ 
+-    nrequest = util_common_calloc_s(sizeof(volume_list_volume_request));
++    nrequest = util_common_calloc_s(sizeof(volume_prune_volume_request));
+     if (nrequest == NULL) {
+         ERROR("Out of memory");
+         return -1;
+diff --git a/src/daemon/modules/service/service_network.c b/src/daemon/modules/service/service_network.c
+index 6754cf1a..fe6dbf1d 100644
+--- a/src/daemon/modules/service/service_network.c
++++ b/src/daemon/modules/service/service_network.c
+@@ -74,7 +74,7 @@ static struct attach_net_conf_list *build_attach_networks(const defs_map_string_
+         return NULL;
+     }
+ 
+-    list = (struct attach_net_conf_list *)util_common_calloc_s(sizeof(struct attach_net_conf));
++    list = (struct attach_net_conf_list *)util_common_calloc_s(sizeof(struct attach_net_conf_list));
+     if (list == NULL) {
+         ERROR("Out of memory");
+         return NULL;
+-- 
+2.25.1
+

0101-use-isula_clean_path-rather-than-realpath.patch

@@ -0,0 +1,26 @@
+From 60a2b15e0090018b7850b37369964bf62e253419 Mon Sep 17 00:00:00 2001
+From: jikai <jikai11@huawei.com>
+Date: Wed, 5 Jun 2024 02:07:23 +0000
+Subject: [PATCH 101/149] use isula_clean_path rather than realpath
+
+Signed-off-by: jikai <jikai11@huawei.com>
+---
+ src/cmd/isulad-shim/process.c | 2 +-
+ 1 file changed, 1 insertion(+), 1 deletion(-)
+
+diff --git a/src/cmd/isulad-shim/process.c b/src/cmd/isulad-shim/process.c
+index 18fae03f..dd41c77f 100644
+--- a/src/cmd/isulad-shim/process.c
++++ b/src/cmd/isulad-shim/process.c
+@@ -489,7 +489,7 @@ static bool attach_fifopath_security_check(process_t *p, const char *fifopath)
+         return false;
+     }
+ 
+-    if (realpath(fifopath, real_path) == NULL) {
++    if (isula_clean_path(fifopath, real_path, sizeof(real_path)) == NULL) {
+         ERROR("Failed to get realpath for '%s': %d.", real_path, SHIM_SYS_ERR(errno));
+         return false;
+     }
+-- 
+2.25.1
+

0102-fix-false-engine-rootpath-reference.patch

@@ -0,0 +1,32 @@
+From fa25a8923e47ed6b65e0bcd08954589f5b26092c Mon Sep 17 00:00:00 2001
+From: jikai <jikai11@huawei.com>
+Date: Wed, 5 Jun 2024 02:10:48 +0000
+Subject: [PATCH 102/149] fix false engine rootpath reference
+
+Signed-off-by: jikai <jikai11@huawei.com>
+---
+ src/daemon/config/isulad_config.c | 4 ++--
+ 1 file changed, 2 insertions(+), 2 deletions(-)
+
+diff --git a/src/daemon/config/isulad_config.c b/src/daemon/config/isulad_config.c
+index 80689bce..d7b54498 100644
+--- a/src/daemon/config/isulad_config.c
++++ b/src/daemon/config/isulad_config.c
+@@ -398,12 +398,12 @@ char *conf_get_sandbox_rootpath(void)
+         ERROR("Get rootpath failed");
+         return epath;
+     }
+-    if (strlen(rootpath) > (PATH_MAX - strlen(ENGINE_ROOTPATH_NAME)) - 2) {
++    if (strlen(rootpath) > (PATH_MAX - strlen(SANDBOX_ROOTPATH_NAME)) - 2) {
+         ERROR("Root path is too long");
+         return epath;
+     }
+     // rootpath + "/" + SANDBOX_ROOTPATH_NAME + "/0"
+-    len = strlen(rootpath) + 1 + strlen(ENGINE_ROOTPATH_NAME) + 1;
++    len = strlen(rootpath) + 1 + strlen(SANDBOX_ROOTPATH_NAME) + 1;
+     epath = util_smart_calloc_s(sizeof(char), len);
+     if (epath == NULL) {
+         ERROR("Out of memory");
+-- 
+2.25.1
+

0103-bugfix-add-note.patch

@@ -0,0 +1,25 @@
+From 8eea40e09aa34da85cfa191f07cfe7e123c9809d Mon Sep 17 00:00:00 2001
+From: liuxu <liuxu156@huawei.com>
+Date: Wed, 5 Jun 2024 10:18:06 +0800
+Subject: [PATCH 103/149] bugfix:add note
+
+Signed-off-by: liuxu <liuxu156@huawei.com>
+---
+ src/utils/cutils/utils_array.h | 1 +
+ 1 file changed, 1 insertion(+)
+
+diff --git a/src/utils/cutils/utils_array.h b/src/utils/cutils/utils_array.h
+index 0c4fd217..553c2c13 100644
+--- a/src/utils/cutils/utils_array.h
++++ b/src/utils/cutils/utils_array.h
+@@ -30,6 +30,7 @@ void util_free_array_by_len(char **array, size_t len);
+ 
+ void util_free_array(char **array);
+ 
++// this function just copies the first len elements of array and does not automatically add NULL element in the end.
+ char **util_copy_array_by_len(char **array, size_t len);
+ 
+ int util_grow_array(char ***orig_array, size_t *orig_capacity, size_t size,
+-- 
+2.25.1
+

0104-bugfix-adapt-network-name-max-len.patch

@@ -0,0 +1,64 @@
+From 174f7d9d959bd129675651ccf7ef460794188b63 Mon Sep 17 00:00:00 2001
+From: liuxu <liuxu156@huawei.com>
+Date: Wed, 5 Jun 2024 10:40:22 +0800
+Subject: [PATCH 104/149] bugfix:adapt network name max len
+
+Signed-off-by: liuxu <liuxu156@huawei.com>
+---
+ CI/test_cases/network_cases/network_create.sh | 5 +++--
+ src/utils/cutils/utils.h                      | 7 ++++++-
+ test/cutils/utils_network/utils_network_ut.cc | 4 ++--
+ 3 files changed, 11 insertions(+), 5 deletions(-)
+
+diff --git a/CI/test_cases/network_cases/network_create.sh b/CI/test_cases/network_cases/network_create.sh
+index 3107da12..da54fdd0 100755
+--- a/CI/test_cases/network_cases/network_create.sh
++++ b/CI/test_cases/network_cases/network_create.sh
+@@ -33,8 +33,9 @@ function test_network_create()
+     local name1="cni1"
+     local name2="cni2"
+     local name3="a"
+-    for i in $(seq 1 7);do
+-        name3=${name3}${name3}
++    local basechar="a"
++    for i in $(seq 1 199);do
++        name3=${name3}${basechar}
+     done
+     local name4=${name3}b
+     local invalid_name=".xx"
+diff --git a/src/utils/cutils/utils.h b/src/utils/cutils/utils.h
+index 4417a165..ce0ca703 100644
+--- a/src/utils/cutils/utils.h
++++ b/src/utils/cutils/utils.h
+@@ -99,7 +99,12 @@ int malloc_trim(size_t pad);
+ #define MAX_IMAGE_REF_LEN 384
+ #define MAX_CONTAINER_NAME_LEN 1024
+ #define MAX_RUNTIME_NAME_LEN 32
+-#define MAX_NETWORK_NAME_LEN 255
++/* 
++ * Linux limits the length of the file name to 255,
++ * isulad will create file by name "${ISULAD_CNI_NETWORK_CONF_FILE_PRE}${network_name}.conflist"
++ * when create native network,so we limit the length of the network name to 200.
++ */
++#define MAX_NETWORK_NAME_LEN 200
+ 
+ #define LOGIN_USERNAME_LEN 255
+ #define LOGIN_PASSWORD_LEN 255
+diff --git a/test/cutils/utils_network/utils_network_ut.cc b/test/cutils/utils_network/utils_network_ut.cc
+index 68f6f011..be85e398 100644
+--- a/test/cutils/utils_network/utils_network_ut.cc
++++ b/test/cutils/utils_network/utils_network_ut.cc
+@@ -344,8 +344,8 @@ TEST(utils_network, test_net_contain_ip)
+ TEST(utils_network, test_validate_network_name)
+ {
+     ASSERT_EQ(util_validate_network_name(nullptr), false);
+-    ASSERT_EQ(util_validate_network_name(std::string(255, 'a').c_str()), true);
+-    ASSERT_EQ(util_validate_network_name(std::string(256, 'a').c_str()), false);
++    ASSERT_EQ(util_validate_network_name(std::string(200, 'a').c_str()), true);
++    ASSERT_EQ(util_validate_network_name(std::string(201, 'a').c_str()), false);
+     ASSERT_EQ(util_validate_network_name(std::string(".abce").c_str()), false);
+ }
+ 
+-- 
+2.25.1
+

0105-start-sandbox-before-setup-network-by-default.patch

@@ -0,0 +1,140 @@
+From e6b3528acff10fb2bc62e2da0c3754f1e36cbd54 Mon Sep 17 00:00:00 2001
+From: zhongtao <zhongtao17@huawei.com>
+Date: Wed, 5 Jun 2024 10:04:59 +0800
+Subject: [PATCH 105/149] start sandbox before setup network by default
+
+Signed-off-by: zhongtao <zhongtao17@huawei.com>
+---
+ src/daemon/common/cri/cri_helpers.cc          | 12 +++++++
+ src/daemon/common/cri/cri_helpers.h           |  3 ++
+ .../cri_pod_sandbox_manager_service.cc        | 34 +++++++++++++------
+ src/daemon/modules/spec/specs.c               |  2 +-
+ src/utils/cutils/utils_file.c                 |  2 +-
+ 5 files changed, 41 insertions(+), 12 deletions(-)
+
+diff --git a/src/daemon/common/cri/cri_helpers.cc b/src/daemon/common/cri/cri_helpers.cc
+index 68d569cc..8117403c 100644
+--- a/src/daemon/common/cri/cri_helpers.cc
++++ b/src/daemon/common/cri/cri_helpers.cc
+@@ -47,6 +47,8 @@ const std::string Constants::DOCKER_IMAGEID_PREFIX { "docker://" };
+ const std::string Constants::DOCKER_PULLABLE_IMAGEID_PREFIX { "docker-pullable://" };
+ const std::string Constants::RUNTIME_READY { "RuntimeReady" };
+ const std::string Constants::NETWORK_READY { "NetworkReady" };
++// Kata 2.x need create network namespace and setup network befoce run podsandbox
++const std::string Constants::NETWORK_SETUP_ANNOTATION_KEY { "cri.sandbox.network.setup.v2" };
+ const std::string Constants::POD_CHECKPOINT_KEY { "cri.sandbox.isulad.checkpoint" };
+ const std::string Constants::CONTAINER_TYPE_ANNOTATION_KEY { "io.kubernetes.cri.container-type" };
+ const std::string Constants::CONTAINER_NAME_ANNOTATION_KEY { "io.kubernetes.cri.container-name" };
+@@ -1140,4 +1142,14 @@ auto GetPodSELinuxLabelOpts(const std::string &selinuxLabel, Errors &error)
+     return fmtiSuladOpts(selinuxOpts, securityOptSep);
+ }
+ 
++bool SetupNetworkFirst(const std::map<std::string, std::string> &annotations)
++{
++    auto iter = annotations.find(CRIHelpers::Constants::NETWORK_SETUP_ANNOTATION_KEY);
++    if (iter == annotations.end()) {
++        return false;
++    }
++
++    return iter->second == std::string("true");
++}
++
+ } // namespace CRIHelpers
+diff --git a/src/daemon/common/cri/cri_helpers.h b/src/daemon/common/cri/cri_helpers.h
+index 5c450b32..11a80b45 100644
+--- a/src/daemon/common/cri/cri_helpers.h
++++ b/src/daemon/common/cri/cri_helpers.h
+@@ -49,6 +49,7 @@ public:
+     static const std::string DOCKER_PULLABLE_IMAGEID_PREFIX;
+     static const std::string RUNTIME_READY;
+     static const std::string NETWORK_READY;
++    static const std::string NETWORK_SETUP_ANNOTATION_KEY;
+     static const std::string POD_CHECKPOINT_KEY;
+     static const size_t MAX_CHECKPOINT_KEY_LEN { 250 };
+     static const std::string CONTAINER_TYPE_ANNOTATION_KEY;
+@@ -151,6 +152,8 @@ auto GetPodSELinuxLabelOpts(const std::string &selinuxLabel, Errors &error) -> s
+ auto GetlegacySeccompiSuladOpts(const std::string &seccompProfile, Errors &error) -> std::vector<iSuladOpt>;
+ 
+ auto GetSeccompiSuladOptsByPath(const char *dstpath, Errors &error) -> std::vector<iSuladOpt>;
++
++bool SetupNetworkFirst(const std::map<std::string, std::string> &annotations);
+ }; // namespace CRIHelpers
+ 
+ #endif // DAEMON_ENTRY_CRI_CRI_HELPERS_H
+diff --git a/src/daemon/entry/cri/v1alpha/cri_pod_sandbox_manager_service.cc b/src/daemon/entry/cri/v1alpha/cri_pod_sandbox_manager_service.cc
+index af6b5fff..f852f4df 100644
+--- a/src/daemon/entry/cri/v1alpha/cri_pod_sandbox_manager_service.cc
++++ b/src/daemon/entry/cri/v1alpha/cri_pod_sandbox_manager_service.cc
+@@ -655,19 +655,33 @@ auto PodSandboxManagerService::RunPodSandbox(const runtime::v1alpha2::PodSandbox
+         }
+     }
+ 
+-    // Step 7: Setup networking for the sandbox.
+-    SetupSandboxNetwork(config, response_id, inspect_data, networkOptions, stdAnnos, network_setting_json, error);
+-    if (error.NotEmpty()) {
+-        goto cleanup_ns;
+-    }
++    // Step 7: According to the annotation and network namespace mode,
++    //         determine the order of start sandbox and setup network.
++    if (CRIHelpers::SetupNetworkFirst(stdAnnos)) {
++        // Step 7.1: Setup networking for the sandbox, and then start the sandbox container.
++        SetupSandboxNetwork(config, response_id, inspect_data, networkOptions, stdAnnos, network_setting_json, error);
++        if (error.NotEmpty()) {
++            goto cleanup_ns;
++        }
+ 
+-    // Step 8: Start the sandbox container.
+-    StartSandboxContainer(response_id, error);
+-    if (error.NotEmpty()) {
+-        goto cleanup_network;
++        StartSandboxContainer(response_id, error);
++        if (error.NotEmpty()) {
++            goto cleanup_network;
++        }
++    } else {
++        // Step 7.2: (Default)Start the sandbox container, and then setup networking for the sandbox.
++        StartSandboxContainer(response_id, error);
++        if (error.NotEmpty()) {
++            goto cleanup_ns;
++        }
++
++        SetupSandboxNetwork(config, response_id, inspect_data, networkOptions, stdAnnos, network_setting_json, error);
++        if (error.NotEmpty()) {
++            goto cleanup_ns;
++        }
+     }
+ 
+-    // Step 9: Save network settings json to disk
++    // Step 8: Save network settings json to disk
+     if (namespace_is_cni(inspect_data->host_config->network_mode)) {
+         Errors tmpErr;
+         UpdatePodSandboxNetworkSettings(response_id, network_setting_json, tmpErr);
+diff --git a/src/daemon/modules/spec/specs.c b/src/daemon/modules/spec/specs.c
+index 122f9992..f0538e26 100644
+--- a/src/daemon/modules/spec/specs.c
++++ b/src/daemon/modules/spec/specs.c
+@@ -1601,7 +1601,7 @@ static int merge_share_network_namespace(const oci_runtime_spec *oci_spec, const
+     int ret = 0;
+     char *ns_path = NULL;
+ 
+-    if (host_spec->network_mode == NULL) {
++    if (host_spec->network_mode == NULL || strlen(host_spec->network_mode) == 0) {
+         return 0;
+     }
+ 
+diff --git a/src/utils/cutils/utils_file.c b/src/utils/cutils/utils_file.c
+index 6fc6852d..90bb156f 100644
+--- a/src/utils/cutils/utils_file.c
++++ b/src/utils/cutils/utils_file.c
+@@ -85,7 +85,7 @@ bool util_file_exists(const char *f)
+     struct stat buf;
+     int nret;
+ 
+-    if (f == NULL) {
++    if (f == NULL || strlen(f) == 0) {
+         return false;
+     }
+ 
+-- 
+2.25.1
+

0106-Revert-use-isula_clean_path-rather-than-realpath.patch

@@ -0,0 +1,28 @@
+From f690c9a2dff298b41dc607e4ea6dd09113a322fb Mon Sep 17 00:00:00 2001
+From: jikai <jikai11@huawei.com>
+Date: Thu, 6 Jun 2024 02:13:36 +0000
+Subject: [PATCH 106/149] Revert "use isula_clean_path rather than realpath"
+
+This reverts commit 60a2b15e0090018b7850b37369964bf62e253419.
+
+Signed-off-by: jikai <jikai11@huawei.com>
+---
+ src/cmd/isulad-shim/process.c | 2 +-
+ 1 file changed, 1 insertion(+), 1 deletion(-)
+
+diff --git a/src/cmd/isulad-shim/process.c b/src/cmd/isulad-shim/process.c
+index dd41c77f..18fae03f 100644
+--- a/src/cmd/isulad-shim/process.c
++++ b/src/cmd/isulad-shim/process.c
+@@ -489,7 +489,7 @@ static bool attach_fifopath_security_check(process_t *p, const char *fifopath)
+         return false;
+     }
+ 
+-    if (isula_clean_path(fifopath, real_path, sizeof(real_path)) == NULL) {
++    if (realpath(fifopath, real_path) == NULL) {
+         ERROR("Failed to get realpath for '%s': %d.", real_path, SHIM_SYS_ERR(errno));
+         return false;
+     }
+-- 
+2.25.1
+

0107-bugfix-for-start-sandbox-before-setup-network-by-def.patch

@@ -0,0 +1,28 @@
+From 63f4f9bc9c36825d85a14f6a33102194d30e12a7 Mon Sep 17 00:00:00 2001
+From: zhongtao <zhongtao17@huawei.com>
+Date: Fri, 7 Jun 2024 02:24:49 +1400
+Subject: [PATCH 107/149] bugfix for start sandbox before setup network by
+ default
+
+Signed-off-by: zhongtao <zhongtao17@huawei.com>
+---
+ .../entry/cri/v1alpha/cri_pod_sandbox_manager_service.cc       | 3 +++
+ 1 file changed, 3 insertions(+)
+
+diff --git a/src/daemon/entry/cri/v1alpha/cri_pod_sandbox_manager_service.cc b/src/daemon/entry/cri/v1alpha/cri_pod_sandbox_manager_service.cc
+index f852f4df..bc3f4031 100644
+--- a/src/daemon/entry/cri/v1alpha/cri_pod_sandbox_manager_service.cc
++++ b/src/daemon/entry/cri/v1alpha/cri_pod_sandbox_manager_service.cc
+@@ -677,6 +677,9 @@ auto PodSandboxManagerService::RunPodSandbox(const runtime::v1alpha2::PodSandbox
+ 
+         SetupSandboxNetwork(config, response_id, inspect_data, networkOptions, stdAnnos, network_setting_json, error);
+         if (error.NotEmpty()) {
++            Errors stopError;
++            StopContainerHelper(response_id, stopError);
++            WARN("Error stop container: %s: %s", response_id.c_str(), stopError.GetCMessage());
+             goto cleanup_ns;
+         }
+     }
+-- 
+2.25.1
+

0108-skip-test-rely-on-docker.io.patch

@@ -0,0 +1,59 @@
+From d3e7b0b0d19ca4937716d835e3627714157d6cc3 Mon Sep 17 00:00:00 2001
+From: zhongtao <zhongtao17@huawei.com>
+Date: Tue, 11 Jun 2024 17:14:58 +0800
+Subject: [PATCH 108/149] skip test rely on docker.io
+
+Signed-off-by: zhongtao <zhongtao17@huawei.com>
+---
+ CI/test_cases/image_cases/image_search.sh      | 4 +++-
+ CI/test_cases/image_cases/integration_check.sh | 2 +-
+ CI/test_cases/image_cases/registry.sh          | 5 +++--
+ 3 files changed, 7 insertions(+), 4 deletions(-)
+
+diff --git a/CI/test_cases/image_cases/image_search.sh b/CI/test_cases/image_cases/image_search.sh
+index 11af02f1..4bf0e099 100755
+--- a/CI/test_cases/image_cases/image_search.sh
++++ b/CI/test_cases/image_cases/image_search.sh
+@@ -76,7 +76,9 @@ function test_image_search()
+ 
+ declare -i ans=0
+ 
+-test_image_search || ((ans++))
++# unable to pull image from docker.io without agent, skip this test
++# registry API v1 is not implemented in https://3laho3y3.mirror.aliyuncs.com and isula search cannot be tested
++# test_image_search || ((ans++))
+ 
+ show_result ${ans} "${curr_path}/${0}"
+ 
+diff --git a/CI/test_cases/image_cases/integration_check.sh b/CI/test_cases/image_cases/integration_check.sh
+index 6ec3ab52..f340348d 100755
+--- a/CI/test_cases/image_cases/integration_check.sh
++++ b/CI/test_cases/image_cases/integration_check.sh
+@@ -27,7 +27,7 @@ image="busybox"
+ function test_image_info()
+ {
+   local ret=0
+-  local uimage="docker.io/library/nats"
++  local uimage="nats"
+   local test="list && inspect image info test => (${FUNCNAME[@]})"
+   local lid
+   local cid
+diff --git a/CI/test_cases/image_cases/registry.sh b/CI/test_cases/image_cases/registry.sh
+index e33983d6..7ea9a0c5 100755
+--- a/CI/test_cases/image_cases/registry.sh
++++ b/CI/test_cases/image_cases/registry.sh
+@@ -74,8 +74,9 @@ function isula_pull()
+     isula run --rm -ti busybox echo hello 2>&1 | grep pulling
+     [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - --pull missing failed" && ((ret++))
+ 
+-    isula pull docker.io/library/busybox:latest
+-    [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - --pull docker.io/library/busybox:latest failed" && ((ret++))
++    # Unable to pull image from docker.io without agent, skip this test
++    # isula pull docker.io/library/busybox:latest
++    # [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - --pull docker.io/library/busybox:latest failed" && ((ret++))
+ 
+     isula pull 3laho3y3.mirror.aliyuncs.com/library/busybox
+     fn_check_eq "$?" "0" "isula pull 3laho3y3.mirror.aliyuncs.com/library/busybox"
+-- 
+2.25.1
+

0109-modify-default-registry-mirrors-in-ci-test.patch

@@ -0,0 +1,26 @@
+From d6284e5e786e1407c2ce5ef098a39c154650bd38 Mon Sep 17 00:00:00 2001
+From: zhongtao <zhongtao17@huawei.com>
+Date: Wed, 12 Jun 2024 10:57:39 +0800
+Subject: [PATCH 109/149] modify default registry mirrors in ci test
+
+Signed-off-by: zhongtao <zhongtao17@huawei.com>
+---
+ CI/test_cases/container_cases/test_data/daemon.json | 2 +-
+ 1 file changed, 1 insertion(+), 1 deletion(-)
+
+diff --git a/CI/test_cases/container_cases/test_data/daemon.json b/CI/test_cases/container_cases/test_data/daemon.json
+index 2664c6b2..ab7d0360 100644
+--- a/CI/test_cases/container_cases/test_data/daemon.json
++++ b/CI/test_cases/container_cases/test_data/daemon.json
+@@ -24,7 +24,7 @@
+         "overlay2.override_kernel_check=true"
+     ],
+     "registry-mirrors": [
+-        "docker.io"
++        "https://3laho3y3.mirror.aliyuncs.com"
+     ],
+     "insecure-registries": [
+     ],
+-- 
+2.25.1
+

0110-add-timestamp-in-PodSandboxStatu-response.patch

@@ -0,0 +1,25 @@
+From 5087d7501308660970aa9e7c12cf5be7a3d9b063 Mon Sep 17 00:00:00 2001
+From: jikai <jikai11@huawei.com>
+Date: Wed, 12 Jun 2024 15:20:17 +0000
+Subject: [PATCH 110/149] add timestamp in PodSandboxStatu response
+
+Signed-off-by: jikai <jikai11@huawei.com>
+---
+ src/daemon/entry/cri/v1/v1_cri_pod_sandbox_manager_service.cc | 1 +
+ 1 file changed, 1 insertion(+)
+
+diff --git a/src/daemon/entry/cri/v1/v1_cri_pod_sandbox_manager_service.cc b/src/daemon/entry/cri/v1/v1_cri_pod_sandbox_manager_service.cc
+index fa726e2c..2a458a6d 100644
+--- a/src/daemon/entry/cri/v1/v1_cri_pod_sandbox_manager_service.cc
++++ b/src/daemon/entry/cri/v1/v1_cri_pod_sandbox_manager_service.cc
+@@ -910,6 +910,7 @@ void PodSandboxManagerService::PodSandboxStatus(const std::string &podSandboxID,
+     for (auto &containerStatus : containerStatuses) {
+         *(reply->add_containers_statuses()) = *containerStatus;
+     }
++    reply->set_timestamp(util_get_now_time_nanos());
+     return;
+ }
+ 
+-- 
+2.25.1
+

0111-bugfix-for-file-param-verify.patch

@@ -0,0 +1,75 @@
+From d0fd2c2bf87d7befaa8810a70d7eb2061664f02f Mon Sep 17 00:00:00 2001
+From: zhongtao <zhongtao17@huawei.com>
+Date: Fri, 14 Jun 2024 09:55:28 +0800
+Subject: [PATCH 111/149] bugfix for file param verify
+
+Signed-off-by: zhongtao <zhongtao17@huawei.com>
+---
+ src/cmd/isula/base/create.c | 19 +++++++++++++++++++
+ src/cmd/isula/images/load.c |  6 ++++++
+ 2 files changed, 25 insertions(+)
+
+diff --git a/src/cmd/isula/base/create.c b/src/cmd/isula/base/create.c
+index 543b8fd6..b04dddb5 100644
+--- a/src/cmd/isula/base/create.c
++++ b/src/cmd/isula/base/create.c
+@@ -292,6 +292,12 @@ static int append_env_variables_to_conf(const char *env_file, isula_container_co
+     int ret = 0;
+     size_t file_size;
+ 
++    if (util_dir_exists(env_file)) {
++        COMMAND_ERROR("Env file is a directory: %s", env_file);
++        ret = -1;
++        goto out;   
++    }
++
+     if (!util_file_exists(env_file)) {
+         COMMAND_ERROR("env file not exists: %s", env_file);
+         ret = -1;
+@@ -427,6 +433,12 @@ static int append_labels_to_conf(const char *label_file, isula_container_config_
+     int ret = 0;
+     size_t file_size;
+ 
++    if (util_dir_exists(label_file)) {
++        COMMAND_ERROR("Label file is a directory: %s", label_file);
++        ret = -1;
++        goto out;   
++    }
++
+     if (!util_file_exists(label_file)) {
+         COMMAND_ERROR("label file not exists: %s", label_file);
+         ret = -1;
+@@ -2357,6 +2369,13 @@ static int create_check_env_target_file(const struct client_arguments *args)
+         ret = -1;
+         goto out;
+     }
++
++    if (util_dir_exists(env_path)) {
++        COMMAND_ERROR("Env target file is a directory: %s", env_path);
++        ret = -1;
++        goto out;   
++    }
++
+     if (!util_file_exists(env_path)) {
+         goto out;
+     }
+diff --git a/src/cmd/isula/images/load.c b/src/cmd/isula/images/load.c
+index 314e5d5e..cb39dee7 100644
+--- a/src/cmd/isula/images/load.c
++++ b/src/cmd/isula/images/load.c
+@@ -162,6 +162,12 @@ int cmd_load_main(int argc, const char **argv)
+         g_cmd_load_args.file = file;
+     }
+ 
++    if (util_dir_exists(g_cmd_load_args.file)) {
++        COMMAND_ERROR("Load file is a directory: %s", g_cmd_load_args.file);
++        ret = -1;
++        exit(exit_code);
++    }
++
+     if (!util_file_exists(g_cmd_load_args.file)) {
+         COMMAND_ERROR("File %s is not exist", g_cmd_load_args.file);
+         exit(exit_code);
+-- 
+2.25.1
+

0112-bugfix-change-cni-log-info.patch

@@ -0,0 +1,26 @@
+From 359a6673e01bef937adcc17f99ee94b67caca32e Mon Sep 17 00:00:00 2001
+From: liuxu <liuxu156@huawei.com>
+Date: Fri, 14 Jun 2024 17:12:58 +0800
+Subject: [PATCH 112/149] bugfix:change cni log info
+
+Signed-off-by: liuxu <liuxu156@huawei.com>
+---
+ .../modules/network/cni_operator/libcni/invoke/libcni_exec.c    | 2 +-
+ 1 file changed, 1 insertion(+), 1 deletion(-)
+
+diff --git a/src/daemon/modules/network/cni_operator/libcni/invoke/libcni_exec.c b/src/daemon/modules/network/cni_operator/libcni/invoke/libcni_exec.c
+index 74d6d74a..1e4a7138 100644
+--- a/src/daemon/modules/network/cni_operator/libcni/invoke/libcni_exec.c
++++ b/src/daemon/modules/network/cni_operator/libcni/invoke/libcni_exec.c
+@@ -247,7 +247,7 @@ static char *env_stringify(char *(*pargs)[2], size_t len)
+     bool invalid_arg = (pargs == NULL || len == 0);
+ 
+     if (invalid_arg) {
+-        ERROR("Invalid arguments");
++        DEBUG("Empty arguments");
+         return NULL;
+     }
+ 
+-- 
+2.25.1
+

0113-move-shutdown-handle-after-init-module.patch

@@ -0,0 +1,43 @@
+From 0ae6244c6bfed229a46d300888977a4967e1d718 Mon Sep 17 00:00:00 2001
+From: zhongtao <zhongtao17@huawei.com>
+Date: Wed, 19 Jun 2024 09:50:51 +0800
+Subject: [PATCH 113/149] move shutdown handle after init module
+
+Signed-off-by: zhongtao <zhongtao17@huawei.com>
+---
+ src/cmd/isulad/main.c | 12 +++++++-----
+ 1 file changed, 7 insertions(+), 5 deletions(-)
+
+diff --git a/src/cmd/isulad/main.c b/src/cmd/isulad/main.c
+index 3e2249d7..52ac3172 100644
+--- a/src/cmd/isulad/main.c
++++ b/src/cmd/isulad/main.c
+@@ -1669,11 +1669,6 @@ static int start_daemon_threads()
+ {
+     int ret = -1;
+ 
+-    if (new_shutdown_handler()) {
+-        ERROR("Create new shutdown handler thread failed");
+-        goto out;
+-    }
+-
+     if (events_module_init() != 0) {
+         goto out;
+     }
+@@ -1825,6 +1820,13 @@ int main(int argc, char **argv)
+         goto failure;
+     }
+ 
++    // after all modules are initialized, enable the shutdown handler to
++    // prevent shutdown handler from cleaning up incompletely initialized modules.
++    if (new_shutdown_handler()) {
++        ERROR("Create new shutdown handler thread failed");
++        goto failure;
++    }
++
+ #ifdef ENABLE_PLUGIN
+     if (start_plugin_manager()) {
+         ERROR("Failed to init plugin_manager");
+-- 
+2.25.1
+

0114-bugfix-for-null-pointer-reference.patch

@@ -0,0 +1,56 @@
+From 701180b53d1c52376f753b94c5cf09987ae789b3 Mon Sep 17 00:00:00 2001
+From: zhongtao <zhongtao17@huawei.com>
+Date: Tue, 18 Jun 2024 16:02:25 +0800
+Subject: [PATCH 114/149] bugfix for null pointer reference
+
+Signed-off-by: zhongtao <zhongtao17@huawei.com>
+---
+ src/daemon/entry/connect/grpc/grpc_service.cc | 14 ++++++++++++--
+ 1 file changed, 12 insertions(+), 2 deletions(-)
+
+diff --git a/src/daemon/entry/connect/grpc/grpc_service.cc b/src/daemon/entry/connect/grpc/grpc_service.cc
+index 1d8de922..300af082 100644
+--- a/src/daemon/entry/connect/grpc/grpc_service.cc
++++ b/src/daemon/entry/connect/grpc/grpc_service.cc
+@@ -100,7 +100,9 @@ public:
+     {
+         // Wait for the server to shutdown. Note that some other thread must be
+         // responsible for shutting down the server for this call to ever return.
+-        m_server->Wait();
++        if (m_server != nullptr) {
++            m_server->Wait();
++        }
+ 
+         // Wait for stream server to shutdown
+         m_criService.Wait();
+@@ -109,7 +111,9 @@ public:
+     void Shutdown(void)
+     {
+         // call CRI to shutdown stream server, shutdown cri first to notify events thread to exit
+-        m_criService.Shutdown();
++        if (m_server != nullptr) {
++            m_server->Shutdown();
++        }
+ 
+         m_server->Shutdown();
+ 
+@@ -242,10 +246,16 @@ int grpc_server_init(const struct service_arguments *args)
+ 
+ void grpc_server_wait(void)
+ {
++    if (g_grpcserver == nullptr) {
++        return;
++    }
+     g_grpcserver->Wait();
+ }
+ 
+ void grpc_server_shutdown(void)
+ {
++    if (g_grpcserver == nullptr) {
++        return;
++    }
+     g_grpcserver->Shutdown();
+ }
+-- 
+2.25.1
+

0115-bugfix-for-m_criService-shutdown.patch

@@ -0,0 +1,33 @@
+From 93b1df1a1d3fcf6d285102f3cc1f79e6241aa393 Mon Sep 17 00:00:00 2001
+From: zhongtao <zhongtao17@huawei.com>
+Date: Thu, 4 Jul 2024 10:58:38 +0800
+Subject: [PATCH 115/149] bugfix for m_criService shutdown
+
+Signed-off-by: zhongtao <zhongtao17@huawei.com>
+---
+ src/daemon/entry/connect/grpc/grpc_service.cc | 6 +++---
+ 1 file changed, 3 insertions(+), 3 deletions(-)
+
+diff --git a/src/daemon/entry/connect/grpc/grpc_service.cc b/src/daemon/entry/connect/grpc/grpc_service.cc
+index 300af082..fb5ec3cb 100644
+--- a/src/daemon/entry/connect/grpc/grpc_service.cc
++++ b/src/daemon/entry/connect/grpc/grpc_service.cc
+@@ -111,12 +111,12 @@ public:
+     void Shutdown(void)
+     {
+         // call CRI to shutdown stream server, shutdown cri first to notify events thread to exit
++        m_criService.Shutdown();
++
+         if (m_server != nullptr) {
+             m_server->Shutdown();
+         }
+-
+-        m_server->Shutdown();
+-
++    
+         // Shutdown daemon, this operation should remove socket file.
+         for (const auto &address : m_socketPath) {
+             if (address.find(UNIX_SOCKET_PREFIX) == 0) {
+-- 
+2.25.1
+

0116-fix-bug-in-ci-test.patch

@@ -0,0 +1,54 @@
+From c7cf33c432b3d9479b2fe365169d4b9a37cae8f7 Mon Sep 17 00:00:00 2001
+From: jikai <jikai11@huawei.com>
+Date: Tue, 9 Jul 2024 12:30:01 +0000
+Subject: [PATCH 116/149] fix bug in ci test
+
+Signed-off-by: jikai <jikai11@huawei.com>
+---
+ CI/test_cases/container_cases/run.sh | 6 +++---
+ CI/test_cases/helpers.sh             | 5 +++--
+ 2 files changed, 6 insertions(+), 5 deletions(-)
+
+diff --git a/CI/test_cases/container_cases/run.sh b/CI/test_cases/container_cases/run.sh
+index 1bfd388b..ef04b547 100755
+--- a/CI/test_cases/container_cases/run.sh
++++ b/CI/test_cases/container_cases/run.sh
+@@ -26,9 +26,9 @@ source ../helpers.sh
+ function do_test_t()
+ {
+     tid=`isula run --runtime $1 -tid --name hostname busybox`
+-    chostname=`isula exec -it $tid hostname`
+-    clean_hostname=$(echo "$hostname" | sed 's/[\x01-\x1F\x7F]//g')
+-    fn_check_eq "${clean_hostname}" "${tid:0:12}" "default hostname is not id of container"
++    # should not use -it option, otherwise the hostname will containe special characters such as '$' or '\r'
++    hostname=`isula exec $tid hostname`
++    fn_check_eq "${hostname}" "${tid:0:12}" "default hostname is not id of container"
+     isula exec -it hostname env | grep HOSTNAME
+     fn_check_eq "$?" "0" "check HOSTNAME env failed"
+     isula stop -t 0 $tid
+diff --git a/CI/test_cases/helpers.sh b/CI/test_cases/helpers.sh
+index c5eba8a2..0288b4ea 100755
+--- a/CI/test_cases/helpers.sh
++++ b/CI/test_cases/helpers.sh
+@@ -52,15 +52,16 @@ function cut_output_lines() {
+     return $retval
+ }
+ 
++# use string compare to check the result
+ function fn_check_eq() {
+-    if [[ "$1" -ne "$2" ]];then
++    if [ "x$1" != "x$2" ];then
+         echo "$3"
+         TC_RET_T=$(($TC_RET_T+1))
+     fi
+ }
+ 
+ function fn_check_ne() {
+-    if [[ "$1" -eq "$2" ]];then
++    if [[ "x$1" == "x$2" ]];then
+         echo "$3"
+         TC_RET_T=$(($TC_RET_T+1))
+     fi
+-- 
+2.25.1
+

0119-skip-calling-cni-plugin-cleanup-when-network-namespa.patch

@@ -0,0 +1,51 @@
+From ee720f966fdf14a99b8ebc685f3948bb8b29ba73 Mon Sep 17 00:00:00 2001
+From: zhongtao <zhongtao17@huawei.com>
+Date: Tue, 13 Aug 2024 10:56:44 +0800
+Subject: [PATCH 119/149] skip calling cni plugin cleanup when network
+ namespace is not mounted
+
+Signed-off-by: zhongtao <zhongtao17@huawei.com>
+---
+ .../entry/cri/v1/v1_cri_pod_sandbox_manager_service.cc     | 7 +++++++
+ .../entry/cri/v1alpha/cri_pod_sandbox_manager_service.cc   | 7 +++++++
+ 2 files changed, 14 insertions(+)
+
+diff --git a/src/daemon/entry/cri/v1/v1_cri_pod_sandbox_manager_service.cc b/src/daemon/entry/cri/v1/v1_cri_pod_sandbox_manager_service.cc
+index 2a458a6d..77faf48a 100644
+--- a/src/daemon/entry/cri/v1/v1_cri_pod_sandbox_manager_service.cc
++++ b/src/daemon/entry/cri/v1/v1_cri_pod_sandbox_manager_service.cc
+@@ -435,6 +435,13 @@ void PodSandboxManagerService::ClearCniNetwork(const std::shared_ptr<sandbox::Sa
+         return;
+     }
+ 
++    // If the network namespace is not mounted, the network has been cleaned up
++    // and there is no need to call the cni plugin.
++    if (!util_detect_mounted(sandboxKey.c_str())) {
++        WARN("Network namespace %s not exist", sandboxKey.c_str());
++        return;
++    }
++
+     const auto config = sandbox->GetSandboxConfig();
+     std::map<std::string, std::string> stdAnnos;
+     CRIHelpers::ProtobufAnnoMapToStd(config.annotations(), stdAnnos);
+diff --git a/src/daemon/entry/cri/v1alpha/cri_pod_sandbox_manager_service.cc b/src/daemon/entry/cri/v1alpha/cri_pod_sandbox_manager_service.cc
+index bc3f4031..5590827e 100644
+--- a/src/daemon/entry/cri/v1alpha/cri_pod_sandbox_manager_service.cc
++++ b/src/daemon/entry/cri/v1alpha/cri_pod_sandbox_manager_service.cc
+@@ -848,6 +848,13 @@ auto PodSandboxManagerService::ClearCniNetwork(const std::string &realSandboxID,
+         goto cleanup;
+     }
+ 
++    // If the network namespace is not mounted, the network has been cleaned up
++    // and there is no need to call the cni plugin.
++    if (!util_detect_mounted(netnsPath.c_str())) {
++        WARN("Network namespace %s not exist", netnsPath.c_str());
++        goto cleanup;
++    }
++
+     stdAnnos.insert(std::pair<std::string, std::string>(CRIHelpers::Constants::POD_SANDBOX_KEY, netnsPath));
+     pluginErr.Clear();
+     m_pluginManager->TearDownPod(ns, name, Network::DEFAULT_NETWORK_INTERFACE_NAME, realSandboxID, stdAnnos,
+-- 
+2.25.1
+

0121-get-realpath-before-ns-mountpoint-verification.patch

@@ -0,0 +1,72 @@
+From 6357caaf6bcf413b58e587fe3df5c508275713ee Mon Sep 17 00:00:00 2001
+From: zhongtao <zhongtao17@huawei.com>
+Date: Thu, 15 Aug 2024 19:21:19 +1400
+Subject: [PATCH 121/149] get realpath before ns mountpoint verification
+
+Signed-off-by: zhongtao <zhongtao17@huawei.com>
+---
+ .../entry/cri/v1/v1_cri_pod_sandbox_manager_service.cc   | 9 +++++++--
+ .../entry/cri/v1alpha/cri_pod_sandbox_manager_service.cc | 9 +++++++--
+ 2 files changed, 14 insertions(+), 4 deletions(-)
+
+diff --git a/src/daemon/entry/cri/v1/v1_cri_pod_sandbox_manager_service.cc b/src/daemon/entry/cri/v1/v1_cri_pod_sandbox_manager_service.cc
+index 77faf48a..3ece885f 100644
+--- a/src/daemon/entry/cri/v1/v1_cri_pod_sandbox_manager_service.cc
++++ b/src/daemon/entry/cri/v1/v1_cri_pod_sandbox_manager_service.cc
+@@ -424,6 +424,7 @@ cleanup_sandbox:
+ 
+ void PodSandboxManagerService::ClearCniNetwork(const std::shared_ptr<sandbox::Sandbox> sandbox, Errors &error)
+ {
++    char real_path[PATH_MAX] = { 0 };
+     std::string networkMode = sandbox->GetNetMode();
+     if (!namespace_is_cni(networkMode.c_str()) || !sandbox->GetNetworkReady()) {
+         return;
+@@ -435,10 +436,14 @@ void PodSandboxManagerService::ClearCniNetwork(const std::shared_ptr<sandbox::Sa
+         return;
+     }
+ 
++    if (realpath(sandboxKey.c_str(), real_path) == NULL) {
++        ERROR("Failed to get %s realpath", sandboxKey.c_str());
++    }
++
+     // If the network namespace is not mounted, the network has been cleaned up
+     // and there is no need to call the cni plugin.
+-    if (!util_detect_mounted(sandboxKey.c_str())) {
+-        WARN("Network namespace %s not exist", sandboxKey.c_str());
++    if (strlen(real_path) != 0 && !util_detect_mounted(real_path)) {
++        ERROR("Network namespace %s not exist", real_path);
+         return;
+     }
+ 
+diff --git a/src/daemon/entry/cri/v1alpha/cri_pod_sandbox_manager_service.cc b/src/daemon/entry/cri/v1alpha/cri_pod_sandbox_manager_service.cc
+index 5590827e..1c343cda 100644
+--- a/src/daemon/entry/cri/v1alpha/cri_pod_sandbox_manager_service.cc
++++ b/src/daemon/entry/cri/v1alpha/cri_pod_sandbox_manager_service.cc
+@@ -826,6 +826,7 @@ auto PodSandboxManagerService::ClearCniNetwork(const std::string &realSandboxID,
+                                                /*error*/) -> int
+ {
+     Errors networkErr;
++    char real_path[PATH_MAX] = { 0 };
+ 
+     bool ready = GetNetworkReady(realSandboxID, networkErr);
+     if (hostNetwork || (!ready && networkErr.Empty())) {
+@@ -848,10 +849,14 @@ auto PodSandboxManagerService::ClearCniNetwork(const std::string &realSandboxID,
+         goto cleanup;
+     }
+ 
++    if (realpath(netnsPath.c_str(), real_path) == NULL) {
++        ERROR("Failed to get %s realpath", netnsPath.c_str());
++    }
++
+     // If the network namespace is not mounted, the network has been cleaned up
+     // and there is no need to call the cni plugin.
+-    if (!util_detect_mounted(netnsPath.c_str())) {
+-        WARN("Network namespace %s not exist", netnsPath.c_str());
++    if (strlen(real_path) != 0 && !util_detect_mounted(real_path)) {
++        ERROR("Network namespace %s not exist", real_path);
+         goto cleanup;
+     }
+ 
+-- 
+2.25.1
+

0123-code-improve-for-codecheck.patch

@@ -0,0 +1,287 @@
+From 8e442712354a9d4f766d1f90b018fd1246cb9ef2 Mon Sep 17 00:00:00 2001
+From: zhongtao <zhongtao17@huawei.com>
+Date: Wed, 4 Sep 2024 16:26:59 +1400
+Subject: [PATCH 123/149] code improve for codecheck
+
+Signed-off-by: zhongtao <zhongtao17@huawei.com>
+---
+ src/daemon/common/sysinfo.h                        | 8 ++++----
+ src/daemon/entry/cri/network_plugin.cc             | 2 +-
+ src/daemon/mailbox/message_queue.h                 | 8 ++++----
+ src/daemon/mailbox/message_subscriber.h            | 8 ++++++++
+ src/daemon/modules/runtime/shim/shim_rt_monitor.cc | 2 ++
+ src/daemon/nri/nri_adaption.h                      | 1 -
+ src/daemon/sandbox/sandbox_manager.cc              | 5 ++---
+ src/utils/cutils/blocking_queue.h                  | 2 +-
+ src/utils/cutils/utils_aes.h                       | 2 +-
+ src/utils/cutils/utils_cap.h                       | 7 +++----
+ src/utils/cutils/utils_fs.h                        | 2 +-
+ src/utils/cutils/utils_network.c                   | 2 ++
+ src/utils/cutils/utils_string.h                    | 3 +--
+ src/utils/tar/util_archive.h                       | 5 ++---
+ src/utils/tar/util_gzip.h                          | 2 +-
+ 15 files changed, 33 insertions(+), 26 deletions(-)
+
+diff --git a/src/daemon/common/sysinfo.h b/src/daemon/common/sysinfo.h
+index 6142487b..e6bb7f95 100644
+--- a/src/daemon/common/sysinfo.h
++++ b/src/daemon/common/sysinfo.h
+@@ -15,16 +15,16 @@
+ #ifndef DAEMON_COMMON_SYSINFO_H
+ #define DAEMON_COMMON_SYSINFO_H
+ 
+-#ifdef __cplusplus
+-extern "C" {
+-#endif
+-
+ #include <stdbool.h>
+ #include <stdint.h>
+ #include <isula_libutils/auto_cleanup.h>
+ 
+ #include "cgroup.h"
+ 
++#ifdef __cplusplus
++extern "C" {
++#endif
++
+ typedef struct {
+     // Number of processors currently online (i.e., available).
+     int ncpus;
+diff --git a/src/daemon/entry/cri/network_plugin.cc b/src/daemon/entry/cri/network_plugin.cc
+index f8f9c7e6..439d0224 100644
+--- a/src/daemon/entry/cri/network_plugin.cc
++++ b/src/daemon/entry/cri/network_plugin.cc
+@@ -198,7 +198,7 @@ void InitNetworkPlugin(std::vector<std::shared_ptr<NetworkPlugin>> *plugins, std
+ 
+     if (networkPluginName.empty()) {
+         DEBUG("network plugin name empty");
+-        *result = std::shared_ptr<NetworkPlugin>(new (std::nothrow) NoopNetworkPlugin);
++        *result = std::make_shared<NoopNetworkPlugin>();
+         if (*result == nullptr) {
+             ERROR("Out of memory");
+             return;
+diff --git a/src/daemon/mailbox/message_queue.h b/src/daemon/mailbox/message_queue.h
+index 7905840f..c9bbc9e2 100644
+--- a/src/daemon/mailbox/message_queue.h
++++ b/src/daemon/mailbox/message_queue.h
+@@ -16,10 +16,6 @@
+ #ifndef DAEMON_MESSAGE_MESSAGE_QUEUE_H
+ #define DAEMON_MESSAGE_MESSAGE_QUEUE_H
+ 
+-#ifdef __cplusplus
+-extern "C" {
+-#endif
+-
+ #include <pthread.h>
+ 
+ #include "blocking_queue.h"
+@@ -27,6 +23,10 @@ extern "C" {
+ #include "map.h"
+ #include "message_subscriber.h"
+ 
++#ifdef __cplusplus
++extern "C" {
++#endif
++
+ typedef struct message_queue {
+     blocking_queue *messages;
+ 
+diff --git a/src/daemon/mailbox/message_subscriber.h b/src/daemon/mailbox/message_subscriber.h
+index de4574d9..2987b60d 100644
+--- a/src/daemon/mailbox/message_subscriber.h
++++ b/src/daemon/mailbox/message_subscriber.h
+@@ -19,6 +19,10 @@
+ #include "blocking_queue.h"
+ #include "mailbox_message.h"
+ 
++#ifdef __cplusplus
++extern "C" {
++#endif
++
+ typedef struct {
+     blocking_queue *queue;
+ } message_subscriber;
+@@ -38,4 +42,8 @@ define_auto_cleanup_callback(message_subscriber_destroy, message_subscriber);
+ // define auto free macro for blocking queue
+ #define __isula_auto_subscriber auto_cleanup_tag(message_subscriber_destroy)
+ 
++#ifdef __cplusplus
++}
++#endif
++
+ #endif
+diff --git a/src/daemon/modules/runtime/shim/shim_rt_monitor.cc b/src/daemon/modules/runtime/shim/shim_rt_monitor.cc
+index 2547a206..97f5cd68 100644
+--- a/src/daemon/modules/runtime/shim/shim_rt_monitor.cc
++++ b/src/daemon/modules/runtime/shim/shim_rt_monitor.cc
+@@ -30,6 +30,8 @@
+ #include "utils.h"
+ #include "error.h"
+ 
++// The shim v2 header file needs to be modified to 
++// use extern "C" to wrap external functions.
+ extern "C" {
+ #include <shim_v2.h>
+ }
+diff --git a/src/daemon/nri/nri_adaption.h b/src/daemon/nri/nri_adaption.h
+index 27a6d93e..6bd41941 100644
+--- a/src/daemon/nri/nri_adaption.h
++++ b/src/daemon/nri/nri_adaption.h
+@@ -16,7 +16,6 @@
+ #ifndef DAEMON_NRI_PLUGIN_NRI_ADAPTION_H
+ #define DAEMON_NRI_PLUGIN_NRI_ADAPTION_H
+ 
+-// #include "read_write_lock.h"
+ #include <isula_libutils/nri_update_containers_request.h>
+ #include <isula_libutils/nri_update_containers_response.h>
+ 
+diff --git a/src/daemon/sandbox/sandbox_manager.cc b/src/daemon/sandbox/sandbox_manager.cc
+index cee444f4..4159993f 100644
+--- a/src/daemon/sandbox/sandbox_manager.cc
++++ b/src/daemon/sandbox/sandbox_manager.cc
+@@ -109,8 +109,7 @@ auto SandboxManager::CreateSandbox(const std::string &name, RuntimeInfo &info, s
+         return nullptr;
+     }
+ 
+-    sandbox = std::shared_ptr<Sandbox>(new Sandbox(id, m_rootdir, m_statedir, name, info, netMode, netNsPath,
+-                                                   sandboxConfig, image));
++    sandbox = std::make_shared<Sandbox>(id, m_rootdir, m_statedir, name, info, netMode, netNsPath, sandboxConfig, image);
+     if (sandbox == nullptr) {
+         ERROR("Failed to malloc for sandbox: %s", name.c_str());
+         error.Errorf("Failed to malloc for sandbox: %s", name.c_str());
+@@ -452,7 +451,7 @@ auto SandboxManager::LoadSandbox(std::string &id) -> std::shared_ptr<Sandbox>
+         return nullptr;
+     }
+ 
+-    sandbox = std::shared_ptr<Sandbox>(new Sandbox(id, m_rootdir, m_statedir));
++    sandbox = std::make_shared<Sandbox>(id, m_rootdir, m_statedir);
+     if (sandbox == nullptr) {
+         ERROR("Failed to malloc for sandboxes: %s", id.c_str());
+         return nullptr;
+diff --git a/src/utils/cutils/blocking_queue.h b/src/utils/cutils/blocking_queue.h
+index 257779c3..e6931501 100644
+--- a/src/utils/cutils/blocking_queue.h
++++ b/src/utils/cutils/blocking_queue.h
+@@ -26,7 +26,7 @@
+ extern "C" {
+ #endif
+ 
+-#define BLOCKING_QUEUE_NO_TIMEOUT -1
++#define BLOCKING_QUEUE_NO_TIMEOUT (-1)
+ 
+ typedef struct blocking_node {
+     void *data;
+diff --git a/src/utils/cutils/utils_aes.h b/src/utils/cutils/utils_aes.h
+index bd2c2065..8ff6dad8 100644
+--- a/src/utils/cutils/utils_aes.h
++++ b/src/utils/cutils/utils_aes.h
+@@ -26,7 +26,7 @@ extern "C" {
+ #define AES_256_CFB_KEY_LEN 32
+ #define AES_256_CFB_IV_LEN 16
+ 
+-int util_aes_key(const char *key_path, bool create, unsigned char *aeskey);
++int util_aes_key(const char *key_file, bool create, unsigned char *aeskey);
+ 
+ // note: Input bytes is "IV+data", "bytes+AES_256_CFB_IV_LEN" is the real data to be encoded.
+ //       The output length is the input "len" and add the '\0' after end of the length.
+diff --git a/src/utils/cutils/utils_cap.h b/src/utils/cutils/utils_cap.h
+index de63d070..c7e78ac2 100644
+--- a/src/utils/cutils/utils_cap.h
++++ b/src/utils/cutils/utils_cap.h
+@@ -16,14 +16,13 @@
+ #ifndef UTILS_CUTILS_UTILS_CAP_H
+ #define UTILS_CUTILS_UTILS_CAP_H
+ 
++#include <stdbool.h>
++#include <stddef.h>
++
+ #ifdef __cplusplus
+ extern "C" {
+ #endif
+ 
+-#include <stdbool.h>
+-#include <stddef.h>
+-#include <linux/capability.h>
+-
+ bool util_valid_cap(const char *cap);
+ 
+ /**
+diff --git a/src/utils/cutils/utils_fs.h b/src/utils/cutils/utils_fs.h
+index c44fed8c..438af416 100644
+--- a/src/utils/cutils/utils_fs.h
++++ b/src/utils/cutils/utils_fs.h
+@@ -34,7 +34,7 @@ bool util_detect_mounted(const char *path);
+ int util_ensure_mounted_as(const char *dst, const char *mntopts);
+ int util_mount_from(const char *base, const char *src, const char *dst, const char *mtype, const char *mntopts);
+ typedef int (*mount_info_call_back_t)(const char *, const char *);
+-bool util_deal_with_mount_info(mount_info_call_back_t cb, const char *);
++bool util_deal_with_mount_info(mount_info_call_back_t cb, const char *pattern);
+ bool util_check_readonly_fs(const char *path);
+ #ifdef __cplusplus
+ }
+diff --git a/src/utils/cutils/utils_network.c b/src/utils/cutils/utils_network.c
+index bb6a2f87..be33ec87 100644
+--- a/src/utils/cutils/utils_network.c
++++ b/src/utils/cutils/utils_network.c
+@@ -801,6 +801,8 @@ static bool is_invalid_char(char c)
+             return true;
+         case ' ':
+             return true;
++        default:
++            return false;
+     }
+     return false;
+ }
+diff --git a/src/utils/cutils/utils_string.h b/src/utils/cutils/utils_string.h
+index 0de2266c..d37343d5 100644
+--- a/src/utils/cutils/utils_string.h
++++ b/src/utils/cutils/utils_string.h
+@@ -17,7 +17,6 @@
+ #define UTILS_CUTILS_UTILS_STRING_H
+ #include <stdbool.h>
+ #include <stddef.h>
+-#include <stdint.h>
+ #include <sys/types.h>
+ 
+ #ifdef __cplusplus
+@@ -48,7 +47,7 @@ char **util_string_split(const char *src_str, char _sep);
+ // note that every delimiter bytes  is considered to be a single delimiter
+ char **util_string_split_multi(const char *src_str, char delim);
+ 
+-char **util_string_split_n(const char *src_str, char delim, size_t n);
++char **util_string_split_n(const char *src, char sep, size_t n);
+ 
+ const char *util_str_skip_str(const char *str, const char *skip);
+ 
+diff --git a/src/utils/tar/util_archive.h b/src/utils/tar/util_archive.h
+index 8f0ab2a4..98597d53 100644
+--- a/src/utils/tar/util_archive.h
++++ b/src/utils/tar/util_archive.h
+@@ -53,9 +53,8 @@ int archive_chroot_tar(const char *path, const char *file, const char *root_dir,
+ 
+ int archive_chroot_tar_stream(const char *chroot_dir, const char *tar_path, const char *src_base,
+                               const char *dst_base, const char *root_dir, struct io_read_wrapper *content);
+-int archive_chroot_untar_stream(const struct io_read_wrapper *content, const char *chroot_dir,
+-                                const char *untar_dir, const char *src_base, const char *dst_base,
+-                                const char *root_dir, char **errmsg);
++int archive_chroot_untar_stream(const struct io_read_wrapper *context, const char *chroot_dir, const char *untar_dir,
++                                const char *src_base, const char *dst_base, const char *root_dir, char **errmsg);
+ 
+ int archive_copy_oci_tar_split_and_ret_size(int src_fd, const char *dist_file, int64_t *ret_size);
+ 
+diff --git a/src/utils/tar/util_gzip.h b/src/utils/tar/util_gzip.h
+index 7d881e92..7797c5f9 100644
+--- a/src/utils/tar/util_gzip.h
++++ b/src/utils/tar/util_gzip.h
+@@ -26,7 +26,7 @@ extern "C" {
+ int util_gzip_z(const char *srcfile, const char *dstfile, const mode_t mode);
+ 
+ // Decompress
+-int util_gzip_d(const char *srcfile, const FILE *destfp);
++int util_gzip_d(const char *srcfile, const FILE *dstfp);
+ 
+ /*
+  * compress file.
+-- 
+2.25.1
+

0124-change-pull-registry-to-hub.oepkgs.net.patch

@@ -0,0 +1,137 @@
+From d6f7f7d3e2d644d2208ccc35f1de225b54c452a7 Mon Sep 17 00:00:00 2001
+From: zhongtao <zhongtao17@huawei.com>
+Date: Fri, 6 Sep 2024 17:45:58 +0800
+Subject: [PATCH 124/149] change pull registry to hub.oepkgs.net
+
+Signed-off-by: zhongtao <zhongtao17@huawei.com>
+---
+ CI/make-and-install.sh                        |  4 ++--
+ .../container_cases/test_data/daemon.json     |  2 +-
+ CI/test_cases/image_cases/image_digest.sh     |  6 ++---
+ CI/test_cases/image_cases/image_search.sh     |  2 +-
+ CI/test_cases/image_cases/registry.sh         | 22 +++++++++----------
+ 5 files changed, 18 insertions(+), 18 deletions(-)
+
+diff --git a/CI/make-and-install.sh b/CI/make-and-install.sh
+index 61281965..2c2a4241 100755
+--- a/CI/make-and-install.sh
++++ b/CI/make-and-install.sh
+@@ -95,7 +95,7 @@ cmake -DLIB_INSTALL_DIR=${restbuilddir}/lib -DCMAKE_INSTALL_PREFIX=${restbuilddi
+ make -j $(nproc)
+ make install
+ sed -i 's/"log-driver": "stdout"/"log-driver": "file"/g' ${restbuilddir}/etc/isulad/daemon.json
+-sed -i "/registry-mirrors/a\        \"https://3laho3y3.mirror.aliyuncs.com\"" ${restbuilddir}/etc/isulad/daemon.json
++sed -i "/registry-mirrors/a\        \"https://hub.oepkgs.net\"" ${restbuilddir}/etc/isulad/daemon.json
+ 
+ #build grpc version
+ cd $ISULAD_COPY_PATH
+@@ -110,4 +110,4 @@ fi
+ make -j $(nproc)
+ make install
+ sed -i 's/"log-driver": "stdout"/"log-driver": "file"/g' ${builddir}/etc/isulad/daemon.json
+-sed -i "/registry-mirrors/a\        \"https://3laho3y3.mirror.aliyuncs.com\"" ${builddir}/etc/isulad/daemon.json
++sed -i "/registry-mirrors/a\        \"https://hub.oepkgs.net\"" ${builddir}/etc/isulad/daemon.json
+diff --git a/CI/test_cases/container_cases/test_data/daemon.json b/CI/test_cases/container_cases/test_data/daemon.json
+index ab7d0360..20b001c0 100644
+--- a/CI/test_cases/container_cases/test_data/daemon.json
++++ b/CI/test_cases/container_cases/test_data/daemon.json
+@@ -24,7 +24,7 @@
+         "overlay2.override_kernel_check=true"
+     ],
+     "registry-mirrors": [
+-        "https://3laho3y3.mirror.aliyuncs.com"
++         "https://hub.oepkgs.net"
+     ],
+     "insecure-registries": [
+     ],
+diff --git a/CI/test_cases/image_cases/image_digest.sh b/CI/test_cases/image_cases/image_digest.sh
+index cc8b0e48..20774e07 100755
+--- a/CI/test_cases/image_cases/image_digest.sh
++++ b/CI/test_cases/image_cases/image_digest.sh
+@@ -25,9 +25,9 @@ source ../helpers.sh
+ function test_image_with_digest()
+ {
+   local ret=0
+-  local image="3laho3y3.mirror.aliyuncs.com/library/busybox"
+-  local image2="3laho3y3.mirror.aliyuncs.com/library/ubuntu"
+-  local image_digest="3laho3y3.mirror.aliyuncs.com/library/busybox@sha256:62ffc2ed7554e4c6d360bce40bbcf196573dd27c4ce080641a2c59867e732dee"
++  local image="hub.oepkgs.net/library/busybox"
++  local image2="hub.oepkgs.net/library/ubuntu"
++  local image_digest="hub.oepkgs.net/library/busybox@sha256:6066ca124f8c2686b7ae71aa1d6583b28c6dc3df3bdc386f2c89b92162c597d9"
+   local test="pull && inspect && tag image with digest test => (${FUNCNAME[@]})"
+ 
+   msg_info "${test} starting..."
+diff --git a/CI/test_cases/image_cases/image_search.sh b/CI/test_cases/image_cases/image_search.sh
+index 4bf0e099..9ac680ce 100755
+--- a/CI/test_cases/image_cases/image_search.sh
++++ b/CI/test_cases/image_cases/image_search.sh
+@@ -77,7 +77,7 @@ function test_image_search()
+ declare -i ans=0
+ 
+ # unable to pull image from docker.io without agent, skip this test
+-# registry API v1 is not implemented in https://3laho3y3.mirror.aliyuncs.com and isula search cannot be tested
++# registry API v1 is not implemented in https://hub.oepkgs.net and isula search cannot be tested
+ # test_image_search || ((ans++))
+ 
+ show_result ${ans} "${curr_path}/${0}"
+diff --git a/CI/test_cases/image_cases/registry.sh b/CI/test_cases/image_cases/registry.sh
+index 7ea9a0c5..e56d99d3 100755
+--- a/CI/test_cases/image_cases/registry.sh
++++ b/CI/test_cases/image_cases/registry.sh
+@@ -78,8 +78,8 @@ function isula_pull()
+     # isula pull docker.io/library/busybox:latest
+     # [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - --pull docker.io/library/busybox:latest failed" && ((ret++))
+ 
+-    isula pull 3laho3y3.mirror.aliyuncs.com/library/busybox
+-    fn_check_eq "$?" "0" "isula pull 3laho3y3.mirror.aliyuncs.com/library/busybox"
++    isula pull hub.oepkgs.net/library/busybox
++    fn_check_eq "$?" "0" "isula pull hub.oepkgs.net/library/busybox"
+ 
+     rm -f /etc/isulad/daemon.json.bak
+     cp /etc/isulad/daemon.json /etc/isulad/daemon.json.bak
+@@ -98,7 +98,7 @@ function isula_pull()
+     cp /etc/isulad/daemon.json.bak /etc/isulad/daemon.json
+     rm -f /etc/isulad/daemon.json.bak
+ 
+-    isula rmi 3laho3y3.mirror.aliyuncs.com/library/busybox
++    isula rmi hub.oepkgs.net/library/busybox
+ 
+     check_valgrind_log
+     fn_check_eq "$?" "0" "stop isulad with check valgrind"
+@@ -109,12 +109,12 @@ function isula_pull()
+ 
+ function isula_login()
+ {
+-    isula login -u test -p test 3laho3y3.mirror.aliyuncs.com
+-    fn_check_eq "$?" "0" "isula login -u test -p test 3laho3y3.mirror.aliyuncs.com"
++    isula login -u isulaci -p iSula123 hub.oepkgs.net
++    fn_check_eq "$?" "0" "isula login -u isulaci -p iSula123 hub.oepkgs.net"
+ 
+     # double login for memory leak check
+-    isula login -u test -p test 3laho3y3.mirror.aliyuncs.com
+-    fn_check_eq "$?" "0" "isula login -u test -p test 3laho3y3.mirror.aliyuncs.com"
++    isula login -u isulaci -p iSula123 hub.oepkgs.net
++    fn_check_eq "$?" "0" "isula login -u isulaci -p iSula123 hub.oepkgs.net"
+ 
+     # use username/password to pull busybox for memmory leak check
+     isula pull busybox
+@@ -123,12 +123,12 @@ function isula_login()
+ 
+ function isula_logout()
+ {
+-    isula logout 3laho3y3.mirror.aliyuncs.com
+-    fn_check_eq "$?" "0" "isula logout 3laho3y3.mirror.aliyuncs.com"
++    isula logout hub.oepkgs.net
++    fn_check_eq "$?" "0" "isula logout hub.oepkgs.net"
+ 
+     # double logout for memory leak check
+-    isula logout 3laho3y3.mirror.aliyuncs.com
+-    fn_check_eq "$?" "0" "isula logout 3laho3y3.mirror.aliyuncs.com"
++    isula logout hub.oepkgs.net
++    fn_check_eq "$?" "0" "isula logout hub.oepkgs.net"
+ }
+ 
+ function do_test_t()
+-- 
+2.25.1
+

0125-fix-clang-build-error.patch

@@ -0,0 +1,216 @@
+From d141d8bfc7a602b0f139bef42a1c73dc673687de Mon Sep 17 00:00:00 2001
+From: =?UTF-8?q?=E6=AD=A6=E7=A7=AF=E8=B6=85?= <wujichao1@huawei.com>
+Date: Mon, 21 Oct 2024 19:39:38 +0800
+Subject: [PATCH] fix-clang-build-error
+
+---
+ src/daemon/common/cri/cri_helpers.cc                   |  4 ++--
+ src/daemon/entry/cri/streams/stream_server.h           |  4 ++--
+ .../entry/cri/v1/v1_cri_container_manager_service.cc   |  2 +-
+ .../entry/cri/v1/v1_cri_image_manager_service_impl.cc  |  2 +-
+ .../entry/cri/v1/v1_cri_pod_sandbox_manager_service.cc | 10 +++++-----
+ src/daemon/entry/cri/v1/v1_cri_runtime_service_impl.h  |  2 +-
+ .../entry/cri/v1alpha/cri_container_manager_service.cc |  2 +-
+ .../cri/v1alpha/cri_pod_sandbox_manager_service.cc     |  6 +++---
+ src/daemon/sandbox/sandbox.cc                          |  2 +-
+ src/daemon/sandbox/sandbox_ops.cc                      |  2 +-
+ 10 files changed, 18 insertions(+), 18 deletions(-)
+
+diff --git a/src/daemon/common/cri/cri_helpers.cc b/src/daemon/common/cri/cri_helpers.cc
+index 8117403c..a8cbd996 100644
+--- a/src/daemon/common/cri/cri_helpers.cc
++++ b/src/daemon/common/cri/cri_helpers.cc
+@@ -525,8 +525,8 @@ void RemoveContainerLogSymlink(const std::string &containerID, Errors &error)
+     if (!path.empty()) {
+         // Only remove the symlink when container log path is specified.
+         if (util_path_remove(path.c_str()) != 0 && errno != ENOENT) {
+-            SYSERROR("Failed to remove container %s log symlink %s.", containerID.c_str(), path);
+-            error.Errorf("Failed to remove container %s log symlink %s.", containerID.c_str(), path);
++            SYSERROR("Failed to remove container %s log symlink %s.", containerID.c_str(), path.c_str());
++            error.Errorf("Failed to remove container %s log symlink %s.", containerID.c_str(), path.c_str());
+         }
+     }
+ }
+diff --git a/src/daemon/entry/cri/streams/stream_server.h b/src/daemon/entry/cri/streams/stream_server.h
+index 81aa9987..028dfc84 100644
+--- a/src/daemon/entry/cri/streams/stream_server.h
++++ b/src/daemon/entry/cri/streams/stream_server.h
+@@ -17,6 +17,8 @@
+ #include "errors.h"
+ #include "url.h"
+ 
++url::URLDatum cri_stream_server_url(void);
++
+ #ifdef __cplusplus
+ extern "C" {
+ #endif
+@@ -27,8 +29,6 @@ void cri_stream_server_wait(void);
+ 
+ void cri_stream_server_shutdown(void);
+ 
+-url::URLDatum cri_stream_server_url(void);
+-
+ #ifdef __cplusplus
+ }
+ #endif
+diff --git a/src/daemon/entry/cri/v1/v1_cri_container_manager_service.cc b/src/daemon/entry/cri/v1/v1_cri_container_manager_service.cc
+index d3fdd76a..1e84d14c 100644
+--- a/src/daemon/entry/cri/v1/v1_cri_container_manager_service.cc
++++ b/src/daemon/entry/cri/v1/v1_cri_container_manager_service.cc
+@@ -744,7 +744,7 @@ void ContainerManagerService::ListContainersToGRPC(container_list_response *resp
+             CRIHelpersV1::ContainerStatusToRuntime(Container_Status(response->containers[i]->status));
+         container->set_state(state);
+ 
+-        containers.push_back(move(container));
++        containers.push_back(std::move(container));
+     }
+ }
+ 
+diff --git a/src/daemon/entry/cri/v1/v1_cri_image_manager_service_impl.cc b/src/daemon/entry/cri/v1/v1_cri_image_manager_service_impl.cc
+index 71918706..561a40d5 100644
+--- a/src/daemon/entry/cri/v1/v1_cri_image_manager_service_impl.cc
++++ b/src/daemon/entry/cri/v1/v1_cri_image_manager_service_impl.cc
+@@ -149,7 +149,7 @@ void ImageManagerServiceImpl::list_images_to_grpc(im_list_response *response,
+ 
+         imagetool_image_summary *element = list_images->images[i];
+         conv_image_to_grpc(element, image);
+-        images.push_back(move(image));
++        images.push_back(std::move(image));
+     }
+ }
+ 
+diff --git a/src/daemon/entry/cri/v1/v1_cri_pod_sandbox_manager_service.cc b/src/daemon/entry/cri/v1/v1_cri_pod_sandbox_manager_service.cc
+index b629b1c3..a5f98619 100644
+--- a/src/daemon/entry/cri/v1/v1_cri_pod_sandbox_manager_service.cc
++++ b/src/daemon/entry/cri/v1/v1_cri_pod_sandbox_manager_service.cc
+@@ -536,7 +536,7 @@ auto PodSandboxManagerService::GetContainerListResponse(const std::string &readS
+     if (CRIHelpers::FiltersAddLabel(list_request->filters, CRIHelpers::Constants::SANDBOX_ID_LABEL_KEY,
+                                     readSandboxID) != 0) {
+         std::string tmp_errmsg = "Failed to add label in sandbox" + readSandboxID;
+-        ERROR(tmp_errmsg.c_str());
++        ERROR("%s", tmp_errmsg.c_str());
+         errors.push_back(tmp_errmsg);
+         return nullptr;
+     }
+@@ -551,7 +551,7 @@ auto PodSandboxManagerService::GetContainerListResponse(const std::string &readS
+     }
+     if (ret != 0) {
+         if (list_response != nullptr && list_response->errmsg != nullptr) {
+-            ERROR(list_response->errmsg);
++            ERROR("%s", list_response->errmsg);
+             errors.push_back(list_response->errmsg);
+         } else {
+             ERROR("Failed to call list container callback");
+@@ -1218,7 +1218,7 @@ void PodSandboxManagerService::PodSandboxStatsToGRPC(const std::string &id, cons
+         return;
+     }
+ 
+-    podStats = move(podStatsPtr);
++    podStats = std::move(podStatsPtr);
+     return;
+ }
+ 
+@@ -1227,7 +1227,7 @@ auto PodSandboxManagerService::PodSandboxStats(const std::string &podSandboxID,
+                                                Errors &error) -> std::unique_ptr<runtime::v1::PodSandboxStats>
+ {
+     Errors tmpErr;
+-    cgroup_metrics_t cgroupMetrics { 0 };
++    cgroup_metrics_t cgroupMetrics {{ 0 }};
+     std::vector<Network::NetworkInterfaceStats> netMetrics;
+     std::map<std::string, std::string> annotations;
+     std::unique_ptr<runtime::v1::PodSandboxStats> podStats { nullptr };
+@@ -1368,7 +1368,7 @@ void PodSandboxManagerService::ListPodSandboxStats(const runtime::v1::PodSandbox
+             continue;
+         }
+ 
+-        podsStats.push_back(move(podStats));
++        podsStats.push_back(std::move(podStats));
+     }
+ }
+ 
+diff --git a/src/daemon/entry/cri/v1/v1_cri_runtime_service_impl.h b/src/daemon/entry/cri/v1/v1_cri_runtime_service_impl.h
+index 3d93c7bb..33539a32 100644
+--- a/src/daemon/entry/cri/v1/v1_cri_runtime_service_impl.h
++++ b/src/daemon/entry/cri/v1/v1_cri_runtime_service_impl.h
+@@ -104,7 +104,7 @@ protected:
+ private:
+     std::string m_podSandboxImage;
+     std::shared_ptr<Network::PluginManager> m_pluginManager { nullptr };
+-    bool m_enablePodEvents;
++    [[maybe_unused]] bool m_enablePodEvents;
+ };
+ } // namespace CRIV1
+ #endif // DAEMON_ENTRY_CRI_V1_CRI_RUNTIME_SERVICE_IMPL_H
+diff --git a/src/daemon/entry/cri/v1alpha/cri_container_manager_service.cc b/src/daemon/entry/cri/v1alpha/cri_container_manager_service.cc
+index dbefa143..97acecd9 100644
+--- a/src/daemon/entry/cri/v1alpha/cri_container_manager_service.cc
++++ b/src/daemon/entry/cri/v1alpha/cri_container_manager_service.cc
+@@ -687,7 +687,7 @@ void ContainerManagerService::ListContainersToGRPC(container_list_response *resp
+             CRIHelpersV1Alpha::ContainerStatusToRuntime(Container_Status(response->containers[i]->status));
+         container->set_state(state);
+ 
+-        pods.push_back(move(container));
++        pods.push_back(std::move(container));
+     }
+ }
+ 
+diff --git a/src/daemon/entry/cri/v1alpha/cri_pod_sandbox_manager_service.cc b/src/daemon/entry/cri/v1alpha/cri_pod_sandbox_manager_service.cc
+index 1c343cda..3c128645 100644
+--- a/src/daemon/entry/cri/v1alpha/cri_pod_sandbox_manager_service.cc
++++ b/src/daemon/entry/cri/v1alpha/cri_pod_sandbox_manager_service.cc
+@@ -1573,7 +1573,7 @@ void PodSandboxManagerService::PodSandboxStatsToGRPC(const std::string &id, cons
+         return;
+     }
+ 
+-    podStats = move(podStatsPtr);
++    podStats = std::move(podStatsPtr);
+     return;
+ }
+ 
+@@ -1583,7 +1583,7 @@ auto PodSandboxManagerService::PodSandboxStats(const std::string &podSandboxID,
+ {
+     Errors tmpErr;
+     container_inspect *inspectData { nullptr };
+-    cgroup_metrics_t cgroupMetrics { 0 };
++    cgroup_metrics_t cgroupMetrics {{ 0 }};
+     std::vector<Network::NetworkInterfaceStats> netMetrics;
+     std::map<std::string, std::string> annotations;
+     std::unique_ptr<runtime::v1alpha2::PodSandboxStats> podStats { nullptr };
+@@ -1733,7 +1733,7 @@ void PodSandboxManagerService::ListPodSandboxStats(const runtime::v1alpha2::PodS
+             continue;
+         }
+ 
+-        podsStats.push_back(move(podStats));
++        podsStats.push_back(std::move(podStats));
+     }
+ }
+ 
+diff --git a/src/daemon/sandbox/sandbox.cc b/src/daemon/sandbox/sandbox.cc
+index d44abb99..dec082bc 100644
+--- a/src/daemon/sandbox/sandbox.cc
++++ b/src/daemon/sandbox/sandbox.cc
+@@ -847,7 +847,7 @@ auto Sandbox::SaveState(Errors &error) -> bool
+ 
+     nret = util_atomic_write_file(path.c_str(), stateJson.c_str(), stateJson.length(), CONFIG_FILE_MODE, false);
+     if (nret != 0) {
+-        SYSERROR("Failed to write file %s");
++        SYSERROR("Failed to write file %s", path.c_str());
+         error.Errorf("Failed to write file %s", path.c_str());
+         return false;
+     }
+diff --git a/src/daemon/sandbox/sandbox_ops.cc b/src/daemon/sandbox/sandbox_ops.cc
+index b7fb40bf..22cfea95 100644
+--- a/src/daemon/sandbox/sandbox_ops.cc
++++ b/src/daemon/sandbox/sandbox_ops.cc
+@@ -72,7 +72,7 @@ static int do_sandbox_prepare(const container_config_v2_common_config *config,
+ 
+     params.containerId = config->id;
+     params.execId = (nullptr == exec_id) ? "" : exec_id;
+-    params.spec = std::move(std::unique_ptr<std::string>(new std::string(oci_spec)));
++    params.spec = std::unique_ptr<std::string>(new std::string(oci_spec));
+ 
+     if (generate_ctrl_rootfs(params, config) != 0) {
+         ERROR("Invalid rootfs");
+-- 
+2.25.1
+

0126-add-a-new-registry-to-prevent-missing-mirrors.patch

@@ -0,0 +1,25 @@
+From a7a851f5be6c37665d948ec7587de062b6295bbe Mon Sep 17 00:00:00 2001
+From: zhongtao <zhongtao17@huawei.com>
+Date: Sat, 7 Sep 2024 11:24:44 +0800
+Subject: [PATCH 133/149] add a new registry to prevent missing mirrors
+
+Signed-off-by: zhongtao <zhongtao17@huawei.com>
+---
+ CI/test_cases/container_cases/test_data/daemon.json | 1 +
+ 1 file changed, 1 insertion(+)
+
+diff --git a/CI/test_cases/container_cases/test_data/daemon.json b/CI/test_cases/container_cases/test_data/daemon.json
+index 20b001c0..cf7e0b9d 100644
+--- a/CI/test_cases/container_cases/test_data/daemon.json
++++ b/CI/test_cases/container_cases/test_data/daemon.json
+@@ -24,6 +24,7 @@
+         "overlay2.override_kernel_check=true"
+     ],
+     "registry-mirrors": [
++        "https://docker.chenby.cn",
+          "https://hub.oepkgs.net"
+     ],
+     "insecure-registries": [
+-- 
+2.25.1
+