506 lines
14 KiB
Diff
506 lines
14 KiB
Diff
From ade879e6ccdc4c74a1c153f0750d2cd87ec8a4ec Mon Sep 17 00:00:00 2001
|
|
From: Jingxian He <hejingxian@huawei.com>
|
|
Date: Tue, 30 Nov 2021 10:26:10 +0800
|
|
Subject: [PATCH 52/72] optimization: parallel collecting vmas
|
|
|
|
In order to improve criu dump performance, make the collecting vmas
|
|
operation parallel run with the other collecting operations.
|
|
|
|
In order to prevent the concurrency problem by `find_unused_fd`, only
|
|
the main root task will parallel.
|
|
|
|
Usage:
|
|
criu --parallel
|
|
|
|
Note:
|
|
Ensure criu can use multi-core, otherwise the performance will
|
|
deterioration.
|
|
|
|
Signed-off-by: fu.lin <fulin10@huawei.com>
|
|
Signed-off-by: hewenliang <hewenliang4@huawei.com>
|
|
Signed-off-by: Jingxian He <hejingxian@huawei.com>
|
|
---
|
|
criu/Makefile.crtools | 1 +
|
|
criu/Makefile.packages | 1 +
|
|
criu/config.c | 1 +
|
|
criu/cr-dump.c | 53 +++++++++++-----
|
|
criu/crtools.c | 1 +
|
|
criu/include/cr_options.h | 1 +
|
|
criu/include/pstree.h | 3 +
|
|
criu/include/taskqueue.h | 50 +++++++++++++++
|
|
criu/namespaces.c | 9 ++-
|
|
criu/taskqueue.c | 124 ++++++++++++++++++++++++++++++++++++++
|
|
10 files changed, 228 insertions(+), 16 deletions(-)
|
|
create mode 100644 criu/include/taskqueue.h
|
|
create mode 100644 criu/taskqueue.c
|
|
|
|
diff --git a/criu/Makefile.crtools b/criu/Makefile.crtools
|
|
index 3bb7c19..2ad0207 100644
|
|
--- a/criu/Makefile.crtools
|
|
+++ b/criu/Makefile.crtools
|
|
@@ -97,6 +97,7 @@ obj-y += exit-notify.o
|
|
obj-y += reserved-ports.o
|
|
obj-y += orphan-inode.o
|
|
obj-y += kmsg.o
|
|
+obj-y += taskqueue.o
|
|
obj-$(CONFIG_HAS_LIBBPF) += bpfmap.o
|
|
obj-$(CONFIG_COMPAT) += pie-util-vdso-elf32.o
|
|
CFLAGS_pie-util-vdso-elf32.o += -DCONFIG_VDSO_32
|
|
diff --git a/criu/Makefile.packages b/criu/Makefile.packages
|
|
index 13c346f..851489b 100644
|
|
--- a/criu/Makefile.packages
|
|
+++ b/criu/Makefile.packages
|
|
@@ -31,6 +31,7 @@ REQ-RPM-PKG-TEST-NAMES += $(PYTHON)-pyyaml
|
|
endif
|
|
|
|
export LIBS += -lprotobuf-c -ldl -lnl-3 -lsoccr -Lsoccr/ -lnet
|
|
+export LIBS += -lpthread
|
|
|
|
check-packages-failed:
|
|
$(warning Can not find some of the required libraries)
|
|
diff --git a/criu/config.c b/criu/config.c
|
|
index ae5f81e..fdbc5eb 100644
|
|
--- a/criu/config.c
|
|
+++ b/criu/config.c
|
|
@@ -709,6 +709,7 @@ int parse_options(int argc, char **argv, bool *usage_error, bool *has_exec_cmd,
|
|
{ "share-dst-ports", required_argument, 0, 2000 },
|
|
{ "share-src-ports", required_argument, 0, 2001 },
|
|
{ "reserve-ports", required_argument, 0, 'P' },
|
|
+ BOOL_OPT("parallel", &opts.parallel),
|
|
{},
|
|
};
|
|
|
|
diff --git a/criu/cr-dump.c b/criu/cr-dump.c
|
|
index a8ab61e..ee826c0 100644
|
|
--- a/criu/cr-dump.c
|
|
+++ b/criu/cr-dump.c
|
|
@@ -17,6 +17,7 @@
|
|
|
|
#include <sched.h>
|
|
#include <sys/resource.h>
|
|
+#include <sys/sysinfo.h>
|
|
|
|
#include "types.h"
|
|
#include "protobuf.h"
|
|
@@ -90,6 +91,7 @@
|
|
#include "notifier.h"
|
|
#include "files-chr.h"
|
|
#include "reserved-ports.h"
|
|
+#include "taskqueue.h"
|
|
|
|
/*
|
|
* Architectures can overwrite this function to restore register sets that
|
|
@@ -424,7 +426,7 @@ static int dump_pid_misc(pid_t pid, TaskCoreEntry *tc)
|
|
return 0;
|
|
}
|
|
|
|
-static int dump_filemap(struct vma_area *vma_area, int fd)
|
|
+int dump_filemap(struct vma_area *vma_area, int fd)
|
|
{
|
|
struct fd_parms p = FD_PARMS_INIT;
|
|
VmaEntry *vma = vma_area->e;
|
|
@@ -1504,7 +1506,7 @@ err_cure:
|
|
static int dump_one_task(struct pstree_item *item, InventoryEntry *parent_ie)
|
|
{
|
|
pid_t pid = item->pid->real;
|
|
- struct vm_area_list vmas;
|
|
+ struct vm_area_list *vmas = NULL;
|
|
struct parasite_ctl *parasite_ctl;
|
|
int ret, exit_code = -1;
|
|
struct parasite_dump_misc misc;
|
|
@@ -1513,8 +1515,6 @@ static int dump_one_task(struct pstree_item *item, InventoryEntry *parent_ie)
|
|
struct proc_posix_timers_stat proc_args;
|
|
struct mem_dump_ctl mdc;
|
|
|
|
- vm_area_list_init(&vmas);
|
|
-
|
|
pr_info("========================================\n");
|
|
pr_info("Dumping task (pid: %d)\n", pid);
|
|
pr_info("========================================\n");
|
|
@@ -1525,12 +1525,23 @@ static int dump_one_task(struct pstree_item *item, InventoryEntry *parent_ie)
|
|
*/
|
|
return 0;
|
|
|
|
+ if (!opts.parallel || root_item->pid->real != item->pid->real ) {
|
|
+ vmas = xmalloc(sizeof(struct vm_area_list));
|
|
+ if (vmas == NULL) {
|
|
+ pr_err("xmalloc no memory\n");
|
|
+ return -1;
|
|
+ }
|
|
+ vm_area_list_init(vmas);
|
|
+ } else
|
|
+ vmas = item->maps_info.vmas;
|
|
+
|
|
pr_info("Obtaining task stat ... \n");
|
|
ret = parse_pid_stat(pid, &pps_buf);
|
|
if (ret < 0)
|
|
goto err;
|
|
|
|
- ret = collect_mappings(pid, &vmas, dump_filemap);
|
|
+ ret = (opts.parallel && root_item->pid->real == item->pid->real) ?
|
|
+ 0 : collect_mappings(pid, vmas, dump_filemap);
|
|
if (ret) {
|
|
pr_err("Collect mappings (pid: %d) failed with %d\n", pid, ret);
|
|
goto err;
|
|
@@ -1570,7 +1581,10 @@ static int dump_one_task(struct pstree_item *item, InventoryEntry *parent_ie)
|
|
goto err;
|
|
}
|
|
|
|
- parasite_ctl = parasite_infect_seized(pid, item, &vmas);
|
|
+ if (opts.parallel && end_collect_mappings_thread(item))
|
|
+ goto err;
|
|
+
|
|
+ parasite_ctl = parasite_infect_seized(pid, item, vmas);
|
|
if (!parasite_ctl) {
|
|
pr_err("Can't infect (pid: %d) with parasite\n", pid);
|
|
goto err;
|
|
@@ -1600,13 +1614,13 @@ static int dump_one_task(struct pstree_item *item, InventoryEntry *parent_ie)
|
|
goto err_cure_imgset;
|
|
}
|
|
|
|
- ret = parasite_fixup_vdso(parasite_ctl, pid, &vmas);
|
|
+ ret = parasite_fixup_vdso(parasite_ctl, pid, vmas);
|
|
if (ret) {
|
|
pr_err("Can't fixup vdso VMAs (pid: %d)\n", pid);
|
|
goto err_cure_imgset;
|
|
}
|
|
|
|
- ret = parasite_collect_aios(parasite_ctl, &vmas); /* FIXME -- merge with above */
|
|
+ ret = parasite_collect_aios(parasite_ctl, vmas); /* FIXME -- merge with above */
|
|
if (ret) {
|
|
pr_err("Failed to check aio rings (pid: %d)\n", pid);
|
|
goto err_cure_imgset;
|
|
@@ -1658,7 +1672,7 @@ static int dump_one_task(struct pstree_item *item, InventoryEntry *parent_ie)
|
|
mdc.stat = &pps_buf;
|
|
mdc.parent_ie = parent_ie;
|
|
|
|
- ret = parasite_dump_pages_seized(item, &vmas, &mdc, parasite_ctl);
|
|
+ ret = parasite_dump_pages_seized(item, vmas, &mdc, parasite_ctl);
|
|
if (ret)
|
|
goto err_cure;
|
|
|
|
@@ -1719,7 +1733,7 @@ static int dump_one_task(struct pstree_item *item, InventoryEntry *parent_ie)
|
|
goto err;
|
|
}
|
|
|
|
- ret = dump_task_mm(pid, &pps_buf, &misc, &vmas, cr_imgset);
|
|
+ ret = dump_task_mm(pid, &pps_buf, &misc, vmas, cr_imgset);
|
|
if (ret) {
|
|
pr_err("Dump mappings (pid: %d) failed with %d\n", pid, ret);
|
|
goto err;
|
|
@@ -1735,7 +1749,8 @@ static int dump_one_task(struct pstree_item *item, InventoryEntry *parent_ie)
|
|
exit_code = 0;
|
|
err:
|
|
close_pid_proc();
|
|
- free_mappings(&vmas);
|
|
+ free_mappings(vmas);
|
|
+ free(vmas);
|
|
xfree(dfds);
|
|
return exit_code;
|
|
|
|
@@ -1893,6 +1908,9 @@ int cr_pre_dump_tasks(pid_t pid)
|
|
if (opts.dump_char_dev && parse_devname() < 0)
|
|
goto err;
|
|
|
|
+ if (opts.parallel && init_parallel_env() != 0)
|
|
+ goto err;
|
|
+
|
|
root_item = alloc_pstree_item();
|
|
if (!root_item)
|
|
goto err;
|
|
@@ -2107,6 +2125,13 @@ static int cr_dump_finish(int ret)
|
|
write_stats(DUMP_STATS);
|
|
pr_info("Dumping finished successfully\n");
|
|
}
|
|
+
|
|
+ /*
|
|
+ * Don't care threads' status and ignore unfree resources, use
|
|
+ * `exit_group()` to ensure exit all threads.
|
|
+ */
|
|
+ syscall(SYS_exit_group, post_dump_ret ? : (ret != 0));
|
|
+
|
|
return post_dump_ret ?: (ret != 0);
|
|
}
|
|
|
|
@@ -2203,13 +2228,13 @@ int cr_dump_tasks(pid_t pid)
|
|
if (collect_file_locks())
|
|
goto err;
|
|
|
|
- if (collect_namespaces(true) < 0)
|
|
- goto err;
|
|
-
|
|
glob_imgset = cr_glob_imgset_open(O_DUMP);
|
|
if (!glob_imgset)
|
|
goto err;
|
|
|
|
+ if (collect_namespaces(true) < 0)
|
|
+ goto err;
|
|
+
|
|
if (seccomp_collect_dump_filters() < 0)
|
|
goto err;
|
|
|
|
diff --git a/criu/crtools.c b/criu/crtools.c
|
|
index cc0a18f..c20b3b7 100644
|
|
--- a/criu/crtools.c
|
|
+++ b/criu/crtools.c
|
|
@@ -472,6 +472,7 @@ usage:
|
|
" --weak-file-check Allow file size and mod larger than dumping value\n"
|
|
" --file-locks-repair Use repair mode to dump and restore file locks\n"
|
|
" --reserve-ports Reserve src ports in kernel\n"
|
|
+ " --parallel Collect smaps parallel to accellrate dumping speed\n"
|
|
"\n"
|
|
"Check options:\n"
|
|
" Without options, \"criu check\" checks availability of absolutely required\n"
|
|
diff --git a/criu/include/cr_options.h b/criu/include/cr_options.h
|
|
index 3b61c6b..6478d4d 100644
|
|
--- a/criu/include/cr_options.h
|
|
+++ b/criu/include/cr_options.h
|
|
@@ -201,6 +201,7 @@ struct cr_options {
|
|
char *share_dst_ports;
|
|
char *share_src_ports;
|
|
int reserve_ports;
|
|
+ int parallel;
|
|
};
|
|
|
|
extern struct cr_options opts;
|
|
diff --git a/criu/include/pstree.h b/criu/include/pstree.h
|
|
index 97bef11..87e4c47 100644
|
|
--- a/criu/include/pstree.h
|
|
+++ b/criu/include/pstree.h
|
|
@@ -1,6 +1,8 @@
|
|
#ifndef __CR_PSTREE_H__
|
|
#define __CR_PSTREE_H__
|
|
|
|
+#include "taskqueue.h"
|
|
+
|
|
#include "common/list.h"
|
|
#include "common/lock.h"
|
|
#include "pid.h"
|
|
@@ -31,6 +33,7 @@ struct pstree_item {
|
|
futex_t task_st;
|
|
unsigned long task_st_le_bits;
|
|
};
|
|
+ struct mappings_info maps_info;
|
|
};
|
|
|
|
static inline pid_t vpid(const struct pstree_item *i)
|
|
diff --git a/criu/include/taskqueue.h b/criu/include/taskqueue.h
|
|
new file mode 100644
|
|
index 0000000..16f9e3d
|
|
--- /dev/null
|
|
+++ b/criu/include/taskqueue.h
|
|
@@ -0,0 +1,50 @@
|
|
+#ifndef __CR_TASKQUEUE_H__
|
|
+#define __CR_TASKQUEUE_H__
|
|
+
|
|
+#include <stdbool.h>
|
|
+#include <pthread.h>
|
|
+#include <semaphore.h>
|
|
+
|
|
+#include "vma.h"
|
|
+#include "pstree.h"
|
|
+
|
|
+#include "common/list.h"
|
|
+
|
|
+#define TASKQUEUE_HASH_SIZE 8
|
|
+
|
|
+struct taskqueue {
|
|
+ pthread_t task;
|
|
+ void *(*routine)(void *);
|
|
+ void *arg;
|
|
+ int result;
|
|
+};
|
|
+#define queue_task queue.task
|
|
+#define queue_routine queue.routine
|
|
+#define queue_arg queue.arg
|
|
+#define queue_result queue.result
|
|
+
|
|
+int init_parallel_env(void);
|
|
+
|
|
+static inline int taskqueue_create(struct taskqueue *queue)
|
|
+{
|
|
+ return pthread_create(&queue->task, NULL, queue->routine, queue->arg);
|
|
+}
|
|
+
|
|
+static inline int taskqueue_join(struct taskqueue *queue)
|
|
+{
|
|
+ return pthread_join(queue->task, NULL);
|
|
+}
|
|
+
|
|
+/* parallel collect smaps */
|
|
+struct mappings_info {
|
|
+ struct hlist_node hash;
|
|
+ pid_t pid;
|
|
+ struct vm_area_list *vmas;
|
|
+ dump_filemap_t dump_file;
|
|
+ struct taskqueue queue;
|
|
+};
|
|
+
|
|
+int start_collect_mappings_thread(void);
|
|
+int end_collect_mappings_thread(struct pstree_item *item);
|
|
+
|
|
+#endif /* __CR_TASKQUEUE_H__ */
|
|
diff --git a/criu/namespaces.c b/criu/namespaces.c
|
|
index 7fa5868..05e6732 100644
|
|
--- a/criu/namespaces.c
|
|
+++ b/criu/namespaces.c
|
|
@@ -28,6 +28,7 @@
|
|
#include "cgroup.h"
|
|
#include "fdstore.h"
|
|
#include "kerndat.h"
|
|
+#include "taskqueue.h"
|
|
|
|
#include "protobuf.h"
|
|
#include "util.h"
|
|
@@ -1607,11 +1608,15 @@ int collect_namespaces(bool for_dump)
|
|
{
|
|
int ret;
|
|
|
|
- ret = collect_user_namespaces(for_dump);
|
|
+ ret = collect_mnt_namespaces(for_dump);
|
|
if (ret < 0)
|
|
return ret;
|
|
|
|
- ret = collect_mnt_namespaces(for_dump);
|
|
+ /* need mnt info provided by `mntinfo` */
|
|
+ if (opts.parallel && start_collect_mappings_thread())
|
|
+ return -1;
|
|
+
|
|
+ ret = collect_user_namespaces(for_dump);
|
|
if (ret < 0)
|
|
return ret;
|
|
|
|
diff --git a/criu/taskqueue.c b/criu/taskqueue.c
|
|
new file mode 100644
|
|
index 0000000..1196a5e
|
|
--- /dev/null
|
|
+++ b/criu/taskqueue.c
|
|
@@ -0,0 +1,124 @@
|
|
+/*
|
|
+ * Target:
|
|
+ * parallel dump process
|
|
+ */
|
|
+
|
|
+#include <string.h>
|
|
+#include <errno.h>
|
|
+#include <pthread.h>
|
|
+#include <sys/sysinfo.h>
|
|
+
|
|
+#include "pstree.h"
|
|
+#include "log.h"
|
|
+#include "taskqueue.h"
|
|
+
|
|
+/*
|
|
+ * Sometimes, only one cpu can be used which is bad for parallel routine.
|
|
+ * Therefore, set cpu affinity for criu routine.
|
|
+ */
|
|
+static int set_cpuaffinity(void)
|
|
+{
|
|
+ cpu_set_t *set;
|
|
+ int num_cpus = get_nprocs_conf();
|
|
+ size_t cpusetsize = CPU_ALLOC_SIZE(num_cpus);
|
|
+ int retval;
|
|
+
|
|
+ set = CPU_ALLOC(num_cpus);
|
|
+ memset(set, 0xff, cpusetsize);
|
|
+
|
|
+ retval = sched_setaffinity(getpid(), cpusetsize, set);
|
|
+ if (retval != 0)
|
|
+ pr_err("sched_setaffinity failed: %s\n", strerror(errno));
|
|
+
|
|
+ CPU_FREE(set);
|
|
+
|
|
+ return retval;
|
|
+}
|
|
+
|
|
+int init_parallel_env(void)
|
|
+{
|
|
+ return set_cpuaffinity();
|
|
+}
|
|
+
|
|
+static void *collect_mappings_routine(void *_arg)
|
|
+{
|
|
+ struct mappings_info *info = _arg;
|
|
+
|
|
+ info->queue_result = collect_mappings(info->pid, info->vmas, info->dump_file);
|
|
+ return NULL;
|
|
+}
|
|
+
|
|
+int dump_filemap(struct vma_area *vma_area, int fd); /* defined in criu/cr-dump.c */
|
|
+
|
|
+int start_collect_mappings_thread(void)
|
|
+{
|
|
+ struct pstree_item *pi;
|
|
+ struct mappings_info *info;
|
|
+
|
|
+ for_each_pstree_item(pi) {
|
|
+ /* disable parallel collect for non-root item because of the
|
|
+ * concurrence.
|
|
+ */
|
|
+ if (pi->pid->real != root_item->pid->real)
|
|
+ continue;
|
|
+
|
|
+ info = &pi->maps_info;
|
|
+
|
|
+ info->vmas = xmalloc(sizeof(struct vm_area_list));
|
|
+ if (info->vmas == NULL) {
|
|
+ pr_err("xzalloc vmas no memory\n");
|
|
+ return -1;
|
|
+ }
|
|
+ vm_area_list_init(info->vmas);
|
|
+
|
|
+ info->pid = pi->pid->real;
|
|
+ info->dump_file = dump_filemap;
|
|
+ info->queue_routine = collect_mappings_routine;
|
|
+ info->queue_arg = info;
|
|
+
|
|
+ pr_info("Start thread to collect %d mappings\n", info->pid);
|
|
+
|
|
+ if (taskqueue_create(&info->queue) < 0) {
|
|
+ pr_err("parallel_collect_mappings failed: %s\n", strerror(errno));
|
|
+ free(info->vmas);
|
|
+ /*
|
|
+ * Don't care other threads status, use `exit_group()`
|
|
+ * to ensure all threads exit.
|
|
+ */
|
|
+ return -1;
|
|
+ }
|
|
+ }
|
|
+
|
|
+ return 0;
|
|
+}
|
|
+
|
|
+int end_collect_mappings_thread(struct pstree_item *item)
|
|
+{
|
|
+ struct mappings_info *info = &item->maps_info;
|
|
+ int retval;
|
|
+
|
|
+ /* disable parallel collect for non-root item because of the
|
|
+ * concurrence.
|
|
+ */
|
|
+ if (root_item->pid->real != item->pid->real)
|
|
+ return 0;
|
|
+
|
|
+ retval = taskqueue_join(&info->queue);
|
|
+ if (retval != 0 || info->queue_result != 0) {
|
|
+ pr_err("taskqueue_join failed, retval %d(errno %d: %s),"
|
|
+ " queue_result: %d\n",
|
|
+ retval,
|
|
+ retval == 0 ? 0 : errno,
|
|
+ retval == 0 ? "nil" : strerror(errno),
|
|
+ info->queue_result);
|
|
+ retval = -1;
|
|
+ }
|
|
+
|
|
+ pr_info("End thread to collect %d mappings\n", info->pid);
|
|
+
|
|
+ /*
|
|
+ * Don't care other threads status, use `exit_group()` to ensure all
|
|
+ * threads exit.
|
|
+ */
|
|
+ return retval;
|
|
+}
|
|
--
|
|
2.34.1
|
|
|