From 5ff0e810f04de4b31f605ba3179dec3b3777978a Mon Sep 17 00:00:00 2001 From: "fu.lin" Date: Mon, 8 Nov 2021 15:08:12 +0800 Subject: [PATCH 28/72] infiniband: fix the infiniband fd conflict Phenomenon: Operating uverbs device will generate anonymous fd named `anon_inode:[infinibandevent]`. When `anon_inode:[infinibandevent]` fd is the last opened fd, and some kind of unix socket fd exist, which is generated by syscalls like `socketpair()` at the same tim, `anon_inode:[infinibandevent]` will restore fail probabilistically. log as the following: ``` (00.254523) 63959: open file flags:1 (00.254526) 63959: unix: Opening standalone (stage 0 id 0x1ff ino 1019605 peer 0) (00.254571) 63959: *******flags: 0 (00.254575) 63959: Create fd for 1408 # the fake fd (00.254578) 63959: *******flags: 1 (00.254580) 63959: Create fd for 445 # the restoration fd ``` Reason: During the restoration of unix socket, `socketpair()` will generate two fds, one is used to the current restoration, another is called fake fd which fd nr is owned by `find_unused_fd()`. When `anon_inode:[infinibandevent]` fd is the last one, criu don't dump the fd information for `anon_inode:[infinibandevent]` in original implementation, and criu think the fd nr which should belong to `anon_inode:[infinibandevent]` isn't used. Therefore, it cause the `anon_inode:[infinibandevent]` restoration fail. This patch fix the above problem. Core: dump `anon_inode:[infinibandevent]` fd information, make the criu is aware that that fd nr is used. Conflict:NA Reference:https://gitee.com/src-openeuler/criu/pulls/21 Signed-off-by: fu.lin --- criu/files-chr.c | 57 ++++++++++++++++++++++++++++++++++++ criu/files.c | 10 +++---- criu/include/files-chr.h | 8 +++++ criu/include/image-desc.h | 1 + criu/include/protobuf-desc.h | 1 + images/chr.proto | 3 ++ images/fdinfo.proto | 2 ++ 7 files changed, 76 insertions(+), 6 deletions(-) diff --git a/criu/files-chr.c b/criu/files-chr.c index 95d93e1..6d87c33 100644 --- a/criu/files-chr.c +++ b/criu/files-chr.c @@ -215,3 +215,60 @@ static int handle_hisi_vma(struct list_head *fds, struct vma_area *vma) return 0; } + +static void pr_info_infiniband(char *action, InfinibandEntry *infiniband) +{ + pr_info("%sinfiniband: id %#08x\n", action, infiniband->id); +} + +static int dump_one_infiniband(int lfd, u32 id, const struct fd_parms *p) +{ + FileEntry fe = FILE_ENTRY__INIT; + InfinibandEntry infiniband = INFINIBAND_ENTRY__INIT; + + infiniband.id = id; + + fe.type = FD_TYPES__INFINIBAND; + fe.id = infiniband.id; + fe.infiniband = &infiniband; + + pr_info_infiniband("Dumping ", &infiniband); + + return pb_write_one(img_from_set(glob_imgset, CR_FD_FILES), &fe, PB_FILE); +} + +const struct fdtype_ops infiniband_dump_ops = { + .type = FD_TYPES__INFINIBAND, + .dump = dump_one_infiniband, +}; + +static int infiniband_open(struct file_desc *d, int *new_fd) { + /* + * `*new_fd == -1` at this time, it means this open operation shouldn't + * be served out, which is why this function does nothing here. + */ + return 0; +}; + +static struct file_desc_ops infiniband_desc_ops = { + .type = FD_TYPES__INFINIBAND, + .open = infiniband_open, +}; + +static int collect_one_infiniband(void *o, ProtobufCMessage *base, struct cr_img *i) +{ + struct infiniband_file_info *info = o; + + info->infiniband = pb_msg(base, InfinibandEntry); + pr_info_infiniband("Collected ", info->infiniband); + + /* add the fd to `file_desc_hash` list to prevent from NULL pointer */ + return file_desc_add(&info->d, info->infiniband->id, &infiniband_desc_ops); +} + +struct collect_image_info infiniband_cinfo = { + .fd_type = CR_FD_INFINIBAND, + .pb_type = PB_INFINIBAND, + .priv_size = sizeof(struct infiniband_file_info), + .collect = collect_one_infiniband, +}; diff --git a/criu/files.c b/criu/files.c index 7b688f5..1ec5281 100644 --- a/criu/files.c +++ b/criu/files.c @@ -639,7 +639,7 @@ static int dump_one_file(struct pid *pid, int fd, int lfd, struct fd_opts *opts, else if (is_timerfd_link(link)) ops = &timerfd_dump_ops; else if (is_infiniband_link(link)) - return 1; + ops = &infiniband_dump_ops; #ifdef CONFIG_HAS_LIBBPF else if (is_bpfmap_link(link)) ops = &bpfmap_dump_ops; @@ -744,11 +744,6 @@ int dump_task_files_seized(struct parasite_ctl *ctl, struct pstree_item *item, s ret = dump_one_file(item->pid, dfds->fds[i + off], lfds[i], opts + i, ctl, &e, dfds); if (ret) break; - /* infiniband link file */ - if (ret > 0) { - ret = 0; - continue; - } ret = pb_write_one(img, &e, PB_FDINFO); if (ret) @@ -1864,6 +1859,9 @@ static int collect_one_file(void *o, ProtobufCMessage *base, struct cr_img *i) case FD_TYPES__CHR: ret = collect_one_file_entry(fe, fe->chr->id, &fe->chr->base, &chrfile_cinfo); break; + case FD_TYPES__INFINIBAND: + ret = collect_one_file_entry(fe, fe->infiniband->id, &fe->infiniband->base, &infiniband_cinfo); + break; } return ret; diff --git a/criu/include/files-chr.h b/criu/include/files-chr.h index 26b8fb2..261c4b2 100644 --- a/criu/include/files-chr.h +++ b/criu/include/files-chr.h @@ -38,4 +38,12 @@ enum hisi_sec_dev { HISI_SEC_DUS = 0x2000, }; +struct infiniband_file_info { + InfinibandEntry *infiniband; + struct file_desc d; +}; + +extern const struct fdtype_ops infiniband_dump_ops; +extern struct collect_image_info infiniband_cinfo; + #endif /* __CRIU_FILES_CHR_H__ */ diff --git a/criu/include/image-desc.h b/criu/include/image-desc.h index e35f8b2..9ad5fa0 100644 --- a/criu/include/image-desc.h +++ b/criu/include/image-desc.h @@ -116,6 +116,7 @@ enum { CR_FD_AUTOFS, CR_FD_CHRFILE, + CR_FD_INFINIBAND, CR_FD_MAX }; diff --git a/criu/include/protobuf-desc.h b/criu/include/protobuf-desc.h index 2468e8f..72a9e1d 100644 --- a/criu/include/protobuf-desc.h +++ b/criu/include/protobuf-desc.h @@ -71,6 +71,7 @@ enum { PB_BPFMAP_DATA, PB_APPARMOR, PB_CHRFILE, + PB_INFINIBAND, /* PB_AUTOGEN_STOP */ diff --git a/images/chr.proto b/images/chr.proto index 67929db..ed65005 100644 --- a/images/chr.proto +++ b/images/chr.proto @@ -10,3 +10,6 @@ message chrfile_entry { required bool repair = 5; }; +message infiniband_entry { + required uint32 id = 1; +}; diff --git a/images/fdinfo.proto b/images/fdinfo.proto index 6549472..eb52f35 100644 --- a/images/fdinfo.proto +++ b/images/fdinfo.proto @@ -44,6 +44,7 @@ enum fd_types { MEMFD = 18; BPFMAP = 19; CHR = 21; + INFINIBAND = 22; /* Any number above the real used. Not stored to image */ CTL_TTY = 65534; @@ -81,4 +82,5 @@ message file_entry { optional memfd_file_entry memfd = 20; optional bpfmap_file_entry bpf = 21; optional chrfile_entry chr = 23; + optional infiniband_entry infiniband = 25; } -- 2.34.1