From aac63cee766bb6840326d008ed1b1993bb7c629a Mon Sep 17 00:00:00 2001 From: Liu Chao Date: Mon, 19 Jul 2021 03:19:30 +0000 Subject: [PATCH 41/72] tcp: save src ports to ip_local_reserved_ports when dump tasks and retore it when restore tasks Signed-off-by: Liu Chao Signed-off-by: fu.lin --- criu/Makefile.crtools | 1 + criu/config.c | 8 ++- criu/cr-dump.c | 4 ++ criu/crtools.c | 1 + criu/include/cr_options.h | 1 + criu/include/reserved-ports.h | 10 ++++ criu/net.c | 6 +++ criu/reserved-ports.c | 98 +++++++++++++++++++++++++++++++++++ criu/sk-tcp.c | 2 +- 9 files changed, 129 insertions(+), 2 deletions(-) create mode 100644 criu/include/reserved-ports.h create mode 100644 criu/reserved-ports.c diff --git a/criu/Makefile.crtools b/criu/Makefile.crtools index 65cc215..3e522b4 100644 --- a/criu/Makefile.crtools +++ b/criu/Makefile.crtools @@ -94,6 +94,7 @@ obj-y += pin-mem.o obj-y += devname.o obj-y += files-chr.o obj-y += exit-notify.o +obj-y += reserved-ports.o obj-$(CONFIG_HAS_LIBBPF) += bpfmap.o obj-$(CONFIG_COMPAT) += pie-util-vdso-elf32.o CFLAGS_pie-util-vdso-elf32.o += -DCONFIG_VDSO_32 diff --git a/criu/config.c b/criu/config.c index 7e92731..ae5f81e 100644 --- a/criu/config.c +++ b/criu/config.c @@ -615,7 +615,7 @@ int parse_options(int argc, char **argv, bool *usage_error, bool *has_exec_cmd, "no-" OPT_NAME, no_argument, SAVE_TO, false \ } - static const char short_opts[] = "dSsRt:hD:o:v::x::Vr:jJ:lW:L:M:"; + static const char short_opts[] = "dSsRt:hD:o:v::x::Vr:jJ:lW:L:M:P:"; static struct option long_opts[] = { { "tree", required_argument, 0, 't' }, { "leave-stopped", no_argument, 0, 's' }, @@ -708,6 +708,7 @@ int parse_options(int argc, char **argv, bool *usage_error, bool *has_exec_cmd, BOOL_OPT("file-locks-repair", &opts.file_locks_repair), { "share-dst-ports", required_argument, 0, 2000 }, { "share-src-ports", required_argument, 0, 2001 }, + { "reserve-ports", required_argument, 0, 'P' }, {}, }; @@ -1057,6 +1058,11 @@ int parse_options(int argc, char **argv, bool *usage_error, bool *has_exec_cmd, case 'h': *usage_error = false; return 2; + case 'P': + opts.reserve_ports = atoi(optarg); + if (opts.reserve_ports < 0) + goto bad_arg; + break; default: return 2; } diff --git a/criu/cr-dump.c b/criu/cr-dump.c index 607eac2..a8ab61e 100644 --- a/criu/cr-dump.c +++ b/criu/cr-dump.c @@ -89,6 +89,7 @@ #include "pin-mem.h" #include "notifier.h" #include "files-chr.h" +#include "reserved-ports.h" /* * Architectures can overwrite this function to restore register sets that @@ -2223,6 +2224,9 @@ int cr_dump_tasks(pid_t pid) goto err; } + if (opts.reserve_ports > 0) + set_reserved_ports(); + if (parent_ie) { inventory_entry__free_unpacked(parent_ie, NULL); parent_ie = NULL; diff --git a/criu/crtools.c b/criu/crtools.c index cfa149a..ae858e8 100644 --- a/criu/crtools.c +++ b/criu/crtools.c @@ -460,6 +460,7 @@ usage: " --mask-exit-notify Mask task exit notify during dump and restore\n" " --weak-file-check Allow file size and mod larger than dumping value\n" " --file-locks-repair Use repair mode to dump and restore file locks\n" + " --reserve-ports Reserve src ports in kernel\n" "\n" "Check options:\n" " Without options, \"criu check\" checks availability of absolutely required\n" diff --git a/criu/include/cr_options.h b/criu/include/cr_options.h index b7c1e34..3b61c6b 100644 --- a/criu/include/cr_options.h +++ b/criu/include/cr_options.h @@ -200,6 +200,7 @@ struct cr_options { int file_locks_repair; char *share_dst_ports; char *share_src_ports; + int reserve_ports; }; extern struct cr_options opts; diff --git a/criu/include/reserved-ports.h b/criu/include/reserved-ports.h new file mode 100644 index 0000000..b614482 --- /dev/null +++ b/criu/include/reserved-ports.h @@ -0,0 +1,10 @@ +#ifndef __CRIU_RESERVED_PORTS_H__ +#define __CRIU_RESERVED_PORTS_H__ + +#define RESERVED_PORTS_PATH "/proc/sys/net/ipv4/ip_local_reserved_ports" + +extern void read_reserved_ports(char *path); +extern void write_reserved_ports(char *path); +extern void set_reserved_ports(void); + +#endif /* __CRIU_RESERVED_PORTS_H__ */ diff --git a/criu/net.c b/criu/net.c index 7b45f06..fff4c85 100644 --- a/criu/net.c +++ b/criu/net.c @@ -46,6 +46,7 @@ #include "external.h" #include "fdstore.h" #include "netfilter.h" +#include "reserved-ports.h" #include "protobuf.h" #include "images/netdev.pb-c.h" @@ -3193,6 +3194,11 @@ void network_unlock(void) { pr_info("Unlock network\n"); + if (opts.reserve_ports) { + read_reserved_ports("ip_local_reserved_ports"); + write_reserved_ports(RESERVED_PORTS_PATH); + } + cpt_unlock_tcp_connections(); rst_unlock_tcp_connections(); diff --git a/criu/reserved-ports.c b/criu/reserved-ports.c new file mode 100644 index 0000000..b4996ab --- /dev/null +++ b/criu/reserved-ports.c @@ -0,0 +1,98 @@ +#include +#include +#include +#include +#include + +#include "log.h" +#include "cr_options.h" +#include "util.h" +#include "sk-inet.h" +#include "reserved-ports.h" + +#include "common/list.h" + +static char* reserved_ports; +static int reserved_ports_num; +extern struct list_head cpt_tcp_repair_sockets; + +void read_reserved_ports(char *path) +{ + FILE *file = NULL; + char *ch = NULL; + size_t size = 0; + + if (reserved_ports) { + free(reserved_ports); + reserved_ports = NULL; + } + + file = fopen(path, "r"); + if (!file) { + pr_err("Cannot fopen %s\n", path); + return; + } + + if (getline(&reserved_ports, &size, file) <= 0) + pr_err("Cannot getline from %s\n", path); + fclose(file); + + if (!reserved_ports) + return; + + ch = strstr(reserved_ports, "\n"); + if (ch) + *ch = '\0'; +} + +void write_reserved_ports(char *path) +{ + int fd = -1; + char buf[PATH_MAX]; + + fd = open(path, O_RDWR | O_CREAT, 0640); + if (fd < 0) { + pr_err("Cannot open %s ret %d cwd: %s\n", path, fd, buf); + return; + } + + cr_system(-1, fd, -1, "/usr/bin/echo", + (char *[]) { "echo", reserved_ports, NULL}, 0); + close(fd); +} + +static int add_reserved_ports(struct inet_sk_desc *sk) +{ + if (reserved_ports_num >= opts.reserve_ports) + return -1; + + if (strlen(reserved_ports) == 0) + snprintf(reserved_ports, 6, "%u", sk->src_port); + else + snprintf(reserved_ports + strlen(reserved_ports), 7, ",%u", sk->src_port); + reserved_ports_num++; + + return 0; +} + +void set_reserved_ports(void) +{ + struct inet_sk_desc *sk = NULL; + size_t size = 0; + + read_reserved_ports(RESERVED_PORTS_PATH); + + write_reserved_ports("ip_local_reserved_ports"); + + size = strlen(reserved_ports) + 6 * opts.reserve_ports + 1; + if (xrealloc_safe(&reserved_ports, size)) + exit(1); + + list_for_each_entry(sk, &cpt_tcp_repair_sockets, rlist) + add_reserved_ports(sk); + + write_reserved_ports(RESERVED_PORTS_PATH); + + free(reserved_ports); + reserved_ports = NULL; +} diff --git a/criu/sk-tcp.c b/criu/sk-tcp.c index 0afecd2..38889d7 100644 --- a/criu/sk-tcp.c +++ b/criu/sk-tcp.c @@ -30,7 +30,7 @@ #undef LOG_PREFIX #define LOG_PREFIX "tcp: " -static LIST_HEAD(cpt_tcp_repair_sockets); +LIST_HEAD(cpt_tcp_repair_sockets); static LIST_HEAD(rst_tcp_repair_sockets); static int lock_connection(struct inet_sk_desc *sk) -- 2.34.1