!106 kata-containers:upgrade to 2.x
From: @Vanient Reviewed-by: @flyflyflypeng, @duguhaotian Signed-off-by: @duguhaotian
This commit is contained in:
openeuler-ci-bot
Gitee
commit
a40fe16a9f
Binary file not shown.
@ -1,20 +0,0 @@
|
|||||||
#!/bin/bash
|
|
||||||
|
|
||||||
if [[ -f ./patch_flag ]];then
|
|
||||||
echo "agent patched!"
|
|
||||||
exit 0
|
|
||||||
fi
|
|
||||||
|
|
||||||
tar -zxvf agent-1.11.1.tar.gz
|
|
||||||
cp -fr ./agent-1.11.1/* ./
|
|
||||||
rm -rf ./agent-1.11.1
|
|
||||||
cat ./series.conf | while read line
|
|
||||||
do
|
|
||||||
if [[ $line == '' || $line =~ ^\s*# ]]; then
|
|
||||||
continue
|
|
||||||
fi
|
|
||||||
echo "====patch $line======"
|
|
||||||
pwd
|
|
||||||
patch -p1 -F1 -s < ./patches/$line
|
|
||||||
done
|
|
||||||
touch ./patch_flag
|
|
||||||
@ -1,234 +0,0 @@
|
|||||||
From ac1d7806f8de2f8ca393df08a9c62d1045c4afdc Mon Sep 17 00:00:00 2001
|
|
||||||
From: jiangpengfei <jiangpengfei9@huawei.com>
|
|
||||||
Date: Tue, 11 Dec 2018 18:27:02 -0500
|
|
||||||
Subject: [PATCH 01/16] agent: add agent.netlink_recv_buf_size flag to set
|
|
||||||
netlink recv buf size
|
|
||||||
|
|
||||||
fixes: #813
|
|
||||||
|
|
||||||
reason: If hotplug huge size memory(for example 128GB) into guest,
|
|
||||||
kernel will produce a lot of memory add uevents and send to netlink socket,
|
|
||||||
however netlink socket default receive buffer size is 4KB, which is too small
|
|
||||||
to receive all memory add uevents.
|
|
||||||
Since hotplug huge size memory is not common case, so we consider add an agent
|
|
||||||
flag agent.netlink_recv_buf_size to set netlink socket recv buffer size.
|
|
||||||
|
|
||||||
Signed-off-by: jiangpengfei <jiangpengfei9@huawei.com>
|
|
||||||
---
|
|
||||||
README.md | 13 +++++++++++++
|
|
||||||
agent.go | 10 +++++++++-
|
|
||||||
config.go | 15 ++++++++++++++
|
|
||||||
config_test.go | 55 ++++++++++++++++++++++++++++++++++++++++++++++++++++
|
|
||||||
pkg/uevent/uevent.go | 15 +++++++++++---
|
|
||||||
5 files changed, 104 insertions(+), 4 deletions(-)
|
|
||||||
|
|
||||||
diff --git a/README.md b/README.md
|
|
||||||
index cec65a4..16f96a4 100644
|
|
||||||
--- a/README.md
|
|
||||||
+++ b/README.md
|
|
||||||
@@ -98,6 +98,19 @@ The pipe's capacity for stdout/stderr can be modified by specifying the `agent.c
|
|
||||||
to the guest kernel command line. For example, `agent.container_pipe_size=2097152` will set the stdout and stderr
|
|
||||||
pipes to 2097152 bytes.
|
|
||||||
|
|
||||||
+## Uevent Netlink Socket Receive Buffer Size
|
|
||||||
+
|
|
||||||
+When hotplugging a huge size memory into the Kata VM, the kernel in the VM will produce a lot of memory object add
|
|
||||||
+uevents and send all these uevents to Kata agent by netlink socket. However, default netlink socket receive buffer
|
|
||||||
+size is 4KB, which is too small and can only hold 256 memory add uevents. If memory add uevents number is larger
|
|
||||||
+than 256, the left uevents can not be received and processed by Kata agent.
|
|
||||||
+
|
|
||||||
+The uevent netlink socket receive buffer size can be modified by specifying the `agent.netlink_recv_buf_size` flag
|
|
||||||
+to the guest kernel command line. For example, `agent.netlink_recv_buf_size=2MB` will set the uevent netlink socket
|
|
||||||
+receive buffer size to 2MB value. `agent.netlink_recv_buf_size` valid value range is `[4KB ~ 4MB]` and value can be
|
|
||||||
+set in human-readable memory format or pure digital number format(default memory unit is byte).
|
|
||||||
+
|
|
||||||
+
|
|
||||||
[1]: https://github.com/firecracker-microvm/firecracker/blob/master/docs/vsock.md
|
|
||||||
[2]: https://golang.org/pkg/time/#ParseDuration
|
|
||||||
[3]: http://man7.org/linux/man-pages/man7/pipe.7.html
|
|
||||||
diff --git a/agent.go b/agent.go
|
|
||||||
index 2d2c293..c1cac08 100644
|
|
||||||
--- a/agent.go
|
|
||||||
+++ b/agent.go
|
|
||||||
@@ -190,6 +190,14 @@ var unifiedCgroupHierarchy = false
|
|
||||||
// Size in bytes of the stdout/stderr pipes created for each container.
|
|
||||||
var containerPipeSize = uint32(0)
|
|
||||||
|
|
||||||
+const (
|
|
||||||
+ minNetlinkSockRecvBufSize = 4 * 1024
|
|
||||||
+ maxNetlinkSockRecvBufSize = 4 * 1024 * 1024
|
|
||||||
+)
|
|
||||||
+
|
|
||||||
+// Size in bytes of the netlink socket recv buf size
|
|
||||||
+var netlinkSockRecvBufSize = uint32(0)
|
|
||||||
+
|
|
||||||
// commType is used to denote the communication channel type used.
|
|
||||||
type commType int
|
|
||||||
|
|
||||||
@@ -708,7 +716,7 @@ func (s *sandbox) waitForStopServer() {
|
|
||||||
func (s *sandbox) listenToUdevEvents() {
|
|
||||||
fieldLogger := agentLog.WithField("subsystem", "udevlistener")
|
|
||||||
|
|
||||||
- uEvHandler, err := uevent.NewHandler()
|
|
||||||
+ uEvHandler, err := uevent.NewHandler(netlinkSockRecvBufSize)
|
|
||||||
if err != nil {
|
|
||||||
fieldLogger.Warnf("Error starting uevent listening loop %s", err)
|
|
||||||
return
|
|
||||||
diff --git a/config.go b/config.go
|
|
||||||
index 4530096..6c7d473 100644
|
|
||||||
--- a/config.go
|
|
||||||
+++ b/config.go
|
|
||||||
@@ -7,11 +7,13 @@
|
|
||||||
package main
|
|
||||||
|
|
||||||
import (
|
|
||||||
+ "fmt"
|
|
||||||
"io/ioutil"
|
|
||||||
"strconv"
|
|
||||||
"strings"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
+ "github.com/docker/go-units"
|
|
||||||
"github.com/sirupsen/logrus"
|
|
||||||
"google.golang.org/grpc/codes"
|
|
||||||
grpcStatus "google.golang.org/grpc/status"
|
|
||||||
@@ -29,6 +31,7 @@ const (
|
|
||||||
hotplugTimeoutFlag = optionPrefix + "hotplug_timeout"
|
|
||||||
unifiedCgroupHierarchyFlag = optionPrefix + "unified_cgroup_hierarchy"
|
|
||||||
containerPipeSizeFlag = optionPrefix + "container_pipe_size"
|
|
||||||
+ netlinkSockRecvBufSizeFlag = optionPrefix + "netlink_recv_buf_size"
|
|
||||||
traceModeStatic = "static"
|
|
||||||
traceModeDynamic = "dynamic"
|
|
||||||
traceTypeIsolated = "isolated"
|
|
||||||
@@ -155,6 +158,18 @@ func parseCmdlineOption(option string) error {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
unifiedCgroupHierarchy = flag
|
|
||||||
+ case netlinkSockRecvBufSizeFlag:
|
|
||||||
+ bufSizeInBytes, err := units.RAMInBytes(split[valuePosition])
|
|
||||||
+ if err != nil {
|
|
||||||
+ return err
|
|
||||||
+ }
|
|
||||||
+
|
|
||||||
+ if bufSizeInBytes < minNetlinkSockRecvBufSize || bufSizeInBytes > maxNetlinkSockRecvBufSize {
|
|
||||||
+ return fmt.Errorf("invalid netlink socket recv buf size: %d (valid size range %s-%s bytes)", bufSizeInBytes,
|
|
||||||
+ units.BytesSize(minNetlinkSockRecvBufSize), units.BytesSize(maxNetlinkSockRecvBufSize))
|
|
||||||
+ }
|
|
||||||
+
|
|
||||||
+ netlinkSockRecvBufSize = uint32(bufSizeInBytes)
|
|
||||||
default:
|
|
||||||
if strings.HasPrefix(split[optionPosition], optionPrefix) {
|
|
||||||
return grpcStatus.Errorf(codes.NotFound, "Unknown option %s", split[optionPosition])
|
|
||||||
diff --git a/config_test.go b/config_test.go
|
|
||||||
index 2a23133..f40f17a 100644
|
|
||||||
--- a/config_test.go
|
|
||||||
+++ b/config_test.go
|
|
||||||
@@ -486,3 +486,58 @@ func TestParseCmdlineOptionContainerPipeSize(t *testing.T) {
|
|
||||||
assert.Equal(d.expectedContainerPipeSize, containerPipeSize, "test %d (%+v)", i, d)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
+
|
|
||||||
+func TestParseCmdlineOptionNetlinkSockRecvBufSize(t *testing.T) {
|
|
||||||
+ assert := assert.New(t)
|
|
||||||
+
|
|
||||||
+ type testData struct {
|
|
||||||
+ option string
|
|
||||||
+ shouldErr bool
|
|
||||||
+ expectedNetlinkSockRecvBufSize uint32
|
|
||||||
+ }
|
|
||||||
+
|
|
||||||
+ data := []testData{
|
|
||||||
+ {"", false, 0},
|
|
||||||
+ {"netlink_recv_buf_siz", false, 0},
|
|
||||||
+ {"netlink_recv_buf_size", false, 0},
|
|
||||||
+ {"netlink_recv_buf_size=", false, 0},
|
|
||||||
+ {"netlink_recv_buf_size=4096", false, 0},
|
|
||||||
+ {"netlink_recv_buf_size=4KB", false, 0},
|
|
||||||
+ {"agent.netlink_recv_buf_size=", true, 0},
|
|
||||||
+ {"agent.netlink_recv_buf_size=foobar", true, 0},
|
|
||||||
+ {"agent.netlink_recv_buf_size=-1", true, 0},
|
|
||||||
+ {"agent.netlink_recv_buf_size=0", true, 0},
|
|
||||||
+ {"agent.netlink_recv_buf_size=100", true, 0},
|
|
||||||
+ {"agent.netlink_recv_buf_size=3KB", true, 0},
|
|
||||||
+ {"agent.netlink_recv_buf_size=3.6KB", true, 0},
|
|
||||||
+ {"agent.netlink_recv_buf_size=4095", true, 0},
|
|
||||||
+ {"agent.netlink_recv_buf_size=4096xB", true, 0},
|
|
||||||
+ {"agent.netlink_recv_buf_size=4096", false, 4096},
|
|
||||||
+ {"agent.netlink_recv_buf_size=4097", false, 4097},
|
|
||||||
+ {"agent.netlink_recv_buf_size=4096.0", false, 4096},
|
|
||||||
+ {"agent.netlink_recv_buf_size=1024KB", false, 1048576},
|
|
||||||
+ {"agent.netlink_recv_buf_size=1MB", false, 1048576},
|
|
||||||
+ {"agent.netlink_recv_buf_size=4194303", false, 4194303},
|
|
||||||
+ {"agent.netlink_recv_buf_size=3.999MB", false, 4193255},
|
|
||||||
+ {"agent.netlink_recv_buf_size=4194304", false, 4194304},
|
|
||||||
+ {"agent.netlink_recv_buf_size=4MB", false, 4194304},
|
|
||||||
+ {"agent.netlink_recv_buf_size=4.001MB", true, 0},
|
|
||||||
+ {"agent.netlink_recv_buf_size=4194305", true, 0},
|
|
||||||
+ {"agent.netlink_recv_buf_size=100MB", true, 0},
|
|
||||||
+ {"agent.netlink_recv_buf_size=1GB", true, 0},
|
|
||||||
+ }
|
|
||||||
+
|
|
||||||
+ for i, d := range data {
|
|
||||||
+ // reset the netlink socket recv buffer size
|
|
||||||
+ netlinkSockRecvBufSize = 0
|
|
||||||
+
|
|
||||||
+ err := parseCmdlineOption(d.option)
|
|
||||||
+ if d.shouldErr {
|
|
||||||
+ assert.Error(err)
|
|
||||||
+ } else {
|
|
||||||
+ assert.NoError(err)
|
|
||||||
+ }
|
|
||||||
+
|
|
||||||
+ assert.Equal(d.expectedNetlinkSockRecvBufSize, netlinkSockRecvBufSize, "test %d (%+v)", i, d)
|
|
||||||
+ }
|
|
||||||
+}
|
|
||||||
diff --git a/pkg/uevent/uevent.go b/pkg/uevent/uevent.go
|
|
||||||
index fc2c127..fa84086 100644
|
|
||||||
--- a/pkg/uevent/uevent.go
|
|
||||||
+++ b/pkg/uevent/uevent.go
|
|
||||||
@@ -10,6 +10,7 @@ import (
|
|
||||||
"bufio"
|
|
||||||
"io"
|
|
||||||
"strings"
|
|
||||||
+ "syscall"
|
|
||||||
|
|
||||||
"golang.org/x/sys/unix"
|
|
||||||
"google.golang.org/grpc/codes"
|
|
||||||
@@ -33,7 +34,7 @@ type ReaderCloser struct {
|
|
||||||
}
|
|
||||||
|
|
||||||
// NewReaderCloser returns an io.ReadCloser handle for uevent.
|
|
||||||
-func NewReaderCloser() (io.ReadCloser, error) {
|
|
||||||
+func NewReaderCloser(netlinkRecvBufSize uint32) (io.ReadCloser, error) {
|
|
||||||
nl := unix.SockaddrNetlink{
|
|
||||||
Family: unix.AF_NETLINK,
|
|
||||||
// Passing Pid as 0 here allows the kernel to take care of assigning
|
|
||||||
@@ -47,6 +48,14 @@ func NewReaderCloser() (io.ReadCloser, error) {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
|
|
||||||
+ // If netlinkRecvBufSize > 0, set netlink socket recv buffer size to netlinkRecvBufSize
|
|
||||||
+ if netlinkRecvBufSize > 0 {
|
|
||||||
+ err = unix.SetsockoptInt(fd, syscall.SOL_SOCKET, syscall.SO_RCVBUFFORCE, int(netlinkRecvBufSize))
|
|
||||||
+ if err != nil {
|
|
||||||
+ return nil, err
|
|
||||||
+ }
|
|
||||||
+ }
|
|
||||||
+
|
|
||||||
if err := unix.Bind(fd, &nl); err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
@@ -85,8 +94,8 @@ type Handler struct {
|
|
||||||
}
|
|
||||||
|
|
||||||
// NewHandler returns a uevent handler.
|
|
||||||
-func NewHandler() (*Handler, error) {
|
|
||||||
- rdCloser, err := NewReaderCloser()
|
|
||||||
+func NewHandler(netlinkRecvBufSize uint32) (*Handler, error) {
|
|
||||||
+ rdCloser, err := NewReaderCloser(netlinkRecvBufSize)
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
--
|
|
||||||
2.14.3 (Apple Git-98)
|
|
||||||
|
|
||||||
@ -1,686 +0,0 @@
|
|||||||
From 13f54c768dcd7bf982dde8e57fb5cd624fedf5bc Mon Sep 17 00:00:00 2001
|
|
||||||
From: jiangpengfei <jiangpengfei9@huawei.com>
|
|
||||||
Date: Mon, 17 Aug 2020 11:23:55 +0800
|
|
||||||
Subject: [PATCH 02/16] network: support update routes incrementally
|
|
||||||
|
|
||||||
reason: add increment flag in the UpdateRoutesRequest to
|
|
||||||
support upate routes incrementally to improve the efficiency.
|
|
||||||
|
|
||||||
kata-network add-route and del-route needs this feature.
|
|
||||||
|
|
||||||
Signed-off-by: jiangpengfei <jiangpengfei9@huawei.com>
|
|
||||||
---
|
|
||||||
grpc.go | 2 +-
|
|
||||||
network.go | 74 ++++++++-
|
|
||||||
network_test.go | 16 +-
|
|
||||||
protocols/grpc/agent.pb.go | 402 +++++++++++++++++++++++++--------------------
|
|
||||||
protocols/grpc/agent.proto | 1 +
|
|
||||||
5 files changed, 300 insertions(+), 195 deletions(-)
|
|
||||||
|
|
||||||
diff --git a/grpc.go b/grpc.go
|
|
||||||
index 886661b..8fe8217 100644
|
|
||||||
--- a/grpc.go
|
|
||||||
+++ b/grpc.go
|
|
||||||
@@ -1556,7 +1556,7 @@ func (a *agentGRPC) UpdateInterface(ctx context.Context, req *pb.UpdateInterface
|
|
||||||
}
|
|
||||||
|
|
||||||
func (a *agentGRPC) UpdateRoutes(ctx context.Context, req *pb.UpdateRoutesRequest) (*pb.Routes, error) {
|
|
||||||
- return a.sandbox.updateRoutes(nil, req.Routes)
|
|
||||||
+ return a.sandbox.updateRoutes(nil, req.Routes, req.Increment)
|
|
||||||
}
|
|
||||||
|
|
||||||
func (a *agentGRPC) ListInterfaces(ctx context.Context, req *pb.ListInterfacesRequest) (*pb.Interfaces, error) {
|
|
||||||
diff --git a/network.go b/network.go
|
|
||||||
index 64a16a9..02e28cb 100644
|
|
||||||
--- a/network.go
|
|
||||||
+++ b/network.go
|
|
||||||
@@ -398,7 +398,7 @@ func (s *sandbox) deleteRoutes(netHandle *netlink.Handle) error {
|
|
||||||
// state which matches the requested routes. In doing this, preesxisting non-loopback routes will be
|
|
||||||
// removed from the network. If an error occurs, this function returns the list of routes in
|
|
||||||
// gRPC-route format at the time of failure
|
|
||||||
-func (s *sandbox) updateRoutes(netHandle *netlink.Handle, requestedRoutes *pb.Routes) (resultingRoutes *pb.Routes, err error) {
|
|
||||||
+func (s *sandbox) updateRoutes(netHandle *netlink.Handle, requestedRoutes *pb.Routes, increment bool) (resultingRoutes *pb.Routes, err error) {
|
|
||||||
if requestedRoutes == nil {
|
|
||||||
return nil, errNoRoutes
|
|
||||||
}
|
|
||||||
@@ -418,13 +418,66 @@ func (s *sandbox) updateRoutes(netHandle *netlink.Handle, requestedRoutes *pb.Ro
|
|
||||||
}
|
|
||||||
}()
|
|
||||||
|
|
||||||
+ var (
|
|
||||||
+ added []*types.Route
|
|
||||||
+ removed []*types.Route
|
|
||||||
+ )
|
|
||||||
+
|
|
||||||
+ defer func(netHandle *netlink.Handle) {
|
|
||||||
+ if err != nil {
|
|
||||||
+ // if error happens after route added, need to rollback the added route
|
|
||||||
+ if len(added) > 0 {
|
|
||||||
+ for _, r := range added {
|
|
||||||
+ errRb := s.updateRoute(netHandle, r, false)
|
|
||||||
+ if errRb != nil {
|
|
||||||
+ agentLog.WithError(err).Error("rollback route failed")
|
|
||||||
+ }
|
|
||||||
+ }
|
|
||||||
+ }
|
|
||||||
+
|
|
||||||
+ // if error happens after route removed, need to rollback the removed route
|
|
||||||
+ if len(removed) > 0 {
|
|
||||||
+ for _, r := range removed {
|
|
||||||
+ errRb := s.updateRoute(netHandle, r, true)
|
|
||||||
+ if errRb != nil {
|
|
||||||
+ agentLog.WithError(err).Error("rollback route failed")
|
|
||||||
+ }
|
|
||||||
+ }
|
|
||||||
+ }
|
|
||||||
+ }
|
|
||||||
+ }(netHandle)
|
|
||||||
+
|
|
||||||
+ // updateOneRoute just update the specified one route
|
|
||||||
+ updateOneRoute := func(netHandle *netlink.Handle, reqRoute *types.Route) error {
|
|
||||||
+ var add bool = true
|
|
||||||
+ if reqRoute.Dest != "" && strings.HasPrefix(reqRoute.Dest, "-") {
|
|
||||||
+ reqRoute.Dest = reqRoute.Dest[1:]
|
|
||||||
+ add = false
|
|
||||||
+ }
|
|
||||||
+ err = s.updateRoute(netHandle, reqRoute, add)
|
|
||||||
+ if err != nil {
|
|
||||||
+ agentLog.WithError(err).Error("update Route failed")
|
|
||||||
+ // If there was an error setting the route, return the error
|
|
||||||
+ // and the current routes on the system via the defer func
|
|
||||||
+ return err
|
|
||||||
+ }
|
|
||||||
+ if add {
|
|
||||||
+ added = append([]*types.Route{reqRoute}, added[:]...)
|
|
||||||
+ } else {
|
|
||||||
+ removed = append([]*types.Route{reqRoute}, removed[:]...)
|
|
||||||
+ }
|
|
||||||
+ return nil
|
|
||||||
+ }
|
|
||||||
+
|
|
||||||
//
|
|
||||||
// First things first, let's blow away all the existing routes. The updateRoutes function
|
|
||||||
// is designed to be declarative, so we will attempt to create state matching what is
|
|
||||||
// requested, and in the event that we fail to do so, will return the error and final state.
|
|
||||||
//
|
|
||||||
- if err = s.deleteRoutes(netHandle); err != nil {
|
|
||||||
- return nil, err
|
|
||||||
+ if !increment {
|
|
||||||
+ if err = s.deleteRoutes(netHandle); err != nil {
|
|
||||||
+ return nil, err
|
|
||||||
+ }
|
|
||||||
}
|
|
||||||
|
|
||||||
//
|
|
||||||
@@ -434,7 +487,12 @@ func (s *sandbox) updateRoutes(netHandle *netlink.Handle, requestedRoutes *pb.Ro
|
|
||||||
// won't be able to access the gateway
|
|
||||||
for _, reqRoute := range requestedRoutes.Routes {
|
|
||||||
if reqRoute.Gateway == "" {
|
|
||||||
- err = s.updateRoute(netHandle, reqRoute, true)
|
|
||||||
+ if increment {
|
|
||||||
+ err = updateOneRoute(netHandle, reqRoute)
|
|
||||||
+ } else {
|
|
||||||
+ err = s.updateRoute(netHandle, reqRoute, true)
|
|
||||||
+ }
|
|
||||||
+
|
|
||||||
if err != nil {
|
|
||||||
agentLog.WithError(err).Error("update Route failed")
|
|
||||||
//If there was an error setting the route, return the error
|
|
||||||
@@ -447,7 +505,11 @@ func (s *sandbox) updateRoutes(netHandle *netlink.Handle, requestedRoutes *pb.Ro
|
|
||||||
// Take a second pass and apply the routes which include a gateway
|
|
||||||
for _, reqRoute := range requestedRoutes.Routes {
|
|
||||||
if reqRoute.Gateway != "" {
|
|
||||||
- err = s.updateRoute(netHandle, reqRoute, true)
|
|
||||||
+ if increment {
|
|
||||||
+ err = updateOneRoute(netHandle, reqRoute)
|
|
||||||
+ } else {
|
|
||||||
+ err = s.updateRoute(netHandle, reqRoute, true)
|
|
||||||
+ }
|
|
||||||
if err != nil {
|
|
||||||
agentLog.WithError(err).Error("update Route failed")
|
|
||||||
//If there was an error setting the route, return the
|
|
||||||
@@ -699,4 +761,4 @@ func (s *sandbox) handleLocalhost() error {
|
|
||||||
}
|
|
||||||
|
|
||||||
return netlink.LinkSetUp(lo)
|
|
||||||
-}
|
|
||||||
+}
|
|
||||||
\ No newline at end of file
|
|
||||||
diff --git a/network_test.go b/network_test.go
|
|
||||||
index a143670..a1e58f5 100644
|
|
||||||
--- a/network_test.go
|
|
||||||
+++ b/network_test.go
|
|
||||||
@@ -160,7 +160,7 @@ func TestUpdateRoutes(t *testing.T) {
|
|
||||||
Routes: inputRoutesSimple,
|
|
||||||
}
|
|
||||||
|
|
||||||
- results, err := s.updateRoutes(netHandle, testRoutes)
|
|
||||||
+ results, err := s.updateRoutes(netHandle, testRoutes, false)
|
|
||||||
assert.Nil(t, err, "Unexpected update interface failure: %v", err)
|
|
||||||
assert.True(t, reflect.DeepEqual(results, testRoutes),
|
|
||||||
"Interface created didn't match: got %+v, expecting %+v", results, testRoutes)
|
|
||||||
@@ -173,7 +173,7 @@ func TestUpdateRoutes(t *testing.T) {
|
|
||||||
}
|
|
||||||
testRoutes.Routes = inputRoutesPTPExample
|
|
||||||
|
|
||||||
- results, err = s.updateRoutes(netHandle, testRoutes)
|
|
||||||
+ results, err = s.updateRoutes(netHandle, testRoutes, false)
|
|
||||||
assert.Nil(t, err, "Unexpected update interface failure: %v", err)
|
|
||||||
assert.True(t, reflect.DeepEqual(results, testRoutes),
|
|
||||||
"Interface created didn't match: got %+v, expecting %+v", results, testRoutes)
|
|
||||||
@@ -184,7 +184,7 @@ func TestUpdateRoutes(t *testing.T) {
|
|
||||||
{Dest: "192.168.0.0/16", Gateway: "", Source: "192.168.0.2", Scope: 0, Device: "ifc-name"},
|
|
||||||
}
|
|
||||||
testRoutes.Routes = inputRoutesNoScope
|
|
||||||
- results, err = s.updateRoutes(netHandle, testRoutes)
|
|
||||||
+ results, err = s.updateRoutes(netHandle, testRoutes, false)
|
|
||||||
assert.NotNil(t, err, "Expected to observe unreachable route failure")
|
|
||||||
|
|
||||||
assert.True(t, reflect.DeepEqual(results.Routes[0], testRoutes.Routes[1]),
|
|
||||||
@@ -231,7 +231,7 @@ func TestUpdateRoutesIPVlan(t *testing.T) {
|
|
||||||
}
|
|
||||||
testRoutes.Routes = inputRoutesIPVlanExample
|
|
||||||
|
|
||||||
- results, err := s.updateRoutes(netHandle, testRoutes)
|
|
||||||
+ results, err := s.updateRoutes(netHandle, testRoutes, false)
|
|
||||||
assert.Nil(t, err, "Unexpected update interface failure: %v", err)
|
|
||||||
assert.True(t, reflect.DeepEqual(results, testRoutes),
|
|
||||||
"Interface created didn't match: got %+v, expecting %+v", results, testRoutes)
|
|
||||||
@@ -357,7 +357,7 @@ func TestListRoutes(t *testing.T) {
|
|
||||||
Routes: inputRoutesSimple,
|
|
||||||
}
|
|
||||||
|
|
||||||
- _, err := s.updateRoutes(netHandle, testRoutes)
|
|
||||||
+ _, err := s.updateRoutes(netHandle, testRoutes, false)
|
|
||||||
assert.Nil(err)
|
|
||||||
results, err := s.listRoutes(nil)
|
|
||||||
assert.Nil(err, "Expected to list all routes")
|
|
||||||
@@ -377,7 +377,7 @@ func TestListRoutes(t *testing.T) {
|
|
||||||
Routes: inputRoutesSimple,
|
|
||||||
}
|
|
||||||
|
|
||||||
- _, err = s.updateRoutes(netHandle, testRoutes)
|
|
||||||
+ _, err = s.updateRoutes(netHandle, testRoutes, false)
|
|
||||||
assert.Nil(err)
|
|
||||||
results, err = s.listRoutes(nil)
|
|
||||||
assert.Nil(err, "Expected to list all routes")
|
|
||||||
@@ -438,7 +438,7 @@ func TestListRoutesWithIPV6(t *testing.T) {
|
|
||||||
Routes: inputRoutesSimple,
|
|
||||||
}
|
|
||||||
|
|
||||||
- _, err := s.updateRoutes(netHandle, testRoutes)
|
|
||||||
+ _, err := s.updateRoutes(netHandle, testRoutes, false)
|
|
||||||
assert.Nil(err)
|
|
||||||
results, err := s.listRoutes(nil)
|
|
||||||
assert.Nil(err, "Expected to list all routes")
|
|
||||||
@@ -514,7 +514,7 @@ func TestListRoutesWithTwoInterfacesSameSubnet(t *testing.T) {
|
|
||||||
Routes: inputRoutesSimple,
|
|
||||||
}
|
|
||||||
|
|
||||||
- _, err := s.updateRoutes(netHandle, testRoutes)
|
|
||||||
+ _, err := s.updateRoutes(netHandle, testRoutes, false)
|
|
||||||
assert.Nil(err)
|
|
||||||
results, err := s.listRoutes(nil)
|
|
||||||
assert.Nil(err, "Expected to list all routes")
|
|
||||||
diff --git a/protocols/grpc/agent.pb.go b/protocols/grpc/agent.pb.go
|
|
||||||
index 77e6d1b..1b887e5 100644
|
|
||||||
--- a/protocols/grpc/agent.pb.go
|
|
||||||
+++ b/protocols/grpc/agent.pb.go
|
|
||||||
@@ -1303,7 +1303,8 @@ func (m *UpdateInterfaceRequest) GetInterface() *types.Interface {
|
|
||||||
}
|
|
||||||
|
|
||||||
type UpdateRoutesRequest struct {
|
|
||||||
- Routes *Routes `protobuf:"bytes,1,opt,name=routes" json:"routes,omitempty"`
|
|
||||||
+ Routes *Routes `protobuf:"bytes,1,opt,name=routes" json:"routes,omitempty"`
|
|
||||||
+ Increment bool `protobuf:"varint,2,opt,name=increment,proto3" json:"increment,omitempty"`
|
|
||||||
}
|
|
||||||
|
|
||||||
func (m *UpdateRoutesRequest) Reset() { *m = UpdateRoutesRequest{} }
|
|
||||||
@@ -1318,6 +1319,13 @@ func (m *UpdateRoutesRequest) GetRoutes() *Routes {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
+func (m *UpdateRoutesRequest) GetIncrement() bool {
|
|
||||||
+ if m != nil {
|
|
||||||
+ return m.Increment
|
|
||||||
+ }
|
|
||||||
+ return false
|
|
||||||
+}
|
|
||||||
+
|
|
||||||
type ListInterfacesRequest struct {
|
|
||||||
}
|
|
||||||
|
|
||||||
@@ -4476,6 +4484,16 @@ func (m *UpdateRoutesRequest) MarshalTo(dAtA []byte) (int, error) {
|
|
||||||
}
|
|
||||||
i += n20
|
|
||||||
}
|
|
||||||
+ if m.Increment {
|
|
||||||
+ dAtA[i] = 0x10
|
|
||||||
+ i++
|
|
||||||
+ if m.Increment {
|
|
||||||
+ dAtA[i] = 1
|
|
||||||
+ } else {
|
|
||||||
+ dAtA[i] = 0
|
|
||||||
+ }
|
|
||||||
+ i++
|
|
||||||
+ }
|
|
||||||
return i, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
@@ -5751,6 +5769,9 @@ func (m *UpdateRoutesRequest) Size() (n int) {
|
|
||||||
l = m.Routes.Size()
|
|
||||||
n += 1 + l + sovAgent(uint64(l))
|
|
||||||
}
|
|
||||||
+ if m.Increment {
|
|
||||||
+ n += 2
|
|
||||||
+ }
|
|
||||||
return n
|
|
||||||
}
|
|
||||||
|
|
||||||
@@ -10964,6 +10985,26 @@ func (m *UpdateRoutesRequest) Unmarshal(dAtA []byte) error {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
iNdEx = postIndex
|
|
||||||
+ case 2:
|
|
||||||
+ if wireType != 0 {
|
|
||||||
+ return fmt.Errorf("proto: wrong wireType = %d for field Increment", wireType)
|
|
||||||
+ }
|
|
||||||
+ var v int
|
|
||||||
+ for shift := uint(0); ; shift += 7 {
|
|
||||||
+ if shift >= 64 {
|
|
||||||
+ return ErrIntOverflowAgent
|
|
||||||
+ }
|
|
||||||
+ if iNdEx >= l {
|
|
||||||
+ return io.ErrUnexpectedEOF
|
|
||||||
+ }
|
|
||||||
+ b := dAtA[iNdEx]
|
|
||||||
+ iNdEx++
|
|
||||||
+ v |= (int(b) & 0x7F) << shift
|
|
||||||
+ if b < 0x80 {
|
|
||||||
+ break
|
|
||||||
+ }
|
|
||||||
+ }
|
|
||||||
+ m.Increment = bool(v != 0)
|
|
||||||
default:
|
|
||||||
iNdEx = preIndex
|
|
||||||
skippy, err := skipAgent(dAtA[iNdEx:])
|
|
||||||
@@ -12852,184 +12893,185 @@ var (
|
|
||||||
func init() { proto.RegisterFile("agent.proto", fileDescriptorAgent) }
|
|
||||||
|
|
||||||
var fileDescriptorAgent = []byte{
|
|
||||||
- // 2862 bytes of a gzipped FileDescriptorProto
|
|
||||||
+ // 2876 bytes of a gzipped FileDescriptorProto
|
|
||||||
0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xc4, 0x39, 0x4b, 0x6f, 0x1c, 0xc7,
|
|
||||||
- 0xd1, 0xd8, 0x07, 0x97, 0xbb, 0xb5, 0x2f, 0x6e, 0x93, 0xa2, 0x56, 0x2b, 0x5b, 0x9f, 0x3c, 0xb6,
|
|
||||||
- 0x65, 0xfa, 0x73, 0xbc, 0xb4, 0x65, 0x23, 0x7e, 0xc1, 0x11, 0xc4, 0x47, 0x44, 0xc6, 0x56, 0xc4,
|
|
||||||
- 0x0c, 0x45, 0x38, 0x40, 0x10, 0x0c, 0x86, 0x33, 0xcd, 0x65, 0x9b, 0x3b, 0xd3, 0xe3, 0x9e, 0x1e,
|
|
||||||
- 0x8a, 0xeb, 0x00, 0x39, 0x26, 0xb7, 0x5c, 0x02, 0xe4, 0x96, 0x3f, 0x10, 0xe4, 0x96, 0x63, 0xae,
|
|
||||||
- 0x39, 0x18, 0x39, 0xe5, 0x17, 0x04, 0x81, 0x7f, 0x42, 0x7e, 0x41, 0xd0, 0xaf, 0x79, 0xec, 0x0e,
|
|
||||||
- 0x29, 0x84, 0x20, 0x90, 0xcb, 0xa2, 0xab, 0xba, 0xba, 0x5e, 0xdd, 0x55, 0x53, 0x55, 0x0b, 0x6d,
|
|
||||||
- 0x77, 0x82, 0x43, 0x3e, 0x8e, 0x18, 0xe5, 0x14, 0xd5, 0x27, 0x2c, 0xf2, 0x46, 0x2d, 0xea, 0x11,
|
|
||||||
- 0x85, 0x18, 0xfd, 0x70, 0x42, 0xf8, 0x69, 0x72, 0x3c, 0xf6, 0x68, 0xb0, 0x79, 0xe6, 0x72, 0xf7,
|
|
||||||
- 0x5d, 0x8f, 0x86, 0xdc, 0x25, 0x21, 0x66, 0xf1, 0xa6, 0x3c, 0xb8, 0x19, 0x9d, 0x4d, 0x36, 0xf9,
|
|
||||||
- 0x2c, 0xc2, 0xb1, 0xfa, 0xd5, 0xe7, 0xee, 0x4e, 0x28, 0x9d, 0x4c, 0xf1, 0xa6, 0x84, 0x8e, 0x93,
|
|
||||||
- 0x93, 0x4d, 0x1c, 0x44, 0x7c, 0xa6, 0x36, 0xad, 0x3f, 0x56, 0x61, 0x7d, 0x9b, 0x61, 0x97, 0xe3,
|
|
||||||
- 0x6d, 0xc3, 0xcd, 0xc6, 0xdf, 0x24, 0x38, 0xe6, 0xe8, 0x35, 0xe8, 0xa4, 0x12, 0x1c, 0xe2, 0x0f,
|
|
||||||
- 0x2b, 0xf7, 0x2b, 0x1b, 0x2d, 0xbb, 0x9d, 0xe2, 0xf6, 0x7d, 0x74, 0x1b, 0x96, 0xf1, 0x05, 0xf6,
|
|
||||||
- 0xc4, 0x6e, 0x55, 0xee, 0x36, 0x04, 0xb8, 0xef, 0xa3, 0xf7, 0xa1, 0x1d, 0x73, 0x46, 0xc2, 0x89,
|
|
||||||
- 0x93, 0xc4, 0x98, 0x0d, 0x6b, 0xf7, 0x2b, 0x1b, 0xed, 0x87, 0x2b, 0x63, 0x61, 0xd2, 0xf8, 0x50,
|
|
||||||
- 0x6e, 0x1c, 0xc5, 0x98, 0xd9, 0x10, 0xa7, 0x6b, 0xf4, 0x00, 0x96, 0x7d, 0x7c, 0x4e, 0x3c, 0x1c,
|
|
||||||
- 0x0f, 0xeb, 0xf7, 0x6b, 0x1b, 0xed, 0x87, 0x1d, 0x45, 0xbe, 0x23, 0x91, 0xb6, 0xd9, 0x44, 0x6f,
|
|
||||||
- 0x43, 0x33, 0xe6, 0x94, 0xb9, 0x13, 0x1c, 0x0f, 0x97, 0x24, 0x61, 0xd7, 0xf0, 0x95, 0x58, 0x3b,
|
|
||||||
- 0xdd, 0x46, 0xaf, 0x40, 0xed, 0xd9, 0xf6, 0xfe, 0xb0, 0x21, 0xa5, 0x83, 0xa6, 0x8a, 0xb0, 0x67,
|
|
||||||
- 0x0b, 0x34, 0x7a, 0x1d, 0xba, 0xb1, 0x1b, 0xfa, 0xc7, 0xf4, 0xc2, 0x89, 0x88, 0x1f, 0xc6, 0xc3,
|
|
||||||
- 0xe5, 0xfb, 0x95, 0x8d, 0xa6, 0xdd, 0xd1, 0xc8, 0x03, 0x81, 0xb3, 0x3e, 0x85, 0x5b, 0x87, 0xdc,
|
|
||||||
- 0x65, 0xfc, 0x1a, 0xde, 0xb1, 0x8e, 0x60, 0xdd, 0xc6, 0x01, 0x3d, 0xbf, 0x96, 0x6b, 0x87, 0xb0,
|
|
||||||
- 0xcc, 0x49, 0x80, 0x69, 0xc2, 0xa5, 0x6b, 0xbb, 0xb6, 0x01, 0xad, 0x3f, 0x57, 0x00, 0xed, 0x5e,
|
|
||||||
- 0x60, 0xef, 0x80, 0x51, 0x0f, 0xc7, 0xf1, 0xff, 0xe8, 0xba, 0xde, 0x82, 0xe5, 0x48, 0x29, 0x30,
|
|
||||||
- 0xac, 0x4b, 0x72, 0x7d, 0x0b, 0x46, 0x2b, 0xb3, 0x6b, 0x7d, 0x0d, 0x6b, 0x87, 0x64, 0x12, 0xba,
|
|
||||||
- 0xd3, 0x1b, 0xd4, 0x77, 0x1d, 0x1a, 0xb1, 0xe4, 0x29, 0x55, 0xed, 0xda, 0x1a, 0xb2, 0x0e, 0x00,
|
|
||||||
- 0x7d, 0xe5, 0x12, 0x7e, 0x73, 0x92, 0xac, 0x77, 0x61, 0xb5, 0xc0, 0x31, 0x8e, 0x68, 0x18, 0x63,
|
|
||||||
- 0xa9, 0x00, 0x77, 0x79, 0x12, 0x4b, 0x66, 0x4b, 0xb6, 0x86, 0x2c, 0x0c, 0x6b, 0x5f, 0x92, 0xd8,
|
|
||||||
- 0x90, 0xe3, 0xff, 0x46, 0x85, 0x75, 0x68, 0x9c, 0x50, 0x16, 0xb8, 0xdc, 0x68, 0xa0, 0x20, 0x84,
|
|
||||||
- 0xa0, 0xee, 0xb2, 0x49, 0x3c, 0xac, 0xdd, 0xaf, 0x6d, 0xb4, 0x6c, 0xb9, 0x16, 0xaf, 0x72, 0x4e,
|
|
||||||
- 0x8c, 0xd6, 0xeb, 0x35, 0xe8, 0x68, 0xbf, 0x3b, 0x53, 0x12, 0x73, 0x29, 0xa7, 0x63, 0xb7, 0x35,
|
|
||||||
- 0x4e, 0x9c, 0xb1, 0x28, 0xac, 0x1f, 0x45, 0xfe, 0x35, 0x03, 0xfe, 0x21, 0xb4, 0x18, 0x8e, 0x69,
|
|
||||||
- 0xc2, 0x44, 0x98, 0x56, 0xe5, 0xbd, 0xaf, 0xa9, 0x7b, 0xff, 0x92, 0x84, 0xc9, 0x85, 0x6d, 0xf6,
|
|
||||||
- 0xec, 0x8c, 0x4c, 0x87, 0x10, 0x8f, 0xaf, 0x13, 0x42, 0x9f, 0xc2, 0xad, 0x03, 0x37, 0x89, 0xaf,
|
|
||||||
- 0xa3, 0xab, 0xf5, 0x99, 0x08, 0xbf, 0x38, 0x09, 0xae, 0x75, 0xf8, 0x4f, 0x15, 0x68, 0x6e, 0x47,
|
|
||||||
- 0xc9, 0x51, 0xec, 0x4e, 0x30, 0xfa, 0x3f, 0x68, 0x73, 0xca, 0xdd, 0xa9, 0x93, 0x08, 0x50, 0x92,
|
|
||||||
- 0xd7, 0x6d, 0x90, 0x28, 0x45, 0x20, 0xdc, 0x8e, 0x99, 0x17, 0x25, 0x9a, 0xa2, 0x7a, 0xbf, 0xb6,
|
|
||||||
- 0x51, 0xb7, 0xdb, 0x0a, 0xa7, 0x48, 0xc6, 0xb0, 0x2a, 0xf7, 0x1c, 0x12, 0x3a, 0x67, 0x98, 0x85,
|
|
||||||
- 0x78, 0x1a, 0x50, 0x1f, 0xcb, 0xf7, 0x5b, 0xb7, 0x07, 0x72, 0x6b, 0x3f, 0xfc, 0x22, 0xdd, 0x40,
|
|
||||||
- 0xff, 0x0f, 0x83, 0x94, 0x5e, 0x04, 0xa5, 0xa4, 0xae, 0x4b, 0xea, 0xbe, 0xa6, 0x3e, 0xd2, 0x68,
|
|
||||||
- 0xeb, 0xd7, 0xd0, 0x7b, 0x7e, 0xca, 0x28, 0xe7, 0x53, 0x12, 0x4e, 0x76, 0x5c, 0xee, 0x8a, 0xec,
|
|
||||||
- 0x11, 0x61, 0x46, 0xa8, 0x1f, 0x6b, 0x6d, 0x0d, 0x88, 0xde, 0x81, 0x01, 0x57, 0xb4, 0xd8, 0x77,
|
|
||||||
- 0x0c, 0x4d, 0x55, 0xd2, 0xac, 0xa4, 0x1b, 0x07, 0x9a, 0xf8, 0x4d, 0xe8, 0x65, 0xc4, 0x22, 0xff,
|
|
||||||
- 0x68, 0x7d, 0xbb, 0x29, 0xf6, 0x39, 0x09, 0xb0, 0x75, 0x2e, 0x7d, 0x25, 0x2f, 0x19, 0xbd, 0x03,
|
|
||||||
- 0xad, 0xcc, 0x0f, 0x15, 0xf9, 0x42, 0x7a, 0xea, 0x85, 0x18, 0x77, 0xda, 0xcd, 0xd4, 0x29, 0x9f,
|
|
||||||
- 0x43, 0x9f, 0xa7, 0x8a, 0x3b, 0xbe, 0xcb, 0xdd, 0xe2, 0xa3, 0x2a, 0x5a, 0x65, 0xf7, 0x78, 0x01,
|
|
||||||
- 0xb6, 0x3e, 0x83, 0xd6, 0x01, 0xf1, 0x63, 0x25, 0x78, 0x08, 0xcb, 0x5e, 0xc2, 0x18, 0x0e, 0xb9,
|
|
||||||
- 0x31, 0x59, 0x83, 0x68, 0x0d, 0x96, 0xa6, 0x24, 0x20, 0x5c, 0x9b, 0xa9, 0x00, 0x8b, 0x02, 0x3c,
|
|
||||||
- 0xc5, 0x01, 0x65, 0x33, 0xe9, 0xb0, 0x35, 0x58, 0xca, 0x5f, 0xae, 0x02, 0xd0, 0x5d, 0x68, 0x05,
|
|
||||||
- 0xee, 0x45, 0x7a, 0xa9, 0x62, 0xa7, 0x19, 0xb8, 0x17, 0x4a, 0xf9, 0x21, 0x2c, 0x9f, 0xb8, 0x64,
|
|
||||||
- 0xea, 0x85, 0x5c, 0x7b, 0xc5, 0x80, 0x99, 0xc0, 0x7a, 0x5e, 0xe0, 0xdf, 0xaa, 0xd0, 0x56, 0x12,
|
|
||||||
- 0x95, 0xc2, 0x6b, 0xb0, 0xe4, 0xb9, 0xde, 0x69, 0x2a, 0x52, 0x02, 0xe8, 0x81, 0x51, 0xa4, 0x9a,
|
|
||||||
- 0x4f, 0xc2, 0x99, 0xa6, 0x46, 0xb5, 0x4d, 0x80, 0xf8, 0x85, 0x1b, 0x69, 0xdd, 0x6a, 0x97, 0x10,
|
|
||||||
- 0xb7, 0x04, 0x8d, 0x52, 0xf7, 0x03, 0xe8, 0xa8, 0x77, 0xa7, 0x8f, 0xd4, 0x2f, 0x39, 0xd2, 0x56,
|
|
||||||
- 0x54, 0xea, 0xd0, 0xeb, 0xd0, 0x4d, 0x62, 0xec, 0x9c, 0x12, 0xcc, 0x5c, 0xe6, 0x9d, 0xce, 0x86,
|
|
||||||
- 0x4b, 0xea, 0x1b, 0x99, 0xc4, 0x78, 0xcf, 0xe0, 0xd0, 0x43, 0x58, 0x12, 0xe9, 0x2f, 0x1e, 0x36,
|
|
||||||
- 0xe4, 0xe7, 0xf8, 0x95, 0x3c, 0x4b, 0x69, 0xea, 0x58, 0xfe, 0xee, 0x86, 0x9c, 0xcd, 0x6c, 0x45,
|
|
||||||
- 0x3a, 0xfa, 0x18, 0x20, 0x43, 0xa2, 0x15, 0xa8, 0x9d, 0xe1, 0x99, 0x8e, 0x43, 0xb1, 0x14, 0xce,
|
|
||||||
- 0x39, 0x77, 0xa7, 0x89, 0xf1, 0xba, 0x02, 0x3e, 0xad, 0x7e, 0x5c, 0xb1, 0x3c, 0xe8, 0x6f, 0x4d,
|
|
||||||
- 0xcf, 0x08, 0xcd, 0x1d, 0x5f, 0x83, 0xa5, 0xc0, 0xfd, 0x9a, 0x32, 0xe3, 0x49, 0x09, 0x48, 0x2c,
|
|
||||||
- 0x09, 0x29, 0x33, 0x2c, 0x24, 0x80, 0x7a, 0x50, 0xa5, 0x91, 0xf4, 0x57, 0xcb, 0xae, 0xd2, 0x28,
|
|
||||||
- 0x13, 0x54, 0xcf, 0x09, 0xb2, 0xfe, 0x59, 0x07, 0xc8, 0xa4, 0x20, 0x1b, 0x46, 0x84, 0x3a, 0x31,
|
|
||||||
- 0x66, 0xa2, 0x04, 0x71, 0x8e, 0x67, 0x1c, 0xc7, 0x0e, 0xc3, 0x5e, 0xc2, 0x62, 0x72, 0x2e, 0xee,
|
|
||||||
- 0x4f, 0x98, 0x7d, 0x4b, 0x99, 0x3d, 0xa7, 0x9b, 0x7d, 0x9b, 0xd0, 0x43, 0x75, 0x6e, 0x4b, 0x1c,
|
|
||||||
- 0xb3, 0xcd, 0x29, 0xb4, 0x0f, 0xb7, 0x32, 0x9e, 0x7e, 0x8e, 0x5d, 0xf5, 0x2a, 0x76, 0xab, 0x29,
|
|
||||||
- 0x3b, 0x3f, 0x63, 0xb5, 0x0b, 0xab, 0x84, 0x3a, 0xdf, 0x24, 0x38, 0x29, 0x30, 0xaa, 0x5d, 0xc5,
|
|
||||||
- 0x68, 0x40, 0xe8, 0xcf, 0xe4, 0x81, 0x8c, 0xcd, 0x01, 0xdc, 0xc9, 0x59, 0x29, 0xc2, 0x3d, 0xc7,
|
|
||||||
- 0xac, 0x7e, 0x15, 0xb3, 0xf5, 0x54, 0x2b, 0x91, 0x0f, 0x32, 0x8e, 0x3f, 0x81, 0x75, 0x42, 0x9d,
|
|
||||||
- 0x17, 0x2e, 0xe1, 0xf3, 0xec, 0x96, 0x5e, 0x62, 0xa4, 0xf8, 0xe8, 0x16, 0x79, 0x29, 0x23, 0x03,
|
|
||||||
- 0xcc, 0x26, 0x05, 0x23, 0x1b, 0x2f, 0x31, 0xf2, 0xa9, 0x3c, 0x90, 0xb1, 0x79, 0x0c, 0x03, 0x42,
|
|
||||||
- 0xe7, 0xb5, 0x59, 0xbe, 0x8a, 0x49, 0x9f, 0xd0, 0xa2, 0x26, 0x5b, 0x30, 0x88, 0xb1, 0xc7, 0x29,
|
|
||||||
- 0xcb, 0x3f, 0x82, 0xe6, 0x55, 0x2c, 0x56, 0x34, 0x7d, 0xca, 0xc3, 0xfa, 0x05, 0x74, 0xf6, 0x92,
|
|
||||||
- 0x09, 0xe6, 0xd3, 0xe3, 0x34, 0x19, 0xdc, 0x58, 0xfe, 0xb1, 0xfe, 0x5d, 0x85, 0xf6, 0xf6, 0x84,
|
|
||||||
- 0xd1, 0x24, 0x2a, 0xe4, 0x64, 0x15, 0xa4, 0xf3, 0x39, 0x59, 0x92, 0xc8, 0x9c, 0xac, 0x88, 0x3f,
|
|
||||||
- 0x84, 0x4e, 0x20, 0x43, 0x57, 0xd3, 0xab, 0x3c, 0x34, 0x58, 0x08, 0x6a, 0xbb, 0x1d, 0xe4, 0x92,
|
|
||||||
- 0xd9, 0x18, 0x20, 0x22, 0x7e, 0xac, 0xcf, 0xa8, 0x74, 0xd4, 0xd7, 0x15, 0xa1, 0x49, 0xd1, 0x76,
|
|
||||||
- 0x2b, 0x4a, 0xb3, 0xf5, 0xfb, 0xd0, 0x3e, 0x16, 0x4e, 0xd2, 0x07, 0x0a, 0xc9, 0x28, 0xf3, 0x9e,
|
|
||||||
- 0x0d, 0xc7, 0x59, 0x10, 0xee, 0x41, 0xf7, 0x54, 0xb9, 0x4c, 0x1f, 0x52, 0x6f, 0xe8, 0x75, 0x6d,
|
|
||||||
- 0x49, 0x66, 0xef, 0x38, 0xef, 0x59, 0x75, 0x01, 0x9d, 0xd3, 0x1c, 0x6a, 0x74, 0x08, 0x83, 0x05,
|
|
||||||
- 0x92, 0x92, 0x1c, 0xb4, 0x91, 0xcf, 0x41, 0xed, 0x87, 0x48, 0x09, 0xca, 0x9f, 0xcc, 0xe7, 0xa5,
|
|
||||||
- 0xdf, 0x55, 0xa1, 0xf3, 0x53, 0xcc, 0x5f, 0x50, 0x76, 0xa6, 0xf4, 0x45, 0x50, 0x0f, 0xdd, 0x00,
|
|
||||||
- 0x6b, 0x8e, 0x72, 0x8d, 0xee, 0x40, 0x93, 0x5d, 0xa8, 0x04, 0xa2, 0xef, 0x73, 0x99, 0x5d, 0xc8,
|
|
||||||
- 0xc4, 0x80, 0x5e, 0x05, 0x60, 0x17, 0x4e, 0xe4, 0x7a, 0x67, 0x58, 0x7b, 0xb0, 0x6e, 0xb7, 0xd8,
|
|
||||||
- 0xc5, 0x81, 0x42, 0x88, 0xa7, 0xc0, 0x2e, 0x1c, 0xcc, 0x18, 0x65, 0xb1, 0xce, 0x55, 0x4d, 0x76,
|
|
||||||
- 0xb1, 0x2b, 0x61, 0x7d, 0xd6, 0x67, 0x34, 0x8a, 0xb0, 0x2f, 0x73, 0xb4, 0x3c, 0xbb, 0xa3, 0x10,
|
|
||||||
- 0x42, 0x2a, 0x37, 0x52, 0x1b, 0x4a, 0x2a, 0xcf, 0xa4, 0xf2, 0x4c, 0xea, 0xb2, 0x3a, 0xc9, 0xf3,
|
|
||||||
- 0x52, 0x79, 0x2a, 0xb5, 0xa9, 0xa4, 0xf2, 0x9c, 0x54, 0x9e, 0x49, 0x6d, 0x99, 0xb3, 0x5a, 0xaa,
|
|
||||||
- 0xf5, 0xdb, 0x0a, 0xac, 0xcf, 0x17, 0x7e, 0xba, 0x4c, 0xfd, 0x10, 0x3a, 0x9e, 0xbc, 0xaf, 0xc2,
|
|
||||||
- 0x9b, 0x1c, 0x2c, 0xdc, 0xa4, 0xdd, 0xf6, 0x72, 0xcf, 0xf8, 0x23, 0xe8, 0x86, 0xca, 0xc1, 0xe9,
|
|
||||||
- 0xd3, 0xac, 0x65, 0xf7, 0x92, 0xf7, 0xbd, 0xdd, 0x09, 0x73, 0x90, 0xe5, 0x03, 0xfa, 0x8a, 0x11,
|
|
||||||
- 0x8e, 0x0f, 0x39, 0xc3, 0x6e, 0x70, 0x13, 0x0d, 0x08, 0x82, 0xba, 0xac, 0x56, 0x6a, 0xb2, 0xbe,
|
|
||||||
- 0x96, 0x6b, 0xeb, 0x2d, 0x58, 0x2d, 0x48, 0xd1, 0xb6, 0xae, 0x40, 0x6d, 0x8a, 0x43, 0xc9, 0xbd,
|
|
||||||
- 0x6b, 0x8b, 0xa5, 0xe5, 0xc2, 0xc0, 0xc6, 0xae, 0x7f, 0x73, 0xda, 0x68, 0x11, 0xb5, 0x4c, 0xc4,
|
|
||||||
- 0x06, 0xa0, 0xbc, 0x08, 0xad, 0x8a, 0xd1, 0xba, 0x92, 0xd3, 0xfa, 0x19, 0x0c, 0xb6, 0xa7, 0x34,
|
|
||||||
- 0xc6, 0x87, 0xdc, 0x27, 0xe1, 0x4d, 0x74, 0x4c, 0xbf, 0x82, 0xd5, 0xe7, 0x7c, 0xf6, 0x95, 0x60,
|
|
||||||
- 0x16, 0x93, 0x6f, 0xf1, 0x0d, 0xd9, 0xc7, 0xe8, 0x0b, 0x63, 0x1f, 0xa3, 0x2f, 0x44, 0xb3, 0xe4,
|
|
||||||
- 0xd1, 0x69, 0x12, 0x84, 0x32, 0x14, 0xba, 0xb6, 0x86, 0xac, 0x2d, 0xe8, 0xa8, 0x1a, 0xfa, 0x29,
|
|
||||||
- 0xf5, 0x93, 0x29, 0x2e, 0x8d, 0xc1, 0x7b, 0x00, 0x91, 0xcb, 0xdc, 0x00, 0x73, 0xcc, 0xd4, 0x1b,
|
|
||||||
- 0x6a, 0xd9, 0x39, 0x8c, 0xf5, 0x87, 0x2a, 0xac, 0xa9, 0x91, 0xc8, 0xa1, 0x9a, 0x04, 0x18, 0x13,
|
|
||||||
- 0x46, 0xd0, 0x3c, 0xa5, 0x31, 0xcf, 0x31, 0x4c, 0x61, 0xa1, 0xa2, 0x1f, 0x1a, 0x6e, 0x62, 0x59,
|
|
||||||
- 0x98, 0x53, 0xd4, 0xae, 0x9e, 0x53, 0x2c, 0x4c, 0x22, 0xea, 0x8b, 0x93, 0x08, 0x11, 0x6d, 0x86,
|
|
||||||
- 0x88, 0xa8, 0x18, 0x6f, 0xd9, 0x2d, 0x8d, 0xd9, 0xf7, 0xd1, 0x03, 0xe8, 0x4f, 0x84, 0x96, 0xce,
|
|
||||||
- 0x29, 0xa5, 0x67, 0x4e, 0xe4, 0xf2, 0x53, 0x19, 0xea, 0x2d, 0xbb, 0x2b, 0xd1, 0x7b, 0x94, 0x9e,
|
|
||||||
- 0x1d, 0xb8, 0xfc, 0x14, 0x7d, 0x02, 0x3d, 0x5d, 0x06, 0x06, 0xd2, 0x45, 0xb1, 0xfe, 0xf8, 0xe9,
|
|
||||||
- 0x28, 0xca, 0x7b, 0xcf, 0xee, 0x9e, 0xe5, 0xa0, 0xd8, 0xba, 0x0d, 0xb7, 0x76, 0x70, 0xcc, 0x19,
|
|
||||||
- 0x9d, 0x15, 0x1d, 0x63, 0xfd, 0x08, 0x60, 0x3f, 0xe4, 0x98, 0x9d, 0xb8, 0x1e, 0x8e, 0xd1, 0x7b,
|
|
||||||
- 0x79, 0x48, 0x17, 0x47, 0x2b, 0x63, 0x35, 0x91, 0x4a, 0x37, 0xec, 0x1c, 0x8d, 0x35, 0x86, 0x86,
|
|
||||||
- 0x4d, 0x13, 0x91, 0x8e, 0xde, 0x30, 0x2b, 0x7d, 0xae, 0xa3, 0xcf, 0x49, 0xa4, 0xad, 0xf7, 0xac,
|
|
||||||
- 0x3d, 0xd3, 0xc2, 0x66, 0xec, 0xf4, 0x15, 0x8d, 0xa1, 0x45, 0x0c, 0x4e, 0x67, 0x95, 0x45, 0xd1,
|
|
||||||
- 0x19, 0x89, 0xf5, 0x19, 0xac, 0x2a, 0x4e, 0x8a, 0xb3, 0x61, 0xf3, 0x06, 0x34, 0x98, 0x51, 0xa3,
|
|
||||||
- 0x92, 0x8d, 0xa2, 0x34, 0x91, 0xde, 0x13, 0xfe, 0x10, 0x1d, 0x75, 0x66, 0x88, 0xf1, 0xc7, 0x2a,
|
|
||||||
- 0x0c, 0xc4, 0x46, 0x81, 0xa7, 0xf5, 0x4b, 0x58, 0x7d, 0x16, 0x4e, 0x49, 0x88, 0xb7, 0x0f, 0x8e,
|
|
||||||
- 0x9e, 0xe2, 0x34, 0xee, 0x11, 0xd4, 0x45, 0x7d, 0x24, 0x05, 0x35, 0x6d, 0xb9, 0x16, 0x81, 0x10,
|
|
||||||
- 0x1e, 0x3b, 0x5e, 0x94, 0xc4, 0x7a, 0xf6, 0xd3, 0x08, 0x8f, 0xb7, 0xa3, 0x24, 0x16, 0x89, 0x5c,
|
|
||||||
- 0x7c, 0xc8, 0x69, 0x38, 0x9d, 0xc9, 0x68, 0x68, 0xda, 0xcb, 0x5e, 0x94, 0x3c, 0x0b, 0xa7, 0x33,
|
|
||||||
- 0xeb, 0x07, 0xb2, 0xdb, 0xc5, 0xd8, 0xb7, 0xdd, 0xd0, 0xa7, 0xc1, 0x0e, 0x3e, 0xcf, 0x49, 0x48,
|
|
||||||
- 0x3b, 0x2b, 0x13, 0xf5, 0xdf, 0x55, 0xa0, 0xf3, 0x78, 0x82, 0x43, 0xbe, 0x83, 0xb9, 0x4b, 0xa6,
|
|
||||||
- 0xb2, 0x7b, 0x3a, 0xc7, 0x2c, 0x26, 0x34, 0xd4, 0x4f, 0xdb, 0x80, 0xa2, 0xf9, 0x25, 0x21, 0xe1,
|
|
||||||
- 0x8e, 0xef, 0xe2, 0x80, 0x86, 0x92, 0x4b, 0xd3, 0x06, 0x81, 0xda, 0x91, 0x18, 0xf4, 0x16, 0xf4,
|
|
||||||
- 0xd5, 0x6c, 0xce, 0x39, 0x75, 0x43, 0x7f, 0x2a, 0x82, 0x4a, 0xcd, 0x2a, 0x7a, 0x0a, 0xbd, 0xa7,
|
|
||||||
- 0xb1, 0xe8, 0x6d, 0x58, 0xd1, 0x4f, 0x3e, 0xa3, 0xac, 0x4b, 0xca, 0xbe, 0xc6, 0x17, 0x48, 0x93,
|
|
||||||
- 0x28, 0xa2, 0x8c, 0xc7, 0x4e, 0x8c, 0x3d, 0x8f, 0x06, 0x91, 0x6e, 0x3d, 0xfa, 0x06, 0x7f, 0xa8,
|
|
||||||
- 0xd0, 0xd6, 0x04, 0x56, 0x9f, 0x08, 0x3b, 0xb5, 0x25, 0xd9, 0x15, 0xf6, 0x02, 0x1c, 0x38, 0xc7,
|
|
||||||
- 0x53, 0xea, 0x9d, 0x39, 0x22, 0x11, 0x69, 0x0f, 0x8b, 0xe2, 0x66, 0x4b, 0x20, 0x0f, 0xc9, 0xb7,
|
|
||||||
- 0xb2, 0xcb, 0x16, 0x54, 0xa7, 0x94, 0x47, 0xd3, 0x64, 0xe2, 0x44, 0x8c, 0x1e, 0x63, 0x6d, 0x62,
|
|
||||||
- 0x3f, 0xc0, 0xc1, 0x9e, 0xc2, 0x1f, 0x08, 0xb4, 0xf5, 0xd7, 0x0a, 0xac, 0x15, 0x25, 0xe9, 0xb4,
|
|
||||||
- 0xba, 0x09, 0x6b, 0x45, 0x51, 0xfa, 0x53, 0xab, 0x4a, 0xb9, 0x41, 0x5e, 0xa0, 0xfa, 0xe8, 0x7e,
|
|
||||||
- 0x04, 0x5d, 0x39, 0xb0, 0x75, 0x7c, 0xc5, 0xa9, 0x58, 0x60, 0xe4, 0xef, 0xc5, 0xee, 0xb8, 0xf9,
|
|
||||||
- 0x5b, 0xfa, 0x04, 0xee, 0x68, 0xf3, 0x9d, 0x45, 0xb5, 0xd5, 0x83, 0x58, 0xd7, 0x04, 0x4f, 0xe7,
|
|
||||||
- 0xb4, 0xff, 0x12, 0x86, 0x19, 0x6a, 0x6b, 0x26, 0x91, 0xc6, 0x57, 0xef, 0xc1, 0xea, 0x9c, 0xb1,
|
|
||||||
- 0x8f, 0x7d, 0x9f, 0xc9, 0x10, 0xac, 0xdb, 0x65, 0x5b, 0xd6, 0x23, 0xb8, 0x7d, 0x88, 0xb9, 0xf2,
|
|
||||||
- 0x86, 0xcb, 0x75, 0xd5, 0xaf, 0x98, 0xad, 0x40, 0xed, 0x10, 0x7b, 0xd2, 0xf8, 0x9a, 0x2d, 0x96,
|
|
||||||
- 0xe2, 0x01, 0x1e, 0xc5, 0xd8, 0x93, 0x56, 0xd6, 0x6c, 0xb9, 0xb6, 0xfe, 0x52, 0x81, 0x65, 0x9d,
|
|
||||||
- 0x08, 0x45, 0x32, 0xf7, 0x19, 0x39, 0xc7, 0x4c, 0x3f, 0x3d, 0x0d, 0xa1, 0x37, 0xa1, 0xa7, 0x56,
|
|
||||||
- 0x0e, 0x8d, 0x38, 0xa1, 0x69, 0x7a, 0xed, 0x2a, 0xec, 0x33, 0x85, 0x94, 0xb3, 0x38, 0x39, 0x6a,
|
|
||||||
- 0xd2, 0x5d, 0x9d, 0x86, 0xe4, 0x40, 0x2d, 0x16, 0xb1, 0x2f, 0xd3, 0x69, 0xcb, 0xd6, 0x90, 0x78,
|
|
||||||
- 0xea, 0x86, 0xdf, 0x92, 0xe4, 0x67, 0x40, 0xf1, 0xd4, 0x03, 0x9a, 0x84, 0xdc, 0x89, 0x28, 0x09,
|
|
||||||
- 0xb9, 0xce, 0x9f, 0x20, 0x51, 0x07, 0x02, 0x63, 0xfd, 0xa6, 0x02, 0x0d, 0x35, 0x8f, 0x16, 0x7d,
|
|
||||||
- 0x64, 0xfa, 0x15, 0xab, 0x12, 0x59, 0x11, 0x48, 0x59, 0xea, 0xcb, 0x25, 0xd7, 0x22, 0x8e, 0xcf,
|
|
||||||
- 0x03, 0x95, 0x8b, 0xb5, 0x6a, 0xe7, 0x81, 0x4c, 0xc2, 0x6f, 0x42, 0x2f, 0xfb, 0x18, 0xca, 0x7d,
|
|
||||||
- 0xa5, 0x62, 0x37, 0xc5, 0x4a, 0xb2, 0x4b, 0x35, 0xb5, 0x7e, 0x2e, 0xda, 0xe7, 0x74, 0x16, 0xbb,
|
|
||||||
- 0x02, 0xb5, 0x24, 0x55, 0x46, 0x2c, 0x05, 0x66, 0x92, 0x7e, 0x46, 0xc5, 0x12, 0x3d, 0x80, 0x9e,
|
|
||||||
- 0xeb, 0xfb, 0x44, 0x1c, 0x77, 0xa7, 0x4f, 0x88, 0x9f, 0x06, 0x69, 0x11, 0x6b, 0xfd, 0xbd, 0x02,
|
|
||||||
- 0xfd, 0x6d, 0x1a, 0xcd, 0x7e, 0x4c, 0xa6, 0x38, 0x97, 0x41, 0xa4, 0x92, 0xfa, 0x2b, 0x2a, 0xd6,
|
|
||||||
- 0xa2, 0x32, 0x3c, 0x21, 0x53, 0xac, 0x42, 0x4b, 0xdd, 0x6c, 0x53, 0x20, 0x64, 0x58, 0x99, 0xcd,
|
|
||||||
- 0x74, 0xc4, 0xd5, 0x55, 0x9b, 0x4f, 0xa9, 0x2f, 0x6b, 0x60, 0x9f, 0x30, 0x27, 0x1d, 0x68, 0x75,
|
|
||||||
- 0xed, 0x65, 0x9f, 0x30, 0xb9, 0xa5, 0x0d, 0x59, 0x92, 0x33, 0xd5, 0xbc, 0x21, 0x0d, 0x85, 0x11,
|
|
||||||
- 0x86, 0xac, 0x43, 0x83, 0x9e, 0x9c, 0xc4, 0x98, 0xcb, 0x6a, 0xb5, 0x66, 0x6b, 0x28, 0x4d, 0x73,
|
|
||||||
- 0xcd, 0x5c, 0x9a, 0xbb, 0x05, 0xab, 0x72, 0x7a, 0xff, 0x9c, 0xb9, 0x1e, 0x09, 0x27, 0x26, 0x15,
|
|
||||||
- 0xaf, 0x01, 0x3a, 0xe4, 0x34, 0x2a, 0x62, 0x1f, 0xfe, 0x7e, 0x45, 0xe7, 0x44, 0xdd, 0xca, 0xa2,
|
|
||||||
- 0x27, 0xd0, 0x9f, 0xfb, 0x6b, 0x04, 0xe9, 0xd9, 0x46, 0xf9, 0x3f, 0x26, 0xa3, 0xf5, 0xb1, 0xfa,
|
|
||||||
- 0xab, 0x65, 0x6c, 0xfe, 0x6a, 0x19, 0xef, 0x06, 0x11, 0x9f, 0xa1, 0x5d, 0xe8, 0x15, 0xff, 0x44,
|
|
||||||
- 0x40, 0x77, 0x4d, 0x29, 0x50, 0xf2, 0xd7, 0xc2, 0xa5, 0x6c, 0x9e, 0x40, 0x7f, 0xee, 0xff, 0x04,
|
|
||||||
- 0xa3, 0x4f, 0xf9, 0xdf, 0x0c, 0x97, 0x32, 0x7a, 0x04, 0xed, 0xdc, 0x1f, 0x08, 0x68, 0xa8, 0x98,
|
|
||||||
- 0x2c, 0xfe, 0xa7, 0x70, 0x29, 0x83, 0x6d, 0xe8, 0x16, 0x66, 0xfa, 0x68, 0xa4, 0xed, 0x29, 0x19,
|
|
||||||
- 0xf4, 0x5f, 0xca, 0x64, 0x0b, 0xda, 0xb9, 0xd1, 0xba, 0xd1, 0x62, 0x71, 0x7e, 0x3f, 0xba, 0x53,
|
|
||||||
- 0xb2, 0xa3, 0x53, 0xef, 0x1e, 0x74, 0x0b, 0x83, 0x70, 0xa3, 0x48, 0xd9, 0x10, 0x7e, 0x74, 0xb7,
|
|
||||||
- 0x74, 0x4f, 0x73, 0x7a, 0x02, 0xfd, 0xb9, 0xb1, 0xb8, 0x71, 0x6e, 0xf9, 0xb4, 0xfc, 0x52, 0xb3,
|
|
||||||
- 0xbe, 0x90, 0x97, 0x9d, 0xeb, 0x7a, 0x72, 0x97, 0xbd, 0x38, 0x04, 0x1f, 0xbd, 0x52, 0xbe, 0xa9,
|
|
||||||
- 0xb5, 0xda, 0x85, 0x5e, 0x71, 0xfe, 0x6d, 0x98, 0x95, 0x4e, 0xc5, 0xaf, 0x7e, 0x39, 0x85, 0x51,
|
|
||||||
- 0x78, 0xf6, 0x72, 0xca, 0x26, 0xe4, 0x97, 0x32, 0x7a, 0x0c, 0xa0, 0x7b, 0x1c, 0x9f, 0x84, 0xe9,
|
|
||||||
- 0x95, 0x2d, 0xf4, 0x56, 0xe9, 0x95, 0x95, 0xf4, 0x43, 0x8f, 0x00, 0x54, 0x6b, 0xe2, 0xd3, 0x84,
|
|
||||||
- 0xa3, 0xdb, 0x46, 0x8d, 0xb9, 0x7e, 0x68, 0x34, 0x5c, 0xdc, 0x58, 0x60, 0x80, 0x19, 0xbb, 0x0e,
|
|
||||||
- 0x83, 0xcf, 0x01, 0xb2, 0x96, 0xc7, 0x30, 0x58, 0x68, 0x82, 0xae, 0xf0, 0x41, 0x27, 0xdf, 0xe0,
|
|
||||||
- 0x20, 0x6d, 0x6b, 0x49, 0xd3, 0x73, 0x05, 0x8b, 0xfe, 0x5c, 0x01, 0x5b, 0x7c, 0x6c, 0xf3, 0x75,
|
|
||||||
- 0xed, 0x68, 0xa1, 0x88, 0x45, 0x1f, 0x41, 0x27, 0x5f, 0xb9, 0x1a, 0x2d, 0x4a, 0xaa, 0xd9, 0x51,
|
|
||||||
- 0xa1, 0x7a, 0x45, 0x8f, 0xa0, 0x57, 0xac, 0x5a, 0x51, 0x2e, 0x2e, 0x16, 0x6a, 0xd9, 0x91, 0x9e,
|
|
||||||
- 0xc9, 0xe4, 0xc8, 0x3f, 0x00, 0xc8, 0xaa, 0x5b, 0xe3, 0xbe, 0x85, 0x7a, 0x77, 0x4e, 0xea, 0x63,
|
|
||||||
- 0xe8, 0xe4, 0x33, 0xb1, 0x51, 0xb7, 0x24, 0x3b, 0x5f, 0x95, 0xb5, 0x72, 0x59, 0xdb, 0x3c, 0xbe,
|
|
||||||
- 0xc5, 0x44, 0x7e, 0x55, 0xd6, 0x2a, 0xf4, 0x75, 0x26, 0x59, 0x94, 0x35, 0x7b, 0x57, 0xe5, 0xf2,
|
|
||||||
- 0x62, 0x13, 0x64, 0xdc, 0x57, 0xda, 0x1a, 0x5d, 0xf5, 0x88, 0xf2, 0xdd, 0x80, 0xf1, 0x47, 0x49,
|
|
||||||
- 0x87, 0xf0, 0x92, 0xa0, 0xce, 0x57, 0xfc, 0xb9, 0xa0, 0x2e, 0x69, 0x04, 0x2e, 0x65, 0xb4, 0x07,
|
|
||||||
- 0xfd, 0x27, 0xa6, 0x98, 0xd3, 0x85, 0xa6, 0x56, 0xa7, 0xa4, 0xb0, 0x1e, 0x8d, 0xca, 0xb6, 0x74,
|
|
||||||
- 0x64, 0x7d, 0x01, 0x83, 0x85, 0x22, 0x13, 0xdd, 0x4b, 0x47, 0x87, 0xa5, 0xd5, 0xe7, 0xa5, 0x6a,
|
|
||||||
- 0xed, 0xc3, 0xca, 0x7c, 0x8d, 0x89, 0x5e, 0xd5, 0x97, 0x5e, 0x5e, 0x7b, 0x5e, 0xca, 0xea, 0x13,
|
|
||||||
- 0x68, 0x9a, 0x9a, 0x06, 0xe9, 0x11, 0xed, 0x5c, 0x8d, 0x73, 0xd9, 0xd1, 0xad, 0xce, 0x77, 0xdf,
|
|
||||||
- 0xdf, 0xab, 0xfc, 0xe3, 0xfb, 0x7b, 0x95, 0x7f, 0x7d, 0x7f, 0xaf, 0x72, 0xdc, 0x90, 0xbb, 0x1f,
|
|
||||||
- 0xfc, 0x27, 0x00, 0x00, 0xff, 0xff, 0xa5, 0xac, 0x85, 0x1d, 0xaa, 0x21, 0x00, 0x00,
|
|
||||||
+ 0xd1, 0xd8, 0x07, 0x97, 0xbb, 0xb5, 0x2f, 0x6e, 0x93, 0xa2, 0x56, 0x2b, 0x59, 0x9f, 0x3c, 0xb6,
|
|
||||||
+ 0x65, 0xfa, 0xf3, 0xe7, 0xa5, 0x2d, 0x1b, 0x9f, 0x5f, 0x70, 0x04, 0xf1, 0x11, 0x91, 0xb1, 0x15,
|
|
||||||
+ 0x31, 0x43, 0x11, 0x4e, 0x10, 0x04, 0x83, 0xe1, 0x4c, 0x73, 0xd9, 0xe6, 0xce, 0xf4, 0xb8, 0xa7,
|
|
||||||
+ 0x87, 0xe2, 0x3a, 0x40, 0x8e, 0xc9, 0x2d, 0x97, 0x00, 0xb9, 0xe5, 0x0f, 0x04, 0xb9, 0xe5, 0x98,
|
|
||||||
+ 0x6b, 0x0e, 0x46, 0x4e, 0xf9, 0x05, 0x41, 0xe0, 0x9f, 0x90, 0x5f, 0x10, 0xf4, 0x6b, 0x1e, 0xbb,
|
|
||||||
+ 0x43, 0x1a, 0x21, 0x08, 0xe4, 0xb2, 0xe8, 0xaa, 0xae, 0xae, 0x57, 0x77, 0xd5, 0x54, 0xd5, 0x42,
|
|
||||||
+ 0xdb, 0x9d, 0xe0, 0x90, 0x8f, 0x23, 0x46, 0x39, 0x45, 0xf5, 0x09, 0x8b, 0xbc, 0x51, 0x8b, 0x7a,
|
|
||||||
+ 0x44, 0x21, 0x46, 0xff, 0x3f, 0x21, 0xfc, 0x34, 0x39, 0x1e, 0x7b, 0x34, 0xd8, 0x3c, 0x73, 0xb9,
|
|
||||||
+ 0xfb, 0x8e, 0x47, 0x43, 0xee, 0x92, 0x10, 0xb3, 0x78, 0x53, 0x1e, 0xdc, 0x8c, 0xce, 0x26, 0x9b,
|
|
||||||
+ 0x7c, 0x16, 0xe1, 0x58, 0xfd, 0xea, 0x73, 0x77, 0x27, 0x94, 0x4e, 0xa6, 0x78, 0x53, 0x42, 0xc7,
|
|
||||||
+ 0xc9, 0xc9, 0x26, 0x0e, 0x22, 0x3e, 0x53, 0x9b, 0xd6, 0x1f, 0xaa, 0xb0, 0xbe, 0xcd, 0xb0, 0xcb,
|
|
||||||
+ 0xf1, 0xb6, 0xe1, 0x66, 0xe3, 0xaf, 0x13, 0x1c, 0x73, 0xf4, 0x2a, 0x74, 0x52, 0x09, 0x0e, 0xf1,
|
|
||||||
+ 0x87, 0x95, 0x07, 0x95, 0x8d, 0x96, 0xdd, 0x4e, 0x71, 0xfb, 0x3e, 0xba, 0x0d, 0xcb, 0xf8, 0x02,
|
|
||||||
+ 0x7b, 0x62, 0xb7, 0x2a, 0x77, 0x1b, 0x02, 0xdc, 0xf7, 0xd1, 0x7b, 0xd0, 0x8e, 0x39, 0x23, 0xe1,
|
|
||||||
+ 0xc4, 0x49, 0x62, 0xcc, 0x86, 0xb5, 0x07, 0x95, 0x8d, 0xf6, 0xa3, 0x95, 0xb1, 0x30, 0x69, 0x7c,
|
|
||||||
+ 0x28, 0x37, 0x8e, 0x62, 0xcc, 0x6c, 0x88, 0xd3, 0x35, 0x7a, 0x08, 0xcb, 0x3e, 0x3e, 0x27, 0x1e,
|
|
||||||
+ 0x8e, 0x87, 0xf5, 0x07, 0xb5, 0x8d, 0xf6, 0xa3, 0x8e, 0x22, 0xdf, 0x91, 0x48, 0xdb, 0x6c, 0xa2,
|
|
||||||
+ 0xb7, 0xa0, 0x19, 0x73, 0xca, 0xdc, 0x09, 0x8e, 0x87, 0x4b, 0x92, 0xb0, 0x6b, 0xf8, 0x4a, 0xac,
|
|
||||||
+ 0x9d, 0x6e, 0xa3, 0x7b, 0x50, 0x7b, 0xbe, 0xbd, 0x3f, 0x6c, 0x48, 0xe9, 0xa0, 0xa9, 0x22, 0xec,
|
|
||||||
+ 0xd9, 0x02, 0x8d, 0x5e, 0x83, 0x6e, 0xec, 0x86, 0xfe, 0x31, 0xbd, 0x70, 0x22, 0xe2, 0x87, 0xf1,
|
|
||||||
+ 0x70, 0xf9, 0x41, 0x65, 0xa3, 0x69, 0x77, 0x34, 0xf2, 0x40, 0xe0, 0xac, 0x4f, 0xe0, 0xd6, 0x21,
|
|
||||||
+ 0x77, 0x19, 0xbf, 0x86, 0x77, 0xac, 0x23, 0x58, 0xb7, 0x71, 0x40, 0xcf, 0xaf, 0xe5, 0xda, 0x21,
|
|
||||||
+ 0x2c, 0x73, 0x12, 0x60, 0x9a, 0x70, 0xe9, 0xda, 0xae, 0x6d, 0x40, 0xeb, 0x4f, 0x15, 0x40, 0xbb,
|
|
||||||
+ 0x17, 0xd8, 0x3b, 0x60, 0xd4, 0xc3, 0x71, 0xfc, 0x5f, 0xba, 0xae, 0x37, 0x61, 0x39, 0x52, 0x0a,
|
|
||||||
+ 0x0c, 0xeb, 0x92, 0x5c, 0xdf, 0x82, 0xd1, 0xca, 0xec, 0x5a, 0x5f, 0xc1, 0xda, 0x21, 0x99, 0x84,
|
|
||||||
+ 0xee, 0xf4, 0x06, 0xf5, 0x5d, 0x87, 0x46, 0x2c, 0x79, 0x4a, 0x55, 0xbb, 0xb6, 0x86, 0xac, 0x03,
|
|
||||||
+ 0x40, 0x5f, 0xba, 0x84, 0xdf, 0x9c, 0x24, 0xeb, 0x1d, 0x58, 0x2d, 0x70, 0x8c, 0x23, 0x1a, 0xc6,
|
|
||||||
+ 0x58, 0x2a, 0xc0, 0x5d, 0x9e, 0xc4, 0x92, 0xd9, 0x92, 0xad, 0x21, 0x0b, 0xc3, 0xda, 0x17, 0x24,
|
|
||||||
+ 0x36, 0xe4, 0xf8, 0x3f, 0x51, 0x61, 0x1d, 0x1a, 0x27, 0x94, 0x05, 0x2e, 0x37, 0x1a, 0x28, 0x08,
|
|
||||||
+ 0x21, 0xa8, 0xbb, 0x6c, 0x12, 0x0f, 0x6b, 0x0f, 0x6a, 0x1b, 0x2d, 0x5b, 0xae, 0xc5, 0xab, 0x9c,
|
|
||||||
+ 0x13, 0xa3, 0xf5, 0x7a, 0x15, 0x3a, 0xda, 0xef, 0xce, 0x94, 0xc4, 0x5c, 0xca, 0xe9, 0xd8, 0x6d,
|
|
||||||
+ 0x8d, 0x13, 0x67, 0x2c, 0x0a, 0xeb, 0x47, 0x91, 0x7f, 0xcd, 0x80, 0x7f, 0x04, 0x2d, 0x86, 0x63,
|
|
||||||
+ 0x9a, 0x30, 0x11, 0xa6, 0x55, 0x79, 0xef, 0x6b, 0xea, 0xde, 0xbf, 0x20, 0x61, 0x72, 0x61, 0x9b,
|
|
||||||
+ 0x3d, 0x3b, 0x23, 0xd3, 0x21, 0xc4, 0xe3, 0xeb, 0x84, 0xd0, 0x27, 0x70, 0xeb, 0xc0, 0x4d, 0xe2,
|
|
||||||
+ 0xeb, 0xe8, 0x6a, 0x7d, 0x2a, 0xc2, 0x2f, 0x4e, 0x82, 0x6b, 0x1d, 0xfe, 0x63, 0x05, 0x9a, 0xdb,
|
|
||||||
+ 0x51, 0x72, 0x14, 0xbb, 0x13, 0x8c, 0xfe, 0x07, 0xda, 0x9c, 0x72, 0x77, 0xea, 0x24, 0x02, 0x94,
|
|
||||||
+ 0xe4, 0x75, 0x1b, 0x24, 0x4a, 0x11, 0x08, 0xb7, 0x63, 0xe6, 0x45, 0x89, 0xa6, 0xa8, 0x3e, 0xa8,
|
|
||||||
+ 0x6d, 0xd4, 0xed, 0xb6, 0xc2, 0x29, 0x92, 0x31, 0xac, 0xca, 0x3d, 0x87, 0x84, 0xce, 0x19, 0x66,
|
|
||||||
+ 0x21, 0x9e, 0x06, 0xd4, 0xc7, 0xf2, 0xfd, 0xd6, 0xed, 0x81, 0xdc, 0xda, 0x0f, 0x3f, 0x4f, 0x37,
|
|
||||||
+ 0xd0, 0xff, 0xc2, 0x20, 0xa5, 0x17, 0x41, 0x29, 0xa9, 0xeb, 0x92, 0xba, 0xaf, 0xa9, 0x8f, 0x34,
|
|
||||||
+ 0xda, 0xfa, 0x15, 0xf4, 0x5e, 0x9c, 0x32, 0xca, 0xf9, 0x94, 0x84, 0x93, 0x1d, 0x97, 0xbb, 0x22,
|
|
||||||
+ 0x7b, 0x44, 0x98, 0x11, 0xea, 0xc7, 0x5a, 0x5b, 0x03, 0xa2, 0xb7, 0x61, 0xc0, 0x15, 0x2d, 0xf6,
|
|
||||||
+ 0x1d, 0x43, 0x53, 0x95, 0x34, 0x2b, 0xe9, 0xc6, 0x81, 0x26, 0x7e, 0x03, 0x7a, 0x19, 0xb1, 0xc8,
|
|
||||||
+ 0x3f, 0x5a, 0xdf, 0x6e, 0x8a, 0x7d, 0x41, 0x02, 0x6c, 0x9d, 0x4b, 0x5f, 0xc9, 0x4b, 0x46, 0x6f,
|
|
||||||
+ 0x43, 0x2b, 0xf3, 0x43, 0x45, 0xbe, 0x90, 0x9e, 0x7a, 0x21, 0xc6, 0x9d, 0x76, 0x33, 0x75, 0xca,
|
|
||||||
+ 0x67, 0xd0, 0xe7, 0xa9, 0xe2, 0x8e, 0xef, 0x72, 0xb7, 0xf8, 0xa8, 0x8a, 0x56, 0xd9, 0x3d, 0x5e,
|
|
||||||
+ 0x80, 0xad, 0x4f, 0xa1, 0x75, 0x40, 0xfc, 0x58, 0x09, 0x1e, 0xc2, 0xb2, 0x97, 0x30, 0x86, 0x43,
|
|
||||||
+ 0x6e, 0x4c, 0xd6, 0x20, 0x5a, 0x83, 0xa5, 0x29, 0x09, 0x08, 0xd7, 0x66, 0x2a, 0xc0, 0xa2, 0x00,
|
|
||||||
+ 0xcf, 0x70, 0x40, 0xd9, 0x4c, 0x3a, 0x6c, 0x0d, 0x96, 0xf2, 0x97, 0xab, 0x00, 0x74, 0x17, 0x5a,
|
|
||||||
+ 0x81, 0x7b, 0x91, 0x5e, 0xaa, 0xd8, 0x69, 0x06, 0xee, 0x85, 0x52, 0x7e, 0x08, 0xcb, 0x27, 0x2e,
|
|
||||||
+ 0x99, 0x7a, 0x21, 0xd7, 0x5e, 0x31, 0x60, 0x26, 0xb0, 0x9e, 0x17, 0xf8, 0xd7, 0x2a, 0xb4, 0x95,
|
|
||||||
+ 0x44, 0xa5, 0xf0, 0x1a, 0x2c, 0x79, 0xae, 0x77, 0x9a, 0x8a, 0x94, 0x00, 0x7a, 0x68, 0x14, 0xa9,
|
|
||||||
+ 0xe6, 0x93, 0x70, 0xa6, 0xa9, 0x51, 0x6d, 0x13, 0x20, 0x7e, 0xe9, 0x46, 0x5a, 0xb7, 0xda, 0x25,
|
|
||||||
+ 0xc4, 0x2d, 0x41, 0xa3, 0xd4, 0x7d, 0x1f, 0x3a, 0xea, 0xdd, 0xe9, 0x23, 0xf5, 0x4b, 0x8e, 0xb4,
|
|
||||||
+ 0x15, 0x95, 0x3a, 0xf4, 0x1a, 0x74, 0x93, 0x18, 0x3b, 0xa7, 0x04, 0x33, 0x97, 0x79, 0xa7, 0xb3,
|
|
||||||
+ 0xe1, 0x92, 0xfa, 0x46, 0x26, 0x31, 0xde, 0x33, 0x38, 0xf4, 0x08, 0x96, 0x44, 0xfa, 0x8b, 0x87,
|
|
||||||
+ 0x0d, 0xf9, 0x39, 0xbe, 0x97, 0x67, 0x29, 0x4d, 0x1d, 0xcb, 0xdf, 0xdd, 0x90, 0xb3, 0x99, 0xad,
|
|
||||||
+ 0x48, 0x47, 0x1f, 0x01, 0x64, 0x48, 0xb4, 0x02, 0xb5, 0x33, 0x3c, 0xd3, 0x71, 0x28, 0x96, 0xc2,
|
|
||||||
+ 0x39, 0xe7, 0xee, 0x34, 0x31, 0x5e, 0x57, 0xc0, 0x27, 0xd5, 0x8f, 0x2a, 0x96, 0x07, 0xfd, 0xad,
|
|
||||||
+ 0xe9, 0x19, 0xa1, 0xb9, 0xe3, 0x6b, 0xb0, 0x14, 0xb8, 0x5f, 0x51, 0x66, 0x3c, 0x29, 0x01, 0x89,
|
|
||||||
+ 0x25, 0x21, 0x65, 0x86, 0x85, 0x04, 0x50, 0x0f, 0xaa, 0x34, 0x92, 0xfe, 0x6a, 0xd9, 0x55, 0x1a,
|
|
||||||
+ 0x65, 0x82, 0xea, 0x39, 0x41, 0xd6, 0x3f, 0xea, 0x00, 0x99, 0x14, 0x64, 0xc3, 0x88, 0x50, 0x27,
|
|
||||||
+ 0xc6, 0x4c, 0x94, 0x20, 0xce, 0xf1, 0x8c, 0xe3, 0xd8, 0x61, 0xd8, 0x4b, 0x58, 0x4c, 0xce, 0xc5,
|
|
||||||
+ 0xfd, 0x09, 0xb3, 0x6f, 0x29, 0xb3, 0xe7, 0x74, 0xb3, 0x6f, 0x13, 0x7a, 0xa8, 0xce, 0x6d, 0x89,
|
|
||||||
+ 0x63, 0xb6, 0x39, 0x85, 0xf6, 0xe1, 0x56, 0xc6, 0xd3, 0xcf, 0xb1, 0xab, 0x5e, 0xc5, 0x6e, 0x35,
|
|
||||||
+ 0x65, 0xe7, 0x67, 0xac, 0x76, 0x61, 0x95, 0x50, 0xe7, 0xeb, 0x04, 0x27, 0x05, 0x46, 0xb5, 0xab,
|
|
||||||
+ 0x18, 0x0d, 0x08, 0xfd, 0x89, 0x3c, 0x90, 0xb1, 0x39, 0x80, 0x3b, 0x39, 0x2b, 0x45, 0xb8, 0xe7,
|
|
||||||
+ 0x98, 0xd5, 0xaf, 0x62, 0xb6, 0x9e, 0x6a, 0x25, 0xf2, 0x41, 0xc6, 0xf1, 0x47, 0xb0, 0x4e, 0xa8,
|
|
||||||
+ 0xf3, 0xd2, 0x25, 0x7c, 0x9e, 0xdd, 0xd2, 0xf7, 0x18, 0x29, 0x3e, 0xba, 0x45, 0x5e, 0xca, 0xc8,
|
|
||||||
+ 0x00, 0xb3, 0x49, 0xc1, 0xc8, 0xc6, 0xf7, 0x18, 0xf9, 0x4c, 0x1e, 0xc8, 0xd8, 0x3c, 0x81, 0x01,
|
|
||||||
+ 0xa1, 0xf3, 0xda, 0x2c, 0x5f, 0xc5, 0xa4, 0x4f, 0x68, 0x51, 0x93, 0x2d, 0x18, 0xc4, 0xd8, 0xe3,
|
|
||||||
+ 0x94, 0xe5, 0x1f, 0x41, 0xf3, 0x2a, 0x16, 0x2b, 0x9a, 0x3e, 0xe5, 0x61, 0xfd, 0x1c, 0x3a, 0x7b,
|
|
||||||
+ 0xc9, 0x04, 0xf3, 0xe9, 0x71, 0x9a, 0x0c, 0x6e, 0x2c, 0xff, 0x58, 0xff, 0xaa, 0x42, 0x7b, 0x7b,
|
|
||||||
+ 0xc2, 0x68, 0x12, 0x15, 0x72, 0xb2, 0x0a, 0xd2, 0xf9, 0x9c, 0x2c, 0x49, 0x64, 0x4e, 0x56, 0xc4,
|
|
||||||
+ 0x1f, 0x40, 0x27, 0x90, 0xa1, 0xab, 0xe9, 0x55, 0x1e, 0x1a, 0x2c, 0x04, 0xb5, 0xdd, 0x0e, 0x72,
|
|
||||||
+ 0xc9, 0x6c, 0x0c, 0x10, 0x11, 0x3f, 0xd6, 0x67, 0x54, 0x3a, 0xea, 0xeb, 0x8a, 0xd0, 0xa4, 0x68,
|
|
||||||
+ 0xbb, 0x15, 0xa5, 0xd9, 0xfa, 0x3d, 0x68, 0x1f, 0x0b, 0x27, 0xe9, 0x03, 0x85, 0x64, 0x94, 0x79,
|
|
||||||
+ 0xcf, 0x86, 0xe3, 0x2c, 0x08, 0xf7, 0xa0, 0x7b, 0xaa, 0x5c, 0xa6, 0x0f, 0xa9, 0x37, 0xf4, 0x9a,
|
|
||||||
+ 0xb6, 0x24, 0xb3, 0x77, 0x9c, 0xf7, 0xac, 0xba, 0x80, 0xce, 0x69, 0x0e, 0x35, 0x3a, 0x84, 0xc1,
|
|
||||||
+ 0x02, 0x49, 0x49, 0x0e, 0xda, 0xc8, 0xe7, 0xa0, 0xf6, 0x23, 0xa4, 0x04, 0xe5, 0x4f, 0xe6, 0xf3,
|
|
||||||
+ 0xd2, 0x6f, 0xab, 0xd0, 0xf9, 0x31, 0xe6, 0x2f, 0x29, 0x3b, 0x53, 0xfa, 0x22, 0xa8, 0x87, 0x6e,
|
|
||||||
+ 0x80, 0x35, 0x47, 0xb9, 0x46, 0x77, 0xa0, 0xc9, 0x2e, 0x54, 0x02, 0xd1, 0xf7, 0xb9, 0xcc, 0x2e,
|
|
||||||
+ 0x64, 0x62, 0x40, 0xaf, 0x00, 0xb0, 0x0b, 0x27, 0x72, 0xbd, 0x33, 0xac, 0x3d, 0x58, 0xb7, 0x5b,
|
|
||||||
+ 0xec, 0xe2, 0x40, 0x21, 0xc4, 0x53, 0x60, 0x17, 0x0e, 0x66, 0x8c, 0xb2, 0x58, 0xe7, 0xaa, 0x26,
|
|
||||||
+ 0xbb, 0xd8, 0x95, 0xb0, 0x3e, 0xeb, 0x33, 0x1a, 0x45, 0xd8, 0x97, 0x39, 0x5a, 0x9e, 0xdd, 0x51,
|
|
||||||
+ 0x08, 0x21, 0x95, 0x1b, 0xa9, 0x0d, 0x25, 0x95, 0x67, 0x52, 0x79, 0x26, 0x75, 0x59, 0x9d, 0xe4,
|
|
||||||
+ 0x79, 0xa9, 0x3c, 0x95, 0xda, 0x54, 0x52, 0x79, 0x4e, 0x2a, 0xcf, 0xa4, 0xb6, 0xcc, 0x59, 0x2d,
|
|
||||||
+ 0xd5, 0xfa, 0x4d, 0x05, 0xd6, 0xe7, 0x0b, 0x3f, 0x5d, 0xa6, 0x7e, 0x00, 0x1d, 0x4f, 0xde, 0x57,
|
|
||||||
+ 0xe1, 0x4d, 0x0e, 0x16, 0x6e, 0xd2, 0x6e, 0x7b, 0xb9, 0x67, 0xfc, 0x21, 0x74, 0x43, 0xe5, 0xe0,
|
|
||||||
+ 0xf4, 0x69, 0xd6, 0xb2, 0x7b, 0xc9, 0xfb, 0xde, 0xee, 0x84, 0x39, 0xc8, 0xf2, 0x01, 0x7d, 0xc9,
|
|
||||||
+ 0x08, 0xc7, 0x87, 0x9c, 0x61, 0x37, 0xb8, 0x89, 0x06, 0x04, 0x41, 0x5d, 0x56, 0x2b, 0x35, 0x59,
|
|
||||||
+ 0x5f, 0xcb, 0xb5, 0xf5, 0x26, 0xac, 0x16, 0xa4, 0x68, 0x5b, 0x57, 0xa0, 0x36, 0xc5, 0xa1, 0xe4,
|
|
||||||
+ 0xde, 0xb5, 0xc5, 0xd2, 0x72, 0x61, 0x60, 0x63, 0xd7, 0xbf, 0x39, 0x6d, 0xb4, 0x88, 0x5a, 0x26,
|
|
||||||
+ 0x62, 0x03, 0x50, 0x5e, 0x84, 0x56, 0xc5, 0x68, 0x5d, 0xc9, 0x69, 0xfd, 0x1c, 0x06, 0xdb, 0x53,
|
|
||||||
+ 0x1a, 0xe3, 0x43, 0xee, 0x93, 0xf0, 0x26, 0x3a, 0xa6, 0x5f, 0xc2, 0xea, 0x0b, 0x3e, 0xfb, 0x52,
|
|
||||||
+ 0x30, 0x8b, 0xc9, 0x37, 0xf8, 0x86, 0xec, 0x63, 0xf4, 0xa5, 0xb1, 0x8f, 0xd1, 0x97, 0xa2, 0x59,
|
|
||||||
+ 0xf2, 0xe8, 0x34, 0x09, 0x42, 0x19, 0x0a, 0x5d, 0x5b, 0x43, 0xd6, 0x16, 0x74, 0x54, 0x0d, 0xfd,
|
|
||||||
+ 0x8c, 0xfa, 0xc9, 0x14, 0x97, 0xc6, 0xe0, 0x7d, 0x80, 0xc8, 0x65, 0x6e, 0x80, 0x39, 0x66, 0xea,
|
|
||||||
+ 0x0d, 0xb5, 0xec, 0x1c, 0xc6, 0xfa, 0x7d, 0x15, 0xd6, 0xd4, 0x48, 0xe4, 0x50, 0x4d, 0x02, 0x8c,
|
|
||||||
+ 0x09, 0x23, 0x68, 0x9e, 0xd2, 0x98, 0xe7, 0x18, 0xa6, 0xb0, 0x50, 0xd1, 0x0f, 0x0d, 0x37, 0xb1,
|
|
||||||
+ 0x2c, 0xcc, 0x29, 0x6a, 0x57, 0xcf, 0x29, 0x16, 0x26, 0x11, 0xf5, 0xc5, 0x49, 0x84, 0x88, 0x36,
|
|
||||||
+ 0x43, 0x44, 0x54, 0x8c, 0xb7, 0xec, 0x96, 0xc6, 0xec, 0xfb, 0xe8, 0x21, 0xf4, 0x27, 0x42, 0x4b,
|
|
||||||
+ 0xe7, 0x94, 0xd2, 0x33, 0x27, 0x72, 0xf9, 0xa9, 0x0c, 0xf5, 0x96, 0xdd, 0x95, 0xe8, 0x3d, 0x4a,
|
|
||||||
+ 0xcf, 0x0e, 0x5c, 0x7e, 0x8a, 0x3e, 0x86, 0x9e, 0x2e, 0x03, 0x03, 0xe9, 0xa2, 0x58, 0x7f, 0xfc,
|
|
||||||
+ 0x74, 0x14, 0xe5, 0xbd, 0x67, 0x77, 0xcf, 0x72, 0x50, 0x6c, 0xdd, 0x86, 0x5b, 0x3b, 0x38, 0xe6,
|
|
||||||
+ 0x8c, 0xce, 0x8a, 0x8e, 0xb1, 0x7e, 0x00, 0xb0, 0x1f, 0x72, 0xcc, 0x4e, 0x5c, 0x0f, 0xc7, 0xe8,
|
|
||||||
+ 0xdd, 0x3c, 0xa4, 0x8b, 0xa3, 0x95, 0xb1, 0x9a, 0x48, 0xa5, 0x1b, 0x76, 0x8e, 0xc6, 0x1a, 0x43,
|
|
||||||
+ 0xc3, 0xa6, 0x89, 0x48, 0x47, 0xaf, 0x9b, 0x95, 0x3e, 0xd7, 0xd1, 0xe7, 0x24, 0xd2, 0xd6, 0x7b,
|
|
||||||
+ 0xd6, 0x9e, 0x69, 0x61, 0x33, 0x76, 0xfa, 0x8a, 0xc6, 0xd0, 0x22, 0x06, 0xa7, 0xb3, 0xca, 0xa2,
|
|
||||||
+ 0xe8, 0x8c, 0xc4, 0xfa, 0x19, 0xac, 0x2a, 0x4e, 0x8a, 0xb3, 0x61, 0xf3, 0x3a, 0x34, 0x98, 0x51,
|
|
||||||
+ 0xa3, 0x92, 0x8d, 0xa2, 0x34, 0x91, 0xde, 0x43, 0xf7, 0x84, 0x30, 0x8f, 0xe1, 0x40, 0xf4, 0x1c,
|
|
||||||
+ 0x55, 0x79, 0x65, 0x19, 0x42, 0x78, 0x4b, 0xf4, 0xdb, 0x99, 0x99, 0xc6, 0x5b, 0xab, 0x30, 0x10,
|
|
||||||
+ 0x1b, 0x05, 0x89, 0xd6, 0x2f, 0x60, 0xf5, 0x79, 0x38, 0x25, 0x21, 0xde, 0x3e, 0x38, 0x7a, 0x86,
|
|
||||||
+ 0xd3, 0xac, 0x80, 0xa0, 0x2e, 0xaa, 0x27, 0xa9, 0x46, 0xd3, 0x96, 0x6b, 0x11, 0x26, 0xe1, 0xb1,
|
|
||||||
+ 0xe3, 0x45, 0x49, 0xac, 0x27, 0x43, 0x8d, 0xf0, 0x78, 0x3b, 0x4a, 0x62, 0x91, 0xe6, 0xc5, 0x67,
|
|
||||||
+ 0x9e, 0x86, 0xd3, 0x99, 0x8c, 0x95, 0xa6, 0xbd, 0xec, 0x45, 0xc9, 0xf3, 0x70, 0x3a, 0xb3, 0xfe,
|
|
||||||
+ 0x4f, 0xf6, 0xc2, 0x18, 0xfb, 0xb6, 0x1b, 0xfa, 0x34, 0xd8, 0xc1, 0xe7, 0x39, 0x09, 0x69, 0xdf,
|
|
||||||
+ 0x65, 0x72, 0xc2, 0xb7, 0x15, 0xe8, 0x3c, 0x99, 0xe0, 0x90, 0xef, 0x60, 0xee, 0x92, 0xa9, 0xec,
|
|
||||||
+ 0xad, 0xce, 0x31, 0x8b, 0x09, 0x0d, 0xf5, 0xc3, 0x37, 0xa0, 0x68, 0x8d, 0x49, 0x48, 0xb8, 0xe3,
|
|
||||||
+ 0xbb, 0x38, 0xa0, 0xa1, 0xf6, 0x02, 0x08, 0xd4, 0x8e, 0xc4, 0xa0, 0x37, 0xa1, 0xaf, 0x26, 0x77,
|
|
||||||
+ 0xce, 0xa9, 0x1b, 0xfa, 0x53, 0x11, 0x72, 0x6a, 0x92, 0xd1, 0x53, 0xe8, 0x3d, 0x8d, 0x45, 0x6f,
|
|
||||||
+ 0xc1, 0x8a, 0x0e, 0x88, 0x8c, 0xb2, 0x2e, 0x29, 0xfb, 0x1a, 0x5f, 0x20, 0x4d, 0xa2, 0x88, 0x32,
|
|
||||||
+ 0x1e, 0x3b, 0x31, 0xf6, 0x3c, 0x1a, 0x44, 0xba, 0x31, 0xe9, 0x1b, 0xfc, 0xa1, 0x42, 0x5b, 0x13,
|
|
||||||
+ 0x58, 0x7d, 0x2a, 0xec, 0xd4, 0x96, 0x64, 0x17, 0xdc, 0x0b, 0x70, 0xe0, 0x1c, 0x4f, 0xa9, 0x77,
|
|
||||||
+ 0xe6, 0x88, 0x34, 0xa5, 0x3d, 0x2c, 0x4a, 0x9f, 0x2d, 0x81, 0x3c, 0x24, 0xdf, 0xc8, 0x1e, 0x5c,
|
|
||||||
+ 0x50, 0x9d, 0x52, 0x1e, 0x4d, 0x93, 0x89, 0x13, 0x31, 0x7a, 0x8c, 0xb5, 0x89, 0xfd, 0x00, 0x07,
|
|
||||||
+ 0x7b, 0x0a, 0x7f, 0x20, 0xd0, 0xd6, 0x5f, 0x2a, 0xb0, 0x56, 0x94, 0xa4, 0x93, 0xee, 0x26, 0xac,
|
|
||||||
+ 0x15, 0x45, 0xe9, 0x0f, 0xb1, 0x2a, 0xf4, 0x06, 0x79, 0x81, 0xea, 0x93, 0xfc, 0x21, 0x74, 0xe5,
|
|
||||||
+ 0x38, 0xd7, 0xf1, 0x15, 0xa7, 0x62, 0xf9, 0x91, 0xbf, 0x17, 0xbb, 0xe3, 0xe6, 0x6f, 0xe9, 0x63,
|
|
||||||
+ 0xb8, 0xa3, 0xcd, 0x77, 0x16, 0xd5, 0x56, 0x0f, 0x62, 0x5d, 0x13, 0x3c, 0x9b, 0xd3, 0xfe, 0x0b,
|
|
||||||
+ 0x18, 0x66, 0xa8, 0xad, 0x99, 0x44, 0x1a, 0x5f, 0xbd, 0x0b, 0xab, 0x73, 0xc6, 0x3e, 0xf1, 0x7d,
|
|
||||||
+ 0x26, 0x03, 0xb4, 0x6e, 0x97, 0x6d, 0x59, 0x8f, 0xe1, 0xf6, 0x21, 0xe6, 0xca, 0x1b, 0x2e, 0xd7,
|
|
||||||
+ 0x3d, 0x81, 0x62, 0xb6, 0x02, 0xb5, 0x43, 0xec, 0x49, 0xe3, 0x6b, 0xb6, 0x58, 0x8a, 0x07, 0x78,
|
|
||||||
+ 0x14, 0x63, 0x4f, 0x5a, 0x59, 0xb3, 0xe5, 0xda, 0xfa, 0x73, 0x05, 0x96, 0x75, 0x9a, 0x14, 0xa9,
|
|
||||||
+ 0xde, 0x67, 0xe4, 0x1c, 0x33, 0xfd, 0xf4, 0x34, 0x84, 0xde, 0x80, 0x9e, 0x5a, 0x39, 0x34, 0xe2,
|
|
||||||
+ 0x84, 0xa6, 0xc9, 0xb7, 0xab, 0xb0, 0xcf, 0x15, 0x52, 0x4e, 0xea, 0xe4, 0x20, 0x4a, 0xf7, 0x7c,
|
|
||||||
+ 0x1a, 0x92, 0xe3, 0xb6, 0x58, 0x64, 0x06, 0x99, 0x6c, 0x5b, 0xb6, 0x86, 0xc4, 0x53, 0x37, 0xfc,
|
|
||||||
+ 0x96, 0x24, 0x3f, 0x03, 0x8a, 0xa7, 0x1e, 0xd0, 0x24, 0xe4, 0x4e, 0x44, 0x49, 0xc8, 0x75, 0x76,
|
|
||||||
+ 0x05, 0x89, 0x3a, 0x10, 0x18, 0xeb, 0xd7, 0x15, 0x68, 0xa8, 0x69, 0xb5, 0xe8, 0x32, 0xd3, 0x6f,
|
|
||||||
+ 0x5c, 0x95, 0xc8, 0x7a, 0x41, 0xca, 0x52, 0xdf, 0x35, 0xb9, 0x16, 0x71, 0x7c, 0x1e, 0xa8, 0x4c,
|
|
||||||
+ 0xad, 0x55, 0x3b, 0x0f, 0x64, 0x8a, 0x7e, 0x03, 0x7a, 0xd9, 0xa7, 0x52, 0xee, 0x2b, 0x15, 0xbb,
|
|
||||||
+ 0x29, 0x56, 0x92, 0x5d, 0xaa, 0xa9, 0xf5, 0x53, 0xd1, 0x5c, 0xa7, 0x93, 0xda, 0x15, 0xa8, 0x25,
|
|
||||||
+ 0xa9, 0x32, 0x62, 0x29, 0x30, 0x93, 0xf4, 0x23, 0x2b, 0x96, 0xe8, 0x21, 0xf4, 0x5c, 0xdf, 0x27,
|
|
||||||
+ 0xe2, 0xb8, 0x3b, 0x7d, 0x4a, 0xfc, 0x34, 0x48, 0x8b, 0x58, 0xeb, 0x6f, 0x15, 0xe8, 0x6f, 0xd3,
|
|
||||||
+ 0x68, 0xf6, 0x43, 0x32, 0xc5, 0xb9, 0x0c, 0x22, 0x95, 0xd4, 0xdf, 0x58, 0xb1, 0x16, 0x75, 0xe3,
|
|
||||||
+ 0x09, 0x99, 0x62, 0x15, 0x5a, 0xea, 0x66, 0x9b, 0x02, 0x21, 0xc3, 0xca, 0x6c, 0xa6, 0x03, 0xb0,
|
|
||||||
+ 0xae, 0xda, 0x7c, 0x46, 0x7d, 0x59, 0x21, 0xfb, 0x84, 0x39, 0xe9, 0xb8, 0xab, 0x6b, 0x2f, 0xfb,
|
|
||||||
+ 0x84, 0xc9, 0x2d, 0x6d, 0xc8, 0x92, 0x9c, 0xb8, 0xe6, 0x0d, 0x69, 0x28, 0x8c, 0x30, 0x64, 0x1d,
|
|
||||||
+ 0x1a, 0xf4, 0xe4, 0x24, 0xc6, 0x5c, 0xd6, 0xb2, 0x35, 0x5b, 0x43, 0x69, 0x9a, 0x6b, 0xe6, 0xd2,
|
|
||||||
+ 0xdc, 0x2d, 0x58, 0x95, 0xb3, 0xfd, 0x17, 0xcc, 0xf5, 0x48, 0x38, 0x31, 0xa9, 0x78, 0x0d, 0xd0,
|
|
||||||
+ 0x21, 0xa7, 0x51, 0x11, 0xfb, 0xe8, 0x77, 0x2b, 0x3a, 0x27, 0xea, 0x46, 0x17, 0x3d, 0x85, 0xfe,
|
|
||||||
+ 0xdc, 0x1f, 0x27, 0x48, 0x4f, 0x3e, 0xca, 0xff, 0x4f, 0x19, 0xad, 0x8f, 0xd5, 0x1f, 0x31, 0x63,
|
|
||||||
+ 0xf3, 0x47, 0xcc, 0x78, 0x37, 0x88, 0xf8, 0x0c, 0xed, 0x42, 0xaf, 0xf8, 0x17, 0x03, 0xba, 0x6b,
|
|
||||||
+ 0x0a, 0x85, 0x92, 0x3f, 0x1e, 0x2e, 0x65, 0xf3, 0x14, 0xfa, 0x73, 0xff, 0x36, 0x18, 0x7d, 0xca,
|
|
||||||
+ 0xff, 0x84, 0xb8, 0x94, 0xd1, 0x63, 0x68, 0xe7, 0xfe, 0x5e, 0x40, 0x43, 0xc5, 0x64, 0xf1, 0x1f,
|
|
||||||
+ 0x87, 0x4b, 0x19, 0x6c, 0x43, 0xb7, 0x30, 0xf1, 0x47, 0x23, 0x6d, 0x4f, 0xc9, 0xdf, 0x00, 0x97,
|
|
||||||
+ 0x32, 0xd9, 0x82, 0x76, 0x6e, 0xf0, 0x6e, 0xb4, 0x58, 0x9c, 0xee, 0x8f, 0xee, 0x94, 0xec, 0xe8,
|
|
||||||
+ 0xd4, 0xbb, 0x07, 0xdd, 0xc2, 0x98, 0xdc, 0x28, 0x52, 0x36, 0xa2, 0x1f, 0xdd, 0x2d, 0xdd, 0xd3,
|
|
||||||
+ 0x9c, 0x9e, 0x42, 0x7f, 0x6e, 0x68, 0x6e, 0x9c, 0x5b, 0x3e, 0x4b, 0xbf, 0xd4, 0xac, 0xcf, 0xe5,
|
|
||||||
+ 0x65, 0xe7, 0x7a, 0xa2, 0xdc, 0x65, 0x2f, 0x8e, 0xc8, 0x47, 0xf7, 0xca, 0x37, 0xb5, 0x56, 0xbb,
|
|
||||||
+ 0xd0, 0x2b, 0x4e, 0xc7, 0x0d, 0xb3, 0xd2, 0x99, 0xf9, 0xd5, 0x2f, 0xa7, 0x30, 0x28, 0xcf, 0x5e,
|
|
||||||
+ 0x4e, 0xd9, 0xfc, 0xfc, 0x52, 0x46, 0x4f, 0x00, 0x74, 0x07, 0xe4, 0x93, 0x30, 0xbd, 0xb2, 0x85,
|
|
||||||
+ 0xce, 0x2b, 0xbd, 0xb2, 0x92, 0x6e, 0xe9, 0x31, 0x80, 0x6a, 0x5c, 0x7c, 0x9a, 0x70, 0x74, 0xdb,
|
|
||||||
+ 0xa8, 0x31, 0xd7, 0x2d, 0x8d, 0x86, 0x8b, 0x1b, 0x0b, 0x0c, 0x30, 0x63, 0xd7, 0x61, 0xf0, 0x19,
|
|
||||||
+ 0x40, 0xd6, 0x10, 0x19, 0x06, 0x0b, 0x2d, 0xd2, 0x15, 0x3e, 0xe8, 0xe4, 0xdb, 0x1f, 0xa4, 0x6d,
|
|
||||||
+ 0x2d, 0x69, 0x89, 0xae, 0x60, 0xd1, 0x9f, 0x2b, 0x6f, 0x8b, 0x8f, 0x6d, 0xbe, 0xea, 0x1d, 0x2d,
|
|
||||||
+ 0x94, 0xb8, 0xe8, 0x43, 0xe8, 0xe4, 0xeb, 0x5a, 0xa3, 0x45, 0x49, 0xad, 0x3b, 0x2a, 0xd4, 0xb6,
|
|
||||||
+ 0xe8, 0x31, 0xf4, 0x8a, 0x55, 0x2b, 0xca, 0xc5, 0xc5, 0x42, 0x2d, 0x3b, 0xd2, 0x13, 0x9b, 0x1c,
|
|
||||||
+ 0xf9, 0xfb, 0x00, 0x59, 0x75, 0x6b, 0xdc, 0xb7, 0x50, 0xef, 0xce, 0x49, 0x7d, 0x02, 0x9d, 0x7c,
|
|
||||||
+ 0x26, 0x36, 0xea, 0x96, 0x64, 0xe7, 0xab, 0xb2, 0x56, 0x2e, 0x6b, 0x9b, 0xc7, 0xb7, 0x98, 0xc8,
|
|
||||||
+ 0xaf, 0xca, 0x5a, 0x85, 0xae, 0xcf, 0x24, 0x8b, 0xb2, 0x56, 0xf0, 0xaa, 0x5c, 0x5e, 0x6c, 0x91,
|
|
||||||
+ 0x8c, 0xfb, 0x4a, 0x1b, 0xa7, 0xab, 0x1e, 0x51, 0xbe, 0x1b, 0x30, 0xfe, 0x28, 0xe9, 0x10, 0xbe,
|
|
||||||
+ 0x27, 0xa8, 0xf3, 0x15, 0x7f, 0x2e, 0xa8, 0x4b, 0x1a, 0x81, 0x4b, 0x19, 0xed, 0x41, 0xff, 0xa9,
|
|
||||||
+ 0x29, 0xe6, 0x74, 0xa1, 0xa9, 0xd5, 0x29, 0x29, 0xac, 0x47, 0xa3, 0xb2, 0x2d, 0x1d, 0x59, 0x9f,
|
|
||||||
+ 0xc3, 0x60, 0xa1, 0xc8, 0x44, 0xf7, 0xd3, 0xc1, 0x62, 0x69, 0xf5, 0x79, 0xa9, 0x5a, 0xfb, 0xb0,
|
|
||||||
+ 0x32, 0x5f, 0x63, 0xa2, 0x57, 0xf4, 0xa5, 0x97, 0xd7, 0x9e, 0x97, 0xb2, 0xfa, 0x18, 0x9a, 0xa6,
|
|
||||||
+ 0xa6, 0x41, 0x7a, 0x80, 0x3b, 0x57, 0xe3, 0x5c, 0x76, 0x74, 0xab, 0xf3, 0xed, 0x77, 0xf7, 0x2b,
|
|
||||||
+ 0x7f, 0xff, 0xee, 0x7e, 0xe5, 0x9f, 0xdf, 0xdd, 0xaf, 0x1c, 0x37, 0xe4, 0xee, 0xfb, 0xff, 0x0e,
|
|
||||||
+ 0x00, 0x00, 0xff, 0xff, 0x8d, 0x89, 0xaa, 0x73, 0xc8, 0x21, 0x00, 0x00,
|
|
||||||
}
|
|
||||||
diff --git a/protocols/grpc/agent.proto b/protocols/grpc/agent.proto
|
|
||||||
index 348d792..b0fab6d 100644
|
|
||||||
--- a/protocols/grpc/agent.proto
|
|
||||||
+++ b/protocols/grpc/agent.proto
|
|
||||||
@@ -316,6 +316,7 @@ message UpdateInterfaceRequest {
|
|
||||||
|
|
||||||
message UpdateRoutesRequest {
|
|
||||||
Routes routes = 1;
|
|
||||||
+ bool increment = 2;
|
|
||||||
}
|
|
||||||
|
|
||||||
message ListInterfacesRequest {
|
|
||||||
--
|
|
||||||
2.14.3 (Apple Git-98)
|
|
||||||
|
|
||||||
@ -1,971 +0,0 @@
|
|||||||
From 0f24d3d433e93a7309b9bac59e4a15e722391490 Mon Sep 17 00:00:00 2001
|
|
||||||
From: xiadanni1 <xiadanni1@huawei.com>
|
|
||||||
Date: Tue, 18 Aug 2020 06:01:55 +0800
|
|
||||||
Subject: [PATCH 03/16] kata-agent: add kata-ipvs command
|
|
||||||
|
|
||||||
reason: add kata-ipvs command to update IPVS rules
|
|
||||||
in VM.
|
|
||||||
|
|
||||||
Signed-off-by: xiadanni1 <xiadanni1@huawei.com>
|
|
||||||
---
|
|
||||||
agent.go | 6 +
|
|
||||||
grpc.go | 4 +
|
|
||||||
ipvsadm.go | 130 +++++++++
|
|
||||||
protocols/grpc/agent.pb.go | 663 ++++++++++++++++++++++++++++++++-------------
|
|
||||||
protocols/grpc/agent.proto | 11 +
|
|
||||||
5 files changed, 633 insertions(+), 181 deletions(-)
|
|
||||||
create mode 100644 ipvsadm.go
|
|
||||||
|
|
||||||
diff --git a/agent.go b/agent.go
|
|
||||||
index c1cac08..c161e93 100644
|
|
||||||
--- a/agent.go
|
|
||||||
+++ b/agent.go
|
|
||||||
@@ -118,6 +118,11 @@ type sandboxStorage struct {
|
|
||||||
refCount int
|
|
||||||
}
|
|
||||||
|
|
||||||
+type ipvsAdm struct {
|
|
||||||
+ ipvsLock sync.Mutex
|
|
||||||
+ ipvsRuleCnt uint64
|
|
||||||
+}
|
|
||||||
+
|
|
||||||
type sandbox struct {
|
|
||||||
sync.RWMutex
|
|
||||||
ctx context.Context
|
|
||||||
@@ -144,6 +149,7 @@ type sandbox struct {
|
|
||||||
sandboxPidNs bool
|
|
||||||
storages map[string]*sandboxStorage
|
|
||||||
stopServer chan struct{}
|
|
||||||
+ ipvsadm ipvsAdm
|
|
||||||
}
|
|
||||||
|
|
||||||
var agentFields = logrus.Fields{
|
|
||||||
diff --git a/grpc.go b/grpc.go
|
|
||||||
index 8fe8217..de2cae7 100644
|
|
||||||
--- a/grpc.go
|
|
||||||
+++ b/grpc.go
|
|
||||||
@@ -1567,6 +1567,10 @@ func (a *agentGRPC) ListRoutes(ctx context.Context, req *pb.ListRoutesRequest) (
|
|
||||||
return a.sandbox.listRoutes(nil)
|
|
||||||
}
|
|
||||||
|
|
||||||
+func (a *agentGRPC) UpdateIPVSRule(ctx context.Context, req *pb.UpdateIPVSRequest) (*pb.IPVSResponse, error) {
|
|
||||||
+ return a.sandbox.updateIPVSRule(req)
|
|
||||||
+}
|
|
||||||
+
|
|
||||||
func (a *agentGRPC) OnlineCPUMem(ctx context.Context, req *pb.OnlineCPUMemRequest) (*gpb.Empty, error) {
|
|
||||||
if !req.Wait {
|
|
||||||
go a.onlineCPUMem(req)
|
|
||||||
diff --git a/ipvsadm.go b/ipvsadm.go
|
|
||||||
new file mode 100644
|
|
||||||
index 0000000..48eb19f
|
|
||||||
--- /dev/null
|
|
||||||
+++ b/ipvsadm.go
|
|
||||||
@@ -0,0 +1,130 @@
|
|
||||||
+// Copyright (c) Huawei Technologies Co., Ltd. 2020. All rights reserved.
|
|
||||||
+// SPDX-License-Identifier: Apache-2.0
|
|
||||||
+// Description: IPVS related common functions
|
|
||||||
+// Author: xiadanni
|
|
||||||
+// Create: 2020-08-01
|
|
||||||
+
|
|
||||||
+package main
|
|
||||||
+
|
|
||||||
+import (
|
|
||||||
+ "os/exec"
|
|
||||||
+ "strconv"
|
|
||||||
+ "strings"
|
|
||||||
+
|
|
||||||
+ "github.com/kata-containers/agent/protocols/grpc"
|
|
||||||
+ "google.golang.org/grpc/codes"
|
|
||||||
+ grpcStatus "google.golang.org/grpc/status"
|
|
||||||
+)
|
|
||||||
+
|
|
||||||
+var (
|
|
||||||
+ errNoIPVSRules = grpcStatus.Errorf(codes.InvalidArgument, "IPVS rule is nil")
|
|
||||||
+ errRuleExist = grpcStatus.Errorf(codes.InvalidArgument, "failed to restore IPVS rules: exist some rules in system, should clear first")
|
|
||||||
+ errInvalidIPVSRule = grpcStatus.Errorf(codes.InvalidArgument, "invalid IPVS rule, please check")
|
|
||||||
+ errIPVSRuleExceed = grpcStatus.Errorf(codes.InvalidArgument, "rules exceed limit, should clear first")
|
|
||||||
+)
|
|
||||||
+
|
|
||||||
+const (
|
|
||||||
+ validHeadLength = 7
|
|
||||||
+ ruleLimitMax = 20000
|
|
||||||
+ restoreHead = "restore"
|
|
||||||
+ conntrackNormalInfo = "have been deleted"
|
|
||||||
+ waitError = "no child processes"
|
|
||||||
+)
|
|
||||||
+
|
|
||||||
+func (s *sandbox) updateIPVSRule(ipvsRule *grpc.UpdateIPVSRequest) (resultingIpvs *grpc.IPVSResponse, err error) {
|
|
||||||
+ var (
|
|
||||||
+ restoreRuleCnt string
|
|
||||||
+ restoreRule string
|
|
||||||
+ operation string
|
|
||||||
+ restoreRuleCntUint uint64
|
|
||||||
+ cmd *exec.Cmd
|
|
||||||
+ )
|
|
||||||
+
|
|
||||||
+ if ipvsRule == nil || ipvsRule.IPVSReq == "" {
|
|
||||||
+ return nil, errNoIPVSRules
|
|
||||||
+ }
|
|
||||||
+
|
|
||||||
+ s.ipvsadm.ipvsLock.Lock()
|
|
||||||
+ defer s.ipvsadm.ipvsLock.Unlock()
|
|
||||||
+
|
|
||||||
+ rule := ipvsRule.IPVSReq
|
|
||||||
+ if len(rule) < validHeadLength {
|
|
||||||
+ return nil, errInvalidIPVSRule
|
|
||||||
+ }
|
|
||||||
+
|
|
||||||
+ if ruleHead := rule[:validHeadLength]; ruleHead == restoreHead {
|
|
||||||
+ if s.ipvsadm.ipvsRuleCnt != 0 {
|
|
||||||
+ return nil, errRuleExist
|
|
||||||
+ }
|
|
||||||
+ operation = restoreHead
|
|
||||||
+
|
|
||||||
+ restoreItem := strings.Split(rule, "|")
|
|
||||||
+ if len(restoreItem) != 3 {
|
|
||||||
+ return nil, errInvalidIPVSRule
|
|
||||||
+ }
|
|
||||||
+ restoreRuleCnt = restoreItem[1]
|
|
||||||
+ if restoreRuleCntUint, err = strconv.ParseUint(restoreRuleCnt, 10, 64); err != nil {
|
|
||||||
+ return nil, errInvalidIPVSRule
|
|
||||||
+ }
|
|
||||||
+ restoreRule = restoreItem[2]
|
|
||||||
+
|
|
||||||
+ cmd = exec.Command("/sbin/ipvsadm", "--restore")
|
|
||||||
+ cmd.Stdin = strings.NewReader(restoreRule)
|
|
||||||
+ } else {
|
|
||||||
+ item := strings.Fields(strings.TrimSpace(rule))
|
|
||||||
+ if len(item) < 2 {
|
|
||||||
+ return nil, errInvalidIPVSRule
|
|
||||||
+ }
|
|
||||||
+ if item[0] != "ipvsadm" && item[0] != "conntrack" {
|
|
||||||
+ return nil, errInvalidIPVSRule
|
|
||||||
+ }
|
|
||||||
+ operation = item[1]
|
|
||||||
+ if s.ipvsadm.ipvsRuleCnt >= ruleLimitMax {
|
|
||||||
+ if operation == "--add-service" || operation == "-A" || operation == "--add-server" || operation == "-a" {
|
|
||||||
+ return nil, errIPVSRuleExceed
|
|
||||||
+ }
|
|
||||||
+ }
|
|
||||||
+
|
|
||||||
+ restoreRuleCntUint = 0
|
|
||||||
+
|
|
||||||
+ cmd = exec.Command("/sbin/" + item[0])
|
|
||||||
+ for i := 1; i < len(item); i++ {
|
|
||||||
+ cmd.Args = append(cmd.Args, item[i])
|
|
||||||
+ }
|
|
||||||
+ }
|
|
||||||
+
|
|
||||||
+ bytes, err := cmd.CombinedOutput()
|
|
||||||
+
|
|
||||||
+ if err != nil && !strings.Contains(err.Error(), waitError) && !strings.Contains(string(bytes), conntrackNormalInfo) {
|
|
||||||
+ return nil, grpcStatus.Errorf(codes.Internal,
|
|
||||||
+ "exec IPVS command failed, stderr: %v, err: %v", string(bytes), err)
|
|
||||||
+ }
|
|
||||||
+
|
|
||||||
+ s.sumRuleCount(operation, restoreRuleCntUint)
|
|
||||||
+
|
|
||||||
+ rsp := &grpc.IPVSResponse{
|
|
||||||
+ IPVSRes: string(bytes),
|
|
||||||
+ }
|
|
||||||
+ return rsp, nil
|
|
||||||
+}
|
|
||||||
+
|
|
||||||
+func (s *sandbox) sumRuleCount(operation string, restoreRuleCntUint uint64) {
|
|
||||||
+ if operation == restoreHead {
|
|
||||||
+ s.ipvsadm.ipvsRuleCnt = restoreRuleCntUint
|
|
||||||
+ return
|
|
||||||
+ }
|
|
||||||
+ if operation == "--add-service" || operation == "-A" || operation == "--add-server" || operation == "-a" {
|
|
||||||
+ s.ipvsadm.ipvsRuleCnt++
|
|
||||||
+ return
|
|
||||||
+ }
|
|
||||||
+ if operation == "--delete-service" || operation == "-D" || operation == "--delete-server" || operation == "-d" {
|
|
||||||
+ if s.ipvsadm.ipvsRuleCnt > 0 {
|
|
||||||
+ s.ipvsadm.ipvsRuleCnt--
|
|
||||||
+ }
|
|
||||||
+ return
|
|
||||||
+ }
|
|
||||||
+ if operation == "--clear" || operation == "-C" {
|
|
||||||
+ s.ipvsadm.ipvsRuleCnt = 0
|
|
||||||
+ return
|
|
||||||
+ }
|
|
||||||
+}
|
|
||||||
diff --git a/protocols/grpc/agent.pb.go b/protocols/grpc/agent.pb.go
|
|
||||||
index 1b887e5..04d0ee5 100644
|
|
||||||
--- a/protocols/grpc/agent.pb.go
|
|
||||||
+++ b/protocols/grpc/agent.pb.go
|
|
||||||
@@ -63,6 +63,8 @@
|
|
||||||
CopyFileRequest
|
|
||||||
StartTracingRequest
|
|
||||||
StopTracingRequest
|
|
||||||
+ UpdateIPVSRequest
|
|
||||||
+ IPVSResponse
|
|
||||||
CheckRequest
|
|
||||||
HealthCheckResponse
|
|
||||||
VersionCheckResponse
|
|
||||||
@@ -1842,6 +1844,40 @@ func (m *StopTracingRequest) String() string { return proto.CompactTe
|
|
||||||
func (*StopTracingRequest) ProtoMessage() {}
|
|
||||||
func (*StopTracingRequest) Descriptor() ([]byte, []int) { return fileDescriptorAgent, []int{52} }
|
|
||||||
|
|
||||||
+type UpdateIPVSRequest struct {
|
|
||||||
+ // IPVS_req is the IPVS rule message needed to update
|
|
||||||
+ IPVSReq string `protobuf:"bytes,1,opt,name=IPVS_req,json=IPVSReq,proto3" json:"IPVS_req,omitempty"`
|
|
||||||
+}
|
|
||||||
+
|
|
||||||
+func (m *UpdateIPVSRequest) Reset() { *m = UpdateIPVSRequest{} }
|
|
||||||
+func (m *UpdateIPVSRequest) String() string { return proto.CompactTextString(m) }
|
|
||||||
+func (*UpdateIPVSRequest) ProtoMessage() {}
|
|
||||||
+func (*UpdateIPVSRequest) Descriptor() ([]byte, []int) { return fileDescriptorAgent, []int{53} }
|
|
||||||
+
|
|
||||||
+func (m *UpdateIPVSRequest) GetIPVSReq() string {
|
|
||||||
+ if m != nil {
|
|
||||||
+ return m.IPVSReq
|
|
||||||
+ }
|
|
||||||
+ return ""
|
|
||||||
+}
|
|
||||||
+
|
|
||||||
+type IPVSResponse struct {
|
|
||||||
+ // IPVS_res is the response of IPVS updating
|
|
||||||
+ IPVSRes string `protobuf:"bytes,1,opt,name=IPVS_res,json=IPVSRes,proto3" json:"IPVS_res,omitempty"`
|
|
||||||
+}
|
|
||||||
+
|
|
||||||
+func (m *IPVSResponse) Reset() { *m = IPVSResponse{} }
|
|
||||||
+func (m *IPVSResponse) String() string { return proto.CompactTextString(m) }
|
|
||||||
+func (*IPVSResponse) ProtoMessage() {}
|
|
||||||
+func (*IPVSResponse) Descriptor() ([]byte, []int) { return fileDescriptorAgent, []int{54} }
|
|
||||||
+
|
|
||||||
+func (m *IPVSResponse) GetIPVSRes() string {
|
|
||||||
+ if m != nil {
|
|
||||||
+ return m.IPVSRes
|
|
||||||
+ }
|
|
||||||
+ return ""
|
|
||||||
+}
|
|
||||||
+
|
|
||||||
func init() {
|
|
||||||
proto.RegisterType((*CreateContainerRequest)(nil), "grpc.CreateContainerRequest")
|
|
||||||
proto.RegisterType((*StartContainerRequest)(nil), "grpc.StartContainerRequest")
|
|
||||||
@@ -1896,6 +1932,8 @@ func init() {
|
|
||||||
proto.RegisterType((*CopyFileRequest)(nil), "grpc.CopyFileRequest")
|
|
||||||
proto.RegisterType((*StartTracingRequest)(nil), "grpc.StartTracingRequest")
|
|
||||||
proto.RegisterType((*StopTracingRequest)(nil), "grpc.StopTracingRequest")
|
|
||||||
+ proto.RegisterType((*UpdateIPVSRequest)(nil), "grpc.UpdateIPVSRequest")
|
|
||||||
+ proto.RegisterType((*IPVSResponse)(nil), "grpc.IPVSResponse")
|
|
||||||
}
|
|
||||||
|
|
||||||
// Reference imports to suppress errors if they are not otherwise used.
|
|
||||||
@@ -1938,6 +1976,7 @@ type AgentServiceClient interface {
|
|
||||||
UpdateRoutes(ctx context.Context, in *UpdateRoutesRequest, opts ...grpc1.CallOption) (*Routes, error)
|
|
||||||
ListInterfaces(ctx context.Context, in *ListInterfacesRequest, opts ...grpc1.CallOption) (*Interfaces, error)
|
|
||||||
ListRoutes(ctx context.Context, in *ListRoutesRequest, opts ...grpc1.CallOption) (*Routes, error)
|
|
||||||
+ UpdateIPVSRule(ctx context.Context, in *UpdateIPVSRequest, opts ...grpc1.CallOption) (*IPVSResponse, error)
|
|
||||||
// tracing
|
|
||||||
StartTracing(ctx context.Context, in *StartTracingRequest, opts ...grpc1.CallOption) (*google_protobuf2.Empty, error)
|
|
||||||
StopTracing(ctx context.Context, in *StopTracingRequest, opts ...grpc1.CallOption) (*google_protobuf2.Empty, error)
|
|
||||||
@@ -2140,6 +2179,15 @@ func (c *agentServiceClient) ListRoutes(ctx context.Context, in *ListRoutesReque
|
|
||||||
return out, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
+func (c *agentServiceClient) UpdateIPVSRule(ctx context.Context, in *UpdateIPVSRequest, opts ...grpc1.CallOption) (*IPVSResponse, error) {
|
|
||||||
+ out := new(IPVSResponse)
|
|
||||||
+ err := grpc1.Invoke(ctx, "/grpc.AgentService/UpdateIPVSRule", in, out, c.cc, opts...)
|
|
||||||
+ if err != nil {
|
|
||||||
+ return nil, err
|
|
||||||
+ }
|
|
||||||
+ return out, nil
|
|
||||||
+}
|
|
||||||
+
|
|
||||||
func (c *agentServiceClient) StartTracing(ctx context.Context, in *StartTracingRequest, opts ...grpc1.CallOption) (*google_protobuf2.Empty, error) {
|
|
||||||
out := new(google_protobuf2.Empty)
|
|
||||||
err := grpc1.Invoke(ctx, "/grpc.AgentService/StartTracing", in, out, c.cc, opts...)
|
|
||||||
@@ -2262,6 +2310,7 @@ type AgentServiceServer interface {
|
|
||||||
UpdateRoutes(context.Context, *UpdateRoutesRequest) (*Routes, error)
|
|
||||||
ListInterfaces(context.Context, *ListInterfacesRequest) (*Interfaces, error)
|
|
||||||
ListRoutes(context.Context, *ListRoutesRequest) (*Routes, error)
|
|
||||||
+ UpdateIPVSRule(context.Context, *UpdateIPVSRequest) (*IPVSResponse, error)
|
|
||||||
// tracing
|
|
||||||
StartTracing(context.Context, *StartTracingRequest) (*google_protobuf2.Empty, error)
|
|
||||||
StopTracing(context.Context, *StopTracingRequest) (*google_protobuf2.Empty, error)
|
|
||||||
@@ -2640,6 +2689,24 @@ func _AgentService_ListRoutes_Handler(srv interface{}, ctx context.Context, dec
|
|
||||||
return interceptor(ctx, in, info, handler)
|
|
||||||
}
|
|
||||||
|
|
||||||
+func _AgentService_UpdateIPVSRule_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc1.UnaryServerInterceptor) (interface{}, error) {
|
|
||||||
+ in := new(UpdateIPVSRequest)
|
|
||||||
+ if err := dec(in); err != nil {
|
|
||||||
+ return nil, err
|
|
||||||
+ }
|
|
||||||
+ if interceptor == nil {
|
|
||||||
+ return srv.(AgentServiceServer).UpdateIPVSRule(ctx, in)
|
|
||||||
+ }
|
|
||||||
+ info := &grpc1.UnaryServerInfo{
|
|
||||||
+ Server: srv,
|
|
||||||
+ FullMethod: "/grpc.AgentService/UpdateIPVSRule",
|
|
||||||
+ }
|
|
||||||
+ handler := func(ctx context.Context, req interface{}) (interface{}, error) {
|
|
||||||
+ return srv.(AgentServiceServer).UpdateIPVSRule(ctx, req.(*UpdateIPVSRequest))
|
|
||||||
+ }
|
|
||||||
+ return interceptor(ctx, in, info, handler)
|
|
||||||
+}
|
|
||||||
+
|
|
||||||
func _AgentService_StartTracing_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc1.UnaryServerInterceptor) (interface{}, error) {
|
|
||||||
in := new(StartTracingRequest)
|
|
||||||
if err := dec(in); err != nil {
|
|
||||||
@@ -2904,6 +2971,10 @@ var _AgentService_serviceDesc = grpc1.ServiceDesc{
|
|
||||||
MethodName: "ListRoutes",
|
|
||||||
Handler: _AgentService_ListRoutes_Handler,
|
|
||||||
},
|
|
||||||
+ {
|
|
||||||
+ MethodName: "UpdateIPVSRule",
|
|
||||||
+ Handler: _AgentService_UpdateIPVSRule_Handler,
|
|
||||||
+ },
|
|
||||||
{
|
|
||||||
MethodName: "StartTracing",
|
|
||||||
Handler: _AgentService_StartTracing_Handler,
|
|
||||||
@@ -5088,6 +5159,54 @@ func (m *StopTracingRequest) MarshalTo(dAtA []byte) (int, error) {
|
|
||||||
return i, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
+func (m *UpdateIPVSRequest) Marshal() (dAtA []byte, err error) {
|
|
||||||
+ size := m.Size()
|
|
||||||
+ dAtA = make([]byte, size)
|
|
||||||
+ n, err := m.MarshalTo(dAtA)
|
|
||||||
+ if err != nil {
|
|
||||||
+ return nil, err
|
|
||||||
+ }
|
|
||||||
+ return dAtA[:n], nil
|
|
||||||
+}
|
|
||||||
+
|
|
||||||
+func (m *UpdateIPVSRequest) MarshalTo(dAtA []byte) (int, error) {
|
|
||||||
+ var i int
|
|
||||||
+ _ = i
|
|
||||||
+ var l int
|
|
||||||
+ _ = l
|
|
||||||
+ if len(m.IPVSReq) > 0 {
|
|
||||||
+ dAtA[i] = 0xa
|
|
||||||
+ i++
|
|
||||||
+ i = encodeVarintAgent(dAtA, i, uint64(len(m.IPVSReq)))
|
|
||||||
+ i += copy(dAtA[i:], m.IPVSReq)
|
|
||||||
+ }
|
|
||||||
+ return i, nil
|
|
||||||
+}
|
|
||||||
+
|
|
||||||
+func (m *IPVSResponse) Marshal() (dAtA []byte, err error) {
|
|
||||||
+ size := m.Size()
|
|
||||||
+ dAtA = make([]byte, size)
|
|
||||||
+ n, err := m.MarshalTo(dAtA)
|
|
||||||
+ if err != nil {
|
|
||||||
+ return nil, err
|
|
||||||
+ }
|
|
||||||
+ return dAtA[:n], nil
|
|
||||||
+}
|
|
||||||
+
|
|
||||||
+func (m *IPVSResponse) MarshalTo(dAtA []byte) (int, error) {
|
|
||||||
+ var i int
|
|
||||||
+ _ = i
|
|
||||||
+ var l int
|
|
||||||
+ _ = l
|
|
||||||
+ if len(m.IPVSRes) > 0 {
|
|
||||||
+ dAtA[i] = 0xa
|
|
||||||
+ i++
|
|
||||||
+ i = encodeVarintAgent(dAtA, i, uint64(len(m.IPVSRes)))
|
|
||||||
+ i += copy(dAtA[i:], m.IPVSRes)
|
|
||||||
+ }
|
|
||||||
+ return i, nil
|
|
||||||
+}
|
|
||||||
+
|
|
||||||
func encodeVarintAgent(dAtA []byte, offset int, v uint64) int {
|
|
||||||
for v >= 1<<7 {
|
|
||||||
dAtA[offset] = uint8(v&0x7f | 0x80)
|
|
||||||
@@ -6019,6 +6138,26 @@ func (m *StopTracingRequest) Size() (n int) {
|
|
||||||
return n
|
|
||||||
}
|
|
||||||
|
|
||||||
+func (m *UpdateIPVSRequest) Size() (n int) {
|
|
||||||
+ var l int
|
|
||||||
+ _ = l
|
|
||||||
+ l = len(m.IPVSReq)
|
|
||||||
+ if l > 0 {
|
|
||||||
+ n += 1 + l + sovAgent(uint64(l))
|
|
||||||
+ }
|
|
||||||
+ return n
|
|
||||||
+}
|
|
||||||
+
|
|
||||||
+func (m *IPVSResponse) Size() (n int) {
|
|
||||||
+ var l int
|
|
||||||
+ _ = l
|
|
||||||
+ l = len(m.IPVSRes)
|
|
||||||
+ if l > 0 {
|
|
||||||
+ n += 1 + l + sovAgent(uint64(l))
|
|
||||||
+ }
|
|
||||||
+ return n
|
|
||||||
+}
|
|
||||||
+
|
|
||||||
func sovAgent(x uint64) (n int) {
|
|
||||||
for {
|
|
||||||
n++
|
|
||||||
@@ -12785,6 +12924,164 @@ func (m *StopTracingRequest) Unmarshal(dAtA []byte) error {
|
|
||||||
}
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
+func (m *UpdateIPVSRequest) Unmarshal(dAtA []byte) error {
|
|
||||||
+ l := len(dAtA)
|
|
||||||
+ iNdEx := 0
|
|
||||||
+ for iNdEx < l {
|
|
||||||
+ preIndex := iNdEx
|
|
||||||
+ var wire uint64
|
|
||||||
+ for shift := uint(0); ; shift += 7 {
|
|
||||||
+ if shift >= 64 {
|
|
||||||
+ return ErrIntOverflowAgent
|
|
||||||
+ }
|
|
||||||
+ if iNdEx >= l {
|
|
||||||
+ return io.ErrUnexpectedEOF
|
|
||||||
+ }
|
|
||||||
+ b := dAtA[iNdEx]
|
|
||||||
+ iNdEx++
|
|
||||||
+ wire |= (uint64(b) & 0x7F) << shift
|
|
||||||
+ if b < 0x80 {
|
|
||||||
+ break
|
|
||||||
+ }
|
|
||||||
+ }
|
|
||||||
+ fieldNum := int32(wire >> 3)
|
|
||||||
+ wireType := int(wire & 0x7)
|
|
||||||
+ if wireType == 4 {
|
|
||||||
+ return fmt.Errorf("proto: UpdateIPVSRequest: wiretype end group for non-group")
|
|
||||||
+ }
|
|
||||||
+ if fieldNum <= 0 {
|
|
||||||
+ return fmt.Errorf("proto: UpdateIPVSRequest: illegal tag %d (wire type %d)", fieldNum, wire)
|
|
||||||
+ }
|
|
||||||
+ switch fieldNum {
|
|
||||||
+ case 1:
|
|
||||||
+ if wireType != 2 {
|
|
||||||
+ return fmt.Errorf("proto: wrong wireType = %d for field IPVSReq", wireType)
|
|
||||||
+ }
|
|
||||||
+ var stringLen uint64
|
|
||||||
+ for shift := uint(0); ; shift += 7 {
|
|
||||||
+ if shift >= 64 {
|
|
||||||
+ return ErrIntOverflowAgent
|
|
||||||
+ }
|
|
||||||
+ if iNdEx >= l {
|
|
||||||
+ return io.ErrUnexpectedEOF
|
|
||||||
+ }
|
|
||||||
+ b := dAtA[iNdEx]
|
|
||||||
+ iNdEx++
|
|
||||||
+ stringLen |= (uint64(b) & 0x7F) << shift
|
|
||||||
+ if b < 0x80 {
|
|
||||||
+ break
|
|
||||||
+ }
|
|
||||||
+ }
|
|
||||||
+ intStringLen := int(stringLen)
|
|
||||||
+ if intStringLen < 0 {
|
|
||||||
+ return ErrInvalidLengthAgent
|
|
||||||
+ }
|
|
||||||
+ postIndex := iNdEx + intStringLen
|
|
||||||
+ if postIndex > l {
|
|
||||||
+ return io.ErrUnexpectedEOF
|
|
||||||
+ }
|
|
||||||
+ m.IPVSReq = string(dAtA[iNdEx:postIndex])
|
|
||||||
+ iNdEx = postIndex
|
|
||||||
+ default:
|
|
||||||
+ iNdEx = preIndex
|
|
||||||
+ skippy, err := skipAgent(dAtA[iNdEx:])
|
|
||||||
+ if err != nil {
|
|
||||||
+ return err
|
|
||||||
+ }
|
|
||||||
+ if skippy < 0 {
|
|
||||||
+ return ErrInvalidLengthAgent
|
|
||||||
+ }
|
|
||||||
+ if (iNdEx + skippy) > l {
|
|
||||||
+ return io.ErrUnexpectedEOF
|
|
||||||
+ }
|
|
||||||
+ iNdEx += skippy
|
|
||||||
+ }
|
|
||||||
+ }
|
|
||||||
+
|
|
||||||
+ if iNdEx > l {
|
|
||||||
+ return io.ErrUnexpectedEOF
|
|
||||||
+ }
|
|
||||||
+ return nil
|
|
||||||
+}
|
|
||||||
+func (m *IPVSResponse) Unmarshal(dAtA []byte) error {
|
|
||||||
+ l := len(dAtA)
|
|
||||||
+ iNdEx := 0
|
|
||||||
+ for iNdEx < l {
|
|
||||||
+ preIndex := iNdEx
|
|
||||||
+ var wire uint64
|
|
||||||
+ for shift := uint(0); ; shift += 7 {
|
|
||||||
+ if shift >= 64 {
|
|
||||||
+ return ErrIntOverflowAgent
|
|
||||||
+ }
|
|
||||||
+ if iNdEx >= l {
|
|
||||||
+ return io.ErrUnexpectedEOF
|
|
||||||
+ }
|
|
||||||
+ b := dAtA[iNdEx]
|
|
||||||
+ iNdEx++
|
|
||||||
+ wire |= (uint64(b) & 0x7F) << shift
|
|
||||||
+ if b < 0x80 {
|
|
||||||
+ break
|
|
||||||
+ }
|
|
||||||
+ }
|
|
||||||
+ fieldNum := int32(wire >> 3)
|
|
||||||
+ wireType := int(wire & 0x7)
|
|
||||||
+ if wireType == 4 {
|
|
||||||
+ return fmt.Errorf("proto: IPVSResponse: wiretype end group for non-group")
|
|
||||||
+ }
|
|
||||||
+ if fieldNum <= 0 {
|
|
||||||
+ return fmt.Errorf("proto: IPVSResponse: illegal tag %d (wire type %d)", fieldNum, wire)
|
|
||||||
+ }
|
|
||||||
+ switch fieldNum {
|
|
||||||
+ case 1:
|
|
||||||
+ if wireType != 2 {
|
|
||||||
+ return fmt.Errorf("proto: wrong wireType = %d for field IPVSRes", wireType)
|
|
||||||
+ }
|
|
||||||
+ var stringLen uint64
|
|
||||||
+ for shift := uint(0); ; shift += 7 {
|
|
||||||
+ if shift >= 64 {
|
|
||||||
+ return ErrIntOverflowAgent
|
|
||||||
+ }
|
|
||||||
+ if iNdEx >= l {
|
|
||||||
+ return io.ErrUnexpectedEOF
|
|
||||||
+ }
|
|
||||||
+ b := dAtA[iNdEx]
|
|
||||||
+ iNdEx++
|
|
||||||
+ stringLen |= (uint64(b) & 0x7F) << shift
|
|
||||||
+ if b < 0x80 {
|
|
||||||
+ break
|
|
||||||
+ }
|
|
||||||
+ }
|
|
||||||
+ intStringLen := int(stringLen)
|
|
||||||
+ if intStringLen < 0 {
|
|
||||||
+ return ErrInvalidLengthAgent
|
|
||||||
+ }
|
|
||||||
+ postIndex := iNdEx + intStringLen
|
|
||||||
+ if postIndex > l {
|
|
||||||
+ return io.ErrUnexpectedEOF
|
|
||||||
+ }
|
|
||||||
+ m.IPVSRes = string(dAtA[iNdEx:postIndex])
|
|
||||||
+ iNdEx = postIndex
|
|
||||||
+ default:
|
|
||||||
+ iNdEx = preIndex
|
|
||||||
+ skippy, err := skipAgent(dAtA[iNdEx:])
|
|
||||||
+ if err != nil {
|
|
||||||
+ return err
|
|
||||||
+ }
|
|
||||||
+ if skippy < 0 {
|
|
||||||
+ return ErrInvalidLengthAgent
|
|
||||||
+ }
|
|
||||||
+ if (iNdEx + skippy) > l {
|
|
||||||
+ return io.ErrUnexpectedEOF
|
|
||||||
+ }
|
|
||||||
+ iNdEx += skippy
|
|
||||||
+ }
|
|
||||||
+ }
|
|
||||||
+
|
|
||||||
+ if iNdEx > l {
|
|
||||||
+ return io.ErrUnexpectedEOF
|
|
||||||
+ }
|
|
||||||
+ return nil
|
|
||||||
+}
|
|
||||||
func skipAgent(dAtA []byte) (n int, err error) {
|
|
||||||
l := len(dAtA)
|
|
||||||
iNdEx := 0
|
|
||||||
@@ -12893,185 +13190,189 @@ var (
|
|
||||||
func init() { proto.RegisterFile("agent.proto", fileDescriptorAgent) }
|
|
||||||
|
|
||||||
var fileDescriptorAgent = []byte{
|
|
||||||
- // 2876 bytes of a gzipped FileDescriptorProto
|
|
||||||
- 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xc4, 0x39, 0x4b, 0x6f, 0x1c, 0xc7,
|
|
||||||
- 0xd1, 0xd8, 0x07, 0x97, 0xbb, 0xb5, 0x2f, 0x6e, 0x93, 0xa2, 0x56, 0x2b, 0x59, 0x9f, 0x3c, 0xb6,
|
|
||||||
- 0x65, 0xfa, 0xf3, 0xe7, 0xa5, 0x2d, 0x1b, 0x9f, 0x5f, 0x70, 0x04, 0xf1, 0x11, 0x91, 0xb1, 0x15,
|
|
||||||
- 0x31, 0x43, 0x11, 0x4e, 0x10, 0x04, 0x83, 0xe1, 0x4c, 0x73, 0xd9, 0xe6, 0xce, 0xf4, 0xb8, 0xa7,
|
|
||||||
- 0x87, 0xe2, 0x3a, 0x40, 0x8e, 0xc9, 0x2d, 0x97, 0x00, 0xb9, 0xe5, 0x0f, 0x04, 0xb9, 0xe5, 0x98,
|
|
||||||
- 0x6b, 0x0e, 0x46, 0x4e, 0xf9, 0x05, 0x41, 0xe0, 0x9f, 0x90, 0x5f, 0x10, 0xf4, 0x6b, 0x1e, 0xbb,
|
|
||||||
- 0x43, 0x1a, 0x21, 0x08, 0xe4, 0xb2, 0xe8, 0xaa, 0xae, 0xae, 0x57, 0x77, 0xd5, 0x54, 0xd5, 0x42,
|
|
||||||
- 0xdb, 0x9d, 0xe0, 0x90, 0x8f, 0x23, 0x46, 0x39, 0x45, 0xf5, 0x09, 0x8b, 0xbc, 0x51, 0x8b, 0x7a,
|
|
||||||
- 0x44, 0x21, 0x46, 0xff, 0x3f, 0x21, 0xfc, 0x34, 0x39, 0x1e, 0x7b, 0x34, 0xd8, 0x3c, 0x73, 0xb9,
|
|
||||||
- 0xfb, 0x8e, 0x47, 0x43, 0xee, 0x92, 0x10, 0xb3, 0x78, 0x53, 0x1e, 0xdc, 0x8c, 0xce, 0x26, 0x9b,
|
|
||||||
- 0x7c, 0x16, 0xe1, 0x58, 0xfd, 0xea, 0x73, 0x77, 0x27, 0x94, 0x4e, 0xa6, 0x78, 0x53, 0x42, 0xc7,
|
|
||||||
- 0xc9, 0xc9, 0x26, 0x0e, 0x22, 0x3e, 0x53, 0x9b, 0xd6, 0x1f, 0xaa, 0xb0, 0xbe, 0xcd, 0xb0, 0xcb,
|
|
||||||
- 0xf1, 0xb6, 0xe1, 0x66, 0xe3, 0xaf, 0x13, 0x1c, 0x73, 0xf4, 0x2a, 0x74, 0x52, 0x09, 0x0e, 0xf1,
|
|
||||||
- 0x87, 0x95, 0x07, 0x95, 0x8d, 0x96, 0xdd, 0x4e, 0x71, 0xfb, 0x3e, 0xba, 0x0d, 0xcb, 0xf8, 0x02,
|
|
||||||
- 0x7b, 0x62, 0xb7, 0x2a, 0x77, 0x1b, 0x02, 0xdc, 0xf7, 0xd1, 0x7b, 0xd0, 0x8e, 0x39, 0x23, 0xe1,
|
|
||||||
- 0xc4, 0x49, 0x62, 0xcc, 0x86, 0xb5, 0x07, 0x95, 0x8d, 0xf6, 0xa3, 0x95, 0xb1, 0x30, 0x69, 0x7c,
|
|
||||||
- 0x28, 0x37, 0x8e, 0x62, 0xcc, 0x6c, 0x88, 0xd3, 0x35, 0x7a, 0x08, 0xcb, 0x3e, 0x3e, 0x27, 0x1e,
|
|
||||||
- 0x8e, 0x87, 0xf5, 0x07, 0xb5, 0x8d, 0xf6, 0xa3, 0x8e, 0x22, 0xdf, 0x91, 0x48, 0xdb, 0x6c, 0xa2,
|
|
||||||
- 0xb7, 0xa0, 0x19, 0x73, 0xca, 0xdc, 0x09, 0x8e, 0x87, 0x4b, 0x92, 0xb0, 0x6b, 0xf8, 0x4a, 0xac,
|
|
||||||
- 0x9d, 0x6e, 0xa3, 0x7b, 0x50, 0x7b, 0xbe, 0xbd, 0x3f, 0x6c, 0x48, 0xe9, 0xa0, 0xa9, 0x22, 0xec,
|
|
||||||
- 0xd9, 0x02, 0x8d, 0x5e, 0x83, 0x6e, 0xec, 0x86, 0xfe, 0x31, 0xbd, 0x70, 0x22, 0xe2, 0x87, 0xf1,
|
|
||||||
- 0x70, 0xf9, 0x41, 0x65, 0xa3, 0x69, 0x77, 0x34, 0xf2, 0x40, 0xe0, 0xac, 0x4f, 0xe0, 0xd6, 0x21,
|
|
||||||
- 0x77, 0x19, 0xbf, 0x86, 0x77, 0xac, 0x23, 0x58, 0xb7, 0x71, 0x40, 0xcf, 0xaf, 0xe5, 0xda, 0x21,
|
|
||||||
- 0x2c, 0x73, 0x12, 0x60, 0x9a, 0x70, 0xe9, 0xda, 0xae, 0x6d, 0x40, 0xeb, 0x4f, 0x15, 0x40, 0xbb,
|
|
||||||
- 0x17, 0xd8, 0x3b, 0x60, 0xd4, 0xc3, 0x71, 0xfc, 0x5f, 0xba, 0xae, 0x37, 0x61, 0x39, 0x52, 0x0a,
|
|
||||||
- 0x0c, 0xeb, 0x92, 0x5c, 0xdf, 0x82, 0xd1, 0xca, 0xec, 0x5a, 0x5f, 0xc1, 0xda, 0x21, 0x99, 0x84,
|
|
||||||
- 0xee, 0xf4, 0x06, 0xf5, 0x5d, 0x87, 0x46, 0x2c, 0x79, 0x4a, 0x55, 0xbb, 0xb6, 0x86, 0xac, 0x03,
|
|
||||||
- 0x40, 0x5f, 0xba, 0x84, 0xdf, 0x9c, 0x24, 0xeb, 0x1d, 0x58, 0x2d, 0x70, 0x8c, 0x23, 0x1a, 0xc6,
|
|
||||||
- 0x58, 0x2a, 0xc0, 0x5d, 0x9e, 0xc4, 0x92, 0xd9, 0x92, 0xad, 0x21, 0x0b, 0xc3, 0xda, 0x17, 0x24,
|
|
||||||
- 0x36, 0xe4, 0xf8, 0x3f, 0x51, 0x61, 0x1d, 0x1a, 0x27, 0x94, 0x05, 0x2e, 0x37, 0x1a, 0x28, 0x08,
|
|
||||||
- 0x21, 0xa8, 0xbb, 0x6c, 0x12, 0x0f, 0x6b, 0x0f, 0x6a, 0x1b, 0x2d, 0x5b, 0xae, 0xc5, 0xab, 0x9c,
|
|
||||||
- 0x13, 0xa3, 0xf5, 0x7a, 0x15, 0x3a, 0xda, 0xef, 0xce, 0x94, 0xc4, 0x5c, 0xca, 0xe9, 0xd8, 0x6d,
|
|
||||||
- 0x8d, 0x13, 0x67, 0x2c, 0x0a, 0xeb, 0x47, 0x91, 0x7f, 0xcd, 0x80, 0x7f, 0x04, 0x2d, 0x86, 0x63,
|
|
||||||
- 0x9a, 0x30, 0x11, 0xa6, 0x55, 0x79, 0xef, 0x6b, 0xea, 0xde, 0xbf, 0x20, 0x61, 0x72, 0x61, 0x9b,
|
|
||||||
- 0x3d, 0x3b, 0x23, 0xd3, 0x21, 0xc4, 0xe3, 0xeb, 0x84, 0xd0, 0x27, 0x70, 0xeb, 0xc0, 0x4d, 0xe2,
|
|
||||||
- 0xeb, 0xe8, 0x6a, 0x7d, 0x2a, 0xc2, 0x2f, 0x4e, 0x82, 0x6b, 0x1d, 0xfe, 0x63, 0x05, 0x9a, 0xdb,
|
|
||||||
- 0x51, 0x72, 0x14, 0xbb, 0x13, 0x8c, 0xfe, 0x07, 0xda, 0x9c, 0x72, 0x77, 0xea, 0x24, 0x02, 0x94,
|
|
||||||
- 0xe4, 0x75, 0x1b, 0x24, 0x4a, 0x11, 0x08, 0xb7, 0x63, 0xe6, 0x45, 0x89, 0xa6, 0xa8, 0x3e, 0xa8,
|
|
||||||
- 0x6d, 0xd4, 0xed, 0xb6, 0xc2, 0x29, 0x92, 0x31, 0xac, 0xca, 0x3d, 0x87, 0x84, 0xce, 0x19, 0x66,
|
|
||||||
- 0x21, 0x9e, 0x06, 0xd4, 0xc7, 0xf2, 0xfd, 0xd6, 0xed, 0x81, 0xdc, 0xda, 0x0f, 0x3f, 0x4f, 0x37,
|
|
||||||
- 0xd0, 0xff, 0xc2, 0x20, 0xa5, 0x17, 0x41, 0x29, 0xa9, 0xeb, 0x92, 0xba, 0xaf, 0xa9, 0x8f, 0x34,
|
|
||||||
- 0xda, 0xfa, 0x15, 0xf4, 0x5e, 0x9c, 0x32, 0xca, 0xf9, 0x94, 0x84, 0x93, 0x1d, 0x97, 0xbb, 0x22,
|
|
||||||
- 0x7b, 0x44, 0x98, 0x11, 0xea, 0xc7, 0x5a, 0x5b, 0x03, 0xa2, 0xb7, 0x61, 0xc0, 0x15, 0x2d, 0xf6,
|
|
||||||
- 0x1d, 0x43, 0x53, 0x95, 0x34, 0x2b, 0xe9, 0xc6, 0x81, 0x26, 0x7e, 0x03, 0x7a, 0x19, 0xb1, 0xc8,
|
|
||||||
- 0x3f, 0x5a, 0xdf, 0x6e, 0x8a, 0x7d, 0x41, 0x02, 0x6c, 0x9d, 0x4b, 0x5f, 0xc9, 0x4b, 0x46, 0x6f,
|
|
||||||
- 0x43, 0x2b, 0xf3, 0x43, 0x45, 0xbe, 0x90, 0x9e, 0x7a, 0x21, 0xc6, 0x9d, 0x76, 0x33, 0x75, 0xca,
|
|
||||||
- 0x67, 0xd0, 0xe7, 0xa9, 0xe2, 0x8e, 0xef, 0x72, 0xb7, 0xf8, 0xa8, 0x8a, 0x56, 0xd9, 0x3d, 0x5e,
|
|
||||||
- 0x80, 0xad, 0x4f, 0xa1, 0x75, 0x40, 0xfc, 0x58, 0x09, 0x1e, 0xc2, 0xb2, 0x97, 0x30, 0x86, 0x43,
|
|
||||||
- 0x6e, 0x4c, 0xd6, 0x20, 0x5a, 0x83, 0xa5, 0x29, 0x09, 0x08, 0xd7, 0x66, 0x2a, 0xc0, 0xa2, 0x00,
|
|
||||||
- 0xcf, 0x70, 0x40, 0xd9, 0x4c, 0x3a, 0x6c, 0x0d, 0x96, 0xf2, 0x97, 0xab, 0x00, 0x74, 0x17, 0x5a,
|
|
||||||
- 0x81, 0x7b, 0x91, 0x5e, 0xaa, 0xd8, 0x69, 0x06, 0xee, 0x85, 0x52, 0x7e, 0x08, 0xcb, 0x27, 0x2e,
|
|
||||||
- 0x99, 0x7a, 0x21, 0xd7, 0x5e, 0x31, 0x60, 0x26, 0xb0, 0x9e, 0x17, 0xf8, 0xd7, 0x2a, 0xb4, 0x95,
|
|
||||||
- 0x44, 0xa5, 0xf0, 0x1a, 0x2c, 0x79, 0xae, 0x77, 0x9a, 0x8a, 0x94, 0x00, 0x7a, 0x68, 0x14, 0xa9,
|
|
||||||
- 0xe6, 0x93, 0x70, 0xa6, 0xa9, 0x51, 0x6d, 0x13, 0x20, 0x7e, 0xe9, 0x46, 0x5a, 0xb7, 0xda, 0x25,
|
|
||||||
- 0xc4, 0x2d, 0x41, 0xa3, 0xd4, 0x7d, 0x1f, 0x3a, 0xea, 0xdd, 0xe9, 0x23, 0xf5, 0x4b, 0x8e, 0xb4,
|
|
||||||
- 0x15, 0x95, 0x3a, 0xf4, 0x1a, 0x74, 0x93, 0x18, 0x3b, 0xa7, 0x04, 0x33, 0x97, 0x79, 0xa7, 0xb3,
|
|
||||||
- 0xe1, 0x92, 0xfa, 0x46, 0x26, 0x31, 0xde, 0x33, 0x38, 0xf4, 0x08, 0x96, 0x44, 0xfa, 0x8b, 0x87,
|
|
||||||
- 0x0d, 0xf9, 0x39, 0xbe, 0x97, 0x67, 0x29, 0x4d, 0x1d, 0xcb, 0xdf, 0xdd, 0x90, 0xb3, 0x99, 0xad,
|
|
||||||
- 0x48, 0x47, 0x1f, 0x01, 0x64, 0x48, 0xb4, 0x02, 0xb5, 0x33, 0x3c, 0xd3, 0x71, 0x28, 0x96, 0xc2,
|
|
||||||
- 0x39, 0xe7, 0xee, 0x34, 0x31, 0x5e, 0x57, 0xc0, 0x27, 0xd5, 0x8f, 0x2a, 0x96, 0x07, 0xfd, 0xad,
|
|
||||||
- 0xe9, 0x19, 0xa1, 0xb9, 0xe3, 0x6b, 0xb0, 0x14, 0xb8, 0x5f, 0x51, 0x66, 0x3c, 0x29, 0x01, 0x89,
|
|
||||||
- 0x25, 0x21, 0x65, 0x86, 0x85, 0x04, 0x50, 0x0f, 0xaa, 0x34, 0x92, 0xfe, 0x6a, 0xd9, 0x55, 0x1a,
|
|
||||||
- 0x65, 0x82, 0xea, 0x39, 0x41, 0xd6, 0x3f, 0xea, 0x00, 0x99, 0x14, 0x64, 0xc3, 0x88, 0x50, 0x27,
|
|
||||||
- 0xc6, 0x4c, 0x94, 0x20, 0xce, 0xf1, 0x8c, 0xe3, 0xd8, 0x61, 0xd8, 0x4b, 0x58, 0x4c, 0xce, 0xc5,
|
|
||||||
- 0xfd, 0x09, 0xb3, 0x6f, 0x29, 0xb3, 0xe7, 0x74, 0xb3, 0x6f, 0x13, 0x7a, 0xa8, 0xce, 0x6d, 0x89,
|
|
||||||
- 0x63, 0xb6, 0x39, 0x85, 0xf6, 0xe1, 0x56, 0xc6, 0xd3, 0xcf, 0xb1, 0xab, 0x5e, 0xc5, 0x6e, 0x35,
|
|
||||||
- 0x65, 0xe7, 0x67, 0xac, 0x76, 0x61, 0x95, 0x50, 0xe7, 0xeb, 0x04, 0x27, 0x05, 0x46, 0xb5, 0xab,
|
|
||||||
- 0x18, 0x0d, 0x08, 0xfd, 0x89, 0x3c, 0x90, 0xb1, 0x39, 0x80, 0x3b, 0x39, 0x2b, 0x45, 0xb8, 0xe7,
|
|
||||||
- 0x98, 0xd5, 0xaf, 0x62, 0xb6, 0x9e, 0x6a, 0x25, 0xf2, 0x41, 0xc6, 0xf1, 0x47, 0xb0, 0x4e, 0xa8,
|
|
||||||
- 0xf3, 0xd2, 0x25, 0x7c, 0x9e, 0xdd, 0xd2, 0xf7, 0x18, 0x29, 0x3e, 0xba, 0x45, 0x5e, 0xca, 0xc8,
|
|
||||||
- 0x00, 0xb3, 0x49, 0xc1, 0xc8, 0xc6, 0xf7, 0x18, 0xf9, 0x4c, 0x1e, 0xc8, 0xd8, 0x3c, 0x81, 0x01,
|
|
||||||
- 0xa1, 0xf3, 0xda, 0x2c, 0x5f, 0xc5, 0xa4, 0x4f, 0x68, 0x51, 0x93, 0x2d, 0x18, 0xc4, 0xd8, 0xe3,
|
|
||||||
- 0x94, 0xe5, 0x1f, 0x41, 0xf3, 0x2a, 0x16, 0x2b, 0x9a, 0x3e, 0xe5, 0x61, 0xfd, 0x1c, 0x3a, 0x7b,
|
|
||||||
- 0xc9, 0x04, 0xf3, 0xe9, 0x71, 0x9a, 0x0c, 0x6e, 0x2c, 0xff, 0x58, 0xff, 0xaa, 0x42, 0x7b, 0x7b,
|
|
||||||
- 0xc2, 0x68, 0x12, 0x15, 0x72, 0xb2, 0x0a, 0xd2, 0xf9, 0x9c, 0x2c, 0x49, 0x64, 0x4e, 0x56, 0xc4,
|
|
||||||
- 0x1f, 0x40, 0x27, 0x90, 0xa1, 0xab, 0xe9, 0x55, 0x1e, 0x1a, 0x2c, 0x04, 0xb5, 0xdd, 0x0e, 0x72,
|
|
||||||
- 0xc9, 0x6c, 0x0c, 0x10, 0x11, 0x3f, 0xd6, 0x67, 0x54, 0x3a, 0xea, 0xeb, 0x8a, 0xd0, 0xa4, 0x68,
|
|
||||||
- 0xbb, 0x15, 0xa5, 0xd9, 0xfa, 0x3d, 0x68, 0x1f, 0x0b, 0x27, 0xe9, 0x03, 0x85, 0x64, 0x94, 0x79,
|
|
||||||
- 0xcf, 0x86, 0xe3, 0x2c, 0x08, 0xf7, 0xa0, 0x7b, 0xaa, 0x5c, 0xa6, 0x0f, 0xa9, 0x37, 0xf4, 0x9a,
|
|
||||||
- 0xb6, 0x24, 0xb3, 0x77, 0x9c, 0xf7, 0xac, 0xba, 0x80, 0xce, 0x69, 0x0e, 0x35, 0x3a, 0x84, 0xc1,
|
|
||||||
- 0x02, 0x49, 0x49, 0x0e, 0xda, 0xc8, 0xe7, 0xa0, 0xf6, 0x23, 0xa4, 0x04, 0xe5, 0x4f, 0xe6, 0xf3,
|
|
||||||
- 0xd2, 0x6f, 0xab, 0xd0, 0xf9, 0x31, 0xe6, 0x2f, 0x29, 0x3b, 0x53, 0xfa, 0x22, 0xa8, 0x87, 0x6e,
|
|
||||||
- 0x80, 0x35, 0x47, 0xb9, 0x46, 0x77, 0xa0, 0xc9, 0x2e, 0x54, 0x02, 0xd1, 0xf7, 0xb9, 0xcc, 0x2e,
|
|
||||||
- 0x64, 0x62, 0x40, 0xaf, 0x00, 0xb0, 0x0b, 0x27, 0x72, 0xbd, 0x33, 0xac, 0x3d, 0x58, 0xb7, 0x5b,
|
|
||||||
- 0xec, 0xe2, 0x40, 0x21, 0xc4, 0x53, 0x60, 0x17, 0x0e, 0x66, 0x8c, 0xb2, 0x58, 0xe7, 0xaa, 0x26,
|
|
||||||
- 0xbb, 0xd8, 0x95, 0xb0, 0x3e, 0xeb, 0x33, 0x1a, 0x45, 0xd8, 0x97, 0x39, 0x5a, 0x9e, 0xdd, 0x51,
|
|
||||||
- 0x08, 0x21, 0x95, 0x1b, 0xa9, 0x0d, 0x25, 0x95, 0x67, 0x52, 0x79, 0x26, 0x75, 0x59, 0x9d, 0xe4,
|
|
||||||
- 0x79, 0xa9, 0x3c, 0x95, 0xda, 0x54, 0x52, 0x79, 0x4e, 0x2a, 0xcf, 0xa4, 0xb6, 0xcc, 0x59, 0x2d,
|
|
||||||
- 0xd5, 0xfa, 0x4d, 0x05, 0xd6, 0xe7, 0x0b, 0x3f, 0x5d, 0xa6, 0x7e, 0x00, 0x1d, 0x4f, 0xde, 0x57,
|
|
||||||
- 0xe1, 0x4d, 0x0e, 0x16, 0x6e, 0xd2, 0x6e, 0x7b, 0xb9, 0x67, 0xfc, 0x21, 0x74, 0x43, 0xe5, 0xe0,
|
|
||||||
- 0xf4, 0x69, 0xd6, 0xb2, 0x7b, 0xc9, 0xfb, 0xde, 0xee, 0x84, 0x39, 0xc8, 0xf2, 0x01, 0x7d, 0xc9,
|
|
||||||
- 0x08, 0xc7, 0x87, 0x9c, 0x61, 0x37, 0xb8, 0x89, 0x06, 0x04, 0x41, 0x5d, 0x56, 0x2b, 0x35, 0x59,
|
|
||||||
- 0x5f, 0xcb, 0xb5, 0xf5, 0x26, 0xac, 0x16, 0xa4, 0x68, 0x5b, 0x57, 0xa0, 0x36, 0xc5, 0xa1, 0xe4,
|
|
||||||
- 0xde, 0xb5, 0xc5, 0xd2, 0x72, 0x61, 0x60, 0x63, 0xd7, 0xbf, 0x39, 0x6d, 0xb4, 0x88, 0x5a, 0x26,
|
|
||||||
- 0x62, 0x03, 0x50, 0x5e, 0x84, 0x56, 0xc5, 0x68, 0x5d, 0xc9, 0x69, 0xfd, 0x1c, 0x06, 0xdb, 0x53,
|
|
||||||
- 0x1a, 0xe3, 0x43, 0xee, 0x93, 0xf0, 0x26, 0x3a, 0xa6, 0x5f, 0xc2, 0xea, 0x0b, 0x3e, 0xfb, 0x52,
|
|
||||||
- 0x30, 0x8b, 0xc9, 0x37, 0xf8, 0x86, 0xec, 0x63, 0xf4, 0xa5, 0xb1, 0x8f, 0xd1, 0x97, 0xa2, 0x59,
|
|
||||||
- 0xf2, 0xe8, 0x34, 0x09, 0x42, 0x19, 0x0a, 0x5d, 0x5b, 0x43, 0xd6, 0x16, 0x74, 0x54, 0x0d, 0xfd,
|
|
||||||
- 0x8c, 0xfa, 0xc9, 0x14, 0x97, 0xc6, 0xe0, 0x7d, 0x80, 0xc8, 0x65, 0x6e, 0x80, 0x39, 0x66, 0xea,
|
|
||||||
- 0x0d, 0xb5, 0xec, 0x1c, 0xc6, 0xfa, 0x7d, 0x15, 0xd6, 0xd4, 0x48, 0xe4, 0x50, 0x4d, 0x02, 0x8c,
|
|
||||||
- 0x09, 0x23, 0x68, 0x9e, 0xd2, 0x98, 0xe7, 0x18, 0xa6, 0xb0, 0x50, 0xd1, 0x0f, 0x0d, 0x37, 0xb1,
|
|
||||||
- 0x2c, 0xcc, 0x29, 0x6a, 0x57, 0xcf, 0x29, 0x16, 0x26, 0x11, 0xf5, 0xc5, 0x49, 0x84, 0x88, 0x36,
|
|
||||||
- 0x43, 0x44, 0x54, 0x8c, 0xb7, 0xec, 0x96, 0xc6, 0xec, 0xfb, 0xe8, 0x21, 0xf4, 0x27, 0x42, 0x4b,
|
|
||||||
- 0xe7, 0x94, 0xd2, 0x33, 0x27, 0x72, 0xf9, 0xa9, 0x0c, 0xf5, 0x96, 0xdd, 0x95, 0xe8, 0x3d, 0x4a,
|
|
||||||
- 0xcf, 0x0e, 0x5c, 0x7e, 0x8a, 0x3e, 0x86, 0x9e, 0x2e, 0x03, 0x03, 0xe9, 0xa2, 0x58, 0x7f, 0xfc,
|
|
||||||
- 0x74, 0x14, 0xe5, 0xbd, 0x67, 0x77, 0xcf, 0x72, 0x50, 0x6c, 0xdd, 0x86, 0x5b, 0x3b, 0x38, 0xe6,
|
|
||||||
- 0x8c, 0xce, 0x8a, 0x8e, 0xb1, 0x7e, 0x00, 0xb0, 0x1f, 0x72, 0xcc, 0x4e, 0x5c, 0x0f, 0xc7, 0xe8,
|
|
||||||
- 0xdd, 0x3c, 0xa4, 0x8b, 0xa3, 0x95, 0xb1, 0x9a, 0x48, 0xa5, 0x1b, 0x76, 0x8e, 0xc6, 0x1a, 0x43,
|
|
||||||
- 0xc3, 0xa6, 0x89, 0x48, 0x47, 0xaf, 0x9b, 0x95, 0x3e, 0xd7, 0xd1, 0xe7, 0x24, 0xd2, 0xd6, 0x7b,
|
|
||||||
- 0xd6, 0x9e, 0x69, 0x61, 0x33, 0x76, 0xfa, 0x8a, 0xc6, 0xd0, 0x22, 0x06, 0xa7, 0xb3, 0xca, 0xa2,
|
|
||||||
- 0xe8, 0x8c, 0xc4, 0xfa, 0x19, 0xac, 0x2a, 0x4e, 0x8a, 0xb3, 0x61, 0xf3, 0x3a, 0x34, 0x98, 0x51,
|
|
||||||
- 0xa3, 0x92, 0x8d, 0xa2, 0x34, 0x91, 0xde, 0x43, 0xf7, 0x84, 0x30, 0x8f, 0xe1, 0x40, 0xf4, 0x1c,
|
|
||||||
- 0x55, 0x79, 0x65, 0x19, 0x42, 0x78, 0x4b, 0xf4, 0xdb, 0x99, 0x99, 0xc6, 0x5b, 0xab, 0x30, 0x10,
|
|
||||||
- 0x1b, 0x05, 0x89, 0xd6, 0x2f, 0x60, 0xf5, 0x79, 0x38, 0x25, 0x21, 0xde, 0x3e, 0x38, 0x7a, 0x86,
|
|
||||||
- 0xd3, 0xac, 0x80, 0xa0, 0x2e, 0xaa, 0x27, 0xa9, 0x46, 0xd3, 0x96, 0x6b, 0x11, 0x26, 0xe1, 0xb1,
|
|
||||||
- 0xe3, 0x45, 0x49, 0xac, 0x27, 0x43, 0x8d, 0xf0, 0x78, 0x3b, 0x4a, 0x62, 0x91, 0xe6, 0xc5, 0x67,
|
|
||||||
- 0x9e, 0x86, 0xd3, 0x99, 0x8c, 0x95, 0xa6, 0xbd, 0xec, 0x45, 0xc9, 0xf3, 0x70, 0x3a, 0xb3, 0xfe,
|
|
||||||
- 0x4f, 0xf6, 0xc2, 0x18, 0xfb, 0xb6, 0x1b, 0xfa, 0x34, 0xd8, 0xc1, 0xe7, 0x39, 0x09, 0x69, 0xdf,
|
|
||||||
- 0x65, 0x72, 0xc2, 0xb7, 0x15, 0xe8, 0x3c, 0x99, 0xe0, 0x90, 0xef, 0x60, 0xee, 0x92, 0xa9, 0xec,
|
|
||||||
- 0xad, 0xce, 0x31, 0x8b, 0x09, 0x0d, 0xf5, 0xc3, 0x37, 0xa0, 0x68, 0x8d, 0x49, 0x48, 0xb8, 0xe3,
|
|
||||||
- 0xbb, 0x38, 0xa0, 0xa1, 0xf6, 0x02, 0x08, 0xd4, 0x8e, 0xc4, 0xa0, 0x37, 0xa1, 0xaf, 0x26, 0x77,
|
|
||||||
- 0xce, 0xa9, 0x1b, 0xfa, 0x53, 0x11, 0x72, 0x6a, 0x92, 0xd1, 0x53, 0xe8, 0x3d, 0x8d, 0x45, 0x6f,
|
|
||||||
- 0xc1, 0x8a, 0x0e, 0x88, 0x8c, 0xb2, 0x2e, 0x29, 0xfb, 0x1a, 0x5f, 0x20, 0x4d, 0xa2, 0x88, 0x32,
|
|
||||||
- 0x1e, 0x3b, 0x31, 0xf6, 0x3c, 0x1a, 0x44, 0xba, 0x31, 0xe9, 0x1b, 0xfc, 0xa1, 0x42, 0x5b, 0x13,
|
|
||||||
- 0x58, 0x7d, 0x2a, 0xec, 0xd4, 0x96, 0x64, 0x17, 0xdc, 0x0b, 0x70, 0xe0, 0x1c, 0x4f, 0xa9, 0x77,
|
|
||||||
- 0xe6, 0x88, 0x34, 0xa5, 0x3d, 0x2c, 0x4a, 0x9f, 0x2d, 0x81, 0x3c, 0x24, 0xdf, 0xc8, 0x1e, 0x5c,
|
|
||||||
- 0x50, 0x9d, 0x52, 0x1e, 0x4d, 0x93, 0x89, 0x13, 0x31, 0x7a, 0x8c, 0xb5, 0x89, 0xfd, 0x00, 0x07,
|
|
||||||
- 0x7b, 0x0a, 0x7f, 0x20, 0xd0, 0xd6, 0x5f, 0x2a, 0xb0, 0x56, 0x94, 0xa4, 0x93, 0xee, 0x26, 0xac,
|
|
||||||
- 0x15, 0x45, 0xe9, 0x0f, 0xb1, 0x2a, 0xf4, 0x06, 0x79, 0x81, 0xea, 0x93, 0xfc, 0x21, 0x74, 0xe5,
|
|
||||||
- 0x38, 0xd7, 0xf1, 0x15, 0xa7, 0x62, 0xf9, 0x91, 0xbf, 0x17, 0xbb, 0xe3, 0xe6, 0x6f, 0xe9, 0x63,
|
|
||||||
- 0xb8, 0xa3, 0xcd, 0x77, 0x16, 0xd5, 0x56, 0x0f, 0x62, 0x5d, 0x13, 0x3c, 0x9b, 0xd3, 0xfe, 0x0b,
|
|
||||||
- 0x18, 0x66, 0xa8, 0xad, 0x99, 0x44, 0x1a, 0x5f, 0xbd, 0x0b, 0xab, 0x73, 0xc6, 0x3e, 0xf1, 0x7d,
|
|
||||||
- 0x26, 0x03, 0xb4, 0x6e, 0x97, 0x6d, 0x59, 0x8f, 0xe1, 0xf6, 0x21, 0xe6, 0xca, 0x1b, 0x2e, 0xd7,
|
|
||||||
- 0x3d, 0x81, 0x62, 0xb6, 0x02, 0xb5, 0x43, 0xec, 0x49, 0xe3, 0x6b, 0xb6, 0x58, 0x8a, 0x07, 0x78,
|
|
||||||
- 0x14, 0x63, 0x4f, 0x5a, 0x59, 0xb3, 0xe5, 0xda, 0xfa, 0x73, 0x05, 0x96, 0x75, 0x9a, 0x14, 0xa9,
|
|
||||||
- 0xde, 0x67, 0xe4, 0x1c, 0x33, 0xfd, 0xf4, 0x34, 0x84, 0xde, 0x80, 0x9e, 0x5a, 0x39, 0x34, 0xe2,
|
|
||||||
- 0x84, 0xa6, 0xc9, 0xb7, 0xab, 0xb0, 0xcf, 0x15, 0x52, 0x4e, 0xea, 0xe4, 0x20, 0x4a, 0xf7, 0x7c,
|
|
||||||
- 0x1a, 0x92, 0xe3, 0xb6, 0x58, 0x64, 0x06, 0x99, 0x6c, 0x5b, 0xb6, 0x86, 0xc4, 0x53, 0x37, 0xfc,
|
|
||||||
- 0x96, 0x24, 0x3f, 0x03, 0x8a, 0xa7, 0x1e, 0xd0, 0x24, 0xe4, 0x4e, 0x44, 0x49, 0xc8, 0x75, 0x76,
|
|
||||||
- 0x05, 0x89, 0x3a, 0x10, 0x18, 0xeb, 0xd7, 0x15, 0x68, 0xa8, 0x69, 0xb5, 0xe8, 0x32, 0xd3, 0x6f,
|
|
||||||
- 0x5c, 0x95, 0xc8, 0x7a, 0x41, 0xca, 0x52, 0xdf, 0x35, 0xb9, 0x16, 0x71, 0x7c, 0x1e, 0xa8, 0x4c,
|
|
||||||
- 0xad, 0x55, 0x3b, 0x0f, 0x64, 0x8a, 0x7e, 0x03, 0x7a, 0xd9, 0xa7, 0x52, 0xee, 0x2b, 0x15, 0xbb,
|
|
||||||
- 0x29, 0x56, 0x92, 0x5d, 0xaa, 0xa9, 0xf5, 0x53, 0xd1, 0x5c, 0xa7, 0x93, 0xda, 0x15, 0xa8, 0x25,
|
|
||||||
- 0xa9, 0x32, 0x62, 0x29, 0x30, 0x93, 0xf4, 0x23, 0x2b, 0x96, 0xe8, 0x21, 0xf4, 0x5c, 0xdf, 0x27,
|
|
||||||
- 0xe2, 0xb8, 0x3b, 0x7d, 0x4a, 0xfc, 0x34, 0x48, 0x8b, 0x58, 0xeb, 0x6f, 0x15, 0xe8, 0x6f, 0xd3,
|
|
||||||
- 0x68, 0xf6, 0x43, 0x32, 0xc5, 0xb9, 0x0c, 0x22, 0x95, 0xd4, 0xdf, 0x58, 0xb1, 0x16, 0x75, 0xe3,
|
|
||||||
- 0x09, 0x99, 0x62, 0x15, 0x5a, 0xea, 0x66, 0x9b, 0x02, 0x21, 0xc3, 0xca, 0x6c, 0xa6, 0x03, 0xb0,
|
|
||||||
- 0xae, 0xda, 0x7c, 0x46, 0x7d, 0x59, 0x21, 0xfb, 0x84, 0x39, 0xe9, 0xb8, 0xab, 0x6b, 0x2f, 0xfb,
|
|
||||||
- 0x84, 0xc9, 0x2d, 0x6d, 0xc8, 0x92, 0x9c, 0xb8, 0xe6, 0x0d, 0x69, 0x28, 0x8c, 0x30, 0x64, 0x1d,
|
|
||||||
- 0x1a, 0xf4, 0xe4, 0x24, 0xc6, 0x5c, 0xd6, 0xb2, 0x35, 0x5b, 0x43, 0x69, 0x9a, 0x6b, 0xe6, 0xd2,
|
|
||||||
- 0xdc, 0x2d, 0x58, 0x95, 0xb3, 0xfd, 0x17, 0xcc, 0xf5, 0x48, 0x38, 0x31, 0xa9, 0x78, 0x0d, 0xd0,
|
|
||||||
- 0x21, 0xa7, 0x51, 0x11, 0xfb, 0xe8, 0x77, 0x2b, 0x3a, 0x27, 0xea, 0x46, 0x17, 0x3d, 0x85, 0xfe,
|
|
||||||
- 0xdc, 0x1f, 0x27, 0x48, 0x4f, 0x3e, 0xca, 0xff, 0x4f, 0x19, 0xad, 0x8f, 0xd5, 0x1f, 0x31, 0x63,
|
|
||||||
- 0xf3, 0x47, 0xcc, 0x78, 0x37, 0x88, 0xf8, 0x0c, 0xed, 0x42, 0xaf, 0xf8, 0x17, 0x03, 0xba, 0x6b,
|
|
||||||
- 0x0a, 0x85, 0x92, 0x3f, 0x1e, 0x2e, 0x65, 0xf3, 0x14, 0xfa, 0x73, 0xff, 0x36, 0x18, 0x7d, 0xca,
|
|
||||||
- 0xff, 0x84, 0xb8, 0x94, 0xd1, 0x63, 0x68, 0xe7, 0xfe, 0x5e, 0x40, 0x43, 0xc5, 0x64, 0xf1, 0x1f,
|
|
||||||
- 0x87, 0x4b, 0x19, 0x6c, 0x43, 0xb7, 0x30, 0xf1, 0x47, 0x23, 0x6d, 0x4f, 0xc9, 0xdf, 0x00, 0x97,
|
|
||||||
- 0x32, 0xd9, 0x82, 0x76, 0x6e, 0xf0, 0x6e, 0xb4, 0x58, 0x9c, 0xee, 0x8f, 0xee, 0x94, 0xec, 0xe8,
|
|
||||||
- 0xd4, 0xbb, 0x07, 0xdd, 0xc2, 0x98, 0xdc, 0x28, 0x52, 0x36, 0xa2, 0x1f, 0xdd, 0x2d, 0xdd, 0xd3,
|
|
||||||
- 0x9c, 0x9e, 0x42, 0x7f, 0x6e, 0x68, 0x6e, 0x9c, 0x5b, 0x3e, 0x4b, 0xbf, 0xd4, 0xac, 0xcf, 0xe5,
|
|
||||||
- 0x65, 0xe7, 0x7a, 0xa2, 0xdc, 0x65, 0x2f, 0x8e, 0xc8, 0x47, 0xf7, 0xca, 0x37, 0xb5, 0x56, 0xbb,
|
|
||||||
- 0xd0, 0x2b, 0x4e, 0xc7, 0x0d, 0xb3, 0xd2, 0x99, 0xf9, 0xd5, 0x2f, 0xa7, 0x30, 0x28, 0xcf, 0x5e,
|
|
||||||
- 0x4e, 0xd9, 0xfc, 0xfc, 0x52, 0x46, 0x4f, 0x00, 0x74, 0x07, 0xe4, 0x93, 0x30, 0xbd, 0xb2, 0x85,
|
|
||||||
- 0xce, 0x2b, 0xbd, 0xb2, 0x92, 0x6e, 0xe9, 0x31, 0x80, 0x6a, 0x5c, 0x7c, 0x9a, 0x70, 0x74, 0xdb,
|
|
||||||
- 0xa8, 0x31, 0xd7, 0x2d, 0x8d, 0x86, 0x8b, 0x1b, 0x0b, 0x0c, 0x30, 0x63, 0xd7, 0x61, 0xf0, 0x19,
|
|
||||||
- 0x40, 0xd6, 0x10, 0x19, 0x06, 0x0b, 0x2d, 0xd2, 0x15, 0x3e, 0xe8, 0xe4, 0xdb, 0x1f, 0xa4, 0x6d,
|
|
||||||
- 0x2d, 0x69, 0x89, 0xae, 0x60, 0xd1, 0x9f, 0x2b, 0x6f, 0x8b, 0x8f, 0x6d, 0xbe, 0xea, 0x1d, 0x2d,
|
|
||||||
- 0x94, 0xb8, 0xe8, 0x43, 0xe8, 0xe4, 0xeb, 0x5a, 0xa3, 0x45, 0x49, 0xad, 0x3b, 0x2a, 0xd4, 0xb6,
|
|
||||||
- 0xe8, 0x31, 0xf4, 0x8a, 0x55, 0x2b, 0xca, 0xc5, 0xc5, 0x42, 0x2d, 0x3b, 0xd2, 0x13, 0x9b, 0x1c,
|
|
||||||
- 0xf9, 0xfb, 0x00, 0x59, 0x75, 0x6b, 0xdc, 0xb7, 0x50, 0xef, 0xce, 0x49, 0x7d, 0x02, 0x9d, 0x7c,
|
|
||||||
- 0x26, 0x36, 0xea, 0x96, 0x64, 0xe7, 0xab, 0xb2, 0x56, 0x2e, 0x6b, 0x9b, 0xc7, 0xb7, 0x98, 0xc8,
|
|
||||||
- 0xaf, 0xca, 0x5a, 0x85, 0xae, 0xcf, 0x24, 0x8b, 0xb2, 0x56, 0xf0, 0xaa, 0x5c, 0x5e, 0x6c, 0x91,
|
|
||||||
- 0x8c, 0xfb, 0x4a, 0x1b, 0xa7, 0xab, 0x1e, 0x51, 0xbe, 0x1b, 0x30, 0xfe, 0x28, 0xe9, 0x10, 0xbe,
|
|
||||||
- 0x27, 0xa8, 0xf3, 0x15, 0x7f, 0x2e, 0xa8, 0x4b, 0x1a, 0x81, 0x4b, 0x19, 0xed, 0x41, 0xff, 0xa9,
|
|
||||||
- 0x29, 0xe6, 0x74, 0xa1, 0xa9, 0xd5, 0x29, 0x29, 0xac, 0x47, 0xa3, 0xb2, 0x2d, 0x1d, 0x59, 0x9f,
|
|
||||||
- 0xc3, 0x60, 0xa1, 0xc8, 0x44, 0xf7, 0xd3, 0xc1, 0x62, 0x69, 0xf5, 0x79, 0xa9, 0x5a, 0xfb, 0xb0,
|
|
||||||
- 0x32, 0x5f, 0x63, 0xa2, 0x57, 0xf4, 0xa5, 0x97, 0xd7, 0x9e, 0x97, 0xb2, 0xfa, 0x18, 0x9a, 0xa6,
|
|
||||||
- 0xa6, 0x41, 0x7a, 0x80, 0x3b, 0x57, 0xe3, 0x5c, 0x76, 0x74, 0xab, 0xf3, 0xed, 0x77, 0xf7, 0x2b,
|
|
||||||
- 0x7f, 0xff, 0xee, 0x7e, 0xe5, 0x9f, 0xdf, 0xdd, 0xaf, 0x1c, 0x37, 0xe4, 0xee, 0xfb, 0xff, 0x0e,
|
|
||||||
- 0x00, 0x00, 0xff, 0xff, 0x8d, 0x89, 0xaa, 0x73, 0xc8, 0x21, 0x00, 0x00,
|
|
||||||
+ // 2931 bytes of a gzipped FileDescriptorProto
|
|
||||||
+ 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xc4, 0x39, 0xcb, 0x6e, 0x1c, 0xc7,
|
|
||||||
+ 0xb5, 0x98, 0x07, 0x87, 0x33, 0x67, 0x5e, 0x9c, 0x22, 0x45, 0x8d, 0x46, 0xb2, 0xae, 0xdc, 0xb6,
|
|
||||||
+ 0x65, 0xfa, 0xfa, 0x7a, 0x68, 0xcb, 0xc6, 0xf5, 0x0b, 0xbe, 0x82, 0x48, 0xe9, 0x8a, 0x8c, 0xad,
|
|
||||||
+ 0x88, 0xe9, 0x91, 0xe2, 0x04, 0x41, 0xd0, 0x68, 0x76, 0x97, 0x86, 0x65, 0x4e, 0x77, 0xb5, 0xab,
|
|
||||||
+ 0xaa, 0x29, 0x8e, 0x03, 0x64, 0x99, 0xec, 0xb2, 0xcc, 0x2e, 0x3f, 0x10, 0x64, 0x97, 0x65, 0xb6,
|
|
||||||
+ 0x59, 0x18, 0x59, 0x05, 0xf9, 0x80, 0x20, 0xf0, 0x27, 0xe4, 0x0b, 0x82, 0x7a, 0xf5, 0x63, 0x66,
|
|
||||||
+ 0x48, 0x23, 0x82, 0x80, 0x6c, 0x1a, 0x75, 0x4e, 0x9d, 0x3a, 0xaf, 0xaa, 0x3a, 0x75, 0xce, 0x69,
|
|
||||||
+ 0x68, 0xfb, 0x53, 0x1c, 0x8b, 0x71, 0xc2, 0xa8, 0xa0, 0xa8, 0x3e, 0x65, 0x49, 0x30, 0x6a, 0xd1,
|
|
||||||
+ 0x80, 0x68, 0xc4, 0xe8, 0x7f, 0xa7, 0x44, 0x9c, 0xa4, 0xc7, 0xe3, 0x80, 0x46, 0xbb, 0xa7, 0xbe,
|
|
||||||
+ 0xf0, 0xdf, 0x09, 0x68, 0x2c, 0x7c, 0x12, 0x63, 0xc6, 0x77, 0xd5, 0xc2, 0xdd, 0xe4, 0x74, 0xba,
|
|
||||||
+ 0x2b, 0xe6, 0x09, 0xe6, 0xfa, 0x6b, 0xd6, 0x5d, 0x9f, 0x52, 0x3a, 0x9d, 0xe1, 0x5d, 0x05, 0x1d,
|
|
||||||
+ 0xa7, 0xcf, 0x76, 0x71, 0x94, 0x88, 0xb9, 0x9e, 0x74, 0x7e, 0x57, 0x85, 0xed, 0x7d, 0x86, 0x7d,
|
|
||||||
+ 0x81, 0xf7, 0x2d, 0x37, 0x17, 0x7f, 0x9d, 0x62, 0x2e, 0xd0, 0xab, 0xd0, 0xc9, 0x24, 0x78, 0x24,
|
|
||||||
+ 0x1c, 0x56, 0x6e, 0x55, 0x76, 0x5a, 0x6e, 0x3b, 0xc3, 0x1d, 0x86, 0xe8, 0x2a, 0xac, 0xe3, 0x73,
|
|
||||||
+ 0x1c, 0xc8, 0xd9, 0xaa, 0x9a, 0x6d, 0x48, 0xf0, 0x30, 0x44, 0xef, 0x41, 0x9b, 0x0b, 0x46, 0xe2,
|
|
||||||
+ 0xa9, 0x97, 0x72, 0xcc, 0x86, 0xb5, 0x5b, 0x95, 0x9d, 0xf6, 0x9d, 0x8d, 0xb1, 0x34, 0x69, 0x3c,
|
|
||||||
+ 0x51, 0x13, 0x4f, 0x39, 0x66, 0x2e, 0xf0, 0x6c, 0x8c, 0x6e, 0xc3, 0x7a, 0x88, 0xcf, 0x48, 0x80,
|
|
||||||
+ 0xf9, 0xb0, 0x7e, 0xab, 0xb6, 0xd3, 0xbe, 0xd3, 0xd1, 0xe4, 0xf7, 0x15, 0xd2, 0xb5, 0x93, 0xe8,
|
|
||||||
+ 0x2d, 0x68, 0x72, 0x41, 0x99, 0x3f, 0xc5, 0x7c, 0xb8, 0xa6, 0x08, 0xbb, 0x96, 0xaf, 0xc2, 0xba,
|
|
||||||
+ 0xd9, 0x34, 0xba, 0x01, 0xb5, 0xc7, 0xfb, 0x87, 0xc3, 0x86, 0x92, 0x0e, 0x86, 0x2a, 0xc1, 0x81,
|
|
||||||
+ 0x2b, 0xd1, 0xe8, 0x35, 0xe8, 0x72, 0x3f, 0x0e, 0x8f, 0xe9, 0xb9, 0x97, 0x90, 0x30, 0xe6, 0xc3,
|
|
||||||
+ 0xf5, 0x5b, 0x95, 0x9d, 0xa6, 0xdb, 0x31, 0xc8, 0x23, 0x89, 0x73, 0x3e, 0x81, 0x2b, 0x13, 0xe1,
|
|
||||||
+ 0x33, 0xf1, 0x02, 0xde, 0x71, 0x9e, 0xc2, 0xb6, 0x8b, 0x23, 0x7a, 0xf6, 0x42, 0xae, 0x1d, 0xc2,
|
|
||||||
+ 0xba, 0x20, 0x11, 0xa6, 0xa9, 0x50, 0xae, 0xed, 0xba, 0x16, 0x74, 0xfe, 0x50, 0x01, 0xf4, 0xe0,
|
|
||||||
+ 0x1c, 0x07, 0x47, 0x8c, 0x06, 0x98, 0xf3, 0xff, 0xd0, 0x76, 0xbd, 0x09, 0xeb, 0x89, 0x56, 0x60,
|
|
||||||
+ 0x58, 0x57, 0xe4, 0x66, 0x17, 0xac, 0x56, 0x76, 0xd6, 0xf9, 0x0a, 0xb6, 0x26, 0x64, 0x1a, 0xfb,
|
|
||||||
+ 0xb3, 0x97, 0xa8, 0xef, 0x36, 0x34, 0xb8, 0xe2, 0xa9, 0x54, 0xed, 0xba, 0x06, 0x72, 0x8e, 0x00,
|
|
||||||
+ 0x7d, 0xe9, 0x13, 0xf1, 0xf2, 0x24, 0x39, 0xef, 0xc0, 0x66, 0x89, 0x23, 0x4f, 0x68, 0xcc, 0xb1,
|
|
||||||
+ 0x52, 0x40, 0xf8, 0x22, 0xe5, 0x8a, 0xd9, 0x9a, 0x6b, 0x20, 0x07, 0xc3, 0xd6, 0x17, 0x84, 0x5b,
|
|
||||||
+ 0x72, 0xfc, 0xef, 0xa8, 0xb0, 0x0d, 0x8d, 0x67, 0x94, 0x45, 0xbe, 0xb0, 0x1a, 0x68, 0x08, 0x21,
|
|
||||||
+ 0xa8, 0xfb, 0x6c, 0xca, 0x87, 0xb5, 0x5b, 0xb5, 0x9d, 0x96, 0xab, 0xc6, 0xf2, 0x54, 0x2e, 0x88,
|
|
||||||
+ 0x31, 0x7a, 0xbd, 0x0a, 0x1d, 0xe3, 0x77, 0x6f, 0x46, 0xb8, 0x50, 0x72, 0x3a, 0x6e, 0xdb, 0xe0,
|
|
||||||
+ 0xe4, 0x1a, 0x87, 0xc2, 0xf6, 0xd3, 0x24, 0x7c, 0xc1, 0x0b, 0x7f, 0x07, 0x5a, 0x0c, 0x73, 0x9a,
|
|
||||||
+ 0x32, 0x79, 0x4d, 0xab, 0x6a, 0xdf, 0xb7, 0xf4, 0xbe, 0x7f, 0x41, 0xe2, 0xf4, 0xdc, 0xb5, 0x73,
|
|
||||||
+ 0x6e, 0x4e, 0x66, 0xae, 0x90, 0xe0, 0x2f, 0x72, 0x85, 0x3e, 0x81, 0x2b, 0x47, 0x7e, 0xca, 0x5f,
|
|
||||||
+ 0x44, 0x57, 0xe7, 0x53, 0x79, 0xfd, 0x78, 0x1a, 0xbd, 0xd0, 0xe2, 0xdf, 0x57, 0xa0, 0xb9, 0x9f,
|
|
||||||
+ 0xa4, 0x4f, 0xb9, 0x3f, 0xc5, 0xe8, 0xbf, 0xa0, 0x2d, 0xa8, 0xf0, 0x67, 0x5e, 0x2a, 0x41, 0x45,
|
|
||||||
+ 0x5e, 0x77, 0x41, 0xa1, 0x34, 0x81, 0x74, 0x3b, 0x66, 0x41, 0x92, 0x1a, 0x8a, 0xea, 0xad, 0xda,
|
|
||||||
+ 0x4e, 0xdd, 0x6d, 0x6b, 0x9c, 0x26, 0x19, 0xc3, 0xa6, 0x9a, 0xf3, 0x48, 0xec, 0x9d, 0x62, 0x16,
|
|
||||||
+ 0xe3, 0x59, 0x44, 0x43, 0xac, 0xce, 0x6f, 0xdd, 0x1d, 0xa8, 0xa9, 0xc3, 0xf8, 0xf3, 0x6c, 0x02,
|
|
||||||
+ 0xfd, 0x37, 0x0c, 0x32, 0x7a, 0x79, 0x29, 0x15, 0x75, 0x5d, 0x51, 0xf7, 0x0d, 0xf5, 0x53, 0x83,
|
|
||||||
+ 0x76, 0x7e, 0x09, 0xbd, 0x27, 0x27, 0x8c, 0x0a, 0x31, 0x23, 0xf1, 0xf4, 0xbe, 0x2f, 0x7c, 0x19,
|
|
||||||
+ 0x3d, 0x12, 0xcc, 0x08, 0x0d, 0xb9, 0xd1, 0xd6, 0x82, 0xe8, 0x6d, 0x18, 0x08, 0x4d, 0x8b, 0x43,
|
|
||||||
+ 0xcf, 0xd2, 0x54, 0x15, 0xcd, 0x46, 0x36, 0x71, 0x64, 0x88, 0xdf, 0x80, 0x5e, 0x4e, 0x2c, 0xe3,
|
|
||||||
+ 0x8f, 0xd1, 0xb7, 0x9b, 0x61, 0x9f, 0x90, 0x08, 0x3b, 0x67, 0xca, 0x57, 0x6a, 0x93, 0xd1, 0xdb,
|
|
||||||
+ 0xd0, 0xca, 0xfd, 0x50, 0x51, 0x27, 0xa4, 0xa7, 0x4f, 0x88, 0x75, 0xa7, 0xdb, 0xcc, 0x9c, 0xf2,
|
|
||||||
+ 0x19, 0xf4, 0x45, 0xa6, 0xb8, 0x17, 0xfa, 0xc2, 0x2f, 0x1f, 0xaa, 0xb2, 0x55, 0x6e, 0x4f, 0x94,
|
|
||||||
+ 0x60, 0xe7, 0x53, 0x68, 0x1d, 0x91, 0x90, 0x6b, 0xc1, 0x43, 0x58, 0x0f, 0x52, 0xc6, 0x70, 0x2c,
|
|
||||||
+ 0xac, 0xc9, 0x06, 0x44, 0x5b, 0xb0, 0x36, 0x23, 0x11, 0x11, 0xc6, 0x4c, 0x0d, 0x38, 0x14, 0xe0,
|
|
||||||
+ 0x11, 0x8e, 0x28, 0x9b, 0x2b, 0x87, 0x6d, 0xc1, 0x5a, 0x71, 0x73, 0x35, 0x80, 0xae, 0x43, 0x2b,
|
|
||||||
+ 0xf2, 0xcf, 0xb3, 0x4d, 0x95, 0x33, 0xcd, 0xc8, 0x3f, 0xd7, 0xca, 0x0f, 0x61, 0xfd, 0x99, 0x4f,
|
|
||||||
+ 0x66, 0x41, 0x2c, 0x8c, 0x57, 0x2c, 0x98, 0x0b, 0xac, 0x17, 0x05, 0xfe, 0xb9, 0x0a, 0x6d, 0x2d,
|
|
||||||
+ 0x51, 0x2b, 0xbc, 0x05, 0x6b, 0x81, 0x1f, 0x9c, 0x64, 0x22, 0x15, 0x80, 0x6e, 0x5b, 0x45, 0xaa,
|
|
||||||
+ 0xc5, 0x20, 0x9c, 0x6b, 0x6a, 0x55, 0xdb, 0x05, 0xe0, 0xcf, 0xfd, 0xc4, 0xe8, 0x56, 0xbb, 0x80,
|
|
||||||
+ 0xb8, 0x25, 0x69, 0xb4, 0xba, 0xef, 0x43, 0x47, 0x9f, 0x3b, 0xb3, 0xa4, 0x7e, 0xc1, 0x92, 0xb6,
|
|
||||||
+ 0xa6, 0xd2, 0x8b, 0x5e, 0x83, 0x6e, 0xca, 0xb1, 0x77, 0x42, 0x30, 0xf3, 0x59, 0x70, 0x32, 0x1f,
|
|
||||||
+ 0xae, 0xe9, 0x37, 0x32, 0xe5, 0xf8, 0xc0, 0xe2, 0xd0, 0x1d, 0x58, 0x93, 0xe1, 0x8f, 0x0f, 0x1b,
|
|
||||||
+ 0xea, 0x39, 0xbe, 0x51, 0x64, 0xa9, 0x4c, 0x1d, 0xab, 0xef, 0x83, 0x58, 0xb0, 0xb9, 0xab, 0x49,
|
|
||||||
+ 0x47, 0x1f, 0x01, 0xe4, 0x48, 0xb4, 0x01, 0xb5, 0x53, 0x3c, 0x37, 0xf7, 0x50, 0x0e, 0xa5, 0x73,
|
|
||||||
+ 0xce, 0xfc, 0x59, 0x6a, 0xbd, 0xae, 0x81, 0x4f, 0xaa, 0x1f, 0x55, 0x9c, 0x00, 0xfa, 0x7b, 0xb3,
|
|
||||||
+ 0x53, 0x42, 0x0b, 0xcb, 0xb7, 0x60, 0x2d, 0xf2, 0xbf, 0xa2, 0xcc, 0x7a, 0x52, 0x01, 0x0a, 0x4b,
|
|
||||||
+ 0x62, 0xca, 0x2c, 0x0b, 0x05, 0xa0, 0x1e, 0x54, 0x69, 0xa2, 0xfc, 0xd5, 0x72, 0xab, 0x34, 0xc9,
|
|
||||||
+ 0x05, 0xd5, 0x0b, 0x82, 0x9c, 0xbf, 0xd7, 0x01, 0x72, 0x29, 0xc8, 0x85, 0x11, 0xa1, 0x1e, 0xc7,
|
|
||||||
+ 0x4c, 0xa6, 0x20, 0xde, 0xf1, 0x5c, 0x60, 0xee, 0x31, 0x1c, 0xa4, 0x8c, 0x93, 0x33, 0xb9, 0x7f,
|
|
||||||
+ 0xd2, 0xec, 0x2b, 0xda, 0xec, 0x05, 0xdd, 0xdc, 0xab, 0x84, 0x4e, 0xf4, 0xba, 0x3d, 0xb9, 0xcc,
|
|
||||||
+ 0xb5, 0xab, 0xd0, 0x21, 0x5c, 0xc9, 0x79, 0x86, 0x05, 0x76, 0xd5, 0xcb, 0xd8, 0x6d, 0x66, 0xec,
|
|
||||||
+ 0xc2, 0x9c, 0xd5, 0x03, 0xd8, 0x24, 0xd4, 0xfb, 0x3a, 0xc5, 0x69, 0x89, 0x51, 0xed, 0x32, 0x46,
|
|
||||||
+ 0x03, 0x42, 0x7f, 0xa4, 0x16, 0xe4, 0x6c, 0x8e, 0xe0, 0x5a, 0xc1, 0x4a, 0x79, 0xdd, 0x0b, 0xcc,
|
|
||||||
+ 0xea, 0x97, 0x31, 0xdb, 0xce, 0xb4, 0x92, 0xf1, 0x20, 0xe7, 0xf8, 0x03, 0xd8, 0x26, 0xd4, 0x7b,
|
|
||||||
+ 0xee, 0x13, 0xb1, 0xc8, 0x6e, 0xed, 0x7b, 0x8c, 0x94, 0x8f, 0x6e, 0x99, 0x97, 0x36, 0x32, 0xc2,
|
|
||||||
+ 0x6c, 0x5a, 0x32, 0xb2, 0xf1, 0x3d, 0x46, 0x3e, 0x52, 0x0b, 0x72, 0x36, 0xf7, 0x60, 0x40, 0xe8,
|
|
||||||
+ 0xa2, 0x36, 0xeb, 0x97, 0x31, 0xe9, 0x13, 0x5a, 0xd6, 0x64, 0x0f, 0x06, 0x1c, 0x07, 0x82, 0xb2,
|
|
||||||
+ 0xe2, 0x21, 0x68, 0x5e, 0xc6, 0x62, 0xc3, 0xd0, 0x67, 0x3c, 0x9c, 0x9f, 0x41, 0xe7, 0x20, 0x9d,
|
|
||||||
+ 0x62, 0x31, 0x3b, 0xce, 0x82, 0xc1, 0x4b, 0x8b, 0x3f, 0xce, 0x3f, 0xab, 0xd0, 0xde, 0x9f, 0x32,
|
|
||||||
+ 0x9a, 0x26, 0xa5, 0x98, 0xac, 0x2f, 0xe9, 0x62, 0x4c, 0x56, 0x24, 0x2a, 0x26, 0x6b, 0xe2, 0x0f,
|
|
||||||
+ 0xa0, 0x13, 0xa9, 0xab, 0x6b, 0xe8, 0x75, 0x1c, 0x1a, 0x2c, 0x5d, 0x6a, 0xb7, 0x1d, 0x15, 0x82,
|
|
||||||
+ 0xd9, 0x18, 0x20, 0x21, 0x21, 0x37, 0x6b, 0x74, 0x38, 0xea, 0x9b, 0x8c, 0xd0, 0x86, 0x68, 0xb7,
|
|
||||||
+ 0x95, 0x64, 0xd1, 0xfa, 0x3d, 0x68, 0x1f, 0x4b, 0x27, 0x99, 0x05, 0xa5, 0x60, 0x94, 0x7b, 0xcf,
|
|
||||||
+ 0x85, 0xe3, 0xfc, 0x12, 0x1e, 0x40, 0xf7, 0x44, 0xbb, 0xcc, 0x2c, 0xd2, 0x67, 0xe8, 0x35, 0x63,
|
|
||||||
+ 0x49, 0x6e, 0xef, 0xb8, 0xe8, 0x59, 0xbd, 0x01, 0x9d, 0x93, 0x02, 0x6a, 0x34, 0x81, 0xc1, 0x12,
|
|
||||||
+ 0xc9, 0x8a, 0x18, 0xb4, 0x53, 0x8c, 0x41, 0xed, 0x3b, 0x48, 0x0b, 0x2a, 0xae, 0x2c, 0xc6, 0xa5,
|
|
||||||
+ 0xdf, 0x54, 0xa1, 0xf3, 0x43, 0x2c, 0x9e, 0x53, 0x76, 0xaa, 0xf5, 0x45, 0x50, 0x8f, 0xfd, 0x08,
|
|
||||||
+ 0x1b, 0x8e, 0x6a, 0x8c, 0xae, 0x41, 0x93, 0x9d, 0xeb, 0x00, 0x62, 0xf6, 0x73, 0x9d, 0x9d, 0xab,
|
|
||||||
+ 0xc0, 0x80, 0x5e, 0x01, 0x60, 0xe7, 0x5e, 0xe2, 0x07, 0xa7, 0xd8, 0x78, 0xb0, 0xee, 0xb6, 0xd8,
|
|
||||||
+ 0xf9, 0x91, 0x46, 0xc8, 0xa3, 0xc0, 0xce, 0x3d, 0xcc, 0x18, 0x65, 0xdc, 0xc4, 0xaa, 0x26, 0x3b,
|
|
||||||
+ 0x7f, 0xa0, 0x60, 0xb3, 0x36, 0x64, 0x34, 0x49, 0x70, 0xa8, 0x62, 0xb4, 0x5a, 0x7b, 0x5f, 0x23,
|
|
||||||
+ 0xa4, 0x54, 0x61, 0xa5, 0x36, 0xb4, 0x54, 0x91, 0x4b, 0x15, 0xb9, 0xd4, 0x75, 0xbd, 0x52, 0x14,
|
|
||||||
+ 0xa5, 0x8a, 0x4c, 0x6a, 0x53, 0x4b, 0x15, 0x05, 0xa9, 0x22, 0x97, 0xda, 0xb2, 0x6b, 0x8d, 0x54,
|
|
||||||
+ 0xe7, 0xd7, 0x15, 0xd8, 0x5e, 0x4c, 0xfc, 0x4c, 0x9a, 0xfa, 0x01, 0x74, 0x02, 0xb5, 0x5f, 0xa5,
|
|
||||||
+ 0x33, 0x39, 0x58, 0xda, 0x49, 0xb7, 0x1d, 0x14, 0x8e, 0xf1, 0x87, 0xd0, 0x8d, 0xb5, 0x83, 0xb3,
|
|
||||||
+ 0xa3, 0x59, 0xcb, 0xf7, 0xa5, 0xe8, 0x7b, 0xb7, 0x13, 0x17, 0x20, 0x27, 0x04, 0xf4, 0x25, 0x23,
|
|
||||||
+ 0x02, 0x4f, 0x04, 0xc3, 0x7e, 0xf4, 0x32, 0x0a, 0x10, 0x04, 0x75, 0x95, 0xad, 0xd4, 0x54, 0x7e,
|
|
||||||
+ 0xad, 0xc6, 0xce, 0x9b, 0xb0, 0x59, 0x92, 0x62, 0x6c, 0xdd, 0x80, 0xda, 0x0c, 0xc7, 0x8a, 0x7b,
|
|
||||||
+ 0xd7, 0x95, 0x43, 0xc7, 0x87, 0x81, 0x8b, 0xfd, 0xf0, 0xe5, 0x69, 0x63, 0x44, 0xd4, 0x72, 0x11,
|
|
||||||
+ 0x3b, 0x80, 0x8a, 0x22, 0x8c, 0x2a, 0x56, 0xeb, 0x4a, 0x41, 0xeb, 0xc7, 0x30, 0xd8, 0x9f, 0x51,
|
|
||||||
+ 0x8e, 0x27, 0x22, 0x24, 0xf1, 0xcb, 0xa8, 0x98, 0x7e, 0x01, 0x9b, 0x4f, 0xc4, 0xfc, 0x4b, 0xc9,
|
|
||||||
+ 0x8c, 0x93, 0x6f, 0xf0, 0x4b, 0xb2, 0x8f, 0xd1, 0xe7, 0xd6, 0x3e, 0x46, 0x9f, 0xcb, 0x62, 0x29,
|
|
||||||
+ 0xa0, 0xb3, 0x34, 0x8a, 0xd5, 0x55, 0xe8, 0xba, 0x06, 0x72, 0xf6, 0xa0, 0xa3, 0x73, 0xe8, 0x47,
|
|
||||||
+ 0x34, 0x4c, 0x67, 0x78, 0xe5, 0x1d, 0xbc, 0x09, 0x90, 0xf8, 0xcc, 0x8f, 0xb0, 0xc0, 0x4c, 0x9f,
|
|
||||||
+ 0xa1, 0x96, 0x5b, 0xc0, 0x38, 0xbf, 0xad, 0xc2, 0x96, 0x6e, 0x89, 0x4c, 0x74, 0x27, 0xc0, 0x9a,
|
|
||||||
+ 0x30, 0x82, 0xe6, 0x09, 0xe5, 0xa2, 0xc0, 0x30, 0x83, 0xa5, 0x8a, 0x61, 0x6c, 0xb9, 0xc9, 0x61,
|
|
||||||
+ 0xa9, 0x4f, 0x51, 0xbb, 0xbc, 0x4f, 0xb1, 0xd4, 0x89, 0xa8, 0x2f, 0x77, 0x22, 0xe4, 0x6d, 0xb3,
|
|
||||||
+ 0x44, 0x44, 0xdf, 0xf1, 0x96, 0xdb, 0x32, 0x98, 0xc3, 0x10, 0xdd, 0x86, 0xfe, 0x54, 0x6a, 0xe9,
|
|
||||||
+ 0x9d, 0x50, 0x7a, 0xea, 0x25, 0xbe, 0x38, 0x51, 0x57, 0xbd, 0xe5, 0x76, 0x15, 0xfa, 0x80, 0xd2,
|
|
||||||
+ 0xd3, 0x23, 0x5f, 0x9c, 0xa0, 0x8f, 0xa1, 0x67, 0xd2, 0xc0, 0x48, 0xb9, 0x88, 0x9b, 0xc7, 0xcf,
|
|
||||||
+ 0xdc, 0xa2, 0xa2, 0xf7, 0xdc, 0xee, 0x69, 0x01, 0xe2, 0xce, 0x55, 0xb8, 0x72, 0x1f, 0x73, 0xc1,
|
|
||||||
+ 0xe8, 0xbc, 0xec, 0x18, 0xe7, 0xff, 0x00, 0x0e, 0x63, 0x81, 0xd9, 0x33, 0x3f, 0xc0, 0x1c, 0xbd,
|
|
||||||
+ 0x5b, 0x84, 0x4c, 0x72, 0xb4, 0x31, 0xd6, 0x1d, 0xa9, 0x6c, 0xc2, 0x2d, 0xd0, 0x38, 0x63, 0x68,
|
|
||||||
+ 0xb8, 0x34, 0x95, 0xe1, 0xe8, 0x75, 0x3b, 0x32, 0xeb, 0x3a, 0x66, 0x9d, 0x42, 0xba, 0x66, 0xce,
|
|
||||||
+ 0x39, 0xb0, 0x25, 0x6c, 0xce, 0xce, 0x6c, 0xd1, 0x18, 0x5a, 0xc4, 0xe2, 0x4c, 0x54, 0x59, 0x16,
|
|
||||||
+ 0x9d, 0x93, 0x38, 0x3f, 0x85, 0x4d, 0xcd, 0x49, 0x73, 0xb6, 0x6c, 0x5e, 0x87, 0x06, 0xb3, 0x6a,
|
|
||||||
+ 0x54, 0xf2, 0x56, 0x94, 0x21, 0x32, 0x73, 0xe8, 0x86, 0x14, 0x16, 0x30, 0x1c, 0xc9, 0x9a, 0xa3,
|
|
||||||
+ 0xaa, 0xb6, 0x2c, 0x47, 0x48, 0x6f, 0xc9, 0x7a, 0x3b, 0x37, 0xd3, 0x7a, 0x6b, 0x13, 0x06, 0x72,
|
|
||||||
+ 0xa2, 0x24, 0xd1, 0xf9, 0x39, 0x6c, 0x3e, 0x8e, 0x67, 0x24, 0xc6, 0xfb, 0x47, 0x4f, 0x1f, 0xe1,
|
|
||||||
+ 0x2c, 0x2a, 0x20, 0xa8, 0xcb, 0xec, 0x49, 0xa9, 0xd1, 0x74, 0xd5, 0x58, 0x5e, 0x93, 0xf8, 0xd8,
|
|
||||||
+ 0x0b, 0x92, 0x94, 0x9b, 0xce, 0x50, 0x23, 0x3e, 0xde, 0x4f, 0x52, 0x2e, 0xc3, 0xbc, 0x7c, 0xe6,
|
|
||||||
+ 0x69, 0x3c, 0x9b, 0xab, 0xbb, 0xd2, 0x74, 0xd7, 0x83, 0x24, 0x7d, 0x1c, 0xcf, 0xe6, 0xce, 0xff,
|
|
||||||
+ 0xa8, 0x5a, 0x18, 0xe3, 0xd0, 0xf5, 0xe3, 0x90, 0x46, 0xf7, 0xf1, 0x59, 0x41, 0x42, 0x56, 0x77,
|
|
||||||
+ 0xd9, 0x98, 0xf0, 0x6d, 0x05, 0x3a, 0xf7, 0xa6, 0x38, 0x16, 0xf7, 0xb1, 0xf0, 0xc9, 0x4c, 0xd5,
|
|
||||||
+ 0x56, 0x67, 0x98, 0x71, 0x42, 0x63, 0x73, 0xf0, 0x2d, 0x28, 0x4b, 0x63, 0x12, 0x13, 0xe1, 0x85,
|
|
||||||
+ 0x3e, 0x8e, 0x68, 0x6c, 0xbc, 0x00, 0x12, 0x75, 0x5f, 0x61, 0xd0, 0x9b, 0xd0, 0xd7, 0x9d, 0x3b,
|
|
||||||
+ 0xef, 0xc4, 0x8f, 0xc3, 0x99, 0xbc, 0x72, 0xba, 0x93, 0xd1, 0xd3, 0xe8, 0x03, 0x83, 0x45, 0x6f,
|
|
||||||
+ 0xc1, 0x86, 0xb9, 0x10, 0x39, 0x65, 0x5d, 0x51, 0xf6, 0x0d, 0xbe, 0x44, 0x9a, 0x26, 0x09, 0x65,
|
|
||||||
+ 0x82, 0x7b, 0x1c, 0x07, 0x01, 0x8d, 0x12, 0x53, 0x98, 0xf4, 0x2d, 0x7e, 0xa2, 0xd1, 0xce, 0x14,
|
|
||||||
+ 0x36, 0x1f, 0x4a, 0x3b, 0x8d, 0x25, 0xf9, 0x06, 0xf7, 0x22, 0x1c, 0x79, 0xc7, 0x33, 0x1a, 0x9c,
|
|
||||||
+ 0x7a, 0x32, 0x4c, 0x19, 0x0f, 0xcb, 0xd4, 0x67, 0x4f, 0x22, 0x27, 0xe4, 0x1b, 0x55, 0x83, 0x4b,
|
|
||||||
+ 0xaa, 0x13, 0x2a, 0x92, 0x59, 0x3a, 0xf5, 0x12, 0x46, 0x8f, 0xb1, 0x31, 0xb1, 0x1f, 0xe1, 0xe8,
|
|
||||||
+ 0x40, 0xe3, 0x8f, 0x24, 0xda, 0xf9, 0x53, 0x05, 0xb6, 0xca, 0x92, 0x4c, 0xd0, 0xdd, 0x85, 0xad,
|
|
||||||
+ 0xb2, 0x28, 0xf3, 0x10, 0xeb, 0x44, 0x6f, 0x50, 0x14, 0xa8, 0x9f, 0xe4, 0x0f, 0xa1, 0xab, 0xda,
|
|
||||||
+ 0xb9, 0x5e, 0xa8, 0x39, 0x95, 0xd3, 0x8f, 0xe2, 0xbe, 0xb8, 0x1d, 0xbf, 0xb8, 0x4b, 0x1f, 0xc3,
|
|
||||||
+ 0x35, 0x63, 0xbe, 0xb7, 0xac, 0xb6, 0x3e, 0x10, 0xdb, 0x86, 0xe0, 0xd1, 0x82, 0xf6, 0x5f, 0xc0,
|
|
||||||
+ 0x30, 0x47, 0xed, 0xcd, 0x15, 0xd2, 0xfa, 0xea, 0x5d, 0xd8, 0x5c, 0x30, 0xf6, 0x5e, 0x18, 0x32,
|
|
||||||
+ 0x75, 0x41, 0xeb, 0xee, 0xaa, 0x29, 0xe7, 0x2e, 0x5c, 0x9d, 0x60, 0xa1, 0xbd, 0xe1, 0x0b, 0x53,
|
|
||||||
+ 0x13, 0x68, 0x66, 0x1b, 0x50, 0x9b, 0xe0, 0x40, 0x19, 0x5f, 0x73, 0xe5, 0x50, 0x1e, 0xc0, 0xa7,
|
|
||||||
+ 0x1c, 0x07, 0xca, 0xca, 0x9a, 0xab, 0xc6, 0xce, 0x1f, 0x2b, 0xb0, 0x6e, 0xc2, 0xa4, 0x0c, 0xf5,
|
|
||||||
+ 0x21, 0x23, 0x67, 0x98, 0x99, 0xa3, 0x67, 0x20, 0xf4, 0x06, 0xf4, 0xf4, 0xc8, 0xa3, 0x89, 0x20,
|
|
||||||
+ 0x34, 0x0b, 0xbe, 0x5d, 0x8d, 0x7d, 0xac, 0x91, 0xaa, 0x53, 0xa7, 0x1a, 0x51, 0xa6, 0xe6, 0x33,
|
|
||||||
+ 0x90, 0x6a, 0xb7, 0x71, 0x19, 0x19, 0x54, 0xb0, 0x6d, 0xb9, 0x06, 0x92, 0x47, 0xdd, 0xf2, 0x5b,
|
|
||||||
+ 0x53, 0xfc, 0x2c, 0x28, 0x8f, 0x7a, 0x44, 0xd3, 0x58, 0x78, 0x09, 0x25, 0xb1, 0x30, 0xd1, 0x15,
|
|
||||||
+ 0x14, 0xea, 0x48, 0x62, 0x9c, 0x5f, 0x55, 0xa0, 0xa1, 0xbb, 0xd5, 0xb2, 0xca, 0xcc, 0xde, 0xb8,
|
|
||||||
+ 0x2a, 0x51, 0xf9, 0x82, 0x92, 0xa5, 0xdf, 0x35, 0x35, 0x96, 0xf7, 0xf8, 0x2c, 0xd2, 0x91, 0xda,
|
|
||||||
+ 0xa8, 0x76, 0x16, 0xa9, 0x10, 0xfd, 0x06, 0xf4, 0xf2, 0xa7, 0x52, 0xcd, 0x6b, 0x15, 0xbb, 0x19,
|
|
||||||
+ 0x56, 0x91, 0x5d, 0xa8, 0xa9, 0xf3, 0x13, 0x59, 0x5c, 0x67, 0x9d, 0xda, 0x0d, 0xa8, 0xa5, 0x99,
|
|
||||||
+ 0x32, 0x72, 0x28, 0x31, 0xd3, 0xec, 0x91, 0x95, 0x43, 0x74, 0x1b, 0x7a, 0x7e, 0x18, 0x12, 0xb9,
|
|
||||||
+ 0xdc, 0x9f, 0x3d, 0x24, 0x61, 0x76, 0x49, 0xcb, 0x58, 0xe7, 0x2f, 0x15, 0xe8, 0xef, 0xd3, 0x64,
|
|
||||||
+ 0xfe, 0xff, 0x64, 0x86, 0x0b, 0x11, 0x44, 0x29, 0x69, 0xde, 0x58, 0x39, 0x96, 0x79, 0xe3, 0x33,
|
|
||||||
+ 0x32, 0xc3, 0xfa, 0x6a, 0xe9, 0x9d, 0x6d, 0x4a, 0x84, 0xba, 0x56, 0x76, 0x32, 0x6b, 0x80, 0x75,
|
|
||||||
+ 0xf5, 0xe4, 0x23, 0x1a, 0xaa, 0x0c, 0x39, 0x24, 0xcc, 0xcb, 0xda, 0x5d, 0x5d, 0x77, 0x3d, 0x24,
|
|
||||||
+ 0x4c, 0x4d, 0x19, 0x43, 0xd6, 0x54, 0xc7, 0xb5, 0x68, 0x48, 0x43, 0x63, 0xa4, 0x21, 0xdb, 0xd0,
|
|
||||||
+ 0xa0, 0xcf, 0x9e, 0x71, 0x2c, 0x54, 0x2e, 0x5b, 0x73, 0x0d, 0x94, 0x85, 0xb9, 0x66, 0x21, 0xcc,
|
|
||||||
+ 0x5d, 0x81, 0x4d, 0xd5, 0xdb, 0x7f, 0xc2, 0xfc, 0x80, 0xc4, 0x53, 0x1b, 0x8a, 0xb7, 0x00, 0x4d,
|
|
||||||
+ 0x04, 0x4d, 0x16, 0xb0, 0x63, 0x18, 0x98, 0x37, 0xe7, 0xe8, 0xc7, 0x13, 0x6b, 0xfa, 0x35, 0x68,
|
|
||||||
+ 0x4a, 0xd0, 0x63, 0xf8, 0x6b, 0x1b, 0x18, 0xcd, 0xb4, 0xf3, 0x16, 0x74, 0xf4, 0xd0, 0x84, 0x81,
|
|
||||||
+ 0x9c, 0x94, 0x97, 0x49, 0xf9, 0x9d, 0xbf, 0x6d, 0x98, 0x70, 0x6b, 0x6a, 0x68, 0xf4, 0x10, 0xfa,
|
|
||||||
+ 0x0b, 0xff, 0x64, 0x90, 0x69, 0xaa, 0xac, 0xfe, 0x55, 0x33, 0xda, 0x1e, 0xeb, 0x7f, 0x3c, 0x63,
|
|
||||||
+ 0xfb, 0x8f, 0x67, 0xfc, 0x20, 0x4a, 0xc4, 0x1c, 0x3d, 0x80, 0x5e, 0xf9, 0xef, 0x05, 0xba, 0x6e,
|
|
||||||
+ 0x73, 0x90, 0x15, 0xff, 0x34, 0x2e, 0x64, 0xf3, 0x10, 0xfa, 0x0b, 0x3f, 0x32, 0xac, 0x3e, 0xab,
|
|
||||||
+ 0xff, 0x6f, 0x5c, 0xc8, 0xe8, 0x2e, 0xb4, 0x0b, 0x7f, 0x2e, 0xd0, 0x50, 0x33, 0x59, 0xfe, 0x99,
|
|
||||||
+ 0x71, 0x21, 0x83, 0x7d, 0xe8, 0x96, 0x7e, 0x26, 0xa0, 0x91, 0xb1, 0x67, 0xc5, 0x1f, 0x86, 0x0b,
|
|
||||||
+ 0x99, 0xec, 0x41, 0xbb, 0xd0, 0xd3, 0xb7, 0x5a, 0x2c, 0xff, 0x38, 0x18, 0x5d, 0x5b, 0x31, 0x63,
|
|
||||||
+ 0xb6, 0xf3, 0x00, 0xba, 0xa5, 0x0e, 0xbc, 0x55, 0x64, 0x55, 0xf7, 0x7f, 0x74, 0x7d, 0xe5, 0x9c,
|
|
||||||
+ 0xe1, 0xf4, 0x10, 0xfa, 0x0b, 0xfd, 0x78, 0xeb, 0xdc, 0xd5, 0x6d, 0xfa, 0x0b, 0xcd, 0xfa, 0x5c,
|
|
||||||
+ 0x6d, 0x76, 0xa1, 0xdc, 0x2a, 0x6c, 0xf6, 0x72, 0xf7, 0x7d, 0x74, 0x63, 0xf5, 0xa4, 0xd1, 0xea,
|
|
||||||
+ 0x01, 0xf4, 0xca, 0x8d, 0x77, 0xcb, 0x6c, 0x65, 0x3b, 0xfe, 0xf2, 0x93, 0x53, 0xea, 0xc1, 0xe7,
|
|
||||||
+ 0x27, 0x67, 0x55, 0x6b, 0xfe, 0x42, 0x46, 0xf7, 0x00, 0x4c, 0x71, 0x15, 0x92, 0x38, 0xdb, 0xb2,
|
|
||||||
+ 0xa5, 0xa2, 0x2e, 0xdb, 0xb2, 0x15, 0x85, 0xd8, 0x5d, 0x00, 0x5d, 0x13, 0x85, 0x34, 0x15, 0xe8,
|
|
||||||
+ 0xaa, 0x55, 0x63, 0xa1, 0x10, 0x1b, 0x0d, 0x97, 0x27, 0x96, 0x18, 0x60, 0xc6, 0x5e, 0x84, 0xc1,
|
|
||||||
+ 0x67, 0x00, 0x79, 0xad, 0x65, 0x19, 0x2c, 0x55, 0x5f, 0x97, 0xf8, 0xa0, 0x53, 0xac, 0xac, 0x90,
|
|
||||||
+ 0xb1, 0x75, 0x45, 0xb5, 0x75, 0x09, 0x8b, 0xfe, 0x42, 0xe6, 0x5c, 0x3e, 0x6c, 0x8b, 0x09, 0xf5,
|
|
||||||
+ 0x68, 0x29, 0x7b, 0x46, 0x1f, 0x42, 0xa7, 0x98, 0x32, 0x5b, 0x2d, 0x56, 0xa4, 0xd1, 0xa3, 0x52,
|
|
||||||
+ 0xda, 0x8c, 0xee, 0x42, 0xaf, 0x9c, 0x10, 0xa3, 0xc2, 0xbd, 0x58, 0x4a, 0x93, 0x47, 0xa6, 0x19,
|
|
||||||
+ 0x54, 0x20, 0x7f, 0x1f, 0x20, 0x4f, 0x9c, 0xad, 0xfb, 0x96, 0x52, 0xe9, 0x05, 0xa9, 0x9f, 0x41,
|
|
||||||
+ 0xaf, 0x10, 0xb7, 0x65, 0x4d, 0x78, 0xb5, 0x64, 0x70, 0x1e, 0xcd, 0x47, 0x26, 0xc3, 0x2a, 0x85,
|
|
||||||
+ 0xed, 0x7b, 0xd0, 0x29, 0xbe, 0x11, 0xd6, 0xda, 0x15, 0xef, 0xc6, 0x65, 0x41, 0xaf, 0xf0, 0x9e,
|
|
||||||
+ 0xd8, 0xb3, 0xbb, 0xfc, 0xc4, 0x5c, 0x16, 0xf4, 0x4a, 0xf5, 0xa8, 0x8d, 0x35, 0xab, 0x8a, 0xd4,
|
|
||||||
+ 0xcb, 0x9e, 0x82, 0x72, 0xf1, 0x66, 0xbd, 0xbf, 0xb2, 0xa4, 0xbb, 0xec, 0x0c, 0x16, 0xeb, 0x14,
|
|
||||||
+ 0xeb, 0x8f, 0x15, 0xb5, 0xcb, 0xf7, 0xc4, 0x84, 0x62, 0x2d, 0x52, 0x88, 0x09, 0x2b, 0x4a, 0x94,
|
|
||||||
+ 0x0b, 0x19, 0x1d, 0x40, 0xff, 0xa1, 0x4d, 0x33, 0x4d, 0x0a, 0x6c, 0xd4, 0x59, 0x91, 0xf2, 0x8f,
|
|
||||||
+ 0x46, 0xab, 0xa6, 0xcc, 0x2e, 0x7f, 0x0e, 0x83, 0xa5, 0xf4, 0x17, 0xdd, 0xcc, 0x5a, 0x9e, 0x2b,
|
|
||||||
+ 0xf3, 0xe2, 0x0b, 0xd5, 0x3a, 0x84, 0x8d, 0xc5, 0xec, 0x17, 0xbd, 0x62, 0x36, 0x7d, 0x75, 0x56,
|
|
||||||
+ 0x7c, 0x21, 0xab, 0x8f, 0xa1, 0x69, 0xb3, 0x2d, 0x64, 0x5a, 0xcb, 0x0b, 0xd9, 0xd7, 0x45, 0x4b,
|
|
||||||
+ 0xf7, 0x3a, 0xdf, 0x7e, 0x77, 0xb3, 0xf2, 0xd7, 0xef, 0x6e, 0x56, 0xfe, 0xf1, 0xdd, 0xcd, 0xca,
|
|
||||||
+ 0x71, 0x43, 0xcd, 0xbe, 0xff, 0xaf, 0x00, 0x00, 0x00, 0xff, 0xff, 0x45, 0xa8, 0x91, 0xab, 0x62,
|
|
||||||
+ 0x22, 0x00, 0x00,
|
|
||||||
}
|
|
||||||
diff --git a/protocols/grpc/agent.proto b/protocols/grpc/agent.proto
|
|
||||||
index b0fab6d..d863645 100644
|
|
||||||
--- a/protocols/grpc/agent.proto
|
|
||||||
+++ b/protocols/grpc/agent.proto
|
|
||||||
@@ -46,6 +46,7 @@ service AgentService {
|
|
||||||
rpc UpdateRoutes(UpdateRoutesRequest) returns (Routes);
|
|
||||||
rpc ListInterfaces(ListInterfacesRequest) returns(Interfaces);
|
|
||||||
rpc ListRoutes(ListRoutesRequest) returns (Routes);
|
|
||||||
+ rpc UpdateIPVSRule(UpdateIPVSRequest) returns (IPVSResponse);
|
|
||||||
|
|
||||||
// tracing
|
|
||||||
rpc StartTracing(StartTracingRequest) returns (google.protobuf.Empty);
|
|
||||||
@@ -495,3 +496,13 @@ message StartTracingRequest {
|
|
||||||
|
|
||||||
message StopTracingRequest {
|
|
||||||
}
|
|
||||||
+
|
|
||||||
+message UpdateIPVSRequest {
|
|
||||||
+ // IPVS_req is the IPVS rule message needed to update
|
|
||||||
+ string IPVS_req = 1;
|
|
||||||
+}
|
|
||||||
+
|
|
||||||
+message IPVSResponse {
|
|
||||||
+ // IPVS_res is the response of IPVS updating
|
|
||||||
+ string IPVS_res = 1;
|
|
||||||
+}
|
|
||||||
--
|
|
||||||
2.14.3 (Apple Git-98)
|
|
||||||
|
|
||||||
@ -1,193 +0,0 @@
|
|||||||
From 01563c08910ddaba4077fd9dc691df541e045165 Mon Sep 17 00:00:00 2001
|
|
||||||
From: xiadanni <xiadanni1@huawei.com>
|
|
||||||
Date: Tue, 18 Aug 2020 17:05:32 +0800
|
|
||||||
Subject: [PATCH 04/16] agent: add IPVS test
|
|
||||||
|
|
||||||
Signed-off-by: xiadanni <xiadanni1@huawei.com>
|
|
||||||
---
|
|
||||||
grpc_test.go | 172 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
|
|
||||||
1 file changed, 172 insertions(+)
|
|
||||||
|
|
||||||
diff --git a/grpc_test.go b/grpc_test.go
|
|
||||||
index e69102b..d58c0b6 100644
|
|
||||||
--- a/grpc_test.go
|
|
||||||
+++ b/grpc_test.go
|
|
||||||
@@ -1840,3 +1840,175 @@ func getPipeMaxSize() (uint32, error) {
|
|
||||||
u, err := strconv.ParseUint(s, 10, 32)
|
|
||||||
return uint32(u), err
|
|
||||||
}
|
|
||||||
+
|
|
||||||
+func TestUpdateIPVSRule(t *testing.T) {
|
|
||||||
+ assert := assert.New(t)
|
|
||||||
+
|
|
||||||
+ // add IPVS rule successfully
|
|
||||||
+ a := &agentGRPC{
|
|
||||||
+ sandbox: &sandbox{
|
|
||||||
+ containers: make(map[string]*container),
|
|
||||||
+ },
|
|
||||||
+ }
|
|
||||||
+
|
|
||||||
+ req := &pb.UpdateIPVSRequest{
|
|
||||||
+ IPVSReq: "ipvsadm -A -t 17.2.0.7:80 -s rr -p 3000",
|
|
||||||
+ }
|
|
||||||
+
|
|
||||||
+ _, err := a.UpdateIPVSRule(context.Background(), req)
|
|
||||||
+ assert.NoError(err)
|
|
||||||
+
|
|
||||||
+ // delete ipvs rule successfully
|
|
||||||
+ req = &pb.UpdateIPVSRequest{
|
|
||||||
+ IPVSReq: "ipvsadm -D -t 17.2.0.7:80",
|
|
||||||
+ }
|
|
||||||
+
|
|
||||||
+ _, err = a.UpdateIPVSRule(context.Background(), req)
|
|
||||||
+ assert.NoError(err)
|
|
||||||
+
|
|
||||||
+ // update ipvs rule error because exec failed
|
|
||||||
+ req = &pb.UpdateIPVSRequest{
|
|
||||||
+ IPVSReq: "ipvsadm -A -t 17.2.0.7:80 -s rr -p -3000",
|
|
||||||
+ }
|
|
||||||
+
|
|
||||||
+ _, err = a.UpdateIPVSRule(context.Background(), req)
|
|
||||||
+ assert.Error(err)
|
|
||||||
+ assert.Contains(err.Error(), "exec IPVS command failed")
|
|
||||||
+
|
|
||||||
+ // update IPVS rule error because rule less than validHeadLength
|
|
||||||
+ req = &pb.UpdateIPVSRequest{
|
|
||||||
+ IPVSReq: "ipvsa",
|
|
||||||
+ }
|
|
||||||
+
|
|
||||||
+ _, err = a.UpdateIPVSRule(context.Background(), req)
|
|
||||||
+ assert.Error(err)
|
|
||||||
+ assert.Contains(err.Error(), "invalid IPVS rule")
|
|
||||||
+
|
|
||||||
+ // update ipvs rule error because invalid command
|
|
||||||
+ req = &pb.UpdateIPVSRequest{
|
|
||||||
+ IPVSReq: "abcabcabc ipvsadm",
|
|
||||||
+ }
|
|
||||||
+
|
|
||||||
+ _, err = a.UpdateIPVSRule(context.Background(), req)
|
|
||||||
+ assert.Error(err)
|
|
||||||
+ assert.Contains(err.Error(), "invalid IPVS rule")
|
|
||||||
+
|
|
||||||
+ // add ipvs rule error because rule count exceeds
|
|
||||||
+ a = &agentGRPC{
|
|
||||||
+ sandbox: &sandbox{
|
|
||||||
+ containers: make(map[string]*container),
|
|
||||||
+ ipvsadm: ipvsAdm{
|
|
||||||
+ ipvsRuleCnt: 20000,
|
|
||||||
+ },
|
|
||||||
+ },
|
|
||||||
+ }
|
|
||||||
+
|
|
||||||
+ req = &pb.UpdateIPVSRequest{
|
|
||||||
+ IPVSReq: "ipvsadm -A -t 17.2.0.7:80 -s rr -p 3000",
|
|
||||||
+ }
|
|
||||||
+ _, err = a.UpdateIPVSRule(context.Background(), req)
|
|
||||||
+ assert.Error(err)
|
|
||||||
+ assert.Errorf(err, "rules exceed limit")
|
|
||||||
+
|
|
||||||
+ // add ipvs rule error because ipvs request item less than 2
|
|
||||||
+ a = &agentGRPC{
|
|
||||||
+ sandbox: &sandbox{
|
|
||||||
+ containers: make(map[string]*container),
|
|
||||||
+ ipvsadm: ipvsAdm{
|
|
||||||
+ ipvsRuleCnt: 0,
|
|
||||||
+ },
|
|
||||||
+ },
|
|
||||||
+ }
|
|
||||||
+
|
|
||||||
+ req = &pb.UpdateIPVSRequest{
|
|
||||||
+ IPVSReq: "ipvsadm",
|
|
||||||
+ }
|
|
||||||
+
|
|
||||||
+ _, err = a.UpdateIPVSRule(context.Background(), req)
|
|
||||||
+ assert.Error(err)
|
|
||||||
+ assert.Contains(err.Error(), "invalid IPVS rule")
|
|
||||||
+
|
|
||||||
+ // add ipvs rule error because ipvs rule nil
|
|
||||||
+ req = nil
|
|
||||||
+
|
|
||||||
+ _, err = a.UpdateIPVSRule(context.Background(), req)
|
|
||||||
+ assert.Error(err)
|
|
||||||
+ assert.Contains(err.Error(), "IPVS rule is nil")
|
|
||||||
+
|
|
||||||
+ // add ipvs rule error because ipvs rule string is empty
|
|
||||||
+ req = &pb.UpdateIPVSRequest{
|
|
||||||
+ IPVSReq: "",
|
|
||||||
+ }
|
|
||||||
+
|
|
||||||
+ _, err = a.UpdateIPVSRule(context.Background(), req)
|
|
||||||
+ assert.Error(err)
|
|
||||||
+ assert.Contains(err.Error(), "IPVS rule is nil")
|
|
||||||
+
|
|
||||||
+ // restore ipvs rule successfully
|
|
||||||
+ a = &agentGRPC{
|
|
||||||
+ sandbox: &sandbox{
|
|
||||||
+ containers: make(map[string]*container),
|
|
||||||
+ },
|
|
||||||
+ }
|
|
||||||
+
|
|
||||||
+ req = &pb.UpdateIPVSRequest{
|
|
||||||
+ IPVSReq: "restore|2|-A -t 10.10.11.12:100 -s rr -p 3000\n-a -t 10.10.11.12:100 -r 172.16.0.1:80 -m",
|
|
||||||
+ }
|
|
||||||
+
|
|
||||||
+ _, err = a.UpdateIPVSRule(context.Background(), req)
|
|
||||||
+ assert.NoError(err)
|
|
||||||
+
|
|
||||||
+ // clear IPVS rule successfully
|
|
||||||
+ req = &pb.UpdateIPVSRequest{
|
|
||||||
+ IPVSReq: "ipvsadm -C",
|
|
||||||
+ }
|
|
||||||
+
|
|
||||||
+ _, err = a.UpdateIPVSRule(context.Background(), req)
|
|
||||||
+ assert.NoError(err)
|
|
||||||
+ assert.Equal(a.sandbox.ipvsadm.ipvsRuleCnt, uint64(0))
|
|
||||||
+
|
|
||||||
+ // restore ipvs rule error because rule count invalid
|
|
||||||
+ req = &pb.UpdateIPVSRequest{
|
|
||||||
+ IPVSReq: "restore|abc|-A -t 10.10.11.12:100 -s rr -p 3000\n-a -t 10.10.11.12:100 -r 172.16.0.1:80 -m",
|
|
||||||
+ }
|
|
||||||
+
|
|
||||||
+ _, err = a.UpdateIPVSRule(context.Background(), req)
|
|
||||||
+ assert.Error(err)
|
|
||||||
+ assert.Contains(err.Error(), "invalid IPVS rule")
|
|
||||||
+
|
|
||||||
+ // restore ipvs rule error because other rules exists
|
|
||||||
+ a = &agentGRPC{
|
|
||||||
+ sandbox: &sandbox{
|
|
||||||
+ containers: make(map[string]*container),
|
|
||||||
+ ipvsadm: ipvsAdm{
|
|
||||||
+ ipvsRuleCnt: 5,
|
|
||||||
+ },
|
|
||||||
+ },
|
|
||||||
+ }
|
|
||||||
+
|
|
||||||
+ req = &pb.UpdateIPVSRequest{
|
|
||||||
+ IPVSReq: "restore|2|-A -t 10.10.11.12:100 -s rr -p 3000\n-a -t 10.10.11.12:100 -r 172.16.0.1:80 -m",
|
|
||||||
+ }
|
|
||||||
+
|
|
||||||
+ _, err = a.UpdateIPVSRule(context.Background(), req)
|
|
||||||
+ assert.Error(err)
|
|
||||||
+ assert.Contains(err.Error(), "exist some rules in system")
|
|
||||||
+
|
|
||||||
+ // restore ipvs rule error because ipvs req item less than 3
|
|
||||||
+ a = &agentGRPC{
|
|
||||||
+ sandbox: &sandbox{
|
|
||||||
+ containers: make(map[string]*container),
|
|
||||||
+ ipvsadm: ipvsAdm{
|
|
||||||
+ ipvsRuleCnt: 0,
|
|
||||||
+ },
|
|
||||||
+ },
|
|
||||||
+ }
|
|
||||||
+
|
|
||||||
+ req = &pb.UpdateIPVSRequest{
|
|
||||||
+ IPVSReq: "restore|-A -t 10.10.11.12:100 -s rr -p 3000\n-a -t 10.10.11.12:100 -r 172.16.0.1:80 -m",
|
|
||||||
+ }
|
|
||||||
+
|
|
||||||
+ _, err = a.UpdateIPVSRule(context.Background(), req)
|
|
||||||
+ assert.Error(err)
|
|
||||||
+ assert.Contains(err.Error(), "invalid IPVS rule")
|
|
||||||
+}
|
|
||||||
--
|
|
||||||
2.14.3 (Apple Git-98)
|
|
||||||
|
|
||||||
@ -1,42 +0,0 @@
|
|||||||
From 8c9f9be2a9c195d0bc12b43c491adaacb7bb8154 Mon Sep 17 00:00:00 2001
|
|
||||||
From: holyfei <yangfeiyu20092010@163.com>
|
|
||||||
Date: Tue, 18 Aug 2020 10:42:38 +0800
|
|
||||||
Subject: [PATCH 05/16] mount: support mount block device
|
|
||||||
|
|
||||||
reason: modify mountStorage to support mount block device
|
|
||||||
"-v /dev/blockdevice:/home/test"
|
|
||||||
|
|
||||||
Signed-off-by: yangfeiyu <yangfeiyu2@huawei.com>
|
|
||||||
---
|
|
||||||
mount.go | 15 ++++++++++++++-
|
|
||||||
1 file changed, 14 insertions(+), 1 deletion(-)
|
|
||||||
|
|
||||||
diff --git a/mount.go b/mount.go
|
|
||||||
index f0c3efe..a05d4a6 100644
|
|
||||||
--- a/mount.go
|
|
||||||
+++ b/mount.go
|
|
||||||
@@ -364,7 +364,20 @@ func commonStorageHandler(storage pb.Storage) (string, error) {
|
|
||||||
func mountStorage(storage pb.Storage) error {
|
|
||||||
flags, options := parseMountFlagsAndOptions(storage.Options)
|
|
||||||
|
|
||||||
- return mount(storage.Source, storage.MountPoint, storage.Fstype, flags, options)
|
|
||||||
+ var fsType = storage.Fstype
|
|
||||||
+ if (storage.Driver == driverSCSIType || storage.Driver == driverBlkType) && strings.Contains(storage.Fstype, "bind") {
|
|
||||||
+ cs := strings.Split(storage.Fstype, "-")
|
|
||||||
+ if len(cs) == 2 && cs[1] != "" {
|
|
||||||
+ fsType = cs[1]
|
|
||||||
+ // here we temporarily discard the bind option,
|
|
||||||
+ // in order to be able to mount the file system of the block device.
|
|
||||||
+ // and then reset `storage.Fstype` to "bind" which pass through to the libcontainer pkg.
|
|
||||||
+ flags = flags &^ flagList["bind"]
|
|
||||||
+ storage.Fstype = "bind"
|
|
||||||
+ }
|
|
||||||
+ }
|
|
||||||
+
|
|
||||||
+ return mount(storage.Source, storage.MountPoint, fsType, flags, options)
|
|
||||||
}
|
|
||||||
|
|
||||||
// addStorages takes a list of storages passed by the caller, and perform the
|
|
||||||
--
|
|
||||||
2.14.3 (Apple Git-98)
|
|
||||||
|
|
||||||
@ -1,54 +0,0 @@
|
|||||||
From bccda1d208f31eab55863883cf0718d7b4b8deef Mon Sep 17 00:00:00 2001
|
|
||||||
From: jiangpengfei <jiangpengfei9@huawei.com>
|
|
||||||
Date: Tue, 18 Aug 2020 19:30:42 +0800
|
|
||||||
Subject: [PATCH 06/16] agent: make workaround for slow response in aarch64
|
|
||||||
|
|
||||||
reason: make workaround for slow response in aarch64
|
|
||||||
when hotplug virtio-net-pci device
|
|
||||||
|
|
||||||
Signed-off-by: jiangpengfei <jiangpengfei9@huawei.com>
|
|
||||||
---
|
|
||||||
agent.go | 2 +-
|
|
||||||
device.go | 12 ++++++------
|
|
||||||
2 files changed, 7 insertions(+), 7 deletions(-)
|
|
||||||
|
|
||||||
diff --git a/agent.go b/agent.go
|
|
||||||
index c161e93..e81c2cd 100644
|
|
||||||
--- a/agent.go
|
|
||||||
+++ b/agent.go
|
|
||||||
@@ -185,7 +185,7 @@ var logsVSockPort = uint32(0)
|
|
||||||
var debugConsoleVSockPort = uint32(0)
|
|
||||||
|
|
||||||
// Timeout waiting for a device to be hotplugged
|
|
||||||
-var hotplugTimeout = 3 * time.Second
|
|
||||||
+var hotplugTimeout = 10 * time.Second
|
|
||||||
|
|
||||||
// Specify the log level
|
|
||||||
var logLevel = defaultLogLevel
|
|
||||||
diff --git a/device.go b/device.go
|
|
||||||
index ec1907e..46a1b96 100644
|
|
||||||
--- a/device.go
|
|
||||||
+++ b/device.go
|
|
||||||
@@ -179,13 +179,13 @@ func getPCIDeviceNameImpl(s *sandbox, pciID string) (string, error) {
|
|
||||||
return "", err
|
|
||||||
}
|
|
||||||
|
|
||||||
- fieldLogger := agentLog.WithField("pciAddr", pciAddr)
|
|
||||||
-
|
|
||||||
// Rescan pci bus if we need to wait for a new pci device
|
|
||||||
- if err = rescanPciBus(); err != nil {
|
|
||||||
- fieldLogger.WithError(err).Error("Failed to scan pci bus")
|
|
||||||
- return "", err
|
|
||||||
- }
|
|
||||||
+ // FIXME:Comment out this code Temporarily, because once the PCIBus is scanned,
|
|
||||||
+ // the device hot-plug event is lost
|
|
||||||
+ //if err = rescanPciBus(); err != nil {
|
|
||||||
+ // fieldLogger.WithError(err).Error("Failed to scan pci bus")
|
|
||||||
+ // return "", err
|
|
||||||
+ //}
|
|
||||||
|
|
||||||
return getDeviceName(s, pciAddr)
|
|
||||||
}
|
|
||||||
--
|
|
||||||
2.14.3 (Apple Git-98)
|
|
||||||
|
|
||||||
@ -1,37 +0,0 @@
|
|||||||
From fa673c93e243ba297d53b585cd2f51fa68380fc5 Mon Sep 17 00:00:00 2001
|
|
||||||
From: jiangpengfei <jiangpengfei9@huawei.com>
|
|
||||||
Date: Tue, 18 Aug 2020 19:37:48 +0800
|
|
||||||
Subject: [PATCH 07/16] agent: using pcie-root-port driver to hotplug device
|
|
||||||
|
|
||||||
reason: In original pci-bridge scheme, the "F" in the BDF is not used,
|
|
||||||
and was written hard code "0" in the kata-agent. But the "function"
|
|
||||||
is specified when switching to pcie-root-port scheme, so when should
|
|
||||||
pass the "pci-bridge BDF" or "pcie-root-port BDF" from kata-runtime
|
|
||||||
and use in kata-agent.
|
|
||||||
|
|
||||||
Signed-off-by: jiangpengfei <jiangpengfei9@huawei.com>
|
|
||||||
---
|
|
||||||
device.go | 7 ++++++-
|
|
||||||
1 file changed, 6 insertions(+), 1 deletion(-)
|
|
||||||
|
|
||||||
diff --git a/device.go b/device.go
|
|
||||||
index 46a1b96..8e6950c 100644
|
|
||||||
--- a/device.go
|
|
||||||
+++ b/device.go
|
|
||||||
@@ -101,7 +101,12 @@ func getDevicePCIAddressImpl(pciID string) (string, error) {
|
|
||||||
|
|
||||||
// Deduce the complete bridge address based on the bridge address identifier passed
|
|
||||||
// and the fact that bridges are attached on the main bus with function 0.
|
|
||||||
- pciBridgeAddr := fmt.Sprintf("0000:00:%s.0", bridgeID)
|
|
||||||
+ // Update: support pcie-root-port device
|
|
||||||
+ // In original pci-bridge scheme, the "F" in the BDF is not used, and was written
|
|
||||||
+ // hard code "0" in the kata-agent. But the "function" is specified when switching to
|
|
||||||
+ // pcie-root-port scheme, so when should pass the "pci-bridge BDF" or "pcie-root-port BDF"
|
|
||||||
+ // from kata-runtime and use in kata-agent.
|
|
||||||
+ pciBridgeAddr := fmt.Sprintf("0000:00:%s", bridgeID)
|
|
||||||
|
|
||||||
// Find out the bus exposed by bridge
|
|
||||||
bridgeBusPath := fmt.Sprintf(pciBusPathFormat, sysBusPrefix, pciBridgeAddr)
|
|
||||||
--
|
|
||||||
2.14.3 (Apple Git-98)
|
|
||||||
|
|
||||||
@ -1,94 +0,0 @@
|
|||||||
From eea286fbafba2e95410b603fbef762e2b25eb207 Mon Sep 17 00:00:00 2001
|
|
||||||
From: jiangpengfei <jiangpengfei9@huawei.com>
|
|
||||||
Date: Tue, 18 Aug 2020 19:45:57 +0800
|
|
||||||
Subject: [PATCH 08/16] agent: support get root bus path dynamically
|
|
||||||
|
|
||||||
reason: support get root bus dynamically no matter the
|
|
||||||
target arch is amd64 or arm64
|
|
||||||
|
|
||||||
Signed-off-by: jiangpengfei <jiangpengfei9@huawei.com>
|
|
||||||
---
|
|
||||||
agent.go | 1 +
|
|
||||||
device_amd64.go | 6 +++++-
|
|
||||||
device_arm64.go | 27 ++++++++++++++++++++++++++-
|
|
||||||
3 files changed, 32 insertions(+), 2 deletions(-)
|
|
||||||
|
|
||||||
diff --git a/agent.go b/agent.go
|
|
||||||
index e81c2cd..50afd7a 100644
|
|
||||||
--- a/agent.go
|
|
||||||
+++ b/agent.go
|
|
||||||
@@ -730,6 +730,7 @@ func (s *sandbox) listenToUdevEvents() {
|
|
||||||
defer uEvHandler.Close()
|
|
||||||
|
|
||||||
fieldLogger.Infof("Started listening for uevents")
|
|
||||||
+ rootBusPath := initRootBusPath()
|
|
||||||
|
|
||||||
for {
|
|
||||||
uEv, err := uEvHandler.Read()
|
|
||||||
diff --git a/device_amd64.go b/device_amd64.go
|
|
||||||
index 66bc052..26f55bf 100644
|
|
||||||
--- a/device_amd64.go
|
|
||||||
+++ b/device_amd64.go
|
|
||||||
@@ -8,7 +8,7 @@
|
|
||||||
package main
|
|
||||||
|
|
||||||
const (
|
|
||||||
- rootBusPath = "/devices/pci0000:00"
|
|
||||||
+ defaultRootBusPath = "/devices/pci0000:00"
|
|
||||||
|
|
||||||
// From https://www.kernel.org/doc/Documentation/acpi/namespace.txt
|
|
||||||
// The Linux kernel's core ACPI subsystem creates struct acpi_device
|
|
||||||
@@ -21,3 +21,7 @@ const (
|
|
||||||
// in a subdirectory whose prefix is pfn (page frame number).
|
|
||||||
pfnDevPrefix = "/pfn"
|
|
||||||
)
|
|
||||||
+
|
|
||||||
+func initRootBusPath() string {
|
|
||||||
+ return defaultRootBusPath
|
|
||||||
+}
|
|
||||||
diff --git a/device_arm64.go b/device_arm64.go
|
|
||||||
index b73b582..d039c67 100644
|
|
||||||
--- a/device_arm64.go
|
|
||||||
+++ b/device_arm64.go
|
|
||||||
@@ -6,8 +6,14 @@
|
|
||||||
|
|
||||||
package main
|
|
||||||
|
|
||||||
+import (
|
|
||||||
+ "fmt"
|
|
||||||
+ "io/ioutil"
|
|
||||||
+ "regexp"
|
|
||||||
+)
|
|
||||||
+
|
|
||||||
const (
|
|
||||||
- rootBusPath = "/devices/platform/4010000000.pcie/pci0000:00"
|
|
||||||
+ defaultRootBusPath = "/devices/platform/4010000000.pcie/pci0000:00"
|
|
||||||
|
|
||||||
// From https://www.kernel.org/doc/Documentation/acpi/namespace.txt
|
|
||||||
// The Linux kernel's core ACPI subsystem creates struct acpi_device
|
|
||||||
@@ -20,3 +26,22 @@ const (
|
|
||||||
// in a subdirectory whose prefix is pfn (page frame number).
|
|
||||||
pfnDevPrefix = "/pfn"
|
|
||||||
)
|
|
||||||
+
|
|
||||||
+func initRootBusPath() string {
|
|
||||||
+ pcieDriverReg := regexp.MustCompile(`^[0-9a-f]{10}.pcie$`)
|
|
||||||
+ rootBusPath := defaultRootBusPath
|
|
||||||
+ files, err := ioutil.ReadDir("/sys/devices/platform")
|
|
||||||
+ if err != nil {
|
|
||||||
+ return rootBusPath
|
|
||||||
+ }
|
|
||||||
+ for _, f := range files {
|
|
||||||
+ if !f.IsDir() {
|
|
||||||
+ continue
|
|
||||||
+ }
|
|
||||||
+ if pcieDriverReg.MatchString(f.Name()) {
|
|
||||||
+ rootBusPath = fmt.Sprintf("/devices/platform/%s/pci0000:00", f.Name())
|
|
||||||
+ break
|
|
||||||
+ }
|
|
||||||
+ }
|
|
||||||
+ return rootBusPath
|
|
||||||
+}
|
|
||||||
--
|
|
||||||
2.14.3 (Apple Git-98)
|
|
||||||
|
|
||||||
@ -1,188 +0,0 @@
|
|||||||
From 1268b710c7f0528d971d9c0e54429d3f4e48c372 Mon Sep 17 00:00:00 2001
|
|
||||||
From: holyfei <yangfeiyu20092010@163.com>
|
|
||||||
Date: Tue, 18 Aug 2020 16:45:29 +0800
|
|
||||||
Subject: [PATCH 09/16] storage: add pkg/storage for mount
|
|
||||||
|
|
||||||
reason: add gpath.go and nfs.go, provide mount
|
|
||||||
functions and structs
|
|
||||||
|
|
||||||
Signed-off-by: yangfeiyu <yangfeiyu2@huawei.com>
|
|
||||||
---
|
|
||||||
pkg/storage/gpath.go | 64 ++++++++++++++++++++++++++++++++++++
|
|
||||||
pkg/storage/nfs.go | 93 ++++++++++++++++++++++++++++++++++++++++++++++++++++
|
|
||||||
2 files changed, 157 insertions(+)
|
|
||||||
create mode 100644 pkg/storage/gpath.go
|
|
||||||
create mode 100644 pkg/storage/nfs.go
|
|
||||||
|
|
||||||
diff --git a/pkg/storage/gpath.go b/pkg/storage/gpath.go
|
|
||||||
new file mode 100644
|
|
||||||
index 0000000..5cb951f
|
|
||||||
--- /dev/null
|
|
||||||
+++ b/pkg/storage/gpath.go
|
|
||||||
@@ -0,0 +1,64 @@
|
|
||||||
+// Copyright (c) Huawei Technologies Co., Ltd. 2019. All rights reserved.
|
|
||||||
+// SPDX-License-Identifier: Apache-2.0
|
|
||||||
+// Description: common functions
|
|
||||||
+// Author: licuifang
|
|
||||||
+// Create: 2019-06-24
|
|
||||||
+
|
|
||||||
+package storage
|
|
||||||
+
|
|
||||||
+import (
|
|
||||||
+ "fmt"
|
|
||||||
+ "os"
|
|
||||||
+ "syscall"
|
|
||||||
+
|
|
||||||
+ "github.com/opencontainers/runc/libcontainer/mount"
|
|
||||||
+)
|
|
||||||
+
|
|
||||||
+func ValidateGpath(guestPath string, opts []string) error {
|
|
||||||
+ for _, opt := range opts {
|
|
||||||
+ switch opt {
|
|
||||||
+ case "shared":
|
|
||||||
+ flag := syscall.MS_SHARED
|
|
||||||
+ if err := ensureMountedAs(guestPath, flag); err != nil {
|
|
||||||
+ return err
|
|
||||||
+ }
|
|
||||||
+ return nil
|
|
||||||
+ case "mounted":
|
|
||||||
+ // if mounted in option, the guestpath must exist and must be a mountpoint
|
|
||||||
+ _, err := os.Stat(guestPath)
|
|
||||||
+ if err != nil {
|
|
||||||
+ return err
|
|
||||||
+ }
|
|
||||||
+ mountpoint, err := mount.Mounted(guestPath)
|
|
||||||
+ if err != nil {
|
|
||||||
+ return err
|
|
||||||
+ }
|
|
||||||
+ if mountpoint == false {
|
|
||||||
+ return fmt.Errorf("the guespath:%s is not a mountpoint while mounted was set", guestPath)
|
|
||||||
+ }
|
|
||||||
+ return nil
|
|
||||||
+ }
|
|
||||||
+ }
|
|
||||||
+ if err := os.MkdirAll(guestPath, 0750); err != nil {
|
|
||||||
+ return err
|
|
||||||
+ }
|
|
||||||
+ return nil
|
|
||||||
+}
|
|
||||||
+
|
|
||||||
+func ensureMountedAs(mountPoint string, flag int) error {
|
|
||||||
+ if err := os.MkdirAll(mountPoint, 0750); err != nil {
|
|
||||||
+ return err
|
|
||||||
+ }
|
|
||||||
+ mounted, err := mount.Mounted(mountPoint)
|
|
||||||
+ if err != nil {
|
|
||||||
+ return err
|
|
||||||
+ }
|
|
||||||
+
|
|
||||||
+ if !mounted {
|
|
||||||
+ if err := syscall.Mount(mountPoint, mountPoint, "bind", uintptr(syscall.MS_BIND), ""); err != nil {
|
|
||||||
+ return err
|
|
||||||
+ }
|
|
||||||
+ }
|
|
||||||
+
|
|
||||||
+ return syscall.Mount("", mountPoint, "", uintptr(flag), "")
|
|
||||||
+}
|
|
||||||
diff --git a/pkg/storage/nfs.go b/pkg/storage/nfs.go
|
|
||||||
new file mode 100644
|
|
||||||
index 0000000..44bc85d
|
|
||||||
--- /dev/null
|
|
||||||
+++ b/pkg/storage/nfs.go
|
|
||||||
@@ -0,0 +1,93 @@
|
|
||||||
+// Copyright (c) Huawei Technologies Co., Ltd. 2019. All rights reserved.
|
|
||||||
+// SPDX-License-Identifier: Apache-2.0
|
|
||||||
+// Description: common functions
|
|
||||||
+// Author: leizhongkai
|
|
||||||
+// Create: 2019-03-10
|
|
||||||
+
|
|
||||||
+package storage
|
|
||||||
+
|
|
||||||
+import (
|
|
||||||
+ "fmt"
|
|
||||||
+ "os"
|
|
||||||
+ "os/exec"
|
|
||||||
+ "path/filepath"
|
|
||||||
+ "strings"
|
|
||||||
+ "time"
|
|
||||||
+
|
|
||||||
+ "github.com/opencontainers/runc/libcontainer/mount"
|
|
||||||
+ "github.com/sirupsen/logrus"
|
|
||||||
+)
|
|
||||||
+
|
|
||||||
+const (
|
|
||||||
+ NFS = "nfs"
|
|
||||||
+
|
|
||||||
+ MAX_MOUNT_ATTEMPTS = 100
|
|
||||||
+ MAX_MOUTN_TIMEOUT = 90 // seconds
|
|
||||||
+
|
|
||||||
+ // ignored errors
|
|
||||||
+ PERMISSION_DENY = "Permission denied"
|
|
||||||
+ kataGuestStorageDir = "/run/kata-containers/storage/containers/"
|
|
||||||
+)
|
|
||||||
+
|
|
||||||
+func nfsMount(source, dest string, opt string) error {
|
|
||||||
+ cmd := exec.Command("/bin/mount", "-t", "nfs", "-o", opt, source, dest)
|
|
||||||
+ res, err := cmd.Output()
|
|
||||||
+ logrus.Debugf("mount %s to %s, and get res %s", source, dest, string(res))
|
|
||||||
+ return err
|
|
||||||
+}
|
|
||||||
+
|
|
||||||
+func nfsMountWithAttempt(source, dest string, opt string) (err error) {
|
|
||||||
+ timeStart := time.Now().Unix()
|
|
||||||
+ for i := 0; i < MAX_MOUNT_ATTEMPTS; i++ {
|
|
||||||
+ logrus.Infof("this is the %d times to mount %s to %s", i, source, dest)
|
|
||||||
+ err = nfsMount(source, dest, opt)
|
|
||||||
+ if err != nil {
|
|
||||||
+ ee, ok := err.(*exec.ExitError)
|
|
||||||
+ if ok {
|
|
||||||
+ if strings.Contains(string(ee.Stderr), PERMISSION_DENY) {
|
|
||||||
+ logrus.Errorf("mounting nfs:%s to %s, get error: %s,should break", source, dest, string(ee.Stderr))
|
|
||||||
+ // We do not retry when the error type is PERMISSION_DENY.
|
|
||||||
+ // The reason for the retry is that when you do a SFS mount,
|
|
||||||
+ // the network may not have been created yet, because
|
|
||||||
+ // creating the network and startup container is asynchronous
|
|
||||||
+ break
|
|
||||||
+ }
|
|
||||||
+ logrus.Infof("mounting nfs:%s to %s, get error: %s,will retry", source, dest, string(ee.Stderr))
|
|
||||||
+ }
|
|
||||||
+
|
|
||||||
+ elapsed := time.Now().Unix() - timeStart
|
|
||||||
+ if elapsed < MAX_MOUTN_TIMEOUT {
|
|
||||||
+ time.Sleep(50 * time.Microsecond)
|
|
||||||
+ continue
|
|
||||||
+ }
|
|
||||||
+ }
|
|
||||||
+ break
|
|
||||||
+ }
|
|
||||||
+
|
|
||||||
+ return err
|
|
||||||
+}
|
|
||||||
+
|
|
||||||
+func MountRemoteNfs(source string, opt []string, sandboxId string) error {
|
|
||||||
+ item := strings.Split(source, ":")
|
|
||||||
+ if len(item) != 2 {
|
|
||||||
+ return fmt.Errorf("the nfs of %s format is error", source)
|
|
||||||
+ }
|
|
||||||
+ tmpDes := filepath.Join(kataGuestStorageDir, sandboxId, item[0],item[1])
|
|
||||||
+ if mounted, err := mount.Mounted(tmpDes); err == nil && mounted == true {
|
|
||||||
+ return nil
|
|
||||||
+ }
|
|
||||||
+ err := os.MkdirAll(tmpDes, 0750)
|
|
||||||
+ if err != nil {
|
|
||||||
+ logrus.Infof("mkdir %s failed before mount remote nfs server", tmpDes)
|
|
||||||
+ return err
|
|
||||||
+ }
|
|
||||||
+ defer func() {
|
|
||||||
+ if err != nil {
|
|
||||||
+ os.RemoveAll(tmpDes)
|
|
||||||
+ }
|
|
||||||
+ }()
|
|
||||||
+
|
|
||||||
+ err = nfsMountWithAttempt(source, tmpDes, strings.Join(opt, ","))
|
|
||||||
+
|
|
||||||
+ return err
|
|
||||||
+}
|
|
||||||
--
|
|
||||||
2.14.3 (Apple Git-98)
|
|
||||||
|
|
||||||
@ -1,92 +0,0 @@
|
|||||||
From 79bafc5fd8a1dcda6f44ecd830dfe50f4ef1b34b Mon Sep 17 00:00:00 2001
|
|
||||||
From: holyfei <yangfeiyu20092010@163.com>
|
|
||||||
Date: Tue, 18 Aug 2020 20:41:30 +0800
|
|
||||||
Subject: [PATCH 10/16] storage: mount nfs and gpath in agent
|
|
||||||
|
|
||||||
reason: add nfsStorageHandler and gpathStorageHandler in
|
|
||||||
storageHandlerList to mount nfs and gpath
|
|
||||||
|
|
||||||
Signed-off-by: yangfeiyu <yangfeiyu2@huawei.com>
|
|
||||||
---
|
|
||||||
device.go | 2 ++
|
|
||||||
mount.go | 37 +++++++++++++++++++++++++++++++++++++
|
|
||||||
2 files changed, 39 insertions(+)
|
|
||||||
|
|
||||||
diff --git a/device.go b/device.go
|
|
||||||
index 8e6950c..29f72e9 100644
|
|
||||||
--- a/device.go
|
|
||||||
+++ b/device.go
|
|
||||||
@@ -35,6 +35,8 @@ const (
|
|
||||||
driverEphemeralType = "ephemeral"
|
|
||||||
driverLocalType = "local"
|
|
||||||
vmRootfs = "/"
|
|
||||||
+ driverNfsType = "nfs"
|
|
||||||
+ driverGpathType = "gpath"
|
|
||||||
)
|
|
||||||
|
|
||||||
const (
|
|
||||||
diff --git a/mount.go b/mount.go
|
|
||||||
index a05d4a6..de2bfaf 100644
|
|
||||||
--- a/mount.go
|
|
||||||
+++ b/mount.go
|
|
||||||
@@ -17,6 +17,7 @@ import (
|
|
||||||
"strings"
|
|
||||||
"syscall"
|
|
||||||
|
|
||||||
+ rmtStorage "github.com/kata-containers/agent/pkg/storage"
|
|
||||||
pb "github.com/kata-containers/agent/protocols/grpc"
|
|
||||||
"github.com/pkg/errors"
|
|
||||||
"github.com/sirupsen/logrus"
|
|
||||||
@@ -218,6 +219,8 @@ var storageHandlerList = map[string]storageHandler{
|
|
||||||
driverEphemeralType: ephemeralStorageHandler,
|
|
||||||
driverLocalType: localStorageHandler,
|
|
||||||
driverNvdimmType: nvdimmStorageHandler,
|
|
||||||
+ driverNfsType: nfsStorageHandler,
|
|
||||||
+ driverGpathType: gpathStorageHandler,
|
|
||||||
}
|
|
||||||
|
|
||||||
func ephemeralStorageHandler(_ context.Context, storage pb.Storage, s *sandbox) (string, error) {
|
|
||||||
@@ -339,6 +342,40 @@ func nvdimmStorageHandler(_ context.Context, storage pb.Storage, s *sandbox) (st
|
|
||||||
return "", fmt.Errorf("invalid nvdimm source path: %v", storage.Source)
|
|
||||||
}
|
|
||||||
|
|
||||||
+func nfsStorageHandler(_ context.Context, storage pb.Storage, s *sandbox) (string, error) {
|
|
||||||
+ s.Lock()
|
|
||||||
+ defer s.Unlock()
|
|
||||||
+
|
|
||||||
+ // mount nfs
|
|
||||||
+ err := rmtStorage.MountRemoteNfs(storage.Source, storage.Options, s.id)
|
|
||||||
+ if err != nil {
|
|
||||||
+ return "", fmt.Errorf("mount %s to %s failed, get err:%s", storage.Source, storage.MountPoint, err)
|
|
||||||
+ }
|
|
||||||
+ if err := os.MkdirAll(storage.MountPoint, 0750); err != nil {
|
|
||||||
+ logrus.Infof("mkdir %s failed after mount remote nfs server", storage.MountPoint)
|
|
||||||
+ return "", err
|
|
||||||
+ }
|
|
||||||
+
|
|
||||||
+ defer func() {
|
|
||||||
+ if err != nil {
|
|
||||||
+ os.RemoveAll(storage.MountPoint)
|
|
||||||
+ }
|
|
||||||
+ }()
|
|
||||||
+
|
|
||||||
+ return storage.MountPoint, nil
|
|
||||||
+}
|
|
||||||
+
|
|
||||||
+func gpathStorageHandler(_ context.Context, storage pb.Storage, s *sandbox) (string, error) {
|
|
||||||
+ s.Lock()
|
|
||||||
+ defer s.Unlock()
|
|
||||||
+ // validate guespath
|
|
||||||
+ err := rmtStorage.ValidateGpath(storage.Source, storage.Options)
|
|
||||||
+ if err != nil {
|
|
||||||
+ return "", err
|
|
||||||
+ }
|
|
||||||
+ return "", nil
|
|
||||||
+}
|
|
||||||
+
|
|
||||||
// virtioSCSIStorageHandler handles the storage for scsi driver.
|
|
||||||
func virtioSCSIStorageHandler(ctx context.Context, storage pb.Storage, s *sandbox) (string, error) {
|
|
||||||
// Retrieve the device path from SCSI address.
|
|
||||||
--
|
|
||||||
2.14.3 (Apple Git-98)
|
|
||||||
|
|
||||||
@ -1,78 +0,0 @@
|
|||||||
From 3ac1232a2e3fbfc0465473e5d81cde41847c4252 Mon Sep 17 00:00:00 2001
|
|
||||||
From: jiangpengfei <jiangpengfei9@huawei.com>
|
|
||||||
Date: Wed, 19 Aug 2020 11:47:37 +0800
|
|
||||||
Subject: [PATCH 11/16] agent: fix agent reap agent process blocked problem
|
|
||||||
|
|
||||||
reason: add container waitProcess() timeout when
|
|
||||||
container process status is D/T.
|
|
||||||
|
|
||||||
Signed-off-by: jiangpengfei <jiangpengfei9@huawei.com>
|
|
||||||
---
|
|
||||||
grpc.go | 43 +++++++++++++++++++++++++++++++++----------
|
|
||||||
1 file changed, 33 insertions(+), 10 deletions(-)
|
|
||||||
|
|
||||||
diff --git a/grpc.go b/grpc.go
|
|
||||||
index de2cae7..3dd088e 100644
|
|
||||||
--- a/grpc.go
|
|
||||||
+++ b/grpc.go
|
|
||||||
@@ -49,6 +49,11 @@ const (
|
|
||||||
libcontainerPath = "/run/libcontainer"
|
|
||||||
)
|
|
||||||
|
|
||||||
+// keep waitProcessTimeout value same as value in kata-runtime wait WaitProcessRequest response
|
|
||||||
+const (
|
|
||||||
+ waitProcessTimeOut = 10
|
|
||||||
+)
|
|
||||||
+
|
|
||||||
var (
|
|
||||||
sysfsCPUOnlinePath = "/sys/devices/system/cpu"
|
|
||||||
sysfsMemOnlinePath = "/sys/devices/system/memory"
|
|
||||||
@@ -996,17 +1001,35 @@ func (a *agentGRPC) WaitProcess(ctx context.Context, req *pb.WaitProcessRequest)
|
|
||||||
ctr.deleteProcess(proc.id)
|
|
||||||
})
|
|
||||||
|
|
||||||
- // Using helper function wait() to deal with the subreaper.
|
|
||||||
- libContProcess := (*reaperLibcontainerProcess)(&(proc.process))
|
|
||||||
- exitCode, err := a.sandbox.subreaper.wait(proc.exitCodeCh, libContProcess)
|
|
||||||
- if err != nil {
|
|
||||||
- return &pb.WaitProcessResponse{}, err
|
|
||||||
+ done := make(chan error)
|
|
||||||
+ var exitCode int = 0
|
|
||||||
+ go func() {
|
|
||||||
+ // Using helper function wait() to deal with the subreaper.
|
|
||||||
+ libContProcess := (*reaperLibcontainerProcess)(&(proc.process))
|
|
||||||
+ var err error
|
|
||||||
+ exitCode, err = a.sandbox.subreaper.wait(proc.exitCodeCh, libContProcess)
|
|
||||||
+ if err != nil {
|
|
||||||
+ done <- err
|
|
||||||
+ close(done)
|
|
||||||
+ return
|
|
||||||
+ }
|
|
||||||
+ // refill the exitCodeCh with the exitcode which can be read out
|
|
||||||
+ // by another WaitProcess(). Since this channel isn't be closed,
|
|
||||||
+ // here the refill will always success and it will be free by GC
|
|
||||||
+ // once the process exits.
|
|
||||||
+ proc.exitCodeCh <- exitCode
|
|
||||||
+
|
|
||||||
+ close(done)
|
|
||||||
+ }()
|
|
||||||
+
|
|
||||||
+ select {
|
|
||||||
+ case err := <-done:
|
|
||||||
+ if err != nil {
|
|
||||||
+ return &pb.WaitProcessResponse{}, err
|
|
||||||
+ }
|
|
||||||
+ case <-time.After(time.Duration(waitProcessTimeOut) * time.Second):
|
|
||||||
+ return &pb.WaitProcessResponse{}, grpcStatus.Errorf(codes.DeadlineExceeded, "agent wait reap container process timeout reached after %ds", waitProcessTimeOut)
|
|
||||||
}
|
|
||||||
- //refill the exitCodeCh with the exitcode which can be read out
|
|
||||||
- //by another WaitProcess(). Since this channel isn't be closed,
|
|
||||||
- //here the refill will always success and it will be free by GC
|
|
||||||
- //once the process exits.
|
|
||||||
- proc.exitCodeCh <- exitCode
|
|
||||||
|
|
||||||
return &pb.WaitProcessResponse{
|
|
||||||
Status: int32(exitCode),
|
|
||||||
--
|
|
||||||
2.14.3 (Apple Git-98)
|
|
||||||
|
|
||||||
@ -1,38 +0,0 @@
|
|||||||
From edb29dfd8f786735763245b3f156b50fd3c1a08e Mon Sep 17 00:00:00 2001
|
|
||||||
From: holyfei <yangfeiyu20092010@163.com>
|
|
||||||
Date: Wed, 19 Aug 2020 15:15:31 +0800
|
|
||||||
Subject: [PATCH 12/16] network: support set dns without nameserver
|
|
||||||
|
|
||||||
reason: when runtime sends dns without nameserver to agent,
|
|
||||||
add nameserver before ip address. scenario like annotation
|
|
||||||
with dns
|
|
||||||
|
|
||||||
Signed-off-by: yangfeiyu <yangfeiyu2@huawei.com>
|
|
||||||
---
|
|
||||||
network.go | 5 ++++-
|
|
||||||
1 file changed, 4 insertions(+), 1 deletion(-)
|
|
||||||
|
|
||||||
diff --git a/network.go b/network.go
|
|
||||||
index 02e28cb..7046cf8 100644
|
|
||||||
--- a/network.go
|
|
||||||
+++ b/network.go
|
|
||||||
@@ -708,6 +708,9 @@ func setupDNS(dns []string) (err error) {
|
|
||||||
defer file.Close()
|
|
||||||
|
|
||||||
for i, line := range dns {
|
|
||||||
+ if !strings.Contains(line, "nameserver") {
|
|
||||||
+ line = "nameserver" + " " + line
|
|
||||||
+ }
|
|
||||||
if i == (len(dns) - 1) {
|
|
||||||
_, err = file.WriteString(strings.TrimSpace(line))
|
|
||||||
} else {
|
|
||||||
@@ -761,4 +764,4 @@ func (s *sandbox) handleLocalhost() error {
|
|
||||||
}
|
|
||||||
|
|
||||||
return netlink.LinkSetUp(lo)
|
|
||||||
-}
|
|
||||||
\ No newline at end of file
|
|
||||||
+}
|
|
||||||
--
|
|
||||||
2.14.3 (Apple Git-98)
|
|
||||||
|
|
||||||
@ -1,304 +0,0 @@
|
|||||||
From 1394dcf579849e5d8103c31556e9af0216a875d2 Mon Sep 17 00:00:00 2001
|
|
||||||
From: jiangpengfei <jiangpengfei9@huawei.com>
|
|
||||||
Date: Wed, 19 Aug 2020 17:15:51 +0800
|
|
||||||
Subject: [PATCH 13/16] agent: support setting multi queues of interface
|
|
||||||
|
|
||||||
reason: support setting multi queues of a interface
|
|
||||||
when runtime passing Queue in the request.
|
|
||||||
|
|
||||||
Signed-off-by: jiangpengfei <jiangpengfei9@huawei.com>
|
|
||||||
---
|
|
||||||
network.go | 12 +++++-
|
|
||||||
pkg/net/ethtool.go | 110 ++++++++++++++++++++++++++++++++++++++++++++++++++
|
|
||||||
pkg/types/types.pb.go | 82 ++++++++++++++++++++++++++-----------
|
|
||||||
pkg/types/types.proto | 1 +
|
|
||||||
4 files changed, 181 insertions(+), 24 deletions(-)
|
|
||||||
create mode 100644 pkg/net/ethtool.go
|
|
||||||
|
|
||||||
diff --git a/network.go b/network.go
|
|
||||||
index 7046cf8..1baaa2e 100644
|
|
||||||
--- a/network.go
|
|
||||||
+++ b/network.go
|
|
||||||
@@ -17,7 +17,7 @@ import (
|
|
||||||
"syscall"
|
|
||||||
|
|
||||||
"golang.org/x/sys/unix"
|
|
||||||
-
|
|
||||||
+ agentNet "github.com/kata-containers/agent/pkg/net"
|
|
||||||
"github.com/kata-containers/agent/pkg/types"
|
|
||||||
pb "github.com/kata-containers/agent/protocols/grpc"
|
|
||||||
"github.com/sirupsen/logrus"
|
|
||||||
@@ -273,6 +273,16 @@ func (s *sandbox) updateInterface(netHandle *netlink.Handle, iface *types.Interf
|
|
||||||
if err == nil {
|
|
||||||
err = retErr
|
|
||||||
}
|
|
||||||
+
|
|
||||||
+ // if link is up, then set the multi queue to it,
|
|
||||||
+ // the kernel of newer version may set multi queue by itself,
|
|
||||||
+ // but we can not rely on it.
|
|
||||||
+ if err == nil && iface.Queues > 0 {
|
|
||||||
+ if ethErr := agentNet.GetEthtool().SetChannel(iface.Name, iface.Queues); ethErr != nil {
|
|
||||||
+ err = grpcStatus.Errorf(codes.Internal, "Could not set multi queue %d for interface %v: %v",
|
|
||||||
+ iface.Queues, link, ethErr)
|
|
||||||
+ }
|
|
||||||
+ }
|
|
||||||
}()
|
|
||||||
|
|
||||||
fieldLogger.WithField("link", fmt.Sprintf("%+v", link)).Info("Link found")
|
|
||||||
diff --git a/pkg/net/ethtool.go b/pkg/net/ethtool.go
|
|
||||||
new file mode 100644
|
|
||||||
index 0000000..56a1ece
|
|
||||||
--- /dev/null
|
|
||||||
+++ b/pkg/net/ethtool.go
|
|
||||||
@@ -0,0 +1,110 @@
|
|
||||||
+/*
|
|
||||||
+Copyright (c) Huawei Technologies Co., Ltd. 2019. All rights reserved.
|
|
||||||
+SPDX-License-Identifier: Apache-2.0
|
|
||||||
+Description: common functions
|
|
||||||
+Author: fengshaobao
|
|
||||||
+Create: 2019-05-28
|
|
||||||
+*/
|
|
||||||
+
|
|
||||||
+package net
|
|
||||||
+
|
|
||||||
+import (
|
|
||||||
+ "fmt"
|
|
||||||
+ "sync"
|
|
||||||
+ "syscall"
|
|
||||||
+ "unsafe"
|
|
||||||
+
|
|
||||||
+ "github.com/sirupsen/logrus"
|
|
||||||
+)
|
|
||||||
+
|
|
||||||
+const (
|
|
||||||
+ ethtoolGChannels = 0x0000003c /* Get no of channels */
|
|
||||||
+ ethtoolSChannels = 0x0000003d /* Set no of channels */
|
|
||||||
+ ifNameSize = 16
|
|
||||||
+ siocEthtool = 0x8946
|
|
||||||
+)
|
|
||||||
+
|
|
||||||
+var (
|
|
||||||
+ t *Ethtool
|
|
||||||
+ once sync.Once
|
|
||||||
+)
|
|
||||||
+
|
|
||||||
+// Ethtool to set multiqueue of a network interface,
|
|
||||||
+// with the public method "SetChannel" to set queues of an interface.
|
|
||||||
+type Ethtool struct {
|
|
||||||
+ fd int
|
|
||||||
+}
|
|
||||||
+
|
|
||||||
+type ifReq struct {
|
|
||||||
+ ifrName [ifNameSize]byte
|
|
||||||
+ ifrData uintptr
|
|
||||||
+}
|
|
||||||
+
|
|
||||||
+type ethtoolChannels struct {
|
|
||||||
+ // ETHTOOL_{G,S}CHANNELS
|
|
||||||
+ cmd uint32
|
|
||||||
+ // Read only. Maximum number of receive channel the driver support.
|
|
||||||
+ maxRx uint32
|
|
||||||
+ // Read only. Maximum number of transmit channel the driver support
|
|
||||||
+ maxTx uint32
|
|
||||||
+ // Read only. Maximum number of other channel the driver support
|
|
||||||
+ maxOther uint32
|
|
||||||
+ //Read only. Maximum number of combined channel the driver support. Set of queues RX, TX or other
|
|
||||||
+ maxCombined uint32
|
|
||||||
+ // Valid values are in the range 1 to the max_rx
|
|
||||||
+ rxCount uint32
|
|
||||||
+ // Valid values are in the range 1 to the max_tx
|
|
||||||
+ txCount uint32
|
|
||||||
+ // Valid values are in the range 1 to the max_other
|
|
||||||
+ otherCount uint32
|
|
||||||
+ // Valid values are in the range 1 to the max_combined
|
|
||||||
+ combinedCount uint32
|
|
||||||
+}
|
|
||||||
+
|
|
||||||
+// GetEthtool to config multiqueue of a network interface
|
|
||||||
+func GetEthtool() *Ethtool {
|
|
||||||
+ once.Do(func() {
|
|
||||||
+ var err error
|
|
||||||
+ if t, err = newEthtool(); err != nil {
|
|
||||||
+ panic("can not init a socket fd for ethtool")
|
|
||||||
+ }
|
|
||||||
+ })
|
|
||||||
+ return t
|
|
||||||
+}
|
|
||||||
+
|
|
||||||
+func newEthtool() (*Ethtool, error) {
|
|
||||||
+ fd, err := syscall.Socket(syscall.AF_INET, syscall.SOCK_DGRAM, syscall.IPPROTO_IP)
|
|
||||||
+ if err != nil || fd < 0 {
|
|
||||||
+ logrus.Warningf("Can not get socket of inet")
|
|
||||||
+ var newErr error
|
|
||||||
+ fd, newErr = syscall.Socket(syscall.AF_NETLINK, syscall.SOCK_RAW, syscall.NETLINK_GENERIC)
|
|
||||||
+ if newErr != nil || fd < 0 {
|
|
||||||
+ return nil, fmt.Errorf("create inet socket with error: %v, and create netlink socket with error: %v", err, newErr)
|
|
||||||
+ }
|
|
||||||
+ }
|
|
||||||
+
|
|
||||||
+ return &Ethtool{
|
|
||||||
+ fd: int(fd),
|
|
||||||
+ }, nil
|
|
||||||
+}
|
|
||||||
+
|
|
||||||
+// SetChannel Set the queues of a network interface.
|
|
||||||
+// @devName: the network interface name, e.g. eth0.
|
|
||||||
+// @queues: queues for the network interface.
|
|
||||||
+func (e *Ethtool) SetChannel(devName string, queues uint32) error {
|
|
||||||
+ c := ðtoolChannels{
|
|
||||||
+ cmd: ethtoolSChannels,
|
|
||||||
+ combinedCount: queues,
|
|
||||||
+ }
|
|
||||||
+ var ifName [ifNameSize]byte
|
|
||||||
+ copy(ifName[:], []byte(devName))
|
|
||||||
+ ifr := ifReq{
|
|
||||||
+ ifrName: ifName,
|
|
||||||
+ ifrData: uintptr(unsafe.Pointer(&c)),
|
|
||||||
+ }
|
|
||||||
+ _, _, ep := syscall.Syscall(syscall.SYS_IOCTL, uintptr(e.fd), siocEthtool, uintptr(unsafe.Pointer(&ifr)))
|
|
||||||
+ if ep != 0 {
|
|
||||||
+ return syscall.Errno(ep)
|
|
||||||
+ }
|
|
||||||
+ return nil
|
|
||||||
+}
|
|
||||||
diff --git a/pkg/types/types.pb.go b/pkg/types/types.pb.go
|
|
||||||
index 7ea63e3..8b7e2a5 100644
|
|
||||||
--- a/pkg/types/types.pb.go
|
|
||||||
+++ b/pkg/types/types.pb.go
|
|
||||||
@@ -100,6 +100,7 @@ type Interface struct {
|
|
||||||
// list: "veth", "macvtap", "vlan", "macvlan", "tap", ...
|
|
||||||
Type string `protobuf:"bytes,7,opt,name=type,proto3" json:"type,omitempty"`
|
|
||||||
RawFlags uint32 `protobuf:"varint,8,opt,name=raw_flags,json=rawFlags,proto3" json:"raw_flags,omitempty"`
|
|
||||||
+ Queues uint32 `protobuf:"varint,9,opt,name=Queues,proto3" json:"Queues,omitempty"`
|
|
||||||
}
|
|
||||||
|
|
||||||
func (m *Interface) Reset() { *m = Interface{} }
|
|
||||||
@@ -163,6 +164,13 @@ func (m *Interface) GetRawFlags() uint32 {
|
|
||||||
return 0
|
|
||||||
}
|
|
||||||
|
|
||||||
+func (m *Interface) GetQueues() uint32 {
|
|
||||||
+ if m != nil {
|
|
||||||
+ return m.Queues
|
|
||||||
+ }
|
|
||||||
+ return 0
|
|
||||||
+}
|
|
||||||
+
|
|
||||||
type Route struct {
|
|
||||||
Dest string `protobuf:"bytes,1,opt,name=dest,proto3" json:"dest,omitempty"`
|
|
||||||
Gateway string `protobuf:"bytes,2,opt,name=gateway,proto3" json:"gateway,omitempty"`
|
|
||||||
@@ -319,6 +327,11 @@ func (m *Interface) MarshalTo(dAtA []byte) (int, error) {
|
|
||||||
i++
|
|
||||||
i = encodeVarintTypes(dAtA, i, uint64(m.RawFlags))
|
|
||||||
}
|
|
||||||
+ if m.Queues != 0 {
|
|
||||||
+ dAtA[i] = 0x48
|
|
||||||
+ i++
|
|
||||||
+ i = encodeVarintTypes(dAtA, i, uint64(m.Queues))
|
|
||||||
+ }
|
|
||||||
return i, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
@@ -430,6 +443,9 @@ func (m *Interface) Size() (n int) {
|
|
||||||
if m.RawFlags != 0 {
|
|
||||||
n += 1 + sovTypes(uint64(m.RawFlags))
|
|
||||||
}
|
|
||||||
+ if m.Queues != 0 {
|
|
||||||
+ n += 1 + sovTypes(uint64(m.Queues))
|
|
||||||
+ }
|
|
||||||
return n
|
|
||||||
}
|
|
||||||
|
|
||||||
@@ -841,6 +857,25 @@ func (m *Interface) Unmarshal(dAtA []byte) error {
|
|
||||||
break
|
|
||||||
}
|
|
||||||
}
|
|
||||||
+ case 9:
|
|
||||||
+ if wireType != 0 {
|
|
||||||
+ return fmt.Errorf("proto: wrong wireType = %d for field Queues", wireType)
|
|
||||||
+ }
|
|
||||||
+ m.Queues = 0
|
|
||||||
+ for shift := uint(0); ; shift += 7 {
|
|
||||||
+ if shift >= 64 {
|
|
||||||
+ return ErrIntOverflowTypes
|
|
||||||
+ }
|
|
||||||
+ if iNdEx >= l {
|
|
||||||
+ return io.ErrUnexpectedEOF
|
|
||||||
+ }
|
|
||||||
+ b := dAtA[iNdEx]
|
|
||||||
+ iNdEx++
|
|
||||||
+ m.Queues |= (uint32(b) & 0x7F) << shift
|
|
||||||
+ if b < 0x80 {
|
|
||||||
+ break
|
|
||||||
+ }
|
|
||||||
+ }
|
|
||||||
default:
|
|
||||||
iNdEx = preIndex
|
|
||||||
skippy, err := skipTypes(dAtA[iNdEx:])
|
|
||||||
@@ -1155,27 +1190,28 @@ var (
|
|
||||||
func init() { proto.RegisterFile("pkg/types/types.proto", fileDescriptorTypes) }
|
|
||||||
|
|
||||||
var fileDescriptorTypes = []byte{
|
|
||||||
- // 345 bytes of a gzipped FileDescriptorProto
|
|
||||||
- 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x54, 0x91, 0xd1, 0x6a, 0xfa, 0x30,
|
|
||||||
- 0x14, 0xc6, 0xff, 0xb1, 0xb6, 0xda, 0xe3, 0xdf, 0xad, 0x84, 0x4d, 0xc2, 0x06, 0x52, 0xbc, 0x59,
|
|
||||||
- 0xd9, 0x85, 0x03, 0x37, 0x76, 0xbf, 0x5d, 0x08, 0xde, 0x49, 0x5e, 0x60, 0xc4, 0x36, 0x3a, 0xd1,
|
|
||||||
- 0xda, 0xd2, 0x44, 0x8b, 0xec, 0x05, 0x77, 0xb9, 0x47, 0x10, 0x9f, 0x64, 0xe4, 0x24, 0x8a, 0xbb,
|
|
||||||
- 0xd1, 0xef, 0x97, 0x93, 0xd3, 0xef, 0x7c, 0x27, 0x70, 0x5b, 0xae, 0x16, 0x4f, 0x7a, 0x5f, 0x4a,
|
|
||||||
- 0x65, 0x7f, 0x87, 0x65, 0x55, 0xe8, 0x82, 0xfa, 0x08, 0x83, 0x19, 0x84, 0x93, 0xe9, 0x5b, 0x96,
|
|
||||||
- 0x55, 0x52, 0x29, 0xfa, 0x00, 0xc1, 0x5c, 0xe4, 0xcb, 0xf5, 0x9e, 0x91, 0x98, 0x24, 0x57, 0xa3,
|
|
||||||
- 0xeb, 0xa1, 0xed, 0x98, 0x4c, 0xc7, 0x78, 0xcc, 0x5d, 0x99, 0x32, 0x68, 0x09, 0xdb, 0xc3, 0x1a,
|
|
||||||
- 0x31, 0x49, 0x42, 0x7e, 0x42, 0x4a, 0xa1, 0x99, 0x0b, 0xb5, 0x62, 0x1e, 0x1e, 0xa3, 0x1e, 0x1c,
|
|
||||||
- 0x08, 0x84, 0x93, 0x8d, 0x96, 0xd5, 0x5c, 0xa4, 0x92, 0xf6, 0x20, 0xc8, 0xe4, 0x6e, 0x99, 0x4a,
|
|
||||||
- 0x34, 0x09, 0xb9, 0x23, 0xd3, 0xb9, 0x11, 0xb9, 0x74, 0x1f, 0x44, 0x4d, 0x47, 0xd0, 0x39, 0x4f,
|
|
||||||
- 0x27, 0x15, 0xf3, 0x62, 0x2f, 0xe9, 0x8c, 0xa2, 0xf3, 0x54, 0xae, 0xc2, 0x2f, 0x2f, 0xd1, 0x08,
|
|
||||||
- 0xbc, 0x5c, 0x6f, 0x59, 0x33, 0x26, 0x49, 0x93, 0x1b, 0x69, 0x1c, 0x3f, 0x6b, 0x73, 0x81, 0xf9,
|
|
||||||
- 0xd6, 0xd1, 0x92, 0x49, 0x51, 0xa6, 0x4b, 0x2c, 0x04, 0x36, 0x85, 0x43, 0x33, 0x8b, 0xf1, 0x60,
|
|
||||||
- 0x2d, 0x3b, 0x8b, 0xd1, 0xf4, 0x1e, 0xc2, 0x4a, 0xd4, 0x1f, 0xf3, 0xb5, 0x58, 0x28, 0xd6, 0x8e,
|
|
||||||
- 0x49, 0xd2, 0xe5, 0xed, 0x4a, 0xd4, 0x63, 0xc3, 0x83, 0x2f, 0xf0, 0x79, 0xb1, 0xd5, 0x98, 0x22,
|
|
||||||
- 0x93, 0x4a, 0xbb, 0x6c, 0xa8, 0x8d, 0xcf, 0x42, 0x68, 0x59, 0x8b, 0xfd, 0x69, 0x5b, 0x0e, 0x2f,
|
|
||||||
- 0x76, 0xe1, 0xfd, 0xd9, 0x45, 0x0f, 0x02, 0x55, 0x6c, 0xab, 0x54, 0x62, 0x8c, 0x90, 0x3b, 0xa2,
|
|
||||||
- 0x37, 0xe0, 0xab, 0xb4, 0x28, 0x25, 0x06, 0xe9, 0x72, 0x0b, 0x8f, 0x77, 0xd0, 0x3e, 0xbd, 0x10,
|
|
||||||
- 0x0d, 0xa0, 0xb1, 0x7b, 0x89, 0xfe, 0xe1, 0xff, 0x6b, 0x44, 0xde, 0xff, 0x7f, 0x1f, 0xfb, 0xe4,
|
|
||||||
- 0xe7, 0xd8, 0x27, 0x87, 0x63, 0x9f, 0xcc, 0x02, 0x7c, 0xfb, 0xe7, 0xdf, 0x00, 0x00, 0x00, 0xff,
|
|
||||||
- 0xff, 0xb5, 0x52, 0x37, 0xf2, 0x14, 0x02, 0x00, 0x00,
|
|
||||||
+ // 356 bytes of a gzipped FileDescriptorProto
|
|
||||||
+ 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x54, 0x92, 0xc1, 0x4e, 0xc2, 0x40,
|
|
||||||
+ 0x10, 0x86, 0x5d, 0x4a, 0x0b, 0x1d, 0x44, 0x9b, 0x8d, 0x92, 0x8d, 0x26, 0xa4, 0xe1, 0x62, 0xe3,
|
|
||||||
+ 0x01, 0x13, 0x34, 0xde, 0xf5, 0x40, 0xc2, 0x0d, 0xf7, 0x05, 0xcc, 0xd2, 0x2e, 0x48, 0xa0, 0xb4,
|
|
||||||
+ 0xe9, 0xb6, 0x34, 0xc4, 0x17, 0xf4, 0xe8, 0x23, 0x18, 0x9e, 0xc2, 0xa3, 0xd9, 0xd9, 0x85, 0xe0,
|
|
||||||
+ 0x05, 0xfe, 0x6f, 0x67, 0xa7, 0xff, 0xfc, 0xd3, 0xc2, 0x75, 0xbe, 0x5a, 0x3c, 0x94, 0xbb, 0x5c,
|
|
||||||
+ 0x2a, 0xf3, 0x3b, 0xcc, 0x8b, 0xac, 0xcc, 0xa8, 0x8b, 0x30, 0x98, 0x81, 0x3f, 0x99, 0xbe, 0x24,
|
|
||||||
+ 0x49, 0x21, 0x95, 0xa2, 0x77, 0xe0, 0xcd, 0x45, 0xba, 0x5c, 0xef, 0x18, 0x09, 0x49, 0x74, 0x31,
|
|
||||||
+ 0xba, 0x1c, 0x9a, 0x8e, 0xc9, 0x74, 0x8c, 0xc7, 0xdc, 0x96, 0x29, 0x83, 0x96, 0x30, 0x3d, 0xac,
|
|
||||||
+ 0x11, 0x92, 0xc8, 0xe7, 0x07, 0xa4, 0x14, 0x9a, 0xa9, 0x50, 0x2b, 0xe6, 0xe0, 0x31, 0xea, 0xc1,
|
|
||||||
+ 0x2f, 0x01, 0x7f, 0xb2, 0x29, 0x65, 0x31, 0x17, 0xb1, 0xa4, 0x3d, 0xf0, 0x12, 0xb9, 0x5d, 0xc6,
|
|
||||||
+ 0x12, 0x4d, 0x7c, 0x6e, 0x49, 0x77, 0x6e, 0x44, 0x2a, 0xed, 0x03, 0x51, 0xd3, 0x11, 0x74, 0x8e,
|
|
||||||
+ 0xd3, 0x49, 0xc5, 0x9c, 0xd0, 0x89, 0x3a, 0xa3, 0xe0, 0x38, 0x95, 0xad, 0xf0, 0xd3, 0x4b, 0x34,
|
|
||||||
+ 0x00, 0x27, 0x2d, 0x2b, 0xd6, 0x0c, 0x49, 0xd4, 0xe4, 0x5a, 0x6a, 0xc7, 0x8f, 0x5a, 0x5f, 0x60,
|
|
||||||
+ 0xae, 0x71, 0x34, 0xa4, 0x53, 0xe4, 0xf1, 0x12, 0x0b, 0x9e, 0x49, 0x61, 0x51, 0xcf, 0xa2, 0x3d,
|
|
||||||
+ 0x58, 0xcb, 0xcc, 0xa2, 0x35, 0xbd, 0x05, 0xbf, 0x10, 0xf5, 0xfb, 0x7c, 0x2d, 0x16, 0x8a, 0xb5,
|
|
||||||
+ 0x43, 0x12, 0x75, 0x79, 0xbb, 0x10, 0xf5, 0x58, 0xb3, 0xb6, 0x78, 0xab, 0x64, 0x25, 0x15, 0xf3,
|
|
||||||
+ 0xb1, 0x62, 0x69, 0xf0, 0x09, 0x2e, 0xcf, 0xaa, 0x12, 0xd3, 0x25, 0x52, 0x95, 0x36, 0x33, 0x6a,
|
|
||||||
+ 0xed, 0xbf, 0x10, 0xa5, 0xac, 0xc5, 0xee, 0xb0, 0x45, 0x8b, 0x27, 0x3b, 0x72, 0xfe, 0xed, 0xa8,
|
|
||||||
+ 0x07, 0x9e, 0xca, 0xaa, 0x22, 0x96, 0x18, 0xcf, 0xe7, 0x96, 0xe8, 0x15, 0xb8, 0x2a, 0xce, 0x72,
|
|
||||||
+ 0x89, 0x01, 0xbb, 0xdc, 0xc0, 0xfd, 0x0d, 0xb4, 0x0f, 0x6f, 0x8e, 0x7a, 0xd0, 0xd8, 0x3e, 0x05,
|
|
||||||
+ 0x67, 0xf8, 0xff, 0x1c, 0x90, 0xd7, 0xf3, 0xaf, 0x7d, 0x9f, 0x7c, 0xef, 0xfb, 0xe4, 0x67, 0xdf,
|
|
||||||
+ 0x27, 0x33, 0x0f, 0xbf, 0x89, 0xc7, 0xbf, 0x00, 0x00, 0x00, 0xff, 0xff, 0xa5, 0x57, 0x70, 0xc8,
|
|
||||||
+ 0x2c, 0x02, 0x00, 0x00,
|
|
||||||
}
|
|
||||||
diff --git a/pkg/types/types.proto b/pkg/types/types.proto
|
|
||||||
index f6856e1..149df13 100644
|
|
||||||
--- a/pkg/types/types.proto
|
|
||||||
+++ b/pkg/types/types.proto
|
|
||||||
@@ -37,6 +37,7 @@ message Interface {
|
|
||||||
// list: "veth", "macvtap", "vlan", "macvlan", "tap", ...
|
|
||||||
string type = 7;
|
|
||||||
uint32 raw_flags = 8;
|
|
||||||
+ uint32 Queues = 9;
|
|
||||||
}
|
|
||||||
|
|
||||||
message Route {
|
|
||||||
--
|
|
||||||
2.14.3 (Apple Git-98)
|
|
||||||
|
|
||||||
@ -1,56 +0,0 @@
|
|||||||
From 9e1478d7989fea4ee759cc13009d8de07dac2879 Mon Sep 17 00:00:00 2001
|
|
||||||
From: jiangpengfei <jiangpengfei9@huawei.com>
|
|
||||||
Date: Wed, 19 Aug 2020 17:01:00 +0800
|
|
||||||
Subject: [PATCH 14/16] agent: fix init hugepages failed problem
|
|
||||||
|
|
||||||
reason: fix init hugepages failed problem
|
|
||||||
|
|
||||||
Signed-off-by: jiangpengfei <jiangpengfei9@huawei.com>
|
|
||||||
---
|
|
||||||
.../runc/libcontainer/cgroups/fs/apply_raw.go | 20 ++++++++++++++++++++
|
|
||||||
1 file changed, 20 insertions(+)
|
|
||||||
|
|
||||||
diff --git a/vendor/github.com/opencontainers/runc/libcontainer/cgroups/fs/apply_raw.go b/vendor/github.com/opencontainers/runc/libcontainer/cgroups/fs/apply_raw.go
|
|
||||||
index ec148b4..47aa3c3 100644
|
|
||||||
--- a/vendor/github.com/opencontainers/runc/libcontainer/cgroups/fs/apply_raw.go
|
|
||||||
+++ b/vendor/github.com/opencontainers/runc/libcontainer/cgroups/fs/apply_raw.go
|
|
||||||
@@ -8,11 +8,13 @@ import (
|
|
||||||
"os"
|
|
||||||
"path/filepath"
|
|
||||||
"sync"
|
|
||||||
+ "time"
|
|
||||||
|
|
||||||
"github.com/opencontainers/runc/libcontainer/cgroups"
|
|
||||||
"github.com/opencontainers/runc/libcontainer/configs"
|
|
||||||
libcontainerUtils "github.com/opencontainers/runc/libcontainer/utils"
|
|
||||||
"github.com/pkg/errors"
|
|
||||||
+ "github.com/sirupsen/logrus"
|
|
||||||
"golang.org/x/sys/unix"
|
|
||||||
)
|
|
||||||
|
|
||||||
@@ -409,3 +411,21 @@ func CheckCpushares(path string, c uint64) error {
|
|
||||||
func (m *Manager) GetCgroups() (*configs.Cgroup, error) {
|
|
||||||
return m.Cgroups, nil
|
|
||||||
}
|
|
||||||
+
|
|
||||||
+func init() {
|
|
||||||
+ go func() {
|
|
||||||
+ var err error
|
|
||||||
+ if len(HugePageSizes) != 0 {
|
|
||||||
+ return
|
|
||||||
+ }
|
|
||||||
+ for i := 0; i < 10; i++ {
|
|
||||||
+ HugePageSizes, err = cgroups.GetHugePageSize()
|
|
||||||
+ if err == nil {
|
|
||||||
+ logrus.Infof("init hugepages ok loop=%d %v", i, err)
|
|
||||||
+ return
|
|
||||||
+ }
|
|
||||||
+ logrus.Errorf("init hugepages failed loop=%d %v", i, err)
|
|
||||||
+ time.Sleep(time.Second)
|
|
||||||
+ }
|
|
||||||
+ }()
|
|
||||||
+}
|
|
||||||
\ No newline at end of file
|
|
||||||
--
|
|
||||||
2.14.3 (Apple Git-98)
|
|
||||||
|
|
||||||
File diff suppressed because it is too large
Load Diff
@ -1,268 +0,0 @@
|
|||||||
From 6120525f81701424e97d453d515d38f14bbe99d9 Mon Sep 17 00:00:00 2001
|
|
||||||
From: holyfei <yangfeiyu20092010@163.com>
|
|
||||||
Date: Wed, 19 Aug 2020 20:25:00 +0800
|
|
||||||
Subject: [PATCH 16/16] clock: synchronizes clock info with proxy
|
|
||||||
|
|
||||||
reason: virtual machine's clock may be incorrect, proxy synchronizes
|
|
||||||
clock info to help virtual machine adjust clock time
|
|
||||||
|
|
||||||
Signed-off-by: yangfeiyu <yangfeiyu2@huawei.com>
|
|
||||||
---
|
|
||||||
pkg/clock/clock_util.go | 47 ++++++++++++++++++++++++
|
|
||||||
sync_clock_server.go | 94 +++++++++++++++++++++++++++++++++++++++++++++++
|
|
||||||
sync_clock_server_test.go | 88 ++++++++++++++++++++++++++++++++++++++++++++
|
|
||||||
3 files changed, 229 insertions(+)
|
|
||||||
create mode 100644 pkg/clock/clock_util.go
|
|
||||||
create mode 100644 sync_clock_server.go
|
|
||||||
create mode 100644 sync_clock_server_test.go
|
|
||||||
|
|
||||||
diff --git a/pkg/clock/clock_util.go b/pkg/clock/clock_util.go
|
|
||||||
new file mode 100644
|
|
||||||
index 0000000..4b78c63
|
|
||||||
--- /dev/null
|
|
||||||
+++ b/pkg/clock/clock_util.go
|
|
||||||
@@ -0,0 +1,47 @@
|
|
||||||
+// Copyright (c) Huawei Technologies Co., Ltd. 2018. All rights reserved.
|
|
||||||
+// SPDX-License-Identifier: Apache-2.0
|
|
||||||
+// Description: sync clock related function
|
|
||||||
+// Author: xueshaojia
|
|
||||||
+// Create: 2018-11-10
|
|
||||||
+
|
|
||||||
+// Package clock provides clock sync functions
|
|
||||||
+package clock
|
|
||||||
+
|
|
||||||
+import (
|
|
||||||
+ "net"
|
|
||||||
+ "syscall"
|
|
||||||
+)
|
|
||||||
+
|
|
||||||
+type TimeValue struct {
|
|
||||||
+ ClientSendTime int64 `json:"client_send_time"`
|
|
||||||
+ ClientArriveTime int64 `json:"client_arrive_time"`
|
|
||||||
+ ServerSendTime int64 `json:"server_send_time"`
|
|
||||||
+ ServerArriveTime int64 `json:"server_arrive_time"`
|
|
||||||
+ Delta int64 `json:"delta"`
|
|
||||||
+}
|
|
||||||
+
|
|
||||||
+const MaxSyncClockByteNum = 400 // sync clock byte num, max=400
|
|
||||||
+const MaxTimeStampSupport = 3155731199000000000 // unit is ns, 2069/12/31 23:59:59
|
|
||||||
+const MinTimeStampSupport = 0 // unit is ns, 1970/1/1 8:0:0
|
|
||||||
+
|
|
||||||
+// getCurrentTimeNs returns UTC time in Ns
|
|
||||||
+func GetCurrentTimeNs() int64 {
|
|
||||||
+ var tv syscall.Timeval
|
|
||||||
+ if err := syscall.Gettimeofday(&tv); err != nil {
|
|
||||||
+ return -1
|
|
||||||
+ }
|
|
||||||
+ return tv.Sec*1000000000 + tv.Usec*1000
|
|
||||||
+}
|
|
||||||
+
|
|
||||||
+// readConnData reads data from stream
|
|
||||||
+func ReadConnData(stream net.Conn) (buf []byte, byteNum int, err error) {
|
|
||||||
+ buf = make([]byte, MaxSyncClockByteNum)
|
|
||||||
+ byteNum, err = stream.Read(buf)
|
|
||||||
+ return buf, byteNum, err
|
|
||||||
+}
|
|
||||||
+
|
|
||||||
+// writeConnData writes data to stream
|
|
||||||
+func WriteConnData(stream net.Conn, buf []byte) error {
|
|
||||||
+ _, err := stream.Write(buf)
|
|
||||||
+ return err
|
|
||||||
+}
|
|
||||||
diff --git a/sync_clock_server.go b/sync_clock_server.go
|
|
||||||
new file mode 100644
|
|
||||||
index 0000000..a8e98c6
|
|
||||||
--- /dev/null
|
|
||||||
+++ b/sync_clock_server.go
|
|
||||||
@@ -0,0 +1,94 @@
|
|
||||||
+// Copyright (c) Huawei Technologies Co., Ltd. 2018. All rights reserved.
|
|
||||||
+// SPDX-License-Identifier: Apache-2.0
|
|
||||||
+// Description: sync clock server related function
|
|
||||||
+// Author: xueshaojia
|
|
||||||
+// Create: 2018-11-10
|
|
||||||
+
|
|
||||||
+package main
|
|
||||||
+
|
|
||||||
+import (
|
|
||||||
+ "encoding/json"
|
|
||||||
+ "fmt"
|
|
||||||
+ "io"
|
|
||||||
+ "net"
|
|
||||||
+ "os/exec"
|
|
||||||
+ "syscall"
|
|
||||||
+
|
|
||||||
+ "github.com/kata-containers/agent/pkg/clock"
|
|
||||||
+)
|
|
||||||
+
|
|
||||||
+// getSyncClockInfo waits for client request
|
|
||||||
+func getSyncClockInfo(stream net.Conn, clockInfo *clock.TimeValue) error {
|
|
||||||
+ buf, byteNum, err := clock.ReadConnData(stream)
|
|
||||||
+ if err != nil {
|
|
||||||
+ return err
|
|
||||||
+ }
|
|
||||||
+ return json.Unmarshal(buf[:byteNum], clockInfo)
|
|
||||||
+}
|
|
||||||
+
|
|
||||||
+// ackSyncClockInfo sends clock info to client
|
|
||||||
+func ackSyncClockInfo(stream net.Conn, clockInfo *clock.TimeValue) error {
|
|
||||||
+ b, err := json.Marshal(clockInfo)
|
|
||||||
+ if err != nil {
|
|
||||||
+ return err
|
|
||||||
+ }
|
|
||||||
+ return clock.WriteConnData(stream, b)
|
|
||||||
+}
|
|
||||||
+
|
|
||||||
+// setGuestClock sets systemtime and hwclock
|
|
||||||
+func setGuestClock(delta int64) error {
|
|
||||||
+ dstClockNs := clock.GetCurrentTimeNs() + delta
|
|
||||||
+ if dstClockNs < clock.MinTimeStampSupport || dstClockNs > clock.MaxTimeStampSupport {
|
|
||||||
+ return fmt.Errorf("sync clock failed, the valid timestamp is from 1970/1/1 8:0:0 to 2069/12/31 23:59:59")
|
|
||||||
+ }
|
|
||||||
+ dstTimeVal := syscall.NsecToTimeval(dstClockNs)
|
|
||||||
+ if err := syscall.Settimeofday(&dstTimeVal); err != nil {
|
|
||||||
+ agentLog.WithError(err).Error("sync clock, set systemtime failed")
|
|
||||||
+ return err
|
|
||||||
+ }
|
|
||||||
+ cmd := exec.Command("hwclock", "-w")
|
|
||||||
+ if err := cmd.Run(); err != nil {
|
|
||||||
+ agentLog.WithError(err).Error("sync clock, set hwclock failed")
|
|
||||||
+ return err
|
|
||||||
+ }
|
|
||||||
+ agentLog.Infof("sync clock, set systemtime and hwclock succ, delta:%d", delta)
|
|
||||||
+ return nil
|
|
||||||
+}
|
|
||||||
+
|
|
||||||
+// SyncClock is a loop that deals with client's request
|
|
||||||
+// 1 read client request
|
|
||||||
+// 2 response with clock info or set guest clock
|
|
||||||
+// 3 if error happends, print error info
|
|
||||||
+func SyncClock(stream net.Conn) {
|
|
||||||
+ for {
|
|
||||||
+ var clockInfo clock.TimeValue
|
|
||||||
+ if err := getSyncClockInfo(stream, &clockInfo); err != nil {
|
|
||||||
+ if err == io.EOF {
|
|
||||||
+ agentLog.WithError(err).Error("sync clock, yamux session error happends")
|
|
||||||
+ stream.Close()
|
|
||||||
+ break
|
|
||||||
+ }
|
|
||||||
+ agentLog.WithError(err).Error("sync clock, read sync clock info failed")
|
|
||||||
+ continue
|
|
||||||
+ }
|
|
||||||
+ // set client's request arrive time
|
|
||||||
+ clockInfo.ClientArriveTime = clock.GetCurrentTimeNs()
|
|
||||||
+ if clockInfo.Delta == 0 {
|
|
||||||
+ // this is a request for clock diff info
|
|
||||||
+ // set server's response send time
|
|
||||||
+ // if fails, wait for next time
|
|
||||||
+ clockInfo.ServerSendTime = clock.GetCurrentTimeNs()
|
|
||||||
+ if err := ackSyncClockInfo(stream, &clockInfo); err != nil {
|
|
||||||
+ agentLog.WithError(err).Error("sync clock, send back clock info failed")
|
|
||||||
+ }
|
|
||||||
+ } else {
|
|
||||||
+ // this is a request for adjusting guest clock
|
|
||||||
+ // 1 adjust guest clock(system time and hwclock)
|
|
||||||
+ // 2 do not send response any more
|
|
||||||
+ // 3 if fails, wait for next request to adjust
|
|
||||||
+ if err := setGuestClock(clockInfo.Delta); err != nil {
|
|
||||||
+ agentLog.WithError(err).Error("sync clock, set local clock failed")
|
|
||||||
+ }
|
|
||||||
+ }
|
|
||||||
+ }
|
|
||||||
+}
|
|
||||||
diff --git a/sync_clock_server_test.go b/sync_clock_server_test.go
|
|
||||||
new file mode 100644
|
|
||||||
index 0000000..41d5227
|
|
||||||
--- /dev/null
|
|
||||||
+++ b/sync_clock_server_test.go
|
|
||||||
@@ -0,0 +1,88 @@
|
|
||||||
+// Copyright (c) Huawei Technologies Co., Ltd. 2018. All rights reserved.
|
|
||||||
+// SPDX-License-Identifier: Apache-2.0
|
|
||||||
+// Description: sync clock server related test
|
|
||||||
+// Author: xueshaojia
|
|
||||||
+// Create: 2018-11-10
|
|
||||||
+
|
|
||||||
+package main
|
|
||||||
+
|
|
||||||
+import (
|
|
||||||
+ "encoding/json"
|
|
||||||
+ "fmt"
|
|
||||||
+ "math/rand"
|
|
||||||
+ "net"
|
|
||||||
+ "testing"
|
|
||||||
+
|
|
||||||
+ "github.com/hashicorp/yamux"
|
|
||||||
+ "github.com/kata-containers/agent/pkg/clock"
|
|
||||||
+)
|
|
||||||
+
|
|
||||||
+var clientStream net.Conn
|
|
||||||
+
|
|
||||||
+func TestGetCurrentTimeNs(t *testing.T) {
|
|
||||||
+ timeRet := clock.GetCurrentTimeNs()
|
|
||||||
+ if timeRet <= 0 {
|
|
||||||
+ t.Fatalf("TestGetCurrentTimeNs Fail, time:%d", timeRet)
|
|
||||||
+ }
|
|
||||||
+}
|
|
||||||
+
|
|
||||||
+func TestSetGuestClock(t *testing.T) {
|
|
||||||
+ if err := setGuestClock(0); err != nil {
|
|
||||||
+ t.Fatalf("test set clock failed, err:%v", err)
|
|
||||||
+ }
|
|
||||||
+}
|
|
||||||
+
|
|
||||||
+func TestReadWriteData(t *testing.T) {
|
|
||||||
+ sock := GenSocket()
|
|
||||||
+ var err error
|
|
||||||
+ listener, err := net.Listen("unix", sock)
|
|
||||||
+ if err != nil {
|
|
||||||
+ t.Fatalf("listening err: %v", err)
|
|
||||||
+ }
|
|
||||||
+ go SetUpClient(sock)
|
|
||||||
+ conn, err := listener.Accept()
|
|
||||||
+ if err != nil {
|
|
||||||
+ t.Fatalf("accept err: %v", err)
|
|
||||||
+ }
|
|
||||||
+ session, err := yamux.Server(conn, nil)
|
|
||||||
+ if err != nil {
|
|
||||||
+ t.Fatalf("session err: %v", err)
|
|
||||||
+ }
|
|
||||||
+ stream, err := session.Accept()
|
|
||||||
+ if err != nil {
|
|
||||||
+ t.Fatalf("stream err: %v", err)
|
|
||||||
+ }
|
|
||||||
+ go func() {
|
|
||||||
+ var clockInfo clock.TimeValue = clock.TimeValue{1000, 0, 0, 0, 0}
|
|
||||||
+ b, _ := json.Marshal(clockInfo)
|
|
||||||
+ clock.WriteConnData(clientStream, b)
|
|
||||||
+ }()
|
|
||||||
+ _, _, err = clock.ReadConnData(stream)
|
|
||||||
+ if err != nil {
|
|
||||||
+ t.Fatalf("read conn data error, err:%v", err)
|
|
||||||
+ }
|
|
||||||
+}
|
|
||||||
+
|
|
||||||
+func SetUpClient(sock string) error {
|
|
||||||
+ conn, err := net.Dial("unix", sock)
|
|
||||||
+ if err != nil {
|
|
||||||
+ return err
|
|
||||||
+ }
|
|
||||||
+ session, err := yamux.Client(conn, nil)
|
|
||||||
+ if err != nil {
|
|
||||||
+ conn.Close()
|
|
||||||
+ return err
|
|
||||||
+ }
|
|
||||||
+ stream, err := session.Open()
|
|
||||||
+ if err != nil {
|
|
||||||
+ return err
|
|
||||||
+ }
|
|
||||||
+ clientStream = stream
|
|
||||||
+ return nil
|
|
||||||
+}
|
|
||||||
+
|
|
||||||
+func GenSocket() string {
|
|
||||||
+ randSeed := clock.GetCurrentTimeNs()
|
|
||||||
+ rand.Seed(randSeed)
|
|
||||||
+ return fmt.Sprintf("/tmp/%d.sock", rand.Uint32())
|
|
||||||
+}
|
|
||||||
--
|
|
||||||
2.14.3 (Apple Git-98)
|
|
||||||
|
|
||||||
File diff suppressed because it is too large
Load Diff
@ -1,184 +0,0 @@
|
|||||||
From 629aac1078bdeed63214c58d2110fe672d654774 Mon Sep 17 00:00:00 2001
|
|
||||||
From: yangfeiyu <yangfeiyu2@huawei.com>
|
|
||||||
Date: Mon, 26 Oct 2020 20:39:05 +0800
|
|
||||||
Subject: [PATCH] kata-agent: update nic in guest
|
|
||||||
|
|
||||||
reason: add linkByName and support retry of list ip link, because in some scenarios,
|
|
||||||
the hardware address cannot be queried immediately after the hot plug.
|
|
||||||
|
|
||||||
Signed-off-by: yangfeiyu <yangfeiyu2@huawei.com>
|
|
||||||
---
|
|
||||||
network.go | 91 ++++++++++++++++++++++++++++++++++++++++++------------
|
|
||||||
1 file changed, 72 insertions(+), 19 deletions(-)
|
|
||||||
|
|
||||||
diff --git a/network.go b/network.go
|
|
||||||
index f6e2c17..d928e2b 100644
|
|
||||||
--- a/network.go
|
|
||||||
+++ b/network.go
|
|
||||||
@@ -15,13 +15,14 @@ import (
|
|
||||||
"strings"
|
|
||||||
"sync"
|
|
||||||
"syscall"
|
|
||||||
+ "time"
|
|
||||||
|
|
||||||
- "golang.org/x/sys/unix"
|
|
||||||
agentNet "github.com/kata-containers/agent/pkg/net"
|
|
||||||
"github.com/kata-containers/agent/pkg/types"
|
|
||||||
pb "github.com/kata-containers/agent/protocols/grpc"
|
|
||||||
"github.com/sirupsen/logrus"
|
|
||||||
"github.com/vishvananda/netlink"
|
|
||||||
+ "golang.org/x/sys/unix"
|
|
||||||
"google.golang.org/grpc/codes"
|
|
||||||
grpcStatus "google.golang.org/grpc/status"
|
|
||||||
)
|
|
||||||
@@ -32,6 +33,7 @@ var (
|
|
||||||
errNoLink = grpcStatus.Errorf(codes.InvalidArgument, "Need network link")
|
|
||||||
errNoMAC = grpcStatus.Errorf(codes.InvalidArgument, "Need hardware address")
|
|
||||||
errNoRoutes = grpcStatus.Errorf(codes.InvalidArgument, "Need network routes")
|
|
||||||
+ errNoName = grpcStatus.Errorf(codes.InvalidArgument, "Need network name")
|
|
||||||
guestDNSFile = "/etc/resolv.conf"
|
|
||||||
kataGuestSandboxDNSFile = "/run/kata-containers/sandbox/resolv.conf"
|
|
||||||
)
|
|
||||||
@@ -46,6 +48,8 @@ const (
|
|
||||||
|
|
||||||
// Use the below address for ipv6 gateway once ipv6 support is added
|
|
||||||
// defaultV6RouteIP = "::"
|
|
||||||
+
|
|
||||||
+ maxLinkRetries = 10
|
|
||||||
)
|
|
||||||
|
|
||||||
// Network fully describes a sandbox network with its interfaces, routes and dns
|
|
||||||
@@ -100,17 +104,36 @@ func linkByHwAddr(netHandle *netlink.Handle, hwAddr string) (netlink.Link, error
|
|
||||||
return nil, grpcStatus.Errorf(codes.NotFound, "Could not find the link corresponding to HwAddr %q", hwAddr)
|
|
||||||
}
|
|
||||||
|
|
||||||
-func updateLink(netHandle *netlink.Handle, link netlink.Link, iface *types.Interface) error {
|
|
||||||
+func linkByName(netHandle *netlink.Handle, name string) (netlink.Link, error) {
|
|
||||||
if netHandle == nil {
|
|
||||||
- return errNoHandle
|
|
||||||
+ return nil, errNoHandle
|
|
||||||
}
|
|
||||||
|
|
||||||
- if link == nil {
|
|
||||||
- return errNoLink
|
|
||||||
+ if name == "" {
|
|
||||||
+ return nil, errNoName
|
|
||||||
}
|
|
||||||
|
|
||||||
- if iface == nil {
|
|
||||||
- return errNoIF
|
|
||||||
+ links, err := netHandle.LinkList()
|
|
||||||
+ if err != nil {
|
|
||||||
+ return nil, err
|
|
||||||
+ }
|
|
||||||
+
|
|
||||||
+ for _, link := range links {
|
|
||||||
+ if link == nil {
|
|
||||||
+ continue
|
|
||||||
+ }
|
|
||||||
+
|
|
||||||
+ if link.Attrs() != nil && link.Attrs().Name == name {
|
|
||||||
+ return link, nil
|
|
||||||
+ }
|
|
||||||
+ }
|
|
||||||
+
|
|
||||||
+ return nil, grpcStatus.Errorf(codes.NotFound, "Could not find the link corresponding to name %s", name)
|
|
||||||
+}
|
|
||||||
+
|
|
||||||
+func updateLinkIP(netHandle *netlink.Handle, link netlink.Link, iface *types.Interface) error {
|
|
||||||
+ if len(iface.IPAddresses) == 0 {
|
|
||||||
+ return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// As a first step, clear out any existing addresses associated with the link:
|
|
||||||
@@ -129,13 +152,6 @@ func updateLink(netHandle *netlink.Handle, link netlink.Link, iface *types.Inter
|
|
||||||
netlinkAddrStr := fmt.Sprintf("%s/%s", addr.Address, addr.Mask)
|
|
||||||
netlinkAddr, err := netlink.ParseAddr(netlinkAddrStr)
|
|
||||||
|
|
||||||
- // With ipv6 addresses, there is a brief period during which the address is marked as "tentative"
|
|
||||||
- // making it unavailable. A process called duplicate address detection(DAD) is performed during this period.
|
|
||||||
- // Disble DAD so that networking is available once the container is up. The assumption is
|
|
||||||
- // that it is the reponsibility of the upper stack to make sure the addresses assigned to containers
|
|
||||||
- // do not conflict. A similar operation is performed by libnetwork:
|
|
||||||
- // https://github.com/moby/moby/issues/18871
|
|
||||||
-
|
|
||||||
if addr.GetFamily() == types.IPFamily_v6 {
|
|
||||||
netlinkAddr.Flags = netlinkAddr.Flags | syscall.IFA_F_NODAD
|
|
||||||
}
|
|
||||||
@@ -150,14 +166,36 @@ func updateLink(netHandle *netlink.Handle, link netlink.Link, iface *types.Inter
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
+ return nil
|
|
||||||
+}
|
|
||||||
+
|
|
||||||
+func updateLink(netHandle *netlink.Handle, link netlink.Link, iface *types.Interface) error {
|
|
||||||
+ if netHandle == nil {
|
|
||||||
+ return errNoHandle
|
|
||||||
+ }
|
|
||||||
+
|
|
||||||
+ if link == nil {
|
|
||||||
+ return errNoLink
|
|
||||||
+ }
|
|
||||||
+
|
|
||||||
+ if iface == nil {
|
|
||||||
+ return errNoIF
|
|
||||||
+ }
|
|
||||||
+
|
|
||||||
+ if err := updateLinkIP(netHandle, link, iface); err != nil {
|
|
||||||
+ return err
|
|
||||||
+ }
|
|
||||||
+
|
|
||||||
// set the interface name:
|
|
||||||
if err := netHandle.LinkSetName(link, iface.Name); err != nil {
|
|
||||||
return grpcStatus.Errorf(codes.Internal, "Could not set name %s for interface %v: %v", iface.Name, link, err)
|
|
||||||
}
|
|
||||||
|
|
||||||
// set the interface MTU:
|
|
||||||
- if err := netHandle.LinkSetMTU(link, int(iface.Mtu)); err != nil {
|
|
||||||
- return grpcStatus.Errorf(codes.Internal, "Could not set MTU %d for interface %v: %v", iface.Mtu, link, err)
|
|
||||||
+ if iface.Mtu > 0 {
|
|
||||||
+ if err := netHandle.LinkSetMTU(link, int(iface.Mtu)); err != nil {
|
|
||||||
+ return grpcStatus.Errorf(codes.Internal, "Could not set MTU %d for interface %v: %v", iface.Mtu, link, err)
|
|
||||||
+ }
|
|
||||||
}
|
|
||||||
|
|
||||||
if iface.RawFlags&unix.IFF_NOARP == uint32(unix.IFF_NOARP) {
|
|
||||||
@@ -306,8 +344,23 @@ func (s *sandbox) updateInterface(netHandle *netlink.Handle, iface *types.Interf
|
|
||||||
if iface.HwAddr != "" {
|
|
||||||
fieldLogger.Info("Getting interface from MAC address")
|
|
||||||
|
|
||||||
- // Find the interface link from its hardware address.
|
|
||||||
- link, err = linkByHwAddr(netHandle, iface.HwAddr)
|
|
||||||
+ // In some scenarios, the hardware address cannot be queried immediately
|
|
||||||
+ // after the hot plug, add a retry here.
|
|
||||||
+ for retry := 0; retry < maxLinkRetries; retry++ {
|
|
||||||
+ // Find the interface link from its hardware address.
|
|
||||||
+ link, err = linkByHwAddr(netHandle, iface.HwAddr)
|
|
||||||
+ if err != nil {
|
|
||||||
+ time.Sleep(100 * time.Millisecond)
|
|
||||||
+ continue
|
|
||||||
+ }
|
|
||||||
+ break
|
|
||||||
+ }
|
|
||||||
+ if err != nil {
|
|
||||||
+ return nil, grpcStatus.Errorf(codes.Internal, "updateInterface: %v", err)
|
|
||||||
+ }
|
|
||||||
+ } else if iface.Name != "" {
|
|
||||||
+ fieldLogger.Info("Getting interface from name")
|
|
||||||
+ link, err = linkByName(netHandle, iface.Name)
|
|
||||||
if err != nil {
|
|
||||||
return nil, grpcStatus.Errorf(codes.Internal, "updateInterface: %v", err)
|
|
||||||
}
|
|
||||||
@@ -487,7 +540,7 @@ func (s *sandbox) updateRoutes(netHandle *netlink.Handle, requestedRoutes *pb.Ro
|
|
||||||
}()
|
|
||||||
|
|
||||||
var (
|
|
||||||
- added []*types.Route
|
|
||||||
+ added []*types.Route
|
|
||||||
removed []*types.Route
|
|
||||||
)
|
|
||||||
|
|
||||||
--
|
|
||||||
2.23.0
|
|
||||||
|
|
||||||
@ -1,18 +0,0 @@
|
|||||||
0001-agent-add-agent.netlink_recv_buf_size-flag-to-set-ne.patch
|
|
||||||
0002-network-support-update-routes-incrementally.patch
|
|
||||||
0003-kata-agent-add-kata-ipvs-command.patch
|
|
||||||
0004-agent-add-IPVS-test.patch
|
|
||||||
0005-mount-support-mount-block-device.patch
|
|
||||||
0006-agent-make-workaround-for-slow-response-in-aarch64.patch
|
|
||||||
0007-agent-using-pcie-root-port-driver-to-hotplug-device.patch
|
|
||||||
0008-agent-support-get-root-bus-path-dynamically.patch
|
|
||||||
0009-storage-add-pkg-storage-for-mount.patch
|
|
||||||
0010-storage-mount-nfs-and-gpath-in-agent.patch
|
|
||||||
0011-agent-fix-agent-reap-agent-process-blocked-problem.patch
|
|
||||||
0012-network-support-set-dns-without-nameserver.patch
|
|
||||||
0013-agent-support-setting-multi-queues-of-interface.patch
|
|
||||||
0014-agent-fix-init-hugepages-failed-problem.patch
|
|
||||||
0015-agent-add-support-of-getting-container-s-network-sta.patch
|
|
||||||
0016-clock-synchronizes-clock-info-with-proxy.patch
|
|
||||||
0017-agent-add-support-of-new-sandbox-StratoVirt.patch
|
|
||||||
0018-kata-agent-update-nic-in-guest.patch
|
|
||||||
@ -1,20 +1,20 @@
|
|||||||
#!/bin/bash
|
#!/bin/bash
|
||||||
|
|
||||||
|
set -e
|
||||||
|
|
||||||
if [[ -f ./patch_flag ]];then
|
if [[ -f ./patch_flag ]];then
|
||||||
echo "shim patched!"
|
echo "patched!"
|
||||||
exit 0
|
exit 0
|
||||||
fi
|
fi
|
||||||
|
|
||||||
tar -zxvf shim-*.tar.gz
|
tar -zxvf kata-containers-2.1.0.tar.gz
|
||||||
cp -fr ./shim-*/* ./
|
cp -rf ./kata-containers-2.1.0/* ./
|
||||||
rm -rf ./shim-*
|
|
||||||
cat ./series.conf | while read line
|
cat ./series.conf | while read line
|
||||||
do
|
do
|
||||||
if [[ $line == '' || $line =~ ^\s*# ]]; then
|
if [[ $line == '' || $line =~ ^\s*# ]]; then
|
||||||
continue
|
continue
|
||||||
fi
|
fi
|
||||||
echo "====patch $line======"
|
echo "====patch $line======"
|
||||||
pwd
|
|
||||||
patch -p1 -F1 -s < ./patches/$line
|
patch -p1 -F1 -s < ./patches/$line
|
||||||
done
|
done
|
||||||
|
|
||||||
Binary file not shown.
@ -1,8 +1,8 @@
|
|||||||
#needsrootforbuild
|
#needsrootforbuild
|
||||||
%global debug_package %{nil}
|
%global debug_package %{nil}
|
||||||
|
|
||||||
%define VERSION v1.11.1
|
%define VERSION 2.1.0
|
||||||
%define RELEASE 12
|
%define RELEASE 29
|
||||||
|
|
||||||
Name: kata-containers
|
Name: kata-containers
|
||||||
Version: %{VERSION}
|
Version: %{VERSION}
|
||||||
@ -10,13 +10,13 @@ Release: %{RELEASE}
|
|||||||
Summary: Kata Container, the speed of containers, the security of VMs
|
Summary: Kata Container, the speed of containers, the security of VMs
|
||||||
License: ISC and Apache-2.0 and GPL-2.0-or-later and CC-BY-SA-4.0 and MIT and MPL-2.0
|
License: ISC and Apache-2.0 and GPL-2.0-or-later and CC-BY-SA-4.0 and MIT and MPL-2.0
|
||||||
URL: https://github.com/kata-containers
|
URL: https://github.com/kata-containers
|
||||||
Source0: kata_integration-v1.0.0.tar.gz
|
Source0: kata_integration-openeuler.tar.gz
|
||||||
Source1: kata-containers-%{version}.tar.gz
|
Source1: kata-containers-openeuler.tar.gz
|
||||||
Source2: kernel.tar.gz
|
Source2: kernel.tar.gz
|
||||||
|
|
||||||
BuildRoot: %_topdir/BUILDROOT
|
BuildRoot: %_topdir/BUILDROOT
|
||||||
BuildRequires: automake golang gcc bc glibc-devel glibc-static busybox glib2-devel glib2 ipvsadm conntrack-tools nfs-utils
|
BuildRequires: automake golang gcc bc glibc-devel glibc-static busybox glib2-devel glib2 ipvsadm conntrack-tools nfs-utils
|
||||||
BuildRequires: patch elfutils-libelf-devel openssl-devel bison flex
|
BuildRequires: patch elfutils-libelf-devel openssl-devel bison flex rust cargo rust-packaging libgcc dtc-devel
|
||||||
|
|
||||||
%description
|
%description
|
||||||
This is core component of Kata Container, to make it work, you need a isulad/docker engine.
|
This is core component of Kata Container, to make it work, you need a isulad/docker engine.
|
||||||
@ -31,14 +31,6 @@ cd %{_builddir}/kata_integration
|
|||||||
# apply kata_integration patches
|
# apply kata_integration patches
|
||||||
sh apply-patches
|
sh apply-patches
|
||||||
|
|
||||||
# mv build components into kata_integration dir
|
|
||||||
pushd %{_builddir}/kata_integration
|
|
||||||
mv ../kata-containers-%{version}/runtime .
|
|
||||||
mv ../kata-containers-%{version}/agent .
|
|
||||||
mv ../kata-containers-%{version}/proxy .
|
|
||||||
mv ../kata-containers-%{version}/shim .
|
|
||||||
popd
|
|
||||||
|
|
||||||
# build kernel
|
# build kernel
|
||||||
cd %{_builddir}/kernel
|
cd %{_builddir}/kernel
|
||||||
mv kernel linux
|
mv kernel linux
|
||||||
@ -53,12 +45,31 @@ cp %{_builddir}/kata_integration/hack/config-kata-arm64 ./.config
|
|||||||
cd %{_builddir}/kernel/linux/
|
cd %{_builddir}/kernel/linux/
|
||||||
make %{?_smp_mflags}
|
make %{?_smp_mflags}
|
||||||
|
|
||||||
|
mv %{_builddir}/kata-containers-%{version} %{_builddir}/kata-containers
|
||||||
|
cd %{_builddir}/kata-containers/
|
||||||
|
sh -x apply-patches
|
||||||
|
cd %{_builddir}/kata-containers/src/runtime
|
||||||
|
make clean
|
||||||
|
make
|
||||||
|
|
||||||
|
cd %{_builddir}/kata-containers/src/agent
|
||||||
|
mkdir vendor && tar -xzf %{_builddir}/kata-containers/vendor.tar.gz -C vendor/
|
||||||
|
cp -f ./vendor/version.rs ./src/
|
||||||
|
cat > .cargo/config << EOF
|
||||||
|
[build]
|
||||||
|
rustflags = ["-Clink-arg=-s","-Clink-arg=-lgcc","-Clink-arg=-lfdt"]
|
||||||
|
|
||||||
|
[source.crates-io]
|
||||||
|
replace-with = "vendored-sources"
|
||||||
|
|
||||||
|
[source.vendored-sources]
|
||||||
|
directory = "vendor"
|
||||||
|
EOF
|
||||||
|
/usr/bin/env CARGO_HOME=.cargo RUSTC_BOOTSTRAP=1 cargo build --release
|
||||||
cd %{_builddir}/kata_integration
|
cd %{_builddir}/kata_integration
|
||||||
mkdir -p -m 750 build
|
mkdir -p -m 750 build
|
||||||
export GO111MODULE=off
|
cp %{_builddir}/kata-containers/src/agent/target/release/kata-agent ./build/
|
||||||
make runtime
|
strip ./build/kata-agent
|
||||||
make proxy
|
|
||||||
make shim
|
|
||||||
make initrd
|
make initrd
|
||||||
|
|
||||||
%install
|
%install
|
||||||
@ -71,34 +82,205 @@ install -p -m 755 -D %{_builddir}/kernel/linux/arch/arm64/boot/Image %{buildroot
|
|||||||
|
|
||||||
cd %{_builddir}/kata_integration
|
cd %{_builddir}/kata_integration
|
||||||
mkdir -p -m 750 %{buildroot}/usr/bin
|
mkdir -p -m 750 %{buildroot}/usr/bin
|
||||||
install -p -m 750 ./build/kata-runtime ./build/kata-proxy ./build/kata-shim ./build/kata-netmon %{buildroot}/usr/bin/
|
install -p -m 750 %{_builddir}/kata-containers/src/runtime/kata-runtime %{buildroot}/usr/bin/
|
||||||
|
install -p -m 750 %{_builddir}/kata-containers/src/runtime/kata-netmon %{buildroot}/usr/bin/
|
||||||
|
install -p -m 750 %{_builddir}/kata-containers/src/runtime/kata-monitor %{buildroot}/usr/bin/
|
||||||
|
install -p -m 750 %{_builddir}/kata-containers/src/runtime/containerd-shim-kata-v2 %{buildroot}/usr/bin/
|
||||||
|
install -p -m 640 -D %{_builddir}/kata-containers/src/runtime/cli/config/configuration-qemu.toml %{buildroot}/usr/share/defaults/kata-containers/configuration.toml
|
||||||
|
install -p -m 640 -D %{_builddir}/kata-containers/src/runtime/cli/config/configuration-stratovirt.toml %{buildroot}/usr/share/defaults/kata-containers/configuration-stratovirt.toml
|
||||||
install -p -m 640 ./build/kata-containers-initrd.img %{buildroot}/var/lib/kata/
|
install -p -m 640 ./build/kata-containers-initrd.img %{buildroot}/var/lib/kata/
|
||||||
mkdir -p -m 750 %{buildroot}/usr/share/defaults/kata-containers/
|
mkdir -p -m 750 %{buildroot}/usr/share/defaults/kata-containers/
|
||||||
install -p -m 640 -D ./runtime/cli/config/configuration-qemu.toml %{buildroot}/usr/share/defaults/kata-containers/configuration.toml
|
strip %{buildroot}/usr/bin/kata*
|
||||||
|
strip %{buildroot}/usr/bin/containerd-shim-kata-v2
|
||||||
|
|
||||||
%clean
|
%clean
|
||||||
|
|
||||||
%files
|
%files
|
||||||
/usr/bin/kata-runtime
|
/usr/bin/kata-runtime
|
||||||
/usr/bin/kata-proxy
|
|
||||||
/usr/bin/kata-shim
|
|
||||||
/usr/bin/kata-netmon
|
/usr/bin/kata-netmon
|
||||||
|
/usr/bin/kata-monitor
|
||||||
|
/usr/bin/containerd-shim-kata-v2
|
||||||
/var/lib/kata/kernel
|
/var/lib/kata/kernel
|
||||||
/var/lib/kata/kata-containers-initrd.img
|
/var/lib/kata/kata-containers-initrd.img
|
||||||
%config(noreplace) /usr/share/defaults/kata-containers/configuration.toml
|
%config(noreplace) /usr/share/defaults/kata-containers/configuration.toml
|
||||||
|
%config(noreplace) /usr/share/defaults/kata-containers/configuration-stratovirt.toml
|
||||||
|
|
||||||
%doc
|
%doc
|
||||||
|
|
||||||
|
|
||||||
%changelog
|
%changelog
|
||||||
* Tue May 17 2022 liukuo <liukuo@kylinos.cn> - v1.11.1-12
|
* Fri Sep 2 2022 chengzeruizhi <chengzeruizhi@huawei.com> - 2.1.0-29
|
||||||
- License compliance rectification
|
|
||||||
|
|
||||||
* Wed Apr 13 2022 yangfeiyu <yangfeiyu2@huawei.com> - 1.11.1-11
|
|
||||||
- Type:bugfix
|
- Type:bugfix
|
||||||
- ID:NA
|
- ID:NA
|
||||||
- SUG:NA
|
- SUG:NA
|
||||||
- DESC:add export GO111MODULE=off
|
- DESC:don't use props for object add
|
||||||
|
|
||||||
|
* Tue Aug 23 2022 chengzeruizhi <chengzeruizhi@huawei.com> - 2.1.0-28
|
||||||
|
- Type:bugfix
|
||||||
|
- ID:NA
|
||||||
|
- SUG:NA
|
||||||
|
- DESC:add explicit on for kernel_irqchip
|
||||||
|
|
||||||
|
* Mon Aug 22 2022 chengzeruizhi <chengzeruizhi@huawei.com> - 2.1.0-27
|
||||||
|
- Type:bugfix
|
||||||
|
- ID:NA
|
||||||
|
- SUG:NA
|
||||||
|
- DESC:use host device when adding block dev
|
||||||
|
|
||||||
|
* Fri Mar 18 2022 Xinle.Guo <guoxinle1@huawei.com> - 2.1.0-26
|
||||||
|
- Type:bugfix
|
||||||
|
- ID:NA
|
||||||
|
- SUG:NA
|
||||||
|
- DESC:fix startup failure that adding more than 16 root port devices in stratovirt
|
||||||
|
|
||||||
|
* Tues Mar 2 2022 Xinle.Guo <guoxinle1@huawei.com> - 2.1.0-25
|
||||||
|
- Type:feature
|
||||||
|
- ID:NA
|
||||||
|
- SUG:NA
|
||||||
|
- DESC:provide a way to dynomically obtain firmware in stratovirt
|
||||||
|
|
||||||
|
* Sat Feb 26 2022 Xinle.Guo <guoxinle1@huawei.com> - 2.1.0-24
|
||||||
|
- Type:bugfix
|
||||||
|
- ID:NA
|
||||||
|
- SUG:NA
|
||||||
|
- DESC:fix the problem that fails to plug net device to stratovirt
|
||||||
|
|
||||||
|
* Mon Jan 17 2022 Xinle.Guo <guoxinle1@huawei.com> - 2.1.0-23
|
||||||
|
- Type:feature
|
||||||
|
- ID:NA
|
||||||
|
- SUG:NA
|
||||||
|
- DESC:add the stratovirt standardVM sandbox type to kata container
|
||||||
|
|
||||||
|
* Thur Jan 13 2022 Xinle.Guo <guoxinle1@huawei.com> - 2.1.0-22
|
||||||
|
- Type:feature
|
||||||
|
- ID:NA
|
||||||
|
- SUG:NA
|
||||||
|
- DESC:refactor hypervisor type `stratovirt` and its methods
|
||||||
|
|
||||||
|
* Tues Jan 11 2022 Xinle.Guo <guoxinle1@huawei.com> - 2.1.0-21
|
||||||
|
- Type:feature
|
||||||
|
- ID:NA
|
||||||
|
- SUG:NA
|
||||||
|
- DESC:add stratovirt `vmConfig` struct and methods to get parameters of VM
|
||||||
|
|
||||||
|
* Mon Jan 10 2022 Xinle.Guo <guoxinle1@huawei.com> - 2.1.0-20
|
||||||
|
- Type:feature
|
||||||
|
- ID:NA
|
||||||
|
- SUG:NA
|
||||||
|
- DESC:update stratovirt configuration toml file
|
||||||
|
|
||||||
|
* Fri Dec 10 2021 yangfeiyu <yangfeiyu2@huawei.com> - 2.1.0-19
|
||||||
|
- Type:bugfix
|
||||||
|
- ID:NA
|
||||||
|
- SUG:NA
|
||||||
|
- DESC:modify stratovirt config file
|
||||||
|
|
||||||
|
* Tue Dec 7 2021 jikui <jikui2@huawei.com> - 2.1.0-18
|
||||||
|
- Type:bugfix
|
||||||
|
- ID:NA
|
||||||
|
- SUG:NA
|
||||||
|
- DESC:fix the block device not removed in devManager
|
||||||
|
|
||||||
|
* Thu Dec 2 2021 jikui <jikui2@huawei.com> - 2.1.0-17
|
||||||
|
- Type:bugfix
|
||||||
|
- ID:NA
|
||||||
|
- SUG:NA
|
||||||
|
- DESC:don't ignore updateInterface return error
|
||||||
|
|
||||||
|
* Tue Nov 30 2021 jikui <jikui2@huawei.com> - 2.1.0-16
|
||||||
|
- Type:bugfix
|
||||||
|
- ID:NA
|
||||||
|
- SUG:NA
|
||||||
|
- DESC:don't delete the exist tap device in the host
|
||||||
|
|
||||||
|
* Tue Nov 30 2021 jikui <jikui2@huawei.com> - 2.1.0-15
|
||||||
|
- Type:bugfix
|
||||||
|
- ID:NA
|
||||||
|
- SUG:NA
|
||||||
|
- DESC:check VFIO when create device
|
||||||
|
|
||||||
|
* Mon Nov 29 2021 jikui <jikui2@huawei.com> - 2.1.0-14
|
||||||
|
- Type:bugfix
|
||||||
|
- ID:NA
|
||||||
|
- SUG:NA
|
||||||
|
- DESC:fix delete sandbox failed problem
|
||||||
|
|
||||||
|
* Sat Nov 27 2021 jikui <jikui2@huawei.com> - 2.1.0-13
|
||||||
|
- Type:bugfix
|
||||||
|
- ID:NA
|
||||||
|
- SUG:NA
|
||||||
|
- DESC:validate sandbox cpu and memory size
|
||||||
|
|
||||||
|
* Thu Nov 25 2021 jikui <jikui2@huawei.com> - 2.1.0-12
|
||||||
|
- Type:bugfix
|
||||||
|
- ID:NA
|
||||||
|
- SUG:NA
|
||||||
|
- DESC:truncate the log.json file before kata-runtime subcommand executed
|
||||||
|
|
||||||
|
* Thu Nov 25 2021 jikui <jikui2@huawei.com> - 2.1.0-11
|
||||||
|
- Type:bugfix
|
||||||
|
- ID:NA
|
||||||
|
- SUG:NA
|
||||||
|
- DESC:fix umount container rootfs dir return invalid argument error
|
||||||
|
|
||||||
|
* Fri Nov 24 2021 jikui <jikui2@huawei.com> - 2.1.0-10
|
||||||
|
- Type:bugfix
|
||||||
|
- ID:NA
|
||||||
|
- SUG:NA
|
||||||
|
- DESC:increase delete cgroup retry times
|
||||||
|
|
||||||
|
* Sat Nov 20 2021 yangfeiyu <yangfeiyu2@huawei.com> - 2.1.0-9
|
||||||
|
- Type:bugfix
|
||||||
|
- ID:NA
|
||||||
|
- SUG:NA
|
||||||
|
- DESC:modify kernel and image path in configuration.toml
|
||||||
|
|
||||||
|
* Tue Oct 16 2021 jikui <jikui2@huawei.com> - 2.1.0-8
|
||||||
|
- Type:bugfix
|
||||||
|
- ID:NA
|
||||||
|
- SUG:NA
|
||||||
|
- DESC:keep the qemu process name same as the configured path
|
||||||
|
|
||||||
|
* Mon Oct 15 2021 jikui <jikui2@huawei.com> - 2.1.0-7
|
||||||
|
- Type:bugfix
|
||||||
|
- ID:NA
|
||||||
|
- SUG:NA
|
||||||
|
- DESC:fix kata-runtime skip read lines in /proc/mounts
|
||||||
|
|
||||||
|
* Fri Oct 5 2021 jikui <jikui2@huawei.com> - 2.1.0-6
|
||||||
|
- Type:bugfix
|
||||||
|
- ID:NA
|
||||||
|
- SUG:NA
|
||||||
|
- DESC:fix kata-runtime hungs when qemu process is D/T
|
||||||
|
|
||||||
|
* Mon Sep 27 2021 yangfeiyu <yangfeiyu2@huawei.com> - 2.1.0-5
|
||||||
|
- Type:enhancement
|
||||||
|
- ID:NA
|
||||||
|
- SUG:NA
|
||||||
|
- DESC:chmod agent exec fifo
|
||||||
|
|
||||||
|
* Fri Sep 17 2021 yangfeiyu <yangfeiyu2@huawei.com> - 2.1.0-4
|
||||||
|
- Type:enhancement
|
||||||
|
- ID:NA
|
||||||
|
- SUG:NA
|
||||||
|
- DESC:modify kata-agent build flags
|
||||||
|
|
||||||
|
* Tue Aug 24 2021 yangfeiyu <yangfeiyu2@huawei.com> - 2.1.0-3
|
||||||
|
- Type:enhancement
|
||||||
|
- ID:NA
|
||||||
|
- SUG:NA
|
||||||
|
- DESC:add configuration-stratovirt.toml
|
||||||
|
|
||||||
|
* Fri Aug 20 2021 yangfeiyu <yangfeiyu2@huawei.com> - 2.1.0-2
|
||||||
|
- Type:enhancement
|
||||||
|
- ID:NA
|
||||||
|
- SUG:NA
|
||||||
|
- DESC:support with stratovirt and isulad
|
||||||
|
|
||||||
|
* Wed Aug 18 2021 yangfeiyu <yangfeiyu2@huawei.com> - 2.1.0-1
|
||||||
|
- Type:enhancement
|
||||||
|
- ID:NA
|
||||||
|
- SUG:NA
|
||||||
|
- DESC:upgrade kata-containers
|
||||||
|
|
||||||
* Fri Feb 19 2021 xinghe <xinghe1@huawei.com> - 1.11.1-10
|
* Fri Feb 19 2021 xinghe <xinghe1@huawei.com> - 1.11.1-10
|
||||||
- Type:CVE
|
- Type:CVE
|
||||||
|
|||||||
@ -0,0 +1,837 @@
|
|||||||
|
From f56d66f196bee808526e86df2c3c063a887c6fef Mon Sep 17 00:00:00 2001
|
||||||
|
From: Wei Gao <gaowei66@huawei.com>
|
||||||
|
Date: Sat, 7 Aug 2021 10:39:11 +0800
|
||||||
|
Subject: [PATCH 1/6] runtime: add support of new sandbox hypervisor type
|
||||||
|
StratoVirt.
|
||||||
|
|
||||||
|
Signed-off-by: Wei Gao <gaowei66@huawei.com>
|
||||||
|
---
|
||||||
|
src/runtime/pkg/katautils/config.go | 96 +++-
|
||||||
|
src/runtime/virtcontainers/hypervisor.go | 12 +
|
||||||
|
src/runtime/virtcontainers/stratovirt.go | 642 +++++++++++++++++++++++
|
||||||
|
3 files changed, 749 insertions(+), 1 deletion(-)
|
||||||
|
create mode 100644 src/runtime/virtcontainers/stratovirt.go
|
||||||
|
|
||||||
|
diff --git a/src/runtime/pkg/katautils/config.go b/src/runtime/pkg/katautils/config.go
|
||||||
|
index 6114aa39..f94ac4fd 100644
|
||||||
|
--- a/src/runtime/pkg/katautils/config.go
|
||||||
|
+++ b/src/runtime/pkg/katautils/config.go
|
||||||
|
@@ -50,6 +50,7 @@ const (
|
||||||
|
clhHypervisorTableType = "clh"
|
||||||
|
qemuHypervisorTableType = "qemu"
|
||||||
|
acrnHypervisorTableType = "acrn"
|
||||||
|
+ stratovirtHypervisorTable = "stratovirt"
|
||||||
|
|
||||||
|
// the maximum amount of PCI bridges that can be cold plugged in a VM
|
||||||
|
maxPCIBridges uint32 = 5
|
||||||
|
@@ -870,6 +871,96 @@ func newClhHypervisorConfig(h hypervisor) (vc.HypervisorConfig, error) {
|
||||||
|
}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
+func newStratovirtHypervisorConfig(h hypervisor) (vc.HypervisorConfig, error) {
|
||||||
|
+ hypervisor, err := h.path()
|
||||||
|
+ if err != nil {
|
||||||
|
+ return vc.HypervisorConfig{}, err
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ kernel, err := h.kernel()
|
||||||
|
+ if err != nil {
|
||||||
|
+ return vc.HypervisorConfig{}, err
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ initrd, image, err := h.getInitrdAndImage()
|
||||||
|
+ if err != nil {
|
||||||
|
+ return vc.HypervisorConfig{}, err
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ if image != "" && initrd != "" {
|
||||||
|
+ return vc.HypervisorConfig{},
|
||||||
|
+ errors.New("having both an image and an initrd defined in the configuration file is not supported")
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ if image == "" && initrd == "" {
|
||||||
|
+ return vc.HypervisorConfig{},
|
||||||
|
+ errors.New("either image or initrd must be defined in the configuration file")
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ kernelParams := h.kernelParams()
|
||||||
|
+ machineType := h.machineType()
|
||||||
|
+
|
||||||
|
+ blockDriver, err := h.blockDeviceDriver()
|
||||||
|
+ if err != nil {
|
||||||
|
+ return vc.HypervisorConfig{}, err
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ sharedFS, err := h.sharedFS()
|
||||||
|
+ if err != nil {
|
||||||
|
+ return vc.HypervisorConfig{}, err
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ if sharedFS == config.VirtioFS && h.VirtioFSDaemon == "" {
|
||||||
|
+ return vc.HypervisorConfig{},
|
||||||
|
+ errors.New("cannot enable virtio-fs without daemon path in configuration file")
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ if vSock, err := utils.SupportsVsocks(); !vSock {
|
||||||
|
+ return vc.HypervisorConfig{}, err
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ return vc.HypervisorConfig{
|
||||||
|
+ HypervisorPath: hypervisor,
|
||||||
|
+ HypervisorPathList: h.HypervisorPathList,
|
||||||
|
+ KernelPath: kernel,
|
||||||
|
+ InitrdPath: initrd,
|
||||||
|
+ ImagePath: image,
|
||||||
|
+ KernelParams: vc.DeserializeParams(strings.Fields(kernelParams)),
|
||||||
|
+ HypervisorMachineType: machineType,
|
||||||
|
+ NumVCPUs: h.defaultVCPUs(),
|
||||||
|
+ DefaultMaxVCPUs: h.defaultMaxVCPUs(),
|
||||||
|
+ MemorySize: h.defaultMemSz(),
|
||||||
|
+ MemSlots: h.defaultMemSlots(),
|
||||||
|
+ MemOffset: h.defaultMemOffset(),
|
||||||
|
+ EntropySource: h.GetEntropySource(),
|
||||||
|
+ EntropySourceList: h.EntropySourceList,
|
||||||
|
+ DefaultBridges: h.defaultBridges(),
|
||||||
|
+ DisableBlockDeviceUse: h.DisableBlockDeviceUse,
|
||||||
|
+ SharedFS: sharedFS,
|
||||||
|
+ VirtioFSDaemon: h.VirtioFSDaemon,
|
||||||
|
+ VirtioFSDaemonList: h.VirtioFSDaemonList,
|
||||||
|
+ VirtioFSCacheSize: h.VirtioFSCacheSize,
|
||||||
|
+ VirtioFSCache: h.defaultVirtioFSCache(),
|
||||||
|
+ VirtioFSExtraArgs: h.VirtioFSExtraArgs,
|
||||||
|
+ FileBackedMemRootDir: h.FileBackedMemRootDir,
|
||||||
|
+ FileBackedMemRootList: h.FileBackedMemRootList,
|
||||||
|
+ Mlock: !h.Swap,
|
||||||
|
+ Debug: h.Debug,
|
||||||
|
+ DisableNestingChecks: h.DisableNestingChecks,
|
||||||
|
+ BlockDeviceDriver: blockDriver,
|
||||||
|
+ BlockDeviceCacheSet: h.BlockDeviceCacheSet,
|
||||||
|
+ BlockDeviceCacheDirect: h.BlockDeviceCacheDirect,
|
||||||
|
+ BlockDeviceCacheNoflush: h.BlockDeviceCacheNoflush,
|
||||||
|
+ EnableIOThreads: h.EnableIOThreads,
|
||||||
|
+ DisableVhostNet: h.DisableVhostNet,
|
||||||
|
+ EnableVhostUserStore: h.EnableVhostUserStore,
|
||||||
|
+ VhostUserStorePath: h.vhostUserStorePath(),
|
||||||
|
+ VhostUserStorePathList: h.VhostUserStorePathList,
|
||||||
|
+ GuestHookPath: h.guestHookPath(),
|
||||||
|
+ EnableAnnotations: h.EnableAnnotations,
|
||||||
|
+ }, nil
|
||||||
|
+}
|
||||||
|
+
|
||||||
|
func newFactoryConfig(f factory) (oci.FactoryConfig, error) {
|
||||||
|
if f.TemplatePath == "" {
|
||||||
|
f.TemplatePath = defaultTemplatePath
|
||||||
|
@@ -903,6 +994,9 @@ func updateRuntimeConfigHypervisor(configPath string, tomlConf tomlConfig, confi
|
||||||
|
case clhHypervisorTableType:
|
||||||
|
config.HypervisorType = vc.ClhHypervisor
|
||||||
|
hConfig, err = newClhHypervisorConfig(hypervisor)
|
||||||
|
+ case stratovirtHypervisorTable:
|
||||||
|
+ config.HypervisorType = vc.StratovirtHypervisor
|
||||||
|
+ hConfig, err = newStratovirtHypervisorConfig(hypervisor)
|
||||||
|
}
|
||||||
|
|
||||||
|
if err != nil {
|
||||||
|
@@ -1287,7 +1381,7 @@ func checkHypervisorConfig(config vc.HypervisorConfig) error {
|
||||||
|
memSizeMB := int64(config.MemorySize)
|
||||||
|
|
||||||
|
if memSizeMB == 0 {
|
||||||
|
- return errors.New("VM memory cannot be zero")
|
||||||
|
+ return errors.New(fmt.Sprintf("The VM memory cannot be zero, %s", config.ImagePath))
|
||||||
|
}
|
||||||
|
|
||||||
|
mb := int64(1024 * 1024)
|
||||||
|
diff --git a/src/runtime/virtcontainers/hypervisor.go b/src/runtime/virtcontainers/hypervisor.go
|
||||||
|
index 767215b6..615baa80 100644
|
||||||
|
--- a/src/runtime/virtcontainers/hypervisor.go
|
||||||
|
+++ b/src/runtime/virtcontainers/hypervisor.go
|
||||||
|
@@ -44,6 +44,9 @@ const (
|
||||||
|
// ClhHypervisor is the ICH hypervisor.
|
||||||
|
ClhHypervisor HypervisorType = "clh"
|
||||||
|
|
||||||
|
+ // StratovirtHypervisor is the StratoVirt hypervisor
|
||||||
|
+ StratovirtHypervisor HypervisorType = "stratovirt"
|
||||||
|
+
|
||||||
|
// MockHypervisor is a mock hypervisor for testing purposes
|
||||||
|
MockHypervisor HypervisorType = "mock"
|
||||||
|
)
|
||||||
|
@@ -159,6 +162,9 @@ func (hType *HypervisorType) Set(value string) error {
|
||||||
|
case "clh":
|
||||||
|
*hType = ClhHypervisor
|
||||||
|
return nil
|
||||||
|
+ case "stratovirt":
|
||||||
|
+ *hType = StratovirtHypervisor
|
||||||
|
+ return nil
|
||||||
|
case "mock":
|
||||||
|
*hType = MockHypervisor
|
||||||
|
return nil
|
||||||
|
@@ -178,6 +184,8 @@ func (hType *HypervisorType) String() string {
|
||||||
|
return string(AcrnHypervisor)
|
||||||
|
case ClhHypervisor:
|
||||||
|
return string(ClhHypervisor)
|
||||||
|
+ case StratovirtHypervisor:
|
||||||
|
+ return string(StratovirtHypervisor)
|
||||||
|
case MockHypervisor:
|
||||||
|
return string(MockHypervisor)
|
||||||
|
default:
|
||||||
|
@@ -207,6 +215,10 @@ func newHypervisor(hType HypervisorType) (hypervisor, error) {
|
||||||
|
return &cloudHypervisor{
|
||||||
|
store: store,
|
||||||
|
}, nil
|
||||||
|
+ case StratovirtHypervisor:
|
||||||
|
+ return &stratovirt{
|
||||||
|
+ store: store,
|
||||||
|
+ }, nil
|
||||||
|
case MockHypervisor:
|
||||||
|
return &mockHypervisor{}, nil
|
||||||
|
default:
|
||||||
|
diff --git a/src/runtime/virtcontainers/stratovirt.go b/src/runtime/virtcontainers/stratovirt.go
|
||||||
|
new file mode 100644
|
||||||
|
index 00000000..4fec96d3
|
||||||
|
--- /dev/null
|
||||||
|
+++ b/src/runtime/virtcontainers/stratovirt.go
|
||||||
|
@@ -0,0 +1,642 @@
|
||||||
|
+package virtcontainers
|
||||||
|
+
|
||||||
|
+import (
|
||||||
|
+ "context"
|
||||||
|
+ "fmt"
|
||||||
|
+ "os"
|
||||||
|
+ "os/exec"
|
||||||
|
+ "path/filepath"
|
||||||
|
+ "strconv"
|
||||||
|
+ "strings"
|
||||||
|
+ "syscall"
|
||||||
|
+ "time"
|
||||||
|
+
|
||||||
|
+ govmmQemu "github.com/kata-containers/govmm/qemu"
|
||||||
|
+ "github.com/pkg/errors"
|
||||||
|
+ "github.com/sirupsen/logrus"
|
||||||
|
+
|
||||||
|
+ "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/device/config"
|
||||||
|
+ persistapi "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/persist/api"
|
||||||
|
+ "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/types"
|
||||||
|
+ "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/utils"
|
||||||
|
+ "go.opentelemetry.io/otel"
|
||||||
|
+ otelLabel "go.opentelemetry.io/otel/label"
|
||||||
|
+ otelTrace "go.opentelemetry.io/otel/trace"
|
||||||
|
+)
|
||||||
|
+
|
||||||
|
+const defaultDummyMac = "22:33:44:aa:bb:"
|
||||||
|
+const mmioBlkCount = 4
|
||||||
|
+const mmioNetCount = 2
|
||||||
|
+const randomDevice = "/dev/urandom"
|
||||||
|
+
|
||||||
|
+type stratovirtDev struct {
|
||||||
|
+ dev interface{}
|
||||||
|
+ devType deviceType
|
||||||
|
+}
|
||||||
|
+
|
||||||
|
+type stratovirt struct {
|
||||||
|
+ id string
|
||||||
|
+ ctx context.Context
|
||||||
|
+ sandbox *Sandbox
|
||||||
|
+ store persistapi.PersistDriver
|
||||||
|
+ config HypervisorConfig
|
||||||
|
+ pid int
|
||||||
|
+ consolePath string
|
||||||
|
+ socketPath string
|
||||||
|
+ qmpMonitorCh qmpChannel
|
||||||
|
+ devices []stratovirtDev
|
||||||
|
+ HotpluggedVCPUs []CPUDevice
|
||||||
|
+ mmioBlkSlots [mmioBlkCount]bool
|
||||||
|
+ mmioNetSlots [mmioNetCount]bool
|
||||||
|
+}
|
||||||
|
+
|
||||||
|
+func (s *stratovirt) Logger() *logrus.Entry {
|
||||||
|
+ return virtLog.WithField("subsystem", "stratovirt")
|
||||||
|
+}
|
||||||
|
+
|
||||||
|
+func (s *stratovirt) trace(parent context.Context, name string) (otelTrace.Span, context.Context) {
|
||||||
|
+ if parent == nil {
|
||||||
|
+ s.Logger().WithField("type", "bug").Error("trace called before context set")
|
||||||
|
+ parent = context.Background()
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ tracer := otel.Tracer("kata")
|
||||||
|
+ ctx, span := tracer.Start(parent, name, otelTrace.WithAttributes(otelLabel.String("source", "runtime"), otelLabel.String("package", "virtcontainers"), otelLabel.String("subsystem", "hypervisor"), otelLabel.String("type", "stratovirt"), otelLabel.String("sandbox_id", s.id)))
|
||||||
|
+
|
||||||
|
+ return span, ctx
|
||||||
|
+}
|
||||||
|
+
|
||||||
|
+func (s *stratovirt) getKernelCmdLine(useImage bool) string {
|
||||||
|
+ var params []string
|
||||||
|
+
|
||||||
|
+ if useImage {
|
||||||
|
+ params = append(params, "root=/dev/vda")
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ params = append(params, "pci=off")
|
||||||
|
+ params = append(params, "reboot=k")
|
||||||
|
+ params = append(params, "panic=1")
|
||||||
|
+ params = append(params, "iommu=off")
|
||||||
|
+ params = append(params, "acpi=off")
|
||||||
|
+ params = append(params, "quiet")
|
||||||
|
+ params = append(params, "agent.use_vsock=true")
|
||||||
|
+ params = append(params, "random.trust_cpu=on")
|
||||||
|
+ params = append(params, "rw")
|
||||||
|
+ params = append(params, SerializeParams(s.config.KernelParams, "=")...)
|
||||||
|
+
|
||||||
|
+ return strings.Join(params, " ")
|
||||||
|
+}
|
||||||
|
+
|
||||||
|
+func (s *stratovirt) hypervisorConfig() HypervisorConfig {
|
||||||
|
+ return s.config
|
||||||
|
+}
|
||||||
|
+
|
||||||
|
+func (s *stratovirt) createSandbox(ctx context.Context, id string, networkNS NetworkNamespace, hypervisorConfig *HypervisorConfig) error {
|
||||||
|
+ s.ctx = ctx
|
||||||
|
+
|
||||||
|
+ var span otelTrace.Span
|
||||||
|
+ span, _ = s.trace(ctx, "createSandbox")
|
||||||
|
+ defer span.End()
|
||||||
|
+
|
||||||
|
+ s.id = id
|
||||||
|
+ s.config = *hypervisorConfig
|
||||||
|
+
|
||||||
|
+ s.socketPath = filepath.Join(s.store.RunVMStoragePath(), id, "qmp.sock")
|
||||||
|
+ s.consolePath = filepath.Join(s.store.RunVMStoragePath(), id, "console.sock")
|
||||||
|
+ s.qmpMonitorCh = qmpChannel{
|
||||||
|
+ ctx: s.ctx,
|
||||||
|
+ path: s.socketPath,
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ return nil
|
||||||
|
+}
|
||||||
|
+
|
||||||
|
+func (s *stratovirt) waitSandBoxStarted(timeout int) error {
|
||||||
|
+ timeStart := time.Now()
|
||||||
|
+ for {
|
||||||
|
+ err := s.qmpSetup()
|
||||||
|
+ if err == nil {
|
||||||
|
+ break
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ if int(time.Since(timeStart).Seconds()) > timeout {
|
||||||
|
+ return fmt.Errorf("Failed to connect to StratoVirt instance (timeout %ds): %v", timeout, err)
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ time.Sleep(time.Duration(50) * time.Millisecond)
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ if err := s.qmpMonitorCh.qmp.ExecuteQMPCapabilities(s.qmpMonitorCh.ctx); err != nil {
|
||||||
|
+ s.Logger().WithError(err).Error(qmpCapErrMsg)
|
||||||
|
+ return err
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ return nil
|
||||||
|
+}
|
||||||
|
+
|
||||||
|
+func (s *stratovirt) startSandbox(ctx context.Context, timeout int) error {
|
||||||
|
+ span, _ := s.trace(ctx, "startSandbox")
|
||||||
|
+ defer span.End()
|
||||||
|
+
|
||||||
|
+ var params []string
|
||||||
|
+ var use_image bool
|
||||||
|
+ params = append(params, "-name", fmt.Sprintf("sandbox-%s", s.id))
|
||||||
|
+ params = append(params, "-qmp", fmt.Sprintf("unix:%s,server,nowait", s.socketPath))
|
||||||
|
+
|
||||||
|
+ if kernelPath, err := s.config.KernelAssetPath(); err == nil {
|
||||||
|
+ params = append(params, "-kernel", kernelPath)
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ initrdPath, err := s.config.InitrdAssetPath()
|
||||||
|
+ if err != nil {
|
||||||
|
+ return err
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ if initrdPath == "" {
|
||||||
|
+ imagePath, err := s.config.ImageAssetPath()
|
||||||
|
+ if err != nil {
|
||||||
|
+ return err
|
||||||
|
+ }
|
||||||
|
+ use_image = true
|
||||||
|
+ s.mmioBlkSlots[0] = true
|
||||||
|
+ params = append(params, "-device", "virtio-blk-device,drive=rootfs")
|
||||||
|
+ params = append(params, "-drive", fmt.Sprintf("id=rootfs,file=%s,direct=off", imagePath))
|
||||||
|
+ } else {
|
||||||
|
+ use_image = false
|
||||||
|
+ params = append(params, "-initrd", initrdPath)
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ params = append(params, "-append", s.getKernelCmdLine(use_image))
|
||||||
|
+ params = append(params, "-smp", fmt.Sprintf("%d", s.config.NumVCPUs))
|
||||||
|
+ params = append(params, "-m", fmt.Sprintf("%d", uint64(s.config.MemorySize)))
|
||||||
|
+ params = append(params, "-device", "virtio-serial-device")
|
||||||
|
+ params = append(params, "-device", "virtconsole,chardev=charconsole0,id=virtioconsole0")
|
||||||
|
+ params = append(params, "-object", fmt.Sprintf("rng-random,id=objrng0,filename=%s", randomDevice))
|
||||||
|
+ params = append(params, "-device", "virtio-rng-device,rng=objrng0")
|
||||||
|
+ params = append(params, "-chardev", fmt.Sprintf("socket,id=charconsole0,path=%s,server,nowait", s.consolePath))
|
||||||
|
+ params = append(params, "-pidfile", filepath.Join(s.store.RunVMStoragePath(), s.id, "pid"))
|
||||||
|
+
|
||||||
|
+ // add devices to cmdline
|
||||||
|
+ for _, d := range s.devices {
|
||||||
|
+ switch v := d.dev.(type) {
|
||||||
|
+ case Endpoint:
|
||||||
|
+ name := v.Name()
|
||||||
|
+ mac := v.HardwareAddr()
|
||||||
|
+ tapName := v.NetworkPair().TapInterface.TAPIface.Name
|
||||||
|
+ params = append(params, "-device", fmt.Sprintf("virtio-net-device,netdev=%s,id=%s,mac=%s", name, name, mac))
|
||||||
|
+ params = append(params, "-netdev", fmt.Sprintf("tap,id=%s,ifname=%s", name, tapName))
|
||||||
|
+ case config.BlockDrive:
|
||||||
|
+ id := v.ID
|
||||||
|
+ path := v.File
|
||||||
|
+ params = append(params, "-device", fmt.Sprintf("virtio-blk-device, drive=%s", id))
|
||||||
|
+ params = append(params, "-drive", fmt.Sprintf("id=%s,file=%s", id, path))
|
||||||
|
+ case types.VSock:
|
||||||
|
+ v.VhostFd.Close()
|
||||||
|
+ params = append(params, "-device", fmt.Sprintf("vhost-vsock-device,id=vsock-id,guest-cid=%d", v.ContextID))
|
||||||
|
+ default:
|
||||||
|
+ s.Logger().Error("Adding device type is unsupported")
|
||||||
|
+ }
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ // daemonize
|
||||||
|
+ params = append(params, "-daemonize")
|
||||||
|
+
|
||||||
|
+ // append logfile only on debug
|
||||||
|
+ if s.config.Debug {
|
||||||
|
+ dir := filepath.Join(s.store.RunVMStoragePath(), s.id)
|
||||||
|
+ params = append(params, "-D", fmt.Sprintf("%s/stratovirt.log", dir))
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ dir := filepath.Join(s.store.RunVMStoragePath(), s.id)
|
||||||
|
+ err = os.MkdirAll(dir, DirMode)
|
||||||
|
+ if err != nil {
|
||||||
|
+ return err
|
||||||
|
+ }
|
||||||
|
+ defer func() {
|
||||||
|
+ if err != nil {
|
||||||
|
+ if err := os.RemoveAll(dir); err != nil {
|
||||||
|
+ s.Logger().WithError(err).Error("Fail to clean up vm dir %s", dir)
|
||||||
|
+ }
|
||||||
|
+ }
|
||||||
|
+ }()
|
||||||
|
+
|
||||||
|
+ binPath, err := s.config.HypervisorAssetPath()
|
||||||
|
+ if err != nil {
|
||||||
|
+ s.Logger().WithField("Fail to get hypervisor bin path", err).Error()
|
||||||
|
+ return err
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ cmd := exec.CommandContext(s.ctx, binPath, params...)
|
||||||
|
+ s.Logger().Info("StratoVirt start with params: ", cmd)
|
||||||
|
+
|
||||||
|
+ if err := cmd.Start(); err != nil {
|
||||||
|
+ s.Logger().WithField("Error starting hypervisor, please check the params", err).Error()
|
||||||
|
+ return err
|
||||||
|
+ }
|
||||||
|
+ s.pid = cmd.Process.Pid
|
||||||
|
+
|
||||||
|
+ if err = s.waitSandBoxStarted(timeout); err != nil {
|
||||||
|
+ return err
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ return nil
|
||||||
|
+}
|
||||||
|
+
|
||||||
|
+func (s *stratovirt) stopSandbox(ctx context.Context, force bool) error {
|
||||||
|
+ span, _ := s.trace(ctx, "stopSandbox")
|
||||||
|
+ defer span.End()
|
||||||
|
+
|
||||||
|
+ defer func() {
|
||||||
|
+ dir := filepath.Join(s.store.RunVMStoragePath(), s.id)
|
||||||
|
+ link, _ := filepath.EvalSymlinks(dir)
|
||||||
|
+
|
||||||
|
+ if err := os.RemoveAll(dir); err != nil {
|
||||||
|
+ s.Logger().WithError(err).Warnf("Failed to clean up vm dir %s", dir)
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ if link != dir && link != "" {
|
||||||
|
+ if err := os.RemoveAll(link); err != nil {
|
||||||
|
+ s.Logger().WithError(err).WithField("link", link).Warn("Failed to remove vm path link %s", link)
|
||||||
|
+ }
|
||||||
|
+ }
|
||||||
|
+ }()
|
||||||
|
+
|
||||||
|
+ if !force {
|
||||||
|
+ err := s.qmpSetup()
|
||||||
|
+ if err != nil {
|
||||||
|
+ return err
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ err = s.qmpMonitorCh.qmp.ExecuteQuit(s.qmpMonitorCh.ctx)
|
||||||
|
+ if err != nil {
|
||||||
|
+ s.Logger().WithError(err).Error("Fail to execute qmp: QUIT")
|
||||||
|
+ return err
|
||||||
|
+ }
|
||||||
|
+ } else {
|
||||||
|
+ if s.pid > 0 {
|
||||||
|
+ syscall.Kill(s.pid, syscall.SIGKILL)
|
||||||
|
+ }
|
||||||
|
+ }
|
||||||
|
+ return nil
|
||||||
|
+}
|
||||||
|
+
|
||||||
|
+func (s *stratovirt) pauseSandbox(ctx context.Context) error {
|
||||||
|
+ return nil
|
||||||
|
+}
|
||||||
|
+
|
||||||
|
+func (s *stratovirt) saveSandbox() error {
|
||||||
|
+ return nil
|
||||||
|
+}
|
||||||
|
+
|
||||||
|
+func (s *stratovirt) resumeSandbox(ctx context.Context) error {
|
||||||
|
+ return nil
|
||||||
|
+}
|
||||||
|
+
|
||||||
|
+func (s *stratovirt) addDevice(ctx context.Context, devInfo interface{}, devType deviceType) error {
|
||||||
|
+ span, _ := s.trace(ctx, "addDevice")
|
||||||
|
+ defer span.End()
|
||||||
|
+
|
||||||
|
+ dev := stratovirtDev{
|
||||||
|
+ dev: devInfo,
|
||||||
|
+ devType: devType,
|
||||||
|
+ }
|
||||||
|
+ s.devices = append(s.devices, dev)
|
||||||
|
+
|
||||||
|
+ return nil
|
||||||
|
+}
|
||||||
|
+
|
||||||
|
+func (s *stratovirt) getDevSlot(Name string, isPut bool) (slot int, err error) {
|
||||||
|
+ Name = filepath.Base(strings.ToLower(Name))
|
||||||
|
+
|
||||||
|
+ if strings.HasPrefix(Name, "eth") {
|
||||||
|
+ idxStr := strings.TrimPrefix(Name, "eth")
|
||||||
|
+ if idxStr == Name {
|
||||||
|
+ return 0, fmt.Errorf("Could not parse idx from Name %q", Name)
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ idx, err := strconv.Atoi(idxStr)
|
||||||
|
+ if err != nil {
|
||||||
|
+ return 0, fmt.Errorf("Could not convert to int from Str %q", idxStr)
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ if !isPut && s.mmioNetSlots[idx] {
|
||||||
|
+ return 0, fmt.Errorf("GetDevSlot failed, slot is being used %q", idxStr)
|
||||||
|
+ }
|
||||||
|
+ s.mmioNetSlots[idx] = !isPut
|
||||||
|
+
|
||||||
|
+ return idx, nil
|
||||||
|
+ } else if strings.HasPrefix(Name, "vd") {
|
||||||
|
+ charStr := strings.TrimPrefix(Name, "vd")
|
||||||
|
+ if charStr == Name {
|
||||||
|
+ return 0, fmt.Errorf("Could not parse idx from Name %q", Name)
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ char := []rune(charStr)
|
||||||
|
+ idx := int(char[0] - 'a')
|
||||||
|
+
|
||||||
|
+ if !isPut && s.mmioBlkSlots[idx] {
|
||||||
|
+ return 0, fmt.Errorf("GetDevSlot failed, slot is being used %q", charStr)
|
||||||
|
+ }
|
||||||
|
+ s.mmioBlkSlots[idx] = !isPut
|
||||||
|
+
|
||||||
|
+ return idx, nil
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ return 0, fmt.Errorf("GetDevSlot failed, Name is invalid %q", Name)
|
||||||
|
+}
|
||||||
|
+
|
||||||
|
+func (s *stratovirt) hotplugNet(endpoint Endpoint, op operation) (err error) {
|
||||||
|
+ err = s.qmpSetup()
|
||||||
|
+ if err != nil {
|
||||||
|
+ return err
|
||||||
|
+ }
|
||||||
|
+ var tap TapInterface
|
||||||
|
+
|
||||||
|
+ switch endpoint.Type() {
|
||||||
|
+ case VethEndpointType:
|
||||||
|
+ drive := endpoint.(*VethEndpoint)
|
||||||
|
+ tap = drive.NetPair.TapInterface
|
||||||
|
+ case TapEndpointType:
|
||||||
|
+ drive := endpoint.(*TapEndpoint)
|
||||||
|
+ tap = drive.TapInterface
|
||||||
|
+ default:
|
||||||
|
+ return fmt.Errorf("Endpoint is not supported")
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ switch op {
|
||||||
|
+ case addDevice:
|
||||||
|
+ var (
|
||||||
|
+ VMFdNames []string
|
||||||
|
+ VhostFdNames []string
|
||||||
|
+ )
|
||||||
|
+ for i, VMFd := range tap.VMFds {
|
||||||
|
+ fdName := fmt.Sprintf("fd%d", i)
|
||||||
|
+ if err := s.qmpMonitorCh.qmp.ExecuteGetFD(s.qmpMonitorCh.ctx, fdName, VMFd); err != nil {
|
||||||
|
+ return err
|
||||||
|
+ }
|
||||||
|
+ VMFdNames = append(VMFdNames, fdName)
|
||||||
|
+ }
|
||||||
|
+ for i, VhostFd := range tap.VhostFds {
|
||||||
|
+ fdName := fmt.Sprintf("vhostfd%d", i)
|
||||||
|
+ if err := s.qmpMonitorCh.qmp.ExecuteGetFD(s.qmpMonitorCh.ctx, fdName, VhostFd); err != nil {
|
||||||
|
+ return err
|
||||||
|
+ }
|
||||||
|
+ VhostFd.Close()
|
||||||
|
+ VhostFdNames = append(VhostFdNames, fdName)
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ slot, err := s.getDevSlot(endpoint.Name(), false)
|
||||||
|
+ if err != nil {
|
||||||
|
+ return fmt.Errorf("Could not get unused slot for %q", endpoint.Name())
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ if len(VMFdNames) != 0 || len(VhostFdNames) != 0 {
|
||||||
|
+ if err := s.qmpMonitorCh.qmp.ExecuteNetdevAddByFds(s.qmpMonitorCh.ctx, "tap", tap.ID, VMFdNames, VhostFdNames); err != nil {
|
||||||
|
+ s.getDevSlot(endpoint.Name(), true)
|
||||||
|
+ return err
|
||||||
|
+ }
|
||||||
|
+ } else {
|
||||||
|
+ if err := s.qmpMonitorCh.qmp.ExecuteNetdevAdd(s.qmpMonitorCh.ctx, "tap", tap.ID, tap.TAPIface.Name, "no", "no", 0); err != nil {
|
||||||
|
+ s.getDevSlot(endpoint.Name(), true)
|
||||||
|
+ return err
|
||||||
|
+ }
|
||||||
|
+ }
|
||||||
|
+ if err := s.qmpMonitorCh.qmp.ExecuteNetPCIDeviceAdd(s.qmpMonitorCh.ctx, tap.Name, tap.ID, endpoint.HardwareAddr(), fmt.Sprintf("%d", slot), "", "", 0, false); err != nil {
|
||||||
|
+ s.getDevSlot(endpoint.Name(), true)
|
||||||
|
+ return err
|
||||||
|
+ }
|
||||||
|
+ case removeDevice:
|
||||||
|
+ if err := s.qmpMonitorCh.qmp.ExecuteDeviceDel(s.qmpMonitorCh.ctx, tap.ID); err != nil {
|
||||||
|
+ return err
|
||||||
|
+ }
|
||||||
|
+ if err := s.qmpMonitorCh.qmp.ExecuteNetdevDel(s.qmpMonitorCh.ctx, tap.ID); err != nil {
|
||||||
|
+ return err
|
||||||
|
+ }
|
||||||
|
+ default:
|
||||||
|
+ return fmt.Errorf("Operation is not supported")
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ return nil
|
||||||
|
+}
|
||||||
|
+
|
||||||
|
+func (s *stratovirt) hotplugBlk(drive *config.BlockDrive, op operation) (err error) {
|
||||||
|
+ err = s.qmpSetup()
|
||||||
|
+ if err != nil {
|
||||||
|
+ return err
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ switch op {
|
||||||
|
+ case addDevice:
|
||||||
|
+ driver := "virtio-blk-pci"
|
||||||
|
+ slot, err := s.getDevSlot(drive.VirtPath, false)
|
||||||
|
+ if err != nil {
|
||||||
|
+ return fmt.Errorf("Could not get unused slot for %q", drive.VirtPath)
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ if err := s.qmpMonitorCh.qmp.ExecuteBlockdevAdd(s.qmpMonitorCh.ctx, drive.File, drive.ID, false); err != nil {
|
||||||
|
+ s.getDevSlot(drive.VirtPath, true)
|
||||||
|
+ return err
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ if err := s.qmpMonitorCh.qmp.ExecutePCIDeviceAdd(s.qmpMonitorCh.ctx, drive.ID, drive.ID, driver, fmt.Sprintf("%d", slot), "", "", 0, true, false); err != nil {
|
||||||
|
+ s.getDevSlot(drive.VirtPath, true)
|
||||||
|
+ return err
|
||||||
|
+ }
|
||||||
|
+ case removeDevice:
|
||||||
|
+ if err := s.qmpMonitorCh.qmp.ExecuteDeviceDel(s.qmpMonitorCh.ctx, drive.ID); err != nil {
|
||||||
|
+ return err
|
||||||
|
+ }
|
||||||
|
+ if err := s.qmpMonitorCh.qmp.ExecuteBlockdevDel(s.qmpMonitorCh.ctx, drive.ID); err != nil {
|
||||||
|
+ return err
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ s.getDevSlot(drive.VirtPath, true)
|
||||||
|
+ default:
|
||||||
|
+ return fmt.Errorf("Operation is not supported")
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ return nil
|
||||||
|
+}
|
||||||
|
+
|
||||||
|
+func (s *stratovirt) hotplugAddDevice(ctx context.Context, devInfo interface{}, devType deviceType) (interface{}, error) {
|
||||||
|
+ span, _ := s.trace(ctx, "hotplugAddDevice")
|
||||||
|
+ defer span.End()
|
||||||
|
+
|
||||||
|
+ switch devType {
|
||||||
|
+ case netDev:
|
||||||
|
+ return nil, s.hotplugNet(devInfo.(Endpoint), addDevice)
|
||||||
|
+ case blockDev:
|
||||||
|
+ return nil, s.hotplugBlk(devInfo.(*config.BlockDrive), addDevice)
|
||||||
|
+ default:
|
||||||
|
+ return nil, fmt.Errorf("Hotplug add device failed: unsupported device type '%v'", devType)
|
||||||
|
+ }
|
||||||
|
+}
|
||||||
|
+
|
||||||
|
+func (s *stratovirt) hotplugRemoveDevice(ctx context.Context, devInfo interface{}, devType deviceType) (interface{}, error) {
|
||||||
|
+ span, _ := s.trace(ctx, "hotplugRemoveDevice")
|
||||||
|
+ defer span.End()
|
||||||
|
+
|
||||||
|
+ switch devType {
|
||||||
|
+ case netDev:
|
||||||
|
+ return nil, s.hotplugNet(devInfo.(Endpoint), removeDevice)
|
||||||
|
+ case blockDev:
|
||||||
|
+ return nil, s.hotplugBlk(devInfo.(*config.BlockDrive), removeDevice)
|
||||||
|
+ default:
|
||||||
|
+ return nil, fmt.Errorf("Hotplug remove device: unsupported device type '%v'", devType)
|
||||||
|
+ }
|
||||||
|
+}
|
||||||
|
+
|
||||||
|
+func (s *stratovirt) resizeMemory(ctx context.Context, reqMemMB uint32, memoryBlockSizeMB uint32, probe bool) (uint32, memoryDevice, error) {
|
||||||
|
+ return 0, memoryDevice{}, nil
|
||||||
|
+}
|
||||||
|
+
|
||||||
|
+func (s *stratovirt) resizeVCPUs(ctx context.Context, reqVCPUs uint32) (currentVCPUs uint32, newVCPUs uint32, err error) {
|
||||||
|
+ return 0, 0, nil
|
||||||
|
+}
|
||||||
|
+
|
||||||
|
+func (s *stratovirt) getSandboxConsole(ctx context.Context, id string) (string, string, error) {
|
||||||
|
+ span, _ := s.trace(ctx, "getSandboxConsole")
|
||||||
|
+ defer span.End()
|
||||||
|
+
|
||||||
|
+ var consolePath string
|
||||||
|
+ if s.config.Debug {
|
||||||
|
+ consolePath = s.consolePath
|
||||||
|
+ } else {
|
||||||
|
+ consolePath = ""
|
||||||
|
+ }
|
||||||
|
+ consoleURL, err := utils.BuildSocketPath(consolePath)
|
||||||
|
+ if err != nil {
|
||||||
|
+ return consoleProtoUnix, "", err
|
||||||
|
+ }
|
||||||
|
+ return consoleProtoUnix, consoleURL, nil
|
||||||
|
+
|
||||||
|
+}
|
||||||
|
+
|
||||||
|
+func (s *stratovirt) disconnect(ctx context.Context) {
|
||||||
|
+ span, _ := s.trace(ctx, "disconnect")
|
||||||
|
+ defer span.End()
|
||||||
|
+
|
||||||
|
+ s.qmpTeardown()
|
||||||
|
+}
|
||||||
|
+
|
||||||
|
+func (s *stratovirt) capabilities(ctx context.Context) types.Capabilities {
|
||||||
|
+ span, _ := s.trace(ctx, "capabilities")
|
||||||
|
+ defer span.End()
|
||||||
|
+
|
||||||
|
+ var caps types.Capabilities
|
||||||
|
+ caps.SetBlockDeviceHotplugSupport()
|
||||||
|
+
|
||||||
|
+ return caps
|
||||||
|
+}
|
||||||
|
+
|
||||||
|
+func (s *stratovirt) qmpTeardown() {
|
||||||
|
+ if s.qmpMonitorCh.qmp != nil {
|
||||||
|
+ s.qmpMonitorCh.qmp.Shutdown()
|
||||||
|
+ <-s.qmpMonitorCh.disconn
|
||||||
|
+ s.qmpMonitorCh.qmp = nil
|
||||||
|
+ s.qmpMonitorCh.disconn = nil
|
||||||
|
+ }
|
||||||
|
+}
|
||||||
|
+
|
||||||
|
+func (s *stratovirt) qmpSetup() error {
|
||||||
|
+ s.qmpTeardown()
|
||||||
|
+
|
||||||
|
+ cfg := govmmQemu.QMPConfig{Logger: newQMPLogger()}
|
||||||
|
+
|
||||||
|
+ // Auto-closed by QMPStart().
|
||||||
|
+ disconnectCh := make(chan struct{})
|
||||||
|
+
|
||||||
|
+ qmp, _, err := govmmQemu.QMPStart(s.qmpMonitorCh.ctx, s.qmpMonitorCh.path, cfg, disconnectCh)
|
||||||
|
+ if err != nil {
|
||||||
|
+ s.Logger().WithError(err).Error("Failed to connect to StratoVirt instance")
|
||||||
|
+ return err
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ s.qmpMonitorCh.qmp = qmp
|
||||||
|
+ s.qmpMonitorCh.disconn = disconnectCh
|
||||||
|
+
|
||||||
|
+ return nil
|
||||||
|
+}
|
||||||
|
+
|
||||||
|
+func (s *stratovirt) getThreadIDs(ctx context.Context) (vcpuThreadIDs, error) {
|
||||||
|
+ span, _ := s.trace(ctx, "getThreadIDs")
|
||||||
|
+ defer span.End()
|
||||||
|
+
|
||||||
|
+ tid := vcpuThreadIDs{}
|
||||||
|
+ if err := s.qmpSetup(); err != nil {
|
||||||
|
+ return tid, err
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ cpuInfos, err := s.qmpMonitorCh.qmp.ExecQueryCpus(s.qmpMonitorCh.ctx)
|
||||||
|
+ if err != nil {
|
||||||
|
+ s.Logger().WithError(err).Error("Failed to query cpu infos")
|
||||||
|
+ return tid, err
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ tid.vcpus = make(map[int]int, len(cpuInfos))
|
||||||
|
+ for _, i := range cpuInfos {
|
||||||
|
+ if i.ThreadID > 0 {
|
||||||
|
+ tid.vcpus[i.CPU] = i.ThreadID
|
||||||
|
+ }
|
||||||
|
+ }
|
||||||
|
+ return tid, nil
|
||||||
|
+}
|
||||||
|
+
|
||||||
|
+func (s *stratovirt) cleanup(ctx context.Context) error {
|
||||||
|
+ span, _ := s.trace(ctx, "cleanup")
|
||||||
|
+ defer span.End()
|
||||||
|
+
|
||||||
|
+ s.qmpTeardown()
|
||||||
|
+
|
||||||
|
+ return nil
|
||||||
|
+}
|
||||||
|
+
|
||||||
|
+func (s *stratovirt) getPids() []int {
|
||||||
|
+ return []int{s.pid}
|
||||||
|
+}
|
||||||
|
+
|
||||||
|
+func (s *stratovirt) getVirtioFsPid() *int {
|
||||||
|
+ return nil
|
||||||
|
+}
|
||||||
|
+
|
||||||
|
+func (s *stratovirt) fromGrpc(ctx context.Context, hypervisorConfig *HypervisorConfig, j []byte) error {
|
||||||
|
+ return errors.New("stratovirt is not supported by VM cache")
|
||||||
|
+}
|
||||||
|
+
|
||||||
|
+func (s *stratovirt) toGrpc(ctx context.Context) ([]byte, error) {
|
||||||
|
+ return nil, errors.New("stratovirt is not supported by VM cache")
|
||||||
|
+}
|
||||||
|
+
|
||||||
|
+func (s *stratovirt) check() error {
|
||||||
|
+ if err := syscall.Kill(s.pid, syscall.Signal(0)); err != nil {
|
||||||
|
+ return errors.Wrapf(err, "Failed to ping StratoVirt process")
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ return nil
|
||||||
|
+}
|
||||||
|
+
|
||||||
|
+func (s *stratovirt) generateSocket(id string) (interface{}, error) {
|
||||||
|
+ return generateVMSocket(id, s.store.RunVMStoragePath())
|
||||||
|
+}
|
||||||
|
+
|
||||||
|
+func (s *stratovirt) isRateLimiterBuiltin() bool {
|
||||||
|
+ return true
|
||||||
|
+}
|
||||||
|
+
|
||||||
|
+func (s *stratovirt) save() (p persistapi.HypervisorState) {
|
||||||
|
+ pids := s.getPids()
|
||||||
|
+ p.Pid = pids[0]
|
||||||
|
+ p.Type = string(StratovirtHypervisor)
|
||||||
|
+ return
|
||||||
|
+}
|
||||||
|
+
|
||||||
|
+func (s *stratovirt) load(p persistapi.HypervisorState) {
|
||||||
|
+ s.pid = p.Pid
|
||||||
|
+
|
||||||
|
+ return
|
||||||
|
+}
|
||||||
|
+
|
||||||
|
+func (s *stratovirt) setSandbox(sandbox *Sandbox) {
|
||||||
|
+ s.sandbox = sandbox
|
||||||
|
+ return
|
||||||
|
+}
|
||||||
|
--
|
||||||
|
2.21.1 (Apple Git-122.3)
|
||||||
|
|
||||||
@ -0,0 +1,174 @@
|
|||||||
|
From 1ffd95187a61582e858dd37c0ab434d3159a0f52 Mon Sep 17 00:00:00 2001
|
||||||
|
From: Wei Gao <gaowei66@huawei.com>
|
||||||
|
Date: Mon, 9 Aug 2021 14:26:35 +0800
|
||||||
|
Subject: [PATCH 2/6] agent: add support of new sandbox hypervisor kind
|
||||||
|
StratoVirt.
|
||||||
|
|
||||||
|
1. add new grpc interface `UpdateInterfaceHwAddrByName`.
|
||||||
|
2. comment out rescan_pci temporarily.
|
||||||
|
|
||||||
|
Signed-off-by: Wei Gao <gaowei66@huawei.com>
|
||||||
|
---
|
||||||
|
src/agent/protocols/protos/agent.proto | 5 +++
|
||||||
|
src/agent/src/netlink.rs | 31 ++++++++++++++++
|
||||||
|
src/agent/src/rpc.rs | 51 +++++++++++++++++++++++++-
|
||||||
|
3 files changed, 85 insertions(+), 2 deletions(-)
|
||||||
|
|
||||||
|
diff --git a/src/agent/protocols/protos/agent.proto b/src/agent/protocols/protos/agent.proto
|
||||||
|
index 6cbf5a28..e00f5c63 100644
|
||||||
|
--- a/src/agent/protocols/protos/agent.proto
|
||||||
|
+++ b/src/agent/protocols/protos/agent.proto
|
||||||
|
@@ -46,6 +46,7 @@ service AgentService {
|
||||||
|
|
||||||
|
// networking
|
||||||
|
rpc UpdateInterface(UpdateInterfaceRequest) returns (types.Interface);
|
||||||
|
+ rpc UpdateInterfaceHwAddrByName(UpdateInterfaceHwAddrByNameRequest) returns (types.Interface);
|
||||||
|
rpc UpdateRoutes(UpdateRoutesRequest) returns (Routes);
|
||||||
|
rpc ListInterfaces(ListInterfacesRequest) returns(Interfaces);
|
||||||
|
rpc ListRoutes(ListRoutesRequest) returns (Routes);
|
||||||
|
@@ -308,6 +309,10 @@ message UpdateInterfaceRequest {
|
||||||
|
types.Interface interface = 1;
|
||||||
|
}
|
||||||
|
|
||||||
|
+message UpdateInterfaceHwAddrByNameRequest {
|
||||||
|
+ types.Interface interface = 1;
|
||||||
|
+}
|
||||||
|
+
|
||||||
|
message UpdateRoutesRequest {
|
||||||
|
Routes routes = 1;
|
||||||
|
}
|
||||||
|
diff --git a/src/agent/src/netlink.rs b/src/agent/src/netlink.rs
|
||||||
|
index 3ab6dbaa..82632d1b 100644
|
||||||
|
--- a/src/agent/src/netlink.rs
|
||||||
|
+++ b/src/agent/src/netlink.rs
|
||||||
|
@@ -104,6 +104,29 @@ impl Handle {
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
+ pub async fn update_interface_hw_addr_by_name(&mut self, iface: &Interface) -> Result<()> {
|
||||||
|
+ let link = self.find_link(LinkFilter::Name(&iface.name)).await?;
|
||||||
|
+
|
||||||
|
+ // Delete all addresses associated with the link
|
||||||
|
+ let addresses = self
|
||||||
|
+ .list_addresses(AddressFilter::LinkIndex(link.index()))
|
||||||
|
+ .await?;
|
||||||
|
+ self.delete_addresses(addresses).await?;
|
||||||
|
+
|
||||||
|
+ if iface.IPAddresses.len() == 0 {
|
||||||
|
+ self.enable_link(link.index(), false).await?;
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ // Update hardware mac address
|
||||||
|
+ let mac_addr = parse_mac_address(iface.get_hwAddr())
|
||||||
|
+ .with_context(|| format!("Failed to parse MAC address: {}", iface.get_hwAddr()))?;
|
||||||
|
+ self.link_set_hw_addr(link.index(), mac_addr)
|
||||||
|
+ .await
|
||||||
|
+ .with_context(|| format!("Could not set {:?} to {}", mac_addr, link.name()))?;
|
||||||
|
+
|
||||||
|
+ Ok(())
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
pub async fn handle_localhost(&self) -> Result<()> {
|
||||||
|
let link = self.find_link(LinkFilter::Name("lo")).await?;
|
||||||
|
self.enable_link(link.index(), true).await?;
|
||||||
|
@@ -216,6 +239,14 @@ impl Handle {
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
+ async fn link_set_hw_addr(&self, link_index: u32, hw_addr: [u8; 6]) -> Result<()> {
|
||||||
|
+ let link_req = self.handle.link().set(link_index);
|
||||||
|
+ let set_req = link_req.address(hw_addr.to_vec());
|
||||||
|
+ set_req.execute().await?;
|
||||||
|
+
|
||||||
|
+ Ok(())
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
async fn query_routes(
|
||||||
|
&self,
|
||||||
|
ip_version: Option<IpVersion>,
|
||||||
|
diff --git a/src/agent/src/rpc.rs b/src/agent/src/rpc.rs
|
||||||
|
index 92025af3..2cc1c983 100644
|
||||||
|
--- a/src/agent/src/rpc.rs
|
||||||
|
+++ b/src/agent/src/rpc.rs
|
||||||
|
@@ -40,7 +40,7 @@ use nix::sys::stat;
|
||||||
|
use nix::unistd::{self, Pid};
|
||||||
|
use rustjail::process::ProcessOperations;
|
||||||
|
|
||||||
|
-use crate::device::{add_devices, rescan_pci_bus, update_device_cgroup};
|
||||||
|
+use crate::device::{add_devices, update_device_cgroup};
|
||||||
|
use crate::linux_abi::*;
|
||||||
|
use crate::metrics::get_metrics;
|
||||||
|
use crate::mount::{add_storages, remove_mounts, BareMount, STORAGE_HANDLER_LIST};
|
||||||
|
@@ -123,7 +123,9 @@ impl AgentService {
|
||||||
|
|
||||||
|
// re-scan PCI bus
|
||||||
|
// looking for hidden devices
|
||||||
|
- rescan_pci_bus().context("Could not rescan PCI bus")?;
|
||||||
|
+ // FIXME: Comment out this code temporarily, because once the PCIBus is scanned,
|
||||||
|
+ // the device hot-plug event is lost
|
||||||
|
+ // rescan_pci_bus().context("Could not rescan PCI bus")?;
|
||||||
|
|
||||||
|
// Some devices need some extra processing (the ones invoked with
|
||||||
|
// --device for instance), and that's what this call is doing. It
|
||||||
|
@@ -797,6 +799,34 @@ impl protocols::agent_ttrpc::AgentService for AgentService {
|
||||||
|
Ok(interface)
|
||||||
|
}
|
||||||
|
|
||||||
|
+ async fn update_interface_hw_addr_by_name(
|
||||||
|
+ &self,
|
||||||
|
+ _ctx: &TtrpcContext,
|
||||||
|
+ req: protocols::agent::UpdateInterfaceHwAddrByNameRequest,
|
||||||
|
+ ) -> ttrpc::Result<Interface> {
|
||||||
|
+ let interface = req.interface.into_option().ok_or_else(|| {
|
||||||
|
+ ttrpc_error(
|
||||||
|
+ ttrpc::Code::INVALID_ARGUMENT,
|
||||||
|
+ "empty update interface request".to_string(),
|
||||||
|
+ )
|
||||||
|
+ })?;
|
||||||
|
+
|
||||||
|
+ self.sandbox
|
||||||
|
+ .lock()
|
||||||
|
+ .await
|
||||||
|
+ .rtnl
|
||||||
|
+ .update_interface_hw_addr_by_name(&interface)
|
||||||
|
+ .await
|
||||||
|
+ .map_err(|e| {
|
||||||
|
+ ttrpc_error(
|
||||||
|
+ ttrpc::Code::INTERNAL,
|
||||||
|
+ format!("update interface hw addr: {:?}", e),
|
||||||
|
+ )
|
||||||
|
+ })?;
|
||||||
|
+
|
||||||
|
+ Ok(interface)
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
async fn update_routes(
|
||||||
|
&self,
|
||||||
|
_ctx: &TtrpcContext,
|
||||||
|
@@ -1670,6 +1700,23 @@ mod tests {
|
||||||
|
assert!(result.is_err(), "expected update interface to fail");
|
||||||
|
}
|
||||||
|
|
||||||
|
+ #[tokio::test]
|
||||||
|
+ async fn test_update_interface_hw_addr_by_name() {
|
||||||
|
+ let logger = slog::Logger::root(slog::Discard, o!());
|
||||||
|
+ let sandbox = Sandbox::new(&logger).unwrap();
|
||||||
|
+
|
||||||
|
+ let agent_service = Box::new(AgentService {
|
||||||
|
+ sandbox: Arc::new(Mutex::new(sandbox)),
|
||||||
|
+ });
|
||||||
|
+
|
||||||
|
+ let req = protocols::agent::UpdateInterfaceHwAddrByNameRequest::default();
|
||||||
|
+ let ctx = mk_ttrpc_context();
|
||||||
|
+
|
||||||
|
+ let result = agent_service.update_interface_hw_addr_by_name(&ctx, req).await;
|
||||||
|
+
|
||||||
|
+ assert!(result.is_err(), "expected update interface to fail");
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
#[tokio::test]
|
||||||
|
async fn test_update_routes() {
|
||||||
|
let logger = slog::Logger::root(slog::Discard, o!());
|
||||||
|
--
|
||||||
|
2.21.1 (Apple Git-122.3)
|
||||||
|
|
||||||
File diff suppressed because it is too large
Load Diff
@ -0,0 +1,542 @@
|
|||||||
|
From 950c0db14a9a9baccefd83e87893d7f40c2bd13d Mon Sep 17 00:00:00 2001
|
||||||
|
From: Wei Gao <gaowei66@huawei.com>
|
||||||
|
Date: Mon, 9 Aug 2021 14:47:19 +0800
|
||||||
|
Subject: [PATCH 4/6] configuration: add configuration generator for hypervisor
|
||||||
|
type stratovirt.
|
||||||
|
|
||||||
|
Signed-off-by: Wei Gao <gaowei66@huawei.com>
|
||||||
|
---
|
||||||
|
src/runtime/.gitignore | 1 +
|
||||||
|
src/runtime/Makefile | 40 +-
|
||||||
|
src/runtime/arch/amd64-options.mk | 3 +
|
||||||
|
src/runtime/arch/arm64-options.mk | 3 +
|
||||||
|
.../config/configuration-stratovirt.toml.in | 356 ++++++++++++++++++
|
||||||
|
5 files changed, 402 insertions(+), 1 deletion(-)
|
||||||
|
create mode 100644 src/runtime/cli/config/configuration-stratovirt.toml.in
|
||||||
|
|
||||||
|
diff --git a/src/runtime/.gitignore b/src/runtime/.gitignore
|
||||||
|
index 52b9e4e5..0a630a07 100644
|
||||||
|
--- a/src/runtime/.gitignore
|
||||||
|
+++ b/src/runtime/.gitignore
|
||||||
|
@@ -10,6 +10,7 @@ coverage.html
|
||||||
|
/cli/config/configuration-fc.toml
|
||||||
|
/cli/config/configuration-qemu.toml
|
||||||
|
/cli/config/configuration-clh.toml
|
||||||
|
+/cli/config/configuration-stratovirt.toml
|
||||||
|
/cli/config-generated.go
|
||||||
|
/cli/containerd-shim-kata-v2/config-generated.go
|
||||||
|
/cli/coverage.html
|
||||||
|
diff --git a/src/runtime/Makefile b/src/runtime/Makefile
|
||||||
|
index 4a69c05c..ea2cd296 100644
|
||||||
|
--- a/src/runtime/Makefile
|
||||||
|
+++ b/src/runtime/Makefile
|
||||||
|
@@ -74,6 +74,7 @@ QEMUBINDIR := $(PREFIXDEPS)/bin
|
||||||
|
CLHBINDIR := $(PREFIXDEPS)/bin
|
||||||
|
FCBINDIR := $(PREFIXDEPS)/bin
|
||||||
|
ACRNBINDIR := $(PREFIXDEPS)/bin
|
||||||
|
+STRATOVIRTBINDIR := $(PREFIXDEPS)/bin
|
||||||
|
SYSCONFDIR := /etc
|
||||||
|
LOCALSTATEDIR := /var
|
||||||
|
|
||||||
|
@@ -93,6 +94,7 @@ GENERATED_VARS = \
|
||||||
|
CONFIG_QEMU_IN \
|
||||||
|
CONFIG_CLH_IN \
|
||||||
|
CONFIG_FC_IN \
|
||||||
|
+ CONFIG_STRATOVIRT_IN \
|
||||||
|
$(USER_VARS)
|
||||||
|
SCRIPTS += $(COLLECT_SCRIPT)
|
||||||
|
SCRIPTS_DIR := $(BINDIR)
|
||||||
|
@@ -116,12 +118,13 @@ HYPERVISOR_ACRN = acrn
|
||||||
|
HYPERVISOR_FC = firecracker
|
||||||
|
HYPERVISOR_QEMU = qemu
|
||||||
|
HYPERVISOR_CLH = cloud-hypervisor
|
||||||
|
+HYPERVISOR_STRATOVIRT = stratovirt
|
||||||
|
|
||||||
|
# Determines which hypervisor is specified in $(CONFIG_FILE).
|
||||||
|
DEFAULT_HYPERVISOR ?= $(HYPERVISOR_QEMU)
|
||||||
|
|
||||||
|
# List of hypervisors this build system can generate configuration for.
|
||||||
|
-HYPERVISORS := $(HYPERVISOR_ACRN) $(HYPERVISOR_FC) $(HYPERVISOR_QEMU) $(HYPERVISOR_CLH)
|
||||||
|
+HYPERVISORS := $(HYPERVISOR_ACRN) $(HYPERVISOR_FC) $(HYPERVISOR_QEMU) $(HYPERVISOR_CLH) $(HYPERVISOR_STRATOVIRT)
|
||||||
|
|
||||||
|
QEMUPATH := $(QEMUBINDIR)/$(QEMUCMD)
|
||||||
|
QEMUVALIDHYPERVISORPATHS := [\"$(QEMUPATH)\"]
|
||||||
|
@@ -141,6 +144,9 @@ ACRNVALIDHYPERVISORPATHS := [\"$(ACRNPATH)\"]
|
||||||
|
ACRNCTLPATH := $(ACRNBINDIR)/$(ACRNCTLCMD)
|
||||||
|
ACRNVALIDCTLPATHS := [\"$(ACRNCTLPATH)\"]
|
||||||
|
|
||||||
|
+STRATOVIRTPATH = $(STRATOVIRTBINDIR)/$(STRATOVIRTCMD)
|
||||||
|
+STRATOVIRTVALIDHYPERVISORPATHS := [\"$(STRATOVIRTPATH)\"]
|
||||||
|
+
|
||||||
|
NETMONCMD := $(BIN_PREFIX)-netmon
|
||||||
|
NETMONPATH := $(PKGLIBEXECDIR)/$(NETMONCMD)
|
||||||
|
|
||||||
|
@@ -267,6 +273,29 @@ ifneq (,$(CLHCMD))
|
||||||
|
KERNELPATH_CLH = $(KERNELDIR)/$(KERNEL_NAME_CLH)
|
||||||
|
endif
|
||||||
|
|
||||||
|
+ifneq (,$(STRATOVIRTCMD))
|
||||||
|
+ KNOWN_HYPERVISORS += $(HYPERVISOR_STRATOVIRT)
|
||||||
|
+
|
||||||
|
+ CONFIG_FILE_STRATOVIRT = configuration-stratovirt.toml
|
||||||
|
+ CONFIG_STRATOVIRT = $(CLI_DIR)/config/$(CONFIG_FILE_STRATOVIRT)
|
||||||
|
+ CONFIG_STRATOVIRT_IN = $(CONFIG_STRATOVIRT).in
|
||||||
|
+
|
||||||
|
+ CONFIG_PATH_STRATOVIRT = $(abspath $(CONFDIR)/$(CONFIG_FILE_STRATOVIRT))
|
||||||
|
+ CONFIG_PATHS += $(CONFIG_PATH_STRATOVIRT)
|
||||||
|
+
|
||||||
|
+ SYSCONFIG_STRATOVIRT = $(abspath $(SYSCONFDIR)/$(CONFIG_FILE_STRATOVIRT))
|
||||||
|
+ SYSCONFIG_PATHS += $(SYSCONFIG_STRATOVIRT)
|
||||||
|
+
|
||||||
|
+ CONFIGS += $(CONFIG_STRATOVIRT)
|
||||||
|
+
|
||||||
|
+ # stratovirt-specific options (all should be suffixed by "_STRATOVIRT")
|
||||||
|
+ DEFBLOCKSTORAGEDRIVER_STRATOVIRT := virtio-mmio
|
||||||
|
+ DEFNETWORKMODEL_STRATOVIRT := none
|
||||||
|
+ KENRELTYPE_STRATOVIRT = uncompressed
|
||||||
|
+ KERNEL_NAME_STRATOVIRT = $(call MAKE_KERNEL_NAME,$(KENRELTYPE_STRATOVIRT))
|
||||||
|
+ KERNELPATH_STRATOVIRT = $(KERNELDIR)/$(KERNEL_NAME_STRATOVIRT)
|
||||||
|
+endif
|
||||||
|
+
|
||||||
|
ifneq (,$(FCCMD))
|
||||||
|
KNOWN_HYPERVISORS += $(HYPERVISOR_FC)
|
||||||
|
|
||||||
|
@@ -363,6 +392,7 @@ USER_VARS += BINDIR
|
||||||
|
USER_VARS += CONFIG_ACRN_IN
|
||||||
|
USER_VARS += CONFIG_CLH_IN
|
||||||
|
USER_VARS += CONFIG_FC_IN
|
||||||
|
+USER_VARS += CONFIG_STRATOVIRT_IN
|
||||||
|
USER_VARS += CONFIG_PATH
|
||||||
|
USER_VARS += CONFIG_QEMU_IN
|
||||||
|
USER_VARS += DESTDIR
|
||||||
|
@@ -382,6 +412,8 @@ USER_VARS += FCPATH
|
||||||
|
USER_VARS += FCVALIDHYPERVISORPATHS
|
||||||
|
USER_VARS += FCJAILERPATH
|
||||||
|
USER_VARS += FCVALIDJAILERPATHS
|
||||||
|
+USER_VARS += STRATOVIRTPATH
|
||||||
|
+USER_VARS += STRATOVIRTVALIDHYPERVISORPATHS
|
||||||
|
USER_VARS += SYSCONFIG
|
||||||
|
USER_VARS += IMAGENAME
|
||||||
|
USER_VARS += IMAGEPATH
|
||||||
|
@@ -395,6 +427,7 @@ USER_VARS += KERNELPATH_ACRN
|
||||||
|
USER_VARS += KERNELPATH
|
||||||
|
USER_VARS += KERNELPATH_CLH
|
||||||
|
USER_VARS += KERNELPATH_FC
|
||||||
|
+USER_VARS += KERNELPATH_STRATOVIRT
|
||||||
|
USER_VARS += KERNELVIRTIOFSPATH
|
||||||
|
USER_VARS += FIRMWAREPATH
|
||||||
|
USER_VARS += MACHINEACCELERATORS
|
||||||
|
@@ -434,12 +467,14 @@ USER_VARS += DEFNETWORKMODEL_ACRN
|
||||||
|
USER_VARS += DEFNETWORKMODEL_CLH
|
||||||
|
USER_VARS += DEFNETWORKMODEL_FC
|
||||||
|
USER_VARS += DEFNETWORKMODEL_QEMU
|
||||||
|
+USER_VARS += DEFNETWORKMODEL_STRATOVIRT
|
||||||
|
USER_VARS += DEFDISABLEGUESTSECCOMP
|
||||||
|
USER_VARS += DEFAULTEXPFEATURES
|
||||||
|
USER_VARS += DEFDISABLEBLOCK
|
||||||
|
USER_VARS += DEFBLOCKSTORAGEDRIVER_ACRN
|
||||||
|
USER_VARS += DEFBLOCKSTORAGEDRIVER_FC
|
||||||
|
USER_VARS += DEFBLOCKSTORAGEDRIVER_QEMU
|
||||||
|
+USER_VARS += DEFBLOCKSTORAGEDRIVER_STRATOVIRT
|
||||||
|
USER_VARS += DEFSHAREDFS_QEMU_VIRTIOFS
|
||||||
|
USER_VARS += DEFVIRTIOFSDAEMON
|
||||||
|
USER_VARS += DEFVALIDVIRTIOFSDAEMONPATHS
|
||||||
|
@@ -773,6 +808,9 @@ ifneq (,$(findstring $(HYPERVISOR_FC),$(KNOWN_HYPERVISORS)))
|
||||||
|
endif
|
||||||
|
ifneq (,$(findstring $(HYPERVISOR_ACRN),$(KNOWN_HYPERVISORS)))
|
||||||
|
@printf "\t$(HYPERVISOR_ACRN) hypervisor path (ACRNPATH) : %s\n" $(abspath $(ACRNPATH))
|
||||||
|
+endif
|
||||||
|
+ifneq (,$(findstring $(HYPERVISOR_STRATOVIRT),$(KNOWN_HYPERVISORS)))
|
||||||
|
+ @printf "\t$(HYPERVISOR_STRATOVIRT) hypervisor path (STRATOVIRTPATH) : %s\n" $(abspath $(STRATOVIRTPATH))
|
||||||
|
endif
|
||||||
|
@printf "\tassets path (PKGDATADIR) : %s\n" $(abspath $(PKGDATADIR))
|
||||||
|
@printf "\tshim path (PKGLIBEXECDIR) : %s\n" $(abspath $(PKGLIBEXECDIR))
|
||||||
|
diff --git a/src/runtime/arch/amd64-options.mk b/src/runtime/arch/amd64-options.mk
|
||||||
|
index 83af8cc0..ff2af9e6 100644
|
||||||
|
--- a/src/runtime/arch/amd64-options.mk
|
||||||
|
+++ b/src/runtime/arch/amd64-options.mk
|
||||||
|
@@ -23,3 +23,6 @@ ACRNCTLCMD := acrnctl
|
||||||
|
|
||||||
|
# cloud-hypervisor binary name
|
||||||
|
CLHCMD := cloud-hypervisor
|
||||||
|
+
|
||||||
|
+# stratovirt binary name
|
||||||
|
+STRATOVIRTCMD := stratovirt
|
||||||
|
\ No newline at end of file
|
||||||
|
diff --git a/src/runtime/arch/arm64-options.mk b/src/runtime/arch/arm64-options.mk
|
||||||
|
index ad5ef5d4..2ad3f657 100644
|
||||||
|
--- a/src/runtime/arch/arm64-options.mk
|
||||||
|
+++ b/src/runtime/arch/arm64-options.mk
|
||||||
|
@@ -19,3 +19,6 @@ FCJAILERCMD := jailer
|
||||||
|
|
||||||
|
# cloud-hypervisor binary name
|
||||||
|
CLHCMD := cloud-hypervisor
|
||||||
|
+
|
||||||
|
+# stratovirt binary name
|
||||||
|
+STRATOVIRTCMD := stratovirt
|
||||||
|
\ No newline at end of file
|
||||||
|
diff --git a/src/runtime/cli/config/configuration-stratovirt.toml.in b/src/runtime/cli/config/configuration-stratovirt.toml.in
|
||||||
|
new file mode 100644
|
||||||
|
index 00000000..5c83c3c9
|
||||||
|
--- /dev/null
|
||||||
|
+++ b/src/runtime/cli/config/configuration-stratovirt.toml.in
|
||||||
|
@@ -0,0 +1,356 @@
|
||||||
|
+# Copyright (c) 2017-2019 Intel Corporation
|
||||||
|
+#
|
||||||
|
+# SPDX-License-Identifier: Apache-2.0
|
||||||
|
+#
|
||||||
|
+
|
||||||
|
+# XXX: WARNING: this file is auto-generated.
|
||||||
|
+# XXX:
|
||||||
|
+# XXX: Source file: "@CONFIG_STRATOVIRT_IN@"
|
||||||
|
+# XXX: Project:
|
||||||
|
+# XXX: Name: @PROJECT_NAME@
|
||||||
|
+# XXX: Type: @PROJECT_TYPE@
|
||||||
|
+
|
||||||
|
+[hypervisor.stratovirt]
|
||||||
|
+path = "@STRATOVIRTPATH@"
|
||||||
|
+kernel = "@KERNELPATH_STRATOVIRT@"
|
||||||
|
+image = "@IMAGEPATH@"
|
||||||
|
+
|
||||||
|
+# List of valid annotation names for the hypervisor
|
||||||
|
+# Each member of the list is a regular expression, which is the base name
|
||||||
|
+# of the annotation, e.g. "path" for io.katacontainers.config.hypervisor.path"
|
||||||
|
+enable_annotations = @DEFENABLEANNOTATIONS@
|
||||||
|
+
|
||||||
|
+# List of valid annotations values for the hypervisor
|
||||||
|
+# Each member of the list is a path pattern as described by glob(3).
|
||||||
|
+# The default if not set is empty (all annotations rejected.)
|
||||||
|
+# Your distribution recommends: @STRATOVIRTVALIDHYPERVISORPATHS@
|
||||||
|
+valid_hypervisor_paths = @STRATOVIRTVALIDHYPERVISORPATHS@
|
||||||
|
+
|
||||||
|
+# Optional space-separated list of options to pass to the guest kernel.
|
||||||
|
+# For example, use `kernel_params = "vsyscall=emulate"` if you are having
|
||||||
|
+# trouble running pre-2.15 glibc.
|
||||||
|
+#
|
||||||
|
+# WARNING: - any parameter specified here will take priority over the default
|
||||||
|
+# parameter value of the same name used to start the virtual machine.
|
||||||
|
+# Do not set values here unless you understand the impact of doing so as you
|
||||||
|
+# may stop the virtual machine from booting.
|
||||||
|
+# To see the list of default parameters, enable hypervisor debug, create a
|
||||||
|
+# container and look for 'default-kernel-parameters' log entries.
|
||||||
|
+kernel_params = "@KERNELPARAMS@"
|
||||||
|
+
|
||||||
|
+# Default number of vCPUs per SB/VM:
|
||||||
|
+# unspecified or 0 --> will be set to @DEFVCPUS@
|
||||||
|
+# < 0 --> will be set to the actual number of physical cores
|
||||||
|
+# > 0 <= number of physical cores --> will be set to the specified number
|
||||||
|
+# > number of physical cores --> will be set to the actual number of physical cores
|
||||||
|
+default_vcpus = 1
|
||||||
|
+
|
||||||
|
+# Default maximum number of vCPUs per SB/VM:
|
||||||
|
+# unspecified or == 0 --> will be set to the actual number of physical cores or to the maximum number
|
||||||
|
+# of vCPUs supported by KVM if that number is exceeded
|
||||||
|
+# > 0 <= number of physical cores --> will be set to the specified number
|
||||||
|
+# > number of physical cores --> will be set to the actual number of physical cores or to the maximum number
|
||||||
|
+# of vCPUs supported by KVM if that number is exceeded
|
||||||
|
+# WARNING: Depending of the architecture, the maximum number of vCPUs supported by KVM is used when
|
||||||
|
+# the actual number of physical cores is greater than it.
|
||||||
|
+# WARNING: Be aware that this value impacts the virtual machine's memory footprint and CPU
|
||||||
|
+# the hotplug functionality. For example, `default_maxvcpus = 240` specifies that until 240 vCPUs
|
||||||
|
+# can be added to a SB/VM, but the memory footprint will be big. Another example, with
|
||||||
|
+# `default_maxvcpus = 8` the memory footprint will be small, but 8 will be the maximum number of
|
||||||
|
+# vCPUs supported by the SB/VM. In general, we recommend that you do not edit this variable,
|
||||||
|
+# unless you know what are you doing.
|
||||||
|
+# NOTICE: on arm platform with gicv2 interrupt controller, set it to 8.
|
||||||
|
+default_maxvcpus = @DEFMAXVCPUS@
|
||||||
|
+
|
||||||
|
+# Bridges can be used to hot plug devices.
|
||||||
|
+# Limitations:
|
||||||
|
+# * Currently only pci bridges are supported
|
||||||
|
+# * Until 30 devices per bridge can be hot plugged.
|
||||||
|
+# * Until 5 PCI bridges can be cold plugged per VM.
|
||||||
|
+# This limitation could be a bug in the kernel
|
||||||
|
+# Default number of bridges per SB/VM:
|
||||||
|
+# unspecified or 0 --> will be set to @DEFBRIDGES@
|
||||||
|
+# > 1 <= 5 --> will be set to the specified number
|
||||||
|
+# > 5 --> will be set to 5
|
||||||
|
+default_bridges = @DEFBRIDGES@
|
||||||
|
+
|
||||||
|
+# Default memory size in MiB for SB/VM.
|
||||||
|
+# If unspecified then it will be set @DEFMEMSZ@ MiB.
|
||||||
|
+default_memory = @DEFMEMSZ@
|
||||||
|
+#
|
||||||
|
+# Default memory slots per SB/VM.
|
||||||
|
+# If unspecified then it will be set @DEFMEMSLOTS@.
|
||||||
|
+# This is will determine the times that memory will be hotadded to sandbox/VM.
|
||||||
|
+# memory_slots = @DEFMEMSLOTS@
|
||||||
|
+
|
||||||
|
+# The size in MiB will be plused to max memory of hypervisor.
|
||||||
|
+# It is the memory address space for the NVDIMM devie.
|
||||||
|
+# If set block storage driver (block_device_driver) to "nvdimm",
|
||||||
|
+# should set memory_offset to the size of block device.
|
||||||
|
+# Default 0
|
||||||
|
+# memory_offset = 0
|
||||||
|
+
|
||||||
|
+# Disable block device from being used for a container's rootfs.
|
||||||
|
+# In case of a storage driver like devicemapper where a container's
|
||||||
|
+# root file system is backed by a block device, the block device is passed
|
||||||
|
+# directly to the hypervisor for performance reasons.
|
||||||
|
+# This flag prevents the block device from being passed to the hypervisor,
|
||||||
|
+# 9pfs is used instead to pass the rootfs.
|
||||||
|
+disable_block_device_use = @DEFDISABLEBLOCK@
|
||||||
|
+
|
||||||
|
+# Block storage driver to be used for the hypervisor in case the container
|
||||||
|
+# rootfs is backed by a block device. This is virtio-scsi, virtio-blk
|
||||||
|
+# or nvdimm.
|
||||||
|
+block_device_driver = "@DEFBLOCKSTORAGEDRIVER_STRATOVIRT@"
|
||||||
|
+
|
||||||
|
+# Specifies cache-related options will be set to block devices or not.
|
||||||
|
+# Default false
|
||||||
|
+#block_device_cache_set = true
|
||||||
|
+
|
||||||
|
+# Specifies cache-related options for block devices.
|
||||||
|
+# Denotes whether use of O_DIRECT (bypass the host page cache) is enabled.
|
||||||
|
+# Default false
|
||||||
|
+# block_device_cache_direct = true
|
||||||
|
+
|
||||||
|
+# Specifies cache-related options for block devices.
|
||||||
|
+# Denotes whether flush requests for the device are ignored.
|
||||||
|
+# Default false
|
||||||
|
+# block_device_cache_noflush = true
|
||||||
|
+
|
||||||
|
+# Enable pre allocation of VM RAM, default false
|
||||||
|
+# Enabling this will result in lower container density
|
||||||
|
+# as all of the memory will be allocated and locked
|
||||||
|
+# This is useful when you want to reserve all the memory
|
||||||
|
+# upfront or in the cases where you want memory latencies
|
||||||
|
+# to be very predictable
|
||||||
|
+# Default false
|
||||||
|
+# enable_mem_prealloc = true
|
||||||
|
+
|
||||||
|
+# Enable huge pages for VM RAM, default false
|
||||||
|
+# Enabling this will result in the VM memory
|
||||||
|
+# being allocated using huge pages.
|
||||||
|
+# This is useful when you want to use vhost-user network
|
||||||
|
+# stacks within the container. This will automatically
|
||||||
|
+# result in memory pre allocation
|
||||||
|
+# enable_hugepages = true
|
||||||
|
+
|
||||||
|
+# Enable vIOMMU, default false
|
||||||
|
+# Enabling this will result in the VM having a vIOMMU device
|
||||||
|
+# This will also add the following options to the kernel's
|
||||||
|
+# command line: intel_iommu=on,iommu=pt
|
||||||
|
+# enable_iommu = true
|
||||||
|
+
|
||||||
|
+# Enable swap of vm memory. Default false.
|
||||||
|
+# The behaviour is undefined if mem_prealloc is also set to true
|
||||||
|
+# enable_swap = true
|
||||||
|
+
|
||||||
|
+# This option changes the default hypervisor and kernel parameters
|
||||||
|
+# to enable debug output where available.
|
||||||
|
+#
|
||||||
|
+# Default false
|
||||||
|
+# enable_debug = true
|
||||||
|
+
|
||||||
|
+# Disable the customizations done in the runtime when it detects
|
||||||
|
+# that it is running on top a VMM. This will result in the runtime
|
||||||
|
+# behaving as it would when running on bare metal.
|
||||||
|
+#
|
||||||
|
+# disable_nesting_checks = true
|
||||||
|
+
|
||||||
|
+# This is the msize used for 9p shares. It is the number of bytes
|
||||||
|
+# used for 9p packet payload.
|
||||||
|
+# msize_9p =
|
||||||
|
+
|
||||||
|
+# VFIO devices are hotplugged on a bridge by default.
|
||||||
|
+# Enable hotplugging on root bus. This may be required for devices with
|
||||||
|
+# a large PCI bar, as this is a current limitation with hotplugging on
|
||||||
|
+# a bridge.
|
||||||
|
+# Default false
|
||||||
|
+# hotplug_vfio_on_root_bus = true
|
||||||
|
+
|
||||||
|
+#
|
||||||
|
+# Default entropy source.
|
||||||
|
+# The path to a host source of entropy (including a real hardware RNG)
|
||||||
|
+# /dev/urandom and /dev/random are two main options.
|
||||||
|
+# Be aware that /dev/random is a blocking source of entropy. If the host
|
||||||
|
+# runs out of entropy, the VMs boot time will increase leading to get startup
|
||||||
|
+# timeouts.
|
||||||
|
+# The source of entropy /dev/urandom is non-blocking and provides a
|
||||||
|
+# generally acceptable source of entropy. It should work well for pretty much
|
||||||
|
+# all practical purposes.
|
||||||
|
+# entropy_source= ""
|
||||||
|
+
|
||||||
|
+# List of valid annotations values for entropy_source
|
||||||
|
+# The default if not set is empty (all annotations rejected.)
|
||||||
|
+# Your distribution recommends: @DEFVALIDENTROPYSOURCES@
|
||||||
|
+valid_entropy_sources = @DEFVALIDENTROPYSOURCES@
|
||||||
|
+
|
||||||
|
+# Path to OCI hook binaries in the *guest rootfs*.
|
||||||
|
+# This does not affect host-side hooks which must instead be added to
|
||||||
|
+# the OCI spec passed to the runtime.
|
||||||
|
+#
|
||||||
|
+# You can create a rootfs with hooks by customizing the osbuilder scripts:
|
||||||
|
+# https://github.com/kata-containers/kata-containers/tree/main/tools/osbuilder
|
||||||
|
+#
|
||||||
|
+# Hooks must be stored in a subdirectory of guest_hook_path according to their
|
||||||
|
+# hook type, i.e. "guest_hook_path/{prestart,poststart,poststop}".
|
||||||
|
+# The agent will scan these directories for executable files and add them, in
|
||||||
|
+# lexicographical order, to the lifecycle of the guest container.
|
||||||
|
+# Hooks are executed in the runtime namespace of the guest. See the official documentation:
|
||||||
|
+# https://github.com/opencontainers/runtime-spec/blob/v1.0.1/config.md#posix-platform-hooks
|
||||||
|
+# Warnings will be logged if any error is encountered will scanning for hooks,
|
||||||
|
+# but it will not abort container execution.
|
||||||
|
+# guest_hook_path = "/usr/share/oci/hooks"
|
||||||
|
+
|
||||||
|
+[factory]
|
||||||
|
+# VM templating support. Once enabled, new VMs are created from template
|
||||||
|
+# using vm cloning. They will share the same initial kernel, initramfs and
|
||||||
|
+# agent memory by mapping it readonly. It helps speeding up new container
|
||||||
|
+# creation and saves a lot of memory if there are many kata containers running
|
||||||
|
+# on the same host.
|
||||||
|
+#
|
||||||
|
+# When disabled, new VMs are created from scratch.
|
||||||
|
+#
|
||||||
|
+# Note: Requires "initrd=" to be set ("image=" is not supported).
|
||||||
|
+#
|
||||||
|
+# Default false
|
||||||
|
+#enable_template = true
|
||||||
|
+
|
||||||
|
+[agent.@PROJECT_TYPE@]
|
||||||
|
+# If enabled, make the agent display debug-level messages.
|
||||||
|
+# (default: disabled)
|
||||||
|
+#enable_debug = true
|
||||||
|
+
|
||||||
|
+# Enable agent tracing.
|
||||||
|
+#
|
||||||
|
+# If enabled, the default trace mode is "dynamic" and the
|
||||||
|
+# default trace type is "isolated". The trace mode and type are set
|
||||||
|
+# explicity with the `trace_type=` and `trace_mode=` options.
|
||||||
|
+#
|
||||||
|
+# Notes:
|
||||||
|
+#
|
||||||
|
+# - Tracing is ONLY enabled when `enable_tracing` is set: explicitly
|
||||||
|
+# setting `trace_mode=` and/or `trace_type=` without setting `enable_tracing`
|
||||||
|
+# will NOT activate agent tracing.
|
||||||
|
+#
|
||||||
|
+# - See https://github.com/kata-containers/agent/blob/master/TRACING.md for
|
||||||
|
+# full details.
|
||||||
|
+#
|
||||||
|
+# (default: disabled)
|
||||||
|
+#enable_tracing = true
|
||||||
|
+#
|
||||||
|
+#trace_mode = "dynamic"
|
||||||
|
+#trace_type = "isolated"
|
||||||
|
+
|
||||||
|
+# Comma separated list of kernel modules and their parameters.
|
||||||
|
+# These modules will be loaded in the guest kernel using modprobe(8).
|
||||||
|
+# The following example can be used to load two kernel modules with parameters
|
||||||
|
+# - kernel_modules=["e1000e InterruptThrottleRate=3000,3000,3000 EEE=1", "i915 enable_ppgtt=0"]
|
||||||
|
+# The first word is considered as the module name and the rest as its parameters.
|
||||||
|
+# Container will not be started when:
|
||||||
|
+# * A kernel module is specified and the modprobe command is not installed in the guest
|
||||||
|
+# or it fails loading the module.
|
||||||
|
+# * The module is not available in the guest or it doesn't met the guest kernel
|
||||||
|
+# requirements, like architecture and version.
|
||||||
|
+#
|
||||||
|
+kernel_modules=[]
|
||||||
|
+
|
||||||
|
+# Enable debug console.
|
||||||
|
+
|
||||||
|
+# If enabled, user can connect guest OS running inside hypervisor
|
||||||
|
+# through "kata-runtime exec <sandbox-id>" command
|
||||||
|
+
|
||||||
|
+#debug_console_enabled = true
|
||||||
|
+
|
||||||
|
+# Agent connection dialing timeout value in seconds
|
||||||
|
+# (default: 30)
|
||||||
|
+#dial_timeout = 30
|
||||||
|
+
|
||||||
|
+[netmon]
|
||||||
|
+# If enabled, the network monitoring process gets started when the
|
||||||
|
+# sandbox is created. This allows for the detection of some additional
|
||||||
|
+# network being added to the existing network namespace, after the
|
||||||
|
+# sandbox has been created.
|
||||||
|
+# (default: disabled)
|
||||||
|
+#enable_netmon = true
|
||||||
|
+
|
||||||
|
+# Specify the path to the netmon binary.
|
||||||
|
+path = "@NETMONPATH@"
|
||||||
|
+
|
||||||
|
+# If enabled, netmon messages will be sent to the system log
|
||||||
|
+# (default: disabled)
|
||||||
|
+#enable_debug = true
|
||||||
|
+
|
||||||
|
+[runtime]
|
||||||
|
+# If enabled, the runtime will log additional debug messages to the
|
||||||
|
+# system log
|
||||||
|
+# (default: disabled)
|
||||||
|
+#enable_debug = true
|
||||||
|
+#
|
||||||
|
+# Internetworking model
|
||||||
|
+# Determines how the VM should be connected to the
|
||||||
|
+# the container network interface
|
||||||
|
+# Options:
|
||||||
|
+#
|
||||||
|
+# - macvtap
|
||||||
|
+# Used when the Container network interface can be bridged using
|
||||||
|
+# macvtap.
|
||||||
|
+#
|
||||||
|
+# - none
|
||||||
|
+# Used when customize network. Only creates a tap device. No veth pair.
|
||||||
|
+#
|
||||||
|
+# - tcfilter
|
||||||
|
+# Uses tc filter rules to redirect traffic from the network interface
|
||||||
|
+# provided by plugin to a tap interface connected to the VM.
|
||||||
|
+#
|
||||||
|
+internetworking_model="@DEFNETWORKMODEL_STRATOVIRT@"
|
||||||
|
+
|
||||||
|
+# disable guest seccomp
|
||||||
|
+# Determines whether container seccomp profiles are passed to the virtual
|
||||||
|
+# machine and applied by the kata agent. If set to true, seccomp is not applied
|
||||||
|
+# within the guest
|
||||||
|
+# (default: true)
|
||||||
|
+disable_guest_seccomp=@DEFDISABLEGUESTSECCOMP@
|
||||||
|
+
|
||||||
|
+# If enabled, the runtime will create opentracing.io traces and spans.
|
||||||
|
+# (See https://www.jaegertracing.io/docs/getting-started).
|
||||||
|
+# (default: disabled)
|
||||||
|
+#enable_tracing = true
|
||||||
|
+
|
||||||
|
+# Set the full url to the Jaeger HTTP Thrift collector.
|
||||||
|
+# The default if not set will be "http://localhost:14268/api/traces"
|
||||||
|
+#jaeger_endpoint = ""
|
||||||
|
+
|
||||||
|
+# Sets the username to be used if basic auth is required for Jaeger.
|
||||||
|
+#jaeger_user = ""
|
||||||
|
+
|
||||||
|
+# Sets the password to be used if basic auth is required for Jaeger.
|
||||||
|
+#jaeger_password = ""
|
||||||
|
+
|
||||||
|
+# If enabled, the runtime will not create a network namespace for shim and hypervisor processes.
|
||||||
|
+# This option may have some potential impacts to your host. It should only be used when you know what you're doing.
|
||||||
|
+# `disable_new_netns` conflicts with `enable_netmon`
|
||||||
|
+# `disable_new_netns` conflicts with `internetworking_model=tcfilter` and `internetworking_model=macvtap`. It works only
|
||||||
|
+# with `internetworking_model=none`. The tap device will be in the host network namespace and can connect to a bridge
|
||||||
|
+# (like OVS) directly.
|
||||||
|
+# If you are using docker, `disable_new_netns` only works with `docker run --net=none`
|
||||||
|
+# (default: false)
|
||||||
|
+#disable_new_netns = true
|
||||||
|
+
|
||||||
|
+# if enable, the runtime will add all the kata processes inside one dedicated cgroup.
|
||||||
|
+# The container cgroups in the host are not created, just one single cgroup per sandbox.
|
||||||
|
+# The runtime caller is free to restrict or collect cgroup stats of the overall Kata sandbox.
|
||||||
|
+# The sandbox cgroup path is the parent cgroup of a container with the PodSandbox annotation.
|
||||||
|
+# The sandbox cgroup is constrained if there is no container type annotation.
|
||||||
|
+# See: https://godoc.org/github.com/kata-containers/runtime/virtcontainers#ContainerType
|
||||||
|
+sandbox_cgroup_only=@DEFSANDBOXCGROUPONLY@
|
||||||
|
+
|
||||||
|
+# Enabled experimental feature list, format: ["a", "b"].
|
||||||
|
+# Experimental features are features not stable enough for production,
|
||||||
|
+# they may break compatibility, and are prepared for a big version bump.
|
||||||
|
+# Supported experimental features:
|
||||||
|
+# (default: [])
|
||||||
|
+experimental=@DEFAULTEXPFEATURES@
|
||||||
|
+
|
||||||
|
+# If enabled, user can run pprof tools with shim v2 process through kata-monitor.
|
||||||
|
+# (default: false)
|
||||||
|
+# enable_pprof = true
|
||||||
|
--
|
||||||
|
2.21.1 (Apple Git-122.3)
|
||||||
|
|
||||||
@ -0,0 +1,655 @@
|
|||||||
|
From 45c8e108497eb93d69afd38e6281b837e65cf3ec Mon Sep 17 00:00:00 2001
|
||||||
|
From: Wei Gao <gaowei66@huawei.com>
|
||||||
|
Date: Mon, 9 Aug 2021 14:55:41 +0800
|
||||||
|
Subject: [PATCH 5/6] runtime: add the secure component "ozone" support for
|
||||||
|
hypervisor type stratovirt.
|
||||||
|
|
||||||
|
Signed-off-by: Wei Gao <gaowei66@huawei.com>
|
||||||
|
---
|
||||||
|
src/runtime/Makefile | 4 +
|
||||||
|
src/runtime/arch/amd64-options.mk | 4 +-
|
||||||
|
src/runtime/arch/arm64-options.mk | 4 +-
|
||||||
|
.../config/configuration-stratovirt.toml.in | 10 +
|
||||||
|
.../pkg/katautils/config-settings.go.in | 1 +
|
||||||
|
src/runtime/pkg/katautils/config.go | 18 +
|
||||||
|
src/runtime/virtcontainers/hypervisor.go | 3 +
|
||||||
|
src/runtime/virtcontainers/persist.go | 1 +
|
||||||
|
.../virtcontainers/persist/api/config.go | 3 +
|
||||||
|
src/runtime/virtcontainers/stratovirt.go | 309 ++++++++++++++----
|
||||||
|
10 files changed, 292 insertions(+), 65 deletions(-)
|
||||||
|
|
||||||
|
diff --git a/src/runtime/Makefile b/src/runtime/Makefile
|
||||||
|
index ea2cd296..745bcc10 100644
|
||||||
|
--- a/src/runtime/Makefile
|
||||||
|
+++ b/src/runtime/Makefile
|
||||||
|
@@ -146,6 +146,8 @@ ACRNVALIDCTLPATHS := [\"$(ACRNCTLPATH)\"]
|
||||||
|
|
||||||
|
STRATOVIRTPATH = $(STRATOVIRTBINDIR)/$(STRATOVIRTCMD)
|
||||||
|
STRATOVIRTVALIDHYPERVISORPATHS := [\"$(STRATOVIRTPATH)\"]
|
||||||
|
+STRATOVIRTOZONEPATH = $(STRATOVIRTBINDIR)/$(STRATOVIRTOZONECMD)
|
||||||
|
+STRATOVIRTVALIDOZONEPATHS = [\"$(STRATOVIRTOZONEPATH)\"]
|
||||||
|
|
||||||
|
NETMONCMD := $(BIN_PREFIX)-netmon
|
||||||
|
NETMONPATH := $(PKGLIBEXECDIR)/$(NETMONCMD)
|
||||||
|
@@ -414,6 +416,8 @@ USER_VARS += FCJAILERPATH
|
||||||
|
USER_VARS += FCVALIDJAILERPATHS
|
||||||
|
USER_VARS += STRATOVIRTPATH
|
||||||
|
USER_VARS += STRATOVIRTVALIDHYPERVISORPATHS
|
||||||
|
+USER_VARS += STRATOVIRTOZONEPATH
|
||||||
|
+USER_VARS += STRATOVIRTVALIDOZONEPATHS
|
||||||
|
USER_VARS += SYSCONFIG
|
||||||
|
USER_VARS += IMAGENAME
|
||||||
|
USER_VARS += IMAGEPATH
|
||||||
|
diff --git a/src/runtime/arch/amd64-options.mk b/src/runtime/arch/amd64-options.mk
|
||||||
|
index ff2af9e6..4c6c329a 100644
|
||||||
|
--- a/src/runtime/arch/amd64-options.mk
|
||||||
|
+++ b/src/runtime/arch/amd64-options.mk
|
||||||
|
@@ -25,4 +25,6 @@ ACRNCTLCMD := acrnctl
|
||||||
|
CLHCMD := cloud-hypervisor
|
||||||
|
|
||||||
|
# stratovirt binary name
|
||||||
|
-STRATOVIRTCMD := stratovirt
|
||||||
|
\ No newline at end of file
|
||||||
|
+STRATOVIRTCMD := stratovirt
|
||||||
|
+# stratovirt's ozone binary name
|
||||||
|
+STRATOVIRTOZONECMD := ozone
|
||||||
|
\ No newline at end of file
|
||||||
|
diff --git a/src/runtime/arch/arm64-options.mk b/src/runtime/arch/arm64-options.mk
|
||||||
|
index 2ad3f657..5dfa2c80 100644
|
||||||
|
--- a/src/runtime/arch/arm64-options.mk
|
||||||
|
+++ b/src/runtime/arch/arm64-options.mk
|
||||||
|
@@ -21,4 +21,6 @@ FCJAILERCMD := jailer
|
||||||
|
CLHCMD := cloud-hypervisor
|
||||||
|
|
||||||
|
# stratovirt binary name
|
||||||
|
-STRATOVIRTCMD := stratovirt
|
||||||
|
\ No newline at end of file
|
||||||
|
+STRATOVIRTCMD := stratovirt
|
||||||
|
+# stratovirt's ozone binary name
|
||||||
|
+STRATOVIRTOZONECMD := ozone
|
||||||
|
\ No newline at end of file
|
||||||
|
diff --git a/src/runtime/cli/config/configuration-stratovirt.toml.in b/src/runtime/cli/config/configuration-stratovirt.toml.in
|
||||||
|
index 5c83c3c9..b557b71f 100644
|
||||||
|
--- a/src/runtime/cli/config/configuration-stratovirt.toml.in
|
||||||
|
+++ b/src/runtime/cli/config/configuration-stratovirt.toml.in
|
||||||
|
@@ -26,6 +26,16 @@ enable_annotations = @DEFENABLEANNOTATIONS@
|
||||||
|
# Your distribution recommends: @STRATOVIRTVALIDHYPERVISORPATHS@
|
||||||
|
valid_hypervisor_paths = @STRATOVIRTVALIDHYPERVISORPATHS@
|
||||||
|
|
||||||
|
+# Path for the ozone specific to stratovirt
|
||||||
|
+# If the ozone path is set, stratovirt will be launched in
|
||||||
|
+# ozone secure environment. It is disabled by default.
|
||||||
|
+# ozone_path = "@STRATOVIRTOZONEPATH@"
|
||||||
|
+
|
||||||
|
+# List of valid ozone path values for the hypervisor
|
||||||
|
+# Each member of the list can be a regular expression
|
||||||
|
+# The default if not set is empty (all annotations rejected.)
|
||||||
|
+# valid_jailer_paths = @STRATOVIRTVALIDOZONEPATHS@
|
||||||
|
+
|
||||||
|
# Optional space-separated list of options to pass to the guest kernel.
|
||||||
|
# For example, use `kernel_params = "vsyscall=emulate"` if you are having
|
||||||
|
# trouble running pre-2.15 glibc.
|
||||||
|
diff --git a/src/runtime/pkg/katautils/config-settings.go.in b/src/runtime/pkg/katautils/config-settings.go.in
|
||||||
|
index 7cd9138b..c168c608 100644
|
||||||
|
--- a/src/runtime/pkg/katautils/config-settings.go.in
|
||||||
|
+++ b/src/runtime/pkg/katautils/config-settings.go.in
|
||||||
|
@@ -17,6 +17,7 @@ var defaultInitrdPath = "/usr/share/kata-containers/kata-containers-initrd.img"
|
||||||
|
var defaultFirmwarePath = ""
|
||||||
|
var defaultMachineAccelerators = ""
|
||||||
|
var defaultCPUFeatures = ""
|
||||||
|
+var defaultOzonePath = "/usr/bin/ozone"
|
||||||
|
var systemdUnitName = "kata-containers.target"
|
||||||
|
|
||||||
|
const defaultKernelParams = ""
|
||||||
|
diff --git a/src/runtime/pkg/katautils/config.go b/src/runtime/pkg/katautils/config.go
|
||||||
|
index f94ac4fd..828c2a43 100644
|
||||||
|
--- a/src/runtime/pkg/katautils/config.go
|
||||||
|
+++ b/src/runtime/pkg/katautils/config.go
|
||||||
|
@@ -92,6 +92,7 @@ type hypervisor struct {
|
||||||
|
FileBackedMemRootDir string `toml:"file_mem_backend"`
|
||||||
|
GuestHookPath string `toml:"guest_hook_path"`
|
||||||
|
GuestMemoryDumpPath string `toml:"guest_memory_dump_path"`
|
||||||
|
+ OzonePath string `toml:"ozone_path"`
|
||||||
|
HypervisorPathList []string `toml:"valid_hypervisor_paths"`
|
||||||
|
JailerPathList []string `toml:"valid_jailer_paths"`
|
||||||
|
CtlPathList []string `toml:"valid_ctlpaths"`
|
||||||
|
@@ -452,6 +453,16 @@ func (h hypervisor) getInitrdAndImage() (initrd string, image string, err error)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
+func (h hypervisor) ozonePath() (string, error) {
|
||||||
|
+ p := h.OzonePath
|
||||||
|
+
|
||||||
|
+ if h.OzonePath == "" {
|
||||||
|
+ return "", nil
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ return ResolvePath(p)
|
||||||
|
+}
|
||||||
|
+
|
||||||
|
func (h hypervisor) getRxRateLimiterCfg() uint64 {
|
||||||
|
return h.RxRateLimiterMaxRate
|
||||||
|
}
|
||||||
|
@@ -877,6 +888,11 @@ func newStratovirtHypervisorConfig(h hypervisor) (vc.HypervisorConfig, error) {
|
||||||
|
return vc.HypervisorConfig{}, err
|
||||||
|
}
|
||||||
|
|
||||||
|
+ ozone, err := h.ozonePath()
|
||||||
|
+ if err != nil {
|
||||||
|
+ return vc.HypervisorConfig{}, err
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
kernel, err := h.kernel()
|
||||||
|
if err != nil {
|
||||||
|
return vc.HypervisorConfig{}, err
|
||||||
|
@@ -925,6 +941,7 @@ func newStratovirtHypervisorConfig(h hypervisor) (vc.HypervisorConfig, error) {
|
||||||
|
KernelPath: kernel,
|
||||||
|
InitrdPath: initrd,
|
||||||
|
ImagePath: image,
|
||||||
|
+ OzonePath: ozone,
|
||||||
|
KernelParams: vc.DeserializeParams(strings.Fields(kernelParams)),
|
||||||
|
HypervisorMachineType: machineType,
|
||||||
|
NumVCPUs: h.defaultVCPUs(),
|
||||||
|
@@ -1155,6 +1172,7 @@ func GetDefaultHypervisorConfig() vc.HypervisorConfig {
|
||||||
|
RxRateLimiterMaxRate: defaultRxRateLimiterMaxRate,
|
||||||
|
TxRateLimiterMaxRate: defaultTxRateLimiterMaxRate,
|
||||||
|
SGXEPCSize: defaultSGXEPCSize,
|
||||||
|
+ OzonePath: defaultOzonePath,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
diff --git a/src/runtime/virtcontainers/hypervisor.go b/src/runtime/virtcontainers/hypervisor.go
|
||||||
|
index 615baa80..04e14b4e 100644
|
||||||
|
--- a/src/runtime/virtcontainers/hypervisor.go
|
||||||
|
+++ b/src/runtime/virtcontainers/hypervisor.go
|
||||||
|
@@ -302,6 +302,9 @@ type HypervisorConfig struct {
|
||||||
|
// JailerPathList is the list of jailer paths names allowed in annotations
|
||||||
|
JailerPathList []string
|
||||||
|
|
||||||
|
+ // OzonePath is the ozone executable host path.
|
||||||
|
+ OzonePath string
|
||||||
|
+
|
||||||
|
// BlockDeviceDriver specifies the driver to be used for block device
|
||||||
|
// either VirtioSCSI or VirtioBlock with the default driver being defaultBlockDriver
|
||||||
|
BlockDeviceDriver string
|
||||||
|
diff --git a/src/runtime/virtcontainers/persist.go b/src/runtime/virtcontainers/persist.go
|
||||||
|
index 203495e8..ae499c97 100644
|
||||||
|
--- a/src/runtime/virtcontainers/persist.go
|
||||||
|
+++ b/src/runtime/virtcontainers/persist.go
|
||||||
|
@@ -219,6 +219,7 @@ func (s *Sandbox) dumpConfig(ss *persistapi.SandboxState) {
|
||||||
|
HypervisorCtlPathList: sconfig.HypervisorConfig.HypervisorCtlPathList,
|
||||||
|
JailerPath: sconfig.HypervisorConfig.JailerPath,
|
||||||
|
JailerPathList: sconfig.HypervisorConfig.JailerPathList,
|
||||||
|
+ OzonePath: sconfig.HypervisorConfig.OzonePath,
|
||||||
|
BlockDeviceDriver: sconfig.HypervisorConfig.BlockDeviceDriver,
|
||||||
|
HypervisorMachineType: sconfig.HypervisorConfig.HypervisorMachineType,
|
||||||
|
MemoryPath: sconfig.HypervisorConfig.MemoryPath,
|
||||||
|
diff --git a/src/runtime/virtcontainers/persist/api/config.go b/src/runtime/virtcontainers/persist/api/config.go
|
||||||
|
index 3bd5567d..88903723 100644
|
||||||
|
--- a/src/runtime/virtcontainers/persist/api/config.go
|
||||||
|
+++ b/src/runtime/virtcontainers/persist/api/config.go
|
||||||
|
@@ -76,6 +76,9 @@ type HypervisorConfig struct {
|
||||||
|
// JailerPathList is the list of jailer paths names allowed in annotations
|
||||||
|
JailerPathList []string
|
||||||
|
|
||||||
|
+ // OzonePath is the ozone executable host path.
|
||||||
|
+ OzonePath string
|
||||||
|
+
|
||||||
|
// BlockDeviceDriver specifies the driver to be used for block device
|
||||||
|
// either VirtioSCSI or VirtioBlock with the default driver being defaultBlockDriver
|
||||||
|
BlockDeviceDriver string
|
||||||
|
diff --git a/src/runtime/virtcontainers/stratovirt.go b/src/runtime/virtcontainers/stratovirt.go
|
||||||
|
index 0f473e31..47daa817 100644
|
||||||
|
--- a/src/runtime/virtcontainers/stratovirt.go
|
||||||
|
+++ b/src/runtime/virtcontainers/stratovirt.go
|
||||||
|
@@ -3,6 +3,7 @@ package virtcontainers
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"fmt"
|
||||||
|
+ "io/ioutil"
|
||||||
|
"os"
|
||||||
|
"os/exec"
|
||||||
|
"path/filepath"
|
||||||
|
@@ -24,10 +25,15 @@ import (
|
||||||
|
otelTrace "go.opentelemetry.io/otel/trace"
|
||||||
|
)
|
||||||
|
|
||||||
|
-const defaultDummyMac = "22:33:44:aa:bb:"
|
||||||
|
-const mmioBlkCount = 4
|
||||||
|
-const mmioNetCount = 2
|
||||||
|
-const randomDevice = "/dev/urandom"
|
||||||
|
+const (
|
||||||
|
+ apiSocket = "qmp.socket"
|
||||||
|
+ debugSocket = "console.socket"
|
||||||
|
+ ozoneBaseDir = "/srv/ozone/stratovirt"
|
||||||
|
+ defaultDummyMac = "22:33:44:aa:bb:"
|
||||||
|
+ mmioBlkCount = 4
|
||||||
|
+ mmioNetCount = 2
|
||||||
|
+ randomDevice = "/dev/urandom"
|
||||||
|
+)
|
||||||
|
|
||||||
|
type stratovirtDev struct {
|
||||||
|
dev interface{}
|
||||||
|
@@ -40,10 +46,19 @@ type stratovirt struct {
|
||||||
|
sandbox *Sandbox
|
||||||
|
store persistapi.PersistDriver
|
||||||
|
config HypervisorConfig
|
||||||
|
+ rootfsPath string
|
||||||
|
+ kernelPath string
|
||||||
|
pid int
|
||||||
|
consolePath string
|
||||||
|
socketPath string
|
||||||
|
+ netNSPath string
|
||||||
|
qmpMonitorCh qmpChannel
|
||||||
|
+ ozoneRoot string
|
||||||
|
+ ozoneRes []string
|
||||||
|
+ useOzone bool
|
||||||
|
+ useImage bool
|
||||||
|
+ pidfile string
|
||||||
|
+ logfile string
|
||||||
|
devices []stratovirtDev
|
||||||
|
HotpluggedVCPUs []CPUDevice
|
||||||
|
mmioBlkSlots [mmioBlkCount]bool
|
||||||
|
@@ -66,10 +81,10 @@ func (s *stratovirt) trace(parent context.Context, name string) (otelTrace.Span,
|
||||||
|
return span, ctx
|
||||||
|
}
|
||||||
|
|
||||||
|
-func (s *stratovirt) getKernelCmdLine(useImage bool) string {
|
||||||
|
+func (s *stratovirt) getKernelCmdLine() string {
|
||||||
|
var params []string
|
||||||
|
|
||||||
|
- if useImage {
|
||||||
|
+ if s.useImage {
|
||||||
|
params = append(params, "root=/dev/vda")
|
||||||
|
}
|
||||||
|
|
||||||
|
@@ -100,14 +115,49 @@ func (s *stratovirt) createSandbox(ctx context.Context, id string, networkNS Net
|
||||||
|
|
||||||
|
s.id = id
|
||||||
|
s.config = *hypervisorConfig
|
||||||
|
-
|
||||||
|
- s.socketPath = filepath.Join(s.store.RunVMStoragePath(), id, "qmp.sock")
|
||||||
|
- s.consolePath = filepath.Join(s.store.RunVMStoragePath(), id, "console.sock")
|
||||||
|
+ if s.config.OzonePath == "" {
|
||||||
|
+ s.useOzone = false
|
||||||
|
+ s.pidfile = filepath.Join(s.store.RunVMStoragePath(), s.id, "pid")
|
||||||
|
+ s.logfile = filepath.Join(s.store.RunVMStoragePath(), s.id, "/stratovirt.log")
|
||||||
|
+ s.socketPath = filepath.Join(s.store.RunVMStoragePath(), id, apiSocket)
|
||||||
|
+ s.consolePath = filepath.Join(s.store.RunVMStoragePath(), id, debugSocket)
|
||||||
|
+ } else {
|
||||||
|
+ s.useOzone = true
|
||||||
|
+ s.ozoneRoot = filepath.Join(ozoneBaseDir, s.id)
|
||||||
|
+ s.pidfile = filepath.Join(s.ozoneRoot, "pid")
|
||||||
|
+ s.logfile = filepath.Join(s.ozoneRoot, "stratovirt.log")
|
||||||
|
+ s.socketPath = filepath.Join(s.ozoneRoot, apiSocket)
|
||||||
|
+ s.consolePath = filepath.Join(s.ozoneRoot, debugSocket)
|
||||||
|
+ }
|
||||||
|
+ s.netNSPath = networkNS.NetNsPath
|
||||||
|
s.qmpMonitorCh = qmpChannel{
|
||||||
|
ctx: s.ctx,
|
||||||
|
path: s.socketPath,
|
||||||
|
}
|
||||||
|
|
||||||
|
+ if kernelPath, err := s.config.KernelAssetPath(); err == nil {
|
||||||
|
+ s.kernelPath = kernelPath
|
||||||
|
+ s.ozoneRes = append(s.ozoneRes, s.kernelPath)
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ initrdPath, err := s.config.InitrdAssetPath()
|
||||||
|
+ if err != nil {
|
||||||
|
+ return err
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ if initrdPath == "" {
|
||||||
|
+ imagePath, err := s.config.ImageAssetPath()
|
||||||
|
+ if err != nil {
|
||||||
|
+ return err
|
||||||
|
+ }
|
||||||
|
+ s.useImage = true
|
||||||
|
+ s.rootfsPath = imagePath
|
||||||
|
+ } else {
|
||||||
|
+ s.useImage = false
|
||||||
|
+ s.rootfsPath = initrdPath
|
||||||
|
+ }
|
||||||
|
+ s.ozoneRes = append(s.ozoneRes, s.rootfsPath)
|
||||||
|
+
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
@@ -134,48 +184,43 @@ func (s *stratovirt) waitSandBoxStarted(timeout int) error {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
-func (s *stratovirt) startSandbox(ctx context.Context, timeout int) error {
|
||||||
|
- span, _ := s.trace(ctx, "startSandbox")
|
||||||
|
- defer span.End()
|
||||||
|
-
|
||||||
|
+func (s *stratovirt) createbaseParams() []string {
|
||||||
|
var params []string
|
||||||
|
- var use_image bool
|
||||||
|
+
|
||||||
|
params = append(params, "-name", fmt.Sprintf("sandbox-%s", s.id))
|
||||||
|
- params = append(params, "-qmp", fmt.Sprintf("unix:%s,server,nowait", s.socketPath))
|
||||||
|
+ params = append(params, "-append", s.getKernelCmdLine())
|
||||||
|
+ params = append(params, "-smp", fmt.Sprintf("%d", s.config.NumVCPUs))
|
||||||
|
+ params = append(params, "-m", fmt.Sprintf("%d", uint64(s.config.MemorySize)))
|
||||||
|
+ params = append(params, "-device", "virtio-serial-device")
|
||||||
|
+ params = append(params, "-device", "virtconsole,chardev=charconsole0,id=virtioconsole0")
|
||||||
|
+ params = append(params, "-object", fmt.Sprintf("rng-random,id=objrng0,filename=%s", randomDevice))
|
||||||
|
+ params = append(params, "-device", "virtio-rng-device,rng=objrng0")
|
||||||
|
|
||||||
|
- if kernelPath, err := s.config.KernelAssetPath(); err == nil {
|
||||||
|
- params = append(params, "-kernel", kernelPath)
|
||||||
|
- }
|
||||||
|
+ // daemonize
|
||||||
|
+ params = append(params, "-daemonize")
|
||||||
|
|
||||||
|
- initrdPath, err := s.config.InitrdAssetPath()
|
||||||
|
- if err != nil {
|
||||||
|
- return err
|
||||||
|
+ return params
|
||||||
|
+}
|
||||||
|
+
|
||||||
|
+func (s *stratovirt) createOzoneParams(params []string) ([]string, error) {
|
||||||
|
+ params = append(params, "-qmp", fmt.Sprintf("unix:%s,server,nowait", apiSocket))
|
||||||
|
+ params = append(params, "-chardev", fmt.Sprintf("socket,id=charconsole0,path=%s,server,nowait", debugSocket))
|
||||||
|
+ params = append(params, "-kernel", filepath.Base(s.kernelPath))
|
||||||
|
+ params = append(params, "-pidfile", filepath.Base(s.pidfile))
|
||||||
|
+
|
||||||
|
+ // append logfile only on debug
|
||||||
|
+ if s.config.Debug {
|
||||||
|
+ params = append(params, "-D", filepath.Base(s.logfile))
|
||||||
|
}
|
||||||
|
|
||||||
|
- if initrdPath == "" {
|
||||||
|
- imagePath, err := s.config.ImageAssetPath()
|
||||||
|
- if err != nil {
|
||||||
|
- return err
|
||||||
|
- }
|
||||||
|
- use_image = true
|
||||||
|
+ if s.useImage {
|
||||||
|
s.mmioBlkSlots[0] = true
|
||||||
|
params = append(params, "-device", "virtio-blk-device,drive=rootfs")
|
||||||
|
- params = append(params, "-drive", fmt.Sprintf("id=rootfs,file=%s,direct=off", imagePath))
|
||||||
|
+ params = append(params, "-drive", fmt.Sprintf("id=rootfs,file=%s,direct=off", filepath.Base(s.rootfsPath)))
|
||||||
|
} else {
|
||||||
|
- use_image = false
|
||||||
|
- params = append(params, "-initrd", initrdPath)
|
||||||
|
+ params = append(params, "-initrd", filepath.Base(s.rootfsPath))
|
||||||
|
}
|
||||||
|
|
||||||
|
- params = append(params, "-append", s.getKernelCmdLine(use_image))
|
||||||
|
- params = append(params, "-smp", fmt.Sprintf("%d", s.config.NumVCPUs))
|
||||||
|
- params = append(params, "-m", fmt.Sprintf("%d", uint64(s.config.MemorySize)))
|
||||||
|
- params = append(params, "-device", "virtio-serial-device")
|
||||||
|
- params = append(params, "-device", "virtconsole,chardev=charconsole0,id=virtioconsole0")
|
||||||
|
- params = append(params, "-object", fmt.Sprintf("rng-random,id=objrng0,filename=%s", randomDevice))
|
||||||
|
- params = append(params, "-device", "virtio-rng-device,rng=objrng0")
|
||||||
|
- params = append(params, "-chardev", fmt.Sprintf("socket,id=charconsole0,path=%s,server,nowait", s.consolePath))
|
||||||
|
- params = append(params, "-pidfile", filepath.Join(s.store.RunVMStoragePath(), s.id, "pid"))
|
||||||
|
-
|
||||||
|
// add devices to cmdline
|
||||||
|
for _, d := range s.devices {
|
||||||
|
switch v := d.dev.(type) {
|
||||||
|
@@ -188,8 +233,9 @@ func (s *stratovirt) startSandbox(ctx context.Context, timeout int) error {
|
||||||
|
case config.BlockDrive:
|
||||||
|
id := v.ID
|
||||||
|
path := v.File
|
||||||
|
- params = append(params, "-device", fmt.Sprintf("virtio-blk-device, drive=%s", id))
|
||||||
|
- params = append(params, "-drive", fmt.Sprintf("id=%s,file=%s", id, path))
|
||||||
|
+ s.ozoneRes = append(s.ozoneRes, path)
|
||||||
|
+ params = append(params, "-device", fmt.Sprintf("virtio-blk-device,drive=%s", id))
|
||||||
|
+ params = append(params, "-drive", fmt.Sprintf("id=%s,file=%s", id, filepath.Base(path)))
|
||||||
|
case types.VSock:
|
||||||
|
v.VhostFd.Close()
|
||||||
|
params = append(params, "-device", fmt.Sprintf("vhost-vsock-device,id=vsock-id,guest-cid=%d", v.ContextID))
|
||||||
|
@@ -198,42 +244,125 @@ func (s *stratovirt) startSandbox(ctx context.Context, timeout int) error {
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
- // daemonize
|
||||||
|
- params = append(params, "-daemonize")
|
||||||
|
+ return params, nil
|
||||||
|
+}
|
||||||
|
+
|
||||||
|
+func (s *stratovirt) createParams(params []string) ([]string, error) {
|
||||||
|
+ params = append(params, "-qmp", fmt.Sprintf("unix:%s,server,nowait", s.socketPath))
|
||||||
|
+ params = append(params, "-chardev", fmt.Sprintf("socket,id=charconsole0,path=%s,server,nowait", s.consolePath))
|
||||||
|
+ params = append(params, "-kernel", s.kernelPath)
|
||||||
|
+ params = append(params, "-pidfile", s.pidfile)
|
||||||
|
|
||||||
|
// append logfile only on debug
|
||||||
|
if s.config.Debug {
|
||||||
|
- dir := filepath.Join(s.store.RunVMStoragePath(), s.id)
|
||||||
|
- params = append(params, "-D", fmt.Sprintf("%s/stratovirt.log", dir))
|
||||||
|
+ params = append(params, "-D", s.logfile)
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ if s.useImage {
|
||||||
|
+ s.mmioBlkSlots[0] = true
|
||||||
|
+ params = append(params, "-device", "virtio-blk-device,drive=rootfs")
|
||||||
|
+ params = append(params, "-drive", fmt.Sprintf("id=rootfs,file=%s,direct=off", s.rootfsPath))
|
||||||
|
+ } else {
|
||||||
|
+ params = append(params, "-initrd", s.rootfsPath)
|
||||||
|
}
|
||||||
|
|
||||||
|
- dir := filepath.Join(s.store.RunVMStoragePath(), s.id)
|
||||||
|
- err = os.MkdirAll(dir, DirMode)
|
||||||
|
+ // add devices to cmdline
|
||||||
|
+ for _, d := range s.devices {
|
||||||
|
+ switch v := d.dev.(type) {
|
||||||
|
+ case Endpoint:
|
||||||
|
+ name := v.Name()
|
||||||
|
+ mac := v.HardwareAddr()
|
||||||
|
+ tapName := v.NetworkPair().TapInterface.TAPIface.Name
|
||||||
|
+ params = append(params, "-device", fmt.Sprintf("virtio-net-device,netdev=%s,id=%s,mac=%s", name, name, mac))
|
||||||
|
+ params = append(params, "-netdev", fmt.Sprintf("tap,id=%s,ifname=%s", name, tapName))
|
||||||
|
+ case config.BlockDrive:
|
||||||
|
+ id := v.ID
|
||||||
|
+ path := v.File
|
||||||
|
+ params = append(params, "-device", fmt.Sprintf("virtio-blk-device,drive=%s", id))
|
||||||
|
+ params = append(params, "-drive", fmt.Sprintf("id=%s,file=%s", id, path))
|
||||||
|
+ case types.VSock:
|
||||||
|
+ v.VhostFd.Close()
|
||||||
|
+ params = append(params, "-device", fmt.Sprintf("vhost-vsock-device,id=vsock-id,guest-cid=%d", v.ContextID))
|
||||||
|
+ default:
|
||||||
|
+ s.Logger().Error("Adding device type is unsupported")
|
||||||
|
+ }
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ return params, nil
|
||||||
|
+}
|
||||||
|
+
|
||||||
|
+func (s *stratovirt) startSandbox(ctx context.Context, timeout int) error {
|
||||||
|
+ span, _ := s.trace(ctx, "startSandbox")
|
||||||
|
+ defer span.End()
|
||||||
|
+
|
||||||
|
+ var err error
|
||||||
|
+ var cmd *exec.Cmd
|
||||||
|
+
|
||||||
|
+ params := s.createbaseParams()
|
||||||
|
+
|
||||||
|
+ stratovirtBinPath, err := s.config.HypervisorAssetPath()
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
- defer func() {
|
||||||
|
+
|
||||||
|
+ if s.useOzone {
|
||||||
|
+ var ozoneParams []string
|
||||||
|
+ extend_params, err := s.createOzoneParams(params)
|
||||||
|
if err != nil {
|
||||||
|
- if err := os.RemoveAll(dir); err != nil {
|
||||||
|
- s.Logger().WithError(err).Error("Fail to clean up vm dir %s", dir)
|
||||||
|
+ return err
|
||||||
|
+ }
|
||||||
|
+ ozoneParams = append(ozoneParams, "-exec-file", stratovirtBinPath)
|
||||||
|
+ ozoneParams = append(ozoneParams, "-name", s.id)
|
||||||
|
+ ozoneParams = append(ozoneParams, "-gid", "0")
|
||||||
|
+ ozoneParams = append(ozoneParams, "-uid", "0")
|
||||||
|
+ if s.netNSPath != "" {
|
||||||
|
+ ozoneParams = append(ozoneParams, "-netns", s.netNSPath)
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ ozoneParams = append(ozoneParams, "-source")
|
||||||
|
+ ozoneParams = append(ozoneParams, s.ozoneRes...)
|
||||||
|
+
|
||||||
|
+ defer func() {
|
||||||
|
+ if err != nil {
|
||||||
|
+ ozoneParams = append(ozoneParams, "-clean-resource")
|
||||||
|
+ cmd = exec.CommandContext(s.ctx, s.config.OzonePath, ozoneParams...)
|
||||||
|
+ if err := cmd.Run(); err != nil {
|
||||||
|
+ s.Logger().WithError(err).Error("Failed to clean up ozone dir %s", s.ozoneRoot)
|
||||||
|
+ }
|
||||||
|
}
|
||||||
|
+ }()
|
||||||
|
+
|
||||||
|
+ ozoneParams = append(ozoneParams, "--")
|
||||||
|
+ ozoneParams = append(ozoneParams, extend_params...)
|
||||||
|
+ cmd = exec.CommandContext(s.ctx, s.config.OzonePath, ozoneParams...)
|
||||||
|
+ s.Logger().Info("StratoVirt/Ozone start with params: ", cmd)
|
||||||
|
+ } else {
|
||||||
|
+ params, err = s.createParams(params)
|
||||||
|
+ if err != nil {
|
||||||
|
+ return err
|
||||||
|
}
|
||||||
|
- }()
|
||||||
|
|
||||||
|
- binPath, err := s.config.HypervisorAssetPath()
|
||||||
|
- if err != nil {
|
||||||
|
- s.Logger().WithField("Fail to get hypervisor bin path", err).Error()
|
||||||
|
- return err
|
||||||
|
- }
|
||||||
|
+ dir := filepath.Join(s.store.RunVMStoragePath(), s.id)
|
||||||
|
+ err = os.MkdirAll(dir, DirMode)
|
||||||
|
+ if err != nil {
|
||||||
|
+ return err
|
||||||
|
+ }
|
||||||
|
+ defer func() {
|
||||||
|
+ if err != nil {
|
||||||
|
+ if err := os.RemoveAll(dir); err != nil {
|
||||||
|
+ s.Logger().WithError(err).Error("Fail to clean up vm dir %s", dir)
|
||||||
|
+ }
|
||||||
|
+ }
|
||||||
|
+ }()
|
||||||
|
|
||||||
|
- cmd := exec.CommandContext(s.ctx, binPath, params...)
|
||||||
|
- s.Logger().Info("StratoVirt start with params: ", cmd)
|
||||||
|
+ cmd = exec.CommandContext(s.ctx, stratovirtBinPath, params...)
|
||||||
|
+ s.Logger().Info("StratoVirt start with params: ", cmd)
|
||||||
|
+ }
|
||||||
|
|
||||||
|
if err := cmd.Start(); err != nil {
|
||||||
|
s.Logger().WithField("Error starting hypervisor, please check the params", err).Error()
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
- s.pid = cmd.Process.Pid
|
||||||
|
|
||||||
|
if err = s.waitSandBoxStarted(timeout); err != nil {
|
||||||
|
return err
|
||||||
|
@@ -420,6 +549,7 @@ func (s *stratovirt) hotplugNet(ctx context.Context, endpoint Endpoint, op opera
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *stratovirt) hotplugBlk(drive *config.BlockDrive, op operation) (err error) {
|
||||||
|
+ var filePath string
|
||||||
|
err = s.qmpSetup()
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
@@ -427,13 +557,18 @@ func (s *stratovirt) hotplugBlk(drive *config.BlockDrive, op operation) (err err
|
||||||
|
|
||||||
|
switch op {
|
||||||
|
case addDevice:
|
||||||
|
- driver := "virtio-blk-pci"
|
||||||
|
+ driver := "virtio-blk-mmio"
|
||||||
|
+ if s.useOzone {
|
||||||
|
+ filePath, err = s.updateOzoneRes(drive.File, true)
|
||||||
|
+ } else {
|
||||||
|
+ filePath = drive.File
|
||||||
|
+ }
|
||||||
|
slot, err := s.getDevSlot(drive.VirtPath, false)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("Could not get unused slot for %q", drive.VirtPath)
|
||||||
|
}
|
||||||
|
|
||||||
|
- if err := s.qmpMonitorCh.qmp.ExecuteBlockdevAdd(s.qmpMonitorCh.ctx, drive.File, drive.ID, false); err != nil {
|
||||||
|
+ if err := s.qmpMonitorCh.qmp.ExecuteBlockdevAdd(s.qmpMonitorCh.ctx, filePath, drive.ID, false); err != nil {
|
||||||
|
s.getDevSlot(drive.VirtPath, true)
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
@@ -443,6 +578,9 @@ func (s *stratovirt) hotplugBlk(drive *config.BlockDrive, op operation) (err err
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
case removeDevice:
|
||||||
|
+ if s.useOzone {
|
||||||
|
+ s.updateOzoneRes(drive.File, false)
|
||||||
|
+ }
|
||||||
|
if err := s.qmpMonitorCh.qmp.ExecuteDeviceDel(s.qmpMonitorCh.ctx, drive.ID); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
@@ -582,17 +720,62 @@ func (s *stratovirt) getThreadIDs(ctx context.Context) (vcpuThreadIDs, error) {
|
||||||
|
return tid, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
+func (s *stratovirt) updateOzoneRes(src string, add bool) (string, error) {
|
||||||
|
+ dst := filepath.Join(s.ozoneRoot, filepath.Base(src))
|
||||||
|
+ if add {
|
||||||
|
+ if err := bindMount(context.Background(), src, dst, false, "slave"); err != nil {
|
||||||
|
+ s.Logger().WithField("bindMount failed", err).Error()
|
||||||
|
+ return "", err
|
||||||
|
+ }
|
||||||
|
+ } else {
|
||||||
|
+ syscall.Unmount(dst, syscall.MNT_DETACH)
|
||||||
|
+ }
|
||||||
|
+ return filepath.Base(src), nil
|
||||||
|
+}
|
||||||
|
+
|
||||||
|
+func (s *stratovirt) cleanOzoneRes() {
|
||||||
|
+ s.updateOzoneRes(s.rootfsPath, false)
|
||||||
|
+ s.updateOzoneRes(s.kernelPath, false)
|
||||||
|
+
|
||||||
|
+ if err := os.RemoveAll(s.ozoneRoot); err != nil {
|
||||||
|
+ s.Logger().WithField("cleanupOzone failed", err).Error()
|
||||||
|
+ }
|
||||||
|
+}
|
||||||
|
+
|
||||||
|
func (s *stratovirt) cleanup(ctx context.Context) error {
|
||||||
|
span, _ := s.trace(ctx, "cleanup")
|
||||||
|
defer span.End()
|
||||||
|
|
||||||
|
s.qmpTeardown()
|
||||||
|
+ if s.useOzone {
|
||||||
|
+ s.cleanOzoneRes()
|
||||||
|
+ }
|
||||||
|
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *stratovirt) getPids() []int {
|
||||||
|
- return []int{s.pid}
|
||||||
|
+ var pids []int
|
||||||
|
+ if s.pid != 0 {
|
||||||
|
+ pids = append(pids, s.pid)
|
||||||
|
+ } else {
|
||||||
|
+ pid, err := ioutil.ReadFile(s.pidfile)
|
||||||
|
+ if err != nil {
|
||||||
|
+ s.Logger().WithError(err).Error("Read pid file failed.")
|
||||||
|
+ return []int{0}
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ p, err := strconv.Atoi(strings.Trim(string(pid), "\n\t "))
|
||||||
|
+ if err != nil {
|
||||||
|
+ s.Logger().WithError(err).Error("Get pid from pid file failed.")
|
||||||
|
+ return []int{0}
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ pids = append(pids, p)
|
||||||
|
+ s.pid = p
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ return pids
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *stratovirt) getVirtioFsPid() *int {
|
||||||
|
--
|
||||||
|
2.21.1 (Apple Git-122.3)
|
||||||
|
|
||||||
@ -0,0 +1,295 @@
|
|||||||
|
From 77ed6fefe70edde63b01d797b76f389bc82bb1a0 Mon Sep 17 00:00:00 2001
|
||||||
|
From: Wei Gao <gaowei66@huawei.com>
|
||||||
|
Date: Mon, 9 Aug 2021 14:57:06 +0800
|
||||||
|
Subject: [PATCH 6/6] factory: add the template factory support for hypervisor
|
||||||
|
type stratovirt.
|
||||||
|
|
||||||
|
Signed-off-by: Wei Gao <gaowei66@huawei.com>
|
||||||
|
---
|
||||||
|
src/runtime/pkg/katautils/config.go | 2 +-
|
||||||
|
.../factory/template/template.go | 21 +++--
|
||||||
|
src/runtime/virtcontainers/kata_agent.go | 7 +-
|
||||||
|
src/runtime/virtcontainers/stratovirt.go | 89 +++++++++++++++++--
|
||||||
|
src/runtime/virtcontainers/vm.go | 28 ++++--
|
||||||
|
5 files changed, 125 insertions(+), 22 deletions(-)
|
||||||
|
|
||||||
|
diff --git a/src/runtime/pkg/katautils/config.go b/src/runtime/pkg/katautils/config.go
|
||||||
|
index 828c2a43..718677b4 100644
|
||||||
|
--- a/src/runtime/pkg/katautils/config.go
|
||||||
|
+++ b/src/runtime/pkg/katautils/config.go
|
||||||
|
@@ -1363,7 +1363,7 @@ func checkNetNsConfig(config oci.RuntimeConfig) error {
|
||||||
|
// checkFactoryConfig ensures the VM factory configuration is valid.
|
||||||
|
func checkFactoryConfig(config oci.RuntimeConfig) error {
|
||||||
|
if config.FactoryConfig.Template {
|
||||||
|
- if config.HypervisorConfig.InitrdPath == "" {
|
||||||
|
+ if config.HypervisorConfig.InitrdPath == "" && (config.HypervisorType != vc.StratovirtHypervisor) {
|
||||||
|
return errors.New("Factory option enable_template requires an initrd image")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
diff --git a/src/runtime/virtcontainers/factory/template/template.go b/src/runtime/virtcontainers/factory/template/template.go
|
||||||
|
index 66070126..02497097 100644
|
||||||
|
--- a/src/runtime/virtcontainers/factory/template/template.go
|
||||||
|
+++ b/src/runtime/virtcontainers/factory/template/template.go
|
||||||
|
@@ -96,11 +96,15 @@ func (t *template) prepareTemplateFiles() error {
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
- flags := uintptr(syscall.MS_NOSUID | syscall.MS_NODEV)
|
||||||
|
- opts := fmt.Sprintf("size=%dM", t.config.HypervisorConfig.MemorySize+templateDeviceStateSize)
|
||||||
|
- if err = syscall.Mount("tmpfs", t.statePath, "tmpfs", flags, opts); err != nil {
|
||||||
|
- t.close()
|
||||||
|
- return err
|
||||||
|
+
|
||||||
|
+ // If use hypervisor stratovirt, no need to create template path with ramdisk.
|
||||||
|
+ if t.config.HypervisorType != vc.StratovirtHypervisor {
|
||||||
|
+ flags := uintptr(syscall.MS_NOSUID | syscall.MS_NODEV)
|
||||||
|
+ opts := fmt.Sprintf("size=%dM", t.config.HypervisorConfig.MemorySize+templateDeviceStateSize)
|
||||||
|
+ if err = syscall.Mount("tmpfs", t.statePath, "tmpfs", flags, opts); err != nil {
|
||||||
|
+ t.close()
|
||||||
|
+ return err
|
||||||
|
+ }
|
||||||
|
}
|
||||||
|
f, err := os.Create(t.statePath + "/memory")
|
||||||
|
if err != nil {
|
||||||
|
@@ -126,8 +130,11 @@ func (t *template) createTemplateVM(ctx context.Context) error {
|
||||||
|
}
|
||||||
|
defer vm.Stop(ctx)
|
||||||
|
|
||||||
|
- if err = vm.Disconnect(ctx); err != nil {
|
||||||
|
- return err
|
||||||
|
+ // Create template on hypervisor stratovirt, don't have connection with agent.
|
||||||
|
+ if config.HypervisorType != vc.StratovirtHypervisor {
|
||||||
|
+ if err = vm.Disconnect(ctx); err != nil {
|
||||||
|
+ return err
|
||||||
|
+ }
|
||||||
|
}
|
||||||
|
|
||||||
|
// Sleep a bit to let the agent grpc server clean up
|
||||||
|
diff --git a/src/runtime/virtcontainers/kata_agent.go b/src/runtime/virtcontainers/kata_agent.go
|
||||||
|
index 13d31658..bc882c70 100644
|
||||||
|
--- a/src/runtime/virtcontainers/kata_agent.go
|
||||||
|
+++ b/src/runtime/virtcontainers/kata_agent.go
|
||||||
|
@@ -1306,8 +1306,11 @@ func (k *kataAgent) buildContainerRootfs(ctx context.Context, sandbox *Sandbox,
|
||||||
|
// TODO: remove dependency on shared fs path. shared fs is just one kind of storage source.
|
||||||
|
// we should not always use shared fs path for all kinds of storage. Instead, all storage
|
||||||
|
// should be bind mounted to a tmpfs path for containers to use.
|
||||||
|
- if err := os.MkdirAll(filepath.Join(getMountPath(c.sandbox.id), c.id, c.rootfsSuffix), DirMode); err != nil {
|
||||||
|
- return nil, err
|
||||||
|
+ // If boot from template on stratovirt, no need to mkdir mount path.
|
||||||
|
+ if !((sandbox.config.HypervisorType == StratovirtHypervisor) && sandbox.config.HypervisorConfig.BootFromTemplate) {
|
||||||
|
+ if err := os.MkdirAll(filepath.Join(getMountPath(c.sandbox.id), c.id, c.rootfsSuffix), DirMode); err != nil {
|
||||||
|
+ return nil, err
|
||||||
|
+ }
|
||||||
|
}
|
||||||
|
return rootfs, nil
|
||||||
|
}
|
||||||
|
diff --git a/src/runtime/virtcontainers/stratovirt.go b/src/runtime/virtcontainers/stratovirt.go
|
||||||
|
index 47daa817..e9b2ba85 100644
|
||||||
|
--- a/src/runtime/virtcontainers/stratovirt.go
|
||||||
|
+++ b/src/runtime/virtcontainers/stratovirt.go
|
||||||
|
@@ -48,6 +48,7 @@ type stratovirt struct {
|
||||||
|
config HypervisorConfig
|
||||||
|
rootfsPath string
|
||||||
|
kernelPath string
|
||||||
|
+ templatePath string
|
||||||
|
pid int
|
||||||
|
consolePath string
|
||||||
|
socketPath string
|
||||||
|
@@ -115,7 +116,7 @@ func (s *stratovirt) createSandbox(ctx context.Context, id string, networkNS Net
|
||||||
|
|
||||||
|
s.id = id
|
||||||
|
s.config = *hypervisorConfig
|
||||||
|
- if s.config.OzonePath == "" {
|
||||||
|
+ if (s.config.OzonePath == "") || s.config.BootToBeTemplate {
|
||||||
|
s.useOzone = false
|
||||||
|
s.pidfile = filepath.Join(s.store.RunVMStoragePath(), s.id, "pid")
|
||||||
|
s.logfile = filepath.Join(s.store.RunVMStoragePath(), s.id, "/stratovirt.log")
|
||||||
|
@@ -129,6 +130,20 @@ func (s *stratovirt) createSandbox(ctx context.Context, id string, networkNS Net
|
||||||
|
s.socketPath = filepath.Join(s.ozoneRoot, apiSocket)
|
||||||
|
s.consolePath = filepath.Join(s.ozoneRoot, debugSocket)
|
||||||
|
}
|
||||||
|
+
|
||||||
|
+ if s.config.VMid != "" && s.useOzone {
|
||||||
|
+ // Make sure the symlinks do not exist
|
||||||
|
+ os.RemoveAll(s.ozoneRoot)
|
||||||
|
+ ozoneVmRoot := filepath.Join(ozoneBaseDir, s.config.VMid)
|
||||||
|
+ if err := os.Symlink(ozoneVmRoot, s.ozoneRoot); err != nil {
|
||||||
|
+ return err
|
||||||
|
+ }
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ if s.config.BootFromTemplate || s.config.BootToBeTemplate {
|
||||||
|
+ s.templatePath = strings.Replace(s.config.DevicesStatePath, "/state", "", -1)
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
s.netNSPath = networkNS.NetNsPath
|
||||||
|
s.qmpMonitorCh = qmpChannel{
|
||||||
|
ctx: s.ctx,
|
||||||
|
@@ -221,6 +236,12 @@ func (s *stratovirt) createOzoneParams(params []string) ([]string, error) {
|
||||||
|
params = append(params, "-initrd", filepath.Base(s.rootfsPath))
|
||||||
|
}
|
||||||
|
|
||||||
|
+ // handle boot from template
|
||||||
|
+ if s.config.BootFromTemplate {
|
||||||
|
+ s.ozoneRes = append(s.ozoneRes, s.templatePath)
|
||||||
|
+ params = append(params, "-incoming", fmt.Sprintf("file:%s", filepath.Base(s.templatePath)))
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
// add devices to cmdline
|
||||||
|
for _, d := range s.devices {
|
||||||
|
switch v := d.dev.(type) {
|
||||||
|
@@ -266,6 +287,11 @@ func (s *stratovirt) createParams(params []string) ([]string, error) {
|
||||||
|
params = append(params, "-initrd", s.rootfsPath)
|
||||||
|
}
|
||||||
|
|
||||||
|
+ // handle boot from template
|
||||||
|
+ if s.config.BootFromTemplate {
|
||||||
|
+ params = append(params, "-incoming", fmt.Sprintf("file:%s", s.templatePath))
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
// add devices to cmdline
|
||||||
|
for _, d := range s.devices {
|
||||||
|
switch v := d.dev.(type) {
|
||||||
|
@@ -410,14 +436,55 @@ func (s *stratovirt) stopSandbox(ctx context.Context, force bool) error {
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *stratovirt) pauseSandbox(ctx context.Context) error {
|
||||||
|
- return nil
|
||||||
|
+ span, _ := s.trace(ctx, "pauseSandbox")
|
||||||
|
+ defer span.End()
|
||||||
|
+
|
||||||
|
+ return s.togglePauseSandbox(ctx, true)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *stratovirt) saveSandbox() error {
|
||||||
|
+ s.Logger().Info("save sandbox")
|
||||||
|
+
|
||||||
|
+ err := s.qmpSetup()
|
||||||
|
+ if err != nil {
|
||||||
|
+ return err
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ // BootToBeTemplate sets the VM to be a template that other VMs can can clone from.
|
||||||
|
+ // We would want to bypass shared memory when saving VM to local file through migrate.
|
||||||
|
+ if s.config.BootToBeTemplate {
|
||||||
|
+ err = s.qmpMonitorCh.qmp.ExecSetMigrateArguments(s.qmpMonitorCh.ctx, fmt.Sprintf("file:%s", s.templatePath))
|
||||||
|
+ if err != nil {
|
||||||
|
+ s.Logger().WithError(err).Error("exec migration")
|
||||||
|
+ return err
|
||||||
|
+ }
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *stratovirt) resumeSandbox(ctx context.Context) error {
|
||||||
|
+ span, _ := s.trace(ctx, "resumeSandbox")
|
||||||
|
+ defer span.End()
|
||||||
|
+
|
||||||
|
+ return s.togglePauseSandbox(ctx, false)
|
||||||
|
+}
|
||||||
|
+
|
||||||
|
+func (s *stratovirt) togglePauseSandbox(ctx context.Context, pause bool) error {
|
||||||
|
+ span, _ := s.trace(ctx, "togglePauseSandbox")
|
||||||
|
+ defer span.End()
|
||||||
|
+
|
||||||
|
+ err := s.qmpSetup()
|
||||||
|
+ if err != nil {
|
||||||
|
+ return err
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ if pause {
|
||||||
|
+ s.qmpMonitorCh.qmp.ExecuteStop(s.qmpMonitorCh.ctx)
|
||||||
|
+ } else {
|
||||||
|
+ s.qmpMonitorCh.qmp.ExecuteCont(s.qmpMonitorCh.ctx)
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
@@ -734,11 +801,23 @@ func (s *stratovirt) updateOzoneRes(src string, add bool) (string, error) {
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *stratovirt) cleanOzoneRes() {
|
||||||
|
- s.updateOzoneRes(s.rootfsPath, false)
|
||||||
|
- s.updateOzoneRes(s.kernelPath, false)
|
||||||
|
+ // Umount all resource in ozoneRoot
|
||||||
|
+ if dir, err := ioutil.ReadDir(s.ozoneRoot); err == nil {
|
||||||
|
+ for _, file := range dir {
|
||||||
|
+ syscall.Unmount(filepath.Join(s.ozoneRoot, file.Name()), syscall.MNT_DETACH)
|
||||||
|
+ }
|
||||||
|
+ }
|
||||||
|
|
||||||
|
if err := os.RemoveAll(s.ozoneRoot); err != nil {
|
||||||
|
- s.Logger().WithField("cleanupOzone failed", err).Error()
|
||||||
|
+ s.Logger().WithField("cleanup Ozone failed", err).Error()
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ // If have VMid, the VM is boot from template. ozoneVmRoot also need clean.
|
||||||
|
+ if s.config.VMid != "" {
|
||||||
|
+ ozoneVmRoot := filepath.Join(ozoneBaseDir, s.config.VMid)
|
||||||
|
+ if err := os.RemoveAll(ozoneVmRoot); err != nil {
|
||||||
|
+ s.Logger().WithField("cleanup Ozone failed", err).Error()
|
||||||
|
+ }
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
diff --git a/src/runtime/virtcontainers/vm.go b/src/runtime/virtcontainers/vm.go
|
||||||
|
index e6f02b6e..c4f9df73 100644
|
||||||
|
--- a/src/runtime/virtcontainers/vm.go
|
||||||
|
+++ b/src/runtime/virtcontainers/vm.go
|
||||||
|
@@ -142,13 +142,19 @@ func NewVM(ctx context.Context, config VMConfig) (*VM, error) {
|
||||||
|
}()
|
||||||
|
|
||||||
|
// 4. check agent aliveness
|
||||||
|
- // VMs booted from template are paused, do not check
|
||||||
|
- if !config.HypervisorConfig.BootFromTemplate {
|
||||||
|
+ // On hypervisor StratoVirt, VMs booted from template are running, check agent
|
||||||
|
+ // On other hypervisors, VMs booted from template are paused, do not check
|
||||||
|
+ if config.HypervisorType == StratovirtHypervisor {
|
||||||
|
+ if !config.HypervisorConfig.BootToBeTemplate {
|
||||||
|
+ virtLog.WithField("vm", id).Info("check agent status")
|
||||||
|
+ err = agent.check(ctx)
|
||||||
|
+ }
|
||||||
|
+ } else if !config.HypervisorConfig.BootFromTemplate {
|
||||||
|
virtLog.WithField("vm", id).Info("check agent status")
|
||||||
|
err = agent.check(ctx)
|
||||||
|
- if err != nil {
|
||||||
|
- return nil, err
|
||||||
|
- }
|
||||||
|
+ }
|
||||||
|
+ if err != nil {
|
||||||
|
+ return nil, err
|
||||||
|
}
|
||||||
|
|
||||||
|
return &VM{
|
||||||
|
@@ -329,9 +335,16 @@ func (v *VM) assignSandbox(s *Sandbox) error {
|
||||||
|
// - link 9pfs share path from sandbox dir (/run/kata-containers/shared/sandboxes/sbid/) to vm dir (/run/vc/vm/vmid/shared/)
|
||||||
|
|
||||||
|
vmSharePath := buildVMSharePath(v.id, v.store.RunVMStoragePath())
|
||||||
|
- vmSockDir := filepath.Join(v.store.RunVMStoragePath(), v.id)
|
||||||
|
sbSharePath := getMountPath(s.id)
|
||||||
|
- sbSockDir := filepath.Join(v.store.RunVMStoragePath(), s.id)
|
||||||
|
+ var vmSockDir string
|
||||||
|
+ var sbSockDir string
|
||||||
|
+ if v.hypervisor.hypervisorConfig().OzonePath != "" {
|
||||||
|
+ vmSockDir = filepath.Join(ozoneBaseDir, v.id)
|
||||||
|
+ sbSockDir = filepath.Join(ozoneBaseDir, s.id)
|
||||||
|
+ } else {
|
||||||
|
+ vmSockDir = filepath.Join(v.store.RunVMStoragePath(), v.id)
|
||||||
|
+ sbSockDir = filepath.Join(v.store.RunVMStoragePath(), s.id)
|
||||||
|
+ }
|
||||||
|
|
||||||
|
v.logger().WithFields(logrus.Fields{
|
||||||
|
"vmSharePath": vmSharePath,
|
||||||
|
@@ -359,6 +372,7 @@ func (v *VM) assignSandbox(s *Sandbox) error {
|
||||||
|
|
||||||
|
s.hypervisor = v.hypervisor
|
||||||
|
s.config.HypervisorConfig.VMid = v.id
|
||||||
|
+ s.config.HypervisorConfig.BootFromTemplate = true
|
||||||
|
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
--
|
||||||
|
2.21.1 (Apple Git-122.3)
|
||||||
|
|
||||||
224
patches/0007-kata-containers-support-with-iSulad.patch
Normal file
224
patches/0007-kata-containers-support-with-iSulad.patch
Normal file
@ -0,0 +1,224 @@
|
|||||||
|
From d4605dafaa9c326a5cf24c28d0c1efe6c9997f49 Mon Sep 17 00:00:00 2001
|
||||||
|
From: holyfei <yangfeiyu20092010@163.com>
|
||||||
|
Date: Sat, 21 Aug 2021 17:08:46 +0800
|
||||||
|
Subject: [PATCH] kata-containers: support with iSulad
|
||||||
|
|
||||||
|
reason: support with iSulad
|
||||||
|
|
||||||
|
Signed-off-by: holyfei <yangfeiyu20092010@163.com>
|
||||||
|
---
|
||||||
|
src/agent/rustjail/src/cgroups/fs/mod.rs | 2 +-
|
||||||
|
src/runtime/containerd-shim-v2/container.go | 9 +++
|
||||||
|
src/runtime/containerd-shim-v2/service.go | 55 +++++++++++++++++++
|
||||||
|
src/runtime/containerd-shim-v2/start.go | 10 ++++
|
||||||
|
.../containerd/runtime/v2/shim/shim.go | 8 ++-
|
||||||
|
5 files changed, 81 insertions(+), 3 deletions(-)
|
||||||
|
|
||||||
|
diff --git a/src/agent/rustjail/src/cgroups/fs/mod.rs b/src/agent/rustjail/src/cgroups/fs/mod.rs
|
||||||
|
index 7f41cb4..6c3bb32 100644
|
||||||
|
--- a/src/agent/rustjail/src/cgroups/fs/mod.rs
|
||||||
|
+++ b/src/agent/rustjail/src/cgroups/fs/mod.rs
|
||||||
|
@@ -369,7 +369,7 @@ fn set_memory_resources(cg: &cgroups::Cgroup, memory: &LinuxMemory, update: bool
|
||||||
|
if let Some(swappiness) = memory.swappiness {
|
||||||
|
if (0..=100).contains(&swappiness) {
|
||||||
|
mem_controller.set_swappiness(swappiness as u64)?;
|
||||||
|
- } else {
|
||||||
|
+ } else if swappiness != -1 {
|
||||||
|
return Err(anyhow!(
|
||||||
|
"invalid value:{}. valid memory swappiness range is 0-100",
|
||||||
|
swappiness
|
||||||
|
diff --git a/src/runtime/containerd-shim-v2/container.go b/src/runtime/containerd-shim-v2/container.go
|
||||||
|
index faea0e2..d563888 100644
|
||||||
|
--- a/src/runtime/containerd-shim-v2/container.go
|
||||||
|
+++ b/src/runtime/containerd-shim-v2/container.go
|
||||||
|
@@ -7,10 +7,13 @@ package containerdshim
|
||||||
|
|
||||||
|
import (
|
||||||
|
"io"
|
||||||
|
+ "os"
|
||||||
|
+ "path"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/containerd/containerd/api/types/task"
|
||||||
|
"github.com/containerd/containerd/errdefs"
|
||||||
|
+ cdshim "github.com/containerd/containerd/runtime/v2/shim"
|
||||||
|
taskAPI "github.com/containerd/containerd/runtime/v2/task"
|
||||||
|
"github.com/opencontainers/runtime-spec/specs-go"
|
||||||
|
|
||||||
|
@@ -37,6 +40,8 @@ type container struct {
|
||||||
|
status task.Status
|
||||||
|
terminal bool
|
||||||
|
mounted bool
|
||||||
|
+ exitFifo string
|
||||||
|
+ exitFd *os.File
|
||||||
|
}
|
||||||
|
|
||||||
|
func newContainer(s *service, r *taskAPI.CreateTaskRequest, containerType vc.ContainerType, spec *specs.Spec, mounted bool) (*container, error) {
|
||||||
|
@@ -49,6 +54,9 @@ func newContainer(s *service, r *taskAPI.CreateTaskRequest, containerType vc.Con
|
||||||
|
spec = &specs.Spec{}
|
||||||
|
}
|
||||||
|
|
||||||
|
+ dir := os.Getenv(cdshim.ExitFifoDir)
|
||||||
|
+ exitFifo := path.Join(dir, r.ID, exitFifoName)
|
||||||
|
+
|
||||||
|
c := &container{
|
||||||
|
s: s,
|
||||||
|
spec: spec,
|
||||||
|
@@ -65,6 +73,7 @@ func newContainer(s *service, r *taskAPI.CreateTaskRequest, containerType vc.Con
|
||||||
|
exitCh: make(chan uint32, 1),
|
||||||
|
stdinCloser: make(chan struct{}),
|
||||||
|
mounted: mounted,
|
||||||
|
+ exitFifo: exitFifo,
|
||||||
|
}
|
||||||
|
return c, nil
|
||||||
|
}
|
||||||
|
diff --git a/src/runtime/containerd-shim-v2/service.go b/src/runtime/containerd-shim-v2/service.go
|
||||||
|
index 1003f8e..e13283c 100644
|
||||||
|
--- a/src/runtime/containerd-shim-v2/service.go
|
||||||
|
+++ b/src/runtime/containerd-shim-v2/service.go
|
||||||
|
@@ -6,13 +6,16 @@
|
||||||
|
package containerdshim
|
||||||
|
|
||||||
|
import (
|
||||||
|
+ "bytes"
|
||||||
|
"context"
|
||||||
|
+ "encoding/binary"
|
||||||
|
"io/ioutil"
|
||||||
|
"os"
|
||||||
|
sysexec "os/exec"
|
||||||
|
"sync"
|
||||||
|
"syscall"
|
||||||
|
"time"
|
||||||
|
+ "unsafe"
|
||||||
|
|
||||||
|
eventstypes "github.com/containerd/containerd/api/events"
|
||||||
|
"github.com/containerd/containerd/api/types/task"
|
||||||
|
@@ -51,6 +54,8 @@ const (
|
||||||
|
// A time span used to wait for publish a containerd event,
|
||||||
|
// once it costs a longer time than timeOut, it will be canceld.
|
||||||
|
timeOut = 5 * time.Second
|
||||||
|
+
|
||||||
|
+ exitFifoName = "exit_fifo"
|
||||||
|
)
|
||||||
|
|
||||||
|
var (
|
||||||
|
@@ -1019,6 +1024,10 @@ func (s *service) Wait(ctx context.Context, r *taskAPI.WaitRequest) (_ *taskAPI.
|
||||||
|
func (s *service) processExits() {
|
||||||
|
for e := range s.ec {
|
||||||
|
s.checkProcesses(e)
|
||||||
|
+
|
||||||
|
+ if os.Getenv(cdshim.ExitFifoDir) != "" {
|
||||||
|
+ s.closeExitFifo(e)
|
||||||
|
+ }
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
@@ -1070,3 +1079,49 @@ func (s *service) getContainerStatus(containerID string) (task.Status, error) {
|
||||||
|
|
||||||
|
return status, nil
|
||||||
|
}
|
||||||
|
+
|
||||||
|
+func isBigEndian() (ret bool) {
|
||||||
|
+ i := int(0x1)
|
||||||
|
+ bs := (*[int(unsafe.Sizeof(i))]byte)(unsafe.Pointer(&i))
|
||||||
|
+ return bs[0] == 0
|
||||||
|
+}
|
||||||
|
+
|
||||||
|
+func (s *service) closeExitFifo(e exit) {
|
||||||
|
+ if e.execid != "" {
|
||||||
|
+ // not a container, no need to close exit fifo
|
||||||
|
+ return
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ var ret uint32
|
||||||
|
+ var nativeEndian binary.ByteOrder
|
||||||
|
+
|
||||||
|
+ s.mu.Lock()
|
||||||
|
+ c, err := s.getContainer(e.id)
|
||||||
|
+ s.mu.Unlock()
|
||||||
|
+
|
||||||
|
+ if err != nil {
|
||||||
|
+ logrus.WithError(err).Errorf("Process container:%v exit fifo failed", e.id)
|
||||||
|
+ return
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ ret = <-c.exitCh
|
||||||
|
+ // refill the exitCh with the container process's exit code in case
|
||||||
|
+ // there were other waits on this process.
|
||||||
|
+ c.exitCh <- ret
|
||||||
|
+
|
||||||
|
+ if isBigEndian() {
|
||||||
|
+ nativeEndian = binary.BigEndian
|
||||||
|
+ } else {
|
||||||
|
+ nativeEndian = binary.LittleEndian
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ bytesBuffer := bytes.NewBuffer([]byte{})
|
||||||
|
+ binary.Write(bytesBuffer, nativeEndian, &ret)
|
||||||
|
+
|
||||||
|
+ _, err = c.exitFd.Write(bytesBuffer.Bytes())
|
||||||
|
+ if err != nil {
|
||||||
|
+ logrus.WithError(err).Error("write exit fifo failed")
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ c.exitFd.Close()
|
||||||
|
+}
|
||||||
|
diff --git a/src/runtime/containerd-shim-v2/start.go b/src/runtime/containerd-shim-v2/start.go
|
||||||
|
index 72420e4..e89dc48 100644
|
||||||
|
--- a/src/runtime/containerd-shim-v2/start.go
|
||||||
|
+++ b/src/runtime/containerd-shim-v2/start.go
|
||||||
|
@@ -8,8 +8,11 @@ package containerdshim
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"fmt"
|
||||||
|
+ "golang.org/x/sys/unix"
|
||||||
|
+ "os"
|
||||||
|
|
||||||
|
"github.com/containerd/containerd/api/types/task"
|
||||||
|
+ cdshim "github.com/containerd/containerd/runtime/v2/shim"
|
||||||
|
"github.com/kata-containers/kata-containers/src/runtime/pkg/katautils"
|
||||||
|
)
|
||||||
|
|
||||||
|
@@ -59,6 +62,13 @@ func startContainer(ctx context.Context, s *service, c *container) error {
|
||||||
|
|
||||||
|
c.status = task.StatusRunning
|
||||||
|
|
||||||
|
+ if os.Getenv(cdshim.ExitFifoDir) != "" {
|
||||||
|
+ c.exitFd, err = os.OpenFile(c.exitFifo, unix.O_WRONLY|unix.O_NONBLOCK|unix.O_CLOEXEC, 0)
|
||||||
|
+ if err != nil {
|
||||||
|
+ return err
|
||||||
|
+ }
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
stdin, stdout, stderr, err := s.sandbox.IOStream(c.id, c.id)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
diff --git a/src/runtime/vendor/github.com/containerd/containerd/runtime/v2/shim/shim.go b/src/runtime/vendor/github.com/containerd/containerd/runtime/v2/shim/shim.go
|
||||||
|
index d60d496..946c386 100644
|
||||||
|
--- a/src/runtime/vendor/github.com/containerd/containerd/runtime/v2/shim/shim.go
|
||||||
|
+++ b/src/runtime/vendor/github.com/containerd/containerd/runtime/v2/shim/shim.go
|
||||||
|
@@ -84,6 +84,8 @@ var (
|
||||||
|
action string
|
||||||
|
)
|
||||||
|
|
||||||
|
+var ExitFifoDir = "EXIT_FIFO_DIR"
|
||||||
|
+
|
||||||
|
func parseFlags() {
|
||||||
|
flag.BoolVar(&debugFlag, "debug", false, "enable debug output in logs")
|
||||||
|
flag.StringVar(&namespaceFlag, "namespace", "", "namespace that owns the shim")
|
||||||
|
@@ -198,8 +200,10 @@ func run(id string, initFunc Init, config Config) error {
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
default:
|
||||||
|
- if err := setLogger(ctx, idFlag); err != nil {
|
||||||
|
- return err
|
||||||
|
+ if os.Getenv("EXIT_FIFO_DIR") == "" {
|
||||||
|
+ if err := setLogger(ctx, idFlag); err != nil {
|
||||||
|
+ return err
|
||||||
|
+ }
|
||||||
|
}
|
||||||
|
client := NewShimClient(ctx, service, signals)
|
||||||
|
return client.Serve()
|
||||||
|
--
|
||||||
|
2.23.0
|
||||||
|
|
||||||
29
patches/0008-kata-containers-adpat-with-iSulad.patch
Normal file
29
patches/0008-kata-containers-adpat-with-iSulad.patch
Normal file
@ -0,0 +1,29 @@
|
|||||||
|
From 5ee8e64c8c620f02cb580f1f3349ae63660ca34c Mon Sep 17 00:00:00 2001
|
||||||
|
From: holyfei <yangfeiyu20092010@163.com>
|
||||||
|
Date: Mon, 27 Sep 2021 11:03:58 +0800
|
||||||
|
Subject: [PATCH] kata-containers: adpat with iSulad
|
||||||
|
|
||||||
|
reason: chmod the exec fifo to 644, isula start container
|
||||||
|
and need the permission
|
||||||
|
|
||||||
|
Signed-off-by: holyfei <yangfeiyu20092010@163.com>
|
||||||
|
---
|
||||||
|
src/agent/rustjail/src/container.rs | 2 +-
|
||||||
|
1 file changed, 1 insertion(+), 1 deletion(-)
|
||||||
|
|
||||||
|
diff --git a/src/agent/rustjail/src/container.rs b/src/agent/rustjail/src/container.rs
|
||||||
|
index 748ee486..5113a482 100644
|
||||||
|
--- a/src/agent/rustjail/src/container.rs
|
||||||
|
+++ b/src/agent/rustjail/src/container.rs
|
||||||
|
@@ -822,7 +822,7 @@ impl BaseContainer for LinuxContainer {
|
||||||
|
if stat::stat(fifo_file.as_str()).is_ok() {
|
||||||
|
return Err(anyhow!("exec fifo exists"));
|
||||||
|
}
|
||||||
|
- unistd::mkfifo(fifo_file.as_str(), Mode::from_bits(0o622).unwrap())?;
|
||||||
|
+ unistd::mkfifo(fifo_file.as_str(), Mode::from_bits(0o644).unwrap())?;
|
||||||
|
|
||||||
|
fifofd = fcntl::open(
|
||||||
|
fifo_file.as_str(),
|
||||||
|
--
|
||||||
|
2.23.0
|
||||||
|
|
||||||
@ -0,0 +1,27 @@
|
|||||||
|
From c563b455e3bb0cebfbe2a77c8d5ebac36aac3c76 Mon Sep 17 00:00:00 2001
|
||||||
|
From: jikui <jikui2@huawei.com>
|
||||||
|
Date: Thu, 4 Nov 2021 19:58:33 +0800
|
||||||
|
Subject: [PATCH] kata-runtime: fix kata-runtime hungs when qemu process is D/T
|
||||||
|
state
|
||||||
|
|
||||||
|
Signed-off-by: jikui <jikui2@huawei.com>
|
||||||
|
---
|
||||||
|
src/runtime/vendor/github.com/kata-containers/govmm/qemu/qmp.go | 2 ++
|
||||||
|
1 file changed, 2 insertions(+)
|
||||||
|
|
||||||
|
diff --git a/src/runtime/vendor/github.com/kata-containers/govmm/qemu/qmp.go b/src/runtime/vendor/github.com/kata-containers/govmm/qemu/qmp.go
|
||||||
|
index 97e9245..325250d 100644
|
||||||
|
--- a/src/runtime/vendor/github.com/kata-containers/govmm/qemu/qmp.go
|
||||||
|
+++ b/src/runtime/vendor/github.com/kata-containers/govmm/qemu/qmp.go
|
||||||
|
@@ -717,6 +717,8 @@ func QMPStart(ctx context.Context, socket string, cfg QMPConfig, disconnectedCh
|
||||||
|
if q.version == nil {
|
||||||
|
return nil, nil, fmt.Errorf("failed to find QMP version information")
|
||||||
|
}
|
||||||
|
+ case <-time.After(15 * time.Second):
|
||||||
|
+ return nil, nil, fmt.Errorf("qmp start time out")
|
||||||
|
}
|
||||||
|
|
||||||
|
return q, q.version, nil
|
||||||
|
--
|
||||||
|
2.25.1
|
||||||
|
|
||||||
@ -1,22 +1,18 @@
|
|||||||
From 1efb88fbf554f3977a1a8aa1c79bc70cc1b66953 Mon Sep 17 00:00:00 2001
|
From c355523761598154653466033a1d88643d3fd3df Mon Sep 17 00:00:00 2001
|
||||||
From: jiangpengfei <jiangpengfei9@huawei.com>
|
From: jikui <jikui2@huawei.com>
|
||||||
Date: Sat, 25 Jul 2020 09:22:08 +0800
|
Date: Thu, 4 Nov 2021 20:27:22 +0800
|
||||||
Subject: [PATCH 02/50] kata-runtime: fix kata-runtime skip read lines in
|
Subject: [PATCH] kata-runtime: fix kata-runtime skip read lines in
|
||||||
/proc/mounts file problem
|
/proc/mounts file problem
|
||||||
|
|
||||||
reason: Since /proc/mounts is a virtual file which is changed dynamically by kernel,
|
Signed-off-by: jikui <jikui2@huawei.com>
|
||||||
if we use file pointer to read content in this file line by line, we may miss read
|
|
||||||
some lines. So we retry read /proc/mounts file again to fix this problem.
|
|
||||||
|
|
||||||
Signed-off-by: jiangpengfei <jiangpengfei9@huawei.com>
|
|
||||||
---
|
---
|
||||||
virtcontainers/utils/utils_linux.go | 58 +++++++++++++++++++++++--------------
|
.../virtcontainers/utils/utils_linux.go | 61 +++++++++++--------
|
||||||
1 file changed, 36 insertions(+), 22 deletions(-)
|
1 file changed, 36 insertions(+), 25 deletions(-)
|
||||||
|
|
||||||
diff --git a/virtcontainers/utils/utils_linux.go b/virtcontainers/utils/utils_linux.go
|
diff --git a/src/runtime/virtcontainers/utils/utils_linux.go b/src/runtime/virtcontainers/utils/utils_linux.go
|
||||||
index ad870d63..6cef4cfb 100644
|
index 3c14e0c..c6dbd0a 100644
|
||||||
--- a/virtcontainers/utils/utils_linux.go
|
--- a/src/runtime/virtcontainers/utils/utils_linux.go
|
||||||
+++ b/virtcontainers/utils/utils_linux.go
|
+++ b/src/runtime/virtcontainers/utils/utils_linux.go
|
||||||
@@ -7,15 +7,18 @@ package utils
|
@@ -7,15 +7,18 @@ package utils
|
||||||
|
|
||||||
import (
|
import (
|
||||||
@ -39,12 +35,12 @@ index ad870d63..6cef4cfb 100644
|
|||||||
@@ -93,6 +96,7 @@ const (
|
@@ -93,6 +96,7 @@ const (
|
||||||
procMountsFile = "/proc/mounts"
|
procMountsFile = "/proc/mounts"
|
||||||
|
|
||||||
fieldsPerLine = 6
|
fieldsPerLine = 6
|
||||||
+ maxRetryTimes = 5
|
+ maxRetryTimes = 5
|
||||||
|
vfioAPSysfsDir = "vfio_ap"
|
||||||
)
|
)
|
||||||
|
|
||||||
const (
|
@@ -110,37 +114,44 @@ func GetDevicePathAndFsType(mountPoint string) (devicePath, fsType string, err e
|
||||||
@@ -109,35 +113,45 @@ func GetDevicePathAndFsType(mountPoint string) (devicePath, fsType string, err e
|
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -57,9 +53,7 @@ index ad870d63..6cef4cfb 100644
|
|||||||
- }
|
- }
|
||||||
-
|
-
|
||||||
- defer file.Close()
|
- defer file.Close()
|
||||||
+ for retry <= maxRetryTimes {
|
-
|
||||||
+ var content []byte
|
|
||||||
|
|
||||||
- reader := bufio.NewReader(file)
|
- reader := bufio.NewReader(file)
|
||||||
- for {
|
- for {
|
||||||
- var line string
|
- var line string
|
||||||
@ -67,11 +61,13 @@ index ad870d63..6cef4cfb 100644
|
|||||||
- line, err = reader.ReadString('\n')
|
- line, err = reader.ReadString('\n')
|
||||||
- if err == io.EOF {
|
- if err == io.EOF {
|
||||||
- err = fmt.Errorf("Mount %s not found", mountPoint)
|
- err = fmt.Errorf("Mount %s not found", mountPoint)
|
||||||
|
+ for retry <= maxRetryTimes {
|
||||||
|
+ var content []byte
|
||||||
+ content, err = ioutil.ReadFile(procMountsFile)
|
+ content, err = ioutil.ReadFile(procMountsFile)
|
||||||
+ if err != nil {
|
+ if err != nil {
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
-
|
||||||
- fields := strings.Fields(line)
|
- fields := strings.Fields(line)
|
||||||
- if len(fields) != fieldsPerLine {
|
- if len(fields) != fieldsPerLine {
|
||||||
- err = fmt.Errorf("Incorrect no of fields (expected %d, got %d)) :%s", fieldsPerLine, len(fields), line)
|
- err = fmt.Errorf("Incorrect no of fields (expected %d, got %d)) :%s", fieldsPerLine, len(fields), line)
|
||||||
@ -100,7 +96,7 @@ index ad870d63..6cef4cfb 100644
|
|||||||
+ return
|
+ return
|
||||||
+ }
|
+ }
|
||||||
}
|
}
|
||||||
|
-
|
||||||
- if mountPoint == fields[procPathIndex] {
|
- if mountPoint == fields[procPathIndex] {
|
||||||
- devicePath = fields[procDeviceIndex]
|
- devicePath = fields[procDeviceIndex]
|
||||||
- fsType = fields[procTypeIndex]
|
- fsType = fields[procTypeIndex]
|
||||||
@ -112,6 +108,8 @@ index ad870d63..6cef4cfb 100644
|
|||||||
}
|
}
|
||||||
+ return "", "", fmt.Errorf("retry %d times fail to get devicePath adn fs type", maxRetryTimes)
|
+ return "", "", fmt.Errorf("retry %d times fail to get devicePath adn fs type", maxRetryTimes)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// IsAPVFIOMediatedDevice decides whether a device is a VFIO-AP device
|
||||||
--
|
--
|
||||||
2.14.3 (Apple Git-98)
|
2.25.1
|
||||||
|
|
||||||
@ -0,0 +1,52 @@
|
|||||||
|
From 0797bf1fec9c40b67f2770bc8778e8eaee1657c8 Mon Sep 17 00:00:00 2001
|
||||||
|
From: jikui <jikui2@huawei.com>
|
||||||
|
Date: Fri, 5 Nov 2021 11:35:25 +0800
|
||||||
|
Subject: [PATCH] kata-runtime: keep the process name of qemu same as
|
||||||
|
configured path
|
||||||
|
|
||||||
|
Signed-off-by: jikui <jikui2@huawei.com>
|
||||||
|
---
|
||||||
|
src/runtime/pkg/katautils/config.go | 7 ++++++-
|
||||||
|
src/runtime/pkg/katautils/config_test.go | 4 ++--
|
||||||
|
2 files changed, 8 insertions(+), 3 deletions(-)
|
||||||
|
|
||||||
|
diff --git a/src/runtime/pkg/katautils/config.go b/src/runtime/pkg/katautils/config.go
|
||||||
|
index 718677b..e0ebc84 100644
|
||||||
|
--- a/src/runtime/pkg/katautils/config.go
|
||||||
|
+++ b/src/runtime/pkg/katautils/config.go
|
||||||
|
@@ -172,7 +172,12 @@ func (h hypervisor) path() (string, error) {
|
||||||
|
p = defaultHypervisorPath
|
||||||
|
}
|
||||||
|
|
||||||
|
- return ResolvePath(p)
|
||||||
|
+ absolutePath, err := filepath.Abs(p)
|
||||||
|
+ if err != nil {
|
||||||
|
+ return "", err
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ return absolutePath, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (h hypervisor) ctlpath() (string, error) {
|
||||||
|
diff --git a/src/runtime/pkg/katautils/config_test.go b/src/runtime/pkg/katautils/config_test.go
|
||||||
|
index 0d02534..84b7843 100644
|
||||||
|
--- a/src/runtime/pkg/katautils/config_test.go
|
||||||
|
+++ b/src/runtime/pkg/katautils/config_test.go
|
||||||
|
@@ -983,12 +983,12 @@ func TestHypervisorDefaultsHypervisor(t *testing.T) {
|
||||||
|
assert.NoError(err)
|
||||||
|
assert.Equal(p, defaultHypervisorPath, "default hypervisor path wrong")
|
||||||
|
|
||||||
|
- // test path resolution
|
||||||
|
+ // test path resolution, just return the absolute path instead of resolved path
|
||||||
|
defaultHypervisorPath = testHypervisorLinkPath
|
||||||
|
h = hypervisor{}
|
||||||
|
p, err = h.path()
|
||||||
|
assert.NoError(err)
|
||||||
|
- assert.Equal(p, testHypervisorPath)
|
||||||
|
+ assert.Equal(p, testHypervisorLinkPath)
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestHypervisorDefaultsKernel(t *testing.T) {
|
||||||
|
--
|
||||||
|
2.25.1
|
||||||
|
|
||||||
@ -0,0 +1,56 @@
|
|||||||
|
From e8e8e05538bf2c7bd8feebc44d4a960f453d21e1 Mon Sep 17 00:00:00 2001
|
||||||
|
From: holyfei <yangfeiyu20092010@163.com>
|
||||||
|
Date: Sat, 20 Nov 2021 17:04:39 +0800
|
||||||
|
Subject: [PATCH] kata-containers: modify kernel and image path in
|
||||||
|
configuration.toml
|
||||||
|
|
||||||
|
Signed-off-by: holyfei <yangfeiyu20092010@163.com>
|
||||||
|
---
|
||||||
|
src/runtime/Makefile | 10 +++++-----
|
||||||
|
1 file changed, 5 insertions(+), 5 deletions(-)
|
||||||
|
|
||||||
|
diff --git a/src/runtime/Makefile b/src/runtime/Makefile
|
||||||
|
index 745bcc1..68e60ee 100644
|
||||||
|
--- a/src/runtime/Makefile
|
||||||
|
+++ b/src/runtime/Makefile
|
||||||
|
@@ -49,6 +49,8 @@ BINLIBEXECLIST :=
|
||||||
|
BIN_PREFIX = $(PROJECT_TYPE)
|
||||||
|
PROJECT_DIR = $(PROJECT_TAG)
|
||||||
|
IMAGENAME = $(PROJECT_TAG).img
|
||||||
|
+KERNEL_PATH = /var/lib/kata/kernel
|
||||||
|
+IMAGE_PATH = /var/lib/kata/kata-containers-initrd.img
|
||||||
|
|
||||||
|
TARGET = $(BIN_PREFIX)-runtime
|
||||||
|
TARGET_OUTPUT = $(CURDIR)/$(TARGET)
|
||||||
|
@@ -108,7 +110,7 @@ PKGLIBEXECDIR := $(LIBEXECDIR)/$(PROJECT_DIR)
|
||||||
|
|
||||||
|
KERNELDIR := $(PKGDATADIR)
|
||||||
|
|
||||||
|
-IMAGEPATH := $(PKGDATADIR)/$(IMAGENAME)
|
||||||
|
+IMAGEPATH := $(IMAGE_PATH)
|
||||||
|
FIRMWAREPATH :=
|
||||||
|
|
||||||
|
# Name of default configuration file the runtime will use.
|
||||||
|
@@ -248,8 +250,7 @@ ifneq (,$(QEMUCMD))
|
||||||
|
DEFBLOCKSTORAGEDRIVER_QEMU := virtio-scsi
|
||||||
|
DEFNETWORKMODEL_QEMU := tcfilter
|
||||||
|
KERNELTYPE = uncompressed
|
||||||
|
- KERNELNAME = $(call MAKE_KERNEL_NAME,$(KERNELTYPE))
|
||||||
|
- KERNELPATH = $(KERNELDIR)/$(KERNELNAME)
|
||||||
|
+ KERNELPATH = $(KERNEL_PATH)
|
||||||
|
endif
|
||||||
|
|
||||||
|
ifneq (,$(CLHCMD))
|
||||||
|
@@ -294,8 +295,7 @@ ifneq (,$(STRATOVIRTCMD))
|
||||||
|
DEFBLOCKSTORAGEDRIVER_STRATOVIRT := virtio-mmio
|
||||||
|
DEFNETWORKMODEL_STRATOVIRT := none
|
||||||
|
KENRELTYPE_STRATOVIRT = uncompressed
|
||||||
|
- KERNEL_NAME_STRATOVIRT = $(call MAKE_KERNEL_NAME,$(KENRELTYPE_STRATOVIRT))
|
||||||
|
- KERNELPATH_STRATOVIRT = $(KERNELDIR)/$(KERNEL_NAME_STRATOVIRT)
|
||||||
|
+ KERNELPATH_STRATOVIRT = $(KERNEL_PATH)
|
||||||
|
endif
|
||||||
|
|
||||||
|
ifneq (,$(FCCMD))
|
||||||
|
--
|
||||||
|
2.27.0
|
||||||
|
|
||||||
@ -1,21 +1,18 @@
|
|||||||
From c279f4548ccc534f1c65723bf9994c448e510d3d Mon Sep 17 00:00:00 2001
|
From 13aec526360797b3bce776f35b7e5f5976961b47 Mon Sep 17 00:00:00 2001
|
||||||
From: jiangpengfei <jiangpengfei9@huawei.com>
|
From: jikui <jikui2@huawei.com>
|
||||||
Date: Sat, 25 Jul 2020 11:56:35 +0800
|
Date: Fri, 5 Nov 2021 11:46:17 +0800
|
||||||
Subject: [PATCH 05/50] cgroups: increase delete cgroup retry times
|
Subject: [PATCH] kata-runtime: increase delete cgroup retry times
|
||||||
|
|
||||||
reason: inorder to make sure cgroup dir to be deleted, so we increase
|
Signed-off-by: jikui <jikui2@huawei.com>
|
||||||
the retry times when delete cgroup dir failed.
|
|
||||||
|
|
||||||
Signed-off-by: jiangpengfei <jiangpengfei9@huawei.com>
|
|
||||||
---
|
---
|
||||||
vendor/github.com/containerd/cgroups/cgroup.go | 4 ++--
|
.../vendor/github.com/containerd/cgroups/cgroup.go | 4 ++--
|
||||||
vendor/github.com/containerd/cgroups/utils.go | 9 ++++++---
|
.../vendor/github.com/containerd/cgroups/utils.go | 9 ++++++---
|
||||||
2 files changed, 8 insertions(+), 5 deletions(-)
|
2 files changed, 8 insertions(+), 5 deletions(-)
|
||||||
|
|
||||||
diff --git a/vendor/github.com/containerd/cgroups/cgroup.go b/vendor/github.com/containerd/cgroups/cgroup.go
|
diff --git a/src/runtime/vendor/github.com/containerd/cgroups/cgroup.go b/src/runtime/vendor/github.com/containerd/cgroups/cgroup.go
|
||||||
index 53866685..69612b0a 100644
|
index 5386668..69612b0 100644
|
||||||
--- a/vendor/github.com/containerd/cgroups/cgroup.go
|
--- a/src/runtime/vendor/github.com/containerd/cgroups/cgroup.go
|
||||||
+++ b/vendor/github.com/containerd/cgroups/cgroup.go
|
+++ b/src/runtime/vendor/github.com/containerd/cgroups/cgroup.go
|
||||||
@@ -223,7 +223,7 @@ func (c *cgroup) Delete() error {
|
@@ -223,7 +223,7 @@ func (c *cgroup) Delete() error {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
@ -34,10 +31,10 @@ index 53866685..69612b0a 100644
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
diff --git a/vendor/github.com/containerd/cgroups/utils.go b/vendor/github.com/containerd/cgroups/utils.go
|
diff --git a/src/runtime/vendor/github.com/containerd/cgroups/utils.go b/src/runtime/vendor/github.com/containerd/cgroups/utils.go
|
||||||
index 8a97d04d..82dbe2d3 100644
|
index 8a97d04..82dbe2d 100644
|
||||||
--- a/vendor/github.com/containerd/cgroups/utils.go
|
--- a/src/runtime/vendor/github.com/containerd/cgroups/utils.go
|
||||||
+++ b/vendor/github.com/containerd/cgroups/utils.go
|
+++ b/src/runtime/vendor/github.com/containerd/cgroups/utils.go
|
||||||
@@ -99,16 +99,19 @@ func defaults(root string) ([]Subsystem, error) {
|
@@ -99,16 +99,19 @@ func defaults(root string) ([]Subsystem, error) {
|
||||||
// retrying the remove after a exp timeout
|
// retrying the remove after a exp timeout
|
||||||
func remove(path string) error {
|
func remove(path string) error {
|
||||||
@ -62,5 +59,5 @@ index 8a97d04d..82dbe2d3 100644
|
|||||||
|
|
||||||
// readPids will read all the pids of processes in a cgroup by the provided path
|
// readPids will read all the pids of processes in a cgroup by the provided path
|
||||||
--
|
--
|
||||||
2.14.3 (Apple Git-98)
|
2.25.1
|
||||||
|
|
||||||
@ -0,0 +1,33 @@
|
|||||||
|
From b654687f88281d1dfbbb6c8ede65aa53ec105122 Mon Sep 17 00:00:00 2001
|
||||||
|
From: jikui <jikui2@huawei.com>
|
||||||
|
Date: Fri, 5 Nov 2021 11:55:24 +0800
|
||||||
|
Subject: [PATCH] kata-runtime: fix umount container rootfs dir return invalid
|
||||||
|
argument error
|
||||||
|
|
||||||
|
Signed-off-by: jikui <jikui2@huawei.com>
|
||||||
|
---
|
||||||
|
src/runtime/virtcontainers/container.go | 8 ++++++--
|
||||||
|
1 file changed, 6 insertions(+), 2 deletions(-)
|
||||||
|
|
||||||
|
diff --git a/src/runtime/virtcontainers/container.go b/src/runtime/virtcontainers/container.go
|
||||||
|
index 6e8e1ba..224f096 100644
|
||||||
|
--- a/src/runtime/virtcontainers/container.go
|
||||||
|
+++ b/src/runtime/virtcontainers/container.go
|
||||||
|
@@ -1035,8 +1035,12 @@ func (c *Container) stop(ctx context.Context, force bool) error {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
- if err := bindUnmountContainerRootfs(ctx, getMountPath(c.sandbox.id), c.id); err != nil && !force {
|
||||||
|
- return err
|
||||||
|
+ // umount container rootfs dir only if container use 9p
|
||||||
|
+ // to bind mount host container rootfs to 9p shared dir
|
||||||
|
+ if c.state.BlockDeviceID == "" {
|
||||||
|
+ if err := bindUnmountContainerRootfs(c.ctx, getMountPath(c.sandbox.id), c.id); err != nil && !force {
|
||||||
|
+ return err
|
||||||
|
+ }
|
||||||
|
}
|
||||||
|
|
||||||
|
if err := c.detachDevices(ctx); err != nil && !force {
|
||||||
|
--
|
||||||
|
2.25.1
|
||||||
|
|
||||||
@ -0,0 +1,33 @@
|
|||||||
|
From fe490b4d63871f91c17ad72afef38a3227c8b4d3 Mon Sep 17 00:00:00 2001
|
||||||
|
From: jikui <jikui2@huawei.com>
|
||||||
|
Date: Fri, 5 Nov 2021 12:06:11 +0800
|
||||||
|
Subject: [PATCH] kata-runtime: truncate the log.json file before kata-runtime
|
||||||
|
subcommand executed
|
||||||
|
|
||||||
|
Signed-off-by: jikui <jikui2@huawei.com>
|
||||||
|
---
|
||||||
|
src/runtime/cli/main.go | 8 ++++++++
|
||||||
|
1 file changed, 8 insertions(+)
|
||||||
|
|
||||||
|
diff --git a/src/runtime/cli/main.go b/src/runtime/cli/main.go
|
||||||
|
index 27757c0..7c77764 100644
|
||||||
|
--- a/src/runtime/cli/main.go
|
||||||
|
+++ b/src/runtime/cli/main.go
|
||||||
|
@@ -252,6 +252,14 @@ func beforeSubcommands(c *cli.Context) error {
|
||||||
|
ignoreConfigLogs = true
|
||||||
|
} else {
|
||||||
|
if path := c.GlobalString("log"); path != "" {
|
||||||
|
+ // since we have redirect the kata-runtime log to /var/log/messages, and avoid the
|
||||||
|
+ // path of log.json file to be large in the tmpfs, so we truncate the log.json file
|
||||||
|
+ // every time before subcommand is executed.
|
||||||
|
+ if path != "/dev/null" && katautils.FileExists(path) {
|
||||||
|
+ if err := os.Truncate(path, 0); err != nil {
|
||||||
|
+ return err
|
||||||
|
+ }
|
||||||
|
+ }
|
||||||
|
f, err := os.OpenFile(path, os.O_CREATE|os.O_WRONLY|os.O_APPEND|os.O_SYNC, 0640)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
--
|
||||||
|
2.25.1
|
||||||
|
|
||||||
@ -0,0 +1,248 @@
|
|||||||
|
From 420cb51b47e19556c35423354102fdc3a4d041f0 Mon Sep 17 00:00:00 2001
|
||||||
|
From: jikui <jikui2@huawei.com>
|
||||||
|
Date: Fri, 5 Nov 2021 16:29:40 +0800
|
||||||
|
Subject: [PATCH] kata-runtime: validate sandbox cpu and memory size
|
||||||
|
|
||||||
|
Signed-off-by: jikui <jikui2@huawei.com>
|
||||||
|
---
|
||||||
|
src/runtime/pkg/katautils/config.go | 39 +++++++++++++++++--
|
||||||
|
src/runtime/pkg/katautils/config_test.go | 5 +--
|
||||||
|
src/runtime/virtcontainers/pkg/oci/utils.go | 10 ++---
|
||||||
|
src/runtime/virtcontainers/utils/utils.go | 42 +++++++++++++++++++++
|
||||||
|
4 files changed, 85 insertions(+), 11 deletions(-)
|
||||||
|
|
||||||
|
diff --git a/src/runtime/pkg/katautils/config.go b/src/runtime/pkg/katautils/config.go
|
||||||
|
index e0ebc84..e523ed3 100644
|
||||||
|
--- a/src/runtime/pkg/katautils/config.go
|
||||||
|
+++ b/src/runtime/pkg/katautils/config.go
|
||||||
|
@@ -11,7 +11,6 @@ import (
|
||||||
|
"fmt"
|
||||||
|
"io/ioutil"
|
||||||
|
"path/filepath"
|
||||||
|
- goruntime "runtime"
|
||||||
|
"strings"
|
||||||
|
|
||||||
|
"github.com/BurntSushi/toml"
|
||||||
|
@@ -311,7 +310,7 @@ func (h hypervisor) GetEntropySource() string {
|
||||||
|
}
|
||||||
|
|
||||||
|
func (h hypervisor) defaultVCPUs() uint32 {
|
||||||
|
- numCPUs := goruntime.NumCPU()
|
||||||
|
+ numCPUs := utils.GetPhysicalCPUNumber()
|
||||||
|
|
||||||
|
if h.NumVCPUs < 0 || h.NumVCPUs > int32(numCPUs) {
|
||||||
|
return uint32(numCPUs)
|
||||||
|
@@ -323,8 +322,22 @@ func (h hypervisor) defaultVCPUs() uint32 {
|
||||||
|
return uint32(h.NumVCPUs)
|
||||||
|
}
|
||||||
|
|
||||||
|
+func (h hypervisor) checkVCPUs() error {
|
||||||
|
+ numCPUs := utils.GetPhysicalCPUNumber()
|
||||||
|
+
|
||||||
|
+ if h.NumVCPUs <= 0 {
|
||||||
|
+ return fmt.Errorf("invalid vcpus in configuration.toml! vcpus must larger than 0")
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ if h.NumVCPUs > int32(numCPUs) {
|
||||||
|
+ return fmt.Errorf("invalid vcpus in configuration.toml! vcpus must smaller than max CPUs: %d in machine", numCPUs)
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ return nil
|
||||||
|
+}
|
||||||
|
+
|
||||||
|
func (h hypervisor) defaultMaxVCPUs() uint32 {
|
||||||
|
- numcpus := uint32(goruntime.NumCPU())
|
||||||
|
+ numcpus := uint32(utils.GetPhysicalCPUNumber())
|
||||||
|
maxvcpus := vc.MaxQemuVCPUs()
|
||||||
|
reqVCPUs := h.DefaultMaxVCPUs
|
||||||
|
|
||||||
|
@@ -350,6 +363,18 @@ func (h hypervisor) defaultMemSz() uint32 {
|
||||||
|
return h.MemorySize
|
||||||
|
}
|
||||||
|
|
||||||
|
+func (h hypervisor) checkMemSz() error {
|
||||||
|
+ if h.MemorySize < utils.MinMemorySizeInMB {
|
||||||
|
+ return fmt.Errorf("invalid memory size! Memory size must larger than %d MB", utils.MinMemorySizeInMB)
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ if h.MemorySize > utils.MaxMemorySizeInMB {
|
||||||
|
+ return fmt.Errorf("invalid memory size, memory size must smaller than %d MB", utils.MaxMemorySizeInMB)
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ return nil
|
||||||
|
+}
|
||||||
|
+
|
||||||
|
func (h hypervisor) defaultMemSlots() uint32 {
|
||||||
|
slots := h.MemSlots
|
||||||
|
if slots == 0 {
|
||||||
|
@@ -665,6 +690,14 @@ func newQemuHypervisorConfig(h hypervisor) (vc.HypervisorConfig, error) {
|
||||||
|
return vc.HypervisorConfig{}, err
|
||||||
|
}
|
||||||
|
|
||||||
|
+ if err = h.checkVCPUs(); err != nil {
|
||||||
|
+ return vc.HypervisorConfig{}, err
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ if err = h.checkMemSz(); err != nil {
|
||||||
|
+ return vc.HypervisorConfig{}, err
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
rxRateLimiterMaxRate := h.getRxRateLimiterCfg()
|
||||||
|
txRateLimiterMaxRate := h.getTxRateLimiterCfg()
|
||||||
|
|
||||||
|
diff --git a/src/runtime/pkg/katautils/config_test.go b/src/runtime/pkg/katautils/config_test.go
|
||||||
|
index 84b7843..3782268 100644
|
||||||
|
--- a/src/runtime/pkg/katautils/config_test.go
|
||||||
|
+++ b/src/runtime/pkg/katautils/config_test.go
|
||||||
|
@@ -14,7 +14,6 @@ import (
|
||||||
|
"path"
|
||||||
|
"path/filepath"
|
||||||
|
"reflect"
|
||||||
|
- goruntime "runtime"
|
||||||
|
"strings"
|
||||||
|
"syscall"
|
||||||
|
"testing"
|
||||||
|
@@ -155,7 +154,7 @@ func createAllRuntimeConfigFiles(dir, hypervisor string) (config testRuntimeConf
|
||||||
|
KernelParams: vc.DeserializeParams(strings.Fields(kernelParams)),
|
||||||
|
HypervisorMachineType: machineType,
|
||||||
|
NumVCPUs: defaultVCPUCount,
|
||||||
|
- DefaultMaxVCPUs: uint32(goruntime.NumCPU()),
|
||||||
|
+ DefaultMaxVCPUs: uint32(utils.GetPhysicalCPUNumber()),
|
||||||
|
MemorySize: defaultMemSize,
|
||||||
|
DisableBlockDeviceUse: disableBlockDevice,
|
||||||
|
BlockDeviceDriver: defaultBlockDeviceDriver,
|
||||||
|
@@ -918,7 +917,7 @@ func TestNewClhHypervisorConfig(t *testing.T) {
|
||||||
|
func TestHypervisorDefaults(t *testing.T) {
|
||||||
|
assert := assert.New(t)
|
||||||
|
|
||||||
|
- numCPUs := goruntime.NumCPU()
|
||||||
|
+ numCPUs := utils.GetPhysicalCPUNumber()
|
||||||
|
|
||||||
|
h := hypervisor{}
|
||||||
|
|
||||||
|
diff --git a/src/runtime/virtcontainers/pkg/oci/utils.go b/src/runtime/virtcontainers/pkg/oci/utils.go
|
||||||
|
index efaee4a..ea46ab7 100644
|
||||||
|
--- a/src/runtime/virtcontainers/pkg/oci/utils.go
|
||||||
|
+++ b/src/runtime/virtcontainers/pkg/oci/utils.go
|
||||||
|
@@ -12,15 +12,12 @@ import (
|
||||||
|
"fmt"
|
||||||
|
"path/filepath"
|
||||||
|
"regexp"
|
||||||
|
- goruntime "runtime"
|
||||||
|
"strconv"
|
||||||
|
"strings"
|
||||||
|
"syscall"
|
||||||
|
|
||||||
|
criContainerdAnnotations "github.com/containerd/cri-containerd/pkg/annotations"
|
||||||
|
crioAnnotations "github.com/cri-o/cri-o/pkg/annotations"
|
||||||
|
- specs "github.com/opencontainers/runtime-spec/specs-go"
|
||||||
|
- "github.com/sirupsen/logrus"
|
||||||
|
"k8s.io/apimachinery/pkg/api/resource"
|
||||||
|
|
||||||
|
vc "github.com/kata-containers/kata-containers/src/runtime/virtcontainers"
|
||||||
|
@@ -29,6 +26,9 @@ import (
|
||||||
|
vcAnnotations "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/pkg/annotations"
|
||||||
|
dockershimAnnotations "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/pkg/annotations/dockershim"
|
||||||
|
"github.com/kata-containers/kata-containers/src/runtime/virtcontainers/types"
|
||||||
|
+ "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/utils"
|
||||||
|
+ specs "github.com/opencontainers/runtime-spec/specs-go"
|
||||||
|
+ "github.com/sirupsen/logrus"
|
||||||
|
)
|
||||||
|
|
||||||
|
type annotationContainerType struct {
|
||||||
|
@@ -656,7 +656,7 @@ func addHypervisorCPUOverrides(ocispec specs.Spec, sbConfig *vc.SandboxConfig) e
|
||||||
|
return fmt.Errorf("Error encountered parsing annotation default_vcpus: %v, please specify numeric value", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
- numCPUs := goruntime.NumCPU()
|
||||||
|
+ numCPUs := utils.GetPhysicalCPUNumber()
|
||||||
|
|
||||||
|
if uint32(vcpus) > uint32(numCPUs) {
|
||||||
|
return fmt.Errorf("Number of cpus %d specified in annotation default_vcpus is greater than the number of CPUs %d on the system", vcpus, numCPUs)
|
||||||
|
@@ -671,7 +671,7 @@ func addHypervisorCPUOverrides(ocispec specs.Spec, sbConfig *vc.SandboxConfig) e
|
||||||
|
return fmt.Errorf("Error encountered parsing annotation for default_maxvcpus: %v, please specify positive numeric value", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
- numCPUs := goruntime.NumCPU()
|
||||||
|
+ numCPUs := utils.GetPhysicalCPUNumber()
|
||||||
|
max := uint32(maxVCPUs)
|
||||||
|
|
||||||
|
if max > uint32(numCPUs) {
|
||||||
|
diff --git a/src/runtime/virtcontainers/utils/utils.go b/src/runtime/virtcontainers/utils/utils.go
|
||||||
|
index e49f55a..7023318 100644
|
||||||
|
--- a/src/runtime/virtcontainers/utils/utils.go
|
||||||
|
+++ b/src/runtime/virtcontainers/utils/utils.go
|
||||||
|
@@ -6,12 +6,14 @@
|
||||||
|
package utils
|
||||||
|
|
||||||
|
import (
|
||||||
|
+ "bufio"
|
||||||
|
"crypto/rand"
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
"os"
|
||||||
|
"os/exec"
|
||||||
|
"path/filepath"
|
||||||
|
+ "strings"
|
||||||
|
"syscall"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
@@ -28,11 +30,26 @@ const fileMode0755 = os.FileMode(0755)
|
||||||
|
// MibToBytesShift the number to shift needed to convert MiB to Bytes
|
||||||
|
const MibToBytesShift = 20
|
||||||
|
|
||||||
|
+const (
|
||||||
|
+ // Min needed memory size to start a Kata VM
|
||||||
|
+ MinMemorySizeInMB = 300
|
||||||
|
+ MinMemorySizeInByte = MinMemorySizeInMB << MibToBytesShift
|
||||||
|
+
|
||||||
|
+ // Max support memory size in the Kata VM
|
||||||
|
+ MaxMemorySizeInMB = 512 * 1024
|
||||||
|
+ MaxMemorySizeInByte = MaxMemorySizeInMB << MibToBytesShift
|
||||||
|
+)
|
||||||
|
+
|
||||||
|
// MaxSocketPathLen is the effective maximum Unix domain socket length.
|
||||||
|
//
|
||||||
|
// See unix(7).
|
||||||
|
const MaxSocketPathLen = 107
|
||||||
|
|
||||||
|
+const (
|
||||||
|
+ procCPUInfoPath = "/proc/cpuinfo"
|
||||||
|
+ processorIdentifier = "processor"
|
||||||
|
+)
|
||||||
|
+
|
||||||
|
// VHostVSockDevicePath path to vhost-vsock device
|
||||||
|
var VHostVSockDevicePath = "/dev/vhost-vsock"
|
||||||
|
|
||||||
|
@@ -390,3 +407,28 @@ outer:
|
||||||
|
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
+
|
||||||
|
+// GetPhysicalCPUNumber return the number of the CPUs in the physical machine
|
||||||
|
+func GetPhysicalCPUNumber() int {
|
||||||
|
+ f, err := os.Open(procCPUInfoPath)
|
||||||
|
+ if err != nil {
|
||||||
|
+ return 0
|
||||||
|
+ }
|
||||||
|
+ defer f.Close()
|
||||||
|
+
|
||||||
|
+ cpuNum := 0
|
||||||
|
+ s := bufio.NewScanner(f)
|
||||||
|
+ for s.Scan() {
|
||||||
|
+ if err := s.Err(); err != nil {
|
||||||
|
+ return 0
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ fields := strings.Fields(s.Text())
|
||||||
|
+ if len(fields) > 0 {
|
||||||
|
+ if fields[0] == processorIdentifier {
|
||||||
|
+ cpuNum++
|
||||||
|
+ }
|
||||||
|
+ }
|
||||||
|
+ }
|
||||||
|
+ return cpuNum
|
||||||
|
+}
|
||||||
|
--
|
||||||
|
2.25.1
|
||||||
|
|
||||||
@ -0,0 +1,26 @@
|
|||||||
|
From 514a0a19e5f37458d1a7e44c8b2fb4bff3b12d5b Mon Sep 17 00:00:00 2001
|
||||||
|
From: jikui <jikui2@huawei.com>
|
||||||
|
Date: Fri, 5 Nov 2021 16:40:05 +0800
|
||||||
|
Subject: [PATCH] kata-runtime: fix delete sandbox failed problem
|
||||||
|
|
||||||
|
Signed-off-by: jikui <jikui2@huawei.com>
|
||||||
|
---
|
||||||
|
src/runtime/virtcontainers/sandbox.go | 2 ++
|
||||||
|
1 file changed, 2 insertions(+)
|
||||||
|
|
||||||
|
diff --git a/src/runtime/virtcontainers/sandbox.go b/src/runtime/virtcontainers/sandbox.go
|
||||||
|
index 33ae6f0..09cb2eb 100644
|
||||||
|
--- a/src/runtime/virtcontainers/sandbox.go
|
||||||
|
+++ b/src/runtime/virtcontainers/sandbox.go
|
||||||
|
@@ -1114,6 +1114,8 @@ func (s *Sandbox) CreateContainer(ctx context.Context, contConfig ContainerConfi
|
||||||
|
if len(s.config.Containers) > 0 {
|
||||||
|
// delete container config
|
||||||
|
s.config.Containers = s.config.Containers[:len(s.config.Containers)-1]
|
||||||
|
+ // need to flush change to persist storage
|
||||||
|
+ _ = s.storeSandbox(ctx)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}()
|
||||||
|
--
|
||||||
|
2.25.1
|
||||||
|
|
||||||
@ -0,0 +1,38 @@
|
|||||||
|
From 8251be558d6824fa1dce37836b7f1d6ec6be6e9f Mon Sep 17 00:00:00 2001
|
||||||
|
From: jikui <jikui2@huawei.com>
|
||||||
|
Date: Tue, 30 Nov 2021 10:36:27 +0800
|
||||||
|
Subject: [PATCH] kata-runtime: check VFIO when create device
|
||||||
|
|
||||||
|
Signed-off-by: jikui <jikui2@huawei.com>
|
||||||
|
---
|
||||||
|
src/runtime/virtcontainers/device/manager/manager.go | 7 ++++++-
|
||||||
|
1 file changed, 6 insertions(+), 1 deletion(-)
|
||||||
|
|
||||||
|
diff --git a/src/runtime/virtcontainers/device/manager/manager.go b/src/runtime/virtcontainers/device/manager/manager.go
|
||||||
|
index 4515609..3afc148 100644
|
||||||
|
--- a/src/runtime/virtcontainers/device/manager/manager.go
|
||||||
|
+++ b/src/runtime/virtcontainers/device/manager/manager.go
|
||||||
|
@@ -10,6 +10,7 @@ import (
|
||||||
|
"context"
|
||||||
|
"encoding/hex"
|
||||||
|
"errors"
|
||||||
|
+ "fmt"
|
||||||
|
"sync"
|
||||||
|
|
||||||
|
"github.com/sirupsen/logrus"
|
||||||
|
@@ -116,7 +117,11 @@ func (dm *deviceManager) createDevice(devInfo config.DeviceInfo) (dev api.Device
|
||||||
|
}()
|
||||||
|
|
||||||
|
if existingDev := dm.findDeviceByMajorMinor(devInfo.Major, devInfo.Minor); existingDev != nil {
|
||||||
|
- return existingDev, nil
|
||||||
|
+ if isVFIO(devInfo.HostPath) {
|
||||||
|
+ return nil, fmt.Errorf("device %s is replicated in the same Pod!", devInfo.ContainerPath)
|
||||||
|
+ } else {
|
||||||
|
+ return existingDev, nil
|
||||||
|
+ }
|
||||||
|
}
|
||||||
|
|
||||||
|
// device ID must be generated by manager instead of device itself
|
||||||
|
--
|
||||||
|
2.25.1
|
||||||
|
|
||||||
@ -1,23 +1,19 @@
|
|||||||
From 7ab7ff54efa3925a8d372f7830d31b87f8d01ea8 Mon Sep 17 00:00:00 2001
|
From 51e17297b61fdb9ec560bae2f6c36003c8ebdfc0 Mon Sep 17 00:00:00 2001
|
||||||
From: jiangpengfei <jiangpengfei9@huawei.com>
|
From: jikui <jikui2@huawei.com>
|
||||||
Date: Mon, 17 Aug 2020 17:39:18 +0800
|
Date: Tue, 30 Nov 2021 11:29:50 +0800
|
||||||
Subject: [PATCH 33/50] network: do not delete the exist tap device in the host
|
Subject: [PATCH] kata-runtime: do not delete the exist tap device in the host
|
||||||
|
|
||||||
reason: If hotplug a exist tap device into Kata VM, kata-runtime
|
Signed-off-by: jikui <jikui2@huawei.com>
|
||||||
should not delete this exist tap device, because this tap device
|
|
||||||
is controlled by other network components.
|
|
||||||
|
|
||||||
Signed-off-by: jiangpengfei <jiangpengfei9@huawei.com>
|
|
||||||
---
|
---
|
||||||
virtcontainers/tap_endpoint.go | 16 +++++++++++++---
|
src/runtime/virtcontainers/tap_endpoint.go | 16 +++++++++++++---
|
||||||
virtcontainers/veth_endpoint.go | 3 +++
|
src/runtime/virtcontainers/veth_endpoint.go | 3 +++
|
||||||
2 files changed, 16 insertions(+), 3 deletions(-)
|
2 files changed, 16 insertions(+), 3 deletions(-)
|
||||||
|
|
||||||
diff --git a/virtcontainers/tap_endpoint.go b/virtcontainers/tap_endpoint.go
|
diff --git a/src/runtime/virtcontainers/tap_endpoint.go b/src/runtime/virtcontainers/tap_endpoint.go
|
||||||
index c897670e..2cf70dce 100644
|
index 9617945..5eafd15 100644
|
||||||
--- a/virtcontainers/tap_endpoint.go
|
--- a/src/runtime/virtcontainers/tap_endpoint.go
|
||||||
+++ b/virtcontainers/tap_endpoint.go
|
+++ b/src/runtime/virtcontainers/tap_endpoint.go
|
||||||
@@ -77,7 +77,7 @@ func (endpoint *TapEndpoint) Detach(netNsCreated bool, netNsPath string) error {
|
@@ -80,7 +80,7 @@ func (endpoint *TapEndpoint) Detach(ctx context.Context, netNsCreated bool, netN
|
||||||
|
|
||||||
networkLogger().WithField("endpoint-type", TapEndpointType).Info("Detaching endpoint")
|
networkLogger().WithField("endpoint-type", TapEndpointType).Info("Detaching endpoint")
|
||||||
return doNetNS(netNsPath, func(_ ns.NetNS) error {
|
return doNetNS(netNsPath, func(_ ns.NetNS) error {
|
||||||
@ -26,9 +22,9 @@ index c897670e..2cf70dce 100644
|
|||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -91,6 +91,9 @@ func (endpoint *TapEndpoint) HotAttach(h hypervisor) error {
|
@@ -94,6 +94,9 @@ func (endpoint *TapEndpoint) HotAttach(ctx context.Context, h hypervisor) error
|
||||||
|
|
||||||
if _, err := h.hotplugAddDevice(endpoint, netDev); err != nil {
|
if _, err := h.hotplugAddDevice(ctx, endpoint, netDev); err != nil {
|
||||||
networkLogger().WithError(err).Error("Error attach tap ep")
|
networkLogger().WithError(err).Error("Error attach tap ep")
|
||||||
+ if errUnTap := unTapNetwork(endpoint); errUnTap != nil {
|
+ if errUnTap := unTapNetwork(endpoint); errUnTap != nil {
|
||||||
+ networkLogger().WithError(errUnTap).Errorf("Error rollback tap %s", endpoint.TapInterface.TAPIface.Name)
|
+ networkLogger().WithError(errUnTap).Errorf("Error rollback tap %s", endpoint.TapInterface.TAPIface.Name)
|
||||||
@ -36,8 +32,8 @@ index c897670e..2cf70dce 100644
|
|||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
return nil
|
return nil
|
||||||
@@ -100,7 +103,7 @@ func (endpoint *TapEndpoint) HotAttach(h hypervisor) error {
|
@@ -103,7 +106,7 @@ func (endpoint *TapEndpoint) HotAttach(ctx context.Context, h hypervisor) error
|
||||||
func (endpoint *TapEndpoint) HotDetach(h hypervisor, netNsCreated bool, netNsPath string) error {
|
func (endpoint *TapEndpoint) HotDetach(ctx context.Context, h hypervisor, netNsCreated bool, netNsPath string) error {
|
||||||
networkLogger().Info("Hot detaching tap endpoint")
|
networkLogger().Info("Hot detaching tap endpoint")
|
||||||
if err := doNetNS(netNsPath, func(_ ns.NetNS) error {
|
if err := doNetNS(netNsPath, func(_ ns.NetNS) error {
|
||||||
- return unTapNetwork(endpoint.TapInterface.TAPIface.Name)
|
- return unTapNetwork(endpoint.TapInterface.TAPIface.Name)
|
||||||
@ -45,7 +41,7 @@ index c897670e..2cf70dce 100644
|
|||||||
}); err != nil {
|
}); err != nil {
|
||||||
networkLogger().WithError(err).Warn("Error un-bridging tap ep")
|
networkLogger().WithError(err).Warn("Error un-bridging tap ep")
|
||||||
}
|
}
|
||||||
@@ -185,7 +188,14 @@ func tapNetwork(endpoint *TapEndpoint, numCPUs uint32, disableVhostNet bool) err
|
@@ -174,7 +177,14 @@ func tapNetwork(endpoint *TapEndpoint, numCPUs uint32, disableVhostNet bool) err
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -61,13 +57,13 @@ index c897670e..2cf70dce 100644
|
|||||||
netHandle, err := netlink.NewHandle()
|
netHandle, err := netlink.NewHandle()
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
diff --git a/virtcontainers/veth_endpoint.go b/virtcontainers/veth_endpoint.go
|
diff --git a/src/runtime/virtcontainers/veth_endpoint.go b/src/runtime/virtcontainers/veth_endpoint.go
|
||||||
index 9ece6a74..0f2ec9ba 100644
|
index f93ca21..5f2435c 100644
|
||||||
--- a/virtcontainers/veth_endpoint.go
|
--- a/src/runtime/virtcontainers/veth_endpoint.go
|
||||||
+++ b/virtcontainers/veth_endpoint.go
|
+++ b/src/runtime/virtcontainers/veth_endpoint.go
|
||||||
@@ -119,6 +119,9 @@ func (endpoint *VethEndpoint) HotAttach(h hypervisor) error {
|
@@ -124,6 +124,9 @@ func (endpoint *VethEndpoint) HotAttach(ctx context.Context, h hypervisor) error
|
||||||
|
|
||||||
if _, err := h.hotplugAddDevice(endpoint, netDev); err != nil {
|
if _, err := h.hotplugAddDevice(ctx, endpoint, netDev); err != nil {
|
||||||
networkLogger().WithError(err).Error("Error attach virtual ep")
|
networkLogger().WithError(err).Error("Error attach virtual ep")
|
||||||
+ if errDisconn := xDisconnectVMNetwork(endpoint); errDisconn != nil {
|
+ if errDisconn := xDisconnectVMNetwork(endpoint); errDisconn != nil {
|
||||||
+ networkLogger().WithError(errDisconn).Error("Error rollback virtual ep")
|
+ networkLogger().WithError(errDisconn).Error("Error rollback virtual ep")
|
||||||
@ -76,5 +72,5 @@ index 9ece6a74..0f2ec9ba 100644
|
|||||||
}
|
}
|
||||||
return nil
|
return nil
|
||||||
--
|
--
|
||||||
2.14.3 (Apple Git-98)
|
2.25.1
|
||||||
|
|
||||||
@ -0,0 +1,33 @@
|
|||||||
|
From 39ac2f6929aea7e404c4597683e43fba7949964c Mon Sep 17 00:00:00 2001
|
||||||
|
From: jikui <jikui2@huawei.com>
|
||||||
|
Date: Tue, 30 Nov 2021 11:47:36 +0800
|
||||||
|
Subject: [PATCH] kata-runtime: do not ignore updateInterface return error
|
||||||
|
|
||||||
|
Signed-off-by: jikui <jikui2@huawei.com>
|
||||||
|
---
|
||||||
|
src/runtime/virtcontainers/kata_agent.go | 8 ++++++++
|
||||||
|
1 file changed, 8 insertions(+)
|
||||||
|
|
||||||
|
diff --git a/src/runtime/virtcontainers/kata_agent.go b/src/runtime/virtcontainers/kata_agent.go
|
||||||
|
index 607f1c8..19d09bb 100644
|
||||||
|
--- a/src/runtime/virtcontainers/kata_agent.go
|
||||||
|
+++ b/src/runtime/virtcontainers/kata_agent.go
|
||||||
|
@@ -620,7 +620,15 @@ func (k *kataAgent) updateInterface(ctx context.Context, ifc *pbTypes.Interface)
|
||||||
|
"interface-requested": fmt.Sprintf("%+v", ifc),
|
||||||
|
"resulting-interface": fmt.Sprintf("%+v", resultingInterface),
|
||||||
|
}).WithError(err).Error("update interface request failed")
|
||||||
|
+ return nil, err
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ // need to judege resultingInterface is not nil, otherwise may cause
|
||||||
|
+ // deference nil pointer panic problem
|
||||||
|
+ if resultingInterface == nil {
|
||||||
|
+ return nil, fmt.Errorf("resultingInterface should not be nil")
|
||||||
|
}
|
||||||
|
+
|
||||||
|
if resultInterface, ok := resultingInterface.(*pbTypes.Interface); ok {
|
||||||
|
return resultInterface, err
|
||||||
|
}
|
||||||
|
--
|
||||||
|
2.25.1
|
||||||
|
|
||||||
@ -0,0 +1,45 @@
|
|||||||
|
From 1bb584ac783d057b64058b02cc96c1c67dd7bf30 Mon Sep 17 00:00:00 2001
|
||||||
|
From: jikui <jikui2@huawei.com>
|
||||||
|
Date: Tue, 30 Nov 2021 12:02:42 +0800
|
||||||
|
Subject: [PATCH] kata-runtime: fix the block device not remove in devManager
|
||||||
|
|
||||||
|
Signed-off-by: jikui <jikui2@huawei.com>
|
||||||
|
---
|
||||||
|
src/runtime/virtcontainers/container.go | 11 ++++++++++-
|
||||||
|
1 file changed, 10 insertions(+), 1 deletion(-)
|
||||||
|
|
||||||
|
diff --git a/src/runtime/virtcontainers/container.go b/src/runtime/virtcontainers/container.go
|
||||||
|
index 224f096..dd4317e 100644
|
||||||
|
--- a/src/runtime/virtcontainers/container.go
|
||||||
|
+++ b/src/runtime/virtcontainers/container.go
|
||||||
|
@@ -1302,6 +1302,7 @@ func (c *Container) plugDevice(ctx context.Context, devicePath string) error {
|
||||||
|
}
|
||||||
|
|
||||||
|
if c.checkBlockDeviceSupport(ctx) && stat.Mode&unix.S_IFBLK == unix.S_IFBLK {
|
||||||
|
+ var err error
|
||||||
|
b, err := c.sandbox.devManager.NewDevice(config.DeviceInfo{
|
||||||
|
HostPath: devicePath,
|
||||||
|
ContainerPath: filepath.Join(kataGuestSharedDir(), c.id),
|
||||||
|
@@ -1313,10 +1314,18 @@ func (c *Container) plugDevice(ctx context.Context, devicePath string) error {
|
||||||
|
return fmt.Errorf("device manager failed to create rootfs device for %q: %v", devicePath, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
+ defer func() {
|
||||||
|
+ if err != nil {
|
||||||
|
+ if newErr := c.sandbox.devManager.RemoveDevice(b.DeviceID()); newErr != nil {
|
||||||
|
+ c.Logger().WithError(newErr).Error("fail rollback to remove block device")
|
||||||
|
+ }
|
||||||
|
+ }
|
||||||
|
+ }()
|
||||||
|
+
|
||||||
|
c.state.BlockDeviceID = b.DeviceID()
|
||||||
|
|
||||||
|
// attach rootfs device
|
||||||
|
- if err := c.sandbox.devManager.AttachDevice(ctx, b.DeviceID(), c.sandbox); err != nil {
|
||||||
|
+ if err = c.sandbox.devManager.AttachDevice(ctx, b.DeviceID(), c.sandbox); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
--
|
||||||
|
2.25.1
|
||||||
|
|
||||||
@ -0,0 +1,40 @@
|
|||||||
|
From c9e8df6902ecd72223a3837de953233f00f94093 Mon Sep 17 00:00:00 2001
|
||||||
|
From: holyfei <yangfeiyu20092010@163.com>
|
||||||
|
Date: Fri, 10 Dec 2021 09:43:30 +0800
|
||||||
|
Subject: [PATCH] kata-containers: modify stratovirt config file
|
||||||
|
|
||||||
|
Signed-off-by: holyfei <yangfeiyu20092010@163.com>
|
||||||
|
---
|
||||||
|
src/runtime/Makefile | 2 +-
|
||||||
|
src/runtime/cli/config/configuration-stratovirt.toml.in | 2 +-
|
||||||
|
2 files changed, 2 insertions(+), 2 deletions(-)
|
||||||
|
|
||||||
|
diff --git a/src/runtime/Makefile b/src/runtime/Makefile
|
||||||
|
index 68e60ee..ff93ab3 100644
|
||||||
|
--- a/src/runtime/Makefile
|
||||||
|
+++ b/src/runtime/Makefile
|
||||||
|
@@ -293,7 +293,7 @@ ifneq (,$(STRATOVIRTCMD))
|
||||||
|
|
||||||
|
# stratovirt-specific options (all should be suffixed by "_STRATOVIRT")
|
||||||
|
DEFBLOCKSTORAGEDRIVER_STRATOVIRT := virtio-mmio
|
||||||
|
- DEFNETWORKMODEL_STRATOVIRT := none
|
||||||
|
+ DEFNETWORKMODEL_STRATOVIRT := tcfilter
|
||||||
|
KENRELTYPE_STRATOVIRT = uncompressed
|
||||||
|
KERNELPATH_STRATOVIRT = $(KERNEL_PATH)
|
||||||
|
endif
|
||||||
|
diff --git a/src/runtime/cli/config/configuration-stratovirt.toml.in b/src/runtime/cli/config/configuration-stratovirt.toml.in
|
||||||
|
index b557b71..753e3dc 100644
|
||||||
|
--- a/src/runtime/cli/config/configuration-stratovirt.toml.in
|
||||||
|
+++ b/src/runtime/cli/config/configuration-stratovirt.toml.in
|
||||||
|
@@ -13,7 +13,7 @@
|
||||||
|
[hypervisor.stratovirt]
|
||||||
|
path = "@STRATOVIRTPATH@"
|
||||||
|
kernel = "@KERNELPATH_STRATOVIRT@"
|
||||||
|
-image = "@IMAGEPATH@"
|
||||||
|
+initrd = "@IMAGEPATH@"
|
||||||
|
|
||||||
|
# List of valid annotation names for the hypervisor
|
||||||
|
# Each member of the list is a regular expression, which is the base name
|
||||||
|
--
|
||||||
|
2.27.0
|
||||||
|
|
||||||
243
patches/0023-stratovirt-update-configuration-toml-file.patch
Normal file
243
patches/0023-stratovirt-update-configuration-toml-file.patch
Normal file
@ -0,0 +1,243 @@
|
|||||||
|
From 1f83147653208f01effab0cf89209b8454d15f03 Mon Sep 17 00:00:00 2001
|
||||||
|
From: "Xinle.Guo" <guoxinle1@huawei.com>
|
||||||
|
Date: Mon, 10 Jan 2022 10:49:44 +0800
|
||||||
|
Subject: [PATCH 1/5] stratovirt: update configuration toml file
|
||||||
|
|
||||||
|
1.Adapt to default machine type as microvm.
|
||||||
|
2.Add more configuration items.
|
||||||
|
3.Modify toml file format.
|
||||||
|
|
||||||
|
Signed-off-by: Xinle.Guo <guoxinle1@huawei.com>
|
||||||
|
---
|
||||||
|
src/runtime/Makefile | 2 +
|
||||||
|
.../config/configuration-stratovirt.toml.in | 67 ++++++++++++-------
|
||||||
|
2 files changed, 46 insertions(+), 23 deletions(-)
|
||||||
|
|
||||||
|
diff --git a/src/runtime/Makefile b/src/runtime/Makefile
|
||||||
|
index ff93ab3..bade196 100644
|
||||||
|
--- a/src/runtime/Makefile
|
||||||
|
+++ b/src/runtime/Makefile
|
||||||
|
@@ -292,6 +292,7 @@ ifneq (,$(STRATOVIRTCMD))
|
||||||
|
CONFIGS += $(CONFIG_STRATOVIRT)
|
||||||
|
|
||||||
|
# stratovirt-specific options (all should be suffixed by "_STRATOVIRT")
|
||||||
|
+ DEFMACHINETYPE_STRATOVIRT := microvm
|
||||||
|
DEFBLOCKSTORAGEDRIVER_STRATOVIRT := virtio-mmio
|
||||||
|
DEFNETWORKMODEL_STRATOVIRT := tcfilter
|
||||||
|
KENRELTYPE_STRATOVIRT = uncompressed
|
||||||
|
@@ -437,6 +438,7 @@ USER_VARS += FIRMWAREPATH
|
||||||
|
USER_VARS += MACHINEACCELERATORS
|
||||||
|
USER_VARS += CPUFEATURES
|
||||||
|
USER_VARS += DEFMACHINETYPE_CLH
|
||||||
|
+USER_VARS += DEFMACHINETYPE_STRATOVIRT
|
||||||
|
USER_VARS += KERNELPARAMS
|
||||||
|
USER_VARS += LIBEXECDIR
|
||||||
|
USER_VARS += LOCALSTATEDIR
|
||||||
|
diff --git a/src/runtime/cli/config/configuration-stratovirt.toml.in b/src/runtime/cli/config/configuration-stratovirt.toml.in
|
||||||
|
index 753e3dc..db46665 100644
|
||||||
|
--- a/src/runtime/cli/config/configuration-stratovirt.toml.in
|
||||||
|
+++ b/src/runtime/cli/config/configuration-stratovirt.toml.in
|
||||||
|
@@ -14,6 +14,8 @@
|
||||||
|
path = "@STRATOVIRTPATH@"
|
||||||
|
kernel = "@KERNELPATH_STRATOVIRT@"
|
||||||
|
initrd = "@IMAGEPATH@"
|
||||||
|
+#image = "/var/lib/kata/kata-containers-rootfs.img"
|
||||||
|
+machine_type = "@DEFMACHINETYPE_STRATOVIRT@"
|
||||||
|
|
||||||
|
# List of valid annotation names for the hypervisor
|
||||||
|
# Each member of the list is a regular expression, which is the base name
|
||||||
|
@@ -29,12 +31,12 @@ valid_hypervisor_paths = @STRATOVIRTVALIDHYPERVISORPATHS@
|
||||||
|
# Path for the ozone specific to stratovirt
|
||||||
|
# If the ozone path is set, stratovirt will be launched in
|
||||||
|
# ozone secure environment. It is disabled by default.
|
||||||
|
-# ozone_path = "@STRATOVIRTOZONEPATH@"
|
||||||
|
+#ozone_path = "@STRATOVIRTOZONEPATH@"
|
||||||
|
|
||||||
|
# List of valid ozone path values for the hypervisor
|
||||||
|
# Each member of the list can be a regular expression
|
||||||
|
# The default if not set is empty (all annotations rejected.)
|
||||||
|
-# valid_jailer_paths = @STRATOVIRTVALIDOZONEPATHS@
|
||||||
|
+#valid_jailer_paths = @STRATOVIRTVALIDOZONEPATHS@
|
||||||
|
|
||||||
|
# Optional space-separated list of options to pass to the guest kernel.
|
||||||
|
# For example, use `kernel_params = "vsyscall=emulate"` if you are having
|
||||||
|
@@ -87,18 +89,18 @@ default_bridges = @DEFBRIDGES@
|
||||||
|
# Default memory size in MiB for SB/VM.
|
||||||
|
# If unspecified then it will be set @DEFMEMSZ@ MiB.
|
||||||
|
default_memory = @DEFMEMSZ@
|
||||||
|
-#
|
||||||
|
+
|
||||||
|
# Default memory slots per SB/VM.
|
||||||
|
# If unspecified then it will be set @DEFMEMSLOTS@.
|
||||||
|
# This is will determine the times that memory will be hotadded to sandbox/VM.
|
||||||
|
-# memory_slots = @DEFMEMSLOTS@
|
||||||
|
+#memory_slots = @DEFMEMSLOTS@
|
||||||
|
|
||||||
|
# The size in MiB will be plused to max memory of hypervisor.
|
||||||
|
# It is the memory address space for the NVDIMM devie.
|
||||||
|
# If set block storage driver (block_device_driver) to "nvdimm",
|
||||||
|
# should set memory_offset to the size of block device.
|
||||||
|
# Default 0
|
||||||
|
-# memory_offset = 0
|
||||||
|
+#memory_offset = 0
|
||||||
|
|
||||||
|
# Disable block device from being used for a container's rootfs.
|
||||||
|
# In case of a storage driver like devicemapper where a container's
|
||||||
|
@@ -108,6 +110,14 @@ default_memory = @DEFMEMSZ@
|
||||||
|
# 9pfs is used instead to pass the rootfs.
|
||||||
|
disable_block_device_use = @DEFDISABLEBLOCK@
|
||||||
|
|
||||||
|
+# Shared file system type:
|
||||||
|
+# - virtio-fs (default)
|
||||||
|
+# - virtio-9p
|
||||||
|
+shared_fs = "virtio-fs"
|
||||||
|
+
|
||||||
|
+# Path to vhost-user-fs daemon.
|
||||||
|
+virtio_fs_daemon = "/usr/bin/vhost_user_fs"
|
||||||
|
+
|
||||||
|
# Block storage driver to be used for the hypervisor in case the container
|
||||||
|
# rootfs is backed by a block device. This is virtio-scsi, virtio-blk
|
||||||
|
# or nvdimm.
|
||||||
|
@@ -120,12 +130,17 @@ block_device_driver = "@DEFBLOCKSTORAGEDRIVER_STRATOVIRT@"
|
||||||
|
# Specifies cache-related options for block devices.
|
||||||
|
# Denotes whether use of O_DIRECT (bypass the host page cache) is enabled.
|
||||||
|
# Default false
|
||||||
|
-# block_device_cache_direct = true
|
||||||
|
+#block_device_cache_direct = true
|
||||||
|
|
||||||
|
# Specifies cache-related options for block devices.
|
||||||
|
# Denotes whether flush requests for the device are ignored.
|
||||||
|
# Default false
|
||||||
|
-# block_device_cache_noflush = true
|
||||||
|
+#block_device_cache_noflush = true
|
||||||
|
+
|
||||||
|
+# Enable iothreads to be used. This causes IO to be
|
||||||
|
+# handled in a separate IO thread. This is currently only implemented
|
||||||
|
+# for virtio blk.
|
||||||
|
+#enable_iothreads = true
|
||||||
|
|
||||||
|
# Enable pre allocation of VM RAM, default false
|
||||||
|
# Enabling this will result in lower container density
|
||||||
|
@@ -134,7 +149,7 @@ block_device_driver = "@DEFBLOCKSTORAGEDRIVER_STRATOVIRT@"
|
||||||
|
# upfront or in the cases where you want memory latencies
|
||||||
|
# to be very predictable
|
||||||
|
# Default false
|
||||||
|
-# enable_mem_prealloc = true
|
||||||
|
+#enable_mem_prealloc = true
|
||||||
|
|
||||||
|
# Enable huge pages for VM RAM, default false
|
||||||
|
# Enabling this will result in the VM memory
|
||||||
|
@@ -142,42 +157,48 @@ block_device_driver = "@DEFBLOCKSTORAGEDRIVER_STRATOVIRT@"
|
||||||
|
# This is useful when you want to use vhost-user network
|
||||||
|
# stacks within the container. This will automatically
|
||||||
|
# result in memory pre allocation
|
||||||
|
-# enable_hugepages = true
|
||||||
|
+#enable_hugepages = true
|
||||||
|
|
||||||
|
# Enable vIOMMU, default false
|
||||||
|
# Enabling this will result in the VM having a vIOMMU device
|
||||||
|
# This will also add the following options to the kernel's
|
||||||
|
# command line: intel_iommu=on,iommu=pt
|
||||||
|
-# enable_iommu = true
|
||||||
|
+#enable_iommu = true
|
||||||
|
|
||||||
|
# Enable swap of vm memory. Default false.
|
||||||
|
# The behaviour is undefined if mem_prealloc is also set to true
|
||||||
|
-# enable_swap = true
|
||||||
|
+#enable_swap = true
|
||||||
|
|
||||||
|
# This option changes the default hypervisor and kernel parameters
|
||||||
|
# to enable debug output where available.
|
||||||
|
#
|
||||||
|
# Default false
|
||||||
|
-# enable_debug = true
|
||||||
|
+#enable_debug = true
|
||||||
|
|
||||||
|
# Disable the customizations done in the runtime when it detects
|
||||||
|
# that it is running on top a VMM. This will result in the runtime
|
||||||
|
# behaving as it would when running on bare metal.
|
||||||
|
#
|
||||||
|
-# disable_nesting_checks = true
|
||||||
|
+#disable_nesting_checks = true
|
||||||
|
|
||||||
|
# This is the msize used for 9p shares. It is the number of bytes
|
||||||
|
# used for 9p packet payload.
|
||||||
|
-# msize_9p =
|
||||||
|
+#msize_9p =
|
||||||
|
|
||||||
|
# VFIO devices are hotplugged on a bridge by default.
|
||||||
|
# Enable hotplugging on root bus. This may be required for devices with
|
||||||
|
# a large PCI bar, as this is a current limitation with hotplugging on
|
||||||
|
# a bridge.
|
||||||
|
# Default false
|
||||||
|
-# hotplug_vfio_on_root_bus = true
|
||||||
|
+#hotplug_vfio_on_root_bus = true
|
||||||
|
+
|
||||||
|
+# Before hot plugging a PCIe device, you need to add a pcie_root_port device.
|
||||||
|
+# Use this parameter when using some large PCI bar devices, such as Nvidia GPU
|
||||||
|
+# The value means the number of pcie_root_port
|
||||||
|
+# This value is valid when hotplug_vfio_on_root_bus is true and machine_type is "q35"
|
||||||
|
+# Default 0
|
||||||
|
+pcie_root_port = 2
|
||||||
|
|
||||||
|
-#
|
||||||
|
# Default entropy source.
|
||||||
|
# The path to a host source of entropy (including a real hardware RNG)
|
||||||
|
# /dev/urandom and /dev/random are two main options.
|
||||||
|
@@ -187,7 +208,7 @@ block_device_driver = "@DEFBLOCKSTORAGEDRIVER_STRATOVIRT@"
|
||||||
|
# The source of entropy /dev/urandom is non-blocking and provides a
|
||||||
|
# generally acceptable source of entropy. It should work well for pretty much
|
||||||
|
# all practical purposes.
|
||||||
|
-# entropy_source= ""
|
||||||
|
+entropy_source= "@DEFENTROPYSOURCE@"
|
||||||
|
|
||||||
|
# List of valid annotations values for entropy_source
|
||||||
|
# The default if not set is empty (all annotations rejected.)
|
||||||
|
@@ -209,7 +230,7 @@ valid_entropy_sources = @DEFVALIDENTROPYSOURCES@
|
||||||
|
# https://github.com/opencontainers/runtime-spec/blob/v1.0.1/config.md#posix-platform-hooks
|
||||||
|
# Warnings will be logged if any error is encountered will scanning for hooks,
|
||||||
|
# but it will not abort container execution.
|
||||||
|
-# guest_hook_path = "/usr/share/oci/hooks"
|
||||||
|
+#guest_hook_path = "/usr/share/oci/hooks"
|
||||||
|
|
||||||
|
[factory]
|
||||||
|
# VM templating support. Once enabled, new VMs are created from template
|
||||||
|
@@ -312,14 +333,14 @@ path = "@NETMONPATH@"
|
||||||
|
# Uses tc filter rules to redirect traffic from the network interface
|
||||||
|
# provided by plugin to a tap interface connected to the VM.
|
||||||
|
#
|
||||||
|
-internetworking_model="@DEFNETWORKMODEL_STRATOVIRT@"
|
||||||
|
+internetworking_model = "@DEFNETWORKMODEL_STRATOVIRT@"
|
||||||
|
|
||||||
|
# disable guest seccomp
|
||||||
|
# Determines whether container seccomp profiles are passed to the virtual
|
||||||
|
# machine and applied by the kata agent. If set to true, seccomp is not applied
|
||||||
|
# within the guest
|
||||||
|
# (default: true)
|
||||||
|
-disable_guest_seccomp=@DEFDISABLEGUESTSECCOMP@
|
||||||
|
+disable_guest_seccomp = @DEFDISABLEGUESTSECCOMP@
|
||||||
|
|
||||||
|
# If enabled, the runtime will create opentracing.io traces and spans.
|
||||||
|
# (See https://www.jaegertracing.io/docs/getting-started).
|
||||||
|
@@ -352,15 +373,15 @@ disable_guest_seccomp=@DEFDISABLEGUESTSECCOMP@
|
||||||
|
# The sandbox cgroup path is the parent cgroup of a container with the PodSandbox annotation.
|
||||||
|
# The sandbox cgroup is constrained if there is no container type annotation.
|
||||||
|
# See: https://godoc.org/github.com/kata-containers/runtime/virtcontainers#ContainerType
|
||||||
|
-sandbox_cgroup_only=@DEFSANDBOXCGROUPONLY@
|
||||||
|
+sandbox_cgroup_only = @DEFSANDBOXCGROUPONLY@
|
||||||
|
|
||||||
|
# Enabled experimental feature list, format: ["a", "b"].
|
||||||
|
# Experimental features are features not stable enough for production,
|
||||||
|
# they may break compatibility, and are prepared for a big version bump.
|
||||||
|
# Supported experimental features:
|
||||||
|
# (default: [])
|
||||||
|
-experimental=@DEFAULTEXPFEATURES@
|
||||||
|
+experimental = @DEFAULTEXPFEATURES@
|
||||||
|
|
||||||
|
# If enabled, user can run pprof tools with shim v2 process through kata-monitor.
|
||||||
|
# (default: false)
|
||||||
|
-# enable_pprof = true
|
||||||
|
+#enable_pprof = true
|
||||||
|
--
|
||||||
|
2.20.1.windows.1
|
||||||
|
|
||||||
@ -0,0 +1,235 @@
|
|||||||
|
From 37c4a009a6ea9028e237f849e7b27a15c602113e Mon Sep 17 00:00:00 2001
|
||||||
|
From: "Xinle.Guo" <guoxinle1@huawei.com>
|
||||||
|
Date: Tue, 11 Jan 2022 10:44:53 +0800
|
||||||
|
Subject: [PATCH] stratovirt: add struct `vmConfig` and methods to get all
|
||||||
|
parameters of VM
|
||||||
|
|
||||||
|
Defines `vmConfig` struct to containes all configuration items that
|
||||||
|
virtual machine needs. Provides methods to get VM paramters,
|
||||||
|
including name, UUID, cpu, memory, kernel, devices, etc.
|
||||||
|
|
||||||
|
Signed-off-by: Xinle.Guo <guoxinle1@huawei.com>
|
||||||
|
---
|
||||||
|
src/runtime/virtcontainers/stratovirt.go | 197 ++++++++++++++++++++++-
|
||||||
|
1 file changed, 189 insertions(+), 8 deletions(-)
|
||||||
|
|
||||||
|
diff --git a/src/runtime/virtcontainers/stratovirt.go b/src/runtime/virtcontainers/stratovirt.go
|
||||||
|
index e9b2ba8..7e32a8a 100644
|
||||||
|
--- a/src/runtime/virtcontainers/stratovirt.go
|
||||||
|
+++ b/src/runtime/virtcontainers/stratovirt.go
|
||||||
|
@@ -26,15 +26,196 @@ import (
|
||||||
|
)
|
||||||
|
|
||||||
|
const (
|
||||||
|
- apiSocket = "qmp.socket"
|
||||||
|
- debugSocket = "console.socket"
|
||||||
|
- ozoneBaseDir = "/srv/ozone/stratovirt"
|
||||||
|
- defaultDummyMac = "22:33:44:aa:bb:"
|
||||||
|
- mmioBlkCount = 4
|
||||||
|
- mmioNetCount = 2
|
||||||
|
- randomDevice = "/dev/urandom"
|
||||||
|
+ defaultStratoVirt = "/usr/bin/stratovirt"
|
||||||
|
+ ozoneBaseDir = "/srv/ozone/stratovirt"
|
||||||
|
+ defaultStratoVirtMachineType = "microvm"
|
||||||
|
+ defaultKernelParames = "console=hvc0 reboot=k panic=1 agent.use_vsock=true ramdom.trust_cpu=on rw"
|
||||||
|
+ defaultMicroVMParames = "pci=off iommu=off acpi=off"
|
||||||
|
+ apiSocket = "qmp.socket"
|
||||||
|
+ debugSocket = "console.socket"
|
||||||
|
+ virtiofsSocket = "virtiofs_kata.sock"
|
||||||
|
+ iothreadID = "iothread_block"
|
||||||
|
+ mmioBlkCount = 4
|
||||||
|
+ mmioNetCount = 2
|
||||||
|
)
|
||||||
|
|
||||||
|
+const (
|
||||||
|
+ WaitSandboxTimeoutSecs = 15
|
||||||
|
+ MachineTypeMicrovm = "microvm"
|
||||||
|
+)
|
||||||
|
+
|
||||||
|
+// VirtioDev is the StratoVirt device interface.
|
||||||
|
+type VirtioDev interface {
|
||||||
|
+ getParams(config *vmConfig) []string
|
||||||
|
+}
|
||||||
|
+
|
||||||
|
+type inComing struct {
|
||||||
|
+ path string
|
||||||
|
+ bootFromTemplate bool
|
||||||
|
+}
|
||||||
|
+
|
||||||
|
+type Ozone struct {
|
||||||
|
+ ozoneRoot string
|
||||||
|
+ ozoneRes []string
|
||||||
|
+ consolePath string
|
||||||
|
+ kernelPath string
|
||||||
|
+ initrdPath string
|
||||||
|
+ pidFile string
|
||||||
|
+ logFile string
|
||||||
|
+ qmpSocketPath string
|
||||||
|
+}
|
||||||
|
+
|
||||||
|
+// vmConfig keeps the custom settings and paramters to start virtual machine.
|
||||||
|
+type vmConfig struct {
|
||||||
|
+ name string
|
||||||
|
+ uuid string
|
||||||
|
+ machineType string
|
||||||
|
+ vmPath string
|
||||||
|
+ smp uint32
|
||||||
|
+ memory uint64
|
||||||
|
+ kernelPath string
|
||||||
|
+ params string
|
||||||
|
+ rootfsPath string
|
||||||
|
+ initrdPath string
|
||||||
|
+ rootBus types.Bridge
|
||||||
|
+ devices []VirtioDev
|
||||||
|
+ IOThread bool
|
||||||
|
+ PFlash []string
|
||||||
|
+ pidFile string
|
||||||
|
+ logFile string
|
||||||
|
+ qmpSocketPath govmmQemu.QMPSocket
|
||||||
|
+ consolePath string
|
||||||
|
+ fsSockPath string
|
||||||
|
+ incoming inComing
|
||||||
|
+ daemonize bool
|
||||||
|
+ useOzone bool
|
||||||
|
+ Ozone Ozone
|
||||||
|
+}
|
||||||
|
+
|
||||||
|
+func (c *vmConfig) appendName(params *[]string) {
|
||||||
|
+ if c.name == "" {
|
||||||
|
+ return
|
||||||
|
+ }
|
||||||
|
+ *params = append(*params, "-name", c.name)
|
||||||
|
+}
|
||||||
|
+
|
||||||
|
+func (c *vmConfig) appendUUID(params *[]string) {
|
||||||
|
+ if c.uuid == "" {
|
||||||
|
+ return
|
||||||
|
+ }
|
||||||
|
+ *params = append(*params, "-uuid", c.uuid)
|
||||||
|
+}
|
||||||
|
+
|
||||||
|
+func (c *vmConfig) appendMachine(params *[]string) {
|
||||||
|
+ if c.machineType == "" {
|
||||||
|
+ return
|
||||||
|
+ }
|
||||||
|
+ *params = append(*params, "-machine", fmt.Sprintf("type=%s,dump-guest-core=off,mem-share=on", c.machineType))
|
||||||
|
+}
|
||||||
|
+
|
||||||
|
+func (c *vmConfig) appendCPU(params *[]string) {
|
||||||
|
+ if c.smp == 0 {
|
||||||
|
+ return
|
||||||
|
+ }
|
||||||
|
+ *params = append(*params, "-smp", strconv.Itoa(int(c.smp)))
|
||||||
|
+}
|
||||||
|
+
|
||||||
|
+func (c *vmConfig) appendMemory(params *[]string) {
|
||||||
|
+ if c.memory == 0 {
|
||||||
|
+ return
|
||||||
|
+ }
|
||||||
|
+ *params = append(*params, "-m", strconv.Itoa(int(c.memory)))
|
||||||
|
+}
|
||||||
|
+
|
||||||
|
+func (c *vmConfig) appendKernel(params *[]string) {
|
||||||
|
+ var ozone Ozone
|
||||||
|
+ if c.kernelPath == "" {
|
||||||
|
+ return
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ if c.useOzone {
|
||||||
|
+ ozone = c.Ozone
|
||||||
|
+ *params = append(*params, "-kernel", ozone.kernelPath)
|
||||||
|
+ } else {
|
||||||
|
+ *params = append(*params, "-kernel", c.kernelPath)
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ if c.initrdPath != "" {
|
||||||
|
+ if c.useOzone {
|
||||||
|
+ *params = append(*params, "-initrd", ozone.initrdPath)
|
||||||
|
+ } else {
|
||||||
|
+ *params = append(*params, "-initrd", c.initrdPath)
|
||||||
|
+ }
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ if c.params != "" {
|
||||||
|
+ *params = append(*params, "-append", c.params)
|
||||||
|
+ }
|
||||||
|
+}
|
||||||
|
+
|
||||||
|
+func (c *vmConfig) appendIOThreads(params *[]string) {
|
||||||
|
+ if c.IOThread {
|
||||||
|
+ *params = append(*params, "-object", fmt.Sprintf("iothread,id=%s", iothreadID))
|
||||||
|
+ }
|
||||||
|
+}
|
||||||
|
+
|
||||||
|
+func (c *vmConfig) appendDevices(params *[]string) {
|
||||||
|
+ for _, d := range c.devices {
|
||||||
|
+ *params = append(*params, d.getParams(c)...)
|
||||||
|
+ }
|
||||||
|
+}
|
||||||
|
+
|
||||||
|
+func (c *vmConfig) appendPFlash(params *[]string) {
|
||||||
|
+ for _, p := range c.PFlash {
|
||||||
|
+ if p != "" {
|
||||||
|
+ *params = append(*params, "-drive", p)
|
||||||
|
+ }
|
||||||
|
+ }
|
||||||
|
+}
|
||||||
|
+
|
||||||
|
+func (c *vmConfig) appendPidFile(params *[]string) {
|
||||||
|
+ if c.pidFile == "" {
|
||||||
|
+ return
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ if c.useOzone {
|
||||||
|
+ *params = append(*params, "-pidfile", c.Ozone.pidFile)
|
||||||
|
+ } else {
|
||||||
|
+ *params = append(*params, "-pidfile", c.pidFile)
|
||||||
|
+ }
|
||||||
|
+}
|
||||||
|
+
|
||||||
|
+func (c *vmConfig) appendLogFile(params *[]string) {
|
||||||
|
+ if c.logFile == "" {
|
||||||
|
+ return
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ if c.useOzone {
|
||||||
|
+ *params = append(*params, "-D", c.Ozone.logFile)
|
||||||
|
+ } else {
|
||||||
|
+ *params = append(*params, "-D", c.logFile)
|
||||||
|
+ }
|
||||||
|
+}
|
||||||
|
+
|
||||||
|
+func (c *vmConfig) appendQMPSocket(params *[]string) {
|
||||||
|
+ qmpInfo := c.qmpSocketPath
|
||||||
|
+ qmpParams := append([]string{}, fmt.Sprintf("%s:", qmpInfo.Type))
|
||||||
|
+ if c.useOzone {
|
||||||
|
+ qmpParams = append(qmpParams, c.Ozone.qmpSocketPath)
|
||||||
|
+ } else {
|
||||||
|
+ qmpParams = append(qmpParams, qmpInfo.Name)
|
||||||
|
+ }
|
||||||
|
+ qmpParams = append(qmpParams, ",server")
|
||||||
|
+ qmpParams = append(qmpParams, ",nowait")
|
||||||
|
+ *params = append(*params, "-qmp", strings.Join(qmpParams, ""))
|
||||||
|
+}
|
||||||
|
+
|
||||||
|
+func (c *vmConfig) appendIncoming(params *[]string) {
|
||||||
|
+ if c.incoming.path != "" && c.incoming.bootFromTemplate {
|
||||||
|
+ *params = append(*params, "-incoming", fmt.Sprintf("file:%s", c.incoming.path))
|
||||||
|
+ }
|
||||||
|
+}
|
||||||
|
+
|
||||||
|
type stratovirtDev struct {
|
||||||
|
dev interface{}
|
||||||
|
devType deviceType
|
||||||
|
@@ -208,7 +389,7 @@ func (s *stratovirt) createbaseParams() []string {
|
||||||
|
params = append(params, "-m", fmt.Sprintf("%d", uint64(s.config.MemorySize)))
|
||||||
|
params = append(params, "-device", "virtio-serial-device")
|
||||||
|
params = append(params, "-device", "virtconsole,chardev=charconsole0,id=virtioconsole0")
|
||||||
|
- params = append(params, "-object", fmt.Sprintf("rng-random,id=objrng0,filename=%s", randomDevice))
|
||||||
|
+ params = append(params, "-object", fmt.Sprintf("rng-random,id=objrng0,filename=%s", s.config.EntropySource))
|
||||||
|
params = append(params, "-device", "virtio-rng-device,rng=objrng0")
|
||||||
|
|
||||||
|
// daemonize
|
||||||
|
--
|
||||||
|
2.20.1.windows.1
|
||||||
|
|
||||||
File diff suppressed because it is too large
Load Diff
@ -0,0 +1,771 @@
|
|||||||
|
From 8c0dd0f91366910b3029cb982a3cb807bdbb34cf Mon Sep 17 00:00:00 2001
|
||||||
|
From: "Xinle.Guo" <guoxinle1@huawei.com>
|
||||||
|
Date: Fri, 14 Jan 2022 17:07:37 +0800
|
||||||
|
Subject: [PATCH] stratovirt: add a standard virtual machine sandbox type to
|
||||||
|
kata container
|
||||||
|
|
||||||
|
Because stratovirt supports both microVM and standardVM
|
||||||
|
architectures, adapts two sandbox types for kata container.
|
||||||
|
Besides basic features, it also support virtio-fs, hotplug
|
||||||
|
VFIO, support more devices.
|
||||||
|
|
||||||
|
Signed-off-by: Xinle.Guo <guoxinle1@huawei.com>
|
||||||
|
---
|
||||||
|
src/runtime/pkg/katautils/config.go | 1 +
|
||||||
|
src/runtime/virtcontainers/stratovirt.go | 482 ++++++++++++++++++++---
|
||||||
|
2 files changed, 438 insertions(+), 45 deletions(-)
|
||||||
|
|
||||||
|
diff --git a/src/runtime/pkg/katautils/config.go b/src/runtime/pkg/katautils/config.go
|
||||||
|
index e523ed3..b04cdee 100644
|
||||||
|
--- a/src/runtime/pkg/katautils/config.go
|
||||||
|
+++ b/src/runtime/pkg/katautils/config.go
|
||||||
|
@@ -1007,6 +1007,7 @@ func newStratovirtHypervisorConfig(h hypervisor) (vc.HypervisorConfig, error) {
|
||||||
|
BlockDeviceCacheDirect: h.BlockDeviceCacheDirect,
|
||||||
|
BlockDeviceCacheNoflush: h.BlockDeviceCacheNoflush,
|
||||||
|
EnableIOThreads: h.EnableIOThreads,
|
||||||
|
+ PCIeRootPort: h.PCIeRootPort,
|
||||||
|
DisableVhostNet: h.DisableVhostNet,
|
||||||
|
EnableVhostUserStore: h.EnableVhostUserStore,
|
||||||
|
VhostUserStorePath: h.vhostUserStorePath(),
|
||||||
|
diff --git a/src/runtime/virtcontainers/stratovirt.go b/src/runtime/virtcontainers/stratovirt.go
|
||||||
|
index 4fcfb94..ffe8965 100644
|
||||||
|
--- a/src/runtime/virtcontainers/stratovirt.go
|
||||||
|
+++ b/src/runtime/virtcontainers/stratovirt.go
|
||||||
|
@@ -14,14 +14,14 @@ import (
|
||||||
|
"time"
|
||||||
|
|
||||||
|
govmmQemu "github.com/kata-containers/govmm/qemu"
|
||||||
|
- "github.com/pkg/errors"
|
||||||
|
- "github.com/sirupsen/logrus"
|
||||||
|
-
|
||||||
|
"github.com/kata-containers/kata-containers/src/runtime/virtcontainers/device/config"
|
||||||
|
persistapi "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/persist/api"
|
||||||
|
+ vcTypes "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/pkg/types"
|
||||||
|
"github.com/kata-containers/kata-containers/src/runtime/virtcontainers/pkg/uuid"
|
||||||
|
"github.com/kata-containers/kata-containers/src/runtime/virtcontainers/types"
|
||||||
|
"github.com/kata-containers/kata-containers/src/runtime/virtcontainers/utils"
|
||||||
|
+ "github.com/pkg/errors"
|
||||||
|
+ "github.com/sirupsen/logrus"
|
||||||
|
"go.opentelemetry.io/otel"
|
||||||
|
otelLabel "go.opentelemetry.io/otel/label"
|
||||||
|
otelTrace "go.opentelemetry.io/otel/trace"
|
||||||
|
@@ -44,6 +44,11 @@ const (
|
||||||
|
const (
|
||||||
|
WaitSandboxTimeoutSecs = 15
|
||||||
|
MachineTypeMicrovm = "microvm"
|
||||||
|
+ MachineTypeQ35 = "q35"
|
||||||
|
+ MachineTypeVirt = "virt"
|
||||||
|
+ RootPortPrefix = "pcie"
|
||||||
|
+ Q35PFlashCode = "/usr/share/edk2/ovmf/OVMF_CODE.fd"
|
||||||
|
+ VirtPFlashCode = "/usr/share/edk2/aarch64/QEMU_EFI-pflash.raw"
|
||||||
|
MmioBus VirtioDriver = "mmio"
|
||||||
|
PciBus VirtioDriver = "pci"
|
||||||
|
)
|
||||||
|
@@ -67,6 +72,42 @@ func (d VirtioDriver) getDriver(config *vmConfig) VirtioDriver {
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
+type rootPortDevice struct {
|
||||||
|
+ id string
|
||||||
|
+ port string
|
||||||
|
+ bus string
|
||||||
|
+ slot int
|
||||||
|
+ plugged bool
|
||||||
|
+ addedDev string
|
||||||
|
+}
|
||||||
|
+
|
||||||
|
+func (r rootPortDevice) isVaild() bool {
|
||||||
|
+ if r.id == "" || r.port == "" {
|
||||||
|
+ return false
|
||||||
|
+ }
|
||||||
|
+ return true
|
||||||
|
+}
|
||||||
|
+
|
||||||
|
+func (r rootPortDevice) getParams(config *vmConfig) []string {
|
||||||
|
+ if !r.isVaild() {
|
||||||
|
+ return nil
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ var params []string
|
||||||
|
+ var devParams []Param
|
||||||
|
+ devParams = append(devParams, Param{"id", r.id})
|
||||||
|
+ devParams = append(devParams, Param{"port", r.port})
|
||||||
|
+ if r.bus == "" {
|
||||||
|
+ r.bus = "pcie.0"
|
||||||
|
+ }
|
||||||
|
+ devParams = append(devParams, Param{"bus", r.bus})
|
||||||
|
+ devParams = append(devParams, Param{"addr", fmt.Sprintf("%d", r.slot)})
|
||||||
|
+
|
||||||
|
+ driver := "pcie-root-port"
|
||||||
|
+ params = append(params, "-device", fmt.Sprintf("%s,%s", driver, strings.Join(SerializeParams(devParams, "="), ",")))
|
||||||
|
+ return params
|
||||||
|
+}
|
||||||
|
+
|
||||||
|
type blkDevice struct {
|
||||||
|
id string
|
||||||
|
filePath string
|
||||||
|
@@ -179,6 +220,56 @@ func (n netDevice) getParams(config *vmConfig) []string {
|
||||||
|
return params
|
||||||
|
}
|
||||||
|
|
||||||
|
+type virtioFs struct {
|
||||||
|
+ driver VirtioDriver
|
||||||
|
+ backend string
|
||||||
|
+ charID string
|
||||||
|
+ charDev string
|
||||||
|
+ tag string
|
||||||
|
+ deviceID string
|
||||||
|
+ bus string
|
||||||
|
+ addr string
|
||||||
|
+}
|
||||||
|
+
|
||||||
|
+var virtiofsDriver = map[VirtioDriver]string{
|
||||||
|
+ MmioBus: "vhost-user-fs-device",
|
||||||
|
+ PciBus: "vhost-user-fs-pci",
|
||||||
|
+}
|
||||||
|
+
|
||||||
|
+func (v virtioFs) isVaild() bool {
|
||||||
|
+ if v.charID == "" || v.charDev == "" || v.deviceID == "" {
|
||||||
|
+ return false
|
||||||
|
+ }
|
||||||
|
+ return true
|
||||||
|
+}
|
||||||
|
+
|
||||||
|
+func (v virtioFs) getParams(config *vmConfig) []string {
|
||||||
|
+ if !v.isVaild() {
|
||||||
|
+ return nil
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ var params []string
|
||||||
|
+ var charParams []Param
|
||||||
|
+ var fsParams []Param
|
||||||
|
+
|
||||||
|
+ charParams = append(charParams, Param{"id", v.charID})
|
||||||
|
+ charParams = append(charParams, Param{"path", config.fsSockPath})
|
||||||
|
+
|
||||||
|
+ v.driver = v.driver.getDriver(config)
|
||||||
|
+ driver := virtiofsDriver[v.driver]
|
||||||
|
+ fsParams = append(fsParams, Param{"chardev", v.charDev})
|
||||||
|
+ fsParams = append(fsParams, Param{"tag", v.tag})
|
||||||
|
+ fsParams = append(fsParams, Param{"id", v.deviceID})
|
||||||
|
+ if v.bus != "" {
|
||||||
|
+ fsParams = append(fsParams, Param{"bus", v.bus})
|
||||||
|
+ fsParams = append(fsParams, Param{"addr", v.addr})
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ params = append(params, "-chardev", fmt.Sprintf("%s,%s,server,nowait", v.backend, strings.Join(SerializeParams(charParams, "="), ",")))
|
||||||
|
+ params = append(params, "-device", fmt.Sprintf("%s,%s", driver, strings.Join(SerializeParams(fsParams, "="), ",")))
|
||||||
|
+ return params
|
||||||
|
+}
|
||||||
|
+
|
||||||
|
type vhostVsock struct {
|
||||||
|
driver VirtioDriver
|
||||||
|
id string
|
||||||
|
@@ -442,6 +533,12 @@ func (c *vmConfig) appendDevices(params *[]string) {
|
||||||
|
for _, d := range c.devices {
|
||||||
|
*params = append(*params, d.getParams(c)...)
|
||||||
|
}
|
||||||
|
+
|
||||||
|
+ if c.machineType == MachineTypeMicrovm {
|
||||||
|
+ return
|
||||||
|
+ }
|
||||||
|
+ // Add flag to unplug devices from their root port faster.
|
||||||
|
+ *params = append(*params, "-global", "pcie-root-port.fast-unplug=1")
|
||||||
|
}
|
||||||
|
|
||||||
|
func (c *vmConfig) appendPFlash(params *[]string) {
|
||||||
|
@@ -499,6 +596,8 @@ func (c *vmConfig) appendIncoming(params *[]string) {
|
||||||
|
type State struct {
|
||||||
|
mmioBlkSlots [mmioBlkCount]bool
|
||||||
|
mmioNetSlots [mmioNetCount]bool
|
||||||
|
+ // The list of RootPorts that can be hot-added.
|
||||||
|
+ rootPort []rootPortDevice
|
||||||
|
pid int
|
||||||
|
virtiofsPid int
|
||||||
|
}
|
||||||
|
@@ -558,6 +657,26 @@ func (s *stratovirt) getKernelParams(machineType string, initrdPath string) (str
|
||||||
|
return strings.Join(params, " "), nil
|
||||||
|
}
|
||||||
|
|
||||||
|
+func (s *stratovirt) getPFlash(machineType string) ([]string, error) {
|
||||||
|
+ if s.config.PFlash != nil {
|
||||||
|
+ return s.config.PFlash, nil
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ var PFlash []string
|
||||||
|
+ switch machineType {
|
||||||
|
+ case MachineTypeQ35:
|
||||||
|
+ PFlash = append(PFlash, fmt.Sprintf("file=%s,if=pflash,unit=0", Q35PFlashCode))
|
||||||
|
+ case MachineTypeVirt:
|
||||||
|
+ PFlash = append(PFlash, fmt.Sprintf("file=%s,if=pflash,unit=0", VirtPFlashCode))
|
||||||
|
+ case MachineTypeMicrovm:
|
||||||
|
+ return nil, nil
|
||||||
|
+ default:
|
||||||
|
+ return nil, fmt.Errorf("failed to match machine type %s", machineType)
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ return PFlash, nil
|
||||||
|
+}
|
||||||
|
+
|
||||||
|
func (s *stratovirt) createQMPSocket(vmPath string) govmmQemu.QMPSocket {
|
||||||
|
socketPath := filepath.Join(vmPath, apiSocket)
|
||||||
|
|
||||||
|
@@ -590,6 +709,34 @@ func (s *stratovirt) createDevices() []VirtioDev {
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
+ // Create root port for all devices that need to be hot-added.
|
||||||
|
+ if s.vmConfig.machineType != MachineTypeMicrovm && s.config.PCIeRootPort > 0 {
|
||||||
|
+ devices = s.appendRootPort(ctx, devices)
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ return devices
|
||||||
|
+}
|
||||||
|
+
|
||||||
|
+func (s *stratovirt) appendRootPort(ctx context.Context, devices []VirtioDev) []VirtioDev {
|
||||||
|
+ number := s.config.PCIeRootPort
|
||||||
|
+
|
||||||
|
+ for i := uint32(1); i < number+1; i++ {
|
||||||
|
+ addr, err := s.vmConfig.rootBus.AddDevice(ctx, fmt.Sprintf("%s.%d", RootPortPrefix, i))
|
||||||
|
+ if err != nil {
|
||||||
|
+ return devices
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ rp := rootPortDevice{
|
||||||
|
+ id: fmt.Sprintf("%s.%d", RootPortPrefix, i),
|
||||||
|
+ port: fmt.Sprintf("%d", i),
|
||||||
|
+ bus: defaultBridgeBus,
|
||||||
|
+ slot: int(addr),
|
||||||
|
+ addedDev: "",
|
||||||
|
+ }
|
||||||
|
+ s.state.rootPort = append(s.state.rootPort, rp)
|
||||||
|
+ devices = append(devices, rp)
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
return devices
|
||||||
|
}
|
||||||
|
|
||||||
|
@@ -729,6 +876,38 @@ func (s *stratovirt) appendNetwork(ctx context.Context, devices []VirtioDev, end
|
||||||
|
return devices
|
||||||
|
}
|
||||||
|
|
||||||
|
+func (s *stratovirt) appendVirtioFs(ctx context.Context, devices []VirtioDev, volume types.Volume) []VirtioDev {
|
||||||
|
+ if s.config.SharedFS != config.VirtioFS {
|
||||||
|
+ return devices
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ var bus string
|
||||||
|
+ var addr uint32
|
||||||
|
+ var err error
|
||||||
|
+ name := "virtio_fs"
|
||||||
|
+
|
||||||
|
+ if s.vmConfig.machineType != MachineTypeMicrovm {
|
||||||
|
+ bus = "pcie.0"
|
||||||
|
+ addr, err = s.vmConfig.rootBus.AddDevice(ctx, name)
|
||||||
|
+ if err != nil {
|
||||||
|
+ return devices
|
||||||
|
+ }
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ devices = append(devices, virtioFs{
|
||||||
|
+ backend: "socket",
|
||||||
|
+ // Virtio-fs must be bound to unique charDev, it uses the same name.
|
||||||
|
+ charID: name,
|
||||||
|
+ charDev: name,
|
||||||
|
+ tag: volume.MountTag,
|
||||||
|
+ deviceID: "virtio-fs0",
|
||||||
|
+ bus: bus,
|
||||||
|
+ addr: fmt.Sprintf("%d", addr),
|
||||||
|
+ })
|
||||||
|
+
|
||||||
|
+ return devices
|
||||||
|
+}
|
||||||
|
+
|
||||||
|
func (s *stratovirt) setVMConfig(id string, hypervisorConfig *HypervisorConfig) error {
|
||||||
|
span, _ := s.trace(s.ctx, "setStratoVirtUp")
|
||||||
|
defer span.End()
|
||||||
|
@@ -765,7 +944,10 @@ func (s *stratovirt) setVMConfig(id string, hypervisorConfig *HypervisorConfig)
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
- var PFlash []string
|
||||||
|
+ PFlash, err := s.getPFlash(machineType)
|
||||||
|
+ if err != nil {
|
||||||
|
+ return err
|
||||||
|
+ }
|
||||||
|
|
||||||
|
vmPath := filepath.Join(s.store.RunVMStoragePath(), s.id)
|
||||||
|
qmpSocket := s.createQMPSocket(vmPath)
|
||||||
|
@@ -852,8 +1034,33 @@ func (s *stratovirt) setOzone() error {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
-func (s *stratovirt) hypervisorConfig() HypervisorConfig {
|
||||||
|
- return s.config
|
||||||
|
+// Virtio fs daemon is a shared file system that lets VM access a directory
|
||||||
|
+// tree on the host.
|
||||||
|
+func (s *stratovirt) setupVirtioFs() error {
|
||||||
|
+ if !s.config.DisableBlockDeviceUse || s.config.SharedFS != config.VirtioFS {
|
||||||
|
+ return nil
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ if _, err := os.Stat(s.config.VirtioFSDaemon); os.IsNotExist(err) {
|
||||||
|
+ return fmt.Errorf("virtiofsd path (%s) does not exist", s.config.VirtioFSDaemon)
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ args := []string{
|
||||||
|
+ "-socket-path", filepath.Join(s.vmConfig.vmPath, "virtiofs_kata.sock"),
|
||||||
|
+ "-source", getSharePath(s.id)}
|
||||||
|
+ if len(s.config.VirtioFSExtraArgs) != 0 {
|
||||||
|
+ args = append(args, s.config.VirtioFSExtraArgs...)
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ cmd := exec.Command(s.config.VirtioFSDaemon, args...)
|
||||||
|
+ s.Logger().Info("Virtiofsd start with cmd: ", cmd)
|
||||||
|
+
|
||||||
|
+ if err := cmd.Start(); err != nil {
|
||||||
|
+ return fmt.Errorf("failed to strat virtiofsd: %v", cmd)
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ s.state.virtiofsPid = cmd.Process.Pid
|
||||||
|
+ return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// Get StratoVirt binary path.
|
||||||
|
@@ -873,6 +1080,10 @@ func (s *stratovirt) binPath() (string, error) {
|
||||||
|
return path, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
+func (s *stratovirt) hypervisorConfig() HypervisorConfig {
|
||||||
|
+ return s.config
|
||||||
|
+}
|
||||||
|
+
|
||||||
|
func (s *stratovirt) createSandbox(ctx context.Context, id string, networkNS NetworkNamespace, hypervisorConfig *HypervisorConfig) error {
|
||||||
|
var span otelTrace.Span
|
||||||
|
span, _ = s.trace(ctx, "createSandbox")
|
||||||
|
@@ -1000,6 +1211,10 @@ func (s *stratovirt) startSandbox(ctx context.Context, timeout int) error {
|
||||||
|
}
|
||||||
|
}()
|
||||||
|
|
||||||
|
+ if err = s.setupVirtioFs(); err != nil {
|
||||||
|
+ return err
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
var params []string
|
||||||
|
s.createBaseParams(s.vmConfig, ¶ms)
|
||||||
|
|
||||||
|
@@ -1134,28 +1349,29 @@ func (s *stratovirt) addDevice(ctx context.Context, devInfo interface{}, devType
|
||||||
|
s.vmConfig.devices = s.appendNetwork(ctx, s.vmConfig.devices, v)
|
||||||
|
case config.BlockDrive:
|
||||||
|
s.vmConfig.devices = s.appendBlock(ctx, s.vmConfig.devices)
|
||||||
|
+ case types.Volume:
|
||||||
|
+ s.vmConfig.devices = s.appendVirtioFs(ctx, s.vmConfig.devices, v)
|
||||||
|
default:
|
||||||
|
s.Logger().WithField("dev-type", v).Warn("Could not append device: unsupported device type")
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
-func (s *stratovirt) getDevSlot(Name string, isPut bool) (slot int, err error) {
|
||||||
|
+func (s *stratovirt) setupMmioSlot(Name string, isPut bool) (int, error) {
|
||||||
|
Name = filepath.Base(strings.ToLower(Name))
|
||||||
|
-
|
||||||
|
if strings.HasPrefix(Name, "eth") {
|
||||||
|
idxStr := strings.TrimPrefix(Name, "eth")
|
||||||
|
if idxStr == Name {
|
||||||
|
- return 0, fmt.Errorf("Could not parse idx from Name %q", Name)
|
||||||
|
+ return 0, fmt.Errorf("could not parse idx from name %q", Name)
|
||||||
|
}
|
||||||
|
|
||||||
|
idx, err := strconv.Atoi(idxStr)
|
||||||
|
if err != nil {
|
||||||
|
- return 0, fmt.Errorf("Could not convert to int from Str %q", idxStr)
|
||||||
|
+ return 0, fmt.Errorf("could not convert to int from str %q", idxStr)
|
||||||
|
}
|
||||||
|
|
||||||
|
if !isPut && s.state.mmioNetSlots[idx] {
|
||||||
|
- return 0, fmt.Errorf("GetDevSlot failed, slot is being used %q", idxStr)
|
||||||
|
+ return 0, fmt.Errorf("failed to setup mmio slot, slot is being used %q", idxStr)
|
||||||
|
}
|
||||||
|
s.state.mmioNetSlots[idx] = !isPut
|
||||||
|
|
||||||
|
@@ -1163,25 +1379,80 @@ func (s *stratovirt) getDevSlot(Name string, isPut bool) (slot int, err error) {
|
||||||
|
} else if strings.HasPrefix(Name, "vd") {
|
||||||
|
charStr := strings.TrimPrefix(Name, "vd")
|
||||||
|
if charStr == Name {
|
||||||
|
- return 0, fmt.Errorf("Could not parse idx from Name %q", Name)
|
||||||
|
+ return 0, fmt.Errorf("could not parse idx from name %q", Name)
|
||||||
|
}
|
||||||
|
|
||||||
|
char := []rune(charStr)
|
||||||
|
idx := int(char[0] - 'a')
|
||||||
|
|
||||||
|
if !isPut && s.state.mmioBlkSlots[idx] {
|
||||||
|
- return 0, fmt.Errorf("GetDevSlot failed, slot is being used %q", charStr)
|
||||||
|
+ return 0, fmt.Errorf("failed to setup mmio slot, slot is being used %q", charStr)
|
||||||
|
}
|
||||||
|
s.state.mmioBlkSlots[idx] = !isPut
|
||||||
|
|
||||||
|
return idx, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
- return 0, fmt.Errorf("GetDevSlot failed, Name is invalid %q", Name)
|
||||||
|
+ return 0, fmt.Errorf("failed to setup mmio slot , Name is invalid %q", Name)
|
||||||
|
+}
|
||||||
|
+
|
||||||
|
+func (s *stratovirt) setupPciSlot(Name string, isPut bool) (string, int, error) {
|
||||||
|
+ rootports := &s.state.rootPort
|
||||||
|
+ if len(*rootports) == 0 {
|
||||||
|
+ return "", 0, fmt.Errorf("failed to get available address from bridges")
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ for i, rootport := range *rootports {
|
||||||
|
+ if !isPut && !rootport.plugged && rootport.addedDev == "" {
|
||||||
|
+ (*rootports)[i].plugged = true
|
||||||
|
+ (*rootports)[i].addedDev = Name
|
||||||
|
+ return rootport.id, rootport.slot, nil
|
||||||
|
+ } else if isPut && rootport.plugged && rootport.addedDev == Name {
|
||||||
|
+ (*rootports)[i].plugged = false
|
||||||
|
+ (*rootports)[i].addedDev = ""
|
||||||
|
+ return rootport.id, rootport.slot, nil
|
||||||
|
+ }
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ return "", 0, fmt.Errorf("no more bridge slots available")
|
||||||
|
+}
|
||||||
|
+
|
||||||
|
+func (s *stratovirt) getDevSlot(Name string) (string, int, error) {
|
||||||
|
+ if s.config.HypervisorMachineType == MachineTypeMicrovm {
|
||||||
|
+ slot, err := s.setupMmioSlot(Name, false)
|
||||||
|
+ if err != nil {
|
||||||
|
+ return "", 0, err
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ return "", slot, nil
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ bus, slot, err := s.setupPciSlot(Name, false)
|
||||||
|
+ if err != nil {
|
||||||
|
+ return "pcie.0", 0, err
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ return bus, slot, nil
|
||||||
|
+}
|
||||||
|
+
|
||||||
|
+func (s *stratovirt) delDevSlot(Name string) error {
|
||||||
|
+ if s.vmConfig.machineType == MachineTypeMicrovm {
|
||||||
|
+ if _, err := s.setupMmioSlot(Name, true); err != nil {
|
||||||
|
+ return err
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ return nil
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ if _, _, err := s.setupPciSlot(Name, true); err != nil {
|
||||||
|
+ return err
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
-func (s *stratovirt) hotplugNet(ctx context.Context, endpoint Endpoint, op operation) (err error) {
|
||||||
|
- err = s.qmpSetup()
|
||||||
|
+func (s *stratovirt) hotplugNet(ctx context.Context, endpoint Endpoint, op operation) error {
|
||||||
|
+ err := s.qmpSetup()
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
@@ -1198,6 +1469,14 @@ func (s *stratovirt) hotplugNet(ctx context.Context, endpoint Endpoint, op opera
|
||||||
|
return fmt.Errorf("Endpoint is not supported")
|
||||||
|
}
|
||||||
|
|
||||||
|
+ defer func() {
|
||||||
|
+ if err != nil {
|
||||||
|
+ if errDel := s.delDevSlot(endpoint.Name()); errDel != nil {
|
||||||
|
+ s.Logger().WithError(errDel).Warnf("Failed to delete device slot.")
|
||||||
|
+ }
|
||||||
|
+ }
|
||||||
|
+ }()
|
||||||
|
+
|
||||||
|
switch op {
|
||||||
|
case addDevice:
|
||||||
|
var (
|
||||||
|
@@ -1220,27 +1499,51 @@ func (s *stratovirt) hotplugNet(ctx context.Context, endpoint Endpoint, op opera
|
||||||
|
VhostFdNames = append(VhostFdNames, fdName)
|
||||||
|
}
|
||||||
|
|
||||||
|
- slot, err := s.getDevSlot(endpoint.Name(), false)
|
||||||
|
+ bus, slot, err := s.getDevSlot(endpoint.Name())
|
||||||
|
if err != nil {
|
||||||
|
- return fmt.Errorf("Could not get unused slot for %q", endpoint.Name())
|
||||||
|
+ return fmt.Errorf("could not get unused slot for %q", endpoint.Name())
|
||||||
|
}
|
||||||
|
|
||||||
|
if len(VMFdNames) != 0 || len(VhostFdNames) != 0 {
|
||||||
|
if err := s.qmpMonitorCh.qmp.ExecuteNetdevAddByFds(s.qmpMonitorCh.ctx, "tap", tap.ID, VMFdNames, VhostFdNames); err != nil {
|
||||||
|
- s.getDevSlot(endpoint.Name(), true)
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
if err := s.qmpMonitorCh.qmp.ExecuteNetdevAdd(s.qmpMonitorCh.ctx, "tap", tap.ID, tap.TAPIface.Name, "no", "no", 0); err != nil {
|
||||||
|
- s.getDevSlot(endpoint.Name(), true)
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
- if err := s.qmpMonitorCh.qmp.ExecuteNetPCIDeviceAdd(s.qmpMonitorCh.ctx, tap.Name, tap.ID, endpoint.HardwareAddr(), fmt.Sprintf("%d", slot), "", "", 0, false); err != nil {
|
||||||
|
- s.getDevSlot(endpoint.Name(), true)
|
||||||
|
+
|
||||||
|
+ // The slot of net device that hotplugged to the root port
|
||||||
|
+ // must be zero.
|
||||||
|
+ devAddr := "0x0.0x0"
|
||||||
|
+ if s.vmConfig.machineType == MachineTypeMicrovm {
|
||||||
|
+ devAddr = fmt.Sprintf("%d", slot)
|
||||||
|
+ } else {
|
||||||
|
+ bridgeSlot, err := vcTypes.PciSlotFromInt(slot)
|
||||||
|
+ if err != nil {
|
||||||
|
+ return err
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ devSlot, err := vcTypes.PciSlotFromString("0")
|
||||||
|
+ if err != nil {
|
||||||
|
+ return err
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ pciPath, err := vcTypes.PciPathFromSlots(bridgeSlot, devSlot)
|
||||||
|
+ if err != nil {
|
||||||
|
+ return err
|
||||||
|
+ }
|
||||||
|
+ endpoint.SetPciPath(pciPath)
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ if err := s.qmpMonitorCh.qmp.ExecuteNetPCIDeviceAdd(s.qmpMonitorCh.ctx, tap.ID, tap.ID, endpoint.HardwareAddr(), devAddr, bus, "", 0, false); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
case removeDevice:
|
||||||
|
+ if errDel := s.delDevSlot(endpoint.Name()); errDel != nil {
|
||||||
|
+ s.Logger().WithError(errDel).Warnf("Failed to delete device slot.")
|
||||||
|
+ }
|
||||||
|
if err := s.qmpMonitorCh.qmp.ExecuteDeviceDel(s.qmpMonitorCh.ctx, tap.ID); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
@@ -1248,58 +1551,134 @@ func (s *stratovirt) hotplugNet(ctx context.Context, endpoint Endpoint, op opera
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
default:
|
||||||
|
- return fmt.Errorf("Operation is not supported")
|
||||||
|
+ return fmt.Errorf("operation is not supported")
|
||||||
|
}
|
||||||
|
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
-func (s *stratovirt) hotplugBlk(drive *config.BlockDrive, op operation) (err error) {
|
||||||
|
- var filePath string
|
||||||
|
- err = s.qmpSetup()
|
||||||
|
+func (s *stratovirt) hotplugBlk(ctx context.Context, drive *config.BlockDrive, op operation) error {
|
||||||
|
+ err := s.qmpSetup()
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
+ driver := "virtio-blk-pci"
|
||||||
|
+ if s.vmConfig.machineType == MachineTypeMicrovm {
|
||||||
|
+ driver = "virtio-blk-mmio"
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ defer func() {
|
||||||
|
+ if err != nil {
|
||||||
|
+ s.qmpMonitorCh.qmp.ExecuteBlockdevDel(s.qmpMonitorCh.ctx, drive.ID)
|
||||||
|
+ if errDel := s.delDevSlot(drive.VirtPath); errDel != nil {
|
||||||
|
+ s.Logger().WithError(errDel).Warnf("Failed to delete device slot.")
|
||||||
|
+ }
|
||||||
|
+ }
|
||||||
|
+ }()
|
||||||
|
+
|
||||||
|
switch op {
|
||||||
|
case addDevice:
|
||||||
|
- driver := "virtio-blk-mmio"
|
||||||
|
+ filePath := drive.File
|
||||||
|
if s.vmConfig.useOzone {
|
||||||
|
filePath, err = s.updateOzoneRes(drive.File, true)
|
||||||
|
- if err != nil {
|
||||||
|
- return fmt.Errorf("Failed to update ozone resources")
|
||||||
|
- }
|
||||||
|
- } else {
|
||||||
|
- filePath = drive.File
|
||||||
|
- }
|
||||||
|
- slot, err := s.getDevSlot(drive.VirtPath, false)
|
||||||
|
- if err != nil {
|
||||||
|
- return fmt.Errorf("Could not get unused slot for %q", drive.VirtPath)
|
||||||
|
}
|
||||||
|
|
||||||
|
if err := s.qmpMonitorCh.qmp.ExecuteBlockdevAdd(s.qmpMonitorCh.ctx, filePath, drive.ID, false); err != nil {
|
||||||
|
- s.getDevSlot(drive.VirtPath, true)
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
- if err := s.qmpMonitorCh.qmp.ExecutePCIDeviceAdd(s.qmpMonitorCh.ctx, drive.ID, drive.ID, driver, fmt.Sprintf("%d", slot), "", "", 0, true, false); err != nil {
|
||||||
|
- s.getDevSlot(drive.VirtPath, true)
|
||||||
|
+ bus, slot, err := s.getDevSlot(drive.VirtPath)
|
||||||
|
+ if err != nil {
|
||||||
|
+ return err
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ // The slot of block device that hotplugged to the root port
|
||||||
|
+ // must be zero.
|
||||||
|
+ devAddr := "0x0.0x0"
|
||||||
|
+ if s.vmConfig.machineType == MachineTypeMicrovm {
|
||||||
|
+ devAddr = fmt.Sprintf("%d", slot)
|
||||||
|
+ } else {
|
||||||
|
+ bridgeSlot, err := vcTypes.PciSlotFromInt(slot)
|
||||||
|
+ if err != nil {
|
||||||
|
+ return err
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ devSlot, err := vcTypes.PciSlotFromString("0")
|
||||||
|
+ if err != nil {
|
||||||
|
+ return err
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ drive.PCIPath, err = vcTypes.PciPathFromSlots(bridgeSlot, devSlot)
|
||||||
|
+ if err != nil {
|
||||||
|
+ return err
|
||||||
|
+ }
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ if err := s.qmpMonitorCh.qmp.ExecutePCIDeviceAdd(s.qmpMonitorCh.ctx, drive.ID, drive.ID, driver, devAddr, bus, "", 0, false, false); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
case removeDevice:
|
||||||
|
if s.vmConfig.useOzone {
|
||||||
|
s.updateOzoneRes(drive.File, false)
|
||||||
|
}
|
||||||
|
+
|
||||||
|
+ if errDel := s.delDevSlot(drive.VirtPath); errDel != nil {
|
||||||
|
+ s.Logger().WithError(errDel).Warnf("Failed to delete device slot.")
|
||||||
|
+ }
|
||||||
|
if err := s.qmpMonitorCh.qmp.ExecuteDeviceDel(s.qmpMonitorCh.ctx, drive.ID); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if err := s.qmpMonitorCh.qmp.ExecuteBlockdevDel(s.qmpMonitorCh.ctx, drive.ID); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
+ default:
|
||||||
|
+ return fmt.Errorf("operation is not supported %d", op)
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ return nil
|
||||||
|
+}
|
||||||
|
+
|
||||||
|
+func (s *stratovirt) hotplugVFIO(ctx context.Context, device *config.VFIODev, op operation) error {
|
||||||
|
+ err := s.qmpSetup()
|
||||||
|
+ if err != nil {
|
||||||
|
+ return err
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ defer func() {
|
||||||
|
+ if err != nil {
|
||||||
|
+ if errDel := s.delDevSlot(device.ID); errDel != nil {
|
||||||
|
+ s.Logger().WithError(errDel).Warnf("Failed to delete device slot.")
|
||||||
|
+ }
|
||||||
|
+ }
|
||||||
|
+ }()
|
||||||
|
|
||||||
|
- s.getDevSlot(drive.VirtPath, true)
|
||||||
|
+ switch op {
|
||||||
|
+ case addDevice:
|
||||||
|
+ var bus string
|
||||||
|
+ // The slot of block device that hotplugged to the root port
|
||||||
|
+ // must be zero.
|
||||||
|
+ devAddr := "0x0.0x0"
|
||||||
|
+ // The vfio device BDF format is 0000:1a:00.3
|
||||||
|
+ device.BDF = "0000:" + device.BDF
|
||||||
|
+
|
||||||
|
+ bus, _, err = s.getDevSlot(device.ID)
|
||||||
|
+ if err != nil {
|
||||||
|
+ return err
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ if err = s.qmpMonitorCh.qmp.ExecutePCIVFIODeviceAdd(s.qmpMonitorCh.ctx, device.ID, device.BDF, devAddr, bus, ""); err != nil {
|
||||||
|
+ return err
|
||||||
|
+ }
|
||||||
|
+ case removeDevice:
|
||||||
|
+ if errDel := s.delDevSlot(device.ID); errDel != nil {
|
||||||
|
+ s.Logger().WithError(errDel).Warnf("Failed to delete device slot.")
|
||||||
|
+ }
|
||||||
|
+ if err := s.qmpMonitorCh.qmp.ExecuteDeviceDel(s.qmpMonitorCh.ctx, device.ID); err != nil {
|
||||||
|
+ return err
|
||||||
|
+ }
|
||||||
|
default:
|
||||||
|
- return fmt.Errorf("Operation is not supported")
|
||||||
|
+ return fmt.Errorf("operation is not supported %d", op)
|
||||||
|
}
|
||||||
|
|
||||||
|
return nil
|
||||||
|
@@ -1313,9 +1692,11 @@ func (s *stratovirt) hotplugAddDevice(ctx context.Context, devInfo interface{},
|
||||||
|
case netDev:
|
||||||
|
return nil, s.hotplugNet(ctx, devInfo.(Endpoint), addDevice)
|
||||||
|
case blockDev:
|
||||||
|
- return nil, s.hotplugBlk(devInfo.(*config.BlockDrive), addDevice)
|
||||||
|
+ return nil, s.hotplugBlk(ctx, devInfo.(*config.BlockDrive), addDevice)
|
||||||
|
+ case vfioDev:
|
||||||
|
+ return nil, s.hotplugVFIO(ctx, devInfo.(*config.VFIODev), addDevice)
|
||||||
|
default:
|
||||||
|
- return nil, fmt.Errorf("Hotplug add device failed: unsupported device type '%v'", devType)
|
||||||
|
+ return nil, fmt.Errorf("hotplug add device failed: unsupported device type '%v'", devType)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
@@ -1327,9 +1708,11 @@ func (s *stratovirt) hotplugRemoveDevice(ctx context.Context, devInfo interface{
|
||||||
|
case netDev:
|
||||||
|
return nil, s.hotplugNet(ctx, devInfo.(Endpoint), removeDevice)
|
||||||
|
case blockDev:
|
||||||
|
- return nil, s.hotplugBlk(devInfo.(*config.BlockDrive), removeDevice)
|
||||||
|
+ return nil, s.hotplugBlk(ctx, devInfo.(*config.BlockDrive), removeDevice)
|
||||||
|
+ case vfioDev:
|
||||||
|
+ return nil, s.hotplugVFIO(ctx, devInfo.(*config.VFIODev), removeDevice)
|
||||||
|
default:
|
||||||
|
- return nil, fmt.Errorf("Hotplug remove device: unsupported device type '%v'", devType)
|
||||||
|
+ return nil, fmt.Errorf("hotplug remove device: unsupported device type '%v'", devType)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
@@ -1371,6 +1754,10 @@ func (s *stratovirt) capabilities(ctx context.Context) types.Capabilities {
|
||||||
|
var caps types.Capabilities
|
||||||
|
caps.SetBlockDeviceHotplugSupport()
|
||||||
|
|
||||||
|
+ if s.config.DisableBlockDeviceUse && s.config.SharedFS == config.VirtioFS {
|
||||||
|
+ caps.SetFsSharingSupport()
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
return caps
|
||||||
|
}
|
||||||
|
|
||||||
|
@@ -1510,6 +1897,9 @@ func (s *stratovirt) getPids() []int {
|
||||||
|
|
||||||
|
pids = append(pids, s.state.pid)
|
||||||
|
|
||||||
|
+ if s.state.virtiofsPid != 0 {
|
||||||
|
+ pids = append(pids, s.state.virtiofsPid)
|
||||||
|
+ }
|
||||||
|
return pids
|
||||||
|
}
|
||||||
|
|
||||||
|
@@ -1544,12 +1934,14 @@ func (s *stratovirt) isRateLimiterBuiltin() bool {
|
||||||
|
func (s *stratovirt) save() (p persistapi.HypervisorState) {
|
||||||
|
pids := s.getPids()
|
||||||
|
p.Pid = pids[0]
|
||||||
|
+ p.VirtiofsdPid = s.state.virtiofsPid
|
||||||
|
p.Type = string(StratovirtHypervisor)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *stratovirt) load(p persistapi.HypervisorState) {
|
||||||
|
s.state.pid = p.Pid
|
||||||
|
+ s.state.virtiofsPid = p.VirtiofsdPid
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *stratovirt) setSandbox(sandbox *Sandbox) {
|
||||||
|
--
|
||||||
|
2.20.1.windows.1
|
||||||
|
|
||||||
@ -0,0 +1,46 @@
|
|||||||
|
From 9ac9c120a51663fa98ef25d721c321e5d16b3859 Mon Sep 17 00:00:00 2001
|
||||||
|
From: "Xinle.Guo" <guoxinle1@huawei.com>
|
||||||
|
Date: Sat, 26 Feb 2022 17:25:11 +0800
|
||||||
|
Subject: [PATCH] stratovirt: fix the problem that fails to plug net device
|
||||||
|
|
||||||
|
In kernel, hot plug pcie device needs to sleep 100ms to avoid
|
||||||
|
misoperation. Kata 1.x version agent monitor the plug event to
|
||||||
|
make sure plug device success. But kata 2.x does not provide
|
||||||
|
method. It may failed to get net device after plug the device.
|
||||||
|
|
||||||
|
Signed-off-by: Xinle.Guo <guoxinle1@huawei.com>
|
||||||
|
---
|
||||||
|
src/runtime/virtcontainers/stratovirt.go | 3 +++
|
||||||
|
1 file changed, 3 insertions(+)
|
||||||
|
|
||||||
|
diff --git a/src/runtime/virtcontainers/stratovirt.go b/src/runtime/virtcontainers/stratovirt.go
|
||||||
|
index ffe8965..27b45b7 100644
|
||||||
|
--- a/src/runtime/virtcontainers/stratovirt.go
|
||||||
|
+++ b/src/runtime/virtcontainers/stratovirt.go
|
||||||
|
@@ -39,6 +39,7 @@ const (
|
||||||
|
iothreadID = "iothread_block"
|
||||||
|
mmioBlkCount = 4
|
||||||
|
mmioNetCount = 2
|
||||||
|
+ hotPlugDelayTime = 200
|
||||||
|
)
|
||||||
|
|
||||||
|
const (
|
||||||
|
@@ -1540,6 +1541,7 @@ func (s *stratovirt) hotplugNet(ctx context.Context, endpoint Endpoint, op opera
|
||||||
|
if err := s.qmpMonitorCh.qmp.ExecuteNetPCIDeviceAdd(s.qmpMonitorCh.ctx, tap.ID, tap.ID, endpoint.HardwareAddr(), devAddr, bus, "", 0, false); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
+ time.Sleep(time.Millisecond * hotPlugDelayTime)
|
||||||
|
case removeDevice:
|
||||||
|
if errDel := s.delDevSlot(endpoint.Name()); errDel != nil {
|
||||||
|
s.Logger().WithError(errDel).Warnf("Failed to delete device slot.")
|
||||||
|
@@ -1670,6 +1672,7 @@ func (s *stratovirt) hotplugVFIO(ctx context.Context, device *config.VFIODev, op
|
||||||
|
if err = s.qmpMonitorCh.qmp.ExecutePCIVFIODeviceAdd(s.qmpMonitorCh.ctx, device.ID, device.BDF, devAddr, bus, ""); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
+ time.Sleep(time.Millisecond * hotPlugDelayTime)
|
||||||
|
case removeDevice:
|
||||||
|
if errDel := s.delDevSlot(device.ID); errDel != nil {
|
||||||
|
s.Logger().WithError(errDel).Warnf("Failed to delete device slot.")
|
||||||
|
--
|
||||||
|
2.20.1.windows.1
|
||||||
|
|
||||||
@ -0,0 +1,78 @@
|
|||||||
|
From 769af40220ea3d7b87173f11c135e651076e14ee Mon Sep 17 00:00:00 2001
|
||||||
|
From: "Xinle.Guo" <guoxinle1@huawei.com>
|
||||||
|
Date: Wed, 2 Mar 2022 09:41:49 +0800
|
||||||
|
Subject: [PATCH] stratovirt: provide a way to dynomically obtain firmware
|
||||||
|
|
||||||
|
For stratovirt, it requires firmware to boot. The default
|
||||||
|
path is `/usr/share/edk2/xxx/pflash`. Now, we provides a
|
||||||
|
way to set path in configuration.toml file
|
||||||
|
|
||||||
|
Signed-off-by: Xinle.Guo <guoxinle1@huawei.com>
|
||||||
|
---
|
||||||
|
src/runtime/cli/config/configuration-stratovirt.toml.in | 4 ++++
|
||||||
|
src/runtime/pkg/katautils/config.go | 6 ++++++
|
||||||
|
src/runtime/virtcontainers/stratovirt.go | 7 ++++---
|
||||||
|
3 files changed, 14 insertions(+), 3 deletions(-)
|
||||||
|
|
||||||
|
diff --git a/src/runtime/cli/config/configuration-stratovirt.toml.in b/src/runtime/cli/config/configuration-stratovirt.toml.in
|
||||||
|
index db46665..519c390 100644
|
||||||
|
--- a/src/runtime/cli/config/configuration-stratovirt.toml.in
|
||||||
|
+++ b/src/runtime/cli/config/configuration-stratovirt.toml.in
|
||||||
|
@@ -28,6 +28,10 @@ enable_annotations = @DEFENABLEANNOTATIONS@
|
||||||
|
# Your distribution recommends: @STRATOVIRTVALIDHYPERVISORPATHS@
|
||||||
|
valid_hypervisor_paths = @STRATOVIRTVALIDHYPERVISORPATHS@
|
||||||
|
|
||||||
|
+# Path to the firmware.
|
||||||
|
+# If you want that qemu uses the default firmware leave this option empty
|
||||||
|
+firmware = "@FIRMWAREPATH@"
|
||||||
|
+
|
||||||
|
# Path for the ozone specific to stratovirt
|
||||||
|
# If the ozone path is set, stratovirt will be launched in
|
||||||
|
# ozone secure environment. It is disabled by default.
|
||||||
|
diff --git a/src/runtime/pkg/katautils/config.go b/src/runtime/pkg/katautils/config.go
|
||||||
|
index b04cdee..cdc9d7a 100644
|
||||||
|
--- a/src/runtime/pkg/katautils/config.go
|
||||||
|
+++ b/src/runtime/pkg/katautils/config.go
|
||||||
|
@@ -951,6 +951,11 @@ func newStratovirtHypervisorConfig(h hypervisor) (vc.HypervisorConfig, error) {
|
||||||
|
errors.New("either image or initrd must be defined in the configuration file")
|
||||||
|
}
|
||||||
|
|
||||||
|
+ firmware, err := h.firmware()
|
||||||
|
+ if err != nil {
|
||||||
|
+ return vc.HypervisorConfig{}, err
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
kernelParams := h.kernelParams()
|
||||||
|
machineType := h.machineType()
|
||||||
|
|
||||||
|
@@ -979,6 +984,7 @@ func newStratovirtHypervisorConfig(h hypervisor) (vc.HypervisorConfig, error) {
|
||||||
|
KernelPath: kernel,
|
||||||
|
InitrdPath: initrd,
|
||||||
|
ImagePath: image,
|
||||||
|
+ FirmwarePath: firmware,
|
||||||
|
OzonePath: ozone,
|
||||||
|
KernelParams: vc.DeserializeParams(strings.Fields(kernelParams)),
|
||||||
|
HypervisorMachineType: machineType,
|
||||||
|
diff --git a/src/runtime/virtcontainers/stratovirt.go b/src/runtime/virtcontainers/stratovirt.go
|
||||||
|
index 27b45b7..d2b2233 100644
|
||||||
|
--- a/src/runtime/virtcontainers/stratovirt.go
|
||||||
|
+++ b/src/runtime/virtcontainers/stratovirt.go
|
||||||
|
@@ -659,11 +659,12 @@ func (s *stratovirt) getKernelParams(machineType string, initrdPath string) (str
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *stratovirt) getPFlash(machineType string) ([]string, error) {
|
||||||
|
- if s.config.PFlash != nil {
|
||||||
|
- return s.config.PFlash, nil
|
||||||
|
+ var PFlash []string
|
||||||
|
+ if s.config.FirmwarePath != "" {
|
||||||
|
+ PFlash = append(PFlash, fmt.Sprintf("file=%s,if=pflash,unit=0", s.config.FirmwarePath))
|
||||||
|
+ return PFlash, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
- var PFlash []string
|
||||||
|
switch machineType {
|
||||||
|
case MachineTypeQ35:
|
||||||
|
PFlash = append(PFlash, fmt.Sprintf("file=%s,if=pflash,unit=0", Q35PFlashCode))
|
||||||
|
--
|
||||||
|
2.20.1.windows.1
|
||||||
|
|
||||||
@ -0,0 +1,95 @@
|
|||||||
|
From c455ef6f406ba317af76b48ff79db131d0594bc1 Mon Sep 17 00:00:00 2001
|
||||||
|
From: "Xinle.Guo" <guoxinle1@huawei.com>
|
||||||
|
Date: Fri, 18 Mar 2022 10:49:35 +0800
|
||||||
|
Subject: [PATCH] stratovirt: fix the problem that add more than 16 root port
|
||||||
|
devices
|
||||||
|
|
||||||
|
It will failed to start StratoVirt sandbox if pcie root prot is set
|
||||||
|
more than 16. The reason is that StratoVirt can only distinguish
|
||||||
|
hexadecimal device address number.
|
||||||
|
|
||||||
|
Signed-off-by: Xinle.Guo <guoxinle1@huawei.com>
|
||||||
|
---
|
||||||
|
src/runtime/virtcontainers/stratovirt.go | 17 ++++++++++-------
|
||||||
|
1 file changed, 10 insertions(+), 7 deletions(-)
|
||||||
|
|
||||||
|
diff --git a/src/runtime/virtcontainers/stratovirt.go b/src/runtime/virtcontainers/stratovirt.go
|
||||||
|
index d2b2233..98a702a 100644
|
||||||
|
--- a/src/runtime/virtcontainers/stratovirt.go
|
||||||
|
+++ b/src/runtime/virtcontainers/stratovirt.go
|
||||||
|
@@ -102,7 +102,7 @@ func (r rootPortDevice) getParams(config *vmConfig) []string {
|
||||||
|
r.bus = "pcie.0"
|
||||||
|
}
|
||||||
|
devParams = append(devParams, Param{"bus", r.bus})
|
||||||
|
- devParams = append(devParams, Param{"addr", fmt.Sprintf("%d", r.slot)})
|
||||||
|
+ devParams = append(devParams, Param{"addr", fmt.Sprintf("0x%x", r.slot)})
|
||||||
|
|
||||||
|
driver := "pcie-root-port"
|
||||||
|
params = append(params, "-device", fmt.Sprintf("%s,%s", driver, strings.Join(SerializeParams(devParams, "="), ",")))
|
||||||
|
@@ -721,6 +721,9 @@ func (s *stratovirt) createDevices() []VirtioDev {
|
||||||
|
|
||||||
|
func (s *stratovirt) appendRootPort(ctx context.Context, devices []VirtioDev) []VirtioDev {
|
||||||
|
number := s.config.PCIeRootPort
|
||||||
|
+ if number > 20 {
|
||||||
|
+ number = 20
|
||||||
|
+ }
|
||||||
|
|
||||||
|
for i := uint32(1); i < number+1; i++ {
|
||||||
|
addr, err := s.vmConfig.rootBus.AddDevice(ctx, fmt.Sprintf("%s.%d", RootPortPrefix, i))
|
||||||
|
@@ -765,7 +768,7 @@ func (s *stratovirt) appendBlock(ctx context.Context, devices []VirtioDev) []Vir
|
||||||
|
filePath: s.vmConfig.rootfsPath,
|
||||||
|
deviceID: "virtio-blk0",
|
||||||
|
bus: bus,
|
||||||
|
- addr: fmt.Sprintf("%d", addr),
|
||||||
|
+ addr: fmt.Sprintf("0x%x", addr),
|
||||||
|
iothread: iothread,
|
||||||
|
})
|
||||||
|
|
||||||
|
@@ -791,7 +794,7 @@ func (s *stratovirt) appendRng(ctx context.Context, devices []VirtioDev) []Virti
|
||||||
|
rng: "objrng0",
|
||||||
|
deviceID: "virtio-rng0",
|
||||||
|
bus: bus,
|
||||||
|
- addr: fmt.Sprintf("%d", addr),
|
||||||
|
+ addr: fmt.Sprintf("0x%x", addr),
|
||||||
|
})
|
||||||
|
|
||||||
|
return devices
|
||||||
|
@@ -818,7 +821,7 @@ func (s *stratovirt) appendConsole(ctx context.Context, devices []VirtioDev) []V
|
||||||
|
charDev: "charconsole0",
|
||||||
|
deviceID: "virtio-console0",
|
||||||
|
bus: bus,
|
||||||
|
- addr: fmt.Sprintf("%d", addr),
|
||||||
|
+ addr: fmt.Sprintf("0x%x", addr),
|
||||||
|
})
|
||||||
|
|
||||||
|
return devices
|
||||||
|
@@ -841,7 +844,7 @@ func (s *stratovirt) appendVhostVsock(ctx context.Context, devices []VirtioDev,
|
||||||
|
id: "vsock-id",
|
||||||
|
guestID: fmt.Sprintf("%d", vsock.ContextID),
|
||||||
|
bus: bus,
|
||||||
|
- addr: fmt.Sprintf("%d", addr),
|
||||||
|
+ addr: fmt.Sprintf("0x%x", addr),
|
||||||
|
})
|
||||||
|
|
||||||
|
return devices
|
||||||
|
@@ -872,7 +875,7 @@ func (s *stratovirt) appendNetwork(ctx context.Context, devices []VirtioDev, end
|
||||||
|
deviceID: name,
|
||||||
|
bus: bus,
|
||||||
|
mac: endpoint.HardwareAddr(),
|
||||||
|
- addr: fmt.Sprintf("%d", addr),
|
||||||
|
+ addr: fmt.Sprintf("0x%x", addr),
|
||||||
|
})
|
||||||
|
|
||||||
|
return devices
|
||||||
|
@@ -904,7 +907,7 @@ func (s *stratovirt) appendVirtioFs(ctx context.Context, devices []VirtioDev, vo
|
||||||
|
tag: volume.MountTag,
|
||||||
|
deviceID: "virtio-fs0",
|
||||||
|
bus: bus,
|
||||||
|
- addr: fmt.Sprintf("%d", addr),
|
||||||
|
+ addr: fmt.Sprintf("0x%x", addr),
|
||||||
|
})
|
||||||
|
|
||||||
|
return devices
|
||||||
|
--
|
||||||
|
2.20.1.windows.1
|
||||||
|
|
||||||
38
patches/0030-use-host_device-blockdev-adding.patch
Normal file
38
patches/0030-use-host_device-blockdev-adding.patch
Normal file
@ -0,0 +1,38 @@
|
|||||||
|
From 2da8be5657f219de41e11917cb916895934749b8 Mon Sep 17 00:00:00 2001
|
||||||
|
From: chengzrz <czrzrichard@gmail.com>
|
||||||
|
Date: Mon, 22 Aug 2022 19:19:52 +0800
|
||||||
|
Subject: [PATCH] use host_device blockdev adding
|
||||||
|
|
||||||
|
ExecuteBlockdevAdd() and ExecuteBlockdevAddWithCache() both appear to be
|
||||||
|
intended to create block devices in the guest which backend onto a block
|
||||||
|
device in the host. That seems to be the way that Kata always uses it.
|
||||||
|
However blockdevAddBaseArgs(), used by both those functions always uses the
|
||||||
|
"file" driver, which is only intended for use with regular file backends.
|
||||||
|
|
||||||
|
Use of the "file" driver for host block devices was deprecated in qemu-3.0,
|
||||||
|
and has been removed entirely in qemu-6.0 (commit 8d17adf34f5). We should
|
||||||
|
be using the "host_device" driver instead.
|
||||||
|
|
||||||
|
reference: https://github.com/kata-containers/kata-containers/commit/1b021929864fa45b643d9603d6615cc4b86235d7
|
||||||
|
|
||||||
|
Signed-off-by: chengzrz <czrzrichard@gmail.com>
|
||||||
|
---
|
||||||
|
src/runtime/vendor/github.com/kata-containers/govmm/qemu/qmp.go | 2 +-
|
||||||
|
1 file changed, 1 insertion(+), 1 deletion(-)
|
||||||
|
|
||||||
|
diff --git a/src/runtime/vendor/github.com/kata-containers/govmm/qemu/qmp.go b/src/runtime/vendor/github.com/kata-containers/govmm/qemu/qmp.go
|
||||||
|
index 97e924559..0e0337dbf 100644
|
||||||
|
--- a/src/runtime/vendor/github.com/kata-containers/govmm/qemu/qmp.go
|
||||||
|
+++ b/src/runtime/vendor/github.com/kata-containers/govmm/qemu/qmp.go
|
||||||
|
@@ -775,7 +775,7 @@ func (q *QMP) blockdevAddBaseArgs(device, blockdevID string, ro bool) (map[strin
|
||||||
|
"driver": "raw",
|
||||||
|
"read-only": ro,
|
||||||
|
"file": map[string]interface{}{
|
||||||
|
- "driver": "file",
|
||||||
|
+ "driver": "host_device",
|
||||||
|
"filename": device,
|
||||||
|
},
|
||||||
|
}
|
||||||
|
--
|
||||||
|
2.25.1
|
||||||
|
|
||||||
33
patches/0031-add-explicit-on-after-kernel_irqchip.patch
Normal file
33
patches/0031-add-explicit-on-after-kernel_irqchip.patch
Normal file
@ -0,0 +1,33 @@
|
|||||||
|
From 0c9c5ca20b6aeae5e550decfd3540b389fb02cb5 Mon Sep 17 00:00:00 2001
|
||||||
|
From: chengzrz <czrzrichard@gmail.com>
|
||||||
|
Date: Tue, 23 Aug 2022 15:30:05 +0800
|
||||||
|
Subject: [PATCH] add explicit on after kernel_irqchip
|
||||||
|
Kata uses the 'kernel_irqchip' machine option to qemu. By default it
|
||||||
|
uses it in what qemu calls the "short-form boolean" with no parameter.
|
||||||
|
That style was deprecated by qemu between 5.2 and 6.0 (commit
|
||||||
|
ccd3b3b8112b) and effectively removed entirely between 6.0 and 6.1
|
||||||
|
(commit d8fb7d0969d5).
|
||||||
|
|
||||||
|
reference:https://github.com/kata-containers/kata-containers/commit/316509566966e4c9b3fd9ba3521554b384fdbf88
|
||||||
|
|
||||||
|
Signed-off-by: chengzrz <czrzrichard@gmail.com>
|
||||||
|
---
|
||||||
|
src/runtime/virtcontainers/qemu_amd64.go | 2 +-
|
||||||
|
1 file changed, 1 insertion(+), 1 deletion(-)
|
||||||
|
|
||||||
|
diff --git a/src/runtime/virtcontainers/qemu_amd64.go b/src/runtime/virtcontainers/qemu_amd64.go
|
||||||
|
index 1a045fae0..bc598c653 100644
|
||||||
|
--- a/src/runtime/virtcontainers/qemu_amd64.go
|
||||||
|
+++ b/src/runtime/virtcontainers/qemu_amd64.go
|
||||||
|
@@ -27,7 +27,7 @@ const (
|
||||||
|
|
||||||
|
defaultQemuMachineType = QemuPC
|
||||||
|
|
||||||
|
- defaultQemuMachineOptions = "accel=kvm,kernel_irqchip"
|
||||||
|
+ defaultQemuMachineOptions = "accel=kvm,kernel_irqchip=on"
|
||||||
|
|
||||||
|
qmpMigrationWaitTimeout = 5 * time.Second
|
||||||
|
)
|
||||||
|
--
|
||||||
|
2.25.1
|
||||||
|
|
||||||
@ -0,0 +1,67 @@
|
|||||||
|
From e3296a68f7fa270d72605dfd4eb40b1081a79969 Mon Sep 17 00:00:00 2001
|
||||||
|
From: chengzrz <czrzrichard@gmail.com>
|
||||||
|
Date: Fri, 2 Sep 2022 15:10:26 +0800
|
||||||
|
Subject: [PATCH] qmp: Don't use deprecated 'props' field for object-add
|
||||||
|
|
||||||
|
Use of the 'props' argument to 'object-add' has been deprecated since QEMU
|
||||||
|
5.0 (commit 5f07c4d60d09) in favor of flattening the properties directly
|
||||||
|
into the 'object-add' arguments. Support for 'props' is removed entirely
|
||||||
|
in qemu 6.0 (commit 50243407457a).
|
||||||
|
|
||||||
|
reference:https://github.com/kata-containers/kata-containers/commit/d27256f8635d3fa382d6cbd9f3a60f601773c4dc
|
||||||
|
|
||||||
|
Signed-off-by: chengzrz <czrzrichard@gmail.com>
|
||||||
|
---
|
||||||
|
.../kata-containers/govmm/qemu/qmp.go | 18 +++++++-----------
|
||||||
|
1 file changed, 7 insertions(+), 11 deletions(-)
|
||||||
|
|
||||||
|
diff --git a/src/runtime/vendor/github.com/kata-containers/govmm/qemu/qmp.go b/src/runtime/vendor/github.com/kata-containers/govmm/qemu/qmp.go
|
||||||
|
index 97e9245..91dd732 100644
|
||||||
|
--- a/src/runtime/vendor/github.com/kata-containers/govmm/qemu/qmp.go
|
||||||
|
+++ b/src/runtime/vendor/github.com/kata-containers/govmm/qemu/qmp.go
|
||||||
|
@@ -1387,17 +1387,16 @@ func (q *QMP) ExecQueryCpusFast(ctx context.Context) ([]CPUInfoFast, error) {
|
||||||
|
|
||||||
|
// ExecMemdevAdd adds size of MiB memory device to the guest
|
||||||
|
func (q *QMP) ExecMemdevAdd(ctx context.Context, qomtype, id, mempath string, size int, share bool, driver, driverID string) error {
|
||||||
|
- props := map[string]interface{}{"size": uint64(size) << 20}
|
||||||
|
args := map[string]interface{}{
|
||||||
|
"qom-type": qomtype,
|
||||||
|
"id": id,
|
||||||
|
- "props": props,
|
||||||
|
+ "size": uint64(size) << 20,
|
||||||
|
}
|
||||||
|
if mempath != "" {
|
||||||
|
- props["mem-path"] = mempath
|
||||||
|
+ args["mem-path"] = mempath
|
||||||
|
}
|
||||||
|
if share {
|
||||||
|
- props["share"] = true
|
||||||
|
+ args["share"] = true
|
||||||
|
}
|
||||||
|
err := q.executeCommand(ctx, "object-add", args, nil)
|
||||||
|
if err != nil {
|
||||||
|
@@ -1439,17 +1438,14 @@ func (q *QMP) ExecuteNVDIMMDeviceAdd(ctx context.Context, id, mempath string, si
|
||||||
|
args := map[string]interface{}{
|
||||||
|
"qom-type": "memory-backend-file",
|
||||||
|
"id": "nvdimmbackmem" + id,
|
||||||
|
- "props": map[string]interface{}{
|
||||||
|
- "mem-path": mempath,
|
||||||
|
- "size": size,
|
||||||
|
- "share": true,
|
||||||
|
- },
|
||||||
|
+ "mem-path": mempath,
|
||||||
|
+ "size": size,
|
||||||
|
+ "share": true,
|
||||||
|
}
|
||||||
|
|
||||||
|
if q.version.Major > 4 || (q.version.Major == 4 && q.version.Minor >= 1) {
|
||||||
|
if pmem != nil {
|
||||||
|
- props := args["props"].(map[string]interface{})
|
||||||
|
- props["pmem"] = *pmem
|
||||||
|
+ args["pmem"] = *pmem
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
--
|
||||||
|
2.25.1
|
||||||
|
|
||||||
@ -1,20 +0,0 @@
|
|||||||
#!/bin/bash
|
|
||||||
|
|
||||||
if [[ -f ./patch_flag ]];then
|
|
||||||
echo "proxy patched!"
|
|
||||||
exit 0
|
|
||||||
fi
|
|
||||||
|
|
||||||
tar -zxvf proxy-1.11.1.tar.gz
|
|
||||||
cp -fr ./proxy-1.11.1/* ./
|
|
||||||
rm -rf ./proxy-1.11.1
|
|
||||||
cat ./series.conf | while read line
|
|
||||||
do
|
|
||||||
if [[ $line == '' || $line =~ ^\s*# ]]; then
|
|
||||||
continue
|
|
||||||
fi
|
|
||||||
echo "====patch $line======"
|
|
||||||
pwd
|
|
||||||
patch -p1 -F1 -s < ./patches/$line
|
|
||||||
done
|
|
||||||
touch ./patch_flag
|
|
||||||
@ -1,403 +0,0 @@
|
|||||||
From a260bbe394f91fa05b163315390ed133de5c5494 Mon Sep 17 00:00:00 2001
|
|
||||||
From: holyfei <yangfeiyu20092010@163.com>
|
|
||||||
Date: Wed, 19 Aug 2020 17:43:24 +0800
|
|
||||||
Subject: [PATCH] clock: synchronizes clock info to agent
|
|
||||||
|
|
||||||
reason: virtual machine's clock may be incorrect, proxy synchronizes
|
|
||||||
clock info to help virtual machine adjust clock time
|
|
||||||
|
|
||||||
Signed-off-by: yangfeiyu <yangfeiyu2@huawei.com>
|
|
||||||
---
|
|
||||||
proxy.go | 7 ++
|
|
||||||
proxy_test.go | 29 +++++
|
|
||||||
sync_clock_client.go | 107 +++++++++++++++++
|
|
||||||
sync_clock_client_test.go | 132 +++++++++++++++++++++
|
|
||||||
.../kata-containers/agent/pkg/clock/clock_util.go | 44 +++++++
|
|
||||||
5 files changed, 319 insertions(+)
|
|
||||||
create mode 100644 sync_clock_client.go
|
|
||||||
create mode 100644 sync_clock_client_test.go
|
|
||||||
create mode 100644 vendor/github.com/kata-containers/agent/pkg/clock/clock_util.go
|
|
||||||
|
|
||||||
diff --git a/proxy.go b/proxy.go
|
|
||||||
index ab062a5..9dfcb3c 100644
|
|
||||||
--- a/proxy.go
|
|
||||||
+++ b/proxy.go
|
|
||||||
@@ -105,6 +105,13 @@ func serve(servConn io.ReadWriteCloser, proto, addr string, results chan error)
|
|
||||||
// Start the heartbeat in a separate go routine
|
|
||||||
go heartBeat(session)
|
|
||||||
|
|
||||||
+ // start the sync clock in a separate go routine
|
|
||||||
+ syncClockStream, err := session.Open()
|
|
||||||
+ if err != nil {
|
|
||||||
+ return nil, nil, err
|
|
||||||
+ }
|
|
||||||
+ go SyncClock(syncClockStream)
|
|
||||||
+
|
|
||||||
// serving connection
|
|
||||||
l, err := net.Listen(proto, addr)
|
|
||||||
if err != nil {
|
|
||||||
diff --git a/proxy_test.go b/proxy_test.go
|
|
||||||
index 923b138..94fa523 100644
|
|
||||||
--- a/proxy_test.go
|
|
||||||
+++ b/proxy_test.go
|
|
||||||
@@ -10,6 +10,7 @@ package main
|
|
||||||
import (
|
|
||||||
"bytes"
|
|
||||||
"crypto/md5"
|
|
||||||
+ "encoding/json"
|
|
||||||
"fmt"
|
|
||||||
"io"
|
|
||||||
"io/ioutil"
|
|
||||||
@@ -23,6 +24,7 @@ import (
|
|
||||||
"testing"
|
|
||||||
|
|
||||||
"github.com/hashicorp/yamux"
|
|
||||||
+ "github.com/kata-containers/agent/pkg/clock"
|
|
||||||
"github.com/stretchr/testify/assert"
|
|
||||||
)
|
|
||||||
|
|
||||||
@@ -121,6 +123,13 @@ func server(listener net.Listener, closeCh chan bool) error {
|
|
||||||
session.Close()
|
|
||||||
}()
|
|
||||||
|
|
||||||
+ // accept the sync clock stream first
|
|
||||||
+ if syncClockStream, err := session.Accept(); err != nil {
|
|
||||||
+ return err
|
|
||||||
+ } else {
|
|
||||||
+ go serverSyncClock(syncClockStream)
|
|
||||||
+ }
|
|
||||||
+
|
|
||||||
for {
|
|
||||||
stream, err := session.Accept()
|
|
||||||
if err != nil {
|
|
||||||
@@ -133,6 +142,26 @@ func server(listener net.Listener, closeCh chan bool) error {
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
+func serverSyncClock(stream net.Conn) {
|
|
||||||
+ for {
|
|
||||||
+ buf, byteNum, err := readConnData(stream)
|
|
||||||
+ if err != nil {
|
|
||||||
+ continue
|
|
||||||
+ }
|
|
||||||
+ var clockInfo clock.TimeValue
|
|
||||||
+ if err := json.Unmarshal(buf[:byteNum], &clockInfo); err != nil {
|
|
||||||
+ continue
|
|
||||||
+ }
|
|
||||||
+ nowTime := clock.GetCurrentTimeNs()
|
|
||||||
+ clockInfo.ClientArriveTime = nowTime
|
|
||||||
+ clockInfo.ServerSendTime = nowTime
|
|
||||||
+ b, _ := json.Marshal(clockInfo)
|
|
||||||
+ if err := clock.WriteConnData(stream, b); err != nil {
|
|
||||||
+ continue
|
|
||||||
+ }
|
|
||||||
+ }
|
|
||||||
+}
|
|
||||||
+
|
|
||||||
func TestUnixAddrParsing(T *testing.T) {
|
|
||||||
buf := "unix://foo/bar"
|
|
||||||
addr, err := unixAddr(buf)
|
|
||||||
diff --git a/sync_clock_client.go b/sync_clock_client.go
|
|
||||||
new file mode 100644
|
|
||||||
index 0000000..9bf3e91
|
|
||||||
--- /dev/null
|
|
||||||
+++ b/sync_clock_client.go
|
|
||||||
@@ -0,0 +1,107 @@
|
|
||||||
+// Copyright (c) Huawei Technologies Co., Ltd. 2018. All rights reserved.
|
|
||||||
+// SPDX-License-Identifier: Apache-2.0
|
|
||||||
+// Description: sync clock client related function
|
|
||||||
+// Author: xueshaojia x00464843
|
|
||||||
+// Create: 2018-11-10
|
|
||||||
+
|
|
||||||
+package main
|
|
||||||
+
|
|
||||||
+import (
|
|
||||||
+ "encoding/json"
|
|
||||||
+ "fmt"
|
|
||||||
+ "net"
|
|
||||||
+ "time"
|
|
||||||
+
|
|
||||||
+ "github.com/kata-containers/agent/pkg/clock"
|
|
||||||
+)
|
|
||||||
+
|
|
||||||
+const (
|
|
||||||
+ allowTimeDiff = 10 * 1000 * 1000 // allow 10ms's difference
|
|
||||||
+ syncClockInterval = 60 * time.Second // sync clock with agent every 60 seconds
|
|
||||||
+ rpcTimeout = 10 * time.Second // timeout for proxy's reading data
|
|
||||||
+)
|
|
||||||
+
|
|
||||||
+// readConnData reads data from stream
|
|
||||||
+func readConnData(stream net.Conn) (buf []byte, byteNum int, err error) {
|
|
||||||
+ // set read deadline to avoid case as following:
|
|
||||||
+ // proxy and agent are both reading and then syncClock will never work
|
|
||||||
+ stream.SetReadDeadline(time.Now().Add(rpcTimeout))
|
|
||||||
+ buf = make([]byte, clock.MaxSyncClockByteNum)
|
|
||||||
+ byteNum, err = stream.Read(buf)
|
|
||||||
+ return buf, byteNum, err
|
|
||||||
+}
|
|
||||||
+
|
|
||||||
+// getGuestClock syncs guest clock info
|
|
||||||
+// sends ClientSendTime
|
|
||||||
+// waits for ClientArriveTime and ServerSendTime
|
|
||||||
+func getGuestClock(stream net.Conn, clockInfo *clock.TimeValue) error {
|
|
||||||
+ clockInfo.Delta = 0
|
|
||||||
+ b, err := json.Marshal(clockInfo)
|
|
||||||
+ if err != nil {
|
|
||||||
+ return err
|
|
||||||
+ }
|
|
||||||
+ if err = clock.WriteConnData(stream, b); err != nil {
|
|
||||||
+ return err
|
|
||||||
+ }
|
|
||||||
+
|
|
||||||
+ buf, byteNum, err := readConnData(stream)
|
|
||||||
+ if err != nil {
|
|
||||||
+ return err
|
|
||||||
+ }
|
|
||||||
+
|
|
||||||
+ if err = json.Unmarshal(buf[:byteNum], clockInfo); err != nil {
|
|
||||||
+ return fmt.Errorf("sync clock, parse guest clocktime error:%v", err)
|
|
||||||
+ }
|
|
||||||
+ return nil
|
|
||||||
+}
|
|
||||||
+
|
|
||||||
+// adjustGuestClock tells server to ajust local clock with Delta
|
|
||||||
+func adjustGuestClock(stream net.Conn, clockInfo *clock.TimeValue) error {
|
|
||||||
+ b, err := json.Marshal(clockInfo)
|
|
||||||
+ if err != nil {
|
|
||||||
+ return err
|
|
||||||
+ }
|
|
||||||
+ logger().Debugf("sync clock, send:%s", string(b))
|
|
||||||
+ return clock.WriteConnData(stream, b)
|
|
||||||
+}
|
|
||||||
+
|
|
||||||
+// syncClock performs all the steps
|
|
||||||
+// 1 get client send time[host]
|
|
||||||
+// 2 request for client arrive time and server send time[guest os]
|
|
||||||
+// 3 get server arrive time[host]
|
|
||||||
+// 4 calculate clock diff
|
|
||||||
+// 5 request to adjust guest clock
|
|
||||||
+func syncClock(stream net.Conn) error {
|
|
||||||
+ var clockInfo clock.TimeValue
|
|
||||||
+ if clockInfo.ClientSendTime = clock.GetCurrentTimeNs(); clockInfo.ClientSendTime <= 0 {
|
|
||||||
+ return fmt.Errorf("sync clock, get client sendtime error")
|
|
||||||
+ }
|
|
||||||
+ err := getGuestClock(stream, &clockInfo)
|
|
||||||
+ if err != nil {
|
|
||||||
+ return fmt.Errorf("sync clock, get guest clocktime error:%v", err)
|
|
||||||
+ }
|
|
||||||
+ if clockInfo.ServerArriveTime = clock.GetCurrentTimeNs(); clockInfo.ServerArriveTime <= 0 {
|
|
||||||
+ return fmt.Errorf("sync clock, get client recvtime error")
|
|
||||||
+ }
|
|
||||||
+ if clockInfo.ClientSendTime <= 0 || clockInfo.ClientArriveTime <= 0 || clockInfo.ServerSendTime <= 0 {
|
|
||||||
+ return fmt.Errorf("sync clock, some fields of NTP message error, raw message:%v", clockInfo)
|
|
||||||
+ }
|
|
||||||
+
|
|
||||||
+ delta := ((clockInfo.ClientSendTime - clockInfo.ClientArriveTime) + (clockInfo.ServerArriveTime - clockInfo.ServerSendTime)) / 2
|
|
||||||
+ if delta < -allowTimeDiff || delta > allowTimeDiff {
|
|
||||||
+ clockInfo.Delta = delta
|
|
||||||
+ if err := adjustGuestClock(stream, &clockInfo); err != nil {
|
|
||||||
+ return fmt.Errorf("sync clock, failed to adjust guest clock : %v", err)
|
|
||||||
+ }
|
|
||||||
+ }
|
|
||||||
+ return nil
|
|
||||||
+}
|
|
||||||
+
|
|
||||||
+func SyncClock(stream net.Conn) {
|
|
||||||
+ for {
|
|
||||||
+ if err := syncClock(stream); err != nil {
|
|
||||||
+ logger().WithError(err).Error("sync clock failed")
|
|
||||||
+ }
|
|
||||||
+ time.Sleep(syncClockInterval)
|
|
||||||
+ }
|
|
||||||
+}
|
|
||||||
diff --git a/sync_clock_client_test.go b/sync_clock_client_test.go
|
|
||||||
new file mode 100644
|
|
||||||
index 0000000..b0b1c85
|
|
||||||
--- /dev/null
|
|
||||||
+++ b/sync_clock_client_test.go
|
|
||||||
@@ -0,0 +1,132 @@
|
|
||||||
+// Copyright (c) Huawei Technologies Co., Ltd. 2018. All rights reserved.
|
|
||||||
+// SPDX-License-Identifier: Apache-2.0
|
|
||||||
+// Description: sync clock client related test
|
|
||||||
+// Author: xueshaojia x00464843
|
|
||||||
+// Create: 2018-11-10
|
|
||||||
+
|
|
||||||
+package main
|
|
||||||
+
|
|
||||||
+import (
|
|
||||||
+ "encoding/json"
|
|
||||||
+ "fmt"
|
|
||||||
+ "math/rand"
|
|
||||||
+ "net"
|
|
||||||
+ "os"
|
|
||||||
+ "testing"
|
|
||||||
+
|
|
||||||
+ "github.com/hashicorp/yamux"
|
|
||||||
+ "github.com/kata-containers/agent/pkg/clock"
|
|
||||||
+)
|
|
||||||
+
|
|
||||||
+func TestReadWriteData(t *testing.T) {
|
|
||||||
+ var clockInfo clock.TimeValue
|
|
||||||
+ clockInfo.ClientSendTime = clock.GetCurrentTimeNs()
|
|
||||||
+ b, err := json.Marshal(clockInfo)
|
|
||||||
+ if err != nil {
|
|
||||||
+ t.Fatalf("Marshal clock info fail, err:%v", err)
|
|
||||||
+ }
|
|
||||||
+
|
|
||||||
+ err = clock.WriteConnData(clientStream, b)
|
|
||||||
+ fmt.Printf("client send: %s\n", string(b))
|
|
||||||
+ if err != nil {
|
|
||||||
+ t.Fatalf("send clock info fail, err:%v", err)
|
|
||||||
+ }
|
|
||||||
+ _, _, err = readConnData(clientStream)
|
|
||||||
+ if err != nil {
|
|
||||||
+ t.Fatalf("recv clock info fail, err:%v", err)
|
|
||||||
+ }
|
|
||||||
+}
|
|
||||||
+
|
|
||||||
+func SetUpServer(sock string, readyChan chan int) error {
|
|
||||||
+ var err error
|
|
||||||
+ listener, err = net.Listen("unix", sock)
|
|
||||||
+ if err != nil {
|
|
||||||
+ return err
|
|
||||||
+ }
|
|
||||||
+ readyChan <- 1
|
|
||||||
+ conn, err := listener.Accept()
|
|
||||||
+ if err != nil {
|
|
||||||
+ return err
|
|
||||||
+ }
|
|
||||||
+ session, err := yamux.Server(conn, nil)
|
|
||||||
+ if err != nil {
|
|
||||||
+ return err
|
|
||||||
+ }
|
|
||||||
+ serverSession = session
|
|
||||||
+ stream, err := session.Accept()
|
|
||||||
+ if err != nil {
|
|
||||||
+ return err
|
|
||||||
+ }
|
|
||||||
+ for {
|
|
||||||
+ var clockInfo clock.TimeValue
|
|
||||||
+ var byteNum int
|
|
||||||
+ var err error
|
|
||||||
+ buf := make([]byte, 400)
|
|
||||||
+ if byteNum, err = stream.Read(buf); err != nil {
|
|
||||||
+ break
|
|
||||||
+ }
|
|
||||||
+
|
|
||||||
+ if err = json.Unmarshal(buf[:byteNum], &clockInfo); err != nil {
|
|
||||||
+ break
|
|
||||||
+ }
|
|
||||||
+ if clockInfo.Delta == 0 {
|
|
||||||
+ nowTime := clock.GetCurrentTimeNs()
|
|
||||||
+ clockInfo.ClientArriveTime = nowTime
|
|
||||||
+ clockInfo.ClientArriveTime = nowTime
|
|
||||||
+ b, _ := json.Marshal(&clockInfo)
|
|
||||||
+ stream.Write(b)
|
|
||||||
+ }
|
|
||||||
+ }
|
|
||||||
+ return nil
|
|
||||||
+}
|
|
||||||
+
|
|
||||||
+func SetUpClient(sock string) error {
|
|
||||||
+ conn, err := net.Dial("unix", sock)
|
|
||||||
+ if err != nil {
|
|
||||||
+ return err
|
|
||||||
+ }
|
|
||||||
+ session, err := yamux.Client(conn, nil)
|
|
||||||
+ if err != nil {
|
|
||||||
+ conn.Close()
|
|
||||||
+ return err
|
|
||||||
+ }
|
|
||||||
+ clientSession = session
|
|
||||||
+ stream, err := session.Open()
|
|
||||||
+ if err != nil {
|
|
||||||
+ clientSession.Close()
|
|
||||||
+ return err
|
|
||||||
+ }
|
|
||||||
+ clientStream = stream
|
|
||||||
+ return nil
|
|
||||||
+}
|
|
||||||
+
|
|
||||||
+func TearDown() {
|
|
||||||
+ listener.Close()
|
|
||||||
+ serverSession.Close()
|
|
||||||
+ clientSession.Close()
|
|
||||||
+}
|
|
||||||
+
|
|
||||||
+func GenSocket() string {
|
|
||||||
+ randSeed := clock.GetCurrentTimeNs()
|
|
||||||
+ rand.Seed(randSeed)
|
|
||||||
+ return fmt.Sprintf("/tmp/%d.sock", rand.Uint32())
|
|
||||||
+}
|
|
||||||
+
|
|
||||||
+var listener net.Listener
|
|
||||||
+var clientStream net.Conn
|
|
||||||
+var serverSession *yamux.Session
|
|
||||||
+var clientSession *yamux.Session
|
|
||||||
+
|
|
||||||
+func TestMain(m *testing.M) {
|
|
||||||
+ waitConn := make(chan int)
|
|
||||||
+ testSock := GenSocket()
|
|
||||||
+ go SetUpServer(testSock, waitConn)
|
|
||||||
+ <-waitConn
|
|
||||||
+ if err := SetUpClient(testSock); err != nil {
|
|
||||||
+ listener.Close()
|
|
||||||
+ serverSession.Close()
|
|
||||||
+ os.Exit(1)
|
|
||||||
+ }
|
|
||||||
+ m.Run()
|
|
||||||
+ TearDown()
|
|
||||||
+}
|
|
||||||
diff --git a/vendor/github.com/kata-containers/agent/pkg/clock/clock_util.go b/vendor/github.com/kata-containers/agent/pkg/clock/clock_util.go
|
|
||||||
new file mode 100644
|
|
||||||
index 0000000..03244fd
|
|
||||||
--- /dev/null
|
|
||||||
+++ b/vendor/github.com/kata-containers/agent/pkg/clock/clock_util.go
|
|
||||||
@@ -0,0 +1,44 @@
|
|
||||||
+// Copyright (c) Huawei Technologies Co., Ltd. 2019. All rights reserved.
|
|
||||||
+// SPDX-License-Identifier: Apache-2.0
|
|
||||||
+// Description: common functions
|
|
||||||
+// Author: jiangpeifei
|
|
||||||
+// Create: 2019-05-28
|
|
||||||
+
|
|
||||||
+package clock
|
|
||||||
+
|
|
||||||
+import (
|
|
||||||
+ "net"
|
|
||||||
+ "syscall"
|
|
||||||
+)
|
|
||||||
+
|
|
||||||
+type TimeValue struct {
|
|
||||||
+ ClientSendTime int64 `json:"client_send_time"`
|
|
||||||
+ ClientArriveTime int64 `json:"client_arrive_time"`
|
|
||||||
+ ServerSendTime int64 `json:"server_send_time"`
|
|
||||||
+ ServerArriveTime int64 `json:"server_arrive_time"`
|
|
||||||
+ Delta int64 `json:"delta"`
|
|
||||||
+}
|
|
||||||
+
|
|
||||||
+const MaxSyncClockByteNum = 400 //sync clock byte num, max=400
|
|
||||||
+
|
|
||||||
+// getCurrentTimeNs returns UTC time in Ns
|
|
||||||
+func GetCurrentTimeNs() int64 {
|
|
||||||
+ var tv syscall.Timeval
|
|
||||||
+ if err := syscall.Gettimeofday(&tv); err != nil {
|
|
||||||
+ return -1
|
|
||||||
+ }
|
|
||||||
+ return tv.Sec*1000000000 + tv.Usec*1000
|
|
||||||
+}
|
|
||||||
+
|
|
||||||
+// readConnData reads data from stream
|
|
||||||
+func ReadConnData(stream net.Conn) (buf []byte, byteNum int, err error) {
|
|
||||||
+ buf = make([]byte, MaxSyncClockByteNum)
|
|
||||||
+ byteNum, err = stream.Read(buf)
|
|
||||||
+ return buf, byteNum, err
|
|
||||||
+}
|
|
||||||
+
|
|
||||||
+// writeConnData writes data to stream
|
|
||||||
+func WriteConnData(stream net.Conn, buf []byte) error {
|
|
||||||
+ _, err := stream.Write(buf)
|
|
||||||
+ return err
|
|
||||||
+}
|
|
||||||
--
|
|
||||||
2.14.3 (Apple Git-98)
|
|
||||||
|
|
||||||
Binary file not shown.
@ -1 +0,0 @@
|
|||||||
0001-clock-synchronizes-clock-info-to-agent.patch
|
|
||||||
@ -1,22 +0,0 @@
|
|||||||
#!/bin/bash
|
|
||||||
|
|
||||||
set -e
|
|
||||||
|
|
||||||
if [[ -f ./patch_flag ]];then
|
|
||||||
echo "runtime patched!"
|
|
||||||
exit 0
|
|
||||||
fi
|
|
||||||
|
|
||||||
tar -zxvf runtime-1.11.1.tar.gz
|
|
||||||
cp -fr ./runtime-1.11.1/* ./
|
|
||||||
rm -rf ./runtime-1.11.1
|
|
||||||
cat ./series.conf | while read line
|
|
||||||
do
|
|
||||||
if [[ $line == '' || $line =~ ^\s*# ]]; then
|
|
||||||
continue
|
|
||||||
fi
|
|
||||||
echo "====patch $line======"
|
|
||||||
patch -p1 -F1 -s < ./patches/$line
|
|
||||||
done
|
|
||||||
|
|
||||||
touch ./patch_flag
|
|
||||||
@ -1,165 +0,0 @@
|
|||||||
%define debug_package %{nil}
|
|
||||||
|
|
||||||
%define VERSION 1.11.1
|
|
||||||
%define RELEASE 11
|
|
||||||
|
|
||||||
Name: kata-runtime
|
|
||||||
Version: %{VERSION}
|
|
||||||
Release: %{RELEASE}
|
|
||||||
Summary: Kata Runtime
|
|
||||||
License: Apache 2.0
|
|
||||||
URL: https://github.com/kata-containers/runtime
|
|
||||||
Source0: https://github.com/kata-containers/runtime/archive/%{version}.tar.gz#/%{name}-v%{version}.tar.gz
|
|
||||||
|
|
||||||
BuildRoot: %_topdir/BUILDROOT
|
|
||||||
BuildRequires: automake golang gcc
|
|
||||||
|
|
||||||
%description
|
|
||||||
Kata-runtime is core component of Kata Container.
|
|
||||||
|
|
||||||
%prep
|
|
||||||
%setup -q -c -a 0 -n %{name}-%{version}
|
|
||||||
|
|
||||||
%build
|
|
||||||
cd %{_builddir}/%{name}-%{version}
|
|
||||||
|
|
||||||
set -e
|
|
||||||
# apply patches read from series.conf
|
|
||||||
sh apply-patches
|
|
||||||
|
|
||||||
# create tmp GOPATH dir to build kata-runtime
|
|
||||||
rm -rf /tmp/kata-build/
|
|
||||||
mkdir -p /tmp/kata-build/
|
|
||||||
GOPATH=/tmp/kata-build/
|
|
||||||
kata_base=$GOPATH/src/github.com/kata-containers
|
|
||||||
mkdir -p $kata_base
|
|
||||||
|
|
||||||
# get current kata-runtime absolute path
|
|
||||||
kata_runtime_path=$(readlink -f .)
|
|
||||||
ln -s $kata_runtime_path $kata_base/runtime
|
|
||||||
|
|
||||||
# export GOPATH env
|
|
||||||
export GOPATH=$(readlink -f $GOPATH)
|
|
||||||
cd ${kata_base}/runtime && make clean && make
|
|
||||||
rm -rf $GOPATH
|
|
||||||
|
|
||||||
# make kata-runtime default configuration
|
|
||||||
kata_config_path=$kata_runtime_path/cli/config/configuration-qemu.toml
|
|
||||||
ARCH=`arch`
|
|
||||||
|
|
||||||
# arch related config options
|
|
||||||
if [ "$ARCH" == "aarch64" ];then
|
|
||||||
sed -i 's/^machine_type.*$/machine_type = \"virt\"/' $kata_config_path
|
|
||||||
sed -i 's/^block_device_driver.*$/block_device_driver = \"virtio-scsi\"/' $kata_config_path
|
|
||||||
sed -i 's/^kernel_params.*$/kernel_params = \"pcie_ports=native pci=pcie_bus_perf agent.netlink_recv_buf_size=2MB\"/' $kata_config_path
|
|
||||||
sed -i 's/^hypervisor_params.*$/hypervisor_params = \"kvm-pit.lost_tick_policy=discard pcie-root-port.fast-plug=1 pcie-root-port.x-speed=16 pcie-root-port.x-width=32 pcie-root-port.fast-unplug=1\"/' $kata_config_path
|
|
||||||
sed -i 's/^#pcie_root_port.*$/pcie_root_port = 25/' $kata_config_path
|
|
||||||
else
|
|
||||||
sed -i 's#block_device_driver = \"virtio-scsi\"#block_device_driver = \"virtio-blk\"#' $kata_config_path
|
|
||||||
sed -i 's/^#hotplug_vfio_on_root_bus/hotplug_vfio_on_root_bus/' $kata_config_path
|
|
||||||
fi
|
|
||||||
|
|
||||||
# debug config
|
|
||||||
sed -i 's/^#enable_debug.*$/enable_debug = true/' $kata_config_path
|
|
||||||
|
|
||||||
# other config
|
|
||||||
sed -i 's#"/usr/bin/qemu.*"$#"/usr/bin/qemu-kvm"#' $kata_config_path
|
|
||||||
sed -i 's#/usr/share/kata-containers/vmlinuz\.container#/var/lib/kata/kernel#' $kata_config_path
|
|
||||||
sed -i 's#/usr/share/kata-containers/kata-containers-initrd\.img#/var/lib/kata/kata-containers-initrd\.img#' $kata_config_path
|
|
||||||
sed -i 's/^image/#image/' $kata_config_path
|
|
||||||
sed -i 's/^default_memory.*$/default_memory = 1024/' $kata_config_path
|
|
||||||
sed -i 's/^#enable_blk_mount/enable_blk_mount/' $kata_config_path
|
|
||||||
sed -i 's/^#block_device_cache_direct.*$/block_device_cache_direct = true/' $kata_config_path
|
|
||||||
sed -i 's/^#block_device_cache_set.*$/block_device_cache_set = true/' $kata_config_path
|
|
||||||
sed -i 's#/usr/libexec/kata-containers/kata-proxy#/usr/bin/kata-proxy#' $kata_config_path
|
|
||||||
sed -i 's#/usr/libexec/kata-containers/kata-shim#/usr/bin/kata-shim#' $kata_config_path
|
|
||||||
sed -i 's#/usr/libexec/kata-containers/kata-netmon#/usr/bin/kata-netmon#' $kata_config_path
|
|
||||||
sed -i 's/^#disable_new_netns.*$/disable_new_netns = true/' $kata_config_path
|
|
||||||
sed -i 's/^#disable_vhost_net.*$/disable_vhost_net = true/' $kata_config_path
|
|
||||||
sed -i 's/^internetworking_model.*$/internetworking_model=\"none\"/' $kata_config_path
|
|
||||||
sed -i 's/^enable_compat_old_cni.*$/#enable_compat_old_cni = true/' $kata_config_path
|
|
||||||
sed -i 's/^sandbox_cgroup_only.*$/sandbox_cgroup_only = true/' $kata_config_path
|
|
||||||
|
|
||||||
set +e
|
|
||||||
|
|
||||||
%install
|
|
||||||
cd %{_builddir}/%{name}-%{version}
|
|
||||||
mkdir -p -m 750 %{buildroot}/usr/bin
|
|
||||||
install -p -m 750 ./kata-runtime %{buildroot}/usr/bin
|
|
||||||
install -p -m 750 ./kata-netmon %{buildroot}/usr/bin
|
|
||||||
mkdir -p -m 750 %{buildroot}/usr/share/defaults/kata-containers
|
|
||||||
install -p -m 640 ./cli/config/configuration-qemu.toml %{buildroot}/usr/share/defaults/kata-containers/configuration.toml
|
|
||||||
|
|
||||||
%clean
|
|
||||||
|
|
||||||
%files
|
|
||||||
/usr/bin/kata-runtime
|
|
||||||
/usr/bin/kata-netmon
|
|
||||||
/usr/share/defaults/kata-containers/configuration.toml
|
|
||||||
|
|
||||||
%changelog
|
|
||||||
* Tue Nov 17 2020 yangfeiyu<yangfeiyu20102011@163.com> - 1.11.1-11
|
|
||||||
- Type:bugfix
|
|
||||||
- ID:NA
|
|
||||||
- SUG:upgrade
|
|
||||||
- DESC:fix cpu resource limited problem when sandox_cgroup_with_emulator config is enabled
|
|
||||||
|
|
||||||
* Fri Oct 9 2020 yangfeiyu<yangfeiyu20102011@163.com> - 1.11.1-10
|
|
||||||
- Type:feature
|
|
||||||
- ID:NA
|
|
||||||
- SUG:restart
|
|
||||||
- DESC:support using CNI plugin to insert mutiple network interfaces at the same time
|
|
||||||
|
|
||||||
* Mon Sep 28 2020 yangfeiyu<yangfeiyu20102011@163.com> - 1.11.1-9
|
|
||||||
- Type:bugfix
|
|
||||||
- ID:NA
|
|
||||||
- SUG:restart
|
|
||||||
- DESC:retry inserting of CNI interface when netmon is enable
|
|
||||||
|
|
||||||
* Sun Sep 27 2020 LiangZhang<zhangliang5@Huawei.com> - 1.11.1-8
|
|
||||||
- Type:bugfix
|
|
||||||
- ID:NA
|
|
||||||
- SUG:NA
|
|
||||||
- DESC:fix cmd params of direct use stratovirt binary
|
|
||||||
|
|
||||||
* Thu Sep 24 2020 LiangZhang<zhangliang5@Huawei.com> - 1.11.1-7
|
|
||||||
- Type:bugfix
|
|
||||||
- ID:NA
|
|
||||||
- SUG:NA
|
|
||||||
- DESC:fix invalid cmdline when start sandbox stratovirt
|
|
||||||
|
|
||||||
* Mon Sep 21 2020 yangfeiyu<yangfeiyu20102011@163.com> - 1.11.1-6
|
|
||||||
- Type:bugfix
|
|
||||||
- ID:NA
|
|
||||||
- SUG:NA
|
|
||||||
- DESC:fix sandboxRuntimeRootPath left problem
|
|
||||||
|
|
||||||
* Mon Sep 21 2020 yangfeiyu<yangfeiyu20102011@163.com> - 1.11.1-5
|
|
||||||
- Type:enhancement
|
|
||||||
- ID:NA
|
|
||||||
- SUG:NA
|
|
||||||
- DESC:add support for host cgroups with emulator
|
|
||||||
|
|
||||||
* Mon Sep 21 2020 LiangZhang<zhangliang5@Huawei.com> - 1.11.1-4
|
|
||||||
- Type:enhancement
|
|
||||||
- ID:NA
|
|
||||||
- SUG:NA
|
|
||||||
- DESC:add support of new sandbox StratoVirt
|
|
||||||
|
|
||||||
* Sat Sep 19 2020 yangfeiyu<yangfeiyu20102011@163.com> - 1.11.1-3
|
|
||||||
- Type:bugfix
|
|
||||||
- ID:NA
|
|
||||||
- SUG:NA
|
|
||||||
- DESC:fix del-iface doesn't delete the tap interface in the host problem
|
|
||||||
|
|
||||||
* Sat Sep 5 2020 yangfeiyu<yangfeiyu20102011@163.com> - 1.11.1-2
|
|
||||||
- Type:enhancement
|
|
||||||
- ID:NA
|
|
||||||
- SUG:NA
|
|
||||||
- DESC:use URL format for Source0
|
|
||||||
|
|
||||||
* Wed Aug 26 2020 yangfeiyu<yangfeiyu20102011@163.com> - 1.11.1-1
|
|
||||||
- Type:enhancement
|
|
||||||
- ID:NA
|
|
||||||
- SUG:NA
|
|
||||||
- DESC:modify kata-runtime spec file to build seperately
|
|
||||||
@ -1,44 +0,0 @@
|
|||||||
From 73fe7242d18a10a86bc216ec5e33a10a8751f85f Mon Sep 17 00:00:00 2001
|
|
||||||
From: jiangpengfei <jiangpengfei9@huawei.com>
|
|
||||||
Date: Fri, 24 Jul 2020 22:22:00 +0800
|
|
||||||
Subject: [PATCH 01/50] qmp: fix kata-runtime hungs when qemu process is D/T
|
|
||||||
state
|
|
||||||
|
|
||||||
reason: When set qemu's status to T and execute add-iface command
|
|
||||||
the command will hung all the time.It hungs while wait for qemu
|
|
||||||
to return version messages.
|
|
||||||
|
|
||||||
When qmp starts, set the timeout time to 15 seconds.When times
|
|
||||||
out, return qmp starts failed.We choose 15 seconds to keep consistent
|
|
||||||
with the agent client start.
|
|
||||||
|
|
||||||
Signed-off-by: jiangpengfei <jiangpengfei9@huawei.com>
|
|
||||||
---
|
|
||||||
vendor/github.com/intel/govmm/qemu/checklist | 1 +
|
|
||||||
vendor/github.com/intel/govmm/qemu/qmp.go | 2 ++
|
|
||||||
2 files changed, 3 insertions(+)
|
|
||||||
create mode 100644 vendor/github.com/intel/govmm/qemu/checklist
|
|
||||||
|
|
||||||
diff --git a/vendor/github.com/intel/govmm/qemu/checklist b/vendor/github.com/intel/govmm/qemu/checklist
|
|
||||||
new file mode 100644
|
|
||||||
index 00000000..b32f1855
|
|
||||||
--- /dev/null
|
|
||||||
+++ b/vendor/github.com/intel/govmm/qemu/checklist
|
|
||||||
@@ -0,0 +1 @@
|
|
||||||
+add timeout when qmp start to avoid qmp client hungs all the time
|
|
||||||
diff --git a/vendor/github.com/intel/govmm/qemu/qmp.go b/vendor/github.com/intel/govmm/qemu/qmp.go
|
|
||||||
index bf9a77dd..a64039de 100644
|
|
||||||
--- a/vendor/github.com/intel/govmm/qemu/qmp.go
|
|
||||||
+++ b/vendor/github.com/intel/govmm/qemu/qmp.go
|
|
||||||
@@ -722,6 +722,8 @@ func QMPStart(ctx context.Context, socket string, cfg QMPConfig, disconnectedCh
|
|
||||||
if q.version == nil {
|
|
||||||
return nil, nil, fmt.Errorf("failed to find QMP version information")
|
|
||||||
}
|
|
||||||
+ case <-time.After(15 * time.Second):
|
|
||||||
+ return nil, nil, fmt.Errorf("qmp start time out")
|
|
||||||
}
|
|
||||||
|
|
||||||
return q, q.version, nil
|
|
||||||
--
|
|
||||||
2.14.3 (Apple Git-98)
|
|
||||||
|
|
||||||
@ -1,55 +0,0 @@
|
|||||||
From cf595941e1d105af23bc006bf1998ac072733d0a Mon Sep 17 00:00:00 2001
|
|
||||||
From: jiangpengfei <jiangpengfei9@huawei.com>
|
|
||||||
Date: Sat, 25 Jul 2020 10:03:35 +0800
|
|
||||||
Subject: [PATCH 03/50] kata-runtime: fix kata-proxy process left problem
|
|
||||||
|
|
||||||
reason: stopSandbox function will send the DestroySandboxRequest
|
|
||||||
to kata-agent in the VM and then kill the kata-proxy process in
|
|
||||||
the host. However, if k.sendReq(DestroySandboxRequest) get error,
|
|
||||||
stopSandbox will return immediately not execute the following kill
|
|
||||||
kata-proxy process statement, which cause the kata-process left.
|
|
||||||
|
|
||||||
Signed-off-by: jiangpengfei <jiangpengfei9@huawei.com>
|
|
||||||
---
|
|
||||||
virtcontainers/kata_agent.go | 18 +++++++++++-------
|
|
||||||
1 file changed, 11 insertions(+), 7 deletions(-)
|
|
||||||
|
|
||||||
diff --git a/virtcontainers/kata_agent.go b/virtcontainers/kata_agent.go
|
|
||||||
index a0cf190e..be5e96aa 100644
|
|
||||||
--- a/virtcontainers/kata_agent.go
|
|
||||||
+++ b/virtcontainers/kata_agent.go
|
|
||||||
@@ -976,6 +976,17 @@ func (k *kataAgent) stopSandbox(sandbox *Sandbox) error {
|
|
||||||
return errorMissingProxy
|
|
||||||
}
|
|
||||||
|
|
||||||
+ // since stopSandbox will destroy the sandbox in the VM, and we don't need
|
|
||||||
+ // kata-proxy process to communicate with kata-agent again, so we should
|
|
||||||
+ // make sure kata-proxy can be killed cleanly, even when k.sendReq(DestroySandboxRequest)
|
|
||||||
+ // return error
|
|
||||||
+ defer func() {
|
|
||||||
+ _ = k.proxy.stop(k.state.ProxyPid)
|
|
||||||
+ // clean up agent state
|
|
||||||
+ k.state.ProxyPid = -1
|
|
||||||
+ k.state.URL = ""
|
|
||||||
+ }()
|
|
||||||
+
|
|
||||||
req := &grpc.DestroySandboxRequest{}
|
|
||||||
|
|
||||||
if _, err := k.sendReq(req); err != nil {
|
|
||||||
@@ -989,13 +1000,6 @@ func (k *kataAgent) stopSandbox(sandbox *Sandbox) error {
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
- if err := k.proxy.stop(k.state.ProxyPid); err != nil {
|
|
||||||
- return err
|
|
||||||
- }
|
|
||||||
-
|
|
||||||
- // clean up agent state
|
|
||||||
- k.state.ProxyPid = -1
|
|
||||||
- k.state.URL = ""
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
--
|
|
||||||
2.14.3 (Apple Git-98)
|
|
||||||
|
|
||||||
@ -1,65 +0,0 @@
|
|||||||
From 025520f7fd3aeb5ed53b468b5e494b1bbb6674ae Mon Sep 17 00:00:00 2001
|
|
||||||
From: jiangpengfei <jiangpengfei9@huawei.com>
|
|
||||||
Date: Sat, 25 Jul 2020 11:25:34 +0800
|
|
||||||
Subject: [PATCH 04/50] kata-runtime: keep the process name of qemu same as
|
|
||||||
configured path
|
|
||||||
|
|
||||||
reason: inorder to make testcase scripts can use the same hypervisor
|
|
||||||
name no matter what version hypervisor use, keep the process name of
|
|
||||||
hypervisor same as configured path in the configuration.toml file
|
|
||||||
instead of the resolved path of symbol link.
|
|
||||||
|
|
||||||
Signed-off-by: jiangpengfei <jiangpengfei9@huawei.com>
|
|
||||||
---
|
|
||||||
pkg/katautils/config.go | 8 +++++++-
|
|
||||||
pkg/katautils/config_test.go | 4 ++--
|
|
||||||
2 files changed, 9 insertions(+), 3 deletions(-)
|
|
||||||
|
|
||||||
diff --git a/pkg/katautils/config.go b/pkg/katautils/config.go
|
|
||||||
index 14794a24..349e667f 100644
|
|
||||||
--- a/pkg/katautils/config.go
|
|
||||||
+++ b/pkg/katautils/config.go
|
|
||||||
@@ -10,6 +10,7 @@ import (
|
|
||||||
"errors"
|
|
||||||
"fmt"
|
|
||||||
"io/ioutil"
|
|
||||||
+ "path/filepath"
|
|
||||||
goruntime "runtime"
|
|
||||||
"strings"
|
|
||||||
|
|
||||||
@@ -172,7 +173,12 @@ func (h hypervisor) path() (string, error) {
|
|
||||||
p = defaultHypervisorPath
|
|
||||||
}
|
|
||||||
|
|
||||||
- return ResolvePath(p)
|
|
||||||
+ absolutePath, err := filepath.Abs(p)
|
|
||||||
+ if err != nil {
|
|
||||||
+ return "", err
|
|
||||||
+ }
|
|
||||||
+
|
|
||||||
+ return absolutePath, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func (h hypervisor) ctlpath() (string, error) {
|
|
||||||
diff --git a/pkg/katautils/config_test.go b/pkg/katautils/config_test.go
|
|
||||||
index 221a4b55..2eae1f6a 100644
|
|
||||||
--- a/pkg/katautils/config_test.go
|
|
||||||
+++ b/pkg/katautils/config_test.go
|
|
||||||
@@ -1061,12 +1061,12 @@ func TestHypervisorDefaultsHypervisor(t *testing.T) {
|
|
||||||
assert.NoError(err)
|
|
||||||
assert.Equal(p, defaultHypervisorPath, "default hypervisor path wrong")
|
|
||||||
|
|
||||||
- // test path resolution
|
|
||||||
+ // test path resolution, just return the absolute path instead of resolved path
|
|
||||||
defaultHypervisorPath = testHypervisorLinkPath
|
|
||||||
h = hypervisor{}
|
|
||||||
p, err = h.path()
|
|
||||||
assert.NoError(err)
|
|
||||||
- assert.Equal(p, testHypervisorPath)
|
|
||||||
+ assert.Equal(p, testHypervisorLinkPath)
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestHypervisorDefaultsKernel(t *testing.T) {
|
|
||||||
--
|
|
||||||
2.14.3 (Apple Git-98)
|
|
||||||
|
|
||||||
@ -1,39 +0,0 @@
|
|||||||
From 3dc10421f177900c0ee94fc49b32ec66a46d9331 Mon Sep 17 00:00:00 2001
|
|
||||||
From: jiangpengfei <jiangpengfei9@huawei.com>
|
|
||||||
Date: Mon, 27 Jul 2020 19:18:50 +0800
|
|
||||||
Subject: [PATCH 06/50] kata-runtime: fix umount container rootfs dir return
|
|
||||||
ivalid argument error
|
|
||||||
|
|
||||||
reason: If sandbox hypervisor doesn't use block device driver for hotplugging container
|
|
||||||
rootfs block device into guest, kata-runtime will bind mount container rootfs dir to 9p
|
|
||||||
kataShared dir. However, container stop() function will always call bindUnmountContainerRootfs
|
|
||||||
function no matter block device driver is used or not. So we just need to call
|
|
||||||
bindUnmountContainerRootfs only if rootfs is bind mount to guest by 9p.
|
|
||||||
|
|
||||||
Signed-off-by: jiangpengfei <jiangpengfei9@huawei.com>
|
|
||||||
---
|
|
||||||
virtcontainers/container.go | 8 ++++++--
|
|
||||||
1 file changed, 6 insertions(+), 2 deletions(-)
|
|
||||||
|
|
||||||
diff --git a/virtcontainers/container.go b/virtcontainers/container.go
|
|
||||||
index 9e2d1e94..b42cc6e9 100644
|
|
||||||
--- a/virtcontainers/container.go
|
|
||||||
+++ b/virtcontainers/container.go
|
|
||||||
@@ -1120,8 +1120,12 @@ func (c *Container) stop(force bool) error {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
- if err := bindUnmountContainerRootfs(c.ctx, getMountPath(c.sandbox.id), c.id); err != nil && !force {
|
|
||||||
- return err
|
|
||||||
+ // umount container rootfs dir only if container use 9p
|
|
||||||
+ // to bind mount host container rootfs to 9p shared dir
|
|
||||||
+ if c.state.BlockDeviceID == "" {
|
|
||||||
+ if err := bindUnmountContainerRootfs(c.ctx, getMountPath(c.sandbox.id), c.id); err != nil && !force {
|
|
||||||
+ return err
|
|
||||||
+ }
|
|
||||||
}
|
|
||||||
|
|
||||||
if err := c.detachDevices(); err != nil && !force {
|
|
||||||
--
|
|
||||||
2.14.3 (Apple Git-98)
|
|
||||||
|
|
||||||
@ -1,855 +0,0 @@
|
|||||||
From d93da1875ed7f1a6061cffb13475506d73c86003 Mon Sep 17 00:00:00 2001
|
|
||||||
From: jiangpengfei <jiangpengfei9@huawei.com>
|
|
||||||
Date: Sat, 25 Jul 2020 16:04:19 +0800
|
|
||||||
Subject: [PATCH 07/50] kata-runtime: enhance reliability when kata related
|
|
||||||
process
|
|
||||||
|
|
||||||
reason: enhance the reliability when kata related processes is abnormal,
|
|
||||||
make kata-container still destroy the sandbox and clean up all resources.
|
|
||||||
|
|
||||||
Signed-off-by: jiangpengfei <jiangpengfei9@huawei.com>
|
|
||||||
---
|
|
||||||
cli/delete.go | 6 ++
|
|
||||||
cli/kill.go | 3 +-
|
|
||||||
virtcontainers/acrn.go | 2 +-
|
|
||||||
virtcontainers/agent.go | 3 +
|
|
||||||
virtcontainers/api.go | 67 +++++++++++++++++++++
|
|
||||||
virtcontainers/clh.go | 2 +-
|
|
||||||
virtcontainers/container.go | 55 +++++++++++++++--
|
|
||||||
virtcontainers/fc.go | 2 +-
|
|
||||||
virtcontainers/hypervisor.go | 2 +-
|
|
||||||
virtcontainers/kata_agent.go | 31 ++++++----
|
|
||||||
virtcontainers/mock_hypervisor.go | 2 +-
|
|
||||||
virtcontainers/mock_hypervisor_test.go | 2 +-
|
|
||||||
virtcontainers/noop_agent.go | 4 ++
|
|
||||||
virtcontainers/pkg/oci/utils.go | 5 ++
|
|
||||||
virtcontainers/qemu.go | 51 +++++++++-------
|
|
||||||
virtcontainers/sandbox.go | 106 +++++++++++++++++++++++++++++----
|
|
||||||
virtcontainers/types/sandbox.go | 14 ++++-
|
|
||||||
virtcontainers/utils/utils.go | 46 ++++++++++++++
|
|
||||||
virtcontainers/vm.go | 4 +-
|
|
||||||
19 files changed, 348 insertions(+), 59 deletions(-)
|
|
||||||
|
|
||||||
diff --git a/cli/delete.go b/cli/delete.go
|
|
||||||
index c2ce52a4..2f5586e5 100644
|
|
||||||
--- a/cli/delete.go
|
|
||||||
+++ b/cli/delete.go
|
|
||||||
@@ -110,6 +110,12 @@ func delete(ctx context.Context, containerID string, force bool) error {
|
|
||||||
forceStop = true
|
|
||||||
}
|
|
||||||
|
|
||||||
+ if oci.StateToOCIState(status.State.State) == oci.StateUnhealthy {
|
|
||||||
+ // Set forceStop and force bool flag to true to force delete everything
|
|
||||||
+ forceStop = true
|
|
||||||
+ force = true
|
|
||||||
+ }
|
|
||||||
+
|
|
||||||
switch containerType {
|
|
||||||
case vc.PodSandbox:
|
|
||||||
if err := deleteSandbox(ctx, sandboxID, force); err != nil {
|
|
||||||
diff --git a/cli/kill.go b/cli/kill.go
|
|
||||||
index 60fa41e0..b228205f 100644
|
|
||||||
--- a/cli/kill.go
|
|
||||||
+++ b/cli/kill.go
|
|
||||||
@@ -133,11 +133,12 @@ func kill(ctx context.Context, containerID, signal string, all bool) error {
|
|
||||||
kataLog.WithField("signal", signal).WithField("container state", status.State.State).Info("kill")
|
|
||||||
|
|
||||||
// container MUST be created, running or paused
|
|
||||||
+ // If container state is unhealthy, should process this exceptional case separately
|
|
||||||
if status.State.State == types.StateReady || status.State.State == types.StateRunning || status.State.State == types.StatePaused {
|
|
||||||
if err := vci.KillContainer(ctx, sandboxID, containerID, signum, all); err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
- } else if !all {
|
|
||||||
+ } else if !all && status.State.State != types.StateUnhealthy {
|
|
||||||
return fmt.Errorf("container not running")
|
|
||||||
}
|
|
||||||
|
|
||||||
diff --git a/virtcontainers/acrn.go b/virtcontainers/acrn.go
|
|
||||||
index 761eda03..10cae06f 100644
|
|
||||||
--- a/virtcontainers/acrn.go
|
|
||||||
+++ b/virtcontainers/acrn.go
|
|
||||||
@@ -475,7 +475,7 @@ func (a *Acrn) waitSandbox(timeoutSecs int) error {
|
|
||||||
}
|
|
||||||
|
|
||||||
// stopSandbox will stop the Sandbox's VM.
|
|
||||||
-func (a *Acrn) stopSandbox() (err error) {
|
|
||||||
+func (a *Acrn) stopSandbox(force bool) (err error) {
|
|
||||||
span, _ := a.trace("stopSandbox")
|
|
||||||
defer span.Finish()
|
|
||||||
|
|
||||||
diff --git a/virtcontainers/agent.go b/virtcontainers/agent.go
|
|
||||||
index c62107ec..be9526c7 100644
|
|
||||||
--- a/virtcontainers/agent.go
|
|
||||||
+++ b/virtcontainers/agent.go
|
|
||||||
@@ -259,4 +259,7 @@ type agent interface {
|
|
||||||
|
|
||||||
// load data from disk
|
|
||||||
load(persistapi.AgentState)
|
|
||||||
+
|
|
||||||
+ // get proxy process pid
|
|
||||||
+ getProxyPid() int
|
|
||||||
}
|
|
||||||
diff --git a/virtcontainers/api.go b/virtcontainers/api.go
|
|
||||||
index de569713..fa82d163 100644
|
|
||||||
--- a/virtcontainers/api.go
|
|
||||||
+++ b/virtcontainers/api.go
|
|
||||||
@@ -7,8 +7,10 @@ package virtcontainers
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
+ "fmt"
|
|
||||||
"os"
|
|
||||||
"runtime"
|
|
||||||
+ "strings"
|
|
||||||
"syscall"
|
|
||||||
|
|
||||||
deviceApi "github.com/kata-containers/runtime/virtcontainers/device/api"
|
|
||||||
@@ -18,6 +20,7 @@ import (
|
|
||||||
vcTypes "github.com/kata-containers/runtime/virtcontainers/pkg/types"
|
|
||||||
"github.com/kata-containers/runtime/virtcontainers/store"
|
|
||||||
"github.com/kata-containers/runtime/virtcontainers/types"
|
|
||||||
+ "github.com/kata-containers/runtime/virtcontainers/utils"
|
|
||||||
specs "github.com/opencontainers/runtime-spec/specs-go"
|
|
||||||
opentracing "github.com/opentracing/opentracing-go"
|
|
||||||
"github.com/sirupsen/logrus"
|
|
||||||
@@ -597,20 +600,51 @@ func statusContainer(sandbox *Sandbox, containerID string) (ContainerStatus, err
|
|
||||||
container.state.State == types.StatePaused) &&
|
|
||||||
container.process.Pid > 0 {
|
|
||||||
|
|
||||||
+ // If container state is active, however kata-proxy and qemu process all exit already
|
|
||||||
+ // which means sandbox has beed stopped exceptionally, then we should force delete
|
|
||||||
+ // sandbox and container state files in sandbox.Store
|
|
||||||
+ if sandbox.shouldForceDelete() {
|
|
||||||
+ virtLog.Logger.Warn("sandbox status is abnormal, sandbox should be force deleted")
|
|
||||||
+ sandbox.forceDeleteSandbox()
|
|
||||||
+ return ContainerStatus{}, fmt.Errorf("sandbox has beed stopped exceptionally")
|
|
||||||
+ }
|
|
||||||
+
|
|
||||||
running, err := isShimRunning(container.process.Pid)
|
|
||||||
if err != nil {
|
|
||||||
return ContainerStatus{}, err
|
|
||||||
}
|
|
||||||
|
|
||||||
+ // If kata-shim process exit or be killed, need to stop the container
|
|
||||||
if !running {
|
|
||||||
virtLog.WithFields(logrus.Fields{
|
|
||||||
"state": container.state.State,
|
|
||||||
"pid": container.process.Pid}).
|
|
||||||
Info("container isn't running")
|
|
||||||
+
|
|
||||||
if err := container.stop(true); err != nil {
|
|
||||||
return ContainerStatus{}, err
|
|
||||||
}
|
|
||||||
}
|
|
||||||
+
|
|
||||||
+ isPodSandbox := (containerID == sandbox.id)
|
|
||||||
+
|
|
||||||
+ // If sandbox is unhealthy, process it correctly
|
|
||||||
+ if !sandbox.health() {
|
|
||||||
+ // process podSandbox container type case
|
|
||||||
+ if isPodSandbox {
|
|
||||||
+ if err := processUnhealthySandbox(sandbox, container); err != nil {
|
|
||||||
+ return ContainerStatus{}, err
|
|
||||||
+ }
|
|
||||||
+ } else {
|
|
||||||
+ // If container type is pod_container, which means container operations can not be
|
|
||||||
+ // processed successfully, we should return the error as soon as possible
|
|
||||||
+ if err := container.setContainerState(types.StateUnhealthy); err != nil {
|
|
||||||
+ return ContainerStatus{}, err
|
|
||||||
+ }
|
|
||||||
+
|
|
||||||
+ return ContainerStatus{}, fmt.Errorf("container status is unhealthy, stop container failed")
|
|
||||||
+ }
|
|
||||||
+ }
|
|
||||||
}
|
|
||||||
|
|
||||||
return ContainerStatus{
|
|
||||||
@@ -1016,3 +1050,36 @@ func CleanupContainer(ctx context.Context, sandboxID, containerID string, force
|
|
||||||
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
+
|
|
||||||
+// procesUnhealthySandbox only change sandbox state to unhealthy
|
|
||||||
+// when caller is kata-runtime kill or kata-runtime delete
|
|
||||||
+func processUnhealthySandbox(sandbox *Sandbox, container *Container) error {
|
|
||||||
+ // Set all containers state to unhealthy
|
|
||||||
+ if err := sandbox.setContainersState(types.StateUnhealthy); err != nil {
|
|
||||||
+ container.Logger().WithError(err).Warn("set all containers state to unhealthy fail")
|
|
||||||
+ }
|
|
||||||
+
|
|
||||||
+ // Set sandbox state to unhealthy
|
|
||||||
+ if err := sandbox.setSandboxState(types.StateUnhealthy); err != nil {
|
|
||||||
+ container.Logger().WithError(err).Warn("set sandbox state to unhealthy fail")
|
|
||||||
+ }
|
|
||||||
+
|
|
||||||
+ forceDelete := false
|
|
||||||
+
|
|
||||||
+ // If process is kata-runtime kill or kata-runtime delete,
|
|
||||||
+ // we should kill or delete sandbox forcefully
|
|
||||||
+ if cmdline, err := utils.GetProcessCmdline(os.Getpid()); err != nil {
|
|
||||||
+ container.Logger().WithError(err).Warn("fail to get process cmdline info")
|
|
||||||
+ } else {
|
|
||||||
+ forceDelete = strings.Contains(cmdline, "kill") || strings.Contains(cmdline, "delete")
|
|
||||||
+ }
|
|
||||||
+
|
|
||||||
+ if forceDelete {
|
|
||||||
+ // force stop podSandbox type container's kata-shim process
|
|
||||||
+ if err := stopShim(container.process.Pid); err != nil {
|
|
||||||
+ container.Logger().WithError(err).Warn("fail to stop podSandbox type container kata-shim")
|
|
||||||
+ }
|
|
||||||
+ }
|
|
||||||
+
|
|
||||||
+ return nil
|
|
||||||
+}
|
|
||||||
diff --git a/virtcontainers/clh.go b/virtcontainers/clh.go
|
|
||||||
index d40b698b..59510b02 100644
|
|
||||||
--- a/virtcontainers/clh.go
|
|
||||||
+++ b/virtcontainers/clh.go
|
|
||||||
@@ -569,7 +569,7 @@ func (clh *cloudHypervisor) resumeSandbox() error {
|
|
||||||
}
|
|
||||||
|
|
||||||
// stopSandbox will stop the Sandbox's VM.
|
|
||||||
-func (clh *cloudHypervisor) stopSandbox() (err error) {
|
|
||||||
+func (clh *cloudHypervisor) stopSandbox(force bool) (err error) {
|
|
||||||
span, _ := clh.trace("stopSandbox")
|
|
||||||
defer span.Finish()
|
|
||||||
clh.Logger().WithField("function", "stopSandbox").Info("Stop Sandbox")
|
|
||||||
diff --git a/virtcontainers/container.go b/virtcontainers/container.go
|
|
||||||
index b42cc6e9..9485e708 100644
|
|
||||||
--- a/virtcontainers/container.go
|
|
||||||
+++ b/virtcontainers/container.go
|
|
||||||
@@ -17,8 +17,12 @@ import (
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"github.com/containerd/cgroups"
|
|
||||||
+ "github.com/kata-containers/runtime/virtcontainers/device/config"
|
|
||||||
+ "github.com/kata-containers/runtime/virtcontainers/device/manager"
|
|
||||||
vccgroups "github.com/kata-containers/runtime/virtcontainers/pkg/cgroups"
|
|
||||||
+ "github.com/kata-containers/runtime/virtcontainers/pkg/rootless"
|
|
||||||
vcTypes "github.com/kata-containers/runtime/virtcontainers/pkg/types"
|
|
||||||
+ "github.com/kata-containers/runtime/virtcontainers/store"
|
|
||||||
"github.com/kata-containers/runtime/virtcontainers/types"
|
|
||||||
"github.com/kata-containers/runtime/virtcontainers/utils"
|
|
||||||
specs "github.com/opencontainers/runtime-spec/specs-go"
|
|
||||||
@@ -26,11 +30,6 @@ import (
|
|
||||||
"github.com/pkg/errors"
|
|
||||||
"github.com/sirupsen/logrus"
|
|
||||||
"golang.org/x/sys/unix"
|
|
||||||
-
|
|
||||||
- "github.com/kata-containers/runtime/virtcontainers/device/config"
|
|
||||||
- "github.com/kata-containers/runtime/virtcontainers/device/manager"
|
|
||||||
- "github.com/kata-containers/runtime/virtcontainers/pkg/rootless"
|
|
||||||
- "github.com/kata-containers/runtime/virtcontainers/store"
|
|
||||||
)
|
|
||||||
|
|
||||||
// https://github.com/torvalds/linux/blob/master/include/uapi/linux/major.h
|
|
||||||
@@ -1047,6 +1046,13 @@ func (c *Container) stop(force bool) error {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
+ // If container state is unhealthy, just force kill the container
|
|
||||||
+ if c.state.State == types.StateUnhealthy {
|
|
||||||
+ c.forceKillContainer()
|
|
||||||
+ // after force kill container, then change container state to stopped
|
|
||||||
+ return c.setContainerState(types.StateStopped)
|
|
||||||
+ }
|
|
||||||
+
|
|
||||||
if err := c.state.ValidTransition(c.state.State, types.StateStopped); err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
@@ -1063,6 +1069,8 @@ func (c *Container) stop(force bool) error {
|
|
||||||
if err := stopShim(c.process.Pid); err != nil {
|
|
||||||
l.WithError(err).Warn("failed to stop shim")
|
|
||||||
}
|
|
||||||
+
|
|
||||||
+ c.forceKillContainer()
|
|
||||||
}
|
|
||||||
|
|
||||||
}()
|
|
||||||
@@ -1096,7 +1104,9 @@ func (c *Container) stop(force bool) error {
|
|
||||||
// this signal will ensure the container will get killed to match
|
|
||||||
// the state of the shim. This will allow the following call to
|
|
||||||
// stopContainer() to succeed in such particular case.
|
|
||||||
- c.kill(syscall.SIGKILL, true)
|
|
||||||
+ if err := c.kill(syscall.SIGKILL, true); err != nil {
|
|
||||||
+ c.Logger().Errorf("send signal to container failed: %v", err)
|
|
||||||
+ }
|
|
||||||
|
|
||||||
// Since the agent has supported the MultiWaitProcess, it's better to
|
|
||||||
// wait the process here to make sure the process has exited before to
|
|
||||||
@@ -1582,3 +1592,36 @@ func (c *Container) cgroupsUpdate(resources specs.LinuxResources) error {
|
|
||||||
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
+
|
|
||||||
+// forceDeleteContainer force clean container mount info and resources stored in the disk
|
|
||||||
+func (c *Container) forceDeleteContainer() {
|
|
||||||
+ if err := c.unmountHostMounts(); err != nil {
|
|
||||||
+ c.Logger().WithError(err).Warn("container force delete umount host mounts fail")
|
|
||||||
+ }
|
|
||||||
+
|
|
||||||
+ if err := c.sandbox.removeContainer(c.id); err != nil {
|
|
||||||
+ c.Logger().WithError(err).Warn("sandbox removeContainer fail")
|
|
||||||
+ }
|
|
||||||
+
|
|
||||||
+ if err := c.store.Delete(); err != nil {
|
|
||||||
+ c.Logger().WithError(err).Warn("force delete container store fail")
|
|
||||||
+ }
|
|
||||||
+}
|
|
||||||
+
|
|
||||||
+func (c *Container) forceKillContainer() {
|
|
||||||
+ if err := c.setContainerState(types.StateStopped); err != nil {
|
|
||||||
+ c.Logger().WithError(err).Warn("force kill container: change container state to StateStopped failed")
|
|
||||||
+ }
|
|
||||||
+
|
|
||||||
+ if err := c.unmountHostMounts(); err != nil {
|
|
||||||
+ c.Logger().WithError(err).Warn("force kill container: umount container host mounts failed")
|
|
||||||
+ }
|
|
||||||
+
|
|
||||||
+ if err := c.detachDevices(); err != nil {
|
|
||||||
+ c.Logger().WithError(err).Warn("force kill container: detach container devices failed")
|
|
||||||
+ }
|
|
||||||
+
|
|
||||||
+ if err := c.removeDrive(); err != nil {
|
|
||||||
+ c.Logger().WithError(err).Warn("force kill container: remove container drive failed")
|
|
||||||
+ }
|
|
||||||
+}
|
|
||||||
diff --git a/virtcontainers/fc.go b/virtcontainers/fc.go
|
|
||||||
index 97ef5ffc..72a8e192 100644
|
|
||||||
--- a/virtcontainers/fc.go
|
|
||||||
+++ b/virtcontainers/fc.go
|
|
||||||
@@ -864,7 +864,7 @@ func (fc *firecracker) cleanupJail() {
|
|
||||||
}
|
|
||||||
|
|
||||||
// stopSandbox will stop the Sandbox's VM.
|
|
||||||
-func (fc *firecracker) stopSandbox() (err error) {
|
|
||||||
+func (fc *firecracker) stopSandbox(force bool) (err error) {
|
|
||||||
span, _ := fc.trace("stopSandbox")
|
|
||||||
defer span.Finish()
|
|
||||||
|
|
||||||
diff --git a/virtcontainers/hypervisor.go b/virtcontainers/hypervisor.go
|
|
||||||
index 4b3dd3d0..fd7d1f8e 100644
|
|
||||||
--- a/virtcontainers/hypervisor.go
|
|
||||||
+++ b/virtcontainers/hypervisor.go
|
|
||||||
@@ -766,7 +766,7 @@ func generateVMSocket(id string, useVsock bool, vmStogarePath string) (interface
|
|
||||||
type hypervisor interface {
|
|
||||||
createSandbox(ctx context.Context, id string, networkNS NetworkNamespace, hypervisorConfig *HypervisorConfig, stateful bool) error
|
|
||||||
startSandbox(timeout int) error
|
|
||||||
- stopSandbox() error
|
|
||||||
+ stopSandbox(force bool) error
|
|
||||||
pauseSandbox() error
|
|
||||||
saveSandbox() error
|
|
||||||
resumeSandbox() error
|
|
||||||
diff --git a/virtcontainers/kata_agent.go b/virtcontainers/kata_agent.go
|
|
||||||
index be5e96aa..7575d326 100644
|
|
||||||
--- a/virtcontainers/kata_agent.go
|
|
||||||
+++ b/virtcontainers/kata_agent.go
|
|
||||||
@@ -57,8 +57,9 @@ const (
|
|
||||||
)
|
|
||||||
|
|
||||||
var (
|
|
||||||
- checkRequestTimeout = 30 * time.Second
|
|
||||||
- defaultRequestTimeout = 60 * time.Second
|
|
||||||
+ checkRequestTimeout = 10 * time.Second
|
|
||||||
+ defaultRequestTimeout = 10 * time.Second
|
|
||||||
+ createContainerTimeout = 120 * time.Second
|
|
||||||
errorMissingProxy = errors.New("Missing proxy pointer")
|
|
||||||
errorMissingOCISpec = errors.New("Missing OCI specification")
|
|
||||||
defaultKataHostSharedDir = "/run/kata-containers/shared/sandboxes/"
|
|
||||||
@@ -987,17 +988,21 @@ func (k *kataAgent) stopSandbox(sandbox *Sandbox) error {
|
|
||||||
k.state.URL = ""
|
|
||||||
}()
|
|
||||||
|
|
||||||
- req := &grpc.DestroySandboxRequest{}
|
|
||||||
+ // If sandbox.state.State is unhealthy, we don't need to send DestroySandboxRequest
|
|
||||||
+ // to kata-agent, just force stop the sandbox
|
|
||||||
+ if sandbox.state.State != types.StateUnhealthy {
|
|
||||||
+ req := &grpc.DestroySandboxRequest{}
|
|
||||||
|
|
||||||
- if _, err := k.sendReq(req); err != nil {
|
|
||||||
- return err
|
|
||||||
- }
|
|
||||||
-
|
|
||||||
- if k.dynamicTracing {
|
|
||||||
- _, err := k.sendReq(&grpc.StopTracingRequest{})
|
|
||||||
- if err != nil {
|
|
||||||
+ if _, err := k.sendReq(req); err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
+
|
|
||||||
+ if k.dynamicTracing {
|
|
||||||
+ _, err := k.sendReq(&grpc.StopTracingRequest{})
|
|
||||||
+ if err != nil {
|
|
||||||
+ return err
|
|
||||||
+ }
|
|
||||||
+ }
|
|
||||||
}
|
|
||||||
|
|
||||||
return nil
|
|
||||||
@@ -2062,6 +2067,8 @@ func (k *kataAgent) getReqContext(reqName string) (ctx context.Context, cancel c
|
|
||||||
// Wait has no timeout
|
|
||||||
case grpcCheckRequest:
|
|
||||||
ctx, cancel = context.WithTimeout(ctx, checkRequestTimeout)
|
|
||||||
+ case grpcCreateContainerRequest:
|
|
||||||
+ ctx, cancel = context.WithTimeout(ctx, createContainerTimeout)
|
|
||||||
default:
|
|
||||||
ctx, cancel = context.WithTimeout(ctx, defaultRequestTimeout)
|
|
||||||
}
|
|
||||||
@@ -2382,3 +2389,7 @@ func (k *kataAgent) load(s persistapi.AgentState) {
|
|
||||||
k.state.ProxyPid = s.ProxyPid
|
|
||||||
k.state.URL = s.URL
|
|
||||||
}
|
|
||||||
+
|
|
||||||
+func (k *kataAgent) getProxyPid() int {
|
|
||||||
+ return k.state.ProxyPid
|
|
||||||
+}
|
|
||||||
diff --git a/virtcontainers/mock_hypervisor.go b/virtcontainers/mock_hypervisor.go
|
|
||||||
index 0c84e43c..a5b67491 100644
|
|
||||||
--- a/virtcontainers/mock_hypervisor.go
|
|
||||||
+++ b/virtcontainers/mock_hypervisor.go
|
|
||||||
@@ -39,7 +39,7 @@ func (m *mockHypervisor) startSandbox(timeout int) error {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
-func (m *mockHypervisor) stopSandbox() error {
|
|
||||||
+func (m *mockHypervisor) stopSandbox(force bool) error {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
diff --git a/virtcontainers/mock_hypervisor_test.go b/virtcontainers/mock_hypervisor_test.go
|
|
||||||
index b73b28f2..827e3192 100644
|
|
||||||
--- a/virtcontainers/mock_hypervisor_test.go
|
|
||||||
+++ b/virtcontainers/mock_hypervisor_test.go
|
|
||||||
@@ -53,7 +53,7 @@ func TestMockHypervisorStartSandbox(t *testing.T) {
|
|
||||||
func TestMockHypervisorStopSandbox(t *testing.T) {
|
|
||||||
var m *mockHypervisor
|
|
||||||
|
|
||||||
- assert.NoError(t, m.stopSandbox())
|
|
||||||
+ assert.NoError(t, m.stopSandbox(false))
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestMockHypervisorAddDevice(t *testing.T) {
|
|
||||||
diff --git a/virtcontainers/noop_agent.go b/virtcontainers/noop_agent.go
|
|
||||||
index 189f6b3f..8a7cd337 100644
|
|
||||||
--- a/virtcontainers/noop_agent.go
|
|
||||||
+++ b/virtcontainers/noop_agent.go
|
|
||||||
@@ -236,3 +236,7 @@ func (n *noopAgent) save() (s persistapi.AgentState) {
|
|
||||||
|
|
||||||
// load is the Noop agent state loader. It does nothing.
|
|
||||||
func (n *noopAgent) load(s persistapi.AgentState) {}
|
|
||||||
+
|
|
||||||
+func (n *noopAgent) getProxyPid() int {
|
|
||||||
+ return -1
|
|
||||||
+}
|
|
||||||
\ No newline at end of file
|
|
||||||
diff --git a/virtcontainers/pkg/oci/utils.go b/virtcontainers/pkg/oci/utils.go
|
|
||||||
index 5348c57d..cd8d48ce 100644
|
|
||||||
--- a/virtcontainers/pkg/oci/utils.go
|
|
||||||
+++ b/virtcontainers/pkg/oci/utils.go
|
|
||||||
@@ -70,6 +70,9 @@ const (
|
|
||||||
|
|
||||||
// StatePaused represents a container that has been paused.
|
|
||||||
StatePaused = "paused"
|
|
||||||
+
|
|
||||||
+ // StateUnhealthy represents a container that is unhealthy
|
|
||||||
+ StateUnhealthy = "unhealthy"
|
|
||||||
)
|
|
||||||
|
|
||||||
const KernelModulesSeparator = ";"
|
|
||||||
@@ -964,6 +967,8 @@ func StateToOCIState(state types.StateString) string {
|
|
||||||
return StateStopped
|
|
||||||
case types.StatePaused:
|
|
||||||
return StatePaused
|
|
||||||
+ case types.StateUnhealthy:
|
|
||||||
+ return StateUnhealthy
|
|
||||||
default:
|
|
||||||
return ""
|
|
||||||
}
|
|
||||||
diff --git a/virtcontainers/qemu.go b/virtcontainers/qemu.go
|
|
||||||
index ca286550..4b15d968 100644
|
|
||||||
--- a/virtcontainers/qemu.go
|
|
||||||
+++ b/virtcontainers/qemu.go
|
|
||||||
@@ -687,7 +687,7 @@ func (q *qemu) setupVirtiofsd() (err error) {
|
|
||||||
q.Logger().Info("virtiofsd quits")
|
|
||||||
// Wait to release resources of virtiofsd process
|
|
||||||
cmd.Process.Wait()
|
|
||||||
- q.stopSandbox()
|
|
||||||
+ q.stopSandbox(false)
|
|
||||||
}()
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
@@ -922,11 +922,11 @@ func (q *qemu) waitSandbox(timeout int) error {
|
|
||||||
}
|
|
||||||
|
|
||||||
// stopSandbox will stop the Sandbox's VM.
|
|
||||||
-func (q *qemu) stopSandbox() error {
|
|
||||||
+func (q *qemu) stopSandbox(force bool) error {
|
|
||||||
span, _ := q.trace("stopSandbox")
|
|
||||||
defer span.Finish()
|
|
||||||
|
|
||||||
- q.Logger().Info("Stopping Sandbox")
|
|
||||||
+ q.Logger().Infof("force stopping Sandbox: %v", force)
|
|
||||||
if q.stopped {
|
|
||||||
q.Logger().Info("Already stopped")
|
|
||||||
return nil
|
|
||||||
@@ -937,28 +937,37 @@ func (q *qemu) stopSandbox() error {
|
|
||||||
q.stopped = true
|
|
||||||
}()
|
|
||||||
|
|
||||||
- if q.config.Debug && q.qemuConfig.LogFile != "" {
|
|
||||||
- f, err := os.OpenFile(q.qemuConfig.LogFile, os.O_RDONLY, 0)
|
|
||||||
- if err == nil {
|
|
||||||
- scanner := bufio.NewScanner(f)
|
|
||||||
- for scanner.Scan() {
|
|
||||||
- q.Logger().Debug(scanner.Text())
|
|
||||||
- }
|
|
||||||
- if err := scanner.Err(); err != nil {
|
|
||||||
- q.Logger().WithError(err).Debug("read qemu log failed")
|
|
||||||
+ if !force {
|
|
||||||
+ if q.config.Debug && q.qemuConfig.LogFile != "" {
|
|
||||||
+ f, err := os.OpenFile(q.qemuConfig.LogFile, os.O_RDONLY, 0)
|
|
||||||
+ if err == nil {
|
|
||||||
+ scanner := bufio.NewScanner(f)
|
|
||||||
+ for scanner.Scan() {
|
|
||||||
+ q.Logger().Debug(scanner.Text())
|
|
||||||
+ }
|
|
||||||
+ if err := scanner.Err(); err != nil {
|
|
||||||
+ q.Logger().WithError(err).Debug("read qemu log failed")
|
|
||||||
+ }
|
|
||||||
}
|
|
||||||
}
|
|
||||||
- }
|
|
||||||
|
|
||||||
- err := q.qmpSetup()
|
|
||||||
- if err != nil {
|
|
||||||
- return err
|
|
||||||
- }
|
|
||||||
+ err := q.qmpSetup()
|
|
||||||
+ if err != nil {
|
|
||||||
+ return err
|
|
||||||
+ }
|
|
||||||
|
|
||||||
- err = q.qmpMonitorCh.qmp.ExecuteQuit(q.qmpMonitorCh.ctx)
|
|
||||||
- if err != nil {
|
|
||||||
- q.Logger().WithError(err).Error("Fail to execute qmp QUIT")
|
|
||||||
- return err
|
|
||||||
+ err = q.qmpMonitorCh.qmp.ExecuteQuit(q.qmpMonitorCh.ctx)
|
|
||||||
+ if err != nil {
|
|
||||||
+ q.Logger().WithError(err).Error("Fail to execute qmp QUIT")
|
|
||||||
+ return err
|
|
||||||
+ }
|
|
||||||
+ } else {
|
|
||||||
+ qemuMainPid := q.getPids()[0]
|
|
||||||
+ if qemuMainPid <= 1 {
|
|
||||||
+ return fmt.Errorf("force kill qemu process pid is invalid")
|
|
||||||
+ }
|
|
||||||
+
|
|
||||||
+ _ = syscall.Kill(qemuMainPid, syscall.SIGKILL)
|
|
||||||
}
|
|
||||||
|
|
||||||
return nil
|
|
||||||
diff --git a/virtcontainers/sandbox.go b/virtcontainers/sandbox.go
|
|
||||||
index edd1af5b..78188ed7 100644
|
|
||||||
--- a/virtcontainers/sandbox.go
|
|
||||||
+++ b/virtcontainers/sandbox.go
|
|
||||||
@@ -12,19 +12,13 @@ import (
|
|
||||||
"math"
|
|
||||||
"net"
|
|
||||||
"os"
|
|
||||||
+ "path/filepath"
|
|
||||||
"strings"
|
|
||||||
"sync"
|
|
||||||
"syscall"
|
|
||||||
|
|
||||||
"github.com/containerd/cgroups"
|
|
||||||
"github.com/containernetworking/plugins/pkg/ns"
|
|
||||||
- "github.com/opencontainers/runc/libcontainer/configs"
|
|
||||||
- specs "github.com/opencontainers/runtime-spec/specs-go"
|
|
||||||
- opentracing "github.com/opentracing/opentracing-go"
|
|
||||||
- "github.com/pkg/errors"
|
|
||||||
- "github.com/sirupsen/logrus"
|
|
||||||
- "github.com/vishvananda/netlink"
|
|
||||||
-
|
|
||||||
"github.com/kata-containers/agent/protocols/grpc"
|
|
||||||
"github.com/kata-containers/runtime/virtcontainers/device/api"
|
|
||||||
"github.com/kata-containers/runtime/virtcontainers/device/config"
|
|
||||||
@@ -41,6 +35,12 @@ import (
|
|
||||||
"github.com/kata-containers/runtime/virtcontainers/store"
|
|
||||||
"github.com/kata-containers/runtime/virtcontainers/types"
|
|
||||||
"github.com/kata-containers/runtime/virtcontainers/utils"
|
|
||||||
+ "github.com/opencontainers/runc/libcontainer/configs"
|
|
||||||
+ specs "github.com/opencontainers/runtime-spec/specs-go"
|
|
||||||
+ opentracing "github.com/opentracing/opentracing-go"
|
|
||||||
+ "github.com/pkg/errors"
|
|
||||||
+ "github.com/sirupsen/logrus"
|
|
||||||
+ "github.com/vishvananda/netlink"
|
|
||||||
)
|
|
||||||
|
|
||||||
const (
|
|
||||||
@@ -50,6 +50,9 @@ const (
|
|
||||||
|
|
||||||
// DirMode is the permission bits used for creating a directory
|
|
||||||
DirMode = os.FileMode(0750) | os.ModeDir
|
|
||||||
+
|
|
||||||
+ // kata-proxy proces name
|
|
||||||
+ KataProxyProcessName = "kata-proxy"
|
|
||||||
)
|
|
||||||
|
|
||||||
// SandboxStatus describes a sandbox status.
|
|
||||||
@@ -1037,7 +1040,7 @@ func (s *Sandbox) startVM() (err error) {
|
|
||||||
|
|
||||||
defer func() {
|
|
||||||
if err != nil {
|
|
||||||
- s.hypervisor.stopSandbox()
|
|
||||||
+ s.hypervisor.stopSandbox(false)
|
|
||||||
}
|
|
||||||
}()
|
|
||||||
|
|
||||||
@@ -1090,7 +1093,12 @@ func (s *Sandbox) stopVM() error {
|
|
||||||
}
|
|
||||||
|
|
||||||
s.Logger().Info("Stopping VM")
|
|
||||||
- return s.hypervisor.stopSandbox()
|
|
||||||
+ forceStop := false
|
|
||||||
+ if s.state.State == types.StateUnhealthy {
|
|
||||||
+ forceStop = true
|
|
||||||
+ }
|
|
||||||
+
|
|
||||||
+ return s.hypervisor.stopSandbox(forceStop)
|
|
||||||
}
|
|
||||||
|
|
||||||
func (s *Sandbox) addContainer(c *Container) error {
|
|
||||||
@@ -1591,13 +1599,15 @@ func (s *Sandbox) setSandboxState(state types.StateString) error {
|
|
||||||
return vcTypes.ErrNeedState
|
|
||||||
}
|
|
||||||
|
|
||||||
+ s.Logger().Debugf("Setting sandbox state from %v to %v", s.state.State, state)
|
|
||||||
// update in-memory state
|
|
||||||
s.state.State = state
|
|
||||||
|
|
||||||
if useOldStore(s.ctx) {
|
|
||||||
return s.store.Store(store.State, s.state)
|
|
||||||
+ } else {
|
|
||||||
+ return s.Save()
|
|
||||||
}
|
|
||||||
- return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
const maxBlockIndex = 65535
|
|
||||||
@@ -2207,3 +2217,79 @@ func (s *Sandbox) GetPatchedOCISpec() *specs.Spec {
|
|
||||||
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
+
|
|
||||||
+// health return current sandbox healthy or not
|
|
||||||
+// If qemu/kata-proxy/kata-agent process is abnormal,
|
|
||||||
+// s.agent.check() will return false
|
|
||||||
+func (s *Sandbox) health() bool {
|
|
||||||
+ err := s.agent.check()
|
|
||||||
+ if err != nil {
|
|
||||||
+ return false
|
|
||||||
+ }
|
|
||||||
+
|
|
||||||
+ return true
|
|
||||||
+}
|
|
||||||
+
|
|
||||||
+// shouldForceDelete force delete the sandbox when kata-proxy and hypervisor process exit
|
|
||||||
+// already and current process is kata-runtime kill or kata-runtime delete
|
|
||||||
+func (s *Sandbox) shouldForceDelete() bool {
|
|
||||||
+ cmdline, err := utils.GetProcessCmdline(os.Getpid())
|
|
||||||
+ if err != nil {
|
|
||||||
+ s.Logger().Errorf("fail to get process cmdline: %v", err)
|
|
||||||
+ return false
|
|
||||||
+ }
|
|
||||||
+
|
|
||||||
+ proxyPid := s.agent.getProxyPid()
|
|
||||||
+ hypervisorPids := s.hypervisor.getPids()
|
|
||||||
+ if len(hypervisorPids) <= 0 {
|
|
||||||
+ s.Logger().Warnf("get hypervisor main pid fail")
|
|
||||||
+ return false
|
|
||||||
+ }
|
|
||||||
+ hypervisorMainPid := hypervisorPids[0]
|
|
||||||
+ hypervisorPath := s.hypervisor.hypervisorConfig().HypervisorPath
|
|
||||||
+ hypervisorName := filepath.Base(hypervisorPath)
|
|
||||||
+
|
|
||||||
+ if !utils.IsProcessRunning(proxyPid, KataProxyProcessName, s.id) && !utils.IsProcessRunning(hypervisorMainPid, hypervisorName, s.id) &&
|
|
||||||
+ strings.Contains(cmdline, "delete") && strings.Contains(cmdline, "force") {
|
|
||||||
+ return true
|
|
||||||
+ }
|
|
||||||
+
|
|
||||||
+ return false
|
|
||||||
+}
|
|
||||||
+
|
|
||||||
+func (s *Sandbox) forceDeleteSandbox() {
|
|
||||||
+ for _, c := range s.containers {
|
|
||||||
+ // force delete all containers in the sandbox
|
|
||||||
+ c.forceDeleteContainer()
|
|
||||||
+ }
|
|
||||||
+
|
|
||||||
+ globalSandboxList.removeSandbox(s.id)
|
|
||||||
+
|
|
||||||
+ if s.monitor != nil {
|
|
||||||
+ s.monitor.stop()
|
|
||||||
+ }
|
|
||||||
+
|
|
||||||
+ if err := s.hypervisor.cleanup(); err != nil {
|
|
||||||
+ s.Logger().WithError(err).Error("failed to force cleanup hypervisor resource")
|
|
||||||
+ }
|
|
||||||
+
|
|
||||||
+ s.agent.cleanup(s)
|
|
||||||
+
|
|
||||||
+ if err := s.store.Delete(); err != nil {
|
|
||||||
+ s.Logger().WithError(err).Warn("sandbox force delete store failed")
|
|
||||||
+ }
|
|
||||||
+}
|
|
||||||
+
|
|
||||||
+func (s *Sandbox) setContainersState(state types.StateString) error {
|
|
||||||
+ if state == "" {
|
|
||||||
+ return vcTypes.ErrNeedState
|
|
||||||
+ }
|
|
||||||
+
|
|
||||||
+ for _, c := range s.containers {
|
|
||||||
+ if err := c.setContainerState(state); err != nil {
|
|
||||||
+ return err
|
|
||||||
+ }
|
|
||||||
+ }
|
|
||||||
+
|
|
||||||
+ return nil
|
|
||||||
+}
|
|
||||||
diff --git a/virtcontainers/types/sandbox.go b/virtcontainers/types/sandbox.go
|
|
||||||
index 3b64b20a..5d586b21 100644
|
|
||||||
--- a/virtcontainers/types/sandbox.go
|
|
||||||
+++ b/virtcontainers/types/sandbox.go
|
|
||||||
@@ -28,6 +28,9 @@ const (
|
|
||||||
|
|
||||||
// StateStopped represents a sandbox/container that has been stopped.
|
|
||||||
StateStopped StateString = "stopped"
|
|
||||||
+
|
|
||||||
+ // StateUnhealthy represents a sandbox/container that's in abnormal state.
|
|
||||||
+ StateUnhealthy StateString = "unhealthy"
|
|
||||||
)
|
|
||||||
|
|
||||||
const (
|
|
||||||
@@ -90,17 +93,17 @@ func (state *StateString) validTransition(oldState StateString, newState StateSt
|
|
||||||
|
|
||||||
switch *state {
|
|
||||||
case StateReady:
|
|
||||||
- if newState == StateRunning || newState == StateStopped {
|
|
||||||
+ if newState == StateRunning || newState == StateStopped || newState == StateUnhealthy {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
case StateRunning:
|
|
||||||
- if newState == StatePaused || newState == StateStopped {
|
|
||||||
+ if newState == StatePaused || newState == StateStopped || newState == StateUnhealthy {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
case StatePaused:
|
|
||||||
- if newState == StateRunning || newState == StateStopped {
|
|
||||||
+ if newState == StateRunning || newState == StateStopped || newState == StateUnhealthy {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
@@ -108,6 +111,11 @@ func (state *StateString) validTransition(oldState StateString, newState StateSt
|
|
||||||
if newState == StateRunning {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
+
|
|
||||||
+ case StateUnhealthy:
|
|
||||||
+ if newState == StateStopped {
|
|
||||||
+ return nil
|
|
||||||
+ }
|
|
||||||
}
|
|
||||||
|
|
||||||
return fmt.Errorf("Can not move from %v to %v",
|
|
||||||
diff --git a/virtcontainers/utils/utils.go b/virtcontainers/utils/utils.go
|
|
||||||
index 85c55489..2b555ebb 100644
|
|
||||||
--- a/virtcontainers/utils/utils.go
|
|
||||||
+++ b/virtcontainers/utils/utils.go
|
|
||||||
@@ -9,9 +9,13 @@ import (
|
|
||||||
"crypto/rand"
|
|
||||||
"errors"
|
|
||||||
"fmt"
|
|
||||||
+ "io/ioutil"
|
|
||||||
"os"
|
|
||||||
"os/exec"
|
|
||||||
"path/filepath"
|
|
||||||
+ "strconv"
|
|
||||||
+ "strings"
|
|
||||||
+ "syscall"
|
|
||||||
)
|
|
||||||
|
|
||||||
const cpBinaryName = "cp"
|
|
||||||
@@ -275,3 +279,45 @@ const (
|
|
||||||
MiB = KiB << 10
|
|
||||||
GiB = MiB << 10
|
|
||||||
)
|
|
||||||
+
|
|
||||||
+// Get process cmdline info by read /proc/<pid>/cmdline file
|
|
||||||
+func GetProcessCmdline(pid int) (cmdline string, err error) {
|
|
||||||
+ if pid <= 1 {
|
|
||||||
+ return "", fmt.Errorf("invalid pid number")
|
|
||||||
+ }
|
|
||||||
+
|
|
||||||
+ bytes, err := ioutil.ReadFile(filepath.Join("/proc", strconv.Itoa(pid), "cmdline"))
|
|
||||||
+ if err != nil {
|
|
||||||
+ return "", err
|
|
||||||
+ }
|
|
||||||
+
|
|
||||||
+ return string(bytes), nil
|
|
||||||
+}
|
|
||||||
+
|
|
||||||
+func IsProcessRunning(pid int, processName string, sandboxID string) bool {
|
|
||||||
+ if pid <= 0 {
|
|
||||||
+ return false
|
|
||||||
+ }
|
|
||||||
+
|
|
||||||
+ process, err := os.FindProcess(pid)
|
|
||||||
+ if err != nil {
|
|
||||||
+ return false
|
|
||||||
+ }
|
|
||||||
+
|
|
||||||
+ if err := process.Signal(syscall.Signal(0)); err != nil {
|
|
||||||
+ return false
|
|
||||||
+ }
|
|
||||||
+
|
|
||||||
+ cmdline, err := GetProcessCmdline(pid)
|
|
||||||
+ if err != nil {
|
|
||||||
+ return false
|
|
||||||
+ }
|
|
||||||
+
|
|
||||||
+ // If process's cmdline contains processName and sandboxID keyword,
|
|
||||||
+ // We think this process isn't be reused
|
|
||||||
+ if strings.Contains(cmdline, processName) && strings.Contains(cmdline, sandboxID) {
|
|
||||||
+ return true
|
|
||||||
+ }
|
|
||||||
+
|
|
||||||
+ return false
|
|
||||||
+}
|
|
||||||
diff --git a/virtcontainers/vm.go b/virtcontainers/vm.go
|
|
||||||
index fcda1e97..8d27b1fe 100644
|
|
||||||
--- a/virtcontainers/vm.go
|
|
||||||
+++ b/virtcontainers/vm.go
|
|
||||||
@@ -191,7 +191,7 @@ func NewVM(ctx context.Context, config VMConfig) (*VM, error) {
|
|
||||||
defer func() {
|
|
||||||
if err != nil {
|
|
||||||
virtLog.WithField("vm", id).WithError(err).Info("clean up vm")
|
|
||||||
- hypervisor.stopSandbox()
|
|
||||||
+ hypervisor.stopSandbox(false)
|
|
||||||
}
|
|
||||||
}()
|
|
||||||
|
|
||||||
@@ -333,7 +333,7 @@ func (v *VM) Disconnect() error {
|
|
||||||
func (v *VM) Stop() error {
|
|
||||||
v.logger().Info("stop vm")
|
|
||||||
|
|
||||||
- if err := v.hypervisor.stopSandbox(); err != nil {
|
|
||||||
+ if err := v.hypervisor.stopSandbox(false); err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
--
|
|
||||||
2.14.3 (Apple Git-98)
|
|
||||||
|
|
||||||
@ -1,177 +0,0 @@
|
|||||||
From a1bf2e1c696b703935f4b81ca087a60cc2559464 Mon Sep 17 00:00:00 2001
|
|
||||||
From: jiangpengfei <jiangpengfei9@huawei.com>
|
|
||||||
Date: Mon, 27 Jul 2020 21:45:25 +0800
|
|
||||||
Subject: [PATCH 08/50] kata-runtime: fix kata-runtime resource left problem
|
|
||||||
|
|
||||||
reason: fix the following resource left problem
|
|
||||||
- sandbox stored files and directory are deleted before work container delete
|
|
||||||
- ignore StatusSandbox error, let stopSandbox function call be executed
|
|
||||||
|
|
||||||
Signed-off-by: jiangpengfei <jiangpengfei9@huawei.com>
|
|
||||||
---
|
|
||||||
cli/delete.go | 25 +++++++++++++++++++++++--
|
|
||||||
cli/delete_test.go | 15 +++++++++++++++
|
|
||||||
cli/state.go | 6 ++++++
|
|
||||||
virtcontainers/api.go | 4 +++-
|
|
||||||
4 files changed, 47 insertions(+), 3 deletions(-)
|
|
||||||
|
|
||||||
diff --git a/cli/delete.go b/cli/delete.go
|
|
||||||
index 2f5586e5..871ac40d 100644
|
|
||||||
--- a/cli/delete.go
|
|
||||||
+++ b/cli/delete.go
|
|
||||||
@@ -10,6 +10,7 @@ import (
|
|
||||||
"context"
|
|
||||||
"fmt"
|
|
||||||
"os"
|
|
||||||
+ "strings"
|
|
||||||
|
|
||||||
"github.com/kata-containers/runtime/pkg/katautils"
|
|
||||||
vc "github.com/kata-containers/runtime/virtcontainers"
|
|
||||||
@@ -68,6 +69,12 @@ func delete(ctx context.Context, containerID string, force bool) error {
|
|
||||||
setExternalLoggers(ctx, kataLog)
|
|
||||||
span.SetTag("container", containerID)
|
|
||||||
|
|
||||||
+ // Remove the containerID to sandboxID mapping
|
|
||||||
+ // no matter what error return
|
|
||||||
+ defer func() {
|
|
||||||
+ _ = katautils.DelContainerIDMapping(ctx, containerID)
|
|
||||||
+ }()
|
|
||||||
+
|
|
||||||
// Checks the MUST and MUST NOT from OCI runtime specification
|
|
||||||
status, sandboxID, err := getExistingContainerInfo(ctx, containerID)
|
|
||||||
if err != nil {
|
|
||||||
@@ -75,6 +82,15 @@ func delete(ctx context.Context, containerID string, force bool) error {
|
|
||||||
kataLog.Warnf("Failed to get container, force will not fail: %s", err)
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
+
|
|
||||||
+ // If err info containers "no such file or directory, because pod_sandbox type
|
|
||||||
+ // container is deleted before pod_container type container, just return nil
|
|
||||||
+ // and let containerd delete container operations continue
|
|
||||||
+ if strings.Contains(err.Error(), "no such file or directory") {
|
|
||||||
+ kataLog.Warnf("pod_sandbox deleted before pod_container: %v", err)
|
|
||||||
+ return nil
|
|
||||||
+ }
|
|
||||||
+
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
@@ -123,7 +139,12 @@ func delete(ctx context.Context, containerID string, force bool) error {
|
|
||||||
}
|
|
||||||
case vc.PodContainer:
|
|
||||||
if err := deleteContainer(ctx, sandboxID, containerID, forceStop); err != nil {
|
|
||||||
- return err
|
|
||||||
+ // If err info containers "no such file or directory, because pod_sandbox type
|
|
||||||
+ // container is deleted before pod_container type container, just return nil
|
|
||||||
+ // and let containerd delete container operations continue
|
|
||||||
+ if !strings.Contains(err.Error(), "no such file or directory") {
|
|
||||||
+ return err
|
|
||||||
+ }
|
|
||||||
}
|
|
||||||
default:
|
|
||||||
return fmt.Errorf("Invalid container type found")
|
|
||||||
@@ -134,7 +155,7 @@ func delete(ctx context.Context, containerID string, force bool) error {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
- return katautils.DelContainerIDMapping(ctx, containerID)
|
|
||||||
+ return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func deleteSandbox(ctx context.Context, sandboxID string, force bool) error {
|
|
||||||
diff --git a/cli/delete_test.go b/cli/delete_test.go
|
|
||||||
index a2455dee..ae421cd7 100644
|
|
||||||
--- a/cli/delete_test.go
|
|
||||||
+++ b/cli/delete_test.go
|
|
||||||
@@ -184,6 +184,9 @@ func TestDeleteSandbox(t *testing.T) {
|
|
||||||
assert.Error(err)
|
|
||||||
assert.True(vcmock.IsMockError(err))
|
|
||||||
|
|
||||||
+ path, err = createTempContainerIDMapping(sandbox.ID(), sandbox.ID())
|
|
||||||
+ assert.NoError(err)
|
|
||||||
+
|
|
||||||
testingImpl.StatusSandboxFunc = func(ctx context.Context, sandboxID string) (vc.SandboxStatus, error) {
|
|
||||||
return vc.SandboxStatus{
|
|
||||||
ID: sandbox.ID(),
|
|
||||||
@@ -201,6 +204,9 @@ func TestDeleteSandbox(t *testing.T) {
|
|
||||||
assert.Error(err)
|
|
||||||
assert.True(vcmock.IsMockError(err))
|
|
||||||
|
|
||||||
+ path, err = createTempContainerIDMapping(sandbox.ID(), sandbox.ID())
|
|
||||||
+ assert.NoError(err)
|
|
||||||
+
|
|
||||||
testingImpl.StopSandboxFunc = func(ctx context.Context, sandboxID string, force bool) (vc.VCSandbox, error) {
|
|
||||||
return sandbox, nil
|
|
||||||
}
|
|
||||||
@@ -213,6 +219,9 @@ func TestDeleteSandbox(t *testing.T) {
|
|
||||||
assert.Error(err)
|
|
||||||
assert.True(vcmock.IsMockError(err))
|
|
||||||
|
|
||||||
+ path, err = createTempContainerIDMapping(sandbox.ID(), sandbox.ID())
|
|
||||||
+ assert.NoError(err)
|
|
||||||
+
|
|
||||||
testingImpl.DeleteSandboxFunc = func(ctx context.Context, sandboxID string) (vc.VCSandbox, error) {
|
|
||||||
return sandbox, nil
|
|
||||||
}
|
|
||||||
@@ -302,6 +311,9 @@ func TestDeleteSandboxRunning(t *testing.T) {
|
|
||||||
assert.Error(err)
|
|
||||||
assert.False(vcmock.IsMockError(err))
|
|
||||||
|
|
||||||
+ path, err = createTempContainerIDMapping(sandbox.ID(), sandbox.ID())
|
|
||||||
+ assert.NoError(err)
|
|
||||||
+
|
|
||||||
testingImpl.StatusSandboxFunc = func(ctx context.Context, sandboxID string) (vc.SandboxStatus, error) {
|
|
||||||
return vc.SandboxStatus{
|
|
||||||
ID: sandbox.ID(),
|
|
||||||
@@ -325,6 +337,9 @@ func TestDeleteSandboxRunning(t *testing.T) {
|
|
||||||
assert.Error(err)
|
|
||||||
assert.True(vcmock.IsMockError(err))
|
|
||||||
|
|
||||||
+ path, err = createTempContainerIDMapping(sandbox.ID(), sandbox.ID())
|
|
||||||
+ assert.NoError(err)
|
|
||||||
+
|
|
||||||
testingImpl.DeleteSandboxFunc = func(ctx context.Context, sandboxID string) (vc.VCSandbox, error) {
|
|
||||||
return sandbox, nil
|
|
||||||
}
|
|
||||||
diff --git a/cli/state.go b/cli/state.go
|
|
||||||
index a2fcc12e..de843d34 100644
|
|
||||||
--- a/cli/state.go
|
|
||||||
+++ b/cli/state.go
|
|
||||||
@@ -11,6 +11,7 @@ import (
|
|
||||||
"encoding/json"
|
|
||||||
"fmt"
|
|
||||||
"os"
|
|
||||||
+ "strings"
|
|
||||||
|
|
||||||
"github.com/kata-containers/runtime/pkg/katautils"
|
|
||||||
"github.com/kata-containers/runtime/virtcontainers/pkg/oci"
|
|
||||||
@@ -52,6 +53,11 @@ func state(ctx context.Context, containerID string) error {
|
|
||||||
// Checks the MUST and MUST NOT from OCI runtime specification
|
|
||||||
status, _, err := getExistingContainerInfo(ctx, containerID)
|
|
||||||
if err != nil {
|
|
||||||
+ // If err info containers "no such file or directory, because pod_sandbox type
|
|
||||||
+ // container is deleted before pod_container type container, just return nil
|
|
||||||
+ if strings.Contains(err.Error(), "no such file or directory") {
|
|
||||||
+ return nil
|
|
||||||
+ }
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
diff --git a/virtcontainers/api.go b/virtcontainers/api.go
|
|
||||||
index fa82d163..449a03e0 100644
|
|
||||||
--- a/virtcontainers/api.go
|
|
||||||
+++ b/virtcontainers/api.go
|
|
||||||
@@ -374,7 +374,9 @@ func StatusSandbox(ctx context.Context, sandboxID string) (SandboxStatus, error)
|
|
||||||
for _, container := range s.containers {
|
|
||||||
contStatus, err := statusContainer(s, container.id)
|
|
||||||
if err != nil {
|
|
||||||
- return SandboxStatus{}, err
|
|
||||||
+ // Since statusContainer may get error because of qemu process D or T,
|
|
||||||
+ // So just ignore this error and let StatusSandbox function return actually SandboxStatus
|
|
||||||
+ s.Logger().Warnf("Status container's status in sandbox get error: %v", contStatus)
|
|
||||||
}
|
|
||||||
|
|
||||||
contStatusList = append(contStatusList, contStatus)
|
|
||||||
--
|
|
||||||
2.14.3 (Apple Git-98)
|
|
||||||
|
|
||||||
@ -1,132 +0,0 @@
|
|||||||
From 508fd9b94b1b12be3167342b03a47f8f97245e9c Mon Sep 17 00:00:00 2001
|
|
||||||
From: jiangpengfei <jiangpengfei9@huawei.com>
|
|
||||||
Date: Tue, 28 Jul 2020 10:53:30 +0800
|
|
||||||
Subject: [PATCH 09/50] kata-runtime: add kata-runtime global flag --debug
|
|
||||||
|
|
||||||
reason: add the same debug flag with runc to adapt to
|
|
||||||
containerd 1.2.0
|
|
||||||
|
|
||||||
Signed-off-by: jiangpengfei <jiangpengfei9@huawei.com>
|
|
||||||
---
|
|
||||||
cli/kata-env_test.go | 2 +-
|
|
||||||
cli/main.go | 6 +++++-
|
|
||||||
containerd-shim-v2/create.go | 2 +-
|
|
||||||
pkg/katautils/config.go | 4 ++--
|
|
||||||
pkg/katautils/config_test.go | 8 ++++----
|
|
||||||
5 files changed, 13 insertions(+), 9 deletions(-)
|
|
||||||
|
|
||||||
diff --git a/cli/kata-env_test.go b/cli/kata-env_test.go
|
|
||||||
index b31b6cc2..75bb697f 100644
|
|
||||||
--- a/cli/kata-env_test.go
|
|
||||||
+++ b/cli/kata-env_test.go
|
|
||||||
@@ -178,7 +178,7 @@ func makeRuntimeConfig(prefixDir string) (configFile string, config oci.RuntimeC
|
|
||||||
return "", oci.RuntimeConfig{}, err
|
|
||||||
}
|
|
||||||
|
|
||||||
- _, config, err = katautils.LoadConfiguration(configFile, true, false)
|
|
||||||
+ _, config, err = katautils.LoadConfiguration(configFile, true, false, false)
|
|
||||||
if err != nil {
|
|
||||||
return "", oci.RuntimeConfig{}, err
|
|
||||||
}
|
|
||||||
diff --git a/cli/main.go b/cli/main.go
|
|
||||||
index df16c5f3..1362a8fb 100644
|
|
||||||
--- a/cli/main.go
|
|
||||||
+++ b/cli/main.go
|
|
||||||
@@ -115,6 +115,10 @@ var runtimeFlags = []cli.Flag{
|
|
||||||
Name: "systemd-cgroup",
|
|
||||||
Usage: "enable systemd cgroup support, expects cgroupsPath to be of form \"slice:prefix:name\" for e.g. \"system.slice:runc:434234\"",
|
|
||||||
},
|
|
||||||
+ cli.BoolFlag{
|
|
||||||
+ Name: "debug",
|
|
||||||
+ Usage: "enable debug output for logging",
|
|
||||||
+ },
|
|
||||||
}
|
|
||||||
|
|
||||||
// runtimeCommands is the list of supported command-line (sub-)
|
|
||||||
@@ -334,7 +338,7 @@ func beforeSubcommands(c *cli.Context) error {
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
- configFile, runtimeConfig, err = katautils.LoadConfiguration(c.GlobalString(configFilePathOption), ignoreConfigLogs, false)
|
|
||||||
+ configFile, runtimeConfig, err = katautils.LoadConfiguration(c.GlobalString(configFilePathOption), ignoreConfigLogs, false, c.GlobalBool("debug"))
|
|
||||||
if err != nil {
|
|
||||||
fatal(err)
|
|
||||||
}
|
|
||||||
diff --git a/containerd-shim-v2/create.go b/containerd-shim-v2/create.go
|
|
||||||
index affdbae2..9749073d 100644
|
|
||||||
--- a/containerd-shim-v2/create.go
|
|
||||||
+++ b/containerd-shim-v2/create.go
|
|
||||||
@@ -167,7 +167,7 @@ func loadRuntimeConfig(s *service, r *taskAPI.CreateTaskRequest, anno map[string
|
|
||||||
configPath = os.Getenv("KATA_CONF_FILE")
|
|
||||||
}
|
|
||||||
|
|
||||||
- _, runtimeConfig, err := katautils.LoadConfiguration(configPath, false, true)
|
|
||||||
+ _, runtimeConfig, err := katautils.LoadConfiguration(configPath, false, true, false)
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
diff --git a/pkg/katautils/config.go b/pkg/katautils/config.go
|
|
||||||
index 349e667f..448d23ac 100644
|
|
||||||
--- a/pkg/katautils/config.go
|
|
||||||
+++ b/pkg/katautils/config.go
|
|
||||||
@@ -1141,7 +1141,7 @@ func initConfig() (config oci.RuntimeConfig, err error) {
|
|
||||||
//
|
|
||||||
// All paths are resolved fully meaning if this function does not return an
|
|
||||||
// error, all paths are valid at the time of the call.
|
|
||||||
-func LoadConfiguration(configPath string, ignoreLogging, builtIn bool) (resolvedConfigPath string, config oci.RuntimeConfig, err error) {
|
|
||||||
+func LoadConfiguration(configPath string, ignoreLogging, builtIn bool, debugFlag bool) (resolvedConfigPath string, config oci.RuntimeConfig, err error) {
|
|
||||||
|
|
||||||
config, err = initConfig()
|
|
||||||
if err != nil {
|
|
||||||
@@ -1154,7 +1154,7 @@ func LoadConfiguration(configPath string, ignoreLogging, builtIn bool) (resolved
|
|
||||||
}
|
|
||||||
|
|
||||||
config.Debug = tomlConf.Runtime.Debug
|
|
||||||
- if !tomlConf.Runtime.Debug {
|
|
||||||
+ if !tomlConf.Runtime.Debug && !debugFlag {
|
|
||||||
// If debug is not required, switch back to the original
|
|
||||||
// default log priority, otherwise continue in debug mode.
|
|
||||||
kataUtilsLogger.Logger.Level = originalLoggerLevel
|
|
||||||
diff --git a/pkg/katautils/config_test.go b/pkg/katautils/config_test.go
|
|
||||||
index 2eae1f6a..31afcca6 100644
|
|
||||||
--- a/pkg/katautils/config_test.go
|
|
||||||
+++ b/pkg/katautils/config_test.go
|
|
||||||
@@ -264,7 +264,7 @@ func testLoadConfiguration(t *testing.T, dir string,
|
|
||||||
assert.NoError(t, err)
|
|
||||||
}
|
|
||||||
|
|
||||||
- resolvedConfigPath, config, err := LoadConfiguration(file, ignoreLogging, false)
|
|
||||||
+ resolvedConfigPath, config, err := LoadConfiguration(file, ignoreLogging, false, false)
|
|
||||||
if expectFail {
|
|
||||||
assert.Error(t, err)
|
|
||||||
|
|
||||||
@@ -578,7 +578,7 @@ func TestMinimalRuntimeConfig(t *testing.T) {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
|
|
||||||
- _, config, err := LoadConfiguration(configPath, false, false)
|
|
||||||
+ _, config, err := LoadConfiguration(configPath, false, false, false)
|
|
||||||
if err == nil {
|
|
||||||
t.Fatalf("Expected loadConfiguration to fail as shim path does not exist: %+v", config)
|
|
||||||
}
|
|
||||||
@@ -608,7 +608,7 @@ func TestMinimalRuntimeConfig(t *testing.T) {
|
|
||||||
t.Error(err)
|
|
||||||
}
|
|
||||||
|
|
||||||
- _, config, err = LoadConfiguration(configPath, false, false)
|
|
||||||
+ _, config, err = LoadConfiguration(configPath, false, false, false)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
@@ -748,7 +748,7 @@ func TestMinimalRuntimeConfigWithVsock(t *testing.T) {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
|
|
||||||
- _, config, err := LoadConfiguration(configPath, false, false)
|
|
||||||
+ _, config, err := LoadConfiguration(configPath, false, false, false)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
--
|
|
||||||
2.14.3 (Apple Git-98)
|
|
||||||
|
|
||||||
@ -1,172 +0,0 @@
|
|||||||
From 76cbca91608e94c1855705ad1a8d06ffa2273115 Mon Sep 17 00:00:00 2001
|
|
||||||
From: jiangpengfei <jiangpengfei9@huawei.com>
|
|
||||||
Date: Tue, 28 Jul 2020 18:18:54 +0800
|
|
||||||
Subject: [PATCH 10/50] kata-runtime: fix kata-shim pid reused problem
|
|
||||||
|
|
||||||
reason: If kata-shim process exit and it's pid reused by other process,
|
|
||||||
it may cause kill other proecss and cause some problem.
|
|
||||||
|
|
||||||
Signed-off-by: jiangpengfei <jiangpengfei9@huawei.com>
|
|
||||||
---
|
|
||||||
virtcontainers/api.go | 2 +-
|
|
||||||
virtcontainers/container.go | 6 +++---
|
|
||||||
virtcontainers/shim.go | 21 +++++++++++++++++----
|
|
||||||
virtcontainers/shim_test.go | 10 +++++-----
|
|
||||||
4 files changed, 26 insertions(+), 13 deletions(-)
|
|
||||||
|
|
||||||
diff --git a/virtcontainers/api.go b/virtcontainers/api.go
|
|
||||||
index 449a03e0..5e8c9c9e 100644
|
|
||||||
--- a/virtcontainers/api.go
|
|
||||||
+++ b/virtcontainers/api.go
|
|
||||||
@@ -611,7 +611,7 @@ func statusContainer(sandbox *Sandbox, containerID string) (ContainerStatus, err
|
|
||||||
return ContainerStatus{}, fmt.Errorf("sandbox has beed stopped exceptionally")
|
|
||||||
}
|
|
||||||
|
|
||||||
- running, err := isShimRunning(container.process.Pid)
|
|
||||||
+ running, err := isShimRunning(container.process.Pid, containerID)
|
|
||||||
if err != nil {
|
|
||||||
return ContainerStatus{}, err
|
|
||||||
}
|
|
||||||
diff --git a/virtcontainers/container.go b/virtcontainers/container.go
|
|
||||||
index 9485e708..75f590eb 100644
|
|
||||||
--- a/virtcontainers/container.go
|
|
||||||
+++ b/virtcontainers/container.go
|
|
||||||
@@ -1063,7 +1063,7 @@ func (c *Container) stop(force bool) error {
|
|
||||||
|
|
||||||
// If shim is still running something went wrong
|
|
||||||
// Make sure we stop the shim process
|
|
||||||
- if running, _ := isShimRunning(c.process.Pid); running {
|
|
||||||
+ if running, _ := isShimRunning(c.process.Pid, c.id); running {
|
|
||||||
l := c.Logger()
|
|
||||||
l.Error("Failed to stop container so stopping dangling shim")
|
|
||||||
if err := stopShim(c.process.Pid); err != nil {
|
|
||||||
@@ -1081,7 +1081,7 @@ func (c *Container) stop(force bool) error {
|
|
||||||
// However, if the signal didn't reach its goal, the caller still
|
|
||||||
// expects this container to be stopped, that's why we should not
|
|
||||||
// return an error, but instead try to kill it forcefully.
|
|
||||||
- if err := waitForShim(c.process.Pid); err != nil {
|
|
||||||
+ if err := waitForShim(c.process.Pid, c.id); err != nil {
|
|
||||||
// Force the container to be killed.
|
|
||||||
if err := c.kill(syscall.SIGKILL, true); err != nil && !force {
|
|
||||||
return err
|
|
||||||
@@ -1091,7 +1091,7 @@ func (c *Container) stop(force bool) error {
|
|
||||||
// to succeed. Indeed, we have already given a second chance
|
|
||||||
// to the container by trying to kill it with SIGKILL, there
|
|
||||||
// is no reason to try to go further if we got an error.
|
|
||||||
- if err := waitForShim(c.process.Pid); err != nil && !force {
|
|
||||||
+ if err := waitForShim(c.process.Pid, c.id); err != nil && !force {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
}
|
|
||||||
diff --git a/virtcontainers/shim.go b/virtcontainers/shim.go
|
|
||||||
index 8ec7458b..6f784a03 100644
|
|
||||||
--- a/virtcontainers/shim.go
|
|
||||||
+++ b/virtcontainers/shim.go
|
|
||||||
@@ -9,11 +9,13 @@ import (
|
|
||||||
"fmt"
|
|
||||||
"os"
|
|
||||||
"os/exec"
|
|
||||||
+ "strings"
|
|
||||||
"syscall"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
ns "github.com/kata-containers/runtime/virtcontainers/pkg/nsenter"
|
|
||||||
"github.com/kata-containers/runtime/virtcontainers/types"
|
|
||||||
+ "github.com/kata-containers/runtime/virtcontainers/utils"
|
|
||||||
"github.com/mitchellh/mapstructure"
|
|
||||||
"github.com/sirupsen/logrus"
|
|
||||||
)
|
|
||||||
@@ -227,7 +229,7 @@ func startShim(args []string, params ShimParams) (int, error) {
|
|
||||||
return cmd.Process.Pid, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
-func isShimRunning(pid int) (bool, error) {
|
|
||||||
+func isShimRunning(pid int, containerID string) (bool, error) {
|
|
||||||
if pid <= 0 {
|
|
||||||
return false, nil
|
|
||||||
}
|
|
||||||
@@ -241,19 +243,30 @@ func isShimRunning(pid int) (bool, error) {
|
|
||||||
return false, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
- return true, nil
|
|
||||||
+ cmdline, err := utils.GetProcessCmdline(pid)
|
|
||||||
+ if err != nil {
|
|
||||||
+ return false, nil
|
|
||||||
+ }
|
|
||||||
+
|
|
||||||
+ // If process's cmdline contains kata-shim and containerID keyword, we think this process pid isn't be reused
|
|
||||||
+ if strings.Contains(cmdline, "kata-shim") && strings.Contains(cmdline, containerID) {
|
|
||||||
+ return true, nil
|
|
||||||
+ }
|
|
||||||
+
|
|
||||||
+ shimLogger().Errorf("%d process isn't a kata-shim process", pid)
|
|
||||||
+ return false, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// waitForShim waits for the end of the shim unless it reaches the timeout
|
|
||||||
// first, returning an error in that case.
|
|
||||||
-func waitForShim(pid int) error {
|
|
||||||
+func waitForShim(pid int, containerID string) error {
|
|
||||||
if pid <= 0 {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
tInit := time.Now()
|
|
||||||
for {
|
|
||||||
- running, err := isShimRunning(pid)
|
|
||||||
+ running, err := isShimRunning(pid, containerID)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
diff --git a/virtcontainers/shim_test.go b/virtcontainers/shim_test.go
|
|
||||||
index e9bd027c..62471311 100644
|
|
||||||
--- a/virtcontainers/shim_test.go
|
|
||||||
+++ b/virtcontainers/shim_test.go
|
|
||||||
@@ -190,7 +190,7 @@ func TestStopShimSuccessfulProcessRunning(t *testing.T) {
|
|
||||||
|
|
||||||
func testIsShimRunning(t *testing.T, pid int, expected bool) {
|
|
||||||
assert := assert.New(t)
|
|
||||||
- running, err := isShimRunning(pid)
|
|
||||||
+ running, err := isShimRunning(pid, containerID)
|
|
||||||
assert.NoError(err)
|
|
||||||
assert.Equal(running, expected)
|
|
||||||
}
|
|
||||||
@@ -205,7 +205,7 @@ func TestIsShimRunningTrue(t *testing.T) {
|
|
||||||
cmd := testRunSleep999AndGetCmd(t)
|
|
||||||
assert := assert.New(t)
|
|
||||||
|
|
||||||
- testIsShimRunning(t, cmd.Process.Pid, true)
|
|
||||||
+ testIsShimRunning(t, cmd.Process.Pid, false)
|
|
||||||
|
|
||||||
err := syscall.Kill(cmd.Process.Pid, syscall.SIGKILL)
|
|
||||||
assert.NoError(err)
|
|
||||||
@@ -216,7 +216,7 @@ func TestWaitForShimInvalidPidSuccessful(t *testing.T) {
|
|
||||||
assert := assert.New(t)
|
|
||||||
|
|
||||||
for _, val := range wrongValuesList {
|
|
||||||
- err := waitForShim(val)
|
|
||||||
+ err := waitForShim(val, containerID)
|
|
||||||
assert.NoError(err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -224,7 +224,7 @@ func TestWaitForShimInvalidPidSuccessful(t *testing.T) {
|
|
||||||
func TestWaitForShimNotRunningSuccessful(t *testing.T) {
|
|
||||||
pid := testRunSleep0AndGetPid(t)
|
|
||||||
assert := assert.New(t)
|
|
||||||
- assert.NoError(waitForShim(pid))
|
|
||||||
+ assert.NoError(waitForShim(pid, containerID))
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestWaitForShimRunningForTooLongFailure(t *testing.T) {
|
|
||||||
@@ -232,6 +232,6 @@ func TestWaitForShimRunningForTooLongFailure(t *testing.T) {
|
|
||||||
assert := assert.New(t)
|
|
||||||
|
|
||||||
waitForShimTimeout = 0.1
|
|
||||||
- assert.Error(waitForShim(cmd.Process.Pid))
|
|
||||||
+ assert.NoError(waitForShim(cmd.Process.Pid, containerID))
|
|
||||||
assert.NoError(syscall.Kill(cmd.Process.Pid, syscall.SIGKILL))
|
|
||||||
}
|
|
||||||
--
|
|
||||||
2.14.3 (Apple Git-98)
|
|
||||||
|
|
||||||
@ -1,119 +0,0 @@
|
|||||||
From 0aeff2632eac58eefdc8ae438891303332831ec5 Mon Sep 17 00:00:00 2001
|
|
||||||
From: jiangpengfei <jiangpengfei9@huawei.com>
|
|
||||||
Date: Tue, 28 Jul 2020 20:48:24 +0800
|
|
||||||
Subject: [PATCH 11/50] kata-runtime: check the process info before send
|
|
||||||
SIGKILL
|
|
||||||
|
|
||||||
reason: In order to avoid the pid reuse problem, check the
|
|
||||||
process info before send SIGKILL signal to process.
|
|
||||||
|
|
||||||
Signed-off-by: jiangpengfei <jiangpengfei9@huawei.com>
|
|
||||||
---
|
|
||||||
virtcontainers/kata_proxy.go | 18 ++++++++++++++++++
|
|
||||||
virtcontainers/qemu.go | 5 +++++
|
|
||||||
virtcontainers/shim.go | 9 +++++++++
|
|
||||||
virtcontainers/shim_test.go | 8 ++++----
|
|
||||||
4 files changed, 36 insertions(+), 4 deletions(-)
|
|
||||||
|
|
||||||
diff --git a/virtcontainers/kata_proxy.go b/virtcontainers/kata_proxy.go
|
|
||||||
index e04b4cff..ed272bad 100644
|
|
||||||
--- a/virtcontainers/kata_proxy.go
|
|
||||||
+++ b/virtcontainers/kata_proxy.go
|
|
||||||
@@ -6,8 +6,12 @@
|
|
||||||
package virtcontainers
|
|
||||||
|
|
||||||
import (
|
|
||||||
+ "fmt"
|
|
||||||
"os/exec"
|
|
||||||
+ "strings"
|
|
||||||
"syscall"
|
|
||||||
+
|
|
||||||
+ "github.com/kata-containers/runtime/virtcontainers/utils"
|
|
||||||
)
|
|
||||||
|
|
||||||
// This is the Kata Containers implementation of the proxy interface.
|
|
||||||
@@ -61,6 +65,20 @@ func (p *kataProxy) start(params proxyParams) (int, string, error) {
|
|
||||||
|
|
||||||
// stop is kataProxy stop implementation for proxy interface.
|
|
||||||
func (p *kataProxy) stop(pid int) error {
|
|
||||||
+ if pid <= 1 {
|
|
||||||
+ return nil
|
|
||||||
+ }
|
|
||||||
+
|
|
||||||
+ // check process info before send SIGKILL signal
|
|
||||||
+ cmdline, err := utils.GetProcessCmdline(pid)
|
|
||||||
+ if err != nil {
|
|
||||||
+ return fmt.Errorf("get kata-proxy %d cmdline error: %v", pid, err)
|
|
||||||
+ }
|
|
||||||
+
|
|
||||||
+ if !strings.Contains(cmdline, KataProxyProcessName) {
|
|
||||||
+ return fmt.Errorf("%d is not kata-proxy process, don't kill wrong process", pid)
|
|
||||||
+ }
|
|
||||||
+
|
|
||||||
// Signal the proxy with SIGTERM.
|
|
||||||
return syscall.Kill(pid, syscall.SIGTERM)
|
|
||||||
}
|
|
||||||
diff --git a/virtcontainers/qemu.go b/virtcontainers/qemu.go
|
|
||||||
index 4b15d968..4789101d 100644
|
|
||||||
--- a/virtcontainers/qemu.go
|
|
||||||
+++ b/virtcontainers/qemu.go
|
|
||||||
@@ -967,6 +967,11 @@ func (q *qemu) stopSandbox(force bool) error {
|
|
||||||
return fmt.Errorf("force kill qemu process pid is invalid")
|
|
||||||
}
|
|
||||||
|
|
||||||
+ cmdline, _ := utils.GetProcessCmdline(qemuMainPid)
|
|
||||||
+ if !strings.Contains(cmdline, string(QemuHypervisor)) {
|
|
||||||
+ return fmt.Errorf("force kill %d process is not qemu process, don't kill wrong process", qemuMainPid)
|
|
||||||
+ }
|
|
||||||
+
|
|
||||||
_ = syscall.Kill(qemuMainPid, syscall.SIGKILL)
|
|
||||||
}
|
|
||||||
|
|
||||||
diff --git a/virtcontainers/shim.go b/virtcontainers/shim.go
|
|
||||||
index 6f784a03..b192b258 100644
|
|
||||||
--- a/virtcontainers/shim.go
|
|
||||||
+++ b/virtcontainers/shim.go
|
|
||||||
@@ -143,6 +143,15 @@ func stopShim(pid int) error {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
+ cmdline, err := utils.GetProcessCmdline(pid)
|
|
||||||
+ if err != nil {
|
|
||||||
+ return err
|
|
||||||
+ }
|
|
||||||
+
|
|
||||||
+ if !strings.Contains(cmdline, "kata-shim") {
|
|
||||||
+ return fmt.Errorf("%d process is not kata-shim process, don't kill wrong process", pid)
|
|
||||||
+ }
|
|
||||||
+
|
|
||||||
if err := signalShim(pid, syscall.SIGKILL); err != nil && err != syscall.ESRCH {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
diff --git a/virtcontainers/shim_test.go b/virtcontainers/shim_test.go
|
|
||||||
index 62471311..dc15eab0 100644
|
|
||||||
--- a/virtcontainers/shim_test.go
|
|
||||||
+++ b/virtcontainers/shim_test.go
|
|
||||||
@@ -176,16 +176,16 @@ func testRunSleep999AndGetCmd(t *testing.T) *exec.Cmd {
|
|
||||||
return cmd
|
|
||||||
}
|
|
||||||
|
|
||||||
-func TestStopShimSuccessfulProcessNotRunning(t *testing.T) {
|
|
||||||
+func TestStopShimFailProcessNotRunning(t *testing.T) {
|
|
||||||
assert := assert.New(t)
|
|
||||||
pid := testRunSleep0AndGetPid(t)
|
|
||||||
- assert.NoError(stopShim(pid))
|
|
||||||
+ assert.Error(stopShim(pid))
|
|
||||||
}
|
|
||||||
|
|
||||||
-func TestStopShimSuccessfulProcessRunning(t *testing.T) {
|
|
||||||
+func TestStopShimFailProcessRunning(t *testing.T) {
|
|
||||||
assert := assert.New(t)
|
|
||||||
cmd := testRunSleep999AndGetCmd(t)
|
|
||||||
- assert.NoError(stopShim(cmd.Process.Pid))
|
|
||||||
+ assert.Error(stopShim(cmd.Process.Pid))
|
|
||||||
}
|
|
||||||
|
|
||||||
func testIsShimRunning(t *testing.T, pid int, expected bool) {
|
|
||||||
--
|
|
||||||
2.14.3 (Apple Git-98)
|
|
||||||
|
|
||||||
@ -1,37 +0,0 @@
|
|||||||
From 441d80f55f4dc5efb4c92d91608a3c8db3d087cb Mon Sep 17 00:00:00 2001
|
|
||||||
From: jiangpengfei <jiangpengfei9@huawei.com>
|
|
||||||
Date: Tue, 28 Jul 2020 21:43:15 +0800
|
|
||||||
Subject: [PATCH 12/50] kata-runtime: truncate the log.json file before
|
|
||||||
kata-runtime subcommand executed
|
|
||||||
|
|
||||||
reason: since we have redirect the kata-runtime log to /var/log/messages, and avoid the
|
|
||||||
path of log.json file to be large in the tmpfs, so we truncate the log.json file every
|
|
||||||
time before subcommand is executed.
|
|
||||||
|
|
||||||
Signed-off-by: jiangpengfei <jiangpengfei9@huawei.com>
|
|
||||||
---
|
|
||||||
cli/main.go | 8 ++++++++
|
|
||||||
1 file changed, 8 insertions(+)
|
|
||||||
|
|
||||||
diff --git a/cli/main.go b/cli/main.go
|
|
||||||
index 1362a8fb..5eb2fb19 100644
|
|
||||||
--- a/cli/main.go
|
|
||||||
+++ b/cli/main.go
|
|
||||||
@@ -300,6 +300,14 @@ func beforeSubcommands(c *cli.Context) error {
|
|
||||||
ignoreConfigLogs = true
|
|
||||||
} else {
|
|
||||||
if path := c.GlobalString("log"); path != "" {
|
|
||||||
+ // since we have redirect the kata-runtime log to /var/log/messages, and avoid the
|
|
||||||
+ // path of log.json file to be large in the tmpfs, so we truncate the log.json file
|
|
||||||
+ // every time before subcommand is executed.
|
|
||||||
+ if path != "/dev/null" && katautils.FileExists(path) {
|
|
||||||
+ if err := os.Truncate(path, 0); err != nil {
|
|
||||||
+ return err
|
|
||||||
+ }
|
|
||||||
+ }
|
|
||||||
f, err := os.OpenFile(path, os.O_CREATE|os.O_WRONLY|os.O_APPEND|os.O_SYNC, 0640)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
--
|
|
||||||
2.14.3 (Apple Git-98)
|
|
||||||
|
|
||||||
@ -1,98 +0,0 @@
|
|||||||
From fd63d26a5b0542f35d61b0c19c80795f052b4518 Mon Sep 17 00:00:00 2001
|
|
||||||
From: jiangpengfei <jiangpengfei9@huawei.com>
|
|
||||||
Date: Tue, 28 Jul 2020 22:05:44 +0800
|
|
||||||
Subject: [PATCH 13/50] kata-runtime: get container info by containerID prefix
|
|
||||||
|
|
||||||
reason: get container info by containerID prefix, so we just
|
|
||||||
need to input the prefix containerID when call kata-runtime
|
|
||||||
subcommand
|
|
||||||
|
|
||||||
Signed-off-by: jiangpengfei <jiangpengfei9@huawei.com>
|
|
||||||
---
|
|
||||||
cli/oci.go | 35 +++++++++++++++++++++++++++++++++++
|
|
||||||
pkg/katautils/oci.go | 5 +++++
|
|
||||||
2 files changed, 40 insertions(+)
|
|
||||||
|
|
||||||
diff --git a/cli/oci.go b/cli/oci.go
|
|
||||||
index 8ffac2df..bf962d03 100644
|
|
||||||
--- a/cli/oci.go
|
|
||||||
+++ b/cli/oci.go
|
|
||||||
@@ -9,6 +9,7 @@ import (
|
|
||||||
"bufio"
|
|
||||||
"context"
|
|
||||||
"fmt"
|
|
||||||
+ "io/ioutil"
|
|
||||||
"net"
|
|
||||||
"os"
|
|
||||||
"path/filepath"
|
|
||||||
@@ -25,6 +26,8 @@ const (
|
|
||||||
// Filesystem type corresponding to CGROUP_SUPER_MAGIC as listed
|
|
||||||
// here: http://man7.org/linux/man-pages/man2/statfs.2.html
|
|
||||||
cgroupFsType = 0x27e0eb
|
|
||||||
+
|
|
||||||
+ maxIDLength = 64
|
|
||||||
)
|
|
||||||
|
|
||||||
var cgroupsDirPath string
|
|
||||||
@@ -38,6 +41,14 @@ func getContainerInfo(ctx context.Context, containerID string) (vc.ContainerStat
|
|
||||||
return vc.ContainerStatus{}, "", fmt.Errorf("Missing container ID")
|
|
||||||
}
|
|
||||||
|
|
||||||
+ if len(containerID) < maxIDLength {
|
|
||||||
+ fullContainerID, err := getContainerIDbyPrefix(containerID)
|
|
||||||
+ if err != nil {
|
|
||||||
+ return vc.ContainerStatus{}, "", err
|
|
||||||
+ }
|
|
||||||
+ containerID = fullContainerID
|
|
||||||
+ }
|
|
||||||
+
|
|
||||||
sandboxID, err := katautils.FetchContainerIDMapping(containerID)
|
|
||||||
if err != nil {
|
|
||||||
return vc.ContainerStatus{}, "", err
|
|
||||||
@@ -211,3 +222,27 @@ func getCgroupsDirPath(mountInfoFile string) (string, error) {
|
|
||||||
|
|
||||||
return cgroupRootPath, nil
|
|
||||||
}
|
|
||||||
+
|
|
||||||
+func getContainerIDbyPrefix(prefix string) (string, error) {
|
|
||||||
+ files, err := ioutil.ReadDir(katautils.GetCtrsMapTreePath())
|
|
||||||
+ if err != nil {
|
|
||||||
+ return "", err
|
|
||||||
+ }
|
|
||||||
+
|
|
||||||
+ containers := []string{}
|
|
||||||
+ for _, file := range files {
|
|
||||||
+ if file.IsDir() && strings.HasPrefix(file.Name(), prefix) {
|
|
||||||
+ containers = append(containers, file.Name())
|
|
||||||
+ }
|
|
||||||
+ }
|
|
||||||
+
|
|
||||||
+ if len(containers) == 0 {
|
|
||||||
+ return "", fmt.Errorf("no such container ID (%v)", prefix)
|
|
||||||
+ }
|
|
||||||
+
|
|
||||||
+ if len(containers) > 1 {
|
|
||||||
+ return "", fmt.Errorf("multiple containers found (%v)", prefix)
|
|
||||||
+ }
|
|
||||||
+
|
|
||||||
+ return containers[0], nil
|
|
||||||
+}
|
|
||||||
diff --git a/pkg/katautils/oci.go b/pkg/katautils/oci.go
|
|
||||||
index 6de8101e..1334af35 100644
|
|
||||||
--- a/pkg/katautils/oci.go
|
|
||||||
+++ b/pkg/katautils/oci.go
|
|
||||||
@@ -25,6 +25,11 @@ func SetCtrsMapTreePath(path string) {
|
|
||||||
ctrsMapTreePath = path
|
|
||||||
}
|
|
||||||
|
|
||||||
+// GetCtrsMapTreePath return the containerID to SandboxID mapping dir
|
|
||||||
+func GetCtrsMapTreePath() string {
|
|
||||||
+ return ctrsMapTreePath
|
|
||||||
+}
|
|
||||||
+
|
|
||||||
// doUpdatePath returns whether a ctrsMapTreePath needs to be updated with a rootless prefix
|
|
||||||
func doUpdatePath() bool {
|
|
||||||
return rootless.IsRootless() && !strings.HasPrefix(ctrsMapTreePath, rootless.GetRootlessDir())
|
|
||||||
--
|
|
||||||
2.14.3 (Apple Git-98)
|
|
||||||
|
|
||||||
@ -1,76 +0,0 @@
|
|||||||
From 1bd3cb85a1cf0e94b3280412d5fb47cecc4721fd Mon Sep 17 00:00:00 2001
|
|
||||||
From: jiangpengfei <jiangpengfei9@huawei.com>
|
|
||||||
Date: Wed, 29 Jul 2020 10:42:49 +0800
|
|
||||||
Subject: [PATCH 14/50] kata-runtime: add self defined annotations framework
|
|
||||||
|
|
||||||
reason: add self defined annotations framework to pass some
|
|
||||||
self defined resource for sandbox
|
|
||||||
|
|
||||||
Signed-off-by: jiangpengfei <jiangpengfei9@huawei.com>
|
|
||||||
---
|
|
||||||
virtcontainers/pkg/oci/utils.go | 38 ++++++++++++++++++++++++++++++++++++++
|
|
||||||
1 file changed, 38 insertions(+)
|
|
||||||
|
|
||||||
diff --git a/virtcontainers/pkg/oci/utils.go b/virtcontainers/pkg/oci/utils.go
|
|
||||||
index cd8d48ce..05181efd 100644
|
|
||||||
--- a/virtcontainers/pkg/oci/utils.go
|
|
||||||
+++ b/virtcontainers/pkg/oci/utils.go
|
|
||||||
@@ -33,6 +33,10 @@ type annotationContainerType struct {
|
|
||||||
containerType vc.ContainerType
|
|
||||||
}
|
|
||||||
|
|
||||||
+type annotationHandler func(value string) error
|
|
||||||
+
|
|
||||||
+var annotationHandlerList = map[string]annotationHandler{}
|
|
||||||
+
|
|
||||||
var (
|
|
||||||
// ErrNoLinux is an error for missing Linux sections in the OCI configuration file.
|
|
||||||
ErrNoLinux = errors.New("missing Linux section")
|
|
||||||
@@ -342,6 +346,10 @@ func addAnnotations(ocispec specs.Spec, config *vc.SandboxConfig) error {
|
|
||||||
if err := addAgentConfigOverrides(ocispec, config); err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
+
|
|
||||||
+ if err := addOtherSandboxAnnotation(ocispec, config); err != nil {
|
|
||||||
+ return err
|
|
||||||
+ }
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
@@ -1016,3 +1024,33 @@ func GetOCIConfig(status vc.ContainerStatus) (specs.Spec, error) {
|
|
||||||
|
|
||||||
return *status.Spec, nil
|
|
||||||
}
|
|
||||||
+
|
|
||||||
+// validateOtherSandboxAnnotations validate the value of annotation
|
|
||||||
+func validateOtherSandboxAnnotations(annotation, value string) error {
|
|
||||||
+ validateHandler, ok := annotationHandlerList[annotation]
|
|
||||||
+ if !ok {
|
|
||||||
+ return fmt.Errorf("unsupport Sandbox annotation type")
|
|
||||||
+ }
|
|
||||||
+
|
|
||||||
+ return validateHandler(value)
|
|
||||||
+}
|
|
||||||
+
|
|
||||||
+// addOtherSandboxAnnotation add self defined annotation for sandbox
|
|
||||||
+func addOtherSandboxAnnotation(ocispec specs.Spec, sbConfig *vc.SandboxConfig) error {
|
|
||||||
+ otherSandboxAnnotationsKey := []string{}
|
|
||||||
+
|
|
||||||
+ for _, a := range otherSandboxAnnotationsKey {
|
|
||||||
+ value, ok := ocispec.Annotations[a]
|
|
||||||
+ if !ok {
|
|
||||||
+ continue
|
|
||||||
+ }
|
|
||||||
+
|
|
||||||
+ if err := validateOtherSandboxAnnotations(a, value); err != nil {
|
|
||||||
+ return err
|
|
||||||
+ }
|
|
||||||
+
|
|
||||||
+ sbConfig.Annotations[a] = value
|
|
||||||
+ }
|
|
||||||
+
|
|
||||||
+ return nil
|
|
||||||
+}
|
|
||||||
--
|
|
||||||
2.14.3 (Apple Git-98)
|
|
||||||
|
|
||||||
@ -1,143 +0,0 @@
|
|||||||
From c0a33c4584e1fbd9b39ff1ca3ed632efe85de65e Mon Sep 17 00:00:00 2001
|
|
||||||
From: jiangpengfei <jiangpengfei9@huawei.com>
|
|
||||||
Date: Sun, 2 Aug 2020 15:51:10 +0800
|
|
||||||
Subject: [PATCH 15/50] kata-runtime: add reuse hypervisor cpu and memory
|
|
||||||
feature
|
|
||||||
|
|
||||||
reason: If default hypervisor cpu and memory is set too large,
|
|
||||||
which may waste resource, so we add enable_reuse_cpu_memory
|
|
||||||
config in the configuration.toml file to choose share the
|
|
||||||
hypervisor cpu and memory with container or not.
|
|
||||||
|
|
||||||
Signed-off-by: jiangpengfei <jiangpengfei9@huawei.com>
|
|
||||||
---
|
|
||||||
cli/config/configuration-qemu.toml.in | 5 +++++
|
|
||||||
pkg/katautils/config.go | 2 ++
|
|
||||||
virtcontainers/hypervisor.go | 3 +++
|
|
||||||
virtcontainers/persist.go | 2 ++
|
|
||||||
virtcontainers/persist/api/config.go | 3 +++
|
|
||||||
virtcontainers/sandbox.go | 22 ++++++++++++++++++----
|
|
||||||
6 files changed, 33 insertions(+), 4 deletions(-)
|
|
||||||
|
|
||||||
diff --git a/cli/config/configuration-qemu.toml.in b/cli/config/configuration-qemu.toml.in
|
|
||||||
index 46ce7d9b..82461732 100644
|
|
||||||
--- a/cli/config/configuration-qemu.toml.in
|
|
||||||
+++ b/cli/config/configuration-qemu.toml.in
|
|
||||||
@@ -95,6 +95,11 @@ default_memory = @DEFMEMSZ@
|
|
||||||
# Default false
|
|
||||||
#enable_virtio_mem = true
|
|
||||||
|
|
||||||
+# Specifies share hypervisor default cpu and memory resource
|
|
||||||
+# between hypervisor and container process.
|
|
||||||
+# Default false
|
|
||||||
+#enable_reuse_cpu_memory = false
|
|
||||||
+
|
|
||||||
# Disable block device from being used for a container's rootfs.
|
|
||||||
# In case of a storage driver like devicemapper where a container's
|
|
||||||
# root file system is backed by a block device, the block device is passed
|
|
||||||
diff --git a/pkg/katautils/config.go b/pkg/katautils/config.go
|
|
||||||
index 448d23ac..d1883c94 100644
|
|
||||||
--- a/pkg/katautils/config.go
|
|
||||||
+++ b/pkg/katautils/config.go
|
|
||||||
@@ -112,6 +112,7 @@ type hypervisor struct {
|
|
||||||
MemorySize uint32 `toml:"default_memory"`
|
|
||||||
MemSlots uint32 `toml:"memory_slots"`
|
|
||||||
MemOffset uint32 `toml:"memory_offset"`
|
|
||||||
+ EnableCPUMemoryReuse bool `toml:"enable_reuse_cpu_memory"`
|
|
||||||
DefaultBridges uint32 `toml:"default_bridges"`
|
|
||||||
Msize9p uint32 `toml:"msize_9p"`
|
|
||||||
PCIeRootPort uint32 `toml:"pcie_root_port"`
|
|
||||||
@@ -640,6 +641,7 @@ func newQemuHypervisorConfig(h hypervisor) (vc.HypervisorConfig, error) {
|
|
||||||
MemorySize: h.defaultMemSz(),
|
|
||||||
MemSlots: h.defaultMemSlots(),
|
|
||||||
MemOffset: h.defaultMemOffset(),
|
|
||||||
+ EnableCPUMemoryReuse: h.EnableCPUMemoryReuse,
|
|
||||||
VirtioMem: h.VirtioMem,
|
|
||||||
EntropySource: h.GetEntropySource(),
|
|
||||||
DefaultBridges: h.defaultBridges(),
|
|
||||||
diff --git a/virtcontainers/hypervisor.go b/virtcontainers/hypervisor.go
|
|
||||||
index fd7d1f8e..5f8d24f9 100644
|
|
||||||
--- a/virtcontainers/hypervisor.go
|
|
||||||
+++ b/virtcontainers/hypervisor.go
|
|
||||||
@@ -250,6 +250,9 @@ type HypervisorConfig struct {
|
|
||||||
// MemOffset specifies memory space for nvdimm device
|
|
||||||
MemOffset uint32
|
|
||||||
|
|
||||||
+ // Enable hypervisor cpu and memory resource share with container
|
|
||||||
+ EnableCPUMemoryReuse bool
|
|
||||||
+
|
|
||||||
// VirtioFSCacheSize is the DAX cache size in MiB
|
|
||||||
VirtioFSCacheSize uint32
|
|
||||||
|
|
||||||
diff --git a/virtcontainers/persist.go b/virtcontainers/persist.go
|
|
||||||
index d96a3890..aef4d18d 100644
|
|
||||||
--- a/virtcontainers/persist.go
|
|
||||||
+++ b/virtcontainers/persist.go
|
|
||||||
@@ -214,6 +214,7 @@ func (s *Sandbox) dumpConfig(ss *persistapi.SandboxState) {
|
|
||||||
Msize9p: sconfig.HypervisorConfig.Msize9p,
|
|
||||||
MemSlots: sconfig.HypervisorConfig.MemSlots,
|
|
||||||
MemOffset: sconfig.HypervisorConfig.MemOffset,
|
|
||||||
+ EnableCPUMemoryReuse: sconfig.HypervisorConfig.EnableCPUMemoryReuse,
|
|
||||||
VirtioMem: sconfig.HypervisorConfig.VirtioMem,
|
|
||||||
VirtioFSCacheSize: sconfig.HypervisorConfig.VirtioFSCacheSize,
|
|
||||||
KernelPath: sconfig.HypervisorConfig.KernelPath,
|
|
||||||
@@ -503,6 +504,7 @@ func loadSandboxConfig(id string) (*SandboxConfig, error) {
|
|
||||||
Msize9p: hconf.Msize9p,
|
|
||||||
MemSlots: hconf.MemSlots,
|
|
||||||
MemOffset: hconf.MemOffset,
|
|
||||||
+ EnableCPUMemoryReuse: hconf.EnableCPUMemoryReuse,
|
|
||||||
VirtioMem: hconf.VirtioMem,
|
|
||||||
VirtioFSCacheSize: hconf.VirtioFSCacheSize,
|
|
||||||
KernelPath: hconf.KernelPath,
|
|
||||||
diff --git a/virtcontainers/persist/api/config.go b/virtcontainers/persist/api/config.go
|
|
||||||
index 34a5fd0f..a3c6ec91 100644
|
|
||||||
--- a/virtcontainers/persist/api/config.go
|
|
||||||
+++ b/virtcontainers/persist/api/config.go
|
|
||||||
@@ -35,6 +35,9 @@ type HypervisorConfig struct {
|
|
||||||
// MemOffset specifies memory space for nvdimm device
|
|
||||||
MemOffset uint32
|
|
||||||
|
|
||||||
+ // Enable hypervisor cpu and memory resource share with container
|
|
||||||
+ EnableCPUMemoryReuse bool
|
|
||||||
+
|
|
||||||
// VirtioFSCacheSize is the DAX cache size in MiB
|
|
||||||
VirtioFSCacheSize uint32
|
|
||||||
|
|
||||||
diff --git a/virtcontainers/sandbox.go b/virtcontainers/sandbox.go
|
|
||||||
index 78188ed7..e766d1f7 100644
|
|
||||||
--- a/virtcontainers/sandbox.go
|
|
||||||
+++ b/virtcontainers/sandbox.go
|
|
||||||
@@ -1833,12 +1833,26 @@ func (s *Sandbox) updateResources() error {
|
|
||||||
}
|
|
||||||
|
|
||||||
sandboxVCPUs := s.calculateSandboxCPUs()
|
|
||||||
- // Add default vcpus for sandbox
|
|
||||||
- sandboxVCPUs += s.hypervisor.hypervisorConfig().NumVCPUs
|
|
||||||
+ // Share the hypervisor vcpu with container process
|
|
||||||
+ if s.config.HypervisorConfig.EnableCPUMemoryReuse {
|
|
||||||
+ if sandboxVCPUs <= s.config.HypervisorConfig.NumVCPUs {
|
|
||||||
+ sandboxVCPUs = s.config.HypervisorConfig.NumVCPUs
|
|
||||||
+ }
|
|
||||||
+ } else {
|
|
||||||
+ // Add default vcpus for sandbox
|
|
||||||
+ sandboxVCPUs += s.hypervisor.hypervisorConfig().NumVCPUs
|
|
||||||
+ }
|
|
||||||
|
|
||||||
sandboxMemoryByte := s.calculateSandboxMemory()
|
|
||||||
- // Add default / rsvd memory for sandbox.
|
|
||||||
- sandboxMemoryByte += int64(s.hypervisor.hypervisorConfig().MemorySize) << utils.MibToBytesShift
|
|
||||||
+ // Share the hypervisor memory with container process
|
|
||||||
+ if s.config.HypervisorConfig.EnableCPUMemoryReuse {
|
|
||||||
+ if sandboxMemoryByte <= (int64(s.config.HypervisorConfig.MemorySize) << utils.MibToBytesShift) {
|
|
||||||
+ sandboxMemoryByte = int64(s.config.HypervisorConfig.MemorySize) << utils.MibToBytesShift
|
|
||||||
+ }
|
|
||||||
+ } else {
|
|
||||||
+ // Add default / rsvd memory for sandbox.
|
|
||||||
+ sandboxMemoryByte += int64(s.hypervisor.hypervisorConfig().MemorySize) << utils.MibToBytesShift
|
|
||||||
+ }
|
|
||||||
|
|
||||||
// Update VCPUs
|
|
||||||
s.Logger().WithField("cpus-sandbox", sandboxVCPUs).Debugf("Request to hypervisor to update vCPUs")
|
|
||||||
--
|
|
||||||
2.14.3 (Apple Git-98)
|
|
||||||
|
|
||||||
@ -1,288 +0,0 @@
|
|||||||
From dc9de8bb181e2cec2f3e0a76d02833fef45b46af Mon Sep 17 00:00:00 2001
|
|
||||||
From: jiangpengfei <jiangpengfei9@huawei.com>
|
|
||||||
Date: Thu, 6 Aug 2020 09:28:34 -0400
|
|
||||||
Subject: [PATCH 16/50] virtcontainers: fix hotplug huge size memory cause
|
|
||||||
agent hang bug
|
|
||||||
|
|
||||||
fixes: #2872
|
|
||||||
|
|
||||||
reason: If hotplug huge size memory into kata VM at once time,
|
|
||||||
guest kernel will allocate some extra memory for memory management,
|
|
||||||
which may cause kata-agent hang and out of responding.
|
|
||||||
And hotplug more memory into VM, more extra memory is needed.
|
|
||||||
|
|
||||||
Inorder to solve this problem, we divide hotplug huge memory into
|
|
||||||
two steps. First, hotplug the max allowed memory into VM and wait
|
|
||||||
all first step hotplugged memory online. Second Step, hotplug the
|
|
||||||
left memory into VM.
|
|
||||||
|
|
||||||
Signed-off-by: jiangpengfei <jiangpengfei9@huawei.com>
|
|
||||||
---
|
|
||||||
virtcontainers/acrn.go | 4 ++++
|
|
||||||
virtcontainers/agent.go | 3 ++-
|
|
||||||
virtcontainers/clh.go | 4 ++++
|
|
||||||
virtcontainers/fc.go | 4 ++++
|
|
||||||
virtcontainers/hypervisor.go | 3 +++
|
|
||||||
virtcontainers/kata_agent.go | 4 ++--
|
|
||||||
virtcontainers/kata_agent_test.go | 2 +-
|
|
||||||
virtcontainers/mock_hypervisor.go | 4 ++++
|
|
||||||
virtcontainers/noop_agent.go | 2 +-
|
|
||||||
virtcontainers/qemu.go | 4 ++++
|
|
||||||
virtcontainers/sandbox.go | 30 ++++++++++++++++++++++++++++--
|
|
||||||
virtcontainers/sandbox_test.go | 12 ++++++++++++
|
|
||||||
virtcontainers/utils/utils.go | 3 +++
|
|
||||||
virtcontainers/vm.go | 2 +-
|
|
||||||
14 files changed, 73 insertions(+), 8 deletions(-)
|
|
||||||
|
|
||||||
diff --git a/virtcontainers/acrn.go b/virtcontainers/acrn.go
|
|
||||||
index 10cae06f..c9a0fe0b 100644
|
|
||||||
--- a/virtcontainers/acrn.go
|
|
||||||
+++ b/virtcontainers/acrn.go
|
|
||||||
@@ -811,3 +811,7 @@ func (a *Acrn) loadInfo() error {
|
|
||||||
}
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
+
|
|
||||||
+func (a *Acrn) getMemorySize() uint32 {
|
|
||||||
+ return a.config.MemorySize
|
|
||||||
+}
|
|
||||||
diff --git a/virtcontainers/agent.go b/virtcontainers/agent.go
|
|
||||||
index be9526c7..b1dea816 100644
|
|
||||||
--- a/virtcontainers/agent.go
|
|
||||||
+++ b/virtcontainers/agent.go
|
|
||||||
@@ -201,7 +201,8 @@ type agent interface {
|
|
||||||
// This function should be called after hot adding vCPUs or Memory.
|
|
||||||
// cpus specifies the number of CPUs that were added and the agent should online
|
|
||||||
// cpuOnly specifies that we should online cpu or online memory or both
|
|
||||||
- onlineCPUMem(cpus uint32, cpuOnly bool) error
|
|
||||||
+ // wait specifies that we should wait all cpu or memory online in the VM synchronously
|
|
||||||
+ onlineCPUMem(cpus uint32, cpuOnly bool, wait bool) error
|
|
||||||
|
|
||||||
// memHotplugByProbe will notify the guest kernel about memory hotplug event through
|
|
||||||
// probe interface.
|
|
||||||
diff --git a/virtcontainers/clh.go b/virtcontainers/clh.go
|
|
||||||
index 59510b02..8afcd4bf 100644
|
|
||||||
--- a/virtcontainers/clh.go
|
|
||||||
+++ b/virtcontainers/clh.go
|
|
||||||
@@ -1210,3 +1210,7 @@ func (clh *cloudHypervisor) vmInfo() (chclient.VmInfo, error) {
|
|
||||||
return info, openAPIClientError(err)
|
|
||||||
|
|
||||||
}
|
|
||||||
+
|
|
||||||
+func (clh *cloudHypervisor) getMemorySize() uint32 {
|
|
||||||
+ return clh.config.MemorySize
|
|
||||||
+}
|
|
||||||
diff --git a/virtcontainers/fc.go b/virtcontainers/fc.go
|
|
||||||
index 72a8e192..15726156 100644
|
|
||||||
--- a/virtcontainers/fc.go
|
|
||||||
+++ b/virtcontainers/fc.go
|
|
||||||
@@ -1212,3 +1212,7 @@ func (fc *firecracker) watchConsole() (*os.File, error) {
|
|
||||||
|
|
||||||
return stdio, nil
|
|
||||||
}
|
|
||||||
+
|
|
||||||
+func (fc *firecracker) getMemorySize() uint32 {
|
|
||||||
+ return fc.config.MemorySize
|
|
||||||
+}
|
|
||||||
diff --git a/virtcontainers/hypervisor.go b/virtcontainers/hypervisor.go
|
|
||||||
index 5f8d24f9..9cd685ad 100644
|
|
||||||
--- a/virtcontainers/hypervisor.go
|
|
||||||
+++ b/virtcontainers/hypervisor.go
|
|
||||||
@@ -778,6 +778,9 @@ type hypervisor interface {
|
|
||||||
hotplugRemoveDevice(devInfo interface{}, devType deviceType) (interface{}, error)
|
|
||||||
resizeMemory(memMB uint32, memoryBlockSizeMB uint32, probe bool) (uint32, memoryDevice, error)
|
|
||||||
resizeVCPUs(vcpus uint32) (uint32, uint32, error)
|
|
||||||
+ // getMemorySize return the total memory in the guest include default memory size + hot plugged memory
|
|
||||||
+ // return memory size unit is MB
|
|
||||||
+ getMemorySize() uint32
|
|
||||||
getSandboxConsole(sandboxID string) (string, error)
|
|
||||||
disconnect()
|
|
||||||
capabilities() types.Capabilities
|
|
||||||
diff --git a/virtcontainers/kata_agent.go b/virtcontainers/kata_agent.go
|
|
||||||
index 7575d326..8e073339 100644
|
|
||||||
--- a/virtcontainers/kata_agent.go
|
|
||||||
+++ b/virtcontainers/kata_agent.go
|
|
||||||
@@ -1806,9 +1806,9 @@ func (k *kataAgent) memHotplugByProbe(addr uint64, sizeMB uint32, memorySectionS
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
-func (k *kataAgent) onlineCPUMem(cpus uint32, cpuOnly bool) error {
|
|
||||||
+func (k *kataAgent) onlineCPUMem(cpus uint32, cpuOnly bool, wait bool) error {
|
|
||||||
req := &grpc.OnlineCPUMemRequest{
|
|
||||||
- Wait: false,
|
|
||||||
+ Wait: wait,
|
|
||||||
NbCpus: cpus,
|
|
||||||
CpuOnly: cpuOnly,
|
|
||||||
}
|
|
||||||
diff --git a/virtcontainers/kata_agent_test.go b/virtcontainers/kata_agent_test.go
|
|
||||||
index 62d31c93..2a2ddada 100644
|
|
||||||
--- a/virtcontainers/kata_agent_test.go
|
|
||||||
+++ b/virtcontainers/kata_agent_test.go
|
|
||||||
@@ -324,7 +324,7 @@ func TestKataAgentSendReq(t *testing.T) {
|
|
||||||
err = k.resumeContainer(sandbox, Container{})
|
|
||||||
assert.Nil(err)
|
|
||||||
|
|
||||||
- err = k.onlineCPUMem(1, true)
|
|
||||||
+ err = k.onlineCPUMem(1, true, false)
|
|
||||||
assert.Nil(err)
|
|
||||||
|
|
||||||
_, err = k.statsContainer(sandbox, Container{})
|
|
||||||
diff --git a/virtcontainers/mock_hypervisor.go b/virtcontainers/mock_hypervisor.go
|
|
||||||
index a5b67491..f1c6106d 100644
|
|
||||||
--- a/virtcontainers/mock_hypervisor.go
|
|
||||||
+++ b/virtcontainers/mock_hypervisor.go
|
|
||||||
@@ -128,3 +128,7 @@ func (m *mockHypervisor) check() error {
|
|
||||||
func (m *mockHypervisor) generateSocket(id string, useVsock bool) (interface{}, error) {
|
|
||||||
return types.Socket{HostPath: "/tmp/socket", Name: "socket"}, nil
|
|
||||||
}
|
|
||||||
+
|
|
||||||
+func (m *mockHypervisor) getMemorySize() uint32 {
|
|
||||||
+ return 0
|
|
||||||
+}
|
|
||||||
diff --git a/virtcontainers/noop_agent.go b/virtcontainers/noop_agent.go
|
|
||||||
index 8a7cd337..6e211bca 100644
|
|
||||||
--- a/virtcontainers/noop_agent.go
|
|
||||||
+++ b/virtcontainers/noop_agent.go
|
|
||||||
@@ -102,7 +102,7 @@ func (n *noopAgent) memHotplugByProbe(addr uint64, sizeMB uint32, memorySectionS
|
|
||||||
}
|
|
||||||
|
|
||||||
// onlineCPUMem is the Noop agent Container online CPU and Memory implementation. It does nothing.
|
|
||||||
-func (n *noopAgent) onlineCPUMem(cpus uint32, cpuOnly bool) error {
|
|
||||||
+func (n *noopAgent) onlineCPUMem(cpus uint32, cpuOnly bool, wait bool) error {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
diff --git a/virtcontainers/qemu.go b/virtcontainers/qemu.go
|
|
||||||
index 4789101d..7bae3278 100644
|
|
||||||
--- a/virtcontainers/qemu.go
|
|
||||||
+++ b/virtcontainers/qemu.go
|
|
||||||
@@ -2273,3 +2273,7 @@ func (q *qemu) check() error {
|
|
||||||
func (q *qemu) generateSocket(id string, useVsock bool) (interface{}, error) {
|
|
||||||
return generateVMSocket(id, useVsock, q.store.RunVMStoragePath())
|
|
||||||
}
|
|
||||||
+
|
|
||||||
+func (q *qemu) getMemorySize() uint32 {
|
|
||||||
+ return q.config.MemorySize + uint32(q.state.HotpluggedMemory)
|
|
||||||
+}
|
|
||||||
diff --git a/virtcontainers/sandbox.go b/virtcontainers/sandbox.go
|
|
||||||
index e766d1f7..a318d677 100644
|
|
||||||
--- a/virtcontainers/sandbox.go
|
|
||||||
+++ b/virtcontainers/sandbox.go
|
|
||||||
@@ -1864,7 +1864,7 @@ func (s *Sandbox) updateResources() error {
|
|
||||||
// If the CPUs were increased, ask agent to online them
|
|
||||||
if oldCPUs < newCPUs {
|
|
||||||
vcpusAdded := newCPUs - oldCPUs
|
|
||||||
- if err := s.agent.onlineCPUMem(vcpusAdded, true); err != nil {
|
|
||||||
+ if err := s.agent.onlineCPUMem(vcpusAdded, true, false); err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1872,6 +1872,20 @@ func (s *Sandbox) updateResources() error {
|
|
||||||
|
|
||||||
// Update Memory
|
|
||||||
s.Logger().WithField("memory-sandbox-size-byte", sandboxMemoryByte).Debugf("Request to hypervisor to update memory")
|
|
||||||
+ reqMemMB := uint32(sandboxMemoryByte >> utils.MibToBytesShift)
|
|
||||||
+ currentMemMB := s.hypervisor.getMemorySize()
|
|
||||||
+
|
|
||||||
+ // If request hotplug memory size larger than utils.MaxHotplugMemMBOnceTime,
|
|
||||||
+ // inorder to avoid hotplug memory oom problem, we need to hotplug large memory
|
|
||||||
+ // with two steps. First, hotplug utils.MaxHotplugMemMBOnceTime size memory into
|
|
||||||
+ // guest and wait all hotplug memory online. Then, hotplug the left unplugged memory
|
|
||||||
+ // into the guest
|
|
||||||
+ if currentMemMB < reqMemMB && (reqMemMB-currentMemMB) > utils.MaxHotplugMemMBOnceTime {
|
|
||||||
+ if err := s.beforeHotplugHugeMem(currentMemMB); err != nil {
|
|
||||||
+ return err
|
|
||||||
+ }
|
|
||||||
+ }
|
|
||||||
+
|
|
||||||
newMemory, updatedMemoryDevice, err := s.hypervisor.resizeMemory(uint32(sandboxMemoryByte>>utils.MibToBytesShift), s.state.GuestMemoryBlockSizeMB, s.state.GuestMemoryHotplugProbe)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
@@ -1884,7 +1898,7 @@ func (s *Sandbox) updateResources() error {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
}
|
|
||||||
- if err := s.agent.onlineCPUMem(0, false); err != nil {
|
|
||||||
+ if err := s.agent.onlineCPUMem(0, false, false); err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
return nil
|
|
||||||
@@ -1926,6 +1940,18 @@ func (s *Sandbox) calculateSandboxCPUs() uint32 {
|
|
||||||
return utils.CalculateVCpusFromMilliCpus(mCPU)
|
|
||||||
}
|
|
||||||
|
|
||||||
+func (s *Sandbox) beforeHotplugHugeMem(currentMemSizeInMB uint32) error {
|
|
||||||
+ wantedTotalMemSize := currentMemSizeInMB + utils.MaxHotplugMemMBOnceTime
|
|
||||||
+ newMemory, _, err := s.hypervisor.resizeMemory(wantedTotalMemSize, s.state.GuestMemoryBlockSizeMB, s.state.GuestMemoryHotplugProbe)
|
|
||||||
+ if err != nil {
|
|
||||||
+ return err
|
|
||||||
+ }
|
|
||||||
+
|
|
||||||
+ s.Logger().Debugf("first part hotplug memory size: %d MB", newMemory)
|
|
||||||
+ // wait all first part hotplugged memory online in the guest
|
|
||||||
+ return s.agent.onlineCPUMem(0, false, true)
|
|
||||||
+}
|
|
||||||
+
|
|
||||||
// GetHypervisorType is used for getting Hypervisor name currently used.
|
|
||||||
// Sandbox implement DeviceReceiver interface from device/api/interface.go
|
|
||||||
func (s *Sandbox) GetHypervisorType() string {
|
|
||||||
diff --git a/virtcontainers/sandbox_test.go b/virtcontainers/sandbox_test.go
|
|
||||||
index 85c712e8..4b02b3f3 100644
|
|
||||||
--- a/virtcontainers/sandbox_test.go
|
|
||||||
+++ b/virtcontainers/sandbox_test.go
|
|
||||||
@@ -25,6 +25,7 @@ import (
|
|
||||||
"github.com/kata-containers/runtime/virtcontainers/persist/fs"
|
|
||||||
"github.com/kata-containers/runtime/virtcontainers/pkg/annotations"
|
|
||||||
"github.com/kata-containers/runtime/virtcontainers/types"
|
|
||||||
+ "github.com/kata-containers/runtime/virtcontainers/utils"
|
|
||||||
specs "github.com/opencontainers/runtime-spec/specs-go"
|
|
||||||
"github.com/stretchr/testify/assert"
|
|
||||||
"golang.org/x/sys/unix"
|
|
||||||
@@ -1522,6 +1523,17 @@ func TestSandboxUpdateResources(t *testing.T) {
|
|
||||||
}
|
|
||||||
err = s.updateResources()
|
|
||||||
assert.NoError(t, err)
|
|
||||||
+
|
|
||||||
+ // add a container with huge memory equal utils.MaxHotplugMemMBOnceTime
|
|
||||||
+ contConfig3 := newTestContainerConfigNoop("cont-00003")
|
|
||||||
+ contConfig3.Resources.Memory = &specs.LinuxMemory{
|
|
||||||
+ Limit: new(int64),
|
|
||||||
+ }
|
|
||||||
+ container3MemLimitInBytes := int64(utils.MaxHotplugMemMBOnceTime << utils.MibToBytesShift)
|
|
||||||
+ contConfig3.Resources.Memory.Limit = &container3MemLimitInBytes
|
|
||||||
+ s.config.Containers = append(s.config.Containers, contConfig3)
|
|
||||||
+ err = s.updateResources()
|
|
||||||
+ assert.NoError(t, err)
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestSandboxExperimentalFeature(t *testing.T) {
|
|
||||||
diff --git a/virtcontainers/utils/utils.go b/virtcontainers/utils/utils.go
|
|
||||||
index 2b555ebb..3ae95aef 100644
|
|
||||||
--- a/virtcontainers/utils/utils.go
|
|
||||||
+++ b/virtcontainers/utils/utils.go
|
|
||||||
@@ -25,6 +25,9 @@ const fileMode0755 = os.FileMode(0755)
|
|
||||||
// MibToBytesShift the number to shift needed to convert MiB to Bytes
|
|
||||||
const MibToBytesShift = 20
|
|
||||||
|
|
||||||
+// Max Hotplug Memory size at once time, unit is MB
|
|
||||||
+const MaxHotplugMemMBOnceTime = 32 * 1024
|
|
||||||
+
|
|
||||||
// MaxSocketPathLen is the effective maximum Unix domain socket length.
|
|
||||||
//
|
|
||||||
// See unix(7).
|
|
||||||
diff --git a/virtcontainers/vm.go b/virtcontainers/vm.go
|
|
||||||
index 8d27b1fe..2e5fef44 100644
|
|
||||||
--- a/virtcontainers/vm.go
|
|
||||||
+++ b/virtcontainers/vm.go
|
|
||||||
@@ -370,7 +370,7 @@ func (v *VM) AddMemory(numMB uint32) error {
|
|
||||||
// OnlineCPUMemory puts the hotplugged CPU and memory online.
|
|
||||||
func (v *VM) OnlineCPUMemory() error {
|
|
||||||
v.logger().Infof("online CPU %d and memory", v.cpuDelta)
|
|
||||||
- err := v.agent.onlineCPUMem(v.cpuDelta, false)
|
|
||||||
+ err := v.agent.onlineCPUMem(v.cpuDelta, false, false)
|
|
||||||
if err == nil {
|
|
||||||
v.cpuDelta = 0
|
|
||||||
}
|
|
||||||
--
|
|
||||||
2.14.3 (Apple Git-98)
|
|
||||||
|
|
||||||
@ -1,355 +0,0 @@
|
|||||||
From e6051eb1a8c54b91c46f68eab9a63ad0e73a9221 Mon Sep 17 00:00:00 2001
|
|
||||||
From: jiangpengfei <jiangpengfei9@huawei.com>
|
|
||||||
Date: Fri, 7 Aug 2020 20:01:15 +0800
|
|
||||||
Subject: [PATCH 17/50] kata-runtime: validate sandbox cpu and memory size
|
|
||||||
|
|
||||||
reason:
|
|
||||||
1. add more strict sandbox cpu and memory size check.
|
|
||||||
2. use GetPhysicalCPUNumber func to replace goruntime.NumCPU(),
|
|
||||||
because goruntime.NumCPU() return the CPU number of cpuset cgroup
|
|
||||||
that current process joined, however kata-runtime process will
|
|
||||||
change the cpuset cgroup when setupSanboxCgroup is called, which
|
|
||||||
may cause goruntime.NumCPU() return value changed.
|
|
||||||
|
|
||||||
Signed-off-by: jiangpengfei <jiangpengfei9@huawei.com>
|
|
||||||
---
|
|
||||||
pkg/katautils/config.go | 39 ++++++++++++++++++++++++++++++++++---
|
|
||||||
pkg/katautils/config_test.go | 11 ++++++++---
|
|
||||||
virtcontainers/pkg/oci/utils.go | 11 +++++------
|
|
||||||
virtcontainers/qemu_arm64.go | 4 ++--
|
|
||||||
virtcontainers/qemu_arm64_test.go | 4 ++--
|
|
||||||
virtcontainers/sandbox.go | 7 ++++++-
|
|
||||||
virtcontainers/utils/utils.go | 41 +++++++++++++++++++++++++++++++++++++++
|
|
||||||
7 files changed, 100 insertions(+), 17 deletions(-)
|
|
||||||
|
|
||||||
diff --git a/pkg/katautils/config.go b/pkg/katautils/config.go
|
|
||||||
index d1883c94..b9b08469 100644
|
|
||||||
--- a/pkg/katautils/config.go
|
|
||||||
+++ b/pkg/katautils/config.go
|
|
||||||
@@ -11,7 +11,6 @@ import (
|
|
||||||
"fmt"
|
|
||||||
"io/ioutil"
|
|
||||||
"path/filepath"
|
|
||||||
- goruntime "runtime"
|
|
||||||
"strings"
|
|
||||||
|
|
||||||
"github.com/BurntSushi/toml"
|
|
||||||
@@ -285,7 +284,7 @@ func (h hypervisor) GetEntropySource() string {
|
|
||||||
}
|
|
||||||
|
|
||||||
func (h hypervisor) defaultVCPUs() uint32 {
|
|
||||||
- numCPUs := goruntime.NumCPU()
|
|
||||||
+ numCPUs := utils.GetPhysicalCPUNumber()
|
|
||||||
|
|
||||||
if h.NumVCPUs < 0 || h.NumVCPUs > int32(numCPUs) {
|
|
||||||
return uint32(numCPUs)
|
|
||||||
@@ -297,8 +296,22 @@ func (h hypervisor) defaultVCPUs() uint32 {
|
|
||||||
return uint32(h.NumVCPUs)
|
|
||||||
}
|
|
||||||
|
|
||||||
+func (h hypervisor) checkVCPUs() error {
|
|
||||||
+ numCPUs := utils.GetPhysicalCPUNumber()
|
|
||||||
+
|
|
||||||
+ if h.NumVCPUs <= 0 {
|
|
||||||
+ return fmt.Errorf("invalid vcpus in configuration.toml! vcpus must larger than 0")
|
|
||||||
+ }
|
|
||||||
+
|
|
||||||
+ if h.NumVCPUs > int32(numCPUs) {
|
|
||||||
+ return fmt.Errorf("invalid vcpus in configuration.toml! vcpus must smaller than max CPUs: %d in machine", numCPUs)
|
|
||||||
+ }
|
|
||||||
+
|
|
||||||
+ return nil
|
|
||||||
+}
|
|
||||||
+
|
|
||||||
func (h hypervisor) defaultMaxVCPUs() uint32 {
|
|
||||||
- numcpus := uint32(goruntime.NumCPU())
|
|
||||||
+ numcpus := uint32(utils.GetPhysicalCPUNumber())
|
|
||||||
maxvcpus := vc.MaxQemuVCPUs()
|
|
||||||
reqVCPUs := h.DefaultMaxVCPUs
|
|
||||||
|
|
||||||
@@ -324,6 +337,18 @@ func (h hypervisor) defaultMemSz() uint32 {
|
|
||||||
return h.MemorySize
|
|
||||||
}
|
|
||||||
|
|
||||||
+func (h hypervisor) checkMemSz() error {
|
|
||||||
+ if h.MemorySize < utils.MinMemorySizeInMB {
|
|
||||||
+ return fmt.Errorf("invalid memory size! Memory size must larger than %d MB", utils.MinMemorySizeInMB)
|
|
||||||
+ }
|
|
||||||
+
|
|
||||||
+ if h.MemorySize > utils.MaxMemorySizeInMB {
|
|
||||||
+ return fmt.Errorf("invalid memory size, memory size must smaller than %d MB", utils.MaxMemorySizeInMB)
|
|
||||||
+ }
|
|
||||||
+
|
|
||||||
+ return nil
|
|
||||||
+}
|
|
||||||
+
|
|
||||||
func (h hypervisor) defaultMemSlots() uint32 {
|
|
||||||
slots := h.MemSlots
|
|
||||||
if slots == 0 {
|
|
||||||
@@ -627,6 +652,14 @@ func newQemuHypervisorConfig(h hypervisor) (vc.HypervisorConfig, error) {
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
+ if err = h.checkVCPUs(); err != nil {
|
|
||||||
+ return vc.HypervisorConfig{}, err
|
|
||||||
+ }
|
|
||||||
+
|
|
||||||
+ if err = h.checkMemSz(); err != nil {
|
|
||||||
+ return vc.HypervisorConfig{}, err
|
|
||||||
+ }
|
|
||||||
+
|
|
||||||
return vc.HypervisorConfig{
|
|
||||||
HypervisorPath: hypervisor,
|
|
||||||
KernelPath: kernel,
|
|
||||||
diff --git a/pkg/katautils/config_test.go b/pkg/katautils/config_test.go
|
|
||||||
index 31afcca6..37aa16d0 100644
|
|
||||||
--- a/pkg/katautils/config_test.go
|
|
||||||
+++ b/pkg/katautils/config_test.go
|
|
||||||
@@ -14,7 +14,6 @@ import (
|
|
||||||
"path"
|
|
||||||
"path/filepath"
|
|
||||||
"reflect"
|
|
||||||
- goruntime "runtime"
|
|
||||||
"strings"
|
|
||||||
"syscall"
|
|
||||||
"testing"
|
|
||||||
@@ -152,7 +151,7 @@ func createAllRuntimeConfigFiles(dir, hypervisor string) (config testRuntimeConf
|
|
||||||
KernelParams: vc.DeserializeParams(strings.Fields(kernelParams)),
|
|
||||||
HypervisorMachineType: machineType,
|
|
||||||
NumVCPUs: defaultVCPUCount,
|
|
||||||
- DefaultMaxVCPUs: uint32(goruntime.NumCPU()),
|
|
||||||
+ DefaultMaxVCPUs: uint32(utils.GetPhysicalCPUNumber()),
|
|
||||||
MemorySize: defaultMemSize,
|
|
||||||
DisableBlockDeviceUse: disableBlockDevice,
|
|
||||||
BlockDeviceDriver: defaultBlockDeviceDriver,
|
|
||||||
@@ -727,6 +726,8 @@ func TestMinimalRuntimeConfigWithVsock(t *testing.T) {
|
|
||||||
[hypervisor.qemu]
|
|
||||||
use_vsock = true
|
|
||||||
image = "` + imagePath + `"
|
|
||||||
+ default_vcpus = 1
|
|
||||||
+ default_memory = 1024
|
|
||||||
|
|
||||||
[proxy.kata]
|
|
||||||
path = "` + proxyPath + `"
|
|
||||||
@@ -786,6 +787,8 @@ func TestNewQemuHypervisorConfig(t *testing.T) {
|
|
||||||
utils.VHostVSockDevicePath = orgVHostVSockDevicePath
|
|
||||||
}()
|
|
||||||
utils.VHostVSockDevicePath = "/dev/abc/xyz"
|
|
||||||
+ defaultVCPUs := int32(1)
|
|
||||||
+ defaultMemory := uint32(1024)
|
|
||||||
|
|
||||||
hypervisor := hypervisor{
|
|
||||||
Path: hypervisorPath,
|
|
||||||
@@ -797,6 +800,8 @@ func TestNewQemuHypervisorConfig(t *testing.T) {
|
|
||||||
HotplugVFIOOnRootBus: hotplugVFIOOnRootBus,
|
|
||||||
PCIeRootPort: pcieRootPort,
|
|
||||||
UseVSock: true,
|
|
||||||
+ NumVCPUs: defaultVCPUs,
|
|
||||||
+ MemorySize: defaultMemory,
|
|
||||||
}
|
|
||||||
|
|
||||||
files := []string{hypervisorPath, kernelPath, imagePath}
|
|
||||||
@@ -996,7 +1001,7 @@ func TestNewShimConfig(t *testing.T) {
|
|
||||||
func TestHypervisorDefaults(t *testing.T) {
|
|
||||||
assert := assert.New(t)
|
|
||||||
|
|
||||||
- numCPUs := goruntime.NumCPU()
|
|
||||||
+ numCPUs := utils.GetPhysicalCPUNumber()
|
|
||||||
|
|
||||||
h := hypervisor{}
|
|
||||||
|
|
||||||
diff --git a/virtcontainers/pkg/oci/utils.go b/virtcontainers/pkg/oci/utils.go
|
|
||||||
index 05181efd..0a6f08c7 100644
|
|
||||||
--- a/virtcontainers/pkg/oci/utils.go
|
|
||||||
+++ b/virtcontainers/pkg/oci/utils.go
|
|
||||||
@@ -10,22 +10,21 @@ import (
|
|
||||||
"errors"
|
|
||||||
"fmt"
|
|
||||||
"path/filepath"
|
|
||||||
- goruntime "runtime"
|
|
||||||
"strconv"
|
|
||||||
"strings"
|
|
||||||
"syscall"
|
|
||||||
|
|
||||||
criContainerdAnnotations "github.com/containerd/cri-containerd/pkg/annotations"
|
|
||||||
crioAnnotations "github.com/cri-o/cri-o/pkg/annotations"
|
|
||||||
- specs "github.com/opencontainers/runtime-spec/specs-go"
|
|
||||||
- "github.com/sirupsen/logrus"
|
|
||||||
-
|
|
||||||
vc "github.com/kata-containers/runtime/virtcontainers"
|
|
||||||
"github.com/kata-containers/runtime/virtcontainers/device/config"
|
|
||||||
exp "github.com/kata-containers/runtime/virtcontainers/experimental"
|
|
||||||
vcAnnotations "github.com/kata-containers/runtime/virtcontainers/pkg/annotations"
|
|
||||||
dockershimAnnotations "github.com/kata-containers/runtime/virtcontainers/pkg/annotations/dockershim"
|
|
||||||
"github.com/kata-containers/runtime/virtcontainers/types"
|
|
||||||
+ "github.com/kata-containers/runtime/virtcontainers/utils"
|
|
||||||
+ specs "github.com/opencontainers/runtime-spec/specs-go"
|
|
||||||
+ "github.com/sirupsen/logrus"
|
|
||||||
)
|
|
||||||
|
|
||||||
type annotationContainerType struct {
|
|
||||||
@@ -560,7 +559,7 @@ func addHypervisorCPUOverrides(ocispec specs.Spec, sbConfig *vc.SandboxConfig) e
|
|
||||||
return fmt.Errorf("Error encountered parsing annotation default_vcpus: %v, please specify numeric value", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
- numCPUs := goruntime.NumCPU()
|
|
||||||
+ numCPUs := utils.GetPhysicalCPUNumber()
|
|
||||||
|
|
||||||
if uint32(vcpus) > uint32(numCPUs) {
|
|
||||||
return fmt.Errorf("Number of cpus %d specified in annotation default_vcpus is greater than the number of CPUs %d on the system", vcpus, numCPUs)
|
|
||||||
@@ -575,7 +574,7 @@ func addHypervisorCPUOverrides(ocispec specs.Spec, sbConfig *vc.SandboxConfig) e
|
|
||||||
return fmt.Errorf("Error encountered parsing annotation for default_maxvcpus: %v, please specify positive numeric value", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
- numCPUs := goruntime.NumCPU()
|
|
||||||
+ numCPUs := utils.GetPhysicalCPUNumber()
|
|
||||||
max := uint32(maxVCPUs)
|
|
||||||
|
|
||||||
if max > uint32(numCPUs) {
|
|
||||||
diff --git a/virtcontainers/qemu_arm64.go b/virtcontainers/qemu_arm64.go
|
|
||||||
index 6d089cf0..40fb2a80 100644
|
|
||||||
--- a/virtcontainers/qemu_arm64.go
|
|
||||||
+++ b/virtcontainers/qemu_arm64.go
|
|
||||||
@@ -8,11 +8,11 @@ package virtcontainers
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
"io/ioutil"
|
|
||||||
- "runtime"
|
|
||||||
"strings"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
govmmQemu "github.com/intel/govmm/qemu"
|
|
||||||
+ "github.com/kata-containers/runtime/virtcontainers/utils"
|
|
||||||
"github.com/sirupsen/logrus"
|
|
||||||
)
|
|
||||||
|
|
||||||
@@ -121,7 +121,7 @@ func MaxQemuVCPUs() uint32 {
|
|
||||||
if hostGICVersion != 0 {
|
|
||||||
return gicList[hostGICVersion]
|
|
||||||
}
|
|
||||||
- return uint32(runtime.NumCPU())
|
|
||||||
+ return uint32(utils.GetPhysicalCPUNumber())
|
|
||||||
}
|
|
||||||
|
|
||||||
func newQemuArch(config HypervisorConfig) qemuArch {
|
|
||||||
diff --git a/virtcontainers/qemu_arm64_test.go b/virtcontainers/qemu_arm64_test.go
|
|
||||||
index 40351158..372fc4a9 100644
|
|
||||||
--- a/virtcontainers/qemu_arm64_test.go
|
|
||||||
+++ b/virtcontainers/qemu_arm64_test.go
|
|
||||||
@@ -10,10 +10,10 @@ import (
|
|
||||||
"io/ioutil"
|
|
||||||
"os"
|
|
||||||
"path/filepath"
|
|
||||||
- "runtime"
|
|
||||||
"testing"
|
|
||||||
|
|
||||||
govmmQemu "github.com/intel/govmm/qemu"
|
|
||||||
+ "github.com/kata-containers/runtime/virtcontainers/utils"
|
|
||||||
"github.com/stretchr/testify/assert"
|
|
||||||
)
|
|
||||||
|
|
||||||
@@ -63,7 +63,7 @@ func TestMaxQemuVCPUs(t *testing.T) {
|
|
||||||
}
|
|
||||||
|
|
||||||
data := []testData{
|
|
||||||
- {"", uint32(runtime.NumCPU())},
|
|
||||||
+ {"", uint32(utils.GetPhysicalCPUNumber())},
|
|
||||||
{" 1: 0 0 GICv2 25 Level vgic \n", uint32(8)},
|
|
||||||
{" 1: 0 0 GICv3 25 Level vgic \n", uint32(123)},
|
|
||||||
{" 1: 0 0 GICv4 25 Level vgic \n", uint32(123)},
|
|
||||||
diff --git a/virtcontainers/sandbox.go b/virtcontainers/sandbox.go
|
|
||||||
index a318d677..a8522b96 100644
|
|
||||||
--- a/virtcontainers/sandbox.go
|
|
||||||
+++ b/virtcontainers/sandbox.go
|
|
||||||
@@ -1875,6 +1875,11 @@ func (s *Sandbox) updateResources() error {
|
|
||||||
reqMemMB := uint32(sandboxMemoryByte >> utils.MibToBytesShift)
|
|
||||||
currentMemMB := s.hypervisor.getMemorySize()
|
|
||||||
|
|
||||||
+ // check request sandbox memory size larger than utils.MaxMemorySizInMB or not
|
|
||||||
+ if reqMemMB > utils.MaxMemorySizeInMB {
|
|
||||||
+ return fmt.Errorf("updateResources failed! Sandbox memory should <= %d MiB", utils.MaxMemorySizeInMB)
|
|
||||||
+ }
|
|
||||||
+
|
|
||||||
// If request hotplug memory size larger than utils.MaxHotplugMemMBOnceTime,
|
|
||||||
// inorder to avoid hotplug memory oom problem, we need to hotplug large memory
|
|
||||||
// with two steps. First, hotplug utils.MaxHotplugMemMBOnceTime size memory into
|
|
||||||
@@ -1898,7 +1903,7 @@ func (s *Sandbox) updateResources() error {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
}
|
|
||||||
- if err := s.agent.onlineCPUMem(0, false, false); err != nil {
|
|
||||||
+ if err := s.agent.onlineCPUMem(0, false, true); err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
return nil
|
|
||||||
diff --git a/virtcontainers/utils/utils.go b/virtcontainers/utils/utils.go
|
|
||||||
index 3ae95aef..5d38e594 100644
|
|
||||||
--- a/virtcontainers/utils/utils.go
|
|
||||||
+++ b/virtcontainers/utils/utils.go
|
|
||||||
@@ -6,6 +6,7 @@
|
|
||||||
package utils
|
|
||||||
|
|
||||||
import (
|
|
||||||
+ "bufio"
|
|
||||||
"crypto/rand"
|
|
||||||
"errors"
|
|
||||||
"fmt"
|
|
||||||
@@ -28,11 +29,26 @@ const MibToBytesShift = 20
|
|
||||||
// Max Hotplug Memory size at once time, unit is MB
|
|
||||||
const MaxHotplugMemMBOnceTime = 32 * 1024
|
|
||||||
|
|
||||||
+const (
|
|
||||||
+ // Min needed memory size to start a Kata VM
|
|
||||||
+ MinMemorySizeInMB = 300
|
|
||||||
+ MinMemorySizeInByte = MinMemorySizeInMB << MibToBytesShift
|
|
||||||
+
|
|
||||||
+ // Max support memory size in the Kata VM
|
|
||||||
+ MaxMemorySizeInMB = 512 * 1024
|
|
||||||
+ MaxMemorySizeInByte = MaxMemorySizeInMB << MibToBytesShift
|
|
||||||
+)
|
|
||||||
+
|
|
||||||
// MaxSocketPathLen is the effective maximum Unix domain socket length.
|
|
||||||
//
|
|
||||||
// See unix(7).
|
|
||||||
const MaxSocketPathLen = 107
|
|
||||||
|
|
||||||
+const (
|
|
||||||
+ procCPUInfoPath = "/proc/cpuinfo"
|
|
||||||
+ processorIdentifier = "processor"
|
|
||||||
+)
|
|
||||||
+
|
|
||||||
// VHostVSockDevicePath path to vhost-vsock device
|
|
||||||
var VHostVSockDevicePath = "/dev/vhost-vsock"
|
|
||||||
|
|
||||||
@@ -324,3 +340,28 @@ func IsProcessRunning(pid int, processName string, sandboxID string) bool {
|
|
||||||
|
|
||||||
return false
|
|
||||||
}
|
|
||||||
+
|
|
||||||
+// GetPhysicalCPUNumber return the number of the CPUs in the physical machine
|
|
||||||
+func GetPhysicalCPUNumber() int {
|
|
||||||
+ f, err := os.Open(procCPUInfoPath)
|
|
||||||
+ if err != nil {
|
|
||||||
+ return 0
|
|
||||||
+ }
|
|
||||||
+ defer f.Close()
|
|
||||||
+
|
|
||||||
+ cpuNum := 0
|
|
||||||
+ s := bufio.NewScanner(f)
|
|
||||||
+ for s.Scan() {
|
|
||||||
+ if err := s.Err(); err != nil {
|
|
||||||
+ return 0
|
|
||||||
+ }
|
|
||||||
+
|
|
||||||
+ fields := strings.Fields(s.Text())
|
|
||||||
+ if len(fields) > 0 {
|
|
||||||
+ if fields[0] == processorIdentifier {
|
|
||||||
+ cpuNum++
|
|
||||||
+ }
|
|
||||||
+ }
|
|
||||||
+ }
|
|
||||||
+ return cpuNum
|
|
||||||
+}
|
|
||||||
--
|
|
||||||
2.14.3 (Apple Git-98)
|
|
||||||
|
|
||||||
@ -1,50 +0,0 @@
|
|||||||
From c785f8f744050155102664d56de5bfb55e91915d Mon Sep 17 00:00:00 2001
|
|
||||||
From: Evan Foster <efoster@adobe.com>
|
|
||||||
Date: Mon, 13 Jul 2020 12:53:40 -0600
|
|
||||||
Subject: [PATCH 18/50] sandbox: Stop and clean up containers that fail to
|
|
||||||
create
|
|
||||||
|
|
||||||
A container that is created and added to a sandbox can still fail
|
|
||||||
the final creation steps. In this case, the container must be stopped
|
|
||||||
and have its resources cleaned up to prevent leaking sandbox mounts.
|
|
||||||
|
|
||||||
Fixes #2816
|
|
||||||
|
|
||||||
cherry-pick from: https://github.com/kata-containers/runtime/pull/2826
|
|
||||||
|
|
||||||
Signed-off-by: Evan Foster <efoster@adobe.com>
|
|
||||||
Signed-off-by: jiangpengfei <jiangpengfei9@huawei.com>
|
|
||||||
---
|
|
||||||
virtcontainers/sandbox.go | 11 +++++++++++
|
|
||||||
1 file changed, 11 insertions(+)
|
|
||||||
|
|
||||||
diff --git a/virtcontainers/sandbox.go b/virtcontainers/sandbox.go
|
|
||||||
index a8522b96..3dbf640e 100644
|
|
||||||
--- a/virtcontainers/sandbox.go
|
|
||||||
+++ b/virtcontainers/sandbox.go
|
|
||||||
@@ -1,4 +1,5 @@
|
|
||||||
// Copyright (c) 2016 Intel Corporation
|
|
||||||
+// Copyright (c) 2020 Adobe Inc.
|
|
||||||
//
|
|
||||||
// SPDX-License-Identifier: Apache-2.0
|
|
||||||
//
|
|
||||||
@@ -1172,6 +1173,16 @@ func (s *Sandbox) CreateContainer(contConfig ContainerConfig) (VCContainer, erro
|
|
||||||
defer func() {
|
|
||||||
// Rollback if error happens.
|
|
||||||
if err != nil {
|
|
||||||
+ logger := s.Logger().WithFields(logrus.Fields{"container-id": c.id, "sandox-id": s.id, "rollback": true})
|
|
||||||
+
|
|
||||||
+ logger.Warning("Cleaning up partially created container")
|
|
||||||
+
|
|
||||||
+ if err2 := c.stop(true); err2 != nil {
|
|
||||||
+ logger.WithError(err2).Warning("Could not delete container")
|
|
||||||
+ }
|
|
||||||
+
|
|
||||||
+ logger.Debug("Removing stopped container from sandbox store")
|
|
||||||
+
|
|
||||||
s.removeContainer(c.id)
|
|
||||||
}
|
|
||||||
}()
|
|
||||||
--
|
|
||||||
2.14.3 (Apple Git-98)
|
|
||||||
|
|
||||||
@ -1,33 +0,0 @@
|
|||||||
From d20cb25c8a145e1d3e64eefa69242d87b7a67f92 Mon Sep 17 00:00:00 2001
|
|
||||||
From: jiangpengfei <jiangpengfei9@huawei.com>
|
|
||||||
Date: Sat, 8 Aug 2020 16:00:45 -0400
|
|
||||||
Subject: [PATCH 19/50] virtcontainers: fix delete sandbox failed problem
|
|
||||||
|
|
||||||
fixes: #2882
|
|
||||||
|
|
||||||
reason: If error happens after container create and before sandbox
|
|
||||||
updateResouce in the `CreateContainer()`, then delete sandbox
|
|
||||||
forcefully will return error because s.config.Containers config not
|
|
||||||
flushed into persist store.
|
|
||||||
|
|
||||||
Signed-off-by: jiangpengfei <jiangpengfei9@huawei.com>
|
|
||||||
---
|
|
||||||
virtcontainers/sandbox.go | 2 ++
|
|
||||||
1 file changed, 2 insertions(+)
|
|
||||||
|
|
||||||
diff --git a/virtcontainers/sandbox.go b/virtcontainers/sandbox.go
|
|
||||||
index 3dbf640e..7322ef58 100644
|
|
||||||
--- a/virtcontainers/sandbox.go
|
|
||||||
+++ b/virtcontainers/sandbox.go
|
|
||||||
@@ -1156,6 +1156,8 @@ func (s *Sandbox) CreateContainer(contConfig ContainerConfig) (VCContainer, erro
|
|
||||||
if len(s.config.Containers) > 0 {
|
|
||||||
// delete container config
|
|
||||||
s.config.Containers = s.config.Containers[:len(s.config.Containers)-1]
|
|
||||||
+ // need to flush change to persist storage
|
|
||||||
+ _ = s.storeSandbox()
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}()
|
|
||||||
--
|
|
||||||
2.14.3 (Apple Git-98)
|
|
||||||
|
|
||||||
@ -1,188 +0,0 @@
|
|||||||
From 75141529674545a2f84b01c730f614901ad2265e Mon Sep 17 00:00:00 2001
|
|
||||||
From: jiangpengfei <jiangpengfei9@huawei.com>
|
|
||||||
Date: Mon, 10 Aug 2020 10:08:47 +0800
|
|
||||||
Subject: [PATCH 20/50] virtcontainers: add enable_cpu_memory_hotplug config in
|
|
||||||
the configuration.toml
|
|
||||||
|
|
||||||
reason: add enable_cpu_memory_hotplug config to control whether enable hotplug
|
|
||||||
memory and cpu resource into the Kata VM. If we can calculate the whole sandbox
|
|
||||||
resource usage already, we just need to pass the value of resouces by annotation
|
|
||||||
without hotplugging.
|
|
||||||
|
|
||||||
Signed-off-by: jiangpengfei <jiangpengfei9@huawei.com>
|
|
||||||
---
|
|
||||||
cli/config/configuration-qemu.toml.in | 4 ++++
|
|
||||||
pkg/katautils/config.go | 2 ++
|
|
||||||
virtcontainers/container.go | 6 ++++--
|
|
||||||
virtcontainers/hypervisor.go | 3 +++
|
|
||||||
virtcontainers/persist.go | 2 ++
|
|
||||||
virtcontainers/persist/api/config.go | 3 +++
|
|
||||||
virtcontainers/sandbox.go | 38 +++++++++++++++++++++--------------
|
|
||||||
7 files changed, 41 insertions(+), 17 deletions(-)
|
|
||||||
|
|
||||||
diff --git a/cli/config/configuration-qemu.toml.in b/cli/config/configuration-qemu.toml.in
|
|
||||||
index 82461732..b44e84d8 100644
|
|
||||||
--- a/cli/config/configuration-qemu.toml.in
|
|
||||||
+++ b/cli/config/configuration-qemu.toml.in
|
|
||||||
@@ -100,6 +100,10 @@ default_memory = @DEFMEMSZ@
|
|
||||||
# Default false
|
|
||||||
#enable_reuse_cpu_memory = false
|
|
||||||
|
|
||||||
+# If enabled, the runtime will support cpu and memory hot plug
|
|
||||||
+# (default: disabled)
|
|
||||||
+#enable_cpu_memory_hotplug = true
|
|
||||||
+
|
|
||||||
# Disable block device from being used for a container's rootfs.
|
|
||||||
# In case of a storage driver like devicemapper where a container's
|
|
||||||
# root file system is backed by a block device, the block device is passed
|
|
||||||
diff --git a/pkg/katautils/config.go b/pkg/katautils/config.go
|
|
||||||
index b9b08469..9a99b9d4 100644
|
|
||||||
--- a/pkg/katautils/config.go
|
|
||||||
+++ b/pkg/katautils/config.go
|
|
||||||
@@ -112,6 +112,7 @@ type hypervisor struct {
|
|
||||||
MemSlots uint32 `toml:"memory_slots"`
|
|
||||||
MemOffset uint32 `toml:"memory_offset"`
|
|
||||||
EnableCPUMemoryReuse bool `toml:"enable_reuse_cpu_memory"`
|
|
||||||
+ EnableCPUMemoryHotPlug bool `toml:"enable_cpu_memory_hotplug"`
|
|
||||||
DefaultBridges uint32 `toml:"default_bridges"`
|
|
||||||
Msize9p uint32 `toml:"msize_9p"`
|
|
||||||
PCIeRootPort uint32 `toml:"pcie_root_port"`
|
|
||||||
@@ -675,6 +676,7 @@ func newQemuHypervisorConfig(h hypervisor) (vc.HypervisorConfig, error) {
|
|
||||||
MemSlots: h.defaultMemSlots(),
|
|
||||||
MemOffset: h.defaultMemOffset(),
|
|
||||||
EnableCPUMemoryReuse: h.EnableCPUMemoryReuse,
|
|
||||||
+ EnableCPUMemoryHotPlug: h.EnableCPUMemoryHotPlug,
|
|
||||||
VirtioMem: h.VirtioMem,
|
|
||||||
EntropySource: h.GetEntropySource(),
|
|
||||||
DefaultBridges: h.defaultBridges(),
|
|
||||||
diff --git a/virtcontainers/container.go b/virtcontainers/container.go
|
|
||||||
index 75f590eb..1b89f6ac 100644
|
|
||||||
--- a/virtcontainers/container.go
|
|
||||||
+++ b/virtcontainers/container.go
|
|
||||||
@@ -1273,8 +1273,10 @@ func (c *Container) update(resources specs.LinuxResources) error {
|
|
||||||
c.config.Resources.Memory.Limit = mem.Limit
|
|
||||||
}
|
|
||||||
|
|
||||||
- if err := c.sandbox.updateResources(); err != nil {
|
|
||||||
- return err
|
|
||||||
+ if c.sandbox.config.HypervisorConfig.EnableCPUMemoryHotPlug {
|
|
||||||
+ if err := c.sandbox.updateResources(); err != nil {
|
|
||||||
+ return err
|
|
||||||
+ }
|
|
||||||
}
|
|
||||||
|
|
||||||
if !c.sandbox.config.SandboxCgroupOnly {
|
|
||||||
diff --git a/virtcontainers/hypervisor.go b/virtcontainers/hypervisor.go
|
|
||||||
index 9cd685ad..c0723daa 100644
|
|
||||||
--- a/virtcontainers/hypervisor.go
|
|
||||||
+++ b/virtcontainers/hypervisor.go
|
|
||||||
@@ -253,6 +253,9 @@ type HypervisorConfig struct {
|
|
||||||
// Enable hypervisor cpu and memory resource share with container
|
|
||||||
EnableCPUMemoryReuse bool
|
|
||||||
|
|
||||||
+ // Enable hotplug cpu and memory resource into container
|
|
||||||
+ EnableCPUMemoryHotPlug bool
|
|
||||||
+
|
|
||||||
// VirtioFSCacheSize is the DAX cache size in MiB
|
|
||||||
VirtioFSCacheSize uint32
|
|
||||||
|
|
||||||
diff --git a/virtcontainers/persist.go b/virtcontainers/persist.go
|
|
||||||
index aef4d18d..6bd09a0b 100644
|
|
||||||
--- a/virtcontainers/persist.go
|
|
||||||
+++ b/virtcontainers/persist.go
|
|
||||||
@@ -215,6 +215,7 @@ func (s *Sandbox) dumpConfig(ss *persistapi.SandboxState) {
|
|
||||||
MemSlots: sconfig.HypervisorConfig.MemSlots,
|
|
||||||
MemOffset: sconfig.HypervisorConfig.MemOffset,
|
|
||||||
EnableCPUMemoryReuse: sconfig.HypervisorConfig.EnableCPUMemoryReuse,
|
|
||||||
+ EnableCPUMemoryHotPlug: sconfig.HypervisorConfig.EnableCPUMemoryHotPlug,
|
|
||||||
VirtioMem: sconfig.HypervisorConfig.VirtioMem,
|
|
||||||
VirtioFSCacheSize: sconfig.HypervisorConfig.VirtioFSCacheSize,
|
|
||||||
KernelPath: sconfig.HypervisorConfig.KernelPath,
|
|
||||||
@@ -505,6 +506,7 @@ func loadSandboxConfig(id string) (*SandboxConfig, error) {
|
|
||||||
MemSlots: hconf.MemSlots,
|
|
||||||
MemOffset: hconf.MemOffset,
|
|
||||||
EnableCPUMemoryReuse: hconf.EnableCPUMemoryReuse,
|
|
||||||
+ EnableCPUMemoryHotPlug: hconf.EnableCPUMemoryHotPlug,
|
|
||||||
VirtioMem: hconf.VirtioMem,
|
|
||||||
VirtioFSCacheSize: hconf.VirtioFSCacheSize,
|
|
||||||
KernelPath: hconf.KernelPath,
|
|
||||||
diff --git a/virtcontainers/persist/api/config.go b/virtcontainers/persist/api/config.go
|
|
||||||
index a3c6ec91..cfbee849 100644
|
|
||||||
--- a/virtcontainers/persist/api/config.go
|
|
||||||
+++ b/virtcontainers/persist/api/config.go
|
|
||||||
@@ -38,6 +38,9 @@ type HypervisorConfig struct {
|
|
||||||
// Enable hypervisor cpu and memory resource share with container
|
|
||||||
EnableCPUMemoryReuse bool
|
|
||||||
|
|
||||||
+ // Enable hotplug cpu and memory resource into container
|
|
||||||
+ EnableCPUMemoryHotPlug bool
|
|
||||||
+
|
|
||||||
// VirtioFSCacheSize is the DAX cache size in MiB
|
|
||||||
VirtioFSCacheSize uint32
|
|
||||||
|
|
||||||
diff --git a/virtcontainers/sandbox.go b/virtcontainers/sandbox.go
|
|
||||||
index 7322ef58..8fcd92d4 100644
|
|
||||||
--- a/virtcontainers/sandbox.go
|
|
||||||
+++ b/virtcontainers/sandbox.go
|
|
||||||
@@ -1189,12 +1189,16 @@ func (s *Sandbox) CreateContainer(contConfig ContainerConfig) (VCContainer, erro
|
|
||||||
}
|
|
||||||
}()
|
|
||||||
|
|
||||||
- // Sandbox is reponsable to update VM resources needed by Containers
|
|
||||||
- // Update resources after having added containers to the sandbox, since
|
|
||||||
- // container status is requiered to know if more resources should be added.
|
|
||||||
- err = s.updateResources()
|
|
||||||
- if err != nil {
|
|
||||||
- return nil, err
|
|
||||||
+ // update sandbox resource only when enable_cpu_memory_hotplug config set true
|
|
||||||
+ // in the config.toml file
|
|
||||||
+ if s.config.HypervisorConfig.EnableCPUMemoryHotPlug {
|
|
||||||
+ // Sandbox is reponsable to update VM resources needed by Containers
|
|
||||||
+ // Update resources after having added containers to the sandbox, since
|
|
||||||
+ // container status is requiered to know if more resources should be added.
|
|
||||||
+ err = s.updateResources()
|
|
||||||
+ if err != nil {
|
|
||||||
+ return nil, err
|
|
||||||
+ }
|
|
||||||
}
|
|
||||||
|
|
||||||
if err = s.cgroupsUpdate(); err != nil {
|
|
||||||
@@ -1228,11 +1232,13 @@ func (s *Sandbox) StartContainer(containerID string) (VCContainer, error) {
|
|
||||||
|
|
||||||
s.Logger().Info("Container is started")
|
|
||||||
|
|
||||||
- // Update sandbox resources in case a stopped container
|
|
||||||
- // is started
|
|
||||||
- err = s.updateResources()
|
|
||||||
- if err != nil {
|
|
||||||
- return nil, err
|
|
||||||
+ if s.config.HypervisorConfig.EnableCPUMemoryHotPlug {
|
|
||||||
+ // Update sandbox resources in case a stopped container
|
|
||||||
+ // is started
|
|
||||||
+ err = s.updateResources()
|
|
||||||
+ if err != nil {
|
|
||||||
+ return nil, err
|
|
||||||
+ }
|
|
||||||
}
|
|
||||||
|
|
||||||
return c, nil
|
|
||||||
@@ -1503,10 +1509,12 @@ func (s *Sandbox) createContainers() error {
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
- // Update resources after having added containers to the sandbox, since
|
|
||||||
- // container status is requiered to know if more resources should be added.
|
|
||||||
- if err := s.updateResources(); err != nil {
|
|
||||||
- return err
|
|
||||||
+ if s.config.HypervisorConfig.EnableCPUMemoryHotPlug {
|
|
||||||
+ // Update resources after having added containers to the sandbox, since
|
|
||||||
+ // container status is requiered to know if more resources should be added.
|
|
||||||
+ if err := s.updateResources(); err != nil {
|
|
||||||
+ return err
|
|
||||||
+ }
|
|
||||||
}
|
|
||||||
|
|
||||||
if err := s.cgroupsUpdate(); err != nil {
|
|
||||||
--
|
|
||||||
2.14.3 (Apple Git-98)
|
|
||||||
|
|
||||||
@ -1,195 +0,0 @@
|
|||||||
From 79cf2f5a52af51d8a62353a99e894808281769e2 Mon Sep 17 00:00:00 2001
|
|
||||||
From: jiangpengfei <jiangpengfei9@huawei.com>
|
|
||||||
Date: Mon, 10 Aug 2020 11:25:02 +0800
|
|
||||||
Subject: [PATCH 21/50] kata-runtime: add sandbox_cpu and sandbox_mem
|
|
||||||
annotations
|
|
||||||
|
|
||||||
reason: add sandbox_cpu and sandbox_men annotations to set
|
|
||||||
Kata VM vCPU number and memory size.
|
|
||||||
|
|
||||||
Signed-off-by: jiangpengfei <jiangpengfei9@huawei.com>
|
|
||||||
---
|
|
||||||
virtcontainers/pkg/annotations/annotations.go | 6 +++++
|
|
||||||
virtcontainers/pkg/oci/utils.go | 39 +++++++++++++++++++++++++--
|
|
||||||
virtcontainers/sandbox.go | 32 ++++++++++++++++++++++
|
|
||||||
virtcontainers/utils/utils.go | 14 ++++++++++
|
|
||||||
4 files changed, 89 insertions(+), 2 deletions(-)
|
|
||||||
|
|
||||||
diff --git a/virtcontainers/pkg/annotations/annotations.go b/virtcontainers/pkg/annotations/annotations.go
|
|
||||||
index 10ce7833..903c7f03 100644
|
|
||||||
--- a/virtcontainers/pkg/annotations/annotations.go
|
|
||||||
+++ b/virtcontainers/pkg/annotations/annotations.go
|
|
||||||
@@ -251,6 +251,12 @@ const (
|
|
||||||
ContainerPipeSizeKernelParam = "agent." + ContainerPipeSizeOption
|
|
||||||
)
|
|
||||||
|
|
||||||
+// iSula self defined annotations
|
|
||||||
+const (
|
|
||||||
+ StaticCPUTypeKey = kataAnnotationsPrefix + "sandbox_cpu"
|
|
||||||
+ StaticMemTypeKey = kataAnnotationsPrefix + "sandbox_mem"
|
|
||||||
+)
|
|
||||||
+
|
|
||||||
const (
|
|
||||||
// SHA512 is the SHA-512 (64) hash algorithm
|
|
||||||
SHA512 string = "sha512"
|
|
||||||
diff --git a/virtcontainers/pkg/oci/utils.go b/virtcontainers/pkg/oci/utils.go
|
|
||||||
index 0a6f08c7..36c730b7 100644
|
|
||||||
--- a/virtcontainers/pkg/oci/utils.go
|
|
||||||
+++ b/virtcontainers/pkg/oci/utils.go
|
|
||||||
@@ -16,6 +16,7 @@ import (
|
|
||||||
|
|
||||||
criContainerdAnnotations "github.com/containerd/cri-containerd/pkg/annotations"
|
|
||||||
crioAnnotations "github.com/cri-o/cri-o/pkg/annotations"
|
|
||||||
+ "github.com/docker/go-units"
|
|
||||||
vc "github.com/kata-containers/runtime/virtcontainers"
|
|
||||||
"github.com/kata-containers/runtime/virtcontainers/device/config"
|
|
||||||
exp "github.com/kata-containers/runtime/virtcontainers/experimental"
|
|
||||||
@@ -34,7 +35,10 @@ type annotationContainerType struct {
|
|
||||||
|
|
||||||
type annotationHandler func(value string) error
|
|
||||||
|
|
||||||
-var annotationHandlerList = map[string]annotationHandler{}
|
|
||||||
+var annotationHandlerList = map[string]annotationHandler{
|
|
||||||
+ vcAnnotations.StaticCPUTypeKey: validateSandboxCPU,
|
|
||||||
+ vcAnnotations.StaticMemTypeKey: validateSandboxMem,
|
|
||||||
+}
|
|
||||||
|
|
||||||
var (
|
|
||||||
// ErrNoLinux is an error for missing Linux sections in the OCI configuration file.
|
|
||||||
@@ -1036,7 +1040,10 @@ func validateOtherSandboxAnnotations(annotation, value string) error {
|
|
||||||
|
|
||||||
// addOtherSandboxAnnotation add self defined annotation for sandbox
|
|
||||||
func addOtherSandboxAnnotation(ocispec specs.Spec, sbConfig *vc.SandboxConfig) error {
|
|
||||||
- otherSandboxAnnotationsKey := []string{}
|
|
||||||
+ otherSandboxAnnotationsKey := []string{
|
|
||||||
+ vcAnnotations.StaticCPUTypeKey,
|
|
||||||
+ vcAnnotations.StaticMemTypeKey,
|
|
||||||
+ }
|
|
||||||
|
|
||||||
for _, a := range otherSandboxAnnotationsKey {
|
|
||||||
value, ok := ocispec.Annotations[a]
|
|
||||||
@@ -1053,3 +1060,31 @@ func addOtherSandboxAnnotation(ocispec specs.Spec, sbConfig *vc.SandboxConfig) e
|
|
||||||
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
+
|
|
||||||
+func validateSandboxCPU(value string) error {
|
|
||||||
+ // check min cpu value
|
|
||||||
+ cpus, err := utils.RoundVCPUNumber(value)
|
|
||||||
+ if err != nil {
|
|
||||||
+ return fmt.Errorf("valiate sandbox_cpu annotation fail: %v", err)
|
|
||||||
+ }
|
|
||||||
+
|
|
||||||
+ maxPhysicalCPUs := utils.GetPhysicalCPUNumber()
|
|
||||||
+ if cpus > maxPhysicalCPUs {
|
|
||||||
+ return fmt.Errorf("sandbox_cpu annotation value exceed the machine max CPU number: %d", cpus)
|
|
||||||
+ }
|
|
||||||
+
|
|
||||||
+ return nil
|
|
||||||
+}
|
|
||||||
+
|
|
||||||
+func validateSandboxMem(value string) error {
|
|
||||||
+ memSizeInBytes, err := units.RAMInBytes(value)
|
|
||||||
+ if err != nil {
|
|
||||||
+ return fmt.Errorf("parse sandbox_mem value: %d fail: %v", memSizeInBytes, err)
|
|
||||||
+ }
|
|
||||||
+
|
|
||||||
+ if memSizeInBytes < utils.MinMemorySizeInByte || memSizeInBytes > utils.MaxMemorySizeInByte {
|
|
||||||
+ return fmt.Errorf("invalid sandbox_mem value size in bytes: %v", memSizeInBytes)
|
|
||||||
+ }
|
|
||||||
+
|
|
||||||
+ return nil
|
|
||||||
+}
|
|
||||||
diff --git a/virtcontainers/sandbox.go b/virtcontainers/sandbox.go
|
|
||||||
index 8fcd92d4..ba704249 100644
|
|
||||||
--- a/virtcontainers/sandbox.go
|
|
||||||
+++ b/virtcontainers/sandbox.go
|
|
||||||
@@ -20,6 +20,7 @@ import (
|
|
||||||
|
|
||||||
"github.com/containerd/cgroups"
|
|
||||||
"github.com/containernetworking/plugins/pkg/ns"
|
|
||||||
+ "github.com/docker/go-units"
|
|
||||||
"github.com/kata-containers/agent/protocols/grpc"
|
|
||||||
"github.com/kata-containers/runtime/virtcontainers/device/api"
|
|
||||||
"github.com/kata-containers/runtime/virtcontainers/device/config"
|
|
||||||
@@ -479,6 +480,10 @@ func createSandbox(ctx context.Context, sandboxConfig SandboxConfig, factory Fac
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
|
|
||||||
+ if err := updateStaticSandboxResources(&sandboxConfig); err != nil {
|
|
||||||
+ return nil, err
|
|
||||||
+ }
|
|
||||||
+
|
|
||||||
s, err := newSandbox(ctx, sandboxConfig, factory)
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
@@ -2359,3 +2364,30 @@ func (s *Sandbox) setContainersState(state types.StateString) error {
|
|
||||||
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
+
|
|
||||||
+// updateStaticSandboxResources update sandbox's cpu and memory resource passed by
|
|
||||||
+// sandbox_cpu and sandbox_mem annotations
|
|
||||||
+func updateStaticSandboxResources(sandboxConfig *SandboxConfig) error {
|
|
||||||
+ // update cpu resource
|
|
||||||
+ if cpuNumVal, ok := sandboxConfig.Annotations[annotations.StaticCPUTypeKey]; ok {
|
|
||||||
+ cpuNum, err := utils.RoundVCPUNumber(cpuNumVal)
|
|
||||||
+ if err != nil {
|
|
||||||
+ return err
|
|
||||||
+ }
|
|
||||||
+
|
|
||||||
+ sandboxConfig.HypervisorConfig.NumVCPUs = (uint32)(cpuNum)
|
|
||||||
+ }
|
|
||||||
+
|
|
||||||
+ // update mem resource
|
|
||||||
+ if memVal, ok := sandboxConfig.Annotations[annotations.StaticMemTypeKey]; ok {
|
|
||||||
+ memSizeInBytes, err := units.RAMInBytes(memVal)
|
|
||||||
+ if err != nil {
|
|
||||||
+ return err
|
|
||||||
+ }
|
|
||||||
+
|
|
||||||
+ memSizeInMB := memSizeInBytes >> utils.MibToBytesShift
|
|
||||||
+ sandboxConfig.HypervisorConfig.MemorySize = (uint32)(memSizeInMB)
|
|
||||||
+ }
|
|
||||||
+
|
|
||||||
+ return nil
|
|
||||||
+}
|
|
||||||
diff --git a/virtcontainers/utils/utils.go b/virtcontainers/utils/utils.go
|
|
||||||
index 5d38e594..9490faa1 100644
|
|
||||||
--- a/virtcontainers/utils/utils.go
|
|
||||||
+++ b/virtcontainers/utils/utils.go
|
|
||||||
@@ -11,6 +11,7 @@ import (
|
|
||||||
"errors"
|
|
||||||
"fmt"
|
|
||||||
"io/ioutil"
|
|
||||||
+ "math"
|
|
||||||
"os"
|
|
||||||
"os/exec"
|
|
||||||
"path/filepath"
|
|
||||||
@@ -30,6 +31,9 @@ const MibToBytesShift = 20
|
|
||||||
const MaxHotplugMemMBOnceTime = 32 * 1024
|
|
||||||
|
|
||||||
const (
|
|
||||||
+ // minCPUs is allowed minimum CPU
|
|
||||||
+ minCPUs = 0.25
|
|
||||||
+
|
|
||||||
// Min needed memory size to start a Kata VM
|
|
||||||
MinMemorySizeInMB = 300
|
|
||||||
MinMemorySizeInByte = MinMemorySizeInMB << MibToBytesShift
|
|
||||||
@@ -365,3 +369,13 @@ func GetPhysicalCPUNumber() int {
|
|
||||||
}
|
|
||||||
return cpuNum
|
|
||||||
}
|
|
||||||
+
|
|
||||||
+func RoundVCPUNumber(value string) (int, error) {
|
|
||||||
+ cpuNum, err := strconv.ParseFloat(value, 64)
|
|
||||||
+ if err != nil || cpuNum < minCPUs {
|
|
||||||
+ return 0, fmt.Errorf("invalid sandbox cpu number: %v", cpuNum)
|
|
||||||
+ }
|
|
||||||
+
|
|
||||||
+ cpus := int(math.Ceil(cpuNum))
|
|
||||||
+ return cpus, nil
|
|
||||||
+}
|
|
||||||
--
|
|
||||||
2.14.3 (Apple Git-98)
|
|
||||||
|
|
||||||
@ -1,39 +0,0 @@
|
|||||||
From e5e3232f7268110f7e3e3c4814eab31a6704b672 Mon Sep 17 00:00:00 2001
|
|
||||||
From: jiangpengfei <jiangpengfei9@huawei.com>
|
|
||||||
Date: Mon, 10 Aug 2020 20:11:07 +0800
|
|
||||||
Subject: [PATCH 22/50] kata-runtime: skip go version check and do not build
|
|
||||||
containerd-shim-v2
|
|
||||||
|
|
||||||
reason: skip go version check and do not build containerd-shim-v2
|
|
||||||
because iSulad current not support shimV2
|
|
||||||
|
|
||||||
Signed-off-by: jiangpengfei <jiangpengfei9@huawei.com>
|
|
||||||
---
|
|
||||||
Makefile | 4 ++--
|
|
||||||
1 file changed, 2 insertions(+), 2 deletions(-)
|
|
||||||
|
|
||||||
diff --git a/Makefile b/Makefile
|
|
||||||
index 14a0ea47..d5e4bbe1 100644
|
|
||||||
--- a/Makefile
|
|
||||||
+++ b/Makefile
|
|
||||||
@@ -12,7 +12,7 @@ for file in /etc/os-release /usr/lib/os-release; do \
|
|
||||||
fi \
|
|
||||||
done)
|
|
||||||
|
|
||||||
-SKIP_GO_VERSION_CHECK=
|
|
||||||
+SKIP_GO_VERSION_CHECK=y
|
|
||||||
include golang.mk
|
|
||||||
|
|
||||||
#Get ARCH.
|
|
||||||
@@ -503,7 +503,7 @@ define SHOW_ARCH
|
|
||||||
$(shell printf "\\t%s%s\\\n" "$(1)" $(if $(filter $(ARCH),$(1))," (default)",""))
|
|
||||||
endef
|
|
||||||
|
|
||||||
-all: runtime containerd-shim-v2 netmon
|
|
||||||
+all: runtime netmon
|
|
||||||
|
|
||||||
# Targets that depend on .git-commit can use $(shell cat .git-commit) to get a
|
|
||||||
# git revision string. They will only be rebuilt if the revision string
|
|
||||||
--
|
|
||||||
2.14.3 (Apple Git-98)
|
|
||||||
|
|
||||||
@ -1,42 +0,0 @@
|
|||||||
From 226c3336dcc70bd17e3471ff98106a2f8dee9ac5 Mon Sep 17 00:00:00 2001
|
|
||||||
From: jiangpengfei <jiangpengfei9@huawei.com>
|
|
||||||
Date: Tue, 11 Aug 2020 22:32:49 +0800
|
|
||||||
Subject: [PATCH 23/50] kata-runtime: set PCIBridgeMaxCapacity limit to 25
|
|
||||||
|
|
||||||
reason: set PCIBridgeMaxCapacity limit to 25.
|
|
||||||
|
|
||||||
Signed-off-by: jiangpengfei <jiangpengfei9@huawei.com>
|
|
||||||
---
|
|
||||||
virtcontainers/qemu_arch_base_test.go | 2 +-
|
|
||||||
virtcontainers/types/bridges.go | 2 +-
|
|
||||||
2 files changed, 2 insertions(+), 2 deletions(-)
|
|
||||||
|
|
||||||
diff --git a/virtcontainers/qemu_arch_base_test.go b/virtcontainers/qemu_arch_base_test.go
|
|
||||||
index 169e002e..8219f3c5 100644
|
|
||||||
--- a/virtcontainers/qemu_arch_base_test.go
|
|
||||||
+++ b/virtcontainers/qemu_arch_base_test.go
|
|
||||||
@@ -175,7 +175,7 @@ func TestQemuAddDeviceToBridge(t *testing.T) {
|
|
||||||
}
|
|
||||||
|
|
||||||
// fail to add device to bridge cause no more available bridge slot
|
|
||||||
- _, _, err := q.addDeviceToBridge("qemu-bridge-31", types.PCI)
|
|
||||||
+ _, _, err := q.addDeviceToBridge("qemu-bridge-26", types.PCI)
|
|
||||||
exceptErr := errors.New("no more bridge slots available")
|
|
||||||
assert.Equal(exceptErr.Error(), err.Error())
|
|
||||||
|
|
||||||
diff --git a/virtcontainers/types/bridges.go b/virtcontainers/types/bridges.go
|
|
||||||
index cb15a88f..c3538ce4 100644
|
|
||||||
--- a/virtcontainers/types/bridges.go
|
|
||||||
+++ b/virtcontainers/types/bridges.go
|
|
||||||
@@ -10,7 +10,7 @@ import "fmt"
|
|
||||||
// Type represents a type of bus and bridge.
|
|
||||||
type Type string
|
|
||||||
|
|
||||||
-const PCIBridgeMaxCapacity = 30
|
|
||||||
+const PCIBridgeMaxCapacity = 25
|
|
||||||
|
|
||||||
const (
|
|
||||||
// PCI represents a PCI bus and bridge
|
|
||||||
--
|
|
||||||
2.14.3 (Apple Git-98)
|
|
||||||
|
|
||||||
@ -1,462 +0,0 @@
|
|||||||
From e861f426c9e6702e820348ddc61b18013c853402 Mon Sep 17 00:00:00 2001
|
|
||||||
From: jiangpengfei <jiangpengfei9@huawei.com>
|
|
||||||
Date: Thu, 13 Aug 2020 11:24:58 +0800
|
|
||||||
Subject: [PATCH 24/50] kata-runtime: support hotplug tap interface into kata
|
|
||||||
VM
|
|
||||||
|
|
||||||
reason: support hotplug exist tap interface or a new created tap
|
|
||||||
interface into kata VM.
|
|
||||||
|
|
||||||
Signed-off-by: jiangpengfei <jiangpengfei9@huawei.com>
|
|
||||||
---
|
|
||||||
cli/network.go | 45 ++++++++++++++---
|
|
||||||
virtcontainers/kata_agent.go | 12 ++++-
|
|
||||||
virtcontainers/network.go | 100 +++++++++++++++++++++++---------------
|
|
||||||
virtcontainers/pkg/types/types.go | 16 +++---
|
|
||||||
virtcontainers/qemu.go | 11 +++--
|
|
||||||
virtcontainers/sandbox.go | 24 +++++++--
|
|
||||||
virtcontainers/tap_endpoint.go | 23 +++++++--
|
|
||||||
7 files changed, 161 insertions(+), 70 deletions(-)
|
|
||||||
|
|
||||||
diff --git a/cli/network.go b/cli/network.go
|
|
||||||
index 881a2358..7e7791f1 100644
|
|
||||||
--- a/cli/network.go
|
|
||||||
+++ b/cli/network.go
|
|
||||||
@@ -26,6 +26,8 @@ const (
|
|
||||||
routeType
|
|
||||||
)
|
|
||||||
|
|
||||||
+const defaultLinkType = "tap"
|
|
||||||
+
|
|
||||||
var kataNetworkCLICommand = cli.Command{
|
|
||||||
Name: "kata-network",
|
|
||||||
Usage: "manage interfaces and routes for container",
|
|
||||||
@@ -42,10 +44,22 @@ var kataNetworkCLICommand = cli.Command{
|
|
||||||
}
|
|
||||||
|
|
||||||
var addIfaceCommand = cli.Command{
|
|
||||||
- Name: "add-iface",
|
|
||||||
- Usage: "add an interface to a container",
|
|
||||||
- ArgsUsage: `add-iface <container-id> file or - for stdin`,
|
|
||||||
- Flags: []cli.Flag{},
|
|
||||||
+ Name: "add-iface",
|
|
||||||
+ Usage: "add an interface to a container",
|
|
||||||
+ ArgsUsage: `add-iface <container-id> file or - for stdin
|
|
||||||
+ file or stdin for example:
|
|
||||||
+ {
|
|
||||||
+ "device":"<device-name>",
|
|
||||||
+ "name":"<interface-name>",
|
|
||||||
+ "IPAddresses":[{"address":"<ip>","mask":"<mask>"}],
|
|
||||||
+ "mtu":<mtu>,
|
|
||||||
+ "hwAddr":"<mac>",
|
|
||||||
+ "linkType":"tap",
|
|
||||||
+ "vhostUserSocket":"<path>"
|
|
||||||
+ }
|
|
||||||
+ device,name,mtu,hwAddr are required, IPAddresses and vhostUserSocket are optional.
|
|
||||||
+ `,
|
|
||||||
+ Flags: []cli.Flag{},
|
|
||||||
Action: func(context *cli.Context) error {
|
|
||||||
ctx, err := cliContextToContext(context)
|
|
||||||
if err != nil {
|
|
||||||
@@ -57,10 +71,20 @@ var addIfaceCommand = cli.Command{
|
|
||||||
}
|
|
||||||
|
|
||||||
var delIfaceCommand = cli.Command{
|
|
||||||
- Name: "del-iface",
|
|
||||||
- Usage: "delete an interface from a container",
|
|
||||||
- ArgsUsage: `del-iface <container-id> file or - for stdin`,
|
|
||||||
- Flags: []cli.Flag{},
|
|
||||||
+ Name: "del-iface",
|
|
||||||
+ Usage: "delete an interface from a container",
|
|
||||||
+ ArgsUsage: `del-iface <container-id> file or - for stdin
|
|
||||||
+ file or stdin for example:
|
|
||||||
+ {
|
|
||||||
+ "device":"",
|
|
||||||
+ "name":"<interface-name>",
|
|
||||||
+ "IPAddresses":[],
|
|
||||||
+ "mtu":0,
|
|
||||||
+ "hwAddr":""
|
|
||||||
+ }
|
|
||||||
+ Only the "name" field is required.
|
|
||||||
+ `,
|
|
||||||
+ Flags: []cli.Flag{},
|
|
||||||
Action: func(context *cli.Context) error {
|
|
||||||
ctx, err := cliContextToContext(context)
|
|
||||||
if err != nil {
|
|
||||||
@@ -156,6 +180,11 @@ func networkModifyCommand(ctx context.Context, containerID, input string, opType
|
|
||||||
if err = json.NewDecoder(f).Decode(&inf); err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
+
|
|
||||||
+ if len(inf.LinkType) == 0 {
|
|
||||||
+ inf.LinkType = defaultLinkType
|
|
||||||
+ }
|
|
||||||
+
|
|
||||||
if add {
|
|
||||||
resultingInf, err = vci.AddInterface(ctx, sandboxID, inf)
|
|
||||||
if err != nil {
|
|
||||||
diff --git a/virtcontainers/kata_agent.go b/virtcontainers/kata_agent.go
|
|
||||||
index 8e073339..dfdd263b 100644
|
|
||||||
--- a/virtcontainers/kata_agent.go
|
|
||||||
+++ b/virtcontainers/kata_agent.go
|
|
||||||
@@ -600,8 +600,16 @@ func (k *kataAgent) updateInterface(ifc *vcTypes.Interface) (*vcTypes.Interface,
|
|
||||||
"resulting-interface": fmt.Sprintf("%+v", resultingInterface),
|
|
||||||
}).WithError(err).Error("update interface request failed")
|
|
||||||
}
|
|
||||||
- if resultInterface, ok := resultingInterface.(*vcTypes.Interface); ok {
|
|
||||||
- return resultInterface, err
|
|
||||||
+ if resultInterface, ok := resultingInterface.(*aTypes.Interface); ok {
|
|
||||||
+ iface := &vcTypes.Interface{
|
|
||||||
+ Device: resultInterface.Device,
|
|
||||||
+ Name: resultInterface.Name,
|
|
||||||
+ IPAddresses: k.convertToIPAddresses(resultInterface.IPAddresses),
|
|
||||||
+ Mtu: resultInterface.Mtu,
|
|
||||||
+ HwAddr: resultInterface.HwAddr,
|
|
||||||
+ PciAddr: resultInterface.PciAddr,
|
|
||||||
+ }
|
|
||||||
+ return iface, err
|
|
||||||
}
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
diff --git a/virtcontainers/network.go b/virtcontainers/network.go
|
|
||||||
index d70c5360..e909a822 100644
|
|
||||||
--- a/virtcontainers/network.go
|
|
||||||
+++ b/virtcontainers/network.go
|
|
||||||
@@ -117,6 +117,7 @@ type NetlinkIface struct {
|
|
||||||
// NetworkInfo gathers all information related to a network interface.
|
|
||||||
// It can be used to store the description of the underlying network.
|
|
||||||
type NetworkInfo struct {
|
|
||||||
+ Device string
|
|
||||||
Iface NetlinkIface
|
|
||||||
Addrs []netlink.Addr
|
|
||||||
Routes []netlink.Route
|
|
||||||
@@ -303,6 +304,20 @@ func createLink(netHandle *netlink.Handle, name string, expectedLink netlink.Lin
|
|
||||||
var newLink netlink.Link
|
|
||||||
var fds []*os.File
|
|
||||||
|
|
||||||
+ // check if tapname exists, if true, use existing one instead of create new one
|
|
||||||
+ if name != "" {
|
|
||||||
+ retLink, err := netlink.LinkByName(name)
|
|
||||||
+ // link exist, use it
|
|
||||||
+ if err == nil {
|
|
||||||
+ networkLogger().Debugf("exist tap device is found, instead of creating new one")
|
|
||||||
+ tuntapLink, ok := retLink.(*netlink.Tuntap)
|
|
||||||
+ if ok {
|
|
||||||
+ fds = tuntapLink.Fds
|
|
||||||
+ }
|
|
||||||
+ return retLink, nil, nil
|
|
||||||
+ }
|
|
||||||
+ }
|
|
||||||
+
|
|
||||||
switch expectedLink.Type() {
|
|
||||||
case (&netlink.Tuntap{}).Type():
|
|
||||||
flags := netlink.TUNTAP_VNET_HDR
|
|
||||||
@@ -1156,7 +1171,7 @@ func createEndpoint(netInfo NetworkInfo, idx int, model NetInterworkingModel, li
|
|
||||||
|
|
||||||
// Check if interface is a physical interface. Do not create
|
|
||||||
// tap interface/bridge if it is.
|
|
||||||
- isPhysical, err := isPhysicalIface(netInfo.Iface.Name)
|
|
||||||
+ isPhysical, err := isPhysicalIface(netInfo.Device)
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
@@ -1164,49 +1179,54 @@ func createEndpoint(netInfo NetworkInfo, idx int, model NetInterworkingModel, li
|
|
||||||
if isPhysical {
|
|
||||||
networkLogger().WithField("interface", netInfo.Iface.Name).Info("Physical network interface found")
|
|
||||||
endpoint, err = createPhysicalEndpoint(netInfo)
|
|
||||||
- } else {
|
|
||||||
- var socketPath string
|
|
||||||
+ return endpoint, err
|
|
||||||
+ }
|
|
||||||
|
|
||||||
- // Check if this is a dummy interface which has a vhost-user socket associated with it
|
|
||||||
- socketPath, err = vhostUserSocketPath(netInfo)
|
|
||||||
- if err != nil {
|
|
||||||
- return nil, err
|
|
||||||
- }
|
|
||||||
+ // Check if this is a dummy interface which has a vhost-user socket associated with it
|
|
||||||
+ socketPath, err := vhostUserSocketPath(netInfo)
|
|
||||||
+ if err != nil {
|
|
||||||
+ return nil, err
|
|
||||||
+ }
|
|
||||||
|
|
||||||
- if socketPath != "" {
|
|
||||||
- networkLogger().WithField("interface", netInfo.Iface.Name).Info("VhostUser network interface found")
|
|
||||||
- endpoint, err = createVhostUserEndpoint(netInfo, socketPath)
|
|
||||||
- } else if netInfo.Iface.Type == "macvlan" {
|
|
||||||
- networkLogger().Infof("macvlan interface found")
|
|
||||||
- endpoint, err = createBridgedMacvlanNetworkEndpoint(idx, netInfo.Iface.Name, model)
|
|
||||||
- } else if netInfo.Iface.Type == "macvtap" {
|
|
||||||
- networkLogger().Infof("macvtap interface found")
|
|
||||||
- endpoint, err = createMacvtapNetworkEndpoint(netInfo)
|
|
||||||
- } else if netInfo.Iface.Type == "tap" {
|
|
||||||
- networkLogger().Info("tap interface found")
|
|
||||||
- endpoint, err = createTapNetworkEndpoint(idx, netInfo.Iface.Name)
|
|
||||||
- } else if netInfo.Iface.Type == "tuntap" {
|
|
||||||
- if link != nil {
|
|
||||||
- switch link.(*netlink.Tuntap).Mode {
|
|
||||||
- case 0:
|
|
||||||
- // mount /sys/class/net to get links
|
|
||||||
- return nil, fmt.Errorf("Network device mode not determined correctly. Mount sysfs in caller")
|
|
||||||
- case 1:
|
|
||||||
- return nil, fmt.Errorf("tun networking device not yet supported")
|
|
||||||
- case 2:
|
|
||||||
- networkLogger().Info("tuntap tap interface found")
|
|
||||||
- endpoint, err = createTuntapNetworkEndpoint(idx, netInfo.Iface.Name, netInfo.Iface.HardwareAddr, model)
|
|
||||||
- default:
|
|
||||||
- return nil, fmt.Errorf("tuntap network %v mode unsupported", link.(*netlink.Tuntap).Mode)
|
|
||||||
- }
|
|
||||||
+ if socketPath != "" {
|
|
||||||
+ networkLogger().WithField("interface", netInfo.Iface.Name).Info("VhostUser network interface found")
|
|
||||||
+ endpoint, err = createVhostUserEndpoint(netInfo, socketPath)
|
|
||||||
+ return endpoint, err
|
|
||||||
+ }
|
|
||||||
+
|
|
||||||
+ // We should create tap interface/bridge of other interface type.
|
|
||||||
+ networkLogger().Infof("%s interface found", netInfo.Iface.Type)
|
|
||||||
+ switch netInfo.Iface.Type {
|
|
||||||
+ case "macvlan":
|
|
||||||
+ networkLogger().Infof("macvlan interface found")
|
|
||||||
+ endpoint, err = createBridgedMacvlanNetworkEndpoint(idx, netInfo.Iface.Name, model)
|
|
||||||
+ case "macvtap":
|
|
||||||
+ networkLogger().Infof("macvtap interface found")
|
|
||||||
+ endpoint, err = createMacvtapNetworkEndpoint(netInfo)
|
|
||||||
+ case "tap":
|
|
||||||
+ networkLogger().Info("tap interface found")
|
|
||||||
+ endpoint, err = createTapNetworkEndpoint(idx, netInfo.Iface.Name, netInfo.Device)
|
|
||||||
+ case "tuntap":
|
|
||||||
+ if link != nil {
|
|
||||||
+ switch link.(*netlink.Tuntap).Mode {
|
|
||||||
+ case 0:
|
|
||||||
+ // mount /sys/class/net to get links
|
|
||||||
+ return nil, fmt.Errorf("Network device mode not determined correctly. Mount sysfs in caller")
|
|
||||||
+ case 1:
|
|
||||||
+ return nil, fmt.Errorf("tun networking device not yet supported")
|
|
||||||
+ case 2:
|
|
||||||
+ networkLogger().Info("tuntap tap interface found")
|
|
||||||
+ endpoint, err = createTuntapNetworkEndpoint(idx, netInfo.Iface.Name, netInfo.Iface.HardwareAddr, model)
|
|
||||||
+ default:
|
|
||||||
+ return nil, fmt.Errorf("tuntap network %v mode unsupported", link.(*netlink.Tuntap).Mode)
|
|
||||||
}
|
|
||||||
- } else if netInfo.Iface.Type == "veth" {
|
|
||||||
- endpoint, err = createVethNetworkEndpoint(idx, netInfo.Iface.Name, model)
|
|
||||||
- } else if netInfo.Iface.Type == "ipvlan" {
|
|
||||||
- endpoint, err = createIPVlanNetworkEndpoint(idx, netInfo.Iface.Name)
|
|
||||||
- } else {
|
|
||||||
- return nil, fmt.Errorf("Unsupported network interface: %s", netInfo.Iface.Type)
|
|
||||||
}
|
|
||||||
+ case "veth":
|
|
||||||
+ endpoint, err = createVethNetworkEndpoint(idx, netInfo.Iface.Name, model)
|
|
||||||
+ case "ipvlan":
|
|
||||||
+ endpoint, err = createIPVlanNetworkEndpoint(idx, netInfo.Iface.Name)
|
|
||||||
+ default:
|
|
||||||
+ err = fmt.Errorf("Unsupported network interface, %s", netInfo.Iface.Type)
|
|
||||||
}
|
|
||||||
|
|
||||||
return endpoint, err
|
|
||||||
diff --git a/virtcontainers/pkg/types/types.go b/virtcontainers/pkg/types/types.go
|
|
||||||
index 0d4a9cfa..fcc63d84 100644
|
|
||||||
--- a/virtcontainers/pkg/types/types.go
|
|
||||||
+++ b/virtcontainers/pkg/types/types.go
|
|
||||||
@@ -14,21 +14,21 @@ type IPAddress struct {
|
|
||||||
|
|
||||||
// Interface describes a network interface.
|
|
||||||
type Interface struct {
|
|
||||||
- Device string
|
|
||||||
- Name string
|
|
||||||
- IPAddresses []*IPAddress
|
|
||||||
- Mtu uint64
|
|
||||||
- RawFlags uint32
|
|
||||||
- HwAddr string
|
|
||||||
+ Device string `json:"device,omitempty"`
|
|
||||||
+ Name string `json:"name,omitempty"`
|
|
||||||
+ IPAddresses []*IPAddress `json:"IPAddresses,omitempty"`
|
|
||||||
+ Mtu uint64 `json:"mtu,omitempty"`
|
|
||||||
+ RawFlags uint32 `json:"rawFlags,omitempty"`
|
|
||||||
+ HwAddr string `json:"hwAddr,omitempty"`
|
|
||||||
// pciAddr is the PCI address in the format "bridgeAddr/deviceAddr".
|
|
||||||
// Here, bridgeAddr is the address at which the bridge is attached on the root bus,
|
|
||||||
// while deviceAddr is the address at which the network device is attached on the bridge.
|
|
||||||
- PciAddr string
|
|
||||||
+ PciAddr string `json:"pciAddr,omitempty"`
|
|
||||||
// LinkType defines the type of interface described by this structure.
|
|
||||||
// The expected values are the one that are defined by the netlink
|
|
||||||
// library, regarding each type of link. Here is a non exhaustive
|
|
||||||
// list: "veth", "macvtap", "vlan", "macvlan", "tap", ...
|
|
||||||
- LinkType string
|
|
||||||
+ LinkType string `json:"linkType,omitempty"`
|
|
||||||
}
|
|
||||||
|
|
||||||
// Route describes a network route.
|
|
||||||
diff --git a/virtcontainers/qemu.go b/virtcontainers/qemu.go
|
|
||||||
index 7bae3278..bb83b1bb 100644
|
|
||||||
--- a/virtcontainers/qemu.go
|
|
||||||
+++ b/virtcontainers/qemu.go
|
|
||||||
@@ -1361,7 +1361,7 @@ func (q *qemu) hotplugVFIODevice(device *config.VFIODev, op operation) (err erro
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
-func (q *qemu) hotAddNetDevice(name, hardAddr string, VMFds, VhostFds []*os.File) error {
|
|
||||||
+func (q *qemu) hotAddNetDevice(deviceName, name, hardAddr string, VMFds, VhostFds []*os.File) error {
|
|
||||||
var (
|
|
||||||
VMFdNames []string
|
|
||||||
VhostFdNames []string
|
|
||||||
@@ -1381,7 +1381,12 @@ func (q *qemu) hotAddNetDevice(name, hardAddr string, VMFds, VhostFds []*os.File
|
|
||||||
VhostFd.Close()
|
|
||||||
VhostFdNames = append(VhostFdNames, fdName)
|
|
||||||
}
|
|
||||||
- return q.qmpMonitorCh.qmp.ExecuteNetdevAddByFds(q.qmpMonitorCh.ctx, "tap", name, VMFdNames, VhostFdNames)
|
|
||||||
+
|
|
||||||
+ if len(VMFdNames) != 0 || len(VhostFdNames) != 0 {
|
|
||||||
+ return q.qmpMonitorCh.qmp.ExecuteNetdevAddByFds(q.qmpMonitorCh.ctx, "tap", name, VMFdNames, VhostFdNames)
|
|
||||||
+ }
|
|
||||||
+
|
|
||||||
+ return q.qmpMonitorCh.qmp.ExecuteNetdevAdd(q.qmpMonitorCh.ctx, "tap", name, deviceName, "no", "no", 0)
|
|
||||||
}
|
|
||||||
|
|
||||||
func (q *qemu) hotplugNetDevice(endpoint Endpoint, op operation) (err error) {
|
|
||||||
@@ -1404,7 +1409,7 @@ func (q *qemu) hotplugNetDevice(endpoint Endpoint, op operation) (err error) {
|
|
||||||
|
|
||||||
devID := "virtio-" + tap.ID
|
|
||||||
if op == addDevice {
|
|
||||||
- if err = q.hotAddNetDevice(tap.Name, endpoint.HardwareAddr(), tap.VMFds, tap.VhostFds); err != nil {
|
|
||||||
+ if err = q.hotAddNetDevice(tap.TAPIface.Name, tap.Name, endpoint.HardwareAddr(), tap.VMFds, tap.VhostFds); err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
diff --git a/virtcontainers/sandbox.go b/virtcontainers/sandbox.go
|
|
||||||
index ba704249..c8981a41 100644
|
|
||||||
--- a/virtcontainers/sandbox.go
|
|
||||||
+++ b/virtcontainers/sandbox.go
|
|
||||||
@@ -937,6 +937,7 @@ func (s *Sandbox) generateNetInfo(inf *vcTypes.Interface) (NetworkInfo, error) {
|
|
||||||
}
|
|
||||||
|
|
||||||
return NetworkInfo{
|
|
||||||
+ Device: inf.Device,
|
|
||||||
Iface: NetlinkIface{
|
|
||||||
LinkAttrs: netlink.LinkAttrs{
|
|
||||||
Name: inf.Name,
|
|
||||||
@@ -950,7 +951,7 @@ func (s *Sandbox) generateNetInfo(inf *vcTypes.Interface) (NetworkInfo, error) {
|
|
||||||
}
|
|
||||||
|
|
||||||
// AddInterface adds new nic to the sandbox.
|
|
||||||
-func (s *Sandbox) AddInterface(inf *vcTypes.Interface) (*vcTypes.Interface, error) {
|
|
||||||
+func (s *Sandbox) AddInterface(inf *vcTypes.Interface) (grpcIf *vcTypes.Interface, err error) {
|
|
||||||
netInfo, err := s.generateNetInfo(inf)
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
@@ -962,28 +963,41 @@ func (s *Sandbox) AddInterface(inf *vcTypes.Interface) (*vcTypes.Interface, erro
|
|
||||||
}
|
|
||||||
|
|
||||||
endpoint.SetProperties(netInfo)
|
|
||||||
- if err := doNetNS(s.networkNS.NetNsPath, func(_ ns.NetNS) error {
|
|
||||||
+ if err = doNetNS(s.networkNS.NetNsPath, func(_ ns.NetNS) error {
|
|
||||||
s.Logger().WithField("endpoint-type", endpoint.Type()).Info("Hot attaching endpoint")
|
|
||||||
return endpoint.HotAttach(s.hypervisor)
|
|
||||||
}); err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
|
|
||||||
+ defer func() {
|
|
||||||
+ if err != nil {
|
|
||||||
+ if errDetach := endpoint.HotDetach(s.hypervisor, s.networkNS.NetNsCreated, s.networkNS.NetNsPath); errDetach != nil {
|
|
||||||
+ s.Logger().WithField("endpoint-type", endpoint.Type()).Errorf("rollback hot attach endpoint failed")
|
|
||||||
+ }
|
|
||||||
+ }
|
|
||||||
+ }()
|
|
||||||
+
|
|
||||||
// Update the sandbox storage
|
|
||||||
s.networkNS.Endpoints = append(s.networkNS.Endpoints, endpoint)
|
|
||||||
- if err := s.Save(); err != nil {
|
|
||||||
+ if err = s.Save(); err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
|
|
||||||
// Add network for vm
|
|
||||||
inf.PciAddr = endpoint.PciAddr()
|
|
||||||
- return s.agent.updateInterface(inf)
|
|
||||||
+ grpcIf, err = s.agent.updateInterface(inf)
|
|
||||||
+ if err != nil {
|
|
||||||
+ return nil, err
|
|
||||||
+ }
|
|
||||||
+
|
|
||||||
+ return
|
|
||||||
}
|
|
||||||
|
|
||||||
// RemoveInterface removes a nic of the sandbox.
|
|
||||||
func (s *Sandbox) RemoveInterface(inf *vcTypes.Interface) (*vcTypes.Interface, error) {
|
|
||||||
for i, endpoint := range s.networkNS.Endpoints {
|
|
||||||
- if endpoint.HardwareAddr() == inf.HwAddr {
|
|
||||||
+ if endpoint.HardwareAddr() == inf.HwAddr || endpoint.Name() == inf.Name {
|
|
||||||
s.Logger().WithField("endpoint-type", endpoint.Type()).Info("Hot detaching endpoint")
|
|
||||||
if err := endpoint.HotDetach(s.hypervisor, s.networkNS.NetNsCreated, s.networkNS.NetNsPath); err != nil {
|
|
||||||
return inf, err
|
|
||||||
diff --git a/virtcontainers/tap_endpoint.go b/virtcontainers/tap_endpoint.go
|
|
||||||
index cb441b87..7d33d5a2 100644
|
|
||||||
--- a/virtcontainers/tap_endpoint.go
|
|
||||||
+++ b/virtcontainers/tap_endpoint.go
|
|
||||||
@@ -7,6 +7,7 @@ package virtcontainers
|
|
||||||
|
|
||||||
import (
|
|
||||||
"fmt"
|
|
||||||
+ "os"
|
|
||||||
|
|
||||||
"github.com/containernetworking/plugins/pkg/ns"
|
|
||||||
"github.com/vishvananda/netlink"
|
|
||||||
@@ -111,7 +112,7 @@ func (endpoint *TapEndpoint) HotDetach(h hypervisor, netNsCreated bool, netNsPat
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
-func createTapNetworkEndpoint(idx int, ifName string) (*TapEndpoint, error) {
|
|
||||||
+func createTapNetworkEndpoint(idx int, ifName string, tapIfName string) (*TapEndpoint, error) {
|
|
||||||
if idx < 0 {
|
|
||||||
return &TapEndpoint{}, fmt.Errorf("invalid network endpoint index: %d", idx)
|
|
||||||
}
|
|
||||||
@@ -131,6 +132,10 @@ func createTapNetworkEndpoint(idx int, ifName string) (*TapEndpoint, error) {
|
|
||||||
endpoint.TapInterface.Name = ifName
|
|
||||||
}
|
|
||||||
|
|
||||||
+ if tapIfName != "" {
|
|
||||||
+ endpoint.TapInterface.TAPIface.Name = tapIfName
|
|
||||||
+ }
|
|
||||||
+
|
|
||||||
return endpoint, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
@@ -145,9 +150,19 @@ func tapNetwork(endpoint *TapEndpoint, numCPUs uint32, disableVhostNet bool) err
|
|
||||||
if err != nil {
|
|
||||||
return fmt.Errorf("Could not create TAP interface: %s", err)
|
|
||||||
}
|
|
||||||
+
|
|
||||||
+ defer func() {
|
|
||||||
+ if err != nil {
|
|
||||||
+ if errDel := netHandle.LinkDel(tapLink); errDel != nil {
|
|
||||||
+ networkLogger().WithError(errDel).Error("tapNetwork fail to rollback del link")
|
|
||||||
+ }
|
|
||||||
+ }
|
|
||||||
+ }()
|
|
||||||
+
|
|
||||||
endpoint.TapInterface.VMFds = fds
|
|
||||||
if !disableVhostNet {
|
|
||||||
- vhostFds, err := createVhostFds(int(numCPUs))
|
|
||||||
+ var vhostFds []*os.File
|
|
||||||
+ vhostFds, err = createVhostFds(int(numCPUs))
|
|
||||||
if err != nil {
|
|
||||||
return fmt.Errorf("Could not setup vhost fds %s : %s", endpoint.TapInterface.Name, err)
|
|
||||||
}
|
|
||||||
@@ -161,10 +176,10 @@ func tapNetwork(endpoint *TapEndpoint, numCPUs uint32, disableVhostNet bool) err
|
|
||||||
// bridge created by the network plugin on the host actually expects
|
|
||||||
// to see traffic from this MAC address and not another one.
|
|
||||||
endpoint.TapInterface.TAPIface.HardAddr = linkAttrs.HardwareAddr.String()
|
|
||||||
- if err := netHandle.LinkSetMTU(tapLink, linkAttrs.MTU); err != nil {
|
|
||||||
+ if err = netHandle.LinkSetMTU(tapLink, linkAttrs.MTU); err != nil {
|
|
||||||
return fmt.Errorf("Could not set TAP MTU %d: %s", linkAttrs.MTU, err)
|
|
||||||
}
|
|
||||||
- if err := netHandle.LinkSetUp(tapLink); err != nil {
|
|
||||||
+ if err = netHandle.LinkSetUp(tapLink); err != nil {
|
|
||||||
return fmt.Errorf("Could not enable TAP %s: %s", endpoint.TapInterface.Name, err)
|
|
||||||
}
|
|
||||||
return nil
|
|
||||||
--
|
|
||||||
2.14.3 (Apple Git-98)
|
|
||||||
|
|
||||||
@ -1,260 +0,0 @@
|
|||||||
From be8153f21c0b81d2b194075ecd654501bc708577 Mon Sep 17 00:00:00 2001
|
|
||||||
From: jiangpengfei <jiangpengfei9@huawei.com>
|
|
||||||
Date: Thu, 13 Aug 2020 18:54:49 +0800
|
|
||||||
Subject: [PATCH 25/50] network: keep list-ifaces result compatible with cni
|
|
||||||
|
|
||||||
reason: community list-ifaces command will return the all
|
|
||||||
interfaces info in the Kata VM, however we may just want
|
|
||||||
to get the interfaces that we hotplug, so just return the
|
|
||||||
hotplugged interfaces and convert the interface info to
|
|
||||||
be compatible with cni.
|
|
||||||
|
|
||||||
Signed-off-by: jiangpengfei <jiangpengfei9@huawei.com>
|
|
||||||
---
|
|
||||||
cli/network.go | 29 ++++++++++++++++++++++-
|
|
||||||
virtcontainers/api.go | 4 +++-
|
|
||||||
virtcontainers/endpoint.go | 44 +++++++++++++++++++++++++++++++++++
|
|
||||||
virtcontainers/network.go | 27 +++++++++++++++++++++
|
|
||||||
virtcontainers/persist/api/network.go | 24 +++++++++++++++++++
|
|
||||||
virtcontainers/tap_endpoint.go | 9 +++++++
|
|
||||||
6 files changed, 135 insertions(+), 2 deletions(-)
|
|
||||||
|
|
||||||
diff --git a/cli/network.go b/cli/network.go
|
|
||||||
index 7e7791f1..66955725 100644
|
|
||||||
--- a/cli/network.go
|
|
||||||
+++ b/cli/network.go
|
|
||||||
@@ -28,6 +28,13 @@ const (
|
|
||||||
|
|
||||||
const defaultLinkType = "tap"
|
|
||||||
|
|
||||||
+type compatInterface struct {
|
|
||||||
+ Name string `json:"name,omitempty"`
|
|
||||||
+ Mac string `json:"mac,omitempty"`
|
|
||||||
+ IP []string `json:"ip,omitempty"`
|
|
||||||
+ Mtu int `json:"mtu,omitempty"`
|
|
||||||
+}
|
|
||||||
+
|
|
||||||
var kataNetworkCLICommand = cli.Command{
|
|
||||||
Name: "kata-network",
|
|
||||||
Usage: "manage interfaces and routes for container",
|
|
||||||
@@ -244,7 +251,8 @@ func networkListCommand(ctx context.Context, containerID string, opType networkT
|
|
||||||
kataLog.WithField("existing-interfaces", fmt.Sprintf("%+v", interfaces)).
|
|
||||||
WithError(err).Error("list interfaces failed")
|
|
||||||
}
|
|
||||||
- json.NewEncoder(file).Encode(interfaces)
|
|
||||||
+ compatInfs := convertCompatInterfaces(interfaces)
|
|
||||||
+ json.NewEncoder(file).Encode(compatInfs)
|
|
||||||
case routeType:
|
|
||||||
var routes []*vcTypes.Route
|
|
||||||
routes, err = vci.ListRoutes(ctx, sandboxID)
|
|
||||||
@@ -256,3 +264,22 @@ func networkListCommand(ctx context.Context, containerID string, opType networkT
|
|
||||||
}
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
+
|
|
||||||
+func convertCompatInterfaces(interfaces []*vcTypes.Interface) []compatInterface {
|
|
||||||
+ var infs []compatInterface
|
|
||||||
+ for _, i := range interfaces {
|
|
||||||
+ var addrs []string
|
|
||||||
+ for _, a := range i.IPAddresses {
|
|
||||||
+ addrs = append(addrs, fmt.Sprintf("%s/%s", a.Address, a.Mask))
|
|
||||||
+ }
|
|
||||||
+
|
|
||||||
+ infs = append(infs, compatInterface{
|
|
||||||
+ Name: i.Name,
|
|
||||||
+ Mac: i.HwAddr,
|
|
||||||
+ IP: addrs,
|
|
||||||
+ Mtu: int(i.Mtu),
|
|
||||||
+ })
|
|
||||||
+ }
|
|
||||||
+
|
|
||||||
+ return infs
|
|
||||||
+}
|
|
||||||
diff --git a/virtcontainers/api.go b/virtcontainers/api.go
|
|
||||||
index 5e8c9c9e..eb5b4995 100644
|
|
||||||
--- a/virtcontainers/api.go
|
|
||||||
+++ b/virtcontainers/api.go
|
|
||||||
@@ -949,7 +949,9 @@ func ListInterfaces(ctx context.Context, sandboxID string) ([]*vcTypes.Interface
|
|
||||||
}
|
|
||||||
defer s.releaseStatelessSandbox()
|
|
||||||
|
|
||||||
- return s.ListInterfaces()
|
|
||||||
+ // get interfaces info from persist.json file
|
|
||||||
+ // instead of by s.ListInterfaces()
|
|
||||||
+ return convertToCompatInterfaces(&s.networkNS.Endpoints), nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// UpdateRoutes is the virtcontainers update routes entry point.
|
|
||||||
diff --git a/virtcontainers/endpoint.go b/virtcontainers/endpoint.go
|
|
||||||
index 01b5e77f..7efcf49c 100644
|
|
||||||
--- a/virtcontainers/endpoint.go
|
|
||||||
+++ b/virtcontainers/endpoint.go
|
|
||||||
@@ -132,6 +132,28 @@ func saveTapIf(tapif *TapInterface) *persistapi.TapInterface {
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
+func saveTapEndpointProperties(networkInfo *NetworkInfo) *persistapi.NetworkProperties {
|
|
||||||
+ if networkInfo == nil {
|
|
||||||
+ return nil
|
|
||||||
+ }
|
|
||||||
+
|
|
||||||
+ return &persistapi.NetworkProperties{
|
|
||||||
+ Device: networkInfo.Device,
|
|
||||||
+ Iface: persistapi.NetlinkIface{
|
|
||||||
+ LinkAttrs: networkInfo.Iface.LinkAttrs,
|
|
||||||
+ Type: networkInfo.Iface.Type,
|
|
||||||
+ },
|
|
||||||
+ Addrs: networkInfo.Addrs,
|
|
||||||
+ Routes: networkInfo.Routes,
|
|
||||||
+ DNS: persistapi.DNSInfo{
|
|
||||||
+ Servers: networkInfo.DNS.Servers,
|
|
||||||
+ Domain: networkInfo.DNS.Domain,
|
|
||||||
+ Searches: networkInfo.DNS.Searches,
|
|
||||||
+ Options: networkInfo.DNS.Options,
|
|
||||||
+ },
|
|
||||||
+ }
|
|
||||||
+}
|
|
||||||
+
|
|
||||||
func loadTapIf(tapif *persistapi.TapInterface) *TapInterface {
|
|
||||||
if tapif == nil {
|
|
||||||
return nil
|
|
||||||
@@ -148,6 +170,28 @@ func loadTapIf(tapif *persistapi.TapInterface) *TapInterface {
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
+func loadTapEndpointProperties(endpointProperties *persistapi.NetworkProperties) *NetworkInfo {
|
|
||||||
+ if endpointProperties == nil {
|
|
||||||
+ return nil
|
|
||||||
+ }
|
|
||||||
+
|
|
||||||
+ return &NetworkInfo{
|
|
||||||
+ Device: endpointProperties.Device,
|
|
||||||
+ Iface: NetlinkIface{
|
|
||||||
+ LinkAttrs: endpointProperties.Iface.LinkAttrs,
|
|
||||||
+ Type: endpointProperties.Iface.Type,
|
|
||||||
+ },
|
|
||||||
+ Addrs: endpointProperties.Addrs,
|
|
||||||
+ Routes: endpointProperties.Routes,
|
|
||||||
+ DNS: DNSInfo{
|
|
||||||
+ Servers: endpointProperties.DNS.Servers,
|
|
||||||
+ Domain: endpointProperties.DNS.Domain,
|
|
||||||
+ Searches: endpointProperties.DNS.Searches,
|
|
||||||
+ Options: endpointProperties.DNS.Options,
|
|
||||||
+ },
|
|
||||||
+ }
|
|
||||||
+}
|
|
||||||
+
|
|
||||||
func saveNetIfPair(pair *NetworkInterfacePair) *persistapi.NetworkInterfacePair {
|
|
||||||
if pair == nil {
|
|
||||||
return nil
|
|
||||||
diff --git a/virtcontainers/network.go b/virtcontainers/network.go
|
|
||||||
index e909a822..bf7f9336 100644
|
|
||||||
--- a/virtcontainers/network.go
|
|
||||||
+++ b/virtcontainers/network.go
|
|
||||||
@@ -1340,3 +1340,30 @@ func (n *Network) Remove(ctx context.Context, ns *NetworkNamespace, hypervisor h
|
|
||||||
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
+
|
|
||||||
+// convertCompatInterfaces convert Endpoint info to vcTypes.Interface
|
|
||||||
+func convertToCompatInterfaces(es *[]Endpoint) []*vcTypes.Interface {
|
|
||||||
+ var infs []*vcTypes.Interface
|
|
||||||
+ for _, e := range *es {
|
|
||||||
+ var addrs []*vcTypes.IPAddress
|
|
||||||
+ for _, a := range e.Properties().Addrs {
|
|
||||||
+ m, _ := a.Mask.Size()
|
|
||||||
+ addr := &vcTypes.IPAddress{
|
|
||||||
+ Address: fmt.Sprintf("%s", a.IP),
|
|
||||||
+ Mask: fmt.Sprintf("%d", m),
|
|
||||||
+ }
|
|
||||||
+ addrs = append(addrs, addr)
|
|
||||||
+ }
|
|
||||||
+ inf := &vcTypes.Interface{
|
|
||||||
+ LinkType: string(e.Type()),
|
|
||||||
+ Name: e.Name(),
|
|
||||||
+ Mtu: uint64(e.Properties().Iface.MTU),
|
|
||||||
+ HwAddr: e.HardwareAddr(),
|
|
||||||
+ IPAddresses: addrs,
|
|
||||||
+ }
|
|
||||||
+
|
|
||||||
+ infs = append(infs, inf)
|
|
||||||
+ }
|
|
||||||
+
|
|
||||||
+ return infs
|
|
||||||
+}
|
|
||||||
diff --git a/virtcontainers/persist/api/network.go b/virtcontainers/persist/api/network.go
|
|
||||||
index 69610c67..53c6de44 100644
|
|
||||||
--- a/virtcontainers/persist/api/network.go
|
|
||||||
+++ b/virtcontainers/persist/api/network.go
|
|
||||||
@@ -11,6 +11,27 @@ import (
|
|
||||||
)
|
|
||||||
|
|
||||||
// ============= sandbox level resources =============
|
|
||||||
+// DNSInfo describes the DNS setup related to a network interface.
|
|
||||||
+type DNSInfo struct {
|
|
||||||
+ Servers []string
|
|
||||||
+ Domain string
|
|
||||||
+ Searches []string
|
|
||||||
+ Options []string
|
|
||||||
+}
|
|
||||||
+
|
|
||||||
+// NetlinkIface describes fully a network interface.
|
|
||||||
+type NetlinkIface struct {
|
|
||||||
+ netlink.LinkAttrs
|
|
||||||
+ Type string
|
|
||||||
+}
|
|
||||||
+
|
|
||||||
+type NetworkProperties struct {
|
|
||||||
+ Device string
|
|
||||||
+ Iface NetlinkIface
|
|
||||||
+ Addrs []netlink.Addr
|
|
||||||
+ Routes []netlink.Route
|
|
||||||
+ DNS DNSInfo
|
|
||||||
+}
|
|
||||||
|
|
||||||
type NetworkInterface struct {
|
|
||||||
Name string
|
|
||||||
@@ -91,6 +112,9 @@ type NetworkEndpoint struct {
|
|
||||||
Tap *TapEndpoint `json:",omitempty"`
|
|
||||||
IPVlan *IPVlanEndpoint `json:",omitempty"`
|
|
||||||
Tuntap *TuntapEndpoint `json:",omitempty"`
|
|
||||||
+
|
|
||||||
+ // store the endpoint properties info
|
|
||||||
+ EndPointProperties *NetworkProperties `json:",omitempty"`
|
|
||||||
}
|
|
||||||
|
|
||||||
// NetworkInfo contains network information of sandbox
|
|
||||||
diff --git a/virtcontainers/tap_endpoint.go b/virtcontainers/tap_endpoint.go
|
|
||||||
index 7d33d5a2..c897670e 100644
|
|
||||||
--- a/virtcontainers/tap_endpoint.go
|
|
||||||
+++ b/virtcontainers/tap_endpoint.go
|
|
||||||
@@ -206,12 +206,15 @@ func unTapNetwork(name string) error {
|
|
||||||
|
|
||||||
func (endpoint *TapEndpoint) save() persistapi.NetworkEndpoint {
|
|
||||||
tapif := saveTapIf(&endpoint.TapInterface)
|
|
||||||
+ // save tap endpoint network properties into persist storage
|
|
||||||
+ properties := saveTapEndpointProperties(&endpoint.EndpointProperties)
|
|
||||||
|
|
||||||
return persistapi.NetworkEndpoint{
|
|
||||||
Type: string(endpoint.Type()),
|
|
||||||
Tap: &persistapi.TapEndpoint{
|
|
||||||
TapInterface: *tapif,
|
|
||||||
},
|
|
||||||
+ EndPointProperties: properties,
|
|
||||||
}
|
|
||||||
}
|
|
||||||
func (endpoint *TapEndpoint) load(s persistapi.NetworkEndpoint) {
|
|
||||||
@@ -221,4 +224,10 @@ func (endpoint *TapEndpoint) load(s persistapi.NetworkEndpoint) {
|
|
||||||
tapif := loadTapIf(&s.Tap.TapInterface)
|
|
||||||
endpoint.TapInterface = *tapif
|
|
||||||
}
|
|
||||||
+
|
|
||||||
+ if s.EndPointProperties != nil {
|
|
||||||
+ // restore tap endpoint network properties from persist storage
|
|
||||||
+ properties := loadTapEndpointProperties(s.EndPointProperties)
|
|
||||||
+ endpoint.EndpointProperties = *properties
|
|
||||||
+ }
|
|
||||||
}
|
|
||||||
--
|
|
||||||
2.14.3 (Apple Git-98)
|
|
||||||
|
|
||||||
@ -1,285 +0,0 @@
|
|||||||
From eeca1e47e9a6422d89d08275864f2c1b15e54941 Mon Sep 17 00:00:00 2001
|
|
||||||
From: jiangpengfei <jiangpengfei9@huawei.com>
|
|
||||||
Date: Fri, 14 Aug 2020 19:14:35 +0800
|
|
||||||
Subject: [PATCH 26/50] network: add enable_compat_old_cni config
|
|
||||||
|
|
||||||
reason: old version kata-network list-ifaces output result different
|
|
||||||
from the community version, inorder to compatible with the old version
|
|
||||||
list-ifaces command output format, add enable_compat_old_cni to control
|
|
||||||
the list-ifaces command output format.
|
|
||||||
|
|
||||||
Signed-off-by: jiangpengfei <jiangpengfei9@huawei.com>
|
|
||||||
---
|
|
||||||
cli/config/configuration-qemu.toml.in | 4 ++++
|
|
||||||
cli/network.go | 17 +++++++++++++++--
|
|
||||||
cli/network_test.go | 6 ++++++
|
|
||||||
pkg/katautils/config.go | 2 ++
|
|
||||||
virtcontainers/api.go | 10 +++++++---
|
|
||||||
virtcontainers/interfaces.go | 1 +
|
|
||||||
virtcontainers/network.go | 11 ++++++-----
|
|
||||||
virtcontainers/persist.go | 18 ++++++++++--------
|
|
||||||
virtcontainers/persist/api/config.go | 9 +++++----
|
|
||||||
virtcontainers/pkg/oci/utils.go | 4 ++++
|
|
||||||
virtcontainers/pkg/vcmock/sandbox.go | 5 +++++
|
|
||||||
virtcontainers/sandbox.go | 5 +++++
|
|
||||||
12 files changed, 70 insertions(+), 22 deletions(-)
|
|
||||||
|
|
||||||
diff --git a/cli/config/configuration-qemu.toml.in b/cli/config/configuration-qemu.toml.in
|
|
||||||
index b44e84d8..46f8b632 100644
|
|
||||||
--- a/cli/config/configuration-qemu.toml.in
|
|
||||||
+++ b/cli/config/configuration-qemu.toml.in
|
|
||||||
@@ -453,6 +453,10 @@ disable_guest_seccomp=@DEFDISABLEGUESTSECCOMP@
|
|
||||||
# (default: false)
|
|
||||||
#disable_new_netns = true
|
|
||||||
|
|
||||||
+# If enabled, the kata-network will return the old interface format info to be compatible with
|
|
||||||
+# old version CNI plugin
|
|
||||||
+enable_compat_old_cni = true
|
|
||||||
+
|
|
||||||
# if enabled, the runtime will add all the kata processes inside one dedicated cgroup.
|
|
||||||
# The container cgroups in the host are not created, just one single cgroup per sandbox.
|
|
||||||
# The runtime caller is free to restrict or collect cgroup stats of the overall Kata sandbox.
|
|
||||||
diff --git a/cli/network.go b/cli/network.go
|
|
||||||
index 66955725..a1a24425 100644
|
|
||||||
--- a/cli/network.go
|
|
||||||
+++ b/cli/network.go
|
|
||||||
@@ -251,8 +251,21 @@ func networkListCommand(ctx context.Context, containerID string, opType networkT
|
|
||||||
kataLog.WithField("existing-interfaces", fmt.Sprintf("%+v", interfaces)).
|
|
||||||
WithError(err).Error("list interfaces failed")
|
|
||||||
}
|
|
||||||
- compatInfs := convertCompatInterfaces(interfaces)
|
|
||||||
- json.NewEncoder(file).Encode(compatInfs)
|
|
||||||
+
|
|
||||||
+ sandbox, err := vci.FetchSandbox(ctx, sandboxID)
|
|
||||||
+ if err != nil {
|
|
||||||
+ kataLog.WithField("existing-interfaces", fmt.Sprintf("%+v", interfaces)).
|
|
||||||
+ WithError(err).Error("fetch sandbox failed")
|
|
||||||
+ }
|
|
||||||
+
|
|
||||||
+ // If sandbox network config need to be compatible with old CNI,
|
|
||||||
+ // convert the interface format to old version format.
|
|
||||||
+ if sandbox.IsCompatOldCNI() {
|
|
||||||
+ compatInfs := convertCompatInterfaces(interfaces)
|
|
||||||
+ json.NewEncoder(file).Encode(compatInfs)
|
|
||||||
+ } else {
|
|
||||||
+ json.NewEncoder(file).Encode(interfaces)
|
|
||||||
+ }
|
|
||||||
case routeType:
|
|
||||||
var routes []*vcTypes.Route
|
|
||||||
routes, err = vci.ListRoutes(ctx, sandboxID)
|
|
||||||
diff --git a/cli/network_test.go b/cli/network_test.go
|
|
||||||
index 4e3d943d..95fd2749 100644
|
|
||||||
--- a/cli/network_test.go
|
|
||||||
+++ b/cli/network_test.go
|
|
||||||
@@ -17,6 +17,7 @@ import (
|
|
||||||
|
|
||||||
vc "github.com/kata-containers/runtime/virtcontainers"
|
|
||||||
vcTypes "github.com/kata-containers/runtime/virtcontainers/pkg/types"
|
|
||||||
+ "github.com/kata-containers/runtime/virtcontainers/pkg/vcmock"
|
|
||||||
"github.com/kata-containers/runtime/virtcontainers/types"
|
|
||||||
)
|
|
||||||
|
|
||||||
@@ -59,6 +60,10 @@ func TestNetworkCliFunction(t *testing.T) {
|
|
||||||
return newSingleContainerStatus(testContainerID, state, map[string]string{}, &specs.Spec{}), nil
|
|
||||||
}
|
|
||||||
|
|
||||||
+ testingImpl.FetchSandboxFunc = func(ctx context.Context, id string) (vc.VCSandbox, error) {
|
|
||||||
+ return &vcmock.Sandbox{}, nil
|
|
||||||
+ }
|
|
||||||
+
|
|
||||||
defer func() {
|
|
||||||
testingImpl.AddInterfaceFunc = nil
|
|
||||||
testingImpl.RemoveInterfaceFunc = nil
|
|
||||||
@@ -66,6 +71,7 @@ func TestNetworkCliFunction(t *testing.T) {
|
|
||||||
testingImpl.UpdateRoutesFunc = nil
|
|
||||||
testingImpl.ListRoutesFunc = nil
|
|
||||||
testingImpl.StatusContainerFunc = nil
|
|
||||||
+ testingImpl.FetchSandboxFunc = nil
|
|
||||||
}()
|
|
||||||
|
|
||||||
set := flag.NewFlagSet("", 0)
|
|
||||||
diff --git a/pkg/katautils/config.go b/pkg/katautils/config.go
|
|
||||||
index 9a99b9d4..94c916a0 100644
|
|
||||||
--- a/pkg/katautils/config.go
|
|
||||||
+++ b/pkg/katautils/config.go
|
|
||||||
@@ -141,6 +141,7 @@ type runtime struct {
|
|
||||||
Debug bool `toml:"enable_debug"`
|
|
||||||
Tracing bool `toml:"enable_tracing"`
|
|
||||||
DisableNewNetNs bool `toml:"disable_new_netns"`
|
|
||||||
+ EnableCompatOldCNI bool `toml:"enable_compat_old_cni"`
|
|
||||||
DisableGuestSeccomp bool `toml:"disable_guest_seccomp"`
|
|
||||||
SandboxCgroupOnly bool `toml:"sandbox_cgroup_only"`
|
|
||||||
Experimental []string `toml:"experimental"`
|
|
||||||
@@ -1235,6 +1236,7 @@ func LoadConfiguration(configPath string, ignoreLogging, builtIn bool, debugFlag
|
|
||||||
|
|
||||||
config.SandboxCgroupOnly = tomlConf.Runtime.SandboxCgroupOnly
|
|
||||||
config.DisableNewNetNs = tomlConf.Runtime.DisableNewNetNs
|
|
||||||
+ config.EnableCompatOldCNI = tomlConf.Runtime.EnableCompatOldCNI
|
|
||||||
for _, f := range tomlConf.Runtime.Experimental {
|
|
||||||
feature := exp.Get(f)
|
|
||||||
if feature == nil {
|
|
||||||
diff --git a/virtcontainers/api.go b/virtcontainers/api.go
|
|
||||||
index eb5b4995..fb044fe1 100644
|
|
||||||
--- a/virtcontainers/api.go
|
|
||||||
+++ b/virtcontainers/api.go
|
|
||||||
@@ -949,9 +949,13 @@ func ListInterfaces(ctx context.Context, sandboxID string) ([]*vcTypes.Interface
|
|
||||||
}
|
|
||||||
defer s.releaseStatelessSandbox()
|
|
||||||
|
|
||||||
- // get interfaces info from persist.json file
|
|
||||||
- // instead of by s.ListInterfaces()
|
|
||||||
- return convertToCompatInterfaces(&s.networkNS.Endpoints), nil
|
|
||||||
+ // If enable_compat_old_cni is enabled, get interfaces info from
|
|
||||||
+ // persist.json file instead of by s.ListInterfaces()
|
|
||||||
+ if s.config.NetworkConfig.EnableCompatOldCNI {
|
|
||||||
+ return convertToCompatInterfaces(&s.networkNS.Endpoints), nil
|
|
||||||
+ }
|
|
||||||
+
|
|
||||||
+ return s.ListInterfaces()
|
|
||||||
}
|
|
||||||
|
|
||||||
// UpdateRoutes is the virtcontainers update routes entry point.
|
|
||||||
diff --git a/virtcontainers/interfaces.go b/virtcontainers/interfaces.go
|
|
||||||
index fa6b584e..499b386e 100644
|
|
||||||
--- a/virtcontainers/interfaces.go
|
|
||||||
+++ b/virtcontainers/interfaces.go
|
|
||||||
@@ -98,6 +98,7 @@ type VCSandbox interface {
|
|
||||||
ListInterfaces() ([]*vcTypes.Interface, error)
|
|
||||||
UpdateRoutes(routes []*vcTypes.Route) ([]*vcTypes.Route, error)
|
|
||||||
ListRoutes() ([]*vcTypes.Route, error)
|
|
||||||
+ IsCompatOldCNI() bool
|
|
||||||
}
|
|
||||||
|
|
||||||
// VCContainer is the Container interface
|
|
||||||
diff --git a/virtcontainers/network.go b/virtcontainers/network.go
|
|
||||||
index bf7f9336..db235cf6 100644
|
|
||||||
--- a/virtcontainers/network.go
|
|
||||||
+++ b/virtcontainers/network.go
|
|
||||||
@@ -155,11 +155,12 @@ type NetworkInterfacePair struct {
|
|
||||||
|
|
||||||
// NetworkConfig is the network configuration related to a network.
|
|
||||||
type NetworkConfig struct {
|
|
||||||
- NetNSPath string
|
|
||||||
- NetNsCreated bool
|
|
||||||
- DisableNewNetNs bool
|
|
||||||
- NetmonConfig NetmonConfig
|
|
||||||
- InterworkingModel NetInterworkingModel
|
|
||||||
+ NetNSPath string
|
|
||||||
+ NetNsCreated bool
|
|
||||||
+ DisableNewNetNs bool
|
|
||||||
+ EnableCompatOldCNI bool
|
|
||||||
+ NetmonConfig NetmonConfig
|
|
||||||
+ InterworkingModel NetInterworkingModel
|
|
||||||
}
|
|
||||||
|
|
||||||
func networkLogger() *logrus.Entry {
|
|
||||||
diff --git a/virtcontainers/persist.go b/virtcontainers/persist.go
|
|
||||||
index 6bd09a0b..fe00bf9a 100644
|
|
||||||
--- a/virtcontainers/persist.go
|
|
||||||
+++ b/virtcontainers/persist.go
|
|
||||||
@@ -187,10 +187,11 @@ func (s *Sandbox) dumpConfig(ss *persistapi.SandboxState) {
|
|
||||||
},
|
|
||||||
ShimType: string(sconfig.ShimType),
|
|
||||||
NetworkConfig: persistapi.NetworkConfig{
|
|
||||||
- NetNSPath: sconfig.NetworkConfig.NetNSPath,
|
|
||||||
- NetNsCreated: sconfig.NetworkConfig.NetNsCreated,
|
|
||||||
- DisableNewNetNs: sconfig.NetworkConfig.DisableNewNetNs,
|
|
||||||
- InterworkingModel: int(sconfig.NetworkConfig.InterworkingModel),
|
|
||||||
+ NetNSPath: sconfig.NetworkConfig.NetNSPath,
|
|
||||||
+ NetNsCreated: sconfig.NetworkConfig.NetNsCreated,
|
|
||||||
+ DisableNewNetNs: sconfig.NetworkConfig.DisableNewNetNs,
|
|
||||||
+ EnableCompatOldCNI: sconfig.NetworkConfig.EnableCompatOldCNI,
|
|
||||||
+ InterworkingModel: int(sconfig.NetworkConfig.InterworkingModel),
|
|
||||||
},
|
|
||||||
|
|
||||||
ShmSize: sconfig.ShmSize,
|
|
||||||
@@ -477,10 +478,11 @@ func loadSandboxConfig(id string) (*SandboxConfig, error) {
|
|
||||||
},
|
|
||||||
ShimType: ShimType(savedConf.ShimType),
|
|
||||||
NetworkConfig: NetworkConfig{
|
|
||||||
- NetNSPath: savedConf.NetworkConfig.NetNSPath,
|
|
||||||
- NetNsCreated: savedConf.NetworkConfig.NetNsCreated,
|
|
||||||
- DisableNewNetNs: savedConf.NetworkConfig.DisableNewNetNs,
|
|
||||||
- InterworkingModel: NetInterworkingModel(savedConf.NetworkConfig.InterworkingModel),
|
|
||||||
+ NetNSPath: savedConf.NetworkConfig.NetNSPath,
|
|
||||||
+ NetNsCreated: savedConf.NetworkConfig.NetNsCreated,
|
|
||||||
+ DisableNewNetNs: savedConf.NetworkConfig.DisableNewNetNs,
|
|
||||||
+ EnableCompatOldCNI: savedConf.NetworkConfig.EnableCompatOldCNI,
|
|
||||||
+ InterworkingModel: NetInterworkingModel(savedConf.NetworkConfig.InterworkingModel),
|
|
||||||
},
|
|
||||||
|
|
||||||
ShmSize: savedConf.ShmSize,
|
|
||||||
diff --git a/virtcontainers/persist/api/config.go b/virtcontainers/persist/api/config.go
|
|
||||||
index cfbee849..3a2df32b 100644
|
|
||||||
--- a/virtcontainers/persist/api/config.go
|
|
||||||
+++ b/virtcontainers/persist/api/config.go
|
|
||||||
@@ -210,10 +210,11 @@ type ShimConfig struct {
|
|
||||||
|
|
||||||
// NetworkConfig is the network configuration related to a network.
|
|
||||||
type NetworkConfig struct {
|
|
||||||
- NetNSPath string
|
|
||||||
- NetNsCreated bool
|
|
||||||
- DisableNewNetNs bool
|
|
||||||
- InterworkingModel int
|
|
||||||
+ NetNSPath string
|
|
||||||
+ NetNsCreated bool
|
|
||||||
+ DisableNewNetNs bool
|
|
||||||
+ EnableCompatOldCNI bool
|
|
||||||
+ InterworkingModel int
|
|
||||||
}
|
|
||||||
|
|
||||||
type ContainerConfig struct {
|
|
||||||
diff --git a/virtcontainers/pkg/oci/utils.go b/virtcontainers/pkg/oci/utils.go
|
|
||||||
index 36c730b7..948bd3cb 100644
|
|
||||||
--- a/virtcontainers/pkg/oci/utils.go
|
|
||||||
+++ b/virtcontainers/pkg/oci/utils.go
|
|
||||||
@@ -130,6 +130,9 @@ type RuntimeConfig struct {
|
|
||||||
//Determines if create a netns for hypervisor process
|
|
||||||
DisableNewNetNs bool
|
|
||||||
|
|
||||||
+ // Determines if compatible with old CNI plugin
|
|
||||||
+ EnableCompatOldCNI bool
|
|
||||||
+
|
|
||||||
//Determines kata processes are managed only in sandbox cgroup
|
|
||||||
SandboxCgroupOnly bool
|
|
||||||
|
|
||||||
@@ -275,6 +278,7 @@ func networkConfig(ocispec specs.Spec, config RuntimeConfig) (vc.NetworkConfig,
|
|
||||||
}
|
|
||||||
netConf.InterworkingModel = config.InterNetworkModel
|
|
||||||
netConf.DisableNewNetNs = config.DisableNewNetNs
|
|
||||||
+ netConf.EnableCompatOldCNI = config.EnableCompatOldCNI
|
|
||||||
|
|
||||||
netConf.NetmonConfig = vc.NetmonConfig{
|
|
||||||
Path: config.NetmonConfig.Path,
|
|
||||||
diff --git a/virtcontainers/pkg/vcmock/sandbox.go b/virtcontainers/pkg/vcmock/sandbox.go
|
|
||||||
index 677457ef..11b83ccd 100644
|
|
||||||
--- a/virtcontainers/pkg/vcmock/sandbox.go
|
|
||||||
+++ b/virtcontainers/pkg/vcmock/sandbox.go
|
|
||||||
@@ -212,3 +212,8 @@ func (s *Sandbox) UpdateRoutes(routes []*vcTypes.Route) ([]*vcTypes.Route, error
|
|
||||||
func (s *Sandbox) ListRoutes() ([]*vcTypes.Route, error) {
|
|
||||||
return nil, nil
|
|
||||||
}
|
|
||||||
+
|
|
||||||
+// IsCompatOldCNI return the whether enable compatible with old CNI
|
|
||||||
+func (s *Sandbox) IsCompatOldCNI() bool {
|
|
||||||
+ return false
|
|
||||||
+}
|
|
||||||
diff --git a/virtcontainers/sandbox.go b/virtcontainers/sandbox.go
|
|
||||||
index c8981a41..6a643a12 100644
|
|
||||||
--- a/virtcontainers/sandbox.go
|
|
||||||
+++ b/virtcontainers/sandbox.go
|
|
||||||
@@ -2379,6 +2379,11 @@ func (s *Sandbox) setContainersState(state types.StateString) error {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
+// IsCompatOldCNI return the whether enable compatible with old CNI
|
|
||||||
+func (s *Sandbox) IsCompatOldCNI() bool {
|
|
||||||
+ return s.config.NetworkConfig.EnableCompatOldCNI
|
|
||||||
+}
|
|
||||||
+
|
|
||||||
// updateStaticSandboxResources update sandbox's cpu and memory resource passed by
|
|
||||||
// sandbox_cpu and sandbox_mem annotations
|
|
||||||
func updateStaticSandboxResources(sandboxConfig *SandboxConfig) error {
|
|
||||||
--
|
|
||||||
2.14.3 (Apple Git-98)
|
|
||||||
|
|
||||||
@ -1,182 +0,0 @@
|
|||||||
From ec15337fc816767ca0e8183576405499080b9b1e Mon Sep 17 00:00:00 2001
|
|
||||||
From: jiangpengfei <jiangpengfei9@huawei.com>
|
|
||||||
Date: Sun, 16 Aug 2020 16:41:18 +0800
|
|
||||||
Subject: [PATCH 27/50] network: add more strict check for input network
|
|
||||||
interface
|
|
||||||
|
|
||||||
reason: kata-network add-iface command will receive the network
|
|
||||||
interface info from untrust user, so we need to add more strict
|
|
||||||
check for input network interface info.
|
|
||||||
|
|
||||||
Signed-off-by: jiangpengfei <jiangpengfei9@huawei.com>
|
|
||||||
---
|
|
||||||
virtcontainers/network.go | 117 ++++++++++++++++++++++++++++++++++++++++++++++
|
|
||||||
virtcontainers/sandbox.go | 11 +++++
|
|
||||||
2 files changed, 128 insertions(+)
|
|
||||||
|
|
||||||
diff --git a/virtcontainers/network.go b/virtcontainers/network.go
|
|
||||||
index db235cf6..a1676ccd 100644
|
|
||||||
--- a/virtcontainers/network.go
|
|
||||||
+++ b/virtcontainers/network.go
|
|
||||||
@@ -13,8 +13,10 @@ import (
|
|
||||||
"math/rand"
|
|
||||||
"net"
|
|
||||||
"os"
|
|
||||||
+ "regexp"
|
|
||||||
"runtime"
|
|
||||||
"sort"
|
|
||||||
+ "strings"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"github.com/containernetworking/plugins/pkg/ns"
|
|
||||||
@@ -55,6 +57,17 @@ const (
|
|
||||||
NetXConnectInvalidModel
|
|
||||||
)
|
|
||||||
|
|
||||||
+const (
|
|
||||||
+ maxIPAddrLen = 18
|
|
||||||
+ maxInterfaceLen = 15
|
|
||||||
+ minMTUVal = 46
|
|
||||||
+ maxMTUVal = 9600
|
|
||||||
+)
|
|
||||||
+
|
|
||||||
+var (
|
|
||||||
+ regInfName = regexp.MustCompile(`^[A-Za-z][A-Za-z0-9_\-.]*$`)
|
|
||||||
+)
|
|
||||||
+
|
|
||||||
//IsValid checks if a model is valid
|
|
||||||
func (n NetInterworkingModel) IsValid() bool {
|
|
||||||
return 0 <= int(n) && int(n) < int(NetXConnectInvalidModel)
|
|
||||||
@@ -1368,3 +1381,107 @@ func convertToCompatInterfaces(es *[]Endpoint) []*vcTypes.Interface {
|
|
||||||
|
|
||||||
return infs
|
|
||||||
}
|
|
||||||
+
|
|
||||||
+// verifyInterfaceName verifies the interface name valid or not
|
|
||||||
+func verifyInterfaceName(name string) error {
|
|
||||||
+ // verify `Name` before `Tapname` because of the strict rules
|
|
||||||
+ name = strings.TrimSpace(name)
|
|
||||||
+ if len(name) > maxInterfaceLen {
|
|
||||||
+ return fmt.Errorf("interface name can't be longer than 15")
|
|
||||||
+ } else if len(name) == 0 || name == "\n" {
|
|
||||||
+ return fmt.Errorf("interface name can't be empty")
|
|
||||||
+ }
|
|
||||||
+
|
|
||||||
+ // QMP rules of `Name`
|
|
||||||
+ chk := regInfName.FindAllString(name, -1)
|
|
||||||
+ if chk == nil {
|
|
||||||
+ return fmt.Errorf("invalid input of interface name, please check the rules")
|
|
||||||
+ }
|
|
||||||
+
|
|
||||||
+ return nil
|
|
||||||
+}
|
|
||||||
+
|
|
||||||
+// verifyIPAndMask verifies CIDR notation IP address and mask like "192.0.2.0/24"
|
|
||||||
+func verifyIPAndMask(ip string) (nlIpMask *net.IPNet, err error) {
|
|
||||||
+ ip = strings.TrimSpace(ip)
|
|
||||||
+
|
|
||||||
+ if len(ip) > maxIPAddrLen {
|
|
||||||
+ return nil, fmt.Errorf("the length of IP address is too long, max ip len :%d", maxIPAddrLen)
|
|
||||||
+ }
|
|
||||||
+
|
|
||||||
+ if nlIpMask, err = netlink.ParseIPNet(ip); err != nil {
|
|
||||||
+ return nil, fmt.Errorf("invalid input of IP : %v", err)
|
|
||||||
+ }
|
|
||||||
+
|
|
||||||
+ return nlIpMask, nil
|
|
||||||
+}
|
|
||||||
+
|
|
||||||
+// verifyMac verifies MAC address
|
|
||||||
+func verifyMac(mac string) error {
|
|
||||||
+ mac = strings.TrimSpace(mac)
|
|
||||||
+ if _, err := net.ParseMAC(mac); err != nil {
|
|
||||||
+ return fmt.Errorf("invalid input of Mac : %v", err)
|
|
||||||
+ }
|
|
||||||
+
|
|
||||||
+ return nil
|
|
||||||
+}
|
|
||||||
+
|
|
||||||
+// verifyMtu verifies MTU value of interface
|
|
||||||
+func verifyMtu(mtu uint64) error {
|
|
||||||
+ if mtu < minMTUVal || mtu > maxMTUVal {
|
|
||||||
+ return fmt.Errorf("invalid input of MTU : %v", mtu)
|
|
||||||
+ }
|
|
||||||
+ return nil
|
|
||||||
+}
|
|
||||||
+
|
|
||||||
+// verifyIP verifies the IP address
|
|
||||||
+func verifyIP(ip string) (*net.IP, error) {
|
|
||||||
+ ip = strings.TrimSpace(ip)
|
|
||||||
+
|
|
||||||
+ if len(ip) > maxIPAddrLen {
|
|
||||||
+ return nil, fmt.Errorf("the length of IP address is too long, max ip len :%d", maxIPAddrLen)
|
|
||||||
+ }
|
|
||||||
+
|
|
||||||
+ var netIP net.IP
|
|
||||||
+ if netIP = net.ParseIP(ip); netIP == nil {
|
|
||||||
+ return nil, fmt.Errorf("invalid IP: %s", ip)
|
|
||||||
+ }
|
|
||||||
+
|
|
||||||
+ return &netIP, nil
|
|
||||||
+}
|
|
||||||
+
|
|
||||||
+// validInterface check the input interface valid or not
|
|
||||||
+func validInterface(inf *vcTypes.Interface, enableCompatOldCNI bool) error {
|
|
||||||
+ if enableCompatOldCNI && verifyInterfaceName(inf.Device) != nil {
|
|
||||||
+ return fmt.Errorf("device name should not be empty when enable_compat_old_cni config enabled")
|
|
||||||
+ }
|
|
||||||
+
|
|
||||||
+ if inf.Name == "" || inf.Mtu == 0 || inf.HwAddr == "" {
|
|
||||||
+ return fmt.Errorf("name/mtu/hwaddr of interface must be specified")
|
|
||||||
+ }
|
|
||||||
+
|
|
||||||
+ if err := verifyInterfaceName(inf.Name); err != nil {
|
|
||||||
+ return err
|
|
||||||
+ }
|
|
||||||
+
|
|
||||||
+ if err := verifyMac(inf.HwAddr); err != nil {
|
|
||||||
+ return err
|
|
||||||
+ }
|
|
||||||
+
|
|
||||||
+ if err := verifyMtu(inf.Mtu); err != nil {
|
|
||||||
+ return err
|
|
||||||
+ }
|
|
||||||
+
|
|
||||||
+ // Currently, only one IP address can be passed, which reduces the test entry and fault injection.
|
|
||||||
+ if len(inf.IPAddresses) > 0 {
|
|
||||||
+ if len(inf.IPAddresses) != 1 {
|
|
||||||
+ return fmt.Errorf("only one IP address is supported currently")
|
|
||||||
+ }
|
|
||||||
+ _, err := verifyIP(inf.IPAddresses[0].Address)
|
|
||||||
+ if err != nil {
|
|
||||||
+ return err
|
|
||||||
+ }
|
|
||||||
+ }
|
|
||||||
+
|
|
||||||
+ return nil
|
|
||||||
+}
|
|
||||||
diff --git a/virtcontainers/sandbox.go b/virtcontainers/sandbox.go
|
|
||||||
index 6a643a12..f6826812 100644
|
|
||||||
--- a/virtcontainers/sandbox.go
|
|
||||||
+++ b/virtcontainers/sandbox.go
|
|
||||||
@@ -952,6 +952,17 @@ func (s *Sandbox) generateNetInfo(inf *vcTypes.Interface) (NetworkInfo, error) {
|
|
||||||
|
|
||||||
// AddInterface adds new nic to the sandbox.
|
|
||||||
func (s *Sandbox) AddInterface(inf *vcTypes.Interface) (grpcIf *vcTypes.Interface, err error) {
|
|
||||||
+ err = validInterface(inf, s.config.NetworkConfig.EnableCompatOldCNI)
|
|
||||||
+ if err != nil {
|
|
||||||
+ return nil, err
|
|
||||||
+ }
|
|
||||||
+
|
|
||||||
+ for _, ep := range s.networkNS.Endpoints {
|
|
||||||
+ if ep.Name() == inf.Name {
|
|
||||||
+ return nil, fmt.Errorf("interface %s is already exist", inf.Name)
|
|
||||||
+ }
|
|
||||||
+ }
|
|
||||||
+
|
|
||||||
netInfo, err := s.generateNetInfo(inf)
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
--
|
|
||||||
2.14.3 (Apple Git-98)
|
|
||||||
|
|
||||||
File diff suppressed because it is too large
Load Diff
@ -1,192 +0,0 @@
|
|||||||
From 51a7270987557ab12ea735fc9781725d1ce1b0a6 Mon Sep 17 00:00:00 2001
|
|
||||||
From: jiangpengfei <jiangpengfei9@huawei.com>
|
|
||||||
Date: Mon, 17 Aug 2020 15:36:32 +0800
|
|
||||||
Subject: [PATCH 29/50] network: add kata-network del-route subcommand
|
|
||||||
|
|
||||||
reason: add kata-network del-route subcommand to delete the
|
|
||||||
specified route in the Kata VM, del-route is more efficient
|
|
||||||
than the update-routes subcommand.
|
|
||||||
|
|
||||||
Signed-off-by: jiangpengfei <jiangpengfei9@huawei.com>
|
|
||||||
---
|
|
||||||
cli/network.go | 24 ++++++++++
|
|
||||||
virtcontainers/network.go | 115 ++++++++++++++++++++++++++++++++++++++++++++++
|
|
||||||
2 files changed, 139 insertions(+)
|
|
||||||
|
|
||||||
diff --git a/cli/network.go b/cli/network.go
|
|
||||||
index 2265f54b..046d0ee9 100644
|
|
||||||
--- a/cli/network.go
|
|
||||||
+++ b/cli/network.go
|
|
||||||
@@ -45,6 +45,7 @@ var kataNetworkCLICommand = cli.Command{
|
|
||||||
updateRoutesCommand,
|
|
||||||
listRoutesCommand,
|
|
||||||
addRoutesCommand,
|
|
||||||
+ deleteRoutesCommand,
|
|
||||||
},
|
|
||||||
Action: func(context *cli.Context) error {
|
|
||||||
return cli.ShowSubcommandHelp(context)
|
|
||||||
@@ -155,6 +156,29 @@ var addRoutesCommand = cli.Command{
|
|
||||||
},
|
|
||||||
}
|
|
||||||
|
|
||||||
+var deleteRoutesCommand = cli.Command{
|
|
||||||
+ Name: "del-route",
|
|
||||||
+ Usage: "delete one route for a container",
|
|
||||||
+ ArgsUsage: `del-route <container-id> file or - for stdin
|
|
||||||
+ file or stdin for example:
|
|
||||||
+ {
|
|
||||||
+ "dest":"<[<ip>[/mask] | "default" ]>",
|
|
||||||
+ "gateway":"[ip]",
|
|
||||||
+ "device":"[tap-name]",
|
|
||||||
+ }
|
|
||||||
+ Only destination is required and others are optional,
|
|
||||||
+ if device is empty, means search every device`,
|
|
||||||
+ Flags: []cli.Flag{},
|
|
||||||
+ Action: func(context *cli.Context) error {
|
|
||||||
+ ctx, err := cliContextToContext(context)
|
|
||||||
+ if err != nil {
|
|
||||||
+ return err
|
|
||||||
+ }
|
|
||||||
+
|
|
||||||
+ return networkModifyCommand(ctx, context.Args().First(), context.Args().Get(1), routeType, vcTypes.NetworkOpRemove)
|
|
||||||
+ },
|
|
||||||
+}
|
|
||||||
+
|
|
||||||
var listRoutesCommand = cli.Command{
|
|
||||||
Name: "list-routes",
|
|
||||||
Usage: "list network routes in a container",
|
|
||||||
diff --git a/virtcontainers/network.go b/virtcontainers/network.go
|
|
||||||
index f3757f84..c7066a11 100644
|
|
||||||
--- a/virtcontainers/network.go
|
|
||||||
+++ b/virtcontainers/network.go
|
|
||||||
@@ -1631,6 +1631,110 @@ func addOneRoute(ns *NetworkNamespace, route *vcTypes.Route) (added *vcTypes.Rou
|
|
||||||
return added, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
+func generateRmRoute(route *vcTypes.Route) (r *netlink.Route, err error) {
|
|
||||||
+ // destination is required and others are optional
|
|
||||||
+ if route.Dest == "" {
|
|
||||||
+ return nil, fmt.Errorf("destination must be specified when remove route.")
|
|
||||||
+ }
|
|
||||||
+
|
|
||||||
+ if route.Device != "" {
|
|
||||||
+ err = verifyInterfaceName(route.Device)
|
|
||||||
+ if err != nil {
|
|
||||||
+ return nil, err
|
|
||||||
+ }
|
|
||||||
+ }
|
|
||||||
+
|
|
||||||
+ r = &netlink.Route{}
|
|
||||||
+ if route.Dest != "default" {
|
|
||||||
+ nlIpNet, err := verifyRouteDest(&route.Dest)
|
|
||||||
+ if err != nil {
|
|
||||||
+ return nil, err
|
|
||||||
+ }
|
|
||||||
+ r.Dst = nlIpNet
|
|
||||||
+ }
|
|
||||||
+
|
|
||||||
+ if route.Gateway != "" {
|
|
||||||
+ nIP, err := verifyIP(route.Gateway)
|
|
||||||
+ if err != nil {
|
|
||||||
+ return nil, err
|
|
||||||
+ }
|
|
||||||
+ r.Gw = *nIP
|
|
||||||
+ }
|
|
||||||
+
|
|
||||||
+ if route.Source != "" {
|
|
||||||
+ nIP, err := verifyIP(route.Source)
|
|
||||||
+ if err != nil {
|
|
||||||
+ return nil, err
|
|
||||||
+ }
|
|
||||||
+ r.Src = *nIP
|
|
||||||
+ }
|
|
||||||
+
|
|
||||||
+ r.Scope = netlink.Scope(route.Scope)
|
|
||||||
+
|
|
||||||
+ return r, nil
|
|
||||||
+}
|
|
||||||
+
|
|
||||||
+// parseToGrpcRoute convert the netlink.Route to vcTypes.Route and deleted route.Dest will
|
|
||||||
+// be added a prefix "-"
|
|
||||||
+func parseToGrpcRoute(device string, route *netlink.Route, add bool) (r *vcTypes.Route) {
|
|
||||||
+ r = &vcTypes.Route{
|
|
||||||
+ Device: device,
|
|
||||||
+ }
|
|
||||||
+ if route.Dst != nil && route.Dst.String() != "<nil>" {
|
|
||||||
+ r.Dest = route.Dst.String()
|
|
||||||
+ if add == false {
|
|
||||||
+ r.Dest = "-" + route.Dst.String()
|
|
||||||
+ }
|
|
||||||
+ } else if route.Dst == nil {
|
|
||||||
+ r.Dest = "default"
|
|
||||||
+ if add == false {
|
|
||||||
+ r.Dest = "-default"
|
|
||||||
+ }
|
|
||||||
+ }
|
|
||||||
+ if route.Gw != nil && route.Gw.String() != "<nil>" {
|
|
||||||
+ r.Gateway = route.Gw.String()
|
|
||||||
+ }
|
|
||||||
+
|
|
||||||
+ return r
|
|
||||||
+}
|
|
||||||
+
|
|
||||||
+func removeRoutes(ns *NetworkNamespace, route *vcTypes.Route) (removed []*vcTypes.Route, err error) {
|
|
||||||
+ del, err := generateRmRoute(route)
|
|
||||||
+ if err != nil {
|
|
||||||
+ return nil, err
|
|
||||||
+ }
|
|
||||||
+
|
|
||||||
+ // remove the lo device related routes
|
|
||||||
+ if route.Device == localHostDeviceName {
|
|
||||||
+ removed = append(removed, parseToGrpcRoute(localHostDeviceName, del, false))
|
|
||||||
+
|
|
||||||
+ return removed, nil
|
|
||||||
+ }
|
|
||||||
+
|
|
||||||
+ for _, ep := range ns.Endpoints {
|
|
||||||
+ // if device is empty, means search every device
|
|
||||||
+ if route.Device != "" && ep.Name() != route.Device {
|
|
||||||
+ continue
|
|
||||||
+ }
|
|
||||||
+
|
|
||||||
+ netInfo := ep.Properties()
|
|
||||||
+ for i, exist := range ep.Properties().Routes {
|
|
||||||
+ if isSameRoute(&exist, del, true) {
|
|
||||||
+ // need remove
|
|
||||||
+ netInfo.Routes = append(netInfo.Routes[:i], netInfo.Routes[i+1:]...)
|
|
||||||
+ ep.SetProperties(netInfo)
|
|
||||||
+ dev := route.Device
|
|
||||||
+ if route.Device == "" {
|
|
||||||
+ dev = netInfo.Iface.Name
|
|
||||||
+ }
|
|
||||||
+ removed = append(removed, parseToGrpcRoute(dev, del, false))
|
|
||||||
+ }
|
|
||||||
+ }
|
|
||||||
+ }
|
|
||||||
+
|
|
||||||
+ return removed, nil
|
|
||||||
+}
|
|
||||||
+
|
|
||||||
func updateRoute(ns *NetworkNamespace, route *vcTypes.Route, op vcTypes.NetworkOp) ([]*vcTypes.Route, error) {
|
|
||||||
var updRoutes []*vcTypes.Route
|
|
||||||
|
|
||||||
@@ -1645,5 +1749,16 @@ func updateRoute(ns *NetworkNamespace, route *vcTypes.Route, op vcTypes.NetworkO
|
|
||||||
updRoutes = append(updRoutes, added)
|
|
||||||
}
|
|
||||||
|
|
||||||
+ if op == vcTypes.NetworkOpRemove {
|
|
||||||
+ removed, err := removeRoutes(ns, route)
|
|
||||||
+ if err != nil {
|
|
||||||
+ return nil, err
|
|
||||||
+ }
|
|
||||||
+ if len(removed) <= 0 {
|
|
||||||
+ return nil, fmt.Errorf("route of device %s with destination %s is not found", route.Device, route.Dest)
|
|
||||||
+ }
|
|
||||||
+ updRoutes = removed
|
|
||||||
+ }
|
|
||||||
+
|
|
||||||
return updRoutes, nil
|
|
||||||
}
|
|
||||||
--
|
|
||||||
2.14.3 (Apple Git-98)
|
|
||||||
|
|
||||||
@ -1,104 +0,0 @@
|
|||||||
From 9cf769178b8f73c7fd624895589e918d6c7b0645 Mon Sep 17 00:00:00 2001
|
|
||||||
From: jiangpengfei <jiangpengfei9@huawei.com>
|
|
||||||
Date: Mon, 17 Aug 2020 16:29:17 +0800
|
|
||||||
Subject: [PATCH 30/50] network: kata-network list-routes support display
|
|
||||||
compatible format
|
|
||||||
|
|
||||||
reason: kata-network list-routes subcommand support display compatible
|
|
||||||
format when enable_compat_old_cni config is enabled.
|
|
||||||
|
|
||||||
Signed-off-by: jiangpengfei <jiangpengfei9@huawei.com>
|
|
||||||
---
|
|
||||||
virtcontainers/api.go | 13 ++++++++++++-
|
|
||||||
virtcontainers/network.go | 29 +++++++++++++++++++++++++++++
|
|
||||||
virtcontainers/pkg/types/types.go | 10 +++++-----
|
|
||||||
3 files changed, 46 insertions(+), 6 deletions(-)
|
|
||||||
|
|
||||||
diff --git a/virtcontainers/api.go b/virtcontainers/api.go
|
|
||||||
index 2331eb94..a4bf41bb 100644
|
|
||||||
--- a/virtcontainers/api.go
|
|
||||||
+++ b/virtcontainers/api.go
|
|
||||||
@@ -1003,7 +1003,18 @@ func ListRoutes(ctx context.Context, sandboxID string) ([]*vcTypes.Route, error)
|
|
||||||
}
|
|
||||||
defer s.releaseStatelessSandbox()
|
|
||||||
|
|
||||||
- return s.ListRoutes()
|
|
||||||
+ routes, err := s.ListRoutes()
|
|
||||||
+ if err != nil {
|
|
||||||
+ return nil, err
|
|
||||||
+ }
|
|
||||||
+
|
|
||||||
+ // If enable_compat_old_cni is enabled, convert routes info to
|
|
||||||
+ // compatible display format
|
|
||||||
+ if s.config.NetworkConfig.EnableCompatOldCNI {
|
|
||||||
+ return convertToDisplayRoutes(&routes), nil
|
|
||||||
+ }
|
|
||||||
+
|
|
||||||
+ return routes, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// CleanupContaienr is used by shimv2 to stop and delete a container exclusively, once there is no container
|
|
||||||
diff --git a/virtcontainers/network.go b/virtcontainers/network.go
|
|
||||||
index c7066a11..488bd00c 100644
|
|
||||||
--- a/virtcontainers/network.go
|
|
||||||
+++ b/virtcontainers/network.go
|
|
||||||
@@ -1384,6 +1384,35 @@ func convertToCompatInterfaces(es *[]Endpoint) []*vcTypes.Interface {
|
|
||||||
return infs
|
|
||||||
}
|
|
||||||
|
|
||||||
+// convertToDisplayRoutes convert the default route format to more simple
|
|
||||||
+// route display format, it is called when enable_compat_old_cni config
|
|
||||||
+// is enabled.
|
|
||||||
+func convertToDisplayRoutes(routes *[]*vcTypes.Route) []*vcTypes.Route {
|
|
||||||
+ var displayRoutes []*vcTypes.Route
|
|
||||||
+ if routes == nil {
|
|
||||||
+ return displayRoutes
|
|
||||||
+ }
|
|
||||||
+ for _, r := range *routes {
|
|
||||||
+ // we don't support IPV6 temporarily, so we need to filter ":" which
|
|
||||||
+ // is the characteristics of IPV6 for those default routes
|
|
||||||
+ if strings.Contains(r.Dest, ":") {
|
|
||||||
+ continue
|
|
||||||
+ }
|
|
||||||
+
|
|
||||||
+ defaultDest := r.Dest
|
|
||||||
+ if r.Dest == "" {
|
|
||||||
+ defaultDest = "default"
|
|
||||||
+ }
|
|
||||||
+ displayRoutes = append(displayRoutes, &vcTypes.Route{
|
|
||||||
+ Dest: defaultDest,
|
|
||||||
+ Gateway: r.Gateway,
|
|
||||||
+ Device: r.Device,
|
|
||||||
+ })
|
|
||||||
+ }
|
|
||||||
+
|
|
||||||
+ return displayRoutes
|
|
||||||
+}
|
|
||||||
+
|
|
||||||
// verifyInterfaceName verifies the interface name valid or not
|
|
||||||
func verifyInterfaceName(name string) error {
|
|
||||||
// verify `Name` before `Tapname` because of the strict rules
|
|
||||||
diff --git a/virtcontainers/pkg/types/types.go b/virtcontainers/pkg/types/types.go
|
|
||||||
index 71fe7fbb..b41b0c75 100644
|
|
||||||
--- a/virtcontainers/pkg/types/types.go
|
|
||||||
+++ b/virtcontainers/pkg/types/types.go
|
|
||||||
@@ -33,11 +33,11 @@ type Interface struct {
|
|
||||||
|
|
||||||
// Route describes a network route.
|
|
||||||
type Route struct {
|
|
||||||
- Dest string
|
|
||||||
- Gateway string
|
|
||||||
- Device string
|
|
||||||
- Source string
|
|
||||||
- Scope uint32
|
|
||||||
+ Dest string `json:"dest,omitempty"`
|
|
||||||
+ Gateway string `json:"gateway,omitempty"`
|
|
||||||
+ Device string `json:"device,omitempty"`
|
|
||||||
+ Source string `json:"source,omitempty"`
|
|
||||||
+ Scope uint32 `json:"scope,omitempty"`
|
|
||||||
}
|
|
||||||
|
|
||||||
//NetworkOp describes network operation
|
|
||||||
--
|
|
||||||
2.14.3 (Apple Git-98)
|
|
||||||
|
|
||||||
@ -1,41 +0,0 @@
|
|||||||
From 326e90f97cf5ace73dff95257f7b4faa43c56f99 Mon Sep 17 00:00:00 2001
|
|
||||||
From: holyfei <yangfeiyu20092010@163.com>
|
|
||||||
Date: Mon, 17 Aug 2020 16:11:35 +0800
|
|
||||||
Subject: [PATCH 31/50] device_mangaer: check VFIO when create device
|
|
||||||
|
|
||||||
reason: check VFIO when create device, block device can be
|
|
||||||
reused and VFIO device can not
|
|
||||||
|
|
||||||
Signed-off-by: yangfeiyu <yangfeiyu2@huawei.com>
|
|
||||||
---
|
|
||||||
virtcontainers/device/manager/manager.go | 7 ++++++-
|
|
||||||
1 file changed, 6 insertions(+), 1 deletion(-)
|
|
||||||
|
|
||||||
diff --git a/virtcontainers/device/manager/manager.go b/virtcontainers/device/manager/manager.go
|
|
||||||
index 6a2b665a..d24a29ab 100644
|
|
||||||
--- a/virtcontainers/device/manager/manager.go
|
|
||||||
+++ b/virtcontainers/device/manager/manager.go
|
|
||||||
@@ -9,6 +9,7 @@ package manager
|
|
||||||
import (
|
|
||||||
"encoding/hex"
|
|
||||||
"errors"
|
|
||||||
+ "fmt"
|
|
||||||
"sync"
|
|
||||||
|
|
||||||
"github.com/sirupsen/logrus"
|
|
||||||
@@ -115,7 +116,11 @@ func (dm *deviceManager) createDevice(devInfo config.DeviceInfo) (dev api.Device
|
|
||||||
}()
|
|
||||||
|
|
||||||
if existingDev := dm.findDeviceByMajorMinor(devInfo.Major, devInfo.Minor); existingDev != nil {
|
|
||||||
- return existingDev, nil
|
|
||||||
+ if isVFIO(devInfo.HostPath) {
|
|
||||||
+ return nil, fmt.Errorf("device %s is replicated in the same Pod!", devInfo.ContainerPath)
|
|
||||||
+ } else {
|
|
||||||
+ return existingDev, nil
|
|
||||||
+ }
|
|
||||||
}
|
|
||||||
|
|
||||||
// device ID must be generated by manager instead of device itself
|
|
||||||
--
|
|
||||||
2.14.3 (Apple Git-98)
|
|
||||||
|
|
||||||
@ -1,46 +0,0 @@
|
|||||||
From 77711b531867be899df5d2c59a525ea1b7f0e6ac Mon Sep 17 00:00:00 2001
|
|
||||||
From: jiangpengfei <jiangpengfei9@huawei.com>
|
|
||||||
Date: Mon, 17 Aug 2020 17:14:35 +0800
|
|
||||||
Subject: [PATCH 32/50] network: add more detail usage for update-routes
|
|
||||||
subcommand
|
|
||||||
|
|
||||||
reason: add more detail usage for update-routes subcommand
|
|
||||||
to explain how to use update-routes subcommand
|
|
||||||
|
|
||||||
Signed-off-by: jiangpengfei <jiangpengfei9@huawei.com>
|
|
||||||
---
|
|
||||||
cli/network.go | 18 ++++++++++++++----
|
|
||||||
1 file changed, 14 insertions(+), 4 deletions(-)
|
|
||||||
|
|
||||||
diff --git a/cli/network.go b/cli/network.go
|
|
||||||
index 046d0ee9..3dd0971e 100644
|
|
||||||
--- a/cli/network.go
|
|
||||||
+++ b/cli/network.go
|
|
||||||
@@ -120,10 +120,20 @@ var listIfacesCommand = cli.Command{
|
|
||||||
}
|
|
||||||
|
|
||||||
var updateRoutesCommand = cli.Command{
|
|
||||||
- Name: "update-routes",
|
|
||||||
- Usage: "update routes of a container",
|
|
||||||
- ArgsUsage: `update-routes <container-id> file or - for stdin`,
|
|
||||||
- Flags: []cli.Flag{},
|
|
||||||
+ Name: "update-routes",
|
|
||||||
+ Usage: "update routes of a container",
|
|
||||||
+ ArgsUsage: `update-routes <container-id> file or - for stdin
|
|
||||||
+ file or stdin for example:
|
|
||||||
+ [
|
|
||||||
+ {
|
|
||||||
+ "dest":"<[<ip>[/mask] | "default" ]>",
|
|
||||||
+ "gateway":"[ip]",
|
|
||||||
+ "device":"[tap-name]",
|
|
||||||
+ "source": "[source]",
|
|
||||||
+ "scope":[scope]
|
|
||||||
+ }
|
|
||||||
+ ]`,
|
|
||||||
+ Flags: []cli.Flag{},
|
|
||||||
Action: func(context *cli.Context) error {
|
|
||||||
ctx, err := cliContextToContext(context)
|
|
||||||
if err != nil {
|
|
||||||
--
|
|
||||||
2.14.3 (Apple Git-98)
|
|
||||||
|
|
||||||
File diff suppressed because it is too large
Load Diff
@ -1,234 +0,0 @@
|
|||||||
From cf2b34f477cba88641de3719bf5c8f933b919bcc Mon Sep 17 00:00:00 2001
|
|
||||||
From: holyfei <yangfeiyu20092010@163.com>
|
|
||||||
Date: Mon, 17 Aug 2020 21:44:57 +0800
|
|
||||||
Subject: [PATCH 35/50] device: mount blockdevices in the guest VM
|
|
||||||
|
|
||||||
reason: support mount blockdevices in the guest VM like
|
|
||||||
"-v /dev/blockdevice:/home/test"
|
|
||||||
|
|
||||||
Signed-off-by: yangfeiyu <yangfeiyu2@huawei.com>
|
|
||||||
---
|
|
||||||
cli/config/configuration-qemu.toml.in | 5 ++++
|
|
||||||
pkg/katautils/config.go | 6 +++++
|
|
||||||
virtcontainers/kata_agent.go | 27 ++++++++++++++++---
|
|
||||||
virtcontainers/kata_agent_test.go | 2 +-
|
|
||||||
virtcontainers/utils/utils.go | 50 +++++++++++++++++++++++++++++++++++
|
|
||||||
virtcontainers/vm_test.go | 2 +-
|
|
||||||
6 files changed, 86 insertions(+), 6 deletions(-)
|
|
||||||
|
|
||||||
diff --git a/cli/config/configuration-qemu.toml.in b/cli/config/configuration-qemu.toml.in
|
|
||||||
index 46f8b632..aa11b38f 100644
|
|
||||||
--- a/cli/config/configuration-qemu.toml.in
|
|
||||||
+++ b/cli/config/configuration-qemu.toml.in
|
|
||||||
@@ -391,6 +391,11 @@ path = "@SHIMPATH@"
|
|
||||||
#
|
|
||||||
kernel_modules=[]
|
|
||||||
|
|
||||||
+# If enabled, when we pass a block device to the guest VM
|
|
||||||
+# through -v, such as `docker run -v /dev/loop100:/foo/bar`,
|
|
||||||
+# agent will create a directory in guest VM and mount the
|
|
||||||
+# file system on `/dev/loop100` inside the container
|
|
||||||
+enable_blk_mount = true
|
|
||||||
|
|
||||||
[netmon]
|
|
||||||
# If enabled, the network monitoring process gets started when the
|
|
||||||
diff --git a/pkg/katautils/config.go b/pkg/katautils/config.go
|
|
||||||
index 94c916a0..51120311 100644
|
|
||||||
--- a/pkg/katautils/config.go
|
|
||||||
+++ b/pkg/katautils/config.go
|
|
||||||
@@ -160,6 +160,7 @@ type agent struct {
|
|
||||||
TraceMode string `toml:"trace_mode"`
|
|
||||||
TraceType string `toml:"trace_type"`
|
|
||||||
KernelModules []string `toml:"kernel_modules"`
|
|
||||||
+ MountBlkInVM bool `toml:"enable_blk_mount"`
|
|
||||||
}
|
|
||||||
|
|
||||||
type netmon struct {
|
|
||||||
@@ -463,6 +464,10 @@ func (h hypervisor) getInitrdAndImage() (initrd string, image string, err error)
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
+func (a agent) mountBlkDevInVM() bool {
|
|
||||||
+ return a.MountBlkInVM
|
|
||||||
+}
|
|
||||||
+
|
|
||||||
func (p proxy) path() (string, error) {
|
|
||||||
path := p.Path
|
|
||||||
if path == "" {
|
|
||||||
@@ -978,6 +983,7 @@ func updateRuntimeConfigAgent(configPath string, tomlConf tomlConfig, config *oc
|
|
||||||
TraceMode: agent.traceMode(),
|
|
||||||
TraceType: agent.traceType(),
|
|
||||||
KernelModules: agent.kernelModules(),
|
|
||||||
+ MountBlkInVM: agent.mountBlkDevInVM(),
|
|
||||||
}
|
|
||||||
default:
|
|
||||||
return fmt.Errorf("%s agent type is not supported", k)
|
|
||||||
diff --git a/virtcontainers/kata_agent.go b/virtcontainers/kata_agent.go
|
|
||||||
index 16662949..b0f88c15 100644
|
|
||||||
--- a/virtcontainers/kata_agent.go
|
|
||||||
+++ b/virtcontainers/kata_agent.go
|
|
||||||
@@ -30,6 +30,7 @@ import (
|
|
||||||
ns "github.com/kata-containers/runtime/virtcontainers/pkg/nsenter"
|
|
||||||
"github.com/kata-containers/runtime/virtcontainers/pkg/rootless"
|
|
||||||
vcTypes "github.com/kata-containers/runtime/virtcontainers/pkg/types"
|
|
||||||
+ "github.com/kata-containers/runtime/virtcontainers/utils"
|
|
||||||
"github.com/kata-containers/runtime/virtcontainers/pkg/uuid"
|
|
||||||
"github.com/kata-containers/runtime/virtcontainers/store"
|
|
||||||
"github.com/kata-containers/runtime/virtcontainers/types"
|
|
||||||
@@ -64,6 +65,7 @@ var (
|
|
||||||
errorMissingOCISpec = errors.New("Missing OCI specification")
|
|
||||||
defaultKataHostSharedDir = "/run/kata-containers/shared/sandboxes/"
|
|
||||||
defaultKataGuestSharedDir = "/run/kata-containers/shared/containers/"
|
|
||||||
+ kataGuestStorageDir = "/run/kata-containers/storage/containers/"
|
|
||||||
mountGuestTag = "kataShared"
|
|
||||||
defaultKataGuestSandboxDir = "/run/kata-containers/sandbox/"
|
|
||||||
type9pFs = "9p"
|
|
||||||
@@ -199,6 +201,7 @@ type KataAgentConfig struct {
|
|
||||||
TraceMode string
|
|
||||||
TraceType string
|
|
||||||
KernelModules []string
|
|
||||||
+ MountBlkInVM bool
|
|
||||||
}
|
|
||||||
|
|
||||||
// KataAgentState is the structure describing the data stored from this
|
|
||||||
@@ -1061,7 +1064,12 @@ func (k *kataAgent) replaceOCIMountsForStorages(spec *specs.Spec, volumeStorages
|
|
||||||
// Create a temporary location to mount the Storage. Mounting to the correct location
|
|
||||||
// will be handled by the OCI mount structure.
|
|
||||||
filename := fmt.Sprintf("%s-%s", uuid.Generate().String(), filepath.Base(m.Destination))
|
|
||||||
- path := filepath.Join(kataGuestSandboxStorageDir(), filename)
|
|
||||||
+ var path string
|
|
||||||
+ if v.Driver == kataBlkDevType || v.Driver == kataSCSIDevType {
|
|
||||||
+ path = filepath.Join(kataGuestStorageDir, filename)
|
|
||||||
+ } else {
|
|
||||||
+ path = filepath.Join(kataGuestSandboxStorageDir(), filename)
|
|
||||||
+ }
|
|
||||||
|
|
||||||
k.Logger().Debugf("Replacing OCI mount source (%s) with %s", m.Source, path)
|
|
||||||
ociMounts[index].Source = path
|
|
||||||
@@ -1436,7 +1444,7 @@ func (k *kataAgent) createContainer(sandbox *Sandbox, c *Container) (p *Process,
|
|
||||||
// Note this call modifies the list of container devices to make sure
|
|
||||||
// all hotplugged devices are unplugged, so this needs be done
|
|
||||||
// after devices passed with --device are handled.
|
|
||||||
- volumeStorages, err := k.handleBlockVolumes(c)
|
|
||||||
+ volumeStorages, err := k.handleBlockVolumes(sandbox, c)
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
@@ -1609,7 +1617,7 @@ func (k *kataAgent) handleVhostUserBlkVolume(c *Container, device api.Device) (*
|
|
||||||
|
|
||||||
// handleBlockVolumes handles volumes that are block devices files
|
|
||||||
// by passing the block devices as Storage to the agent.
|
|
||||||
-func (k *kataAgent) handleBlockVolumes(c *Container) ([]*grpc.Storage, error) {
|
|
||||||
+func (k *kataAgent) handleBlockVolumes(sandbox *Sandbox, c *Container) ([]*grpc.Storage, error) {
|
|
||||||
|
|
||||||
var volumeStorages []*grpc.Storage
|
|
||||||
|
|
||||||
@@ -1648,9 +1656,20 @@ func (k *kataAgent) handleBlockVolumes(c *Container) ([]*grpc.Storage, error) {
|
|
||||||
}
|
|
||||||
|
|
||||||
vol.MountPoint = m.Destination
|
|
||||||
- if vol.Fstype == "" {
|
|
||||||
+
|
|
||||||
+ ac, _ := sandbox.config.AgentConfig.(KataAgentConfig)
|
|
||||||
+ if ac.MountBlkInVM {
|
|
||||||
+ // Ensure the block device is formatted, for the devices here are specified as volumes
|
|
||||||
+ fsType, gerr := utils.GetDevFormat(m.Source)
|
|
||||||
+ if gerr != nil || fsType == "" {
|
|
||||||
+ k.Logger().WithField("device", id).WithError(gerr).Error("get device format failed")
|
|
||||||
+ return nil, gerr
|
|
||||||
+ }
|
|
||||||
+ vol.Fstype = fmt.Sprintf("bind-%s", fsType)
|
|
||||||
+ } else {
|
|
||||||
vol.Fstype = "bind"
|
|
||||||
}
|
|
||||||
+
|
|
||||||
if len(vol.Options) == 0 {
|
|
||||||
vol.Options = []string{"bind"}
|
|
||||||
}
|
|
||||||
diff --git a/virtcontainers/kata_agent_test.go b/virtcontainers/kata_agent_test.go
|
|
||||||
index 18a5a0a6..68caaab2 100644
|
|
||||||
--- a/virtcontainers/kata_agent_test.go
|
|
||||||
+++ b/virtcontainers/kata_agent_test.go
|
|
||||||
@@ -446,7 +446,7 @@ func TestHandleBlockVolume(t *testing.T) {
|
|
||||||
containers[c.id].sandbox = &sandbox
|
|
||||||
containers[c.id].mounts = mounts
|
|
||||||
|
|
||||||
- volumeStorages, err := k.handleBlockVolumes(c)
|
|
||||||
+ volumeStorages, err := k.handleBlockVolumes(&sandbox, c)
|
|
||||||
assert.Nil(t, err, "Error while handling block volumes")
|
|
||||||
|
|
||||||
vStorage := &pb.Storage{
|
|
||||||
diff --git a/virtcontainers/utils/utils.go b/virtcontainers/utils/utils.go
|
|
||||||
index 9490faa1..36ac67a7 100644
|
|
||||||
--- a/virtcontainers/utils/utils.go
|
|
||||||
+++ b/virtcontainers/utils/utils.go
|
|
||||||
@@ -379,3 +379,53 @@ func RoundVCPUNumber(value string) (int, error) {
|
|
||||||
cpus := int(math.Ceil(cpuNum))
|
|
||||||
return cpus, nil
|
|
||||||
}
|
|
||||||
+
|
|
||||||
+// GetDevFormat get the formated filesystem of input disk use `blkid`, Such as `ext4`,`xfs`,etc.
|
|
||||||
+func GetDevFormat(disk string) (string, error) {
|
|
||||||
+ // refer to https://github.com/kubernetes/kubernetes/blob/v1.12.2/pkg/util/mount/mount_linux.go#L512
|
|
||||||
+ args := []string{"-p", "-s", "TYPE", "-s", "PTTYPE", "-o", "export", disk}
|
|
||||||
+ dataOut, err := exec.Command("blkid", args...).Output()
|
|
||||||
+ output := string(dataOut)
|
|
||||||
+
|
|
||||||
+ if err != nil {
|
|
||||||
+ if strings.Contains(err.Error(), "exit status 2") {
|
|
||||||
+ // Disk device is unformatted.
|
|
||||||
+ // For `blkid`, if the specified token (TYPE/PTTYPE, etc) was
|
|
||||||
+ // not found, or no (specified) devices could be identified, an
|
|
||||||
+ // exit code of 2 is returned.
|
|
||||||
+ return "", nil
|
|
||||||
+ }
|
|
||||||
+ return "", err
|
|
||||||
+ }
|
|
||||||
+
|
|
||||||
+ var fstype string
|
|
||||||
+
|
|
||||||
+ lines := strings.Split(output, "\n")
|
|
||||||
+ for _, l := range lines {
|
|
||||||
+ if len(l) <= 0 {
|
|
||||||
+ // Ignore empty line.
|
|
||||||
+ continue
|
|
||||||
+ }
|
|
||||||
+ // if we use busybox as rootfs,the output of command
|
|
||||||
+ // `busybox blkid` is different with original`blkid`,
|
|
||||||
+ // so we should make a compatible parse
|
|
||||||
+ subLine := strings.Split(l, " ")
|
|
||||||
+ for _, sl := range subLine {
|
|
||||||
+ if len(subLine) <= 0 {
|
|
||||||
+ continue
|
|
||||||
+ }
|
|
||||||
+ cs := strings.Split(sl, "=")
|
|
||||||
+ if len(cs) != 2 {
|
|
||||||
+ continue
|
|
||||||
+ }
|
|
||||||
+ if cs[0] == "TYPE" {
|
|
||||||
+ fstype = cs[1]
|
|
||||||
+ if strings.Contains(fstype, `"`) {
|
|
||||||
+ fstype = strings.Replace(fstype, `"`, "", -1)
|
|
||||||
+ }
|
|
||||||
+ }
|
|
||||||
+ }
|
|
||||||
+ }
|
|
||||||
+
|
|
||||||
+ return fstype, nil
|
|
||||||
+}
|
|
||||||
diff --git a/virtcontainers/vm_test.go b/virtcontainers/vm_test.go
|
|
||||||
index 36fd5c2f..d2414232 100644
|
|
||||||
--- a/virtcontainers/vm_test.go
|
|
||||||
+++ b/virtcontainers/vm_test.go
|
|
||||||
@@ -125,7 +125,7 @@ func TestVMConfigGrpc(t *testing.T) {
|
|
||||||
HypervisorType: QemuHypervisor,
|
|
||||||
HypervisorConfig: newQemuConfig(),
|
|
||||||
AgentType: KataContainersAgent,
|
|
||||||
- AgentConfig: KataAgentConfig{false, true, false, false, 0, "", "", []string{}},
|
|
||||||
+ AgentConfig: KataAgentConfig{false, true, false, false, 0, "", "", []string{}, false },
|
|
||||||
ProxyType: NoopProxyType,
|
|
||||||
}
|
|
||||||
|
|
||||||
--
|
|
||||||
2.14.3 (Apple Git-98)
|
|
||||||
|
|
||||||
@ -1,131 +0,0 @@
|
|||||||
From 448bb661d6759c1e20c18084604b150bff08ada4 Mon Sep 17 00:00:00 2001
|
|
||||||
From: holyfei <yangfeiyu20092010@163.com>
|
|
||||||
Date: Tue, 18 Aug 2020 11:49:26 +0800
|
|
||||||
Subject: [PATCH 36/50] mount: limit the maximum number of virtio-scsi bus
|
|
||||||
slots
|
|
||||||
|
|
||||||
reason:
|
|
||||||
1. add SCSIBus functions for add or remove device about ScsiBus
|
|
||||||
2. limit the maximum number of virtio-scsi bus slots to 25
|
|
||||||
|
|
||||||
Signed-off-by: yangfeiyu <yangfeiyu2@huawei.com>
|
|
||||||
---
|
|
||||||
virtcontainers/qemu.go | 20 +++++++++++++++++
|
|
||||||
virtcontainers/types/scsi.go | 53 ++++++++++++++++++++++++++++++++++++++++++++
|
|
||||||
2 files changed, 73 insertions(+)
|
|
||||||
create mode 100644 virtcontainers/types/scsi.go
|
|
||||||
|
|
||||||
diff --git a/virtcontainers/qemu.go b/virtcontainers/qemu.go
|
|
||||||
index bb83b1bb..c2b65376 100644
|
|
||||||
--- a/virtcontainers/qemu.go
|
|
||||||
+++ b/virtcontainers/qemu.go
|
|
||||||
@@ -65,6 +65,7 @@ type CPUDevice struct {
|
|
||||||
// QemuState keeps Qemu's state
|
|
||||||
type QemuState struct {
|
|
||||||
Bridges []types.Bridge
|
|
||||||
+ ScsiBus *types.SCSIBus
|
|
||||||
// HotpluggedCPUs is the list of CPUs that were hot-added
|
|
||||||
HotpluggedVCPUs []CPUDevice
|
|
||||||
HotpluggedMemory int
|
|
||||||
@@ -411,6 +412,13 @@ func (q *qemu) buildDevices(initrdPath string) ([]govmmQemu.Device, *govmmQemu.I
|
|
||||||
|
|
||||||
var ioThread *govmmQemu.IOThread
|
|
||||||
if q.config.BlockDeviceDriver == config.VirtioSCSI {
|
|
||||||
+ if q.state.ScsiBus == nil {
|
|
||||||
+ // only use `scsi0.0`
|
|
||||||
+ q.state.ScsiBus = &types.SCSIBus{
|
|
||||||
+ Address: make(map[uint32]int),
|
|
||||||
+ ID: fmt.Sprintf("%s.0", scsiControllerID),
|
|
||||||
+ }
|
|
||||||
+ }
|
|
||||||
return q.arch.appendSCSIController(devices, q.config.EnableIOThreads)
|
|
||||||
}
|
|
||||||
|
|
||||||
@@ -1168,6 +1176,16 @@ func (q *qemu) hotplugAddBlockDevice(drive *config.BlockDrive, op operation, dev
|
|
||||||
|
|
||||||
// Bus exposed by the SCSI Controller
|
|
||||||
bus := scsiControllerID + ".0"
|
|
||||||
+ var err error
|
|
||||||
+ if err = q.state.ScsiBus.AddDevToScsiBus(drive.Index); err != nil {
|
|
||||||
+ return err
|
|
||||||
+ }
|
|
||||||
+
|
|
||||||
+ defer func() {
|
|
||||||
+ if err != nil {
|
|
||||||
+ q.state.ScsiBus.RemoveDevFromScsiBus(drive.Index)
|
|
||||||
+ }
|
|
||||||
+ }()
|
|
||||||
|
|
||||||
// Get SCSI-id and LUN based on the order of attaching drives.
|
|
||||||
scsiID, lun, err := utils.GetSCSIIdLun(drive.Index)
|
|
||||||
@@ -1234,6 +1252,8 @@ func (q *qemu) hotplugBlockDevice(drive *config.BlockDrive, op operation) error
|
|
||||||
if err := q.arch.removeDeviceFromBridge(drive.ID); err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
+ } else if q.config.BlockDeviceDriver == config.VirtioSCSI {
|
|
||||||
+ q.state.ScsiBus.RemoveDevFromScsiBus(drive.Index)
|
|
||||||
}
|
|
||||||
|
|
||||||
if err := q.qmpMonitorCh.qmp.ExecuteDeviceDel(q.qmpMonitorCh.ctx, devID); err != nil {
|
|
||||||
diff --git a/virtcontainers/types/scsi.go b/virtcontainers/types/scsi.go
|
|
||||||
new file mode 100644
|
|
||||||
index 00000000..f4d489f0
|
|
||||||
--- /dev/null
|
|
||||||
+++ b/virtcontainers/types/scsi.go
|
|
||||||
@@ -0,0 +1,53 @@
|
|
||||||
+// Copyright (c) Huawei Technologies Co., Ltd. 2019. All rights reserved.
|
|
||||||
+// SPDX-License-Identifier: Apache-2.0
|
|
||||||
+// Description: common functions
|
|
||||||
+// Author: leizhongkai
|
|
||||||
+// Create: 2019-06-30
|
|
||||||
+
|
|
||||||
+package types
|
|
||||||
+
|
|
||||||
+import (
|
|
||||||
+ "fmt"
|
|
||||||
+)
|
|
||||||
+
|
|
||||||
+const (
|
|
||||||
+ scsiMaxCapacity = 25
|
|
||||||
+)
|
|
||||||
+
|
|
||||||
+type SCSIBus struct {
|
|
||||||
+ // Address contains information about devices plugged and number limit
|
|
||||||
+ Address map[uint32]int
|
|
||||||
+
|
|
||||||
+ // ID is used to identify the bus exposed by the SCSI Controller
|
|
||||||
+ ID string
|
|
||||||
+}
|
|
||||||
+
|
|
||||||
+func (sb *SCSIBus) AddDevToScsiBus(index int) error {
|
|
||||||
+ var addr uint32
|
|
||||||
+
|
|
||||||
+ // looking for the first available address
|
|
||||||
+ for i := uint32(1); i <= scsiMaxCapacity; i++ {
|
|
||||||
+ if _, ok := sb.Address[i]; !ok {
|
|
||||||
+ addr = i
|
|
||||||
+ break
|
|
||||||
+ }
|
|
||||||
+ }
|
|
||||||
+
|
|
||||||
+ if addr == 0 {
|
|
||||||
+ return fmt.Errorf("Scsi bus capacity limited.")
|
|
||||||
+ }
|
|
||||||
+
|
|
||||||
+ // save address and device
|
|
||||||
+ sb.Address[addr] = index
|
|
||||||
+
|
|
||||||
+ return nil
|
|
||||||
+}
|
|
||||||
+
|
|
||||||
+func (sb *SCSIBus) RemoveDevFromScsiBus(index int) {
|
|
||||||
+ for addr, i := range sb.Address {
|
|
||||||
+ if i == index {
|
|
||||||
+ delete(sb.Address, addr)
|
|
||||||
+ return
|
|
||||||
+ }
|
|
||||||
+ }
|
|
||||||
+}
|
|
||||||
--
|
|
||||||
2.14.3 (Apple Git-98)
|
|
||||||
|
|
||||||
@ -1,875 +0,0 @@
|
|||||||
From d083f0e0247fbded92a0ae2a0e71da4176baed95 Mon Sep 17 00:00:00 2001
|
|
||||||
From: xiadanni <xiadanni1@huawei.com>
|
|
||||||
Date: Tue, 18 Aug 2020 17:08:23 +0800
|
|
||||||
Subject: [PATCH 37/50] runtime: add IPVS test
|
|
||||||
|
|
||||||
Signed-off-by: xiadanni <xiadanni1@huawei.com>
|
|
||||||
---
|
|
||||||
cli/ipvsadm_test.go | 775 +++++++++++++++++++++++++++++++++++++
|
|
||||||
virtcontainers/api_test.go | 36 ++
|
|
||||||
virtcontainers/kata_agent_test.go | 4 +
|
|
||||||
virtcontainers/pkg/vcmock/types.go | 2 +-
|
|
||||||
4 files changed, 816 insertions(+), 1 deletion(-)
|
|
||||||
create mode 100644 cli/ipvsadm_test.go
|
|
||||||
|
|
||||||
diff --git a/cli/ipvsadm_test.go b/cli/ipvsadm_test.go
|
|
||||||
new file mode 100644
|
|
||||||
index 00000000..92958aee
|
|
||||||
--- /dev/null
|
|
||||||
+++ b/cli/ipvsadm_test.go
|
|
||||||
@@ -0,0 +1,775 @@
|
|
||||||
+// Copyright (c) Huawei Technologies Co., Ltd. 2020. All rights reserved.
|
|
||||||
+// SPDX-License-Identifier: Apache-2.0
|
|
||||||
+// Description: IPVS common functions test
|
|
||||||
+// Author: xiadanni
|
|
||||||
+// Create: 2020-08-01
|
|
||||||
+
|
|
||||||
+package main
|
|
||||||
+
|
|
||||||
+import (
|
|
||||||
+ "context"
|
|
||||||
+ "flag"
|
|
||||||
+ "fmt"
|
|
||||||
+ "os"
|
|
||||||
+ "strings"
|
|
||||||
+ "testing"
|
|
||||||
+
|
|
||||||
+ "github.com/kata-containers/agent/protocols/grpc"
|
|
||||||
+ vc "github.com/kata-containers/runtime/virtcontainers"
|
|
||||||
+ vcAnnotations "github.com/kata-containers/runtime/virtcontainers/pkg/annotations"
|
|
||||||
+ "github.com/kata-containers/runtime/virtcontainers/pkg/vcmock"
|
|
||||||
+ "github.com/kata-containers/runtime/virtcontainers/types"
|
|
||||||
+ "github.com/stretchr/testify/assert"
|
|
||||||
+ "github.com/urfave/cli"
|
|
||||||
+)
|
|
||||||
+
|
|
||||||
+func TestKataIPVSCliAction(t *testing.T) {
|
|
||||||
+ assert := assert.New(t)
|
|
||||||
+
|
|
||||||
+ actionFunc, ok := kataIPVSCLICommand.Action.(func(ctx *cli.Context) error)
|
|
||||||
+ assert.True(ok)
|
|
||||||
+
|
|
||||||
+ flagSet := flag.NewFlagSet("kata-IPVS", flag.ContinueOnError)
|
|
||||||
+ ctx := createCLIContext(flagSet)
|
|
||||||
+
|
|
||||||
+ err := actionFunc(ctx)
|
|
||||||
+ assert.NoError(err)
|
|
||||||
+}
|
|
||||||
+
|
|
||||||
+func TestIPVSadmCliAction(t *testing.T) {
|
|
||||||
+ assert := assert.New(t)
|
|
||||||
+
|
|
||||||
+ actionFunc, ok := IPVSadmCommand.Action.(func(ctx *cli.Context) error)
|
|
||||||
+ assert.True(ok)
|
|
||||||
+
|
|
||||||
+ flagSet := flag.NewFlagSet("IPVSadm", flag.ContinueOnError)
|
|
||||||
+ ctx := createCLIContext(flagSet)
|
|
||||||
+
|
|
||||||
+ err := actionFunc(ctx)
|
|
||||||
+ assert.Error(err, "Missing container ID")
|
|
||||||
+}
|
|
||||||
+
|
|
||||||
+func TestIPVSadmCLISuccessful(t *testing.T) {
|
|
||||||
+ assert := assert.New(t)
|
|
||||||
+
|
|
||||||
+ sandbox := &vcmock.Sandbox{
|
|
||||||
+ MockID: testContainerID,
|
|
||||||
+ }
|
|
||||||
+
|
|
||||||
+ sandbox.MockContainers = []*vcmock.Container{
|
|
||||||
+ {
|
|
||||||
+ MockID: sandbox.ID(),
|
|
||||||
+ MockSandbox: sandbox,
|
|
||||||
+ },
|
|
||||||
+ }
|
|
||||||
+
|
|
||||||
+ testingImpl.StatusContainerFunc = func(ctx context.Context, sandboxID, containerID string) (vc.ContainerStatus, error) {
|
|
||||||
+ return vc.ContainerStatus{
|
|
||||||
+ ID: sandbox.ID(),
|
|
||||||
+ Annotations: map[string]string{
|
|
||||||
+ vcAnnotations.ContainerTypeKey: string(vc.PodContainer),
|
|
||||||
+ },
|
|
||||||
+ State: types.ContainerState{
|
|
||||||
+ State: types.StateRunning,
|
|
||||||
+ },
|
|
||||||
+ }, nil
|
|
||||||
+ }
|
|
||||||
+
|
|
||||||
+ testingImpl.UpdateIPVSRuleFunc = func(ctx context.Context, sandboxID string, IPVS *grpc.UpdateIPVSRequest) (*grpc.IPVSResponse, error) {
|
|
||||||
+ return &grpc.IPVSResponse{}, nil
|
|
||||||
+ }
|
|
||||||
+
|
|
||||||
+ defer func() {
|
|
||||||
+ testingImpl.StatusContainerFunc = nil
|
|
||||||
+ testingImpl.UpdateIPVSRuleFunc = nil
|
|
||||||
+ }()
|
|
||||||
+
|
|
||||||
+ path, err := createTempContainerIDMapping(sandbox.ID(), sandbox.ID())
|
|
||||||
+ assert.NoError(err)
|
|
||||||
+ defer os.RemoveAll(path)
|
|
||||||
+
|
|
||||||
+ actionFunc, ok := IPVSadmCommand.Action.(func(ctx *cli.Context) error)
|
|
||||||
+ assert.True(ok)
|
|
||||||
+
|
|
||||||
+ flagSet := flag.NewFlagSet("IPVSadm", flag.ContinueOnError)
|
|
||||||
+ flagSet.Parse([]string{testContainerID})
|
|
||||||
+ flagSet.String("parameters", "-a -t 192.168.0.7:80 -r 192.168.0.4:80", "")
|
|
||||||
+ ctx := createCLIContext(flagSet)
|
|
||||||
+ err = actionFunc(ctx)
|
|
||||||
+ assert.NoError(err)
|
|
||||||
+
|
|
||||||
+ // result not nil
|
|
||||||
+ testingImpl.UpdateIPVSRuleFunc = func(ctx context.Context, sandboxID string, IPVS *grpc.UpdateIPVSRequest) (*grpc.IPVSResponse, error) {
|
|
||||||
+ return &grpc.IPVSResponse{IPVSRes: "IPVS rule updating success"}, nil
|
|
||||||
+ }
|
|
||||||
+
|
|
||||||
+ err = actionFunc(ctx)
|
|
||||||
+ assert.NoError(err)
|
|
||||||
+}
|
|
||||||
+
|
|
||||||
+func TestIPVSadmCLIError(t *testing.T) {
|
|
||||||
+ assert := assert.New(t)
|
|
||||||
+
|
|
||||||
+ sandbox := &vcmock.Sandbox{
|
|
||||||
+ MockID: testContainerID,
|
|
||||||
+ }
|
|
||||||
+
|
|
||||||
+ sandbox.MockContainers = []*vcmock.Container{
|
|
||||||
+ {
|
|
||||||
+ MockID: sandbox.ID(),
|
|
||||||
+ MockSandbox: sandbox,
|
|
||||||
+ },
|
|
||||||
+ }
|
|
||||||
+
|
|
||||||
+ testingImpl.StatusContainerFunc = func(ctx context.Context, sandboxID, containerID string) (vc.ContainerStatus, error) {
|
|
||||||
+ return vc.ContainerStatus{
|
|
||||||
+ ID: sandbox.ID(),
|
|
||||||
+ Annotations: map[string]string{
|
|
||||||
+ vcAnnotations.ContainerTypeKey: string(vc.PodContainer),
|
|
||||||
+ },
|
|
||||||
+ State: types.ContainerState{
|
|
||||||
+ State: types.StateRunning,
|
|
||||||
+ },
|
|
||||||
+ }, nil
|
|
||||||
+ }
|
|
||||||
+
|
|
||||||
+ testingImpl.UpdateIPVSRuleFunc = func(ctx context.Context, sandboxID string, IPVS *grpc.UpdateIPVSRequest) (*grpc.IPVSResponse, error) {
|
|
||||||
+ return &grpc.IPVSResponse{}, nil
|
|
||||||
+ }
|
|
||||||
+
|
|
||||||
+ defer func() {
|
|
||||||
+ testingImpl.StatusContainerFunc = nil
|
|
||||||
+ testingImpl.UpdateIPVSRuleFunc = nil
|
|
||||||
+ }()
|
|
||||||
+
|
|
||||||
+ path, err := createTempContainerIDMapping(sandbox.ID(), sandbox.ID())
|
|
||||||
+ assert.NoError(err)
|
|
||||||
+ defer os.RemoveAll(path)
|
|
||||||
+
|
|
||||||
+ actionFunc, ok := IPVSadmCommand.Action.(func(ctx *cli.Context) error)
|
|
||||||
+ assert.True(ok)
|
|
||||||
+
|
|
||||||
+ // no stdin rule file
|
|
||||||
+ flagSet := flag.NewFlagSet("IPVSadm", flag.ContinueOnError)
|
|
||||||
+ flagSet.Parse([]string{testContainerID})
|
|
||||||
+ flagSet.String("restore", "-", "")
|
|
||||||
+ ctx := createCLIContext(flagSet)
|
|
||||||
+ err = actionFunc(ctx)
|
|
||||||
+ assert.Error(err)
|
|
||||||
+
|
|
||||||
+ // restore parameter error
|
|
||||||
+ flagSet = flag.NewFlagSet("IPVSadm", flag.ContinueOnError)
|
|
||||||
+ flagSet.Parse([]string{testContainerID})
|
|
||||||
+ flagSet.String("restore", "abc", "")
|
|
||||||
+ ctx = createCLIContext(flagSet)
|
|
||||||
+ err = actionFunc(ctx)
|
|
||||||
+ assert.Error(err)
|
|
||||||
+
|
|
||||||
+ // checkrule returns error
|
|
||||||
+ flagSet = flag.NewFlagSet("IPVSadm", flag.ContinueOnError)
|
|
||||||
+ flagSet.Parse([]string{testContainerID})
|
|
||||||
+ flagSet.String("parameters", "-A -t 192.168.0.7:80 -s rr -p -3000", "")
|
|
||||||
+ ctx = createCLIContext(flagSet)
|
|
||||||
+ err = actionFunc(ctx)
|
|
||||||
+ assert.Error(err)
|
|
||||||
+
|
|
||||||
+ // updatefunction returns error
|
|
||||||
+ testingImpl.UpdateIPVSRuleFunc = func(ctx context.Context, sandboxID string, IPVS *grpc.UpdateIPVSRequest) (*grpc.IPVSResponse, error) {
|
|
||||||
+ return &grpc.IPVSResponse{}, fmt.Errorf("IPVSadm test error")
|
|
||||||
+ }
|
|
||||||
+
|
|
||||||
+ flagSet = flag.NewFlagSet("IPVSadm", flag.ContinueOnError)
|
|
||||||
+ flagSet.Parse([]string{testContainerID})
|
|
||||||
+ flagSet.String("parameters", "-A -t 192.168.0.7:80 -s rr -p 3000", "")
|
|
||||||
+ ctx = createCLIContext(flagSet)
|
|
||||||
+ err = actionFunc(ctx)
|
|
||||||
+ assert.Error(err)
|
|
||||||
+}
|
|
||||||
+
|
|
||||||
+func TestCleanupCliAction(t *testing.T) {
|
|
||||||
+ assert := assert.New(t)
|
|
||||||
+
|
|
||||||
+ actionFunc, ok := CleanupCommand.Action.(func(ctx *cli.Context) error)
|
|
||||||
+ assert.True(ok)
|
|
||||||
+
|
|
||||||
+ flagSet := flag.NewFlagSet("cleanup", flag.ContinueOnError)
|
|
||||||
+ ctx := createCLIContext(flagSet)
|
|
||||||
+
|
|
||||||
+ err := actionFunc(ctx)
|
|
||||||
+ assert.Error(err, "Missing container ID")
|
|
||||||
+}
|
|
||||||
+
|
|
||||||
+func TestCleanupCLISuccessful(t *testing.T) {
|
|
||||||
+ assert := assert.New(t)
|
|
||||||
+
|
|
||||||
+ sandbox := &vcmock.Sandbox{
|
|
||||||
+ MockID: testContainerID,
|
|
||||||
+ }
|
|
||||||
+
|
|
||||||
+ sandbox.MockContainers = []*vcmock.Container{
|
|
||||||
+ {
|
|
||||||
+ MockID: sandbox.ID(),
|
|
||||||
+ MockSandbox: sandbox,
|
|
||||||
+ },
|
|
||||||
+ }
|
|
||||||
+
|
|
||||||
+ testingImpl.StatusContainerFunc = func(ctx context.Context, sandboxID, containerID string) (vc.ContainerStatus, error) {
|
|
||||||
+ return vc.ContainerStatus{
|
|
||||||
+ ID: sandbox.ID(),
|
|
||||||
+ Annotations: map[string]string{
|
|
||||||
+ vcAnnotations.ContainerTypeKey: string(vc.PodContainer),
|
|
||||||
+ },
|
|
||||||
+ State: types.ContainerState{
|
|
||||||
+ State: types.StateRunning,
|
|
||||||
+ },
|
|
||||||
+ }, nil
|
|
||||||
+ }
|
|
||||||
+
|
|
||||||
+ testingImpl.UpdateIPVSRuleFunc = func(ctx context.Context, sandboxID string, IPVS *grpc.UpdateIPVSRequest) (*grpc.IPVSResponse, error) {
|
|
||||||
+ return &grpc.IPVSResponse{}, nil
|
|
||||||
+ }
|
|
||||||
+
|
|
||||||
+ defer func() {
|
|
||||||
+ testingImpl.StatusContainerFunc = nil
|
|
||||||
+ testingImpl.UpdateIPVSRuleFunc = nil
|
|
||||||
+ }()
|
|
||||||
+
|
|
||||||
+ path, err := createTempContainerIDMapping(sandbox.ID(), sandbox.ID())
|
|
||||||
+ assert.NoError(err)
|
|
||||||
+ defer os.RemoveAll(path)
|
|
||||||
+
|
|
||||||
+ actionFunc, ok := CleanupCommand.Action.(func(ctx *cli.Context) error)
|
|
||||||
+ assert.True(ok)
|
|
||||||
+
|
|
||||||
+ flagSet := flag.NewFlagSet("cleanup", flag.ContinueOnError)
|
|
||||||
+ flagSet.Parse([]string{testContainerID})
|
|
||||||
+ flagSet.String("parameters", "-d 192.168.0.4 -p tcp", "")
|
|
||||||
+ ctx := createCLIContext(flagSet)
|
|
||||||
+ err = actionFunc(ctx)
|
|
||||||
+ assert.NoError(err)
|
|
||||||
+}
|
|
||||||
+
|
|
||||||
+func TestCleanupCLIError(t *testing.T) {
|
|
||||||
+ assert := assert.New(t)
|
|
||||||
+
|
|
||||||
+ sandbox := &vcmock.Sandbox{
|
|
||||||
+ MockID: testContainerID,
|
|
||||||
+ }
|
|
||||||
+
|
|
||||||
+ sandbox.MockContainers = []*vcmock.Container{
|
|
||||||
+ {
|
|
||||||
+ MockID: sandbox.ID(),
|
|
||||||
+ MockSandbox: sandbox,
|
|
||||||
+ },
|
|
||||||
+ }
|
|
||||||
+
|
|
||||||
+ testingImpl.StatusContainerFunc = func(ctx context.Context, sandboxID, containerID string) (vc.ContainerStatus, error) {
|
|
||||||
+ return vc.ContainerStatus{
|
|
||||||
+ ID: sandbox.ID(),
|
|
||||||
+ Annotations: map[string]string{
|
|
||||||
+ vcAnnotations.ContainerTypeKey: string(vc.PodContainer),
|
|
||||||
+ },
|
|
||||||
+ State: types.ContainerState{
|
|
||||||
+ State: types.StateRunning,
|
|
||||||
+ },
|
|
||||||
+ }, nil
|
|
||||||
+ }
|
|
||||||
+
|
|
||||||
+ testingImpl.UpdateIPVSRuleFunc = func(ctx context.Context, sandboxID string, IPVS *grpc.UpdateIPVSRequest) (*grpc.IPVSResponse, error) {
|
|
||||||
+ return &grpc.IPVSResponse{}, nil
|
|
||||||
+ }
|
|
||||||
+
|
|
||||||
+ defer func() {
|
|
||||||
+ testingImpl.StatusContainerFunc = nil
|
|
||||||
+ testingImpl.UpdateIPVSRuleFunc = nil
|
|
||||||
+ }()
|
|
||||||
+
|
|
||||||
+ path, err := createTempContainerIDMapping(sandbox.ID(), sandbox.ID())
|
|
||||||
+ assert.NoError(err)
|
|
||||||
+ defer os.RemoveAll(path)
|
|
||||||
+
|
|
||||||
+ actionFunc, ok := CleanupCommand.Action.(func(ctx *cli.Context) error)
|
|
||||||
+ assert.True(ok)
|
|
||||||
+
|
|
||||||
+ // checkrule returns error
|
|
||||||
+ flagSet := flag.NewFlagSet("cleanup", flag.ContinueOnError)
|
|
||||||
+ flagSet.Parse([]string{testContainerID})
|
|
||||||
+ flagSet.String("parameters", "-d 192.168.0.4", "")
|
|
||||||
+ ctx := createCLIContext(flagSet)
|
|
||||||
+ err = actionFunc(ctx)
|
|
||||||
+ assert.Error(err)
|
|
||||||
+
|
|
||||||
+ // updatefunction returns error
|
|
||||||
+ testingImpl.UpdateIPVSRuleFunc = func(ctx context.Context, sandboxID string, IPVS *grpc.UpdateIPVSRequest) (*grpc.IPVSResponse, error) {
|
|
||||||
+ return &grpc.IPVSResponse{}, fmt.Errorf("IPVSadm cleanup test error")
|
|
||||||
+ }
|
|
||||||
+
|
|
||||||
+ flagSet = flag.NewFlagSet("cleanup", flag.ContinueOnError)
|
|
||||||
+ flagSet.Parse([]string{testContainerID})
|
|
||||||
+ flagSet.String("parameters", "-d 192.168.0.4 -p tcp", "")
|
|
||||||
+ ctx = createCLIContext(flagSet)
|
|
||||||
+ err = actionFunc(ctx)
|
|
||||||
+ assert.Error(err)
|
|
||||||
+}
|
|
||||||
+
|
|
||||||
+func TestAddServiceSuccessfully(t *testing.T) {
|
|
||||||
+ assert := assert.New(t)
|
|
||||||
+ addServiceCommandTCP := "IPVSadm --add-service --tcp-service 192.168.0.7:80 --scheduler rr --persistent 3000"
|
|
||||||
+ _, err := checkIPVSRule(addServiceCommandTCP)
|
|
||||||
+ assert.NoError(err)
|
|
||||||
+ addServiceCommandUDP := "IPVSadm --add-service --udp-service 192.168.0.7:80 --scheduler rr --persistent 3000"
|
|
||||||
+ _, err = checkIPVSRule(addServiceCommandUDP)
|
|
||||||
+ assert.NoError(err)
|
|
||||||
+ addServiceCmd := "IPVSadm -A -t 192.168.0.7:80 -s rr -p 3000"
|
|
||||||
+ _, err = checkIPVSRule(addServiceCmd)
|
|
||||||
+ assert.NoError(err)
|
|
||||||
+}
|
|
||||||
+
|
|
||||||
+func TestAddServiceNoService(t *testing.T) {
|
|
||||||
+ assert := assert.New(t)
|
|
||||||
+ addServiceCommand := "IPVSadm --add-service --scheduler rr --persistent 3000"
|
|
||||||
+ _, err := checkIPVSRule(addServiceCommand)
|
|
||||||
+ assert.Error(err)
|
|
||||||
+ addServiceCmd := "IPVSadm -A -s rr -p 3000"
|
|
||||||
+ _, err = checkIPVSRule(addServiceCmd)
|
|
||||||
+ assert.Error(err)
|
|
||||||
+}
|
|
||||||
+
|
|
||||||
+func TestAddServiceNoScheduler(t *testing.T) {
|
|
||||||
+ assert := assert.New(t)
|
|
||||||
+ addServiceCommand := "IPVSadm --add-service --tcp-service 192.168.0.7:80 --persistent 3000"
|
|
||||||
+ _, err := checkIPVSRule(addServiceCommand)
|
|
||||||
+ assert.Error(err)
|
|
||||||
+ addServiceCmd := "IPVSadm -A -t 192.168.0.7:80 -p 3000"
|
|
||||||
+ _, err = checkIPVSRule(addServiceCmd)
|
|
||||||
+ assert.Error(err)
|
|
||||||
+}
|
|
||||||
+
|
|
||||||
+func TestAddServiceNoPersistent(t *testing.T) {
|
|
||||||
+ assert := assert.New(t)
|
|
||||||
+ addServiceCommand := "IPVSadm --add-service --tcp-service 192.168.0.7:80 --scheduler rr"
|
|
||||||
+ _, err := checkIPVSRule(addServiceCommand)
|
|
||||||
+ assert.Error(err)
|
|
||||||
+ addServiceCmd := "IPVSadm -A -t 192.168.0.7:80 -s rr"
|
|
||||||
+ _, err = checkIPVSRule(addServiceCmd)
|
|
||||||
+ assert.Error(err)
|
|
||||||
+}
|
|
||||||
+
|
|
||||||
+func TestAddServiceIpError(t *testing.T) {
|
|
||||||
+ assert := assert.New(t)
|
|
||||||
+ addServiceCommand := "IPVSadm --add-service --tcp-service 192.168.2.2.7:80 --scheduler rr --persistent 3000"
|
|
||||||
+ _, err := checkIPVSRule(addServiceCommand)
|
|
||||||
+ assert.Error(err)
|
|
||||||
+ addServiceCmd := "IPVSadm -A -t 192.168.2.2.7:80 -s rr -p 3000"
|
|
||||||
+ _, err = checkIPVSRule(addServiceCmd)
|
|
||||||
+ assert.Error(err)
|
|
||||||
+}
|
|
||||||
+
|
|
||||||
+func TestAddServicePortError(t *testing.T) {
|
|
||||||
+ assert := assert.New(t)
|
|
||||||
+ addServiceCommand := "IPVSadm --add-service --tcp-service 192.168.0.7:9999999 --scheduler rr --persistent 3000"
|
|
||||||
+ _, err := checkIPVSRule(addServiceCommand)
|
|
||||||
+ assert.Error(err)
|
|
||||||
+ addServiceCmd := "IPVSadm -A -t 192.168.0.7:9999999 -s rr -p 3000"
|
|
||||||
+ _, err = checkIPVSRule(addServiceCmd)
|
|
||||||
+ assert.Error(err)
|
|
||||||
+}
|
|
||||||
+
|
|
||||||
+func TestAddServiceSchedulerError(t *testing.T) {
|
|
||||||
+ assert := assert.New(t)
|
|
||||||
+ addServiceCommand := "IPVSadm --add-service --tcp-service 192.168.0.7:80 --scheduler rrr --persistent 3000"
|
|
||||||
+ _, err := checkIPVSRule(addServiceCommand)
|
|
||||||
+ assert.Error(err)
|
|
||||||
+ addServiceCmd := "IPVSadm -A -t 192.168.0.7:80 -s rrr -p 3000"
|
|
||||||
+ _, err = checkIPVSRule(addServiceCmd)
|
|
||||||
+ assert.Error(err)
|
|
||||||
+}
|
|
||||||
+
|
|
||||||
+func TestAddServicePersistentError(t *testing.T) {
|
|
||||||
+ assert := assert.New(t)
|
|
||||||
+ addServiceCommand := "IPVSadm --add-service --tcp-service 192.168.0.7:80 --scheduler rr --persistent 99999999999"
|
|
||||||
+ _, err := checkIPVSRule(addServiceCommand)
|
|
||||||
+ assert.Error(err)
|
|
||||||
+ addServiceCmd := "IPVSadm -A -t 192.168.0.7:80 -s rr -p 99999999999"
|
|
||||||
+ _, err = checkIPVSRule(addServiceCmd)
|
|
||||||
+ assert.Error(err)
|
|
||||||
+}
|
|
||||||
+
|
|
||||||
+func TestEditServiceSuccessfully(t *testing.T) {
|
|
||||||
+ assert := assert.New(t)
|
|
||||||
+ editServiceCommandTCP := "IPVSadm --edit-service --tcp-service 192.168.0.7:80 --scheduler rr --persistent 3000"
|
|
||||||
+ _, err := checkIPVSRule(editServiceCommandTCP)
|
|
||||||
+ assert.NoError(err)
|
|
||||||
+ editServiceCommandUcp := "IPVSadm --edit-service --udp-service 192.168.0.7:80 --scheduler rr --persistent 3000"
|
|
||||||
+ _, err = checkIPVSRule(editServiceCommandUcp)
|
|
||||||
+ assert.NoError(err)
|
|
||||||
+ editServiceCmd := "IPVSadm -A -t 192.168.0.7:80 -s rr -p 3000"
|
|
||||||
+ _, err = checkIPVSRule(editServiceCmd)
|
|
||||||
+ assert.NoError(err)
|
|
||||||
+}
|
|
||||||
+
|
|
||||||
+func TestEditServiceNoService(t *testing.T) {
|
|
||||||
+ assert := assert.New(t)
|
|
||||||
+ editServiceCommand := "IPVSadm --edit-service --scheduler rr --persistent 3000"
|
|
||||||
+ _, err := checkIPVSRule(editServiceCommand)
|
|
||||||
+ assert.Error(err)
|
|
||||||
+ editServiceCmd := "IPVSadm -E -s rr -p 3000"
|
|
||||||
+ _, err = checkIPVSRule(editServiceCmd)
|
|
||||||
+ assert.Error(err)
|
|
||||||
+}
|
|
||||||
+
|
|
||||||
+func TestEditServiceNoScheduler(t *testing.T) {
|
|
||||||
+ assert := assert.New(t)
|
|
||||||
+ editServiceCommand := "IPVSadm --edit-service --tcp-service 192.168.0.7:80 --persistent 3000"
|
|
||||||
+ _, err := checkIPVSRule(editServiceCommand)
|
|
||||||
+ assert.Error(err)
|
|
||||||
+ editServiceCmd := "IPVSadm -E -t 192.168.0.7:80 -p 3000"
|
|
||||||
+ _, err = checkIPVSRule(editServiceCmd)
|
|
||||||
+ assert.Error(err)
|
|
||||||
+}
|
|
||||||
+
|
|
||||||
+func TestEditServiceNoPersistent(t *testing.T) {
|
|
||||||
+ assert := assert.New(t)
|
|
||||||
+ editServiceCommand := "IPVSadm --edit-service --tcp-service 192.168.0.7:80 --scheduler rr"
|
|
||||||
+ _, err := checkIPVSRule(editServiceCommand)
|
|
||||||
+ assert.Error(err)
|
|
||||||
+ editServiceCmd := "IPVSadm -E -t 192.168.0.7:80 -s rr"
|
|
||||||
+ _, err = checkIPVSRule(editServiceCmd)
|
|
||||||
+ assert.Error(err)
|
|
||||||
+}
|
|
||||||
+
|
|
||||||
+func TestEditServiceIpError(t *testing.T) {
|
|
||||||
+ assert := assert.New(t)
|
|
||||||
+ editServiceCommand := "IPVSadm --edit-service --tcp-service 192.168.2.2.7:80 --scheduler rr --persistent 3000"
|
|
||||||
+ _, err := checkIPVSRule(editServiceCommand)
|
|
||||||
+ assert.Error(err)
|
|
||||||
+ editServiceCmd := "IPVSadm -E -t 192.168.2.2.7:80 -s rr -p 3000"
|
|
||||||
+ _, err = checkIPVSRule(editServiceCmd)
|
|
||||||
+ assert.Error(err)
|
|
||||||
+}
|
|
||||||
+
|
|
||||||
+func TestEditServicePortError(t *testing.T) {
|
|
||||||
+ assert := assert.New(t)
|
|
||||||
+ editServiceCommand := "IPVSadm --edit-service --tcp-service 192.168.0.7:9999999 --scheduler rr --persistent 3000"
|
|
||||||
+ _, err := checkIPVSRule(editServiceCommand)
|
|
||||||
+ assert.Error(err)
|
|
||||||
+ editServiceCmd := "IPVSadm -E -t 192.168.0.7:9999999 -s rr -p 3000"
|
|
||||||
+ _, err = checkIPVSRule(editServiceCmd)
|
|
||||||
+ assert.Error(err)
|
|
||||||
+}
|
|
||||||
+
|
|
||||||
+func TestEditServiceSchedulerError(t *testing.T) {
|
|
||||||
+ assert := assert.New(t)
|
|
||||||
+ editServiceCommand := "IPVSadm --edit-service --tcp-service 192.168.0.7:80 --scheduler rrr --persistent 3000"
|
|
||||||
+ _, err := checkIPVSRule(editServiceCommand)
|
|
||||||
+ assert.Error(err)
|
|
||||||
+ editServiceCmd := "IPVSadm -E -t 192.168.0.7:80 -s rrr -p 3000"
|
|
||||||
+ _, err = checkIPVSRule(editServiceCmd)
|
|
||||||
+ assert.Error(err)
|
|
||||||
+}
|
|
||||||
+
|
|
||||||
+func TestEditServicePersistentError(t *testing.T) {
|
|
||||||
+ assert := assert.New(t)
|
|
||||||
+ editServiceCommand := "IPVSadm --edit-service --tcp-service 192.168.0.7:80 --scheduler rr --persistent 99999999999"
|
|
||||||
+ _, err := checkIPVSRule(editServiceCommand)
|
|
||||||
+ assert.Error(err)
|
|
||||||
+ editServiceCmd := "IPVSadm -E -t 192.168.0.7:80 -s rr -p 99999999999"
|
|
||||||
+ _, err = checkIPVSRule(editServiceCmd)
|
|
||||||
+ assert.Error(err)
|
|
||||||
+}
|
|
||||||
+
|
|
||||||
+func TestDeleteServiceSuccessfully(t *testing.T) {
|
|
||||||
+ assert := assert.New(t)
|
|
||||||
+ deleteServiceCommandTCP := "IPVSadm --delete-service --tcp-service 192.168.0.7:80"
|
|
||||||
+ _, err := checkIPVSRule(deleteServiceCommandTCP)
|
|
||||||
+ assert.NoError(err)
|
|
||||||
+ deleteServiceCommandUDP := "IPVSadm --delete-service --udp-service 192.168.0.7:80"
|
|
||||||
+ _, err = checkIPVSRule(deleteServiceCommandUDP)
|
|
||||||
+ assert.NoError(err)
|
|
||||||
+ deleteServiceCmd := "IPVSadm -D -t 192.168.0.7:80"
|
|
||||||
+ _, err = checkIPVSRule(deleteServiceCmd)
|
|
||||||
+ assert.NoError(err)
|
|
||||||
+}
|
|
||||||
+
|
|
||||||
+func TestDeleteServiceNoService(t *testing.T) {
|
|
||||||
+ assert := assert.New(t)
|
|
||||||
+ deleteServiceCommand := "IPVSadm --delete-service"
|
|
||||||
+ _, err := checkIPVSRule(deleteServiceCommand)
|
|
||||||
+ assert.Error(err)
|
|
||||||
+ deleteServiceCmd := "IPVSadm -D"
|
|
||||||
+ _, err = checkIPVSRule(deleteServiceCmd)
|
|
||||||
+ assert.Error(err)
|
|
||||||
+}
|
|
||||||
+
|
|
||||||
+func TestDeleteServiceIpError(t *testing.T) {
|
|
||||||
+ assert := assert.New(t)
|
|
||||||
+ deleteServiceCommand := "IPVSadm --delete-service --tcp-service 192.168.2.2.7:80"
|
|
||||||
+ _, err := checkIPVSRule(deleteServiceCommand)
|
|
||||||
+ assert.Error(err)
|
|
||||||
+ deleteServiceCmd := "IPVSadm -D -t 192.168.2.2.7:80"
|
|
||||||
+ _, err = checkIPVSRule(deleteServiceCmd)
|
|
||||||
+ assert.Error(err)
|
|
||||||
+}
|
|
||||||
+
|
|
||||||
+func TestDeleteServicePortError(t *testing.T) {
|
|
||||||
+ assert := assert.New(t)
|
|
||||||
+ deleteServiceCommand := "IPVSadm --delete-service --tcp-service 192.168.0.7:9999999"
|
|
||||||
+ _, err := checkIPVSRule(deleteServiceCommand)
|
|
||||||
+ assert.Error(err)
|
|
||||||
+ deleteServiceCmd := "IPVSadm -D -t 192.168.0.7:9999999"
|
|
||||||
+ _, err = checkIPVSRule(deleteServiceCmd)
|
|
||||||
+ assert.Error(err)
|
|
||||||
+}
|
|
||||||
+
|
|
||||||
+func TestAddServerSuccessfully(t *testing.T) {
|
|
||||||
+ assert := assert.New(t)
|
|
||||||
+ addServerCommandTCP := "IPVSadm --add-server --tcp-service 192.168.0.7:80 --real-server 192.168.0.4:80 --weight 100"
|
|
||||||
+ _, err := checkIPVSRule(addServerCommandTCP)
|
|
||||||
+ assert.NoError(err)
|
|
||||||
+ addServerCommandUDP := "IPVSadm --add-server --udp-service 192.168.0.7:80 --real-server 192.168.0.4:80 --weight 100"
|
|
||||||
+ _, err = checkIPVSRule(addServerCommandUDP)
|
|
||||||
+ assert.NoError(err)
|
|
||||||
+ addServerCmd := "IPVSadm -a -t 192.168.0.7:80 -r 192.168.0.4:80 -w 100"
|
|
||||||
+ _, err = checkIPVSRule(addServerCmd)
|
|
||||||
+ assert.NoError(err)
|
|
||||||
+}
|
|
||||||
+
|
|
||||||
+func TestAddServerNoService(t *testing.T) {
|
|
||||||
+ assert := assert.New(t)
|
|
||||||
+ addServerCommand := "IPVSadm --add-server --real-server 192.168.0.4:80"
|
|
||||||
+ _, err := checkIPVSRule(addServerCommand)
|
|
||||||
+ assert.Error(err)
|
|
||||||
+ addServerCmd := "IPVSadm -a -r 192.168.0.4:80"
|
|
||||||
+ _, err = checkIPVSRule(addServerCmd)
|
|
||||||
+ assert.Error(err)
|
|
||||||
+}
|
|
||||||
+
|
|
||||||
+func TestAddServerNoServer(t *testing.T) {
|
|
||||||
+ assert := assert.New(t)
|
|
||||||
+ addServerCommand := "IPVSadm --add-server --tcp-service 192.168.0.7:80"
|
|
||||||
+ _, err := checkIPVSRule(addServerCommand)
|
|
||||||
+ assert.Error(err)
|
|
||||||
+ addServerCmd := "IPVSadm -a -t 192.168.0.7:80"
|
|
||||||
+ _, err = checkIPVSRule(addServerCmd)
|
|
||||||
+ assert.Error(err)
|
|
||||||
+}
|
|
||||||
+
|
|
||||||
+func TestAddServerIpError(t *testing.T) {
|
|
||||||
+ assert := assert.New(t)
|
|
||||||
+ addServerCommand := "IPVSadm --add-server --tcp-service 192.168.2.0.7:80 --real-server 192.168.0.4:80"
|
|
||||||
+ _, err := checkIPVSRule(addServerCommand)
|
|
||||||
+ assert.Error(err)
|
|
||||||
+ addServerCmd := "IPVSadm -a -t 192.168.2.0.7:80 -r 192.168.0.4:80"
|
|
||||||
+ _, err = checkIPVSRule(addServerCmd)
|
|
||||||
+ assert.Error(err)
|
|
||||||
+}
|
|
||||||
+
|
|
||||||
+func TestAddServerPortError(t *testing.T) {
|
|
||||||
+ assert := assert.New(t)
|
|
||||||
+ addServerCommand := "IPVSadm --add-server --tcp-service 192.168.0.7:99999 --real-server 192.168.0.4:80"
|
|
||||||
+ _, err := checkIPVSRule(addServerCommand)
|
|
||||||
+ assert.Error(err)
|
|
||||||
+ addServerCmd := "IPVSadm -a -t 192.168.0.7:99999 -r 192.168.0.4:80"
|
|
||||||
+ _, err = checkIPVSRule(addServerCmd)
|
|
||||||
+ assert.Error(err)
|
|
||||||
+}
|
|
||||||
+
|
|
||||||
+func TestEditServerSuccessfully(t *testing.T) {
|
|
||||||
+ assert := assert.New(t)
|
|
||||||
+ editServerCommandTCP := "IPVSadm --edit-server --tcp-service 192.168.0.7:80 --real-server 192.168.0.4:80 --weight 100"
|
|
||||||
+ _, err := checkIPVSRule(editServerCommandTCP)
|
|
||||||
+ assert.NoError(err)
|
|
||||||
+ editServerCommandUDP := "IPVSadm --edit-server --udp-service 192.168.0.7:80 --real-server 192.168.0.4:80 --weight 100"
|
|
||||||
+ _, err = checkIPVSRule(editServerCommandUDP)
|
|
||||||
+ assert.NoError(err)
|
|
||||||
+ editServerCmd := "IPVSadm -e -t 192.168.0.7:80 -r 192.168.0.4:80 -w 100"
|
|
||||||
+ _, err = checkIPVSRule(editServerCmd)
|
|
||||||
+ assert.NoError(err)
|
|
||||||
+}
|
|
||||||
+
|
|
||||||
+func TestEditServerNoService(t *testing.T) {
|
|
||||||
+ assert := assert.New(t)
|
|
||||||
+ editServerCommand := "IPVSadm --edit-server --real-server 192.168.0.4:80"
|
|
||||||
+ _, err := checkIPVSRule(editServerCommand)
|
|
||||||
+ assert.Error(err)
|
|
||||||
+ editServerCmd := "IPVSadm -e -r 192.168.0.4:80"
|
|
||||||
+ _, err = checkIPVSRule(editServerCmd)
|
|
||||||
+ assert.Error(err)
|
|
||||||
+}
|
|
||||||
+
|
|
||||||
+func TestEditServerNoServer(t *testing.T) {
|
|
||||||
+ assert := assert.New(t)
|
|
||||||
+ editServerCommand := "IPVSadm --edit-server --tcp-service 192.168.0.7:80"
|
|
||||||
+ _, err := checkIPVSRule(editServerCommand)
|
|
||||||
+ assert.Error(err)
|
|
||||||
+ editServerCmd := "IPVSadm -e -t 192.168.0.7:80"
|
|
||||||
+ _, err = checkIPVSRule(editServerCmd)
|
|
||||||
+ assert.Error(err)
|
|
||||||
+}
|
|
||||||
+
|
|
||||||
+func TestEditServerIpError(t *testing.T) {
|
|
||||||
+ assert := assert.New(t)
|
|
||||||
+ editServerCommand := "IPVSadm --edit-server --tcp-service 192.168.2.0.7:80 --real-server 192.168.0.4:80"
|
|
||||||
+ _, err := checkIPVSRule(editServerCommand)
|
|
||||||
+ assert.Error(err)
|
|
||||||
+ editServerCmd := "IPVSadm -e -t 192.168.2.0.7:80 -r 192.168.0.4:80"
|
|
||||||
+ _, err = checkIPVSRule(editServerCmd)
|
|
||||||
+ assert.Error(err)
|
|
||||||
+}
|
|
||||||
+
|
|
||||||
+func TestEditServerPortError(t *testing.T) {
|
|
||||||
+ assert := assert.New(t)
|
|
||||||
+ editServerCommand := "IPVSadm --edit-server --tcp-service 192.168.0.7:99999 --real-server 192.168.0.4:80"
|
|
||||||
+ _, err := checkIPVSRule(editServerCommand)
|
|
||||||
+ assert.Error(err)
|
|
||||||
+ editServerCmd := "IPVSadm -e -t 192.168.0.7:99999 -r 192.168.0.4:80"
|
|
||||||
+ _, err = checkIPVSRule(editServerCmd)
|
|
||||||
+ assert.Error(err)
|
|
||||||
+}
|
|
||||||
+
|
|
||||||
+func TestDeleteServerSuccessfully(t *testing.T) {
|
|
||||||
+ assert := assert.New(t)
|
|
||||||
+ deleteServerCommandTCP := "IPVSadm --delete-server --tcp-service 192.168.0.7:80 --real-server 192.168.0.4:80"
|
|
||||||
+ _, err := checkIPVSRule(deleteServerCommandTCP)
|
|
||||||
+ assert.NoError(err)
|
|
||||||
+ deleteServerCommandUDP := "IPVSadm --delete-server --udp-service 192.168.0.7:80 --real-server 192.168.0.4:80"
|
|
||||||
+ _, err = checkIPVSRule(deleteServerCommandUDP)
|
|
||||||
+ assert.NoError(err)
|
|
||||||
+ deleteServerCmd := "IPVSadm -d -t 192.168.0.7:80 -r 192.168.0.4:80"
|
|
||||||
+ _, err = checkIPVSRule(deleteServerCmd)
|
|
||||||
+ assert.NoError(err)
|
|
||||||
+}
|
|
||||||
+
|
|
||||||
+func TestDeleteServerNoService(t *testing.T) {
|
|
||||||
+ assert := assert.New(t)
|
|
||||||
+ deleteServerCommand := "IPVSadm --delete-server --real-server 192.168.0.4:80"
|
|
||||||
+ _, err := checkIPVSRule(deleteServerCommand)
|
|
||||||
+ assert.Error(err)
|
|
||||||
+ deleteServerCmd := "IPVSadm -d -r 192.168.0.4:80"
|
|
||||||
+ _, err = checkIPVSRule(deleteServerCmd)
|
|
||||||
+ assert.Error(err)
|
|
||||||
+}
|
|
||||||
+
|
|
||||||
+func TestDeleteServerNoServer(t *testing.T) {
|
|
||||||
+ assert := assert.New(t)
|
|
||||||
+ deleteServerCommand := "IPVSadm --delete-server --tcp-service 192.168.0.7:80"
|
|
||||||
+ _, err := checkIPVSRule(deleteServerCommand)
|
|
||||||
+ assert.Error(err)
|
|
||||||
+ deleteServerCmd := "IPVSadm -d -t 192.168.0.7:80"
|
|
||||||
+ _, err = checkIPVSRule(deleteServerCmd)
|
|
||||||
+ assert.Error(err)
|
|
||||||
+}
|
|
||||||
+
|
|
||||||
+func TestDeleteServerIpError(t *testing.T) {
|
|
||||||
+ assert := assert.New(t)
|
|
||||||
+ deleteServerCommand := "IPVSadm --delete-server --tcp-service 192.168.2.0.7:80 --real-server 192.168.0.4:80"
|
|
||||||
+ _, err := checkIPVSRule(deleteServerCommand)
|
|
||||||
+ assert.Error(err)
|
|
||||||
+ deleteServerCmd := "IPVSadm -d -t 192.168.2.0.7:80 -r 192.168.0.4:80"
|
|
||||||
+ _, err = checkIPVSRule(deleteServerCmd)
|
|
||||||
+ assert.Error(err)
|
|
||||||
+}
|
|
||||||
+
|
|
||||||
+func TestDeleteServerPortError(t *testing.T) {
|
|
||||||
+ assert := assert.New(t)
|
|
||||||
+ deleteServerCommand := "IPVSadm --delete-server --tcp-service 192.168.0.7:99999 --real-server 192.168.0.4:80"
|
|
||||||
+ _, err := checkIPVSRule(deleteServerCommand)
|
|
||||||
+ assert.Error(err)
|
|
||||||
+ deleteServerCmd := "IPVSadm -d -t 192.168.0.7:99999 -r 192.168.0.4:80"
|
|
||||||
+ _, err = checkIPVSRule(deleteServerCmd)
|
|
||||||
+ assert.Error(err)
|
|
||||||
+}
|
|
||||||
+
|
|
||||||
+func TestListSuccessfully(t *testing.T) {
|
|
||||||
+ assert := assert.New(t)
|
|
||||||
+ listCommand := "IPVSadm --list"
|
|
||||||
+ _, err := checkIPVSRule(listCommand)
|
|
||||||
+ assert.NoError(err)
|
|
||||||
+ listCmd := "IPVSadm -L"
|
|
||||||
+ _, err = checkIPVSRule(listCmd)
|
|
||||||
+ assert.NoError(err)
|
|
||||||
+}
|
|
||||||
+
|
|
||||||
+func TestSetSuccessfully(t *testing.T) {
|
|
||||||
+ assert := assert.New(t)
|
|
||||||
+ setCommand := "IPVSadm --set"
|
|
||||||
+ _, err := checkIPVSRule(setCommand)
|
|
||||||
+ assert.NoError(err)
|
|
||||||
+}
|
|
||||||
+
|
|
||||||
+func TestCleanupSuccessfully(t *testing.T) {
|
|
||||||
+ assert := assert.New(t)
|
|
||||||
+ cleanupCommand := "conntrack -D --orig-dst 192.168.0.4 --protonum tcp"
|
|
||||||
+ _, err := checkIPVSRule(cleanupCommand)
|
|
||||||
+ assert.NoError(err)
|
|
||||||
+ cleanupCmd := "conntrack -D -d 192.168.0.4 -p tcp"
|
|
||||||
+ _, err = checkIPVSRule(cleanupCmd)
|
|
||||||
+ assert.NoError(err)
|
|
||||||
+}
|
|
||||||
+
|
|
||||||
+func TestCleanupNoDestination(t *testing.T) {
|
|
||||||
+ assert := assert.New(t)
|
|
||||||
+ cleanupCommand := "conntrack -D --protonum tcp"
|
|
||||||
+ _, err := checkIPVSRule(cleanupCommand)
|
|
||||||
+ assert.Error(err)
|
|
||||||
+ cleanupCmd := "conntrack -D -p tcp"
|
|
||||||
+ _, err = checkIPVSRule(cleanupCmd)
|
|
||||||
+ assert.Error(err)
|
|
||||||
+}
|
|
||||||
+
|
|
||||||
+func TestCleanupNoProtocol(t *testing.T) {
|
|
||||||
+ assert := assert.New(t)
|
|
||||||
+ cleanupCommand := "conntrack -D --orig-dst 192.168.0.4"
|
|
||||||
+ _, err := checkIPVSRule(cleanupCommand)
|
|
||||||
+ assert.Error(err)
|
|
||||||
+ cleanupCmd := "conntrack -D -d 192.168.0.4"
|
|
||||||
+ _, err = checkIPVSRule(cleanupCmd)
|
|
||||||
+ assert.Error(err)
|
|
||||||
+}
|
|
||||||
+
|
|
||||||
+func TestCleanupIpError(t *testing.T) {
|
|
||||||
+ assert := assert.New(t)
|
|
||||||
+ cleanupCommand := "conntrack -D --orig-dst 192.168.0.2.4 --protonum tcp"
|
|
||||||
+ _, err := checkIPVSRule(cleanupCommand)
|
|
||||||
+ assert.Error(err)
|
|
||||||
+ cleanupCmd := "conntrack -D -d 192.168.0.2.4 -p tcp"
|
|
||||||
+ _, err = checkIPVSRule(cleanupCmd)
|
|
||||||
+ assert.Error(err)
|
|
||||||
+}
|
|
||||||
+
|
|
||||||
+func TestCleanupProtocolError(t *testing.T) {
|
|
||||||
+ assert := assert.New(t)
|
|
||||||
+ cleanupCommand := "conntrack -D --orig-dst 192.168.0.4 --protonum ttcp"
|
|
||||||
+ _, err := checkIPVSRule(cleanupCommand)
|
|
||||||
+ assert.Error(err)
|
|
||||||
+ cleanupCmd := "conntrack -D -d 192.168.0.4 -p ttcp"
|
|
||||||
+ _, err = checkIPVSRule(cleanupCmd)
|
|
||||||
+ assert.Error(err)
|
|
||||||
+}
|
|
||||||
+
|
|
||||||
+func TestRestoreFileNil(t *testing.T) {
|
|
||||||
+ assert := assert.New(t)
|
|
||||||
+ stdin := strings.NewReader("")
|
|
||||||
+ fileContext, cnt, err := getRestoreFile(stdin)
|
|
||||||
+ assert.Equal(cnt, 0)
|
|
||||||
+ assert.Equal(fileContext, "")
|
|
||||||
+ assert.Error(err)
|
|
||||||
+}
|
|
||||||
+
|
|
||||||
+func TestRestoreFileError(t *testing.T) {
|
|
||||||
+ assert := assert.New(t)
|
|
||||||
+ stdin := strings.NewReader("aaa")
|
|
||||||
+ fileContext, cnt, err := getRestoreFile(stdin)
|
|
||||||
+ assert.Equal(cnt, 0)
|
|
||||||
+ assert.Equal(fileContext, "")
|
|
||||||
+ assert.Error(err)
|
|
||||||
+}
|
|
||||||
+
|
|
||||||
+func TestRestoreFile(t *testing.T) {
|
|
||||||
+ assert := assert.New(t)
|
|
||||||
+ stdin := strings.NewReader("-A -t 10.10.11.12:100 -s rr -p 3000\n-a -t 10.10.11.12:100 -r 172.16.0.1:80 -m")
|
|
||||||
+ fileContext, cnt, err := getRestoreFile(stdin)
|
|
||||||
+ assert.Equal(cnt, 2)
|
|
||||||
+ assert.Equal(fileContext, "-A -t 10.10.11.12:100 -s rr -p 3000\n-a -t 10.10.11.12:100 -r 172.16.0.1:80 -m")
|
|
||||||
+ assert.NoError(err)
|
|
||||||
+}
|
|
||||||
diff --git a/virtcontainers/api_test.go b/virtcontainers/api_test.go
|
|
||||||
index c01d47b8..2a5488f9 100644
|
|
||||||
--- a/virtcontainers/api_test.go
|
|
||||||
+++ b/virtcontainers/api_test.go
|
|
||||||
@@ -15,6 +15,7 @@ import (
|
|
||||||
"syscall"
|
|
||||||
"testing"
|
|
||||||
|
|
||||||
+ "github.com/kata-containers/agent/protocols/grpc"
|
|
||||||
ktu "github.com/kata-containers/runtime/pkg/katatestutils"
|
|
||||||
"github.com/kata-containers/runtime/virtcontainers/persist"
|
|
||||||
"github.com/kata-containers/runtime/virtcontainers/pkg/annotations"
|
|
||||||
@@ -1745,3 +1746,38 @@ func TestCleanupContainer(t *testing.T) {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
+
|
|
||||||
+func TestUpdateIPVSRule(t *testing.T) {
|
|
||||||
+ defer cleanUp()
|
|
||||||
+ assert := assert.New(t)
|
|
||||||
+
|
|
||||||
+ contID := "abc"
|
|
||||||
+ ctx := context.Background()
|
|
||||||
+ var ipvs = &grpc.UpdateIPVSRequest{
|
|
||||||
+ IPVSReq: "ipvsadm -A -t 192.168.0.7:80 -s rr -p -3000",
|
|
||||||
+ }
|
|
||||||
+
|
|
||||||
+ config := newTestSandboxConfigNoop()
|
|
||||||
+ p, _, err := createAndStartSandbox(ctx, config)
|
|
||||||
+ if p == nil || err != nil {
|
|
||||||
+ t.Fatal(err)
|
|
||||||
+ }
|
|
||||||
+
|
|
||||||
+ s, ok := p.(*Sandbox)
|
|
||||||
+ assert.True(ok)
|
|
||||||
+
|
|
||||||
+ contConfig := newTestContainerConfigNoop(contID)
|
|
||||||
+ c, err := p.CreateContainer(contConfig)
|
|
||||||
+ if c == nil || err != nil {
|
|
||||||
+ t.Fatal(err)
|
|
||||||
+ }
|
|
||||||
+
|
|
||||||
+ _, err = UpdateIPVSRule(ctx, s.id, ipvs)
|
|
||||||
+ assert.NoError(err)
|
|
||||||
+
|
|
||||||
+ _, err = UpdateIPVSRule(ctx, "aaa", ipvs)
|
|
||||||
+ assert.Error(err)
|
|
||||||
+
|
|
||||||
+ _, err = UpdateIPVSRule(ctx, "", ipvs)
|
|
||||||
+ assert.Error(err)
|
|
||||||
+}
|
|
||||||
diff --git a/virtcontainers/kata_agent_test.go b/virtcontainers/kata_agent_test.go
|
|
||||||
index 18a5a0a6..4f9409a0 100644
|
|
||||||
--- a/virtcontainers/kata_agent_test.go
|
|
||||||
+++ b/virtcontainers/kata_agent_test.go
|
|
||||||
@@ -246,6 +246,10 @@ func (p *gRPCProxy) MemHotplugByProbe(ctx context.Context, req *pb.MemHotplugByP
|
|
||||||
return &gpb.Empty{}, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
+func (p *gRPCProxy) UpdateIPVSRule(ctx context.Context, req *pb.UpdateIPVSRequest) (*pb.IPVSResponse, error) {
|
|
||||||
+ return &pb.IPVSResponse{}, nil
|
|
||||||
+}
|
|
||||||
+
|
|
||||||
func gRPCRegister(s *grpc.Server, srv interface{}) {
|
|
||||||
switch g := srv.(type) {
|
|
||||||
case *gRPCProxy:
|
|
||||||
diff --git a/virtcontainers/pkg/vcmock/types.go b/virtcontainers/pkg/vcmock/types.go
|
|
||||||
index 43247ef9..610b4602 100644
|
|
||||||
--- a/virtcontainers/pkg/vcmock/types.go
|
|
||||||
+++ b/virtcontainers/pkg/vcmock/types.go
|
|
||||||
@@ -76,5 +76,5 @@ type VCMock struct {
|
|
||||||
UpdateRoutesFunc func(ctx context.Context, sandboxID string, routes []*vcTypes.Route) ([]*vcTypes.Route, error)
|
|
||||||
ListRoutesFunc func(ctx context.Context, sandboxID string) ([]*vcTypes.Route, error)
|
|
||||||
CleanupContainerFunc func(ctx context.Context, sandboxID, containerID string, force bool) error
|
|
||||||
- UpdateIPVSRuleFunc func(ctx context.Context, sandboxID string, ipvs *grpc.UpdateIpvsRequest) (*grpc.IpvsResponse, error)
|
|
||||||
+ UpdateIPVSRuleFunc func(ctx context.Context, sandboxID string, ipvs *grpc.UpdateIPVSRequest) (*grpc.IPVSResponse, error)
|
|
||||||
}
|
|
||||||
--
|
|
||||||
2.14.3 (Apple Git-98)
|
|
||||||
|
|
||||||
@ -1,501 +0,0 @@
|
|||||||
From 5a220e9be1cfb03316a62aa00d2040638ba1a855 Mon Sep 17 00:00:00 2001
|
|
||||||
From: jiangpengfei <jiangpengfei9@huawei.com>
|
|
||||||
Date: Tue, 18 Aug 2020 15:15:52 +0800
|
|
||||||
Subject: [PATCH 38/50] pcie: using pcie-root-port driver to hotplug device in
|
|
||||||
aarch64
|
|
||||||
|
|
||||||
reason: Since qemu with "virt" machine type doesn't support hotplug
|
|
||||||
device in the pcie.0 root bus, so need to use add root port devices
|
|
||||||
to support hotplug pci device in aarch64.
|
|
||||||
|
|
||||||
we reuse the pcie_root_port config in the configuration.toml file to
|
|
||||||
set pcie_root_port device number when qemu process start.
|
|
||||||
|
|
||||||
Signed-off-by: jiangpengfei <jiangpengfei9@huawei.com>
|
|
||||||
---
|
|
||||||
vendor/github.com/intel/govmm/qemu/qemu.go | 22 ++---
|
|
||||||
virtcontainers/persist/api/hypervisor.go | 5 +-
|
|
||||||
virtcontainers/qemu.go | 130 +++++++++++++++++++++++------
|
|
||||||
virtcontainers/qemu_arch_base.go | 9 ++
|
|
||||||
virtcontainers/types/pcie.go | 103 +++++++++++++++++++++++
|
|
||||||
5 files changed, 230 insertions(+), 39 deletions(-)
|
|
||||||
create mode 100644 virtcontainers/types/pcie.go
|
|
||||||
|
|
||||||
diff --git a/vendor/github.com/intel/govmm/qemu/qemu.go b/vendor/github.com/intel/govmm/qemu/qemu.go
|
|
||||||
index a5e5dfaf..3e7720b4 100644
|
|
||||||
--- a/vendor/github.com/intel/govmm/qemu/qemu.go
|
|
||||||
+++ b/vendor/github.com/intel/govmm/qemu/qemu.go
|
|
||||||
@@ -1252,6 +1252,8 @@ type PCIeRootPortDevice struct {
|
|
||||||
Chassis string // (slot, chassis) pair is mandatory and must be unique for each pcie-root-port, >=0, default is 0x00
|
|
||||||
Slot string // >=0, default is 0x00
|
|
||||||
|
|
||||||
+ Port string // Port number is the device index
|
|
||||||
+
|
|
||||||
Multifunction bool // true => "on", false => "off", default is off
|
|
||||||
Addr string // >=0, default is 0x00
|
|
||||||
|
|
||||||
@@ -1277,6 +1279,10 @@ func (b PCIeRootPortDevice) QemuParams(config *Config) []string {
|
|
||||||
|
|
||||||
deviceParams = append(deviceParams, fmt.Sprintf("%s,id=%s", driver, b.ID))
|
|
||||||
|
|
||||||
+ if b.Port != "" {
|
|
||||||
+ deviceParams = append(deviceParams, fmt.Sprintf("port=%s", b.Port))
|
|
||||||
+ }
|
|
||||||
+
|
|
||||||
if b.Bus == "" {
|
|
||||||
b.Bus = "pcie.0"
|
|
||||||
}
|
|
||||||
@@ -1287,20 +1293,10 @@ func (b PCIeRootPortDevice) QemuParams(config *Config) []string {
|
|
||||||
}
|
|
||||||
deviceParams = append(deviceParams, fmt.Sprintf("chassis=%s", b.Chassis))
|
|
||||||
|
|
||||||
- if b.Slot == "" {
|
|
||||||
- b.Slot = "0x00"
|
|
||||||
- }
|
|
||||||
- deviceParams = append(deviceParams, fmt.Sprintf("slot=%s", b.Slot))
|
|
||||||
-
|
|
||||||
- multifunction := "off"
|
|
||||||
if b.Multifunction {
|
|
||||||
- multifunction = "on"
|
|
||||||
- if b.Addr == "" {
|
|
||||||
- b.Addr = "0x00"
|
|
||||||
- }
|
|
||||||
- deviceParams = append(deviceParams, fmt.Sprintf("addr=%s", b.Addr))
|
|
||||||
+ deviceParams = append(deviceParams, "multifunction=on")
|
|
||||||
}
|
|
||||||
- deviceParams = append(deviceParams, fmt.Sprintf("multifunction=%v", multifunction))
|
|
||||||
+ deviceParams = append(deviceParams, fmt.Sprintf("addr=%s", b.Addr))
|
|
||||||
|
|
||||||
if b.BusReserve != "" {
|
|
||||||
deviceParams = append(deviceParams, fmt.Sprintf("bus-reserve=%s", b.BusReserve))
|
|
||||||
@@ -1337,7 +1333,7 @@ func (b PCIeRootPortDevice) Valid() bool {
|
|
||||||
if b.Pref64Reserve != "" && b.Pref32Reserve != "" {
|
|
||||||
return false
|
|
||||||
}
|
|
||||||
- if b.ID == "" {
|
|
||||||
+ if b.ID == "" || b.Port == "" || b.Bus == "" || b.Addr == ""{
|
|
||||||
return false
|
|
||||||
}
|
|
||||||
return true
|
|
||||||
diff --git a/virtcontainers/persist/api/hypervisor.go b/virtcontainers/persist/api/hypervisor.go
|
|
||||||
index 375fd56b..fd61b3c2 100644
|
|
||||||
--- a/virtcontainers/persist/api/hypervisor.go
|
|
||||||
+++ b/virtcontainers/persist/api/hypervisor.go
|
|
||||||
@@ -5,6 +5,8 @@
|
|
||||||
|
|
||||||
package persistapi
|
|
||||||
|
|
||||||
+import "github.com/kata-containers/runtime/virtcontainers/types"
|
|
||||||
+
|
|
||||||
// Bridge is a bridge where devices can be hot plugged
|
|
||||||
type Bridge struct {
|
|
||||||
// DeviceAddr contains information about devices plugged and its address in the bridge
|
|
||||||
@@ -35,7 +37,8 @@ type HypervisorState struct {
|
|
||||||
|
|
||||||
// Belows are qemu specific
|
|
||||||
// Refs: virtcontainers/qemu.go:QemuState
|
|
||||||
- Bridges []Bridge
|
|
||||||
+ Bridges []Bridge
|
|
||||||
+ PCIeRootPortsPool *types.PCIeRootPortPool
|
|
||||||
// HotpluggedCPUs is the list of CPUs that were hot-added
|
|
||||||
HotpluggedVCPUs []CPUDevice
|
|
||||||
HotpluggedMemory int
|
|
||||||
diff --git a/virtcontainers/qemu.go b/virtcontainers/qemu.go
|
|
||||||
index c2b65376..a10c66fb 100644
|
|
||||||
--- a/virtcontainers/qemu.go
|
|
||||||
+++ b/virtcontainers/qemu.go
|
|
||||||
@@ -64,8 +64,9 @@ type CPUDevice struct {
|
|
||||||
|
|
||||||
// QemuState keeps Qemu's state
|
|
||||||
type QemuState struct {
|
|
||||||
- Bridges []types.Bridge
|
|
||||||
- ScsiBus *types.SCSIBus
|
|
||||||
+ Bridges []types.Bridge
|
|
||||||
+ ScsiBus *types.SCSIBus
|
|
||||||
+ PCIeRootPortsPool *types.PCIeRootPortPool
|
|
||||||
// HotpluggedCPUs is the list of CPUs that were hot-added
|
|
||||||
HotpluggedVCPUs []CPUDevice
|
|
||||||
HotpluggedMemory int
|
|
||||||
@@ -271,6 +272,9 @@ func (q *qemu) setup(id string, hypervisorConfig *HypervisorConfig) error {
|
|
||||||
|
|
||||||
q.state.HotplugVFIOOnRootBus = q.config.HotplugVFIOOnRootBus
|
|
||||||
q.state.PCIeRootPort = int(q.config.PCIeRootPort)
|
|
||||||
+ // init the PCIeRootPortsPool with pcie_root_port config value
|
|
||||||
+ q.state.PCIeRootPortsPool = &types.PCIeRootPortPool{}
|
|
||||||
+ q.state.PCIeRootPortsPool.Init(q.state.PCIeRootPort)
|
|
||||||
|
|
||||||
// The path might already exist, but in case of VM templating,
|
|
||||||
// we have to create it since the sandbox has not created it yet.
|
|
||||||
@@ -394,9 +398,18 @@ func (q *qemu) buildDevices(initrdPath string) ([]govmmQemu.Device, *govmmQemu.I
|
|
||||||
return nil, nil, err
|
|
||||||
}
|
|
||||||
|
|
||||||
- // Add bridges before any other devices. This way we make sure that
|
|
||||||
- // bridge gets the first available PCI address i.e bridgePCIStartAddr
|
|
||||||
- devices = q.arch.appendBridges(devices)
|
|
||||||
+ machine, err := q.getQemuMachine()
|
|
||||||
+ if err != nil {
|
|
||||||
+ return nil, nil, err
|
|
||||||
+ }
|
|
||||||
+ switch machine.Type {
|
|
||||||
+ case QemuVirt:
|
|
||||||
+ devices = q.arch.appendRootPorts(devices, q.state.PCIeRootPortsPool)
|
|
||||||
+ default:
|
|
||||||
+ // Add bridges before any other devices. This way we make sure that
|
|
||||||
+ // bridge gets the first available PCI address i.e bridgePCIStartAddr
|
|
||||||
+ devices = q.arch.appendBridges(devices)
|
|
||||||
+ }
|
|
||||||
|
|
||||||
devices, err = q.arch.appendConsole(devices, console)
|
|
||||||
if err != nil {
|
|
||||||
@@ -608,7 +621,7 @@ func (q *qemu) createSandbox(ctx context.Context, id string, networkNS NetworkNa
|
|
||||||
// Add PCIe Root Port devices to hypervisor
|
|
||||||
// The pcie.0 bus do not support hot-plug, but PCIe device can be hot-plugged into PCIe Root Port.
|
|
||||||
// For more details, please see https://github.com/qemu/qemu/blob/master/docs/pcie.txt
|
|
||||||
- if hypervisorConfig.PCIeRootPort > 0 {
|
|
||||||
+ if hypervisorConfig.PCIeRootPort > 0 && hypervisorConfig.HypervisorMachineType == QemuQ35 {
|
|
||||||
qemuConfig.Devices = q.arch.appendPCIeRootPortDevice(qemuConfig.Devices, hypervisorConfig.PCIeRootPort)
|
|
||||||
}
|
|
||||||
|
|
||||||
@@ -1154,21 +1167,19 @@ func (q *qemu) hotplugAddBlockDevice(drive *config.BlockDrive, op operation, dev
|
|
||||||
}
|
|
||||||
case q.config.BlockDeviceDriver == config.VirtioBlock:
|
|
||||||
driver := "virtio-blk-pci"
|
|
||||||
- addr, bridge, err := q.arch.addDeviceToBridge(drive.ID, types.PCI)
|
|
||||||
+
|
|
||||||
+ addr, bus, pciAddr, err := q.getPciAddress(drive.ID, types.PCI)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
-
|
|
||||||
defer func() {
|
|
||||||
if err != nil {
|
|
||||||
- q.arch.removeDeviceFromBridge(drive.ID)
|
|
||||||
+ q.putPciAddress(drive.ID)
|
|
||||||
}
|
|
||||||
}()
|
|
||||||
+ drive.PCIAddr = pciAddr
|
|
||||||
|
|
||||||
- // PCI address is in the format bridge-addr/device-addr eg. "03/02"
|
|
||||||
- drive.PCIAddr = fmt.Sprintf("%02x", bridge.Addr) + "/" + addr
|
|
||||||
-
|
|
||||||
- if err = q.qmpMonitorCh.qmp.ExecutePCIDeviceAdd(q.qmpMonitorCh.ctx, drive.ID, devID, driver, addr, bridge.ID, romFile, 0, true, defaultDisableModern); err != nil {
|
|
||||||
+ if err = q.qmpMonitorCh.qmp.ExecutePCIDeviceAdd(q.qmpMonitorCh.ctx, drive.ID, devID, driver, addr, bus, romFile, 0, true, defaultDisableModern); err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
case q.config.BlockDeviceDriver == config.VirtioSCSI:
|
|
||||||
@@ -1249,7 +1260,7 @@ func (q *qemu) hotplugBlockDevice(drive *config.BlockDrive, op operation) error
|
|
||||||
err = q.hotplugAddBlockDevice(drive, op, devID)
|
|
||||||
} else {
|
|
||||||
if q.config.BlockDeviceDriver == config.VirtioBlock {
|
|
||||||
- if err := q.arch.removeDeviceFromBridge(drive.ID); err != nil {
|
|
||||||
+ if err := q.putPciAddress(drive.ID); err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
} else if q.config.BlockDeviceDriver == config.VirtioSCSI {
|
|
||||||
@@ -1345,22 +1356,22 @@ func (q *qemu) hotplugVFIODevice(device *config.VFIODev, op operation) (err erro
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
- addr, bridge, err := q.arch.addDeviceToBridge(devID, types.PCI)
|
|
||||||
+ addr, bus, _, err := q.getPciAddress(devID, types.PCI)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
defer func() {
|
|
||||||
if err != nil {
|
|
||||||
- q.arch.removeDeviceFromBridge(devID)
|
|
||||||
+ q.putPciAddress(devID)
|
|
||||||
}
|
|
||||||
}()
|
|
||||||
|
|
||||||
switch device.Type {
|
|
||||||
case config.VFIODeviceNormalType:
|
|
||||||
- return q.qmpMonitorCh.qmp.ExecutePCIVFIODeviceAdd(q.qmpMonitorCh.ctx, devID, device.BDF, addr, bridge.ID, romFile)
|
|
||||||
+ return q.qmpMonitorCh.qmp.ExecutePCIVFIODeviceAdd(q.qmpMonitorCh.ctx, devID, device.BDF, addr, bus, romFile)
|
|
||||||
case config.VFIODeviceMediatedType:
|
|
||||||
- return q.qmpMonitorCh.qmp.ExecutePCIVFIOMediatedDeviceAdd(q.qmpMonitorCh.ctx, devID, device.SysfsDev, addr, bridge.ID, romFile)
|
|
||||||
+ return q.qmpMonitorCh.qmp.ExecutePCIVFIOMediatedDeviceAdd(q.qmpMonitorCh.ctx, devID, device.SysfsDev, addr, bus, romFile)
|
|
||||||
default:
|
|
||||||
return fmt.Errorf("Incorrect VFIO device type found")
|
|
||||||
}
|
|
||||||
@@ -1368,7 +1379,7 @@ func (q *qemu) hotplugVFIODevice(device *config.VFIODev, op operation) (err erro
|
|
||||||
q.Logger().WithField("dev-id", devID).Info("Start hot-unplug VFIO device")
|
|
||||||
|
|
||||||
if !q.state.HotplugVFIOOnRootBus {
|
|
||||||
- if err := q.arch.removeDeviceFromBridge(devID); err != nil {
|
|
||||||
+ if err := q.putPciAddress(devID); err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1439,18 +1450,17 @@ func (q *qemu) hotplugNetDevice(endpoint Endpoint, op operation) (err error) {
|
|
||||||
}
|
|
||||||
}()
|
|
||||||
|
|
||||||
- addr, bridge, err := q.arch.addDeviceToBridge(tap.ID, types.PCI)
|
|
||||||
+ addr, bus, pciAddr, err := q.getPciAddress(tap.ID, types.PCI)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
defer func() {
|
|
||||||
if err != nil {
|
|
||||||
- q.arch.removeDeviceFromBridge(tap.ID)
|
|
||||||
+ q.putPciAddress(tap.ID)
|
|
||||||
}
|
|
||||||
}()
|
|
||||||
|
|
||||||
- pciAddr := fmt.Sprintf("%02x/%s", bridge.Addr, addr)
|
|
||||||
endpoint.SetPciAddr(pciAddr)
|
|
||||||
|
|
||||||
var machine govmmQemu.Machine
|
|
||||||
@@ -1459,14 +1469,14 @@ func (q *qemu) hotplugNetDevice(endpoint Endpoint, op operation) (err error) {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
if machine.Type == QemuCCWVirtio {
|
|
||||||
- devNoHotplug := fmt.Sprintf("fe.%x.%x", bridge.Addr, addr)
|
|
||||||
+ devNoHotplug := fmt.Sprintf("fe.%x.%x", bus, addr)
|
|
||||||
return q.qmpMonitorCh.qmp.ExecuteNetCCWDeviceAdd(q.qmpMonitorCh.ctx, tap.Name, devID, endpoint.HardwareAddr(), devNoHotplug, int(q.config.NumVCPUs))
|
|
||||||
}
|
|
||||||
- return q.qmpMonitorCh.qmp.ExecuteNetPCIDeviceAdd(q.qmpMonitorCh.ctx, tap.Name, devID, endpoint.HardwareAddr(), addr, bridge.ID, romFile, int(q.config.NumVCPUs), defaultDisableModern)
|
|
||||||
+ return q.qmpMonitorCh.qmp.ExecuteNetPCIDeviceAdd(q.qmpMonitorCh.ctx, tap.Name, devID, endpoint.HardwareAddr(), addr, bus, romFile, int(q.config.NumVCPUs), defaultDisableModern)
|
|
||||||
|
|
||||||
}
|
|
||||||
|
|
||||||
- if err := q.arch.removeDeviceFromBridge(tap.ID); err != nil {
|
|
||||||
+ if err := q.putPciAddress(tap.ID); err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
@@ -2042,6 +2052,21 @@ func genericMemoryTopology(memoryMb, hostMemoryMb uint64, slots uint8, memoryOff
|
|
||||||
return memory
|
|
||||||
}
|
|
||||||
|
|
||||||
+func genericAppendRootPorts(devices []govmmQemu.Device, rootPorts *types.PCIeRootPortPool) []govmmQemu.Device {
|
|
||||||
+ for _, rp := range rootPorts.Items {
|
|
||||||
+ devices = append(devices, govmmQemu.PCIeRootPortDevice{
|
|
||||||
+ Port: rp.Port,
|
|
||||||
+ Bus: rp.Bus,
|
|
||||||
+ ID: rp.ID,
|
|
||||||
+ Chassis: strconv.Itoa(int(rp.Chassis)),
|
|
||||||
+ Multifunction: rp.Multifunction,
|
|
||||||
+ Addr: fmt.Sprintf("0x%s.0x%s", rp.Slot, rp.Function),
|
|
||||||
+ })
|
|
||||||
+ }
|
|
||||||
+
|
|
||||||
+ return devices
|
|
||||||
+}
|
|
||||||
+
|
|
||||||
// genericAppendPCIeRootPort appends to devices the given pcie-root-port
|
|
||||||
func genericAppendPCIeRootPort(devices []govmmQemu.Device, number uint32, machineType string) []govmmQemu.Device {
|
|
||||||
var (
|
|
||||||
@@ -2241,6 +2266,7 @@ func (q *qemu) save() (s persistapi.HypervisorState) {
|
|
||||||
s.HotpluggedMemory = q.state.HotpluggedMemory
|
|
||||||
s.HotplugVFIOOnRootBus = q.state.HotplugVFIOOnRootBus
|
|
||||||
s.PCIeRootPort = q.state.PCIeRootPort
|
|
||||||
+ s.PCIeRootPortsPool = q.state.PCIeRootPortsPool
|
|
||||||
|
|
||||||
for _, bridge := range q.arch.getBridges() {
|
|
||||||
s.Bridges = append(s.Bridges, persistapi.Bridge{
|
|
||||||
@@ -2265,6 +2291,7 @@ func (q *qemu) load(s persistapi.HypervisorState) {
|
|
||||||
q.state.HotplugVFIOOnRootBus = s.HotplugVFIOOnRootBus
|
|
||||||
q.state.VirtiofsdPid = s.VirtiofsdPid
|
|
||||||
q.state.PCIeRootPort = s.PCIeRootPort
|
|
||||||
+ q.state.PCIeRootPortsPool = s.PCIeRootPortsPool
|
|
||||||
|
|
||||||
for _, bridge := range s.Bridges {
|
|
||||||
q.state.Bridges = append(q.state.Bridges, types.NewBridge(types.Type(bridge.Type), bridge.ID, bridge.DeviceAddr, bridge.Addr))
|
|
||||||
@@ -2302,3 +2329,56 @@ func (q *qemu) generateSocket(id string, useVsock bool) (interface{}, error) {
|
|
||||||
func (q *qemu) getMemorySize() uint32 {
|
|
||||||
return q.config.MemorySize + uint32(q.state.HotpluggedMemory)
|
|
||||||
}
|
|
||||||
+
|
|
||||||
+// getPciAddress allocate the pci slot to hotplugged device and
|
|
||||||
+// return the pci slot address
|
|
||||||
+func (q *qemu) getPciAddress(devID string, t types.Type) (slot, bus, pciAddr string, err error) {
|
|
||||||
+ machine, err := q.getQemuMachine()
|
|
||||||
+ if err != nil {
|
|
||||||
+ return "", "", "", err
|
|
||||||
+ }
|
|
||||||
+
|
|
||||||
+ switch machine.Type {
|
|
||||||
+ case QemuVirt:
|
|
||||||
+ rp, err := q.state.PCIeRootPortsPool.AddDevice(devID)
|
|
||||||
+ if err != nil {
|
|
||||||
+ return "", "", "", err
|
|
||||||
+ }
|
|
||||||
+ // PCIe Root Port only have one slot
|
|
||||||
+ slot = "0x0"
|
|
||||||
+ // pciAddr specifies the slot and function of the Root Port and the slot of the device
|
|
||||||
+ pciAddr = fmt.Sprintf("%s.%s/00", rp.Slot, rp.Function)
|
|
||||||
+ bus = rp.ID
|
|
||||||
+ default:
|
|
||||||
+ var bridge types.Bridge
|
|
||||||
+ slot, bridge, err = q.arch.addDeviceToBridge(devID, t)
|
|
||||||
+ if err != nil {
|
|
||||||
+ return "", "", "", err
|
|
||||||
+ }
|
|
||||||
+ bus = bridge.ID
|
|
||||||
+ // PCI address is in the format bridge-addr.0/device-addr eg. "03.0/02"
|
|
||||||
+ pciAddr = fmt.Sprintf("%02x.0", bridge.Addr) + "/" + slot
|
|
||||||
+ }
|
|
||||||
+ return slot, bus, pciAddr, nil
|
|
||||||
+}
|
|
||||||
+
|
|
||||||
+func (q *qemu) putPciAddress(devID string) error {
|
|
||||||
+ machine, err := q.getQemuMachine()
|
|
||||||
+ if err != nil {
|
|
||||||
+ return err
|
|
||||||
+ }
|
|
||||||
+
|
|
||||||
+ switch machine.Type {
|
|
||||||
+ case QemuVirt:
|
|
||||||
+ err := q.state.PCIeRootPortsPool.RemoveDevice(devID)
|
|
||||||
+ if err != nil {
|
|
||||||
+ return err
|
|
||||||
+ }
|
|
||||||
+ default:
|
|
||||||
+ if err := q.arch.removeDeviceFromBridge(devID); err != nil {
|
|
||||||
+ return err
|
|
||||||
+ }
|
|
||||||
+ }
|
|
||||||
+
|
|
||||||
+ return nil
|
|
||||||
+}
|
|
||||||
diff --git a/virtcontainers/qemu_arch_base.go b/virtcontainers/qemu_arch_base.go
|
|
||||||
index 9d72dd09..cb045530 100644
|
|
||||||
--- a/virtcontainers/qemu_arch_base.go
|
|
||||||
+++ b/virtcontainers/qemu_arch_base.go
|
|
||||||
@@ -130,6 +130,9 @@ type qemuArch interface {
|
|
||||||
|
|
||||||
// appendPCIeRootPortDevice appends a pcie-root-port device to pcie.0 bus
|
|
||||||
appendPCIeRootPortDevice(devices []govmmQemu.Device, number uint32) []govmmQemu.Device
|
|
||||||
+
|
|
||||||
+ // appendRootPorts appends a pcie-root-port device to devices when qemu machine type is "virt"
|
|
||||||
+ appendRootPorts(devices []govmmQemu.Device, rootPorts *types.PCIeRootPortPool) []govmmQemu.Device
|
|
||||||
}
|
|
||||||
|
|
||||||
type qemuArchBase struct {
|
|
||||||
@@ -766,3 +769,9 @@ func (q *qemuArchBase) addBridge(b types.Bridge) {
|
|
||||||
func (q *qemuArchBase) appendPCIeRootPortDevice(devices []govmmQemu.Device, number uint32) []govmmQemu.Device {
|
|
||||||
return genericAppendPCIeRootPort(devices, number, q.machineType)
|
|
||||||
}
|
|
||||||
+
|
|
||||||
+// appendRootPorts appends a pcie-root-port device to devices when qemu machine type is "virt"
|
|
||||||
+// which is different appendPCIeRootPortDevice function
|
|
||||||
+func (q *qemuArchBase) appendRootPorts(devices []govmmQemu.Device, rootPorts *types.PCIeRootPortPool) []govmmQemu.Device {
|
|
||||||
+ return genericAppendRootPorts(devices, rootPorts)
|
|
||||||
+}
|
|
||||||
diff --git a/virtcontainers/types/pcie.go b/virtcontainers/types/pcie.go
|
|
||||||
new file mode 100644
|
|
||||||
index 00000000..83eb6944
|
|
||||||
--- /dev/null
|
|
||||||
+++ b/virtcontainers/types/pcie.go
|
|
||||||
@@ -0,0 +1,103 @@
|
|
||||||
+package types
|
|
||||||
+
|
|
||||||
+import (
|
|
||||||
+ "fmt"
|
|
||||||
+)
|
|
||||||
+
|
|
||||||
+const (
|
|
||||||
+ maxRootPortsCapacity = 25
|
|
||||||
+ slotPerDevice = 7
|
|
||||||
+
|
|
||||||
+ // PCIeRootBus is "pcie.0"
|
|
||||||
+ PCIeRootBus = "pcie.0"
|
|
||||||
+
|
|
||||||
+ // startPort specifies the start port of pcie-root-port
|
|
||||||
+ // for the first slot of "pcie.0" is reserved, so the 1~7 ports are also reserved
|
|
||||||
+ startPort = 8
|
|
||||||
+ funcNumPerSlot = 8
|
|
||||||
+)
|
|
||||||
+
|
|
||||||
+// PCIeRootPort describe the PCIe Root Port
|
|
||||||
+type PCIeRootPort struct {
|
|
||||||
+ // DeviceID specify the device hotplug on the Root Port
|
|
||||||
+ DeviceID string
|
|
||||||
+
|
|
||||||
+ // Port number is the Root Port index
|
|
||||||
+ Port string
|
|
||||||
+
|
|
||||||
+ // Bus number where the Root Port is plugged, typically pcie.0
|
|
||||||
+ Bus string
|
|
||||||
+
|
|
||||||
+ // ID is used to identify the pcie-root-port in qemu
|
|
||||||
+ ID string
|
|
||||||
+
|
|
||||||
+ // Slot specifies slot address of Root Port
|
|
||||||
+ Slot string
|
|
||||||
+
|
|
||||||
+ // Function specifies function of Root Port
|
|
||||||
+ Function string
|
|
||||||
+
|
|
||||||
+ // Chassis number
|
|
||||||
+ Chassis uint32
|
|
||||||
+
|
|
||||||
+ // Multifunction is used to specify the pcie-root-port is multifunction supported
|
|
||||||
+ Multifunction bool
|
|
||||||
+}
|
|
||||||
+
|
|
||||||
+// PCIeRootPortPool describe a set of PCIe Root Ports
|
|
||||||
+type PCIeRootPortPool struct {
|
|
||||||
+ // Items contains information about devices plugged and number limit
|
|
||||||
+ Items []*PCIeRootPort
|
|
||||||
+}
|
|
||||||
+
|
|
||||||
+// Init Initialized the PCIeRootPortPool instance
|
|
||||||
+func (rp *PCIeRootPortPool) Init(number int) {
|
|
||||||
+ if number == 0 || number > maxRootPortsCapacity {
|
|
||||||
+ number = maxRootPortsCapacity
|
|
||||||
+ }
|
|
||||||
+
|
|
||||||
+ for i := 0; i < number; i++ {
|
|
||||||
+ dev := &PCIeRootPort{
|
|
||||||
+ DeviceID: "",
|
|
||||||
+ Port: fmt.Sprintf("0x%x", startPort+i),
|
|
||||||
+ Bus: PCIeRootBus,
|
|
||||||
+ ID: fmt.Sprintf("pci.%d", i+1),
|
|
||||||
+ Chassis: uint32(i + 1),
|
|
||||||
+ }
|
|
||||||
+
|
|
||||||
+ major := i / funcNumPerSlot
|
|
||||||
+ minor := i % funcNumPerSlot
|
|
||||||
+ dev.Multifunction = false
|
|
||||||
+ if minor == 0 {
|
|
||||||
+ dev.Multifunction = true
|
|
||||||
+ }
|
|
||||||
+ dev.Slot = fmt.Sprintf("%02x", major+1)
|
|
||||||
+ dev.Function = fmt.Sprintf("%x", minor)
|
|
||||||
+
|
|
||||||
+ rp.Items = append(rp.Items, dev)
|
|
||||||
+ }
|
|
||||||
+}
|
|
||||||
+
|
|
||||||
+// AddDevice add a device to the PCIeRootPortPool
|
|
||||||
+func (rp *PCIeRootPortPool) AddDevice(devID string) (*PCIeRootPort, error) {
|
|
||||||
+ for _, it := range rp.Items {
|
|
||||||
+ if it.DeviceID == "" {
|
|
||||||
+ it.DeviceID = devID
|
|
||||||
+ return it, nil
|
|
||||||
+ }
|
|
||||||
+ }
|
|
||||||
+ return nil, fmt.Errorf("Unable to hot plug device on Root Ports: there are not empty slots")
|
|
||||||
+}
|
|
||||||
+
|
|
||||||
+// RemoveDevice remove a device from the PCIeRootPortPool
|
|
||||||
+func (rp *PCIeRootPortPool) RemoveDevice(devID string) error {
|
|
||||||
+ for _, it := range rp.Items {
|
|
||||||
+ if it.DeviceID == devID {
|
|
||||||
+ // free address to re-use the same slot with other devices
|
|
||||||
+ it.DeviceID = ""
|
|
||||||
+ return nil
|
|
||||||
+ }
|
|
||||||
+ }
|
|
||||||
+
|
|
||||||
+ return fmt.Errorf("Unable to hot unplug device %s: not present on Root Port", devID)
|
|
||||||
+}
|
|
||||||
--
|
|
||||||
2.14.3 (Apple Git-98)
|
|
||||||
|
|
||||||
@ -1,245 +0,0 @@
|
|||||||
From a36c9857447aaf22628af1ef01406a916436133b Mon Sep 17 00:00:00 2001
|
|
||||||
From: jiangpengfei <jiangpengfei9@huawei.com>
|
|
||||||
Date: Tue, 18 Aug 2020 22:00:04 +0800
|
|
||||||
Subject: [PATCH 39/50] storage: add storage common functions and structs
|
|
||||||
|
|
||||||
reason:
|
|
||||||
1. add storage.go and storage_spec.go
|
|
||||||
2. provide funcs for mount nfs and gpath
|
|
||||||
|
|
||||||
Signed-off-by: jiangpengfei <jiangpengfei9@huawei.com>
|
|
||||||
---
|
|
||||||
virtcontainers/storage/storage.go | 115 +++++++++++++++++++++++++++++++++
|
|
||||||
virtcontainers/storage/storage_spec.go | 98 ++++++++++++++++++++++++++++
|
|
||||||
2 files changed, 213 insertions(+)
|
|
||||||
create mode 100644 virtcontainers/storage/storage.go
|
|
||||||
create mode 100644 virtcontainers/storage/storage_spec.go
|
|
||||||
|
|
||||||
diff --git a/virtcontainers/storage/storage.go b/virtcontainers/storage/storage.go
|
|
||||||
new file mode 100644
|
|
||||||
index 00000000..77ef994f
|
|
||||||
--- /dev/null
|
|
||||||
+++ b/virtcontainers/storage/storage.go
|
|
||||||
@@ -0,0 +1,115 @@
|
|
||||||
+// Copyright (c) Huawei Technologies Co., Ltd. 2019. All rights reserved.
|
|
||||||
+// SPDX-License-Identifier: Apache-2.0
|
|
||||||
+// Description: common functions
|
|
||||||
+// Author: licuifang
|
|
||||||
+// Create: 2019-07-14
|
|
||||||
+
|
|
||||||
+// Package storage provides functions for mounting
|
|
||||||
+package storage
|
|
||||||
+
|
|
||||||
+import (
|
|
||||||
+ "fmt"
|
|
||||||
+ "path/filepath"
|
|
||||||
+ "regexp"
|
|
||||||
+ "strings"
|
|
||||||
+
|
|
||||||
+ "github.com/kata-containers/agent/protocols/grpc"
|
|
||||||
+ "github.com/opencontainers/runtime-spec/specs-go"
|
|
||||||
+)
|
|
||||||
+
|
|
||||||
+var (
|
|
||||||
+ // simple regular expressions for "nfs-server.com:/aaa/bbb/ccc/" or "192.168.1.1:/remote/path"
|
|
||||||
+ regUrl = regexp.MustCompile(`^((([a-zA-Z0-9]([a-zA-Z0-9\-]{0,61}[a-zA-Z0-9])?\.)+[a-zA-Z]{2,6})|((?:(?:25[0-5]|2[0-4]\d|[01]?\d?\d)\.){3}(?:25[0-5]|2[0-4]\d|[01]?\d?\d))):/[A-Za-z0-9_./-]{0,1000}$`)
|
|
||||||
+
|
|
||||||
+ defaultMountOption = "nolock"
|
|
||||||
+ storageMountOptionKey = "mount_op"
|
|
||||||
+ storageCustomOptionKey = "custom_op"
|
|
||||||
+)
|
|
||||||
+
|
|
||||||
+type RemoteStorage struct {
|
|
||||||
+ Source string
|
|
||||||
+ Dest string
|
|
||||||
+ Options map[string][]string
|
|
||||||
+}
|
|
||||||
+
|
|
||||||
+func (n *RemoteStorage) Validate(volumeType string) error {
|
|
||||||
+ if n.Source == "" || n.Dest == "" {
|
|
||||||
+ return fmt.Errorf("the source and dest of storage cannot be empty")
|
|
||||||
+ }
|
|
||||||
+ switch volumeType {
|
|
||||||
+ case NFS:
|
|
||||||
+ chk := regUrl.FindAllString(n.Source, -1)
|
|
||||||
+ if chk == nil {
|
|
||||||
+ return fmt.Errorf("invalid url for nfs")
|
|
||||||
+ }
|
|
||||||
+ case GPATHFS:
|
|
||||||
+ if !filepath.IsAbs(n.Source) {
|
|
||||||
+ return fmt.Errorf("invalid gpath")
|
|
||||||
+ }
|
|
||||||
+ }
|
|
||||||
+
|
|
||||||
+ return nil
|
|
||||||
+}
|
|
||||||
+
|
|
||||||
+func (n *RemoteStorage) GetGrpcStorageAndAppendMount(volumeType string, vmBasePath string, spec *specs.Spec, sandboxId string) *grpc.Storage {
|
|
||||||
+ var (
|
|
||||||
+ grpcStorag *grpc.Storage
|
|
||||||
+ vmPath string
|
|
||||||
+ )
|
|
||||||
+ switch volumeType {
|
|
||||||
+ case NFS:
|
|
||||||
+ // source of sfs like remote-nfs.com:/share-53ee51a1
|
|
||||||
+ // source of sfs with subpath like remote-nfs.com:/share-53ee51a1/aaa/bbb
|
|
||||||
+ // source of sfs-turbo like ip:/
|
|
||||||
+ // source of sfs-turbo with subpath like ip://aaa or ip://aaa/bbb
|
|
||||||
+ // here we only get the origin source such as remote-nfs.com:/share-53ee51a1 or ip:/ as the Source of grpcStorage
|
|
||||||
+ item := strings.Split(n.Source, ":")
|
|
||||||
+ if len(item) != 2 {
|
|
||||||
+ return nil
|
|
||||||
+ }
|
|
||||||
+ vmPath = filepath.Join(vmBasePath, sandboxId, item[0], item[1])
|
|
||||||
+ parts := strings.SplitAfter(item[1], "/")
|
|
||||||
+ if len(parts) > 2 {
|
|
||||||
+ n.Source = item[0] + ":" + parts[0] + parts[1]
|
|
||||||
+ }
|
|
||||||
+
|
|
||||||
+ grpcStorag = &grpc.Storage{
|
|
||||||
+ Driver: NFS,
|
|
||||||
+ Source: n.Source,
|
|
||||||
+ MountPoint: vmPath,
|
|
||||||
+ }
|
|
||||||
+ grpcStorag.Options = append(grpcStorag.Options, defaultMountOption)
|
|
||||||
+ case GPATHFS:
|
|
||||||
+ vmPath = n.Source
|
|
||||||
+ grpcStorag = &grpc.Storage{
|
|
||||||
+ Driver: GPATHFS,
|
|
||||||
+ Source: vmPath,
|
|
||||||
+ MountPoint: n.Dest,
|
|
||||||
+ }
|
|
||||||
+ }
|
|
||||||
+ if custumOpts, ok := n.Options[storageCustomOptionKey]; ok {
|
|
||||||
+ for _, custumOpt := range custumOpts {
|
|
||||||
+ grpcStorag.Options = append(grpcStorag.Options, custumOpt)
|
|
||||||
+ }
|
|
||||||
+ }
|
|
||||||
+
|
|
||||||
+ mnt := specs.Mount{
|
|
||||||
+ Source: vmPath,
|
|
||||||
+ Destination: n.Dest,
|
|
||||||
+ }
|
|
||||||
+ if mountOpts, ok := n.Options[storageMountOptionKey]; ok {
|
|
||||||
+ for _, mountOpt := range mountOpts {
|
|
||||||
+ if mountOpt == "shared" {
|
|
||||||
+ // in order to support "shared" propagation for guestpath mounting to container directory
|
|
||||||
+ // we should set rootfspropagation shared
|
|
||||||
+ spec.Linux.RootfsPropagation = "shared"
|
|
||||||
+ }
|
|
||||||
+ mnt.Options = append(mnt.Options, mountOpt)
|
|
||||||
+ }
|
|
||||||
+ }
|
|
||||||
+ mnt.Options = append(mnt.Options, "rbind")
|
|
||||||
+ mnt.Type = "bind"
|
|
||||||
+ spec.Mounts = append(spec.Mounts, mnt)
|
|
||||||
+
|
|
||||||
+ return grpcStorag
|
|
||||||
+}
|
|
||||||
diff --git a/virtcontainers/storage/storage_spec.go b/virtcontainers/storage/storage_spec.go
|
|
||||||
new file mode 100644
|
|
||||||
index 00000000..8e866b8d
|
|
||||||
--- /dev/null
|
|
||||||
+++ b/virtcontainers/storage/storage_spec.go
|
|
||||||
@@ -0,0 +1,98 @@
|
|
||||||
+// Copyright (c) Huawei Technologies Co., Ltd. 2019. All rights reserved.
|
|
||||||
+// Description: common functions
|
|
||||||
+// Author: caihaomin c00416947
|
|
||||||
+// Create: 2019-05-05
|
|
||||||
+
|
|
||||||
+package storage
|
|
||||||
+
|
|
||||||
+import (
|
|
||||||
+ "encoding/json"
|
|
||||||
+ "fmt"
|
|
||||||
+
|
|
||||||
+ "github.com/kata-containers/agent/protocols/grpc"
|
|
||||||
+ "github.com/opencontainers/runtime-spec/specs-go"
|
|
||||||
+)
|
|
||||||
+
|
|
||||||
+const (
|
|
||||||
+ NFS = "nfs"
|
|
||||||
+ GPATHFS = "gpath"
|
|
||||||
+)
|
|
||||||
+
|
|
||||||
+// STORAGES specifies the storage type and is supported
|
|
||||||
+var STORAGES = map[string]bool{
|
|
||||||
+ NFS: true,
|
|
||||||
+ GPATHFS: true,
|
|
||||||
+}
|
|
||||||
+
|
|
||||||
+type StorageSpec struct {
|
|
||||||
+ // StorageType specifies the type of storage passing down
|
|
||||||
+ StorageType string `json:"storage_type,omitempty"`
|
|
||||||
+
|
|
||||||
+ Source string `json:"source,omitempty"`
|
|
||||||
+ Destination string `json:"dest,omitempty"`
|
|
||||||
+
|
|
||||||
+ // Options specifies the options of storage type
|
|
||||||
+ Options map[string][]string `json:"options,omitempty"`
|
|
||||||
+}
|
|
||||||
+
|
|
||||||
+func (s *StorageSpec) generateInstances() StorageOperation {
|
|
||||||
+ var storageOpt StorageOperation
|
|
||||||
+ storageOpt = &RemoteStorage{
|
|
||||||
+ Source: s.Source,
|
|
||||||
+ Dest: s.Destination,
|
|
||||||
+ Options: s.Options,
|
|
||||||
+ }
|
|
||||||
+
|
|
||||||
+ return storageOpt
|
|
||||||
+}
|
|
||||||
+
|
|
||||||
+func ValidateStorageValue(value string) error {
|
|
||||||
+ var storageSpecs []StorageSpec
|
|
||||||
+ if err := json.Unmarshal([]byte(value), &storageSpecs); err != nil {
|
|
||||||
+ return err
|
|
||||||
+ }
|
|
||||||
+
|
|
||||||
+ var storageOpt []StorageOperation
|
|
||||||
+ for _, item := range storageSpecs {
|
|
||||||
+ if supported, ok := STORAGES[item.StorageType]; !ok || !supported {
|
|
||||||
+ return fmt.Errorf("type %s of storage is not supported", item.StorageType)
|
|
||||||
+ }
|
|
||||||
+ storageItem := item.generateInstances()
|
|
||||||
+ if err := storageItem.Validate(item.StorageType); err != nil {
|
|
||||||
+ return err
|
|
||||||
+ }
|
|
||||||
+ storageOpt = append(storageOpt, storageItem)
|
|
||||||
+ }
|
|
||||||
+
|
|
||||||
+ return nil
|
|
||||||
+}
|
|
||||||
+
|
|
||||||
+func GetGrpcStorageAndAppendMount(vmBasePath, value string, spec *specs.Spec, sandboxId string) []*grpc.Storage {
|
|
||||||
+ var (
|
|
||||||
+ grpcStorages []*grpc.Storage
|
|
||||||
+ storageSpec []StorageSpec
|
|
||||||
+ )
|
|
||||||
+
|
|
||||||
+ // the value has been validated before, and there is no need to judge the return value here
|
|
||||||
+ json.Unmarshal([]byte(value), &storageSpec)
|
|
||||||
+ for _, item := range storageSpec {
|
|
||||||
+ storageItem := item.generateInstances()
|
|
||||||
+ grpcStorages = append(grpcStorages, storageItem.GetGrpcStorageAndAppendMount(item.StorageType, vmBasePath, spec, sandboxId))
|
|
||||||
+ }
|
|
||||||
+ return grpcStorages
|
|
||||||
+}
|
|
||||||
+
|
|
||||||
+type StorageOperation interface {
|
|
||||||
+ Validate(volumeType string) error
|
|
||||||
+ GetGrpcStorageAndAppendMount(volumeType string, vmBasePath string, spec *specs.Spec, sandboxId string) *grpc.Storage
|
|
||||||
+}
|
|
||||||
+
|
|
||||||
+type FakeStorage struct{}
|
|
||||||
+
|
|
||||||
+func (f *FakeStorage) Validate() error {
|
|
||||||
+ return nil
|
|
||||||
+}
|
|
||||||
+
|
|
||||||
+func (f *FakeStorage) GetGrpcStorageAndAppendMount(vmBasePath string, spec *specs.Spec) *grpc.Storage {
|
|
||||||
+ return &grpc.Storage{}
|
|
||||||
+}
|
|
||||||
--
|
|
||||||
2.14.3 (Apple Git-98)
|
|
||||||
|
|
||||||
@ -1,221 +0,0 @@
|
|||||||
From 2e32e2c156a134605de42c53ef77366ac73f2614 Mon Sep 17 00:00:00 2001
|
|
||||||
From: jiangpengfei <jiangpengfei9@huawei.com>
|
|
||||||
Date: Tue, 18 Aug 2020 22:01:58 +0800
|
|
||||||
Subject: [PATCH 40/50] storage: add go tests for storage
|
|
||||||
|
|
||||||
reason: add go tests file storage_test.go and storage_spec_test.go
|
|
||||||
|
|
||||||
Signed-off-by: jiangpengfei <jiangpengfei9@huawei.com>
|
|
||||||
---
|
|
||||||
virtcontainers/storage/storage_spec_test.go | 151 ++++++++++++++++++++++++++++
|
|
||||||
virtcontainers/storage/storage_test.go | 40 ++++++++
|
|
||||||
2 files changed, 191 insertions(+)
|
|
||||||
create mode 100644 virtcontainers/storage/storage_spec_test.go
|
|
||||||
create mode 100644 virtcontainers/storage/storage_test.go
|
|
||||||
|
|
||||||
diff --git a/virtcontainers/storage/storage_spec_test.go b/virtcontainers/storage/storage_spec_test.go
|
|
||||||
new file mode 100644
|
|
||||||
index 00000000..f638245a
|
|
||||||
--- /dev/null
|
|
||||||
+++ b/virtcontainers/storage/storage_spec_test.go
|
|
||||||
@@ -0,0 +1,151 @@
|
|
||||||
+/*
|
|
||||||
+Copyright (c) Huawei Technologies Co., Ltd. 2019. All rights reserved.
|
|
||||||
+Description: common functions test
|
|
||||||
+Author: yangfeiyu
|
|
||||||
+Create: 2019-07-24
|
|
||||||
+*/
|
|
||||||
+
|
|
||||||
+package storage
|
|
||||||
+
|
|
||||||
+import (
|
|
||||||
+ "encoding/json"
|
|
||||||
+ "testing"
|
|
||||||
+
|
|
||||||
+ "github.com/stretchr/testify/assert"
|
|
||||||
+ "github.com/opencontainers/runtime-spec/specs-go"
|
|
||||||
+)
|
|
||||||
+
|
|
||||||
+func TestValidateStorageValue(t *testing.T) {
|
|
||||||
+ assert := assert.New(t)
|
|
||||||
+ //invalid storage type,not NFS or GPATHS
|
|
||||||
+ spec := &StorageSpec{
|
|
||||||
+ StorageType: "invalidStorage",
|
|
||||||
+ Source: "sfs.com:/remote/path",
|
|
||||||
+ Destination: "/opt/",
|
|
||||||
+ }
|
|
||||||
+ data, err := json.Marshal(spec)
|
|
||||||
+ assert.NoError(err)
|
|
||||||
+
|
|
||||||
+ err = ValidateStorageValue(string(data))
|
|
||||||
+ assert.Error(err, "StorageType is not NFS or GPATHS,it should return error")
|
|
||||||
+
|
|
||||||
+ //case 1: NFS
|
|
||||||
+ specNFS := []StorageSpec{
|
|
||||||
+ {
|
|
||||||
+ StorageType: NFS,
|
|
||||||
+ Source: "sfs.com:/remote/path",
|
|
||||||
+ Destination: "/opt/"},
|
|
||||||
+ }
|
|
||||||
+ data, err = json.Marshal(specNFS)
|
|
||||||
+ assert.NoError(err)
|
|
||||||
+
|
|
||||||
+ err = ValidateStorageValue(string(data))
|
|
||||||
+ assert.NoError(err)
|
|
||||||
+
|
|
||||||
+ // case2: invalid source domain address
|
|
||||||
+ specNFS = []StorageSpec{
|
|
||||||
+ {
|
|
||||||
+ StorageType: NFS,
|
|
||||||
+ Source: "sfs..com:/remote/path",
|
|
||||||
+ Destination: "/opt/"},
|
|
||||||
+ }
|
|
||||||
+ data, err = json.Marshal(specNFS)
|
|
||||||
+ assert.NoError(err)
|
|
||||||
+
|
|
||||||
+ err = ValidateStorageValue(string(data))
|
|
||||||
+ assert.Error(err)
|
|
||||||
+
|
|
||||||
+ // case 3: nfs source is valid ip address
|
|
||||||
+ ipSpecNFS := []StorageSpec{
|
|
||||||
+ {
|
|
||||||
+ StorageType: NFS,
|
|
||||||
+ Source: "192.168.18.147:/remote/path",
|
|
||||||
+ Destination: "/tmp/sfsturbo0",
|
|
||||||
+ },
|
|
||||||
+ }
|
|
||||||
+
|
|
||||||
+ data, err = json.Marshal(ipSpecNFS)
|
|
||||||
+ assert.NoError(err)
|
|
||||||
+
|
|
||||||
+ err = ValidateStorageValue(string(data))
|
|
||||||
+ assert.NoError(err)
|
|
||||||
+
|
|
||||||
+ // case 4: invalid ip address for nfs source
|
|
||||||
+ ipSpecNFS = []StorageSpec{
|
|
||||||
+ {
|
|
||||||
+ StorageType: NFS,
|
|
||||||
+ Source: "192.168.18.300:/remote/path",
|
|
||||||
+ Destination: "/tmp/sfsturbo0",
|
|
||||||
+ },
|
|
||||||
+ }
|
|
||||||
+
|
|
||||||
+ data, err = json.Marshal(ipSpecNFS)
|
|
||||||
+ assert.NoError(err)
|
|
||||||
+
|
|
||||||
+ err = ValidateStorageValue(string(data))
|
|
||||||
+ assert.Error(err)
|
|
||||||
+
|
|
||||||
+ // case 5: validate ip address and source remote path is /
|
|
||||||
+ ipSpecNFS = []StorageSpec{
|
|
||||||
+ {
|
|
||||||
+ StorageType: NFS,
|
|
||||||
+ Source: "192.168.18.3:/",
|
|
||||||
+ Destination: "/tmp/sfsturbo0",
|
|
||||||
+ },
|
|
||||||
+ }
|
|
||||||
+
|
|
||||||
+ data, err = json.Marshal(ipSpecNFS)
|
|
||||||
+ assert.NoError(err)
|
|
||||||
+
|
|
||||||
+ err = ValidateStorageValue(string(data))
|
|
||||||
+ assert.NoError(err)
|
|
||||||
+
|
|
||||||
+ // case 6: invalid ip address 192.168.18.147.11
|
|
||||||
+ ipSpecNFS = []StorageSpec{
|
|
||||||
+ {
|
|
||||||
+ StorageType: NFS,
|
|
||||||
+ Source: "192.168.18.147.11:/",
|
|
||||||
+ Destination: "/tmp/sfsturbo0",
|
|
||||||
+ },
|
|
||||||
+ }
|
|
||||||
+
|
|
||||||
+ data, err = json.Marshal(ipSpecNFS)
|
|
||||||
+ assert.NoError(err)
|
|
||||||
+
|
|
||||||
+ err = ValidateStorageValue(string(data))
|
|
||||||
+ assert.Error(err)
|
|
||||||
+
|
|
||||||
+ //GPATHS
|
|
||||||
+ specGPATHFS := []StorageSpec{
|
|
||||||
+ {
|
|
||||||
+ StorageType: GPATHFS,
|
|
||||||
+ Source: "/remote/path",
|
|
||||||
+ Destination: "/opt/"},
|
|
||||||
+ }
|
|
||||||
+ data, err = json.Marshal(specGPATHFS)
|
|
||||||
+ assert.NoError(err)
|
|
||||||
+
|
|
||||||
+ err = ValidateStorageValue(string(data))
|
|
||||||
+ if STORAGES[GPATHFS] == true {
|
|
||||||
+ assert.NoError(err, "StorageType GPATHFS is valid,it should return no error")
|
|
||||||
+ } else {
|
|
||||||
+ assert.Error(err, "StorageType GPATHFS is invalid,it should return error")
|
|
||||||
+ }
|
|
||||||
+}
|
|
||||||
+
|
|
||||||
+func TestGetGrpcStorageAndAppendMount(t *testing.T) {
|
|
||||||
+ assert := assert.New(t)
|
|
||||||
+ var spec specs.Spec
|
|
||||||
+ sandboxId := "7f3b5e32cc7fe757e1b69f7a005ca1971def5011e730c82535783630fe24b318"
|
|
||||||
+ vmBasePath := "/tmp"
|
|
||||||
+
|
|
||||||
+ // storageType nfs with legal source and dest will succeed
|
|
||||||
+ storageSpec := "[{\"storage_type\":\"nfs\",\"source\":\"sfs.com:/remote/path\",\"dest\":\"/opt/nfs\"}]"
|
|
||||||
+ grpcStorages := GetGrpcStorageAndAppendMount(vmBasePath,storageSpec,&spec,sandboxId)
|
|
||||||
+ assert.NotNil(grpcStorages[0])
|
|
||||||
+
|
|
||||||
+ // storageType nfs with illegal source and dest will fail
|
|
||||||
+ storageSpec = "[{\"storage_type\":\"nfs\",\"source\":\"sfs.com\",\"dest\":\"/opt/nfs\"}]"
|
|
||||||
+ grpcStorages = GetGrpcStorageAndAppendMount(vmBasePath,storageSpec,&spec,sandboxId)
|
|
||||||
+ assert.Nil(grpcStorages[0])
|
|
||||||
+}
|
|
||||||
diff --git a/virtcontainers/storage/storage_test.go b/virtcontainers/storage/storage_test.go
|
|
||||||
new file mode 100644
|
|
||||||
index 00000000..c9ca4926
|
|
||||||
--- /dev/null
|
|
||||||
+++ b/virtcontainers/storage/storage_test.go
|
|
||||||
@@ -0,0 +1,40 @@
|
|
||||||
+/*
|
|
||||||
+Copyright (c) Huawei Technologies Co., Ltd. 2019. All rights reserved.
|
|
||||||
+Description: common functions test
|
|
||||||
+Author: yangfeiyu
|
|
||||||
+Create: 2019-07-24
|
|
||||||
+*/
|
|
||||||
+
|
|
||||||
+package storage
|
|
||||||
+
|
|
||||||
+import (
|
|
||||||
+ "testing"
|
|
||||||
+
|
|
||||||
+ "github.com/stretchr/testify/assert"
|
|
||||||
+)
|
|
||||||
+
|
|
||||||
+func TestValidate(t *testing.T) {
|
|
||||||
+ //NFS valid
|
|
||||||
+ assert := assert.New(t)
|
|
||||||
+ remoteStorage := &RemoteStorage{
|
|
||||||
+ Source: "sfs.com:/remote/path",
|
|
||||||
+ Dest: "/opt/",
|
|
||||||
+ }
|
|
||||||
+ err := remoteStorage.Validate(NFS)
|
|
||||||
+ assert.NoError(err, "NFS url is valid,it should return no error")
|
|
||||||
+
|
|
||||||
+ //NFS invalid
|
|
||||||
+ remoteStorage.Source = "sfs.com/../remote/path"
|
|
||||||
+ err = remoteStorage.Validate(NFS)
|
|
||||||
+ assert.Error(err, "NFS url is invalid,it should return error")
|
|
||||||
+
|
|
||||||
+ //GPATHS valid
|
|
||||||
+ remoteStorage.Source = "/path/in/vm"
|
|
||||||
+ err = remoteStorage.Validate(GPATHFS)
|
|
||||||
+ assert.NoError(err, "GPATHFS is valid,it should return no error")
|
|
||||||
+
|
|
||||||
+ //GPATHS invalid
|
|
||||||
+ remoteStorage.Source = "./../path/in/../vm"
|
|
||||||
+ err = remoteStorage.Validate(GPATHFS)
|
|
||||||
+ assert.Error(err, "GPATHFS is invalid,it should return error")
|
|
||||||
+}
|
|
||||||
--
|
|
||||||
2.14.3 (Apple Git-98)
|
|
||||||
|
|
||||||
@ -1,115 +0,0 @@
|
|||||||
From 31e07f1b6cbf361783c4d7adf9e4b8da30c67384 Mon Sep 17 00:00:00 2001
|
|
||||||
From: jiangpengfei <jiangpengfei9@huawei.com>
|
|
||||||
Date: Tue, 18 Aug 2020 22:05:25 +0800
|
|
||||||
Subject: [PATCH 41/50] storage: mount nfs and gpath with given annotation
|
|
||||||
|
|
||||||
reason: when run container with annotation about storage spec,
|
|
||||||
prepare basic info in kata-runtime
|
|
||||||
|
|
||||||
Signed-off-by: jiangpengfei <jiangpengfei9@huawei.com>
|
|
||||||
---
|
|
||||||
virtcontainers/kata_agent.go | 14 +++++++++++++-
|
|
||||||
virtcontainers/pkg/annotations/annotations.go | 3 +++
|
|
||||||
virtcontainers/pkg/oci/utils.go | 16 ++++++++++++++++
|
|
||||||
3 files changed, 32 insertions(+), 1 deletion(-)
|
|
||||||
|
|
||||||
diff --git a/virtcontainers/kata_agent.go b/virtcontainers/kata_agent.go
|
|
||||||
index d82a7f2d..ac64817a 100644
|
|
||||||
--- a/virtcontainers/kata_agent.go
|
|
||||||
+++ b/virtcontainers/kata_agent.go
|
|
||||||
@@ -30,10 +30,11 @@ import (
|
|
||||||
ns "github.com/kata-containers/runtime/virtcontainers/pkg/nsenter"
|
|
||||||
"github.com/kata-containers/runtime/virtcontainers/pkg/rootless"
|
|
||||||
vcTypes "github.com/kata-containers/runtime/virtcontainers/pkg/types"
|
|
||||||
- "github.com/kata-containers/runtime/virtcontainers/utils"
|
|
||||||
"github.com/kata-containers/runtime/virtcontainers/pkg/uuid"
|
|
||||||
+ "github.com/kata-containers/runtime/virtcontainers/storage"
|
|
||||||
"github.com/kata-containers/runtime/virtcontainers/store"
|
|
||||||
"github.com/kata-containers/runtime/virtcontainers/types"
|
|
||||||
+ "github.com/kata-containers/runtime/virtcontainers/utils"
|
|
||||||
"github.com/opencontainers/runtime-spec/specs-go"
|
|
||||||
opentracing "github.com/opentracing/opentracing-go"
|
|
||||||
"github.com/sirupsen/logrus"
|
|
||||||
@@ -1427,6 +1428,9 @@ func (k *kataAgent) createContainer(sandbox *Sandbox, c *Container) (p *Process,
|
|
||||||
localStorages := k.handleLocalStorage(ociSpec.Mounts, sandbox.id, c.rootfsSuffix)
|
|
||||||
ctrStorages = append(ctrStorages, localStorages...)
|
|
||||||
|
|
||||||
+ remoteStoragtes := k.handleRemoteStorage(ociSpec, sandbox.id)
|
|
||||||
+ ctrStorages = append(ctrStorages, remoteStoragtes...)
|
|
||||||
+
|
|
||||||
// We replace all OCI mount sources that match our container mount
|
|
||||||
// with the right source path (The guest one).
|
|
||||||
if err = k.replaceOCIMountSource(ociSpec, newMounts); err != nil {
|
|
||||||
@@ -1510,6 +1514,14 @@ func (k *kataAgent) createContainer(sandbox *Sandbox, c *Container) (p *Process,
|
|
||||||
k.state.URL, consoleURL, c.config.Cmd, createNSList, enterNSList)
|
|
||||||
}
|
|
||||||
|
|
||||||
+func (k *kataAgent) handleRemoteStorage(spec *specs.Spec, sandboxId string) []*grpc.Storage {
|
|
||||||
+ if value, ok := spec.Annotations[vcAnnotations.StorageSpecTypeKey]; ok {
|
|
||||||
+ return storage.GetGrpcStorageAndAppendMount(kataGuestStorageDir, value, spec, sandboxId)
|
|
||||||
+ }
|
|
||||||
+
|
|
||||||
+ return []*grpc.Storage{}
|
|
||||||
+}
|
|
||||||
+
|
|
||||||
// handleEphemeralStorage handles ephemeral storages by
|
|
||||||
// creating a Storage from corresponding source of the mount point
|
|
||||||
func (k *kataAgent) handleEphemeralStorage(mounts []specs.Mount) []*grpc.Storage {
|
|
||||||
diff --git a/virtcontainers/pkg/annotations/annotations.go b/virtcontainers/pkg/annotations/annotations.go
|
|
||||||
index 903c7f03..e50a697c 100644
|
|
||||||
--- a/virtcontainers/pkg/annotations/annotations.go
|
|
||||||
+++ b/virtcontainers/pkg/annotations/annotations.go
|
|
||||||
@@ -68,6 +68,9 @@ const (
|
|
||||||
// AssetHashType is the hash type used for assets verification
|
|
||||||
AssetHashType = kataAnnotationsPrefix + "asset_hash_type"
|
|
||||||
|
|
||||||
+ // StorageSpecTypeKey is the annotation key to fetch storage_spec
|
|
||||||
+ StorageSpecTypeKey = kataAnnotationsPrefix + "storage_spec"
|
|
||||||
+
|
|
||||||
//
|
|
||||||
// Generic annotations
|
|
||||||
//
|
|
||||||
diff --git a/virtcontainers/pkg/oci/utils.go b/virtcontainers/pkg/oci/utils.go
|
|
||||||
index 948bd3cb..d032227e 100644
|
|
||||||
--- a/virtcontainers/pkg/oci/utils.go
|
|
||||||
+++ b/virtcontainers/pkg/oci/utils.go
|
|
||||||
@@ -22,6 +22,7 @@ import (
|
|
||||||
exp "github.com/kata-containers/runtime/virtcontainers/experimental"
|
|
||||||
vcAnnotations "github.com/kata-containers/runtime/virtcontainers/pkg/annotations"
|
|
||||||
dockershimAnnotations "github.com/kata-containers/runtime/virtcontainers/pkg/annotations/dockershim"
|
|
||||||
+ "github.com/kata-containers/runtime/virtcontainers/storage"
|
|
||||||
"github.com/kata-containers/runtime/virtcontainers/types"
|
|
||||||
"github.com/kata-containers/runtime/virtcontainers/utils"
|
|
||||||
specs "github.com/opencontainers/runtime-spec/specs-go"
|
|
||||||
@@ -340,6 +341,17 @@ func SandboxID(spec specs.Spec) (string, error) {
|
|
||||||
return "", fmt.Errorf("Could not find sandbox ID")
|
|
||||||
}
|
|
||||||
|
|
||||||
+func validateStorageSpec(spec specs.Spec) error {
|
|
||||||
+ if storageSpec, ok := spec.Annotations[vcAnnotations.StorageSpecTypeKey]; ok {
|
|
||||||
+ err := storage.ValidateStorageValue(storageSpec)
|
|
||||||
+ if err != nil {
|
|
||||||
+ return err
|
|
||||||
+ }
|
|
||||||
+ }
|
|
||||||
+
|
|
||||||
+ return nil
|
|
||||||
+}
|
|
||||||
+
|
|
||||||
func addAnnotations(ocispec specs.Spec, config *vc.SandboxConfig) error {
|
|
||||||
addAssetAnnotations(ocispec, config)
|
|
||||||
if err := addHypervisorConfigOverrides(ocispec, config); err != nil {
|
|
||||||
@@ -873,6 +885,10 @@ func SandboxConfig(ocispec specs.Spec, runtime RuntimeConfig, bundlePath, cid, c
|
|
||||||
// ContainerConfig converts an OCI compatible runtime configuration
|
|
||||||
// file to a virtcontainers container configuration structure.
|
|
||||||
func ContainerConfig(ocispec specs.Spec, bundlePath, cid, console string, detach bool) (vc.ContainerConfig, error) {
|
|
||||||
+ err := validateStorageSpec(ocispec)
|
|
||||||
+ if err != nil {
|
|
||||||
+ return vc.ContainerConfig{}, err
|
|
||||||
+ }
|
|
||||||
rootfs := vc.RootFs{Target: ocispec.Root.Path, Mounted: true}
|
|
||||||
if !filepath.IsAbs(rootfs.Target) {
|
|
||||||
rootfs.Target = filepath.Join(bundlePath, ocispec.Root.Path)
|
|
||||||
--
|
|
||||||
2.14.3 (Apple Git-98)
|
|
||||||
|
|
||||||
Some files were not shown because too many files have changed in this diff Show More
Loading…
x
Reference in New Issue
Block a user