!106 kata-containers:upgrade to 2.x
From: @Vanient Reviewed-by: @flyflyflypeng, @duguhaotian Signed-off-by: @duguhaotian
This commit is contained in:
openeuler-ci-bot
Gitee
commit
a40fe16a9f
Binary file not shown.
@ -1,20 +0,0 @@
|
||||
#!/bin/bash
|
||||
|
||||
if [[ -f ./patch_flag ]];then
|
||||
echo "agent patched!"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
tar -zxvf agent-1.11.1.tar.gz
|
||||
cp -fr ./agent-1.11.1/* ./
|
||||
rm -rf ./agent-1.11.1
|
||||
cat ./series.conf | while read line
|
||||
do
|
||||
if [[ $line == '' || $line =~ ^\s*# ]]; then
|
||||
continue
|
||||
fi
|
||||
echo "====patch $line======"
|
||||
pwd
|
||||
patch -p1 -F1 -s < ./patches/$line
|
||||
done
|
||||
touch ./patch_flag
|
||||
@ -1,234 +0,0 @@
|
||||
From ac1d7806f8de2f8ca393df08a9c62d1045c4afdc Mon Sep 17 00:00:00 2001
|
||||
From: jiangpengfei <jiangpengfei9@huawei.com>
|
||||
Date: Tue, 11 Dec 2018 18:27:02 -0500
|
||||
Subject: [PATCH 01/16] agent: add agent.netlink_recv_buf_size flag to set
|
||||
netlink recv buf size
|
||||
|
||||
fixes: #813
|
||||
|
||||
reason: If hotplug huge size memory(for example 128GB) into guest,
|
||||
kernel will produce a lot of memory add uevents and send to netlink socket,
|
||||
however netlink socket default receive buffer size is 4KB, which is too small
|
||||
to receive all memory add uevents.
|
||||
Since hotplug huge size memory is not common case, so we consider add an agent
|
||||
flag agent.netlink_recv_buf_size to set netlink socket recv buffer size.
|
||||
|
||||
Signed-off-by: jiangpengfei <jiangpengfei9@huawei.com>
|
||||
---
|
||||
README.md | 13 +++++++++++++
|
||||
agent.go | 10 +++++++++-
|
||||
config.go | 15 ++++++++++++++
|
||||
config_test.go | 55 ++++++++++++++++++++++++++++++++++++++++++++++++++++
|
||||
pkg/uevent/uevent.go | 15 +++++++++++---
|
||||
5 files changed, 104 insertions(+), 4 deletions(-)
|
||||
|
||||
diff --git a/README.md b/README.md
|
||||
index cec65a4..16f96a4 100644
|
||||
--- a/README.md
|
||||
+++ b/README.md
|
||||
@@ -98,6 +98,19 @@ The pipe's capacity for stdout/stderr can be modified by specifying the `agent.c
|
||||
to the guest kernel command line. For example, `agent.container_pipe_size=2097152` will set the stdout and stderr
|
||||
pipes to 2097152 bytes.
|
||||
|
||||
+## Uevent Netlink Socket Receive Buffer Size
|
||||
+
|
||||
+When hotplugging a huge size memory into the Kata VM, the kernel in the VM will produce a lot of memory object add
|
||||
+uevents and send all these uevents to Kata agent by netlink socket. However, default netlink socket receive buffer
|
||||
+size is 4KB, which is too small and can only hold 256 memory add uevents. If memory add uevents number is larger
|
||||
+than 256, the left uevents can not be received and processed by Kata agent.
|
||||
+
|
||||
+The uevent netlink socket receive buffer size can be modified by specifying the `agent.netlink_recv_buf_size` flag
|
||||
+to the guest kernel command line. For example, `agent.netlink_recv_buf_size=2MB` will set the uevent netlink socket
|
||||
+receive buffer size to 2MB value. `agent.netlink_recv_buf_size` valid value range is `[4KB ~ 4MB]` and value can be
|
||||
+set in human-readable memory format or pure digital number format(default memory unit is byte).
|
||||
+
|
||||
+
|
||||
[1]: https://github.com/firecracker-microvm/firecracker/blob/master/docs/vsock.md
|
||||
[2]: https://golang.org/pkg/time/#ParseDuration
|
||||
[3]: http://man7.org/linux/man-pages/man7/pipe.7.html
|
||||
diff --git a/agent.go b/agent.go
|
||||
index 2d2c293..c1cac08 100644
|
||||
--- a/agent.go
|
||||
+++ b/agent.go
|
||||
@@ -190,6 +190,14 @@ var unifiedCgroupHierarchy = false
|
||||
// Size in bytes of the stdout/stderr pipes created for each container.
|
||||
var containerPipeSize = uint32(0)
|
||||
|
||||
+const (
|
||||
+ minNetlinkSockRecvBufSize = 4 * 1024
|
||||
+ maxNetlinkSockRecvBufSize = 4 * 1024 * 1024
|
||||
+)
|
||||
+
|
||||
+// Size in bytes of the netlink socket recv buf size
|
||||
+var netlinkSockRecvBufSize = uint32(0)
|
||||
+
|
||||
// commType is used to denote the communication channel type used.
|
||||
type commType int
|
||||
|
||||
@@ -708,7 +716,7 @@ func (s *sandbox) waitForStopServer() {
|
||||
func (s *sandbox) listenToUdevEvents() {
|
||||
fieldLogger := agentLog.WithField("subsystem", "udevlistener")
|
||||
|
||||
- uEvHandler, err := uevent.NewHandler()
|
||||
+ uEvHandler, err := uevent.NewHandler(netlinkSockRecvBufSize)
|
||||
if err != nil {
|
||||
fieldLogger.Warnf("Error starting uevent listening loop %s", err)
|
||||
return
|
||||
diff --git a/config.go b/config.go
|
||||
index 4530096..6c7d473 100644
|
||||
--- a/config.go
|
||||
+++ b/config.go
|
||||
@@ -7,11 +7,13 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
+ "fmt"
|
||||
"io/ioutil"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
+ "github.com/docker/go-units"
|
||||
"github.com/sirupsen/logrus"
|
||||
"google.golang.org/grpc/codes"
|
||||
grpcStatus "google.golang.org/grpc/status"
|
||||
@@ -29,6 +31,7 @@ const (
|
||||
hotplugTimeoutFlag = optionPrefix + "hotplug_timeout"
|
||||
unifiedCgroupHierarchyFlag = optionPrefix + "unified_cgroup_hierarchy"
|
||||
containerPipeSizeFlag = optionPrefix + "container_pipe_size"
|
||||
+ netlinkSockRecvBufSizeFlag = optionPrefix + "netlink_recv_buf_size"
|
||||
traceModeStatic = "static"
|
||||
traceModeDynamic = "dynamic"
|
||||
traceTypeIsolated = "isolated"
|
||||
@@ -155,6 +158,18 @@ func parseCmdlineOption(option string) error {
|
||||
return err
|
||||
}
|
||||
unifiedCgroupHierarchy = flag
|
||||
+ case netlinkSockRecvBufSizeFlag:
|
||||
+ bufSizeInBytes, err := units.RAMInBytes(split[valuePosition])
|
||||
+ if err != nil {
|
||||
+ return err
|
||||
+ }
|
||||
+
|
||||
+ if bufSizeInBytes < minNetlinkSockRecvBufSize || bufSizeInBytes > maxNetlinkSockRecvBufSize {
|
||||
+ return fmt.Errorf("invalid netlink socket recv buf size: %d (valid size range %s-%s bytes)", bufSizeInBytes,
|
||||
+ units.BytesSize(minNetlinkSockRecvBufSize), units.BytesSize(maxNetlinkSockRecvBufSize))
|
||||
+ }
|
||||
+
|
||||
+ netlinkSockRecvBufSize = uint32(bufSizeInBytes)
|
||||
default:
|
||||
if strings.HasPrefix(split[optionPosition], optionPrefix) {
|
||||
return grpcStatus.Errorf(codes.NotFound, "Unknown option %s", split[optionPosition])
|
||||
diff --git a/config_test.go b/config_test.go
|
||||
index 2a23133..f40f17a 100644
|
||||
--- a/config_test.go
|
||||
+++ b/config_test.go
|
||||
@@ -486,3 +486,58 @@ func TestParseCmdlineOptionContainerPipeSize(t *testing.T) {
|
||||
assert.Equal(d.expectedContainerPipeSize, containerPipeSize, "test %d (%+v)", i, d)
|
||||
}
|
||||
}
|
||||
+
|
||||
+func TestParseCmdlineOptionNetlinkSockRecvBufSize(t *testing.T) {
|
||||
+ assert := assert.New(t)
|
||||
+
|
||||
+ type testData struct {
|
||||
+ option string
|
||||
+ shouldErr bool
|
||||
+ expectedNetlinkSockRecvBufSize uint32
|
||||
+ }
|
||||
+
|
||||
+ data := []testData{
|
||||
+ {"", false, 0},
|
||||
+ {"netlink_recv_buf_siz", false, 0},
|
||||
+ {"netlink_recv_buf_size", false, 0},
|
||||
+ {"netlink_recv_buf_size=", false, 0},
|
||||
+ {"netlink_recv_buf_size=4096", false, 0},
|
||||
+ {"netlink_recv_buf_size=4KB", false, 0},
|
||||
+ {"agent.netlink_recv_buf_size=", true, 0},
|
||||
+ {"agent.netlink_recv_buf_size=foobar", true, 0},
|
||||
+ {"agent.netlink_recv_buf_size=-1", true, 0},
|
||||
+ {"agent.netlink_recv_buf_size=0", true, 0},
|
||||
+ {"agent.netlink_recv_buf_size=100", true, 0},
|
||||
+ {"agent.netlink_recv_buf_size=3KB", true, 0},
|
||||
+ {"agent.netlink_recv_buf_size=3.6KB", true, 0},
|
||||
+ {"agent.netlink_recv_buf_size=4095", true, 0},
|
||||
+ {"agent.netlink_recv_buf_size=4096xB", true, 0},
|
||||
+ {"agent.netlink_recv_buf_size=4096", false, 4096},
|
||||
+ {"agent.netlink_recv_buf_size=4097", false, 4097},
|
||||
+ {"agent.netlink_recv_buf_size=4096.0", false, 4096},
|
||||
+ {"agent.netlink_recv_buf_size=1024KB", false, 1048576},
|
||||
+ {"agent.netlink_recv_buf_size=1MB", false, 1048576},
|
||||
+ {"agent.netlink_recv_buf_size=4194303", false, 4194303},
|
||||
+ {"agent.netlink_recv_buf_size=3.999MB", false, 4193255},
|
||||
+ {"agent.netlink_recv_buf_size=4194304", false, 4194304},
|
||||
+ {"agent.netlink_recv_buf_size=4MB", false, 4194304},
|
||||
+ {"agent.netlink_recv_buf_size=4.001MB", true, 0},
|
||||
+ {"agent.netlink_recv_buf_size=4194305", true, 0},
|
||||
+ {"agent.netlink_recv_buf_size=100MB", true, 0},
|
||||
+ {"agent.netlink_recv_buf_size=1GB", true, 0},
|
||||
+ }
|
||||
+
|
||||
+ for i, d := range data {
|
||||
+ // reset the netlink socket recv buffer size
|
||||
+ netlinkSockRecvBufSize = 0
|
||||
+
|
||||
+ err := parseCmdlineOption(d.option)
|
||||
+ if d.shouldErr {
|
||||
+ assert.Error(err)
|
||||
+ } else {
|
||||
+ assert.NoError(err)
|
||||
+ }
|
||||
+
|
||||
+ assert.Equal(d.expectedNetlinkSockRecvBufSize, netlinkSockRecvBufSize, "test %d (%+v)", i, d)
|
||||
+ }
|
||||
+}
|
||||
diff --git a/pkg/uevent/uevent.go b/pkg/uevent/uevent.go
|
||||
index fc2c127..fa84086 100644
|
||||
--- a/pkg/uevent/uevent.go
|
||||
+++ b/pkg/uevent/uevent.go
|
||||
@@ -10,6 +10,7 @@ import (
|
||||
"bufio"
|
||||
"io"
|
||||
"strings"
|
||||
+ "syscall"
|
||||
|
||||
"golang.org/x/sys/unix"
|
||||
"google.golang.org/grpc/codes"
|
||||
@@ -33,7 +34,7 @@ type ReaderCloser struct {
|
||||
}
|
||||
|
||||
// NewReaderCloser returns an io.ReadCloser handle for uevent.
|
||||
-func NewReaderCloser() (io.ReadCloser, error) {
|
||||
+func NewReaderCloser(netlinkRecvBufSize uint32) (io.ReadCloser, error) {
|
||||
nl := unix.SockaddrNetlink{
|
||||
Family: unix.AF_NETLINK,
|
||||
// Passing Pid as 0 here allows the kernel to take care of assigning
|
||||
@@ -47,6 +48,14 @@ func NewReaderCloser() (io.ReadCloser, error) {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
+ // If netlinkRecvBufSize > 0, set netlink socket recv buffer size to netlinkRecvBufSize
|
||||
+ if netlinkRecvBufSize > 0 {
|
||||
+ err = unix.SetsockoptInt(fd, syscall.SOL_SOCKET, syscall.SO_RCVBUFFORCE, int(netlinkRecvBufSize))
|
||||
+ if err != nil {
|
||||
+ return nil, err
|
||||
+ }
|
||||
+ }
|
||||
+
|
||||
if err := unix.Bind(fd, &nl); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@@ -85,8 +94,8 @@ type Handler struct {
|
||||
}
|
||||
|
||||
// NewHandler returns a uevent handler.
|
||||
-func NewHandler() (*Handler, error) {
|
||||
- rdCloser, err := NewReaderCloser()
|
||||
+func NewHandler(netlinkRecvBufSize uint32) (*Handler, error) {
|
||||
+ rdCloser, err := NewReaderCloser(netlinkRecvBufSize)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
--
|
||||
2.14.3 (Apple Git-98)
|
||||
|
||||
@ -1,686 +0,0 @@
|
||||
From 13f54c768dcd7bf982dde8e57fb5cd624fedf5bc Mon Sep 17 00:00:00 2001
|
||||
From: jiangpengfei <jiangpengfei9@huawei.com>
|
||||
Date: Mon, 17 Aug 2020 11:23:55 +0800
|
||||
Subject: [PATCH 02/16] network: support update routes incrementally
|
||||
|
||||
reason: add increment flag in the UpdateRoutesRequest to
|
||||
support upate routes incrementally to improve the efficiency.
|
||||
|
||||
kata-network add-route and del-route needs this feature.
|
||||
|
||||
Signed-off-by: jiangpengfei <jiangpengfei9@huawei.com>
|
||||
---
|
||||
grpc.go | 2 +-
|
||||
network.go | 74 ++++++++-
|
||||
network_test.go | 16 +-
|
||||
protocols/grpc/agent.pb.go | 402 +++++++++++++++++++++++++--------------------
|
||||
protocols/grpc/agent.proto | 1 +
|
||||
5 files changed, 300 insertions(+), 195 deletions(-)
|
||||
|
||||
diff --git a/grpc.go b/grpc.go
|
||||
index 886661b..8fe8217 100644
|
||||
--- a/grpc.go
|
||||
+++ b/grpc.go
|
||||
@@ -1556,7 +1556,7 @@ func (a *agentGRPC) UpdateInterface(ctx context.Context, req *pb.UpdateInterface
|
||||
}
|
||||
|
||||
func (a *agentGRPC) UpdateRoutes(ctx context.Context, req *pb.UpdateRoutesRequest) (*pb.Routes, error) {
|
||||
- return a.sandbox.updateRoutes(nil, req.Routes)
|
||||
+ return a.sandbox.updateRoutes(nil, req.Routes, req.Increment)
|
||||
}
|
||||
|
||||
func (a *agentGRPC) ListInterfaces(ctx context.Context, req *pb.ListInterfacesRequest) (*pb.Interfaces, error) {
|
||||
diff --git a/network.go b/network.go
|
||||
index 64a16a9..02e28cb 100644
|
||||
--- a/network.go
|
||||
+++ b/network.go
|
||||
@@ -398,7 +398,7 @@ func (s *sandbox) deleteRoutes(netHandle *netlink.Handle) error {
|
||||
// state which matches the requested routes. In doing this, preesxisting non-loopback routes will be
|
||||
// removed from the network. If an error occurs, this function returns the list of routes in
|
||||
// gRPC-route format at the time of failure
|
||||
-func (s *sandbox) updateRoutes(netHandle *netlink.Handle, requestedRoutes *pb.Routes) (resultingRoutes *pb.Routes, err error) {
|
||||
+func (s *sandbox) updateRoutes(netHandle *netlink.Handle, requestedRoutes *pb.Routes, increment bool) (resultingRoutes *pb.Routes, err error) {
|
||||
if requestedRoutes == nil {
|
||||
return nil, errNoRoutes
|
||||
}
|
||||
@@ -418,13 +418,66 @@ func (s *sandbox) updateRoutes(netHandle *netlink.Handle, requestedRoutes *pb.Ro
|
||||
}
|
||||
}()
|
||||
|
||||
+ var (
|
||||
+ added []*types.Route
|
||||
+ removed []*types.Route
|
||||
+ )
|
||||
+
|
||||
+ defer func(netHandle *netlink.Handle) {
|
||||
+ if err != nil {
|
||||
+ // if error happens after route added, need to rollback the added route
|
||||
+ if len(added) > 0 {
|
||||
+ for _, r := range added {
|
||||
+ errRb := s.updateRoute(netHandle, r, false)
|
||||
+ if errRb != nil {
|
||||
+ agentLog.WithError(err).Error("rollback route failed")
|
||||
+ }
|
||||
+ }
|
||||
+ }
|
||||
+
|
||||
+ // if error happens after route removed, need to rollback the removed route
|
||||
+ if len(removed) > 0 {
|
||||
+ for _, r := range removed {
|
||||
+ errRb := s.updateRoute(netHandle, r, true)
|
||||
+ if errRb != nil {
|
||||
+ agentLog.WithError(err).Error("rollback route failed")
|
||||
+ }
|
||||
+ }
|
||||
+ }
|
||||
+ }
|
||||
+ }(netHandle)
|
||||
+
|
||||
+ // updateOneRoute just update the specified one route
|
||||
+ updateOneRoute := func(netHandle *netlink.Handle, reqRoute *types.Route) error {
|
||||
+ var add bool = true
|
||||
+ if reqRoute.Dest != "" && strings.HasPrefix(reqRoute.Dest, "-") {
|
||||
+ reqRoute.Dest = reqRoute.Dest[1:]
|
||||
+ add = false
|
||||
+ }
|
||||
+ err = s.updateRoute(netHandle, reqRoute, add)
|
||||
+ if err != nil {
|
||||
+ agentLog.WithError(err).Error("update Route failed")
|
||||
+ // If there was an error setting the route, return the error
|
||||
+ // and the current routes on the system via the defer func
|
||||
+ return err
|
||||
+ }
|
||||
+ if add {
|
||||
+ added = append([]*types.Route{reqRoute}, added[:]...)
|
||||
+ } else {
|
||||
+ removed = append([]*types.Route{reqRoute}, removed[:]...)
|
||||
+ }
|
||||
+ return nil
|
||||
+ }
|
||||
+
|
||||
//
|
||||
// First things first, let's blow away all the existing routes. The updateRoutes function
|
||||
// is designed to be declarative, so we will attempt to create state matching what is
|
||||
// requested, and in the event that we fail to do so, will return the error and final state.
|
||||
//
|
||||
- if err = s.deleteRoutes(netHandle); err != nil {
|
||||
- return nil, err
|
||||
+ if !increment {
|
||||
+ if err = s.deleteRoutes(netHandle); err != nil {
|
||||
+ return nil, err
|
||||
+ }
|
||||
}
|
||||
|
||||
//
|
||||
@@ -434,7 +487,12 @@ func (s *sandbox) updateRoutes(netHandle *netlink.Handle, requestedRoutes *pb.Ro
|
||||
// won't be able to access the gateway
|
||||
for _, reqRoute := range requestedRoutes.Routes {
|
||||
if reqRoute.Gateway == "" {
|
||||
- err = s.updateRoute(netHandle, reqRoute, true)
|
||||
+ if increment {
|
||||
+ err = updateOneRoute(netHandle, reqRoute)
|
||||
+ } else {
|
||||
+ err = s.updateRoute(netHandle, reqRoute, true)
|
||||
+ }
|
||||
+
|
||||
if err != nil {
|
||||
agentLog.WithError(err).Error("update Route failed")
|
||||
//If there was an error setting the route, return the error
|
||||
@@ -447,7 +505,11 @@ func (s *sandbox) updateRoutes(netHandle *netlink.Handle, requestedRoutes *pb.Ro
|
||||
// Take a second pass and apply the routes which include a gateway
|
||||
for _, reqRoute := range requestedRoutes.Routes {
|
||||
if reqRoute.Gateway != "" {
|
||||
- err = s.updateRoute(netHandle, reqRoute, true)
|
||||
+ if increment {
|
||||
+ err = updateOneRoute(netHandle, reqRoute)
|
||||
+ } else {
|
||||
+ err = s.updateRoute(netHandle, reqRoute, true)
|
||||
+ }
|
||||
if err != nil {
|
||||
agentLog.WithError(err).Error("update Route failed")
|
||||
//If there was an error setting the route, return the
|
||||
@@ -699,4 +761,4 @@ func (s *sandbox) handleLocalhost() error {
|
||||
}
|
||||
|
||||
return netlink.LinkSetUp(lo)
|
||||
-}
|
||||
+}
|
||||
\ No newline at end of file
|
||||
diff --git a/network_test.go b/network_test.go
|
||||
index a143670..a1e58f5 100644
|
||||
--- a/network_test.go
|
||||
+++ b/network_test.go
|
||||
@@ -160,7 +160,7 @@ func TestUpdateRoutes(t *testing.T) {
|
||||
Routes: inputRoutesSimple,
|
||||
}
|
||||
|
||||
- results, err := s.updateRoutes(netHandle, testRoutes)
|
||||
+ results, err := s.updateRoutes(netHandle, testRoutes, false)
|
||||
assert.Nil(t, err, "Unexpected update interface failure: %v", err)
|
||||
assert.True(t, reflect.DeepEqual(results, testRoutes),
|
||||
"Interface created didn't match: got %+v, expecting %+v", results, testRoutes)
|
||||
@@ -173,7 +173,7 @@ func TestUpdateRoutes(t *testing.T) {
|
||||
}
|
||||
testRoutes.Routes = inputRoutesPTPExample
|
||||
|
||||
- results, err = s.updateRoutes(netHandle, testRoutes)
|
||||
+ results, err = s.updateRoutes(netHandle, testRoutes, false)
|
||||
assert.Nil(t, err, "Unexpected update interface failure: %v", err)
|
||||
assert.True(t, reflect.DeepEqual(results, testRoutes),
|
||||
"Interface created didn't match: got %+v, expecting %+v", results, testRoutes)
|
||||
@@ -184,7 +184,7 @@ func TestUpdateRoutes(t *testing.T) {
|
||||
{Dest: "192.168.0.0/16", Gateway: "", Source: "192.168.0.2", Scope: 0, Device: "ifc-name"},
|
||||
}
|
||||
testRoutes.Routes = inputRoutesNoScope
|
||||
- results, err = s.updateRoutes(netHandle, testRoutes)
|
||||
+ results, err = s.updateRoutes(netHandle, testRoutes, false)
|
||||
assert.NotNil(t, err, "Expected to observe unreachable route failure")
|
||||
|
||||
assert.True(t, reflect.DeepEqual(results.Routes[0], testRoutes.Routes[1]),
|
||||
@@ -231,7 +231,7 @@ func TestUpdateRoutesIPVlan(t *testing.T) {
|
||||
}
|
||||
testRoutes.Routes = inputRoutesIPVlanExample
|
||||
|
||||
- results, err := s.updateRoutes(netHandle, testRoutes)
|
||||
+ results, err := s.updateRoutes(netHandle, testRoutes, false)
|
||||
assert.Nil(t, err, "Unexpected update interface failure: %v", err)
|
||||
assert.True(t, reflect.DeepEqual(results, testRoutes),
|
||||
"Interface created didn't match: got %+v, expecting %+v", results, testRoutes)
|
||||
@@ -357,7 +357,7 @@ func TestListRoutes(t *testing.T) {
|
||||
Routes: inputRoutesSimple,
|
||||
}
|
||||
|
||||
- _, err := s.updateRoutes(netHandle, testRoutes)
|
||||
+ _, err := s.updateRoutes(netHandle, testRoutes, false)
|
||||
assert.Nil(err)
|
||||
results, err := s.listRoutes(nil)
|
||||
assert.Nil(err, "Expected to list all routes")
|
||||
@@ -377,7 +377,7 @@ func TestListRoutes(t *testing.T) {
|
||||
Routes: inputRoutesSimple,
|
||||
}
|
||||
|
||||
- _, err = s.updateRoutes(netHandle, testRoutes)
|
||||
+ _, err = s.updateRoutes(netHandle, testRoutes, false)
|
||||
assert.Nil(err)
|
||||
results, err = s.listRoutes(nil)
|
||||
assert.Nil(err, "Expected to list all routes")
|
||||
@@ -438,7 +438,7 @@ func TestListRoutesWithIPV6(t *testing.T) {
|
||||
Routes: inputRoutesSimple,
|
||||
}
|
||||
|
||||
- _, err := s.updateRoutes(netHandle, testRoutes)
|
||||
+ _, err := s.updateRoutes(netHandle, testRoutes, false)
|
||||
assert.Nil(err)
|
||||
results, err := s.listRoutes(nil)
|
||||
assert.Nil(err, "Expected to list all routes")
|
||||
@@ -514,7 +514,7 @@ func TestListRoutesWithTwoInterfacesSameSubnet(t *testing.T) {
|
||||
Routes: inputRoutesSimple,
|
||||
}
|
||||
|
||||
- _, err := s.updateRoutes(netHandle, testRoutes)
|
||||
+ _, err := s.updateRoutes(netHandle, testRoutes, false)
|
||||
assert.Nil(err)
|
||||
results, err := s.listRoutes(nil)
|
||||
assert.Nil(err, "Expected to list all routes")
|
||||
diff --git a/protocols/grpc/agent.pb.go b/protocols/grpc/agent.pb.go
|
||||
index 77e6d1b..1b887e5 100644
|
||||
--- a/protocols/grpc/agent.pb.go
|
||||
+++ b/protocols/grpc/agent.pb.go
|
||||
@@ -1303,7 +1303,8 @@ func (m *UpdateInterfaceRequest) GetInterface() *types.Interface {
|
||||
}
|
||||
|
||||
type UpdateRoutesRequest struct {
|
||||
- Routes *Routes `protobuf:"bytes,1,opt,name=routes" json:"routes,omitempty"`
|
||||
+ Routes *Routes `protobuf:"bytes,1,opt,name=routes" json:"routes,omitempty"`
|
||||
+ Increment bool `protobuf:"varint,2,opt,name=increment,proto3" json:"increment,omitempty"`
|
||||
}
|
||||
|
||||
func (m *UpdateRoutesRequest) Reset() { *m = UpdateRoutesRequest{} }
|
||||
@@ -1318,6 +1319,13 @@ func (m *UpdateRoutesRequest) GetRoutes() *Routes {
|
||||
return nil
|
||||
}
|
||||
|
||||
+func (m *UpdateRoutesRequest) GetIncrement() bool {
|
||||
+ if m != nil {
|
||||
+ return m.Increment
|
||||
+ }
|
||||
+ return false
|
||||
+}
|
||||
+
|
||||
type ListInterfacesRequest struct {
|
||||
}
|
||||
|
||||
@@ -4476,6 +4484,16 @@ func (m *UpdateRoutesRequest) MarshalTo(dAtA []byte) (int, error) {
|
||||
}
|
||||
i += n20
|
||||
}
|
||||
+ if m.Increment {
|
||||
+ dAtA[i] = 0x10
|
||||
+ i++
|
||||
+ if m.Increment {
|
||||
+ dAtA[i] = 1
|
||||
+ } else {
|
||||
+ dAtA[i] = 0
|
||||
+ }
|
||||
+ i++
|
||||
+ }
|
||||
return i, nil
|
||||
}
|
||||
|
||||
@@ -5751,6 +5769,9 @@ func (m *UpdateRoutesRequest) Size() (n int) {
|
||||
l = m.Routes.Size()
|
||||
n += 1 + l + sovAgent(uint64(l))
|
||||
}
|
||||
+ if m.Increment {
|
||||
+ n += 2
|
||||
+ }
|
||||
return n
|
||||
}
|
||||
|
||||
@@ -10964,6 +10985,26 @@ func (m *UpdateRoutesRequest) Unmarshal(dAtA []byte) error {
|
||||
return err
|
||||
}
|
||||
iNdEx = postIndex
|
||||
+ case 2:
|
||||
+ if wireType != 0 {
|
||||
+ return fmt.Errorf("proto: wrong wireType = %d for field Increment", wireType)
|
||||
+ }
|
||||
+ var v int
|
||||
+ for shift := uint(0); ; shift += 7 {
|
||||
+ if shift >= 64 {
|
||||
+ return ErrIntOverflowAgent
|
||||
+ }
|
||||
+ if iNdEx >= l {
|
||||
+ return io.ErrUnexpectedEOF
|
||||
+ }
|
||||
+ b := dAtA[iNdEx]
|
||||
+ iNdEx++
|
||||
+ v |= (int(b) & 0x7F) << shift
|
||||
+ if b < 0x80 {
|
||||
+ break
|
||||
+ }
|
||||
+ }
|
||||
+ m.Increment = bool(v != 0)
|
||||
default:
|
||||
iNdEx = preIndex
|
||||
skippy, err := skipAgent(dAtA[iNdEx:])
|
||||
@@ -12852,184 +12893,185 @@ var (
|
||||
func init() { proto.RegisterFile("agent.proto", fileDescriptorAgent) }
|
||||
|
||||
var fileDescriptorAgent = []byte{
|
||||
- // 2862 bytes of a gzipped FileDescriptorProto
|
||||
+ // 2876 bytes of a gzipped FileDescriptorProto
|
||||
0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xc4, 0x39, 0x4b, 0x6f, 0x1c, 0xc7,
|
||||
- 0xd1, 0xd8, 0x07, 0x97, 0xbb, 0xb5, 0x2f, 0x6e, 0x93, 0xa2, 0x56, 0x2b, 0x5b, 0x9f, 0x3c, 0xb6,
|
||||
- 0x65, 0xfa, 0x73, 0xbc, 0xb4, 0x65, 0x23, 0x7e, 0xc1, 0x11, 0xc4, 0x47, 0x44, 0xc6, 0x56, 0xc4,
|
||||
- 0x0c, 0x45, 0x38, 0x40, 0x10, 0x0c, 0x86, 0x33, 0xcd, 0x65, 0x9b, 0x3b, 0xd3, 0xe3, 0x9e, 0x1e,
|
||||
- 0x8a, 0xeb, 0x00, 0x39, 0x26, 0xb7, 0x5c, 0x02, 0xe4, 0x96, 0x3f, 0x10, 0xe4, 0x96, 0x63, 0xae,
|
||||
- 0x39, 0x18, 0x39, 0xe5, 0x17, 0x04, 0x81, 0x7f, 0x42, 0x7e, 0x41, 0xd0, 0xaf, 0x79, 0xec, 0x0e,
|
||||
- 0x29, 0x84, 0x20, 0x90, 0xcb, 0xa2, 0xab, 0xba, 0xba, 0x5e, 0xdd, 0x55, 0x53, 0x55, 0x0b, 0x6d,
|
||||
- 0x77, 0x82, 0x43, 0x3e, 0x8e, 0x18, 0xe5, 0x14, 0xd5, 0x27, 0x2c, 0xf2, 0x46, 0x2d, 0xea, 0x11,
|
||||
- 0x85, 0x18, 0xfd, 0x70, 0x42, 0xf8, 0x69, 0x72, 0x3c, 0xf6, 0x68, 0xb0, 0x79, 0xe6, 0x72, 0xf7,
|
||||
- 0x5d, 0x8f, 0x86, 0xdc, 0x25, 0x21, 0x66, 0xf1, 0xa6, 0x3c, 0xb8, 0x19, 0x9d, 0x4d, 0x36, 0xf9,
|
||||
- 0x2c, 0xc2, 0xb1, 0xfa, 0xd5, 0xe7, 0xee, 0x4e, 0x28, 0x9d, 0x4c, 0xf1, 0xa6, 0x84, 0x8e, 0x93,
|
||||
- 0x93, 0x4d, 0x1c, 0x44, 0x7c, 0xa6, 0x36, 0xad, 0x3f, 0x56, 0x61, 0x7d, 0x9b, 0x61, 0x97, 0xe3,
|
||||
- 0x6d, 0xc3, 0xcd, 0xc6, 0xdf, 0x24, 0x38, 0xe6, 0xe8, 0x35, 0xe8, 0xa4, 0x12, 0x1c, 0xe2, 0x0f,
|
||||
- 0x2b, 0xf7, 0x2b, 0x1b, 0x2d, 0xbb, 0x9d, 0xe2, 0xf6, 0x7d, 0x74, 0x1b, 0x96, 0xf1, 0x05, 0xf6,
|
||||
- 0xc4, 0x6e, 0x55, 0xee, 0x36, 0x04, 0xb8, 0xef, 0xa3, 0xf7, 0xa1, 0x1d, 0x73, 0x46, 0xc2, 0x89,
|
||||
- 0x93, 0xc4, 0x98, 0x0d, 0x6b, 0xf7, 0x2b, 0x1b, 0xed, 0x87, 0x2b, 0x63, 0x61, 0xd2, 0xf8, 0x50,
|
||||
- 0x6e, 0x1c, 0xc5, 0x98, 0xd9, 0x10, 0xa7, 0x6b, 0xf4, 0x00, 0x96, 0x7d, 0x7c, 0x4e, 0x3c, 0x1c,
|
||||
- 0x0f, 0xeb, 0xf7, 0x6b, 0x1b, 0xed, 0x87, 0x1d, 0x45, 0xbe, 0x23, 0x91, 0xb6, 0xd9, 0x44, 0x6f,
|
||||
- 0x43, 0x33, 0xe6, 0x94, 0xb9, 0x13, 0x1c, 0x0f, 0x97, 0x24, 0x61, 0xd7, 0xf0, 0x95, 0x58, 0x3b,
|
||||
- 0xdd, 0x46, 0xaf, 0x40, 0xed, 0xd9, 0xf6, 0xfe, 0xb0, 0x21, 0xa5, 0x83, 0xa6, 0x8a, 0xb0, 0x67,
|
||||
- 0x0b, 0x34, 0x7a, 0x1d, 0xba, 0xb1, 0x1b, 0xfa, 0xc7, 0xf4, 0xc2, 0x89, 0x88, 0x1f, 0xc6, 0xc3,
|
||||
- 0xe5, 0xfb, 0x95, 0x8d, 0xa6, 0xdd, 0xd1, 0xc8, 0x03, 0x81, 0xb3, 0x3e, 0x85, 0x5b, 0x87, 0xdc,
|
||||
- 0x65, 0xfc, 0x1a, 0xde, 0xb1, 0x8e, 0x60, 0xdd, 0xc6, 0x01, 0x3d, 0xbf, 0x96, 0x6b, 0x87, 0xb0,
|
||||
- 0xcc, 0x49, 0x80, 0x69, 0xc2, 0xa5, 0x6b, 0xbb, 0xb6, 0x01, 0xad, 0x3f, 0x57, 0x00, 0xed, 0x5e,
|
||||
- 0x60, 0xef, 0x80, 0x51, 0x0f, 0xc7, 0xf1, 0xff, 0xe8, 0xba, 0xde, 0x82, 0xe5, 0x48, 0x29, 0x30,
|
||||
- 0xac, 0x4b, 0x72, 0x7d, 0x0b, 0x46, 0x2b, 0xb3, 0x6b, 0x7d, 0x0d, 0x6b, 0x87, 0x64, 0x12, 0xba,
|
||||
- 0xd3, 0x1b, 0xd4, 0x77, 0x1d, 0x1a, 0xb1, 0xe4, 0x29, 0x55, 0xed, 0xda, 0x1a, 0xb2, 0x0e, 0x00,
|
||||
- 0x7d, 0xe5, 0x12, 0x7e, 0x73, 0x92, 0xac, 0x77, 0x61, 0xb5, 0xc0, 0x31, 0x8e, 0x68, 0x18, 0x63,
|
||||
- 0xa9, 0x00, 0x77, 0x79, 0x12, 0x4b, 0x66, 0x4b, 0xb6, 0x86, 0x2c, 0x0c, 0x6b, 0x5f, 0x92, 0xd8,
|
||||
- 0x90, 0xe3, 0xff, 0x46, 0x85, 0x75, 0x68, 0x9c, 0x50, 0x16, 0xb8, 0xdc, 0x68, 0xa0, 0x20, 0x84,
|
||||
- 0xa0, 0xee, 0xb2, 0x49, 0x3c, 0xac, 0xdd, 0xaf, 0x6d, 0xb4, 0x6c, 0xb9, 0x16, 0xaf, 0x72, 0x4e,
|
||||
- 0x8c, 0xd6, 0xeb, 0x35, 0xe8, 0x68, 0xbf, 0x3b, 0x53, 0x12, 0x73, 0x29, 0xa7, 0x63, 0xb7, 0x35,
|
||||
- 0x4e, 0x9c, 0xb1, 0x28, 0xac, 0x1f, 0x45, 0xfe, 0x35, 0x03, 0xfe, 0x21, 0xb4, 0x18, 0x8e, 0x69,
|
||||
- 0xc2, 0x44, 0x98, 0x56, 0xe5, 0xbd, 0xaf, 0xa9, 0x7b, 0xff, 0x92, 0x84, 0xc9, 0x85, 0x6d, 0xf6,
|
||||
- 0xec, 0x8c, 0x4c, 0x87, 0x10, 0x8f, 0xaf, 0x13, 0x42, 0x9f, 0xc2, 0xad, 0x03, 0x37, 0x89, 0xaf,
|
||||
- 0xa3, 0xab, 0xf5, 0x99, 0x08, 0xbf, 0x38, 0x09, 0xae, 0x75, 0xf8, 0x4f, 0x15, 0x68, 0x6e, 0x47,
|
||||
- 0xc9, 0x51, 0xec, 0x4e, 0x30, 0xfa, 0x3f, 0x68, 0x73, 0xca, 0xdd, 0xa9, 0x93, 0x08, 0x50, 0x92,
|
||||
- 0xd7, 0x6d, 0x90, 0x28, 0x45, 0x20, 0xdc, 0x8e, 0x99, 0x17, 0x25, 0x9a, 0xa2, 0x7a, 0xbf, 0xb6,
|
||||
- 0x51, 0xb7, 0xdb, 0x0a, 0xa7, 0x48, 0xc6, 0xb0, 0x2a, 0xf7, 0x1c, 0x12, 0x3a, 0x67, 0x98, 0x85,
|
||||
- 0x78, 0x1a, 0x50, 0x1f, 0xcb, 0xf7, 0x5b, 0xb7, 0x07, 0x72, 0x6b, 0x3f, 0xfc, 0x22, 0xdd, 0x40,
|
||||
- 0xff, 0x0f, 0x83, 0x94, 0x5e, 0x04, 0xa5, 0xa4, 0xae, 0x4b, 0xea, 0xbe, 0xa6, 0x3e, 0xd2, 0x68,
|
||||
- 0xeb, 0xd7, 0xd0, 0x7b, 0x7e, 0xca, 0x28, 0xe7, 0x53, 0x12, 0x4e, 0x76, 0x5c, 0xee, 0x8a, 0xec,
|
||||
- 0x11, 0x61, 0x46, 0xa8, 0x1f, 0x6b, 0x6d, 0x0d, 0x88, 0xde, 0x81, 0x01, 0x57, 0xb4, 0xd8, 0x77,
|
||||
- 0x0c, 0x4d, 0x55, 0xd2, 0xac, 0xa4, 0x1b, 0x07, 0x9a, 0xf8, 0x4d, 0xe8, 0x65, 0xc4, 0x22, 0xff,
|
||||
- 0x68, 0x7d, 0xbb, 0x29, 0xf6, 0x39, 0x09, 0xb0, 0x75, 0x2e, 0x7d, 0x25, 0x2f, 0x19, 0xbd, 0x03,
|
||||
- 0xad, 0xcc, 0x0f, 0x15, 0xf9, 0x42, 0x7a, 0xea, 0x85, 0x18, 0x77, 0xda, 0xcd, 0xd4, 0x29, 0x9f,
|
||||
- 0x43, 0x9f, 0xa7, 0x8a, 0x3b, 0xbe, 0xcb, 0xdd, 0xe2, 0xa3, 0x2a, 0x5a, 0x65, 0xf7, 0x78, 0x01,
|
||||
- 0xb6, 0x3e, 0x83, 0xd6, 0x01, 0xf1, 0x63, 0x25, 0x78, 0x08, 0xcb, 0x5e, 0xc2, 0x18, 0x0e, 0xb9,
|
||||
- 0x31, 0x59, 0x83, 0x68, 0x0d, 0x96, 0xa6, 0x24, 0x20, 0x5c, 0x9b, 0xa9, 0x00, 0x8b, 0x02, 0x3c,
|
||||
- 0xc5, 0x01, 0x65, 0x33, 0xe9, 0xb0, 0x35, 0x58, 0xca, 0x5f, 0xae, 0x02, 0xd0, 0x5d, 0x68, 0x05,
|
||||
- 0xee, 0x45, 0x7a, 0xa9, 0x62, 0xa7, 0x19, 0xb8, 0x17, 0x4a, 0xf9, 0x21, 0x2c, 0x9f, 0xb8, 0x64,
|
||||
- 0xea, 0x85, 0x5c, 0x7b, 0xc5, 0x80, 0x99, 0xc0, 0x7a, 0x5e, 0xe0, 0xdf, 0xaa, 0xd0, 0x56, 0x12,
|
||||
- 0x95, 0xc2, 0x6b, 0xb0, 0xe4, 0xb9, 0xde, 0x69, 0x2a, 0x52, 0x02, 0xe8, 0x81, 0x51, 0xa4, 0x9a,
|
||||
- 0x4f, 0xc2, 0x99, 0xa6, 0x46, 0xb5, 0x4d, 0x80, 0xf8, 0x85, 0x1b, 0x69, 0xdd, 0x6a, 0x97, 0x10,
|
||||
- 0xb7, 0x04, 0x8d, 0x52, 0xf7, 0x03, 0xe8, 0xa8, 0x77, 0xa7, 0x8f, 0xd4, 0x2f, 0x39, 0xd2, 0x56,
|
||||
- 0x54, 0xea, 0xd0, 0xeb, 0xd0, 0x4d, 0x62, 0xec, 0x9c, 0x12, 0xcc, 0x5c, 0xe6, 0x9d, 0xce, 0x86,
|
||||
- 0x4b, 0xea, 0x1b, 0x99, 0xc4, 0x78, 0xcf, 0xe0, 0xd0, 0x43, 0x58, 0x12, 0xe9, 0x2f, 0x1e, 0x36,
|
||||
- 0xe4, 0xe7, 0xf8, 0x95, 0x3c, 0x4b, 0x69, 0xea, 0x58, 0xfe, 0xee, 0x86, 0x9c, 0xcd, 0x6c, 0x45,
|
||||
- 0x3a, 0xfa, 0x18, 0x20, 0x43, 0xa2, 0x15, 0xa8, 0x9d, 0xe1, 0x99, 0x8e, 0x43, 0xb1, 0x14, 0xce,
|
||||
- 0x39, 0x77, 0xa7, 0x89, 0xf1, 0xba, 0x02, 0x3e, 0xad, 0x7e, 0x5c, 0xb1, 0x3c, 0xe8, 0x6f, 0x4d,
|
||||
- 0xcf, 0x08, 0xcd, 0x1d, 0x5f, 0x83, 0xa5, 0xc0, 0xfd, 0x9a, 0x32, 0xe3, 0x49, 0x09, 0x48, 0x2c,
|
||||
- 0x09, 0x29, 0x33, 0x2c, 0x24, 0x80, 0x7a, 0x50, 0xa5, 0x91, 0xf4, 0x57, 0xcb, 0xae, 0xd2, 0x28,
|
||||
- 0x13, 0x54, 0xcf, 0x09, 0xb2, 0xfe, 0x59, 0x07, 0xc8, 0xa4, 0x20, 0x1b, 0x46, 0x84, 0x3a, 0x31,
|
||||
- 0x66, 0xa2, 0x04, 0x71, 0x8e, 0x67, 0x1c, 0xc7, 0x0e, 0xc3, 0x5e, 0xc2, 0x62, 0x72, 0x2e, 0xee,
|
||||
- 0x4f, 0x98, 0x7d, 0x4b, 0x99, 0x3d, 0xa7, 0x9b, 0x7d, 0x9b, 0xd0, 0x43, 0x75, 0x6e, 0x4b, 0x1c,
|
||||
- 0xb3, 0xcd, 0x29, 0xb4, 0x0f, 0xb7, 0x32, 0x9e, 0x7e, 0x8e, 0x5d, 0xf5, 0x2a, 0x76, 0xab, 0x29,
|
||||
- 0x3b, 0x3f, 0x63, 0xb5, 0x0b, 0xab, 0x84, 0x3a, 0xdf, 0x24, 0x38, 0x29, 0x30, 0xaa, 0x5d, 0xc5,
|
||||
- 0x68, 0x40, 0xe8, 0xcf, 0xe4, 0x81, 0x8c, 0xcd, 0x01, 0xdc, 0xc9, 0x59, 0x29, 0xc2, 0x3d, 0xc7,
|
||||
- 0xac, 0x7e, 0x15, 0xb3, 0xf5, 0x54, 0x2b, 0x91, 0x0f, 0x32, 0x8e, 0x3f, 0x81, 0x75, 0x42, 0x9d,
|
||||
- 0x17, 0x2e, 0xe1, 0xf3, 0xec, 0x96, 0x5e, 0x62, 0xa4, 0xf8, 0xe8, 0x16, 0x79, 0x29, 0x23, 0x03,
|
||||
- 0xcc, 0x26, 0x05, 0x23, 0x1b, 0x2f, 0x31, 0xf2, 0xa9, 0x3c, 0x90, 0xb1, 0x79, 0x0c, 0x03, 0x42,
|
||||
- 0xe7, 0xb5, 0x59, 0xbe, 0x8a, 0x49, 0x9f, 0xd0, 0xa2, 0x26, 0x5b, 0x30, 0x88, 0xb1, 0xc7, 0x29,
|
||||
- 0xcb, 0x3f, 0x82, 0xe6, 0x55, 0x2c, 0x56, 0x34, 0x7d, 0xca, 0xc3, 0xfa, 0x05, 0x74, 0xf6, 0x92,
|
||||
- 0x09, 0xe6, 0xd3, 0xe3, 0x34, 0x19, 0xdc, 0x58, 0xfe, 0xb1, 0xfe, 0x5d, 0x85, 0xf6, 0xf6, 0x84,
|
||||
- 0xd1, 0x24, 0x2a, 0xe4, 0x64, 0x15, 0xa4, 0xf3, 0x39, 0x59, 0x92, 0xc8, 0x9c, 0xac, 0x88, 0x3f,
|
||||
- 0x84, 0x4e, 0x20, 0x43, 0x57, 0xd3, 0xab, 0x3c, 0x34, 0x58, 0x08, 0x6a, 0xbb, 0x1d, 0xe4, 0x92,
|
||||
- 0xd9, 0x18, 0x20, 0x22, 0x7e, 0xac, 0xcf, 0xa8, 0x74, 0xd4, 0xd7, 0x15, 0xa1, 0x49, 0xd1, 0x76,
|
||||
- 0x2b, 0x4a, 0xb3, 0xf5, 0xfb, 0xd0, 0x3e, 0x16, 0x4e, 0xd2, 0x07, 0x0a, 0xc9, 0x28, 0xf3, 0x9e,
|
||||
- 0x0d, 0xc7, 0x59, 0x10, 0xee, 0x41, 0xf7, 0x54, 0xb9, 0x4c, 0x1f, 0x52, 0x6f, 0xe8, 0x75, 0x6d,
|
||||
- 0x49, 0x66, 0xef, 0x38, 0xef, 0x59, 0x75, 0x01, 0x9d, 0xd3, 0x1c, 0x6a, 0x74, 0x08, 0x83, 0x05,
|
||||
- 0x92, 0x92, 0x1c, 0xb4, 0x91, 0xcf, 0x41, 0xed, 0x87, 0x48, 0x09, 0xca, 0x9f, 0xcc, 0xe7, 0xa5,
|
||||
- 0xdf, 0x55, 0xa1, 0xf3, 0x53, 0xcc, 0x5f, 0x50, 0x76, 0xa6, 0xf4, 0x45, 0x50, 0x0f, 0xdd, 0x00,
|
||||
- 0x6b, 0x8e, 0x72, 0x8d, 0xee, 0x40, 0x93, 0x5d, 0xa8, 0x04, 0xa2, 0xef, 0x73, 0x99, 0x5d, 0xc8,
|
||||
- 0xc4, 0x80, 0x5e, 0x05, 0x60, 0x17, 0x4e, 0xe4, 0x7a, 0x67, 0x58, 0x7b, 0xb0, 0x6e, 0xb7, 0xd8,
|
||||
- 0xc5, 0x81, 0x42, 0x88, 0xa7, 0xc0, 0x2e, 0x1c, 0xcc, 0x18, 0x65, 0xb1, 0xce, 0x55, 0x4d, 0x76,
|
||||
- 0xb1, 0x2b, 0x61, 0x7d, 0xd6, 0x67, 0x34, 0x8a, 0xb0, 0x2f, 0x73, 0xb4, 0x3c, 0xbb, 0xa3, 0x10,
|
||||
- 0x42, 0x2a, 0x37, 0x52, 0x1b, 0x4a, 0x2a, 0xcf, 0xa4, 0xf2, 0x4c, 0xea, 0xb2, 0x3a, 0xc9, 0xf3,
|
||||
- 0x52, 0x79, 0x2a, 0xb5, 0xa9, 0xa4, 0xf2, 0x9c, 0x54, 0x9e, 0x49, 0x6d, 0x99, 0xb3, 0x5a, 0xaa,
|
||||
- 0xf5, 0xdb, 0x0a, 0xac, 0xcf, 0x17, 0x7e, 0xba, 0x4c, 0xfd, 0x10, 0x3a, 0x9e, 0xbc, 0xaf, 0xc2,
|
||||
- 0x9b, 0x1c, 0x2c, 0xdc, 0xa4, 0xdd, 0xf6, 0x72, 0xcf, 0xf8, 0x23, 0xe8, 0x86, 0xca, 0xc1, 0xe9,
|
||||
- 0xd3, 0xac, 0x65, 0xf7, 0x92, 0xf7, 0xbd, 0xdd, 0x09, 0x73, 0x90, 0xe5, 0x03, 0xfa, 0x8a, 0x11,
|
||||
- 0x8e, 0x0f, 0x39, 0xc3, 0x6e, 0x70, 0x13, 0x0d, 0x08, 0x82, 0xba, 0xac, 0x56, 0x6a, 0xb2, 0xbe,
|
||||
- 0x96, 0x6b, 0xeb, 0x2d, 0x58, 0x2d, 0x48, 0xd1, 0xb6, 0xae, 0x40, 0x6d, 0x8a, 0x43, 0xc9, 0xbd,
|
||||
- 0x6b, 0x8b, 0xa5, 0xe5, 0xc2, 0xc0, 0xc6, 0xae, 0x7f, 0x73, 0xda, 0x68, 0x11, 0xb5, 0x4c, 0xc4,
|
||||
- 0x06, 0xa0, 0xbc, 0x08, 0xad, 0x8a, 0xd1, 0xba, 0x92, 0xd3, 0xfa, 0x19, 0x0c, 0xb6, 0xa7, 0x34,
|
||||
- 0xc6, 0x87, 0xdc, 0x27, 0xe1, 0x4d, 0x74, 0x4c, 0xbf, 0x82, 0xd5, 0xe7, 0x7c, 0xf6, 0x95, 0x60,
|
||||
- 0x16, 0x93, 0x6f, 0xf1, 0x0d, 0xd9, 0xc7, 0xe8, 0x0b, 0x63, 0x1f, 0xa3, 0x2f, 0x44, 0xb3, 0xe4,
|
||||
- 0xd1, 0x69, 0x12, 0x84, 0x32, 0x14, 0xba, 0xb6, 0x86, 0xac, 0x2d, 0xe8, 0xa8, 0x1a, 0xfa, 0x29,
|
||||
- 0xf5, 0x93, 0x29, 0x2e, 0x8d, 0xc1, 0x7b, 0x00, 0x91, 0xcb, 0xdc, 0x00, 0x73, 0xcc, 0xd4, 0x1b,
|
||||
- 0x6a, 0xd9, 0x39, 0x8c, 0xf5, 0x87, 0x2a, 0xac, 0xa9, 0x91, 0xc8, 0xa1, 0x9a, 0x04, 0x18, 0x13,
|
||||
- 0x46, 0xd0, 0x3c, 0xa5, 0x31, 0xcf, 0x31, 0x4c, 0x61, 0xa1, 0xa2, 0x1f, 0x1a, 0x6e, 0x62, 0x59,
|
||||
- 0x98, 0x53, 0xd4, 0xae, 0x9e, 0x53, 0x2c, 0x4c, 0x22, 0xea, 0x8b, 0x93, 0x08, 0x11, 0x6d, 0x86,
|
||||
- 0x88, 0xa8, 0x18, 0x6f, 0xd9, 0x2d, 0x8d, 0xd9, 0xf7, 0xd1, 0x03, 0xe8, 0x4f, 0x84, 0x96, 0xce,
|
||||
- 0x29, 0xa5, 0x67, 0x4e, 0xe4, 0xf2, 0x53, 0x19, 0xea, 0x2d, 0xbb, 0x2b, 0xd1, 0x7b, 0x94, 0x9e,
|
||||
- 0x1d, 0xb8, 0xfc, 0x14, 0x7d, 0x02, 0x3d, 0x5d, 0x06, 0x06, 0xd2, 0x45, 0xb1, 0xfe, 0xf8, 0xe9,
|
||||
- 0x28, 0xca, 0x7b, 0xcf, 0xee, 0x9e, 0xe5, 0xa0, 0xd8, 0xba, 0x0d, 0xb7, 0x76, 0x70, 0xcc, 0x19,
|
||||
- 0x9d, 0x15, 0x1d, 0x63, 0xfd, 0x08, 0x60, 0x3f, 0xe4, 0x98, 0x9d, 0xb8, 0x1e, 0x8e, 0xd1, 0x7b,
|
||||
- 0x79, 0x48, 0x17, 0x47, 0x2b, 0x63, 0x35, 0x91, 0x4a, 0x37, 0xec, 0x1c, 0x8d, 0x35, 0x86, 0x86,
|
||||
- 0x4d, 0x13, 0x91, 0x8e, 0xde, 0x30, 0x2b, 0x7d, 0xae, 0xa3, 0xcf, 0x49, 0xa4, 0xad, 0xf7, 0xac,
|
||||
- 0x3d, 0xd3, 0xc2, 0x66, 0xec, 0xf4, 0x15, 0x8d, 0xa1, 0x45, 0x0c, 0x4e, 0x67, 0x95, 0x45, 0xd1,
|
||||
- 0x19, 0x89, 0xf5, 0x19, 0xac, 0x2a, 0x4e, 0x8a, 0xb3, 0x61, 0xf3, 0x06, 0x34, 0x98, 0x51, 0xa3,
|
||||
- 0x92, 0x8d, 0xa2, 0x34, 0x91, 0xde, 0x13, 0xfe, 0x10, 0x1d, 0x75, 0x66, 0x88, 0xf1, 0xc7, 0x2a,
|
||||
- 0x0c, 0xc4, 0x46, 0x81, 0xa7, 0xf5, 0x4b, 0x58, 0x7d, 0x16, 0x4e, 0x49, 0x88, 0xb7, 0x0f, 0x8e,
|
||||
- 0x9e, 0xe2, 0x34, 0xee, 0x11, 0xd4, 0x45, 0x7d, 0x24, 0x05, 0x35, 0x6d, 0xb9, 0x16, 0x81, 0x10,
|
||||
- 0x1e, 0x3b, 0x5e, 0x94, 0xc4, 0x7a, 0xf6, 0xd3, 0x08, 0x8f, 0xb7, 0xa3, 0x24, 0x16, 0x89, 0x5c,
|
||||
- 0x7c, 0xc8, 0x69, 0x38, 0x9d, 0xc9, 0x68, 0x68, 0xda, 0xcb, 0x5e, 0x94, 0x3c, 0x0b, 0xa7, 0x33,
|
||||
- 0xeb, 0x07, 0xb2, 0xdb, 0xc5, 0xd8, 0xb7, 0xdd, 0xd0, 0xa7, 0xc1, 0x0e, 0x3e, 0xcf, 0x49, 0x48,
|
||||
- 0x3b, 0x2b, 0x13, 0xf5, 0xdf, 0x55, 0xa0, 0xf3, 0x78, 0x82, 0x43, 0xbe, 0x83, 0xb9, 0x4b, 0xa6,
|
||||
- 0xb2, 0x7b, 0x3a, 0xc7, 0x2c, 0x26, 0x34, 0xd4, 0x4f, 0xdb, 0x80, 0xa2, 0xf9, 0x25, 0x21, 0xe1,
|
||||
- 0x8e, 0xef, 0xe2, 0x80, 0x86, 0x92, 0x4b, 0xd3, 0x06, 0x81, 0xda, 0x91, 0x18, 0xf4, 0x16, 0xf4,
|
||||
- 0xd5, 0x6c, 0xce, 0x39, 0x75, 0x43, 0x7f, 0x2a, 0x82, 0x4a, 0xcd, 0x2a, 0x7a, 0x0a, 0xbd, 0xa7,
|
||||
- 0xb1, 0xe8, 0x6d, 0x58, 0xd1, 0x4f, 0x3e, 0xa3, 0xac, 0x4b, 0xca, 0xbe, 0xc6, 0x17, 0x48, 0x93,
|
||||
- 0x28, 0xa2, 0x8c, 0xc7, 0x4e, 0x8c, 0x3d, 0x8f, 0x06, 0x91, 0x6e, 0x3d, 0xfa, 0x06, 0x7f, 0xa8,
|
||||
- 0xd0, 0xd6, 0x04, 0x56, 0x9f, 0x08, 0x3b, 0xb5, 0x25, 0xd9, 0x15, 0xf6, 0x02, 0x1c, 0x38, 0xc7,
|
||||
- 0x53, 0xea, 0x9d, 0x39, 0x22, 0x11, 0x69, 0x0f, 0x8b, 0xe2, 0x66, 0x4b, 0x20, 0x0f, 0xc9, 0xb7,
|
||||
- 0xb2, 0xcb, 0x16, 0x54, 0xa7, 0x94, 0x47, 0xd3, 0x64, 0xe2, 0x44, 0x8c, 0x1e, 0x63, 0x6d, 0x62,
|
||||
- 0x3f, 0xc0, 0xc1, 0x9e, 0xc2, 0x1f, 0x08, 0xb4, 0xf5, 0xd7, 0x0a, 0xac, 0x15, 0x25, 0xe9, 0xb4,
|
||||
- 0xba, 0x09, 0x6b, 0x45, 0x51, 0xfa, 0x53, 0xab, 0x4a, 0xb9, 0x41, 0x5e, 0xa0, 0xfa, 0xe8, 0x7e,
|
||||
- 0x04, 0x5d, 0x39, 0xb0, 0x75, 0x7c, 0xc5, 0xa9, 0x58, 0x60, 0xe4, 0xef, 0xc5, 0xee, 0xb8, 0xf9,
|
||||
- 0x5b, 0xfa, 0x04, 0xee, 0x68, 0xf3, 0x9d, 0x45, 0xb5, 0xd5, 0x83, 0x58, 0xd7, 0x04, 0x4f, 0xe7,
|
||||
- 0xb4, 0xff, 0x12, 0x86, 0x19, 0x6a, 0x6b, 0x26, 0x91, 0xc6, 0x57, 0xef, 0xc1, 0xea, 0x9c, 0xb1,
|
||||
- 0x8f, 0x7d, 0x9f, 0xc9, 0x10, 0xac, 0xdb, 0x65, 0x5b, 0xd6, 0x23, 0xb8, 0x7d, 0x88, 0xb9, 0xf2,
|
||||
- 0x86, 0xcb, 0x75, 0xd5, 0xaf, 0x98, 0xad, 0x40, 0xed, 0x10, 0x7b, 0xd2, 0xf8, 0x9a, 0x2d, 0x96,
|
||||
- 0xe2, 0x01, 0x1e, 0xc5, 0xd8, 0x93, 0x56, 0xd6, 0x6c, 0xb9, 0xb6, 0xfe, 0x52, 0x81, 0x65, 0x9d,
|
||||
- 0x08, 0x45, 0x32, 0xf7, 0x19, 0x39, 0xc7, 0x4c, 0x3f, 0x3d, 0x0d, 0xa1, 0x37, 0xa1, 0xa7, 0x56,
|
||||
- 0x0e, 0x8d, 0x38, 0xa1, 0x69, 0x7a, 0xed, 0x2a, 0xec, 0x33, 0x85, 0x94, 0xb3, 0x38, 0x39, 0x6a,
|
||||
- 0xd2, 0x5d, 0x9d, 0x86, 0xe4, 0x40, 0x2d, 0x16, 0xb1, 0x2f, 0xd3, 0x69, 0xcb, 0xd6, 0x90, 0x78,
|
||||
- 0xea, 0x86, 0xdf, 0x92, 0xe4, 0x67, 0x40, 0xf1, 0xd4, 0x03, 0x9a, 0x84, 0xdc, 0x89, 0x28, 0x09,
|
||||
- 0xb9, 0xce, 0x9f, 0x20, 0x51, 0x07, 0x02, 0x63, 0xfd, 0xa6, 0x02, 0x0d, 0x35, 0x8f, 0x16, 0x7d,
|
||||
- 0x64, 0xfa, 0x15, 0xab, 0x12, 0x59, 0x11, 0x48, 0x59, 0xea, 0xcb, 0x25, 0xd7, 0x22, 0x8e, 0xcf,
|
||||
- 0x03, 0x95, 0x8b, 0xb5, 0x6a, 0xe7, 0x81, 0x4c, 0xc2, 0x6f, 0x42, 0x2f, 0xfb, 0x18, 0xca, 0x7d,
|
||||
- 0xa5, 0x62, 0x37, 0xc5, 0x4a, 0xb2, 0x4b, 0x35, 0xb5, 0x7e, 0x2e, 0xda, 0xe7, 0x74, 0x16, 0xbb,
|
||||
- 0x02, 0xb5, 0x24, 0x55, 0x46, 0x2c, 0x05, 0x66, 0x92, 0x7e, 0x46, 0xc5, 0x12, 0x3d, 0x80, 0x9e,
|
||||
- 0xeb, 0xfb, 0x44, 0x1c, 0x77, 0xa7, 0x4f, 0x88, 0x9f, 0x06, 0x69, 0x11, 0x6b, 0xfd, 0xbd, 0x02,
|
||||
- 0xfd, 0x6d, 0x1a, 0xcd, 0x7e, 0x4c, 0xa6, 0x38, 0x97, 0x41, 0xa4, 0x92, 0xfa, 0x2b, 0x2a, 0xd6,
|
||||
- 0xa2, 0x32, 0x3c, 0x21, 0x53, 0xac, 0x42, 0x4b, 0xdd, 0x6c, 0x53, 0x20, 0x64, 0x58, 0x99, 0xcd,
|
||||
- 0x74, 0xc4, 0xd5, 0x55, 0x9b, 0x4f, 0xa9, 0x2f, 0x6b, 0x60, 0x9f, 0x30, 0x27, 0x1d, 0x68, 0x75,
|
||||
- 0xed, 0x65, 0x9f, 0x30, 0xb9, 0xa5, 0x0d, 0x59, 0x92, 0x33, 0xd5, 0xbc, 0x21, 0x0d, 0x85, 0x11,
|
||||
- 0x86, 0xac, 0x43, 0x83, 0x9e, 0x9c, 0xc4, 0x98, 0xcb, 0x6a, 0xb5, 0x66, 0x6b, 0x28, 0x4d, 0x73,
|
||||
- 0xcd, 0x5c, 0x9a, 0xbb, 0x05, 0xab, 0x72, 0x7a, 0xff, 0x9c, 0xb9, 0x1e, 0x09, 0x27, 0x26, 0x15,
|
||||
- 0xaf, 0x01, 0x3a, 0xe4, 0x34, 0x2a, 0x62, 0x1f, 0xfe, 0x7e, 0x45, 0xe7, 0x44, 0xdd, 0xca, 0xa2,
|
||||
- 0x27, 0xd0, 0x9f, 0xfb, 0x6b, 0x04, 0xe9, 0xd9, 0x46, 0xf9, 0x3f, 0x26, 0xa3, 0xf5, 0xb1, 0xfa,
|
||||
- 0xab, 0x65, 0x6c, 0xfe, 0x6a, 0x19, 0xef, 0x06, 0x11, 0x9f, 0xa1, 0x5d, 0xe8, 0x15, 0xff, 0x44,
|
||||
- 0x40, 0x77, 0x4d, 0x29, 0x50, 0xf2, 0xd7, 0xc2, 0xa5, 0x6c, 0x9e, 0x40, 0x7f, 0xee, 0xff, 0x04,
|
||||
- 0xa3, 0x4f, 0xf9, 0xdf, 0x0c, 0x97, 0x32, 0x7a, 0x04, 0xed, 0xdc, 0x1f, 0x08, 0x68, 0xa8, 0x98,
|
||||
- 0x2c, 0xfe, 0xa7, 0x70, 0x29, 0x83, 0x6d, 0xe8, 0x16, 0x66, 0xfa, 0x68, 0xa4, 0xed, 0x29, 0x19,
|
||||
- 0xf4, 0x5f, 0xca, 0x64, 0x0b, 0xda, 0xb9, 0xd1, 0xba, 0xd1, 0x62, 0x71, 0x7e, 0x3f, 0xba, 0x53,
|
||||
- 0xb2, 0xa3, 0x53, 0xef, 0x1e, 0x74, 0x0b, 0x83, 0x70, 0xa3, 0x48, 0xd9, 0x10, 0x7e, 0x74, 0xb7,
|
||||
- 0x74, 0x4f, 0x73, 0x7a, 0x02, 0xfd, 0xb9, 0xb1, 0xb8, 0x71, 0x6e, 0xf9, 0xb4, 0xfc, 0x52, 0xb3,
|
||||
- 0xbe, 0x90, 0x97, 0x9d, 0xeb, 0x7a, 0x72, 0x97, 0xbd, 0x38, 0x04, 0x1f, 0xbd, 0x52, 0xbe, 0xa9,
|
||||
- 0xb5, 0xda, 0x85, 0x5e, 0x71, 0xfe, 0x6d, 0x98, 0x95, 0x4e, 0xc5, 0xaf, 0x7e, 0x39, 0x85, 0x51,
|
||||
- 0x78, 0xf6, 0x72, 0xca, 0x26, 0xe4, 0x97, 0x32, 0x7a, 0x0c, 0xa0, 0x7b, 0x1c, 0x9f, 0x84, 0xe9,
|
||||
- 0x95, 0x2d, 0xf4, 0x56, 0xe9, 0x95, 0x95, 0xf4, 0x43, 0x8f, 0x00, 0x54, 0x6b, 0xe2, 0xd3, 0x84,
|
||||
- 0xa3, 0xdb, 0x46, 0x8d, 0xb9, 0x7e, 0x68, 0x34, 0x5c, 0xdc, 0x58, 0x60, 0x80, 0x19, 0xbb, 0x0e,
|
||||
- 0x83, 0xcf, 0x01, 0xb2, 0x96, 0xc7, 0x30, 0x58, 0x68, 0x82, 0xae, 0xf0, 0x41, 0x27, 0xdf, 0xe0,
|
||||
- 0x20, 0x6d, 0x6b, 0x49, 0xd3, 0x73, 0x05, 0x8b, 0xfe, 0x5c, 0x01, 0x5b, 0x7c, 0x6c, 0xf3, 0x75,
|
||||
- 0xed, 0x68, 0xa1, 0x88, 0x45, 0x1f, 0x41, 0x27, 0x5f, 0xb9, 0x1a, 0x2d, 0x4a, 0xaa, 0xd9, 0x51,
|
||||
- 0xa1, 0x7a, 0x45, 0x8f, 0xa0, 0x57, 0xac, 0x5a, 0x51, 0x2e, 0x2e, 0x16, 0x6a, 0xd9, 0x91, 0x9e,
|
||||
- 0xc9, 0xe4, 0xc8, 0x3f, 0x00, 0xc8, 0xaa, 0x5b, 0xe3, 0xbe, 0x85, 0x7a, 0x77, 0x4e, 0xea, 0x63,
|
||||
- 0xe8, 0xe4, 0x33, 0xb1, 0x51, 0xb7, 0x24, 0x3b, 0x5f, 0x95, 0xb5, 0x72, 0x59, 0xdb, 0x3c, 0xbe,
|
||||
- 0xc5, 0x44, 0x7e, 0x55, 0xd6, 0x2a, 0xf4, 0x75, 0x26, 0x59, 0x94, 0x35, 0x7b, 0x57, 0xe5, 0xf2,
|
||||
- 0x62, 0x13, 0x64, 0xdc, 0x57, 0xda, 0x1a, 0x5d, 0xf5, 0x88, 0xf2, 0xdd, 0x80, 0xf1, 0x47, 0x49,
|
||||
- 0x87, 0xf0, 0x92, 0xa0, 0xce, 0x57, 0xfc, 0xb9, 0xa0, 0x2e, 0x69, 0x04, 0x2e, 0x65, 0xb4, 0x07,
|
||||
- 0xfd, 0x27, 0xa6, 0x98, 0xd3, 0x85, 0xa6, 0x56, 0xa7, 0xa4, 0xb0, 0x1e, 0x8d, 0xca, 0xb6, 0x74,
|
||||
- 0x64, 0x7d, 0x01, 0x83, 0x85, 0x22, 0x13, 0xdd, 0x4b, 0x47, 0x87, 0xa5, 0xd5, 0xe7, 0xa5, 0x6a,
|
||||
- 0xed, 0xc3, 0xca, 0x7c, 0x8d, 0x89, 0x5e, 0xd5, 0x97, 0x5e, 0x5e, 0x7b, 0x5e, 0xca, 0xea, 0x13,
|
||||
- 0x68, 0x9a, 0x9a, 0x06, 0xe9, 0x11, 0xed, 0x5c, 0x8d, 0x73, 0xd9, 0xd1, 0xad, 0xce, 0x77, 0xdf,
|
||||
- 0xdf, 0xab, 0xfc, 0xe3, 0xfb, 0x7b, 0x95, 0x7f, 0x7d, 0x7f, 0xaf, 0x72, 0xdc, 0x90, 0xbb, 0x1f,
|
||||
- 0xfc, 0x27, 0x00, 0x00, 0xff, 0xff, 0xa5, 0xac, 0x85, 0x1d, 0xaa, 0x21, 0x00, 0x00,
|
||||
+ 0xd1, 0xd8, 0x07, 0x97, 0xbb, 0xb5, 0x2f, 0x6e, 0x93, 0xa2, 0x56, 0x2b, 0x59, 0x9f, 0x3c, 0xb6,
|
||||
+ 0x65, 0xfa, 0xf3, 0xe7, 0xa5, 0x2d, 0x1b, 0x9f, 0x5f, 0x70, 0x04, 0xf1, 0x11, 0x91, 0xb1, 0x15,
|
||||
+ 0x31, 0x43, 0x11, 0x4e, 0x10, 0x04, 0x83, 0xe1, 0x4c, 0x73, 0xd9, 0xe6, 0xce, 0xf4, 0xb8, 0xa7,
|
||||
+ 0x87, 0xe2, 0x3a, 0x40, 0x8e, 0xc9, 0x2d, 0x97, 0x00, 0xb9, 0xe5, 0x0f, 0x04, 0xb9, 0xe5, 0x98,
|
||||
+ 0x6b, 0x0e, 0x46, 0x4e, 0xf9, 0x05, 0x41, 0xe0, 0x9f, 0x90, 0x5f, 0x10, 0xf4, 0x6b, 0x1e, 0xbb,
|
||||
+ 0x43, 0x1a, 0x21, 0x08, 0xe4, 0xb2, 0xe8, 0xaa, 0xae, 0xae, 0x57, 0x77, 0xd5, 0x54, 0xd5, 0x42,
|
||||
+ 0xdb, 0x9d, 0xe0, 0x90, 0x8f, 0x23, 0x46, 0x39, 0x45, 0xf5, 0x09, 0x8b, 0xbc, 0x51, 0x8b, 0x7a,
|
||||
+ 0x44, 0x21, 0x46, 0xff, 0x3f, 0x21, 0xfc, 0x34, 0x39, 0x1e, 0x7b, 0x34, 0xd8, 0x3c, 0x73, 0xb9,
|
||||
+ 0xfb, 0x8e, 0x47, 0x43, 0xee, 0x92, 0x10, 0xb3, 0x78, 0x53, 0x1e, 0xdc, 0x8c, 0xce, 0x26, 0x9b,
|
||||
+ 0x7c, 0x16, 0xe1, 0x58, 0xfd, 0xea, 0x73, 0x77, 0x27, 0x94, 0x4e, 0xa6, 0x78, 0x53, 0x42, 0xc7,
|
||||
+ 0xc9, 0xc9, 0x26, 0x0e, 0x22, 0x3e, 0x53, 0x9b, 0xd6, 0x1f, 0xaa, 0xb0, 0xbe, 0xcd, 0xb0, 0xcb,
|
||||
+ 0xf1, 0xb6, 0xe1, 0x66, 0xe3, 0xaf, 0x13, 0x1c, 0x73, 0xf4, 0x2a, 0x74, 0x52, 0x09, 0x0e, 0xf1,
|
||||
+ 0x87, 0x95, 0x07, 0x95, 0x8d, 0x96, 0xdd, 0x4e, 0x71, 0xfb, 0x3e, 0xba, 0x0d, 0xcb, 0xf8, 0x02,
|
||||
+ 0x7b, 0x62, 0xb7, 0x2a, 0x77, 0x1b, 0x02, 0xdc, 0xf7, 0xd1, 0x7b, 0xd0, 0x8e, 0x39, 0x23, 0xe1,
|
||||
+ 0xc4, 0x49, 0x62, 0xcc, 0x86, 0xb5, 0x07, 0x95, 0x8d, 0xf6, 0xa3, 0x95, 0xb1, 0x30, 0x69, 0x7c,
|
||||
+ 0x28, 0x37, 0x8e, 0x62, 0xcc, 0x6c, 0x88, 0xd3, 0x35, 0x7a, 0x08, 0xcb, 0x3e, 0x3e, 0x27, 0x1e,
|
||||
+ 0x8e, 0x87, 0xf5, 0x07, 0xb5, 0x8d, 0xf6, 0xa3, 0x8e, 0x22, 0xdf, 0x91, 0x48, 0xdb, 0x6c, 0xa2,
|
||||
+ 0xb7, 0xa0, 0x19, 0x73, 0xca, 0xdc, 0x09, 0x8e, 0x87, 0x4b, 0x92, 0xb0, 0x6b, 0xf8, 0x4a, 0xac,
|
||||
+ 0x9d, 0x6e, 0xa3, 0x7b, 0x50, 0x7b, 0xbe, 0xbd, 0x3f, 0x6c, 0x48, 0xe9, 0xa0, 0xa9, 0x22, 0xec,
|
||||
+ 0xd9, 0x02, 0x8d, 0x5e, 0x83, 0x6e, 0xec, 0x86, 0xfe, 0x31, 0xbd, 0x70, 0x22, 0xe2, 0x87, 0xf1,
|
||||
+ 0x70, 0xf9, 0x41, 0x65, 0xa3, 0x69, 0x77, 0x34, 0xf2, 0x40, 0xe0, 0xac, 0x4f, 0xe0, 0xd6, 0x21,
|
||||
+ 0x77, 0x19, 0xbf, 0x86, 0x77, 0xac, 0x23, 0x58, 0xb7, 0x71, 0x40, 0xcf, 0xaf, 0xe5, 0xda, 0x21,
|
||||
+ 0x2c, 0x73, 0x12, 0x60, 0x9a, 0x70, 0xe9, 0xda, 0xae, 0x6d, 0x40, 0xeb, 0x4f, 0x15, 0x40, 0xbb,
|
||||
+ 0x17, 0xd8, 0x3b, 0x60, 0xd4, 0xc3, 0x71, 0xfc, 0x5f, 0xba, 0xae, 0x37, 0x61, 0x39, 0x52, 0x0a,
|
||||
+ 0x0c, 0xeb, 0x92, 0x5c, 0xdf, 0x82, 0xd1, 0xca, 0xec, 0x5a, 0x5f, 0xc1, 0xda, 0x21, 0x99, 0x84,
|
||||
+ 0xee, 0xf4, 0x06, 0xf5, 0x5d, 0x87, 0x46, 0x2c, 0x79, 0x4a, 0x55, 0xbb, 0xb6, 0x86, 0xac, 0x03,
|
||||
+ 0x40, 0x5f, 0xba, 0x84, 0xdf, 0x9c, 0x24, 0xeb, 0x1d, 0x58, 0x2d, 0x70, 0x8c, 0x23, 0x1a, 0xc6,
|
||||
+ 0x58, 0x2a, 0xc0, 0x5d, 0x9e, 0xc4, 0x92, 0xd9, 0x92, 0xad, 0x21, 0x0b, 0xc3, 0xda, 0x17, 0x24,
|
||||
+ 0x36, 0xe4, 0xf8, 0x3f, 0x51, 0x61, 0x1d, 0x1a, 0x27, 0x94, 0x05, 0x2e, 0x37, 0x1a, 0x28, 0x08,
|
||||
+ 0x21, 0xa8, 0xbb, 0x6c, 0x12, 0x0f, 0x6b, 0x0f, 0x6a, 0x1b, 0x2d, 0x5b, 0xae, 0xc5, 0xab, 0x9c,
|
||||
+ 0x13, 0xa3, 0xf5, 0x7a, 0x15, 0x3a, 0xda, 0xef, 0xce, 0x94, 0xc4, 0x5c, 0xca, 0xe9, 0xd8, 0x6d,
|
||||
+ 0x8d, 0x13, 0x67, 0x2c, 0x0a, 0xeb, 0x47, 0x91, 0x7f, 0xcd, 0x80, 0x7f, 0x04, 0x2d, 0x86, 0x63,
|
||||
+ 0x9a, 0x30, 0x11, 0xa6, 0x55, 0x79, 0xef, 0x6b, 0xea, 0xde, 0xbf, 0x20, 0x61, 0x72, 0x61, 0x9b,
|
||||
+ 0x3d, 0x3b, 0x23, 0xd3, 0x21, 0xc4, 0xe3, 0xeb, 0x84, 0xd0, 0x27, 0x70, 0xeb, 0xc0, 0x4d, 0xe2,
|
||||
+ 0xeb, 0xe8, 0x6a, 0x7d, 0x2a, 0xc2, 0x2f, 0x4e, 0x82, 0x6b, 0x1d, 0xfe, 0x63, 0x05, 0x9a, 0xdb,
|
||||
+ 0x51, 0x72, 0x14, 0xbb, 0x13, 0x8c, 0xfe, 0x07, 0xda, 0x9c, 0x72, 0x77, 0xea, 0x24, 0x02, 0x94,
|
||||
+ 0xe4, 0x75, 0x1b, 0x24, 0x4a, 0x11, 0x08, 0xb7, 0x63, 0xe6, 0x45, 0x89, 0xa6, 0xa8, 0x3e, 0xa8,
|
||||
+ 0x6d, 0xd4, 0xed, 0xb6, 0xc2, 0x29, 0x92, 0x31, 0xac, 0xca, 0x3d, 0x87, 0x84, 0xce, 0x19, 0x66,
|
||||
+ 0x21, 0x9e, 0x06, 0xd4, 0xc7, 0xf2, 0xfd, 0xd6, 0xed, 0x81, 0xdc, 0xda, 0x0f, 0x3f, 0x4f, 0x37,
|
||||
+ 0xd0, 0xff, 0xc2, 0x20, 0xa5, 0x17, 0x41, 0x29, 0xa9, 0xeb, 0x92, 0xba, 0xaf, 0xa9, 0x8f, 0x34,
|
||||
+ 0xda, 0xfa, 0x15, 0xf4, 0x5e, 0x9c, 0x32, 0xca, 0xf9, 0x94, 0x84, 0x93, 0x1d, 0x97, 0xbb, 0x22,
|
||||
+ 0x7b, 0x44, 0x98, 0x11, 0xea, 0xc7, 0x5a, 0x5b, 0x03, 0xa2, 0xb7, 0x61, 0xc0, 0x15, 0x2d, 0xf6,
|
||||
+ 0x1d, 0x43, 0x53, 0x95, 0x34, 0x2b, 0xe9, 0xc6, 0x81, 0x26, 0x7e, 0x03, 0x7a, 0x19, 0xb1, 0xc8,
|
||||
+ 0x3f, 0x5a, 0xdf, 0x6e, 0x8a, 0x7d, 0x41, 0x02, 0x6c, 0x9d, 0x4b, 0x5f, 0xc9, 0x4b, 0x46, 0x6f,
|
||||
+ 0x43, 0x2b, 0xf3, 0x43, 0x45, 0xbe, 0x90, 0x9e, 0x7a, 0x21, 0xc6, 0x9d, 0x76, 0x33, 0x75, 0xca,
|
||||
+ 0x67, 0xd0, 0xe7, 0xa9, 0xe2, 0x8e, 0xef, 0x72, 0xb7, 0xf8, 0xa8, 0x8a, 0x56, 0xd9, 0x3d, 0x5e,
|
||||
+ 0x80, 0xad, 0x4f, 0xa1, 0x75, 0x40, 0xfc, 0x58, 0x09, 0x1e, 0xc2, 0xb2, 0x97, 0x30, 0x86, 0x43,
|
||||
+ 0x6e, 0x4c, 0xd6, 0x20, 0x5a, 0x83, 0xa5, 0x29, 0x09, 0x08, 0xd7, 0x66, 0x2a, 0xc0, 0xa2, 0x00,
|
||||
+ 0xcf, 0x70, 0x40, 0xd9, 0x4c, 0x3a, 0x6c, 0x0d, 0x96, 0xf2, 0x97, 0xab, 0x00, 0x74, 0x17, 0x5a,
|
||||
+ 0x81, 0x7b, 0x91, 0x5e, 0xaa, 0xd8, 0x69, 0x06, 0xee, 0x85, 0x52, 0x7e, 0x08, 0xcb, 0x27, 0x2e,
|
||||
+ 0x99, 0x7a, 0x21, 0xd7, 0x5e, 0x31, 0x60, 0x26, 0xb0, 0x9e, 0x17, 0xf8, 0xd7, 0x2a, 0xb4, 0x95,
|
||||
+ 0x44, 0xa5, 0xf0, 0x1a, 0x2c, 0x79, 0xae, 0x77, 0x9a, 0x8a, 0x94, 0x00, 0x7a, 0x68, 0x14, 0xa9,
|
||||
+ 0xe6, 0x93, 0x70, 0xa6, 0xa9, 0x51, 0x6d, 0x13, 0x20, 0x7e, 0xe9, 0x46, 0x5a, 0xb7, 0xda, 0x25,
|
||||
+ 0xc4, 0x2d, 0x41, 0xa3, 0xd4, 0x7d, 0x1f, 0x3a, 0xea, 0xdd, 0xe9, 0x23, 0xf5, 0x4b, 0x8e, 0xb4,
|
||||
+ 0x15, 0x95, 0x3a, 0xf4, 0x1a, 0x74, 0x93, 0x18, 0x3b, 0xa7, 0x04, 0x33, 0x97, 0x79, 0xa7, 0xb3,
|
||||
+ 0xe1, 0x92, 0xfa, 0x46, 0x26, 0x31, 0xde, 0x33, 0x38, 0xf4, 0x08, 0x96, 0x44, 0xfa, 0x8b, 0x87,
|
||||
+ 0x0d, 0xf9, 0x39, 0xbe, 0x97, 0x67, 0x29, 0x4d, 0x1d, 0xcb, 0xdf, 0xdd, 0x90, 0xb3, 0x99, 0xad,
|
||||
+ 0x48, 0x47, 0x1f, 0x01, 0x64, 0x48, 0xb4, 0x02, 0xb5, 0x33, 0x3c, 0xd3, 0x71, 0x28, 0x96, 0xc2,
|
||||
+ 0x39, 0xe7, 0xee, 0x34, 0x31, 0x5e, 0x57, 0xc0, 0x27, 0xd5, 0x8f, 0x2a, 0x96, 0x07, 0xfd, 0xad,
|
||||
+ 0xe9, 0x19, 0xa1, 0xb9, 0xe3, 0x6b, 0xb0, 0x14, 0xb8, 0x5f, 0x51, 0x66, 0x3c, 0x29, 0x01, 0x89,
|
||||
+ 0x25, 0x21, 0x65, 0x86, 0x85, 0x04, 0x50, 0x0f, 0xaa, 0x34, 0x92, 0xfe, 0x6a, 0xd9, 0x55, 0x1a,
|
||||
+ 0x65, 0x82, 0xea, 0x39, 0x41, 0xd6, 0x3f, 0xea, 0x00, 0x99, 0x14, 0x64, 0xc3, 0x88, 0x50, 0x27,
|
||||
+ 0xc6, 0x4c, 0x94, 0x20, 0xce, 0xf1, 0x8c, 0xe3, 0xd8, 0x61, 0xd8, 0x4b, 0x58, 0x4c, 0xce, 0xc5,
|
||||
+ 0xfd, 0x09, 0xb3, 0x6f, 0x29, 0xb3, 0xe7, 0x74, 0xb3, 0x6f, 0x13, 0x7a, 0xa8, 0xce, 0x6d, 0x89,
|
||||
+ 0x63, 0xb6, 0x39, 0x85, 0xf6, 0xe1, 0x56, 0xc6, 0xd3, 0xcf, 0xb1, 0xab, 0x5e, 0xc5, 0x6e, 0x35,
|
||||
+ 0x65, 0xe7, 0x67, 0xac, 0x76, 0x61, 0x95, 0x50, 0xe7, 0xeb, 0x04, 0x27, 0x05, 0x46, 0xb5, 0xab,
|
||||
+ 0x18, 0x0d, 0x08, 0xfd, 0x89, 0x3c, 0x90, 0xb1, 0x39, 0x80, 0x3b, 0x39, 0x2b, 0x45, 0xb8, 0xe7,
|
||||
+ 0x98, 0xd5, 0xaf, 0x62, 0xb6, 0x9e, 0x6a, 0x25, 0xf2, 0x41, 0xc6, 0xf1, 0x47, 0xb0, 0x4e, 0xa8,
|
||||
+ 0xf3, 0xd2, 0x25, 0x7c, 0x9e, 0xdd, 0xd2, 0xf7, 0x18, 0x29, 0x3e, 0xba, 0x45, 0x5e, 0xca, 0xc8,
|
||||
+ 0x00, 0xb3, 0x49, 0xc1, 0xc8, 0xc6, 0xf7, 0x18, 0xf9, 0x4c, 0x1e, 0xc8, 0xd8, 0x3c, 0x81, 0x01,
|
||||
+ 0xa1, 0xf3, 0xda, 0x2c, 0x5f, 0xc5, 0xa4, 0x4f, 0x68, 0x51, 0x93, 0x2d, 0x18, 0xc4, 0xd8, 0xe3,
|
||||
+ 0x94, 0xe5, 0x1f, 0x41, 0xf3, 0x2a, 0x16, 0x2b, 0x9a, 0x3e, 0xe5, 0x61, 0xfd, 0x1c, 0x3a, 0x7b,
|
||||
+ 0xc9, 0x04, 0xf3, 0xe9, 0x71, 0x9a, 0x0c, 0x6e, 0x2c, 0xff, 0x58, 0xff, 0xaa, 0x42, 0x7b, 0x7b,
|
||||
+ 0xc2, 0x68, 0x12, 0x15, 0x72, 0xb2, 0x0a, 0xd2, 0xf9, 0x9c, 0x2c, 0x49, 0x64, 0x4e, 0x56, 0xc4,
|
||||
+ 0x1f, 0x40, 0x27, 0x90, 0xa1, 0xab, 0xe9, 0x55, 0x1e, 0x1a, 0x2c, 0x04, 0xb5, 0xdd, 0x0e, 0x72,
|
||||
+ 0xc9, 0x6c, 0x0c, 0x10, 0x11, 0x3f, 0xd6, 0x67, 0x54, 0x3a, 0xea, 0xeb, 0x8a, 0xd0, 0xa4, 0x68,
|
||||
+ 0xbb, 0x15, 0xa5, 0xd9, 0xfa, 0x3d, 0x68, 0x1f, 0x0b, 0x27, 0xe9, 0x03, 0x85, 0x64, 0x94, 0x79,
|
||||
+ 0xcf, 0x86, 0xe3, 0x2c, 0x08, 0xf7, 0xa0, 0x7b, 0xaa, 0x5c, 0xa6, 0x0f, 0xa9, 0x37, 0xf4, 0x9a,
|
||||
+ 0xb6, 0x24, 0xb3, 0x77, 0x9c, 0xf7, 0xac, 0xba, 0x80, 0xce, 0x69, 0x0e, 0x35, 0x3a, 0x84, 0xc1,
|
||||
+ 0x02, 0x49, 0x49, 0x0e, 0xda, 0xc8, 0xe7, 0xa0, 0xf6, 0x23, 0xa4, 0x04, 0xe5, 0x4f, 0xe6, 0xf3,
|
||||
+ 0xd2, 0x6f, 0xab, 0xd0, 0xf9, 0x31, 0xe6, 0x2f, 0x29, 0x3b, 0x53, 0xfa, 0x22, 0xa8, 0x87, 0x6e,
|
||||
+ 0x80, 0x35, 0x47, 0xb9, 0x46, 0x77, 0xa0, 0xc9, 0x2e, 0x54, 0x02, 0xd1, 0xf7, 0xb9, 0xcc, 0x2e,
|
||||
+ 0x64, 0x62, 0x40, 0xaf, 0x00, 0xb0, 0x0b, 0x27, 0x72, 0xbd, 0x33, 0xac, 0x3d, 0x58, 0xb7, 0x5b,
|
||||
+ 0xec, 0xe2, 0x40, 0x21, 0xc4, 0x53, 0x60, 0x17, 0x0e, 0x66, 0x8c, 0xb2, 0x58, 0xe7, 0xaa, 0x26,
|
||||
+ 0xbb, 0xd8, 0x95, 0xb0, 0x3e, 0xeb, 0x33, 0x1a, 0x45, 0xd8, 0x97, 0x39, 0x5a, 0x9e, 0xdd, 0x51,
|
||||
+ 0x08, 0x21, 0x95, 0x1b, 0xa9, 0x0d, 0x25, 0x95, 0x67, 0x52, 0x79, 0x26, 0x75, 0x59, 0x9d, 0xe4,
|
||||
+ 0x79, 0xa9, 0x3c, 0x95, 0xda, 0x54, 0x52, 0x79, 0x4e, 0x2a, 0xcf, 0xa4, 0xb6, 0xcc, 0x59, 0x2d,
|
||||
+ 0xd5, 0xfa, 0x4d, 0x05, 0xd6, 0xe7, 0x0b, 0x3f, 0x5d, 0xa6, 0x7e, 0x00, 0x1d, 0x4f, 0xde, 0x57,
|
||||
+ 0xe1, 0x4d, 0x0e, 0x16, 0x6e, 0xd2, 0x6e, 0x7b, 0xb9, 0x67, 0xfc, 0x21, 0x74, 0x43, 0xe5, 0xe0,
|
||||
+ 0xf4, 0x69, 0xd6, 0xb2, 0x7b, 0xc9, 0xfb, 0xde, 0xee, 0x84, 0x39, 0xc8, 0xf2, 0x01, 0x7d, 0xc9,
|
||||
+ 0x08, 0xc7, 0x87, 0x9c, 0x61, 0x37, 0xb8, 0x89, 0x06, 0x04, 0x41, 0x5d, 0x56, 0x2b, 0x35, 0x59,
|
||||
+ 0x5f, 0xcb, 0xb5, 0xf5, 0x26, 0xac, 0x16, 0xa4, 0x68, 0x5b, 0x57, 0xa0, 0x36, 0xc5, 0xa1, 0xe4,
|
||||
+ 0xde, 0xb5, 0xc5, 0xd2, 0x72, 0x61, 0x60, 0x63, 0xd7, 0xbf, 0x39, 0x6d, 0xb4, 0x88, 0x5a, 0x26,
|
||||
+ 0x62, 0x03, 0x50, 0x5e, 0x84, 0x56, 0xc5, 0x68, 0x5d, 0xc9, 0x69, 0xfd, 0x1c, 0x06, 0xdb, 0x53,
|
||||
+ 0x1a, 0xe3, 0x43, 0xee, 0x93, 0xf0, 0x26, 0x3a, 0xa6, 0x5f, 0xc2, 0xea, 0x0b, 0x3e, 0xfb, 0x52,
|
||||
+ 0x30, 0x8b, 0xc9, 0x37, 0xf8, 0x86, 0xec, 0x63, 0xf4, 0xa5, 0xb1, 0x8f, 0xd1, 0x97, 0xa2, 0x59,
|
||||
+ 0xf2, 0xe8, 0x34, 0x09, 0x42, 0x19, 0x0a, 0x5d, 0x5b, 0x43, 0xd6, 0x16, 0x74, 0x54, 0x0d, 0xfd,
|
||||
+ 0x8c, 0xfa, 0xc9, 0x14, 0x97, 0xc6, 0xe0, 0x7d, 0x80, 0xc8, 0x65, 0x6e, 0x80, 0x39, 0x66, 0xea,
|
||||
+ 0x0d, 0xb5, 0xec, 0x1c, 0xc6, 0xfa, 0x7d, 0x15, 0xd6, 0xd4, 0x48, 0xe4, 0x50, 0x4d, 0x02, 0x8c,
|
||||
+ 0x09, 0x23, 0x68, 0x9e, 0xd2, 0x98, 0xe7, 0x18, 0xa6, 0xb0, 0x50, 0xd1, 0x0f, 0x0d, 0x37, 0xb1,
|
||||
+ 0x2c, 0xcc, 0x29, 0x6a, 0x57, 0xcf, 0x29, 0x16, 0x26, 0x11, 0xf5, 0xc5, 0x49, 0x84, 0x88, 0x36,
|
||||
+ 0x43, 0x44, 0x54, 0x8c, 0xb7, 0xec, 0x96, 0xc6, 0xec, 0xfb, 0xe8, 0x21, 0xf4, 0x27, 0x42, 0x4b,
|
||||
+ 0xe7, 0x94, 0xd2, 0x33, 0x27, 0x72, 0xf9, 0xa9, 0x0c, 0xf5, 0x96, 0xdd, 0x95, 0xe8, 0x3d, 0x4a,
|
||||
+ 0xcf, 0x0e, 0x5c, 0x7e, 0x8a, 0x3e, 0x86, 0x9e, 0x2e, 0x03, 0x03, 0xe9, 0xa2, 0x58, 0x7f, 0xfc,
|
||||
+ 0x74, 0x14, 0xe5, 0xbd, 0x67, 0x77, 0xcf, 0x72, 0x50, 0x6c, 0xdd, 0x86, 0x5b, 0x3b, 0x38, 0xe6,
|
||||
+ 0x8c, 0xce, 0x8a, 0x8e, 0xb1, 0x7e, 0x00, 0xb0, 0x1f, 0x72, 0xcc, 0x4e, 0x5c, 0x0f, 0xc7, 0xe8,
|
||||
+ 0xdd, 0x3c, 0xa4, 0x8b, 0xa3, 0x95, 0xb1, 0x9a, 0x48, 0xa5, 0x1b, 0x76, 0x8e, 0xc6, 0x1a, 0x43,
|
||||
+ 0xc3, 0xa6, 0x89, 0x48, 0x47, 0xaf, 0x9b, 0x95, 0x3e, 0xd7, 0xd1, 0xe7, 0x24, 0xd2, 0xd6, 0x7b,
|
||||
+ 0xd6, 0x9e, 0x69, 0x61, 0x33, 0x76, 0xfa, 0x8a, 0xc6, 0xd0, 0x22, 0x06, 0xa7, 0xb3, 0xca, 0xa2,
|
||||
+ 0xe8, 0x8c, 0xc4, 0xfa, 0x19, 0xac, 0x2a, 0x4e, 0x8a, 0xb3, 0x61, 0xf3, 0x3a, 0x34, 0x98, 0x51,
|
||||
+ 0xa3, 0x92, 0x8d, 0xa2, 0x34, 0x91, 0xde, 0x43, 0xf7, 0x84, 0x30, 0x8f, 0xe1, 0x40, 0xf4, 0x1c,
|
||||
+ 0x55, 0x79, 0x65, 0x19, 0x42, 0x78, 0x4b, 0xf4, 0xdb, 0x99, 0x99, 0xc6, 0x5b, 0xab, 0x30, 0x10,
|
||||
+ 0x1b, 0x05, 0x89, 0xd6, 0x2f, 0x60, 0xf5, 0x79, 0x38, 0x25, 0x21, 0xde, 0x3e, 0x38, 0x7a, 0x86,
|
||||
+ 0xd3, 0xac, 0x80, 0xa0, 0x2e, 0xaa, 0x27, 0xa9, 0x46, 0xd3, 0x96, 0x6b, 0x11, 0x26, 0xe1, 0xb1,
|
||||
+ 0xe3, 0x45, 0x49, 0xac, 0x27, 0x43, 0x8d, 0xf0, 0x78, 0x3b, 0x4a, 0x62, 0x91, 0xe6, 0xc5, 0x67,
|
||||
+ 0x9e, 0x86, 0xd3, 0x99, 0x8c, 0x95, 0xa6, 0xbd, 0xec, 0x45, 0xc9, 0xf3, 0x70, 0x3a, 0xb3, 0xfe,
|
||||
+ 0x4f, 0xf6, 0xc2, 0x18, 0xfb, 0xb6, 0x1b, 0xfa, 0x34, 0xd8, 0xc1, 0xe7, 0x39, 0x09, 0x69, 0xdf,
|
||||
+ 0x65, 0x72, 0xc2, 0xb7, 0x15, 0xe8, 0x3c, 0x99, 0xe0, 0x90, 0xef, 0x60, 0xee, 0x92, 0xa9, 0xec,
|
||||
+ 0xad, 0xce, 0x31, 0x8b, 0x09, 0x0d, 0xf5, 0xc3, 0x37, 0xa0, 0x68, 0x8d, 0x49, 0x48, 0xb8, 0xe3,
|
||||
+ 0xbb, 0x38, 0xa0, 0xa1, 0xf6, 0x02, 0x08, 0xd4, 0x8e, 0xc4, 0xa0, 0x37, 0xa1, 0xaf, 0x26, 0x77,
|
||||
+ 0xce, 0xa9, 0x1b, 0xfa, 0x53, 0x11, 0x72, 0x6a, 0x92, 0xd1, 0x53, 0xe8, 0x3d, 0x8d, 0x45, 0x6f,
|
||||
+ 0xc1, 0x8a, 0x0e, 0x88, 0x8c, 0xb2, 0x2e, 0x29, 0xfb, 0x1a, 0x5f, 0x20, 0x4d, 0xa2, 0x88, 0x32,
|
||||
+ 0x1e, 0x3b, 0x31, 0xf6, 0x3c, 0x1a, 0x44, 0xba, 0x31, 0xe9, 0x1b, 0xfc, 0xa1, 0x42, 0x5b, 0x13,
|
||||
+ 0x58, 0x7d, 0x2a, 0xec, 0xd4, 0x96, 0x64, 0x17, 0xdc, 0x0b, 0x70, 0xe0, 0x1c, 0x4f, 0xa9, 0x77,
|
||||
+ 0xe6, 0x88, 0x34, 0xa5, 0x3d, 0x2c, 0x4a, 0x9f, 0x2d, 0x81, 0x3c, 0x24, 0xdf, 0xc8, 0x1e, 0x5c,
|
||||
+ 0x50, 0x9d, 0x52, 0x1e, 0x4d, 0x93, 0x89, 0x13, 0x31, 0x7a, 0x8c, 0xb5, 0x89, 0xfd, 0x00, 0x07,
|
||||
+ 0x7b, 0x0a, 0x7f, 0x20, 0xd0, 0xd6, 0x5f, 0x2a, 0xb0, 0x56, 0x94, 0xa4, 0x93, 0xee, 0x26, 0xac,
|
||||
+ 0x15, 0x45, 0xe9, 0x0f, 0xb1, 0x2a, 0xf4, 0x06, 0x79, 0x81, 0xea, 0x93, 0xfc, 0x21, 0x74, 0xe5,
|
||||
+ 0x38, 0xd7, 0xf1, 0x15, 0xa7, 0x62, 0xf9, 0x91, 0xbf, 0x17, 0xbb, 0xe3, 0xe6, 0x6f, 0xe9, 0x63,
|
||||
+ 0xb8, 0xa3, 0xcd, 0x77, 0x16, 0xd5, 0x56, 0x0f, 0x62, 0x5d, 0x13, 0x3c, 0x9b, 0xd3, 0xfe, 0x0b,
|
||||
+ 0x18, 0x66, 0xa8, 0xad, 0x99, 0x44, 0x1a, 0x5f, 0xbd, 0x0b, 0xab, 0x73, 0xc6, 0x3e, 0xf1, 0x7d,
|
||||
+ 0x26, 0x03, 0xb4, 0x6e, 0x97, 0x6d, 0x59, 0x8f, 0xe1, 0xf6, 0x21, 0xe6, 0xca, 0x1b, 0x2e, 0xd7,
|
||||
+ 0x3d, 0x81, 0x62, 0xb6, 0x02, 0xb5, 0x43, 0xec, 0x49, 0xe3, 0x6b, 0xb6, 0x58, 0x8a, 0x07, 0x78,
|
||||
+ 0x14, 0x63, 0x4f, 0x5a, 0x59, 0xb3, 0xe5, 0xda, 0xfa, 0x73, 0x05, 0x96, 0x75, 0x9a, 0x14, 0xa9,
|
||||
+ 0xde, 0x67, 0xe4, 0x1c, 0x33, 0xfd, 0xf4, 0x34, 0x84, 0xde, 0x80, 0x9e, 0x5a, 0x39, 0x34, 0xe2,
|
||||
+ 0x84, 0xa6, 0xc9, 0xb7, 0xab, 0xb0, 0xcf, 0x15, 0x52, 0x4e, 0xea, 0xe4, 0x20, 0x4a, 0xf7, 0x7c,
|
||||
+ 0x1a, 0x92, 0xe3, 0xb6, 0x58, 0x64, 0x06, 0x99, 0x6c, 0x5b, 0xb6, 0x86, 0xc4, 0x53, 0x37, 0xfc,
|
||||
+ 0x96, 0x24, 0x3f, 0x03, 0x8a, 0xa7, 0x1e, 0xd0, 0x24, 0xe4, 0x4e, 0x44, 0x49, 0xc8, 0x75, 0x76,
|
||||
+ 0x05, 0x89, 0x3a, 0x10, 0x18, 0xeb, 0xd7, 0x15, 0x68, 0xa8, 0x69, 0xb5, 0xe8, 0x32, 0xd3, 0x6f,
|
||||
+ 0x5c, 0x95, 0xc8, 0x7a, 0x41, 0xca, 0x52, 0xdf, 0x35, 0xb9, 0x16, 0x71, 0x7c, 0x1e, 0xa8, 0x4c,
|
||||
+ 0xad, 0x55, 0x3b, 0x0f, 0x64, 0x8a, 0x7e, 0x03, 0x7a, 0xd9, 0xa7, 0x52, 0xee, 0x2b, 0x15, 0xbb,
|
||||
+ 0x29, 0x56, 0x92, 0x5d, 0xaa, 0xa9, 0xf5, 0x53, 0xd1, 0x5c, 0xa7, 0x93, 0xda, 0x15, 0xa8, 0x25,
|
||||
+ 0xa9, 0x32, 0x62, 0x29, 0x30, 0x93, 0xf4, 0x23, 0x2b, 0x96, 0xe8, 0x21, 0xf4, 0x5c, 0xdf, 0x27,
|
||||
+ 0xe2, 0xb8, 0x3b, 0x7d, 0x4a, 0xfc, 0x34, 0x48, 0x8b, 0x58, 0xeb, 0x6f, 0x15, 0xe8, 0x6f, 0xd3,
|
||||
+ 0x68, 0xf6, 0x43, 0x32, 0xc5, 0xb9, 0x0c, 0x22, 0x95, 0xd4, 0xdf, 0x58, 0xb1, 0x16, 0x75, 0xe3,
|
||||
+ 0x09, 0x99, 0x62, 0x15, 0x5a, 0xea, 0x66, 0x9b, 0x02, 0x21, 0xc3, 0xca, 0x6c, 0xa6, 0x03, 0xb0,
|
||||
+ 0xae, 0xda, 0x7c, 0x46, 0x7d, 0x59, 0x21, 0xfb, 0x84, 0x39, 0xe9, 0xb8, 0xab, 0x6b, 0x2f, 0xfb,
|
||||
+ 0x84, 0xc9, 0x2d, 0x6d, 0xc8, 0x92, 0x9c, 0xb8, 0xe6, 0x0d, 0x69, 0x28, 0x8c, 0x30, 0x64, 0x1d,
|
||||
+ 0x1a, 0xf4, 0xe4, 0x24, 0xc6, 0x5c, 0xd6, 0xb2, 0x35, 0x5b, 0x43, 0x69, 0x9a, 0x6b, 0xe6, 0xd2,
|
||||
+ 0xdc, 0x2d, 0x58, 0x95, 0xb3, 0xfd, 0x17, 0xcc, 0xf5, 0x48, 0x38, 0x31, 0xa9, 0x78, 0x0d, 0xd0,
|
||||
+ 0x21, 0xa7, 0x51, 0x11, 0xfb, 0xe8, 0x77, 0x2b, 0x3a, 0x27, 0xea, 0x46, 0x17, 0x3d, 0x85, 0xfe,
|
||||
+ 0xdc, 0x1f, 0x27, 0x48, 0x4f, 0x3e, 0xca, 0xff, 0x4f, 0x19, 0xad, 0x8f, 0xd5, 0x1f, 0x31, 0x63,
|
||||
+ 0xf3, 0x47, 0xcc, 0x78, 0x37, 0x88, 0xf8, 0x0c, 0xed, 0x42, 0xaf, 0xf8, 0x17, 0x03, 0xba, 0x6b,
|
||||
+ 0x0a, 0x85, 0x92, 0x3f, 0x1e, 0x2e, 0x65, 0xf3, 0x14, 0xfa, 0x73, 0xff, 0x36, 0x18, 0x7d, 0xca,
|
||||
+ 0xff, 0x84, 0xb8, 0x94, 0xd1, 0x63, 0x68, 0xe7, 0xfe, 0x5e, 0x40, 0x43, 0xc5, 0x64, 0xf1, 0x1f,
|
||||
+ 0x87, 0x4b, 0x19, 0x6c, 0x43, 0xb7, 0x30, 0xf1, 0x47, 0x23, 0x6d, 0x4f, 0xc9, 0xdf, 0x00, 0x97,
|
||||
+ 0x32, 0xd9, 0x82, 0x76, 0x6e, 0xf0, 0x6e, 0xb4, 0x58, 0x9c, 0xee, 0x8f, 0xee, 0x94, 0xec, 0xe8,
|
||||
+ 0xd4, 0xbb, 0x07, 0xdd, 0xc2, 0x98, 0xdc, 0x28, 0x52, 0x36, 0xa2, 0x1f, 0xdd, 0x2d, 0xdd, 0xd3,
|
||||
+ 0x9c, 0x9e, 0x42, 0x7f, 0x6e, 0x68, 0x6e, 0x9c, 0x5b, 0x3e, 0x4b, 0xbf, 0xd4, 0xac, 0xcf, 0xe5,
|
||||
+ 0x65, 0xe7, 0x7a, 0xa2, 0xdc, 0x65, 0x2f, 0x8e, 0xc8, 0x47, 0xf7, 0xca, 0x37, 0xb5, 0x56, 0xbb,
|
||||
+ 0xd0, 0x2b, 0x4e, 0xc7, 0x0d, 0xb3, 0xd2, 0x99, 0xf9, 0xd5, 0x2f, 0xa7, 0x30, 0x28, 0xcf, 0x5e,
|
||||
+ 0x4e, 0xd9, 0xfc, 0xfc, 0x52, 0x46, 0x4f, 0x00, 0x74, 0x07, 0xe4, 0x93, 0x30, 0xbd, 0xb2, 0x85,
|
||||
+ 0xce, 0x2b, 0xbd, 0xb2, 0x92, 0x6e, 0xe9, 0x31, 0x80, 0x6a, 0x5c, 0x7c, 0x9a, 0x70, 0x74, 0xdb,
|
||||
+ 0xa8, 0x31, 0xd7, 0x2d, 0x8d, 0x86, 0x8b, 0x1b, 0x0b, 0x0c, 0x30, 0x63, 0xd7, 0x61, 0xf0, 0x19,
|
||||
+ 0x40, 0xd6, 0x10, 0x19, 0x06, 0x0b, 0x2d, 0xd2, 0x15, 0x3e, 0xe8, 0xe4, 0xdb, 0x1f, 0xa4, 0x6d,
|
||||
+ 0x2d, 0x69, 0x89, 0xae, 0x60, 0xd1, 0x9f, 0x2b, 0x6f, 0x8b, 0x8f, 0x6d, 0xbe, 0xea, 0x1d, 0x2d,
|
||||
+ 0x94, 0xb8, 0xe8, 0x43, 0xe8, 0xe4, 0xeb, 0x5a, 0xa3, 0x45, 0x49, 0xad, 0x3b, 0x2a, 0xd4, 0xb6,
|
||||
+ 0xe8, 0x31, 0xf4, 0x8a, 0x55, 0x2b, 0xca, 0xc5, 0xc5, 0x42, 0x2d, 0x3b, 0xd2, 0x13, 0x9b, 0x1c,
|
||||
+ 0xf9, 0xfb, 0x00, 0x59, 0x75, 0x6b, 0xdc, 0xb7, 0x50, 0xef, 0xce, 0x49, 0x7d, 0x02, 0x9d, 0x7c,
|
||||
+ 0x26, 0x36, 0xea, 0x96, 0x64, 0xe7, 0xab, 0xb2, 0x56, 0x2e, 0x6b, 0x9b, 0xc7, 0xb7, 0x98, 0xc8,
|
||||
+ 0xaf, 0xca, 0x5a, 0x85, 0xae, 0xcf, 0x24, 0x8b, 0xb2, 0x56, 0xf0, 0xaa, 0x5c, 0x5e, 0x6c, 0x91,
|
||||
+ 0x8c, 0xfb, 0x4a, 0x1b, 0xa7, 0xab, 0x1e, 0x51, 0xbe, 0x1b, 0x30, 0xfe, 0x28, 0xe9, 0x10, 0xbe,
|
||||
+ 0x27, 0xa8, 0xf3, 0x15, 0x7f, 0x2e, 0xa8, 0x4b, 0x1a, 0x81, 0x4b, 0x19, 0xed, 0x41, 0xff, 0xa9,
|
||||
+ 0x29, 0xe6, 0x74, 0xa1, 0xa9, 0xd5, 0x29, 0x29, 0xac, 0x47, 0xa3, 0xb2, 0x2d, 0x1d, 0x59, 0x9f,
|
||||
+ 0xc3, 0x60, 0xa1, 0xc8, 0x44, 0xf7, 0xd3, 0xc1, 0x62, 0x69, 0xf5, 0x79, 0xa9, 0x5a, 0xfb, 0xb0,
|
||||
+ 0x32, 0x5f, 0x63, 0xa2, 0x57, 0xf4, 0xa5, 0x97, 0xd7, 0x9e, 0x97, 0xb2, 0xfa, 0x18, 0x9a, 0xa6,
|
||||
+ 0xa6, 0x41, 0x7a, 0x80, 0x3b, 0x57, 0xe3, 0x5c, 0x76, 0x74, 0xab, 0xf3, 0xed, 0x77, 0xf7, 0x2b,
|
||||
+ 0x7f, 0xff, 0xee, 0x7e, 0xe5, 0x9f, 0xdf, 0xdd, 0xaf, 0x1c, 0x37, 0xe4, 0xee, 0xfb, 0xff, 0x0e,
|
||||
+ 0x00, 0x00, 0xff, 0xff, 0x8d, 0x89, 0xaa, 0x73, 0xc8, 0x21, 0x00, 0x00,
|
||||
}
|
||||
diff --git a/protocols/grpc/agent.proto b/protocols/grpc/agent.proto
|
||||
index 348d792..b0fab6d 100644
|
||||
--- a/protocols/grpc/agent.proto
|
||||
+++ b/protocols/grpc/agent.proto
|
||||
@@ -316,6 +316,7 @@ message UpdateInterfaceRequest {
|
||||
|
||||
message UpdateRoutesRequest {
|
||||
Routes routes = 1;
|
||||
+ bool increment = 2;
|
||||
}
|
||||
|
||||
message ListInterfacesRequest {
|
||||
--
|
||||
2.14.3 (Apple Git-98)
|
||||
|
||||
@ -1,971 +0,0 @@
|
||||
From 0f24d3d433e93a7309b9bac59e4a15e722391490 Mon Sep 17 00:00:00 2001
|
||||
From: xiadanni1 <xiadanni1@huawei.com>
|
||||
Date: Tue, 18 Aug 2020 06:01:55 +0800
|
||||
Subject: [PATCH 03/16] kata-agent: add kata-ipvs command
|
||||
|
||||
reason: add kata-ipvs command to update IPVS rules
|
||||
in VM.
|
||||
|
||||
Signed-off-by: xiadanni1 <xiadanni1@huawei.com>
|
||||
---
|
||||
agent.go | 6 +
|
||||
grpc.go | 4 +
|
||||
ipvsadm.go | 130 +++++++++
|
||||
protocols/grpc/agent.pb.go | 663 ++++++++++++++++++++++++++++++++-------------
|
||||
protocols/grpc/agent.proto | 11 +
|
||||
5 files changed, 633 insertions(+), 181 deletions(-)
|
||||
create mode 100644 ipvsadm.go
|
||||
|
||||
diff --git a/agent.go b/agent.go
|
||||
index c1cac08..c161e93 100644
|
||||
--- a/agent.go
|
||||
+++ b/agent.go
|
||||
@@ -118,6 +118,11 @@ type sandboxStorage struct {
|
||||
refCount int
|
||||
}
|
||||
|
||||
+type ipvsAdm struct {
|
||||
+ ipvsLock sync.Mutex
|
||||
+ ipvsRuleCnt uint64
|
||||
+}
|
||||
+
|
||||
type sandbox struct {
|
||||
sync.RWMutex
|
||||
ctx context.Context
|
||||
@@ -144,6 +149,7 @@ type sandbox struct {
|
||||
sandboxPidNs bool
|
||||
storages map[string]*sandboxStorage
|
||||
stopServer chan struct{}
|
||||
+ ipvsadm ipvsAdm
|
||||
}
|
||||
|
||||
var agentFields = logrus.Fields{
|
||||
diff --git a/grpc.go b/grpc.go
|
||||
index 8fe8217..de2cae7 100644
|
||||
--- a/grpc.go
|
||||
+++ b/grpc.go
|
||||
@@ -1567,6 +1567,10 @@ func (a *agentGRPC) ListRoutes(ctx context.Context, req *pb.ListRoutesRequest) (
|
||||
return a.sandbox.listRoutes(nil)
|
||||
}
|
||||
|
||||
+func (a *agentGRPC) UpdateIPVSRule(ctx context.Context, req *pb.UpdateIPVSRequest) (*pb.IPVSResponse, error) {
|
||||
+ return a.sandbox.updateIPVSRule(req)
|
||||
+}
|
||||
+
|
||||
func (a *agentGRPC) OnlineCPUMem(ctx context.Context, req *pb.OnlineCPUMemRequest) (*gpb.Empty, error) {
|
||||
if !req.Wait {
|
||||
go a.onlineCPUMem(req)
|
||||
diff --git a/ipvsadm.go b/ipvsadm.go
|
||||
new file mode 100644
|
||||
index 0000000..48eb19f
|
||||
--- /dev/null
|
||||
+++ b/ipvsadm.go
|
||||
@@ -0,0 +1,130 @@
|
||||
+// Copyright (c) Huawei Technologies Co., Ltd. 2020. All rights reserved.
|
||||
+// SPDX-License-Identifier: Apache-2.0
|
||||
+// Description: IPVS related common functions
|
||||
+// Author: xiadanni
|
||||
+// Create: 2020-08-01
|
||||
+
|
||||
+package main
|
||||
+
|
||||
+import (
|
||||
+ "os/exec"
|
||||
+ "strconv"
|
||||
+ "strings"
|
||||
+
|
||||
+ "github.com/kata-containers/agent/protocols/grpc"
|
||||
+ "google.golang.org/grpc/codes"
|
||||
+ grpcStatus "google.golang.org/grpc/status"
|
||||
+)
|
||||
+
|
||||
+var (
|
||||
+ errNoIPVSRules = grpcStatus.Errorf(codes.InvalidArgument, "IPVS rule is nil")
|
||||
+ errRuleExist = grpcStatus.Errorf(codes.InvalidArgument, "failed to restore IPVS rules: exist some rules in system, should clear first")
|
||||
+ errInvalidIPVSRule = grpcStatus.Errorf(codes.InvalidArgument, "invalid IPVS rule, please check")
|
||||
+ errIPVSRuleExceed = grpcStatus.Errorf(codes.InvalidArgument, "rules exceed limit, should clear first")
|
||||
+)
|
||||
+
|
||||
+const (
|
||||
+ validHeadLength = 7
|
||||
+ ruleLimitMax = 20000
|
||||
+ restoreHead = "restore"
|
||||
+ conntrackNormalInfo = "have been deleted"
|
||||
+ waitError = "no child processes"
|
||||
+)
|
||||
+
|
||||
+func (s *sandbox) updateIPVSRule(ipvsRule *grpc.UpdateIPVSRequest) (resultingIpvs *grpc.IPVSResponse, err error) {
|
||||
+ var (
|
||||
+ restoreRuleCnt string
|
||||
+ restoreRule string
|
||||
+ operation string
|
||||
+ restoreRuleCntUint uint64
|
||||
+ cmd *exec.Cmd
|
||||
+ )
|
||||
+
|
||||
+ if ipvsRule == nil || ipvsRule.IPVSReq == "" {
|
||||
+ return nil, errNoIPVSRules
|
||||
+ }
|
||||
+
|
||||
+ s.ipvsadm.ipvsLock.Lock()
|
||||
+ defer s.ipvsadm.ipvsLock.Unlock()
|
||||
+
|
||||
+ rule := ipvsRule.IPVSReq
|
||||
+ if len(rule) < validHeadLength {
|
||||
+ return nil, errInvalidIPVSRule
|
||||
+ }
|
||||
+
|
||||
+ if ruleHead := rule[:validHeadLength]; ruleHead == restoreHead {
|
||||
+ if s.ipvsadm.ipvsRuleCnt != 0 {
|
||||
+ return nil, errRuleExist
|
||||
+ }
|
||||
+ operation = restoreHead
|
||||
+
|
||||
+ restoreItem := strings.Split(rule, "|")
|
||||
+ if len(restoreItem) != 3 {
|
||||
+ return nil, errInvalidIPVSRule
|
||||
+ }
|
||||
+ restoreRuleCnt = restoreItem[1]
|
||||
+ if restoreRuleCntUint, err = strconv.ParseUint(restoreRuleCnt, 10, 64); err != nil {
|
||||
+ return nil, errInvalidIPVSRule
|
||||
+ }
|
||||
+ restoreRule = restoreItem[2]
|
||||
+
|
||||
+ cmd = exec.Command("/sbin/ipvsadm", "--restore")
|
||||
+ cmd.Stdin = strings.NewReader(restoreRule)
|
||||
+ } else {
|
||||
+ item := strings.Fields(strings.TrimSpace(rule))
|
||||
+ if len(item) < 2 {
|
||||
+ return nil, errInvalidIPVSRule
|
||||
+ }
|
||||
+ if item[0] != "ipvsadm" && item[0] != "conntrack" {
|
||||
+ return nil, errInvalidIPVSRule
|
||||
+ }
|
||||
+ operation = item[1]
|
||||
+ if s.ipvsadm.ipvsRuleCnt >= ruleLimitMax {
|
||||
+ if operation == "--add-service" || operation == "-A" || operation == "--add-server" || operation == "-a" {
|
||||
+ return nil, errIPVSRuleExceed
|
||||
+ }
|
||||
+ }
|
||||
+
|
||||
+ restoreRuleCntUint = 0
|
||||
+
|
||||
+ cmd = exec.Command("/sbin/" + item[0])
|
||||
+ for i := 1; i < len(item); i++ {
|
||||
+ cmd.Args = append(cmd.Args, item[i])
|
||||
+ }
|
||||
+ }
|
||||
+
|
||||
+ bytes, err := cmd.CombinedOutput()
|
||||
+
|
||||
+ if err != nil && !strings.Contains(err.Error(), waitError) && !strings.Contains(string(bytes), conntrackNormalInfo) {
|
||||
+ return nil, grpcStatus.Errorf(codes.Internal,
|
||||
+ "exec IPVS command failed, stderr: %v, err: %v", string(bytes), err)
|
||||
+ }
|
||||
+
|
||||
+ s.sumRuleCount(operation, restoreRuleCntUint)
|
||||
+
|
||||
+ rsp := &grpc.IPVSResponse{
|
||||
+ IPVSRes: string(bytes),
|
||||
+ }
|
||||
+ return rsp, nil
|
||||
+}
|
||||
+
|
||||
+func (s *sandbox) sumRuleCount(operation string, restoreRuleCntUint uint64) {
|
||||
+ if operation == restoreHead {
|
||||
+ s.ipvsadm.ipvsRuleCnt = restoreRuleCntUint
|
||||
+ return
|
||||
+ }
|
||||
+ if operation == "--add-service" || operation == "-A" || operation == "--add-server" || operation == "-a" {
|
||||
+ s.ipvsadm.ipvsRuleCnt++
|
||||
+ return
|
||||
+ }
|
||||
+ if operation == "--delete-service" || operation == "-D" || operation == "--delete-server" || operation == "-d" {
|
||||
+ if s.ipvsadm.ipvsRuleCnt > 0 {
|
||||
+ s.ipvsadm.ipvsRuleCnt--
|
||||
+ }
|
||||
+ return
|
||||
+ }
|
||||
+ if operation == "--clear" || operation == "-C" {
|
||||
+ s.ipvsadm.ipvsRuleCnt = 0
|
||||
+ return
|
||||
+ }
|
||||
+}
|
||||
diff --git a/protocols/grpc/agent.pb.go b/protocols/grpc/agent.pb.go
|
||||
index 1b887e5..04d0ee5 100644
|
||||
--- a/protocols/grpc/agent.pb.go
|
||||
+++ b/protocols/grpc/agent.pb.go
|
||||
@@ -63,6 +63,8 @@
|
||||
CopyFileRequest
|
||||
StartTracingRequest
|
||||
StopTracingRequest
|
||||
+ UpdateIPVSRequest
|
||||
+ IPVSResponse
|
||||
CheckRequest
|
||||
HealthCheckResponse
|
||||
VersionCheckResponse
|
||||
@@ -1842,6 +1844,40 @@ func (m *StopTracingRequest) String() string { return proto.CompactTe
|
||||
func (*StopTracingRequest) ProtoMessage() {}
|
||||
func (*StopTracingRequest) Descriptor() ([]byte, []int) { return fileDescriptorAgent, []int{52} }
|
||||
|
||||
+type UpdateIPVSRequest struct {
|
||||
+ // IPVS_req is the IPVS rule message needed to update
|
||||
+ IPVSReq string `protobuf:"bytes,1,opt,name=IPVS_req,json=IPVSReq,proto3" json:"IPVS_req,omitempty"`
|
||||
+}
|
||||
+
|
||||
+func (m *UpdateIPVSRequest) Reset() { *m = UpdateIPVSRequest{} }
|
||||
+func (m *UpdateIPVSRequest) String() string { return proto.CompactTextString(m) }
|
||||
+func (*UpdateIPVSRequest) ProtoMessage() {}
|
||||
+func (*UpdateIPVSRequest) Descriptor() ([]byte, []int) { return fileDescriptorAgent, []int{53} }
|
||||
+
|
||||
+func (m *UpdateIPVSRequest) GetIPVSReq() string {
|
||||
+ if m != nil {
|
||||
+ return m.IPVSReq
|
||||
+ }
|
||||
+ return ""
|
||||
+}
|
||||
+
|
||||
+type IPVSResponse struct {
|
||||
+ // IPVS_res is the response of IPVS updating
|
||||
+ IPVSRes string `protobuf:"bytes,1,opt,name=IPVS_res,json=IPVSRes,proto3" json:"IPVS_res,omitempty"`
|
||||
+}
|
||||
+
|
||||
+func (m *IPVSResponse) Reset() { *m = IPVSResponse{} }
|
||||
+func (m *IPVSResponse) String() string { return proto.CompactTextString(m) }
|
||||
+func (*IPVSResponse) ProtoMessage() {}
|
||||
+func (*IPVSResponse) Descriptor() ([]byte, []int) { return fileDescriptorAgent, []int{54} }
|
||||
+
|
||||
+func (m *IPVSResponse) GetIPVSRes() string {
|
||||
+ if m != nil {
|
||||
+ return m.IPVSRes
|
||||
+ }
|
||||
+ return ""
|
||||
+}
|
||||
+
|
||||
func init() {
|
||||
proto.RegisterType((*CreateContainerRequest)(nil), "grpc.CreateContainerRequest")
|
||||
proto.RegisterType((*StartContainerRequest)(nil), "grpc.StartContainerRequest")
|
||||
@@ -1896,6 +1932,8 @@ func init() {
|
||||
proto.RegisterType((*CopyFileRequest)(nil), "grpc.CopyFileRequest")
|
||||
proto.RegisterType((*StartTracingRequest)(nil), "grpc.StartTracingRequest")
|
||||
proto.RegisterType((*StopTracingRequest)(nil), "grpc.StopTracingRequest")
|
||||
+ proto.RegisterType((*UpdateIPVSRequest)(nil), "grpc.UpdateIPVSRequest")
|
||||
+ proto.RegisterType((*IPVSResponse)(nil), "grpc.IPVSResponse")
|
||||
}
|
||||
|
||||
// Reference imports to suppress errors if they are not otherwise used.
|
||||
@@ -1938,6 +1976,7 @@ type AgentServiceClient interface {
|
||||
UpdateRoutes(ctx context.Context, in *UpdateRoutesRequest, opts ...grpc1.CallOption) (*Routes, error)
|
||||
ListInterfaces(ctx context.Context, in *ListInterfacesRequest, opts ...grpc1.CallOption) (*Interfaces, error)
|
||||
ListRoutes(ctx context.Context, in *ListRoutesRequest, opts ...grpc1.CallOption) (*Routes, error)
|
||||
+ UpdateIPVSRule(ctx context.Context, in *UpdateIPVSRequest, opts ...grpc1.CallOption) (*IPVSResponse, error)
|
||||
// tracing
|
||||
StartTracing(ctx context.Context, in *StartTracingRequest, opts ...grpc1.CallOption) (*google_protobuf2.Empty, error)
|
||||
StopTracing(ctx context.Context, in *StopTracingRequest, opts ...grpc1.CallOption) (*google_protobuf2.Empty, error)
|
||||
@@ -2140,6 +2179,15 @@ func (c *agentServiceClient) ListRoutes(ctx context.Context, in *ListRoutesReque
|
||||
return out, nil
|
||||
}
|
||||
|
||||
+func (c *agentServiceClient) UpdateIPVSRule(ctx context.Context, in *UpdateIPVSRequest, opts ...grpc1.CallOption) (*IPVSResponse, error) {
|
||||
+ out := new(IPVSResponse)
|
||||
+ err := grpc1.Invoke(ctx, "/grpc.AgentService/UpdateIPVSRule", in, out, c.cc, opts...)
|
||||
+ if err != nil {
|
||||
+ return nil, err
|
||||
+ }
|
||||
+ return out, nil
|
||||
+}
|
||||
+
|
||||
func (c *agentServiceClient) StartTracing(ctx context.Context, in *StartTracingRequest, opts ...grpc1.CallOption) (*google_protobuf2.Empty, error) {
|
||||
out := new(google_protobuf2.Empty)
|
||||
err := grpc1.Invoke(ctx, "/grpc.AgentService/StartTracing", in, out, c.cc, opts...)
|
||||
@@ -2262,6 +2310,7 @@ type AgentServiceServer interface {
|
||||
UpdateRoutes(context.Context, *UpdateRoutesRequest) (*Routes, error)
|
||||
ListInterfaces(context.Context, *ListInterfacesRequest) (*Interfaces, error)
|
||||
ListRoutes(context.Context, *ListRoutesRequest) (*Routes, error)
|
||||
+ UpdateIPVSRule(context.Context, *UpdateIPVSRequest) (*IPVSResponse, error)
|
||||
// tracing
|
||||
StartTracing(context.Context, *StartTracingRequest) (*google_protobuf2.Empty, error)
|
||||
StopTracing(context.Context, *StopTracingRequest) (*google_protobuf2.Empty, error)
|
||||
@@ -2640,6 +2689,24 @@ func _AgentService_ListRoutes_Handler(srv interface{}, ctx context.Context, dec
|
||||
return interceptor(ctx, in, info, handler)
|
||||
}
|
||||
|
||||
+func _AgentService_UpdateIPVSRule_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc1.UnaryServerInterceptor) (interface{}, error) {
|
||||
+ in := new(UpdateIPVSRequest)
|
||||
+ if err := dec(in); err != nil {
|
||||
+ return nil, err
|
||||
+ }
|
||||
+ if interceptor == nil {
|
||||
+ return srv.(AgentServiceServer).UpdateIPVSRule(ctx, in)
|
||||
+ }
|
||||
+ info := &grpc1.UnaryServerInfo{
|
||||
+ Server: srv,
|
||||
+ FullMethod: "/grpc.AgentService/UpdateIPVSRule",
|
||||
+ }
|
||||
+ handler := func(ctx context.Context, req interface{}) (interface{}, error) {
|
||||
+ return srv.(AgentServiceServer).UpdateIPVSRule(ctx, req.(*UpdateIPVSRequest))
|
||||
+ }
|
||||
+ return interceptor(ctx, in, info, handler)
|
||||
+}
|
||||
+
|
||||
func _AgentService_StartTracing_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc1.UnaryServerInterceptor) (interface{}, error) {
|
||||
in := new(StartTracingRequest)
|
||||
if err := dec(in); err != nil {
|
||||
@@ -2904,6 +2971,10 @@ var _AgentService_serviceDesc = grpc1.ServiceDesc{
|
||||
MethodName: "ListRoutes",
|
||||
Handler: _AgentService_ListRoutes_Handler,
|
||||
},
|
||||
+ {
|
||||
+ MethodName: "UpdateIPVSRule",
|
||||
+ Handler: _AgentService_UpdateIPVSRule_Handler,
|
||||
+ },
|
||||
{
|
||||
MethodName: "StartTracing",
|
||||
Handler: _AgentService_StartTracing_Handler,
|
||||
@@ -5088,6 +5159,54 @@ func (m *StopTracingRequest) MarshalTo(dAtA []byte) (int, error) {
|
||||
return i, nil
|
||||
}
|
||||
|
||||
+func (m *UpdateIPVSRequest) Marshal() (dAtA []byte, err error) {
|
||||
+ size := m.Size()
|
||||
+ dAtA = make([]byte, size)
|
||||
+ n, err := m.MarshalTo(dAtA)
|
||||
+ if err != nil {
|
||||
+ return nil, err
|
||||
+ }
|
||||
+ return dAtA[:n], nil
|
||||
+}
|
||||
+
|
||||
+func (m *UpdateIPVSRequest) MarshalTo(dAtA []byte) (int, error) {
|
||||
+ var i int
|
||||
+ _ = i
|
||||
+ var l int
|
||||
+ _ = l
|
||||
+ if len(m.IPVSReq) > 0 {
|
||||
+ dAtA[i] = 0xa
|
||||
+ i++
|
||||
+ i = encodeVarintAgent(dAtA, i, uint64(len(m.IPVSReq)))
|
||||
+ i += copy(dAtA[i:], m.IPVSReq)
|
||||
+ }
|
||||
+ return i, nil
|
||||
+}
|
||||
+
|
||||
+func (m *IPVSResponse) Marshal() (dAtA []byte, err error) {
|
||||
+ size := m.Size()
|
||||
+ dAtA = make([]byte, size)
|
||||
+ n, err := m.MarshalTo(dAtA)
|
||||
+ if err != nil {
|
||||
+ return nil, err
|
||||
+ }
|
||||
+ return dAtA[:n], nil
|
||||
+}
|
||||
+
|
||||
+func (m *IPVSResponse) MarshalTo(dAtA []byte) (int, error) {
|
||||
+ var i int
|
||||
+ _ = i
|
||||
+ var l int
|
||||
+ _ = l
|
||||
+ if len(m.IPVSRes) > 0 {
|
||||
+ dAtA[i] = 0xa
|
||||
+ i++
|
||||
+ i = encodeVarintAgent(dAtA, i, uint64(len(m.IPVSRes)))
|
||||
+ i += copy(dAtA[i:], m.IPVSRes)
|
||||
+ }
|
||||
+ return i, nil
|
||||
+}
|
||||
+
|
||||
func encodeVarintAgent(dAtA []byte, offset int, v uint64) int {
|
||||
for v >= 1<<7 {
|
||||
dAtA[offset] = uint8(v&0x7f | 0x80)
|
||||
@@ -6019,6 +6138,26 @@ func (m *StopTracingRequest) Size() (n int) {
|
||||
return n
|
||||
}
|
||||
|
||||
+func (m *UpdateIPVSRequest) Size() (n int) {
|
||||
+ var l int
|
||||
+ _ = l
|
||||
+ l = len(m.IPVSReq)
|
||||
+ if l > 0 {
|
||||
+ n += 1 + l + sovAgent(uint64(l))
|
||||
+ }
|
||||
+ return n
|
||||
+}
|
||||
+
|
||||
+func (m *IPVSResponse) Size() (n int) {
|
||||
+ var l int
|
||||
+ _ = l
|
||||
+ l = len(m.IPVSRes)
|
||||
+ if l > 0 {
|
||||
+ n += 1 + l + sovAgent(uint64(l))
|
||||
+ }
|
||||
+ return n
|
||||
+}
|
||||
+
|
||||
func sovAgent(x uint64) (n int) {
|
||||
for {
|
||||
n++
|
||||
@@ -12785,6 +12924,164 @@ func (m *StopTracingRequest) Unmarshal(dAtA []byte) error {
|
||||
}
|
||||
return nil
|
||||
}
|
||||
+func (m *UpdateIPVSRequest) Unmarshal(dAtA []byte) error {
|
||||
+ l := len(dAtA)
|
||||
+ iNdEx := 0
|
||||
+ for iNdEx < l {
|
||||
+ preIndex := iNdEx
|
||||
+ var wire uint64
|
||||
+ for shift := uint(0); ; shift += 7 {
|
||||
+ if shift >= 64 {
|
||||
+ return ErrIntOverflowAgent
|
||||
+ }
|
||||
+ if iNdEx >= l {
|
||||
+ return io.ErrUnexpectedEOF
|
||||
+ }
|
||||
+ b := dAtA[iNdEx]
|
||||
+ iNdEx++
|
||||
+ wire |= (uint64(b) & 0x7F) << shift
|
||||
+ if b < 0x80 {
|
||||
+ break
|
||||
+ }
|
||||
+ }
|
||||
+ fieldNum := int32(wire >> 3)
|
||||
+ wireType := int(wire & 0x7)
|
||||
+ if wireType == 4 {
|
||||
+ return fmt.Errorf("proto: UpdateIPVSRequest: wiretype end group for non-group")
|
||||
+ }
|
||||
+ if fieldNum <= 0 {
|
||||
+ return fmt.Errorf("proto: UpdateIPVSRequest: illegal tag %d (wire type %d)", fieldNum, wire)
|
||||
+ }
|
||||
+ switch fieldNum {
|
||||
+ case 1:
|
||||
+ if wireType != 2 {
|
||||
+ return fmt.Errorf("proto: wrong wireType = %d for field IPVSReq", wireType)
|
||||
+ }
|
||||
+ var stringLen uint64
|
||||
+ for shift := uint(0); ; shift += 7 {
|
||||
+ if shift >= 64 {
|
||||
+ return ErrIntOverflowAgent
|
||||
+ }
|
||||
+ if iNdEx >= l {
|
||||
+ return io.ErrUnexpectedEOF
|
||||
+ }
|
||||
+ b := dAtA[iNdEx]
|
||||
+ iNdEx++
|
||||
+ stringLen |= (uint64(b) & 0x7F) << shift
|
||||
+ if b < 0x80 {
|
||||
+ break
|
||||
+ }
|
||||
+ }
|
||||
+ intStringLen := int(stringLen)
|
||||
+ if intStringLen < 0 {
|
||||
+ return ErrInvalidLengthAgent
|
||||
+ }
|
||||
+ postIndex := iNdEx + intStringLen
|
||||
+ if postIndex > l {
|
||||
+ return io.ErrUnexpectedEOF
|
||||
+ }
|
||||
+ m.IPVSReq = string(dAtA[iNdEx:postIndex])
|
||||
+ iNdEx = postIndex
|
||||
+ default:
|
||||
+ iNdEx = preIndex
|
||||
+ skippy, err := skipAgent(dAtA[iNdEx:])
|
||||
+ if err != nil {
|
||||
+ return err
|
||||
+ }
|
||||
+ if skippy < 0 {
|
||||
+ return ErrInvalidLengthAgent
|
||||
+ }
|
||||
+ if (iNdEx + skippy) > l {
|
||||
+ return io.ErrUnexpectedEOF
|
||||
+ }
|
||||
+ iNdEx += skippy
|
||||
+ }
|
||||
+ }
|
||||
+
|
||||
+ if iNdEx > l {
|
||||
+ return io.ErrUnexpectedEOF
|
||||
+ }
|
||||
+ return nil
|
||||
+}
|
||||
+func (m *IPVSResponse) Unmarshal(dAtA []byte) error {
|
||||
+ l := len(dAtA)
|
||||
+ iNdEx := 0
|
||||
+ for iNdEx < l {
|
||||
+ preIndex := iNdEx
|
||||
+ var wire uint64
|
||||
+ for shift := uint(0); ; shift += 7 {
|
||||
+ if shift >= 64 {
|
||||
+ return ErrIntOverflowAgent
|
||||
+ }
|
||||
+ if iNdEx >= l {
|
||||
+ return io.ErrUnexpectedEOF
|
||||
+ }
|
||||
+ b := dAtA[iNdEx]
|
||||
+ iNdEx++
|
||||
+ wire |= (uint64(b) & 0x7F) << shift
|
||||
+ if b < 0x80 {
|
||||
+ break
|
||||
+ }
|
||||
+ }
|
||||
+ fieldNum := int32(wire >> 3)
|
||||
+ wireType := int(wire & 0x7)
|
||||
+ if wireType == 4 {
|
||||
+ return fmt.Errorf("proto: IPVSResponse: wiretype end group for non-group")
|
||||
+ }
|
||||
+ if fieldNum <= 0 {
|
||||
+ return fmt.Errorf("proto: IPVSResponse: illegal tag %d (wire type %d)", fieldNum, wire)
|
||||
+ }
|
||||
+ switch fieldNum {
|
||||
+ case 1:
|
||||
+ if wireType != 2 {
|
||||
+ return fmt.Errorf("proto: wrong wireType = %d for field IPVSRes", wireType)
|
||||
+ }
|
||||
+ var stringLen uint64
|
||||
+ for shift := uint(0); ; shift += 7 {
|
||||
+ if shift >= 64 {
|
||||
+ return ErrIntOverflowAgent
|
||||
+ }
|
||||
+ if iNdEx >= l {
|
||||
+ return io.ErrUnexpectedEOF
|
||||
+ }
|
||||
+ b := dAtA[iNdEx]
|
||||
+ iNdEx++
|
||||
+ stringLen |= (uint64(b) & 0x7F) << shift
|
||||
+ if b < 0x80 {
|
||||
+ break
|
||||
+ }
|
||||
+ }
|
||||
+ intStringLen := int(stringLen)
|
||||
+ if intStringLen < 0 {
|
||||
+ return ErrInvalidLengthAgent
|
||||
+ }
|
||||
+ postIndex := iNdEx + intStringLen
|
||||
+ if postIndex > l {
|
||||
+ return io.ErrUnexpectedEOF
|
||||
+ }
|
||||
+ m.IPVSRes = string(dAtA[iNdEx:postIndex])
|
||||
+ iNdEx = postIndex
|
||||
+ default:
|
||||
+ iNdEx = preIndex
|
||||
+ skippy, err := skipAgent(dAtA[iNdEx:])
|
||||
+ if err != nil {
|
||||
+ return err
|
||||
+ }
|
||||
+ if skippy < 0 {
|
||||
+ return ErrInvalidLengthAgent
|
||||
+ }
|
||||
+ if (iNdEx + skippy) > l {
|
||||
+ return io.ErrUnexpectedEOF
|
||||
+ }
|
||||
+ iNdEx += skippy
|
||||
+ }
|
||||
+ }
|
||||
+
|
||||
+ if iNdEx > l {
|
||||
+ return io.ErrUnexpectedEOF
|
||||
+ }
|
||||
+ return nil
|
||||
+}
|
||||
func skipAgent(dAtA []byte) (n int, err error) {
|
||||
l := len(dAtA)
|
||||
iNdEx := 0
|
||||
@@ -12893,185 +13190,189 @@ var (
|
||||
func init() { proto.RegisterFile("agent.proto", fileDescriptorAgent) }
|
||||
|
||||
var fileDescriptorAgent = []byte{
|
||||
- // 2876 bytes of a gzipped FileDescriptorProto
|
||||
- 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xc4, 0x39, 0x4b, 0x6f, 0x1c, 0xc7,
|
||||
- 0xd1, 0xd8, 0x07, 0x97, 0xbb, 0xb5, 0x2f, 0x6e, 0x93, 0xa2, 0x56, 0x2b, 0x59, 0x9f, 0x3c, 0xb6,
|
||||
- 0x65, 0xfa, 0xf3, 0xe7, 0xa5, 0x2d, 0x1b, 0x9f, 0x5f, 0x70, 0x04, 0xf1, 0x11, 0x91, 0xb1, 0x15,
|
||||
- 0x31, 0x43, 0x11, 0x4e, 0x10, 0x04, 0x83, 0xe1, 0x4c, 0x73, 0xd9, 0xe6, 0xce, 0xf4, 0xb8, 0xa7,
|
||||
- 0x87, 0xe2, 0x3a, 0x40, 0x8e, 0xc9, 0x2d, 0x97, 0x00, 0xb9, 0xe5, 0x0f, 0x04, 0xb9, 0xe5, 0x98,
|
||||
- 0x6b, 0x0e, 0x46, 0x4e, 0xf9, 0x05, 0x41, 0xe0, 0x9f, 0x90, 0x5f, 0x10, 0xf4, 0x6b, 0x1e, 0xbb,
|
||||
- 0x43, 0x1a, 0x21, 0x08, 0xe4, 0xb2, 0xe8, 0xaa, 0xae, 0xae, 0x57, 0x77, 0xd5, 0x54, 0xd5, 0x42,
|
||||
- 0xdb, 0x9d, 0xe0, 0x90, 0x8f, 0x23, 0x46, 0x39, 0x45, 0xf5, 0x09, 0x8b, 0xbc, 0x51, 0x8b, 0x7a,
|
||||
- 0x44, 0x21, 0x46, 0xff, 0x3f, 0x21, 0xfc, 0x34, 0x39, 0x1e, 0x7b, 0x34, 0xd8, 0x3c, 0x73, 0xb9,
|
||||
- 0xfb, 0x8e, 0x47, 0x43, 0xee, 0x92, 0x10, 0xb3, 0x78, 0x53, 0x1e, 0xdc, 0x8c, 0xce, 0x26, 0x9b,
|
||||
- 0x7c, 0x16, 0xe1, 0x58, 0xfd, 0xea, 0x73, 0x77, 0x27, 0x94, 0x4e, 0xa6, 0x78, 0x53, 0x42, 0xc7,
|
||||
- 0xc9, 0xc9, 0x26, 0x0e, 0x22, 0x3e, 0x53, 0x9b, 0xd6, 0x1f, 0xaa, 0xb0, 0xbe, 0xcd, 0xb0, 0xcb,
|
||||
- 0xf1, 0xb6, 0xe1, 0x66, 0xe3, 0xaf, 0x13, 0x1c, 0x73, 0xf4, 0x2a, 0x74, 0x52, 0x09, 0x0e, 0xf1,
|
||||
- 0x87, 0x95, 0x07, 0x95, 0x8d, 0x96, 0xdd, 0x4e, 0x71, 0xfb, 0x3e, 0xba, 0x0d, 0xcb, 0xf8, 0x02,
|
||||
- 0x7b, 0x62, 0xb7, 0x2a, 0x77, 0x1b, 0x02, 0xdc, 0xf7, 0xd1, 0x7b, 0xd0, 0x8e, 0x39, 0x23, 0xe1,
|
||||
- 0xc4, 0x49, 0x62, 0xcc, 0x86, 0xb5, 0x07, 0x95, 0x8d, 0xf6, 0xa3, 0x95, 0xb1, 0x30, 0x69, 0x7c,
|
||||
- 0x28, 0x37, 0x8e, 0x62, 0xcc, 0x6c, 0x88, 0xd3, 0x35, 0x7a, 0x08, 0xcb, 0x3e, 0x3e, 0x27, 0x1e,
|
||||
- 0x8e, 0x87, 0xf5, 0x07, 0xb5, 0x8d, 0xf6, 0xa3, 0x8e, 0x22, 0xdf, 0x91, 0x48, 0xdb, 0x6c, 0xa2,
|
||||
- 0xb7, 0xa0, 0x19, 0x73, 0xca, 0xdc, 0x09, 0x8e, 0x87, 0x4b, 0x92, 0xb0, 0x6b, 0xf8, 0x4a, 0xac,
|
||||
- 0x9d, 0x6e, 0xa3, 0x7b, 0x50, 0x7b, 0xbe, 0xbd, 0x3f, 0x6c, 0x48, 0xe9, 0xa0, 0xa9, 0x22, 0xec,
|
||||
- 0xd9, 0x02, 0x8d, 0x5e, 0x83, 0x6e, 0xec, 0x86, 0xfe, 0x31, 0xbd, 0x70, 0x22, 0xe2, 0x87, 0xf1,
|
||||
- 0x70, 0xf9, 0x41, 0x65, 0xa3, 0x69, 0x77, 0x34, 0xf2, 0x40, 0xe0, 0xac, 0x4f, 0xe0, 0xd6, 0x21,
|
||||
- 0x77, 0x19, 0xbf, 0x86, 0x77, 0xac, 0x23, 0x58, 0xb7, 0x71, 0x40, 0xcf, 0xaf, 0xe5, 0xda, 0x21,
|
||||
- 0x2c, 0x73, 0x12, 0x60, 0x9a, 0x70, 0xe9, 0xda, 0xae, 0x6d, 0x40, 0xeb, 0x4f, 0x15, 0x40, 0xbb,
|
||||
- 0x17, 0xd8, 0x3b, 0x60, 0xd4, 0xc3, 0x71, 0xfc, 0x5f, 0xba, 0xae, 0x37, 0x61, 0x39, 0x52, 0x0a,
|
||||
- 0x0c, 0xeb, 0x92, 0x5c, 0xdf, 0x82, 0xd1, 0xca, 0xec, 0x5a, 0x5f, 0xc1, 0xda, 0x21, 0x99, 0x84,
|
||||
- 0xee, 0xf4, 0x06, 0xf5, 0x5d, 0x87, 0x46, 0x2c, 0x79, 0x4a, 0x55, 0xbb, 0xb6, 0x86, 0xac, 0x03,
|
||||
- 0x40, 0x5f, 0xba, 0x84, 0xdf, 0x9c, 0x24, 0xeb, 0x1d, 0x58, 0x2d, 0x70, 0x8c, 0x23, 0x1a, 0xc6,
|
||||
- 0x58, 0x2a, 0xc0, 0x5d, 0x9e, 0xc4, 0x92, 0xd9, 0x92, 0xad, 0x21, 0x0b, 0xc3, 0xda, 0x17, 0x24,
|
||||
- 0x36, 0xe4, 0xf8, 0x3f, 0x51, 0x61, 0x1d, 0x1a, 0x27, 0x94, 0x05, 0x2e, 0x37, 0x1a, 0x28, 0x08,
|
||||
- 0x21, 0xa8, 0xbb, 0x6c, 0x12, 0x0f, 0x6b, 0x0f, 0x6a, 0x1b, 0x2d, 0x5b, 0xae, 0xc5, 0xab, 0x9c,
|
||||
- 0x13, 0xa3, 0xf5, 0x7a, 0x15, 0x3a, 0xda, 0xef, 0xce, 0x94, 0xc4, 0x5c, 0xca, 0xe9, 0xd8, 0x6d,
|
||||
- 0x8d, 0x13, 0x67, 0x2c, 0x0a, 0xeb, 0x47, 0x91, 0x7f, 0xcd, 0x80, 0x7f, 0x04, 0x2d, 0x86, 0x63,
|
||||
- 0x9a, 0x30, 0x11, 0xa6, 0x55, 0x79, 0xef, 0x6b, 0xea, 0xde, 0xbf, 0x20, 0x61, 0x72, 0x61, 0x9b,
|
||||
- 0x3d, 0x3b, 0x23, 0xd3, 0x21, 0xc4, 0xe3, 0xeb, 0x84, 0xd0, 0x27, 0x70, 0xeb, 0xc0, 0x4d, 0xe2,
|
||||
- 0xeb, 0xe8, 0x6a, 0x7d, 0x2a, 0xc2, 0x2f, 0x4e, 0x82, 0x6b, 0x1d, 0xfe, 0x63, 0x05, 0x9a, 0xdb,
|
||||
- 0x51, 0x72, 0x14, 0xbb, 0x13, 0x8c, 0xfe, 0x07, 0xda, 0x9c, 0x72, 0x77, 0xea, 0x24, 0x02, 0x94,
|
||||
- 0xe4, 0x75, 0x1b, 0x24, 0x4a, 0x11, 0x08, 0xb7, 0x63, 0xe6, 0x45, 0x89, 0xa6, 0xa8, 0x3e, 0xa8,
|
||||
- 0x6d, 0xd4, 0xed, 0xb6, 0xc2, 0x29, 0x92, 0x31, 0xac, 0xca, 0x3d, 0x87, 0x84, 0xce, 0x19, 0x66,
|
||||
- 0x21, 0x9e, 0x06, 0xd4, 0xc7, 0xf2, 0xfd, 0xd6, 0xed, 0x81, 0xdc, 0xda, 0x0f, 0x3f, 0x4f, 0x37,
|
||||
- 0xd0, 0xff, 0xc2, 0x20, 0xa5, 0x17, 0x41, 0x29, 0xa9, 0xeb, 0x92, 0xba, 0xaf, 0xa9, 0x8f, 0x34,
|
||||
- 0xda, 0xfa, 0x15, 0xf4, 0x5e, 0x9c, 0x32, 0xca, 0xf9, 0x94, 0x84, 0x93, 0x1d, 0x97, 0xbb, 0x22,
|
||||
- 0x7b, 0x44, 0x98, 0x11, 0xea, 0xc7, 0x5a, 0x5b, 0x03, 0xa2, 0xb7, 0x61, 0xc0, 0x15, 0x2d, 0xf6,
|
||||
- 0x1d, 0x43, 0x53, 0x95, 0x34, 0x2b, 0xe9, 0xc6, 0x81, 0x26, 0x7e, 0x03, 0x7a, 0x19, 0xb1, 0xc8,
|
||||
- 0x3f, 0x5a, 0xdf, 0x6e, 0x8a, 0x7d, 0x41, 0x02, 0x6c, 0x9d, 0x4b, 0x5f, 0xc9, 0x4b, 0x46, 0x6f,
|
||||
- 0x43, 0x2b, 0xf3, 0x43, 0x45, 0xbe, 0x90, 0x9e, 0x7a, 0x21, 0xc6, 0x9d, 0x76, 0x33, 0x75, 0xca,
|
||||
- 0x67, 0xd0, 0xe7, 0xa9, 0xe2, 0x8e, 0xef, 0x72, 0xb7, 0xf8, 0xa8, 0x8a, 0x56, 0xd9, 0x3d, 0x5e,
|
||||
- 0x80, 0xad, 0x4f, 0xa1, 0x75, 0x40, 0xfc, 0x58, 0x09, 0x1e, 0xc2, 0xb2, 0x97, 0x30, 0x86, 0x43,
|
||||
- 0x6e, 0x4c, 0xd6, 0x20, 0x5a, 0x83, 0xa5, 0x29, 0x09, 0x08, 0xd7, 0x66, 0x2a, 0xc0, 0xa2, 0x00,
|
||||
- 0xcf, 0x70, 0x40, 0xd9, 0x4c, 0x3a, 0x6c, 0x0d, 0x96, 0xf2, 0x97, 0xab, 0x00, 0x74, 0x17, 0x5a,
|
||||
- 0x81, 0x7b, 0x91, 0x5e, 0xaa, 0xd8, 0x69, 0x06, 0xee, 0x85, 0x52, 0x7e, 0x08, 0xcb, 0x27, 0x2e,
|
||||
- 0x99, 0x7a, 0x21, 0xd7, 0x5e, 0x31, 0x60, 0x26, 0xb0, 0x9e, 0x17, 0xf8, 0xd7, 0x2a, 0xb4, 0x95,
|
||||
- 0x44, 0xa5, 0xf0, 0x1a, 0x2c, 0x79, 0xae, 0x77, 0x9a, 0x8a, 0x94, 0x00, 0x7a, 0x68, 0x14, 0xa9,
|
||||
- 0xe6, 0x93, 0x70, 0xa6, 0xa9, 0x51, 0x6d, 0x13, 0x20, 0x7e, 0xe9, 0x46, 0x5a, 0xb7, 0xda, 0x25,
|
||||
- 0xc4, 0x2d, 0x41, 0xa3, 0xd4, 0x7d, 0x1f, 0x3a, 0xea, 0xdd, 0xe9, 0x23, 0xf5, 0x4b, 0x8e, 0xb4,
|
||||
- 0x15, 0x95, 0x3a, 0xf4, 0x1a, 0x74, 0x93, 0x18, 0x3b, 0xa7, 0x04, 0x33, 0x97, 0x79, 0xa7, 0xb3,
|
||||
- 0xe1, 0x92, 0xfa, 0x46, 0x26, 0x31, 0xde, 0x33, 0x38, 0xf4, 0x08, 0x96, 0x44, 0xfa, 0x8b, 0x87,
|
||||
- 0x0d, 0xf9, 0x39, 0xbe, 0x97, 0x67, 0x29, 0x4d, 0x1d, 0xcb, 0xdf, 0xdd, 0x90, 0xb3, 0x99, 0xad,
|
||||
- 0x48, 0x47, 0x1f, 0x01, 0x64, 0x48, 0xb4, 0x02, 0xb5, 0x33, 0x3c, 0xd3, 0x71, 0x28, 0x96, 0xc2,
|
||||
- 0x39, 0xe7, 0xee, 0x34, 0x31, 0x5e, 0x57, 0xc0, 0x27, 0xd5, 0x8f, 0x2a, 0x96, 0x07, 0xfd, 0xad,
|
||||
- 0xe9, 0x19, 0xa1, 0xb9, 0xe3, 0x6b, 0xb0, 0x14, 0xb8, 0x5f, 0x51, 0x66, 0x3c, 0x29, 0x01, 0x89,
|
||||
- 0x25, 0x21, 0x65, 0x86, 0x85, 0x04, 0x50, 0x0f, 0xaa, 0x34, 0x92, 0xfe, 0x6a, 0xd9, 0x55, 0x1a,
|
||||
- 0x65, 0x82, 0xea, 0x39, 0x41, 0xd6, 0x3f, 0xea, 0x00, 0x99, 0x14, 0x64, 0xc3, 0x88, 0x50, 0x27,
|
||||
- 0xc6, 0x4c, 0x94, 0x20, 0xce, 0xf1, 0x8c, 0xe3, 0xd8, 0x61, 0xd8, 0x4b, 0x58, 0x4c, 0xce, 0xc5,
|
||||
- 0xfd, 0x09, 0xb3, 0x6f, 0x29, 0xb3, 0xe7, 0x74, 0xb3, 0x6f, 0x13, 0x7a, 0xa8, 0xce, 0x6d, 0x89,
|
||||
- 0x63, 0xb6, 0x39, 0x85, 0xf6, 0xe1, 0x56, 0xc6, 0xd3, 0xcf, 0xb1, 0xab, 0x5e, 0xc5, 0x6e, 0x35,
|
||||
- 0x65, 0xe7, 0x67, 0xac, 0x76, 0x61, 0x95, 0x50, 0xe7, 0xeb, 0x04, 0x27, 0x05, 0x46, 0xb5, 0xab,
|
||||
- 0x18, 0x0d, 0x08, 0xfd, 0x89, 0x3c, 0x90, 0xb1, 0x39, 0x80, 0x3b, 0x39, 0x2b, 0x45, 0xb8, 0xe7,
|
||||
- 0x98, 0xd5, 0xaf, 0x62, 0xb6, 0x9e, 0x6a, 0x25, 0xf2, 0x41, 0xc6, 0xf1, 0x47, 0xb0, 0x4e, 0xa8,
|
||||
- 0xf3, 0xd2, 0x25, 0x7c, 0x9e, 0xdd, 0xd2, 0xf7, 0x18, 0x29, 0x3e, 0xba, 0x45, 0x5e, 0xca, 0xc8,
|
||||
- 0x00, 0xb3, 0x49, 0xc1, 0xc8, 0xc6, 0xf7, 0x18, 0xf9, 0x4c, 0x1e, 0xc8, 0xd8, 0x3c, 0x81, 0x01,
|
||||
- 0xa1, 0xf3, 0xda, 0x2c, 0x5f, 0xc5, 0xa4, 0x4f, 0x68, 0x51, 0x93, 0x2d, 0x18, 0xc4, 0xd8, 0xe3,
|
||||
- 0x94, 0xe5, 0x1f, 0x41, 0xf3, 0x2a, 0x16, 0x2b, 0x9a, 0x3e, 0xe5, 0x61, 0xfd, 0x1c, 0x3a, 0x7b,
|
||||
- 0xc9, 0x04, 0xf3, 0xe9, 0x71, 0x9a, 0x0c, 0x6e, 0x2c, 0xff, 0x58, 0xff, 0xaa, 0x42, 0x7b, 0x7b,
|
||||
- 0xc2, 0x68, 0x12, 0x15, 0x72, 0xb2, 0x0a, 0xd2, 0xf9, 0x9c, 0x2c, 0x49, 0x64, 0x4e, 0x56, 0xc4,
|
||||
- 0x1f, 0x40, 0x27, 0x90, 0xa1, 0xab, 0xe9, 0x55, 0x1e, 0x1a, 0x2c, 0x04, 0xb5, 0xdd, 0x0e, 0x72,
|
||||
- 0xc9, 0x6c, 0x0c, 0x10, 0x11, 0x3f, 0xd6, 0x67, 0x54, 0x3a, 0xea, 0xeb, 0x8a, 0xd0, 0xa4, 0x68,
|
||||
- 0xbb, 0x15, 0xa5, 0xd9, 0xfa, 0x3d, 0x68, 0x1f, 0x0b, 0x27, 0xe9, 0x03, 0x85, 0x64, 0x94, 0x79,
|
||||
- 0xcf, 0x86, 0xe3, 0x2c, 0x08, 0xf7, 0xa0, 0x7b, 0xaa, 0x5c, 0xa6, 0x0f, 0xa9, 0x37, 0xf4, 0x9a,
|
||||
- 0xb6, 0x24, 0xb3, 0x77, 0x9c, 0xf7, 0xac, 0xba, 0x80, 0xce, 0x69, 0x0e, 0x35, 0x3a, 0x84, 0xc1,
|
||||
- 0x02, 0x49, 0x49, 0x0e, 0xda, 0xc8, 0xe7, 0xa0, 0xf6, 0x23, 0xa4, 0x04, 0xe5, 0x4f, 0xe6, 0xf3,
|
||||
- 0xd2, 0x6f, 0xab, 0xd0, 0xf9, 0x31, 0xe6, 0x2f, 0x29, 0x3b, 0x53, 0xfa, 0x22, 0xa8, 0x87, 0x6e,
|
||||
- 0x80, 0x35, 0x47, 0xb9, 0x46, 0x77, 0xa0, 0xc9, 0x2e, 0x54, 0x02, 0xd1, 0xf7, 0xb9, 0xcc, 0x2e,
|
||||
- 0x64, 0x62, 0x40, 0xaf, 0x00, 0xb0, 0x0b, 0x27, 0x72, 0xbd, 0x33, 0xac, 0x3d, 0x58, 0xb7, 0x5b,
|
||||
- 0xec, 0xe2, 0x40, 0x21, 0xc4, 0x53, 0x60, 0x17, 0x0e, 0x66, 0x8c, 0xb2, 0x58, 0xe7, 0xaa, 0x26,
|
||||
- 0xbb, 0xd8, 0x95, 0xb0, 0x3e, 0xeb, 0x33, 0x1a, 0x45, 0xd8, 0x97, 0x39, 0x5a, 0x9e, 0xdd, 0x51,
|
||||
- 0x08, 0x21, 0x95, 0x1b, 0xa9, 0x0d, 0x25, 0x95, 0x67, 0x52, 0x79, 0x26, 0x75, 0x59, 0x9d, 0xe4,
|
||||
- 0x79, 0xa9, 0x3c, 0x95, 0xda, 0x54, 0x52, 0x79, 0x4e, 0x2a, 0xcf, 0xa4, 0xb6, 0xcc, 0x59, 0x2d,
|
||||
- 0xd5, 0xfa, 0x4d, 0x05, 0xd6, 0xe7, 0x0b, 0x3f, 0x5d, 0xa6, 0x7e, 0x00, 0x1d, 0x4f, 0xde, 0x57,
|
||||
- 0xe1, 0x4d, 0x0e, 0x16, 0x6e, 0xd2, 0x6e, 0x7b, 0xb9, 0x67, 0xfc, 0x21, 0x74, 0x43, 0xe5, 0xe0,
|
||||
- 0xf4, 0x69, 0xd6, 0xb2, 0x7b, 0xc9, 0xfb, 0xde, 0xee, 0x84, 0x39, 0xc8, 0xf2, 0x01, 0x7d, 0xc9,
|
||||
- 0x08, 0xc7, 0x87, 0x9c, 0x61, 0x37, 0xb8, 0x89, 0x06, 0x04, 0x41, 0x5d, 0x56, 0x2b, 0x35, 0x59,
|
||||
- 0x5f, 0xcb, 0xb5, 0xf5, 0x26, 0xac, 0x16, 0xa4, 0x68, 0x5b, 0x57, 0xa0, 0x36, 0xc5, 0xa1, 0xe4,
|
||||
- 0xde, 0xb5, 0xc5, 0xd2, 0x72, 0x61, 0x60, 0x63, 0xd7, 0xbf, 0x39, 0x6d, 0xb4, 0x88, 0x5a, 0x26,
|
||||
- 0x62, 0x03, 0x50, 0x5e, 0x84, 0x56, 0xc5, 0x68, 0x5d, 0xc9, 0x69, 0xfd, 0x1c, 0x06, 0xdb, 0x53,
|
||||
- 0x1a, 0xe3, 0x43, 0xee, 0x93, 0xf0, 0x26, 0x3a, 0xa6, 0x5f, 0xc2, 0xea, 0x0b, 0x3e, 0xfb, 0x52,
|
||||
- 0x30, 0x8b, 0xc9, 0x37, 0xf8, 0x86, 0xec, 0x63, 0xf4, 0xa5, 0xb1, 0x8f, 0xd1, 0x97, 0xa2, 0x59,
|
||||
- 0xf2, 0xe8, 0x34, 0x09, 0x42, 0x19, 0x0a, 0x5d, 0x5b, 0x43, 0xd6, 0x16, 0x74, 0x54, 0x0d, 0xfd,
|
||||
- 0x8c, 0xfa, 0xc9, 0x14, 0x97, 0xc6, 0xe0, 0x7d, 0x80, 0xc8, 0x65, 0x6e, 0x80, 0x39, 0x66, 0xea,
|
||||
- 0x0d, 0xb5, 0xec, 0x1c, 0xc6, 0xfa, 0x7d, 0x15, 0xd6, 0xd4, 0x48, 0xe4, 0x50, 0x4d, 0x02, 0x8c,
|
||||
- 0x09, 0x23, 0x68, 0x9e, 0xd2, 0x98, 0xe7, 0x18, 0xa6, 0xb0, 0x50, 0xd1, 0x0f, 0x0d, 0x37, 0xb1,
|
||||
- 0x2c, 0xcc, 0x29, 0x6a, 0x57, 0xcf, 0x29, 0x16, 0x26, 0x11, 0xf5, 0xc5, 0x49, 0x84, 0x88, 0x36,
|
||||
- 0x43, 0x44, 0x54, 0x8c, 0xb7, 0xec, 0x96, 0xc6, 0xec, 0xfb, 0xe8, 0x21, 0xf4, 0x27, 0x42, 0x4b,
|
||||
- 0xe7, 0x94, 0xd2, 0x33, 0x27, 0x72, 0xf9, 0xa9, 0x0c, 0xf5, 0x96, 0xdd, 0x95, 0xe8, 0x3d, 0x4a,
|
||||
- 0xcf, 0x0e, 0x5c, 0x7e, 0x8a, 0x3e, 0x86, 0x9e, 0x2e, 0x03, 0x03, 0xe9, 0xa2, 0x58, 0x7f, 0xfc,
|
||||
- 0x74, 0x14, 0xe5, 0xbd, 0x67, 0x77, 0xcf, 0x72, 0x50, 0x6c, 0xdd, 0x86, 0x5b, 0x3b, 0x38, 0xe6,
|
||||
- 0x8c, 0xce, 0x8a, 0x8e, 0xb1, 0x7e, 0x00, 0xb0, 0x1f, 0x72, 0xcc, 0x4e, 0x5c, 0x0f, 0xc7, 0xe8,
|
||||
- 0xdd, 0x3c, 0xa4, 0x8b, 0xa3, 0x95, 0xb1, 0x9a, 0x48, 0xa5, 0x1b, 0x76, 0x8e, 0xc6, 0x1a, 0x43,
|
||||
- 0xc3, 0xa6, 0x89, 0x48, 0x47, 0xaf, 0x9b, 0x95, 0x3e, 0xd7, 0xd1, 0xe7, 0x24, 0xd2, 0xd6, 0x7b,
|
||||
- 0xd6, 0x9e, 0x69, 0x61, 0x33, 0x76, 0xfa, 0x8a, 0xc6, 0xd0, 0x22, 0x06, 0xa7, 0xb3, 0xca, 0xa2,
|
||||
- 0xe8, 0x8c, 0xc4, 0xfa, 0x19, 0xac, 0x2a, 0x4e, 0x8a, 0xb3, 0x61, 0xf3, 0x3a, 0x34, 0x98, 0x51,
|
||||
- 0xa3, 0x92, 0x8d, 0xa2, 0x34, 0x91, 0xde, 0x43, 0xf7, 0x84, 0x30, 0x8f, 0xe1, 0x40, 0xf4, 0x1c,
|
||||
- 0x55, 0x79, 0x65, 0x19, 0x42, 0x78, 0x4b, 0xf4, 0xdb, 0x99, 0x99, 0xc6, 0x5b, 0xab, 0x30, 0x10,
|
||||
- 0x1b, 0x05, 0x89, 0xd6, 0x2f, 0x60, 0xf5, 0x79, 0x38, 0x25, 0x21, 0xde, 0x3e, 0x38, 0x7a, 0x86,
|
||||
- 0xd3, 0xac, 0x80, 0xa0, 0x2e, 0xaa, 0x27, 0xa9, 0x46, 0xd3, 0x96, 0x6b, 0x11, 0x26, 0xe1, 0xb1,
|
||||
- 0xe3, 0x45, 0x49, 0xac, 0x27, 0x43, 0x8d, 0xf0, 0x78, 0x3b, 0x4a, 0x62, 0x91, 0xe6, 0xc5, 0x67,
|
||||
- 0x9e, 0x86, 0xd3, 0x99, 0x8c, 0x95, 0xa6, 0xbd, 0xec, 0x45, 0xc9, 0xf3, 0x70, 0x3a, 0xb3, 0xfe,
|
||||
- 0x4f, 0xf6, 0xc2, 0x18, 0xfb, 0xb6, 0x1b, 0xfa, 0x34, 0xd8, 0xc1, 0xe7, 0x39, 0x09, 0x69, 0xdf,
|
||||
- 0x65, 0x72, 0xc2, 0xb7, 0x15, 0xe8, 0x3c, 0x99, 0xe0, 0x90, 0xef, 0x60, 0xee, 0x92, 0xa9, 0xec,
|
||||
- 0xad, 0xce, 0x31, 0x8b, 0x09, 0x0d, 0xf5, 0xc3, 0x37, 0xa0, 0x68, 0x8d, 0x49, 0x48, 0xb8, 0xe3,
|
||||
- 0xbb, 0x38, 0xa0, 0xa1, 0xf6, 0x02, 0x08, 0xd4, 0x8e, 0xc4, 0xa0, 0x37, 0xa1, 0xaf, 0x26, 0x77,
|
||||
- 0xce, 0xa9, 0x1b, 0xfa, 0x53, 0x11, 0x72, 0x6a, 0x92, 0xd1, 0x53, 0xe8, 0x3d, 0x8d, 0x45, 0x6f,
|
||||
- 0xc1, 0x8a, 0x0e, 0x88, 0x8c, 0xb2, 0x2e, 0x29, 0xfb, 0x1a, 0x5f, 0x20, 0x4d, 0xa2, 0x88, 0x32,
|
||||
- 0x1e, 0x3b, 0x31, 0xf6, 0x3c, 0x1a, 0x44, 0xba, 0x31, 0xe9, 0x1b, 0xfc, 0xa1, 0x42, 0x5b, 0x13,
|
||||
- 0x58, 0x7d, 0x2a, 0xec, 0xd4, 0x96, 0x64, 0x17, 0xdc, 0x0b, 0x70, 0xe0, 0x1c, 0x4f, 0xa9, 0x77,
|
||||
- 0xe6, 0x88, 0x34, 0xa5, 0x3d, 0x2c, 0x4a, 0x9f, 0x2d, 0x81, 0x3c, 0x24, 0xdf, 0xc8, 0x1e, 0x5c,
|
||||
- 0x50, 0x9d, 0x52, 0x1e, 0x4d, 0x93, 0x89, 0x13, 0x31, 0x7a, 0x8c, 0xb5, 0x89, 0xfd, 0x00, 0x07,
|
||||
- 0x7b, 0x0a, 0x7f, 0x20, 0xd0, 0xd6, 0x5f, 0x2a, 0xb0, 0x56, 0x94, 0xa4, 0x93, 0xee, 0x26, 0xac,
|
||||
- 0x15, 0x45, 0xe9, 0x0f, 0xb1, 0x2a, 0xf4, 0x06, 0x79, 0x81, 0xea, 0x93, 0xfc, 0x21, 0x74, 0xe5,
|
||||
- 0x38, 0xd7, 0xf1, 0x15, 0xa7, 0x62, 0xf9, 0x91, 0xbf, 0x17, 0xbb, 0xe3, 0xe6, 0x6f, 0xe9, 0x63,
|
||||
- 0xb8, 0xa3, 0xcd, 0x77, 0x16, 0xd5, 0x56, 0x0f, 0x62, 0x5d, 0x13, 0x3c, 0x9b, 0xd3, 0xfe, 0x0b,
|
||||
- 0x18, 0x66, 0xa8, 0xad, 0x99, 0x44, 0x1a, 0x5f, 0xbd, 0x0b, 0xab, 0x73, 0xc6, 0x3e, 0xf1, 0x7d,
|
||||
- 0x26, 0x03, 0xb4, 0x6e, 0x97, 0x6d, 0x59, 0x8f, 0xe1, 0xf6, 0x21, 0xe6, 0xca, 0x1b, 0x2e, 0xd7,
|
||||
- 0x3d, 0x81, 0x62, 0xb6, 0x02, 0xb5, 0x43, 0xec, 0x49, 0xe3, 0x6b, 0xb6, 0x58, 0x8a, 0x07, 0x78,
|
||||
- 0x14, 0x63, 0x4f, 0x5a, 0x59, 0xb3, 0xe5, 0xda, 0xfa, 0x73, 0x05, 0x96, 0x75, 0x9a, 0x14, 0xa9,
|
||||
- 0xde, 0x67, 0xe4, 0x1c, 0x33, 0xfd, 0xf4, 0x34, 0x84, 0xde, 0x80, 0x9e, 0x5a, 0x39, 0x34, 0xe2,
|
||||
- 0x84, 0xa6, 0xc9, 0xb7, 0xab, 0xb0, 0xcf, 0x15, 0x52, 0x4e, 0xea, 0xe4, 0x20, 0x4a, 0xf7, 0x7c,
|
||||
- 0x1a, 0x92, 0xe3, 0xb6, 0x58, 0x64, 0x06, 0x99, 0x6c, 0x5b, 0xb6, 0x86, 0xc4, 0x53, 0x37, 0xfc,
|
||||
- 0x96, 0x24, 0x3f, 0x03, 0x8a, 0xa7, 0x1e, 0xd0, 0x24, 0xe4, 0x4e, 0x44, 0x49, 0xc8, 0x75, 0x76,
|
||||
- 0x05, 0x89, 0x3a, 0x10, 0x18, 0xeb, 0xd7, 0x15, 0x68, 0xa8, 0x69, 0xb5, 0xe8, 0x32, 0xd3, 0x6f,
|
||||
- 0x5c, 0x95, 0xc8, 0x7a, 0x41, 0xca, 0x52, 0xdf, 0x35, 0xb9, 0x16, 0x71, 0x7c, 0x1e, 0xa8, 0x4c,
|
||||
- 0xad, 0x55, 0x3b, 0x0f, 0x64, 0x8a, 0x7e, 0x03, 0x7a, 0xd9, 0xa7, 0x52, 0xee, 0x2b, 0x15, 0xbb,
|
||||
- 0x29, 0x56, 0x92, 0x5d, 0xaa, 0xa9, 0xf5, 0x53, 0xd1, 0x5c, 0xa7, 0x93, 0xda, 0x15, 0xa8, 0x25,
|
||||
- 0xa9, 0x32, 0x62, 0x29, 0x30, 0x93, 0xf4, 0x23, 0x2b, 0x96, 0xe8, 0x21, 0xf4, 0x5c, 0xdf, 0x27,
|
||||
- 0xe2, 0xb8, 0x3b, 0x7d, 0x4a, 0xfc, 0x34, 0x48, 0x8b, 0x58, 0xeb, 0x6f, 0x15, 0xe8, 0x6f, 0xd3,
|
||||
- 0x68, 0xf6, 0x43, 0x32, 0xc5, 0xb9, 0x0c, 0x22, 0x95, 0xd4, 0xdf, 0x58, 0xb1, 0x16, 0x75, 0xe3,
|
||||
- 0x09, 0x99, 0x62, 0x15, 0x5a, 0xea, 0x66, 0x9b, 0x02, 0x21, 0xc3, 0xca, 0x6c, 0xa6, 0x03, 0xb0,
|
||||
- 0xae, 0xda, 0x7c, 0x46, 0x7d, 0x59, 0x21, 0xfb, 0x84, 0x39, 0xe9, 0xb8, 0xab, 0x6b, 0x2f, 0xfb,
|
||||
- 0x84, 0xc9, 0x2d, 0x6d, 0xc8, 0x92, 0x9c, 0xb8, 0xe6, 0x0d, 0x69, 0x28, 0x8c, 0x30, 0x64, 0x1d,
|
||||
- 0x1a, 0xf4, 0xe4, 0x24, 0xc6, 0x5c, 0xd6, 0xb2, 0x35, 0x5b, 0x43, 0x69, 0x9a, 0x6b, 0xe6, 0xd2,
|
||||
- 0xdc, 0x2d, 0x58, 0x95, 0xb3, 0xfd, 0x17, 0xcc, 0xf5, 0x48, 0x38, 0x31, 0xa9, 0x78, 0x0d, 0xd0,
|
||||
- 0x21, 0xa7, 0x51, 0x11, 0xfb, 0xe8, 0x77, 0x2b, 0x3a, 0x27, 0xea, 0x46, 0x17, 0x3d, 0x85, 0xfe,
|
||||
- 0xdc, 0x1f, 0x27, 0x48, 0x4f, 0x3e, 0xca, 0xff, 0x4f, 0x19, 0xad, 0x8f, 0xd5, 0x1f, 0x31, 0x63,
|
||||
- 0xf3, 0x47, 0xcc, 0x78, 0x37, 0x88, 0xf8, 0x0c, 0xed, 0x42, 0xaf, 0xf8, 0x17, 0x03, 0xba, 0x6b,
|
||||
- 0x0a, 0x85, 0x92, 0x3f, 0x1e, 0x2e, 0x65, 0xf3, 0x14, 0xfa, 0x73, 0xff, 0x36, 0x18, 0x7d, 0xca,
|
||||
- 0xff, 0x84, 0xb8, 0x94, 0xd1, 0x63, 0x68, 0xe7, 0xfe, 0x5e, 0x40, 0x43, 0xc5, 0x64, 0xf1, 0x1f,
|
||||
- 0x87, 0x4b, 0x19, 0x6c, 0x43, 0xb7, 0x30, 0xf1, 0x47, 0x23, 0x6d, 0x4f, 0xc9, 0xdf, 0x00, 0x97,
|
||||
- 0x32, 0xd9, 0x82, 0x76, 0x6e, 0xf0, 0x6e, 0xb4, 0x58, 0x9c, 0xee, 0x8f, 0xee, 0x94, 0xec, 0xe8,
|
||||
- 0xd4, 0xbb, 0x07, 0xdd, 0xc2, 0x98, 0xdc, 0x28, 0x52, 0x36, 0xa2, 0x1f, 0xdd, 0x2d, 0xdd, 0xd3,
|
||||
- 0x9c, 0x9e, 0x42, 0x7f, 0x6e, 0x68, 0x6e, 0x9c, 0x5b, 0x3e, 0x4b, 0xbf, 0xd4, 0xac, 0xcf, 0xe5,
|
||||
- 0x65, 0xe7, 0x7a, 0xa2, 0xdc, 0x65, 0x2f, 0x8e, 0xc8, 0x47, 0xf7, 0xca, 0x37, 0xb5, 0x56, 0xbb,
|
||||
- 0xd0, 0x2b, 0x4e, 0xc7, 0x0d, 0xb3, 0xd2, 0x99, 0xf9, 0xd5, 0x2f, 0xa7, 0x30, 0x28, 0xcf, 0x5e,
|
||||
- 0x4e, 0xd9, 0xfc, 0xfc, 0x52, 0x46, 0x4f, 0x00, 0x74, 0x07, 0xe4, 0x93, 0x30, 0xbd, 0xb2, 0x85,
|
||||
- 0xce, 0x2b, 0xbd, 0xb2, 0x92, 0x6e, 0xe9, 0x31, 0x80, 0x6a, 0x5c, 0x7c, 0x9a, 0x70, 0x74, 0xdb,
|
||||
- 0xa8, 0x31, 0xd7, 0x2d, 0x8d, 0x86, 0x8b, 0x1b, 0x0b, 0x0c, 0x30, 0x63, 0xd7, 0x61, 0xf0, 0x19,
|
||||
- 0x40, 0xd6, 0x10, 0x19, 0x06, 0x0b, 0x2d, 0xd2, 0x15, 0x3e, 0xe8, 0xe4, 0xdb, 0x1f, 0xa4, 0x6d,
|
||||
- 0x2d, 0x69, 0x89, 0xae, 0x60, 0xd1, 0x9f, 0x2b, 0x6f, 0x8b, 0x8f, 0x6d, 0xbe, 0xea, 0x1d, 0x2d,
|
||||
- 0x94, 0xb8, 0xe8, 0x43, 0xe8, 0xe4, 0xeb, 0x5a, 0xa3, 0x45, 0x49, 0xad, 0x3b, 0x2a, 0xd4, 0xb6,
|
||||
- 0xe8, 0x31, 0xf4, 0x8a, 0x55, 0x2b, 0xca, 0xc5, 0xc5, 0x42, 0x2d, 0x3b, 0xd2, 0x13, 0x9b, 0x1c,
|
||||
- 0xf9, 0xfb, 0x00, 0x59, 0x75, 0x6b, 0xdc, 0xb7, 0x50, 0xef, 0xce, 0x49, 0x7d, 0x02, 0x9d, 0x7c,
|
||||
- 0x26, 0x36, 0xea, 0x96, 0x64, 0xe7, 0xab, 0xb2, 0x56, 0x2e, 0x6b, 0x9b, 0xc7, 0xb7, 0x98, 0xc8,
|
||||
- 0xaf, 0xca, 0x5a, 0x85, 0xae, 0xcf, 0x24, 0x8b, 0xb2, 0x56, 0xf0, 0xaa, 0x5c, 0x5e, 0x6c, 0x91,
|
||||
- 0x8c, 0xfb, 0x4a, 0x1b, 0xa7, 0xab, 0x1e, 0x51, 0xbe, 0x1b, 0x30, 0xfe, 0x28, 0xe9, 0x10, 0xbe,
|
||||
- 0x27, 0xa8, 0xf3, 0x15, 0x7f, 0x2e, 0xa8, 0x4b, 0x1a, 0x81, 0x4b, 0x19, 0xed, 0x41, 0xff, 0xa9,
|
||||
- 0x29, 0xe6, 0x74, 0xa1, 0xa9, 0xd5, 0x29, 0x29, 0xac, 0x47, 0xa3, 0xb2, 0x2d, 0x1d, 0x59, 0x9f,
|
||||
- 0xc3, 0x60, 0xa1, 0xc8, 0x44, 0xf7, 0xd3, 0xc1, 0x62, 0x69, 0xf5, 0x79, 0xa9, 0x5a, 0xfb, 0xb0,
|
||||
- 0x32, 0x5f, 0x63, 0xa2, 0x57, 0xf4, 0xa5, 0x97, 0xd7, 0x9e, 0x97, 0xb2, 0xfa, 0x18, 0x9a, 0xa6,
|
||||
- 0xa6, 0x41, 0x7a, 0x80, 0x3b, 0x57, 0xe3, 0x5c, 0x76, 0x74, 0xab, 0xf3, 0xed, 0x77, 0xf7, 0x2b,
|
||||
- 0x7f, 0xff, 0xee, 0x7e, 0xe5, 0x9f, 0xdf, 0xdd, 0xaf, 0x1c, 0x37, 0xe4, 0xee, 0xfb, 0xff, 0x0e,
|
||||
- 0x00, 0x00, 0xff, 0xff, 0x8d, 0x89, 0xaa, 0x73, 0xc8, 0x21, 0x00, 0x00,
|
||||
+ // 2931 bytes of a gzipped FileDescriptorProto
|
||||
+ 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xc4, 0x39, 0xcb, 0x6e, 0x1c, 0xc7,
|
||||
+ 0xb5, 0x98, 0x07, 0x87, 0x33, 0x67, 0x5e, 0x9c, 0x22, 0x45, 0x8d, 0x46, 0xb2, 0xae, 0xdc, 0xb6,
|
||||
+ 0x65, 0xfa, 0xfa, 0x7a, 0x68, 0xcb, 0xc6, 0xf5, 0x0b, 0xbe, 0x82, 0x48, 0xe9, 0x8a, 0x8c, 0xad,
|
||||
+ 0x88, 0xe9, 0x91, 0xe2, 0x04, 0x41, 0xd0, 0x68, 0x76, 0x97, 0x86, 0x65, 0x4e, 0x77, 0xb5, 0xab,
|
||||
+ 0xaa, 0x29, 0x8e, 0x03, 0x64, 0x99, 0xec, 0xb2, 0xcc, 0x2e, 0x3f, 0x10, 0x64, 0x97, 0x65, 0xb6,
|
||||
+ 0x59, 0x18, 0x59, 0x05, 0xf9, 0x80, 0x20, 0xf0, 0x27, 0xe4, 0x0b, 0x82, 0x7a, 0xf5, 0x63, 0x66,
|
||||
+ 0x48, 0x23, 0x82, 0x80, 0x6c, 0x1a, 0x75, 0x4e, 0x9d, 0x3a, 0xaf, 0xaa, 0x3a, 0x75, 0xce, 0x69,
|
||||
+ 0x68, 0xfb, 0x53, 0x1c, 0x8b, 0x71, 0xc2, 0xa8, 0xa0, 0xa8, 0x3e, 0x65, 0x49, 0x30, 0x6a, 0xd1,
|
||||
+ 0x80, 0x68, 0xc4, 0xe8, 0x7f, 0xa7, 0x44, 0x9c, 0xa4, 0xc7, 0xe3, 0x80, 0x46, 0xbb, 0xa7, 0xbe,
|
||||
+ 0xf0, 0xdf, 0x09, 0x68, 0x2c, 0x7c, 0x12, 0x63, 0xc6, 0x77, 0xd5, 0xc2, 0xdd, 0xe4, 0x74, 0xba,
|
||||
+ 0x2b, 0xe6, 0x09, 0xe6, 0xfa, 0x6b, 0xd6, 0x5d, 0x9f, 0x52, 0x3a, 0x9d, 0xe1, 0x5d, 0x05, 0x1d,
|
||||
+ 0xa7, 0xcf, 0x76, 0x71, 0x94, 0x88, 0xb9, 0x9e, 0x74, 0x7e, 0x57, 0x85, 0xed, 0x7d, 0x86, 0x7d,
|
||||
+ 0x81, 0xf7, 0x2d, 0x37, 0x17, 0x7f, 0x9d, 0x62, 0x2e, 0xd0, 0xab, 0xd0, 0xc9, 0x24, 0x78, 0x24,
|
||||
+ 0x1c, 0x56, 0x6e, 0x55, 0x76, 0x5a, 0x6e, 0x3b, 0xc3, 0x1d, 0x86, 0xe8, 0x2a, 0xac, 0xe3, 0x73,
|
||||
+ 0x1c, 0xc8, 0xd9, 0xaa, 0x9a, 0x6d, 0x48, 0xf0, 0x30, 0x44, 0xef, 0x41, 0x9b, 0x0b, 0x46, 0xe2,
|
||||
+ 0xa9, 0x97, 0x72, 0xcc, 0x86, 0xb5, 0x5b, 0x95, 0x9d, 0xf6, 0x9d, 0x8d, 0xb1, 0x34, 0x69, 0x3c,
|
||||
+ 0x51, 0x13, 0x4f, 0x39, 0x66, 0x2e, 0xf0, 0x6c, 0x8c, 0x6e, 0xc3, 0x7a, 0x88, 0xcf, 0x48, 0x80,
|
||||
+ 0xf9, 0xb0, 0x7e, 0xab, 0xb6, 0xd3, 0xbe, 0xd3, 0xd1, 0xe4, 0xf7, 0x15, 0xd2, 0xb5, 0x93, 0xe8,
|
||||
+ 0x2d, 0x68, 0x72, 0x41, 0x99, 0x3f, 0xc5, 0x7c, 0xb8, 0xa6, 0x08, 0xbb, 0x96, 0xaf, 0xc2, 0xba,
|
||||
+ 0xd9, 0x34, 0xba, 0x01, 0xb5, 0xc7, 0xfb, 0x87, 0xc3, 0x86, 0x92, 0x0e, 0x86, 0x2a, 0xc1, 0x81,
|
||||
+ 0x2b, 0xd1, 0xe8, 0x35, 0xe8, 0x72, 0x3f, 0x0e, 0x8f, 0xe9, 0xb9, 0x97, 0x90, 0x30, 0xe6, 0xc3,
|
||||
+ 0xf5, 0x5b, 0x95, 0x9d, 0xa6, 0xdb, 0x31, 0xc8, 0x23, 0x89, 0x73, 0x3e, 0x81, 0x2b, 0x13, 0xe1,
|
||||
+ 0x33, 0xf1, 0x02, 0xde, 0x71, 0x9e, 0xc2, 0xb6, 0x8b, 0x23, 0x7a, 0xf6, 0x42, 0xae, 0x1d, 0xc2,
|
||||
+ 0xba, 0x20, 0x11, 0xa6, 0xa9, 0x50, 0xae, 0xed, 0xba, 0x16, 0x74, 0xfe, 0x50, 0x01, 0xf4, 0xe0,
|
||||
+ 0x1c, 0x07, 0x47, 0x8c, 0x06, 0x98, 0xf3, 0xff, 0xd0, 0x76, 0xbd, 0x09, 0xeb, 0x89, 0x56, 0x60,
|
||||
+ 0x58, 0x57, 0xe4, 0x66, 0x17, 0xac, 0x56, 0x76, 0xd6, 0xf9, 0x0a, 0xb6, 0x26, 0x64, 0x1a, 0xfb,
|
||||
+ 0xb3, 0x97, 0xa8, 0xef, 0x36, 0x34, 0xb8, 0xe2, 0xa9, 0x54, 0xed, 0xba, 0x06, 0x72, 0x8e, 0x00,
|
||||
+ 0x7d, 0xe9, 0x13, 0xf1, 0xf2, 0x24, 0x39, 0xef, 0xc0, 0x66, 0x89, 0x23, 0x4f, 0x68, 0xcc, 0xb1,
|
||||
+ 0x52, 0x40, 0xf8, 0x22, 0xe5, 0x8a, 0xd9, 0x9a, 0x6b, 0x20, 0x07, 0xc3, 0xd6, 0x17, 0x84, 0x5b,
|
||||
+ 0x72, 0xfc, 0xef, 0xa8, 0xb0, 0x0d, 0x8d, 0x67, 0x94, 0x45, 0xbe, 0xb0, 0x1a, 0x68, 0x08, 0x21,
|
||||
+ 0xa8, 0xfb, 0x6c, 0xca, 0x87, 0xb5, 0x5b, 0xb5, 0x9d, 0x96, 0xab, 0xc6, 0xf2, 0x54, 0x2e, 0x88,
|
||||
+ 0x31, 0x7a, 0xbd, 0x0a, 0x1d, 0xe3, 0x77, 0x6f, 0x46, 0xb8, 0x50, 0x72, 0x3a, 0x6e, 0xdb, 0xe0,
|
||||
+ 0xe4, 0x1a, 0x87, 0xc2, 0xf6, 0xd3, 0x24, 0x7c, 0xc1, 0x0b, 0x7f, 0x07, 0x5a, 0x0c, 0x73, 0x9a,
|
||||
+ 0x32, 0x79, 0x4d, 0xab, 0x6a, 0xdf, 0xb7, 0xf4, 0xbe, 0x7f, 0x41, 0xe2, 0xf4, 0xdc, 0xb5, 0x73,
|
||||
+ 0x6e, 0x4e, 0x66, 0xae, 0x90, 0xe0, 0x2f, 0x72, 0x85, 0x3e, 0x81, 0x2b, 0x47, 0x7e, 0xca, 0x5f,
|
||||
+ 0x44, 0x57, 0xe7, 0x53, 0x79, 0xfd, 0x78, 0x1a, 0xbd, 0xd0, 0xe2, 0xdf, 0x57, 0xa0, 0xb9, 0x9f,
|
||||
+ 0xa4, 0x4f, 0xb9, 0x3f, 0xc5, 0xe8, 0xbf, 0xa0, 0x2d, 0xa8, 0xf0, 0x67, 0x5e, 0x2a, 0x41, 0x45,
|
||||
+ 0x5e, 0x77, 0x41, 0xa1, 0x34, 0x81, 0x74, 0x3b, 0x66, 0x41, 0x92, 0x1a, 0x8a, 0xea, 0xad, 0xda,
|
||||
+ 0x4e, 0xdd, 0x6d, 0x6b, 0x9c, 0x26, 0x19, 0xc3, 0xa6, 0x9a, 0xf3, 0x48, 0xec, 0x9d, 0x62, 0x16,
|
||||
+ 0xe3, 0x59, 0x44, 0x43, 0xac, 0xce, 0x6f, 0xdd, 0x1d, 0xa8, 0xa9, 0xc3, 0xf8, 0xf3, 0x6c, 0x02,
|
||||
+ 0xfd, 0x37, 0x0c, 0x32, 0x7a, 0x79, 0x29, 0x15, 0x75, 0x5d, 0x51, 0xf7, 0x0d, 0xf5, 0x53, 0x83,
|
||||
+ 0x76, 0x7e, 0x09, 0xbd, 0x27, 0x27, 0x8c, 0x0a, 0x31, 0x23, 0xf1, 0xf4, 0xbe, 0x2f, 0x7c, 0x19,
|
||||
+ 0x3d, 0x12, 0xcc, 0x08, 0x0d, 0xb9, 0xd1, 0xd6, 0x82, 0xe8, 0x6d, 0x18, 0x08, 0x4d, 0x8b, 0x43,
|
||||
+ 0xcf, 0xd2, 0x54, 0x15, 0xcd, 0x46, 0x36, 0x71, 0x64, 0x88, 0xdf, 0x80, 0x5e, 0x4e, 0x2c, 0xe3,
|
||||
+ 0x8f, 0xd1, 0xb7, 0x9b, 0x61, 0x9f, 0x90, 0x08, 0x3b, 0x67, 0xca, 0x57, 0x6a, 0x93, 0xd1, 0xdb,
|
||||
+ 0xd0, 0xca, 0xfd, 0x50, 0x51, 0x27, 0xa4, 0xa7, 0x4f, 0x88, 0x75, 0xa7, 0xdb, 0xcc, 0x9c, 0xf2,
|
||||
+ 0x19, 0xf4, 0x45, 0xa6, 0xb8, 0x17, 0xfa, 0xc2, 0x2f, 0x1f, 0xaa, 0xb2, 0x55, 0x6e, 0x4f, 0x94,
|
||||
+ 0x60, 0xe7, 0x53, 0x68, 0x1d, 0x91, 0x90, 0x6b, 0xc1, 0x43, 0x58, 0x0f, 0x52, 0xc6, 0x70, 0x2c,
|
||||
+ 0xac, 0xc9, 0x06, 0x44, 0x5b, 0xb0, 0x36, 0x23, 0x11, 0x11, 0xc6, 0x4c, 0x0d, 0x38, 0x14, 0xe0,
|
||||
+ 0x11, 0x8e, 0x28, 0x9b, 0x2b, 0x87, 0x6d, 0xc1, 0x5a, 0x71, 0x73, 0x35, 0x80, 0xae, 0x43, 0x2b,
|
||||
+ 0xf2, 0xcf, 0xb3, 0x4d, 0x95, 0x33, 0xcd, 0xc8, 0x3f, 0xd7, 0xca, 0x0f, 0x61, 0xfd, 0x99, 0x4f,
|
||||
+ 0x66, 0x41, 0x2c, 0x8c, 0x57, 0x2c, 0x98, 0x0b, 0xac, 0x17, 0x05, 0xfe, 0xb9, 0x0a, 0x6d, 0x2d,
|
||||
+ 0x51, 0x2b, 0xbc, 0x05, 0x6b, 0x81, 0x1f, 0x9c, 0x64, 0x22, 0x15, 0x80, 0x6e, 0x5b, 0x45, 0xaa,
|
||||
+ 0xc5, 0x20, 0x9c, 0x6b, 0x6a, 0x55, 0xdb, 0x05, 0xe0, 0xcf, 0xfd, 0xc4, 0xe8, 0x56, 0xbb, 0x80,
|
||||
+ 0xb8, 0x25, 0x69, 0xb4, 0xba, 0xef, 0x43, 0x47, 0x9f, 0x3b, 0xb3, 0xa4, 0x7e, 0xc1, 0x92, 0xb6,
|
||||
+ 0xa6, 0xd2, 0x8b, 0x5e, 0x83, 0x6e, 0xca, 0xb1, 0x77, 0x42, 0x30, 0xf3, 0x59, 0x70, 0x32, 0x1f,
|
||||
+ 0xae, 0xe9, 0x37, 0x32, 0xe5, 0xf8, 0xc0, 0xe2, 0xd0, 0x1d, 0x58, 0x93, 0xe1, 0x8f, 0x0f, 0x1b,
|
||||
+ 0xea, 0x39, 0xbe, 0x51, 0x64, 0xa9, 0x4c, 0x1d, 0xab, 0xef, 0x83, 0x58, 0xb0, 0xb9, 0xab, 0x49,
|
||||
+ 0x47, 0x1f, 0x01, 0xe4, 0x48, 0xb4, 0x01, 0xb5, 0x53, 0x3c, 0x37, 0xf7, 0x50, 0x0e, 0xa5, 0x73,
|
||||
+ 0xce, 0xfc, 0x59, 0x6a, 0xbd, 0xae, 0x81, 0x4f, 0xaa, 0x1f, 0x55, 0x9c, 0x00, 0xfa, 0x7b, 0xb3,
|
||||
+ 0x53, 0x42, 0x0b, 0xcb, 0xb7, 0x60, 0x2d, 0xf2, 0xbf, 0xa2, 0xcc, 0x7a, 0x52, 0x01, 0x0a, 0x4b,
|
||||
+ 0x62, 0xca, 0x2c, 0x0b, 0x05, 0xa0, 0x1e, 0x54, 0x69, 0xa2, 0xfc, 0xd5, 0x72, 0xab, 0x34, 0xc9,
|
||||
+ 0x05, 0xd5, 0x0b, 0x82, 0x9c, 0xbf, 0xd7, 0x01, 0x72, 0x29, 0xc8, 0x85, 0x11, 0xa1, 0x1e, 0xc7,
|
||||
+ 0x4c, 0xa6, 0x20, 0xde, 0xf1, 0x5c, 0x60, 0xee, 0x31, 0x1c, 0xa4, 0x8c, 0x93, 0x33, 0xb9, 0x7f,
|
||||
+ 0xd2, 0xec, 0x2b, 0xda, 0xec, 0x05, 0xdd, 0xdc, 0xab, 0x84, 0x4e, 0xf4, 0xba, 0x3d, 0xb9, 0xcc,
|
||||
+ 0xb5, 0xab, 0xd0, 0x21, 0x5c, 0xc9, 0x79, 0x86, 0x05, 0x76, 0xd5, 0xcb, 0xd8, 0x6d, 0x66, 0xec,
|
||||
+ 0xc2, 0x9c, 0xd5, 0x03, 0xd8, 0x24, 0xd4, 0xfb, 0x3a, 0xc5, 0x69, 0x89, 0x51, 0xed, 0x32, 0x46,
|
||||
+ 0x03, 0x42, 0x7f, 0xa4, 0x16, 0xe4, 0x6c, 0x8e, 0xe0, 0x5a, 0xc1, 0x4a, 0x79, 0xdd, 0x0b, 0xcc,
|
||||
+ 0xea, 0x97, 0x31, 0xdb, 0xce, 0xb4, 0x92, 0xf1, 0x20, 0xe7, 0xf8, 0x03, 0xd8, 0x26, 0xd4, 0x7b,
|
||||
+ 0xee, 0x13, 0xb1, 0xc8, 0x6e, 0xed, 0x7b, 0x8c, 0x94, 0x8f, 0x6e, 0x99, 0x97, 0x36, 0x32, 0xc2,
|
||||
+ 0x6c, 0x5a, 0x32, 0xb2, 0xf1, 0x3d, 0x46, 0x3e, 0x52, 0x0b, 0x72, 0x36, 0xf7, 0x60, 0x40, 0xe8,
|
||||
+ 0xa2, 0x36, 0xeb, 0x97, 0x31, 0xe9, 0x13, 0x5a, 0xd6, 0x64, 0x0f, 0x06, 0x1c, 0x07, 0x82, 0xb2,
|
||||
+ 0xe2, 0x21, 0x68, 0x5e, 0xc6, 0x62, 0xc3, 0xd0, 0x67, 0x3c, 0x9c, 0x9f, 0x41, 0xe7, 0x20, 0x9d,
|
||||
+ 0x62, 0x31, 0x3b, 0xce, 0x82, 0xc1, 0x4b, 0x8b, 0x3f, 0xce, 0x3f, 0xab, 0xd0, 0xde, 0x9f, 0x32,
|
||||
+ 0x9a, 0x26, 0xa5, 0x98, 0xac, 0x2f, 0xe9, 0x62, 0x4c, 0x56, 0x24, 0x2a, 0x26, 0x6b, 0xe2, 0x0f,
|
||||
+ 0xa0, 0x13, 0xa9, 0xab, 0x6b, 0xe8, 0x75, 0x1c, 0x1a, 0x2c, 0x5d, 0x6a, 0xb7, 0x1d, 0x15, 0x82,
|
||||
+ 0xd9, 0x18, 0x20, 0x21, 0x21, 0x37, 0x6b, 0x74, 0x38, 0xea, 0x9b, 0x8c, 0xd0, 0x86, 0x68, 0xb7,
|
||||
+ 0x95, 0x64, 0xd1, 0xfa, 0x3d, 0x68, 0x1f, 0x4b, 0x27, 0x99, 0x05, 0xa5, 0x60, 0x94, 0x7b, 0xcf,
|
||||
+ 0x85, 0xe3, 0xfc, 0x12, 0x1e, 0x40, 0xf7, 0x44, 0xbb, 0xcc, 0x2c, 0xd2, 0x67, 0xe8, 0x35, 0x63,
|
||||
+ 0x49, 0x6e, 0xef, 0xb8, 0xe8, 0x59, 0xbd, 0x01, 0x9d, 0x93, 0x02, 0x6a, 0x34, 0x81, 0xc1, 0x12,
|
||||
+ 0xc9, 0x8a, 0x18, 0xb4, 0x53, 0x8c, 0x41, 0xed, 0x3b, 0x48, 0x0b, 0x2a, 0xae, 0x2c, 0xc6, 0xa5,
|
||||
+ 0xdf, 0x54, 0xa1, 0xf3, 0x43, 0x2c, 0x9e, 0x53, 0x76, 0xaa, 0xf5, 0x45, 0x50, 0x8f, 0xfd, 0x08,
|
||||
+ 0x1b, 0x8e, 0x6a, 0x8c, 0xae, 0x41, 0x93, 0x9d, 0xeb, 0x00, 0x62, 0xf6, 0x73, 0x9d, 0x9d, 0xab,
|
||||
+ 0xc0, 0x80, 0x5e, 0x01, 0x60, 0xe7, 0x5e, 0xe2, 0x07, 0xa7, 0xd8, 0x78, 0xb0, 0xee, 0xb6, 0xd8,
|
||||
+ 0xf9, 0x91, 0x46, 0xc8, 0xa3, 0xc0, 0xce, 0x3d, 0xcc, 0x18, 0x65, 0xdc, 0xc4, 0xaa, 0x26, 0x3b,
|
||||
+ 0x7f, 0xa0, 0x60, 0xb3, 0x36, 0x64, 0x34, 0x49, 0x70, 0xa8, 0x62, 0xb4, 0x5a, 0x7b, 0x5f, 0x23,
|
||||
+ 0xa4, 0x54, 0x61, 0xa5, 0x36, 0xb4, 0x54, 0x91, 0x4b, 0x15, 0xb9, 0xd4, 0x75, 0xbd, 0x52, 0x14,
|
||||
+ 0xa5, 0x8a, 0x4c, 0x6a, 0x53, 0x4b, 0x15, 0x05, 0xa9, 0x22, 0x97, 0xda, 0xb2, 0x6b, 0x8d, 0x54,
|
||||
+ 0xe7, 0xd7, 0x15, 0xd8, 0x5e, 0x4c, 0xfc, 0x4c, 0x9a, 0xfa, 0x01, 0x74, 0x02, 0xb5, 0x5f, 0xa5,
|
||||
+ 0x33, 0x39, 0x58, 0xda, 0x49, 0xb7, 0x1d, 0x14, 0x8e, 0xf1, 0x87, 0xd0, 0x8d, 0xb5, 0x83, 0xb3,
|
||||
+ 0xa3, 0x59, 0xcb, 0xf7, 0xa5, 0xe8, 0x7b, 0xb7, 0x13, 0x17, 0x20, 0x27, 0x04, 0xf4, 0x25, 0x23,
|
||||
+ 0x02, 0x4f, 0x04, 0xc3, 0x7e, 0xf4, 0x32, 0x0a, 0x10, 0x04, 0x75, 0x95, 0xad, 0xd4, 0x54, 0x7e,
|
||||
+ 0xad, 0xc6, 0xce, 0x9b, 0xb0, 0x59, 0x92, 0x62, 0x6c, 0xdd, 0x80, 0xda, 0x0c, 0xc7, 0x8a, 0x7b,
|
||||
+ 0xd7, 0x95, 0x43, 0xc7, 0x87, 0x81, 0x8b, 0xfd, 0xf0, 0xe5, 0x69, 0x63, 0x44, 0xd4, 0x72, 0x11,
|
||||
+ 0x3b, 0x80, 0x8a, 0x22, 0x8c, 0x2a, 0x56, 0xeb, 0x4a, 0x41, 0xeb, 0xc7, 0x30, 0xd8, 0x9f, 0x51,
|
||||
+ 0x8e, 0x27, 0x22, 0x24, 0xf1, 0xcb, 0xa8, 0x98, 0x7e, 0x01, 0x9b, 0x4f, 0xc4, 0xfc, 0x4b, 0xc9,
|
||||
+ 0x8c, 0x93, 0x6f, 0xf0, 0x4b, 0xb2, 0x8f, 0xd1, 0xe7, 0xd6, 0x3e, 0x46, 0x9f, 0xcb, 0x62, 0x29,
|
||||
+ 0xa0, 0xb3, 0x34, 0x8a, 0xd5, 0x55, 0xe8, 0xba, 0x06, 0x72, 0xf6, 0xa0, 0xa3, 0x73, 0xe8, 0x47,
|
||||
+ 0x34, 0x4c, 0x67, 0x78, 0xe5, 0x1d, 0xbc, 0x09, 0x90, 0xf8, 0xcc, 0x8f, 0xb0, 0xc0, 0x4c, 0x9f,
|
||||
+ 0xa1, 0x96, 0x5b, 0xc0, 0x38, 0xbf, 0xad, 0xc2, 0x96, 0x6e, 0x89, 0x4c, 0x74, 0x27, 0xc0, 0x9a,
|
||||
+ 0x30, 0x82, 0xe6, 0x09, 0xe5, 0xa2, 0xc0, 0x30, 0x83, 0xa5, 0x8a, 0x61, 0x6c, 0xb9, 0xc9, 0x61,
|
||||
+ 0xa9, 0x4f, 0x51, 0xbb, 0xbc, 0x4f, 0xb1, 0xd4, 0x89, 0xa8, 0x2f, 0x77, 0x22, 0xe4, 0x6d, 0xb3,
|
||||
+ 0x44, 0x44, 0xdf, 0xf1, 0x96, 0xdb, 0x32, 0x98, 0xc3, 0x10, 0xdd, 0x86, 0xfe, 0x54, 0x6a, 0xe9,
|
||||
+ 0x9d, 0x50, 0x7a, 0xea, 0x25, 0xbe, 0x38, 0x51, 0x57, 0xbd, 0xe5, 0x76, 0x15, 0xfa, 0x80, 0xd2,
|
||||
+ 0xd3, 0x23, 0x5f, 0x9c, 0xa0, 0x8f, 0xa1, 0x67, 0xd2, 0xc0, 0x48, 0xb9, 0x88, 0x9b, 0xc7, 0xcf,
|
||||
+ 0xdc, 0xa2, 0xa2, 0xf7, 0xdc, 0xee, 0x69, 0x01, 0xe2, 0xce, 0x55, 0xb8, 0x72, 0x1f, 0x73, 0xc1,
|
||||
+ 0xe8, 0xbc, 0xec, 0x18, 0xe7, 0xff, 0x00, 0x0e, 0x63, 0x81, 0xd9, 0x33, 0x3f, 0xc0, 0x1c, 0xbd,
|
||||
+ 0x5b, 0x84, 0x4c, 0x72, 0xb4, 0x31, 0xd6, 0x1d, 0xa9, 0x6c, 0xc2, 0x2d, 0xd0, 0x38, 0x63, 0x68,
|
||||
+ 0xb8, 0x34, 0x95, 0xe1, 0xe8, 0x75, 0x3b, 0x32, 0xeb, 0x3a, 0x66, 0x9d, 0x42, 0xba, 0x66, 0xce,
|
||||
+ 0x39, 0xb0, 0x25, 0x6c, 0xce, 0xce, 0x6c, 0xd1, 0x18, 0x5a, 0xc4, 0xe2, 0x4c, 0x54, 0x59, 0x16,
|
||||
+ 0x9d, 0x93, 0x38, 0x3f, 0x85, 0x4d, 0xcd, 0x49, 0x73, 0xb6, 0x6c, 0x5e, 0x87, 0x06, 0xb3, 0x6a,
|
||||
+ 0x54, 0xf2, 0x56, 0x94, 0x21, 0x32, 0x73, 0xe8, 0x86, 0x14, 0x16, 0x30, 0x1c, 0xc9, 0x9a, 0xa3,
|
||||
+ 0xaa, 0xb6, 0x2c, 0x47, 0x48, 0x6f, 0xc9, 0x7a, 0x3b, 0x37, 0xd3, 0x7a, 0x6b, 0x13, 0x06, 0x72,
|
||||
+ 0xa2, 0x24, 0xd1, 0xf9, 0x39, 0x6c, 0x3e, 0x8e, 0x67, 0x24, 0xc6, 0xfb, 0x47, 0x4f, 0x1f, 0xe1,
|
||||
+ 0x2c, 0x2a, 0x20, 0xa8, 0xcb, 0xec, 0x49, 0xa9, 0xd1, 0x74, 0xd5, 0x58, 0x5e, 0x93, 0xf8, 0xd8,
|
||||
+ 0x0b, 0x92, 0x94, 0x9b, 0xce, 0x50, 0x23, 0x3e, 0xde, 0x4f, 0x52, 0x2e, 0xc3, 0xbc, 0x7c, 0xe6,
|
||||
+ 0x69, 0x3c, 0x9b, 0xab, 0xbb, 0xd2, 0x74, 0xd7, 0x83, 0x24, 0x7d, 0x1c, 0xcf, 0xe6, 0xce, 0xff,
|
||||
+ 0xa8, 0x5a, 0x18, 0xe3, 0xd0, 0xf5, 0xe3, 0x90, 0x46, 0xf7, 0xf1, 0x59, 0x41, 0x42, 0x56, 0x77,
|
||||
+ 0xd9, 0x98, 0xf0, 0x6d, 0x05, 0x3a, 0xf7, 0xa6, 0x38, 0x16, 0xf7, 0xb1, 0xf0, 0xc9, 0x4c, 0xd5,
|
||||
+ 0x56, 0x67, 0x98, 0x71, 0x42, 0x63, 0x73, 0xf0, 0x2d, 0x28, 0x4b, 0x63, 0x12, 0x13, 0xe1, 0x85,
|
||||
+ 0x3e, 0x8e, 0x68, 0x6c, 0xbc, 0x00, 0x12, 0x75, 0x5f, 0x61, 0xd0, 0x9b, 0xd0, 0xd7, 0x9d, 0x3b,
|
||||
+ 0xef, 0xc4, 0x8f, 0xc3, 0x99, 0xbc, 0x72, 0xba, 0x93, 0xd1, 0xd3, 0xe8, 0x03, 0x83, 0x45, 0x6f,
|
||||
+ 0xc1, 0x86, 0xb9, 0x10, 0x39, 0x65, 0x5d, 0x51, 0xf6, 0x0d, 0xbe, 0x44, 0x9a, 0x26, 0x09, 0x65,
|
||||
+ 0x82, 0x7b, 0x1c, 0x07, 0x01, 0x8d, 0x12, 0x53, 0x98, 0xf4, 0x2d, 0x7e, 0xa2, 0xd1, 0xce, 0x14,
|
||||
+ 0x36, 0x1f, 0x4a, 0x3b, 0x8d, 0x25, 0xf9, 0x06, 0xf7, 0x22, 0x1c, 0x79, 0xc7, 0x33, 0x1a, 0x9c,
|
||||
+ 0x7a, 0x32, 0x4c, 0x19, 0x0f, 0xcb, 0xd4, 0x67, 0x4f, 0x22, 0x27, 0xe4, 0x1b, 0x55, 0x83, 0x4b,
|
||||
+ 0xaa, 0x13, 0x2a, 0x92, 0x59, 0x3a, 0xf5, 0x12, 0x46, 0x8f, 0xb1, 0x31, 0xb1, 0x1f, 0xe1, 0xe8,
|
||||
+ 0x40, 0xe3, 0x8f, 0x24, 0xda, 0xf9, 0x53, 0x05, 0xb6, 0xca, 0x92, 0x4c, 0xd0, 0xdd, 0x85, 0xad,
|
||||
+ 0xb2, 0x28, 0xf3, 0x10, 0xeb, 0x44, 0x6f, 0x50, 0x14, 0xa8, 0x9f, 0xe4, 0x0f, 0xa1, 0xab, 0xda,
|
||||
+ 0xb9, 0x5e, 0xa8, 0x39, 0x95, 0xd3, 0x8f, 0xe2, 0xbe, 0xb8, 0x1d, 0xbf, 0xb8, 0x4b, 0x1f, 0xc3,
|
||||
+ 0x35, 0x63, 0xbe, 0xb7, 0xac, 0xb6, 0x3e, 0x10, 0xdb, 0x86, 0xe0, 0xd1, 0x82, 0xf6, 0x5f, 0xc0,
|
||||
+ 0x30, 0x47, 0xed, 0xcd, 0x15, 0xd2, 0xfa, 0xea, 0x5d, 0xd8, 0x5c, 0x30, 0xf6, 0x5e, 0x18, 0x32,
|
||||
+ 0x75, 0x41, 0xeb, 0xee, 0xaa, 0x29, 0xe7, 0x2e, 0x5c, 0x9d, 0x60, 0xa1, 0xbd, 0xe1, 0x0b, 0x53,
|
||||
+ 0x13, 0x68, 0x66, 0x1b, 0x50, 0x9b, 0xe0, 0x40, 0x19, 0x5f, 0x73, 0xe5, 0x50, 0x1e, 0xc0, 0xa7,
|
||||
+ 0x1c, 0x07, 0xca, 0xca, 0x9a, 0xab, 0xc6, 0xce, 0x1f, 0x2b, 0xb0, 0x6e, 0xc2, 0xa4, 0x0c, 0xf5,
|
||||
+ 0x21, 0x23, 0x67, 0x98, 0x99, 0xa3, 0x67, 0x20, 0xf4, 0x06, 0xf4, 0xf4, 0xc8, 0xa3, 0x89, 0x20,
|
||||
+ 0x34, 0x0b, 0xbe, 0x5d, 0x8d, 0x7d, 0xac, 0x91, 0xaa, 0x53, 0xa7, 0x1a, 0x51, 0xa6, 0xe6, 0x33,
|
||||
+ 0x90, 0x6a, 0xb7, 0x71, 0x19, 0x19, 0x54, 0xb0, 0x6d, 0xb9, 0x06, 0x92, 0x47, 0xdd, 0xf2, 0x5b,
|
||||
+ 0x53, 0xfc, 0x2c, 0x28, 0x8f, 0x7a, 0x44, 0xd3, 0x58, 0x78, 0x09, 0x25, 0xb1, 0x30, 0xd1, 0x15,
|
||||
+ 0x14, 0xea, 0x48, 0x62, 0x9c, 0x5f, 0x55, 0xa0, 0xa1, 0xbb, 0xd5, 0xb2, 0xca, 0xcc, 0xde, 0xb8,
|
||||
+ 0x2a, 0x51, 0xf9, 0x82, 0x92, 0xa5, 0xdf, 0x35, 0x35, 0x96, 0xf7, 0xf8, 0x2c, 0xd2, 0x91, 0xda,
|
||||
+ 0xa8, 0x76, 0x16, 0xa9, 0x10, 0xfd, 0x06, 0xf4, 0xf2, 0xa7, 0x52, 0xcd, 0x6b, 0x15, 0xbb, 0x19,
|
||||
+ 0x56, 0x91, 0x5d, 0xa8, 0xa9, 0xf3, 0x13, 0x59, 0x5c, 0x67, 0x9d, 0xda, 0x0d, 0xa8, 0xa5, 0x99,
|
||||
+ 0x32, 0x72, 0x28, 0x31, 0xd3, 0xec, 0x91, 0x95, 0x43, 0x74, 0x1b, 0x7a, 0x7e, 0x18, 0x12, 0xb9,
|
||||
+ 0xdc, 0x9f, 0x3d, 0x24, 0x61, 0x76, 0x49, 0xcb, 0x58, 0xe7, 0x2f, 0x15, 0xe8, 0xef, 0xd3, 0x64,
|
||||
+ 0xfe, 0xff, 0x64, 0x86, 0x0b, 0x11, 0x44, 0x29, 0x69, 0xde, 0x58, 0x39, 0x96, 0x79, 0xe3, 0x33,
|
||||
+ 0x32, 0xc3, 0xfa, 0x6a, 0xe9, 0x9d, 0x6d, 0x4a, 0x84, 0xba, 0x56, 0x76, 0x32, 0x6b, 0x80, 0x75,
|
||||
+ 0xf5, 0xe4, 0x23, 0x1a, 0xaa, 0x0c, 0x39, 0x24, 0xcc, 0xcb, 0xda, 0x5d, 0x5d, 0x77, 0x3d, 0x24,
|
||||
+ 0x4c, 0x4d, 0x19, 0x43, 0xd6, 0x54, 0xc7, 0xb5, 0x68, 0x48, 0x43, 0x63, 0xa4, 0x21, 0xdb, 0xd0,
|
||||
+ 0xa0, 0xcf, 0x9e, 0x71, 0x2c, 0x54, 0x2e, 0x5b, 0x73, 0x0d, 0x94, 0x85, 0xb9, 0x66, 0x21, 0xcc,
|
||||
+ 0x5d, 0x81, 0x4d, 0xd5, 0xdb, 0x7f, 0xc2, 0xfc, 0x80, 0xc4, 0x53, 0x1b, 0x8a, 0xb7, 0x00, 0x4d,
|
||||
+ 0x04, 0x4d, 0x16, 0xb0, 0x63, 0x18, 0x98, 0x37, 0xe7, 0xe8, 0xc7, 0x13, 0x6b, 0xfa, 0x35, 0x68,
|
||||
+ 0x4a, 0xd0, 0x63, 0xf8, 0x6b, 0x1b, 0x18, 0xcd, 0xb4, 0xf3, 0x16, 0x74, 0xf4, 0xd0, 0x84, 0x81,
|
||||
+ 0x9c, 0x94, 0x97, 0x49, 0xf9, 0x9d, 0xbf, 0x6d, 0x98, 0x70, 0x6b, 0x6a, 0x68, 0xf4, 0x10, 0xfa,
|
||||
+ 0x0b, 0xff, 0x64, 0x90, 0x69, 0xaa, 0xac, 0xfe, 0x55, 0x33, 0xda, 0x1e, 0xeb, 0x7f, 0x3c, 0x63,
|
||||
+ 0xfb, 0x8f, 0x67, 0xfc, 0x20, 0x4a, 0xc4, 0x1c, 0x3d, 0x80, 0x5e, 0xf9, 0xef, 0x05, 0xba, 0x6e,
|
||||
+ 0x73, 0x90, 0x15, 0xff, 0x34, 0x2e, 0x64, 0xf3, 0x10, 0xfa, 0x0b, 0x3f, 0x32, 0xac, 0x3e, 0xab,
|
||||
+ 0xff, 0x6f, 0x5c, 0xc8, 0xe8, 0x2e, 0xb4, 0x0b, 0x7f, 0x2e, 0xd0, 0x50, 0x33, 0x59, 0xfe, 0x99,
|
||||
+ 0x71, 0x21, 0x83, 0x7d, 0xe8, 0x96, 0x7e, 0x26, 0xa0, 0x91, 0xb1, 0x67, 0xc5, 0x1f, 0x86, 0x0b,
|
||||
+ 0x99, 0xec, 0x41, 0xbb, 0xd0, 0xd3, 0xb7, 0x5a, 0x2c, 0xff, 0x38, 0x18, 0x5d, 0x5b, 0x31, 0x63,
|
||||
+ 0xb6, 0xf3, 0x00, 0xba, 0xa5, 0x0e, 0xbc, 0x55, 0x64, 0x55, 0xf7, 0x7f, 0x74, 0x7d, 0xe5, 0x9c,
|
||||
+ 0xe1, 0xf4, 0x10, 0xfa, 0x0b, 0xfd, 0x78, 0xeb, 0xdc, 0xd5, 0x6d, 0xfa, 0x0b, 0xcd, 0xfa, 0x5c,
|
||||
+ 0x6d, 0x76, 0xa1, 0xdc, 0x2a, 0x6c, 0xf6, 0x72, 0xf7, 0x7d, 0x74, 0x63, 0xf5, 0xa4, 0xd1, 0xea,
|
||||
+ 0x01, 0xf4, 0xca, 0x8d, 0x77, 0xcb, 0x6c, 0x65, 0x3b, 0xfe, 0xf2, 0x93, 0x53, 0xea, 0xc1, 0xe7,
|
||||
+ 0x27, 0x67, 0x55, 0x6b, 0xfe, 0x42, 0x46, 0xf7, 0x00, 0x4c, 0x71, 0x15, 0x92, 0x38, 0xdb, 0xb2,
|
||||
+ 0xa5, 0xa2, 0x2e, 0xdb, 0xb2, 0x15, 0x85, 0xd8, 0x5d, 0x00, 0x5d, 0x13, 0x85, 0x34, 0x15, 0xe8,
|
||||
+ 0xaa, 0x55, 0x63, 0xa1, 0x10, 0x1b, 0x0d, 0x97, 0x27, 0x96, 0x18, 0x60, 0xc6, 0x5e, 0x84, 0xc1,
|
||||
+ 0x67, 0x00, 0x79, 0xad, 0x65, 0x19, 0x2c, 0x55, 0x5f, 0x97, 0xf8, 0xa0, 0x53, 0xac, 0xac, 0x90,
|
||||
+ 0xb1, 0x75, 0x45, 0xb5, 0x75, 0x09, 0x8b, 0xfe, 0x42, 0xe6, 0x5c, 0x3e, 0x6c, 0x8b, 0x09, 0xf5,
|
||||
+ 0x68, 0x29, 0x7b, 0x46, 0x1f, 0x42, 0xa7, 0x98, 0x32, 0x5b, 0x2d, 0x56, 0xa4, 0xd1, 0xa3, 0x52,
|
||||
+ 0xda, 0x8c, 0xee, 0x42, 0xaf, 0x9c, 0x10, 0xa3, 0xc2, 0xbd, 0x58, 0x4a, 0x93, 0x47, 0xa6, 0x19,
|
||||
+ 0x54, 0x20, 0x7f, 0x1f, 0x20, 0x4f, 0x9c, 0xad, 0xfb, 0x96, 0x52, 0xe9, 0x05, 0xa9, 0x9f, 0x41,
|
||||
+ 0xaf, 0x10, 0xb7, 0x65, 0x4d, 0x78, 0xb5, 0x64, 0x70, 0x1e, 0xcd, 0x47, 0x26, 0xc3, 0x2a, 0x85,
|
||||
+ 0xed, 0x7b, 0xd0, 0x29, 0xbe, 0x11, 0xd6, 0xda, 0x15, 0xef, 0xc6, 0x65, 0x41, 0xaf, 0xf0, 0x9e,
|
||||
+ 0xd8, 0xb3, 0xbb, 0xfc, 0xc4, 0x5c, 0x16, 0xf4, 0x4a, 0xf5, 0xa8, 0x8d, 0x35, 0xab, 0x8a, 0xd4,
|
||||
+ 0xcb, 0x9e, 0x82, 0x72, 0xf1, 0x66, 0xbd, 0xbf, 0xb2, 0xa4, 0xbb, 0xec, 0x0c, 0x16, 0xeb, 0x14,
|
||||
+ 0xeb, 0x8f, 0x15, 0xb5, 0xcb, 0xf7, 0xc4, 0x84, 0x62, 0x2d, 0x52, 0x88, 0x09, 0x2b, 0x4a, 0x94,
|
||||
+ 0x0b, 0x19, 0x1d, 0x40, 0xff, 0xa1, 0x4d, 0x33, 0x4d, 0x0a, 0x6c, 0xd4, 0x59, 0x91, 0xf2, 0x8f,
|
||||
+ 0x46, 0xab, 0xa6, 0xcc, 0x2e, 0x7f, 0x0e, 0x83, 0xa5, 0xf4, 0x17, 0xdd, 0xcc, 0x5a, 0x9e, 0x2b,
|
||||
+ 0xf3, 0xe2, 0x0b, 0xd5, 0x3a, 0x84, 0x8d, 0xc5, 0xec, 0x17, 0xbd, 0x62, 0x36, 0x7d, 0x75, 0x56,
|
||||
+ 0x7c, 0x21, 0xab, 0x8f, 0xa1, 0x69, 0xb3, 0x2d, 0x64, 0x5a, 0xcb, 0x0b, 0xd9, 0xd7, 0x45, 0x4b,
|
||||
+ 0xf7, 0x3a, 0xdf, 0x7e, 0x77, 0xb3, 0xf2, 0xd7, 0xef, 0x6e, 0x56, 0xfe, 0xf1, 0xdd, 0xcd, 0xca,
|
||||
+ 0x71, 0x43, 0xcd, 0xbe, 0xff, 0xaf, 0x00, 0x00, 0x00, 0xff, 0xff, 0x45, 0xa8, 0x91, 0xab, 0x62,
|
||||
+ 0x22, 0x00, 0x00,
|
||||
}
|
||||
diff --git a/protocols/grpc/agent.proto b/protocols/grpc/agent.proto
|
||||
index b0fab6d..d863645 100644
|
||||
--- a/protocols/grpc/agent.proto
|
||||
+++ b/protocols/grpc/agent.proto
|
||||
@@ -46,6 +46,7 @@ service AgentService {
|
||||
rpc UpdateRoutes(UpdateRoutesRequest) returns (Routes);
|
||||
rpc ListInterfaces(ListInterfacesRequest) returns(Interfaces);
|
||||
rpc ListRoutes(ListRoutesRequest) returns (Routes);
|
||||
+ rpc UpdateIPVSRule(UpdateIPVSRequest) returns (IPVSResponse);
|
||||
|
||||
// tracing
|
||||
rpc StartTracing(StartTracingRequest) returns (google.protobuf.Empty);
|
||||
@@ -495,3 +496,13 @@ message StartTracingRequest {
|
||||
|
||||
message StopTracingRequest {
|
||||
}
|
||||
+
|
||||
+message UpdateIPVSRequest {
|
||||
+ // IPVS_req is the IPVS rule message needed to update
|
||||
+ string IPVS_req = 1;
|
||||
+}
|
||||
+
|
||||
+message IPVSResponse {
|
||||
+ // IPVS_res is the response of IPVS updating
|
||||
+ string IPVS_res = 1;
|
||||
+}
|
||||
--
|
||||
2.14.3 (Apple Git-98)
|
||||
|
||||
@ -1,193 +0,0 @@
|
||||
From 01563c08910ddaba4077fd9dc691df541e045165 Mon Sep 17 00:00:00 2001
|
||||
From: xiadanni <xiadanni1@huawei.com>
|
||||
Date: Tue, 18 Aug 2020 17:05:32 +0800
|
||||
Subject: [PATCH 04/16] agent: add IPVS test
|
||||
|
||||
Signed-off-by: xiadanni <xiadanni1@huawei.com>
|
||||
---
|
||||
grpc_test.go | 172 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
|
||||
1 file changed, 172 insertions(+)
|
||||
|
||||
diff --git a/grpc_test.go b/grpc_test.go
|
||||
index e69102b..d58c0b6 100644
|
||||
--- a/grpc_test.go
|
||||
+++ b/grpc_test.go
|
||||
@@ -1840,3 +1840,175 @@ func getPipeMaxSize() (uint32, error) {
|
||||
u, err := strconv.ParseUint(s, 10, 32)
|
||||
return uint32(u), err
|
||||
}
|
||||
+
|
||||
+func TestUpdateIPVSRule(t *testing.T) {
|
||||
+ assert := assert.New(t)
|
||||
+
|
||||
+ // add IPVS rule successfully
|
||||
+ a := &agentGRPC{
|
||||
+ sandbox: &sandbox{
|
||||
+ containers: make(map[string]*container),
|
||||
+ },
|
||||
+ }
|
||||
+
|
||||
+ req := &pb.UpdateIPVSRequest{
|
||||
+ IPVSReq: "ipvsadm -A -t 17.2.0.7:80 -s rr -p 3000",
|
||||
+ }
|
||||
+
|
||||
+ _, err := a.UpdateIPVSRule(context.Background(), req)
|
||||
+ assert.NoError(err)
|
||||
+
|
||||
+ // delete ipvs rule successfully
|
||||
+ req = &pb.UpdateIPVSRequest{
|
||||
+ IPVSReq: "ipvsadm -D -t 17.2.0.7:80",
|
||||
+ }
|
||||
+
|
||||
+ _, err = a.UpdateIPVSRule(context.Background(), req)
|
||||
+ assert.NoError(err)
|
||||
+
|
||||
+ // update ipvs rule error because exec failed
|
||||
+ req = &pb.UpdateIPVSRequest{
|
||||
+ IPVSReq: "ipvsadm -A -t 17.2.0.7:80 -s rr -p -3000",
|
||||
+ }
|
||||
+
|
||||
+ _, err = a.UpdateIPVSRule(context.Background(), req)
|
||||
+ assert.Error(err)
|
||||
+ assert.Contains(err.Error(), "exec IPVS command failed")
|
||||
+
|
||||
+ // update IPVS rule error because rule less than validHeadLength
|
||||
+ req = &pb.UpdateIPVSRequest{
|
||||
+ IPVSReq: "ipvsa",
|
||||
+ }
|
||||
+
|
||||
+ _, err = a.UpdateIPVSRule(context.Background(), req)
|
||||
+ assert.Error(err)
|
||||
+ assert.Contains(err.Error(), "invalid IPVS rule")
|
||||
+
|
||||
+ // update ipvs rule error because invalid command
|
||||
+ req = &pb.UpdateIPVSRequest{
|
||||
+ IPVSReq: "abcabcabc ipvsadm",
|
||||
+ }
|
||||
+
|
||||
+ _, err = a.UpdateIPVSRule(context.Background(), req)
|
||||
+ assert.Error(err)
|
||||
+ assert.Contains(err.Error(), "invalid IPVS rule")
|
||||
+
|
||||
+ // add ipvs rule error because rule count exceeds
|
||||
+ a = &agentGRPC{
|
||||
+ sandbox: &sandbox{
|
||||
+ containers: make(map[string]*container),
|
||||
+ ipvsadm: ipvsAdm{
|
||||
+ ipvsRuleCnt: 20000,
|
||||
+ },
|
||||
+ },
|
||||
+ }
|
||||
+
|
||||
+ req = &pb.UpdateIPVSRequest{
|
||||
+ IPVSReq: "ipvsadm -A -t 17.2.0.7:80 -s rr -p 3000",
|
||||
+ }
|
||||
+ _, err = a.UpdateIPVSRule(context.Background(), req)
|
||||
+ assert.Error(err)
|
||||
+ assert.Errorf(err, "rules exceed limit")
|
||||
+
|
||||
+ // add ipvs rule error because ipvs request item less than 2
|
||||
+ a = &agentGRPC{
|
||||
+ sandbox: &sandbox{
|
||||
+ containers: make(map[string]*container),
|
||||
+ ipvsadm: ipvsAdm{
|
||||
+ ipvsRuleCnt: 0,
|
||||
+ },
|
||||
+ },
|
||||
+ }
|
||||
+
|
||||
+ req = &pb.UpdateIPVSRequest{
|
||||
+ IPVSReq: "ipvsadm",
|
||||
+ }
|
||||
+
|
||||
+ _, err = a.UpdateIPVSRule(context.Background(), req)
|
||||
+ assert.Error(err)
|
||||
+ assert.Contains(err.Error(), "invalid IPVS rule")
|
||||
+
|
||||
+ // add ipvs rule error because ipvs rule nil
|
||||
+ req = nil
|
||||
+
|
||||
+ _, err = a.UpdateIPVSRule(context.Background(), req)
|
||||
+ assert.Error(err)
|
||||
+ assert.Contains(err.Error(), "IPVS rule is nil")
|
||||
+
|
||||
+ // add ipvs rule error because ipvs rule string is empty
|
||||
+ req = &pb.UpdateIPVSRequest{
|
||||
+ IPVSReq: "",
|
||||
+ }
|
||||
+
|
||||
+ _, err = a.UpdateIPVSRule(context.Background(), req)
|
||||
+ assert.Error(err)
|
||||
+ assert.Contains(err.Error(), "IPVS rule is nil")
|
||||
+
|
||||
+ // restore ipvs rule successfully
|
||||
+ a = &agentGRPC{
|
||||
+ sandbox: &sandbox{
|
||||
+ containers: make(map[string]*container),
|
||||
+ },
|
||||
+ }
|
||||
+
|
||||
+ req = &pb.UpdateIPVSRequest{
|
||||
+ IPVSReq: "restore|2|-A -t 10.10.11.12:100 -s rr -p 3000\n-a -t 10.10.11.12:100 -r 172.16.0.1:80 -m",
|
||||
+ }
|
||||
+
|
||||
+ _, err = a.UpdateIPVSRule(context.Background(), req)
|
||||
+ assert.NoError(err)
|
||||
+
|
||||
+ // clear IPVS rule successfully
|
||||
+ req = &pb.UpdateIPVSRequest{
|
||||
+ IPVSReq: "ipvsadm -C",
|
||||
+ }
|
||||
+
|
||||
+ _, err = a.UpdateIPVSRule(context.Background(), req)
|
||||
+ assert.NoError(err)
|
||||
+ assert.Equal(a.sandbox.ipvsadm.ipvsRuleCnt, uint64(0))
|
||||
+
|
||||
+ // restore ipvs rule error because rule count invalid
|
||||
+ req = &pb.UpdateIPVSRequest{
|
||||
+ IPVSReq: "restore|abc|-A -t 10.10.11.12:100 -s rr -p 3000\n-a -t 10.10.11.12:100 -r 172.16.0.1:80 -m",
|
||||
+ }
|
||||
+
|
||||
+ _, err = a.UpdateIPVSRule(context.Background(), req)
|
||||
+ assert.Error(err)
|
||||
+ assert.Contains(err.Error(), "invalid IPVS rule")
|
||||
+
|
||||
+ // restore ipvs rule error because other rules exists
|
||||
+ a = &agentGRPC{
|
||||
+ sandbox: &sandbox{
|
||||
+ containers: make(map[string]*container),
|
||||
+ ipvsadm: ipvsAdm{
|
||||
+ ipvsRuleCnt: 5,
|
||||
+ },
|
||||
+ },
|
||||
+ }
|
||||
+
|
||||
+ req = &pb.UpdateIPVSRequest{
|
||||
+ IPVSReq: "restore|2|-A -t 10.10.11.12:100 -s rr -p 3000\n-a -t 10.10.11.12:100 -r 172.16.0.1:80 -m",
|
||||
+ }
|
||||
+
|
||||
+ _, err = a.UpdateIPVSRule(context.Background(), req)
|
||||
+ assert.Error(err)
|
||||
+ assert.Contains(err.Error(), "exist some rules in system")
|
||||
+
|
||||
+ // restore ipvs rule error because ipvs req item less than 3
|
||||
+ a = &agentGRPC{
|
||||
+ sandbox: &sandbox{
|
||||
+ containers: make(map[string]*container),
|
||||
+ ipvsadm: ipvsAdm{
|
||||
+ ipvsRuleCnt: 0,
|
||||
+ },
|
||||
+ },
|
||||
+ }
|
||||
+
|
||||
+ req = &pb.UpdateIPVSRequest{
|
||||
+ IPVSReq: "restore|-A -t 10.10.11.12:100 -s rr -p 3000\n-a -t 10.10.11.12:100 -r 172.16.0.1:80 -m",
|
||||
+ }
|
||||
+
|
||||
+ _, err = a.UpdateIPVSRule(context.Background(), req)
|
||||
+ assert.Error(err)
|
||||
+ assert.Contains(err.Error(), "invalid IPVS rule")
|
||||
+}
|
||||
--
|
||||
2.14.3 (Apple Git-98)
|
||||
|
||||
@ -1,42 +0,0 @@
|
||||
From 8c9f9be2a9c195d0bc12b43c491adaacb7bb8154 Mon Sep 17 00:00:00 2001
|
||||
From: holyfei <yangfeiyu20092010@163.com>
|
||||
Date: Tue, 18 Aug 2020 10:42:38 +0800
|
||||
Subject: [PATCH 05/16] mount: support mount block device
|
||||
|
||||
reason: modify mountStorage to support mount block device
|
||||
"-v /dev/blockdevice:/home/test"
|
||||
|
||||
Signed-off-by: yangfeiyu <yangfeiyu2@huawei.com>
|
||||
---
|
||||
mount.go | 15 ++++++++++++++-
|
||||
1 file changed, 14 insertions(+), 1 deletion(-)
|
||||
|
||||
diff --git a/mount.go b/mount.go
|
||||
index f0c3efe..a05d4a6 100644
|
||||
--- a/mount.go
|
||||
+++ b/mount.go
|
||||
@@ -364,7 +364,20 @@ func commonStorageHandler(storage pb.Storage) (string, error) {
|
||||
func mountStorage(storage pb.Storage) error {
|
||||
flags, options := parseMountFlagsAndOptions(storage.Options)
|
||||
|
||||
- return mount(storage.Source, storage.MountPoint, storage.Fstype, flags, options)
|
||||
+ var fsType = storage.Fstype
|
||||
+ if (storage.Driver == driverSCSIType || storage.Driver == driverBlkType) && strings.Contains(storage.Fstype, "bind") {
|
||||
+ cs := strings.Split(storage.Fstype, "-")
|
||||
+ if len(cs) == 2 && cs[1] != "" {
|
||||
+ fsType = cs[1]
|
||||
+ // here we temporarily discard the bind option,
|
||||
+ // in order to be able to mount the file system of the block device.
|
||||
+ // and then reset `storage.Fstype` to "bind" which pass through to the libcontainer pkg.
|
||||
+ flags = flags &^ flagList["bind"]
|
||||
+ storage.Fstype = "bind"
|
||||
+ }
|
||||
+ }
|
||||
+
|
||||
+ return mount(storage.Source, storage.MountPoint, fsType, flags, options)
|
||||
}
|
||||
|
||||
// addStorages takes a list of storages passed by the caller, and perform the
|
||||
--
|
||||
2.14.3 (Apple Git-98)
|
||||
|
||||
@ -1,54 +0,0 @@
|
||||
From bccda1d208f31eab55863883cf0718d7b4b8deef Mon Sep 17 00:00:00 2001
|
||||
From: jiangpengfei <jiangpengfei9@huawei.com>
|
||||
Date: Tue, 18 Aug 2020 19:30:42 +0800
|
||||
Subject: [PATCH 06/16] agent: make workaround for slow response in aarch64
|
||||
|
||||
reason: make workaround for slow response in aarch64
|
||||
when hotplug virtio-net-pci device
|
||||
|
||||
Signed-off-by: jiangpengfei <jiangpengfei9@huawei.com>
|
||||
---
|
||||
agent.go | 2 +-
|
||||
device.go | 12 ++++++------
|
||||
2 files changed, 7 insertions(+), 7 deletions(-)
|
||||
|
||||
diff --git a/agent.go b/agent.go
|
||||
index c161e93..e81c2cd 100644
|
||||
--- a/agent.go
|
||||
+++ b/agent.go
|
||||
@@ -185,7 +185,7 @@ var logsVSockPort = uint32(0)
|
||||
var debugConsoleVSockPort = uint32(0)
|
||||
|
||||
// Timeout waiting for a device to be hotplugged
|
||||
-var hotplugTimeout = 3 * time.Second
|
||||
+var hotplugTimeout = 10 * time.Second
|
||||
|
||||
// Specify the log level
|
||||
var logLevel = defaultLogLevel
|
||||
diff --git a/device.go b/device.go
|
||||
index ec1907e..46a1b96 100644
|
||||
--- a/device.go
|
||||
+++ b/device.go
|
||||
@@ -179,13 +179,13 @@ func getPCIDeviceNameImpl(s *sandbox, pciID string) (string, error) {
|
||||
return "", err
|
||||
}
|
||||
|
||||
- fieldLogger := agentLog.WithField("pciAddr", pciAddr)
|
||||
-
|
||||
// Rescan pci bus if we need to wait for a new pci device
|
||||
- if err = rescanPciBus(); err != nil {
|
||||
- fieldLogger.WithError(err).Error("Failed to scan pci bus")
|
||||
- return "", err
|
||||
- }
|
||||
+ // FIXME:Comment out this code Temporarily, because once the PCIBus is scanned,
|
||||
+ // the device hot-plug event is lost
|
||||
+ //if err = rescanPciBus(); err != nil {
|
||||
+ // fieldLogger.WithError(err).Error("Failed to scan pci bus")
|
||||
+ // return "", err
|
||||
+ //}
|
||||
|
||||
return getDeviceName(s, pciAddr)
|
||||
}
|
||||
--
|
||||
2.14.3 (Apple Git-98)
|
||||
|
||||
@ -1,37 +0,0 @@
|
||||
From fa673c93e243ba297d53b585cd2f51fa68380fc5 Mon Sep 17 00:00:00 2001
|
||||
From: jiangpengfei <jiangpengfei9@huawei.com>
|
||||
Date: Tue, 18 Aug 2020 19:37:48 +0800
|
||||
Subject: [PATCH 07/16] agent: using pcie-root-port driver to hotplug device
|
||||
|
||||
reason: In original pci-bridge scheme, the "F" in the BDF is not used,
|
||||
and was written hard code "0" in the kata-agent. But the "function"
|
||||
is specified when switching to pcie-root-port scheme, so when should
|
||||
pass the "pci-bridge BDF" or "pcie-root-port BDF" from kata-runtime
|
||||
and use in kata-agent.
|
||||
|
||||
Signed-off-by: jiangpengfei <jiangpengfei9@huawei.com>
|
||||
---
|
||||
device.go | 7 ++++++-
|
||||
1 file changed, 6 insertions(+), 1 deletion(-)
|
||||
|
||||
diff --git a/device.go b/device.go
|
||||
index 46a1b96..8e6950c 100644
|
||||
--- a/device.go
|
||||
+++ b/device.go
|
||||
@@ -101,7 +101,12 @@ func getDevicePCIAddressImpl(pciID string) (string, error) {
|
||||
|
||||
// Deduce the complete bridge address based on the bridge address identifier passed
|
||||
// and the fact that bridges are attached on the main bus with function 0.
|
||||
- pciBridgeAddr := fmt.Sprintf("0000:00:%s.0", bridgeID)
|
||||
+ // Update: support pcie-root-port device
|
||||
+ // In original pci-bridge scheme, the "F" in the BDF is not used, and was written
|
||||
+ // hard code "0" in the kata-agent. But the "function" is specified when switching to
|
||||
+ // pcie-root-port scheme, so when should pass the "pci-bridge BDF" or "pcie-root-port BDF"
|
||||
+ // from kata-runtime and use in kata-agent.
|
||||
+ pciBridgeAddr := fmt.Sprintf("0000:00:%s", bridgeID)
|
||||
|
||||
// Find out the bus exposed by bridge
|
||||
bridgeBusPath := fmt.Sprintf(pciBusPathFormat, sysBusPrefix, pciBridgeAddr)
|
||||
--
|
||||
2.14.3 (Apple Git-98)
|
||||
|
||||
@ -1,94 +0,0 @@
|
||||
From eea286fbafba2e95410b603fbef762e2b25eb207 Mon Sep 17 00:00:00 2001
|
||||
From: jiangpengfei <jiangpengfei9@huawei.com>
|
||||
Date: Tue, 18 Aug 2020 19:45:57 +0800
|
||||
Subject: [PATCH 08/16] agent: support get root bus path dynamically
|
||||
|
||||
reason: support get root bus dynamically no matter the
|
||||
target arch is amd64 or arm64
|
||||
|
||||
Signed-off-by: jiangpengfei <jiangpengfei9@huawei.com>
|
||||
---
|
||||
agent.go | 1 +
|
||||
device_amd64.go | 6 +++++-
|
||||
device_arm64.go | 27 ++++++++++++++++++++++++++-
|
||||
3 files changed, 32 insertions(+), 2 deletions(-)
|
||||
|
||||
diff --git a/agent.go b/agent.go
|
||||
index e81c2cd..50afd7a 100644
|
||||
--- a/agent.go
|
||||
+++ b/agent.go
|
||||
@@ -730,6 +730,7 @@ func (s *sandbox) listenToUdevEvents() {
|
||||
defer uEvHandler.Close()
|
||||
|
||||
fieldLogger.Infof("Started listening for uevents")
|
||||
+ rootBusPath := initRootBusPath()
|
||||
|
||||
for {
|
||||
uEv, err := uEvHandler.Read()
|
||||
diff --git a/device_amd64.go b/device_amd64.go
|
||||
index 66bc052..26f55bf 100644
|
||||
--- a/device_amd64.go
|
||||
+++ b/device_amd64.go
|
||||
@@ -8,7 +8,7 @@
|
||||
package main
|
||||
|
||||
const (
|
||||
- rootBusPath = "/devices/pci0000:00"
|
||||
+ defaultRootBusPath = "/devices/pci0000:00"
|
||||
|
||||
// From https://www.kernel.org/doc/Documentation/acpi/namespace.txt
|
||||
// The Linux kernel's core ACPI subsystem creates struct acpi_device
|
||||
@@ -21,3 +21,7 @@ const (
|
||||
// in a subdirectory whose prefix is pfn (page frame number).
|
||||
pfnDevPrefix = "/pfn"
|
||||
)
|
||||
+
|
||||
+func initRootBusPath() string {
|
||||
+ return defaultRootBusPath
|
||||
+}
|
||||
diff --git a/device_arm64.go b/device_arm64.go
|
||||
index b73b582..d039c67 100644
|
||||
--- a/device_arm64.go
|
||||
+++ b/device_arm64.go
|
||||
@@ -6,8 +6,14 @@
|
||||
|
||||
package main
|
||||
|
||||
+import (
|
||||
+ "fmt"
|
||||
+ "io/ioutil"
|
||||
+ "regexp"
|
||||
+)
|
||||
+
|
||||
const (
|
||||
- rootBusPath = "/devices/platform/4010000000.pcie/pci0000:00"
|
||||
+ defaultRootBusPath = "/devices/platform/4010000000.pcie/pci0000:00"
|
||||
|
||||
// From https://www.kernel.org/doc/Documentation/acpi/namespace.txt
|
||||
// The Linux kernel's core ACPI subsystem creates struct acpi_device
|
||||
@@ -20,3 +26,22 @@ const (
|
||||
// in a subdirectory whose prefix is pfn (page frame number).
|
||||
pfnDevPrefix = "/pfn"
|
||||
)
|
||||
+
|
||||
+func initRootBusPath() string {
|
||||
+ pcieDriverReg := regexp.MustCompile(`^[0-9a-f]{10}.pcie$`)
|
||||
+ rootBusPath := defaultRootBusPath
|
||||
+ files, err := ioutil.ReadDir("/sys/devices/platform")
|
||||
+ if err != nil {
|
||||
+ return rootBusPath
|
||||
+ }
|
||||
+ for _, f := range files {
|
||||
+ if !f.IsDir() {
|
||||
+ continue
|
||||
+ }
|
||||
+ if pcieDriverReg.MatchString(f.Name()) {
|
||||
+ rootBusPath = fmt.Sprintf("/devices/platform/%s/pci0000:00", f.Name())
|
||||
+ break
|
||||
+ }
|
||||
+ }
|
||||
+ return rootBusPath
|
||||
+}
|
||||
--
|
||||
2.14.3 (Apple Git-98)
|
||||
|
||||
@ -1,188 +0,0 @@
|
||||
From 1268b710c7f0528d971d9c0e54429d3f4e48c372 Mon Sep 17 00:00:00 2001
|
||||
From: holyfei <yangfeiyu20092010@163.com>
|
||||
Date: Tue, 18 Aug 2020 16:45:29 +0800
|
||||
Subject: [PATCH 09/16] storage: add pkg/storage for mount
|
||||
|
||||
reason: add gpath.go and nfs.go, provide mount
|
||||
functions and structs
|
||||
|
||||
Signed-off-by: yangfeiyu <yangfeiyu2@huawei.com>
|
||||
---
|
||||
pkg/storage/gpath.go | 64 ++++++++++++++++++++++++++++++++++++
|
||||
pkg/storage/nfs.go | 93 ++++++++++++++++++++++++++++++++++++++++++++++++++++
|
||||
2 files changed, 157 insertions(+)
|
||||
create mode 100644 pkg/storage/gpath.go
|
||||
create mode 100644 pkg/storage/nfs.go
|
||||
|
||||
diff --git a/pkg/storage/gpath.go b/pkg/storage/gpath.go
|
||||
new file mode 100644
|
||||
index 0000000..5cb951f
|
||||
--- /dev/null
|
||||
+++ b/pkg/storage/gpath.go
|
||||
@@ -0,0 +1,64 @@
|
||||
+// Copyright (c) Huawei Technologies Co., Ltd. 2019. All rights reserved.
|
||||
+// SPDX-License-Identifier: Apache-2.0
|
||||
+// Description: common functions
|
||||
+// Author: licuifang
|
||||
+// Create: 2019-06-24
|
||||
+
|
||||
+package storage
|
||||
+
|
||||
+import (
|
||||
+ "fmt"
|
||||
+ "os"
|
||||
+ "syscall"
|
||||
+
|
||||
+ "github.com/opencontainers/runc/libcontainer/mount"
|
||||
+)
|
||||
+
|
||||
+func ValidateGpath(guestPath string, opts []string) error {
|
||||
+ for _, opt := range opts {
|
||||
+ switch opt {
|
||||
+ case "shared":
|
||||
+ flag := syscall.MS_SHARED
|
||||
+ if err := ensureMountedAs(guestPath, flag); err != nil {
|
||||
+ return err
|
||||
+ }
|
||||
+ return nil
|
||||
+ case "mounted":
|
||||
+ // if mounted in option, the guestpath must exist and must be a mountpoint
|
||||
+ _, err := os.Stat(guestPath)
|
||||
+ if err != nil {
|
||||
+ return err
|
||||
+ }
|
||||
+ mountpoint, err := mount.Mounted(guestPath)
|
||||
+ if err != nil {
|
||||
+ return err
|
||||
+ }
|
||||
+ if mountpoint == false {
|
||||
+ return fmt.Errorf("the guespath:%s is not a mountpoint while mounted was set", guestPath)
|
||||
+ }
|
||||
+ return nil
|
||||
+ }
|
||||
+ }
|
||||
+ if err := os.MkdirAll(guestPath, 0750); err != nil {
|
||||
+ return err
|
||||
+ }
|
||||
+ return nil
|
||||
+}
|
||||
+
|
||||
+func ensureMountedAs(mountPoint string, flag int) error {
|
||||
+ if err := os.MkdirAll(mountPoint, 0750); err != nil {
|
||||
+ return err
|
||||
+ }
|
||||
+ mounted, err := mount.Mounted(mountPoint)
|
||||
+ if err != nil {
|
||||
+ return err
|
||||
+ }
|
||||
+
|
||||
+ if !mounted {
|
||||
+ if err := syscall.Mount(mountPoint, mountPoint, "bind", uintptr(syscall.MS_BIND), ""); err != nil {
|
||||
+ return err
|
||||
+ }
|
||||
+ }
|
||||
+
|
||||
+ return syscall.Mount("", mountPoint, "", uintptr(flag), "")
|
||||
+}
|
||||
diff --git a/pkg/storage/nfs.go b/pkg/storage/nfs.go
|
||||
new file mode 100644
|
||||
index 0000000..44bc85d
|
||||
--- /dev/null
|
||||
+++ b/pkg/storage/nfs.go
|
||||
@@ -0,0 +1,93 @@
|
||||
+// Copyright (c) Huawei Technologies Co., Ltd. 2019. All rights reserved.
|
||||
+// SPDX-License-Identifier: Apache-2.0
|
||||
+// Description: common functions
|
||||
+// Author: leizhongkai
|
||||
+// Create: 2019-03-10
|
||||
+
|
||||
+package storage
|
||||
+
|
||||
+import (
|
||||
+ "fmt"
|
||||
+ "os"
|
||||
+ "os/exec"
|
||||
+ "path/filepath"
|
||||
+ "strings"
|
||||
+ "time"
|
||||
+
|
||||
+ "github.com/opencontainers/runc/libcontainer/mount"
|
||||
+ "github.com/sirupsen/logrus"
|
||||
+)
|
||||
+
|
||||
+const (
|
||||
+ NFS = "nfs"
|
||||
+
|
||||
+ MAX_MOUNT_ATTEMPTS = 100
|
||||
+ MAX_MOUTN_TIMEOUT = 90 // seconds
|
||||
+
|
||||
+ // ignored errors
|
||||
+ PERMISSION_DENY = "Permission denied"
|
||||
+ kataGuestStorageDir = "/run/kata-containers/storage/containers/"
|
||||
+)
|
||||
+
|
||||
+func nfsMount(source, dest string, opt string) error {
|
||||
+ cmd := exec.Command("/bin/mount", "-t", "nfs", "-o", opt, source, dest)
|
||||
+ res, err := cmd.Output()
|
||||
+ logrus.Debugf("mount %s to %s, and get res %s", source, dest, string(res))
|
||||
+ return err
|
||||
+}
|
||||
+
|
||||
+func nfsMountWithAttempt(source, dest string, opt string) (err error) {
|
||||
+ timeStart := time.Now().Unix()
|
||||
+ for i := 0; i < MAX_MOUNT_ATTEMPTS; i++ {
|
||||
+ logrus.Infof("this is the %d times to mount %s to %s", i, source, dest)
|
||||
+ err = nfsMount(source, dest, opt)
|
||||
+ if err != nil {
|
||||
+ ee, ok := err.(*exec.ExitError)
|
||||
+ if ok {
|
||||
+ if strings.Contains(string(ee.Stderr), PERMISSION_DENY) {
|
||||
+ logrus.Errorf("mounting nfs:%s to %s, get error: %s,should break", source, dest, string(ee.Stderr))
|
||||
+ // We do not retry when the error type is PERMISSION_DENY.
|
||||
+ // The reason for the retry is that when you do a SFS mount,
|
||||
+ // the network may not have been created yet, because
|
||||
+ // creating the network and startup container is asynchronous
|
||||
+ break
|
||||
+ }
|
||||
+ logrus.Infof("mounting nfs:%s to %s, get error: %s,will retry", source, dest, string(ee.Stderr))
|
||||
+ }
|
||||
+
|
||||
+ elapsed := time.Now().Unix() - timeStart
|
||||
+ if elapsed < MAX_MOUTN_TIMEOUT {
|
||||
+ time.Sleep(50 * time.Microsecond)
|
||||
+ continue
|
||||
+ }
|
||||
+ }
|
||||
+ break
|
||||
+ }
|
||||
+
|
||||
+ return err
|
||||
+}
|
||||
+
|
||||
+func MountRemoteNfs(source string, opt []string, sandboxId string) error {
|
||||
+ item := strings.Split(source, ":")
|
||||
+ if len(item) != 2 {
|
||||
+ return fmt.Errorf("the nfs of %s format is error", source)
|
||||
+ }
|
||||
+ tmpDes := filepath.Join(kataGuestStorageDir, sandboxId, item[0],item[1])
|
||||
+ if mounted, err := mount.Mounted(tmpDes); err == nil && mounted == true {
|
||||
+ return nil
|
||||
+ }
|
||||
+ err := os.MkdirAll(tmpDes, 0750)
|
||||
+ if err != nil {
|
||||
+ logrus.Infof("mkdir %s failed before mount remote nfs server", tmpDes)
|
||||
+ return err
|
||||
+ }
|
||||
+ defer func() {
|
||||
+ if err != nil {
|
||||
+ os.RemoveAll(tmpDes)
|
||||
+ }
|
||||
+ }()
|
||||
+
|
||||
+ err = nfsMountWithAttempt(source, tmpDes, strings.Join(opt, ","))
|
||||
+
|
||||
+ return err
|
||||
+}
|
||||
--
|
||||
2.14.3 (Apple Git-98)
|
||||
|
||||
@ -1,92 +0,0 @@
|
||||
From 79bafc5fd8a1dcda6f44ecd830dfe50f4ef1b34b Mon Sep 17 00:00:00 2001
|
||||
From: holyfei <yangfeiyu20092010@163.com>
|
||||
Date: Tue, 18 Aug 2020 20:41:30 +0800
|
||||
Subject: [PATCH 10/16] storage: mount nfs and gpath in agent
|
||||
|
||||
reason: add nfsStorageHandler and gpathStorageHandler in
|
||||
storageHandlerList to mount nfs and gpath
|
||||
|
||||
Signed-off-by: yangfeiyu <yangfeiyu2@huawei.com>
|
||||
---
|
||||
device.go | 2 ++
|
||||
mount.go | 37 +++++++++++++++++++++++++++++++++++++
|
||||
2 files changed, 39 insertions(+)
|
||||
|
||||
diff --git a/device.go b/device.go
|
||||
index 8e6950c..29f72e9 100644
|
||||
--- a/device.go
|
||||
+++ b/device.go
|
||||
@@ -35,6 +35,8 @@ const (
|
||||
driverEphemeralType = "ephemeral"
|
||||
driverLocalType = "local"
|
||||
vmRootfs = "/"
|
||||
+ driverNfsType = "nfs"
|
||||
+ driverGpathType = "gpath"
|
||||
)
|
||||
|
||||
const (
|
||||
diff --git a/mount.go b/mount.go
|
||||
index a05d4a6..de2bfaf 100644
|
||||
--- a/mount.go
|
||||
+++ b/mount.go
|
||||
@@ -17,6 +17,7 @@ import (
|
||||
"strings"
|
||||
"syscall"
|
||||
|
||||
+ rmtStorage "github.com/kata-containers/agent/pkg/storage"
|
||||
pb "github.com/kata-containers/agent/protocols/grpc"
|
||||
"github.com/pkg/errors"
|
||||
"github.com/sirupsen/logrus"
|
||||
@@ -218,6 +219,8 @@ var storageHandlerList = map[string]storageHandler{
|
||||
driverEphemeralType: ephemeralStorageHandler,
|
||||
driverLocalType: localStorageHandler,
|
||||
driverNvdimmType: nvdimmStorageHandler,
|
||||
+ driverNfsType: nfsStorageHandler,
|
||||
+ driverGpathType: gpathStorageHandler,
|
||||
}
|
||||
|
||||
func ephemeralStorageHandler(_ context.Context, storage pb.Storage, s *sandbox) (string, error) {
|
||||
@@ -339,6 +342,40 @@ func nvdimmStorageHandler(_ context.Context, storage pb.Storage, s *sandbox) (st
|
||||
return "", fmt.Errorf("invalid nvdimm source path: %v", storage.Source)
|
||||
}
|
||||
|
||||
+func nfsStorageHandler(_ context.Context, storage pb.Storage, s *sandbox) (string, error) {
|
||||
+ s.Lock()
|
||||
+ defer s.Unlock()
|
||||
+
|
||||
+ // mount nfs
|
||||
+ err := rmtStorage.MountRemoteNfs(storage.Source, storage.Options, s.id)
|
||||
+ if err != nil {
|
||||
+ return "", fmt.Errorf("mount %s to %s failed, get err:%s", storage.Source, storage.MountPoint, err)
|
||||
+ }
|
||||
+ if err := os.MkdirAll(storage.MountPoint, 0750); err != nil {
|
||||
+ logrus.Infof("mkdir %s failed after mount remote nfs server", storage.MountPoint)
|
||||
+ return "", err
|
||||
+ }
|
||||
+
|
||||
+ defer func() {
|
||||
+ if err != nil {
|
||||
+ os.RemoveAll(storage.MountPoint)
|
||||
+ }
|
||||
+ }()
|
||||
+
|
||||
+ return storage.MountPoint, nil
|
||||
+}
|
||||
+
|
||||
+func gpathStorageHandler(_ context.Context, storage pb.Storage, s *sandbox) (string, error) {
|
||||
+ s.Lock()
|
||||
+ defer s.Unlock()
|
||||
+ // validate guespath
|
||||
+ err := rmtStorage.ValidateGpath(storage.Source, storage.Options)
|
||||
+ if err != nil {
|
||||
+ return "", err
|
||||
+ }
|
||||
+ return "", nil
|
||||
+}
|
||||
+
|
||||
// virtioSCSIStorageHandler handles the storage for scsi driver.
|
||||
func virtioSCSIStorageHandler(ctx context.Context, storage pb.Storage, s *sandbox) (string, error) {
|
||||
// Retrieve the device path from SCSI address.
|
||||
--
|
||||
2.14.3 (Apple Git-98)
|
||||
|
||||
@ -1,78 +0,0 @@
|
||||
From 3ac1232a2e3fbfc0465473e5d81cde41847c4252 Mon Sep 17 00:00:00 2001
|
||||
From: jiangpengfei <jiangpengfei9@huawei.com>
|
||||
Date: Wed, 19 Aug 2020 11:47:37 +0800
|
||||
Subject: [PATCH 11/16] agent: fix agent reap agent process blocked problem
|
||||
|
||||
reason: add container waitProcess() timeout when
|
||||
container process status is D/T.
|
||||
|
||||
Signed-off-by: jiangpengfei <jiangpengfei9@huawei.com>
|
||||
---
|
||||
grpc.go | 43 +++++++++++++++++++++++++++++++++----------
|
||||
1 file changed, 33 insertions(+), 10 deletions(-)
|
||||
|
||||
diff --git a/grpc.go b/grpc.go
|
||||
index de2cae7..3dd088e 100644
|
||||
--- a/grpc.go
|
||||
+++ b/grpc.go
|
||||
@@ -49,6 +49,11 @@ const (
|
||||
libcontainerPath = "/run/libcontainer"
|
||||
)
|
||||
|
||||
+// keep waitProcessTimeout value same as value in kata-runtime wait WaitProcessRequest response
|
||||
+const (
|
||||
+ waitProcessTimeOut = 10
|
||||
+)
|
||||
+
|
||||
var (
|
||||
sysfsCPUOnlinePath = "/sys/devices/system/cpu"
|
||||
sysfsMemOnlinePath = "/sys/devices/system/memory"
|
||||
@@ -996,17 +1001,35 @@ func (a *agentGRPC) WaitProcess(ctx context.Context, req *pb.WaitProcessRequest)
|
||||
ctr.deleteProcess(proc.id)
|
||||
})
|
||||
|
||||
- // Using helper function wait() to deal with the subreaper.
|
||||
- libContProcess := (*reaperLibcontainerProcess)(&(proc.process))
|
||||
- exitCode, err := a.sandbox.subreaper.wait(proc.exitCodeCh, libContProcess)
|
||||
- if err != nil {
|
||||
- return &pb.WaitProcessResponse{}, err
|
||||
+ done := make(chan error)
|
||||
+ var exitCode int = 0
|
||||
+ go func() {
|
||||
+ // Using helper function wait() to deal with the subreaper.
|
||||
+ libContProcess := (*reaperLibcontainerProcess)(&(proc.process))
|
||||
+ var err error
|
||||
+ exitCode, err = a.sandbox.subreaper.wait(proc.exitCodeCh, libContProcess)
|
||||
+ if err != nil {
|
||||
+ done <- err
|
||||
+ close(done)
|
||||
+ return
|
||||
+ }
|
||||
+ // refill the exitCodeCh with the exitcode which can be read out
|
||||
+ // by another WaitProcess(). Since this channel isn't be closed,
|
||||
+ // here the refill will always success and it will be free by GC
|
||||
+ // once the process exits.
|
||||
+ proc.exitCodeCh <- exitCode
|
||||
+
|
||||
+ close(done)
|
||||
+ }()
|
||||
+
|
||||
+ select {
|
||||
+ case err := <-done:
|
||||
+ if err != nil {
|
||||
+ return &pb.WaitProcessResponse{}, err
|
||||
+ }
|
||||
+ case <-time.After(time.Duration(waitProcessTimeOut) * time.Second):
|
||||
+ return &pb.WaitProcessResponse{}, grpcStatus.Errorf(codes.DeadlineExceeded, "agent wait reap container process timeout reached after %ds", waitProcessTimeOut)
|
||||
}
|
||||
- //refill the exitCodeCh with the exitcode which can be read out
|
||||
- //by another WaitProcess(). Since this channel isn't be closed,
|
||||
- //here the refill will always success and it will be free by GC
|
||||
- //once the process exits.
|
||||
- proc.exitCodeCh <- exitCode
|
||||
|
||||
return &pb.WaitProcessResponse{
|
||||
Status: int32(exitCode),
|
||||
--
|
||||
2.14.3 (Apple Git-98)
|
||||
|
||||
@ -1,38 +0,0 @@
|
||||
From edb29dfd8f786735763245b3f156b50fd3c1a08e Mon Sep 17 00:00:00 2001
|
||||
From: holyfei <yangfeiyu20092010@163.com>
|
||||
Date: Wed, 19 Aug 2020 15:15:31 +0800
|
||||
Subject: [PATCH 12/16] network: support set dns without nameserver
|
||||
|
||||
reason: when runtime sends dns without nameserver to agent,
|
||||
add nameserver before ip address. scenario like annotation
|
||||
with dns
|
||||
|
||||
Signed-off-by: yangfeiyu <yangfeiyu2@huawei.com>
|
||||
---
|
||||
network.go | 5 ++++-
|
||||
1 file changed, 4 insertions(+), 1 deletion(-)
|
||||
|
||||
diff --git a/network.go b/network.go
|
||||
index 02e28cb..7046cf8 100644
|
||||
--- a/network.go
|
||||
+++ b/network.go
|
||||
@@ -708,6 +708,9 @@ func setupDNS(dns []string) (err error) {
|
||||
defer file.Close()
|
||||
|
||||
for i, line := range dns {
|
||||
+ if !strings.Contains(line, "nameserver") {
|
||||
+ line = "nameserver" + " " + line
|
||||
+ }
|
||||
if i == (len(dns) - 1) {
|
||||
_, err = file.WriteString(strings.TrimSpace(line))
|
||||
} else {
|
||||
@@ -761,4 +764,4 @@ func (s *sandbox) handleLocalhost() error {
|
||||
}
|
||||
|
||||
return netlink.LinkSetUp(lo)
|
||||
-}
|
||||
\ No newline at end of file
|
||||
+}
|
||||
--
|
||||
2.14.3 (Apple Git-98)
|
||||
|
||||
@ -1,304 +0,0 @@
|
||||
From 1394dcf579849e5d8103c31556e9af0216a875d2 Mon Sep 17 00:00:00 2001
|
||||
From: jiangpengfei <jiangpengfei9@huawei.com>
|
||||
Date: Wed, 19 Aug 2020 17:15:51 +0800
|
||||
Subject: [PATCH 13/16] agent: support setting multi queues of interface
|
||||
|
||||
reason: support setting multi queues of a interface
|
||||
when runtime passing Queue in the request.
|
||||
|
||||
Signed-off-by: jiangpengfei <jiangpengfei9@huawei.com>
|
||||
---
|
||||
network.go | 12 +++++-
|
||||
pkg/net/ethtool.go | 110 ++++++++++++++++++++++++++++++++++++++++++++++++++
|
||||
pkg/types/types.pb.go | 82 ++++++++++++++++++++++++++-----------
|
||||
pkg/types/types.proto | 1 +
|
||||
4 files changed, 181 insertions(+), 24 deletions(-)
|
||||
create mode 100644 pkg/net/ethtool.go
|
||||
|
||||
diff --git a/network.go b/network.go
|
||||
index 7046cf8..1baaa2e 100644
|
||||
--- a/network.go
|
||||
+++ b/network.go
|
||||
@@ -17,7 +17,7 @@ import (
|
||||
"syscall"
|
||||
|
||||
"golang.org/x/sys/unix"
|
||||
-
|
||||
+ agentNet "github.com/kata-containers/agent/pkg/net"
|
||||
"github.com/kata-containers/agent/pkg/types"
|
||||
pb "github.com/kata-containers/agent/protocols/grpc"
|
||||
"github.com/sirupsen/logrus"
|
||||
@@ -273,6 +273,16 @@ func (s *sandbox) updateInterface(netHandle *netlink.Handle, iface *types.Interf
|
||||
if err == nil {
|
||||
err = retErr
|
||||
}
|
||||
+
|
||||
+ // if link is up, then set the multi queue to it,
|
||||
+ // the kernel of newer version may set multi queue by itself,
|
||||
+ // but we can not rely on it.
|
||||
+ if err == nil && iface.Queues > 0 {
|
||||
+ if ethErr := agentNet.GetEthtool().SetChannel(iface.Name, iface.Queues); ethErr != nil {
|
||||
+ err = grpcStatus.Errorf(codes.Internal, "Could not set multi queue %d for interface %v: %v",
|
||||
+ iface.Queues, link, ethErr)
|
||||
+ }
|
||||
+ }
|
||||
}()
|
||||
|
||||
fieldLogger.WithField("link", fmt.Sprintf("%+v", link)).Info("Link found")
|
||||
diff --git a/pkg/net/ethtool.go b/pkg/net/ethtool.go
|
||||
new file mode 100644
|
||||
index 0000000..56a1ece
|
||||
--- /dev/null
|
||||
+++ b/pkg/net/ethtool.go
|
||||
@@ -0,0 +1,110 @@
|
||||
+/*
|
||||
+Copyright (c) Huawei Technologies Co., Ltd. 2019. All rights reserved.
|
||||
+SPDX-License-Identifier: Apache-2.0
|
||||
+Description: common functions
|
||||
+Author: fengshaobao
|
||||
+Create: 2019-05-28
|
||||
+*/
|
||||
+
|
||||
+package net
|
||||
+
|
||||
+import (
|
||||
+ "fmt"
|
||||
+ "sync"
|
||||
+ "syscall"
|
||||
+ "unsafe"
|
||||
+
|
||||
+ "github.com/sirupsen/logrus"
|
||||
+)
|
||||
+
|
||||
+const (
|
||||
+ ethtoolGChannels = 0x0000003c /* Get no of channels */
|
||||
+ ethtoolSChannels = 0x0000003d /* Set no of channels */
|
||||
+ ifNameSize = 16
|
||||
+ siocEthtool = 0x8946
|
||||
+)
|
||||
+
|
||||
+var (
|
||||
+ t *Ethtool
|
||||
+ once sync.Once
|
||||
+)
|
||||
+
|
||||
+// Ethtool to set multiqueue of a network interface,
|
||||
+// with the public method "SetChannel" to set queues of an interface.
|
||||
+type Ethtool struct {
|
||||
+ fd int
|
||||
+}
|
||||
+
|
||||
+type ifReq struct {
|
||||
+ ifrName [ifNameSize]byte
|
||||
+ ifrData uintptr
|
||||
+}
|
||||
+
|
||||
+type ethtoolChannels struct {
|
||||
+ // ETHTOOL_{G,S}CHANNELS
|
||||
+ cmd uint32
|
||||
+ // Read only. Maximum number of receive channel the driver support.
|
||||
+ maxRx uint32
|
||||
+ // Read only. Maximum number of transmit channel the driver support
|
||||
+ maxTx uint32
|
||||
+ // Read only. Maximum number of other channel the driver support
|
||||
+ maxOther uint32
|
||||
+ //Read only. Maximum number of combined channel the driver support. Set of queues RX, TX or other
|
||||
+ maxCombined uint32
|
||||
+ // Valid values are in the range 1 to the max_rx
|
||||
+ rxCount uint32
|
||||
+ // Valid values are in the range 1 to the max_tx
|
||||
+ txCount uint32
|
||||
+ // Valid values are in the range 1 to the max_other
|
||||
+ otherCount uint32
|
||||
+ // Valid values are in the range 1 to the max_combined
|
||||
+ combinedCount uint32
|
||||
+}
|
||||
+
|
||||
+// GetEthtool to config multiqueue of a network interface
|
||||
+func GetEthtool() *Ethtool {
|
||||
+ once.Do(func() {
|
||||
+ var err error
|
||||
+ if t, err = newEthtool(); err != nil {
|
||||
+ panic("can not init a socket fd for ethtool")
|
||||
+ }
|
||||
+ })
|
||||
+ return t
|
||||
+}
|
||||
+
|
||||
+func newEthtool() (*Ethtool, error) {
|
||||
+ fd, err := syscall.Socket(syscall.AF_INET, syscall.SOCK_DGRAM, syscall.IPPROTO_IP)
|
||||
+ if err != nil || fd < 0 {
|
||||
+ logrus.Warningf("Can not get socket of inet")
|
||||
+ var newErr error
|
||||
+ fd, newErr = syscall.Socket(syscall.AF_NETLINK, syscall.SOCK_RAW, syscall.NETLINK_GENERIC)
|
||||
+ if newErr != nil || fd < 0 {
|
||||
+ return nil, fmt.Errorf("create inet socket with error: %v, and create netlink socket with error: %v", err, newErr)
|
||||
+ }
|
||||
+ }
|
||||
+
|
||||
+ return &Ethtool{
|
||||
+ fd: int(fd),
|
||||
+ }, nil
|
||||
+}
|
||||
+
|
||||
+// SetChannel Set the queues of a network interface.
|
||||
+// @devName: the network interface name, e.g. eth0.
|
||||
+// @queues: queues for the network interface.
|
||||
+func (e *Ethtool) SetChannel(devName string, queues uint32) error {
|
||||
+ c := ðtoolChannels{
|
||||
+ cmd: ethtoolSChannels,
|
||||
+ combinedCount: queues,
|
||||
+ }
|
||||
+ var ifName [ifNameSize]byte
|
||||
+ copy(ifName[:], []byte(devName))
|
||||
+ ifr := ifReq{
|
||||
+ ifrName: ifName,
|
||||
+ ifrData: uintptr(unsafe.Pointer(&c)),
|
||||
+ }
|
||||
+ _, _, ep := syscall.Syscall(syscall.SYS_IOCTL, uintptr(e.fd), siocEthtool, uintptr(unsafe.Pointer(&ifr)))
|
||||
+ if ep != 0 {
|
||||
+ return syscall.Errno(ep)
|
||||
+ }
|
||||
+ return nil
|
||||
+}
|
||||
diff --git a/pkg/types/types.pb.go b/pkg/types/types.pb.go
|
||||
index 7ea63e3..8b7e2a5 100644
|
||||
--- a/pkg/types/types.pb.go
|
||||
+++ b/pkg/types/types.pb.go
|
||||
@@ -100,6 +100,7 @@ type Interface struct {
|
||||
// list: "veth", "macvtap", "vlan", "macvlan", "tap", ...
|
||||
Type string `protobuf:"bytes,7,opt,name=type,proto3" json:"type,omitempty"`
|
||||
RawFlags uint32 `protobuf:"varint,8,opt,name=raw_flags,json=rawFlags,proto3" json:"raw_flags,omitempty"`
|
||||
+ Queues uint32 `protobuf:"varint,9,opt,name=Queues,proto3" json:"Queues,omitempty"`
|
||||
}
|
||||
|
||||
func (m *Interface) Reset() { *m = Interface{} }
|
||||
@@ -163,6 +164,13 @@ func (m *Interface) GetRawFlags() uint32 {
|
||||
return 0
|
||||
}
|
||||
|
||||
+func (m *Interface) GetQueues() uint32 {
|
||||
+ if m != nil {
|
||||
+ return m.Queues
|
||||
+ }
|
||||
+ return 0
|
||||
+}
|
||||
+
|
||||
type Route struct {
|
||||
Dest string `protobuf:"bytes,1,opt,name=dest,proto3" json:"dest,omitempty"`
|
||||
Gateway string `protobuf:"bytes,2,opt,name=gateway,proto3" json:"gateway,omitempty"`
|
||||
@@ -319,6 +327,11 @@ func (m *Interface) MarshalTo(dAtA []byte) (int, error) {
|
||||
i++
|
||||
i = encodeVarintTypes(dAtA, i, uint64(m.RawFlags))
|
||||
}
|
||||
+ if m.Queues != 0 {
|
||||
+ dAtA[i] = 0x48
|
||||
+ i++
|
||||
+ i = encodeVarintTypes(dAtA, i, uint64(m.Queues))
|
||||
+ }
|
||||
return i, nil
|
||||
}
|
||||
|
||||
@@ -430,6 +443,9 @@ func (m *Interface) Size() (n int) {
|
||||
if m.RawFlags != 0 {
|
||||
n += 1 + sovTypes(uint64(m.RawFlags))
|
||||
}
|
||||
+ if m.Queues != 0 {
|
||||
+ n += 1 + sovTypes(uint64(m.Queues))
|
||||
+ }
|
||||
return n
|
||||
}
|
||||
|
||||
@@ -841,6 +857,25 @@ func (m *Interface) Unmarshal(dAtA []byte) error {
|
||||
break
|
||||
}
|
||||
}
|
||||
+ case 9:
|
||||
+ if wireType != 0 {
|
||||
+ return fmt.Errorf("proto: wrong wireType = %d for field Queues", wireType)
|
||||
+ }
|
||||
+ m.Queues = 0
|
||||
+ for shift := uint(0); ; shift += 7 {
|
||||
+ if shift >= 64 {
|
||||
+ return ErrIntOverflowTypes
|
||||
+ }
|
||||
+ if iNdEx >= l {
|
||||
+ return io.ErrUnexpectedEOF
|
||||
+ }
|
||||
+ b := dAtA[iNdEx]
|
||||
+ iNdEx++
|
||||
+ m.Queues |= (uint32(b) & 0x7F) << shift
|
||||
+ if b < 0x80 {
|
||||
+ break
|
||||
+ }
|
||||
+ }
|
||||
default:
|
||||
iNdEx = preIndex
|
||||
skippy, err := skipTypes(dAtA[iNdEx:])
|
||||
@@ -1155,27 +1190,28 @@ var (
|
||||
func init() { proto.RegisterFile("pkg/types/types.proto", fileDescriptorTypes) }
|
||||
|
||||
var fileDescriptorTypes = []byte{
|
||||
- // 345 bytes of a gzipped FileDescriptorProto
|
||||
- 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x54, 0x91, 0xd1, 0x6a, 0xfa, 0x30,
|
||||
- 0x14, 0xc6, 0xff, 0xb1, 0xb6, 0xda, 0xe3, 0xdf, 0xad, 0x84, 0x4d, 0xc2, 0x06, 0x52, 0xbc, 0x59,
|
||||
- 0xd9, 0x85, 0x03, 0x37, 0x76, 0xbf, 0x5d, 0x08, 0xde, 0x49, 0x5e, 0x60, 0xc4, 0x36, 0x3a, 0xd1,
|
||||
- 0xda, 0xd2, 0x44, 0x8b, 0xec, 0x05, 0x77, 0xb9, 0x47, 0x10, 0x9f, 0x64, 0xe4, 0x24, 0x8a, 0xbb,
|
||||
- 0xd1, 0xef, 0x97, 0x93, 0xd3, 0xef, 0x7c, 0x27, 0x70, 0x5b, 0xae, 0x16, 0x4f, 0x7a, 0x5f, 0x4a,
|
||||
- 0x65, 0x7f, 0x87, 0x65, 0x55, 0xe8, 0x82, 0xfa, 0x08, 0x83, 0x19, 0x84, 0x93, 0xe9, 0x5b, 0x96,
|
||||
- 0x55, 0x52, 0x29, 0xfa, 0x00, 0xc1, 0x5c, 0xe4, 0xcb, 0xf5, 0x9e, 0x91, 0x98, 0x24, 0x57, 0xa3,
|
||||
- 0xeb, 0xa1, 0xed, 0x98, 0x4c, 0xc7, 0x78, 0xcc, 0x5d, 0x99, 0x32, 0x68, 0x09, 0xdb, 0xc3, 0x1a,
|
||||
- 0x31, 0x49, 0x42, 0x7e, 0x42, 0x4a, 0xa1, 0x99, 0x0b, 0xb5, 0x62, 0x1e, 0x1e, 0xa3, 0x1e, 0x1c,
|
||||
- 0x08, 0x84, 0x93, 0x8d, 0x96, 0xd5, 0x5c, 0xa4, 0x92, 0xf6, 0x20, 0xc8, 0xe4, 0x6e, 0x99, 0x4a,
|
||||
- 0x34, 0x09, 0xb9, 0x23, 0xd3, 0xb9, 0x11, 0xb9, 0x74, 0x1f, 0x44, 0x4d, 0x47, 0xd0, 0x39, 0x4f,
|
||||
- 0x27, 0x15, 0xf3, 0x62, 0x2f, 0xe9, 0x8c, 0xa2, 0xf3, 0x54, 0xae, 0xc2, 0x2f, 0x2f, 0xd1, 0x08,
|
||||
- 0xbc, 0x5c, 0x6f, 0x59, 0x33, 0x26, 0x49, 0x93, 0x1b, 0x69, 0x1c, 0x3f, 0x6b, 0x73, 0x81, 0xf9,
|
||||
- 0xd6, 0xd1, 0x92, 0x49, 0x51, 0xa6, 0x4b, 0x2c, 0x04, 0x36, 0x85, 0x43, 0x33, 0x8b, 0xf1, 0x60,
|
||||
- 0x2d, 0x3b, 0x8b, 0xd1, 0xf4, 0x1e, 0xc2, 0x4a, 0xd4, 0x1f, 0xf3, 0xb5, 0x58, 0x28, 0xd6, 0x8e,
|
||||
- 0x49, 0xd2, 0xe5, 0xed, 0x4a, 0xd4, 0x63, 0xc3, 0x83, 0x2f, 0xf0, 0x79, 0xb1, 0xd5, 0x98, 0x22,
|
||||
- 0x93, 0x4a, 0xbb, 0x6c, 0xa8, 0x8d, 0xcf, 0x42, 0x68, 0x59, 0x8b, 0xfd, 0x69, 0x5b, 0x0e, 0x2f,
|
||||
- 0x76, 0xe1, 0xfd, 0xd9, 0x45, 0x0f, 0x02, 0x55, 0x6c, 0xab, 0x54, 0x62, 0x8c, 0x90, 0x3b, 0xa2,
|
||||
- 0x37, 0xe0, 0xab, 0xb4, 0x28, 0x25, 0x06, 0xe9, 0x72, 0x0b, 0x8f, 0x77, 0xd0, 0x3e, 0xbd, 0x10,
|
||||
- 0x0d, 0xa0, 0xb1, 0x7b, 0x89, 0xfe, 0xe1, 0xff, 0x6b, 0x44, 0xde, 0xff, 0x7f, 0x1f, 0xfb, 0xe4,
|
||||
- 0xe7, 0xd8, 0x27, 0x87, 0x63, 0x9f, 0xcc, 0x02, 0x7c, 0xfb, 0xe7, 0xdf, 0x00, 0x00, 0x00, 0xff,
|
||||
- 0xff, 0xb5, 0x52, 0x37, 0xf2, 0x14, 0x02, 0x00, 0x00,
|
||||
+ // 356 bytes of a gzipped FileDescriptorProto
|
||||
+ 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x54, 0x92, 0xc1, 0x4e, 0xc2, 0x40,
|
||||
+ 0x10, 0x86, 0x5d, 0x4a, 0x0b, 0x1d, 0x44, 0x9b, 0x8d, 0x92, 0x8d, 0x26, 0xa4, 0xe1, 0x62, 0xe3,
|
||||
+ 0x01, 0x13, 0x34, 0xde, 0xf5, 0x40, 0xc2, 0x0d, 0xf7, 0x05, 0xcc, 0xd2, 0x2e, 0x48, 0xa0, 0xb4,
|
||||
+ 0xe9, 0xb6, 0x34, 0xc4, 0x17, 0xf4, 0xe8, 0x23, 0x18, 0x9e, 0xc2, 0xa3, 0xd9, 0xd9, 0x85, 0xe0,
|
||||
+ 0x05, 0xfe, 0x6f, 0x67, 0xa7, 0xff, 0xfc, 0xd3, 0xc2, 0x75, 0xbe, 0x5a, 0x3c, 0x94, 0xbb, 0x5c,
|
||||
+ 0x2a, 0xf3, 0x3b, 0xcc, 0x8b, 0xac, 0xcc, 0xa8, 0x8b, 0x30, 0x98, 0x81, 0x3f, 0x99, 0xbe, 0x24,
|
||||
+ 0x49, 0x21, 0x95, 0xa2, 0x77, 0xe0, 0xcd, 0x45, 0xba, 0x5c, 0xef, 0x18, 0x09, 0x49, 0x74, 0x31,
|
||||
+ 0xba, 0x1c, 0x9a, 0x8e, 0xc9, 0x74, 0x8c, 0xc7, 0xdc, 0x96, 0x29, 0x83, 0x96, 0x30, 0x3d, 0xac,
|
||||
+ 0x11, 0x92, 0xc8, 0xe7, 0x07, 0xa4, 0x14, 0x9a, 0xa9, 0x50, 0x2b, 0xe6, 0xe0, 0x31, 0xea, 0xc1,
|
||||
+ 0x2f, 0x01, 0x7f, 0xb2, 0x29, 0x65, 0x31, 0x17, 0xb1, 0xa4, 0x3d, 0xf0, 0x12, 0xb9, 0x5d, 0xc6,
|
||||
+ 0x12, 0x4d, 0x7c, 0x6e, 0x49, 0x77, 0x6e, 0x44, 0x2a, 0xed, 0x03, 0x51, 0xd3, 0x11, 0x74, 0x8e,
|
||||
+ 0xd3, 0x49, 0xc5, 0x9c, 0xd0, 0x89, 0x3a, 0xa3, 0xe0, 0x38, 0x95, 0xad, 0xf0, 0xd3, 0x4b, 0x34,
|
||||
+ 0x00, 0x27, 0x2d, 0x2b, 0xd6, 0x0c, 0x49, 0xd4, 0xe4, 0x5a, 0x6a, 0xc7, 0x8f, 0x5a, 0x5f, 0x60,
|
||||
+ 0xae, 0x71, 0x34, 0xa4, 0x53, 0xe4, 0xf1, 0x12, 0x0b, 0x9e, 0x49, 0x61, 0x51, 0xcf, 0xa2, 0x3d,
|
||||
+ 0x58, 0xcb, 0xcc, 0xa2, 0x35, 0xbd, 0x05, 0xbf, 0x10, 0xf5, 0xfb, 0x7c, 0x2d, 0x16, 0x8a, 0xb5,
|
||||
+ 0x43, 0x12, 0x75, 0x79, 0xbb, 0x10, 0xf5, 0x58, 0xb3, 0xb6, 0x78, 0xab, 0x64, 0x25, 0x15, 0xf3,
|
||||
+ 0xb1, 0x62, 0x69, 0xf0, 0x09, 0x2e, 0xcf, 0xaa, 0x12, 0xd3, 0x25, 0x52, 0x95, 0x36, 0x33, 0x6a,
|
||||
+ 0xed, 0xbf, 0x10, 0xa5, 0xac, 0xc5, 0xee, 0xb0, 0x45, 0x8b, 0x27, 0x3b, 0x72, 0xfe, 0xed, 0xa8,
|
||||
+ 0x07, 0x9e, 0xca, 0xaa, 0x22, 0x96, 0x18, 0xcf, 0xe7, 0x96, 0xe8, 0x15, 0xb8, 0x2a, 0xce, 0x72,
|
||||
+ 0x89, 0x01, 0xbb, 0xdc, 0xc0, 0xfd, 0x0d, 0xb4, 0x0f, 0x6f, 0x8e, 0x7a, 0xd0, 0xd8, 0x3e, 0x05,
|
||||
+ 0x67, 0xf8, 0xff, 0x1c, 0x90, 0xd7, 0xf3, 0xaf, 0x7d, 0x9f, 0x7c, 0xef, 0xfb, 0xe4, 0x67, 0xdf,
|
||||
+ 0x27, 0x33, 0x0f, 0xbf, 0x89, 0xc7, 0xbf, 0x00, 0x00, 0x00, 0xff, 0xff, 0xa5, 0x57, 0x70, 0xc8,
|
||||
+ 0x2c, 0x02, 0x00, 0x00,
|
||||
}
|
||||
diff --git a/pkg/types/types.proto b/pkg/types/types.proto
|
||||
index f6856e1..149df13 100644
|
||||
--- a/pkg/types/types.proto
|
||||
+++ b/pkg/types/types.proto
|
||||
@@ -37,6 +37,7 @@ message Interface {
|
||||
// list: "veth", "macvtap", "vlan", "macvlan", "tap", ...
|
||||
string type = 7;
|
||||
uint32 raw_flags = 8;
|
||||
+ uint32 Queues = 9;
|
||||
}
|
||||
|
||||
message Route {
|
||||
--
|
||||
2.14.3 (Apple Git-98)
|
||||
|
||||
@ -1,56 +0,0 @@
|
||||
From 9e1478d7989fea4ee759cc13009d8de07dac2879 Mon Sep 17 00:00:00 2001
|
||||
From: jiangpengfei <jiangpengfei9@huawei.com>
|
||||
Date: Wed, 19 Aug 2020 17:01:00 +0800
|
||||
Subject: [PATCH 14/16] agent: fix init hugepages failed problem
|
||||
|
||||
reason: fix init hugepages failed problem
|
||||
|
||||
Signed-off-by: jiangpengfei <jiangpengfei9@huawei.com>
|
||||
---
|
||||
.../runc/libcontainer/cgroups/fs/apply_raw.go | 20 ++++++++++++++++++++
|
||||
1 file changed, 20 insertions(+)
|
||||
|
||||
diff --git a/vendor/github.com/opencontainers/runc/libcontainer/cgroups/fs/apply_raw.go b/vendor/github.com/opencontainers/runc/libcontainer/cgroups/fs/apply_raw.go
|
||||
index ec148b4..47aa3c3 100644
|
||||
--- a/vendor/github.com/opencontainers/runc/libcontainer/cgroups/fs/apply_raw.go
|
||||
+++ b/vendor/github.com/opencontainers/runc/libcontainer/cgroups/fs/apply_raw.go
|
||||
@@ -8,11 +8,13 @@ import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sync"
|
||||
+ "time"
|
||||
|
||||
"github.com/opencontainers/runc/libcontainer/cgroups"
|
||||
"github.com/opencontainers/runc/libcontainer/configs"
|
||||
libcontainerUtils "github.com/opencontainers/runc/libcontainer/utils"
|
||||
"github.com/pkg/errors"
|
||||
+ "github.com/sirupsen/logrus"
|
||||
"golang.org/x/sys/unix"
|
||||
)
|
||||
|
||||
@@ -409,3 +411,21 @@ func CheckCpushares(path string, c uint64) error {
|
||||
func (m *Manager) GetCgroups() (*configs.Cgroup, error) {
|
||||
return m.Cgroups, nil
|
||||
}
|
||||
+
|
||||
+func init() {
|
||||
+ go func() {
|
||||
+ var err error
|
||||
+ if len(HugePageSizes) != 0 {
|
||||
+ return
|
||||
+ }
|
||||
+ for i := 0; i < 10; i++ {
|
||||
+ HugePageSizes, err = cgroups.GetHugePageSize()
|
||||
+ if err == nil {
|
||||
+ logrus.Infof("init hugepages ok loop=%d %v", i, err)
|
||||
+ return
|
||||
+ }
|
||||
+ logrus.Errorf("init hugepages failed loop=%d %v", i, err)
|
||||
+ time.Sleep(time.Second)
|
||||
+ }
|
||||
+ }()
|
||||
+}
|
||||
\ No newline at end of file
|
||||
--
|
||||
2.14.3 (Apple Git-98)
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@ -1,268 +0,0 @@
|
||||
From 6120525f81701424e97d453d515d38f14bbe99d9 Mon Sep 17 00:00:00 2001
|
||||
From: holyfei <yangfeiyu20092010@163.com>
|
||||
Date: Wed, 19 Aug 2020 20:25:00 +0800
|
||||
Subject: [PATCH 16/16] clock: synchronizes clock info with proxy
|
||||
|
||||
reason: virtual machine's clock may be incorrect, proxy synchronizes
|
||||
clock info to help virtual machine adjust clock time
|
||||
|
||||
Signed-off-by: yangfeiyu <yangfeiyu2@huawei.com>
|
||||
---
|
||||
pkg/clock/clock_util.go | 47 ++++++++++++++++++++++++
|
||||
sync_clock_server.go | 94 +++++++++++++++++++++++++++++++++++++++++++++++
|
||||
sync_clock_server_test.go | 88 ++++++++++++++++++++++++++++++++++++++++++++
|
||||
3 files changed, 229 insertions(+)
|
||||
create mode 100644 pkg/clock/clock_util.go
|
||||
create mode 100644 sync_clock_server.go
|
||||
create mode 100644 sync_clock_server_test.go
|
||||
|
||||
diff --git a/pkg/clock/clock_util.go b/pkg/clock/clock_util.go
|
||||
new file mode 100644
|
||||
index 0000000..4b78c63
|
||||
--- /dev/null
|
||||
+++ b/pkg/clock/clock_util.go
|
||||
@@ -0,0 +1,47 @@
|
||||
+// Copyright (c) Huawei Technologies Co., Ltd. 2018. All rights reserved.
|
||||
+// SPDX-License-Identifier: Apache-2.0
|
||||
+// Description: sync clock related function
|
||||
+// Author: xueshaojia
|
||||
+// Create: 2018-11-10
|
||||
+
|
||||
+// Package clock provides clock sync functions
|
||||
+package clock
|
||||
+
|
||||
+import (
|
||||
+ "net"
|
||||
+ "syscall"
|
||||
+)
|
||||
+
|
||||
+type TimeValue struct {
|
||||
+ ClientSendTime int64 `json:"client_send_time"`
|
||||
+ ClientArriveTime int64 `json:"client_arrive_time"`
|
||||
+ ServerSendTime int64 `json:"server_send_time"`
|
||||
+ ServerArriveTime int64 `json:"server_arrive_time"`
|
||||
+ Delta int64 `json:"delta"`
|
||||
+}
|
||||
+
|
||||
+const MaxSyncClockByteNum = 400 // sync clock byte num, max=400
|
||||
+const MaxTimeStampSupport = 3155731199000000000 // unit is ns, 2069/12/31 23:59:59
|
||||
+const MinTimeStampSupport = 0 // unit is ns, 1970/1/1 8:0:0
|
||||
+
|
||||
+// getCurrentTimeNs returns UTC time in Ns
|
||||
+func GetCurrentTimeNs() int64 {
|
||||
+ var tv syscall.Timeval
|
||||
+ if err := syscall.Gettimeofday(&tv); err != nil {
|
||||
+ return -1
|
||||
+ }
|
||||
+ return tv.Sec*1000000000 + tv.Usec*1000
|
||||
+}
|
||||
+
|
||||
+// readConnData reads data from stream
|
||||
+func ReadConnData(stream net.Conn) (buf []byte, byteNum int, err error) {
|
||||
+ buf = make([]byte, MaxSyncClockByteNum)
|
||||
+ byteNum, err = stream.Read(buf)
|
||||
+ return buf, byteNum, err
|
||||
+}
|
||||
+
|
||||
+// writeConnData writes data to stream
|
||||
+func WriteConnData(stream net.Conn, buf []byte) error {
|
||||
+ _, err := stream.Write(buf)
|
||||
+ return err
|
||||
+}
|
||||
diff --git a/sync_clock_server.go b/sync_clock_server.go
|
||||
new file mode 100644
|
||||
index 0000000..a8e98c6
|
||||
--- /dev/null
|
||||
+++ b/sync_clock_server.go
|
||||
@@ -0,0 +1,94 @@
|
||||
+// Copyright (c) Huawei Technologies Co., Ltd. 2018. All rights reserved.
|
||||
+// SPDX-License-Identifier: Apache-2.0
|
||||
+// Description: sync clock server related function
|
||||
+// Author: xueshaojia
|
||||
+// Create: 2018-11-10
|
||||
+
|
||||
+package main
|
||||
+
|
||||
+import (
|
||||
+ "encoding/json"
|
||||
+ "fmt"
|
||||
+ "io"
|
||||
+ "net"
|
||||
+ "os/exec"
|
||||
+ "syscall"
|
||||
+
|
||||
+ "github.com/kata-containers/agent/pkg/clock"
|
||||
+)
|
||||
+
|
||||
+// getSyncClockInfo waits for client request
|
||||
+func getSyncClockInfo(stream net.Conn, clockInfo *clock.TimeValue) error {
|
||||
+ buf, byteNum, err := clock.ReadConnData(stream)
|
||||
+ if err != nil {
|
||||
+ return err
|
||||
+ }
|
||||
+ return json.Unmarshal(buf[:byteNum], clockInfo)
|
||||
+}
|
||||
+
|
||||
+// ackSyncClockInfo sends clock info to client
|
||||
+func ackSyncClockInfo(stream net.Conn, clockInfo *clock.TimeValue) error {
|
||||
+ b, err := json.Marshal(clockInfo)
|
||||
+ if err != nil {
|
||||
+ return err
|
||||
+ }
|
||||
+ return clock.WriteConnData(stream, b)
|
||||
+}
|
||||
+
|
||||
+// setGuestClock sets systemtime and hwclock
|
||||
+func setGuestClock(delta int64) error {
|
||||
+ dstClockNs := clock.GetCurrentTimeNs() + delta
|
||||
+ if dstClockNs < clock.MinTimeStampSupport || dstClockNs > clock.MaxTimeStampSupport {
|
||||
+ return fmt.Errorf("sync clock failed, the valid timestamp is from 1970/1/1 8:0:0 to 2069/12/31 23:59:59")
|
||||
+ }
|
||||
+ dstTimeVal := syscall.NsecToTimeval(dstClockNs)
|
||||
+ if err := syscall.Settimeofday(&dstTimeVal); err != nil {
|
||||
+ agentLog.WithError(err).Error("sync clock, set systemtime failed")
|
||||
+ return err
|
||||
+ }
|
||||
+ cmd := exec.Command("hwclock", "-w")
|
||||
+ if err := cmd.Run(); err != nil {
|
||||
+ agentLog.WithError(err).Error("sync clock, set hwclock failed")
|
||||
+ return err
|
||||
+ }
|
||||
+ agentLog.Infof("sync clock, set systemtime and hwclock succ, delta:%d", delta)
|
||||
+ return nil
|
||||
+}
|
||||
+
|
||||
+// SyncClock is a loop that deals with client's request
|
||||
+// 1 read client request
|
||||
+// 2 response with clock info or set guest clock
|
||||
+// 3 if error happends, print error info
|
||||
+func SyncClock(stream net.Conn) {
|
||||
+ for {
|
||||
+ var clockInfo clock.TimeValue
|
||||
+ if err := getSyncClockInfo(stream, &clockInfo); err != nil {
|
||||
+ if err == io.EOF {
|
||||
+ agentLog.WithError(err).Error("sync clock, yamux session error happends")
|
||||
+ stream.Close()
|
||||
+ break
|
||||
+ }
|
||||
+ agentLog.WithError(err).Error("sync clock, read sync clock info failed")
|
||||
+ continue
|
||||
+ }
|
||||
+ // set client's request arrive time
|
||||
+ clockInfo.ClientArriveTime = clock.GetCurrentTimeNs()
|
||||
+ if clockInfo.Delta == 0 {
|
||||
+ // this is a request for clock diff info
|
||||
+ // set server's response send time
|
||||
+ // if fails, wait for next time
|
||||
+ clockInfo.ServerSendTime = clock.GetCurrentTimeNs()
|
||||
+ if err := ackSyncClockInfo(stream, &clockInfo); err != nil {
|
||||
+ agentLog.WithError(err).Error("sync clock, send back clock info failed")
|
||||
+ }
|
||||
+ } else {
|
||||
+ // this is a request for adjusting guest clock
|
||||
+ // 1 adjust guest clock(system time and hwclock)
|
||||
+ // 2 do not send response any more
|
||||
+ // 3 if fails, wait for next request to adjust
|
||||
+ if err := setGuestClock(clockInfo.Delta); err != nil {
|
||||
+ agentLog.WithError(err).Error("sync clock, set local clock failed")
|
||||
+ }
|
||||
+ }
|
||||
+ }
|
||||
+}
|
||||
diff --git a/sync_clock_server_test.go b/sync_clock_server_test.go
|
||||
new file mode 100644
|
||||
index 0000000..41d5227
|
||||
--- /dev/null
|
||||
+++ b/sync_clock_server_test.go
|
||||
@@ -0,0 +1,88 @@
|
||||
+// Copyright (c) Huawei Technologies Co., Ltd. 2018. All rights reserved.
|
||||
+// SPDX-License-Identifier: Apache-2.0
|
||||
+// Description: sync clock server related test
|
||||
+// Author: xueshaojia
|
||||
+// Create: 2018-11-10
|
||||
+
|
||||
+package main
|
||||
+
|
||||
+import (
|
||||
+ "encoding/json"
|
||||
+ "fmt"
|
||||
+ "math/rand"
|
||||
+ "net"
|
||||
+ "testing"
|
||||
+
|
||||
+ "github.com/hashicorp/yamux"
|
||||
+ "github.com/kata-containers/agent/pkg/clock"
|
||||
+)
|
||||
+
|
||||
+var clientStream net.Conn
|
||||
+
|
||||
+func TestGetCurrentTimeNs(t *testing.T) {
|
||||
+ timeRet := clock.GetCurrentTimeNs()
|
||||
+ if timeRet <= 0 {
|
||||
+ t.Fatalf("TestGetCurrentTimeNs Fail, time:%d", timeRet)
|
||||
+ }
|
||||
+}
|
||||
+
|
||||
+func TestSetGuestClock(t *testing.T) {
|
||||
+ if err := setGuestClock(0); err != nil {
|
||||
+ t.Fatalf("test set clock failed, err:%v", err)
|
||||
+ }
|
||||
+}
|
||||
+
|
||||
+func TestReadWriteData(t *testing.T) {
|
||||
+ sock := GenSocket()
|
||||
+ var err error
|
||||
+ listener, err := net.Listen("unix", sock)
|
||||
+ if err != nil {
|
||||
+ t.Fatalf("listening err: %v", err)
|
||||
+ }
|
||||
+ go SetUpClient(sock)
|
||||
+ conn, err := listener.Accept()
|
||||
+ if err != nil {
|
||||
+ t.Fatalf("accept err: %v", err)
|
||||
+ }
|
||||
+ session, err := yamux.Server(conn, nil)
|
||||
+ if err != nil {
|
||||
+ t.Fatalf("session err: %v", err)
|
||||
+ }
|
||||
+ stream, err := session.Accept()
|
||||
+ if err != nil {
|
||||
+ t.Fatalf("stream err: %v", err)
|
||||
+ }
|
||||
+ go func() {
|
||||
+ var clockInfo clock.TimeValue = clock.TimeValue{1000, 0, 0, 0, 0}
|
||||
+ b, _ := json.Marshal(clockInfo)
|
||||
+ clock.WriteConnData(clientStream, b)
|
||||
+ }()
|
||||
+ _, _, err = clock.ReadConnData(stream)
|
||||
+ if err != nil {
|
||||
+ t.Fatalf("read conn data error, err:%v", err)
|
||||
+ }
|
||||
+}
|
||||
+
|
||||
+func SetUpClient(sock string) error {
|
||||
+ conn, err := net.Dial("unix", sock)
|
||||
+ if err != nil {
|
||||
+ return err
|
||||
+ }
|
||||
+ session, err := yamux.Client(conn, nil)
|
||||
+ if err != nil {
|
||||
+ conn.Close()
|
||||
+ return err
|
||||
+ }
|
||||
+ stream, err := session.Open()
|
||||
+ if err != nil {
|
||||
+ return err
|
||||
+ }
|
||||
+ clientStream = stream
|
||||
+ return nil
|
||||
+}
|
||||
+
|
||||
+func GenSocket() string {
|
||||
+ randSeed := clock.GetCurrentTimeNs()
|
||||
+ rand.Seed(randSeed)
|
||||
+ return fmt.Sprintf("/tmp/%d.sock", rand.Uint32())
|
||||
+}
|
||||
--
|
||||
2.14.3 (Apple Git-98)
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@ -1,184 +0,0 @@
|
||||
From 629aac1078bdeed63214c58d2110fe672d654774 Mon Sep 17 00:00:00 2001
|
||||
From: yangfeiyu <yangfeiyu2@huawei.com>
|
||||
Date: Mon, 26 Oct 2020 20:39:05 +0800
|
||||
Subject: [PATCH] kata-agent: update nic in guest
|
||||
|
||||
reason: add linkByName and support retry of list ip link, because in some scenarios,
|
||||
the hardware address cannot be queried immediately after the hot plug.
|
||||
|
||||
Signed-off-by: yangfeiyu <yangfeiyu2@huawei.com>
|
||||
---
|
||||
network.go | 91 ++++++++++++++++++++++++++++++++++++++++++------------
|
||||
1 file changed, 72 insertions(+), 19 deletions(-)
|
||||
|
||||
diff --git a/network.go b/network.go
|
||||
index f6e2c17..d928e2b 100644
|
||||
--- a/network.go
|
||||
+++ b/network.go
|
||||
@@ -15,13 +15,14 @@ import (
|
||||
"strings"
|
||||
"sync"
|
||||
"syscall"
|
||||
+ "time"
|
||||
|
||||
- "golang.org/x/sys/unix"
|
||||
agentNet "github.com/kata-containers/agent/pkg/net"
|
||||
"github.com/kata-containers/agent/pkg/types"
|
||||
pb "github.com/kata-containers/agent/protocols/grpc"
|
||||
"github.com/sirupsen/logrus"
|
||||
"github.com/vishvananda/netlink"
|
||||
+ "golang.org/x/sys/unix"
|
||||
"google.golang.org/grpc/codes"
|
||||
grpcStatus "google.golang.org/grpc/status"
|
||||
)
|
||||
@@ -32,6 +33,7 @@ var (
|
||||
errNoLink = grpcStatus.Errorf(codes.InvalidArgument, "Need network link")
|
||||
errNoMAC = grpcStatus.Errorf(codes.InvalidArgument, "Need hardware address")
|
||||
errNoRoutes = grpcStatus.Errorf(codes.InvalidArgument, "Need network routes")
|
||||
+ errNoName = grpcStatus.Errorf(codes.InvalidArgument, "Need network name")
|
||||
guestDNSFile = "/etc/resolv.conf"
|
||||
kataGuestSandboxDNSFile = "/run/kata-containers/sandbox/resolv.conf"
|
||||
)
|
||||
@@ -46,6 +48,8 @@ const (
|
||||
|
||||
// Use the below address for ipv6 gateway once ipv6 support is added
|
||||
// defaultV6RouteIP = "::"
|
||||
+
|
||||
+ maxLinkRetries = 10
|
||||
)
|
||||
|
||||
// Network fully describes a sandbox network with its interfaces, routes and dns
|
||||
@@ -100,17 +104,36 @@ func linkByHwAddr(netHandle *netlink.Handle, hwAddr string) (netlink.Link, error
|
||||
return nil, grpcStatus.Errorf(codes.NotFound, "Could not find the link corresponding to HwAddr %q", hwAddr)
|
||||
}
|
||||
|
||||
-func updateLink(netHandle *netlink.Handle, link netlink.Link, iface *types.Interface) error {
|
||||
+func linkByName(netHandle *netlink.Handle, name string) (netlink.Link, error) {
|
||||
if netHandle == nil {
|
||||
- return errNoHandle
|
||||
+ return nil, errNoHandle
|
||||
}
|
||||
|
||||
- if link == nil {
|
||||
- return errNoLink
|
||||
+ if name == "" {
|
||||
+ return nil, errNoName
|
||||
}
|
||||
|
||||
- if iface == nil {
|
||||
- return errNoIF
|
||||
+ links, err := netHandle.LinkList()
|
||||
+ if err != nil {
|
||||
+ return nil, err
|
||||
+ }
|
||||
+
|
||||
+ for _, link := range links {
|
||||
+ if link == nil {
|
||||
+ continue
|
||||
+ }
|
||||
+
|
||||
+ if link.Attrs() != nil && link.Attrs().Name == name {
|
||||
+ return link, nil
|
||||
+ }
|
||||
+ }
|
||||
+
|
||||
+ return nil, grpcStatus.Errorf(codes.NotFound, "Could not find the link corresponding to name %s", name)
|
||||
+}
|
||||
+
|
||||
+func updateLinkIP(netHandle *netlink.Handle, link netlink.Link, iface *types.Interface) error {
|
||||
+ if len(iface.IPAddresses) == 0 {
|
||||
+ return nil
|
||||
}
|
||||
|
||||
// As a first step, clear out any existing addresses associated with the link:
|
||||
@@ -129,13 +152,6 @@ func updateLink(netHandle *netlink.Handle, link netlink.Link, iface *types.Inter
|
||||
netlinkAddrStr := fmt.Sprintf("%s/%s", addr.Address, addr.Mask)
|
||||
netlinkAddr, err := netlink.ParseAddr(netlinkAddrStr)
|
||||
|
||||
- // With ipv6 addresses, there is a brief period during which the address is marked as "tentative"
|
||||
- // making it unavailable. A process called duplicate address detection(DAD) is performed during this period.
|
||||
- // Disble DAD so that networking is available once the container is up. The assumption is
|
||||
- // that it is the reponsibility of the upper stack to make sure the addresses assigned to containers
|
||||
- // do not conflict. A similar operation is performed by libnetwork:
|
||||
- // https://github.com/moby/moby/issues/18871
|
||||
-
|
||||
if addr.GetFamily() == types.IPFamily_v6 {
|
||||
netlinkAddr.Flags = netlinkAddr.Flags | syscall.IFA_F_NODAD
|
||||
}
|
||||
@@ -150,14 +166,36 @@ func updateLink(netHandle *netlink.Handle, link netlink.Link, iface *types.Inter
|
||||
}
|
||||
}
|
||||
|
||||
+ return nil
|
||||
+}
|
||||
+
|
||||
+func updateLink(netHandle *netlink.Handle, link netlink.Link, iface *types.Interface) error {
|
||||
+ if netHandle == nil {
|
||||
+ return errNoHandle
|
||||
+ }
|
||||
+
|
||||
+ if link == nil {
|
||||
+ return errNoLink
|
||||
+ }
|
||||
+
|
||||
+ if iface == nil {
|
||||
+ return errNoIF
|
||||
+ }
|
||||
+
|
||||
+ if err := updateLinkIP(netHandle, link, iface); err != nil {
|
||||
+ return err
|
||||
+ }
|
||||
+
|
||||
// set the interface name:
|
||||
if err := netHandle.LinkSetName(link, iface.Name); err != nil {
|
||||
return grpcStatus.Errorf(codes.Internal, "Could not set name %s for interface %v: %v", iface.Name, link, err)
|
||||
}
|
||||
|
||||
// set the interface MTU:
|
||||
- if err := netHandle.LinkSetMTU(link, int(iface.Mtu)); err != nil {
|
||||
- return grpcStatus.Errorf(codes.Internal, "Could not set MTU %d for interface %v: %v", iface.Mtu, link, err)
|
||||
+ if iface.Mtu > 0 {
|
||||
+ if err := netHandle.LinkSetMTU(link, int(iface.Mtu)); err != nil {
|
||||
+ return grpcStatus.Errorf(codes.Internal, "Could not set MTU %d for interface %v: %v", iface.Mtu, link, err)
|
||||
+ }
|
||||
}
|
||||
|
||||
if iface.RawFlags&unix.IFF_NOARP == uint32(unix.IFF_NOARP) {
|
||||
@@ -306,8 +344,23 @@ func (s *sandbox) updateInterface(netHandle *netlink.Handle, iface *types.Interf
|
||||
if iface.HwAddr != "" {
|
||||
fieldLogger.Info("Getting interface from MAC address")
|
||||
|
||||
- // Find the interface link from its hardware address.
|
||||
- link, err = linkByHwAddr(netHandle, iface.HwAddr)
|
||||
+ // In some scenarios, the hardware address cannot be queried immediately
|
||||
+ // after the hot plug, add a retry here.
|
||||
+ for retry := 0; retry < maxLinkRetries; retry++ {
|
||||
+ // Find the interface link from its hardware address.
|
||||
+ link, err = linkByHwAddr(netHandle, iface.HwAddr)
|
||||
+ if err != nil {
|
||||
+ time.Sleep(100 * time.Millisecond)
|
||||
+ continue
|
||||
+ }
|
||||
+ break
|
||||
+ }
|
||||
+ if err != nil {
|
||||
+ return nil, grpcStatus.Errorf(codes.Internal, "updateInterface: %v", err)
|
||||
+ }
|
||||
+ } else if iface.Name != "" {
|
||||
+ fieldLogger.Info("Getting interface from name")
|
||||
+ link, err = linkByName(netHandle, iface.Name)
|
||||
if err != nil {
|
||||
return nil, grpcStatus.Errorf(codes.Internal, "updateInterface: %v", err)
|
||||
}
|
||||
@@ -487,7 +540,7 @@ func (s *sandbox) updateRoutes(netHandle *netlink.Handle, requestedRoutes *pb.Ro
|
||||
}()
|
||||
|
||||
var (
|
||||
- added []*types.Route
|
||||
+ added []*types.Route
|
||||
removed []*types.Route
|
||||
)
|
||||
|
||||
--
|
||||
2.23.0
|
||||
|
||||
@ -1,18 +0,0 @@
|
||||
0001-agent-add-agent.netlink_recv_buf_size-flag-to-set-ne.patch
|
||||
0002-network-support-update-routes-incrementally.patch
|
||||
0003-kata-agent-add-kata-ipvs-command.patch
|
||||
0004-agent-add-IPVS-test.patch
|
||||
0005-mount-support-mount-block-device.patch
|
||||
0006-agent-make-workaround-for-slow-response-in-aarch64.patch
|
||||
0007-agent-using-pcie-root-port-driver-to-hotplug-device.patch
|
||||
0008-agent-support-get-root-bus-path-dynamically.patch
|
||||
0009-storage-add-pkg-storage-for-mount.patch
|
||||
0010-storage-mount-nfs-and-gpath-in-agent.patch
|
||||
0011-agent-fix-agent-reap-agent-process-blocked-problem.patch
|
||||
0012-network-support-set-dns-without-nameserver.patch
|
||||
0013-agent-support-setting-multi-queues-of-interface.patch
|
||||
0014-agent-fix-init-hugepages-failed-problem.patch
|
||||
0015-agent-add-support-of-getting-container-s-network-sta.patch
|
||||
0016-clock-synchronizes-clock-info-with-proxy.patch
|
||||
0017-agent-add-support-of-new-sandbox-StratoVirt.patch
|
||||
0018-kata-agent-update-nic-in-guest.patch
|
||||
@ -1,20 +1,20 @@
|
||||
#!/bin/bash
|
||||
|
||||
set -e
|
||||
|
||||
if [[ -f ./patch_flag ]];then
|
||||
echo "shim patched!"
|
||||
echo "patched!"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
tar -zxvf shim-*.tar.gz
|
||||
cp -fr ./shim-*/* ./
|
||||
rm -rf ./shim-*
|
||||
tar -zxvf kata-containers-2.1.0.tar.gz
|
||||
cp -rf ./kata-containers-2.1.0/* ./
|
||||
cat ./series.conf | while read line
|
||||
do
|
||||
if [[ $line == '' || $line =~ ^\s*# ]]; then
|
||||
continue
|
||||
fi
|
||||
echo "====patch $line======"
|
||||
pwd
|
||||
patch -p1 -F1 -s < ./patches/$line
|
||||
done
|
||||
|
||||
Binary file not shown.
@ -1,8 +1,8 @@
|
||||
#needsrootforbuild
|
||||
%global debug_package %{nil}
|
||||
|
||||
%define VERSION v1.11.1
|
||||
%define RELEASE 12
|
||||
%define VERSION 2.1.0
|
||||
%define RELEASE 29
|
||||
|
||||
Name: kata-containers
|
||||
Version: %{VERSION}
|
||||
@ -10,13 +10,13 @@ Release: %{RELEASE}
|
||||
Summary: Kata Container, the speed of containers, the security of VMs
|
||||
License: ISC and Apache-2.0 and GPL-2.0-or-later and CC-BY-SA-4.0 and MIT and MPL-2.0
|
||||
URL: https://github.com/kata-containers
|
||||
Source0: kata_integration-v1.0.0.tar.gz
|
||||
Source1: kata-containers-%{version}.tar.gz
|
||||
Source0: kata_integration-openeuler.tar.gz
|
||||
Source1: kata-containers-openeuler.tar.gz
|
||||
Source2: kernel.tar.gz
|
||||
|
||||
BuildRoot: %_topdir/BUILDROOT
|
||||
BuildRequires: automake golang gcc bc glibc-devel glibc-static busybox glib2-devel glib2 ipvsadm conntrack-tools nfs-utils
|
||||
BuildRequires: patch elfutils-libelf-devel openssl-devel bison flex
|
||||
BuildRequires: patch elfutils-libelf-devel openssl-devel bison flex rust cargo rust-packaging libgcc dtc-devel
|
||||
|
||||
%description
|
||||
This is core component of Kata Container, to make it work, you need a isulad/docker engine.
|
||||
@ -31,14 +31,6 @@ cd %{_builddir}/kata_integration
|
||||
# apply kata_integration patches
|
||||
sh apply-patches
|
||||
|
||||
# mv build components into kata_integration dir
|
||||
pushd %{_builddir}/kata_integration
|
||||
mv ../kata-containers-%{version}/runtime .
|
||||
mv ../kata-containers-%{version}/agent .
|
||||
mv ../kata-containers-%{version}/proxy .
|
||||
mv ../kata-containers-%{version}/shim .
|
||||
popd
|
||||
|
||||
# build kernel
|
||||
cd %{_builddir}/kernel
|
||||
mv kernel linux
|
||||
@ -53,12 +45,31 @@ cp %{_builddir}/kata_integration/hack/config-kata-arm64 ./.config
|
||||
cd %{_builddir}/kernel/linux/
|
||||
make %{?_smp_mflags}
|
||||
|
||||
mv %{_builddir}/kata-containers-%{version} %{_builddir}/kata-containers
|
||||
cd %{_builddir}/kata-containers/
|
||||
sh -x apply-patches
|
||||
cd %{_builddir}/kata-containers/src/runtime
|
||||
make clean
|
||||
make
|
||||
|
||||
cd %{_builddir}/kata-containers/src/agent
|
||||
mkdir vendor && tar -xzf %{_builddir}/kata-containers/vendor.tar.gz -C vendor/
|
||||
cp -f ./vendor/version.rs ./src/
|
||||
cat > .cargo/config << EOF
|
||||
[build]
|
||||
rustflags = ["-Clink-arg=-s","-Clink-arg=-lgcc","-Clink-arg=-lfdt"]
|
||||
|
||||
[source.crates-io]
|
||||
replace-with = "vendored-sources"
|
||||
|
||||
[source.vendored-sources]
|
||||
directory = "vendor"
|
||||
EOF
|
||||
/usr/bin/env CARGO_HOME=.cargo RUSTC_BOOTSTRAP=1 cargo build --release
|
||||
cd %{_builddir}/kata_integration
|
||||
mkdir -p -m 750 build
|
||||
export GO111MODULE=off
|
||||
make runtime
|
||||
make proxy
|
||||
make shim
|
||||
cp %{_builddir}/kata-containers/src/agent/target/release/kata-agent ./build/
|
||||
strip ./build/kata-agent
|
||||
make initrd
|
||||
|
||||
%install
|
||||
@ -71,34 +82,205 @@ install -p -m 755 -D %{_builddir}/kernel/linux/arch/arm64/boot/Image %{buildroot
|
||||
|
||||
cd %{_builddir}/kata_integration
|
||||
mkdir -p -m 750 %{buildroot}/usr/bin
|
||||
install -p -m 750 ./build/kata-runtime ./build/kata-proxy ./build/kata-shim ./build/kata-netmon %{buildroot}/usr/bin/
|
||||
install -p -m 750 %{_builddir}/kata-containers/src/runtime/kata-runtime %{buildroot}/usr/bin/
|
||||
install -p -m 750 %{_builddir}/kata-containers/src/runtime/kata-netmon %{buildroot}/usr/bin/
|
||||
install -p -m 750 %{_builddir}/kata-containers/src/runtime/kata-monitor %{buildroot}/usr/bin/
|
||||
install -p -m 750 %{_builddir}/kata-containers/src/runtime/containerd-shim-kata-v2 %{buildroot}/usr/bin/
|
||||
install -p -m 640 -D %{_builddir}/kata-containers/src/runtime/cli/config/configuration-qemu.toml %{buildroot}/usr/share/defaults/kata-containers/configuration.toml
|
||||
install -p -m 640 -D %{_builddir}/kata-containers/src/runtime/cli/config/configuration-stratovirt.toml %{buildroot}/usr/share/defaults/kata-containers/configuration-stratovirt.toml
|
||||
install -p -m 640 ./build/kata-containers-initrd.img %{buildroot}/var/lib/kata/
|
||||
mkdir -p -m 750 %{buildroot}/usr/share/defaults/kata-containers/
|
||||
install -p -m 640 -D ./runtime/cli/config/configuration-qemu.toml %{buildroot}/usr/share/defaults/kata-containers/configuration.toml
|
||||
strip %{buildroot}/usr/bin/kata*
|
||||
strip %{buildroot}/usr/bin/containerd-shim-kata-v2
|
||||
|
||||
%clean
|
||||
|
||||
%files
|
||||
/usr/bin/kata-runtime
|
||||
/usr/bin/kata-proxy
|
||||
/usr/bin/kata-shim
|
||||
/usr/bin/kata-netmon
|
||||
/usr/bin/kata-monitor
|
||||
/usr/bin/containerd-shim-kata-v2
|
||||
/var/lib/kata/kernel
|
||||
/var/lib/kata/kata-containers-initrd.img
|
||||
%config(noreplace) /usr/share/defaults/kata-containers/configuration.toml
|
||||
%config(noreplace) /usr/share/defaults/kata-containers/configuration-stratovirt.toml
|
||||
|
||||
%doc
|
||||
|
||||
|
||||
%changelog
|
||||
* Tue May 17 2022 liukuo <liukuo@kylinos.cn> - v1.11.1-12
|
||||
- License compliance rectification
|
||||
|
||||
* Wed Apr 13 2022 yangfeiyu <yangfeiyu2@huawei.com> - 1.11.1-11
|
||||
* Fri Sep 2 2022 chengzeruizhi <chengzeruizhi@huawei.com> - 2.1.0-29
|
||||
- Type:bugfix
|
||||
- ID:NA
|
||||
- SUG:NA
|
||||
- DESC:add export GO111MODULE=off
|
||||
- DESC:don't use props for object add
|
||||
|
||||
* Tue Aug 23 2022 chengzeruizhi <chengzeruizhi@huawei.com> - 2.1.0-28
|
||||
- Type:bugfix
|
||||
- ID:NA
|
||||
- SUG:NA
|
||||
- DESC:add explicit on for kernel_irqchip
|
||||
|
||||
* Mon Aug 22 2022 chengzeruizhi <chengzeruizhi@huawei.com> - 2.1.0-27
|
||||
- Type:bugfix
|
||||
- ID:NA
|
||||
- SUG:NA
|
||||
- DESC:use host device when adding block dev
|
||||
|
||||
* Fri Mar 18 2022 Xinle.Guo <guoxinle1@huawei.com> - 2.1.0-26
|
||||
- Type:bugfix
|
||||
- ID:NA
|
||||
- SUG:NA
|
||||
- DESC:fix startup failure that adding more than 16 root port devices in stratovirt
|
||||
|
||||
* Tues Mar 2 2022 Xinle.Guo <guoxinle1@huawei.com> - 2.1.0-25
|
||||
- Type:feature
|
||||
- ID:NA
|
||||
- SUG:NA
|
||||
- DESC:provide a way to dynomically obtain firmware in stratovirt
|
||||
|
||||
* Sat Feb 26 2022 Xinle.Guo <guoxinle1@huawei.com> - 2.1.0-24
|
||||
- Type:bugfix
|
||||
- ID:NA
|
||||
- SUG:NA
|
||||
- DESC:fix the problem that fails to plug net device to stratovirt
|
||||
|
||||
* Mon Jan 17 2022 Xinle.Guo <guoxinle1@huawei.com> - 2.1.0-23
|
||||
- Type:feature
|
||||
- ID:NA
|
||||
- SUG:NA
|
||||
- DESC:add the stratovirt standardVM sandbox type to kata container
|
||||
|
||||
* Thur Jan 13 2022 Xinle.Guo <guoxinle1@huawei.com> - 2.1.0-22
|
||||
- Type:feature
|
||||
- ID:NA
|
||||
- SUG:NA
|
||||
- DESC:refactor hypervisor type `stratovirt` and its methods
|
||||
|
||||
* Tues Jan 11 2022 Xinle.Guo <guoxinle1@huawei.com> - 2.1.0-21
|
||||
- Type:feature
|
||||
- ID:NA
|
||||
- SUG:NA
|
||||
- DESC:add stratovirt `vmConfig` struct and methods to get parameters of VM
|
||||
|
||||
* Mon Jan 10 2022 Xinle.Guo <guoxinle1@huawei.com> - 2.1.0-20
|
||||
- Type:feature
|
||||
- ID:NA
|
||||
- SUG:NA
|
||||
- DESC:update stratovirt configuration toml file
|
||||
|
||||
* Fri Dec 10 2021 yangfeiyu <yangfeiyu2@huawei.com> - 2.1.0-19
|
||||
- Type:bugfix
|
||||
- ID:NA
|
||||
- SUG:NA
|
||||
- DESC:modify stratovirt config file
|
||||
|
||||
* Tue Dec 7 2021 jikui <jikui2@huawei.com> - 2.1.0-18
|
||||
- Type:bugfix
|
||||
- ID:NA
|
||||
- SUG:NA
|
||||
- DESC:fix the block device not removed in devManager
|
||||
|
||||
* Thu Dec 2 2021 jikui <jikui2@huawei.com> - 2.1.0-17
|
||||
- Type:bugfix
|
||||
- ID:NA
|
||||
- SUG:NA
|
||||
- DESC:don't ignore updateInterface return error
|
||||
|
||||
* Tue Nov 30 2021 jikui <jikui2@huawei.com> - 2.1.0-16
|
||||
- Type:bugfix
|
||||
- ID:NA
|
||||
- SUG:NA
|
||||
- DESC:don't delete the exist tap device in the host
|
||||
|
||||
* Tue Nov 30 2021 jikui <jikui2@huawei.com> - 2.1.0-15
|
||||
- Type:bugfix
|
||||
- ID:NA
|
||||
- SUG:NA
|
||||
- DESC:check VFIO when create device
|
||||
|
||||
* Mon Nov 29 2021 jikui <jikui2@huawei.com> - 2.1.0-14
|
||||
- Type:bugfix
|
||||
- ID:NA
|
||||
- SUG:NA
|
||||
- DESC:fix delete sandbox failed problem
|
||||
|
||||
* Sat Nov 27 2021 jikui <jikui2@huawei.com> - 2.1.0-13
|
||||
- Type:bugfix
|
||||
- ID:NA
|
||||
- SUG:NA
|
||||
- DESC:validate sandbox cpu and memory size
|
||||
|
||||
* Thu Nov 25 2021 jikui <jikui2@huawei.com> - 2.1.0-12
|
||||
- Type:bugfix
|
||||
- ID:NA
|
||||
- SUG:NA
|
||||
- DESC:truncate the log.json file before kata-runtime subcommand executed
|
||||
|
||||
* Thu Nov 25 2021 jikui <jikui2@huawei.com> - 2.1.0-11
|
||||
- Type:bugfix
|
||||
- ID:NA
|
||||
- SUG:NA
|
||||
- DESC:fix umount container rootfs dir return invalid argument error
|
||||
|
||||
* Fri Nov 24 2021 jikui <jikui2@huawei.com> - 2.1.0-10
|
||||
- Type:bugfix
|
||||
- ID:NA
|
||||
- SUG:NA
|
||||
- DESC:increase delete cgroup retry times
|
||||
|
||||
* Sat Nov 20 2021 yangfeiyu <yangfeiyu2@huawei.com> - 2.1.0-9
|
||||
- Type:bugfix
|
||||
- ID:NA
|
||||
- SUG:NA
|
||||
- DESC:modify kernel and image path in configuration.toml
|
||||
|
||||
* Tue Oct 16 2021 jikui <jikui2@huawei.com> - 2.1.0-8
|
||||
- Type:bugfix
|
||||
- ID:NA
|
||||
- SUG:NA
|
||||
- DESC:keep the qemu process name same as the configured path
|
||||
|
||||
* Mon Oct 15 2021 jikui <jikui2@huawei.com> - 2.1.0-7
|
||||
- Type:bugfix
|
||||
- ID:NA
|
||||
- SUG:NA
|
||||
- DESC:fix kata-runtime skip read lines in /proc/mounts
|
||||
|
||||
* Fri Oct 5 2021 jikui <jikui2@huawei.com> - 2.1.0-6
|
||||
- Type:bugfix
|
||||
- ID:NA
|
||||
- SUG:NA
|
||||
- DESC:fix kata-runtime hungs when qemu process is D/T
|
||||
|
||||
* Mon Sep 27 2021 yangfeiyu <yangfeiyu2@huawei.com> - 2.1.0-5
|
||||
- Type:enhancement
|
||||
- ID:NA
|
||||
- SUG:NA
|
||||
- DESC:chmod agent exec fifo
|
||||
|
||||
* Fri Sep 17 2021 yangfeiyu <yangfeiyu2@huawei.com> - 2.1.0-4
|
||||
- Type:enhancement
|
||||
- ID:NA
|
||||
- SUG:NA
|
||||
- DESC:modify kata-agent build flags
|
||||
|
||||
* Tue Aug 24 2021 yangfeiyu <yangfeiyu2@huawei.com> - 2.1.0-3
|
||||
- Type:enhancement
|
||||
- ID:NA
|
||||
- SUG:NA
|
||||
- DESC:add configuration-stratovirt.toml
|
||||
|
||||
* Fri Aug 20 2021 yangfeiyu <yangfeiyu2@huawei.com> - 2.1.0-2
|
||||
- Type:enhancement
|
||||
- ID:NA
|
||||
- SUG:NA
|
||||
- DESC:support with stratovirt and isulad
|
||||
|
||||
* Wed Aug 18 2021 yangfeiyu <yangfeiyu2@huawei.com> - 2.1.0-1
|
||||
- Type:enhancement
|
||||
- ID:NA
|
||||
- SUG:NA
|
||||
- DESC:upgrade kata-containers
|
||||
|
||||
* Fri Feb 19 2021 xinghe <xinghe1@huawei.com> - 1.11.1-10
|
||||
- Type:CVE
|
||||
|
||||
@ -0,0 +1,837 @@
|
||||
From f56d66f196bee808526e86df2c3c063a887c6fef Mon Sep 17 00:00:00 2001
|
||||
From: Wei Gao <gaowei66@huawei.com>
|
||||
Date: Sat, 7 Aug 2021 10:39:11 +0800
|
||||
Subject: [PATCH 1/6] runtime: add support of new sandbox hypervisor type
|
||||
StratoVirt.
|
||||
|
||||
Signed-off-by: Wei Gao <gaowei66@huawei.com>
|
||||
---
|
||||
src/runtime/pkg/katautils/config.go | 96 +++-
|
||||
src/runtime/virtcontainers/hypervisor.go | 12 +
|
||||
src/runtime/virtcontainers/stratovirt.go | 642 +++++++++++++++++++++++
|
||||
3 files changed, 749 insertions(+), 1 deletion(-)
|
||||
create mode 100644 src/runtime/virtcontainers/stratovirt.go
|
||||
|
||||
diff --git a/src/runtime/pkg/katautils/config.go b/src/runtime/pkg/katautils/config.go
|
||||
index 6114aa39..f94ac4fd 100644
|
||||
--- a/src/runtime/pkg/katautils/config.go
|
||||
+++ b/src/runtime/pkg/katautils/config.go
|
||||
@@ -50,6 +50,7 @@ const (
|
||||
clhHypervisorTableType = "clh"
|
||||
qemuHypervisorTableType = "qemu"
|
||||
acrnHypervisorTableType = "acrn"
|
||||
+ stratovirtHypervisorTable = "stratovirt"
|
||||
|
||||
// the maximum amount of PCI bridges that can be cold plugged in a VM
|
||||
maxPCIBridges uint32 = 5
|
||||
@@ -870,6 +871,96 @@ func newClhHypervisorConfig(h hypervisor) (vc.HypervisorConfig, error) {
|
||||
}, nil
|
||||
}
|
||||
|
||||
+func newStratovirtHypervisorConfig(h hypervisor) (vc.HypervisorConfig, error) {
|
||||
+ hypervisor, err := h.path()
|
||||
+ if err != nil {
|
||||
+ return vc.HypervisorConfig{}, err
|
||||
+ }
|
||||
+
|
||||
+ kernel, err := h.kernel()
|
||||
+ if err != nil {
|
||||
+ return vc.HypervisorConfig{}, err
|
||||
+ }
|
||||
+
|
||||
+ initrd, image, err := h.getInitrdAndImage()
|
||||
+ if err != nil {
|
||||
+ return vc.HypervisorConfig{}, err
|
||||
+ }
|
||||
+
|
||||
+ if image != "" && initrd != "" {
|
||||
+ return vc.HypervisorConfig{},
|
||||
+ errors.New("having both an image and an initrd defined in the configuration file is not supported")
|
||||
+ }
|
||||
+
|
||||
+ if image == "" && initrd == "" {
|
||||
+ return vc.HypervisorConfig{},
|
||||
+ errors.New("either image or initrd must be defined in the configuration file")
|
||||
+ }
|
||||
+
|
||||
+ kernelParams := h.kernelParams()
|
||||
+ machineType := h.machineType()
|
||||
+
|
||||
+ blockDriver, err := h.blockDeviceDriver()
|
||||
+ if err != nil {
|
||||
+ return vc.HypervisorConfig{}, err
|
||||
+ }
|
||||
+
|
||||
+ sharedFS, err := h.sharedFS()
|
||||
+ if err != nil {
|
||||
+ return vc.HypervisorConfig{}, err
|
||||
+ }
|
||||
+
|
||||
+ if sharedFS == config.VirtioFS && h.VirtioFSDaemon == "" {
|
||||
+ return vc.HypervisorConfig{},
|
||||
+ errors.New("cannot enable virtio-fs without daemon path in configuration file")
|
||||
+ }
|
||||
+
|
||||
+ if vSock, err := utils.SupportsVsocks(); !vSock {
|
||||
+ return vc.HypervisorConfig{}, err
|
||||
+ }
|
||||
+
|
||||
+ return vc.HypervisorConfig{
|
||||
+ HypervisorPath: hypervisor,
|
||||
+ HypervisorPathList: h.HypervisorPathList,
|
||||
+ KernelPath: kernel,
|
||||
+ InitrdPath: initrd,
|
||||
+ ImagePath: image,
|
||||
+ KernelParams: vc.DeserializeParams(strings.Fields(kernelParams)),
|
||||
+ HypervisorMachineType: machineType,
|
||||
+ NumVCPUs: h.defaultVCPUs(),
|
||||
+ DefaultMaxVCPUs: h.defaultMaxVCPUs(),
|
||||
+ MemorySize: h.defaultMemSz(),
|
||||
+ MemSlots: h.defaultMemSlots(),
|
||||
+ MemOffset: h.defaultMemOffset(),
|
||||
+ EntropySource: h.GetEntropySource(),
|
||||
+ EntropySourceList: h.EntropySourceList,
|
||||
+ DefaultBridges: h.defaultBridges(),
|
||||
+ DisableBlockDeviceUse: h.DisableBlockDeviceUse,
|
||||
+ SharedFS: sharedFS,
|
||||
+ VirtioFSDaemon: h.VirtioFSDaemon,
|
||||
+ VirtioFSDaemonList: h.VirtioFSDaemonList,
|
||||
+ VirtioFSCacheSize: h.VirtioFSCacheSize,
|
||||
+ VirtioFSCache: h.defaultVirtioFSCache(),
|
||||
+ VirtioFSExtraArgs: h.VirtioFSExtraArgs,
|
||||
+ FileBackedMemRootDir: h.FileBackedMemRootDir,
|
||||
+ FileBackedMemRootList: h.FileBackedMemRootList,
|
||||
+ Mlock: !h.Swap,
|
||||
+ Debug: h.Debug,
|
||||
+ DisableNestingChecks: h.DisableNestingChecks,
|
||||
+ BlockDeviceDriver: blockDriver,
|
||||
+ BlockDeviceCacheSet: h.BlockDeviceCacheSet,
|
||||
+ BlockDeviceCacheDirect: h.BlockDeviceCacheDirect,
|
||||
+ BlockDeviceCacheNoflush: h.BlockDeviceCacheNoflush,
|
||||
+ EnableIOThreads: h.EnableIOThreads,
|
||||
+ DisableVhostNet: h.DisableVhostNet,
|
||||
+ EnableVhostUserStore: h.EnableVhostUserStore,
|
||||
+ VhostUserStorePath: h.vhostUserStorePath(),
|
||||
+ VhostUserStorePathList: h.VhostUserStorePathList,
|
||||
+ GuestHookPath: h.guestHookPath(),
|
||||
+ EnableAnnotations: h.EnableAnnotations,
|
||||
+ }, nil
|
||||
+}
|
||||
+
|
||||
func newFactoryConfig(f factory) (oci.FactoryConfig, error) {
|
||||
if f.TemplatePath == "" {
|
||||
f.TemplatePath = defaultTemplatePath
|
||||
@@ -903,6 +994,9 @@ func updateRuntimeConfigHypervisor(configPath string, tomlConf tomlConfig, confi
|
||||
case clhHypervisorTableType:
|
||||
config.HypervisorType = vc.ClhHypervisor
|
||||
hConfig, err = newClhHypervisorConfig(hypervisor)
|
||||
+ case stratovirtHypervisorTable:
|
||||
+ config.HypervisorType = vc.StratovirtHypervisor
|
||||
+ hConfig, err = newStratovirtHypervisorConfig(hypervisor)
|
||||
}
|
||||
|
||||
if err != nil {
|
||||
@@ -1287,7 +1381,7 @@ func checkHypervisorConfig(config vc.HypervisorConfig) error {
|
||||
memSizeMB := int64(config.MemorySize)
|
||||
|
||||
if memSizeMB == 0 {
|
||||
- return errors.New("VM memory cannot be zero")
|
||||
+ return errors.New(fmt.Sprintf("The VM memory cannot be zero, %s", config.ImagePath))
|
||||
}
|
||||
|
||||
mb := int64(1024 * 1024)
|
||||
diff --git a/src/runtime/virtcontainers/hypervisor.go b/src/runtime/virtcontainers/hypervisor.go
|
||||
index 767215b6..615baa80 100644
|
||||
--- a/src/runtime/virtcontainers/hypervisor.go
|
||||
+++ b/src/runtime/virtcontainers/hypervisor.go
|
||||
@@ -44,6 +44,9 @@ const (
|
||||
// ClhHypervisor is the ICH hypervisor.
|
||||
ClhHypervisor HypervisorType = "clh"
|
||||
|
||||
+ // StratovirtHypervisor is the StratoVirt hypervisor
|
||||
+ StratovirtHypervisor HypervisorType = "stratovirt"
|
||||
+
|
||||
// MockHypervisor is a mock hypervisor for testing purposes
|
||||
MockHypervisor HypervisorType = "mock"
|
||||
)
|
||||
@@ -159,6 +162,9 @@ func (hType *HypervisorType) Set(value string) error {
|
||||
case "clh":
|
||||
*hType = ClhHypervisor
|
||||
return nil
|
||||
+ case "stratovirt":
|
||||
+ *hType = StratovirtHypervisor
|
||||
+ return nil
|
||||
case "mock":
|
||||
*hType = MockHypervisor
|
||||
return nil
|
||||
@@ -178,6 +184,8 @@ func (hType *HypervisorType) String() string {
|
||||
return string(AcrnHypervisor)
|
||||
case ClhHypervisor:
|
||||
return string(ClhHypervisor)
|
||||
+ case StratovirtHypervisor:
|
||||
+ return string(StratovirtHypervisor)
|
||||
case MockHypervisor:
|
||||
return string(MockHypervisor)
|
||||
default:
|
||||
@@ -207,6 +215,10 @@ func newHypervisor(hType HypervisorType) (hypervisor, error) {
|
||||
return &cloudHypervisor{
|
||||
store: store,
|
||||
}, nil
|
||||
+ case StratovirtHypervisor:
|
||||
+ return &stratovirt{
|
||||
+ store: store,
|
||||
+ }, nil
|
||||
case MockHypervisor:
|
||||
return &mockHypervisor{}, nil
|
||||
default:
|
||||
diff --git a/src/runtime/virtcontainers/stratovirt.go b/src/runtime/virtcontainers/stratovirt.go
|
||||
new file mode 100644
|
||||
index 00000000..4fec96d3
|
||||
--- /dev/null
|
||||
+++ b/src/runtime/virtcontainers/stratovirt.go
|
||||
@@ -0,0 +1,642 @@
|
||||
+package virtcontainers
|
||||
+
|
||||
+import (
|
||||
+ "context"
|
||||
+ "fmt"
|
||||
+ "os"
|
||||
+ "os/exec"
|
||||
+ "path/filepath"
|
||||
+ "strconv"
|
||||
+ "strings"
|
||||
+ "syscall"
|
||||
+ "time"
|
||||
+
|
||||
+ govmmQemu "github.com/kata-containers/govmm/qemu"
|
||||
+ "github.com/pkg/errors"
|
||||
+ "github.com/sirupsen/logrus"
|
||||
+
|
||||
+ "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/device/config"
|
||||
+ persistapi "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/persist/api"
|
||||
+ "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/types"
|
||||
+ "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/utils"
|
||||
+ "go.opentelemetry.io/otel"
|
||||
+ otelLabel "go.opentelemetry.io/otel/label"
|
||||
+ otelTrace "go.opentelemetry.io/otel/trace"
|
||||
+)
|
||||
+
|
||||
+const defaultDummyMac = "22:33:44:aa:bb:"
|
||||
+const mmioBlkCount = 4
|
||||
+const mmioNetCount = 2
|
||||
+const randomDevice = "/dev/urandom"
|
||||
+
|
||||
+type stratovirtDev struct {
|
||||
+ dev interface{}
|
||||
+ devType deviceType
|
||||
+}
|
||||
+
|
||||
+type stratovirt struct {
|
||||
+ id string
|
||||
+ ctx context.Context
|
||||
+ sandbox *Sandbox
|
||||
+ store persistapi.PersistDriver
|
||||
+ config HypervisorConfig
|
||||
+ pid int
|
||||
+ consolePath string
|
||||
+ socketPath string
|
||||
+ qmpMonitorCh qmpChannel
|
||||
+ devices []stratovirtDev
|
||||
+ HotpluggedVCPUs []CPUDevice
|
||||
+ mmioBlkSlots [mmioBlkCount]bool
|
||||
+ mmioNetSlots [mmioNetCount]bool
|
||||
+}
|
||||
+
|
||||
+func (s *stratovirt) Logger() *logrus.Entry {
|
||||
+ return virtLog.WithField("subsystem", "stratovirt")
|
||||
+}
|
||||
+
|
||||
+func (s *stratovirt) trace(parent context.Context, name string) (otelTrace.Span, context.Context) {
|
||||
+ if parent == nil {
|
||||
+ s.Logger().WithField("type", "bug").Error("trace called before context set")
|
||||
+ parent = context.Background()
|
||||
+ }
|
||||
+
|
||||
+ tracer := otel.Tracer("kata")
|
||||
+ ctx, span := tracer.Start(parent, name, otelTrace.WithAttributes(otelLabel.String("source", "runtime"), otelLabel.String("package", "virtcontainers"), otelLabel.String("subsystem", "hypervisor"), otelLabel.String("type", "stratovirt"), otelLabel.String("sandbox_id", s.id)))
|
||||
+
|
||||
+ return span, ctx
|
||||
+}
|
||||
+
|
||||
+func (s *stratovirt) getKernelCmdLine(useImage bool) string {
|
||||
+ var params []string
|
||||
+
|
||||
+ if useImage {
|
||||
+ params = append(params, "root=/dev/vda")
|
||||
+ }
|
||||
+
|
||||
+ params = append(params, "pci=off")
|
||||
+ params = append(params, "reboot=k")
|
||||
+ params = append(params, "panic=1")
|
||||
+ params = append(params, "iommu=off")
|
||||
+ params = append(params, "acpi=off")
|
||||
+ params = append(params, "quiet")
|
||||
+ params = append(params, "agent.use_vsock=true")
|
||||
+ params = append(params, "random.trust_cpu=on")
|
||||
+ params = append(params, "rw")
|
||||
+ params = append(params, SerializeParams(s.config.KernelParams, "=")...)
|
||||
+
|
||||
+ return strings.Join(params, " ")
|
||||
+}
|
||||
+
|
||||
+func (s *stratovirt) hypervisorConfig() HypervisorConfig {
|
||||
+ return s.config
|
||||
+}
|
||||
+
|
||||
+func (s *stratovirt) createSandbox(ctx context.Context, id string, networkNS NetworkNamespace, hypervisorConfig *HypervisorConfig) error {
|
||||
+ s.ctx = ctx
|
||||
+
|
||||
+ var span otelTrace.Span
|
||||
+ span, _ = s.trace(ctx, "createSandbox")
|
||||
+ defer span.End()
|
||||
+
|
||||
+ s.id = id
|
||||
+ s.config = *hypervisorConfig
|
||||
+
|
||||
+ s.socketPath = filepath.Join(s.store.RunVMStoragePath(), id, "qmp.sock")
|
||||
+ s.consolePath = filepath.Join(s.store.RunVMStoragePath(), id, "console.sock")
|
||||
+ s.qmpMonitorCh = qmpChannel{
|
||||
+ ctx: s.ctx,
|
||||
+ path: s.socketPath,
|
||||
+ }
|
||||
+
|
||||
+ return nil
|
||||
+}
|
||||
+
|
||||
+func (s *stratovirt) waitSandBoxStarted(timeout int) error {
|
||||
+ timeStart := time.Now()
|
||||
+ for {
|
||||
+ err := s.qmpSetup()
|
||||
+ if err == nil {
|
||||
+ break
|
||||
+ }
|
||||
+
|
||||
+ if int(time.Since(timeStart).Seconds()) > timeout {
|
||||
+ return fmt.Errorf("Failed to connect to StratoVirt instance (timeout %ds): %v", timeout, err)
|
||||
+ }
|
||||
+
|
||||
+ time.Sleep(time.Duration(50) * time.Millisecond)
|
||||
+ }
|
||||
+
|
||||
+ if err := s.qmpMonitorCh.qmp.ExecuteQMPCapabilities(s.qmpMonitorCh.ctx); err != nil {
|
||||
+ s.Logger().WithError(err).Error(qmpCapErrMsg)
|
||||
+ return err
|
||||
+ }
|
||||
+
|
||||
+ return nil
|
||||
+}
|
||||
+
|
||||
+func (s *stratovirt) startSandbox(ctx context.Context, timeout int) error {
|
||||
+ span, _ := s.trace(ctx, "startSandbox")
|
||||
+ defer span.End()
|
||||
+
|
||||
+ var params []string
|
||||
+ var use_image bool
|
||||
+ params = append(params, "-name", fmt.Sprintf("sandbox-%s", s.id))
|
||||
+ params = append(params, "-qmp", fmt.Sprintf("unix:%s,server,nowait", s.socketPath))
|
||||
+
|
||||
+ if kernelPath, err := s.config.KernelAssetPath(); err == nil {
|
||||
+ params = append(params, "-kernel", kernelPath)
|
||||
+ }
|
||||
+
|
||||
+ initrdPath, err := s.config.InitrdAssetPath()
|
||||
+ if err != nil {
|
||||
+ return err
|
||||
+ }
|
||||
+
|
||||
+ if initrdPath == "" {
|
||||
+ imagePath, err := s.config.ImageAssetPath()
|
||||
+ if err != nil {
|
||||
+ return err
|
||||
+ }
|
||||
+ use_image = true
|
||||
+ s.mmioBlkSlots[0] = true
|
||||
+ params = append(params, "-device", "virtio-blk-device,drive=rootfs")
|
||||
+ params = append(params, "-drive", fmt.Sprintf("id=rootfs,file=%s,direct=off", imagePath))
|
||||
+ } else {
|
||||
+ use_image = false
|
||||
+ params = append(params, "-initrd", initrdPath)
|
||||
+ }
|
||||
+
|
||||
+ params = append(params, "-append", s.getKernelCmdLine(use_image))
|
||||
+ params = append(params, "-smp", fmt.Sprintf("%d", s.config.NumVCPUs))
|
||||
+ params = append(params, "-m", fmt.Sprintf("%d", uint64(s.config.MemorySize)))
|
||||
+ params = append(params, "-device", "virtio-serial-device")
|
||||
+ params = append(params, "-device", "virtconsole,chardev=charconsole0,id=virtioconsole0")
|
||||
+ params = append(params, "-object", fmt.Sprintf("rng-random,id=objrng0,filename=%s", randomDevice))
|
||||
+ params = append(params, "-device", "virtio-rng-device,rng=objrng0")
|
||||
+ params = append(params, "-chardev", fmt.Sprintf("socket,id=charconsole0,path=%s,server,nowait", s.consolePath))
|
||||
+ params = append(params, "-pidfile", filepath.Join(s.store.RunVMStoragePath(), s.id, "pid"))
|
||||
+
|
||||
+ // add devices to cmdline
|
||||
+ for _, d := range s.devices {
|
||||
+ switch v := d.dev.(type) {
|
||||
+ case Endpoint:
|
||||
+ name := v.Name()
|
||||
+ mac := v.HardwareAddr()
|
||||
+ tapName := v.NetworkPair().TapInterface.TAPIface.Name
|
||||
+ params = append(params, "-device", fmt.Sprintf("virtio-net-device,netdev=%s,id=%s,mac=%s", name, name, mac))
|
||||
+ params = append(params, "-netdev", fmt.Sprintf("tap,id=%s,ifname=%s", name, tapName))
|
||||
+ case config.BlockDrive:
|
||||
+ id := v.ID
|
||||
+ path := v.File
|
||||
+ params = append(params, "-device", fmt.Sprintf("virtio-blk-device, drive=%s", id))
|
||||
+ params = append(params, "-drive", fmt.Sprintf("id=%s,file=%s", id, path))
|
||||
+ case types.VSock:
|
||||
+ v.VhostFd.Close()
|
||||
+ params = append(params, "-device", fmt.Sprintf("vhost-vsock-device,id=vsock-id,guest-cid=%d", v.ContextID))
|
||||
+ default:
|
||||
+ s.Logger().Error("Adding device type is unsupported")
|
||||
+ }
|
||||
+ }
|
||||
+
|
||||
+ // daemonize
|
||||
+ params = append(params, "-daemonize")
|
||||
+
|
||||
+ // append logfile only on debug
|
||||
+ if s.config.Debug {
|
||||
+ dir := filepath.Join(s.store.RunVMStoragePath(), s.id)
|
||||
+ params = append(params, "-D", fmt.Sprintf("%s/stratovirt.log", dir))
|
||||
+ }
|
||||
+
|
||||
+ dir := filepath.Join(s.store.RunVMStoragePath(), s.id)
|
||||
+ err = os.MkdirAll(dir, DirMode)
|
||||
+ if err != nil {
|
||||
+ return err
|
||||
+ }
|
||||
+ defer func() {
|
||||
+ if err != nil {
|
||||
+ if err := os.RemoveAll(dir); err != nil {
|
||||
+ s.Logger().WithError(err).Error("Fail to clean up vm dir %s", dir)
|
||||
+ }
|
||||
+ }
|
||||
+ }()
|
||||
+
|
||||
+ binPath, err := s.config.HypervisorAssetPath()
|
||||
+ if err != nil {
|
||||
+ s.Logger().WithField("Fail to get hypervisor bin path", err).Error()
|
||||
+ return err
|
||||
+ }
|
||||
+
|
||||
+ cmd := exec.CommandContext(s.ctx, binPath, params...)
|
||||
+ s.Logger().Info("StratoVirt start with params: ", cmd)
|
||||
+
|
||||
+ if err := cmd.Start(); err != nil {
|
||||
+ s.Logger().WithField("Error starting hypervisor, please check the params", err).Error()
|
||||
+ return err
|
||||
+ }
|
||||
+ s.pid = cmd.Process.Pid
|
||||
+
|
||||
+ if err = s.waitSandBoxStarted(timeout); err != nil {
|
||||
+ return err
|
||||
+ }
|
||||
+
|
||||
+ return nil
|
||||
+}
|
||||
+
|
||||
+func (s *stratovirt) stopSandbox(ctx context.Context, force bool) error {
|
||||
+ span, _ := s.trace(ctx, "stopSandbox")
|
||||
+ defer span.End()
|
||||
+
|
||||
+ defer func() {
|
||||
+ dir := filepath.Join(s.store.RunVMStoragePath(), s.id)
|
||||
+ link, _ := filepath.EvalSymlinks(dir)
|
||||
+
|
||||
+ if err := os.RemoveAll(dir); err != nil {
|
||||
+ s.Logger().WithError(err).Warnf("Failed to clean up vm dir %s", dir)
|
||||
+ }
|
||||
+
|
||||
+ if link != dir && link != "" {
|
||||
+ if err := os.RemoveAll(link); err != nil {
|
||||
+ s.Logger().WithError(err).WithField("link", link).Warn("Failed to remove vm path link %s", link)
|
||||
+ }
|
||||
+ }
|
||||
+ }()
|
||||
+
|
||||
+ if !force {
|
||||
+ err := s.qmpSetup()
|
||||
+ if err != nil {
|
||||
+ return err
|
||||
+ }
|
||||
+
|
||||
+ err = s.qmpMonitorCh.qmp.ExecuteQuit(s.qmpMonitorCh.ctx)
|
||||
+ if err != nil {
|
||||
+ s.Logger().WithError(err).Error("Fail to execute qmp: QUIT")
|
||||
+ return err
|
||||
+ }
|
||||
+ } else {
|
||||
+ if s.pid > 0 {
|
||||
+ syscall.Kill(s.pid, syscall.SIGKILL)
|
||||
+ }
|
||||
+ }
|
||||
+ return nil
|
||||
+}
|
||||
+
|
||||
+func (s *stratovirt) pauseSandbox(ctx context.Context) error {
|
||||
+ return nil
|
||||
+}
|
||||
+
|
||||
+func (s *stratovirt) saveSandbox() error {
|
||||
+ return nil
|
||||
+}
|
||||
+
|
||||
+func (s *stratovirt) resumeSandbox(ctx context.Context) error {
|
||||
+ return nil
|
||||
+}
|
||||
+
|
||||
+func (s *stratovirt) addDevice(ctx context.Context, devInfo interface{}, devType deviceType) error {
|
||||
+ span, _ := s.trace(ctx, "addDevice")
|
||||
+ defer span.End()
|
||||
+
|
||||
+ dev := stratovirtDev{
|
||||
+ dev: devInfo,
|
||||
+ devType: devType,
|
||||
+ }
|
||||
+ s.devices = append(s.devices, dev)
|
||||
+
|
||||
+ return nil
|
||||
+}
|
||||
+
|
||||
+func (s *stratovirt) getDevSlot(Name string, isPut bool) (slot int, err error) {
|
||||
+ Name = filepath.Base(strings.ToLower(Name))
|
||||
+
|
||||
+ if strings.HasPrefix(Name, "eth") {
|
||||
+ idxStr := strings.TrimPrefix(Name, "eth")
|
||||
+ if idxStr == Name {
|
||||
+ return 0, fmt.Errorf("Could not parse idx from Name %q", Name)
|
||||
+ }
|
||||
+
|
||||
+ idx, err := strconv.Atoi(idxStr)
|
||||
+ if err != nil {
|
||||
+ return 0, fmt.Errorf("Could not convert to int from Str %q", idxStr)
|
||||
+ }
|
||||
+
|
||||
+ if !isPut && s.mmioNetSlots[idx] {
|
||||
+ return 0, fmt.Errorf("GetDevSlot failed, slot is being used %q", idxStr)
|
||||
+ }
|
||||
+ s.mmioNetSlots[idx] = !isPut
|
||||
+
|
||||
+ return idx, nil
|
||||
+ } else if strings.HasPrefix(Name, "vd") {
|
||||
+ charStr := strings.TrimPrefix(Name, "vd")
|
||||
+ if charStr == Name {
|
||||
+ return 0, fmt.Errorf("Could not parse idx from Name %q", Name)
|
||||
+ }
|
||||
+
|
||||
+ char := []rune(charStr)
|
||||
+ idx := int(char[0] - 'a')
|
||||
+
|
||||
+ if !isPut && s.mmioBlkSlots[idx] {
|
||||
+ return 0, fmt.Errorf("GetDevSlot failed, slot is being used %q", charStr)
|
||||
+ }
|
||||
+ s.mmioBlkSlots[idx] = !isPut
|
||||
+
|
||||
+ return idx, nil
|
||||
+ }
|
||||
+
|
||||
+ return 0, fmt.Errorf("GetDevSlot failed, Name is invalid %q", Name)
|
||||
+}
|
||||
+
|
||||
+func (s *stratovirt) hotplugNet(endpoint Endpoint, op operation) (err error) {
|
||||
+ err = s.qmpSetup()
|
||||
+ if err != nil {
|
||||
+ return err
|
||||
+ }
|
||||
+ var tap TapInterface
|
||||
+
|
||||
+ switch endpoint.Type() {
|
||||
+ case VethEndpointType:
|
||||
+ drive := endpoint.(*VethEndpoint)
|
||||
+ tap = drive.NetPair.TapInterface
|
||||
+ case TapEndpointType:
|
||||
+ drive := endpoint.(*TapEndpoint)
|
||||
+ tap = drive.TapInterface
|
||||
+ default:
|
||||
+ return fmt.Errorf("Endpoint is not supported")
|
||||
+ }
|
||||
+
|
||||
+ switch op {
|
||||
+ case addDevice:
|
||||
+ var (
|
||||
+ VMFdNames []string
|
||||
+ VhostFdNames []string
|
||||
+ )
|
||||
+ for i, VMFd := range tap.VMFds {
|
||||
+ fdName := fmt.Sprintf("fd%d", i)
|
||||
+ if err := s.qmpMonitorCh.qmp.ExecuteGetFD(s.qmpMonitorCh.ctx, fdName, VMFd); err != nil {
|
||||
+ return err
|
||||
+ }
|
||||
+ VMFdNames = append(VMFdNames, fdName)
|
||||
+ }
|
||||
+ for i, VhostFd := range tap.VhostFds {
|
||||
+ fdName := fmt.Sprintf("vhostfd%d", i)
|
||||
+ if err := s.qmpMonitorCh.qmp.ExecuteGetFD(s.qmpMonitorCh.ctx, fdName, VhostFd); err != nil {
|
||||
+ return err
|
||||
+ }
|
||||
+ VhostFd.Close()
|
||||
+ VhostFdNames = append(VhostFdNames, fdName)
|
||||
+ }
|
||||
+
|
||||
+ slot, err := s.getDevSlot(endpoint.Name(), false)
|
||||
+ if err != nil {
|
||||
+ return fmt.Errorf("Could not get unused slot for %q", endpoint.Name())
|
||||
+ }
|
||||
+
|
||||
+ if len(VMFdNames) != 0 || len(VhostFdNames) != 0 {
|
||||
+ if err := s.qmpMonitorCh.qmp.ExecuteNetdevAddByFds(s.qmpMonitorCh.ctx, "tap", tap.ID, VMFdNames, VhostFdNames); err != nil {
|
||||
+ s.getDevSlot(endpoint.Name(), true)
|
||||
+ return err
|
||||
+ }
|
||||
+ } else {
|
||||
+ if err := s.qmpMonitorCh.qmp.ExecuteNetdevAdd(s.qmpMonitorCh.ctx, "tap", tap.ID, tap.TAPIface.Name, "no", "no", 0); err != nil {
|
||||
+ s.getDevSlot(endpoint.Name(), true)
|
||||
+ return err
|
||||
+ }
|
||||
+ }
|
||||
+ if err := s.qmpMonitorCh.qmp.ExecuteNetPCIDeviceAdd(s.qmpMonitorCh.ctx, tap.Name, tap.ID, endpoint.HardwareAddr(), fmt.Sprintf("%d", slot), "", "", 0, false); err != nil {
|
||||
+ s.getDevSlot(endpoint.Name(), true)
|
||||
+ return err
|
||||
+ }
|
||||
+ case removeDevice:
|
||||
+ if err := s.qmpMonitorCh.qmp.ExecuteDeviceDel(s.qmpMonitorCh.ctx, tap.ID); err != nil {
|
||||
+ return err
|
||||
+ }
|
||||
+ if err := s.qmpMonitorCh.qmp.ExecuteNetdevDel(s.qmpMonitorCh.ctx, tap.ID); err != nil {
|
||||
+ return err
|
||||
+ }
|
||||
+ default:
|
||||
+ return fmt.Errorf("Operation is not supported")
|
||||
+ }
|
||||
+
|
||||
+ return nil
|
||||
+}
|
||||
+
|
||||
+func (s *stratovirt) hotplugBlk(drive *config.BlockDrive, op operation) (err error) {
|
||||
+ err = s.qmpSetup()
|
||||
+ if err != nil {
|
||||
+ return err
|
||||
+ }
|
||||
+
|
||||
+ switch op {
|
||||
+ case addDevice:
|
||||
+ driver := "virtio-blk-pci"
|
||||
+ slot, err := s.getDevSlot(drive.VirtPath, false)
|
||||
+ if err != nil {
|
||||
+ return fmt.Errorf("Could not get unused slot for %q", drive.VirtPath)
|
||||
+ }
|
||||
+
|
||||
+ if err := s.qmpMonitorCh.qmp.ExecuteBlockdevAdd(s.qmpMonitorCh.ctx, drive.File, drive.ID, false); err != nil {
|
||||
+ s.getDevSlot(drive.VirtPath, true)
|
||||
+ return err
|
||||
+ }
|
||||
+
|
||||
+ if err := s.qmpMonitorCh.qmp.ExecutePCIDeviceAdd(s.qmpMonitorCh.ctx, drive.ID, drive.ID, driver, fmt.Sprintf("%d", slot), "", "", 0, true, false); err != nil {
|
||||
+ s.getDevSlot(drive.VirtPath, true)
|
||||
+ return err
|
||||
+ }
|
||||
+ case removeDevice:
|
||||
+ if err := s.qmpMonitorCh.qmp.ExecuteDeviceDel(s.qmpMonitorCh.ctx, drive.ID); err != nil {
|
||||
+ return err
|
||||
+ }
|
||||
+ if err := s.qmpMonitorCh.qmp.ExecuteBlockdevDel(s.qmpMonitorCh.ctx, drive.ID); err != nil {
|
||||
+ return err
|
||||
+ }
|
||||
+
|
||||
+ s.getDevSlot(drive.VirtPath, true)
|
||||
+ default:
|
||||
+ return fmt.Errorf("Operation is not supported")
|
||||
+ }
|
||||
+
|
||||
+ return nil
|
||||
+}
|
||||
+
|
||||
+func (s *stratovirt) hotplugAddDevice(ctx context.Context, devInfo interface{}, devType deviceType) (interface{}, error) {
|
||||
+ span, _ := s.trace(ctx, "hotplugAddDevice")
|
||||
+ defer span.End()
|
||||
+
|
||||
+ switch devType {
|
||||
+ case netDev:
|
||||
+ return nil, s.hotplugNet(devInfo.(Endpoint), addDevice)
|
||||
+ case blockDev:
|
||||
+ return nil, s.hotplugBlk(devInfo.(*config.BlockDrive), addDevice)
|
||||
+ default:
|
||||
+ return nil, fmt.Errorf("Hotplug add device failed: unsupported device type '%v'", devType)
|
||||
+ }
|
||||
+}
|
||||
+
|
||||
+func (s *stratovirt) hotplugRemoveDevice(ctx context.Context, devInfo interface{}, devType deviceType) (interface{}, error) {
|
||||
+ span, _ := s.trace(ctx, "hotplugRemoveDevice")
|
||||
+ defer span.End()
|
||||
+
|
||||
+ switch devType {
|
||||
+ case netDev:
|
||||
+ return nil, s.hotplugNet(devInfo.(Endpoint), removeDevice)
|
||||
+ case blockDev:
|
||||
+ return nil, s.hotplugBlk(devInfo.(*config.BlockDrive), removeDevice)
|
||||
+ default:
|
||||
+ return nil, fmt.Errorf("Hotplug remove device: unsupported device type '%v'", devType)
|
||||
+ }
|
||||
+}
|
||||
+
|
||||
+func (s *stratovirt) resizeMemory(ctx context.Context, reqMemMB uint32, memoryBlockSizeMB uint32, probe bool) (uint32, memoryDevice, error) {
|
||||
+ return 0, memoryDevice{}, nil
|
||||
+}
|
||||
+
|
||||
+func (s *stratovirt) resizeVCPUs(ctx context.Context, reqVCPUs uint32) (currentVCPUs uint32, newVCPUs uint32, err error) {
|
||||
+ return 0, 0, nil
|
||||
+}
|
||||
+
|
||||
+func (s *stratovirt) getSandboxConsole(ctx context.Context, id string) (string, string, error) {
|
||||
+ span, _ := s.trace(ctx, "getSandboxConsole")
|
||||
+ defer span.End()
|
||||
+
|
||||
+ var consolePath string
|
||||
+ if s.config.Debug {
|
||||
+ consolePath = s.consolePath
|
||||
+ } else {
|
||||
+ consolePath = ""
|
||||
+ }
|
||||
+ consoleURL, err := utils.BuildSocketPath(consolePath)
|
||||
+ if err != nil {
|
||||
+ return consoleProtoUnix, "", err
|
||||
+ }
|
||||
+ return consoleProtoUnix, consoleURL, nil
|
||||
+
|
||||
+}
|
||||
+
|
||||
+func (s *stratovirt) disconnect(ctx context.Context) {
|
||||
+ span, _ := s.trace(ctx, "disconnect")
|
||||
+ defer span.End()
|
||||
+
|
||||
+ s.qmpTeardown()
|
||||
+}
|
||||
+
|
||||
+func (s *stratovirt) capabilities(ctx context.Context) types.Capabilities {
|
||||
+ span, _ := s.trace(ctx, "capabilities")
|
||||
+ defer span.End()
|
||||
+
|
||||
+ var caps types.Capabilities
|
||||
+ caps.SetBlockDeviceHotplugSupport()
|
||||
+
|
||||
+ return caps
|
||||
+}
|
||||
+
|
||||
+func (s *stratovirt) qmpTeardown() {
|
||||
+ if s.qmpMonitorCh.qmp != nil {
|
||||
+ s.qmpMonitorCh.qmp.Shutdown()
|
||||
+ <-s.qmpMonitorCh.disconn
|
||||
+ s.qmpMonitorCh.qmp = nil
|
||||
+ s.qmpMonitorCh.disconn = nil
|
||||
+ }
|
||||
+}
|
||||
+
|
||||
+func (s *stratovirt) qmpSetup() error {
|
||||
+ s.qmpTeardown()
|
||||
+
|
||||
+ cfg := govmmQemu.QMPConfig{Logger: newQMPLogger()}
|
||||
+
|
||||
+ // Auto-closed by QMPStart().
|
||||
+ disconnectCh := make(chan struct{})
|
||||
+
|
||||
+ qmp, _, err := govmmQemu.QMPStart(s.qmpMonitorCh.ctx, s.qmpMonitorCh.path, cfg, disconnectCh)
|
||||
+ if err != nil {
|
||||
+ s.Logger().WithError(err).Error("Failed to connect to StratoVirt instance")
|
||||
+ return err
|
||||
+ }
|
||||
+
|
||||
+ s.qmpMonitorCh.qmp = qmp
|
||||
+ s.qmpMonitorCh.disconn = disconnectCh
|
||||
+
|
||||
+ return nil
|
||||
+}
|
||||
+
|
||||
+func (s *stratovirt) getThreadIDs(ctx context.Context) (vcpuThreadIDs, error) {
|
||||
+ span, _ := s.trace(ctx, "getThreadIDs")
|
||||
+ defer span.End()
|
||||
+
|
||||
+ tid := vcpuThreadIDs{}
|
||||
+ if err := s.qmpSetup(); err != nil {
|
||||
+ return tid, err
|
||||
+ }
|
||||
+
|
||||
+ cpuInfos, err := s.qmpMonitorCh.qmp.ExecQueryCpus(s.qmpMonitorCh.ctx)
|
||||
+ if err != nil {
|
||||
+ s.Logger().WithError(err).Error("Failed to query cpu infos")
|
||||
+ return tid, err
|
||||
+ }
|
||||
+
|
||||
+ tid.vcpus = make(map[int]int, len(cpuInfos))
|
||||
+ for _, i := range cpuInfos {
|
||||
+ if i.ThreadID > 0 {
|
||||
+ tid.vcpus[i.CPU] = i.ThreadID
|
||||
+ }
|
||||
+ }
|
||||
+ return tid, nil
|
||||
+}
|
||||
+
|
||||
+func (s *stratovirt) cleanup(ctx context.Context) error {
|
||||
+ span, _ := s.trace(ctx, "cleanup")
|
||||
+ defer span.End()
|
||||
+
|
||||
+ s.qmpTeardown()
|
||||
+
|
||||
+ return nil
|
||||
+}
|
||||
+
|
||||
+func (s *stratovirt) getPids() []int {
|
||||
+ return []int{s.pid}
|
||||
+}
|
||||
+
|
||||
+func (s *stratovirt) getVirtioFsPid() *int {
|
||||
+ return nil
|
||||
+}
|
||||
+
|
||||
+func (s *stratovirt) fromGrpc(ctx context.Context, hypervisorConfig *HypervisorConfig, j []byte) error {
|
||||
+ return errors.New("stratovirt is not supported by VM cache")
|
||||
+}
|
||||
+
|
||||
+func (s *stratovirt) toGrpc(ctx context.Context) ([]byte, error) {
|
||||
+ return nil, errors.New("stratovirt is not supported by VM cache")
|
||||
+}
|
||||
+
|
||||
+func (s *stratovirt) check() error {
|
||||
+ if err := syscall.Kill(s.pid, syscall.Signal(0)); err != nil {
|
||||
+ return errors.Wrapf(err, "Failed to ping StratoVirt process")
|
||||
+ }
|
||||
+
|
||||
+ return nil
|
||||
+}
|
||||
+
|
||||
+func (s *stratovirt) generateSocket(id string) (interface{}, error) {
|
||||
+ return generateVMSocket(id, s.store.RunVMStoragePath())
|
||||
+}
|
||||
+
|
||||
+func (s *stratovirt) isRateLimiterBuiltin() bool {
|
||||
+ return true
|
||||
+}
|
||||
+
|
||||
+func (s *stratovirt) save() (p persistapi.HypervisorState) {
|
||||
+ pids := s.getPids()
|
||||
+ p.Pid = pids[0]
|
||||
+ p.Type = string(StratovirtHypervisor)
|
||||
+ return
|
||||
+}
|
||||
+
|
||||
+func (s *stratovirt) load(p persistapi.HypervisorState) {
|
||||
+ s.pid = p.Pid
|
||||
+
|
||||
+ return
|
||||
+}
|
||||
+
|
||||
+func (s *stratovirt) setSandbox(sandbox *Sandbox) {
|
||||
+ s.sandbox = sandbox
|
||||
+ return
|
||||
+}
|
||||
--
|
||||
2.21.1 (Apple Git-122.3)
|
||||
|
||||
@ -0,0 +1,174 @@
|
||||
From 1ffd95187a61582e858dd37c0ab434d3159a0f52 Mon Sep 17 00:00:00 2001
|
||||
From: Wei Gao <gaowei66@huawei.com>
|
||||
Date: Mon, 9 Aug 2021 14:26:35 +0800
|
||||
Subject: [PATCH 2/6] agent: add support of new sandbox hypervisor kind
|
||||
StratoVirt.
|
||||
|
||||
1. add new grpc interface `UpdateInterfaceHwAddrByName`.
|
||||
2. comment out rescan_pci temporarily.
|
||||
|
||||
Signed-off-by: Wei Gao <gaowei66@huawei.com>
|
||||
---
|
||||
src/agent/protocols/protos/agent.proto | 5 +++
|
||||
src/agent/src/netlink.rs | 31 ++++++++++++++++
|
||||
src/agent/src/rpc.rs | 51 +++++++++++++++++++++++++-
|
||||
3 files changed, 85 insertions(+), 2 deletions(-)
|
||||
|
||||
diff --git a/src/agent/protocols/protos/agent.proto b/src/agent/protocols/protos/agent.proto
|
||||
index 6cbf5a28..e00f5c63 100644
|
||||
--- a/src/agent/protocols/protos/agent.proto
|
||||
+++ b/src/agent/protocols/protos/agent.proto
|
||||
@@ -46,6 +46,7 @@ service AgentService {
|
||||
|
||||
// networking
|
||||
rpc UpdateInterface(UpdateInterfaceRequest) returns (types.Interface);
|
||||
+ rpc UpdateInterfaceHwAddrByName(UpdateInterfaceHwAddrByNameRequest) returns (types.Interface);
|
||||
rpc UpdateRoutes(UpdateRoutesRequest) returns (Routes);
|
||||
rpc ListInterfaces(ListInterfacesRequest) returns(Interfaces);
|
||||
rpc ListRoutes(ListRoutesRequest) returns (Routes);
|
||||
@@ -308,6 +309,10 @@ message UpdateInterfaceRequest {
|
||||
types.Interface interface = 1;
|
||||
}
|
||||
|
||||
+message UpdateInterfaceHwAddrByNameRequest {
|
||||
+ types.Interface interface = 1;
|
||||
+}
|
||||
+
|
||||
message UpdateRoutesRequest {
|
||||
Routes routes = 1;
|
||||
}
|
||||
diff --git a/src/agent/src/netlink.rs b/src/agent/src/netlink.rs
|
||||
index 3ab6dbaa..82632d1b 100644
|
||||
--- a/src/agent/src/netlink.rs
|
||||
+++ b/src/agent/src/netlink.rs
|
||||
@@ -104,6 +104,29 @@ impl Handle {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
+ pub async fn update_interface_hw_addr_by_name(&mut self, iface: &Interface) -> Result<()> {
|
||||
+ let link = self.find_link(LinkFilter::Name(&iface.name)).await?;
|
||||
+
|
||||
+ // Delete all addresses associated with the link
|
||||
+ let addresses = self
|
||||
+ .list_addresses(AddressFilter::LinkIndex(link.index()))
|
||||
+ .await?;
|
||||
+ self.delete_addresses(addresses).await?;
|
||||
+
|
||||
+ if iface.IPAddresses.len() == 0 {
|
||||
+ self.enable_link(link.index(), false).await?;
|
||||
+ }
|
||||
+
|
||||
+ // Update hardware mac address
|
||||
+ let mac_addr = parse_mac_address(iface.get_hwAddr())
|
||||
+ .with_context(|| format!("Failed to parse MAC address: {}", iface.get_hwAddr()))?;
|
||||
+ self.link_set_hw_addr(link.index(), mac_addr)
|
||||
+ .await
|
||||
+ .with_context(|| format!("Could not set {:?} to {}", mac_addr, link.name()))?;
|
||||
+
|
||||
+ Ok(())
|
||||
+ }
|
||||
+
|
||||
pub async fn handle_localhost(&self) -> Result<()> {
|
||||
let link = self.find_link(LinkFilter::Name("lo")).await?;
|
||||
self.enable_link(link.index(), true).await?;
|
||||
@@ -216,6 +239,14 @@ impl Handle {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
+ async fn link_set_hw_addr(&self, link_index: u32, hw_addr: [u8; 6]) -> Result<()> {
|
||||
+ let link_req = self.handle.link().set(link_index);
|
||||
+ let set_req = link_req.address(hw_addr.to_vec());
|
||||
+ set_req.execute().await?;
|
||||
+
|
||||
+ Ok(())
|
||||
+ }
|
||||
+
|
||||
async fn query_routes(
|
||||
&self,
|
||||
ip_version: Option<IpVersion>,
|
||||
diff --git a/src/agent/src/rpc.rs b/src/agent/src/rpc.rs
|
||||
index 92025af3..2cc1c983 100644
|
||||
--- a/src/agent/src/rpc.rs
|
||||
+++ b/src/agent/src/rpc.rs
|
||||
@@ -40,7 +40,7 @@ use nix::sys::stat;
|
||||
use nix::unistd::{self, Pid};
|
||||
use rustjail::process::ProcessOperations;
|
||||
|
||||
-use crate::device::{add_devices, rescan_pci_bus, update_device_cgroup};
|
||||
+use crate::device::{add_devices, update_device_cgroup};
|
||||
use crate::linux_abi::*;
|
||||
use crate::metrics::get_metrics;
|
||||
use crate::mount::{add_storages, remove_mounts, BareMount, STORAGE_HANDLER_LIST};
|
||||
@@ -123,7 +123,9 @@ impl AgentService {
|
||||
|
||||
// re-scan PCI bus
|
||||
// looking for hidden devices
|
||||
- rescan_pci_bus().context("Could not rescan PCI bus")?;
|
||||
+ // FIXME: Comment out this code temporarily, because once the PCIBus is scanned,
|
||||
+ // the device hot-plug event is lost
|
||||
+ // rescan_pci_bus().context("Could not rescan PCI bus")?;
|
||||
|
||||
// Some devices need some extra processing (the ones invoked with
|
||||
// --device for instance), and that's what this call is doing. It
|
||||
@@ -797,6 +799,34 @@ impl protocols::agent_ttrpc::AgentService for AgentService {
|
||||
Ok(interface)
|
||||
}
|
||||
|
||||
+ async fn update_interface_hw_addr_by_name(
|
||||
+ &self,
|
||||
+ _ctx: &TtrpcContext,
|
||||
+ req: protocols::agent::UpdateInterfaceHwAddrByNameRequest,
|
||||
+ ) -> ttrpc::Result<Interface> {
|
||||
+ let interface = req.interface.into_option().ok_or_else(|| {
|
||||
+ ttrpc_error(
|
||||
+ ttrpc::Code::INVALID_ARGUMENT,
|
||||
+ "empty update interface request".to_string(),
|
||||
+ )
|
||||
+ })?;
|
||||
+
|
||||
+ self.sandbox
|
||||
+ .lock()
|
||||
+ .await
|
||||
+ .rtnl
|
||||
+ .update_interface_hw_addr_by_name(&interface)
|
||||
+ .await
|
||||
+ .map_err(|e| {
|
||||
+ ttrpc_error(
|
||||
+ ttrpc::Code::INTERNAL,
|
||||
+ format!("update interface hw addr: {:?}", e),
|
||||
+ )
|
||||
+ })?;
|
||||
+
|
||||
+ Ok(interface)
|
||||
+ }
|
||||
+
|
||||
async fn update_routes(
|
||||
&self,
|
||||
_ctx: &TtrpcContext,
|
||||
@@ -1670,6 +1700,23 @@ mod tests {
|
||||
assert!(result.is_err(), "expected update interface to fail");
|
||||
}
|
||||
|
||||
+ #[tokio::test]
|
||||
+ async fn test_update_interface_hw_addr_by_name() {
|
||||
+ let logger = slog::Logger::root(slog::Discard, o!());
|
||||
+ let sandbox = Sandbox::new(&logger).unwrap();
|
||||
+
|
||||
+ let agent_service = Box::new(AgentService {
|
||||
+ sandbox: Arc::new(Mutex::new(sandbox)),
|
||||
+ });
|
||||
+
|
||||
+ let req = protocols::agent::UpdateInterfaceHwAddrByNameRequest::default();
|
||||
+ let ctx = mk_ttrpc_context();
|
||||
+
|
||||
+ let result = agent_service.update_interface_hw_addr_by_name(&ctx, req).await;
|
||||
+
|
||||
+ assert!(result.is_err(), "expected update interface to fail");
|
||||
+ }
|
||||
+
|
||||
#[tokio::test]
|
||||
async fn test_update_routes() {
|
||||
let logger = slog::Logger::root(slog::Discard, o!());
|
||||
--
|
||||
2.21.1 (Apple Git-122.3)
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@ -0,0 +1,542 @@
|
||||
From 950c0db14a9a9baccefd83e87893d7f40c2bd13d Mon Sep 17 00:00:00 2001
|
||||
From: Wei Gao <gaowei66@huawei.com>
|
||||
Date: Mon, 9 Aug 2021 14:47:19 +0800
|
||||
Subject: [PATCH 4/6] configuration: add configuration generator for hypervisor
|
||||
type stratovirt.
|
||||
|
||||
Signed-off-by: Wei Gao <gaowei66@huawei.com>
|
||||
---
|
||||
src/runtime/.gitignore | 1 +
|
||||
src/runtime/Makefile | 40 +-
|
||||
src/runtime/arch/amd64-options.mk | 3 +
|
||||
src/runtime/arch/arm64-options.mk | 3 +
|
||||
.../config/configuration-stratovirt.toml.in | 356 ++++++++++++++++++
|
||||
5 files changed, 402 insertions(+), 1 deletion(-)
|
||||
create mode 100644 src/runtime/cli/config/configuration-stratovirt.toml.in
|
||||
|
||||
diff --git a/src/runtime/.gitignore b/src/runtime/.gitignore
|
||||
index 52b9e4e5..0a630a07 100644
|
||||
--- a/src/runtime/.gitignore
|
||||
+++ b/src/runtime/.gitignore
|
||||
@@ -10,6 +10,7 @@ coverage.html
|
||||
/cli/config/configuration-fc.toml
|
||||
/cli/config/configuration-qemu.toml
|
||||
/cli/config/configuration-clh.toml
|
||||
+/cli/config/configuration-stratovirt.toml
|
||||
/cli/config-generated.go
|
||||
/cli/containerd-shim-kata-v2/config-generated.go
|
||||
/cli/coverage.html
|
||||
diff --git a/src/runtime/Makefile b/src/runtime/Makefile
|
||||
index 4a69c05c..ea2cd296 100644
|
||||
--- a/src/runtime/Makefile
|
||||
+++ b/src/runtime/Makefile
|
||||
@@ -74,6 +74,7 @@ QEMUBINDIR := $(PREFIXDEPS)/bin
|
||||
CLHBINDIR := $(PREFIXDEPS)/bin
|
||||
FCBINDIR := $(PREFIXDEPS)/bin
|
||||
ACRNBINDIR := $(PREFIXDEPS)/bin
|
||||
+STRATOVIRTBINDIR := $(PREFIXDEPS)/bin
|
||||
SYSCONFDIR := /etc
|
||||
LOCALSTATEDIR := /var
|
||||
|
||||
@@ -93,6 +94,7 @@ GENERATED_VARS = \
|
||||
CONFIG_QEMU_IN \
|
||||
CONFIG_CLH_IN \
|
||||
CONFIG_FC_IN \
|
||||
+ CONFIG_STRATOVIRT_IN \
|
||||
$(USER_VARS)
|
||||
SCRIPTS += $(COLLECT_SCRIPT)
|
||||
SCRIPTS_DIR := $(BINDIR)
|
||||
@@ -116,12 +118,13 @@ HYPERVISOR_ACRN = acrn
|
||||
HYPERVISOR_FC = firecracker
|
||||
HYPERVISOR_QEMU = qemu
|
||||
HYPERVISOR_CLH = cloud-hypervisor
|
||||
+HYPERVISOR_STRATOVIRT = stratovirt
|
||||
|
||||
# Determines which hypervisor is specified in $(CONFIG_FILE).
|
||||
DEFAULT_HYPERVISOR ?= $(HYPERVISOR_QEMU)
|
||||
|
||||
# List of hypervisors this build system can generate configuration for.
|
||||
-HYPERVISORS := $(HYPERVISOR_ACRN) $(HYPERVISOR_FC) $(HYPERVISOR_QEMU) $(HYPERVISOR_CLH)
|
||||
+HYPERVISORS := $(HYPERVISOR_ACRN) $(HYPERVISOR_FC) $(HYPERVISOR_QEMU) $(HYPERVISOR_CLH) $(HYPERVISOR_STRATOVIRT)
|
||||
|
||||
QEMUPATH := $(QEMUBINDIR)/$(QEMUCMD)
|
||||
QEMUVALIDHYPERVISORPATHS := [\"$(QEMUPATH)\"]
|
||||
@@ -141,6 +144,9 @@ ACRNVALIDHYPERVISORPATHS := [\"$(ACRNPATH)\"]
|
||||
ACRNCTLPATH := $(ACRNBINDIR)/$(ACRNCTLCMD)
|
||||
ACRNVALIDCTLPATHS := [\"$(ACRNCTLPATH)\"]
|
||||
|
||||
+STRATOVIRTPATH = $(STRATOVIRTBINDIR)/$(STRATOVIRTCMD)
|
||||
+STRATOVIRTVALIDHYPERVISORPATHS := [\"$(STRATOVIRTPATH)\"]
|
||||
+
|
||||
NETMONCMD := $(BIN_PREFIX)-netmon
|
||||
NETMONPATH := $(PKGLIBEXECDIR)/$(NETMONCMD)
|
||||
|
||||
@@ -267,6 +273,29 @@ ifneq (,$(CLHCMD))
|
||||
KERNELPATH_CLH = $(KERNELDIR)/$(KERNEL_NAME_CLH)
|
||||
endif
|
||||
|
||||
+ifneq (,$(STRATOVIRTCMD))
|
||||
+ KNOWN_HYPERVISORS += $(HYPERVISOR_STRATOVIRT)
|
||||
+
|
||||
+ CONFIG_FILE_STRATOVIRT = configuration-stratovirt.toml
|
||||
+ CONFIG_STRATOVIRT = $(CLI_DIR)/config/$(CONFIG_FILE_STRATOVIRT)
|
||||
+ CONFIG_STRATOVIRT_IN = $(CONFIG_STRATOVIRT).in
|
||||
+
|
||||
+ CONFIG_PATH_STRATOVIRT = $(abspath $(CONFDIR)/$(CONFIG_FILE_STRATOVIRT))
|
||||
+ CONFIG_PATHS += $(CONFIG_PATH_STRATOVIRT)
|
||||
+
|
||||
+ SYSCONFIG_STRATOVIRT = $(abspath $(SYSCONFDIR)/$(CONFIG_FILE_STRATOVIRT))
|
||||
+ SYSCONFIG_PATHS += $(SYSCONFIG_STRATOVIRT)
|
||||
+
|
||||
+ CONFIGS += $(CONFIG_STRATOVIRT)
|
||||
+
|
||||
+ # stratovirt-specific options (all should be suffixed by "_STRATOVIRT")
|
||||
+ DEFBLOCKSTORAGEDRIVER_STRATOVIRT := virtio-mmio
|
||||
+ DEFNETWORKMODEL_STRATOVIRT := none
|
||||
+ KENRELTYPE_STRATOVIRT = uncompressed
|
||||
+ KERNEL_NAME_STRATOVIRT = $(call MAKE_KERNEL_NAME,$(KENRELTYPE_STRATOVIRT))
|
||||
+ KERNELPATH_STRATOVIRT = $(KERNELDIR)/$(KERNEL_NAME_STRATOVIRT)
|
||||
+endif
|
||||
+
|
||||
ifneq (,$(FCCMD))
|
||||
KNOWN_HYPERVISORS += $(HYPERVISOR_FC)
|
||||
|
||||
@@ -363,6 +392,7 @@ USER_VARS += BINDIR
|
||||
USER_VARS += CONFIG_ACRN_IN
|
||||
USER_VARS += CONFIG_CLH_IN
|
||||
USER_VARS += CONFIG_FC_IN
|
||||
+USER_VARS += CONFIG_STRATOVIRT_IN
|
||||
USER_VARS += CONFIG_PATH
|
||||
USER_VARS += CONFIG_QEMU_IN
|
||||
USER_VARS += DESTDIR
|
||||
@@ -382,6 +412,8 @@ USER_VARS += FCPATH
|
||||
USER_VARS += FCVALIDHYPERVISORPATHS
|
||||
USER_VARS += FCJAILERPATH
|
||||
USER_VARS += FCVALIDJAILERPATHS
|
||||
+USER_VARS += STRATOVIRTPATH
|
||||
+USER_VARS += STRATOVIRTVALIDHYPERVISORPATHS
|
||||
USER_VARS += SYSCONFIG
|
||||
USER_VARS += IMAGENAME
|
||||
USER_VARS += IMAGEPATH
|
||||
@@ -395,6 +427,7 @@ USER_VARS += KERNELPATH_ACRN
|
||||
USER_VARS += KERNELPATH
|
||||
USER_VARS += KERNELPATH_CLH
|
||||
USER_VARS += KERNELPATH_FC
|
||||
+USER_VARS += KERNELPATH_STRATOVIRT
|
||||
USER_VARS += KERNELVIRTIOFSPATH
|
||||
USER_VARS += FIRMWAREPATH
|
||||
USER_VARS += MACHINEACCELERATORS
|
||||
@@ -434,12 +467,14 @@ USER_VARS += DEFNETWORKMODEL_ACRN
|
||||
USER_VARS += DEFNETWORKMODEL_CLH
|
||||
USER_VARS += DEFNETWORKMODEL_FC
|
||||
USER_VARS += DEFNETWORKMODEL_QEMU
|
||||
+USER_VARS += DEFNETWORKMODEL_STRATOVIRT
|
||||
USER_VARS += DEFDISABLEGUESTSECCOMP
|
||||
USER_VARS += DEFAULTEXPFEATURES
|
||||
USER_VARS += DEFDISABLEBLOCK
|
||||
USER_VARS += DEFBLOCKSTORAGEDRIVER_ACRN
|
||||
USER_VARS += DEFBLOCKSTORAGEDRIVER_FC
|
||||
USER_VARS += DEFBLOCKSTORAGEDRIVER_QEMU
|
||||
+USER_VARS += DEFBLOCKSTORAGEDRIVER_STRATOVIRT
|
||||
USER_VARS += DEFSHAREDFS_QEMU_VIRTIOFS
|
||||
USER_VARS += DEFVIRTIOFSDAEMON
|
||||
USER_VARS += DEFVALIDVIRTIOFSDAEMONPATHS
|
||||
@@ -773,6 +808,9 @@ ifneq (,$(findstring $(HYPERVISOR_FC),$(KNOWN_HYPERVISORS)))
|
||||
endif
|
||||
ifneq (,$(findstring $(HYPERVISOR_ACRN),$(KNOWN_HYPERVISORS)))
|
||||
@printf "\t$(HYPERVISOR_ACRN) hypervisor path (ACRNPATH) : %s\n" $(abspath $(ACRNPATH))
|
||||
+endif
|
||||
+ifneq (,$(findstring $(HYPERVISOR_STRATOVIRT),$(KNOWN_HYPERVISORS)))
|
||||
+ @printf "\t$(HYPERVISOR_STRATOVIRT) hypervisor path (STRATOVIRTPATH) : %s\n" $(abspath $(STRATOVIRTPATH))
|
||||
endif
|
||||
@printf "\tassets path (PKGDATADIR) : %s\n" $(abspath $(PKGDATADIR))
|
||||
@printf "\tshim path (PKGLIBEXECDIR) : %s\n" $(abspath $(PKGLIBEXECDIR))
|
||||
diff --git a/src/runtime/arch/amd64-options.mk b/src/runtime/arch/amd64-options.mk
|
||||
index 83af8cc0..ff2af9e6 100644
|
||||
--- a/src/runtime/arch/amd64-options.mk
|
||||
+++ b/src/runtime/arch/amd64-options.mk
|
||||
@@ -23,3 +23,6 @@ ACRNCTLCMD := acrnctl
|
||||
|
||||
# cloud-hypervisor binary name
|
||||
CLHCMD := cloud-hypervisor
|
||||
+
|
||||
+# stratovirt binary name
|
||||
+STRATOVIRTCMD := stratovirt
|
||||
\ No newline at end of file
|
||||
diff --git a/src/runtime/arch/arm64-options.mk b/src/runtime/arch/arm64-options.mk
|
||||
index ad5ef5d4..2ad3f657 100644
|
||||
--- a/src/runtime/arch/arm64-options.mk
|
||||
+++ b/src/runtime/arch/arm64-options.mk
|
||||
@@ -19,3 +19,6 @@ FCJAILERCMD := jailer
|
||||
|
||||
# cloud-hypervisor binary name
|
||||
CLHCMD := cloud-hypervisor
|
||||
+
|
||||
+# stratovirt binary name
|
||||
+STRATOVIRTCMD := stratovirt
|
||||
\ No newline at end of file
|
||||
diff --git a/src/runtime/cli/config/configuration-stratovirt.toml.in b/src/runtime/cli/config/configuration-stratovirt.toml.in
|
||||
new file mode 100644
|
||||
index 00000000..5c83c3c9
|
||||
--- /dev/null
|
||||
+++ b/src/runtime/cli/config/configuration-stratovirt.toml.in
|
||||
@@ -0,0 +1,356 @@
|
||||
+# Copyright (c) 2017-2019 Intel Corporation
|
||||
+#
|
||||
+# SPDX-License-Identifier: Apache-2.0
|
||||
+#
|
||||
+
|
||||
+# XXX: WARNING: this file is auto-generated.
|
||||
+# XXX:
|
||||
+# XXX: Source file: "@CONFIG_STRATOVIRT_IN@"
|
||||
+# XXX: Project:
|
||||
+# XXX: Name: @PROJECT_NAME@
|
||||
+# XXX: Type: @PROJECT_TYPE@
|
||||
+
|
||||
+[hypervisor.stratovirt]
|
||||
+path = "@STRATOVIRTPATH@"
|
||||
+kernel = "@KERNELPATH_STRATOVIRT@"
|
||||
+image = "@IMAGEPATH@"
|
||||
+
|
||||
+# List of valid annotation names for the hypervisor
|
||||
+# Each member of the list is a regular expression, which is the base name
|
||||
+# of the annotation, e.g. "path" for io.katacontainers.config.hypervisor.path"
|
||||
+enable_annotations = @DEFENABLEANNOTATIONS@
|
||||
+
|
||||
+# List of valid annotations values for the hypervisor
|
||||
+# Each member of the list is a path pattern as described by glob(3).
|
||||
+# The default if not set is empty (all annotations rejected.)
|
||||
+# Your distribution recommends: @STRATOVIRTVALIDHYPERVISORPATHS@
|
||||
+valid_hypervisor_paths = @STRATOVIRTVALIDHYPERVISORPATHS@
|
||||
+
|
||||
+# Optional space-separated list of options to pass to the guest kernel.
|
||||
+# For example, use `kernel_params = "vsyscall=emulate"` if you are having
|
||||
+# trouble running pre-2.15 glibc.
|
||||
+#
|
||||
+# WARNING: - any parameter specified here will take priority over the default
|
||||
+# parameter value of the same name used to start the virtual machine.
|
||||
+# Do not set values here unless you understand the impact of doing so as you
|
||||
+# may stop the virtual machine from booting.
|
||||
+# To see the list of default parameters, enable hypervisor debug, create a
|
||||
+# container and look for 'default-kernel-parameters' log entries.
|
||||
+kernel_params = "@KERNELPARAMS@"
|
||||
+
|
||||
+# Default number of vCPUs per SB/VM:
|
||||
+# unspecified or 0 --> will be set to @DEFVCPUS@
|
||||
+# < 0 --> will be set to the actual number of physical cores
|
||||
+# > 0 <= number of physical cores --> will be set to the specified number
|
||||
+# > number of physical cores --> will be set to the actual number of physical cores
|
||||
+default_vcpus = 1
|
||||
+
|
||||
+# Default maximum number of vCPUs per SB/VM:
|
||||
+# unspecified or == 0 --> will be set to the actual number of physical cores or to the maximum number
|
||||
+# of vCPUs supported by KVM if that number is exceeded
|
||||
+# > 0 <= number of physical cores --> will be set to the specified number
|
||||
+# > number of physical cores --> will be set to the actual number of physical cores or to the maximum number
|
||||
+# of vCPUs supported by KVM if that number is exceeded
|
||||
+# WARNING: Depending of the architecture, the maximum number of vCPUs supported by KVM is used when
|
||||
+# the actual number of physical cores is greater than it.
|
||||
+# WARNING: Be aware that this value impacts the virtual machine's memory footprint and CPU
|
||||
+# the hotplug functionality. For example, `default_maxvcpus = 240` specifies that until 240 vCPUs
|
||||
+# can be added to a SB/VM, but the memory footprint will be big. Another example, with
|
||||
+# `default_maxvcpus = 8` the memory footprint will be small, but 8 will be the maximum number of
|
||||
+# vCPUs supported by the SB/VM. In general, we recommend that you do not edit this variable,
|
||||
+# unless you know what are you doing.
|
||||
+# NOTICE: on arm platform with gicv2 interrupt controller, set it to 8.
|
||||
+default_maxvcpus = @DEFMAXVCPUS@
|
||||
+
|
||||
+# Bridges can be used to hot plug devices.
|
||||
+# Limitations:
|
||||
+# * Currently only pci bridges are supported
|
||||
+# * Until 30 devices per bridge can be hot plugged.
|
||||
+# * Until 5 PCI bridges can be cold plugged per VM.
|
||||
+# This limitation could be a bug in the kernel
|
||||
+# Default number of bridges per SB/VM:
|
||||
+# unspecified or 0 --> will be set to @DEFBRIDGES@
|
||||
+# > 1 <= 5 --> will be set to the specified number
|
||||
+# > 5 --> will be set to 5
|
||||
+default_bridges = @DEFBRIDGES@
|
||||
+
|
||||
+# Default memory size in MiB for SB/VM.
|
||||
+# If unspecified then it will be set @DEFMEMSZ@ MiB.
|
||||
+default_memory = @DEFMEMSZ@
|
||||
+#
|
||||
+# Default memory slots per SB/VM.
|
||||
+# If unspecified then it will be set @DEFMEMSLOTS@.
|
||||
+# This is will determine the times that memory will be hotadded to sandbox/VM.
|
||||
+# memory_slots = @DEFMEMSLOTS@
|
||||
+
|
||||
+# The size in MiB will be plused to max memory of hypervisor.
|
||||
+# It is the memory address space for the NVDIMM devie.
|
||||
+# If set block storage driver (block_device_driver) to "nvdimm",
|
||||
+# should set memory_offset to the size of block device.
|
||||
+# Default 0
|
||||
+# memory_offset = 0
|
||||
+
|
||||
+# Disable block device from being used for a container's rootfs.
|
||||
+# In case of a storage driver like devicemapper where a container's
|
||||
+# root file system is backed by a block device, the block device is passed
|
||||
+# directly to the hypervisor for performance reasons.
|
||||
+# This flag prevents the block device from being passed to the hypervisor,
|
||||
+# 9pfs is used instead to pass the rootfs.
|
||||
+disable_block_device_use = @DEFDISABLEBLOCK@
|
||||
+
|
||||
+# Block storage driver to be used for the hypervisor in case the container
|
||||
+# rootfs is backed by a block device. This is virtio-scsi, virtio-blk
|
||||
+# or nvdimm.
|
||||
+block_device_driver = "@DEFBLOCKSTORAGEDRIVER_STRATOVIRT@"
|
||||
+
|
||||
+# Specifies cache-related options will be set to block devices or not.
|
||||
+# Default false
|
||||
+#block_device_cache_set = true
|
||||
+
|
||||
+# Specifies cache-related options for block devices.
|
||||
+# Denotes whether use of O_DIRECT (bypass the host page cache) is enabled.
|
||||
+# Default false
|
||||
+# block_device_cache_direct = true
|
||||
+
|
||||
+# Specifies cache-related options for block devices.
|
||||
+# Denotes whether flush requests for the device are ignored.
|
||||
+# Default false
|
||||
+# block_device_cache_noflush = true
|
||||
+
|
||||
+# Enable pre allocation of VM RAM, default false
|
||||
+# Enabling this will result in lower container density
|
||||
+# as all of the memory will be allocated and locked
|
||||
+# This is useful when you want to reserve all the memory
|
||||
+# upfront or in the cases where you want memory latencies
|
||||
+# to be very predictable
|
||||
+# Default false
|
||||
+# enable_mem_prealloc = true
|
||||
+
|
||||
+# Enable huge pages for VM RAM, default false
|
||||
+# Enabling this will result in the VM memory
|
||||
+# being allocated using huge pages.
|
||||
+# This is useful when you want to use vhost-user network
|
||||
+# stacks within the container. This will automatically
|
||||
+# result in memory pre allocation
|
||||
+# enable_hugepages = true
|
||||
+
|
||||
+# Enable vIOMMU, default false
|
||||
+# Enabling this will result in the VM having a vIOMMU device
|
||||
+# This will also add the following options to the kernel's
|
||||
+# command line: intel_iommu=on,iommu=pt
|
||||
+# enable_iommu = true
|
||||
+
|
||||
+# Enable swap of vm memory. Default false.
|
||||
+# The behaviour is undefined if mem_prealloc is also set to true
|
||||
+# enable_swap = true
|
||||
+
|
||||
+# This option changes the default hypervisor and kernel parameters
|
||||
+# to enable debug output where available.
|
||||
+#
|
||||
+# Default false
|
||||
+# enable_debug = true
|
||||
+
|
||||
+# Disable the customizations done in the runtime when it detects
|
||||
+# that it is running on top a VMM. This will result in the runtime
|
||||
+# behaving as it would when running on bare metal.
|
||||
+#
|
||||
+# disable_nesting_checks = true
|
||||
+
|
||||
+# This is the msize used for 9p shares. It is the number of bytes
|
||||
+# used for 9p packet payload.
|
||||
+# msize_9p =
|
||||
+
|
||||
+# VFIO devices are hotplugged on a bridge by default.
|
||||
+# Enable hotplugging on root bus. This may be required for devices with
|
||||
+# a large PCI bar, as this is a current limitation with hotplugging on
|
||||
+# a bridge.
|
||||
+# Default false
|
||||
+# hotplug_vfio_on_root_bus = true
|
||||
+
|
||||
+#
|
||||
+# Default entropy source.
|
||||
+# The path to a host source of entropy (including a real hardware RNG)
|
||||
+# /dev/urandom and /dev/random are two main options.
|
||||
+# Be aware that /dev/random is a blocking source of entropy. If the host
|
||||
+# runs out of entropy, the VMs boot time will increase leading to get startup
|
||||
+# timeouts.
|
||||
+# The source of entropy /dev/urandom is non-blocking and provides a
|
||||
+# generally acceptable source of entropy. It should work well for pretty much
|
||||
+# all practical purposes.
|
||||
+# entropy_source= ""
|
||||
+
|
||||
+# List of valid annotations values for entropy_source
|
||||
+# The default if not set is empty (all annotations rejected.)
|
||||
+# Your distribution recommends: @DEFVALIDENTROPYSOURCES@
|
||||
+valid_entropy_sources = @DEFVALIDENTROPYSOURCES@
|
||||
+
|
||||
+# Path to OCI hook binaries in the *guest rootfs*.
|
||||
+# This does not affect host-side hooks which must instead be added to
|
||||
+# the OCI spec passed to the runtime.
|
||||
+#
|
||||
+# You can create a rootfs with hooks by customizing the osbuilder scripts:
|
||||
+# https://github.com/kata-containers/kata-containers/tree/main/tools/osbuilder
|
||||
+#
|
||||
+# Hooks must be stored in a subdirectory of guest_hook_path according to their
|
||||
+# hook type, i.e. "guest_hook_path/{prestart,poststart,poststop}".
|
||||
+# The agent will scan these directories for executable files and add them, in
|
||||
+# lexicographical order, to the lifecycle of the guest container.
|
||||
+# Hooks are executed in the runtime namespace of the guest. See the official documentation:
|
||||
+# https://github.com/opencontainers/runtime-spec/blob/v1.0.1/config.md#posix-platform-hooks
|
||||
+# Warnings will be logged if any error is encountered will scanning for hooks,
|
||||
+# but it will not abort container execution.
|
||||
+# guest_hook_path = "/usr/share/oci/hooks"
|
||||
+
|
||||
+[factory]
|
||||
+# VM templating support. Once enabled, new VMs are created from template
|
||||
+# using vm cloning. They will share the same initial kernel, initramfs and
|
||||
+# agent memory by mapping it readonly. It helps speeding up new container
|
||||
+# creation and saves a lot of memory if there are many kata containers running
|
||||
+# on the same host.
|
||||
+#
|
||||
+# When disabled, new VMs are created from scratch.
|
||||
+#
|
||||
+# Note: Requires "initrd=" to be set ("image=" is not supported).
|
||||
+#
|
||||
+# Default false
|
||||
+#enable_template = true
|
||||
+
|
||||
+[agent.@PROJECT_TYPE@]
|
||||
+# If enabled, make the agent display debug-level messages.
|
||||
+# (default: disabled)
|
||||
+#enable_debug = true
|
||||
+
|
||||
+# Enable agent tracing.
|
||||
+#
|
||||
+# If enabled, the default trace mode is "dynamic" and the
|
||||
+# default trace type is "isolated". The trace mode and type are set
|
||||
+# explicity with the `trace_type=` and `trace_mode=` options.
|
||||
+#
|
||||
+# Notes:
|
||||
+#
|
||||
+# - Tracing is ONLY enabled when `enable_tracing` is set: explicitly
|
||||
+# setting `trace_mode=` and/or `trace_type=` without setting `enable_tracing`
|
||||
+# will NOT activate agent tracing.
|
||||
+#
|
||||
+# - See https://github.com/kata-containers/agent/blob/master/TRACING.md for
|
||||
+# full details.
|
||||
+#
|
||||
+# (default: disabled)
|
||||
+#enable_tracing = true
|
||||
+#
|
||||
+#trace_mode = "dynamic"
|
||||
+#trace_type = "isolated"
|
||||
+
|
||||
+# Comma separated list of kernel modules and their parameters.
|
||||
+# These modules will be loaded in the guest kernel using modprobe(8).
|
||||
+# The following example can be used to load two kernel modules with parameters
|
||||
+# - kernel_modules=["e1000e InterruptThrottleRate=3000,3000,3000 EEE=1", "i915 enable_ppgtt=0"]
|
||||
+# The first word is considered as the module name and the rest as its parameters.
|
||||
+# Container will not be started when:
|
||||
+# * A kernel module is specified and the modprobe command is not installed in the guest
|
||||
+# or it fails loading the module.
|
||||
+# * The module is not available in the guest or it doesn't met the guest kernel
|
||||
+# requirements, like architecture and version.
|
||||
+#
|
||||
+kernel_modules=[]
|
||||
+
|
||||
+# Enable debug console.
|
||||
+
|
||||
+# If enabled, user can connect guest OS running inside hypervisor
|
||||
+# through "kata-runtime exec <sandbox-id>" command
|
||||
+
|
||||
+#debug_console_enabled = true
|
||||
+
|
||||
+# Agent connection dialing timeout value in seconds
|
||||
+# (default: 30)
|
||||
+#dial_timeout = 30
|
||||
+
|
||||
+[netmon]
|
||||
+# If enabled, the network monitoring process gets started when the
|
||||
+# sandbox is created. This allows for the detection of some additional
|
||||
+# network being added to the existing network namespace, after the
|
||||
+# sandbox has been created.
|
||||
+# (default: disabled)
|
||||
+#enable_netmon = true
|
||||
+
|
||||
+# Specify the path to the netmon binary.
|
||||
+path = "@NETMONPATH@"
|
||||
+
|
||||
+# If enabled, netmon messages will be sent to the system log
|
||||
+# (default: disabled)
|
||||
+#enable_debug = true
|
||||
+
|
||||
+[runtime]
|
||||
+# If enabled, the runtime will log additional debug messages to the
|
||||
+# system log
|
||||
+# (default: disabled)
|
||||
+#enable_debug = true
|
||||
+#
|
||||
+# Internetworking model
|
||||
+# Determines how the VM should be connected to the
|
||||
+# the container network interface
|
||||
+# Options:
|
||||
+#
|
||||
+# - macvtap
|
||||
+# Used when the Container network interface can be bridged using
|
||||
+# macvtap.
|
||||
+#
|
||||
+# - none
|
||||
+# Used when customize network. Only creates a tap device. No veth pair.
|
||||
+#
|
||||
+# - tcfilter
|
||||
+# Uses tc filter rules to redirect traffic from the network interface
|
||||
+# provided by plugin to a tap interface connected to the VM.
|
||||
+#
|
||||
+internetworking_model="@DEFNETWORKMODEL_STRATOVIRT@"
|
||||
+
|
||||
+# disable guest seccomp
|
||||
+# Determines whether container seccomp profiles are passed to the virtual
|
||||
+# machine and applied by the kata agent. If set to true, seccomp is not applied
|
||||
+# within the guest
|
||||
+# (default: true)
|
||||
+disable_guest_seccomp=@DEFDISABLEGUESTSECCOMP@
|
||||
+
|
||||
+# If enabled, the runtime will create opentracing.io traces and spans.
|
||||
+# (See https://www.jaegertracing.io/docs/getting-started).
|
||||
+# (default: disabled)
|
||||
+#enable_tracing = true
|
||||
+
|
||||
+# Set the full url to the Jaeger HTTP Thrift collector.
|
||||
+# The default if not set will be "http://localhost:14268/api/traces"
|
||||
+#jaeger_endpoint = ""
|
||||
+
|
||||
+# Sets the username to be used if basic auth is required for Jaeger.
|
||||
+#jaeger_user = ""
|
||||
+
|
||||
+# Sets the password to be used if basic auth is required for Jaeger.
|
||||
+#jaeger_password = ""
|
||||
+
|
||||
+# If enabled, the runtime will not create a network namespace for shim and hypervisor processes.
|
||||
+# This option may have some potential impacts to your host. It should only be used when you know what you're doing.
|
||||
+# `disable_new_netns` conflicts with `enable_netmon`
|
||||
+# `disable_new_netns` conflicts with `internetworking_model=tcfilter` and `internetworking_model=macvtap`. It works only
|
||||
+# with `internetworking_model=none`. The tap device will be in the host network namespace and can connect to a bridge
|
||||
+# (like OVS) directly.
|
||||
+# If you are using docker, `disable_new_netns` only works with `docker run --net=none`
|
||||
+# (default: false)
|
||||
+#disable_new_netns = true
|
||||
+
|
||||
+# if enable, the runtime will add all the kata processes inside one dedicated cgroup.
|
||||
+# The container cgroups in the host are not created, just one single cgroup per sandbox.
|
||||
+# The runtime caller is free to restrict or collect cgroup stats of the overall Kata sandbox.
|
||||
+# The sandbox cgroup path is the parent cgroup of a container with the PodSandbox annotation.
|
||||
+# The sandbox cgroup is constrained if there is no container type annotation.
|
||||
+# See: https://godoc.org/github.com/kata-containers/runtime/virtcontainers#ContainerType
|
||||
+sandbox_cgroup_only=@DEFSANDBOXCGROUPONLY@
|
||||
+
|
||||
+# Enabled experimental feature list, format: ["a", "b"].
|
||||
+# Experimental features are features not stable enough for production,
|
||||
+# they may break compatibility, and are prepared for a big version bump.
|
||||
+# Supported experimental features:
|
||||
+# (default: [])
|
||||
+experimental=@DEFAULTEXPFEATURES@
|
||||
+
|
||||
+# If enabled, user can run pprof tools with shim v2 process through kata-monitor.
|
||||
+# (default: false)
|
||||
+# enable_pprof = true
|
||||
--
|
||||
2.21.1 (Apple Git-122.3)
|
||||
|
||||
@ -0,0 +1,655 @@
|
||||
From 45c8e108497eb93d69afd38e6281b837e65cf3ec Mon Sep 17 00:00:00 2001
|
||||
From: Wei Gao <gaowei66@huawei.com>
|
||||
Date: Mon, 9 Aug 2021 14:55:41 +0800
|
||||
Subject: [PATCH 5/6] runtime: add the secure component "ozone" support for
|
||||
hypervisor type stratovirt.
|
||||
|
||||
Signed-off-by: Wei Gao <gaowei66@huawei.com>
|
||||
---
|
||||
src/runtime/Makefile | 4 +
|
||||
src/runtime/arch/amd64-options.mk | 4 +-
|
||||
src/runtime/arch/arm64-options.mk | 4 +-
|
||||
.../config/configuration-stratovirt.toml.in | 10 +
|
||||
.../pkg/katautils/config-settings.go.in | 1 +
|
||||
src/runtime/pkg/katautils/config.go | 18 +
|
||||
src/runtime/virtcontainers/hypervisor.go | 3 +
|
||||
src/runtime/virtcontainers/persist.go | 1 +
|
||||
.../virtcontainers/persist/api/config.go | 3 +
|
||||
src/runtime/virtcontainers/stratovirt.go | 309 ++++++++++++++----
|
||||
10 files changed, 292 insertions(+), 65 deletions(-)
|
||||
|
||||
diff --git a/src/runtime/Makefile b/src/runtime/Makefile
|
||||
index ea2cd296..745bcc10 100644
|
||||
--- a/src/runtime/Makefile
|
||||
+++ b/src/runtime/Makefile
|
||||
@@ -146,6 +146,8 @@ ACRNVALIDCTLPATHS := [\"$(ACRNCTLPATH)\"]
|
||||
|
||||
STRATOVIRTPATH = $(STRATOVIRTBINDIR)/$(STRATOVIRTCMD)
|
||||
STRATOVIRTVALIDHYPERVISORPATHS := [\"$(STRATOVIRTPATH)\"]
|
||||
+STRATOVIRTOZONEPATH = $(STRATOVIRTBINDIR)/$(STRATOVIRTOZONECMD)
|
||||
+STRATOVIRTVALIDOZONEPATHS = [\"$(STRATOVIRTOZONEPATH)\"]
|
||||
|
||||
NETMONCMD := $(BIN_PREFIX)-netmon
|
||||
NETMONPATH := $(PKGLIBEXECDIR)/$(NETMONCMD)
|
||||
@@ -414,6 +416,8 @@ USER_VARS += FCJAILERPATH
|
||||
USER_VARS += FCVALIDJAILERPATHS
|
||||
USER_VARS += STRATOVIRTPATH
|
||||
USER_VARS += STRATOVIRTVALIDHYPERVISORPATHS
|
||||
+USER_VARS += STRATOVIRTOZONEPATH
|
||||
+USER_VARS += STRATOVIRTVALIDOZONEPATHS
|
||||
USER_VARS += SYSCONFIG
|
||||
USER_VARS += IMAGENAME
|
||||
USER_VARS += IMAGEPATH
|
||||
diff --git a/src/runtime/arch/amd64-options.mk b/src/runtime/arch/amd64-options.mk
|
||||
index ff2af9e6..4c6c329a 100644
|
||||
--- a/src/runtime/arch/amd64-options.mk
|
||||
+++ b/src/runtime/arch/amd64-options.mk
|
||||
@@ -25,4 +25,6 @@ ACRNCTLCMD := acrnctl
|
||||
CLHCMD := cloud-hypervisor
|
||||
|
||||
# stratovirt binary name
|
||||
-STRATOVIRTCMD := stratovirt
|
||||
\ No newline at end of file
|
||||
+STRATOVIRTCMD := stratovirt
|
||||
+# stratovirt's ozone binary name
|
||||
+STRATOVIRTOZONECMD := ozone
|
||||
\ No newline at end of file
|
||||
diff --git a/src/runtime/arch/arm64-options.mk b/src/runtime/arch/arm64-options.mk
|
||||
index 2ad3f657..5dfa2c80 100644
|
||||
--- a/src/runtime/arch/arm64-options.mk
|
||||
+++ b/src/runtime/arch/arm64-options.mk
|
||||
@@ -21,4 +21,6 @@ FCJAILERCMD := jailer
|
||||
CLHCMD := cloud-hypervisor
|
||||
|
||||
# stratovirt binary name
|
||||
-STRATOVIRTCMD := stratovirt
|
||||
\ No newline at end of file
|
||||
+STRATOVIRTCMD := stratovirt
|
||||
+# stratovirt's ozone binary name
|
||||
+STRATOVIRTOZONECMD := ozone
|
||||
\ No newline at end of file
|
||||
diff --git a/src/runtime/cli/config/configuration-stratovirt.toml.in b/src/runtime/cli/config/configuration-stratovirt.toml.in
|
||||
index 5c83c3c9..b557b71f 100644
|
||||
--- a/src/runtime/cli/config/configuration-stratovirt.toml.in
|
||||
+++ b/src/runtime/cli/config/configuration-stratovirt.toml.in
|
||||
@@ -26,6 +26,16 @@ enable_annotations = @DEFENABLEANNOTATIONS@
|
||||
# Your distribution recommends: @STRATOVIRTVALIDHYPERVISORPATHS@
|
||||
valid_hypervisor_paths = @STRATOVIRTVALIDHYPERVISORPATHS@
|
||||
|
||||
+# Path for the ozone specific to stratovirt
|
||||
+# If the ozone path is set, stratovirt will be launched in
|
||||
+# ozone secure environment. It is disabled by default.
|
||||
+# ozone_path = "@STRATOVIRTOZONEPATH@"
|
||||
+
|
||||
+# List of valid ozone path values for the hypervisor
|
||||
+# Each member of the list can be a regular expression
|
||||
+# The default if not set is empty (all annotations rejected.)
|
||||
+# valid_jailer_paths = @STRATOVIRTVALIDOZONEPATHS@
|
||||
+
|
||||
# Optional space-separated list of options to pass to the guest kernel.
|
||||
# For example, use `kernel_params = "vsyscall=emulate"` if you are having
|
||||
# trouble running pre-2.15 glibc.
|
||||
diff --git a/src/runtime/pkg/katautils/config-settings.go.in b/src/runtime/pkg/katautils/config-settings.go.in
|
||||
index 7cd9138b..c168c608 100644
|
||||
--- a/src/runtime/pkg/katautils/config-settings.go.in
|
||||
+++ b/src/runtime/pkg/katautils/config-settings.go.in
|
||||
@@ -17,6 +17,7 @@ var defaultInitrdPath = "/usr/share/kata-containers/kata-containers-initrd.img"
|
||||
var defaultFirmwarePath = ""
|
||||
var defaultMachineAccelerators = ""
|
||||
var defaultCPUFeatures = ""
|
||||
+var defaultOzonePath = "/usr/bin/ozone"
|
||||
var systemdUnitName = "kata-containers.target"
|
||||
|
||||
const defaultKernelParams = ""
|
||||
diff --git a/src/runtime/pkg/katautils/config.go b/src/runtime/pkg/katautils/config.go
|
||||
index f94ac4fd..828c2a43 100644
|
||||
--- a/src/runtime/pkg/katautils/config.go
|
||||
+++ b/src/runtime/pkg/katautils/config.go
|
||||
@@ -92,6 +92,7 @@ type hypervisor struct {
|
||||
FileBackedMemRootDir string `toml:"file_mem_backend"`
|
||||
GuestHookPath string `toml:"guest_hook_path"`
|
||||
GuestMemoryDumpPath string `toml:"guest_memory_dump_path"`
|
||||
+ OzonePath string `toml:"ozone_path"`
|
||||
HypervisorPathList []string `toml:"valid_hypervisor_paths"`
|
||||
JailerPathList []string `toml:"valid_jailer_paths"`
|
||||
CtlPathList []string `toml:"valid_ctlpaths"`
|
||||
@@ -452,6 +453,16 @@ func (h hypervisor) getInitrdAndImage() (initrd string, image string, err error)
|
||||
return
|
||||
}
|
||||
|
||||
+func (h hypervisor) ozonePath() (string, error) {
|
||||
+ p := h.OzonePath
|
||||
+
|
||||
+ if h.OzonePath == "" {
|
||||
+ return "", nil
|
||||
+ }
|
||||
+
|
||||
+ return ResolvePath(p)
|
||||
+}
|
||||
+
|
||||
func (h hypervisor) getRxRateLimiterCfg() uint64 {
|
||||
return h.RxRateLimiterMaxRate
|
||||
}
|
||||
@@ -877,6 +888,11 @@ func newStratovirtHypervisorConfig(h hypervisor) (vc.HypervisorConfig, error) {
|
||||
return vc.HypervisorConfig{}, err
|
||||
}
|
||||
|
||||
+ ozone, err := h.ozonePath()
|
||||
+ if err != nil {
|
||||
+ return vc.HypervisorConfig{}, err
|
||||
+ }
|
||||
+
|
||||
kernel, err := h.kernel()
|
||||
if err != nil {
|
||||
return vc.HypervisorConfig{}, err
|
||||
@@ -925,6 +941,7 @@ func newStratovirtHypervisorConfig(h hypervisor) (vc.HypervisorConfig, error) {
|
||||
KernelPath: kernel,
|
||||
InitrdPath: initrd,
|
||||
ImagePath: image,
|
||||
+ OzonePath: ozone,
|
||||
KernelParams: vc.DeserializeParams(strings.Fields(kernelParams)),
|
||||
HypervisorMachineType: machineType,
|
||||
NumVCPUs: h.defaultVCPUs(),
|
||||
@@ -1155,6 +1172,7 @@ func GetDefaultHypervisorConfig() vc.HypervisorConfig {
|
||||
RxRateLimiterMaxRate: defaultRxRateLimiterMaxRate,
|
||||
TxRateLimiterMaxRate: defaultTxRateLimiterMaxRate,
|
||||
SGXEPCSize: defaultSGXEPCSize,
|
||||
+ OzonePath: defaultOzonePath,
|
||||
}
|
||||
}
|
||||
|
||||
diff --git a/src/runtime/virtcontainers/hypervisor.go b/src/runtime/virtcontainers/hypervisor.go
|
||||
index 615baa80..04e14b4e 100644
|
||||
--- a/src/runtime/virtcontainers/hypervisor.go
|
||||
+++ b/src/runtime/virtcontainers/hypervisor.go
|
||||
@@ -302,6 +302,9 @@ type HypervisorConfig struct {
|
||||
// JailerPathList is the list of jailer paths names allowed in annotations
|
||||
JailerPathList []string
|
||||
|
||||
+ // OzonePath is the ozone executable host path.
|
||||
+ OzonePath string
|
||||
+
|
||||
// BlockDeviceDriver specifies the driver to be used for block device
|
||||
// either VirtioSCSI or VirtioBlock with the default driver being defaultBlockDriver
|
||||
BlockDeviceDriver string
|
||||
diff --git a/src/runtime/virtcontainers/persist.go b/src/runtime/virtcontainers/persist.go
|
||||
index 203495e8..ae499c97 100644
|
||||
--- a/src/runtime/virtcontainers/persist.go
|
||||
+++ b/src/runtime/virtcontainers/persist.go
|
||||
@@ -219,6 +219,7 @@ func (s *Sandbox) dumpConfig(ss *persistapi.SandboxState) {
|
||||
HypervisorCtlPathList: sconfig.HypervisorConfig.HypervisorCtlPathList,
|
||||
JailerPath: sconfig.HypervisorConfig.JailerPath,
|
||||
JailerPathList: sconfig.HypervisorConfig.JailerPathList,
|
||||
+ OzonePath: sconfig.HypervisorConfig.OzonePath,
|
||||
BlockDeviceDriver: sconfig.HypervisorConfig.BlockDeviceDriver,
|
||||
HypervisorMachineType: sconfig.HypervisorConfig.HypervisorMachineType,
|
||||
MemoryPath: sconfig.HypervisorConfig.MemoryPath,
|
||||
diff --git a/src/runtime/virtcontainers/persist/api/config.go b/src/runtime/virtcontainers/persist/api/config.go
|
||||
index 3bd5567d..88903723 100644
|
||||
--- a/src/runtime/virtcontainers/persist/api/config.go
|
||||
+++ b/src/runtime/virtcontainers/persist/api/config.go
|
||||
@@ -76,6 +76,9 @@ type HypervisorConfig struct {
|
||||
// JailerPathList is the list of jailer paths names allowed in annotations
|
||||
JailerPathList []string
|
||||
|
||||
+ // OzonePath is the ozone executable host path.
|
||||
+ OzonePath string
|
||||
+
|
||||
// BlockDeviceDriver specifies the driver to be used for block device
|
||||
// either VirtioSCSI or VirtioBlock with the default driver being defaultBlockDriver
|
||||
BlockDeviceDriver string
|
||||
diff --git a/src/runtime/virtcontainers/stratovirt.go b/src/runtime/virtcontainers/stratovirt.go
|
||||
index 0f473e31..47daa817 100644
|
||||
--- a/src/runtime/virtcontainers/stratovirt.go
|
||||
+++ b/src/runtime/virtcontainers/stratovirt.go
|
||||
@@ -3,6 +3,7 @@ package virtcontainers
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
+ "io/ioutil"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
@@ -24,10 +25,15 @@ import (
|
||||
otelTrace "go.opentelemetry.io/otel/trace"
|
||||
)
|
||||
|
||||
-const defaultDummyMac = "22:33:44:aa:bb:"
|
||||
-const mmioBlkCount = 4
|
||||
-const mmioNetCount = 2
|
||||
-const randomDevice = "/dev/urandom"
|
||||
+const (
|
||||
+ apiSocket = "qmp.socket"
|
||||
+ debugSocket = "console.socket"
|
||||
+ ozoneBaseDir = "/srv/ozone/stratovirt"
|
||||
+ defaultDummyMac = "22:33:44:aa:bb:"
|
||||
+ mmioBlkCount = 4
|
||||
+ mmioNetCount = 2
|
||||
+ randomDevice = "/dev/urandom"
|
||||
+)
|
||||
|
||||
type stratovirtDev struct {
|
||||
dev interface{}
|
||||
@@ -40,10 +46,19 @@ type stratovirt struct {
|
||||
sandbox *Sandbox
|
||||
store persistapi.PersistDriver
|
||||
config HypervisorConfig
|
||||
+ rootfsPath string
|
||||
+ kernelPath string
|
||||
pid int
|
||||
consolePath string
|
||||
socketPath string
|
||||
+ netNSPath string
|
||||
qmpMonitorCh qmpChannel
|
||||
+ ozoneRoot string
|
||||
+ ozoneRes []string
|
||||
+ useOzone bool
|
||||
+ useImage bool
|
||||
+ pidfile string
|
||||
+ logfile string
|
||||
devices []stratovirtDev
|
||||
HotpluggedVCPUs []CPUDevice
|
||||
mmioBlkSlots [mmioBlkCount]bool
|
||||
@@ -66,10 +81,10 @@ func (s *stratovirt) trace(parent context.Context, name string) (otelTrace.Span,
|
||||
return span, ctx
|
||||
}
|
||||
|
||||
-func (s *stratovirt) getKernelCmdLine(useImage bool) string {
|
||||
+func (s *stratovirt) getKernelCmdLine() string {
|
||||
var params []string
|
||||
|
||||
- if useImage {
|
||||
+ if s.useImage {
|
||||
params = append(params, "root=/dev/vda")
|
||||
}
|
||||
|
||||
@@ -100,14 +115,49 @@ func (s *stratovirt) createSandbox(ctx context.Context, id string, networkNS Net
|
||||
|
||||
s.id = id
|
||||
s.config = *hypervisorConfig
|
||||
-
|
||||
- s.socketPath = filepath.Join(s.store.RunVMStoragePath(), id, "qmp.sock")
|
||||
- s.consolePath = filepath.Join(s.store.RunVMStoragePath(), id, "console.sock")
|
||||
+ if s.config.OzonePath == "" {
|
||||
+ s.useOzone = false
|
||||
+ s.pidfile = filepath.Join(s.store.RunVMStoragePath(), s.id, "pid")
|
||||
+ s.logfile = filepath.Join(s.store.RunVMStoragePath(), s.id, "/stratovirt.log")
|
||||
+ s.socketPath = filepath.Join(s.store.RunVMStoragePath(), id, apiSocket)
|
||||
+ s.consolePath = filepath.Join(s.store.RunVMStoragePath(), id, debugSocket)
|
||||
+ } else {
|
||||
+ s.useOzone = true
|
||||
+ s.ozoneRoot = filepath.Join(ozoneBaseDir, s.id)
|
||||
+ s.pidfile = filepath.Join(s.ozoneRoot, "pid")
|
||||
+ s.logfile = filepath.Join(s.ozoneRoot, "stratovirt.log")
|
||||
+ s.socketPath = filepath.Join(s.ozoneRoot, apiSocket)
|
||||
+ s.consolePath = filepath.Join(s.ozoneRoot, debugSocket)
|
||||
+ }
|
||||
+ s.netNSPath = networkNS.NetNsPath
|
||||
s.qmpMonitorCh = qmpChannel{
|
||||
ctx: s.ctx,
|
||||
path: s.socketPath,
|
||||
}
|
||||
|
||||
+ if kernelPath, err := s.config.KernelAssetPath(); err == nil {
|
||||
+ s.kernelPath = kernelPath
|
||||
+ s.ozoneRes = append(s.ozoneRes, s.kernelPath)
|
||||
+ }
|
||||
+
|
||||
+ initrdPath, err := s.config.InitrdAssetPath()
|
||||
+ if err != nil {
|
||||
+ return err
|
||||
+ }
|
||||
+
|
||||
+ if initrdPath == "" {
|
||||
+ imagePath, err := s.config.ImageAssetPath()
|
||||
+ if err != nil {
|
||||
+ return err
|
||||
+ }
|
||||
+ s.useImage = true
|
||||
+ s.rootfsPath = imagePath
|
||||
+ } else {
|
||||
+ s.useImage = false
|
||||
+ s.rootfsPath = initrdPath
|
||||
+ }
|
||||
+ s.ozoneRes = append(s.ozoneRes, s.rootfsPath)
|
||||
+
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -134,48 +184,43 @@ func (s *stratovirt) waitSandBoxStarted(timeout int) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
-func (s *stratovirt) startSandbox(ctx context.Context, timeout int) error {
|
||||
- span, _ := s.trace(ctx, "startSandbox")
|
||||
- defer span.End()
|
||||
-
|
||||
+func (s *stratovirt) createbaseParams() []string {
|
||||
var params []string
|
||||
- var use_image bool
|
||||
+
|
||||
params = append(params, "-name", fmt.Sprintf("sandbox-%s", s.id))
|
||||
- params = append(params, "-qmp", fmt.Sprintf("unix:%s,server,nowait", s.socketPath))
|
||||
+ params = append(params, "-append", s.getKernelCmdLine())
|
||||
+ params = append(params, "-smp", fmt.Sprintf("%d", s.config.NumVCPUs))
|
||||
+ params = append(params, "-m", fmt.Sprintf("%d", uint64(s.config.MemorySize)))
|
||||
+ params = append(params, "-device", "virtio-serial-device")
|
||||
+ params = append(params, "-device", "virtconsole,chardev=charconsole0,id=virtioconsole0")
|
||||
+ params = append(params, "-object", fmt.Sprintf("rng-random,id=objrng0,filename=%s", randomDevice))
|
||||
+ params = append(params, "-device", "virtio-rng-device,rng=objrng0")
|
||||
|
||||
- if kernelPath, err := s.config.KernelAssetPath(); err == nil {
|
||||
- params = append(params, "-kernel", kernelPath)
|
||||
- }
|
||||
+ // daemonize
|
||||
+ params = append(params, "-daemonize")
|
||||
|
||||
- initrdPath, err := s.config.InitrdAssetPath()
|
||||
- if err != nil {
|
||||
- return err
|
||||
+ return params
|
||||
+}
|
||||
+
|
||||
+func (s *stratovirt) createOzoneParams(params []string) ([]string, error) {
|
||||
+ params = append(params, "-qmp", fmt.Sprintf("unix:%s,server,nowait", apiSocket))
|
||||
+ params = append(params, "-chardev", fmt.Sprintf("socket,id=charconsole0,path=%s,server,nowait", debugSocket))
|
||||
+ params = append(params, "-kernel", filepath.Base(s.kernelPath))
|
||||
+ params = append(params, "-pidfile", filepath.Base(s.pidfile))
|
||||
+
|
||||
+ // append logfile only on debug
|
||||
+ if s.config.Debug {
|
||||
+ params = append(params, "-D", filepath.Base(s.logfile))
|
||||
}
|
||||
|
||||
- if initrdPath == "" {
|
||||
- imagePath, err := s.config.ImageAssetPath()
|
||||
- if err != nil {
|
||||
- return err
|
||||
- }
|
||||
- use_image = true
|
||||
+ if s.useImage {
|
||||
s.mmioBlkSlots[0] = true
|
||||
params = append(params, "-device", "virtio-blk-device,drive=rootfs")
|
||||
- params = append(params, "-drive", fmt.Sprintf("id=rootfs,file=%s,direct=off", imagePath))
|
||||
+ params = append(params, "-drive", fmt.Sprintf("id=rootfs,file=%s,direct=off", filepath.Base(s.rootfsPath)))
|
||||
} else {
|
||||
- use_image = false
|
||||
- params = append(params, "-initrd", initrdPath)
|
||||
+ params = append(params, "-initrd", filepath.Base(s.rootfsPath))
|
||||
}
|
||||
|
||||
- params = append(params, "-append", s.getKernelCmdLine(use_image))
|
||||
- params = append(params, "-smp", fmt.Sprintf("%d", s.config.NumVCPUs))
|
||||
- params = append(params, "-m", fmt.Sprintf("%d", uint64(s.config.MemorySize)))
|
||||
- params = append(params, "-device", "virtio-serial-device")
|
||||
- params = append(params, "-device", "virtconsole,chardev=charconsole0,id=virtioconsole0")
|
||||
- params = append(params, "-object", fmt.Sprintf("rng-random,id=objrng0,filename=%s", randomDevice))
|
||||
- params = append(params, "-device", "virtio-rng-device,rng=objrng0")
|
||||
- params = append(params, "-chardev", fmt.Sprintf("socket,id=charconsole0,path=%s,server,nowait", s.consolePath))
|
||||
- params = append(params, "-pidfile", filepath.Join(s.store.RunVMStoragePath(), s.id, "pid"))
|
||||
-
|
||||
// add devices to cmdline
|
||||
for _, d := range s.devices {
|
||||
switch v := d.dev.(type) {
|
||||
@@ -188,8 +233,9 @@ func (s *stratovirt) startSandbox(ctx context.Context, timeout int) error {
|
||||
case config.BlockDrive:
|
||||
id := v.ID
|
||||
path := v.File
|
||||
- params = append(params, "-device", fmt.Sprintf("virtio-blk-device, drive=%s", id))
|
||||
- params = append(params, "-drive", fmt.Sprintf("id=%s,file=%s", id, path))
|
||||
+ s.ozoneRes = append(s.ozoneRes, path)
|
||||
+ params = append(params, "-device", fmt.Sprintf("virtio-blk-device,drive=%s", id))
|
||||
+ params = append(params, "-drive", fmt.Sprintf("id=%s,file=%s", id, filepath.Base(path)))
|
||||
case types.VSock:
|
||||
v.VhostFd.Close()
|
||||
params = append(params, "-device", fmt.Sprintf("vhost-vsock-device,id=vsock-id,guest-cid=%d", v.ContextID))
|
||||
@@ -198,42 +244,125 @@ func (s *stratovirt) startSandbox(ctx context.Context, timeout int) error {
|
||||
}
|
||||
}
|
||||
|
||||
- // daemonize
|
||||
- params = append(params, "-daemonize")
|
||||
+ return params, nil
|
||||
+}
|
||||
+
|
||||
+func (s *stratovirt) createParams(params []string) ([]string, error) {
|
||||
+ params = append(params, "-qmp", fmt.Sprintf("unix:%s,server,nowait", s.socketPath))
|
||||
+ params = append(params, "-chardev", fmt.Sprintf("socket,id=charconsole0,path=%s,server,nowait", s.consolePath))
|
||||
+ params = append(params, "-kernel", s.kernelPath)
|
||||
+ params = append(params, "-pidfile", s.pidfile)
|
||||
|
||||
// append logfile only on debug
|
||||
if s.config.Debug {
|
||||
- dir := filepath.Join(s.store.RunVMStoragePath(), s.id)
|
||||
- params = append(params, "-D", fmt.Sprintf("%s/stratovirt.log", dir))
|
||||
+ params = append(params, "-D", s.logfile)
|
||||
+ }
|
||||
+
|
||||
+ if s.useImage {
|
||||
+ s.mmioBlkSlots[0] = true
|
||||
+ params = append(params, "-device", "virtio-blk-device,drive=rootfs")
|
||||
+ params = append(params, "-drive", fmt.Sprintf("id=rootfs,file=%s,direct=off", s.rootfsPath))
|
||||
+ } else {
|
||||
+ params = append(params, "-initrd", s.rootfsPath)
|
||||
}
|
||||
|
||||
- dir := filepath.Join(s.store.RunVMStoragePath(), s.id)
|
||||
- err = os.MkdirAll(dir, DirMode)
|
||||
+ // add devices to cmdline
|
||||
+ for _, d := range s.devices {
|
||||
+ switch v := d.dev.(type) {
|
||||
+ case Endpoint:
|
||||
+ name := v.Name()
|
||||
+ mac := v.HardwareAddr()
|
||||
+ tapName := v.NetworkPair().TapInterface.TAPIface.Name
|
||||
+ params = append(params, "-device", fmt.Sprintf("virtio-net-device,netdev=%s,id=%s,mac=%s", name, name, mac))
|
||||
+ params = append(params, "-netdev", fmt.Sprintf("tap,id=%s,ifname=%s", name, tapName))
|
||||
+ case config.BlockDrive:
|
||||
+ id := v.ID
|
||||
+ path := v.File
|
||||
+ params = append(params, "-device", fmt.Sprintf("virtio-blk-device,drive=%s", id))
|
||||
+ params = append(params, "-drive", fmt.Sprintf("id=%s,file=%s", id, path))
|
||||
+ case types.VSock:
|
||||
+ v.VhostFd.Close()
|
||||
+ params = append(params, "-device", fmt.Sprintf("vhost-vsock-device,id=vsock-id,guest-cid=%d", v.ContextID))
|
||||
+ default:
|
||||
+ s.Logger().Error("Adding device type is unsupported")
|
||||
+ }
|
||||
+ }
|
||||
+
|
||||
+ return params, nil
|
||||
+}
|
||||
+
|
||||
+func (s *stratovirt) startSandbox(ctx context.Context, timeout int) error {
|
||||
+ span, _ := s.trace(ctx, "startSandbox")
|
||||
+ defer span.End()
|
||||
+
|
||||
+ var err error
|
||||
+ var cmd *exec.Cmd
|
||||
+
|
||||
+ params := s.createbaseParams()
|
||||
+
|
||||
+ stratovirtBinPath, err := s.config.HypervisorAssetPath()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
- defer func() {
|
||||
+
|
||||
+ if s.useOzone {
|
||||
+ var ozoneParams []string
|
||||
+ extend_params, err := s.createOzoneParams(params)
|
||||
if err != nil {
|
||||
- if err := os.RemoveAll(dir); err != nil {
|
||||
- s.Logger().WithError(err).Error("Fail to clean up vm dir %s", dir)
|
||||
+ return err
|
||||
+ }
|
||||
+ ozoneParams = append(ozoneParams, "-exec-file", stratovirtBinPath)
|
||||
+ ozoneParams = append(ozoneParams, "-name", s.id)
|
||||
+ ozoneParams = append(ozoneParams, "-gid", "0")
|
||||
+ ozoneParams = append(ozoneParams, "-uid", "0")
|
||||
+ if s.netNSPath != "" {
|
||||
+ ozoneParams = append(ozoneParams, "-netns", s.netNSPath)
|
||||
+ }
|
||||
+
|
||||
+ ozoneParams = append(ozoneParams, "-source")
|
||||
+ ozoneParams = append(ozoneParams, s.ozoneRes...)
|
||||
+
|
||||
+ defer func() {
|
||||
+ if err != nil {
|
||||
+ ozoneParams = append(ozoneParams, "-clean-resource")
|
||||
+ cmd = exec.CommandContext(s.ctx, s.config.OzonePath, ozoneParams...)
|
||||
+ if err := cmd.Run(); err != nil {
|
||||
+ s.Logger().WithError(err).Error("Failed to clean up ozone dir %s", s.ozoneRoot)
|
||||
+ }
|
||||
}
|
||||
+ }()
|
||||
+
|
||||
+ ozoneParams = append(ozoneParams, "--")
|
||||
+ ozoneParams = append(ozoneParams, extend_params...)
|
||||
+ cmd = exec.CommandContext(s.ctx, s.config.OzonePath, ozoneParams...)
|
||||
+ s.Logger().Info("StratoVirt/Ozone start with params: ", cmd)
|
||||
+ } else {
|
||||
+ params, err = s.createParams(params)
|
||||
+ if err != nil {
|
||||
+ return err
|
||||
}
|
||||
- }()
|
||||
|
||||
- binPath, err := s.config.HypervisorAssetPath()
|
||||
- if err != nil {
|
||||
- s.Logger().WithField("Fail to get hypervisor bin path", err).Error()
|
||||
- return err
|
||||
- }
|
||||
+ dir := filepath.Join(s.store.RunVMStoragePath(), s.id)
|
||||
+ err = os.MkdirAll(dir, DirMode)
|
||||
+ if err != nil {
|
||||
+ return err
|
||||
+ }
|
||||
+ defer func() {
|
||||
+ if err != nil {
|
||||
+ if err := os.RemoveAll(dir); err != nil {
|
||||
+ s.Logger().WithError(err).Error("Fail to clean up vm dir %s", dir)
|
||||
+ }
|
||||
+ }
|
||||
+ }()
|
||||
|
||||
- cmd := exec.CommandContext(s.ctx, binPath, params...)
|
||||
- s.Logger().Info("StratoVirt start with params: ", cmd)
|
||||
+ cmd = exec.CommandContext(s.ctx, stratovirtBinPath, params...)
|
||||
+ s.Logger().Info("StratoVirt start with params: ", cmd)
|
||||
+ }
|
||||
|
||||
if err := cmd.Start(); err != nil {
|
||||
s.Logger().WithField("Error starting hypervisor, please check the params", err).Error()
|
||||
return err
|
||||
}
|
||||
- s.pid = cmd.Process.Pid
|
||||
|
||||
if err = s.waitSandBoxStarted(timeout); err != nil {
|
||||
return err
|
||||
@@ -420,6 +549,7 @@ func (s *stratovirt) hotplugNet(ctx context.Context, endpoint Endpoint, op opera
|
||||
}
|
||||
|
||||
func (s *stratovirt) hotplugBlk(drive *config.BlockDrive, op operation) (err error) {
|
||||
+ var filePath string
|
||||
err = s.qmpSetup()
|
||||
if err != nil {
|
||||
return err
|
||||
@@ -427,13 +557,18 @@ func (s *stratovirt) hotplugBlk(drive *config.BlockDrive, op operation) (err err
|
||||
|
||||
switch op {
|
||||
case addDevice:
|
||||
- driver := "virtio-blk-pci"
|
||||
+ driver := "virtio-blk-mmio"
|
||||
+ if s.useOzone {
|
||||
+ filePath, err = s.updateOzoneRes(drive.File, true)
|
||||
+ } else {
|
||||
+ filePath = drive.File
|
||||
+ }
|
||||
slot, err := s.getDevSlot(drive.VirtPath, false)
|
||||
if err != nil {
|
||||
return fmt.Errorf("Could not get unused slot for %q", drive.VirtPath)
|
||||
}
|
||||
|
||||
- if err := s.qmpMonitorCh.qmp.ExecuteBlockdevAdd(s.qmpMonitorCh.ctx, drive.File, drive.ID, false); err != nil {
|
||||
+ if err := s.qmpMonitorCh.qmp.ExecuteBlockdevAdd(s.qmpMonitorCh.ctx, filePath, drive.ID, false); err != nil {
|
||||
s.getDevSlot(drive.VirtPath, true)
|
||||
return err
|
||||
}
|
||||
@@ -443,6 +578,9 @@ func (s *stratovirt) hotplugBlk(drive *config.BlockDrive, op operation) (err err
|
||||
return err
|
||||
}
|
||||
case removeDevice:
|
||||
+ if s.useOzone {
|
||||
+ s.updateOzoneRes(drive.File, false)
|
||||
+ }
|
||||
if err := s.qmpMonitorCh.qmp.ExecuteDeviceDel(s.qmpMonitorCh.ctx, drive.ID); err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -582,17 +720,62 @@ func (s *stratovirt) getThreadIDs(ctx context.Context) (vcpuThreadIDs, error) {
|
||||
return tid, nil
|
||||
}
|
||||
|
||||
+func (s *stratovirt) updateOzoneRes(src string, add bool) (string, error) {
|
||||
+ dst := filepath.Join(s.ozoneRoot, filepath.Base(src))
|
||||
+ if add {
|
||||
+ if err := bindMount(context.Background(), src, dst, false, "slave"); err != nil {
|
||||
+ s.Logger().WithField("bindMount failed", err).Error()
|
||||
+ return "", err
|
||||
+ }
|
||||
+ } else {
|
||||
+ syscall.Unmount(dst, syscall.MNT_DETACH)
|
||||
+ }
|
||||
+ return filepath.Base(src), nil
|
||||
+}
|
||||
+
|
||||
+func (s *stratovirt) cleanOzoneRes() {
|
||||
+ s.updateOzoneRes(s.rootfsPath, false)
|
||||
+ s.updateOzoneRes(s.kernelPath, false)
|
||||
+
|
||||
+ if err := os.RemoveAll(s.ozoneRoot); err != nil {
|
||||
+ s.Logger().WithField("cleanupOzone failed", err).Error()
|
||||
+ }
|
||||
+}
|
||||
+
|
||||
func (s *stratovirt) cleanup(ctx context.Context) error {
|
||||
span, _ := s.trace(ctx, "cleanup")
|
||||
defer span.End()
|
||||
|
||||
s.qmpTeardown()
|
||||
+ if s.useOzone {
|
||||
+ s.cleanOzoneRes()
|
||||
+ }
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *stratovirt) getPids() []int {
|
||||
- return []int{s.pid}
|
||||
+ var pids []int
|
||||
+ if s.pid != 0 {
|
||||
+ pids = append(pids, s.pid)
|
||||
+ } else {
|
||||
+ pid, err := ioutil.ReadFile(s.pidfile)
|
||||
+ if err != nil {
|
||||
+ s.Logger().WithError(err).Error("Read pid file failed.")
|
||||
+ return []int{0}
|
||||
+ }
|
||||
+
|
||||
+ p, err := strconv.Atoi(strings.Trim(string(pid), "\n\t "))
|
||||
+ if err != nil {
|
||||
+ s.Logger().WithError(err).Error("Get pid from pid file failed.")
|
||||
+ return []int{0}
|
||||
+ }
|
||||
+
|
||||
+ pids = append(pids, p)
|
||||
+ s.pid = p
|
||||
+ }
|
||||
+
|
||||
+ return pids
|
||||
}
|
||||
|
||||
func (s *stratovirt) getVirtioFsPid() *int {
|
||||
--
|
||||
2.21.1 (Apple Git-122.3)
|
||||
|
||||
@ -0,0 +1,295 @@
|
||||
From 77ed6fefe70edde63b01d797b76f389bc82bb1a0 Mon Sep 17 00:00:00 2001
|
||||
From: Wei Gao <gaowei66@huawei.com>
|
||||
Date: Mon, 9 Aug 2021 14:57:06 +0800
|
||||
Subject: [PATCH 6/6] factory: add the template factory support for hypervisor
|
||||
type stratovirt.
|
||||
|
||||
Signed-off-by: Wei Gao <gaowei66@huawei.com>
|
||||
---
|
||||
src/runtime/pkg/katautils/config.go | 2 +-
|
||||
.../factory/template/template.go | 21 +++--
|
||||
src/runtime/virtcontainers/kata_agent.go | 7 +-
|
||||
src/runtime/virtcontainers/stratovirt.go | 89 +++++++++++++++++--
|
||||
src/runtime/virtcontainers/vm.go | 28 ++++--
|
||||
5 files changed, 125 insertions(+), 22 deletions(-)
|
||||
|
||||
diff --git a/src/runtime/pkg/katautils/config.go b/src/runtime/pkg/katautils/config.go
|
||||
index 828c2a43..718677b4 100644
|
||||
--- a/src/runtime/pkg/katautils/config.go
|
||||
+++ b/src/runtime/pkg/katautils/config.go
|
||||
@@ -1363,7 +1363,7 @@ func checkNetNsConfig(config oci.RuntimeConfig) error {
|
||||
// checkFactoryConfig ensures the VM factory configuration is valid.
|
||||
func checkFactoryConfig(config oci.RuntimeConfig) error {
|
||||
if config.FactoryConfig.Template {
|
||||
- if config.HypervisorConfig.InitrdPath == "" {
|
||||
+ if config.HypervisorConfig.InitrdPath == "" && (config.HypervisorType != vc.StratovirtHypervisor) {
|
||||
return errors.New("Factory option enable_template requires an initrd image")
|
||||
}
|
||||
}
|
||||
diff --git a/src/runtime/virtcontainers/factory/template/template.go b/src/runtime/virtcontainers/factory/template/template.go
|
||||
index 66070126..02497097 100644
|
||||
--- a/src/runtime/virtcontainers/factory/template/template.go
|
||||
+++ b/src/runtime/virtcontainers/factory/template/template.go
|
||||
@@ -96,11 +96,15 @@ func (t *template) prepareTemplateFiles() error {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
- flags := uintptr(syscall.MS_NOSUID | syscall.MS_NODEV)
|
||||
- opts := fmt.Sprintf("size=%dM", t.config.HypervisorConfig.MemorySize+templateDeviceStateSize)
|
||||
- if err = syscall.Mount("tmpfs", t.statePath, "tmpfs", flags, opts); err != nil {
|
||||
- t.close()
|
||||
- return err
|
||||
+
|
||||
+ // If use hypervisor stratovirt, no need to create template path with ramdisk.
|
||||
+ if t.config.HypervisorType != vc.StratovirtHypervisor {
|
||||
+ flags := uintptr(syscall.MS_NOSUID | syscall.MS_NODEV)
|
||||
+ opts := fmt.Sprintf("size=%dM", t.config.HypervisorConfig.MemorySize+templateDeviceStateSize)
|
||||
+ if err = syscall.Mount("tmpfs", t.statePath, "tmpfs", flags, opts); err != nil {
|
||||
+ t.close()
|
||||
+ return err
|
||||
+ }
|
||||
}
|
||||
f, err := os.Create(t.statePath + "/memory")
|
||||
if err != nil {
|
||||
@@ -126,8 +130,11 @@ func (t *template) createTemplateVM(ctx context.Context) error {
|
||||
}
|
||||
defer vm.Stop(ctx)
|
||||
|
||||
- if err = vm.Disconnect(ctx); err != nil {
|
||||
- return err
|
||||
+ // Create template on hypervisor stratovirt, don't have connection with agent.
|
||||
+ if config.HypervisorType != vc.StratovirtHypervisor {
|
||||
+ if err = vm.Disconnect(ctx); err != nil {
|
||||
+ return err
|
||||
+ }
|
||||
}
|
||||
|
||||
// Sleep a bit to let the agent grpc server clean up
|
||||
diff --git a/src/runtime/virtcontainers/kata_agent.go b/src/runtime/virtcontainers/kata_agent.go
|
||||
index 13d31658..bc882c70 100644
|
||||
--- a/src/runtime/virtcontainers/kata_agent.go
|
||||
+++ b/src/runtime/virtcontainers/kata_agent.go
|
||||
@@ -1306,8 +1306,11 @@ func (k *kataAgent) buildContainerRootfs(ctx context.Context, sandbox *Sandbox,
|
||||
// TODO: remove dependency on shared fs path. shared fs is just one kind of storage source.
|
||||
// we should not always use shared fs path for all kinds of storage. Instead, all storage
|
||||
// should be bind mounted to a tmpfs path for containers to use.
|
||||
- if err := os.MkdirAll(filepath.Join(getMountPath(c.sandbox.id), c.id, c.rootfsSuffix), DirMode); err != nil {
|
||||
- return nil, err
|
||||
+ // If boot from template on stratovirt, no need to mkdir mount path.
|
||||
+ if !((sandbox.config.HypervisorType == StratovirtHypervisor) && sandbox.config.HypervisorConfig.BootFromTemplate) {
|
||||
+ if err := os.MkdirAll(filepath.Join(getMountPath(c.sandbox.id), c.id, c.rootfsSuffix), DirMode); err != nil {
|
||||
+ return nil, err
|
||||
+ }
|
||||
}
|
||||
return rootfs, nil
|
||||
}
|
||||
diff --git a/src/runtime/virtcontainers/stratovirt.go b/src/runtime/virtcontainers/stratovirt.go
|
||||
index 47daa817..e9b2ba85 100644
|
||||
--- a/src/runtime/virtcontainers/stratovirt.go
|
||||
+++ b/src/runtime/virtcontainers/stratovirt.go
|
||||
@@ -48,6 +48,7 @@ type stratovirt struct {
|
||||
config HypervisorConfig
|
||||
rootfsPath string
|
||||
kernelPath string
|
||||
+ templatePath string
|
||||
pid int
|
||||
consolePath string
|
||||
socketPath string
|
||||
@@ -115,7 +116,7 @@ func (s *stratovirt) createSandbox(ctx context.Context, id string, networkNS Net
|
||||
|
||||
s.id = id
|
||||
s.config = *hypervisorConfig
|
||||
- if s.config.OzonePath == "" {
|
||||
+ if (s.config.OzonePath == "") || s.config.BootToBeTemplate {
|
||||
s.useOzone = false
|
||||
s.pidfile = filepath.Join(s.store.RunVMStoragePath(), s.id, "pid")
|
||||
s.logfile = filepath.Join(s.store.RunVMStoragePath(), s.id, "/stratovirt.log")
|
||||
@@ -129,6 +130,20 @@ func (s *stratovirt) createSandbox(ctx context.Context, id string, networkNS Net
|
||||
s.socketPath = filepath.Join(s.ozoneRoot, apiSocket)
|
||||
s.consolePath = filepath.Join(s.ozoneRoot, debugSocket)
|
||||
}
|
||||
+
|
||||
+ if s.config.VMid != "" && s.useOzone {
|
||||
+ // Make sure the symlinks do not exist
|
||||
+ os.RemoveAll(s.ozoneRoot)
|
||||
+ ozoneVmRoot := filepath.Join(ozoneBaseDir, s.config.VMid)
|
||||
+ if err := os.Symlink(ozoneVmRoot, s.ozoneRoot); err != nil {
|
||||
+ return err
|
||||
+ }
|
||||
+ }
|
||||
+
|
||||
+ if s.config.BootFromTemplate || s.config.BootToBeTemplate {
|
||||
+ s.templatePath = strings.Replace(s.config.DevicesStatePath, "/state", "", -1)
|
||||
+ }
|
||||
+
|
||||
s.netNSPath = networkNS.NetNsPath
|
||||
s.qmpMonitorCh = qmpChannel{
|
||||
ctx: s.ctx,
|
||||
@@ -221,6 +236,12 @@ func (s *stratovirt) createOzoneParams(params []string) ([]string, error) {
|
||||
params = append(params, "-initrd", filepath.Base(s.rootfsPath))
|
||||
}
|
||||
|
||||
+ // handle boot from template
|
||||
+ if s.config.BootFromTemplate {
|
||||
+ s.ozoneRes = append(s.ozoneRes, s.templatePath)
|
||||
+ params = append(params, "-incoming", fmt.Sprintf("file:%s", filepath.Base(s.templatePath)))
|
||||
+ }
|
||||
+
|
||||
// add devices to cmdline
|
||||
for _, d := range s.devices {
|
||||
switch v := d.dev.(type) {
|
||||
@@ -266,6 +287,11 @@ func (s *stratovirt) createParams(params []string) ([]string, error) {
|
||||
params = append(params, "-initrd", s.rootfsPath)
|
||||
}
|
||||
|
||||
+ // handle boot from template
|
||||
+ if s.config.BootFromTemplate {
|
||||
+ params = append(params, "-incoming", fmt.Sprintf("file:%s", s.templatePath))
|
||||
+ }
|
||||
+
|
||||
// add devices to cmdline
|
||||
for _, d := range s.devices {
|
||||
switch v := d.dev.(type) {
|
||||
@@ -410,14 +436,55 @@ func (s *stratovirt) stopSandbox(ctx context.Context, force bool) error {
|
||||
}
|
||||
|
||||
func (s *stratovirt) pauseSandbox(ctx context.Context) error {
|
||||
- return nil
|
||||
+ span, _ := s.trace(ctx, "pauseSandbox")
|
||||
+ defer span.End()
|
||||
+
|
||||
+ return s.togglePauseSandbox(ctx, true)
|
||||
}
|
||||
|
||||
func (s *stratovirt) saveSandbox() error {
|
||||
+ s.Logger().Info("save sandbox")
|
||||
+
|
||||
+ err := s.qmpSetup()
|
||||
+ if err != nil {
|
||||
+ return err
|
||||
+ }
|
||||
+
|
||||
+ // BootToBeTemplate sets the VM to be a template that other VMs can can clone from.
|
||||
+ // We would want to bypass shared memory when saving VM to local file through migrate.
|
||||
+ if s.config.BootToBeTemplate {
|
||||
+ err = s.qmpMonitorCh.qmp.ExecSetMigrateArguments(s.qmpMonitorCh.ctx, fmt.Sprintf("file:%s", s.templatePath))
|
||||
+ if err != nil {
|
||||
+ s.Logger().WithError(err).Error("exec migration")
|
||||
+ return err
|
||||
+ }
|
||||
+ }
|
||||
+
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *stratovirt) resumeSandbox(ctx context.Context) error {
|
||||
+ span, _ := s.trace(ctx, "resumeSandbox")
|
||||
+ defer span.End()
|
||||
+
|
||||
+ return s.togglePauseSandbox(ctx, false)
|
||||
+}
|
||||
+
|
||||
+func (s *stratovirt) togglePauseSandbox(ctx context.Context, pause bool) error {
|
||||
+ span, _ := s.trace(ctx, "togglePauseSandbox")
|
||||
+ defer span.End()
|
||||
+
|
||||
+ err := s.qmpSetup()
|
||||
+ if err != nil {
|
||||
+ return err
|
||||
+ }
|
||||
+
|
||||
+ if pause {
|
||||
+ s.qmpMonitorCh.qmp.ExecuteStop(s.qmpMonitorCh.ctx)
|
||||
+ } else {
|
||||
+ s.qmpMonitorCh.qmp.ExecuteCont(s.qmpMonitorCh.ctx)
|
||||
+ }
|
||||
+
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -734,11 +801,23 @@ func (s *stratovirt) updateOzoneRes(src string, add bool) (string, error) {
|
||||
}
|
||||
|
||||
func (s *stratovirt) cleanOzoneRes() {
|
||||
- s.updateOzoneRes(s.rootfsPath, false)
|
||||
- s.updateOzoneRes(s.kernelPath, false)
|
||||
+ // Umount all resource in ozoneRoot
|
||||
+ if dir, err := ioutil.ReadDir(s.ozoneRoot); err == nil {
|
||||
+ for _, file := range dir {
|
||||
+ syscall.Unmount(filepath.Join(s.ozoneRoot, file.Name()), syscall.MNT_DETACH)
|
||||
+ }
|
||||
+ }
|
||||
|
||||
if err := os.RemoveAll(s.ozoneRoot); err != nil {
|
||||
- s.Logger().WithField("cleanupOzone failed", err).Error()
|
||||
+ s.Logger().WithField("cleanup Ozone failed", err).Error()
|
||||
+ }
|
||||
+
|
||||
+ // If have VMid, the VM is boot from template. ozoneVmRoot also need clean.
|
||||
+ if s.config.VMid != "" {
|
||||
+ ozoneVmRoot := filepath.Join(ozoneBaseDir, s.config.VMid)
|
||||
+ if err := os.RemoveAll(ozoneVmRoot); err != nil {
|
||||
+ s.Logger().WithField("cleanup Ozone failed", err).Error()
|
||||
+ }
|
||||
}
|
||||
}
|
||||
|
||||
diff --git a/src/runtime/virtcontainers/vm.go b/src/runtime/virtcontainers/vm.go
|
||||
index e6f02b6e..c4f9df73 100644
|
||||
--- a/src/runtime/virtcontainers/vm.go
|
||||
+++ b/src/runtime/virtcontainers/vm.go
|
||||
@@ -142,13 +142,19 @@ func NewVM(ctx context.Context, config VMConfig) (*VM, error) {
|
||||
}()
|
||||
|
||||
// 4. check agent aliveness
|
||||
- // VMs booted from template are paused, do not check
|
||||
- if !config.HypervisorConfig.BootFromTemplate {
|
||||
+ // On hypervisor StratoVirt, VMs booted from template are running, check agent
|
||||
+ // On other hypervisors, VMs booted from template are paused, do not check
|
||||
+ if config.HypervisorType == StratovirtHypervisor {
|
||||
+ if !config.HypervisorConfig.BootToBeTemplate {
|
||||
+ virtLog.WithField("vm", id).Info("check agent status")
|
||||
+ err = agent.check(ctx)
|
||||
+ }
|
||||
+ } else if !config.HypervisorConfig.BootFromTemplate {
|
||||
virtLog.WithField("vm", id).Info("check agent status")
|
||||
err = agent.check(ctx)
|
||||
- if err != nil {
|
||||
- return nil, err
|
||||
- }
|
||||
+ }
|
||||
+ if err != nil {
|
||||
+ return nil, err
|
||||
}
|
||||
|
||||
return &VM{
|
||||
@@ -329,9 +335,16 @@ func (v *VM) assignSandbox(s *Sandbox) error {
|
||||
// - link 9pfs share path from sandbox dir (/run/kata-containers/shared/sandboxes/sbid/) to vm dir (/run/vc/vm/vmid/shared/)
|
||||
|
||||
vmSharePath := buildVMSharePath(v.id, v.store.RunVMStoragePath())
|
||||
- vmSockDir := filepath.Join(v.store.RunVMStoragePath(), v.id)
|
||||
sbSharePath := getMountPath(s.id)
|
||||
- sbSockDir := filepath.Join(v.store.RunVMStoragePath(), s.id)
|
||||
+ var vmSockDir string
|
||||
+ var sbSockDir string
|
||||
+ if v.hypervisor.hypervisorConfig().OzonePath != "" {
|
||||
+ vmSockDir = filepath.Join(ozoneBaseDir, v.id)
|
||||
+ sbSockDir = filepath.Join(ozoneBaseDir, s.id)
|
||||
+ } else {
|
||||
+ vmSockDir = filepath.Join(v.store.RunVMStoragePath(), v.id)
|
||||
+ sbSockDir = filepath.Join(v.store.RunVMStoragePath(), s.id)
|
||||
+ }
|
||||
|
||||
v.logger().WithFields(logrus.Fields{
|
||||
"vmSharePath": vmSharePath,
|
||||
@@ -359,6 +372,7 @@ func (v *VM) assignSandbox(s *Sandbox) error {
|
||||
|
||||
s.hypervisor = v.hypervisor
|
||||
s.config.HypervisorConfig.VMid = v.id
|
||||
+ s.config.HypervisorConfig.BootFromTemplate = true
|
||||
|
||||
return nil
|
||||
}
|
||||
--
|
||||
2.21.1 (Apple Git-122.3)
|
||||
|
||||
224
patches/0007-kata-containers-support-with-iSulad.patch
Normal file
224
patches/0007-kata-containers-support-with-iSulad.patch
Normal file
@ -0,0 +1,224 @@
|
||||
From d4605dafaa9c326a5cf24c28d0c1efe6c9997f49 Mon Sep 17 00:00:00 2001
|
||||
From: holyfei <yangfeiyu20092010@163.com>
|
||||
Date: Sat, 21 Aug 2021 17:08:46 +0800
|
||||
Subject: [PATCH] kata-containers: support with iSulad
|
||||
|
||||
reason: support with iSulad
|
||||
|
||||
Signed-off-by: holyfei <yangfeiyu20092010@163.com>
|
||||
---
|
||||
src/agent/rustjail/src/cgroups/fs/mod.rs | 2 +-
|
||||
src/runtime/containerd-shim-v2/container.go | 9 +++
|
||||
src/runtime/containerd-shim-v2/service.go | 55 +++++++++++++++++++
|
||||
src/runtime/containerd-shim-v2/start.go | 10 ++++
|
||||
.../containerd/runtime/v2/shim/shim.go | 8 ++-
|
||||
5 files changed, 81 insertions(+), 3 deletions(-)
|
||||
|
||||
diff --git a/src/agent/rustjail/src/cgroups/fs/mod.rs b/src/agent/rustjail/src/cgroups/fs/mod.rs
|
||||
index 7f41cb4..6c3bb32 100644
|
||||
--- a/src/agent/rustjail/src/cgroups/fs/mod.rs
|
||||
+++ b/src/agent/rustjail/src/cgroups/fs/mod.rs
|
||||
@@ -369,7 +369,7 @@ fn set_memory_resources(cg: &cgroups::Cgroup, memory: &LinuxMemory, update: bool
|
||||
if let Some(swappiness) = memory.swappiness {
|
||||
if (0..=100).contains(&swappiness) {
|
||||
mem_controller.set_swappiness(swappiness as u64)?;
|
||||
- } else {
|
||||
+ } else if swappiness != -1 {
|
||||
return Err(anyhow!(
|
||||
"invalid value:{}. valid memory swappiness range is 0-100",
|
||||
swappiness
|
||||
diff --git a/src/runtime/containerd-shim-v2/container.go b/src/runtime/containerd-shim-v2/container.go
|
||||
index faea0e2..d563888 100644
|
||||
--- a/src/runtime/containerd-shim-v2/container.go
|
||||
+++ b/src/runtime/containerd-shim-v2/container.go
|
||||
@@ -7,10 +7,13 @@ package containerdshim
|
||||
|
||||
import (
|
||||
"io"
|
||||
+ "os"
|
||||
+ "path"
|
||||
"time"
|
||||
|
||||
"github.com/containerd/containerd/api/types/task"
|
||||
"github.com/containerd/containerd/errdefs"
|
||||
+ cdshim "github.com/containerd/containerd/runtime/v2/shim"
|
||||
taskAPI "github.com/containerd/containerd/runtime/v2/task"
|
||||
"github.com/opencontainers/runtime-spec/specs-go"
|
||||
|
||||
@@ -37,6 +40,8 @@ type container struct {
|
||||
status task.Status
|
||||
terminal bool
|
||||
mounted bool
|
||||
+ exitFifo string
|
||||
+ exitFd *os.File
|
||||
}
|
||||
|
||||
func newContainer(s *service, r *taskAPI.CreateTaskRequest, containerType vc.ContainerType, spec *specs.Spec, mounted bool) (*container, error) {
|
||||
@@ -49,6 +54,9 @@ func newContainer(s *service, r *taskAPI.CreateTaskRequest, containerType vc.Con
|
||||
spec = &specs.Spec{}
|
||||
}
|
||||
|
||||
+ dir := os.Getenv(cdshim.ExitFifoDir)
|
||||
+ exitFifo := path.Join(dir, r.ID, exitFifoName)
|
||||
+
|
||||
c := &container{
|
||||
s: s,
|
||||
spec: spec,
|
||||
@@ -65,6 +73,7 @@ func newContainer(s *service, r *taskAPI.CreateTaskRequest, containerType vc.Con
|
||||
exitCh: make(chan uint32, 1),
|
||||
stdinCloser: make(chan struct{}),
|
||||
mounted: mounted,
|
||||
+ exitFifo: exitFifo,
|
||||
}
|
||||
return c, nil
|
||||
}
|
||||
diff --git a/src/runtime/containerd-shim-v2/service.go b/src/runtime/containerd-shim-v2/service.go
|
||||
index 1003f8e..e13283c 100644
|
||||
--- a/src/runtime/containerd-shim-v2/service.go
|
||||
+++ b/src/runtime/containerd-shim-v2/service.go
|
||||
@@ -6,13 +6,16 @@
|
||||
package containerdshim
|
||||
|
||||
import (
|
||||
+ "bytes"
|
||||
"context"
|
||||
+ "encoding/binary"
|
||||
"io/ioutil"
|
||||
"os"
|
||||
sysexec "os/exec"
|
||||
"sync"
|
||||
"syscall"
|
||||
"time"
|
||||
+ "unsafe"
|
||||
|
||||
eventstypes "github.com/containerd/containerd/api/events"
|
||||
"github.com/containerd/containerd/api/types/task"
|
||||
@@ -51,6 +54,8 @@ const (
|
||||
// A time span used to wait for publish a containerd event,
|
||||
// once it costs a longer time than timeOut, it will be canceld.
|
||||
timeOut = 5 * time.Second
|
||||
+
|
||||
+ exitFifoName = "exit_fifo"
|
||||
)
|
||||
|
||||
var (
|
||||
@@ -1019,6 +1024,10 @@ func (s *service) Wait(ctx context.Context, r *taskAPI.WaitRequest) (_ *taskAPI.
|
||||
func (s *service) processExits() {
|
||||
for e := range s.ec {
|
||||
s.checkProcesses(e)
|
||||
+
|
||||
+ if os.Getenv(cdshim.ExitFifoDir) != "" {
|
||||
+ s.closeExitFifo(e)
|
||||
+ }
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1070,3 +1079,49 @@ func (s *service) getContainerStatus(containerID string) (task.Status, error) {
|
||||
|
||||
return status, nil
|
||||
}
|
||||
+
|
||||
+func isBigEndian() (ret bool) {
|
||||
+ i := int(0x1)
|
||||
+ bs := (*[int(unsafe.Sizeof(i))]byte)(unsafe.Pointer(&i))
|
||||
+ return bs[0] == 0
|
||||
+}
|
||||
+
|
||||
+func (s *service) closeExitFifo(e exit) {
|
||||
+ if e.execid != "" {
|
||||
+ // not a container, no need to close exit fifo
|
||||
+ return
|
||||
+ }
|
||||
+
|
||||
+ var ret uint32
|
||||
+ var nativeEndian binary.ByteOrder
|
||||
+
|
||||
+ s.mu.Lock()
|
||||
+ c, err := s.getContainer(e.id)
|
||||
+ s.mu.Unlock()
|
||||
+
|
||||
+ if err != nil {
|
||||
+ logrus.WithError(err).Errorf("Process container:%v exit fifo failed", e.id)
|
||||
+ return
|
||||
+ }
|
||||
+
|
||||
+ ret = <-c.exitCh
|
||||
+ // refill the exitCh with the container process's exit code in case
|
||||
+ // there were other waits on this process.
|
||||
+ c.exitCh <- ret
|
||||
+
|
||||
+ if isBigEndian() {
|
||||
+ nativeEndian = binary.BigEndian
|
||||
+ } else {
|
||||
+ nativeEndian = binary.LittleEndian
|
||||
+ }
|
||||
+
|
||||
+ bytesBuffer := bytes.NewBuffer([]byte{})
|
||||
+ binary.Write(bytesBuffer, nativeEndian, &ret)
|
||||
+
|
||||
+ _, err = c.exitFd.Write(bytesBuffer.Bytes())
|
||||
+ if err != nil {
|
||||
+ logrus.WithError(err).Error("write exit fifo failed")
|
||||
+ }
|
||||
+
|
||||
+ c.exitFd.Close()
|
||||
+}
|
||||
diff --git a/src/runtime/containerd-shim-v2/start.go b/src/runtime/containerd-shim-v2/start.go
|
||||
index 72420e4..e89dc48 100644
|
||||
--- a/src/runtime/containerd-shim-v2/start.go
|
||||
+++ b/src/runtime/containerd-shim-v2/start.go
|
||||
@@ -8,8 +8,11 @@ package containerdshim
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
+ "golang.org/x/sys/unix"
|
||||
+ "os"
|
||||
|
||||
"github.com/containerd/containerd/api/types/task"
|
||||
+ cdshim "github.com/containerd/containerd/runtime/v2/shim"
|
||||
"github.com/kata-containers/kata-containers/src/runtime/pkg/katautils"
|
||||
)
|
||||
|
||||
@@ -59,6 +62,13 @@ func startContainer(ctx context.Context, s *service, c *container) error {
|
||||
|
||||
c.status = task.StatusRunning
|
||||
|
||||
+ if os.Getenv(cdshim.ExitFifoDir) != "" {
|
||||
+ c.exitFd, err = os.OpenFile(c.exitFifo, unix.O_WRONLY|unix.O_NONBLOCK|unix.O_CLOEXEC, 0)
|
||||
+ if err != nil {
|
||||
+ return err
|
||||
+ }
|
||||
+ }
|
||||
+
|
||||
stdin, stdout, stderr, err := s.sandbox.IOStream(c.id, c.id)
|
||||
if err != nil {
|
||||
return err
|
||||
diff --git a/src/runtime/vendor/github.com/containerd/containerd/runtime/v2/shim/shim.go b/src/runtime/vendor/github.com/containerd/containerd/runtime/v2/shim/shim.go
|
||||
index d60d496..946c386 100644
|
||||
--- a/src/runtime/vendor/github.com/containerd/containerd/runtime/v2/shim/shim.go
|
||||
+++ b/src/runtime/vendor/github.com/containerd/containerd/runtime/v2/shim/shim.go
|
||||
@@ -84,6 +84,8 @@ var (
|
||||
action string
|
||||
)
|
||||
|
||||
+var ExitFifoDir = "EXIT_FIFO_DIR"
|
||||
+
|
||||
func parseFlags() {
|
||||
flag.BoolVar(&debugFlag, "debug", false, "enable debug output in logs")
|
||||
flag.StringVar(&namespaceFlag, "namespace", "", "namespace that owns the shim")
|
||||
@@ -198,8 +200,10 @@ func run(id string, initFunc Init, config Config) error {
|
||||
}
|
||||
return nil
|
||||
default:
|
||||
- if err := setLogger(ctx, idFlag); err != nil {
|
||||
- return err
|
||||
+ if os.Getenv("EXIT_FIFO_DIR") == "" {
|
||||
+ if err := setLogger(ctx, idFlag); err != nil {
|
||||
+ return err
|
||||
+ }
|
||||
}
|
||||
client := NewShimClient(ctx, service, signals)
|
||||
return client.Serve()
|
||||
--
|
||||
2.23.0
|
||||
|
||||
29
patches/0008-kata-containers-adpat-with-iSulad.patch
Normal file
29
patches/0008-kata-containers-adpat-with-iSulad.patch
Normal file
@ -0,0 +1,29 @@
|
||||
From 5ee8e64c8c620f02cb580f1f3349ae63660ca34c Mon Sep 17 00:00:00 2001
|
||||
From: holyfei <yangfeiyu20092010@163.com>
|
||||
Date: Mon, 27 Sep 2021 11:03:58 +0800
|
||||
Subject: [PATCH] kata-containers: adpat with iSulad
|
||||
|
||||
reason: chmod the exec fifo to 644, isula start container
|
||||
and need the permission
|
||||
|
||||
Signed-off-by: holyfei <yangfeiyu20092010@163.com>
|
||||
---
|
||||
src/agent/rustjail/src/container.rs | 2 +-
|
||||
1 file changed, 1 insertion(+), 1 deletion(-)
|
||||
|
||||
diff --git a/src/agent/rustjail/src/container.rs b/src/agent/rustjail/src/container.rs
|
||||
index 748ee486..5113a482 100644
|
||||
--- a/src/agent/rustjail/src/container.rs
|
||||
+++ b/src/agent/rustjail/src/container.rs
|
||||
@@ -822,7 +822,7 @@ impl BaseContainer for LinuxContainer {
|
||||
if stat::stat(fifo_file.as_str()).is_ok() {
|
||||
return Err(anyhow!("exec fifo exists"));
|
||||
}
|
||||
- unistd::mkfifo(fifo_file.as_str(), Mode::from_bits(0o622).unwrap())?;
|
||||
+ unistd::mkfifo(fifo_file.as_str(), Mode::from_bits(0o644).unwrap())?;
|
||||
|
||||
fifofd = fcntl::open(
|
||||
fifo_file.as_str(),
|
||||
--
|
||||
2.23.0
|
||||
|
||||
@ -0,0 +1,27 @@
|
||||
From c563b455e3bb0cebfbe2a77c8d5ebac36aac3c76 Mon Sep 17 00:00:00 2001
|
||||
From: jikui <jikui2@huawei.com>
|
||||
Date: Thu, 4 Nov 2021 19:58:33 +0800
|
||||
Subject: [PATCH] kata-runtime: fix kata-runtime hungs when qemu process is D/T
|
||||
state
|
||||
|
||||
Signed-off-by: jikui <jikui2@huawei.com>
|
||||
---
|
||||
src/runtime/vendor/github.com/kata-containers/govmm/qemu/qmp.go | 2 ++
|
||||
1 file changed, 2 insertions(+)
|
||||
|
||||
diff --git a/src/runtime/vendor/github.com/kata-containers/govmm/qemu/qmp.go b/src/runtime/vendor/github.com/kata-containers/govmm/qemu/qmp.go
|
||||
index 97e9245..325250d 100644
|
||||
--- a/src/runtime/vendor/github.com/kata-containers/govmm/qemu/qmp.go
|
||||
+++ b/src/runtime/vendor/github.com/kata-containers/govmm/qemu/qmp.go
|
||||
@@ -717,6 +717,8 @@ func QMPStart(ctx context.Context, socket string, cfg QMPConfig, disconnectedCh
|
||||
if q.version == nil {
|
||||
return nil, nil, fmt.Errorf("failed to find QMP version information")
|
||||
}
|
||||
+ case <-time.After(15 * time.Second):
|
||||
+ return nil, nil, fmt.Errorf("qmp start time out")
|
||||
}
|
||||
|
||||
return q, q.version, nil
|
||||
--
|
||||
2.25.1
|
||||
|
||||
@ -1,22 +1,18 @@
|
||||
From 1efb88fbf554f3977a1a8aa1c79bc70cc1b66953 Mon Sep 17 00:00:00 2001
|
||||
From: jiangpengfei <jiangpengfei9@huawei.com>
|
||||
Date: Sat, 25 Jul 2020 09:22:08 +0800
|
||||
Subject: [PATCH 02/50] kata-runtime: fix kata-runtime skip read lines in
|
||||
From c355523761598154653466033a1d88643d3fd3df Mon Sep 17 00:00:00 2001
|
||||
From: jikui <jikui2@huawei.com>
|
||||
Date: Thu, 4 Nov 2021 20:27:22 +0800
|
||||
Subject: [PATCH] kata-runtime: fix kata-runtime skip read lines in
|
||||
/proc/mounts file problem
|
||||
|
||||
reason: Since /proc/mounts is a virtual file which is changed dynamically by kernel,
|
||||
if we use file pointer to read content in this file line by line, we may miss read
|
||||
some lines. So we retry read /proc/mounts file again to fix this problem.
|
||||
|
||||
Signed-off-by: jiangpengfei <jiangpengfei9@huawei.com>
|
||||
Signed-off-by: jikui <jikui2@huawei.com>
|
||||
---
|
||||
virtcontainers/utils/utils_linux.go | 58 +++++++++++++++++++++++--------------
|
||||
1 file changed, 36 insertions(+), 22 deletions(-)
|
||||
.../virtcontainers/utils/utils_linux.go | 61 +++++++++++--------
|
||||
1 file changed, 36 insertions(+), 25 deletions(-)
|
||||
|
||||
diff --git a/virtcontainers/utils/utils_linux.go b/virtcontainers/utils/utils_linux.go
|
||||
index ad870d63..6cef4cfb 100644
|
||||
--- a/virtcontainers/utils/utils_linux.go
|
||||
+++ b/virtcontainers/utils/utils_linux.go
|
||||
diff --git a/src/runtime/virtcontainers/utils/utils_linux.go b/src/runtime/virtcontainers/utils/utils_linux.go
|
||||
index 3c14e0c..c6dbd0a 100644
|
||||
--- a/src/runtime/virtcontainers/utils/utils_linux.go
|
||||
+++ b/src/runtime/virtcontainers/utils/utils_linux.go
|
||||
@@ -7,15 +7,18 @@ package utils
|
||||
|
||||
import (
|
||||
@ -39,12 +35,12 @@ index ad870d63..6cef4cfb 100644
|
||||
@@ -93,6 +96,7 @@ const (
|
||||
procMountsFile = "/proc/mounts"
|
||||
|
||||
fieldsPerLine = 6
|
||||
+ maxRetryTimes = 5
|
||||
fieldsPerLine = 6
|
||||
+ maxRetryTimes = 5
|
||||
vfioAPSysfsDir = "vfio_ap"
|
||||
)
|
||||
|
||||
const (
|
||||
@@ -109,35 +113,45 @@ func GetDevicePathAndFsType(mountPoint string) (devicePath, fsType string, err e
|
||||
@@ -110,37 +114,44 @@ func GetDevicePathAndFsType(mountPoint string) (devicePath, fsType string, err e
|
||||
return
|
||||
}
|
||||
|
||||
@ -57,9 +53,7 @@ index ad870d63..6cef4cfb 100644
|
||||
- }
|
||||
-
|
||||
- defer file.Close()
|
||||
+ for retry <= maxRetryTimes {
|
||||
+ var content []byte
|
||||
|
||||
-
|
||||
- reader := bufio.NewReader(file)
|
||||
- for {
|
||||
- var line string
|
||||
@ -67,11 +61,13 @@ index ad870d63..6cef4cfb 100644
|
||||
- line, err = reader.ReadString('\n')
|
||||
- if err == io.EOF {
|
||||
- err = fmt.Errorf("Mount %s not found", mountPoint)
|
||||
+ for retry <= maxRetryTimes {
|
||||
+ var content []byte
|
||||
+ content, err = ioutil.ReadFile(procMountsFile)
|
||||
+ if err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
-
|
||||
- fields := strings.Fields(line)
|
||||
- if len(fields) != fieldsPerLine {
|
||||
- err = fmt.Errorf("Incorrect no of fields (expected %d, got %d)) :%s", fieldsPerLine, len(fields), line)
|
||||
@ -100,7 +96,7 @@ index ad870d63..6cef4cfb 100644
|
||||
+ return
|
||||
+ }
|
||||
}
|
||||
|
||||
-
|
||||
- if mountPoint == fields[procPathIndex] {
|
||||
- devicePath = fields[procDeviceIndex]
|
||||
- fsType = fields[procTypeIndex]
|
||||
@ -112,6 +108,8 @@ index ad870d63..6cef4cfb 100644
|
||||
}
|
||||
+ return "", "", fmt.Errorf("retry %d times fail to get devicePath adn fs type", maxRetryTimes)
|
||||
}
|
||||
|
||||
// IsAPVFIOMediatedDevice decides whether a device is a VFIO-AP device
|
||||
--
|
||||
2.14.3 (Apple Git-98)
|
||||
2.25.1
|
||||
|
||||
@ -0,0 +1,52 @@
|
||||
From 0797bf1fec9c40b67f2770bc8778e8eaee1657c8 Mon Sep 17 00:00:00 2001
|
||||
From: jikui <jikui2@huawei.com>
|
||||
Date: Fri, 5 Nov 2021 11:35:25 +0800
|
||||
Subject: [PATCH] kata-runtime: keep the process name of qemu same as
|
||||
configured path
|
||||
|
||||
Signed-off-by: jikui <jikui2@huawei.com>
|
||||
---
|
||||
src/runtime/pkg/katautils/config.go | 7 ++++++-
|
||||
src/runtime/pkg/katautils/config_test.go | 4 ++--
|
||||
2 files changed, 8 insertions(+), 3 deletions(-)
|
||||
|
||||
diff --git a/src/runtime/pkg/katautils/config.go b/src/runtime/pkg/katautils/config.go
|
||||
index 718677b..e0ebc84 100644
|
||||
--- a/src/runtime/pkg/katautils/config.go
|
||||
+++ b/src/runtime/pkg/katautils/config.go
|
||||
@@ -172,7 +172,12 @@ func (h hypervisor) path() (string, error) {
|
||||
p = defaultHypervisorPath
|
||||
}
|
||||
|
||||
- return ResolvePath(p)
|
||||
+ absolutePath, err := filepath.Abs(p)
|
||||
+ if err != nil {
|
||||
+ return "", err
|
||||
+ }
|
||||
+
|
||||
+ return absolutePath, nil
|
||||
}
|
||||
|
||||
func (h hypervisor) ctlpath() (string, error) {
|
||||
diff --git a/src/runtime/pkg/katautils/config_test.go b/src/runtime/pkg/katautils/config_test.go
|
||||
index 0d02534..84b7843 100644
|
||||
--- a/src/runtime/pkg/katautils/config_test.go
|
||||
+++ b/src/runtime/pkg/katautils/config_test.go
|
||||
@@ -983,12 +983,12 @@ func TestHypervisorDefaultsHypervisor(t *testing.T) {
|
||||
assert.NoError(err)
|
||||
assert.Equal(p, defaultHypervisorPath, "default hypervisor path wrong")
|
||||
|
||||
- // test path resolution
|
||||
+ // test path resolution, just return the absolute path instead of resolved path
|
||||
defaultHypervisorPath = testHypervisorLinkPath
|
||||
h = hypervisor{}
|
||||
p, err = h.path()
|
||||
assert.NoError(err)
|
||||
- assert.Equal(p, testHypervisorPath)
|
||||
+ assert.Equal(p, testHypervisorLinkPath)
|
||||
}
|
||||
|
||||
func TestHypervisorDefaultsKernel(t *testing.T) {
|
||||
--
|
||||
2.25.1
|
||||
|
||||
@ -0,0 +1,56 @@
|
||||
From e8e8e05538bf2c7bd8feebc44d4a960f453d21e1 Mon Sep 17 00:00:00 2001
|
||||
From: holyfei <yangfeiyu20092010@163.com>
|
||||
Date: Sat, 20 Nov 2021 17:04:39 +0800
|
||||
Subject: [PATCH] kata-containers: modify kernel and image path in
|
||||
configuration.toml
|
||||
|
||||
Signed-off-by: holyfei <yangfeiyu20092010@163.com>
|
||||
---
|
||||
src/runtime/Makefile | 10 +++++-----
|
||||
1 file changed, 5 insertions(+), 5 deletions(-)
|
||||
|
||||
diff --git a/src/runtime/Makefile b/src/runtime/Makefile
|
||||
index 745bcc1..68e60ee 100644
|
||||
--- a/src/runtime/Makefile
|
||||
+++ b/src/runtime/Makefile
|
||||
@@ -49,6 +49,8 @@ BINLIBEXECLIST :=
|
||||
BIN_PREFIX = $(PROJECT_TYPE)
|
||||
PROJECT_DIR = $(PROJECT_TAG)
|
||||
IMAGENAME = $(PROJECT_TAG).img
|
||||
+KERNEL_PATH = /var/lib/kata/kernel
|
||||
+IMAGE_PATH = /var/lib/kata/kata-containers-initrd.img
|
||||
|
||||
TARGET = $(BIN_PREFIX)-runtime
|
||||
TARGET_OUTPUT = $(CURDIR)/$(TARGET)
|
||||
@@ -108,7 +110,7 @@ PKGLIBEXECDIR := $(LIBEXECDIR)/$(PROJECT_DIR)
|
||||
|
||||
KERNELDIR := $(PKGDATADIR)
|
||||
|
||||
-IMAGEPATH := $(PKGDATADIR)/$(IMAGENAME)
|
||||
+IMAGEPATH := $(IMAGE_PATH)
|
||||
FIRMWAREPATH :=
|
||||
|
||||
# Name of default configuration file the runtime will use.
|
||||
@@ -248,8 +250,7 @@ ifneq (,$(QEMUCMD))
|
||||
DEFBLOCKSTORAGEDRIVER_QEMU := virtio-scsi
|
||||
DEFNETWORKMODEL_QEMU := tcfilter
|
||||
KERNELTYPE = uncompressed
|
||||
- KERNELNAME = $(call MAKE_KERNEL_NAME,$(KERNELTYPE))
|
||||
- KERNELPATH = $(KERNELDIR)/$(KERNELNAME)
|
||||
+ KERNELPATH = $(KERNEL_PATH)
|
||||
endif
|
||||
|
||||
ifneq (,$(CLHCMD))
|
||||
@@ -294,8 +295,7 @@ ifneq (,$(STRATOVIRTCMD))
|
||||
DEFBLOCKSTORAGEDRIVER_STRATOVIRT := virtio-mmio
|
||||
DEFNETWORKMODEL_STRATOVIRT := none
|
||||
KENRELTYPE_STRATOVIRT = uncompressed
|
||||
- KERNEL_NAME_STRATOVIRT = $(call MAKE_KERNEL_NAME,$(KENRELTYPE_STRATOVIRT))
|
||||
- KERNELPATH_STRATOVIRT = $(KERNELDIR)/$(KERNEL_NAME_STRATOVIRT)
|
||||
+ KERNELPATH_STRATOVIRT = $(KERNEL_PATH)
|
||||
endif
|
||||
|
||||
ifneq (,$(FCCMD))
|
||||
--
|
||||
2.27.0
|
||||
|
||||
@ -1,21 +1,18 @@
|
||||
From c279f4548ccc534f1c65723bf9994c448e510d3d Mon Sep 17 00:00:00 2001
|
||||
From: jiangpengfei <jiangpengfei9@huawei.com>
|
||||
Date: Sat, 25 Jul 2020 11:56:35 +0800
|
||||
Subject: [PATCH 05/50] cgroups: increase delete cgroup retry times
|
||||
From 13aec526360797b3bce776f35b7e5f5976961b47 Mon Sep 17 00:00:00 2001
|
||||
From: jikui <jikui2@huawei.com>
|
||||
Date: Fri, 5 Nov 2021 11:46:17 +0800
|
||||
Subject: [PATCH] kata-runtime: increase delete cgroup retry times
|
||||
|
||||
reason: inorder to make sure cgroup dir to be deleted, so we increase
|
||||
the retry times when delete cgroup dir failed.
|
||||
|
||||
Signed-off-by: jiangpengfei <jiangpengfei9@huawei.com>
|
||||
Signed-off-by: jikui <jikui2@huawei.com>
|
||||
---
|
||||
vendor/github.com/containerd/cgroups/cgroup.go | 4 ++--
|
||||
vendor/github.com/containerd/cgroups/utils.go | 9 ++++++---
|
||||
.../vendor/github.com/containerd/cgroups/cgroup.go | 4 ++--
|
||||
.../vendor/github.com/containerd/cgroups/utils.go | 9 ++++++---
|
||||
2 files changed, 8 insertions(+), 5 deletions(-)
|
||||
|
||||
diff --git a/vendor/github.com/containerd/cgroups/cgroup.go b/vendor/github.com/containerd/cgroups/cgroup.go
|
||||
index 53866685..69612b0a 100644
|
||||
--- a/vendor/github.com/containerd/cgroups/cgroup.go
|
||||
+++ b/vendor/github.com/containerd/cgroups/cgroup.go
|
||||
diff --git a/src/runtime/vendor/github.com/containerd/cgroups/cgroup.go b/src/runtime/vendor/github.com/containerd/cgroups/cgroup.go
|
||||
index 5386668..69612b0 100644
|
||||
--- a/src/runtime/vendor/github.com/containerd/cgroups/cgroup.go
|
||||
+++ b/src/runtime/vendor/github.com/containerd/cgroups/cgroup.go
|
||||
@@ -223,7 +223,7 @@ func (c *cgroup) Delete() error {
|
||||
return err
|
||||
}
|
||||
@ -34,10 +31,10 @@ index 53866685..69612b0a 100644
|
||||
}
|
||||
}
|
||||
}
|
||||
diff --git a/vendor/github.com/containerd/cgroups/utils.go b/vendor/github.com/containerd/cgroups/utils.go
|
||||
index 8a97d04d..82dbe2d3 100644
|
||||
--- a/vendor/github.com/containerd/cgroups/utils.go
|
||||
+++ b/vendor/github.com/containerd/cgroups/utils.go
|
||||
diff --git a/src/runtime/vendor/github.com/containerd/cgroups/utils.go b/src/runtime/vendor/github.com/containerd/cgroups/utils.go
|
||||
index 8a97d04..82dbe2d 100644
|
||||
--- a/src/runtime/vendor/github.com/containerd/cgroups/utils.go
|
||||
+++ b/src/runtime/vendor/github.com/containerd/cgroups/utils.go
|
||||
@@ -99,16 +99,19 @@ func defaults(root string) ([]Subsystem, error) {
|
||||
// retrying the remove after a exp timeout
|
||||
func remove(path string) error {
|
||||
@ -62,5 +59,5 @@ index 8a97d04d..82dbe2d3 100644
|
||||
|
||||
// readPids will read all the pids of processes in a cgroup by the provided path
|
||||
--
|
||||
2.14.3 (Apple Git-98)
|
||||
2.25.1
|
||||
|
||||
@ -0,0 +1,33 @@
|
||||
From b654687f88281d1dfbbb6c8ede65aa53ec105122 Mon Sep 17 00:00:00 2001
|
||||
From: jikui <jikui2@huawei.com>
|
||||
Date: Fri, 5 Nov 2021 11:55:24 +0800
|
||||
Subject: [PATCH] kata-runtime: fix umount container rootfs dir return invalid
|
||||
argument error
|
||||
|
||||
Signed-off-by: jikui <jikui2@huawei.com>
|
||||
---
|
||||
src/runtime/virtcontainers/container.go | 8 ++++++--
|
||||
1 file changed, 6 insertions(+), 2 deletions(-)
|
||||
|
||||
diff --git a/src/runtime/virtcontainers/container.go b/src/runtime/virtcontainers/container.go
|
||||
index 6e8e1ba..224f096 100644
|
||||
--- a/src/runtime/virtcontainers/container.go
|
||||
+++ b/src/runtime/virtcontainers/container.go
|
||||
@@ -1035,8 +1035,12 @@ func (c *Container) stop(ctx context.Context, force bool) error {
|
||||
return err
|
||||
}
|
||||
|
||||
- if err := bindUnmountContainerRootfs(ctx, getMountPath(c.sandbox.id), c.id); err != nil && !force {
|
||||
- return err
|
||||
+ // umount container rootfs dir only if container use 9p
|
||||
+ // to bind mount host container rootfs to 9p shared dir
|
||||
+ if c.state.BlockDeviceID == "" {
|
||||
+ if err := bindUnmountContainerRootfs(c.ctx, getMountPath(c.sandbox.id), c.id); err != nil && !force {
|
||||
+ return err
|
||||
+ }
|
||||
}
|
||||
|
||||
if err := c.detachDevices(ctx); err != nil && !force {
|
||||
--
|
||||
2.25.1
|
||||
|
||||
@ -0,0 +1,33 @@
|
||||
From fe490b4d63871f91c17ad72afef38a3227c8b4d3 Mon Sep 17 00:00:00 2001
|
||||
From: jikui <jikui2@huawei.com>
|
||||
Date: Fri, 5 Nov 2021 12:06:11 +0800
|
||||
Subject: [PATCH] kata-runtime: truncate the log.json file before kata-runtime
|
||||
subcommand executed
|
||||
|
||||
Signed-off-by: jikui <jikui2@huawei.com>
|
||||
---
|
||||
src/runtime/cli/main.go | 8 ++++++++
|
||||
1 file changed, 8 insertions(+)
|
||||
|
||||
diff --git a/src/runtime/cli/main.go b/src/runtime/cli/main.go
|
||||
index 27757c0..7c77764 100644
|
||||
--- a/src/runtime/cli/main.go
|
||||
+++ b/src/runtime/cli/main.go
|
||||
@@ -252,6 +252,14 @@ func beforeSubcommands(c *cli.Context) error {
|
||||
ignoreConfigLogs = true
|
||||
} else {
|
||||
if path := c.GlobalString("log"); path != "" {
|
||||
+ // since we have redirect the kata-runtime log to /var/log/messages, and avoid the
|
||||
+ // path of log.json file to be large in the tmpfs, so we truncate the log.json file
|
||||
+ // every time before subcommand is executed.
|
||||
+ if path != "/dev/null" && katautils.FileExists(path) {
|
||||
+ if err := os.Truncate(path, 0); err != nil {
|
||||
+ return err
|
||||
+ }
|
||||
+ }
|
||||
f, err := os.OpenFile(path, os.O_CREATE|os.O_WRONLY|os.O_APPEND|os.O_SYNC, 0640)
|
||||
if err != nil {
|
||||
return err
|
||||
--
|
||||
2.25.1
|
||||
|
||||
@ -0,0 +1,248 @@
|
||||
From 420cb51b47e19556c35423354102fdc3a4d041f0 Mon Sep 17 00:00:00 2001
|
||||
From: jikui <jikui2@huawei.com>
|
||||
Date: Fri, 5 Nov 2021 16:29:40 +0800
|
||||
Subject: [PATCH] kata-runtime: validate sandbox cpu and memory size
|
||||
|
||||
Signed-off-by: jikui <jikui2@huawei.com>
|
||||
---
|
||||
src/runtime/pkg/katautils/config.go | 39 +++++++++++++++++--
|
||||
src/runtime/pkg/katautils/config_test.go | 5 +--
|
||||
src/runtime/virtcontainers/pkg/oci/utils.go | 10 ++---
|
||||
src/runtime/virtcontainers/utils/utils.go | 42 +++++++++++++++++++++
|
||||
4 files changed, 85 insertions(+), 11 deletions(-)
|
||||
|
||||
diff --git a/src/runtime/pkg/katautils/config.go b/src/runtime/pkg/katautils/config.go
|
||||
index e0ebc84..e523ed3 100644
|
||||
--- a/src/runtime/pkg/katautils/config.go
|
||||
+++ b/src/runtime/pkg/katautils/config.go
|
||||
@@ -11,7 +11,6 @@ import (
|
||||
"fmt"
|
||||
"io/ioutil"
|
||||
"path/filepath"
|
||||
- goruntime "runtime"
|
||||
"strings"
|
||||
|
||||
"github.com/BurntSushi/toml"
|
||||
@@ -311,7 +310,7 @@ func (h hypervisor) GetEntropySource() string {
|
||||
}
|
||||
|
||||
func (h hypervisor) defaultVCPUs() uint32 {
|
||||
- numCPUs := goruntime.NumCPU()
|
||||
+ numCPUs := utils.GetPhysicalCPUNumber()
|
||||
|
||||
if h.NumVCPUs < 0 || h.NumVCPUs > int32(numCPUs) {
|
||||
return uint32(numCPUs)
|
||||
@@ -323,8 +322,22 @@ func (h hypervisor) defaultVCPUs() uint32 {
|
||||
return uint32(h.NumVCPUs)
|
||||
}
|
||||
|
||||
+func (h hypervisor) checkVCPUs() error {
|
||||
+ numCPUs := utils.GetPhysicalCPUNumber()
|
||||
+
|
||||
+ if h.NumVCPUs <= 0 {
|
||||
+ return fmt.Errorf("invalid vcpus in configuration.toml! vcpus must larger than 0")
|
||||
+ }
|
||||
+
|
||||
+ if h.NumVCPUs > int32(numCPUs) {
|
||||
+ return fmt.Errorf("invalid vcpus in configuration.toml! vcpus must smaller than max CPUs: %d in machine", numCPUs)
|
||||
+ }
|
||||
+
|
||||
+ return nil
|
||||
+}
|
||||
+
|
||||
func (h hypervisor) defaultMaxVCPUs() uint32 {
|
||||
- numcpus := uint32(goruntime.NumCPU())
|
||||
+ numcpus := uint32(utils.GetPhysicalCPUNumber())
|
||||
maxvcpus := vc.MaxQemuVCPUs()
|
||||
reqVCPUs := h.DefaultMaxVCPUs
|
||||
|
||||
@@ -350,6 +363,18 @@ func (h hypervisor) defaultMemSz() uint32 {
|
||||
return h.MemorySize
|
||||
}
|
||||
|
||||
+func (h hypervisor) checkMemSz() error {
|
||||
+ if h.MemorySize < utils.MinMemorySizeInMB {
|
||||
+ return fmt.Errorf("invalid memory size! Memory size must larger than %d MB", utils.MinMemorySizeInMB)
|
||||
+ }
|
||||
+
|
||||
+ if h.MemorySize > utils.MaxMemorySizeInMB {
|
||||
+ return fmt.Errorf("invalid memory size, memory size must smaller than %d MB", utils.MaxMemorySizeInMB)
|
||||
+ }
|
||||
+
|
||||
+ return nil
|
||||
+}
|
||||
+
|
||||
func (h hypervisor) defaultMemSlots() uint32 {
|
||||
slots := h.MemSlots
|
||||
if slots == 0 {
|
||||
@@ -665,6 +690,14 @@ func newQemuHypervisorConfig(h hypervisor) (vc.HypervisorConfig, error) {
|
||||
return vc.HypervisorConfig{}, err
|
||||
}
|
||||
|
||||
+ if err = h.checkVCPUs(); err != nil {
|
||||
+ return vc.HypervisorConfig{}, err
|
||||
+ }
|
||||
+
|
||||
+ if err = h.checkMemSz(); err != nil {
|
||||
+ return vc.HypervisorConfig{}, err
|
||||
+ }
|
||||
+
|
||||
rxRateLimiterMaxRate := h.getRxRateLimiterCfg()
|
||||
txRateLimiterMaxRate := h.getTxRateLimiterCfg()
|
||||
|
||||
diff --git a/src/runtime/pkg/katautils/config_test.go b/src/runtime/pkg/katautils/config_test.go
|
||||
index 84b7843..3782268 100644
|
||||
--- a/src/runtime/pkg/katautils/config_test.go
|
||||
+++ b/src/runtime/pkg/katautils/config_test.go
|
||||
@@ -14,7 +14,6 @@ import (
|
||||
"path"
|
||||
"path/filepath"
|
||||
"reflect"
|
||||
- goruntime "runtime"
|
||||
"strings"
|
||||
"syscall"
|
||||
"testing"
|
||||
@@ -155,7 +154,7 @@ func createAllRuntimeConfigFiles(dir, hypervisor string) (config testRuntimeConf
|
||||
KernelParams: vc.DeserializeParams(strings.Fields(kernelParams)),
|
||||
HypervisorMachineType: machineType,
|
||||
NumVCPUs: defaultVCPUCount,
|
||||
- DefaultMaxVCPUs: uint32(goruntime.NumCPU()),
|
||||
+ DefaultMaxVCPUs: uint32(utils.GetPhysicalCPUNumber()),
|
||||
MemorySize: defaultMemSize,
|
||||
DisableBlockDeviceUse: disableBlockDevice,
|
||||
BlockDeviceDriver: defaultBlockDeviceDriver,
|
||||
@@ -918,7 +917,7 @@ func TestNewClhHypervisorConfig(t *testing.T) {
|
||||
func TestHypervisorDefaults(t *testing.T) {
|
||||
assert := assert.New(t)
|
||||
|
||||
- numCPUs := goruntime.NumCPU()
|
||||
+ numCPUs := utils.GetPhysicalCPUNumber()
|
||||
|
||||
h := hypervisor{}
|
||||
|
||||
diff --git a/src/runtime/virtcontainers/pkg/oci/utils.go b/src/runtime/virtcontainers/pkg/oci/utils.go
|
||||
index efaee4a..ea46ab7 100644
|
||||
--- a/src/runtime/virtcontainers/pkg/oci/utils.go
|
||||
+++ b/src/runtime/virtcontainers/pkg/oci/utils.go
|
||||
@@ -12,15 +12,12 @@ import (
|
||||
"fmt"
|
||||
"path/filepath"
|
||||
"regexp"
|
||||
- goruntime "runtime"
|
||||
"strconv"
|
||||
"strings"
|
||||
"syscall"
|
||||
|
||||
criContainerdAnnotations "github.com/containerd/cri-containerd/pkg/annotations"
|
||||
crioAnnotations "github.com/cri-o/cri-o/pkg/annotations"
|
||||
- specs "github.com/opencontainers/runtime-spec/specs-go"
|
||||
- "github.com/sirupsen/logrus"
|
||||
"k8s.io/apimachinery/pkg/api/resource"
|
||||
|
||||
vc "github.com/kata-containers/kata-containers/src/runtime/virtcontainers"
|
||||
@@ -29,6 +26,9 @@ import (
|
||||
vcAnnotations "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/pkg/annotations"
|
||||
dockershimAnnotations "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/pkg/annotations/dockershim"
|
||||
"github.com/kata-containers/kata-containers/src/runtime/virtcontainers/types"
|
||||
+ "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/utils"
|
||||
+ specs "github.com/opencontainers/runtime-spec/specs-go"
|
||||
+ "github.com/sirupsen/logrus"
|
||||
)
|
||||
|
||||
type annotationContainerType struct {
|
||||
@@ -656,7 +656,7 @@ func addHypervisorCPUOverrides(ocispec specs.Spec, sbConfig *vc.SandboxConfig) e
|
||||
return fmt.Errorf("Error encountered parsing annotation default_vcpus: %v, please specify numeric value", err)
|
||||
}
|
||||
|
||||
- numCPUs := goruntime.NumCPU()
|
||||
+ numCPUs := utils.GetPhysicalCPUNumber()
|
||||
|
||||
if uint32(vcpus) > uint32(numCPUs) {
|
||||
return fmt.Errorf("Number of cpus %d specified in annotation default_vcpus is greater than the number of CPUs %d on the system", vcpus, numCPUs)
|
||||
@@ -671,7 +671,7 @@ func addHypervisorCPUOverrides(ocispec specs.Spec, sbConfig *vc.SandboxConfig) e
|
||||
return fmt.Errorf("Error encountered parsing annotation for default_maxvcpus: %v, please specify positive numeric value", err)
|
||||
}
|
||||
|
||||
- numCPUs := goruntime.NumCPU()
|
||||
+ numCPUs := utils.GetPhysicalCPUNumber()
|
||||
max := uint32(maxVCPUs)
|
||||
|
||||
if max > uint32(numCPUs) {
|
||||
diff --git a/src/runtime/virtcontainers/utils/utils.go b/src/runtime/virtcontainers/utils/utils.go
|
||||
index e49f55a..7023318 100644
|
||||
--- a/src/runtime/virtcontainers/utils/utils.go
|
||||
+++ b/src/runtime/virtcontainers/utils/utils.go
|
||||
@@ -6,12 +6,14 @@
|
||||
package utils
|
||||
|
||||
import (
|
||||
+ "bufio"
|
||||
"crypto/rand"
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
+ "strings"
|
||||
"syscall"
|
||||
"time"
|
||||
|
||||
@@ -28,11 +30,26 @@ const fileMode0755 = os.FileMode(0755)
|
||||
// MibToBytesShift the number to shift needed to convert MiB to Bytes
|
||||
const MibToBytesShift = 20
|
||||
|
||||
+const (
|
||||
+ // Min needed memory size to start a Kata VM
|
||||
+ MinMemorySizeInMB = 300
|
||||
+ MinMemorySizeInByte = MinMemorySizeInMB << MibToBytesShift
|
||||
+
|
||||
+ // Max support memory size in the Kata VM
|
||||
+ MaxMemorySizeInMB = 512 * 1024
|
||||
+ MaxMemorySizeInByte = MaxMemorySizeInMB << MibToBytesShift
|
||||
+)
|
||||
+
|
||||
// MaxSocketPathLen is the effective maximum Unix domain socket length.
|
||||
//
|
||||
// See unix(7).
|
||||
const MaxSocketPathLen = 107
|
||||
|
||||
+const (
|
||||
+ procCPUInfoPath = "/proc/cpuinfo"
|
||||
+ processorIdentifier = "processor"
|
||||
+)
|
||||
+
|
||||
// VHostVSockDevicePath path to vhost-vsock device
|
||||
var VHostVSockDevicePath = "/dev/vhost-vsock"
|
||||
|
||||
@@ -390,3 +407,28 @@ outer:
|
||||
|
||||
return nil
|
||||
}
|
||||
+
|
||||
+// GetPhysicalCPUNumber return the number of the CPUs in the physical machine
|
||||
+func GetPhysicalCPUNumber() int {
|
||||
+ f, err := os.Open(procCPUInfoPath)
|
||||
+ if err != nil {
|
||||
+ return 0
|
||||
+ }
|
||||
+ defer f.Close()
|
||||
+
|
||||
+ cpuNum := 0
|
||||
+ s := bufio.NewScanner(f)
|
||||
+ for s.Scan() {
|
||||
+ if err := s.Err(); err != nil {
|
||||
+ return 0
|
||||
+ }
|
||||
+
|
||||
+ fields := strings.Fields(s.Text())
|
||||
+ if len(fields) > 0 {
|
||||
+ if fields[0] == processorIdentifier {
|
||||
+ cpuNum++
|
||||
+ }
|
||||
+ }
|
||||
+ }
|
||||
+ return cpuNum
|
||||
+}
|
||||
--
|
||||
2.25.1
|
||||
|
||||
@ -0,0 +1,26 @@
|
||||
From 514a0a19e5f37458d1a7e44c8b2fb4bff3b12d5b Mon Sep 17 00:00:00 2001
|
||||
From: jikui <jikui2@huawei.com>
|
||||
Date: Fri, 5 Nov 2021 16:40:05 +0800
|
||||
Subject: [PATCH] kata-runtime: fix delete sandbox failed problem
|
||||
|
||||
Signed-off-by: jikui <jikui2@huawei.com>
|
||||
---
|
||||
src/runtime/virtcontainers/sandbox.go | 2 ++
|
||||
1 file changed, 2 insertions(+)
|
||||
|
||||
diff --git a/src/runtime/virtcontainers/sandbox.go b/src/runtime/virtcontainers/sandbox.go
|
||||
index 33ae6f0..09cb2eb 100644
|
||||
--- a/src/runtime/virtcontainers/sandbox.go
|
||||
+++ b/src/runtime/virtcontainers/sandbox.go
|
||||
@@ -1114,6 +1114,8 @@ func (s *Sandbox) CreateContainer(ctx context.Context, contConfig ContainerConfi
|
||||
if len(s.config.Containers) > 0 {
|
||||
// delete container config
|
||||
s.config.Containers = s.config.Containers[:len(s.config.Containers)-1]
|
||||
+ // need to flush change to persist storage
|
||||
+ _ = s.storeSandbox(ctx)
|
||||
}
|
||||
}
|
||||
}()
|
||||
--
|
||||
2.25.1
|
||||
|
||||
@ -0,0 +1,38 @@
|
||||
From 8251be558d6824fa1dce37836b7f1d6ec6be6e9f Mon Sep 17 00:00:00 2001
|
||||
From: jikui <jikui2@huawei.com>
|
||||
Date: Tue, 30 Nov 2021 10:36:27 +0800
|
||||
Subject: [PATCH] kata-runtime: check VFIO when create device
|
||||
|
||||
Signed-off-by: jikui <jikui2@huawei.com>
|
||||
---
|
||||
src/runtime/virtcontainers/device/manager/manager.go | 7 ++++++-
|
||||
1 file changed, 6 insertions(+), 1 deletion(-)
|
||||
|
||||
diff --git a/src/runtime/virtcontainers/device/manager/manager.go b/src/runtime/virtcontainers/device/manager/manager.go
|
||||
index 4515609..3afc148 100644
|
||||
--- a/src/runtime/virtcontainers/device/manager/manager.go
|
||||
+++ b/src/runtime/virtcontainers/device/manager/manager.go
|
||||
@@ -10,6 +10,7 @@ import (
|
||||
"context"
|
||||
"encoding/hex"
|
||||
"errors"
|
||||
+ "fmt"
|
||||
"sync"
|
||||
|
||||
"github.com/sirupsen/logrus"
|
||||
@@ -116,7 +117,11 @@ func (dm *deviceManager) createDevice(devInfo config.DeviceInfo) (dev api.Device
|
||||
}()
|
||||
|
||||
if existingDev := dm.findDeviceByMajorMinor(devInfo.Major, devInfo.Minor); existingDev != nil {
|
||||
- return existingDev, nil
|
||||
+ if isVFIO(devInfo.HostPath) {
|
||||
+ return nil, fmt.Errorf("device %s is replicated in the same Pod!", devInfo.ContainerPath)
|
||||
+ } else {
|
||||
+ return existingDev, nil
|
||||
+ }
|
||||
}
|
||||
|
||||
// device ID must be generated by manager instead of device itself
|
||||
--
|
||||
2.25.1
|
||||
|
||||
@ -1,23 +1,19 @@
|
||||
From 7ab7ff54efa3925a8d372f7830d31b87f8d01ea8 Mon Sep 17 00:00:00 2001
|
||||
From: jiangpengfei <jiangpengfei9@huawei.com>
|
||||
Date: Mon, 17 Aug 2020 17:39:18 +0800
|
||||
Subject: [PATCH 33/50] network: do not delete the exist tap device in the host
|
||||
From 51e17297b61fdb9ec560bae2f6c36003c8ebdfc0 Mon Sep 17 00:00:00 2001
|
||||
From: jikui <jikui2@huawei.com>
|
||||
Date: Tue, 30 Nov 2021 11:29:50 +0800
|
||||
Subject: [PATCH] kata-runtime: do not delete the exist tap device in the host
|
||||
|
||||
reason: If hotplug a exist tap device into Kata VM, kata-runtime
|
||||
should not delete this exist tap device, because this tap device
|
||||
is controlled by other network components.
|
||||
|
||||
Signed-off-by: jiangpengfei <jiangpengfei9@huawei.com>
|
||||
Signed-off-by: jikui <jikui2@huawei.com>
|
||||
---
|
||||
virtcontainers/tap_endpoint.go | 16 +++++++++++++---
|
||||
virtcontainers/veth_endpoint.go | 3 +++
|
||||
src/runtime/virtcontainers/tap_endpoint.go | 16 +++++++++++++---
|
||||
src/runtime/virtcontainers/veth_endpoint.go | 3 +++
|
||||
2 files changed, 16 insertions(+), 3 deletions(-)
|
||||
|
||||
diff --git a/virtcontainers/tap_endpoint.go b/virtcontainers/tap_endpoint.go
|
||||
index c897670e..2cf70dce 100644
|
||||
--- a/virtcontainers/tap_endpoint.go
|
||||
+++ b/virtcontainers/tap_endpoint.go
|
||||
@@ -77,7 +77,7 @@ func (endpoint *TapEndpoint) Detach(netNsCreated bool, netNsPath string) error {
|
||||
diff --git a/src/runtime/virtcontainers/tap_endpoint.go b/src/runtime/virtcontainers/tap_endpoint.go
|
||||
index 9617945..5eafd15 100644
|
||||
--- a/src/runtime/virtcontainers/tap_endpoint.go
|
||||
+++ b/src/runtime/virtcontainers/tap_endpoint.go
|
||||
@@ -80,7 +80,7 @@ func (endpoint *TapEndpoint) Detach(ctx context.Context, netNsCreated bool, netN
|
||||
|
||||
networkLogger().WithField("endpoint-type", TapEndpointType).Info("Detaching endpoint")
|
||||
return doNetNS(netNsPath, func(_ ns.NetNS) error {
|
||||
@ -26,9 +22,9 @@ index c897670e..2cf70dce 100644
|
||||
})
|
||||
}
|
||||
|
||||
@@ -91,6 +91,9 @@ func (endpoint *TapEndpoint) HotAttach(h hypervisor) error {
|
||||
@@ -94,6 +94,9 @@ func (endpoint *TapEndpoint) HotAttach(ctx context.Context, h hypervisor) error
|
||||
|
||||
if _, err := h.hotplugAddDevice(endpoint, netDev); err != nil {
|
||||
if _, err := h.hotplugAddDevice(ctx, endpoint, netDev); err != nil {
|
||||
networkLogger().WithError(err).Error("Error attach tap ep")
|
||||
+ if errUnTap := unTapNetwork(endpoint); errUnTap != nil {
|
||||
+ networkLogger().WithError(errUnTap).Errorf("Error rollback tap %s", endpoint.TapInterface.TAPIface.Name)
|
||||
@ -36,8 +32,8 @@ index c897670e..2cf70dce 100644
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
@@ -100,7 +103,7 @@ func (endpoint *TapEndpoint) HotAttach(h hypervisor) error {
|
||||
func (endpoint *TapEndpoint) HotDetach(h hypervisor, netNsCreated bool, netNsPath string) error {
|
||||
@@ -103,7 +106,7 @@ func (endpoint *TapEndpoint) HotAttach(ctx context.Context, h hypervisor) error
|
||||
func (endpoint *TapEndpoint) HotDetach(ctx context.Context, h hypervisor, netNsCreated bool, netNsPath string) error {
|
||||
networkLogger().Info("Hot detaching tap endpoint")
|
||||
if err := doNetNS(netNsPath, func(_ ns.NetNS) error {
|
||||
- return unTapNetwork(endpoint.TapInterface.TAPIface.Name)
|
||||
@ -45,7 +41,7 @@ index c897670e..2cf70dce 100644
|
||||
}); err != nil {
|
||||
networkLogger().WithError(err).Warn("Error un-bridging tap ep")
|
||||
}
|
||||
@@ -185,7 +188,14 @@ func tapNetwork(endpoint *TapEndpoint, numCPUs uint32, disableVhostNet bool) err
|
||||
@@ -174,7 +177,14 @@ func tapNetwork(endpoint *TapEndpoint, numCPUs uint32, disableVhostNet bool) err
|
||||
return nil
|
||||
}
|
||||
|
||||
@ -61,13 +57,13 @@ index c897670e..2cf70dce 100644
|
||||
netHandle, err := netlink.NewHandle()
|
||||
if err != nil {
|
||||
return err
|
||||
diff --git a/virtcontainers/veth_endpoint.go b/virtcontainers/veth_endpoint.go
|
||||
index 9ece6a74..0f2ec9ba 100644
|
||||
--- a/virtcontainers/veth_endpoint.go
|
||||
+++ b/virtcontainers/veth_endpoint.go
|
||||
@@ -119,6 +119,9 @@ func (endpoint *VethEndpoint) HotAttach(h hypervisor) error {
|
||||
diff --git a/src/runtime/virtcontainers/veth_endpoint.go b/src/runtime/virtcontainers/veth_endpoint.go
|
||||
index f93ca21..5f2435c 100644
|
||||
--- a/src/runtime/virtcontainers/veth_endpoint.go
|
||||
+++ b/src/runtime/virtcontainers/veth_endpoint.go
|
||||
@@ -124,6 +124,9 @@ func (endpoint *VethEndpoint) HotAttach(ctx context.Context, h hypervisor) error
|
||||
|
||||
if _, err := h.hotplugAddDevice(endpoint, netDev); err != nil {
|
||||
if _, err := h.hotplugAddDevice(ctx, endpoint, netDev); err != nil {
|
||||
networkLogger().WithError(err).Error("Error attach virtual ep")
|
||||
+ if errDisconn := xDisconnectVMNetwork(endpoint); errDisconn != nil {
|
||||
+ networkLogger().WithError(errDisconn).Error("Error rollback virtual ep")
|
||||
@ -76,5 +72,5 @@ index 9ece6a74..0f2ec9ba 100644
|
||||
}
|
||||
return nil
|
||||
--
|
||||
2.14.3 (Apple Git-98)
|
||||
2.25.1
|
||||
|
||||
@ -0,0 +1,33 @@
|
||||
From 39ac2f6929aea7e404c4597683e43fba7949964c Mon Sep 17 00:00:00 2001
|
||||
From: jikui <jikui2@huawei.com>
|
||||
Date: Tue, 30 Nov 2021 11:47:36 +0800
|
||||
Subject: [PATCH] kata-runtime: do not ignore updateInterface return error
|
||||
|
||||
Signed-off-by: jikui <jikui2@huawei.com>
|
||||
---
|
||||
src/runtime/virtcontainers/kata_agent.go | 8 ++++++++
|
||||
1 file changed, 8 insertions(+)
|
||||
|
||||
diff --git a/src/runtime/virtcontainers/kata_agent.go b/src/runtime/virtcontainers/kata_agent.go
|
||||
index 607f1c8..19d09bb 100644
|
||||
--- a/src/runtime/virtcontainers/kata_agent.go
|
||||
+++ b/src/runtime/virtcontainers/kata_agent.go
|
||||
@@ -620,7 +620,15 @@ func (k *kataAgent) updateInterface(ctx context.Context, ifc *pbTypes.Interface)
|
||||
"interface-requested": fmt.Sprintf("%+v", ifc),
|
||||
"resulting-interface": fmt.Sprintf("%+v", resultingInterface),
|
||||
}).WithError(err).Error("update interface request failed")
|
||||
+ return nil, err
|
||||
+ }
|
||||
+
|
||||
+ // need to judege resultingInterface is not nil, otherwise may cause
|
||||
+ // deference nil pointer panic problem
|
||||
+ if resultingInterface == nil {
|
||||
+ return nil, fmt.Errorf("resultingInterface should not be nil")
|
||||
}
|
||||
+
|
||||
if resultInterface, ok := resultingInterface.(*pbTypes.Interface); ok {
|
||||
return resultInterface, err
|
||||
}
|
||||
--
|
||||
2.25.1
|
||||
|
||||
@ -0,0 +1,45 @@
|
||||
From 1bb584ac783d057b64058b02cc96c1c67dd7bf30 Mon Sep 17 00:00:00 2001
|
||||
From: jikui <jikui2@huawei.com>
|
||||
Date: Tue, 30 Nov 2021 12:02:42 +0800
|
||||
Subject: [PATCH] kata-runtime: fix the block device not remove in devManager
|
||||
|
||||
Signed-off-by: jikui <jikui2@huawei.com>
|
||||
---
|
||||
src/runtime/virtcontainers/container.go | 11 ++++++++++-
|
||||
1 file changed, 10 insertions(+), 1 deletion(-)
|
||||
|
||||
diff --git a/src/runtime/virtcontainers/container.go b/src/runtime/virtcontainers/container.go
|
||||
index 224f096..dd4317e 100644
|
||||
--- a/src/runtime/virtcontainers/container.go
|
||||
+++ b/src/runtime/virtcontainers/container.go
|
||||
@@ -1302,6 +1302,7 @@ func (c *Container) plugDevice(ctx context.Context, devicePath string) error {
|
||||
}
|
||||
|
||||
if c.checkBlockDeviceSupport(ctx) && stat.Mode&unix.S_IFBLK == unix.S_IFBLK {
|
||||
+ var err error
|
||||
b, err := c.sandbox.devManager.NewDevice(config.DeviceInfo{
|
||||
HostPath: devicePath,
|
||||
ContainerPath: filepath.Join(kataGuestSharedDir(), c.id),
|
||||
@@ -1313,10 +1314,18 @@ func (c *Container) plugDevice(ctx context.Context, devicePath string) error {
|
||||
return fmt.Errorf("device manager failed to create rootfs device for %q: %v", devicePath, err)
|
||||
}
|
||||
|
||||
+ defer func() {
|
||||
+ if err != nil {
|
||||
+ if newErr := c.sandbox.devManager.RemoveDevice(b.DeviceID()); newErr != nil {
|
||||
+ c.Logger().WithError(newErr).Error("fail rollback to remove block device")
|
||||
+ }
|
||||
+ }
|
||||
+ }()
|
||||
+
|
||||
c.state.BlockDeviceID = b.DeviceID()
|
||||
|
||||
// attach rootfs device
|
||||
- if err := c.sandbox.devManager.AttachDevice(ctx, b.DeviceID(), c.sandbox); err != nil {
|
||||
+ if err = c.sandbox.devManager.AttachDevice(ctx, b.DeviceID(), c.sandbox); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
--
|
||||
2.25.1
|
||||
|
||||
@ -0,0 +1,40 @@
|
||||
From c9e8df6902ecd72223a3837de953233f00f94093 Mon Sep 17 00:00:00 2001
|
||||
From: holyfei <yangfeiyu20092010@163.com>
|
||||
Date: Fri, 10 Dec 2021 09:43:30 +0800
|
||||
Subject: [PATCH] kata-containers: modify stratovirt config file
|
||||
|
||||
Signed-off-by: holyfei <yangfeiyu20092010@163.com>
|
||||
---
|
||||
src/runtime/Makefile | 2 +-
|
||||
src/runtime/cli/config/configuration-stratovirt.toml.in | 2 +-
|
||||
2 files changed, 2 insertions(+), 2 deletions(-)
|
||||
|
||||
diff --git a/src/runtime/Makefile b/src/runtime/Makefile
|
||||
index 68e60ee..ff93ab3 100644
|
||||
--- a/src/runtime/Makefile
|
||||
+++ b/src/runtime/Makefile
|
||||
@@ -293,7 +293,7 @@ ifneq (,$(STRATOVIRTCMD))
|
||||
|
||||
# stratovirt-specific options (all should be suffixed by "_STRATOVIRT")
|
||||
DEFBLOCKSTORAGEDRIVER_STRATOVIRT := virtio-mmio
|
||||
- DEFNETWORKMODEL_STRATOVIRT := none
|
||||
+ DEFNETWORKMODEL_STRATOVIRT := tcfilter
|
||||
KENRELTYPE_STRATOVIRT = uncompressed
|
||||
KERNELPATH_STRATOVIRT = $(KERNEL_PATH)
|
||||
endif
|
||||
diff --git a/src/runtime/cli/config/configuration-stratovirt.toml.in b/src/runtime/cli/config/configuration-stratovirt.toml.in
|
||||
index b557b71..753e3dc 100644
|
||||
--- a/src/runtime/cli/config/configuration-stratovirt.toml.in
|
||||
+++ b/src/runtime/cli/config/configuration-stratovirt.toml.in
|
||||
@@ -13,7 +13,7 @@
|
||||
[hypervisor.stratovirt]
|
||||
path = "@STRATOVIRTPATH@"
|
||||
kernel = "@KERNELPATH_STRATOVIRT@"
|
||||
-image = "@IMAGEPATH@"
|
||||
+initrd = "@IMAGEPATH@"
|
||||
|
||||
# List of valid annotation names for the hypervisor
|
||||
# Each member of the list is a regular expression, which is the base name
|
||||
--
|
||||
2.27.0
|
||||
|
||||
243
patches/0023-stratovirt-update-configuration-toml-file.patch
Normal file
243
patches/0023-stratovirt-update-configuration-toml-file.patch
Normal file
@ -0,0 +1,243 @@
|
||||
From 1f83147653208f01effab0cf89209b8454d15f03 Mon Sep 17 00:00:00 2001
|
||||
From: "Xinle.Guo" <guoxinle1@huawei.com>
|
||||
Date: Mon, 10 Jan 2022 10:49:44 +0800
|
||||
Subject: [PATCH 1/5] stratovirt: update configuration toml file
|
||||
|
||||
1.Adapt to default machine type as microvm.
|
||||
2.Add more configuration items.
|
||||
3.Modify toml file format.
|
||||
|
||||
Signed-off-by: Xinle.Guo <guoxinle1@huawei.com>
|
||||
---
|
||||
src/runtime/Makefile | 2 +
|
||||
.../config/configuration-stratovirt.toml.in | 67 ++++++++++++-------
|
||||
2 files changed, 46 insertions(+), 23 deletions(-)
|
||||
|
||||
diff --git a/src/runtime/Makefile b/src/runtime/Makefile
|
||||
index ff93ab3..bade196 100644
|
||||
--- a/src/runtime/Makefile
|
||||
+++ b/src/runtime/Makefile
|
||||
@@ -292,6 +292,7 @@ ifneq (,$(STRATOVIRTCMD))
|
||||
CONFIGS += $(CONFIG_STRATOVIRT)
|
||||
|
||||
# stratovirt-specific options (all should be suffixed by "_STRATOVIRT")
|
||||
+ DEFMACHINETYPE_STRATOVIRT := microvm
|
||||
DEFBLOCKSTORAGEDRIVER_STRATOVIRT := virtio-mmio
|
||||
DEFNETWORKMODEL_STRATOVIRT := tcfilter
|
||||
KENRELTYPE_STRATOVIRT = uncompressed
|
||||
@@ -437,6 +438,7 @@ USER_VARS += FIRMWAREPATH
|
||||
USER_VARS += MACHINEACCELERATORS
|
||||
USER_VARS += CPUFEATURES
|
||||
USER_VARS += DEFMACHINETYPE_CLH
|
||||
+USER_VARS += DEFMACHINETYPE_STRATOVIRT
|
||||
USER_VARS += KERNELPARAMS
|
||||
USER_VARS += LIBEXECDIR
|
||||
USER_VARS += LOCALSTATEDIR
|
||||
diff --git a/src/runtime/cli/config/configuration-stratovirt.toml.in b/src/runtime/cli/config/configuration-stratovirt.toml.in
|
||||
index 753e3dc..db46665 100644
|
||||
--- a/src/runtime/cli/config/configuration-stratovirt.toml.in
|
||||
+++ b/src/runtime/cli/config/configuration-stratovirt.toml.in
|
||||
@@ -14,6 +14,8 @@
|
||||
path = "@STRATOVIRTPATH@"
|
||||
kernel = "@KERNELPATH_STRATOVIRT@"
|
||||
initrd = "@IMAGEPATH@"
|
||||
+#image = "/var/lib/kata/kata-containers-rootfs.img"
|
||||
+machine_type = "@DEFMACHINETYPE_STRATOVIRT@"
|
||||
|
||||
# List of valid annotation names for the hypervisor
|
||||
# Each member of the list is a regular expression, which is the base name
|
||||
@@ -29,12 +31,12 @@ valid_hypervisor_paths = @STRATOVIRTVALIDHYPERVISORPATHS@
|
||||
# Path for the ozone specific to stratovirt
|
||||
# If the ozone path is set, stratovirt will be launched in
|
||||
# ozone secure environment. It is disabled by default.
|
||||
-# ozone_path = "@STRATOVIRTOZONEPATH@"
|
||||
+#ozone_path = "@STRATOVIRTOZONEPATH@"
|
||||
|
||||
# List of valid ozone path values for the hypervisor
|
||||
# Each member of the list can be a regular expression
|
||||
# The default if not set is empty (all annotations rejected.)
|
||||
-# valid_jailer_paths = @STRATOVIRTVALIDOZONEPATHS@
|
||||
+#valid_jailer_paths = @STRATOVIRTVALIDOZONEPATHS@
|
||||
|
||||
# Optional space-separated list of options to pass to the guest kernel.
|
||||
# For example, use `kernel_params = "vsyscall=emulate"` if you are having
|
||||
@@ -87,18 +89,18 @@ default_bridges = @DEFBRIDGES@
|
||||
# Default memory size in MiB for SB/VM.
|
||||
# If unspecified then it will be set @DEFMEMSZ@ MiB.
|
||||
default_memory = @DEFMEMSZ@
|
||||
-#
|
||||
+
|
||||
# Default memory slots per SB/VM.
|
||||
# If unspecified then it will be set @DEFMEMSLOTS@.
|
||||
# This is will determine the times that memory will be hotadded to sandbox/VM.
|
||||
-# memory_slots = @DEFMEMSLOTS@
|
||||
+#memory_slots = @DEFMEMSLOTS@
|
||||
|
||||
# The size in MiB will be plused to max memory of hypervisor.
|
||||
# It is the memory address space for the NVDIMM devie.
|
||||
# If set block storage driver (block_device_driver) to "nvdimm",
|
||||
# should set memory_offset to the size of block device.
|
||||
# Default 0
|
||||
-# memory_offset = 0
|
||||
+#memory_offset = 0
|
||||
|
||||
# Disable block device from being used for a container's rootfs.
|
||||
# In case of a storage driver like devicemapper where a container's
|
||||
@@ -108,6 +110,14 @@ default_memory = @DEFMEMSZ@
|
||||
# 9pfs is used instead to pass the rootfs.
|
||||
disable_block_device_use = @DEFDISABLEBLOCK@
|
||||
|
||||
+# Shared file system type:
|
||||
+# - virtio-fs (default)
|
||||
+# - virtio-9p
|
||||
+shared_fs = "virtio-fs"
|
||||
+
|
||||
+# Path to vhost-user-fs daemon.
|
||||
+virtio_fs_daemon = "/usr/bin/vhost_user_fs"
|
||||
+
|
||||
# Block storage driver to be used for the hypervisor in case the container
|
||||
# rootfs is backed by a block device. This is virtio-scsi, virtio-blk
|
||||
# or nvdimm.
|
||||
@@ -120,12 +130,17 @@ block_device_driver = "@DEFBLOCKSTORAGEDRIVER_STRATOVIRT@"
|
||||
# Specifies cache-related options for block devices.
|
||||
# Denotes whether use of O_DIRECT (bypass the host page cache) is enabled.
|
||||
# Default false
|
||||
-# block_device_cache_direct = true
|
||||
+#block_device_cache_direct = true
|
||||
|
||||
# Specifies cache-related options for block devices.
|
||||
# Denotes whether flush requests for the device are ignored.
|
||||
# Default false
|
||||
-# block_device_cache_noflush = true
|
||||
+#block_device_cache_noflush = true
|
||||
+
|
||||
+# Enable iothreads to be used. This causes IO to be
|
||||
+# handled in a separate IO thread. This is currently only implemented
|
||||
+# for virtio blk.
|
||||
+#enable_iothreads = true
|
||||
|
||||
# Enable pre allocation of VM RAM, default false
|
||||
# Enabling this will result in lower container density
|
||||
@@ -134,7 +149,7 @@ block_device_driver = "@DEFBLOCKSTORAGEDRIVER_STRATOVIRT@"
|
||||
# upfront or in the cases where you want memory latencies
|
||||
# to be very predictable
|
||||
# Default false
|
||||
-# enable_mem_prealloc = true
|
||||
+#enable_mem_prealloc = true
|
||||
|
||||
# Enable huge pages for VM RAM, default false
|
||||
# Enabling this will result in the VM memory
|
||||
@@ -142,42 +157,48 @@ block_device_driver = "@DEFBLOCKSTORAGEDRIVER_STRATOVIRT@"
|
||||
# This is useful when you want to use vhost-user network
|
||||
# stacks within the container. This will automatically
|
||||
# result in memory pre allocation
|
||||
-# enable_hugepages = true
|
||||
+#enable_hugepages = true
|
||||
|
||||
# Enable vIOMMU, default false
|
||||
# Enabling this will result in the VM having a vIOMMU device
|
||||
# This will also add the following options to the kernel's
|
||||
# command line: intel_iommu=on,iommu=pt
|
||||
-# enable_iommu = true
|
||||
+#enable_iommu = true
|
||||
|
||||
# Enable swap of vm memory. Default false.
|
||||
# The behaviour is undefined if mem_prealloc is also set to true
|
||||
-# enable_swap = true
|
||||
+#enable_swap = true
|
||||
|
||||
# This option changes the default hypervisor and kernel parameters
|
||||
# to enable debug output where available.
|
||||
#
|
||||
# Default false
|
||||
-# enable_debug = true
|
||||
+#enable_debug = true
|
||||
|
||||
# Disable the customizations done in the runtime when it detects
|
||||
# that it is running on top a VMM. This will result in the runtime
|
||||
# behaving as it would when running on bare metal.
|
||||
#
|
||||
-# disable_nesting_checks = true
|
||||
+#disable_nesting_checks = true
|
||||
|
||||
# This is the msize used for 9p shares. It is the number of bytes
|
||||
# used for 9p packet payload.
|
||||
-# msize_9p =
|
||||
+#msize_9p =
|
||||
|
||||
# VFIO devices are hotplugged on a bridge by default.
|
||||
# Enable hotplugging on root bus. This may be required for devices with
|
||||
# a large PCI bar, as this is a current limitation with hotplugging on
|
||||
# a bridge.
|
||||
# Default false
|
||||
-# hotplug_vfio_on_root_bus = true
|
||||
+#hotplug_vfio_on_root_bus = true
|
||||
+
|
||||
+# Before hot plugging a PCIe device, you need to add a pcie_root_port device.
|
||||
+# Use this parameter when using some large PCI bar devices, such as Nvidia GPU
|
||||
+# The value means the number of pcie_root_port
|
||||
+# This value is valid when hotplug_vfio_on_root_bus is true and machine_type is "q35"
|
||||
+# Default 0
|
||||
+pcie_root_port = 2
|
||||
|
||||
-#
|
||||
# Default entropy source.
|
||||
# The path to a host source of entropy (including a real hardware RNG)
|
||||
# /dev/urandom and /dev/random are two main options.
|
||||
@@ -187,7 +208,7 @@ block_device_driver = "@DEFBLOCKSTORAGEDRIVER_STRATOVIRT@"
|
||||
# The source of entropy /dev/urandom is non-blocking and provides a
|
||||
# generally acceptable source of entropy. It should work well for pretty much
|
||||
# all practical purposes.
|
||||
-# entropy_source= ""
|
||||
+entropy_source= "@DEFENTROPYSOURCE@"
|
||||
|
||||
# List of valid annotations values for entropy_source
|
||||
# The default if not set is empty (all annotations rejected.)
|
||||
@@ -209,7 +230,7 @@ valid_entropy_sources = @DEFVALIDENTROPYSOURCES@
|
||||
# https://github.com/opencontainers/runtime-spec/blob/v1.0.1/config.md#posix-platform-hooks
|
||||
# Warnings will be logged if any error is encountered will scanning for hooks,
|
||||
# but it will not abort container execution.
|
||||
-# guest_hook_path = "/usr/share/oci/hooks"
|
||||
+#guest_hook_path = "/usr/share/oci/hooks"
|
||||
|
||||
[factory]
|
||||
# VM templating support. Once enabled, new VMs are created from template
|
||||
@@ -312,14 +333,14 @@ path = "@NETMONPATH@"
|
||||
# Uses tc filter rules to redirect traffic from the network interface
|
||||
# provided by plugin to a tap interface connected to the VM.
|
||||
#
|
||||
-internetworking_model="@DEFNETWORKMODEL_STRATOVIRT@"
|
||||
+internetworking_model = "@DEFNETWORKMODEL_STRATOVIRT@"
|
||||
|
||||
# disable guest seccomp
|
||||
# Determines whether container seccomp profiles are passed to the virtual
|
||||
# machine and applied by the kata agent. If set to true, seccomp is not applied
|
||||
# within the guest
|
||||
# (default: true)
|
||||
-disable_guest_seccomp=@DEFDISABLEGUESTSECCOMP@
|
||||
+disable_guest_seccomp = @DEFDISABLEGUESTSECCOMP@
|
||||
|
||||
# If enabled, the runtime will create opentracing.io traces and spans.
|
||||
# (See https://www.jaegertracing.io/docs/getting-started).
|
||||
@@ -352,15 +373,15 @@ disable_guest_seccomp=@DEFDISABLEGUESTSECCOMP@
|
||||
# The sandbox cgroup path is the parent cgroup of a container with the PodSandbox annotation.
|
||||
# The sandbox cgroup is constrained if there is no container type annotation.
|
||||
# See: https://godoc.org/github.com/kata-containers/runtime/virtcontainers#ContainerType
|
||||
-sandbox_cgroup_only=@DEFSANDBOXCGROUPONLY@
|
||||
+sandbox_cgroup_only = @DEFSANDBOXCGROUPONLY@
|
||||
|
||||
# Enabled experimental feature list, format: ["a", "b"].
|
||||
# Experimental features are features not stable enough for production,
|
||||
# they may break compatibility, and are prepared for a big version bump.
|
||||
# Supported experimental features:
|
||||
# (default: [])
|
||||
-experimental=@DEFAULTEXPFEATURES@
|
||||
+experimental = @DEFAULTEXPFEATURES@
|
||||
|
||||
# If enabled, user can run pprof tools with shim v2 process through kata-monitor.
|
||||
# (default: false)
|
||||
-# enable_pprof = true
|
||||
+#enable_pprof = true
|
||||
--
|
||||
2.20.1.windows.1
|
||||
|
||||
@ -0,0 +1,235 @@
|
||||
From 37c4a009a6ea9028e237f849e7b27a15c602113e Mon Sep 17 00:00:00 2001
|
||||
From: "Xinle.Guo" <guoxinle1@huawei.com>
|
||||
Date: Tue, 11 Jan 2022 10:44:53 +0800
|
||||
Subject: [PATCH] stratovirt: add struct `vmConfig` and methods to get all
|
||||
parameters of VM
|
||||
|
||||
Defines `vmConfig` struct to containes all configuration items that
|
||||
virtual machine needs. Provides methods to get VM paramters,
|
||||
including name, UUID, cpu, memory, kernel, devices, etc.
|
||||
|
||||
Signed-off-by: Xinle.Guo <guoxinle1@huawei.com>
|
||||
---
|
||||
src/runtime/virtcontainers/stratovirt.go | 197 ++++++++++++++++++++++-
|
||||
1 file changed, 189 insertions(+), 8 deletions(-)
|
||||
|
||||
diff --git a/src/runtime/virtcontainers/stratovirt.go b/src/runtime/virtcontainers/stratovirt.go
|
||||
index e9b2ba8..7e32a8a 100644
|
||||
--- a/src/runtime/virtcontainers/stratovirt.go
|
||||
+++ b/src/runtime/virtcontainers/stratovirt.go
|
||||
@@ -26,15 +26,196 @@ import (
|
||||
)
|
||||
|
||||
const (
|
||||
- apiSocket = "qmp.socket"
|
||||
- debugSocket = "console.socket"
|
||||
- ozoneBaseDir = "/srv/ozone/stratovirt"
|
||||
- defaultDummyMac = "22:33:44:aa:bb:"
|
||||
- mmioBlkCount = 4
|
||||
- mmioNetCount = 2
|
||||
- randomDevice = "/dev/urandom"
|
||||
+ defaultStratoVirt = "/usr/bin/stratovirt"
|
||||
+ ozoneBaseDir = "/srv/ozone/stratovirt"
|
||||
+ defaultStratoVirtMachineType = "microvm"
|
||||
+ defaultKernelParames = "console=hvc0 reboot=k panic=1 agent.use_vsock=true ramdom.trust_cpu=on rw"
|
||||
+ defaultMicroVMParames = "pci=off iommu=off acpi=off"
|
||||
+ apiSocket = "qmp.socket"
|
||||
+ debugSocket = "console.socket"
|
||||
+ virtiofsSocket = "virtiofs_kata.sock"
|
||||
+ iothreadID = "iothread_block"
|
||||
+ mmioBlkCount = 4
|
||||
+ mmioNetCount = 2
|
||||
)
|
||||
|
||||
+const (
|
||||
+ WaitSandboxTimeoutSecs = 15
|
||||
+ MachineTypeMicrovm = "microvm"
|
||||
+)
|
||||
+
|
||||
+// VirtioDev is the StratoVirt device interface.
|
||||
+type VirtioDev interface {
|
||||
+ getParams(config *vmConfig) []string
|
||||
+}
|
||||
+
|
||||
+type inComing struct {
|
||||
+ path string
|
||||
+ bootFromTemplate bool
|
||||
+}
|
||||
+
|
||||
+type Ozone struct {
|
||||
+ ozoneRoot string
|
||||
+ ozoneRes []string
|
||||
+ consolePath string
|
||||
+ kernelPath string
|
||||
+ initrdPath string
|
||||
+ pidFile string
|
||||
+ logFile string
|
||||
+ qmpSocketPath string
|
||||
+}
|
||||
+
|
||||
+// vmConfig keeps the custom settings and paramters to start virtual machine.
|
||||
+type vmConfig struct {
|
||||
+ name string
|
||||
+ uuid string
|
||||
+ machineType string
|
||||
+ vmPath string
|
||||
+ smp uint32
|
||||
+ memory uint64
|
||||
+ kernelPath string
|
||||
+ params string
|
||||
+ rootfsPath string
|
||||
+ initrdPath string
|
||||
+ rootBus types.Bridge
|
||||
+ devices []VirtioDev
|
||||
+ IOThread bool
|
||||
+ PFlash []string
|
||||
+ pidFile string
|
||||
+ logFile string
|
||||
+ qmpSocketPath govmmQemu.QMPSocket
|
||||
+ consolePath string
|
||||
+ fsSockPath string
|
||||
+ incoming inComing
|
||||
+ daemonize bool
|
||||
+ useOzone bool
|
||||
+ Ozone Ozone
|
||||
+}
|
||||
+
|
||||
+func (c *vmConfig) appendName(params *[]string) {
|
||||
+ if c.name == "" {
|
||||
+ return
|
||||
+ }
|
||||
+ *params = append(*params, "-name", c.name)
|
||||
+}
|
||||
+
|
||||
+func (c *vmConfig) appendUUID(params *[]string) {
|
||||
+ if c.uuid == "" {
|
||||
+ return
|
||||
+ }
|
||||
+ *params = append(*params, "-uuid", c.uuid)
|
||||
+}
|
||||
+
|
||||
+func (c *vmConfig) appendMachine(params *[]string) {
|
||||
+ if c.machineType == "" {
|
||||
+ return
|
||||
+ }
|
||||
+ *params = append(*params, "-machine", fmt.Sprintf("type=%s,dump-guest-core=off,mem-share=on", c.machineType))
|
||||
+}
|
||||
+
|
||||
+func (c *vmConfig) appendCPU(params *[]string) {
|
||||
+ if c.smp == 0 {
|
||||
+ return
|
||||
+ }
|
||||
+ *params = append(*params, "-smp", strconv.Itoa(int(c.smp)))
|
||||
+}
|
||||
+
|
||||
+func (c *vmConfig) appendMemory(params *[]string) {
|
||||
+ if c.memory == 0 {
|
||||
+ return
|
||||
+ }
|
||||
+ *params = append(*params, "-m", strconv.Itoa(int(c.memory)))
|
||||
+}
|
||||
+
|
||||
+func (c *vmConfig) appendKernel(params *[]string) {
|
||||
+ var ozone Ozone
|
||||
+ if c.kernelPath == "" {
|
||||
+ return
|
||||
+ }
|
||||
+
|
||||
+ if c.useOzone {
|
||||
+ ozone = c.Ozone
|
||||
+ *params = append(*params, "-kernel", ozone.kernelPath)
|
||||
+ } else {
|
||||
+ *params = append(*params, "-kernel", c.kernelPath)
|
||||
+ }
|
||||
+
|
||||
+ if c.initrdPath != "" {
|
||||
+ if c.useOzone {
|
||||
+ *params = append(*params, "-initrd", ozone.initrdPath)
|
||||
+ } else {
|
||||
+ *params = append(*params, "-initrd", c.initrdPath)
|
||||
+ }
|
||||
+ }
|
||||
+
|
||||
+ if c.params != "" {
|
||||
+ *params = append(*params, "-append", c.params)
|
||||
+ }
|
||||
+}
|
||||
+
|
||||
+func (c *vmConfig) appendIOThreads(params *[]string) {
|
||||
+ if c.IOThread {
|
||||
+ *params = append(*params, "-object", fmt.Sprintf("iothread,id=%s", iothreadID))
|
||||
+ }
|
||||
+}
|
||||
+
|
||||
+func (c *vmConfig) appendDevices(params *[]string) {
|
||||
+ for _, d := range c.devices {
|
||||
+ *params = append(*params, d.getParams(c)...)
|
||||
+ }
|
||||
+}
|
||||
+
|
||||
+func (c *vmConfig) appendPFlash(params *[]string) {
|
||||
+ for _, p := range c.PFlash {
|
||||
+ if p != "" {
|
||||
+ *params = append(*params, "-drive", p)
|
||||
+ }
|
||||
+ }
|
||||
+}
|
||||
+
|
||||
+func (c *vmConfig) appendPidFile(params *[]string) {
|
||||
+ if c.pidFile == "" {
|
||||
+ return
|
||||
+ }
|
||||
+
|
||||
+ if c.useOzone {
|
||||
+ *params = append(*params, "-pidfile", c.Ozone.pidFile)
|
||||
+ } else {
|
||||
+ *params = append(*params, "-pidfile", c.pidFile)
|
||||
+ }
|
||||
+}
|
||||
+
|
||||
+func (c *vmConfig) appendLogFile(params *[]string) {
|
||||
+ if c.logFile == "" {
|
||||
+ return
|
||||
+ }
|
||||
+
|
||||
+ if c.useOzone {
|
||||
+ *params = append(*params, "-D", c.Ozone.logFile)
|
||||
+ } else {
|
||||
+ *params = append(*params, "-D", c.logFile)
|
||||
+ }
|
||||
+}
|
||||
+
|
||||
+func (c *vmConfig) appendQMPSocket(params *[]string) {
|
||||
+ qmpInfo := c.qmpSocketPath
|
||||
+ qmpParams := append([]string{}, fmt.Sprintf("%s:", qmpInfo.Type))
|
||||
+ if c.useOzone {
|
||||
+ qmpParams = append(qmpParams, c.Ozone.qmpSocketPath)
|
||||
+ } else {
|
||||
+ qmpParams = append(qmpParams, qmpInfo.Name)
|
||||
+ }
|
||||
+ qmpParams = append(qmpParams, ",server")
|
||||
+ qmpParams = append(qmpParams, ",nowait")
|
||||
+ *params = append(*params, "-qmp", strings.Join(qmpParams, ""))
|
||||
+}
|
||||
+
|
||||
+func (c *vmConfig) appendIncoming(params *[]string) {
|
||||
+ if c.incoming.path != "" && c.incoming.bootFromTemplate {
|
||||
+ *params = append(*params, "-incoming", fmt.Sprintf("file:%s", c.incoming.path))
|
||||
+ }
|
||||
+}
|
||||
+
|
||||
type stratovirtDev struct {
|
||||
dev interface{}
|
||||
devType deviceType
|
||||
@@ -208,7 +389,7 @@ func (s *stratovirt) createbaseParams() []string {
|
||||
params = append(params, "-m", fmt.Sprintf("%d", uint64(s.config.MemorySize)))
|
||||
params = append(params, "-device", "virtio-serial-device")
|
||||
params = append(params, "-device", "virtconsole,chardev=charconsole0,id=virtioconsole0")
|
||||
- params = append(params, "-object", fmt.Sprintf("rng-random,id=objrng0,filename=%s", randomDevice))
|
||||
+ params = append(params, "-object", fmt.Sprintf("rng-random,id=objrng0,filename=%s", s.config.EntropySource))
|
||||
params = append(params, "-device", "virtio-rng-device,rng=objrng0")
|
||||
|
||||
// daemonize
|
||||
--
|
||||
2.20.1.windows.1
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@ -0,0 +1,771 @@
|
||||
From 8c0dd0f91366910b3029cb982a3cb807bdbb34cf Mon Sep 17 00:00:00 2001
|
||||
From: "Xinle.Guo" <guoxinle1@huawei.com>
|
||||
Date: Fri, 14 Jan 2022 17:07:37 +0800
|
||||
Subject: [PATCH] stratovirt: add a standard virtual machine sandbox type to
|
||||
kata container
|
||||
|
||||
Because stratovirt supports both microVM and standardVM
|
||||
architectures, adapts two sandbox types for kata container.
|
||||
Besides basic features, it also support virtio-fs, hotplug
|
||||
VFIO, support more devices.
|
||||
|
||||
Signed-off-by: Xinle.Guo <guoxinle1@huawei.com>
|
||||
---
|
||||
src/runtime/pkg/katautils/config.go | 1 +
|
||||
src/runtime/virtcontainers/stratovirt.go | 482 ++++++++++++++++++++---
|
||||
2 files changed, 438 insertions(+), 45 deletions(-)
|
||||
|
||||
diff --git a/src/runtime/pkg/katautils/config.go b/src/runtime/pkg/katautils/config.go
|
||||
index e523ed3..b04cdee 100644
|
||||
--- a/src/runtime/pkg/katautils/config.go
|
||||
+++ b/src/runtime/pkg/katautils/config.go
|
||||
@@ -1007,6 +1007,7 @@ func newStratovirtHypervisorConfig(h hypervisor) (vc.HypervisorConfig, error) {
|
||||
BlockDeviceCacheDirect: h.BlockDeviceCacheDirect,
|
||||
BlockDeviceCacheNoflush: h.BlockDeviceCacheNoflush,
|
||||
EnableIOThreads: h.EnableIOThreads,
|
||||
+ PCIeRootPort: h.PCIeRootPort,
|
||||
DisableVhostNet: h.DisableVhostNet,
|
||||
EnableVhostUserStore: h.EnableVhostUserStore,
|
||||
VhostUserStorePath: h.vhostUserStorePath(),
|
||||
diff --git a/src/runtime/virtcontainers/stratovirt.go b/src/runtime/virtcontainers/stratovirt.go
|
||||
index 4fcfb94..ffe8965 100644
|
||||
--- a/src/runtime/virtcontainers/stratovirt.go
|
||||
+++ b/src/runtime/virtcontainers/stratovirt.go
|
||||
@@ -14,14 +14,14 @@ import (
|
||||
"time"
|
||||
|
||||
govmmQemu "github.com/kata-containers/govmm/qemu"
|
||||
- "github.com/pkg/errors"
|
||||
- "github.com/sirupsen/logrus"
|
||||
-
|
||||
"github.com/kata-containers/kata-containers/src/runtime/virtcontainers/device/config"
|
||||
persistapi "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/persist/api"
|
||||
+ vcTypes "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/pkg/types"
|
||||
"github.com/kata-containers/kata-containers/src/runtime/virtcontainers/pkg/uuid"
|
||||
"github.com/kata-containers/kata-containers/src/runtime/virtcontainers/types"
|
||||
"github.com/kata-containers/kata-containers/src/runtime/virtcontainers/utils"
|
||||
+ "github.com/pkg/errors"
|
||||
+ "github.com/sirupsen/logrus"
|
||||
"go.opentelemetry.io/otel"
|
||||
otelLabel "go.opentelemetry.io/otel/label"
|
||||
otelTrace "go.opentelemetry.io/otel/trace"
|
||||
@@ -44,6 +44,11 @@ const (
|
||||
const (
|
||||
WaitSandboxTimeoutSecs = 15
|
||||
MachineTypeMicrovm = "microvm"
|
||||
+ MachineTypeQ35 = "q35"
|
||||
+ MachineTypeVirt = "virt"
|
||||
+ RootPortPrefix = "pcie"
|
||||
+ Q35PFlashCode = "/usr/share/edk2/ovmf/OVMF_CODE.fd"
|
||||
+ VirtPFlashCode = "/usr/share/edk2/aarch64/QEMU_EFI-pflash.raw"
|
||||
MmioBus VirtioDriver = "mmio"
|
||||
PciBus VirtioDriver = "pci"
|
||||
)
|
||||
@@ -67,6 +72,42 @@ func (d VirtioDriver) getDriver(config *vmConfig) VirtioDriver {
|
||||
}
|
||||
}
|
||||
|
||||
+type rootPortDevice struct {
|
||||
+ id string
|
||||
+ port string
|
||||
+ bus string
|
||||
+ slot int
|
||||
+ plugged bool
|
||||
+ addedDev string
|
||||
+}
|
||||
+
|
||||
+func (r rootPortDevice) isVaild() bool {
|
||||
+ if r.id == "" || r.port == "" {
|
||||
+ return false
|
||||
+ }
|
||||
+ return true
|
||||
+}
|
||||
+
|
||||
+func (r rootPortDevice) getParams(config *vmConfig) []string {
|
||||
+ if !r.isVaild() {
|
||||
+ return nil
|
||||
+ }
|
||||
+
|
||||
+ var params []string
|
||||
+ var devParams []Param
|
||||
+ devParams = append(devParams, Param{"id", r.id})
|
||||
+ devParams = append(devParams, Param{"port", r.port})
|
||||
+ if r.bus == "" {
|
||||
+ r.bus = "pcie.0"
|
||||
+ }
|
||||
+ devParams = append(devParams, Param{"bus", r.bus})
|
||||
+ devParams = append(devParams, Param{"addr", fmt.Sprintf("%d", r.slot)})
|
||||
+
|
||||
+ driver := "pcie-root-port"
|
||||
+ params = append(params, "-device", fmt.Sprintf("%s,%s", driver, strings.Join(SerializeParams(devParams, "="), ",")))
|
||||
+ return params
|
||||
+}
|
||||
+
|
||||
type blkDevice struct {
|
||||
id string
|
||||
filePath string
|
||||
@@ -179,6 +220,56 @@ func (n netDevice) getParams(config *vmConfig) []string {
|
||||
return params
|
||||
}
|
||||
|
||||
+type virtioFs struct {
|
||||
+ driver VirtioDriver
|
||||
+ backend string
|
||||
+ charID string
|
||||
+ charDev string
|
||||
+ tag string
|
||||
+ deviceID string
|
||||
+ bus string
|
||||
+ addr string
|
||||
+}
|
||||
+
|
||||
+var virtiofsDriver = map[VirtioDriver]string{
|
||||
+ MmioBus: "vhost-user-fs-device",
|
||||
+ PciBus: "vhost-user-fs-pci",
|
||||
+}
|
||||
+
|
||||
+func (v virtioFs) isVaild() bool {
|
||||
+ if v.charID == "" || v.charDev == "" || v.deviceID == "" {
|
||||
+ return false
|
||||
+ }
|
||||
+ return true
|
||||
+}
|
||||
+
|
||||
+func (v virtioFs) getParams(config *vmConfig) []string {
|
||||
+ if !v.isVaild() {
|
||||
+ return nil
|
||||
+ }
|
||||
+
|
||||
+ var params []string
|
||||
+ var charParams []Param
|
||||
+ var fsParams []Param
|
||||
+
|
||||
+ charParams = append(charParams, Param{"id", v.charID})
|
||||
+ charParams = append(charParams, Param{"path", config.fsSockPath})
|
||||
+
|
||||
+ v.driver = v.driver.getDriver(config)
|
||||
+ driver := virtiofsDriver[v.driver]
|
||||
+ fsParams = append(fsParams, Param{"chardev", v.charDev})
|
||||
+ fsParams = append(fsParams, Param{"tag", v.tag})
|
||||
+ fsParams = append(fsParams, Param{"id", v.deviceID})
|
||||
+ if v.bus != "" {
|
||||
+ fsParams = append(fsParams, Param{"bus", v.bus})
|
||||
+ fsParams = append(fsParams, Param{"addr", v.addr})
|
||||
+ }
|
||||
+
|
||||
+ params = append(params, "-chardev", fmt.Sprintf("%s,%s,server,nowait", v.backend, strings.Join(SerializeParams(charParams, "="), ",")))
|
||||
+ params = append(params, "-device", fmt.Sprintf("%s,%s", driver, strings.Join(SerializeParams(fsParams, "="), ",")))
|
||||
+ return params
|
||||
+}
|
||||
+
|
||||
type vhostVsock struct {
|
||||
driver VirtioDriver
|
||||
id string
|
||||
@@ -442,6 +533,12 @@ func (c *vmConfig) appendDevices(params *[]string) {
|
||||
for _, d := range c.devices {
|
||||
*params = append(*params, d.getParams(c)...)
|
||||
}
|
||||
+
|
||||
+ if c.machineType == MachineTypeMicrovm {
|
||||
+ return
|
||||
+ }
|
||||
+ // Add flag to unplug devices from their root port faster.
|
||||
+ *params = append(*params, "-global", "pcie-root-port.fast-unplug=1")
|
||||
}
|
||||
|
||||
func (c *vmConfig) appendPFlash(params *[]string) {
|
||||
@@ -499,6 +596,8 @@ func (c *vmConfig) appendIncoming(params *[]string) {
|
||||
type State struct {
|
||||
mmioBlkSlots [mmioBlkCount]bool
|
||||
mmioNetSlots [mmioNetCount]bool
|
||||
+ // The list of RootPorts that can be hot-added.
|
||||
+ rootPort []rootPortDevice
|
||||
pid int
|
||||
virtiofsPid int
|
||||
}
|
||||
@@ -558,6 +657,26 @@ func (s *stratovirt) getKernelParams(machineType string, initrdPath string) (str
|
||||
return strings.Join(params, " "), nil
|
||||
}
|
||||
|
||||
+func (s *stratovirt) getPFlash(machineType string) ([]string, error) {
|
||||
+ if s.config.PFlash != nil {
|
||||
+ return s.config.PFlash, nil
|
||||
+ }
|
||||
+
|
||||
+ var PFlash []string
|
||||
+ switch machineType {
|
||||
+ case MachineTypeQ35:
|
||||
+ PFlash = append(PFlash, fmt.Sprintf("file=%s,if=pflash,unit=0", Q35PFlashCode))
|
||||
+ case MachineTypeVirt:
|
||||
+ PFlash = append(PFlash, fmt.Sprintf("file=%s,if=pflash,unit=0", VirtPFlashCode))
|
||||
+ case MachineTypeMicrovm:
|
||||
+ return nil, nil
|
||||
+ default:
|
||||
+ return nil, fmt.Errorf("failed to match machine type %s", machineType)
|
||||
+ }
|
||||
+
|
||||
+ return PFlash, nil
|
||||
+}
|
||||
+
|
||||
func (s *stratovirt) createQMPSocket(vmPath string) govmmQemu.QMPSocket {
|
||||
socketPath := filepath.Join(vmPath, apiSocket)
|
||||
|
||||
@@ -590,6 +709,34 @@ func (s *stratovirt) createDevices() []VirtioDev {
|
||||
}
|
||||
}
|
||||
|
||||
+ // Create root port for all devices that need to be hot-added.
|
||||
+ if s.vmConfig.machineType != MachineTypeMicrovm && s.config.PCIeRootPort > 0 {
|
||||
+ devices = s.appendRootPort(ctx, devices)
|
||||
+ }
|
||||
+
|
||||
+ return devices
|
||||
+}
|
||||
+
|
||||
+func (s *stratovirt) appendRootPort(ctx context.Context, devices []VirtioDev) []VirtioDev {
|
||||
+ number := s.config.PCIeRootPort
|
||||
+
|
||||
+ for i := uint32(1); i < number+1; i++ {
|
||||
+ addr, err := s.vmConfig.rootBus.AddDevice(ctx, fmt.Sprintf("%s.%d", RootPortPrefix, i))
|
||||
+ if err != nil {
|
||||
+ return devices
|
||||
+ }
|
||||
+
|
||||
+ rp := rootPortDevice{
|
||||
+ id: fmt.Sprintf("%s.%d", RootPortPrefix, i),
|
||||
+ port: fmt.Sprintf("%d", i),
|
||||
+ bus: defaultBridgeBus,
|
||||
+ slot: int(addr),
|
||||
+ addedDev: "",
|
||||
+ }
|
||||
+ s.state.rootPort = append(s.state.rootPort, rp)
|
||||
+ devices = append(devices, rp)
|
||||
+ }
|
||||
+
|
||||
return devices
|
||||
}
|
||||
|
||||
@@ -729,6 +876,38 @@ func (s *stratovirt) appendNetwork(ctx context.Context, devices []VirtioDev, end
|
||||
return devices
|
||||
}
|
||||
|
||||
+func (s *stratovirt) appendVirtioFs(ctx context.Context, devices []VirtioDev, volume types.Volume) []VirtioDev {
|
||||
+ if s.config.SharedFS != config.VirtioFS {
|
||||
+ return devices
|
||||
+ }
|
||||
+
|
||||
+ var bus string
|
||||
+ var addr uint32
|
||||
+ var err error
|
||||
+ name := "virtio_fs"
|
||||
+
|
||||
+ if s.vmConfig.machineType != MachineTypeMicrovm {
|
||||
+ bus = "pcie.0"
|
||||
+ addr, err = s.vmConfig.rootBus.AddDevice(ctx, name)
|
||||
+ if err != nil {
|
||||
+ return devices
|
||||
+ }
|
||||
+ }
|
||||
+
|
||||
+ devices = append(devices, virtioFs{
|
||||
+ backend: "socket",
|
||||
+ // Virtio-fs must be bound to unique charDev, it uses the same name.
|
||||
+ charID: name,
|
||||
+ charDev: name,
|
||||
+ tag: volume.MountTag,
|
||||
+ deviceID: "virtio-fs0",
|
||||
+ bus: bus,
|
||||
+ addr: fmt.Sprintf("%d", addr),
|
||||
+ })
|
||||
+
|
||||
+ return devices
|
||||
+}
|
||||
+
|
||||
func (s *stratovirt) setVMConfig(id string, hypervisorConfig *HypervisorConfig) error {
|
||||
span, _ := s.trace(s.ctx, "setStratoVirtUp")
|
||||
defer span.End()
|
||||
@@ -765,7 +944,10 @@ func (s *stratovirt) setVMConfig(id string, hypervisorConfig *HypervisorConfig)
|
||||
return err
|
||||
}
|
||||
|
||||
- var PFlash []string
|
||||
+ PFlash, err := s.getPFlash(machineType)
|
||||
+ if err != nil {
|
||||
+ return err
|
||||
+ }
|
||||
|
||||
vmPath := filepath.Join(s.store.RunVMStoragePath(), s.id)
|
||||
qmpSocket := s.createQMPSocket(vmPath)
|
||||
@@ -852,8 +1034,33 @@ func (s *stratovirt) setOzone() error {
|
||||
return nil
|
||||
}
|
||||
|
||||
-func (s *stratovirt) hypervisorConfig() HypervisorConfig {
|
||||
- return s.config
|
||||
+// Virtio fs daemon is a shared file system that lets VM access a directory
|
||||
+// tree on the host.
|
||||
+func (s *stratovirt) setupVirtioFs() error {
|
||||
+ if !s.config.DisableBlockDeviceUse || s.config.SharedFS != config.VirtioFS {
|
||||
+ return nil
|
||||
+ }
|
||||
+
|
||||
+ if _, err := os.Stat(s.config.VirtioFSDaemon); os.IsNotExist(err) {
|
||||
+ return fmt.Errorf("virtiofsd path (%s) does not exist", s.config.VirtioFSDaemon)
|
||||
+ }
|
||||
+
|
||||
+ args := []string{
|
||||
+ "-socket-path", filepath.Join(s.vmConfig.vmPath, "virtiofs_kata.sock"),
|
||||
+ "-source", getSharePath(s.id)}
|
||||
+ if len(s.config.VirtioFSExtraArgs) != 0 {
|
||||
+ args = append(args, s.config.VirtioFSExtraArgs...)
|
||||
+ }
|
||||
+
|
||||
+ cmd := exec.Command(s.config.VirtioFSDaemon, args...)
|
||||
+ s.Logger().Info("Virtiofsd start with cmd: ", cmd)
|
||||
+
|
||||
+ if err := cmd.Start(); err != nil {
|
||||
+ return fmt.Errorf("failed to strat virtiofsd: %v", cmd)
|
||||
+ }
|
||||
+
|
||||
+ s.state.virtiofsPid = cmd.Process.Pid
|
||||
+ return nil
|
||||
}
|
||||
|
||||
// Get StratoVirt binary path.
|
||||
@@ -873,6 +1080,10 @@ func (s *stratovirt) binPath() (string, error) {
|
||||
return path, nil
|
||||
}
|
||||
|
||||
+func (s *stratovirt) hypervisorConfig() HypervisorConfig {
|
||||
+ return s.config
|
||||
+}
|
||||
+
|
||||
func (s *stratovirt) createSandbox(ctx context.Context, id string, networkNS NetworkNamespace, hypervisorConfig *HypervisorConfig) error {
|
||||
var span otelTrace.Span
|
||||
span, _ = s.trace(ctx, "createSandbox")
|
||||
@@ -1000,6 +1211,10 @@ func (s *stratovirt) startSandbox(ctx context.Context, timeout int) error {
|
||||
}
|
||||
}()
|
||||
|
||||
+ if err = s.setupVirtioFs(); err != nil {
|
||||
+ return err
|
||||
+ }
|
||||
+
|
||||
var params []string
|
||||
s.createBaseParams(s.vmConfig, ¶ms)
|
||||
|
||||
@@ -1134,28 +1349,29 @@ func (s *stratovirt) addDevice(ctx context.Context, devInfo interface{}, devType
|
||||
s.vmConfig.devices = s.appendNetwork(ctx, s.vmConfig.devices, v)
|
||||
case config.BlockDrive:
|
||||
s.vmConfig.devices = s.appendBlock(ctx, s.vmConfig.devices)
|
||||
+ case types.Volume:
|
||||
+ s.vmConfig.devices = s.appendVirtioFs(ctx, s.vmConfig.devices, v)
|
||||
default:
|
||||
s.Logger().WithField("dev-type", v).Warn("Could not append device: unsupported device type")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
-func (s *stratovirt) getDevSlot(Name string, isPut bool) (slot int, err error) {
|
||||
+func (s *stratovirt) setupMmioSlot(Name string, isPut bool) (int, error) {
|
||||
Name = filepath.Base(strings.ToLower(Name))
|
||||
-
|
||||
if strings.HasPrefix(Name, "eth") {
|
||||
idxStr := strings.TrimPrefix(Name, "eth")
|
||||
if idxStr == Name {
|
||||
- return 0, fmt.Errorf("Could not parse idx from Name %q", Name)
|
||||
+ return 0, fmt.Errorf("could not parse idx from name %q", Name)
|
||||
}
|
||||
|
||||
idx, err := strconv.Atoi(idxStr)
|
||||
if err != nil {
|
||||
- return 0, fmt.Errorf("Could not convert to int from Str %q", idxStr)
|
||||
+ return 0, fmt.Errorf("could not convert to int from str %q", idxStr)
|
||||
}
|
||||
|
||||
if !isPut && s.state.mmioNetSlots[idx] {
|
||||
- return 0, fmt.Errorf("GetDevSlot failed, slot is being used %q", idxStr)
|
||||
+ return 0, fmt.Errorf("failed to setup mmio slot, slot is being used %q", idxStr)
|
||||
}
|
||||
s.state.mmioNetSlots[idx] = !isPut
|
||||
|
||||
@@ -1163,25 +1379,80 @@ func (s *stratovirt) getDevSlot(Name string, isPut bool) (slot int, err error) {
|
||||
} else if strings.HasPrefix(Name, "vd") {
|
||||
charStr := strings.TrimPrefix(Name, "vd")
|
||||
if charStr == Name {
|
||||
- return 0, fmt.Errorf("Could not parse idx from Name %q", Name)
|
||||
+ return 0, fmt.Errorf("could not parse idx from name %q", Name)
|
||||
}
|
||||
|
||||
char := []rune(charStr)
|
||||
idx := int(char[0] - 'a')
|
||||
|
||||
if !isPut && s.state.mmioBlkSlots[idx] {
|
||||
- return 0, fmt.Errorf("GetDevSlot failed, slot is being used %q", charStr)
|
||||
+ return 0, fmt.Errorf("failed to setup mmio slot, slot is being used %q", charStr)
|
||||
}
|
||||
s.state.mmioBlkSlots[idx] = !isPut
|
||||
|
||||
return idx, nil
|
||||
}
|
||||
|
||||
- return 0, fmt.Errorf("GetDevSlot failed, Name is invalid %q", Name)
|
||||
+ return 0, fmt.Errorf("failed to setup mmio slot , Name is invalid %q", Name)
|
||||
+}
|
||||
+
|
||||
+func (s *stratovirt) setupPciSlot(Name string, isPut bool) (string, int, error) {
|
||||
+ rootports := &s.state.rootPort
|
||||
+ if len(*rootports) == 0 {
|
||||
+ return "", 0, fmt.Errorf("failed to get available address from bridges")
|
||||
+ }
|
||||
+
|
||||
+ for i, rootport := range *rootports {
|
||||
+ if !isPut && !rootport.plugged && rootport.addedDev == "" {
|
||||
+ (*rootports)[i].plugged = true
|
||||
+ (*rootports)[i].addedDev = Name
|
||||
+ return rootport.id, rootport.slot, nil
|
||||
+ } else if isPut && rootport.plugged && rootport.addedDev == Name {
|
||||
+ (*rootports)[i].plugged = false
|
||||
+ (*rootports)[i].addedDev = ""
|
||||
+ return rootport.id, rootport.slot, nil
|
||||
+ }
|
||||
+ }
|
||||
+
|
||||
+ return "", 0, fmt.Errorf("no more bridge slots available")
|
||||
+}
|
||||
+
|
||||
+func (s *stratovirt) getDevSlot(Name string) (string, int, error) {
|
||||
+ if s.config.HypervisorMachineType == MachineTypeMicrovm {
|
||||
+ slot, err := s.setupMmioSlot(Name, false)
|
||||
+ if err != nil {
|
||||
+ return "", 0, err
|
||||
+ }
|
||||
+
|
||||
+ return "", slot, nil
|
||||
+ }
|
||||
+
|
||||
+ bus, slot, err := s.setupPciSlot(Name, false)
|
||||
+ if err != nil {
|
||||
+ return "pcie.0", 0, err
|
||||
+ }
|
||||
+
|
||||
+ return bus, slot, nil
|
||||
+}
|
||||
+
|
||||
+func (s *stratovirt) delDevSlot(Name string) error {
|
||||
+ if s.vmConfig.machineType == MachineTypeMicrovm {
|
||||
+ if _, err := s.setupMmioSlot(Name, true); err != nil {
|
||||
+ return err
|
||||
+ }
|
||||
+
|
||||
+ return nil
|
||||
+ }
|
||||
+
|
||||
+ if _, _, err := s.setupPciSlot(Name, true); err != nil {
|
||||
+ return err
|
||||
+ }
|
||||
+
|
||||
+ return nil
|
||||
}
|
||||
|
||||
-func (s *stratovirt) hotplugNet(ctx context.Context, endpoint Endpoint, op operation) (err error) {
|
||||
- err = s.qmpSetup()
|
||||
+func (s *stratovirt) hotplugNet(ctx context.Context, endpoint Endpoint, op operation) error {
|
||||
+ err := s.qmpSetup()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -1198,6 +1469,14 @@ func (s *stratovirt) hotplugNet(ctx context.Context, endpoint Endpoint, op opera
|
||||
return fmt.Errorf("Endpoint is not supported")
|
||||
}
|
||||
|
||||
+ defer func() {
|
||||
+ if err != nil {
|
||||
+ if errDel := s.delDevSlot(endpoint.Name()); errDel != nil {
|
||||
+ s.Logger().WithError(errDel).Warnf("Failed to delete device slot.")
|
||||
+ }
|
||||
+ }
|
||||
+ }()
|
||||
+
|
||||
switch op {
|
||||
case addDevice:
|
||||
var (
|
||||
@@ -1220,27 +1499,51 @@ func (s *stratovirt) hotplugNet(ctx context.Context, endpoint Endpoint, op opera
|
||||
VhostFdNames = append(VhostFdNames, fdName)
|
||||
}
|
||||
|
||||
- slot, err := s.getDevSlot(endpoint.Name(), false)
|
||||
+ bus, slot, err := s.getDevSlot(endpoint.Name())
|
||||
if err != nil {
|
||||
- return fmt.Errorf("Could not get unused slot for %q", endpoint.Name())
|
||||
+ return fmt.Errorf("could not get unused slot for %q", endpoint.Name())
|
||||
}
|
||||
|
||||
if len(VMFdNames) != 0 || len(VhostFdNames) != 0 {
|
||||
if err := s.qmpMonitorCh.qmp.ExecuteNetdevAddByFds(s.qmpMonitorCh.ctx, "tap", tap.ID, VMFdNames, VhostFdNames); err != nil {
|
||||
- s.getDevSlot(endpoint.Name(), true)
|
||||
return err
|
||||
}
|
||||
} else {
|
||||
if err := s.qmpMonitorCh.qmp.ExecuteNetdevAdd(s.qmpMonitorCh.ctx, "tap", tap.ID, tap.TAPIface.Name, "no", "no", 0); err != nil {
|
||||
- s.getDevSlot(endpoint.Name(), true)
|
||||
return err
|
||||
}
|
||||
}
|
||||
- if err := s.qmpMonitorCh.qmp.ExecuteNetPCIDeviceAdd(s.qmpMonitorCh.ctx, tap.Name, tap.ID, endpoint.HardwareAddr(), fmt.Sprintf("%d", slot), "", "", 0, false); err != nil {
|
||||
- s.getDevSlot(endpoint.Name(), true)
|
||||
+
|
||||
+ // The slot of net device that hotplugged to the root port
|
||||
+ // must be zero.
|
||||
+ devAddr := "0x0.0x0"
|
||||
+ if s.vmConfig.machineType == MachineTypeMicrovm {
|
||||
+ devAddr = fmt.Sprintf("%d", slot)
|
||||
+ } else {
|
||||
+ bridgeSlot, err := vcTypes.PciSlotFromInt(slot)
|
||||
+ if err != nil {
|
||||
+ return err
|
||||
+ }
|
||||
+
|
||||
+ devSlot, err := vcTypes.PciSlotFromString("0")
|
||||
+ if err != nil {
|
||||
+ return err
|
||||
+ }
|
||||
+
|
||||
+ pciPath, err := vcTypes.PciPathFromSlots(bridgeSlot, devSlot)
|
||||
+ if err != nil {
|
||||
+ return err
|
||||
+ }
|
||||
+ endpoint.SetPciPath(pciPath)
|
||||
+ }
|
||||
+
|
||||
+ if err := s.qmpMonitorCh.qmp.ExecuteNetPCIDeviceAdd(s.qmpMonitorCh.ctx, tap.ID, tap.ID, endpoint.HardwareAddr(), devAddr, bus, "", 0, false); err != nil {
|
||||
return err
|
||||
}
|
||||
case removeDevice:
|
||||
+ if errDel := s.delDevSlot(endpoint.Name()); errDel != nil {
|
||||
+ s.Logger().WithError(errDel).Warnf("Failed to delete device slot.")
|
||||
+ }
|
||||
if err := s.qmpMonitorCh.qmp.ExecuteDeviceDel(s.qmpMonitorCh.ctx, tap.ID); err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -1248,58 +1551,134 @@ func (s *stratovirt) hotplugNet(ctx context.Context, endpoint Endpoint, op opera
|
||||
return err
|
||||
}
|
||||
default:
|
||||
- return fmt.Errorf("Operation is not supported")
|
||||
+ return fmt.Errorf("operation is not supported")
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
-func (s *stratovirt) hotplugBlk(drive *config.BlockDrive, op operation) (err error) {
|
||||
- var filePath string
|
||||
- err = s.qmpSetup()
|
||||
+func (s *stratovirt) hotplugBlk(ctx context.Context, drive *config.BlockDrive, op operation) error {
|
||||
+ err := s.qmpSetup()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
+ driver := "virtio-blk-pci"
|
||||
+ if s.vmConfig.machineType == MachineTypeMicrovm {
|
||||
+ driver = "virtio-blk-mmio"
|
||||
+ }
|
||||
+
|
||||
+ defer func() {
|
||||
+ if err != nil {
|
||||
+ s.qmpMonitorCh.qmp.ExecuteBlockdevDel(s.qmpMonitorCh.ctx, drive.ID)
|
||||
+ if errDel := s.delDevSlot(drive.VirtPath); errDel != nil {
|
||||
+ s.Logger().WithError(errDel).Warnf("Failed to delete device slot.")
|
||||
+ }
|
||||
+ }
|
||||
+ }()
|
||||
+
|
||||
switch op {
|
||||
case addDevice:
|
||||
- driver := "virtio-blk-mmio"
|
||||
+ filePath := drive.File
|
||||
if s.vmConfig.useOzone {
|
||||
filePath, err = s.updateOzoneRes(drive.File, true)
|
||||
- if err != nil {
|
||||
- return fmt.Errorf("Failed to update ozone resources")
|
||||
- }
|
||||
- } else {
|
||||
- filePath = drive.File
|
||||
- }
|
||||
- slot, err := s.getDevSlot(drive.VirtPath, false)
|
||||
- if err != nil {
|
||||
- return fmt.Errorf("Could not get unused slot for %q", drive.VirtPath)
|
||||
}
|
||||
|
||||
if err := s.qmpMonitorCh.qmp.ExecuteBlockdevAdd(s.qmpMonitorCh.ctx, filePath, drive.ID, false); err != nil {
|
||||
- s.getDevSlot(drive.VirtPath, true)
|
||||
return err
|
||||
}
|
||||
|
||||
- if err := s.qmpMonitorCh.qmp.ExecutePCIDeviceAdd(s.qmpMonitorCh.ctx, drive.ID, drive.ID, driver, fmt.Sprintf("%d", slot), "", "", 0, true, false); err != nil {
|
||||
- s.getDevSlot(drive.VirtPath, true)
|
||||
+ bus, slot, err := s.getDevSlot(drive.VirtPath)
|
||||
+ if err != nil {
|
||||
+ return err
|
||||
+ }
|
||||
+
|
||||
+ // The slot of block device that hotplugged to the root port
|
||||
+ // must be zero.
|
||||
+ devAddr := "0x0.0x0"
|
||||
+ if s.vmConfig.machineType == MachineTypeMicrovm {
|
||||
+ devAddr = fmt.Sprintf("%d", slot)
|
||||
+ } else {
|
||||
+ bridgeSlot, err := vcTypes.PciSlotFromInt(slot)
|
||||
+ if err != nil {
|
||||
+ return err
|
||||
+ }
|
||||
+
|
||||
+ devSlot, err := vcTypes.PciSlotFromString("0")
|
||||
+ if err != nil {
|
||||
+ return err
|
||||
+ }
|
||||
+
|
||||
+ drive.PCIPath, err = vcTypes.PciPathFromSlots(bridgeSlot, devSlot)
|
||||
+ if err != nil {
|
||||
+ return err
|
||||
+ }
|
||||
+ }
|
||||
+
|
||||
+ if err := s.qmpMonitorCh.qmp.ExecutePCIDeviceAdd(s.qmpMonitorCh.ctx, drive.ID, drive.ID, driver, devAddr, bus, "", 0, false, false); err != nil {
|
||||
return err
|
||||
}
|
||||
case removeDevice:
|
||||
if s.vmConfig.useOzone {
|
||||
s.updateOzoneRes(drive.File, false)
|
||||
}
|
||||
+
|
||||
+ if errDel := s.delDevSlot(drive.VirtPath); errDel != nil {
|
||||
+ s.Logger().WithError(errDel).Warnf("Failed to delete device slot.")
|
||||
+ }
|
||||
if err := s.qmpMonitorCh.qmp.ExecuteDeviceDel(s.qmpMonitorCh.ctx, drive.ID); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := s.qmpMonitorCh.qmp.ExecuteBlockdevDel(s.qmpMonitorCh.ctx, drive.ID); err != nil {
|
||||
return err
|
||||
}
|
||||
+ default:
|
||||
+ return fmt.Errorf("operation is not supported %d", op)
|
||||
+ }
|
||||
+
|
||||
+ return nil
|
||||
+}
|
||||
+
|
||||
+func (s *stratovirt) hotplugVFIO(ctx context.Context, device *config.VFIODev, op operation) error {
|
||||
+ err := s.qmpSetup()
|
||||
+ if err != nil {
|
||||
+ return err
|
||||
+ }
|
||||
+
|
||||
+ defer func() {
|
||||
+ if err != nil {
|
||||
+ if errDel := s.delDevSlot(device.ID); errDel != nil {
|
||||
+ s.Logger().WithError(errDel).Warnf("Failed to delete device slot.")
|
||||
+ }
|
||||
+ }
|
||||
+ }()
|
||||
|
||||
- s.getDevSlot(drive.VirtPath, true)
|
||||
+ switch op {
|
||||
+ case addDevice:
|
||||
+ var bus string
|
||||
+ // The slot of block device that hotplugged to the root port
|
||||
+ // must be zero.
|
||||
+ devAddr := "0x0.0x0"
|
||||
+ // The vfio device BDF format is 0000:1a:00.3
|
||||
+ device.BDF = "0000:" + device.BDF
|
||||
+
|
||||
+ bus, _, err = s.getDevSlot(device.ID)
|
||||
+ if err != nil {
|
||||
+ return err
|
||||
+ }
|
||||
+
|
||||
+ if err = s.qmpMonitorCh.qmp.ExecutePCIVFIODeviceAdd(s.qmpMonitorCh.ctx, device.ID, device.BDF, devAddr, bus, ""); err != nil {
|
||||
+ return err
|
||||
+ }
|
||||
+ case removeDevice:
|
||||
+ if errDel := s.delDevSlot(device.ID); errDel != nil {
|
||||
+ s.Logger().WithError(errDel).Warnf("Failed to delete device slot.")
|
||||
+ }
|
||||
+ if err := s.qmpMonitorCh.qmp.ExecuteDeviceDel(s.qmpMonitorCh.ctx, device.ID); err != nil {
|
||||
+ return err
|
||||
+ }
|
||||
default:
|
||||
- return fmt.Errorf("Operation is not supported")
|
||||
+ return fmt.Errorf("operation is not supported %d", op)
|
||||
}
|
||||
|
||||
return nil
|
||||
@@ -1313,9 +1692,11 @@ func (s *stratovirt) hotplugAddDevice(ctx context.Context, devInfo interface{},
|
||||
case netDev:
|
||||
return nil, s.hotplugNet(ctx, devInfo.(Endpoint), addDevice)
|
||||
case blockDev:
|
||||
- return nil, s.hotplugBlk(devInfo.(*config.BlockDrive), addDevice)
|
||||
+ return nil, s.hotplugBlk(ctx, devInfo.(*config.BlockDrive), addDevice)
|
||||
+ case vfioDev:
|
||||
+ return nil, s.hotplugVFIO(ctx, devInfo.(*config.VFIODev), addDevice)
|
||||
default:
|
||||
- return nil, fmt.Errorf("Hotplug add device failed: unsupported device type '%v'", devType)
|
||||
+ return nil, fmt.Errorf("hotplug add device failed: unsupported device type '%v'", devType)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1327,9 +1708,11 @@ func (s *stratovirt) hotplugRemoveDevice(ctx context.Context, devInfo interface{
|
||||
case netDev:
|
||||
return nil, s.hotplugNet(ctx, devInfo.(Endpoint), removeDevice)
|
||||
case blockDev:
|
||||
- return nil, s.hotplugBlk(devInfo.(*config.BlockDrive), removeDevice)
|
||||
+ return nil, s.hotplugBlk(ctx, devInfo.(*config.BlockDrive), removeDevice)
|
||||
+ case vfioDev:
|
||||
+ return nil, s.hotplugVFIO(ctx, devInfo.(*config.VFIODev), removeDevice)
|
||||
default:
|
||||
- return nil, fmt.Errorf("Hotplug remove device: unsupported device type '%v'", devType)
|
||||
+ return nil, fmt.Errorf("hotplug remove device: unsupported device type '%v'", devType)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1371,6 +1754,10 @@ func (s *stratovirt) capabilities(ctx context.Context) types.Capabilities {
|
||||
var caps types.Capabilities
|
||||
caps.SetBlockDeviceHotplugSupport()
|
||||
|
||||
+ if s.config.DisableBlockDeviceUse && s.config.SharedFS == config.VirtioFS {
|
||||
+ caps.SetFsSharingSupport()
|
||||
+ }
|
||||
+
|
||||
return caps
|
||||
}
|
||||
|
||||
@@ -1510,6 +1897,9 @@ func (s *stratovirt) getPids() []int {
|
||||
|
||||
pids = append(pids, s.state.pid)
|
||||
|
||||
+ if s.state.virtiofsPid != 0 {
|
||||
+ pids = append(pids, s.state.virtiofsPid)
|
||||
+ }
|
||||
return pids
|
||||
}
|
||||
|
||||
@@ -1544,12 +1934,14 @@ func (s *stratovirt) isRateLimiterBuiltin() bool {
|
||||
func (s *stratovirt) save() (p persistapi.HypervisorState) {
|
||||
pids := s.getPids()
|
||||
p.Pid = pids[0]
|
||||
+ p.VirtiofsdPid = s.state.virtiofsPid
|
||||
p.Type = string(StratovirtHypervisor)
|
||||
return
|
||||
}
|
||||
|
||||
func (s *stratovirt) load(p persistapi.HypervisorState) {
|
||||
s.state.pid = p.Pid
|
||||
+ s.state.virtiofsPid = p.VirtiofsdPid
|
||||
}
|
||||
|
||||
func (s *stratovirt) setSandbox(sandbox *Sandbox) {
|
||||
--
|
||||
2.20.1.windows.1
|
||||
|
||||
@ -0,0 +1,46 @@
|
||||
From 9ac9c120a51663fa98ef25d721c321e5d16b3859 Mon Sep 17 00:00:00 2001
|
||||
From: "Xinle.Guo" <guoxinle1@huawei.com>
|
||||
Date: Sat, 26 Feb 2022 17:25:11 +0800
|
||||
Subject: [PATCH] stratovirt: fix the problem that fails to plug net device
|
||||
|
||||
In kernel, hot plug pcie device needs to sleep 100ms to avoid
|
||||
misoperation. Kata 1.x version agent monitor the plug event to
|
||||
make sure plug device success. But kata 2.x does not provide
|
||||
method. It may failed to get net device after plug the device.
|
||||
|
||||
Signed-off-by: Xinle.Guo <guoxinle1@huawei.com>
|
||||
---
|
||||
src/runtime/virtcontainers/stratovirt.go | 3 +++
|
||||
1 file changed, 3 insertions(+)
|
||||
|
||||
diff --git a/src/runtime/virtcontainers/stratovirt.go b/src/runtime/virtcontainers/stratovirt.go
|
||||
index ffe8965..27b45b7 100644
|
||||
--- a/src/runtime/virtcontainers/stratovirt.go
|
||||
+++ b/src/runtime/virtcontainers/stratovirt.go
|
||||
@@ -39,6 +39,7 @@ const (
|
||||
iothreadID = "iothread_block"
|
||||
mmioBlkCount = 4
|
||||
mmioNetCount = 2
|
||||
+ hotPlugDelayTime = 200
|
||||
)
|
||||
|
||||
const (
|
||||
@@ -1540,6 +1541,7 @@ func (s *stratovirt) hotplugNet(ctx context.Context, endpoint Endpoint, op opera
|
||||
if err := s.qmpMonitorCh.qmp.ExecuteNetPCIDeviceAdd(s.qmpMonitorCh.ctx, tap.ID, tap.ID, endpoint.HardwareAddr(), devAddr, bus, "", 0, false); err != nil {
|
||||
return err
|
||||
}
|
||||
+ time.Sleep(time.Millisecond * hotPlugDelayTime)
|
||||
case removeDevice:
|
||||
if errDel := s.delDevSlot(endpoint.Name()); errDel != nil {
|
||||
s.Logger().WithError(errDel).Warnf("Failed to delete device slot.")
|
||||
@@ -1670,6 +1672,7 @@ func (s *stratovirt) hotplugVFIO(ctx context.Context, device *config.VFIODev, op
|
||||
if err = s.qmpMonitorCh.qmp.ExecutePCIVFIODeviceAdd(s.qmpMonitorCh.ctx, device.ID, device.BDF, devAddr, bus, ""); err != nil {
|
||||
return err
|
||||
}
|
||||
+ time.Sleep(time.Millisecond * hotPlugDelayTime)
|
||||
case removeDevice:
|
||||
if errDel := s.delDevSlot(device.ID); errDel != nil {
|
||||
s.Logger().WithError(errDel).Warnf("Failed to delete device slot.")
|
||||
--
|
||||
2.20.1.windows.1
|
||||
|
||||
@ -0,0 +1,78 @@
|
||||
From 769af40220ea3d7b87173f11c135e651076e14ee Mon Sep 17 00:00:00 2001
|
||||
From: "Xinle.Guo" <guoxinle1@huawei.com>
|
||||
Date: Wed, 2 Mar 2022 09:41:49 +0800
|
||||
Subject: [PATCH] stratovirt: provide a way to dynomically obtain firmware
|
||||
|
||||
For stratovirt, it requires firmware to boot. The default
|
||||
path is `/usr/share/edk2/xxx/pflash`. Now, we provides a
|
||||
way to set path in configuration.toml file
|
||||
|
||||
Signed-off-by: Xinle.Guo <guoxinle1@huawei.com>
|
||||
---
|
||||
src/runtime/cli/config/configuration-stratovirt.toml.in | 4 ++++
|
||||
src/runtime/pkg/katautils/config.go | 6 ++++++
|
||||
src/runtime/virtcontainers/stratovirt.go | 7 ++++---
|
||||
3 files changed, 14 insertions(+), 3 deletions(-)
|
||||
|
||||
diff --git a/src/runtime/cli/config/configuration-stratovirt.toml.in b/src/runtime/cli/config/configuration-stratovirt.toml.in
|
||||
index db46665..519c390 100644
|
||||
--- a/src/runtime/cli/config/configuration-stratovirt.toml.in
|
||||
+++ b/src/runtime/cli/config/configuration-stratovirt.toml.in
|
||||
@@ -28,6 +28,10 @@ enable_annotations = @DEFENABLEANNOTATIONS@
|
||||
# Your distribution recommends: @STRATOVIRTVALIDHYPERVISORPATHS@
|
||||
valid_hypervisor_paths = @STRATOVIRTVALIDHYPERVISORPATHS@
|
||||
|
||||
+# Path to the firmware.
|
||||
+# If you want that qemu uses the default firmware leave this option empty
|
||||
+firmware = "@FIRMWAREPATH@"
|
||||
+
|
||||
# Path for the ozone specific to stratovirt
|
||||
# If the ozone path is set, stratovirt will be launched in
|
||||
# ozone secure environment. It is disabled by default.
|
||||
diff --git a/src/runtime/pkg/katautils/config.go b/src/runtime/pkg/katautils/config.go
|
||||
index b04cdee..cdc9d7a 100644
|
||||
--- a/src/runtime/pkg/katautils/config.go
|
||||
+++ b/src/runtime/pkg/katautils/config.go
|
||||
@@ -951,6 +951,11 @@ func newStratovirtHypervisorConfig(h hypervisor) (vc.HypervisorConfig, error) {
|
||||
errors.New("either image or initrd must be defined in the configuration file")
|
||||
}
|
||||
|
||||
+ firmware, err := h.firmware()
|
||||
+ if err != nil {
|
||||
+ return vc.HypervisorConfig{}, err
|
||||
+ }
|
||||
+
|
||||
kernelParams := h.kernelParams()
|
||||
machineType := h.machineType()
|
||||
|
||||
@@ -979,6 +984,7 @@ func newStratovirtHypervisorConfig(h hypervisor) (vc.HypervisorConfig, error) {
|
||||
KernelPath: kernel,
|
||||
InitrdPath: initrd,
|
||||
ImagePath: image,
|
||||
+ FirmwarePath: firmware,
|
||||
OzonePath: ozone,
|
||||
KernelParams: vc.DeserializeParams(strings.Fields(kernelParams)),
|
||||
HypervisorMachineType: machineType,
|
||||
diff --git a/src/runtime/virtcontainers/stratovirt.go b/src/runtime/virtcontainers/stratovirt.go
|
||||
index 27b45b7..d2b2233 100644
|
||||
--- a/src/runtime/virtcontainers/stratovirt.go
|
||||
+++ b/src/runtime/virtcontainers/stratovirt.go
|
||||
@@ -659,11 +659,12 @@ func (s *stratovirt) getKernelParams(machineType string, initrdPath string) (str
|
||||
}
|
||||
|
||||
func (s *stratovirt) getPFlash(machineType string) ([]string, error) {
|
||||
- if s.config.PFlash != nil {
|
||||
- return s.config.PFlash, nil
|
||||
+ var PFlash []string
|
||||
+ if s.config.FirmwarePath != "" {
|
||||
+ PFlash = append(PFlash, fmt.Sprintf("file=%s,if=pflash,unit=0", s.config.FirmwarePath))
|
||||
+ return PFlash, nil
|
||||
}
|
||||
|
||||
- var PFlash []string
|
||||
switch machineType {
|
||||
case MachineTypeQ35:
|
||||
PFlash = append(PFlash, fmt.Sprintf("file=%s,if=pflash,unit=0", Q35PFlashCode))
|
||||
--
|
||||
2.20.1.windows.1
|
||||
|
||||
@ -0,0 +1,95 @@
|
||||
From c455ef6f406ba317af76b48ff79db131d0594bc1 Mon Sep 17 00:00:00 2001
|
||||
From: "Xinle.Guo" <guoxinle1@huawei.com>
|
||||
Date: Fri, 18 Mar 2022 10:49:35 +0800
|
||||
Subject: [PATCH] stratovirt: fix the problem that add more than 16 root port
|
||||
devices
|
||||
|
||||
It will failed to start StratoVirt sandbox if pcie root prot is set
|
||||
more than 16. The reason is that StratoVirt can only distinguish
|
||||
hexadecimal device address number.
|
||||
|
||||
Signed-off-by: Xinle.Guo <guoxinle1@huawei.com>
|
||||
---
|
||||
src/runtime/virtcontainers/stratovirt.go | 17 ++++++++++-------
|
||||
1 file changed, 10 insertions(+), 7 deletions(-)
|
||||
|
||||
diff --git a/src/runtime/virtcontainers/stratovirt.go b/src/runtime/virtcontainers/stratovirt.go
|
||||
index d2b2233..98a702a 100644
|
||||
--- a/src/runtime/virtcontainers/stratovirt.go
|
||||
+++ b/src/runtime/virtcontainers/stratovirt.go
|
||||
@@ -102,7 +102,7 @@ func (r rootPortDevice) getParams(config *vmConfig) []string {
|
||||
r.bus = "pcie.0"
|
||||
}
|
||||
devParams = append(devParams, Param{"bus", r.bus})
|
||||
- devParams = append(devParams, Param{"addr", fmt.Sprintf("%d", r.slot)})
|
||||
+ devParams = append(devParams, Param{"addr", fmt.Sprintf("0x%x", r.slot)})
|
||||
|
||||
driver := "pcie-root-port"
|
||||
params = append(params, "-device", fmt.Sprintf("%s,%s", driver, strings.Join(SerializeParams(devParams, "="), ",")))
|
||||
@@ -721,6 +721,9 @@ func (s *stratovirt) createDevices() []VirtioDev {
|
||||
|
||||
func (s *stratovirt) appendRootPort(ctx context.Context, devices []VirtioDev) []VirtioDev {
|
||||
number := s.config.PCIeRootPort
|
||||
+ if number > 20 {
|
||||
+ number = 20
|
||||
+ }
|
||||
|
||||
for i := uint32(1); i < number+1; i++ {
|
||||
addr, err := s.vmConfig.rootBus.AddDevice(ctx, fmt.Sprintf("%s.%d", RootPortPrefix, i))
|
||||
@@ -765,7 +768,7 @@ func (s *stratovirt) appendBlock(ctx context.Context, devices []VirtioDev) []Vir
|
||||
filePath: s.vmConfig.rootfsPath,
|
||||
deviceID: "virtio-blk0",
|
||||
bus: bus,
|
||||
- addr: fmt.Sprintf("%d", addr),
|
||||
+ addr: fmt.Sprintf("0x%x", addr),
|
||||
iothread: iothread,
|
||||
})
|
||||
|
||||
@@ -791,7 +794,7 @@ func (s *stratovirt) appendRng(ctx context.Context, devices []VirtioDev) []Virti
|
||||
rng: "objrng0",
|
||||
deviceID: "virtio-rng0",
|
||||
bus: bus,
|
||||
- addr: fmt.Sprintf("%d", addr),
|
||||
+ addr: fmt.Sprintf("0x%x", addr),
|
||||
})
|
||||
|
||||
return devices
|
||||
@@ -818,7 +821,7 @@ func (s *stratovirt) appendConsole(ctx context.Context, devices []VirtioDev) []V
|
||||
charDev: "charconsole0",
|
||||
deviceID: "virtio-console0",
|
||||
bus: bus,
|
||||
- addr: fmt.Sprintf("%d", addr),
|
||||
+ addr: fmt.Sprintf("0x%x", addr),
|
||||
})
|
||||
|
||||
return devices
|
||||
@@ -841,7 +844,7 @@ func (s *stratovirt) appendVhostVsock(ctx context.Context, devices []VirtioDev,
|
||||
id: "vsock-id",
|
||||
guestID: fmt.Sprintf("%d", vsock.ContextID),
|
||||
bus: bus,
|
||||
- addr: fmt.Sprintf("%d", addr),
|
||||
+ addr: fmt.Sprintf("0x%x", addr),
|
||||
})
|
||||
|
||||
return devices
|
||||
@@ -872,7 +875,7 @@ func (s *stratovirt) appendNetwork(ctx context.Context, devices []VirtioDev, end
|
||||
deviceID: name,
|
||||
bus: bus,
|
||||
mac: endpoint.HardwareAddr(),
|
||||
- addr: fmt.Sprintf("%d", addr),
|
||||
+ addr: fmt.Sprintf("0x%x", addr),
|
||||
})
|
||||
|
||||
return devices
|
||||
@@ -904,7 +907,7 @@ func (s *stratovirt) appendVirtioFs(ctx context.Context, devices []VirtioDev, vo
|
||||
tag: volume.MountTag,
|
||||
deviceID: "virtio-fs0",
|
||||
bus: bus,
|
||||
- addr: fmt.Sprintf("%d", addr),
|
||||
+ addr: fmt.Sprintf("0x%x", addr),
|
||||
})
|
||||
|
||||
return devices
|
||||
--
|
||||
2.20.1.windows.1
|
||||
|
||||
38
patches/0030-use-host_device-blockdev-adding.patch
Normal file
38
patches/0030-use-host_device-blockdev-adding.patch
Normal file
@ -0,0 +1,38 @@
|
||||
From 2da8be5657f219de41e11917cb916895934749b8 Mon Sep 17 00:00:00 2001
|
||||
From: chengzrz <czrzrichard@gmail.com>
|
||||
Date: Mon, 22 Aug 2022 19:19:52 +0800
|
||||
Subject: [PATCH] use host_device blockdev adding
|
||||
|
||||
ExecuteBlockdevAdd() and ExecuteBlockdevAddWithCache() both appear to be
|
||||
intended to create block devices in the guest which backend onto a block
|
||||
device in the host. That seems to be the way that Kata always uses it.
|
||||
However blockdevAddBaseArgs(), used by both those functions always uses the
|
||||
"file" driver, which is only intended for use with regular file backends.
|
||||
|
||||
Use of the "file" driver for host block devices was deprecated in qemu-3.0,
|
||||
and has been removed entirely in qemu-6.0 (commit 8d17adf34f5). We should
|
||||
be using the "host_device" driver instead.
|
||||
|
||||
reference: https://github.com/kata-containers/kata-containers/commit/1b021929864fa45b643d9603d6615cc4b86235d7
|
||||
|
||||
Signed-off-by: chengzrz <czrzrichard@gmail.com>
|
||||
---
|
||||
src/runtime/vendor/github.com/kata-containers/govmm/qemu/qmp.go | 2 +-
|
||||
1 file changed, 1 insertion(+), 1 deletion(-)
|
||||
|
||||
diff --git a/src/runtime/vendor/github.com/kata-containers/govmm/qemu/qmp.go b/src/runtime/vendor/github.com/kata-containers/govmm/qemu/qmp.go
|
||||
index 97e924559..0e0337dbf 100644
|
||||
--- a/src/runtime/vendor/github.com/kata-containers/govmm/qemu/qmp.go
|
||||
+++ b/src/runtime/vendor/github.com/kata-containers/govmm/qemu/qmp.go
|
||||
@@ -775,7 +775,7 @@ func (q *QMP) blockdevAddBaseArgs(device, blockdevID string, ro bool) (map[strin
|
||||
"driver": "raw",
|
||||
"read-only": ro,
|
||||
"file": map[string]interface{}{
|
||||
- "driver": "file",
|
||||
+ "driver": "host_device",
|
||||
"filename": device,
|
||||
},
|
||||
}
|
||||
--
|
||||
2.25.1
|
||||
|
||||
33
patches/0031-add-explicit-on-after-kernel_irqchip.patch
Normal file
33
patches/0031-add-explicit-on-after-kernel_irqchip.patch
Normal file
@ -0,0 +1,33 @@
|
||||
From 0c9c5ca20b6aeae5e550decfd3540b389fb02cb5 Mon Sep 17 00:00:00 2001
|
||||
From: chengzrz <czrzrichard@gmail.com>
|
||||
Date: Tue, 23 Aug 2022 15:30:05 +0800
|
||||
Subject: [PATCH] add explicit on after kernel_irqchip
|
||||
Kata uses the 'kernel_irqchip' machine option to qemu. By default it
|
||||
uses it in what qemu calls the "short-form boolean" with no parameter.
|
||||
That style was deprecated by qemu between 5.2 and 6.0 (commit
|
||||
ccd3b3b8112b) and effectively removed entirely between 6.0 and 6.1
|
||||
(commit d8fb7d0969d5).
|
||||
|
||||
reference:https://github.com/kata-containers/kata-containers/commit/316509566966e4c9b3fd9ba3521554b384fdbf88
|
||||
|
||||
Signed-off-by: chengzrz <czrzrichard@gmail.com>
|
||||
---
|
||||
src/runtime/virtcontainers/qemu_amd64.go | 2 +-
|
||||
1 file changed, 1 insertion(+), 1 deletion(-)
|
||||
|
||||
diff --git a/src/runtime/virtcontainers/qemu_amd64.go b/src/runtime/virtcontainers/qemu_amd64.go
|
||||
index 1a045fae0..bc598c653 100644
|
||||
--- a/src/runtime/virtcontainers/qemu_amd64.go
|
||||
+++ b/src/runtime/virtcontainers/qemu_amd64.go
|
||||
@@ -27,7 +27,7 @@ const (
|
||||
|
||||
defaultQemuMachineType = QemuPC
|
||||
|
||||
- defaultQemuMachineOptions = "accel=kvm,kernel_irqchip"
|
||||
+ defaultQemuMachineOptions = "accel=kvm,kernel_irqchip=on"
|
||||
|
||||
qmpMigrationWaitTimeout = 5 * time.Second
|
||||
)
|
||||
--
|
||||
2.25.1
|
||||
|
||||
@ -0,0 +1,67 @@
|
||||
From e3296a68f7fa270d72605dfd4eb40b1081a79969 Mon Sep 17 00:00:00 2001
|
||||
From: chengzrz <czrzrichard@gmail.com>
|
||||
Date: Fri, 2 Sep 2022 15:10:26 +0800
|
||||
Subject: [PATCH] qmp: Don't use deprecated 'props' field for object-add
|
||||
|
||||
Use of the 'props' argument to 'object-add' has been deprecated since QEMU
|
||||
5.0 (commit 5f07c4d60d09) in favor of flattening the properties directly
|
||||
into the 'object-add' arguments. Support for 'props' is removed entirely
|
||||
in qemu 6.0 (commit 50243407457a).
|
||||
|
||||
reference:https://github.com/kata-containers/kata-containers/commit/d27256f8635d3fa382d6cbd9f3a60f601773c4dc
|
||||
|
||||
Signed-off-by: chengzrz <czrzrichard@gmail.com>
|
||||
---
|
||||
.../kata-containers/govmm/qemu/qmp.go | 18 +++++++-----------
|
||||
1 file changed, 7 insertions(+), 11 deletions(-)
|
||||
|
||||
diff --git a/src/runtime/vendor/github.com/kata-containers/govmm/qemu/qmp.go b/src/runtime/vendor/github.com/kata-containers/govmm/qemu/qmp.go
|
||||
index 97e9245..91dd732 100644
|
||||
--- a/src/runtime/vendor/github.com/kata-containers/govmm/qemu/qmp.go
|
||||
+++ b/src/runtime/vendor/github.com/kata-containers/govmm/qemu/qmp.go
|
||||
@@ -1387,17 +1387,16 @@ func (q *QMP) ExecQueryCpusFast(ctx context.Context) ([]CPUInfoFast, error) {
|
||||
|
||||
// ExecMemdevAdd adds size of MiB memory device to the guest
|
||||
func (q *QMP) ExecMemdevAdd(ctx context.Context, qomtype, id, mempath string, size int, share bool, driver, driverID string) error {
|
||||
- props := map[string]interface{}{"size": uint64(size) << 20}
|
||||
args := map[string]interface{}{
|
||||
"qom-type": qomtype,
|
||||
"id": id,
|
||||
- "props": props,
|
||||
+ "size": uint64(size) << 20,
|
||||
}
|
||||
if mempath != "" {
|
||||
- props["mem-path"] = mempath
|
||||
+ args["mem-path"] = mempath
|
||||
}
|
||||
if share {
|
||||
- props["share"] = true
|
||||
+ args["share"] = true
|
||||
}
|
||||
err := q.executeCommand(ctx, "object-add", args, nil)
|
||||
if err != nil {
|
||||
@@ -1439,17 +1438,14 @@ func (q *QMP) ExecuteNVDIMMDeviceAdd(ctx context.Context, id, mempath string, si
|
||||
args := map[string]interface{}{
|
||||
"qom-type": "memory-backend-file",
|
||||
"id": "nvdimmbackmem" + id,
|
||||
- "props": map[string]interface{}{
|
||||
- "mem-path": mempath,
|
||||
- "size": size,
|
||||
- "share": true,
|
||||
- },
|
||||
+ "mem-path": mempath,
|
||||
+ "size": size,
|
||||
+ "share": true,
|
||||
}
|
||||
|
||||
if q.version.Major > 4 || (q.version.Major == 4 && q.version.Minor >= 1) {
|
||||
if pmem != nil {
|
||||
- props := args["props"].(map[string]interface{})
|
||||
- props["pmem"] = *pmem
|
||||
+ args["pmem"] = *pmem
|
||||
}
|
||||
}
|
||||
|
||||
--
|
||||
2.25.1
|
||||
|
||||
@ -1,20 +0,0 @@
|
||||
#!/bin/bash
|
||||
|
||||
if [[ -f ./patch_flag ]];then
|
||||
echo "proxy patched!"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
tar -zxvf proxy-1.11.1.tar.gz
|
||||
cp -fr ./proxy-1.11.1/* ./
|
||||
rm -rf ./proxy-1.11.1
|
||||
cat ./series.conf | while read line
|
||||
do
|
||||
if [[ $line == '' || $line =~ ^\s*# ]]; then
|
||||
continue
|
||||
fi
|
||||
echo "====patch $line======"
|
||||
pwd
|
||||
patch -p1 -F1 -s < ./patches/$line
|
||||
done
|
||||
touch ./patch_flag
|
||||
@ -1,403 +0,0 @@
|
||||
From a260bbe394f91fa05b163315390ed133de5c5494 Mon Sep 17 00:00:00 2001
|
||||
From: holyfei <yangfeiyu20092010@163.com>
|
||||
Date: Wed, 19 Aug 2020 17:43:24 +0800
|
||||
Subject: [PATCH] clock: synchronizes clock info to agent
|
||||
|
||||
reason: virtual machine's clock may be incorrect, proxy synchronizes
|
||||
clock info to help virtual machine adjust clock time
|
||||
|
||||
Signed-off-by: yangfeiyu <yangfeiyu2@huawei.com>
|
||||
---
|
||||
proxy.go | 7 ++
|
||||
proxy_test.go | 29 +++++
|
||||
sync_clock_client.go | 107 +++++++++++++++++
|
||||
sync_clock_client_test.go | 132 +++++++++++++++++++++
|
||||
.../kata-containers/agent/pkg/clock/clock_util.go | 44 +++++++
|
||||
5 files changed, 319 insertions(+)
|
||||
create mode 100644 sync_clock_client.go
|
||||
create mode 100644 sync_clock_client_test.go
|
||||
create mode 100644 vendor/github.com/kata-containers/agent/pkg/clock/clock_util.go
|
||||
|
||||
diff --git a/proxy.go b/proxy.go
|
||||
index ab062a5..9dfcb3c 100644
|
||||
--- a/proxy.go
|
||||
+++ b/proxy.go
|
||||
@@ -105,6 +105,13 @@ func serve(servConn io.ReadWriteCloser, proto, addr string, results chan error)
|
||||
// Start the heartbeat in a separate go routine
|
||||
go heartBeat(session)
|
||||
|
||||
+ // start the sync clock in a separate go routine
|
||||
+ syncClockStream, err := session.Open()
|
||||
+ if err != nil {
|
||||
+ return nil, nil, err
|
||||
+ }
|
||||
+ go SyncClock(syncClockStream)
|
||||
+
|
||||
// serving connection
|
||||
l, err := net.Listen(proto, addr)
|
||||
if err != nil {
|
||||
diff --git a/proxy_test.go b/proxy_test.go
|
||||
index 923b138..94fa523 100644
|
||||
--- a/proxy_test.go
|
||||
+++ b/proxy_test.go
|
||||
@@ -10,6 +10,7 @@ package main
|
||||
import (
|
||||
"bytes"
|
||||
"crypto/md5"
|
||||
+ "encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"io/ioutil"
|
||||
@@ -23,6 +24,7 @@ import (
|
||||
"testing"
|
||||
|
||||
"github.com/hashicorp/yamux"
|
||||
+ "github.com/kata-containers/agent/pkg/clock"
|
||||
"github.com/stretchr/testify/assert"
|
||||
)
|
||||
|
||||
@@ -121,6 +123,13 @@ func server(listener net.Listener, closeCh chan bool) error {
|
||||
session.Close()
|
||||
}()
|
||||
|
||||
+ // accept the sync clock stream first
|
||||
+ if syncClockStream, err := session.Accept(); err != nil {
|
||||
+ return err
|
||||
+ } else {
|
||||
+ go serverSyncClock(syncClockStream)
|
||||
+ }
|
||||
+
|
||||
for {
|
||||
stream, err := session.Accept()
|
||||
if err != nil {
|
||||
@@ -133,6 +142,26 @@ func server(listener net.Listener, closeCh chan bool) error {
|
||||
}
|
||||
}
|
||||
|
||||
+func serverSyncClock(stream net.Conn) {
|
||||
+ for {
|
||||
+ buf, byteNum, err := readConnData(stream)
|
||||
+ if err != nil {
|
||||
+ continue
|
||||
+ }
|
||||
+ var clockInfo clock.TimeValue
|
||||
+ if err := json.Unmarshal(buf[:byteNum], &clockInfo); err != nil {
|
||||
+ continue
|
||||
+ }
|
||||
+ nowTime := clock.GetCurrentTimeNs()
|
||||
+ clockInfo.ClientArriveTime = nowTime
|
||||
+ clockInfo.ServerSendTime = nowTime
|
||||
+ b, _ := json.Marshal(clockInfo)
|
||||
+ if err := clock.WriteConnData(stream, b); err != nil {
|
||||
+ continue
|
||||
+ }
|
||||
+ }
|
||||
+}
|
||||
+
|
||||
func TestUnixAddrParsing(T *testing.T) {
|
||||
buf := "unix://foo/bar"
|
||||
addr, err := unixAddr(buf)
|
||||
diff --git a/sync_clock_client.go b/sync_clock_client.go
|
||||
new file mode 100644
|
||||
index 0000000..9bf3e91
|
||||
--- /dev/null
|
||||
+++ b/sync_clock_client.go
|
||||
@@ -0,0 +1,107 @@
|
||||
+// Copyright (c) Huawei Technologies Co., Ltd. 2018. All rights reserved.
|
||||
+// SPDX-License-Identifier: Apache-2.0
|
||||
+// Description: sync clock client related function
|
||||
+// Author: xueshaojia x00464843
|
||||
+// Create: 2018-11-10
|
||||
+
|
||||
+package main
|
||||
+
|
||||
+import (
|
||||
+ "encoding/json"
|
||||
+ "fmt"
|
||||
+ "net"
|
||||
+ "time"
|
||||
+
|
||||
+ "github.com/kata-containers/agent/pkg/clock"
|
||||
+)
|
||||
+
|
||||
+const (
|
||||
+ allowTimeDiff = 10 * 1000 * 1000 // allow 10ms's difference
|
||||
+ syncClockInterval = 60 * time.Second // sync clock with agent every 60 seconds
|
||||
+ rpcTimeout = 10 * time.Second // timeout for proxy's reading data
|
||||
+)
|
||||
+
|
||||
+// readConnData reads data from stream
|
||||
+func readConnData(stream net.Conn) (buf []byte, byteNum int, err error) {
|
||||
+ // set read deadline to avoid case as following:
|
||||
+ // proxy and agent are both reading and then syncClock will never work
|
||||
+ stream.SetReadDeadline(time.Now().Add(rpcTimeout))
|
||||
+ buf = make([]byte, clock.MaxSyncClockByteNum)
|
||||
+ byteNum, err = stream.Read(buf)
|
||||
+ return buf, byteNum, err
|
||||
+}
|
||||
+
|
||||
+// getGuestClock syncs guest clock info
|
||||
+// sends ClientSendTime
|
||||
+// waits for ClientArriveTime and ServerSendTime
|
||||
+func getGuestClock(stream net.Conn, clockInfo *clock.TimeValue) error {
|
||||
+ clockInfo.Delta = 0
|
||||
+ b, err := json.Marshal(clockInfo)
|
||||
+ if err != nil {
|
||||
+ return err
|
||||
+ }
|
||||
+ if err = clock.WriteConnData(stream, b); err != nil {
|
||||
+ return err
|
||||
+ }
|
||||
+
|
||||
+ buf, byteNum, err := readConnData(stream)
|
||||
+ if err != nil {
|
||||
+ return err
|
||||
+ }
|
||||
+
|
||||
+ if err = json.Unmarshal(buf[:byteNum], clockInfo); err != nil {
|
||||
+ return fmt.Errorf("sync clock, parse guest clocktime error:%v", err)
|
||||
+ }
|
||||
+ return nil
|
||||
+}
|
||||
+
|
||||
+// adjustGuestClock tells server to ajust local clock with Delta
|
||||
+func adjustGuestClock(stream net.Conn, clockInfo *clock.TimeValue) error {
|
||||
+ b, err := json.Marshal(clockInfo)
|
||||
+ if err != nil {
|
||||
+ return err
|
||||
+ }
|
||||
+ logger().Debugf("sync clock, send:%s", string(b))
|
||||
+ return clock.WriteConnData(stream, b)
|
||||
+}
|
||||
+
|
||||
+// syncClock performs all the steps
|
||||
+// 1 get client send time[host]
|
||||
+// 2 request for client arrive time and server send time[guest os]
|
||||
+// 3 get server arrive time[host]
|
||||
+// 4 calculate clock diff
|
||||
+// 5 request to adjust guest clock
|
||||
+func syncClock(stream net.Conn) error {
|
||||
+ var clockInfo clock.TimeValue
|
||||
+ if clockInfo.ClientSendTime = clock.GetCurrentTimeNs(); clockInfo.ClientSendTime <= 0 {
|
||||
+ return fmt.Errorf("sync clock, get client sendtime error")
|
||||
+ }
|
||||
+ err := getGuestClock(stream, &clockInfo)
|
||||
+ if err != nil {
|
||||
+ return fmt.Errorf("sync clock, get guest clocktime error:%v", err)
|
||||
+ }
|
||||
+ if clockInfo.ServerArriveTime = clock.GetCurrentTimeNs(); clockInfo.ServerArriveTime <= 0 {
|
||||
+ return fmt.Errorf("sync clock, get client recvtime error")
|
||||
+ }
|
||||
+ if clockInfo.ClientSendTime <= 0 || clockInfo.ClientArriveTime <= 0 || clockInfo.ServerSendTime <= 0 {
|
||||
+ return fmt.Errorf("sync clock, some fields of NTP message error, raw message:%v", clockInfo)
|
||||
+ }
|
||||
+
|
||||
+ delta := ((clockInfo.ClientSendTime - clockInfo.ClientArriveTime) + (clockInfo.ServerArriveTime - clockInfo.ServerSendTime)) / 2
|
||||
+ if delta < -allowTimeDiff || delta > allowTimeDiff {
|
||||
+ clockInfo.Delta = delta
|
||||
+ if err := adjustGuestClock(stream, &clockInfo); err != nil {
|
||||
+ return fmt.Errorf("sync clock, failed to adjust guest clock : %v", err)
|
||||
+ }
|
||||
+ }
|
||||
+ return nil
|
||||
+}
|
||||
+
|
||||
+func SyncClock(stream net.Conn) {
|
||||
+ for {
|
||||
+ if err := syncClock(stream); err != nil {
|
||||
+ logger().WithError(err).Error("sync clock failed")
|
||||
+ }
|
||||
+ time.Sleep(syncClockInterval)
|
||||
+ }
|
||||
+}
|
||||
diff --git a/sync_clock_client_test.go b/sync_clock_client_test.go
|
||||
new file mode 100644
|
||||
index 0000000..b0b1c85
|
||||
--- /dev/null
|
||||
+++ b/sync_clock_client_test.go
|
||||
@@ -0,0 +1,132 @@
|
||||
+// Copyright (c) Huawei Technologies Co., Ltd. 2018. All rights reserved.
|
||||
+// SPDX-License-Identifier: Apache-2.0
|
||||
+// Description: sync clock client related test
|
||||
+// Author: xueshaojia x00464843
|
||||
+// Create: 2018-11-10
|
||||
+
|
||||
+package main
|
||||
+
|
||||
+import (
|
||||
+ "encoding/json"
|
||||
+ "fmt"
|
||||
+ "math/rand"
|
||||
+ "net"
|
||||
+ "os"
|
||||
+ "testing"
|
||||
+
|
||||
+ "github.com/hashicorp/yamux"
|
||||
+ "github.com/kata-containers/agent/pkg/clock"
|
||||
+)
|
||||
+
|
||||
+func TestReadWriteData(t *testing.T) {
|
||||
+ var clockInfo clock.TimeValue
|
||||
+ clockInfo.ClientSendTime = clock.GetCurrentTimeNs()
|
||||
+ b, err := json.Marshal(clockInfo)
|
||||
+ if err != nil {
|
||||
+ t.Fatalf("Marshal clock info fail, err:%v", err)
|
||||
+ }
|
||||
+
|
||||
+ err = clock.WriteConnData(clientStream, b)
|
||||
+ fmt.Printf("client send: %s\n", string(b))
|
||||
+ if err != nil {
|
||||
+ t.Fatalf("send clock info fail, err:%v", err)
|
||||
+ }
|
||||
+ _, _, err = readConnData(clientStream)
|
||||
+ if err != nil {
|
||||
+ t.Fatalf("recv clock info fail, err:%v", err)
|
||||
+ }
|
||||
+}
|
||||
+
|
||||
+func SetUpServer(sock string, readyChan chan int) error {
|
||||
+ var err error
|
||||
+ listener, err = net.Listen("unix", sock)
|
||||
+ if err != nil {
|
||||
+ return err
|
||||
+ }
|
||||
+ readyChan <- 1
|
||||
+ conn, err := listener.Accept()
|
||||
+ if err != nil {
|
||||
+ return err
|
||||
+ }
|
||||
+ session, err := yamux.Server(conn, nil)
|
||||
+ if err != nil {
|
||||
+ return err
|
||||
+ }
|
||||
+ serverSession = session
|
||||
+ stream, err := session.Accept()
|
||||
+ if err != nil {
|
||||
+ return err
|
||||
+ }
|
||||
+ for {
|
||||
+ var clockInfo clock.TimeValue
|
||||
+ var byteNum int
|
||||
+ var err error
|
||||
+ buf := make([]byte, 400)
|
||||
+ if byteNum, err = stream.Read(buf); err != nil {
|
||||
+ break
|
||||
+ }
|
||||
+
|
||||
+ if err = json.Unmarshal(buf[:byteNum], &clockInfo); err != nil {
|
||||
+ break
|
||||
+ }
|
||||
+ if clockInfo.Delta == 0 {
|
||||
+ nowTime := clock.GetCurrentTimeNs()
|
||||
+ clockInfo.ClientArriveTime = nowTime
|
||||
+ clockInfo.ClientArriveTime = nowTime
|
||||
+ b, _ := json.Marshal(&clockInfo)
|
||||
+ stream.Write(b)
|
||||
+ }
|
||||
+ }
|
||||
+ return nil
|
||||
+}
|
||||
+
|
||||
+func SetUpClient(sock string) error {
|
||||
+ conn, err := net.Dial("unix", sock)
|
||||
+ if err != nil {
|
||||
+ return err
|
||||
+ }
|
||||
+ session, err := yamux.Client(conn, nil)
|
||||
+ if err != nil {
|
||||
+ conn.Close()
|
||||
+ return err
|
||||
+ }
|
||||
+ clientSession = session
|
||||
+ stream, err := session.Open()
|
||||
+ if err != nil {
|
||||
+ clientSession.Close()
|
||||
+ return err
|
||||
+ }
|
||||
+ clientStream = stream
|
||||
+ return nil
|
||||
+}
|
||||
+
|
||||
+func TearDown() {
|
||||
+ listener.Close()
|
||||
+ serverSession.Close()
|
||||
+ clientSession.Close()
|
||||
+}
|
||||
+
|
||||
+func GenSocket() string {
|
||||
+ randSeed := clock.GetCurrentTimeNs()
|
||||
+ rand.Seed(randSeed)
|
||||
+ return fmt.Sprintf("/tmp/%d.sock", rand.Uint32())
|
||||
+}
|
||||
+
|
||||
+var listener net.Listener
|
||||
+var clientStream net.Conn
|
||||
+var serverSession *yamux.Session
|
||||
+var clientSession *yamux.Session
|
||||
+
|
||||
+func TestMain(m *testing.M) {
|
||||
+ waitConn := make(chan int)
|
||||
+ testSock := GenSocket()
|
||||
+ go SetUpServer(testSock, waitConn)
|
||||
+ <-waitConn
|
||||
+ if err := SetUpClient(testSock); err != nil {
|
||||
+ listener.Close()
|
||||
+ serverSession.Close()
|
||||
+ os.Exit(1)
|
||||
+ }
|
||||
+ m.Run()
|
||||
+ TearDown()
|
||||
+}
|
||||
diff --git a/vendor/github.com/kata-containers/agent/pkg/clock/clock_util.go b/vendor/github.com/kata-containers/agent/pkg/clock/clock_util.go
|
||||
new file mode 100644
|
||||
index 0000000..03244fd
|
||||
--- /dev/null
|
||||
+++ b/vendor/github.com/kata-containers/agent/pkg/clock/clock_util.go
|
||||
@@ -0,0 +1,44 @@
|
||||
+// Copyright (c) Huawei Technologies Co., Ltd. 2019. All rights reserved.
|
||||
+// SPDX-License-Identifier: Apache-2.0
|
||||
+// Description: common functions
|
||||
+// Author: jiangpeifei
|
||||
+// Create: 2019-05-28
|
||||
+
|
||||
+package clock
|
||||
+
|
||||
+import (
|
||||
+ "net"
|
||||
+ "syscall"
|
||||
+)
|
||||
+
|
||||
+type TimeValue struct {
|
||||
+ ClientSendTime int64 `json:"client_send_time"`
|
||||
+ ClientArriveTime int64 `json:"client_arrive_time"`
|
||||
+ ServerSendTime int64 `json:"server_send_time"`
|
||||
+ ServerArriveTime int64 `json:"server_arrive_time"`
|
||||
+ Delta int64 `json:"delta"`
|
||||
+}
|
||||
+
|
||||
+const MaxSyncClockByteNum = 400 //sync clock byte num, max=400
|
||||
+
|
||||
+// getCurrentTimeNs returns UTC time in Ns
|
||||
+func GetCurrentTimeNs() int64 {
|
||||
+ var tv syscall.Timeval
|
||||
+ if err := syscall.Gettimeofday(&tv); err != nil {
|
||||
+ return -1
|
||||
+ }
|
||||
+ return tv.Sec*1000000000 + tv.Usec*1000
|
||||
+}
|
||||
+
|
||||
+// readConnData reads data from stream
|
||||
+func ReadConnData(stream net.Conn) (buf []byte, byteNum int, err error) {
|
||||
+ buf = make([]byte, MaxSyncClockByteNum)
|
||||
+ byteNum, err = stream.Read(buf)
|
||||
+ return buf, byteNum, err
|
||||
+}
|
||||
+
|
||||
+// writeConnData writes data to stream
|
||||
+func WriteConnData(stream net.Conn, buf []byte) error {
|
||||
+ _, err := stream.Write(buf)
|
||||
+ return err
|
||||
+}
|
||||
--
|
||||
2.14.3 (Apple Git-98)
|
||||
|
||||
Binary file not shown.
@ -1 +0,0 @@
|
||||
0001-clock-synchronizes-clock-info-to-agent.patch
|
||||
@ -1,22 +0,0 @@
|
||||
#!/bin/bash
|
||||
|
||||
set -e
|
||||
|
||||
if [[ -f ./patch_flag ]];then
|
||||
echo "runtime patched!"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
tar -zxvf runtime-1.11.1.tar.gz
|
||||
cp -fr ./runtime-1.11.1/* ./
|
||||
rm -rf ./runtime-1.11.1
|
||||
cat ./series.conf | while read line
|
||||
do
|
||||
if [[ $line == '' || $line =~ ^\s*# ]]; then
|
||||
continue
|
||||
fi
|
||||
echo "====patch $line======"
|
||||
patch -p1 -F1 -s < ./patches/$line
|
||||
done
|
||||
|
||||
touch ./patch_flag
|
||||
@ -1,165 +0,0 @@
|
||||
%define debug_package %{nil}
|
||||
|
||||
%define VERSION 1.11.1
|
||||
%define RELEASE 11
|
||||
|
||||
Name: kata-runtime
|
||||
Version: %{VERSION}
|
||||
Release: %{RELEASE}
|
||||
Summary: Kata Runtime
|
||||
License: Apache 2.0
|
||||
URL: https://github.com/kata-containers/runtime
|
||||
Source0: https://github.com/kata-containers/runtime/archive/%{version}.tar.gz#/%{name}-v%{version}.tar.gz
|
||||
|
||||
BuildRoot: %_topdir/BUILDROOT
|
||||
BuildRequires: automake golang gcc
|
||||
|
||||
%description
|
||||
Kata-runtime is core component of Kata Container.
|
||||
|
||||
%prep
|
||||
%setup -q -c -a 0 -n %{name}-%{version}
|
||||
|
||||
%build
|
||||
cd %{_builddir}/%{name}-%{version}
|
||||
|
||||
set -e
|
||||
# apply patches read from series.conf
|
||||
sh apply-patches
|
||||
|
||||
# create tmp GOPATH dir to build kata-runtime
|
||||
rm -rf /tmp/kata-build/
|
||||
mkdir -p /tmp/kata-build/
|
||||
GOPATH=/tmp/kata-build/
|
||||
kata_base=$GOPATH/src/github.com/kata-containers
|
||||
mkdir -p $kata_base
|
||||
|
||||
# get current kata-runtime absolute path
|
||||
kata_runtime_path=$(readlink -f .)
|
||||
ln -s $kata_runtime_path $kata_base/runtime
|
||||
|
||||
# export GOPATH env
|
||||
export GOPATH=$(readlink -f $GOPATH)
|
||||
cd ${kata_base}/runtime && make clean && make
|
||||
rm -rf $GOPATH
|
||||
|
||||
# make kata-runtime default configuration
|
||||
kata_config_path=$kata_runtime_path/cli/config/configuration-qemu.toml
|
||||
ARCH=`arch`
|
||||
|
||||
# arch related config options
|
||||
if [ "$ARCH" == "aarch64" ];then
|
||||
sed -i 's/^machine_type.*$/machine_type = \"virt\"/' $kata_config_path
|
||||
sed -i 's/^block_device_driver.*$/block_device_driver = \"virtio-scsi\"/' $kata_config_path
|
||||
sed -i 's/^kernel_params.*$/kernel_params = \"pcie_ports=native pci=pcie_bus_perf agent.netlink_recv_buf_size=2MB\"/' $kata_config_path
|
||||
sed -i 's/^hypervisor_params.*$/hypervisor_params = \"kvm-pit.lost_tick_policy=discard pcie-root-port.fast-plug=1 pcie-root-port.x-speed=16 pcie-root-port.x-width=32 pcie-root-port.fast-unplug=1\"/' $kata_config_path
|
||||
sed -i 's/^#pcie_root_port.*$/pcie_root_port = 25/' $kata_config_path
|
||||
else
|
||||
sed -i 's#block_device_driver = \"virtio-scsi\"#block_device_driver = \"virtio-blk\"#' $kata_config_path
|
||||
sed -i 's/^#hotplug_vfio_on_root_bus/hotplug_vfio_on_root_bus/' $kata_config_path
|
||||
fi
|
||||
|
||||
# debug config
|
||||
sed -i 's/^#enable_debug.*$/enable_debug = true/' $kata_config_path
|
||||
|
||||
# other config
|
||||
sed -i 's#"/usr/bin/qemu.*"$#"/usr/bin/qemu-kvm"#' $kata_config_path
|
||||
sed -i 's#/usr/share/kata-containers/vmlinuz\.container#/var/lib/kata/kernel#' $kata_config_path
|
||||
sed -i 's#/usr/share/kata-containers/kata-containers-initrd\.img#/var/lib/kata/kata-containers-initrd\.img#' $kata_config_path
|
||||
sed -i 's/^image/#image/' $kata_config_path
|
||||
sed -i 's/^default_memory.*$/default_memory = 1024/' $kata_config_path
|
||||
sed -i 's/^#enable_blk_mount/enable_blk_mount/' $kata_config_path
|
||||
sed -i 's/^#block_device_cache_direct.*$/block_device_cache_direct = true/' $kata_config_path
|
||||
sed -i 's/^#block_device_cache_set.*$/block_device_cache_set = true/' $kata_config_path
|
||||
sed -i 's#/usr/libexec/kata-containers/kata-proxy#/usr/bin/kata-proxy#' $kata_config_path
|
||||
sed -i 's#/usr/libexec/kata-containers/kata-shim#/usr/bin/kata-shim#' $kata_config_path
|
||||
sed -i 's#/usr/libexec/kata-containers/kata-netmon#/usr/bin/kata-netmon#' $kata_config_path
|
||||
sed -i 's/^#disable_new_netns.*$/disable_new_netns = true/' $kata_config_path
|
||||
sed -i 's/^#disable_vhost_net.*$/disable_vhost_net = true/' $kata_config_path
|
||||
sed -i 's/^internetworking_model.*$/internetworking_model=\"none\"/' $kata_config_path
|
||||
sed -i 's/^enable_compat_old_cni.*$/#enable_compat_old_cni = true/' $kata_config_path
|
||||
sed -i 's/^sandbox_cgroup_only.*$/sandbox_cgroup_only = true/' $kata_config_path
|
||||
|
||||
set +e
|
||||
|
||||
%install
|
||||
cd %{_builddir}/%{name}-%{version}
|
||||
mkdir -p -m 750 %{buildroot}/usr/bin
|
||||
install -p -m 750 ./kata-runtime %{buildroot}/usr/bin
|
||||
install -p -m 750 ./kata-netmon %{buildroot}/usr/bin
|
||||
mkdir -p -m 750 %{buildroot}/usr/share/defaults/kata-containers
|
||||
install -p -m 640 ./cli/config/configuration-qemu.toml %{buildroot}/usr/share/defaults/kata-containers/configuration.toml
|
||||
|
||||
%clean
|
||||
|
||||
%files
|
||||
/usr/bin/kata-runtime
|
||||
/usr/bin/kata-netmon
|
||||
/usr/share/defaults/kata-containers/configuration.toml
|
||||
|
||||
%changelog
|
||||
* Tue Nov 17 2020 yangfeiyu<yangfeiyu20102011@163.com> - 1.11.1-11
|
||||
- Type:bugfix
|
||||
- ID:NA
|
||||
- SUG:upgrade
|
||||
- DESC:fix cpu resource limited problem when sandox_cgroup_with_emulator config is enabled
|
||||
|
||||
* Fri Oct 9 2020 yangfeiyu<yangfeiyu20102011@163.com> - 1.11.1-10
|
||||
- Type:feature
|
||||
- ID:NA
|
||||
- SUG:restart
|
||||
- DESC:support using CNI plugin to insert mutiple network interfaces at the same time
|
||||
|
||||
* Mon Sep 28 2020 yangfeiyu<yangfeiyu20102011@163.com> - 1.11.1-9
|
||||
- Type:bugfix
|
||||
- ID:NA
|
||||
- SUG:restart
|
||||
- DESC:retry inserting of CNI interface when netmon is enable
|
||||
|
||||
* Sun Sep 27 2020 LiangZhang<zhangliang5@Huawei.com> - 1.11.1-8
|
||||
- Type:bugfix
|
||||
- ID:NA
|
||||
- SUG:NA
|
||||
- DESC:fix cmd params of direct use stratovirt binary
|
||||
|
||||
* Thu Sep 24 2020 LiangZhang<zhangliang5@Huawei.com> - 1.11.1-7
|
||||
- Type:bugfix
|
||||
- ID:NA
|
||||
- SUG:NA
|
||||
- DESC:fix invalid cmdline when start sandbox stratovirt
|
||||
|
||||
* Mon Sep 21 2020 yangfeiyu<yangfeiyu20102011@163.com> - 1.11.1-6
|
||||
- Type:bugfix
|
||||
- ID:NA
|
||||
- SUG:NA
|
||||
- DESC:fix sandboxRuntimeRootPath left problem
|
||||
|
||||
* Mon Sep 21 2020 yangfeiyu<yangfeiyu20102011@163.com> - 1.11.1-5
|
||||
- Type:enhancement
|
||||
- ID:NA
|
||||
- SUG:NA
|
||||
- DESC:add support for host cgroups with emulator
|
||||
|
||||
* Mon Sep 21 2020 LiangZhang<zhangliang5@Huawei.com> - 1.11.1-4
|
||||
- Type:enhancement
|
||||
- ID:NA
|
||||
- SUG:NA
|
||||
- DESC:add support of new sandbox StratoVirt
|
||||
|
||||
* Sat Sep 19 2020 yangfeiyu<yangfeiyu20102011@163.com> - 1.11.1-3
|
||||
- Type:bugfix
|
||||
- ID:NA
|
||||
- SUG:NA
|
||||
- DESC:fix del-iface doesn't delete the tap interface in the host problem
|
||||
|
||||
* Sat Sep 5 2020 yangfeiyu<yangfeiyu20102011@163.com> - 1.11.1-2
|
||||
- Type:enhancement
|
||||
- ID:NA
|
||||
- SUG:NA
|
||||
- DESC:use URL format for Source0
|
||||
|
||||
* Wed Aug 26 2020 yangfeiyu<yangfeiyu20102011@163.com> - 1.11.1-1
|
||||
- Type:enhancement
|
||||
- ID:NA
|
||||
- SUG:NA
|
||||
- DESC:modify kata-runtime spec file to build seperately
|
||||
@ -1,44 +0,0 @@
|
||||
From 73fe7242d18a10a86bc216ec5e33a10a8751f85f Mon Sep 17 00:00:00 2001
|
||||
From: jiangpengfei <jiangpengfei9@huawei.com>
|
||||
Date: Fri, 24 Jul 2020 22:22:00 +0800
|
||||
Subject: [PATCH 01/50] qmp: fix kata-runtime hungs when qemu process is D/T
|
||||
state
|
||||
|
||||
reason: When set qemu's status to T and execute add-iface command
|
||||
the command will hung all the time.It hungs while wait for qemu
|
||||
to return version messages.
|
||||
|
||||
When qmp starts, set the timeout time to 15 seconds.When times
|
||||
out, return qmp starts failed.We choose 15 seconds to keep consistent
|
||||
with the agent client start.
|
||||
|
||||
Signed-off-by: jiangpengfei <jiangpengfei9@huawei.com>
|
||||
---
|
||||
vendor/github.com/intel/govmm/qemu/checklist | 1 +
|
||||
vendor/github.com/intel/govmm/qemu/qmp.go | 2 ++
|
||||
2 files changed, 3 insertions(+)
|
||||
create mode 100644 vendor/github.com/intel/govmm/qemu/checklist
|
||||
|
||||
diff --git a/vendor/github.com/intel/govmm/qemu/checklist b/vendor/github.com/intel/govmm/qemu/checklist
|
||||
new file mode 100644
|
||||
index 00000000..b32f1855
|
||||
--- /dev/null
|
||||
+++ b/vendor/github.com/intel/govmm/qemu/checklist
|
||||
@@ -0,0 +1 @@
|
||||
+add timeout when qmp start to avoid qmp client hungs all the time
|
||||
diff --git a/vendor/github.com/intel/govmm/qemu/qmp.go b/vendor/github.com/intel/govmm/qemu/qmp.go
|
||||
index bf9a77dd..a64039de 100644
|
||||
--- a/vendor/github.com/intel/govmm/qemu/qmp.go
|
||||
+++ b/vendor/github.com/intel/govmm/qemu/qmp.go
|
||||
@@ -722,6 +722,8 @@ func QMPStart(ctx context.Context, socket string, cfg QMPConfig, disconnectedCh
|
||||
if q.version == nil {
|
||||
return nil, nil, fmt.Errorf("failed to find QMP version information")
|
||||
}
|
||||
+ case <-time.After(15 * time.Second):
|
||||
+ return nil, nil, fmt.Errorf("qmp start time out")
|
||||
}
|
||||
|
||||
return q, q.version, nil
|
||||
--
|
||||
2.14.3 (Apple Git-98)
|
||||
|
||||
@ -1,55 +0,0 @@
|
||||
From cf595941e1d105af23bc006bf1998ac072733d0a Mon Sep 17 00:00:00 2001
|
||||
From: jiangpengfei <jiangpengfei9@huawei.com>
|
||||
Date: Sat, 25 Jul 2020 10:03:35 +0800
|
||||
Subject: [PATCH 03/50] kata-runtime: fix kata-proxy process left problem
|
||||
|
||||
reason: stopSandbox function will send the DestroySandboxRequest
|
||||
to kata-agent in the VM and then kill the kata-proxy process in
|
||||
the host. However, if k.sendReq(DestroySandboxRequest) get error,
|
||||
stopSandbox will return immediately not execute the following kill
|
||||
kata-proxy process statement, which cause the kata-process left.
|
||||
|
||||
Signed-off-by: jiangpengfei <jiangpengfei9@huawei.com>
|
||||
---
|
||||
virtcontainers/kata_agent.go | 18 +++++++++++-------
|
||||
1 file changed, 11 insertions(+), 7 deletions(-)
|
||||
|
||||
diff --git a/virtcontainers/kata_agent.go b/virtcontainers/kata_agent.go
|
||||
index a0cf190e..be5e96aa 100644
|
||||
--- a/virtcontainers/kata_agent.go
|
||||
+++ b/virtcontainers/kata_agent.go
|
||||
@@ -976,6 +976,17 @@ func (k *kataAgent) stopSandbox(sandbox *Sandbox) error {
|
||||
return errorMissingProxy
|
||||
}
|
||||
|
||||
+ // since stopSandbox will destroy the sandbox in the VM, and we don't need
|
||||
+ // kata-proxy process to communicate with kata-agent again, so we should
|
||||
+ // make sure kata-proxy can be killed cleanly, even when k.sendReq(DestroySandboxRequest)
|
||||
+ // return error
|
||||
+ defer func() {
|
||||
+ _ = k.proxy.stop(k.state.ProxyPid)
|
||||
+ // clean up agent state
|
||||
+ k.state.ProxyPid = -1
|
||||
+ k.state.URL = ""
|
||||
+ }()
|
||||
+
|
||||
req := &grpc.DestroySandboxRequest{}
|
||||
|
||||
if _, err := k.sendReq(req); err != nil {
|
||||
@@ -989,13 +1000,6 @@ func (k *kataAgent) stopSandbox(sandbox *Sandbox) error {
|
||||
}
|
||||
}
|
||||
|
||||
- if err := k.proxy.stop(k.state.ProxyPid); err != nil {
|
||||
- return err
|
||||
- }
|
||||
-
|
||||
- // clean up agent state
|
||||
- k.state.ProxyPid = -1
|
||||
- k.state.URL = ""
|
||||
return nil
|
||||
}
|
||||
|
||||
--
|
||||
2.14.3 (Apple Git-98)
|
||||
|
||||
@ -1,65 +0,0 @@
|
||||
From 025520f7fd3aeb5ed53b468b5e494b1bbb6674ae Mon Sep 17 00:00:00 2001
|
||||
From: jiangpengfei <jiangpengfei9@huawei.com>
|
||||
Date: Sat, 25 Jul 2020 11:25:34 +0800
|
||||
Subject: [PATCH 04/50] kata-runtime: keep the process name of qemu same as
|
||||
configured path
|
||||
|
||||
reason: inorder to make testcase scripts can use the same hypervisor
|
||||
name no matter what version hypervisor use, keep the process name of
|
||||
hypervisor same as configured path in the configuration.toml file
|
||||
instead of the resolved path of symbol link.
|
||||
|
||||
Signed-off-by: jiangpengfei <jiangpengfei9@huawei.com>
|
||||
---
|
||||
pkg/katautils/config.go | 8 +++++++-
|
||||
pkg/katautils/config_test.go | 4 ++--
|
||||
2 files changed, 9 insertions(+), 3 deletions(-)
|
||||
|
||||
diff --git a/pkg/katautils/config.go b/pkg/katautils/config.go
|
||||
index 14794a24..349e667f 100644
|
||||
--- a/pkg/katautils/config.go
|
||||
+++ b/pkg/katautils/config.go
|
||||
@@ -10,6 +10,7 @@ import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"io/ioutil"
|
||||
+ "path/filepath"
|
||||
goruntime "runtime"
|
||||
"strings"
|
||||
|
||||
@@ -172,7 +173,12 @@ func (h hypervisor) path() (string, error) {
|
||||
p = defaultHypervisorPath
|
||||
}
|
||||
|
||||
- return ResolvePath(p)
|
||||
+ absolutePath, err := filepath.Abs(p)
|
||||
+ if err != nil {
|
||||
+ return "", err
|
||||
+ }
|
||||
+
|
||||
+ return absolutePath, nil
|
||||
}
|
||||
|
||||
func (h hypervisor) ctlpath() (string, error) {
|
||||
diff --git a/pkg/katautils/config_test.go b/pkg/katautils/config_test.go
|
||||
index 221a4b55..2eae1f6a 100644
|
||||
--- a/pkg/katautils/config_test.go
|
||||
+++ b/pkg/katautils/config_test.go
|
||||
@@ -1061,12 +1061,12 @@ func TestHypervisorDefaultsHypervisor(t *testing.T) {
|
||||
assert.NoError(err)
|
||||
assert.Equal(p, defaultHypervisorPath, "default hypervisor path wrong")
|
||||
|
||||
- // test path resolution
|
||||
+ // test path resolution, just return the absolute path instead of resolved path
|
||||
defaultHypervisorPath = testHypervisorLinkPath
|
||||
h = hypervisor{}
|
||||
p, err = h.path()
|
||||
assert.NoError(err)
|
||||
- assert.Equal(p, testHypervisorPath)
|
||||
+ assert.Equal(p, testHypervisorLinkPath)
|
||||
}
|
||||
|
||||
func TestHypervisorDefaultsKernel(t *testing.T) {
|
||||
--
|
||||
2.14.3 (Apple Git-98)
|
||||
|
||||
@ -1,39 +0,0 @@
|
||||
From 3dc10421f177900c0ee94fc49b32ec66a46d9331 Mon Sep 17 00:00:00 2001
|
||||
From: jiangpengfei <jiangpengfei9@huawei.com>
|
||||
Date: Mon, 27 Jul 2020 19:18:50 +0800
|
||||
Subject: [PATCH 06/50] kata-runtime: fix umount container rootfs dir return
|
||||
ivalid argument error
|
||||
|
||||
reason: If sandbox hypervisor doesn't use block device driver for hotplugging container
|
||||
rootfs block device into guest, kata-runtime will bind mount container rootfs dir to 9p
|
||||
kataShared dir. However, container stop() function will always call bindUnmountContainerRootfs
|
||||
function no matter block device driver is used or not. So we just need to call
|
||||
bindUnmountContainerRootfs only if rootfs is bind mount to guest by 9p.
|
||||
|
||||
Signed-off-by: jiangpengfei <jiangpengfei9@huawei.com>
|
||||
---
|
||||
virtcontainers/container.go | 8 ++++++--
|
||||
1 file changed, 6 insertions(+), 2 deletions(-)
|
||||
|
||||
diff --git a/virtcontainers/container.go b/virtcontainers/container.go
|
||||
index 9e2d1e94..b42cc6e9 100644
|
||||
--- a/virtcontainers/container.go
|
||||
+++ b/virtcontainers/container.go
|
||||
@@ -1120,8 +1120,12 @@ func (c *Container) stop(force bool) error {
|
||||
return err
|
||||
}
|
||||
|
||||
- if err := bindUnmountContainerRootfs(c.ctx, getMountPath(c.sandbox.id), c.id); err != nil && !force {
|
||||
- return err
|
||||
+ // umount container rootfs dir only if container use 9p
|
||||
+ // to bind mount host container rootfs to 9p shared dir
|
||||
+ if c.state.BlockDeviceID == "" {
|
||||
+ if err := bindUnmountContainerRootfs(c.ctx, getMountPath(c.sandbox.id), c.id); err != nil && !force {
|
||||
+ return err
|
||||
+ }
|
||||
}
|
||||
|
||||
if err := c.detachDevices(); err != nil && !force {
|
||||
--
|
||||
2.14.3 (Apple Git-98)
|
||||
|
||||
@ -1,855 +0,0 @@
|
||||
From d93da1875ed7f1a6061cffb13475506d73c86003 Mon Sep 17 00:00:00 2001
|
||||
From: jiangpengfei <jiangpengfei9@huawei.com>
|
||||
Date: Sat, 25 Jul 2020 16:04:19 +0800
|
||||
Subject: [PATCH 07/50] kata-runtime: enhance reliability when kata related
|
||||
process
|
||||
|
||||
reason: enhance the reliability when kata related processes is abnormal,
|
||||
make kata-container still destroy the sandbox and clean up all resources.
|
||||
|
||||
Signed-off-by: jiangpengfei <jiangpengfei9@huawei.com>
|
||||
---
|
||||
cli/delete.go | 6 ++
|
||||
cli/kill.go | 3 +-
|
||||
virtcontainers/acrn.go | 2 +-
|
||||
virtcontainers/agent.go | 3 +
|
||||
virtcontainers/api.go | 67 +++++++++++++++++++++
|
||||
virtcontainers/clh.go | 2 +-
|
||||
virtcontainers/container.go | 55 +++++++++++++++--
|
||||
virtcontainers/fc.go | 2 +-
|
||||
virtcontainers/hypervisor.go | 2 +-
|
||||
virtcontainers/kata_agent.go | 31 ++++++----
|
||||
virtcontainers/mock_hypervisor.go | 2 +-
|
||||
virtcontainers/mock_hypervisor_test.go | 2 +-
|
||||
virtcontainers/noop_agent.go | 4 ++
|
||||
virtcontainers/pkg/oci/utils.go | 5 ++
|
||||
virtcontainers/qemu.go | 51 +++++++++-------
|
||||
virtcontainers/sandbox.go | 106 +++++++++++++++++++++++++++++----
|
||||
virtcontainers/types/sandbox.go | 14 ++++-
|
||||
virtcontainers/utils/utils.go | 46 ++++++++++++++
|
||||
virtcontainers/vm.go | 4 +-
|
||||
19 files changed, 348 insertions(+), 59 deletions(-)
|
||||
|
||||
diff --git a/cli/delete.go b/cli/delete.go
|
||||
index c2ce52a4..2f5586e5 100644
|
||||
--- a/cli/delete.go
|
||||
+++ b/cli/delete.go
|
||||
@@ -110,6 +110,12 @@ func delete(ctx context.Context, containerID string, force bool) error {
|
||||
forceStop = true
|
||||
}
|
||||
|
||||
+ if oci.StateToOCIState(status.State.State) == oci.StateUnhealthy {
|
||||
+ // Set forceStop and force bool flag to true to force delete everything
|
||||
+ forceStop = true
|
||||
+ force = true
|
||||
+ }
|
||||
+
|
||||
switch containerType {
|
||||
case vc.PodSandbox:
|
||||
if err := deleteSandbox(ctx, sandboxID, force); err != nil {
|
||||
diff --git a/cli/kill.go b/cli/kill.go
|
||||
index 60fa41e0..b228205f 100644
|
||||
--- a/cli/kill.go
|
||||
+++ b/cli/kill.go
|
||||
@@ -133,11 +133,12 @@ func kill(ctx context.Context, containerID, signal string, all bool) error {
|
||||
kataLog.WithField("signal", signal).WithField("container state", status.State.State).Info("kill")
|
||||
|
||||
// container MUST be created, running or paused
|
||||
+ // If container state is unhealthy, should process this exceptional case separately
|
||||
if status.State.State == types.StateReady || status.State.State == types.StateRunning || status.State.State == types.StatePaused {
|
||||
if err := vci.KillContainer(ctx, sandboxID, containerID, signum, all); err != nil {
|
||||
return err
|
||||
}
|
||||
- } else if !all {
|
||||
+ } else if !all && status.State.State != types.StateUnhealthy {
|
||||
return fmt.Errorf("container not running")
|
||||
}
|
||||
|
||||
diff --git a/virtcontainers/acrn.go b/virtcontainers/acrn.go
|
||||
index 761eda03..10cae06f 100644
|
||||
--- a/virtcontainers/acrn.go
|
||||
+++ b/virtcontainers/acrn.go
|
||||
@@ -475,7 +475,7 @@ func (a *Acrn) waitSandbox(timeoutSecs int) error {
|
||||
}
|
||||
|
||||
// stopSandbox will stop the Sandbox's VM.
|
||||
-func (a *Acrn) stopSandbox() (err error) {
|
||||
+func (a *Acrn) stopSandbox(force bool) (err error) {
|
||||
span, _ := a.trace("stopSandbox")
|
||||
defer span.Finish()
|
||||
|
||||
diff --git a/virtcontainers/agent.go b/virtcontainers/agent.go
|
||||
index c62107ec..be9526c7 100644
|
||||
--- a/virtcontainers/agent.go
|
||||
+++ b/virtcontainers/agent.go
|
||||
@@ -259,4 +259,7 @@ type agent interface {
|
||||
|
||||
// load data from disk
|
||||
load(persistapi.AgentState)
|
||||
+
|
||||
+ // get proxy process pid
|
||||
+ getProxyPid() int
|
||||
}
|
||||
diff --git a/virtcontainers/api.go b/virtcontainers/api.go
|
||||
index de569713..fa82d163 100644
|
||||
--- a/virtcontainers/api.go
|
||||
+++ b/virtcontainers/api.go
|
||||
@@ -7,8 +7,10 @@ package virtcontainers
|
||||
|
||||
import (
|
||||
"context"
|
||||
+ "fmt"
|
||||
"os"
|
||||
"runtime"
|
||||
+ "strings"
|
||||
"syscall"
|
||||
|
||||
deviceApi "github.com/kata-containers/runtime/virtcontainers/device/api"
|
||||
@@ -18,6 +20,7 @@ import (
|
||||
vcTypes "github.com/kata-containers/runtime/virtcontainers/pkg/types"
|
||||
"github.com/kata-containers/runtime/virtcontainers/store"
|
||||
"github.com/kata-containers/runtime/virtcontainers/types"
|
||||
+ "github.com/kata-containers/runtime/virtcontainers/utils"
|
||||
specs "github.com/opencontainers/runtime-spec/specs-go"
|
||||
opentracing "github.com/opentracing/opentracing-go"
|
||||
"github.com/sirupsen/logrus"
|
||||
@@ -597,20 +600,51 @@ func statusContainer(sandbox *Sandbox, containerID string) (ContainerStatus, err
|
||||
container.state.State == types.StatePaused) &&
|
||||
container.process.Pid > 0 {
|
||||
|
||||
+ // If container state is active, however kata-proxy and qemu process all exit already
|
||||
+ // which means sandbox has beed stopped exceptionally, then we should force delete
|
||||
+ // sandbox and container state files in sandbox.Store
|
||||
+ if sandbox.shouldForceDelete() {
|
||||
+ virtLog.Logger.Warn("sandbox status is abnormal, sandbox should be force deleted")
|
||||
+ sandbox.forceDeleteSandbox()
|
||||
+ return ContainerStatus{}, fmt.Errorf("sandbox has beed stopped exceptionally")
|
||||
+ }
|
||||
+
|
||||
running, err := isShimRunning(container.process.Pid)
|
||||
if err != nil {
|
||||
return ContainerStatus{}, err
|
||||
}
|
||||
|
||||
+ // If kata-shim process exit or be killed, need to stop the container
|
||||
if !running {
|
||||
virtLog.WithFields(logrus.Fields{
|
||||
"state": container.state.State,
|
||||
"pid": container.process.Pid}).
|
||||
Info("container isn't running")
|
||||
+
|
||||
if err := container.stop(true); err != nil {
|
||||
return ContainerStatus{}, err
|
||||
}
|
||||
}
|
||||
+
|
||||
+ isPodSandbox := (containerID == sandbox.id)
|
||||
+
|
||||
+ // If sandbox is unhealthy, process it correctly
|
||||
+ if !sandbox.health() {
|
||||
+ // process podSandbox container type case
|
||||
+ if isPodSandbox {
|
||||
+ if err := processUnhealthySandbox(sandbox, container); err != nil {
|
||||
+ return ContainerStatus{}, err
|
||||
+ }
|
||||
+ } else {
|
||||
+ // If container type is pod_container, which means container operations can not be
|
||||
+ // processed successfully, we should return the error as soon as possible
|
||||
+ if err := container.setContainerState(types.StateUnhealthy); err != nil {
|
||||
+ return ContainerStatus{}, err
|
||||
+ }
|
||||
+
|
||||
+ return ContainerStatus{}, fmt.Errorf("container status is unhealthy, stop container failed")
|
||||
+ }
|
||||
+ }
|
||||
}
|
||||
|
||||
return ContainerStatus{
|
||||
@@ -1016,3 +1050,36 @@ func CleanupContainer(ctx context.Context, sandboxID, containerID string, force
|
||||
|
||||
return nil
|
||||
}
|
||||
+
|
||||
+// procesUnhealthySandbox only change sandbox state to unhealthy
|
||||
+// when caller is kata-runtime kill or kata-runtime delete
|
||||
+func processUnhealthySandbox(sandbox *Sandbox, container *Container) error {
|
||||
+ // Set all containers state to unhealthy
|
||||
+ if err := sandbox.setContainersState(types.StateUnhealthy); err != nil {
|
||||
+ container.Logger().WithError(err).Warn("set all containers state to unhealthy fail")
|
||||
+ }
|
||||
+
|
||||
+ // Set sandbox state to unhealthy
|
||||
+ if err := sandbox.setSandboxState(types.StateUnhealthy); err != nil {
|
||||
+ container.Logger().WithError(err).Warn("set sandbox state to unhealthy fail")
|
||||
+ }
|
||||
+
|
||||
+ forceDelete := false
|
||||
+
|
||||
+ // If process is kata-runtime kill or kata-runtime delete,
|
||||
+ // we should kill or delete sandbox forcefully
|
||||
+ if cmdline, err := utils.GetProcessCmdline(os.Getpid()); err != nil {
|
||||
+ container.Logger().WithError(err).Warn("fail to get process cmdline info")
|
||||
+ } else {
|
||||
+ forceDelete = strings.Contains(cmdline, "kill") || strings.Contains(cmdline, "delete")
|
||||
+ }
|
||||
+
|
||||
+ if forceDelete {
|
||||
+ // force stop podSandbox type container's kata-shim process
|
||||
+ if err := stopShim(container.process.Pid); err != nil {
|
||||
+ container.Logger().WithError(err).Warn("fail to stop podSandbox type container kata-shim")
|
||||
+ }
|
||||
+ }
|
||||
+
|
||||
+ return nil
|
||||
+}
|
||||
diff --git a/virtcontainers/clh.go b/virtcontainers/clh.go
|
||||
index d40b698b..59510b02 100644
|
||||
--- a/virtcontainers/clh.go
|
||||
+++ b/virtcontainers/clh.go
|
||||
@@ -569,7 +569,7 @@ func (clh *cloudHypervisor) resumeSandbox() error {
|
||||
}
|
||||
|
||||
// stopSandbox will stop the Sandbox's VM.
|
||||
-func (clh *cloudHypervisor) stopSandbox() (err error) {
|
||||
+func (clh *cloudHypervisor) stopSandbox(force bool) (err error) {
|
||||
span, _ := clh.trace("stopSandbox")
|
||||
defer span.Finish()
|
||||
clh.Logger().WithField("function", "stopSandbox").Info("Stop Sandbox")
|
||||
diff --git a/virtcontainers/container.go b/virtcontainers/container.go
|
||||
index b42cc6e9..9485e708 100644
|
||||
--- a/virtcontainers/container.go
|
||||
+++ b/virtcontainers/container.go
|
||||
@@ -17,8 +17,12 @@ import (
|
||||
"time"
|
||||
|
||||
"github.com/containerd/cgroups"
|
||||
+ "github.com/kata-containers/runtime/virtcontainers/device/config"
|
||||
+ "github.com/kata-containers/runtime/virtcontainers/device/manager"
|
||||
vccgroups "github.com/kata-containers/runtime/virtcontainers/pkg/cgroups"
|
||||
+ "github.com/kata-containers/runtime/virtcontainers/pkg/rootless"
|
||||
vcTypes "github.com/kata-containers/runtime/virtcontainers/pkg/types"
|
||||
+ "github.com/kata-containers/runtime/virtcontainers/store"
|
||||
"github.com/kata-containers/runtime/virtcontainers/types"
|
||||
"github.com/kata-containers/runtime/virtcontainers/utils"
|
||||
specs "github.com/opencontainers/runtime-spec/specs-go"
|
||||
@@ -26,11 +30,6 @@ import (
|
||||
"github.com/pkg/errors"
|
||||
"github.com/sirupsen/logrus"
|
||||
"golang.org/x/sys/unix"
|
||||
-
|
||||
- "github.com/kata-containers/runtime/virtcontainers/device/config"
|
||||
- "github.com/kata-containers/runtime/virtcontainers/device/manager"
|
||||
- "github.com/kata-containers/runtime/virtcontainers/pkg/rootless"
|
||||
- "github.com/kata-containers/runtime/virtcontainers/store"
|
||||
)
|
||||
|
||||
// https://github.com/torvalds/linux/blob/master/include/uapi/linux/major.h
|
||||
@@ -1047,6 +1046,13 @@ func (c *Container) stop(force bool) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
+ // If container state is unhealthy, just force kill the container
|
||||
+ if c.state.State == types.StateUnhealthy {
|
||||
+ c.forceKillContainer()
|
||||
+ // after force kill container, then change container state to stopped
|
||||
+ return c.setContainerState(types.StateStopped)
|
||||
+ }
|
||||
+
|
||||
if err := c.state.ValidTransition(c.state.State, types.StateStopped); err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -1063,6 +1069,8 @@ func (c *Container) stop(force bool) error {
|
||||
if err := stopShim(c.process.Pid); err != nil {
|
||||
l.WithError(err).Warn("failed to stop shim")
|
||||
}
|
||||
+
|
||||
+ c.forceKillContainer()
|
||||
}
|
||||
|
||||
}()
|
||||
@@ -1096,7 +1104,9 @@ func (c *Container) stop(force bool) error {
|
||||
// this signal will ensure the container will get killed to match
|
||||
// the state of the shim. This will allow the following call to
|
||||
// stopContainer() to succeed in such particular case.
|
||||
- c.kill(syscall.SIGKILL, true)
|
||||
+ if err := c.kill(syscall.SIGKILL, true); err != nil {
|
||||
+ c.Logger().Errorf("send signal to container failed: %v", err)
|
||||
+ }
|
||||
|
||||
// Since the agent has supported the MultiWaitProcess, it's better to
|
||||
// wait the process here to make sure the process has exited before to
|
||||
@@ -1582,3 +1592,36 @@ func (c *Container) cgroupsUpdate(resources specs.LinuxResources) error {
|
||||
|
||||
return nil
|
||||
}
|
||||
+
|
||||
+// forceDeleteContainer force clean container mount info and resources stored in the disk
|
||||
+func (c *Container) forceDeleteContainer() {
|
||||
+ if err := c.unmountHostMounts(); err != nil {
|
||||
+ c.Logger().WithError(err).Warn("container force delete umount host mounts fail")
|
||||
+ }
|
||||
+
|
||||
+ if err := c.sandbox.removeContainer(c.id); err != nil {
|
||||
+ c.Logger().WithError(err).Warn("sandbox removeContainer fail")
|
||||
+ }
|
||||
+
|
||||
+ if err := c.store.Delete(); err != nil {
|
||||
+ c.Logger().WithError(err).Warn("force delete container store fail")
|
||||
+ }
|
||||
+}
|
||||
+
|
||||
+func (c *Container) forceKillContainer() {
|
||||
+ if err := c.setContainerState(types.StateStopped); err != nil {
|
||||
+ c.Logger().WithError(err).Warn("force kill container: change container state to StateStopped failed")
|
||||
+ }
|
||||
+
|
||||
+ if err := c.unmountHostMounts(); err != nil {
|
||||
+ c.Logger().WithError(err).Warn("force kill container: umount container host mounts failed")
|
||||
+ }
|
||||
+
|
||||
+ if err := c.detachDevices(); err != nil {
|
||||
+ c.Logger().WithError(err).Warn("force kill container: detach container devices failed")
|
||||
+ }
|
||||
+
|
||||
+ if err := c.removeDrive(); err != nil {
|
||||
+ c.Logger().WithError(err).Warn("force kill container: remove container drive failed")
|
||||
+ }
|
||||
+}
|
||||
diff --git a/virtcontainers/fc.go b/virtcontainers/fc.go
|
||||
index 97ef5ffc..72a8e192 100644
|
||||
--- a/virtcontainers/fc.go
|
||||
+++ b/virtcontainers/fc.go
|
||||
@@ -864,7 +864,7 @@ func (fc *firecracker) cleanupJail() {
|
||||
}
|
||||
|
||||
// stopSandbox will stop the Sandbox's VM.
|
||||
-func (fc *firecracker) stopSandbox() (err error) {
|
||||
+func (fc *firecracker) stopSandbox(force bool) (err error) {
|
||||
span, _ := fc.trace("stopSandbox")
|
||||
defer span.Finish()
|
||||
|
||||
diff --git a/virtcontainers/hypervisor.go b/virtcontainers/hypervisor.go
|
||||
index 4b3dd3d0..fd7d1f8e 100644
|
||||
--- a/virtcontainers/hypervisor.go
|
||||
+++ b/virtcontainers/hypervisor.go
|
||||
@@ -766,7 +766,7 @@ func generateVMSocket(id string, useVsock bool, vmStogarePath string) (interface
|
||||
type hypervisor interface {
|
||||
createSandbox(ctx context.Context, id string, networkNS NetworkNamespace, hypervisorConfig *HypervisorConfig, stateful bool) error
|
||||
startSandbox(timeout int) error
|
||||
- stopSandbox() error
|
||||
+ stopSandbox(force bool) error
|
||||
pauseSandbox() error
|
||||
saveSandbox() error
|
||||
resumeSandbox() error
|
||||
diff --git a/virtcontainers/kata_agent.go b/virtcontainers/kata_agent.go
|
||||
index be5e96aa..7575d326 100644
|
||||
--- a/virtcontainers/kata_agent.go
|
||||
+++ b/virtcontainers/kata_agent.go
|
||||
@@ -57,8 +57,9 @@ const (
|
||||
)
|
||||
|
||||
var (
|
||||
- checkRequestTimeout = 30 * time.Second
|
||||
- defaultRequestTimeout = 60 * time.Second
|
||||
+ checkRequestTimeout = 10 * time.Second
|
||||
+ defaultRequestTimeout = 10 * time.Second
|
||||
+ createContainerTimeout = 120 * time.Second
|
||||
errorMissingProxy = errors.New("Missing proxy pointer")
|
||||
errorMissingOCISpec = errors.New("Missing OCI specification")
|
||||
defaultKataHostSharedDir = "/run/kata-containers/shared/sandboxes/"
|
||||
@@ -987,17 +988,21 @@ func (k *kataAgent) stopSandbox(sandbox *Sandbox) error {
|
||||
k.state.URL = ""
|
||||
}()
|
||||
|
||||
- req := &grpc.DestroySandboxRequest{}
|
||||
+ // If sandbox.state.State is unhealthy, we don't need to send DestroySandboxRequest
|
||||
+ // to kata-agent, just force stop the sandbox
|
||||
+ if sandbox.state.State != types.StateUnhealthy {
|
||||
+ req := &grpc.DestroySandboxRequest{}
|
||||
|
||||
- if _, err := k.sendReq(req); err != nil {
|
||||
- return err
|
||||
- }
|
||||
-
|
||||
- if k.dynamicTracing {
|
||||
- _, err := k.sendReq(&grpc.StopTracingRequest{})
|
||||
- if err != nil {
|
||||
+ if _, err := k.sendReq(req); err != nil {
|
||||
return err
|
||||
}
|
||||
+
|
||||
+ if k.dynamicTracing {
|
||||
+ _, err := k.sendReq(&grpc.StopTracingRequest{})
|
||||
+ if err != nil {
|
||||
+ return err
|
||||
+ }
|
||||
+ }
|
||||
}
|
||||
|
||||
return nil
|
||||
@@ -2062,6 +2067,8 @@ func (k *kataAgent) getReqContext(reqName string) (ctx context.Context, cancel c
|
||||
// Wait has no timeout
|
||||
case grpcCheckRequest:
|
||||
ctx, cancel = context.WithTimeout(ctx, checkRequestTimeout)
|
||||
+ case grpcCreateContainerRequest:
|
||||
+ ctx, cancel = context.WithTimeout(ctx, createContainerTimeout)
|
||||
default:
|
||||
ctx, cancel = context.WithTimeout(ctx, defaultRequestTimeout)
|
||||
}
|
||||
@@ -2382,3 +2389,7 @@ func (k *kataAgent) load(s persistapi.AgentState) {
|
||||
k.state.ProxyPid = s.ProxyPid
|
||||
k.state.URL = s.URL
|
||||
}
|
||||
+
|
||||
+func (k *kataAgent) getProxyPid() int {
|
||||
+ return k.state.ProxyPid
|
||||
+}
|
||||
diff --git a/virtcontainers/mock_hypervisor.go b/virtcontainers/mock_hypervisor.go
|
||||
index 0c84e43c..a5b67491 100644
|
||||
--- a/virtcontainers/mock_hypervisor.go
|
||||
+++ b/virtcontainers/mock_hypervisor.go
|
||||
@@ -39,7 +39,7 @@ func (m *mockHypervisor) startSandbox(timeout int) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
-func (m *mockHypervisor) stopSandbox() error {
|
||||
+func (m *mockHypervisor) stopSandbox(force bool) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
diff --git a/virtcontainers/mock_hypervisor_test.go b/virtcontainers/mock_hypervisor_test.go
|
||||
index b73b28f2..827e3192 100644
|
||||
--- a/virtcontainers/mock_hypervisor_test.go
|
||||
+++ b/virtcontainers/mock_hypervisor_test.go
|
||||
@@ -53,7 +53,7 @@ func TestMockHypervisorStartSandbox(t *testing.T) {
|
||||
func TestMockHypervisorStopSandbox(t *testing.T) {
|
||||
var m *mockHypervisor
|
||||
|
||||
- assert.NoError(t, m.stopSandbox())
|
||||
+ assert.NoError(t, m.stopSandbox(false))
|
||||
}
|
||||
|
||||
func TestMockHypervisorAddDevice(t *testing.T) {
|
||||
diff --git a/virtcontainers/noop_agent.go b/virtcontainers/noop_agent.go
|
||||
index 189f6b3f..8a7cd337 100644
|
||||
--- a/virtcontainers/noop_agent.go
|
||||
+++ b/virtcontainers/noop_agent.go
|
||||
@@ -236,3 +236,7 @@ func (n *noopAgent) save() (s persistapi.AgentState) {
|
||||
|
||||
// load is the Noop agent state loader. It does nothing.
|
||||
func (n *noopAgent) load(s persistapi.AgentState) {}
|
||||
+
|
||||
+func (n *noopAgent) getProxyPid() int {
|
||||
+ return -1
|
||||
+}
|
||||
\ No newline at end of file
|
||||
diff --git a/virtcontainers/pkg/oci/utils.go b/virtcontainers/pkg/oci/utils.go
|
||||
index 5348c57d..cd8d48ce 100644
|
||||
--- a/virtcontainers/pkg/oci/utils.go
|
||||
+++ b/virtcontainers/pkg/oci/utils.go
|
||||
@@ -70,6 +70,9 @@ const (
|
||||
|
||||
// StatePaused represents a container that has been paused.
|
||||
StatePaused = "paused"
|
||||
+
|
||||
+ // StateUnhealthy represents a container that is unhealthy
|
||||
+ StateUnhealthy = "unhealthy"
|
||||
)
|
||||
|
||||
const KernelModulesSeparator = ";"
|
||||
@@ -964,6 +967,8 @@ func StateToOCIState(state types.StateString) string {
|
||||
return StateStopped
|
||||
case types.StatePaused:
|
||||
return StatePaused
|
||||
+ case types.StateUnhealthy:
|
||||
+ return StateUnhealthy
|
||||
default:
|
||||
return ""
|
||||
}
|
||||
diff --git a/virtcontainers/qemu.go b/virtcontainers/qemu.go
|
||||
index ca286550..4b15d968 100644
|
||||
--- a/virtcontainers/qemu.go
|
||||
+++ b/virtcontainers/qemu.go
|
||||
@@ -687,7 +687,7 @@ func (q *qemu) setupVirtiofsd() (err error) {
|
||||
q.Logger().Info("virtiofsd quits")
|
||||
// Wait to release resources of virtiofsd process
|
||||
cmd.Process.Wait()
|
||||
- q.stopSandbox()
|
||||
+ q.stopSandbox(false)
|
||||
}()
|
||||
return err
|
||||
}
|
||||
@@ -922,11 +922,11 @@ func (q *qemu) waitSandbox(timeout int) error {
|
||||
}
|
||||
|
||||
// stopSandbox will stop the Sandbox's VM.
|
||||
-func (q *qemu) stopSandbox() error {
|
||||
+func (q *qemu) stopSandbox(force bool) error {
|
||||
span, _ := q.trace("stopSandbox")
|
||||
defer span.Finish()
|
||||
|
||||
- q.Logger().Info("Stopping Sandbox")
|
||||
+ q.Logger().Infof("force stopping Sandbox: %v", force)
|
||||
if q.stopped {
|
||||
q.Logger().Info("Already stopped")
|
||||
return nil
|
||||
@@ -937,28 +937,37 @@ func (q *qemu) stopSandbox() error {
|
||||
q.stopped = true
|
||||
}()
|
||||
|
||||
- if q.config.Debug && q.qemuConfig.LogFile != "" {
|
||||
- f, err := os.OpenFile(q.qemuConfig.LogFile, os.O_RDONLY, 0)
|
||||
- if err == nil {
|
||||
- scanner := bufio.NewScanner(f)
|
||||
- for scanner.Scan() {
|
||||
- q.Logger().Debug(scanner.Text())
|
||||
- }
|
||||
- if err := scanner.Err(); err != nil {
|
||||
- q.Logger().WithError(err).Debug("read qemu log failed")
|
||||
+ if !force {
|
||||
+ if q.config.Debug && q.qemuConfig.LogFile != "" {
|
||||
+ f, err := os.OpenFile(q.qemuConfig.LogFile, os.O_RDONLY, 0)
|
||||
+ if err == nil {
|
||||
+ scanner := bufio.NewScanner(f)
|
||||
+ for scanner.Scan() {
|
||||
+ q.Logger().Debug(scanner.Text())
|
||||
+ }
|
||||
+ if err := scanner.Err(); err != nil {
|
||||
+ q.Logger().WithError(err).Debug("read qemu log failed")
|
||||
+ }
|
||||
}
|
||||
}
|
||||
- }
|
||||
|
||||
- err := q.qmpSetup()
|
||||
- if err != nil {
|
||||
- return err
|
||||
- }
|
||||
+ err := q.qmpSetup()
|
||||
+ if err != nil {
|
||||
+ return err
|
||||
+ }
|
||||
|
||||
- err = q.qmpMonitorCh.qmp.ExecuteQuit(q.qmpMonitorCh.ctx)
|
||||
- if err != nil {
|
||||
- q.Logger().WithError(err).Error("Fail to execute qmp QUIT")
|
||||
- return err
|
||||
+ err = q.qmpMonitorCh.qmp.ExecuteQuit(q.qmpMonitorCh.ctx)
|
||||
+ if err != nil {
|
||||
+ q.Logger().WithError(err).Error("Fail to execute qmp QUIT")
|
||||
+ return err
|
||||
+ }
|
||||
+ } else {
|
||||
+ qemuMainPid := q.getPids()[0]
|
||||
+ if qemuMainPid <= 1 {
|
||||
+ return fmt.Errorf("force kill qemu process pid is invalid")
|
||||
+ }
|
||||
+
|
||||
+ _ = syscall.Kill(qemuMainPid, syscall.SIGKILL)
|
||||
}
|
||||
|
||||
return nil
|
||||
diff --git a/virtcontainers/sandbox.go b/virtcontainers/sandbox.go
|
||||
index edd1af5b..78188ed7 100644
|
||||
--- a/virtcontainers/sandbox.go
|
||||
+++ b/virtcontainers/sandbox.go
|
||||
@@ -12,19 +12,13 @@ import (
|
||||
"math"
|
||||
"net"
|
||||
"os"
|
||||
+ "path/filepath"
|
||||
"strings"
|
||||
"sync"
|
||||
"syscall"
|
||||
|
||||
"github.com/containerd/cgroups"
|
||||
"github.com/containernetworking/plugins/pkg/ns"
|
||||
- "github.com/opencontainers/runc/libcontainer/configs"
|
||||
- specs "github.com/opencontainers/runtime-spec/specs-go"
|
||||
- opentracing "github.com/opentracing/opentracing-go"
|
||||
- "github.com/pkg/errors"
|
||||
- "github.com/sirupsen/logrus"
|
||||
- "github.com/vishvananda/netlink"
|
||||
-
|
||||
"github.com/kata-containers/agent/protocols/grpc"
|
||||
"github.com/kata-containers/runtime/virtcontainers/device/api"
|
||||
"github.com/kata-containers/runtime/virtcontainers/device/config"
|
||||
@@ -41,6 +35,12 @@ import (
|
||||
"github.com/kata-containers/runtime/virtcontainers/store"
|
||||
"github.com/kata-containers/runtime/virtcontainers/types"
|
||||
"github.com/kata-containers/runtime/virtcontainers/utils"
|
||||
+ "github.com/opencontainers/runc/libcontainer/configs"
|
||||
+ specs "github.com/opencontainers/runtime-spec/specs-go"
|
||||
+ opentracing "github.com/opentracing/opentracing-go"
|
||||
+ "github.com/pkg/errors"
|
||||
+ "github.com/sirupsen/logrus"
|
||||
+ "github.com/vishvananda/netlink"
|
||||
)
|
||||
|
||||
const (
|
||||
@@ -50,6 +50,9 @@ const (
|
||||
|
||||
// DirMode is the permission bits used for creating a directory
|
||||
DirMode = os.FileMode(0750) | os.ModeDir
|
||||
+
|
||||
+ // kata-proxy proces name
|
||||
+ KataProxyProcessName = "kata-proxy"
|
||||
)
|
||||
|
||||
// SandboxStatus describes a sandbox status.
|
||||
@@ -1037,7 +1040,7 @@ func (s *Sandbox) startVM() (err error) {
|
||||
|
||||
defer func() {
|
||||
if err != nil {
|
||||
- s.hypervisor.stopSandbox()
|
||||
+ s.hypervisor.stopSandbox(false)
|
||||
}
|
||||
}()
|
||||
|
||||
@@ -1090,7 +1093,12 @@ func (s *Sandbox) stopVM() error {
|
||||
}
|
||||
|
||||
s.Logger().Info("Stopping VM")
|
||||
- return s.hypervisor.stopSandbox()
|
||||
+ forceStop := false
|
||||
+ if s.state.State == types.StateUnhealthy {
|
||||
+ forceStop = true
|
||||
+ }
|
||||
+
|
||||
+ return s.hypervisor.stopSandbox(forceStop)
|
||||
}
|
||||
|
||||
func (s *Sandbox) addContainer(c *Container) error {
|
||||
@@ -1591,13 +1599,15 @@ func (s *Sandbox) setSandboxState(state types.StateString) error {
|
||||
return vcTypes.ErrNeedState
|
||||
}
|
||||
|
||||
+ s.Logger().Debugf("Setting sandbox state from %v to %v", s.state.State, state)
|
||||
// update in-memory state
|
||||
s.state.State = state
|
||||
|
||||
if useOldStore(s.ctx) {
|
||||
return s.store.Store(store.State, s.state)
|
||||
+ } else {
|
||||
+ return s.Save()
|
||||
}
|
||||
- return nil
|
||||
}
|
||||
|
||||
const maxBlockIndex = 65535
|
||||
@@ -2207,3 +2217,79 @@ func (s *Sandbox) GetPatchedOCISpec() *specs.Spec {
|
||||
|
||||
return nil
|
||||
}
|
||||
+
|
||||
+// health return current sandbox healthy or not
|
||||
+// If qemu/kata-proxy/kata-agent process is abnormal,
|
||||
+// s.agent.check() will return false
|
||||
+func (s *Sandbox) health() bool {
|
||||
+ err := s.agent.check()
|
||||
+ if err != nil {
|
||||
+ return false
|
||||
+ }
|
||||
+
|
||||
+ return true
|
||||
+}
|
||||
+
|
||||
+// shouldForceDelete force delete the sandbox when kata-proxy and hypervisor process exit
|
||||
+// already and current process is kata-runtime kill or kata-runtime delete
|
||||
+func (s *Sandbox) shouldForceDelete() bool {
|
||||
+ cmdline, err := utils.GetProcessCmdline(os.Getpid())
|
||||
+ if err != nil {
|
||||
+ s.Logger().Errorf("fail to get process cmdline: %v", err)
|
||||
+ return false
|
||||
+ }
|
||||
+
|
||||
+ proxyPid := s.agent.getProxyPid()
|
||||
+ hypervisorPids := s.hypervisor.getPids()
|
||||
+ if len(hypervisorPids) <= 0 {
|
||||
+ s.Logger().Warnf("get hypervisor main pid fail")
|
||||
+ return false
|
||||
+ }
|
||||
+ hypervisorMainPid := hypervisorPids[0]
|
||||
+ hypervisorPath := s.hypervisor.hypervisorConfig().HypervisorPath
|
||||
+ hypervisorName := filepath.Base(hypervisorPath)
|
||||
+
|
||||
+ if !utils.IsProcessRunning(proxyPid, KataProxyProcessName, s.id) && !utils.IsProcessRunning(hypervisorMainPid, hypervisorName, s.id) &&
|
||||
+ strings.Contains(cmdline, "delete") && strings.Contains(cmdline, "force") {
|
||||
+ return true
|
||||
+ }
|
||||
+
|
||||
+ return false
|
||||
+}
|
||||
+
|
||||
+func (s *Sandbox) forceDeleteSandbox() {
|
||||
+ for _, c := range s.containers {
|
||||
+ // force delete all containers in the sandbox
|
||||
+ c.forceDeleteContainer()
|
||||
+ }
|
||||
+
|
||||
+ globalSandboxList.removeSandbox(s.id)
|
||||
+
|
||||
+ if s.monitor != nil {
|
||||
+ s.monitor.stop()
|
||||
+ }
|
||||
+
|
||||
+ if err := s.hypervisor.cleanup(); err != nil {
|
||||
+ s.Logger().WithError(err).Error("failed to force cleanup hypervisor resource")
|
||||
+ }
|
||||
+
|
||||
+ s.agent.cleanup(s)
|
||||
+
|
||||
+ if err := s.store.Delete(); err != nil {
|
||||
+ s.Logger().WithError(err).Warn("sandbox force delete store failed")
|
||||
+ }
|
||||
+}
|
||||
+
|
||||
+func (s *Sandbox) setContainersState(state types.StateString) error {
|
||||
+ if state == "" {
|
||||
+ return vcTypes.ErrNeedState
|
||||
+ }
|
||||
+
|
||||
+ for _, c := range s.containers {
|
||||
+ if err := c.setContainerState(state); err != nil {
|
||||
+ return err
|
||||
+ }
|
||||
+ }
|
||||
+
|
||||
+ return nil
|
||||
+}
|
||||
diff --git a/virtcontainers/types/sandbox.go b/virtcontainers/types/sandbox.go
|
||||
index 3b64b20a..5d586b21 100644
|
||||
--- a/virtcontainers/types/sandbox.go
|
||||
+++ b/virtcontainers/types/sandbox.go
|
||||
@@ -28,6 +28,9 @@ const (
|
||||
|
||||
// StateStopped represents a sandbox/container that has been stopped.
|
||||
StateStopped StateString = "stopped"
|
||||
+
|
||||
+ // StateUnhealthy represents a sandbox/container that's in abnormal state.
|
||||
+ StateUnhealthy StateString = "unhealthy"
|
||||
)
|
||||
|
||||
const (
|
||||
@@ -90,17 +93,17 @@ func (state *StateString) validTransition(oldState StateString, newState StateSt
|
||||
|
||||
switch *state {
|
||||
case StateReady:
|
||||
- if newState == StateRunning || newState == StateStopped {
|
||||
+ if newState == StateRunning || newState == StateStopped || newState == StateUnhealthy {
|
||||
return nil
|
||||
}
|
||||
|
||||
case StateRunning:
|
||||
- if newState == StatePaused || newState == StateStopped {
|
||||
+ if newState == StatePaused || newState == StateStopped || newState == StateUnhealthy {
|
||||
return nil
|
||||
}
|
||||
|
||||
case StatePaused:
|
||||
- if newState == StateRunning || newState == StateStopped {
|
||||
+ if newState == StateRunning || newState == StateStopped || newState == StateUnhealthy {
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -108,6 +111,11 @@ func (state *StateString) validTransition(oldState StateString, newState StateSt
|
||||
if newState == StateRunning {
|
||||
return nil
|
||||
}
|
||||
+
|
||||
+ case StateUnhealthy:
|
||||
+ if newState == StateStopped {
|
||||
+ return nil
|
||||
+ }
|
||||
}
|
||||
|
||||
return fmt.Errorf("Can not move from %v to %v",
|
||||
diff --git a/virtcontainers/utils/utils.go b/virtcontainers/utils/utils.go
|
||||
index 85c55489..2b555ebb 100644
|
||||
--- a/virtcontainers/utils/utils.go
|
||||
+++ b/virtcontainers/utils/utils.go
|
||||
@@ -9,9 +9,13 @@ import (
|
||||
"crypto/rand"
|
||||
"errors"
|
||||
"fmt"
|
||||
+ "io/ioutil"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
+ "strconv"
|
||||
+ "strings"
|
||||
+ "syscall"
|
||||
)
|
||||
|
||||
const cpBinaryName = "cp"
|
||||
@@ -275,3 +279,45 @@ const (
|
||||
MiB = KiB << 10
|
||||
GiB = MiB << 10
|
||||
)
|
||||
+
|
||||
+// Get process cmdline info by read /proc/<pid>/cmdline file
|
||||
+func GetProcessCmdline(pid int) (cmdline string, err error) {
|
||||
+ if pid <= 1 {
|
||||
+ return "", fmt.Errorf("invalid pid number")
|
||||
+ }
|
||||
+
|
||||
+ bytes, err := ioutil.ReadFile(filepath.Join("/proc", strconv.Itoa(pid), "cmdline"))
|
||||
+ if err != nil {
|
||||
+ return "", err
|
||||
+ }
|
||||
+
|
||||
+ return string(bytes), nil
|
||||
+}
|
||||
+
|
||||
+func IsProcessRunning(pid int, processName string, sandboxID string) bool {
|
||||
+ if pid <= 0 {
|
||||
+ return false
|
||||
+ }
|
||||
+
|
||||
+ process, err := os.FindProcess(pid)
|
||||
+ if err != nil {
|
||||
+ return false
|
||||
+ }
|
||||
+
|
||||
+ if err := process.Signal(syscall.Signal(0)); err != nil {
|
||||
+ return false
|
||||
+ }
|
||||
+
|
||||
+ cmdline, err := GetProcessCmdline(pid)
|
||||
+ if err != nil {
|
||||
+ return false
|
||||
+ }
|
||||
+
|
||||
+ // If process's cmdline contains processName and sandboxID keyword,
|
||||
+ // We think this process isn't be reused
|
||||
+ if strings.Contains(cmdline, processName) && strings.Contains(cmdline, sandboxID) {
|
||||
+ return true
|
||||
+ }
|
||||
+
|
||||
+ return false
|
||||
+}
|
||||
diff --git a/virtcontainers/vm.go b/virtcontainers/vm.go
|
||||
index fcda1e97..8d27b1fe 100644
|
||||
--- a/virtcontainers/vm.go
|
||||
+++ b/virtcontainers/vm.go
|
||||
@@ -191,7 +191,7 @@ func NewVM(ctx context.Context, config VMConfig) (*VM, error) {
|
||||
defer func() {
|
||||
if err != nil {
|
||||
virtLog.WithField("vm", id).WithError(err).Info("clean up vm")
|
||||
- hypervisor.stopSandbox()
|
||||
+ hypervisor.stopSandbox(false)
|
||||
}
|
||||
}()
|
||||
|
||||
@@ -333,7 +333,7 @@ func (v *VM) Disconnect() error {
|
||||
func (v *VM) Stop() error {
|
||||
v.logger().Info("stop vm")
|
||||
|
||||
- if err := v.hypervisor.stopSandbox(); err != nil {
|
||||
+ if err := v.hypervisor.stopSandbox(false); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
--
|
||||
2.14.3 (Apple Git-98)
|
||||
|
||||
@ -1,177 +0,0 @@
|
||||
From a1bf2e1c696b703935f4b81ca087a60cc2559464 Mon Sep 17 00:00:00 2001
|
||||
From: jiangpengfei <jiangpengfei9@huawei.com>
|
||||
Date: Mon, 27 Jul 2020 21:45:25 +0800
|
||||
Subject: [PATCH 08/50] kata-runtime: fix kata-runtime resource left problem
|
||||
|
||||
reason: fix the following resource left problem
|
||||
- sandbox stored files and directory are deleted before work container delete
|
||||
- ignore StatusSandbox error, let stopSandbox function call be executed
|
||||
|
||||
Signed-off-by: jiangpengfei <jiangpengfei9@huawei.com>
|
||||
---
|
||||
cli/delete.go | 25 +++++++++++++++++++++++--
|
||||
cli/delete_test.go | 15 +++++++++++++++
|
||||
cli/state.go | 6 ++++++
|
||||
virtcontainers/api.go | 4 +++-
|
||||
4 files changed, 47 insertions(+), 3 deletions(-)
|
||||
|
||||
diff --git a/cli/delete.go b/cli/delete.go
|
||||
index 2f5586e5..871ac40d 100644
|
||||
--- a/cli/delete.go
|
||||
+++ b/cli/delete.go
|
||||
@@ -10,6 +10,7 @@ import (
|
||||
"context"
|
||||
"fmt"
|
||||
"os"
|
||||
+ "strings"
|
||||
|
||||
"github.com/kata-containers/runtime/pkg/katautils"
|
||||
vc "github.com/kata-containers/runtime/virtcontainers"
|
||||
@@ -68,6 +69,12 @@ func delete(ctx context.Context, containerID string, force bool) error {
|
||||
setExternalLoggers(ctx, kataLog)
|
||||
span.SetTag("container", containerID)
|
||||
|
||||
+ // Remove the containerID to sandboxID mapping
|
||||
+ // no matter what error return
|
||||
+ defer func() {
|
||||
+ _ = katautils.DelContainerIDMapping(ctx, containerID)
|
||||
+ }()
|
||||
+
|
||||
// Checks the MUST and MUST NOT from OCI runtime specification
|
||||
status, sandboxID, err := getExistingContainerInfo(ctx, containerID)
|
||||
if err != nil {
|
||||
@@ -75,6 +82,15 @@ func delete(ctx context.Context, containerID string, force bool) error {
|
||||
kataLog.Warnf("Failed to get container, force will not fail: %s", err)
|
||||
return nil
|
||||
}
|
||||
+
|
||||
+ // If err info containers "no such file or directory, because pod_sandbox type
|
||||
+ // container is deleted before pod_container type container, just return nil
|
||||
+ // and let containerd delete container operations continue
|
||||
+ if strings.Contains(err.Error(), "no such file or directory") {
|
||||
+ kataLog.Warnf("pod_sandbox deleted before pod_container: %v", err)
|
||||
+ return nil
|
||||
+ }
|
||||
+
|
||||
return err
|
||||
}
|
||||
|
||||
@@ -123,7 +139,12 @@ func delete(ctx context.Context, containerID string, force bool) error {
|
||||
}
|
||||
case vc.PodContainer:
|
||||
if err := deleteContainer(ctx, sandboxID, containerID, forceStop); err != nil {
|
||||
- return err
|
||||
+ // If err info containers "no such file or directory, because pod_sandbox type
|
||||
+ // container is deleted before pod_container type container, just return nil
|
||||
+ // and let containerd delete container operations continue
|
||||
+ if !strings.Contains(err.Error(), "no such file or directory") {
|
||||
+ return err
|
||||
+ }
|
||||
}
|
||||
default:
|
||||
return fmt.Errorf("Invalid container type found")
|
||||
@@ -134,7 +155,7 @@ func delete(ctx context.Context, containerID string, force bool) error {
|
||||
return err
|
||||
}
|
||||
|
||||
- return katautils.DelContainerIDMapping(ctx, containerID)
|
||||
+ return nil
|
||||
}
|
||||
|
||||
func deleteSandbox(ctx context.Context, sandboxID string, force bool) error {
|
||||
diff --git a/cli/delete_test.go b/cli/delete_test.go
|
||||
index a2455dee..ae421cd7 100644
|
||||
--- a/cli/delete_test.go
|
||||
+++ b/cli/delete_test.go
|
||||
@@ -184,6 +184,9 @@ func TestDeleteSandbox(t *testing.T) {
|
||||
assert.Error(err)
|
||||
assert.True(vcmock.IsMockError(err))
|
||||
|
||||
+ path, err = createTempContainerIDMapping(sandbox.ID(), sandbox.ID())
|
||||
+ assert.NoError(err)
|
||||
+
|
||||
testingImpl.StatusSandboxFunc = func(ctx context.Context, sandboxID string) (vc.SandboxStatus, error) {
|
||||
return vc.SandboxStatus{
|
||||
ID: sandbox.ID(),
|
||||
@@ -201,6 +204,9 @@ func TestDeleteSandbox(t *testing.T) {
|
||||
assert.Error(err)
|
||||
assert.True(vcmock.IsMockError(err))
|
||||
|
||||
+ path, err = createTempContainerIDMapping(sandbox.ID(), sandbox.ID())
|
||||
+ assert.NoError(err)
|
||||
+
|
||||
testingImpl.StopSandboxFunc = func(ctx context.Context, sandboxID string, force bool) (vc.VCSandbox, error) {
|
||||
return sandbox, nil
|
||||
}
|
||||
@@ -213,6 +219,9 @@ func TestDeleteSandbox(t *testing.T) {
|
||||
assert.Error(err)
|
||||
assert.True(vcmock.IsMockError(err))
|
||||
|
||||
+ path, err = createTempContainerIDMapping(sandbox.ID(), sandbox.ID())
|
||||
+ assert.NoError(err)
|
||||
+
|
||||
testingImpl.DeleteSandboxFunc = func(ctx context.Context, sandboxID string) (vc.VCSandbox, error) {
|
||||
return sandbox, nil
|
||||
}
|
||||
@@ -302,6 +311,9 @@ func TestDeleteSandboxRunning(t *testing.T) {
|
||||
assert.Error(err)
|
||||
assert.False(vcmock.IsMockError(err))
|
||||
|
||||
+ path, err = createTempContainerIDMapping(sandbox.ID(), sandbox.ID())
|
||||
+ assert.NoError(err)
|
||||
+
|
||||
testingImpl.StatusSandboxFunc = func(ctx context.Context, sandboxID string) (vc.SandboxStatus, error) {
|
||||
return vc.SandboxStatus{
|
||||
ID: sandbox.ID(),
|
||||
@@ -325,6 +337,9 @@ func TestDeleteSandboxRunning(t *testing.T) {
|
||||
assert.Error(err)
|
||||
assert.True(vcmock.IsMockError(err))
|
||||
|
||||
+ path, err = createTempContainerIDMapping(sandbox.ID(), sandbox.ID())
|
||||
+ assert.NoError(err)
|
||||
+
|
||||
testingImpl.DeleteSandboxFunc = func(ctx context.Context, sandboxID string) (vc.VCSandbox, error) {
|
||||
return sandbox, nil
|
||||
}
|
||||
diff --git a/cli/state.go b/cli/state.go
|
||||
index a2fcc12e..de843d34 100644
|
||||
--- a/cli/state.go
|
||||
+++ b/cli/state.go
|
||||
@@ -11,6 +11,7 @@ import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"os"
|
||||
+ "strings"
|
||||
|
||||
"github.com/kata-containers/runtime/pkg/katautils"
|
||||
"github.com/kata-containers/runtime/virtcontainers/pkg/oci"
|
||||
@@ -52,6 +53,11 @@ func state(ctx context.Context, containerID string) error {
|
||||
// Checks the MUST and MUST NOT from OCI runtime specification
|
||||
status, _, err := getExistingContainerInfo(ctx, containerID)
|
||||
if err != nil {
|
||||
+ // If err info containers "no such file or directory, because pod_sandbox type
|
||||
+ // container is deleted before pod_container type container, just return nil
|
||||
+ if strings.Contains(err.Error(), "no such file or directory") {
|
||||
+ return nil
|
||||
+ }
|
||||
return err
|
||||
}
|
||||
|
||||
diff --git a/virtcontainers/api.go b/virtcontainers/api.go
|
||||
index fa82d163..449a03e0 100644
|
||||
--- a/virtcontainers/api.go
|
||||
+++ b/virtcontainers/api.go
|
||||
@@ -374,7 +374,9 @@ func StatusSandbox(ctx context.Context, sandboxID string) (SandboxStatus, error)
|
||||
for _, container := range s.containers {
|
||||
contStatus, err := statusContainer(s, container.id)
|
||||
if err != nil {
|
||||
- return SandboxStatus{}, err
|
||||
+ // Since statusContainer may get error because of qemu process D or T,
|
||||
+ // So just ignore this error and let StatusSandbox function return actually SandboxStatus
|
||||
+ s.Logger().Warnf("Status container's status in sandbox get error: %v", contStatus)
|
||||
}
|
||||
|
||||
contStatusList = append(contStatusList, contStatus)
|
||||
--
|
||||
2.14.3 (Apple Git-98)
|
||||
|
||||
@ -1,132 +0,0 @@
|
||||
From 508fd9b94b1b12be3167342b03a47f8f97245e9c Mon Sep 17 00:00:00 2001
|
||||
From: jiangpengfei <jiangpengfei9@huawei.com>
|
||||
Date: Tue, 28 Jul 2020 10:53:30 +0800
|
||||
Subject: [PATCH 09/50] kata-runtime: add kata-runtime global flag --debug
|
||||
|
||||
reason: add the same debug flag with runc to adapt to
|
||||
containerd 1.2.0
|
||||
|
||||
Signed-off-by: jiangpengfei <jiangpengfei9@huawei.com>
|
||||
---
|
||||
cli/kata-env_test.go | 2 +-
|
||||
cli/main.go | 6 +++++-
|
||||
containerd-shim-v2/create.go | 2 +-
|
||||
pkg/katautils/config.go | 4 ++--
|
||||
pkg/katautils/config_test.go | 8 ++++----
|
||||
5 files changed, 13 insertions(+), 9 deletions(-)
|
||||
|
||||
diff --git a/cli/kata-env_test.go b/cli/kata-env_test.go
|
||||
index b31b6cc2..75bb697f 100644
|
||||
--- a/cli/kata-env_test.go
|
||||
+++ b/cli/kata-env_test.go
|
||||
@@ -178,7 +178,7 @@ func makeRuntimeConfig(prefixDir string) (configFile string, config oci.RuntimeC
|
||||
return "", oci.RuntimeConfig{}, err
|
||||
}
|
||||
|
||||
- _, config, err = katautils.LoadConfiguration(configFile, true, false)
|
||||
+ _, config, err = katautils.LoadConfiguration(configFile, true, false, false)
|
||||
if err != nil {
|
||||
return "", oci.RuntimeConfig{}, err
|
||||
}
|
||||
diff --git a/cli/main.go b/cli/main.go
|
||||
index df16c5f3..1362a8fb 100644
|
||||
--- a/cli/main.go
|
||||
+++ b/cli/main.go
|
||||
@@ -115,6 +115,10 @@ var runtimeFlags = []cli.Flag{
|
||||
Name: "systemd-cgroup",
|
||||
Usage: "enable systemd cgroup support, expects cgroupsPath to be of form \"slice:prefix:name\" for e.g. \"system.slice:runc:434234\"",
|
||||
},
|
||||
+ cli.BoolFlag{
|
||||
+ Name: "debug",
|
||||
+ Usage: "enable debug output for logging",
|
||||
+ },
|
||||
}
|
||||
|
||||
// runtimeCommands is the list of supported command-line (sub-)
|
||||
@@ -334,7 +338,7 @@ func beforeSubcommands(c *cli.Context) error {
|
||||
}
|
||||
}
|
||||
|
||||
- configFile, runtimeConfig, err = katautils.LoadConfiguration(c.GlobalString(configFilePathOption), ignoreConfigLogs, false)
|
||||
+ configFile, runtimeConfig, err = katautils.LoadConfiguration(c.GlobalString(configFilePathOption), ignoreConfigLogs, false, c.GlobalBool("debug"))
|
||||
if err != nil {
|
||||
fatal(err)
|
||||
}
|
||||
diff --git a/containerd-shim-v2/create.go b/containerd-shim-v2/create.go
|
||||
index affdbae2..9749073d 100644
|
||||
--- a/containerd-shim-v2/create.go
|
||||
+++ b/containerd-shim-v2/create.go
|
||||
@@ -167,7 +167,7 @@ func loadRuntimeConfig(s *service, r *taskAPI.CreateTaskRequest, anno map[string
|
||||
configPath = os.Getenv("KATA_CONF_FILE")
|
||||
}
|
||||
|
||||
- _, runtimeConfig, err := katautils.LoadConfiguration(configPath, false, true)
|
||||
+ _, runtimeConfig, err := katautils.LoadConfiguration(configPath, false, true, false)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
diff --git a/pkg/katautils/config.go b/pkg/katautils/config.go
|
||||
index 349e667f..448d23ac 100644
|
||||
--- a/pkg/katautils/config.go
|
||||
+++ b/pkg/katautils/config.go
|
||||
@@ -1141,7 +1141,7 @@ func initConfig() (config oci.RuntimeConfig, err error) {
|
||||
//
|
||||
// All paths are resolved fully meaning if this function does not return an
|
||||
// error, all paths are valid at the time of the call.
|
||||
-func LoadConfiguration(configPath string, ignoreLogging, builtIn bool) (resolvedConfigPath string, config oci.RuntimeConfig, err error) {
|
||||
+func LoadConfiguration(configPath string, ignoreLogging, builtIn bool, debugFlag bool) (resolvedConfigPath string, config oci.RuntimeConfig, err error) {
|
||||
|
||||
config, err = initConfig()
|
||||
if err != nil {
|
||||
@@ -1154,7 +1154,7 @@ func LoadConfiguration(configPath string, ignoreLogging, builtIn bool) (resolved
|
||||
}
|
||||
|
||||
config.Debug = tomlConf.Runtime.Debug
|
||||
- if !tomlConf.Runtime.Debug {
|
||||
+ if !tomlConf.Runtime.Debug && !debugFlag {
|
||||
// If debug is not required, switch back to the original
|
||||
// default log priority, otherwise continue in debug mode.
|
||||
kataUtilsLogger.Logger.Level = originalLoggerLevel
|
||||
diff --git a/pkg/katautils/config_test.go b/pkg/katautils/config_test.go
|
||||
index 2eae1f6a..31afcca6 100644
|
||||
--- a/pkg/katautils/config_test.go
|
||||
+++ b/pkg/katautils/config_test.go
|
||||
@@ -264,7 +264,7 @@ func testLoadConfiguration(t *testing.T, dir string,
|
||||
assert.NoError(t, err)
|
||||
}
|
||||
|
||||
- resolvedConfigPath, config, err := LoadConfiguration(file, ignoreLogging, false)
|
||||
+ resolvedConfigPath, config, err := LoadConfiguration(file, ignoreLogging, false, false)
|
||||
if expectFail {
|
||||
assert.Error(t, err)
|
||||
|
||||
@@ -578,7 +578,7 @@ func TestMinimalRuntimeConfig(t *testing.T) {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
- _, config, err := LoadConfiguration(configPath, false, false)
|
||||
+ _, config, err := LoadConfiguration(configPath, false, false, false)
|
||||
if err == nil {
|
||||
t.Fatalf("Expected loadConfiguration to fail as shim path does not exist: %+v", config)
|
||||
}
|
||||
@@ -608,7 +608,7 @@ func TestMinimalRuntimeConfig(t *testing.T) {
|
||||
t.Error(err)
|
||||
}
|
||||
|
||||
- _, config, err = LoadConfiguration(configPath, false, false)
|
||||
+ _, config, err = LoadConfiguration(configPath, false, false, false)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@@ -748,7 +748,7 @@ func TestMinimalRuntimeConfigWithVsock(t *testing.T) {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
- _, config, err := LoadConfiguration(configPath, false, false)
|
||||
+ _, config, err := LoadConfiguration(configPath, false, false, false)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
--
|
||||
2.14.3 (Apple Git-98)
|
||||
|
||||
@ -1,172 +0,0 @@
|
||||
From 76cbca91608e94c1855705ad1a8d06ffa2273115 Mon Sep 17 00:00:00 2001
|
||||
From: jiangpengfei <jiangpengfei9@huawei.com>
|
||||
Date: Tue, 28 Jul 2020 18:18:54 +0800
|
||||
Subject: [PATCH 10/50] kata-runtime: fix kata-shim pid reused problem
|
||||
|
||||
reason: If kata-shim process exit and it's pid reused by other process,
|
||||
it may cause kill other proecss and cause some problem.
|
||||
|
||||
Signed-off-by: jiangpengfei <jiangpengfei9@huawei.com>
|
||||
---
|
||||
virtcontainers/api.go | 2 +-
|
||||
virtcontainers/container.go | 6 +++---
|
||||
virtcontainers/shim.go | 21 +++++++++++++++++----
|
||||
virtcontainers/shim_test.go | 10 +++++-----
|
||||
4 files changed, 26 insertions(+), 13 deletions(-)
|
||||
|
||||
diff --git a/virtcontainers/api.go b/virtcontainers/api.go
|
||||
index 449a03e0..5e8c9c9e 100644
|
||||
--- a/virtcontainers/api.go
|
||||
+++ b/virtcontainers/api.go
|
||||
@@ -611,7 +611,7 @@ func statusContainer(sandbox *Sandbox, containerID string) (ContainerStatus, err
|
||||
return ContainerStatus{}, fmt.Errorf("sandbox has beed stopped exceptionally")
|
||||
}
|
||||
|
||||
- running, err := isShimRunning(container.process.Pid)
|
||||
+ running, err := isShimRunning(container.process.Pid, containerID)
|
||||
if err != nil {
|
||||
return ContainerStatus{}, err
|
||||
}
|
||||
diff --git a/virtcontainers/container.go b/virtcontainers/container.go
|
||||
index 9485e708..75f590eb 100644
|
||||
--- a/virtcontainers/container.go
|
||||
+++ b/virtcontainers/container.go
|
||||
@@ -1063,7 +1063,7 @@ func (c *Container) stop(force bool) error {
|
||||
|
||||
// If shim is still running something went wrong
|
||||
// Make sure we stop the shim process
|
||||
- if running, _ := isShimRunning(c.process.Pid); running {
|
||||
+ if running, _ := isShimRunning(c.process.Pid, c.id); running {
|
||||
l := c.Logger()
|
||||
l.Error("Failed to stop container so stopping dangling shim")
|
||||
if err := stopShim(c.process.Pid); err != nil {
|
||||
@@ -1081,7 +1081,7 @@ func (c *Container) stop(force bool) error {
|
||||
// However, if the signal didn't reach its goal, the caller still
|
||||
// expects this container to be stopped, that's why we should not
|
||||
// return an error, but instead try to kill it forcefully.
|
||||
- if err := waitForShim(c.process.Pid); err != nil {
|
||||
+ if err := waitForShim(c.process.Pid, c.id); err != nil {
|
||||
// Force the container to be killed.
|
||||
if err := c.kill(syscall.SIGKILL, true); err != nil && !force {
|
||||
return err
|
||||
@@ -1091,7 +1091,7 @@ func (c *Container) stop(force bool) error {
|
||||
// to succeed. Indeed, we have already given a second chance
|
||||
// to the container by trying to kill it with SIGKILL, there
|
||||
// is no reason to try to go further if we got an error.
|
||||
- if err := waitForShim(c.process.Pid); err != nil && !force {
|
||||
+ if err := waitForShim(c.process.Pid, c.id); err != nil && !force {
|
||||
return err
|
||||
}
|
||||
}
|
||||
diff --git a/virtcontainers/shim.go b/virtcontainers/shim.go
|
||||
index 8ec7458b..6f784a03 100644
|
||||
--- a/virtcontainers/shim.go
|
||||
+++ b/virtcontainers/shim.go
|
||||
@@ -9,11 +9,13 @@ import (
|
||||
"fmt"
|
||||
"os"
|
||||
"os/exec"
|
||||
+ "strings"
|
||||
"syscall"
|
||||
"time"
|
||||
|
||||
ns "github.com/kata-containers/runtime/virtcontainers/pkg/nsenter"
|
||||
"github.com/kata-containers/runtime/virtcontainers/types"
|
||||
+ "github.com/kata-containers/runtime/virtcontainers/utils"
|
||||
"github.com/mitchellh/mapstructure"
|
||||
"github.com/sirupsen/logrus"
|
||||
)
|
||||
@@ -227,7 +229,7 @@ func startShim(args []string, params ShimParams) (int, error) {
|
||||
return cmd.Process.Pid, nil
|
||||
}
|
||||
|
||||
-func isShimRunning(pid int) (bool, error) {
|
||||
+func isShimRunning(pid int, containerID string) (bool, error) {
|
||||
if pid <= 0 {
|
||||
return false, nil
|
||||
}
|
||||
@@ -241,19 +243,30 @@ func isShimRunning(pid int) (bool, error) {
|
||||
return false, nil
|
||||
}
|
||||
|
||||
- return true, nil
|
||||
+ cmdline, err := utils.GetProcessCmdline(pid)
|
||||
+ if err != nil {
|
||||
+ return false, nil
|
||||
+ }
|
||||
+
|
||||
+ // If process's cmdline contains kata-shim and containerID keyword, we think this process pid isn't be reused
|
||||
+ if strings.Contains(cmdline, "kata-shim") && strings.Contains(cmdline, containerID) {
|
||||
+ return true, nil
|
||||
+ }
|
||||
+
|
||||
+ shimLogger().Errorf("%d process isn't a kata-shim process", pid)
|
||||
+ return false, nil
|
||||
}
|
||||
|
||||
// waitForShim waits for the end of the shim unless it reaches the timeout
|
||||
// first, returning an error in that case.
|
||||
-func waitForShim(pid int) error {
|
||||
+func waitForShim(pid int, containerID string) error {
|
||||
if pid <= 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
tInit := time.Now()
|
||||
for {
|
||||
- running, err := isShimRunning(pid)
|
||||
+ running, err := isShimRunning(pid, containerID)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
diff --git a/virtcontainers/shim_test.go b/virtcontainers/shim_test.go
|
||||
index e9bd027c..62471311 100644
|
||||
--- a/virtcontainers/shim_test.go
|
||||
+++ b/virtcontainers/shim_test.go
|
||||
@@ -190,7 +190,7 @@ func TestStopShimSuccessfulProcessRunning(t *testing.T) {
|
||||
|
||||
func testIsShimRunning(t *testing.T, pid int, expected bool) {
|
||||
assert := assert.New(t)
|
||||
- running, err := isShimRunning(pid)
|
||||
+ running, err := isShimRunning(pid, containerID)
|
||||
assert.NoError(err)
|
||||
assert.Equal(running, expected)
|
||||
}
|
||||
@@ -205,7 +205,7 @@ func TestIsShimRunningTrue(t *testing.T) {
|
||||
cmd := testRunSleep999AndGetCmd(t)
|
||||
assert := assert.New(t)
|
||||
|
||||
- testIsShimRunning(t, cmd.Process.Pid, true)
|
||||
+ testIsShimRunning(t, cmd.Process.Pid, false)
|
||||
|
||||
err := syscall.Kill(cmd.Process.Pid, syscall.SIGKILL)
|
||||
assert.NoError(err)
|
||||
@@ -216,7 +216,7 @@ func TestWaitForShimInvalidPidSuccessful(t *testing.T) {
|
||||
assert := assert.New(t)
|
||||
|
||||
for _, val := range wrongValuesList {
|
||||
- err := waitForShim(val)
|
||||
+ err := waitForShim(val, containerID)
|
||||
assert.NoError(err)
|
||||
}
|
||||
}
|
||||
@@ -224,7 +224,7 @@ func TestWaitForShimInvalidPidSuccessful(t *testing.T) {
|
||||
func TestWaitForShimNotRunningSuccessful(t *testing.T) {
|
||||
pid := testRunSleep0AndGetPid(t)
|
||||
assert := assert.New(t)
|
||||
- assert.NoError(waitForShim(pid))
|
||||
+ assert.NoError(waitForShim(pid, containerID))
|
||||
}
|
||||
|
||||
func TestWaitForShimRunningForTooLongFailure(t *testing.T) {
|
||||
@@ -232,6 +232,6 @@ func TestWaitForShimRunningForTooLongFailure(t *testing.T) {
|
||||
assert := assert.New(t)
|
||||
|
||||
waitForShimTimeout = 0.1
|
||||
- assert.Error(waitForShim(cmd.Process.Pid))
|
||||
+ assert.NoError(waitForShim(cmd.Process.Pid, containerID))
|
||||
assert.NoError(syscall.Kill(cmd.Process.Pid, syscall.SIGKILL))
|
||||
}
|
||||
--
|
||||
2.14.3 (Apple Git-98)
|
||||
|
||||
@ -1,119 +0,0 @@
|
||||
From 0aeff2632eac58eefdc8ae438891303332831ec5 Mon Sep 17 00:00:00 2001
|
||||
From: jiangpengfei <jiangpengfei9@huawei.com>
|
||||
Date: Tue, 28 Jul 2020 20:48:24 +0800
|
||||
Subject: [PATCH 11/50] kata-runtime: check the process info before send
|
||||
SIGKILL
|
||||
|
||||
reason: In order to avoid the pid reuse problem, check the
|
||||
process info before send SIGKILL signal to process.
|
||||
|
||||
Signed-off-by: jiangpengfei <jiangpengfei9@huawei.com>
|
||||
---
|
||||
virtcontainers/kata_proxy.go | 18 ++++++++++++++++++
|
||||
virtcontainers/qemu.go | 5 +++++
|
||||
virtcontainers/shim.go | 9 +++++++++
|
||||
virtcontainers/shim_test.go | 8 ++++----
|
||||
4 files changed, 36 insertions(+), 4 deletions(-)
|
||||
|
||||
diff --git a/virtcontainers/kata_proxy.go b/virtcontainers/kata_proxy.go
|
||||
index e04b4cff..ed272bad 100644
|
||||
--- a/virtcontainers/kata_proxy.go
|
||||
+++ b/virtcontainers/kata_proxy.go
|
||||
@@ -6,8 +6,12 @@
|
||||
package virtcontainers
|
||||
|
||||
import (
|
||||
+ "fmt"
|
||||
"os/exec"
|
||||
+ "strings"
|
||||
"syscall"
|
||||
+
|
||||
+ "github.com/kata-containers/runtime/virtcontainers/utils"
|
||||
)
|
||||
|
||||
// This is the Kata Containers implementation of the proxy interface.
|
||||
@@ -61,6 +65,20 @@ func (p *kataProxy) start(params proxyParams) (int, string, error) {
|
||||
|
||||
// stop is kataProxy stop implementation for proxy interface.
|
||||
func (p *kataProxy) stop(pid int) error {
|
||||
+ if pid <= 1 {
|
||||
+ return nil
|
||||
+ }
|
||||
+
|
||||
+ // check process info before send SIGKILL signal
|
||||
+ cmdline, err := utils.GetProcessCmdline(pid)
|
||||
+ if err != nil {
|
||||
+ return fmt.Errorf("get kata-proxy %d cmdline error: %v", pid, err)
|
||||
+ }
|
||||
+
|
||||
+ if !strings.Contains(cmdline, KataProxyProcessName) {
|
||||
+ return fmt.Errorf("%d is not kata-proxy process, don't kill wrong process", pid)
|
||||
+ }
|
||||
+
|
||||
// Signal the proxy with SIGTERM.
|
||||
return syscall.Kill(pid, syscall.SIGTERM)
|
||||
}
|
||||
diff --git a/virtcontainers/qemu.go b/virtcontainers/qemu.go
|
||||
index 4b15d968..4789101d 100644
|
||||
--- a/virtcontainers/qemu.go
|
||||
+++ b/virtcontainers/qemu.go
|
||||
@@ -967,6 +967,11 @@ func (q *qemu) stopSandbox(force bool) error {
|
||||
return fmt.Errorf("force kill qemu process pid is invalid")
|
||||
}
|
||||
|
||||
+ cmdline, _ := utils.GetProcessCmdline(qemuMainPid)
|
||||
+ if !strings.Contains(cmdline, string(QemuHypervisor)) {
|
||||
+ return fmt.Errorf("force kill %d process is not qemu process, don't kill wrong process", qemuMainPid)
|
||||
+ }
|
||||
+
|
||||
_ = syscall.Kill(qemuMainPid, syscall.SIGKILL)
|
||||
}
|
||||
|
||||
diff --git a/virtcontainers/shim.go b/virtcontainers/shim.go
|
||||
index 6f784a03..b192b258 100644
|
||||
--- a/virtcontainers/shim.go
|
||||
+++ b/virtcontainers/shim.go
|
||||
@@ -143,6 +143,15 @@ func stopShim(pid int) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
+ cmdline, err := utils.GetProcessCmdline(pid)
|
||||
+ if err != nil {
|
||||
+ return err
|
||||
+ }
|
||||
+
|
||||
+ if !strings.Contains(cmdline, "kata-shim") {
|
||||
+ return fmt.Errorf("%d process is not kata-shim process, don't kill wrong process", pid)
|
||||
+ }
|
||||
+
|
||||
if err := signalShim(pid, syscall.SIGKILL); err != nil && err != syscall.ESRCH {
|
||||
return err
|
||||
}
|
||||
diff --git a/virtcontainers/shim_test.go b/virtcontainers/shim_test.go
|
||||
index 62471311..dc15eab0 100644
|
||||
--- a/virtcontainers/shim_test.go
|
||||
+++ b/virtcontainers/shim_test.go
|
||||
@@ -176,16 +176,16 @@ func testRunSleep999AndGetCmd(t *testing.T) *exec.Cmd {
|
||||
return cmd
|
||||
}
|
||||
|
||||
-func TestStopShimSuccessfulProcessNotRunning(t *testing.T) {
|
||||
+func TestStopShimFailProcessNotRunning(t *testing.T) {
|
||||
assert := assert.New(t)
|
||||
pid := testRunSleep0AndGetPid(t)
|
||||
- assert.NoError(stopShim(pid))
|
||||
+ assert.Error(stopShim(pid))
|
||||
}
|
||||
|
||||
-func TestStopShimSuccessfulProcessRunning(t *testing.T) {
|
||||
+func TestStopShimFailProcessRunning(t *testing.T) {
|
||||
assert := assert.New(t)
|
||||
cmd := testRunSleep999AndGetCmd(t)
|
||||
- assert.NoError(stopShim(cmd.Process.Pid))
|
||||
+ assert.Error(stopShim(cmd.Process.Pid))
|
||||
}
|
||||
|
||||
func testIsShimRunning(t *testing.T, pid int, expected bool) {
|
||||
--
|
||||
2.14.3 (Apple Git-98)
|
||||
|
||||
@ -1,37 +0,0 @@
|
||||
From 441d80f55f4dc5efb4c92d91608a3c8db3d087cb Mon Sep 17 00:00:00 2001
|
||||
From: jiangpengfei <jiangpengfei9@huawei.com>
|
||||
Date: Tue, 28 Jul 2020 21:43:15 +0800
|
||||
Subject: [PATCH 12/50] kata-runtime: truncate the log.json file before
|
||||
kata-runtime subcommand executed
|
||||
|
||||
reason: since we have redirect the kata-runtime log to /var/log/messages, and avoid the
|
||||
path of log.json file to be large in the tmpfs, so we truncate the log.json file every
|
||||
time before subcommand is executed.
|
||||
|
||||
Signed-off-by: jiangpengfei <jiangpengfei9@huawei.com>
|
||||
---
|
||||
cli/main.go | 8 ++++++++
|
||||
1 file changed, 8 insertions(+)
|
||||
|
||||
diff --git a/cli/main.go b/cli/main.go
|
||||
index 1362a8fb..5eb2fb19 100644
|
||||
--- a/cli/main.go
|
||||
+++ b/cli/main.go
|
||||
@@ -300,6 +300,14 @@ func beforeSubcommands(c *cli.Context) error {
|
||||
ignoreConfigLogs = true
|
||||
} else {
|
||||
if path := c.GlobalString("log"); path != "" {
|
||||
+ // since we have redirect the kata-runtime log to /var/log/messages, and avoid the
|
||||
+ // path of log.json file to be large in the tmpfs, so we truncate the log.json file
|
||||
+ // every time before subcommand is executed.
|
||||
+ if path != "/dev/null" && katautils.FileExists(path) {
|
||||
+ if err := os.Truncate(path, 0); err != nil {
|
||||
+ return err
|
||||
+ }
|
||||
+ }
|
||||
f, err := os.OpenFile(path, os.O_CREATE|os.O_WRONLY|os.O_APPEND|os.O_SYNC, 0640)
|
||||
if err != nil {
|
||||
return err
|
||||
--
|
||||
2.14.3 (Apple Git-98)
|
||||
|
||||
@ -1,98 +0,0 @@
|
||||
From fd63d26a5b0542f35d61b0c19c80795f052b4518 Mon Sep 17 00:00:00 2001
|
||||
From: jiangpengfei <jiangpengfei9@huawei.com>
|
||||
Date: Tue, 28 Jul 2020 22:05:44 +0800
|
||||
Subject: [PATCH 13/50] kata-runtime: get container info by containerID prefix
|
||||
|
||||
reason: get container info by containerID prefix, so we just
|
||||
need to input the prefix containerID when call kata-runtime
|
||||
subcommand
|
||||
|
||||
Signed-off-by: jiangpengfei <jiangpengfei9@huawei.com>
|
||||
---
|
||||
cli/oci.go | 35 +++++++++++++++++++++++++++++++++++
|
||||
pkg/katautils/oci.go | 5 +++++
|
||||
2 files changed, 40 insertions(+)
|
||||
|
||||
diff --git a/cli/oci.go b/cli/oci.go
|
||||
index 8ffac2df..bf962d03 100644
|
||||
--- a/cli/oci.go
|
||||
+++ b/cli/oci.go
|
||||
@@ -9,6 +9,7 @@ import (
|
||||
"bufio"
|
||||
"context"
|
||||
"fmt"
|
||||
+ "io/ioutil"
|
||||
"net"
|
||||
"os"
|
||||
"path/filepath"
|
||||
@@ -25,6 +26,8 @@ const (
|
||||
// Filesystem type corresponding to CGROUP_SUPER_MAGIC as listed
|
||||
// here: http://man7.org/linux/man-pages/man2/statfs.2.html
|
||||
cgroupFsType = 0x27e0eb
|
||||
+
|
||||
+ maxIDLength = 64
|
||||
)
|
||||
|
||||
var cgroupsDirPath string
|
||||
@@ -38,6 +41,14 @@ func getContainerInfo(ctx context.Context, containerID string) (vc.ContainerStat
|
||||
return vc.ContainerStatus{}, "", fmt.Errorf("Missing container ID")
|
||||
}
|
||||
|
||||
+ if len(containerID) < maxIDLength {
|
||||
+ fullContainerID, err := getContainerIDbyPrefix(containerID)
|
||||
+ if err != nil {
|
||||
+ return vc.ContainerStatus{}, "", err
|
||||
+ }
|
||||
+ containerID = fullContainerID
|
||||
+ }
|
||||
+
|
||||
sandboxID, err := katautils.FetchContainerIDMapping(containerID)
|
||||
if err != nil {
|
||||
return vc.ContainerStatus{}, "", err
|
||||
@@ -211,3 +222,27 @@ func getCgroupsDirPath(mountInfoFile string) (string, error) {
|
||||
|
||||
return cgroupRootPath, nil
|
||||
}
|
||||
+
|
||||
+func getContainerIDbyPrefix(prefix string) (string, error) {
|
||||
+ files, err := ioutil.ReadDir(katautils.GetCtrsMapTreePath())
|
||||
+ if err != nil {
|
||||
+ return "", err
|
||||
+ }
|
||||
+
|
||||
+ containers := []string{}
|
||||
+ for _, file := range files {
|
||||
+ if file.IsDir() && strings.HasPrefix(file.Name(), prefix) {
|
||||
+ containers = append(containers, file.Name())
|
||||
+ }
|
||||
+ }
|
||||
+
|
||||
+ if len(containers) == 0 {
|
||||
+ return "", fmt.Errorf("no such container ID (%v)", prefix)
|
||||
+ }
|
||||
+
|
||||
+ if len(containers) > 1 {
|
||||
+ return "", fmt.Errorf("multiple containers found (%v)", prefix)
|
||||
+ }
|
||||
+
|
||||
+ return containers[0], nil
|
||||
+}
|
||||
diff --git a/pkg/katautils/oci.go b/pkg/katautils/oci.go
|
||||
index 6de8101e..1334af35 100644
|
||||
--- a/pkg/katautils/oci.go
|
||||
+++ b/pkg/katautils/oci.go
|
||||
@@ -25,6 +25,11 @@ func SetCtrsMapTreePath(path string) {
|
||||
ctrsMapTreePath = path
|
||||
}
|
||||
|
||||
+// GetCtrsMapTreePath return the containerID to SandboxID mapping dir
|
||||
+func GetCtrsMapTreePath() string {
|
||||
+ return ctrsMapTreePath
|
||||
+}
|
||||
+
|
||||
// doUpdatePath returns whether a ctrsMapTreePath needs to be updated with a rootless prefix
|
||||
func doUpdatePath() bool {
|
||||
return rootless.IsRootless() && !strings.HasPrefix(ctrsMapTreePath, rootless.GetRootlessDir())
|
||||
--
|
||||
2.14.3 (Apple Git-98)
|
||||
|
||||
@ -1,76 +0,0 @@
|
||||
From 1bd3cb85a1cf0e94b3280412d5fb47cecc4721fd Mon Sep 17 00:00:00 2001
|
||||
From: jiangpengfei <jiangpengfei9@huawei.com>
|
||||
Date: Wed, 29 Jul 2020 10:42:49 +0800
|
||||
Subject: [PATCH 14/50] kata-runtime: add self defined annotations framework
|
||||
|
||||
reason: add self defined annotations framework to pass some
|
||||
self defined resource for sandbox
|
||||
|
||||
Signed-off-by: jiangpengfei <jiangpengfei9@huawei.com>
|
||||
---
|
||||
virtcontainers/pkg/oci/utils.go | 38 ++++++++++++++++++++++++++++++++++++++
|
||||
1 file changed, 38 insertions(+)
|
||||
|
||||
diff --git a/virtcontainers/pkg/oci/utils.go b/virtcontainers/pkg/oci/utils.go
|
||||
index cd8d48ce..05181efd 100644
|
||||
--- a/virtcontainers/pkg/oci/utils.go
|
||||
+++ b/virtcontainers/pkg/oci/utils.go
|
||||
@@ -33,6 +33,10 @@ type annotationContainerType struct {
|
||||
containerType vc.ContainerType
|
||||
}
|
||||
|
||||
+type annotationHandler func(value string) error
|
||||
+
|
||||
+var annotationHandlerList = map[string]annotationHandler{}
|
||||
+
|
||||
var (
|
||||
// ErrNoLinux is an error for missing Linux sections in the OCI configuration file.
|
||||
ErrNoLinux = errors.New("missing Linux section")
|
||||
@@ -342,6 +346,10 @@ func addAnnotations(ocispec specs.Spec, config *vc.SandboxConfig) error {
|
||||
if err := addAgentConfigOverrides(ocispec, config); err != nil {
|
||||
return err
|
||||
}
|
||||
+
|
||||
+ if err := addOtherSandboxAnnotation(ocispec, config); err != nil {
|
||||
+ return err
|
||||
+ }
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -1016,3 +1024,33 @@ func GetOCIConfig(status vc.ContainerStatus) (specs.Spec, error) {
|
||||
|
||||
return *status.Spec, nil
|
||||
}
|
||||
+
|
||||
+// validateOtherSandboxAnnotations validate the value of annotation
|
||||
+func validateOtherSandboxAnnotations(annotation, value string) error {
|
||||
+ validateHandler, ok := annotationHandlerList[annotation]
|
||||
+ if !ok {
|
||||
+ return fmt.Errorf("unsupport Sandbox annotation type")
|
||||
+ }
|
||||
+
|
||||
+ return validateHandler(value)
|
||||
+}
|
||||
+
|
||||
+// addOtherSandboxAnnotation add self defined annotation for sandbox
|
||||
+func addOtherSandboxAnnotation(ocispec specs.Spec, sbConfig *vc.SandboxConfig) error {
|
||||
+ otherSandboxAnnotationsKey := []string{}
|
||||
+
|
||||
+ for _, a := range otherSandboxAnnotationsKey {
|
||||
+ value, ok := ocispec.Annotations[a]
|
||||
+ if !ok {
|
||||
+ continue
|
||||
+ }
|
||||
+
|
||||
+ if err := validateOtherSandboxAnnotations(a, value); err != nil {
|
||||
+ return err
|
||||
+ }
|
||||
+
|
||||
+ sbConfig.Annotations[a] = value
|
||||
+ }
|
||||
+
|
||||
+ return nil
|
||||
+}
|
||||
--
|
||||
2.14.3 (Apple Git-98)
|
||||
|
||||
@ -1,143 +0,0 @@
|
||||
From c0a33c4584e1fbd9b39ff1ca3ed632efe85de65e Mon Sep 17 00:00:00 2001
|
||||
From: jiangpengfei <jiangpengfei9@huawei.com>
|
||||
Date: Sun, 2 Aug 2020 15:51:10 +0800
|
||||
Subject: [PATCH 15/50] kata-runtime: add reuse hypervisor cpu and memory
|
||||
feature
|
||||
|
||||
reason: If default hypervisor cpu and memory is set too large,
|
||||
which may waste resource, so we add enable_reuse_cpu_memory
|
||||
config in the configuration.toml file to choose share the
|
||||
hypervisor cpu and memory with container or not.
|
||||
|
||||
Signed-off-by: jiangpengfei <jiangpengfei9@huawei.com>
|
||||
---
|
||||
cli/config/configuration-qemu.toml.in | 5 +++++
|
||||
pkg/katautils/config.go | 2 ++
|
||||
virtcontainers/hypervisor.go | 3 +++
|
||||
virtcontainers/persist.go | 2 ++
|
||||
virtcontainers/persist/api/config.go | 3 +++
|
||||
virtcontainers/sandbox.go | 22 ++++++++++++++++++----
|
||||
6 files changed, 33 insertions(+), 4 deletions(-)
|
||||
|
||||
diff --git a/cli/config/configuration-qemu.toml.in b/cli/config/configuration-qemu.toml.in
|
||||
index 46ce7d9b..82461732 100644
|
||||
--- a/cli/config/configuration-qemu.toml.in
|
||||
+++ b/cli/config/configuration-qemu.toml.in
|
||||
@@ -95,6 +95,11 @@ default_memory = @DEFMEMSZ@
|
||||
# Default false
|
||||
#enable_virtio_mem = true
|
||||
|
||||
+# Specifies share hypervisor default cpu and memory resource
|
||||
+# between hypervisor and container process.
|
||||
+# Default false
|
||||
+#enable_reuse_cpu_memory = false
|
||||
+
|
||||
# Disable block device from being used for a container's rootfs.
|
||||
# In case of a storage driver like devicemapper where a container's
|
||||
# root file system is backed by a block device, the block device is passed
|
||||
diff --git a/pkg/katautils/config.go b/pkg/katautils/config.go
|
||||
index 448d23ac..d1883c94 100644
|
||||
--- a/pkg/katautils/config.go
|
||||
+++ b/pkg/katautils/config.go
|
||||
@@ -112,6 +112,7 @@ type hypervisor struct {
|
||||
MemorySize uint32 `toml:"default_memory"`
|
||||
MemSlots uint32 `toml:"memory_slots"`
|
||||
MemOffset uint32 `toml:"memory_offset"`
|
||||
+ EnableCPUMemoryReuse bool `toml:"enable_reuse_cpu_memory"`
|
||||
DefaultBridges uint32 `toml:"default_bridges"`
|
||||
Msize9p uint32 `toml:"msize_9p"`
|
||||
PCIeRootPort uint32 `toml:"pcie_root_port"`
|
||||
@@ -640,6 +641,7 @@ func newQemuHypervisorConfig(h hypervisor) (vc.HypervisorConfig, error) {
|
||||
MemorySize: h.defaultMemSz(),
|
||||
MemSlots: h.defaultMemSlots(),
|
||||
MemOffset: h.defaultMemOffset(),
|
||||
+ EnableCPUMemoryReuse: h.EnableCPUMemoryReuse,
|
||||
VirtioMem: h.VirtioMem,
|
||||
EntropySource: h.GetEntropySource(),
|
||||
DefaultBridges: h.defaultBridges(),
|
||||
diff --git a/virtcontainers/hypervisor.go b/virtcontainers/hypervisor.go
|
||||
index fd7d1f8e..5f8d24f9 100644
|
||||
--- a/virtcontainers/hypervisor.go
|
||||
+++ b/virtcontainers/hypervisor.go
|
||||
@@ -250,6 +250,9 @@ type HypervisorConfig struct {
|
||||
// MemOffset specifies memory space for nvdimm device
|
||||
MemOffset uint32
|
||||
|
||||
+ // Enable hypervisor cpu and memory resource share with container
|
||||
+ EnableCPUMemoryReuse bool
|
||||
+
|
||||
// VirtioFSCacheSize is the DAX cache size in MiB
|
||||
VirtioFSCacheSize uint32
|
||||
|
||||
diff --git a/virtcontainers/persist.go b/virtcontainers/persist.go
|
||||
index d96a3890..aef4d18d 100644
|
||||
--- a/virtcontainers/persist.go
|
||||
+++ b/virtcontainers/persist.go
|
||||
@@ -214,6 +214,7 @@ func (s *Sandbox) dumpConfig(ss *persistapi.SandboxState) {
|
||||
Msize9p: sconfig.HypervisorConfig.Msize9p,
|
||||
MemSlots: sconfig.HypervisorConfig.MemSlots,
|
||||
MemOffset: sconfig.HypervisorConfig.MemOffset,
|
||||
+ EnableCPUMemoryReuse: sconfig.HypervisorConfig.EnableCPUMemoryReuse,
|
||||
VirtioMem: sconfig.HypervisorConfig.VirtioMem,
|
||||
VirtioFSCacheSize: sconfig.HypervisorConfig.VirtioFSCacheSize,
|
||||
KernelPath: sconfig.HypervisorConfig.KernelPath,
|
||||
@@ -503,6 +504,7 @@ func loadSandboxConfig(id string) (*SandboxConfig, error) {
|
||||
Msize9p: hconf.Msize9p,
|
||||
MemSlots: hconf.MemSlots,
|
||||
MemOffset: hconf.MemOffset,
|
||||
+ EnableCPUMemoryReuse: hconf.EnableCPUMemoryReuse,
|
||||
VirtioMem: hconf.VirtioMem,
|
||||
VirtioFSCacheSize: hconf.VirtioFSCacheSize,
|
||||
KernelPath: hconf.KernelPath,
|
||||
diff --git a/virtcontainers/persist/api/config.go b/virtcontainers/persist/api/config.go
|
||||
index 34a5fd0f..a3c6ec91 100644
|
||||
--- a/virtcontainers/persist/api/config.go
|
||||
+++ b/virtcontainers/persist/api/config.go
|
||||
@@ -35,6 +35,9 @@ type HypervisorConfig struct {
|
||||
// MemOffset specifies memory space for nvdimm device
|
||||
MemOffset uint32
|
||||
|
||||
+ // Enable hypervisor cpu and memory resource share with container
|
||||
+ EnableCPUMemoryReuse bool
|
||||
+
|
||||
// VirtioFSCacheSize is the DAX cache size in MiB
|
||||
VirtioFSCacheSize uint32
|
||||
|
||||
diff --git a/virtcontainers/sandbox.go b/virtcontainers/sandbox.go
|
||||
index 78188ed7..e766d1f7 100644
|
||||
--- a/virtcontainers/sandbox.go
|
||||
+++ b/virtcontainers/sandbox.go
|
||||
@@ -1833,12 +1833,26 @@ func (s *Sandbox) updateResources() error {
|
||||
}
|
||||
|
||||
sandboxVCPUs := s.calculateSandboxCPUs()
|
||||
- // Add default vcpus for sandbox
|
||||
- sandboxVCPUs += s.hypervisor.hypervisorConfig().NumVCPUs
|
||||
+ // Share the hypervisor vcpu with container process
|
||||
+ if s.config.HypervisorConfig.EnableCPUMemoryReuse {
|
||||
+ if sandboxVCPUs <= s.config.HypervisorConfig.NumVCPUs {
|
||||
+ sandboxVCPUs = s.config.HypervisorConfig.NumVCPUs
|
||||
+ }
|
||||
+ } else {
|
||||
+ // Add default vcpus for sandbox
|
||||
+ sandboxVCPUs += s.hypervisor.hypervisorConfig().NumVCPUs
|
||||
+ }
|
||||
|
||||
sandboxMemoryByte := s.calculateSandboxMemory()
|
||||
- // Add default / rsvd memory for sandbox.
|
||||
- sandboxMemoryByte += int64(s.hypervisor.hypervisorConfig().MemorySize) << utils.MibToBytesShift
|
||||
+ // Share the hypervisor memory with container process
|
||||
+ if s.config.HypervisorConfig.EnableCPUMemoryReuse {
|
||||
+ if sandboxMemoryByte <= (int64(s.config.HypervisorConfig.MemorySize) << utils.MibToBytesShift) {
|
||||
+ sandboxMemoryByte = int64(s.config.HypervisorConfig.MemorySize) << utils.MibToBytesShift
|
||||
+ }
|
||||
+ } else {
|
||||
+ // Add default / rsvd memory for sandbox.
|
||||
+ sandboxMemoryByte += int64(s.hypervisor.hypervisorConfig().MemorySize) << utils.MibToBytesShift
|
||||
+ }
|
||||
|
||||
// Update VCPUs
|
||||
s.Logger().WithField("cpus-sandbox", sandboxVCPUs).Debugf("Request to hypervisor to update vCPUs")
|
||||
--
|
||||
2.14.3 (Apple Git-98)
|
||||
|
||||
@ -1,288 +0,0 @@
|
||||
From dc9de8bb181e2cec2f3e0a76d02833fef45b46af Mon Sep 17 00:00:00 2001
|
||||
From: jiangpengfei <jiangpengfei9@huawei.com>
|
||||
Date: Thu, 6 Aug 2020 09:28:34 -0400
|
||||
Subject: [PATCH 16/50] virtcontainers: fix hotplug huge size memory cause
|
||||
agent hang bug
|
||||
|
||||
fixes: #2872
|
||||
|
||||
reason: If hotplug huge size memory into kata VM at once time,
|
||||
guest kernel will allocate some extra memory for memory management,
|
||||
which may cause kata-agent hang and out of responding.
|
||||
And hotplug more memory into VM, more extra memory is needed.
|
||||
|
||||
Inorder to solve this problem, we divide hotplug huge memory into
|
||||
two steps. First, hotplug the max allowed memory into VM and wait
|
||||
all first step hotplugged memory online. Second Step, hotplug the
|
||||
left memory into VM.
|
||||
|
||||
Signed-off-by: jiangpengfei <jiangpengfei9@huawei.com>
|
||||
---
|
||||
virtcontainers/acrn.go | 4 ++++
|
||||
virtcontainers/agent.go | 3 ++-
|
||||
virtcontainers/clh.go | 4 ++++
|
||||
virtcontainers/fc.go | 4 ++++
|
||||
virtcontainers/hypervisor.go | 3 +++
|
||||
virtcontainers/kata_agent.go | 4 ++--
|
||||
virtcontainers/kata_agent_test.go | 2 +-
|
||||
virtcontainers/mock_hypervisor.go | 4 ++++
|
||||
virtcontainers/noop_agent.go | 2 +-
|
||||
virtcontainers/qemu.go | 4 ++++
|
||||
virtcontainers/sandbox.go | 30 ++++++++++++++++++++++++++++--
|
||||
virtcontainers/sandbox_test.go | 12 ++++++++++++
|
||||
virtcontainers/utils/utils.go | 3 +++
|
||||
virtcontainers/vm.go | 2 +-
|
||||
14 files changed, 73 insertions(+), 8 deletions(-)
|
||||
|
||||
diff --git a/virtcontainers/acrn.go b/virtcontainers/acrn.go
|
||||
index 10cae06f..c9a0fe0b 100644
|
||||
--- a/virtcontainers/acrn.go
|
||||
+++ b/virtcontainers/acrn.go
|
||||
@@ -811,3 +811,7 @@ func (a *Acrn) loadInfo() error {
|
||||
}
|
||||
return nil
|
||||
}
|
||||
+
|
||||
+func (a *Acrn) getMemorySize() uint32 {
|
||||
+ return a.config.MemorySize
|
||||
+}
|
||||
diff --git a/virtcontainers/agent.go b/virtcontainers/agent.go
|
||||
index be9526c7..b1dea816 100644
|
||||
--- a/virtcontainers/agent.go
|
||||
+++ b/virtcontainers/agent.go
|
||||
@@ -201,7 +201,8 @@ type agent interface {
|
||||
// This function should be called after hot adding vCPUs or Memory.
|
||||
// cpus specifies the number of CPUs that were added and the agent should online
|
||||
// cpuOnly specifies that we should online cpu or online memory or both
|
||||
- onlineCPUMem(cpus uint32, cpuOnly bool) error
|
||||
+ // wait specifies that we should wait all cpu or memory online in the VM synchronously
|
||||
+ onlineCPUMem(cpus uint32, cpuOnly bool, wait bool) error
|
||||
|
||||
// memHotplugByProbe will notify the guest kernel about memory hotplug event through
|
||||
// probe interface.
|
||||
diff --git a/virtcontainers/clh.go b/virtcontainers/clh.go
|
||||
index 59510b02..8afcd4bf 100644
|
||||
--- a/virtcontainers/clh.go
|
||||
+++ b/virtcontainers/clh.go
|
||||
@@ -1210,3 +1210,7 @@ func (clh *cloudHypervisor) vmInfo() (chclient.VmInfo, error) {
|
||||
return info, openAPIClientError(err)
|
||||
|
||||
}
|
||||
+
|
||||
+func (clh *cloudHypervisor) getMemorySize() uint32 {
|
||||
+ return clh.config.MemorySize
|
||||
+}
|
||||
diff --git a/virtcontainers/fc.go b/virtcontainers/fc.go
|
||||
index 72a8e192..15726156 100644
|
||||
--- a/virtcontainers/fc.go
|
||||
+++ b/virtcontainers/fc.go
|
||||
@@ -1212,3 +1212,7 @@ func (fc *firecracker) watchConsole() (*os.File, error) {
|
||||
|
||||
return stdio, nil
|
||||
}
|
||||
+
|
||||
+func (fc *firecracker) getMemorySize() uint32 {
|
||||
+ return fc.config.MemorySize
|
||||
+}
|
||||
diff --git a/virtcontainers/hypervisor.go b/virtcontainers/hypervisor.go
|
||||
index 5f8d24f9..9cd685ad 100644
|
||||
--- a/virtcontainers/hypervisor.go
|
||||
+++ b/virtcontainers/hypervisor.go
|
||||
@@ -778,6 +778,9 @@ type hypervisor interface {
|
||||
hotplugRemoveDevice(devInfo interface{}, devType deviceType) (interface{}, error)
|
||||
resizeMemory(memMB uint32, memoryBlockSizeMB uint32, probe bool) (uint32, memoryDevice, error)
|
||||
resizeVCPUs(vcpus uint32) (uint32, uint32, error)
|
||||
+ // getMemorySize return the total memory in the guest include default memory size + hot plugged memory
|
||||
+ // return memory size unit is MB
|
||||
+ getMemorySize() uint32
|
||||
getSandboxConsole(sandboxID string) (string, error)
|
||||
disconnect()
|
||||
capabilities() types.Capabilities
|
||||
diff --git a/virtcontainers/kata_agent.go b/virtcontainers/kata_agent.go
|
||||
index 7575d326..8e073339 100644
|
||||
--- a/virtcontainers/kata_agent.go
|
||||
+++ b/virtcontainers/kata_agent.go
|
||||
@@ -1806,9 +1806,9 @@ func (k *kataAgent) memHotplugByProbe(addr uint64, sizeMB uint32, memorySectionS
|
||||
return err
|
||||
}
|
||||
|
||||
-func (k *kataAgent) onlineCPUMem(cpus uint32, cpuOnly bool) error {
|
||||
+func (k *kataAgent) onlineCPUMem(cpus uint32, cpuOnly bool, wait bool) error {
|
||||
req := &grpc.OnlineCPUMemRequest{
|
||||
- Wait: false,
|
||||
+ Wait: wait,
|
||||
NbCpus: cpus,
|
||||
CpuOnly: cpuOnly,
|
||||
}
|
||||
diff --git a/virtcontainers/kata_agent_test.go b/virtcontainers/kata_agent_test.go
|
||||
index 62d31c93..2a2ddada 100644
|
||||
--- a/virtcontainers/kata_agent_test.go
|
||||
+++ b/virtcontainers/kata_agent_test.go
|
||||
@@ -324,7 +324,7 @@ func TestKataAgentSendReq(t *testing.T) {
|
||||
err = k.resumeContainer(sandbox, Container{})
|
||||
assert.Nil(err)
|
||||
|
||||
- err = k.onlineCPUMem(1, true)
|
||||
+ err = k.onlineCPUMem(1, true, false)
|
||||
assert.Nil(err)
|
||||
|
||||
_, err = k.statsContainer(sandbox, Container{})
|
||||
diff --git a/virtcontainers/mock_hypervisor.go b/virtcontainers/mock_hypervisor.go
|
||||
index a5b67491..f1c6106d 100644
|
||||
--- a/virtcontainers/mock_hypervisor.go
|
||||
+++ b/virtcontainers/mock_hypervisor.go
|
||||
@@ -128,3 +128,7 @@ func (m *mockHypervisor) check() error {
|
||||
func (m *mockHypervisor) generateSocket(id string, useVsock bool) (interface{}, error) {
|
||||
return types.Socket{HostPath: "/tmp/socket", Name: "socket"}, nil
|
||||
}
|
||||
+
|
||||
+func (m *mockHypervisor) getMemorySize() uint32 {
|
||||
+ return 0
|
||||
+}
|
||||
diff --git a/virtcontainers/noop_agent.go b/virtcontainers/noop_agent.go
|
||||
index 8a7cd337..6e211bca 100644
|
||||
--- a/virtcontainers/noop_agent.go
|
||||
+++ b/virtcontainers/noop_agent.go
|
||||
@@ -102,7 +102,7 @@ func (n *noopAgent) memHotplugByProbe(addr uint64, sizeMB uint32, memorySectionS
|
||||
}
|
||||
|
||||
// onlineCPUMem is the Noop agent Container online CPU and Memory implementation. It does nothing.
|
||||
-func (n *noopAgent) onlineCPUMem(cpus uint32, cpuOnly bool) error {
|
||||
+func (n *noopAgent) onlineCPUMem(cpus uint32, cpuOnly bool, wait bool) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
diff --git a/virtcontainers/qemu.go b/virtcontainers/qemu.go
|
||||
index 4789101d..7bae3278 100644
|
||||
--- a/virtcontainers/qemu.go
|
||||
+++ b/virtcontainers/qemu.go
|
||||
@@ -2273,3 +2273,7 @@ func (q *qemu) check() error {
|
||||
func (q *qemu) generateSocket(id string, useVsock bool) (interface{}, error) {
|
||||
return generateVMSocket(id, useVsock, q.store.RunVMStoragePath())
|
||||
}
|
||||
+
|
||||
+func (q *qemu) getMemorySize() uint32 {
|
||||
+ return q.config.MemorySize + uint32(q.state.HotpluggedMemory)
|
||||
+}
|
||||
diff --git a/virtcontainers/sandbox.go b/virtcontainers/sandbox.go
|
||||
index e766d1f7..a318d677 100644
|
||||
--- a/virtcontainers/sandbox.go
|
||||
+++ b/virtcontainers/sandbox.go
|
||||
@@ -1864,7 +1864,7 @@ func (s *Sandbox) updateResources() error {
|
||||
// If the CPUs were increased, ask agent to online them
|
||||
if oldCPUs < newCPUs {
|
||||
vcpusAdded := newCPUs - oldCPUs
|
||||
- if err := s.agent.onlineCPUMem(vcpusAdded, true); err != nil {
|
||||
+ if err := s.agent.onlineCPUMem(vcpusAdded, true, false); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
@@ -1872,6 +1872,20 @@ func (s *Sandbox) updateResources() error {
|
||||
|
||||
// Update Memory
|
||||
s.Logger().WithField("memory-sandbox-size-byte", sandboxMemoryByte).Debugf("Request to hypervisor to update memory")
|
||||
+ reqMemMB := uint32(sandboxMemoryByte >> utils.MibToBytesShift)
|
||||
+ currentMemMB := s.hypervisor.getMemorySize()
|
||||
+
|
||||
+ // If request hotplug memory size larger than utils.MaxHotplugMemMBOnceTime,
|
||||
+ // inorder to avoid hotplug memory oom problem, we need to hotplug large memory
|
||||
+ // with two steps. First, hotplug utils.MaxHotplugMemMBOnceTime size memory into
|
||||
+ // guest and wait all hotplug memory online. Then, hotplug the left unplugged memory
|
||||
+ // into the guest
|
||||
+ if currentMemMB < reqMemMB && (reqMemMB-currentMemMB) > utils.MaxHotplugMemMBOnceTime {
|
||||
+ if err := s.beforeHotplugHugeMem(currentMemMB); err != nil {
|
||||
+ return err
|
||||
+ }
|
||||
+ }
|
||||
+
|
||||
newMemory, updatedMemoryDevice, err := s.hypervisor.resizeMemory(uint32(sandboxMemoryByte>>utils.MibToBytesShift), s.state.GuestMemoryBlockSizeMB, s.state.GuestMemoryHotplugProbe)
|
||||
if err != nil {
|
||||
return err
|
||||
@@ -1884,7 +1898,7 @@ func (s *Sandbox) updateResources() error {
|
||||
return err
|
||||
}
|
||||
}
|
||||
- if err := s.agent.onlineCPUMem(0, false); err != nil {
|
||||
+ if err := s.agent.onlineCPUMem(0, false, false); err != nil {
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
@@ -1926,6 +1940,18 @@ func (s *Sandbox) calculateSandboxCPUs() uint32 {
|
||||
return utils.CalculateVCpusFromMilliCpus(mCPU)
|
||||
}
|
||||
|
||||
+func (s *Sandbox) beforeHotplugHugeMem(currentMemSizeInMB uint32) error {
|
||||
+ wantedTotalMemSize := currentMemSizeInMB + utils.MaxHotplugMemMBOnceTime
|
||||
+ newMemory, _, err := s.hypervisor.resizeMemory(wantedTotalMemSize, s.state.GuestMemoryBlockSizeMB, s.state.GuestMemoryHotplugProbe)
|
||||
+ if err != nil {
|
||||
+ return err
|
||||
+ }
|
||||
+
|
||||
+ s.Logger().Debugf("first part hotplug memory size: %d MB", newMemory)
|
||||
+ // wait all first part hotplugged memory online in the guest
|
||||
+ return s.agent.onlineCPUMem(0, false, true)
|
||||
+}
|
||||
+
|
||||
// GetHypervisorType is used for getting Hypervisor name currently used.
|
||||
// Sandbox implement DeviceReceiver interface from device/api/interface.go
|
||||
func (s *Sandbox) GetHypervisorType() string {
|
||||
diff --git a/virtcontainers/sandbox_test.go b/virtcontainers/sandbox_test.go
|
||||
index 85c712e8..4b02b3f3 100644
|
||||
--- a/virtcontainers/sandbox_test.go
|
||||
+++ b/virtcontainers/sandbox_test.go
|
||||
@@ -25,6 +25,7 @@ import (
|
||||
"github.com/kata-containers/runtime/virtcontainers/persist/fs"
|
||||
"github.com/kata-containers/runtime/virtcontainers/pkg/annotations"
|
||||
"github.com/kata-containers/runtime/virtcontainers/types"
|
||||
+ "github.com/kata-containers/runtime/virtcontainers/utils"
|
||||
specs "github.com/opencontainers/runtime-spec/specs-go"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"golang.org/x/sys/unix"
|
||||
@@ -1522,6 +1523,17 @@ func TestSandboxUpdateResources(t *testing.T) {
|
||||
}
|
||||
err = s.updateResources()
|
||||
assert.NoError(t, err)
|
||||
+
|
||||
+ // add a container with huge memory equal utils.MaxHotplugMemMBOnceTime
|
||||
+ contConfig3 := newTestContainerConfigNoop("cont-00003")
|
||||
+ contConfig3.Resources.Memory = &specs.LinuxMemory{
|
||||
+ Limit: new(int64),
|
||||
+ }
|
||||
+ container3MemLimitInBytes := int64(utils.MaxHotplugMemMBOnceTime << utils.MibToBytesShift)
|
||||
+ contConfig3.Resources.Memory.Limit = &container3MemLimitInBytes
|
||||
+ s.config.Containers = append(s.config.Containers, contConfig3)
|
||||
+ err = s.updateResources()
|
||||
+ assert.NoError(t, err)
|
||||
}
|
||||
|
||||
func TestSandboxExperimentalFeature(t *testing.T) {
|
||||
diff --git a/virtcontainers/utils/utils.go b/virtcontainers/utils/utils.go
|
||||
index 2b555ebb..3ae95aef 100644
|
||||
--- a/virtcontainers/utils/utils.go
|
||||
+++ b/virtcontainers/utils/utils.go
|
||||
@@ -25,6 +25,9 @@ const fileMode0755 = os.FileMode(0755)
|
||||
// MibToBytesShift the number to shift needed to convert MiB to Bytes
|
||||
const MibToBytesShift = 20
|
||||
|
||||
+// Max Hotplug Memory size at once time, unit is MB
|
||||
+const MaxHotplugMemMBOnceTime = 32 * 1024
|
||||
+
|
||||
// MaxSocketPathLen is the effective maximum Unix domain socket length.
|
||||
//
|
||||
// See unix(7).
|
||||
diff --git a/virtcontainers/vm.go b/virtcontainers/vm.go
|
||||
index 8d27b1fe..2e5fef44 100644
|
||||
--- a/virtcontainers/vm.go
|
||||
+++ b/virtcontainers/vm.go
|
||||
@@ -370,7 +370,7 @@ func (v *VM) AddMemory(numMB uint32) error {
|
||||
// OnlineCPUMemory puts the hotplugged CPU and memory online.
|
||||
func (v *VM) OnlineCPUMemory() error {
|
||||
v.logger().Infof("online CPU %d and memory", v.cpuDelta)
|
||||
- err := v.agent.onlineCPUMem(v.cpuDelta, false)
|
||||
+ err := v.agent.onlineCPUMem(v.cpuDelta, false, false)
|
||||
if err == nil {
|
||||
v.cpuDelta = 0
|
||||
}
|
||||
--
|
||||
2.14.3 (Apple Git-98)
|
||||
|
||||
@ -1,355 +0,0 @@
|
||||
From e6051eb1a8c54b91c46f68eab9a63ad0e73a9221 Mon Sep 17 00:00:00 2001
|
||||
From: jiangpengfei <jiangpengfei9@huawei.com>
|
||||
Date: Fri, 7 Aug 2020 20:01:15 +0800
|
||||
Subject: [PATCH 17/50] kata-runtime: validate sandbox cpu and memory size
|
||||
|
||||
reason:
|
||||
1. add more strict sandbox cpu and memory size check.
|
||||
2. use GetPhysicalCPUNumber func to replace goruntime.NumCPU(),
|
||||
because goruntime.NumCPU() return the CPU number of cpuset cgroup
|
||||
that current process joined, however kata-runtime process will
|
||||
change the cpuset cgroup when setupSanboxCgroup is called, which
|
||||
may cause goruntime.NumCPU() return value changed.
|
||||
|
||||
Signed-off-by: jiangpengfei <jiangpengfei9@huawei.com>
|
||||
---
|
||||
pkg/katautils/config.go | 39 ++++++++++++++++++++++++++++++++++---
|
||||
pkg/katautils/config_test.go | 11 ++++++++---
|
||||
virtcontainers/pkg/oci/utils.go | 11 +++++------
|
||||
virtcontainers/qemu_arm64.go | 4 ++--
|
||||
virtcontainers/qemu_arm64_test.go | 4 ++--
|
||||
virtcontainers/sandbox.go | 7 ++++++-
|
||||
virtcontainers/utils/utils.go | 41 +++++++++++++++++++++++++++++++++++++++
|
||||
7 files changed, 100 insertions(+), 17 deletions(-)
|
||||
|
||||
diff --git a/pkg/katautils/config.go b/pkg/katautils/config.go
|
||||
index d1883c94..b9b08469 100644
|
||||
--- a/pkg/katautils/config.go
|
||||
+++ b/pkg/katautils/config.go
|
||||
@@ -11,7 +11,6 @@ import (
|
||||
"fmt"
|
||||
"io/ioutil"
|
||||
"path/filepath"
|
||||
- goruntime "runtime"
|
||||
"strings"
|
||||
|
||||
"github.com/BurntSushi/toml"
|
||||
@@ -285,7 +284,7 @@ func (h hypervisor) GetEntropySource() string {
|
||||
}
|
||||
|
||||
func (h hypervisor) defaultVCPUs() uint32 {
|
||||
- numCPUs := goruntime.NumCPU()
|
||||
+ numCPUs := utils.GetPhysicalCPUNumber()
|
||||
|
||||
if h.NumVCPUs < 0 || h.NumVCPUs > int32(numCPUs) {
|
||||
return uint32(numCPUs)
|
||||
@@ -297,8 +296,22 @@ func (h hypervisor) defaultVCPUs() uint32 {
|
||||
return uint32(h.NumVCPUs)
|
||||
}
|
||||
|
||||
+func (h hypervisor) checkVCPUs() error {
|
||||
+ numCPUs := utils.GetPhysicalCPUNumber()
|
||||
+
|
||||
+ if h.NumVCPUs <= 0 {
|
||||
+ return fmt.Errorf("invalid vcpus in configuration.toml! vcpus must larger than 0")
|
||||
+ }
|
||||
+
|
||||
+ if h.NumVCPUs > int32(numCPUs) {
|
||||
+ return fmt.Errorf("invalid vcpus in configuration.toml! vcpus must smaller than max CPUs: %d in machine", numCPUs)
|
||||
+ }
|
||||
+
|
||||
+ return nil
|
||||
+}
|
||||
+
|
||||
func (h hypervisor) defaultMaxVCPUs() uint32 {
|
||||
- numcpus := uint32(goruntime.NumCPU())
|
||||
+ numcpus := uint32(utils.GetPhysicalCPUNumber())
|
||||
maxvcpus := vc.MaxQemuVCPUs()
|
||||
reqVCPUs := h.DefaultMaxVCPUs
|
||||
|
||||
@@ -324,6 +337,18 @@ func (h hypervisor) defaultMemSz() uint32 {
|
||||
return h.MemorySize
|
||||
}
|
||||
|
||||
+func (h hypervisor) checkMemSz() error {
|
||||
+ if h.MemorySize < utils.MinMemorySizeInMB {
|
||||
+ return fmt.Errorf("invalid memory size! Memory size must larger than %d MB", utils.MinMemorySizeInMB)
|
||||
+ }
|
||||
+
|
||||
+ if h.MemorySize > utils.MaxMemorySizeInMB {
|
||||
+ return fmt.Errorf("invalid memory size, memory size must smaller than %d MB", utils.MaxMemorySizeInMB)
|
||||
+ }
|
||||
+
|
||||
+ return nil
|
||||
+}
|
||||
+
|
||||
func (h hypervisor) defaultMemSlots() uint32 {
|
||||
slots := h.MemSlots
|
||||
if slots == 0 {
|
||||
@@ -627,6 +652,14 @@ func newQemuHypervisorConfig(h hypervisor) (vc.HypervisorConfig, error) {
|
||||
}
|
||||
}
|
||||
|
||||
+ if err = h.checkVCPUs(); err != nil {
|
||||
+ return vc.HypervisorConfig{}, err
|
||||
+ }
|
||||
+
|
||||
+ if err = h.checkMemSz(); err != nil {
|
||||
+ return vc.HypervisorConfig{}, err
|
||||
+ }
|
||||
+
|
||||
return vc.HypervisorConfig{
|
||||
HypervisorPath: hypervisor,
|
||||
KernelPath: kernel,
|
||||
diff --git a/pkg/katautils/config_test.go b/pkg/katautils/config_test.go
|
||||
index 31afcca6..37aa16d0 100644
|
||||
--- a/pkg/katautils/config_test.go
|
||||
+++ b/pkg/katautils/config_test.go
|
||||
@@ -14,7 +14,6 @@ import (
|
||||
"path"
|
||||
"path/filepath"
|
||||
"reflect"
|
||||
- goruntime "runtime"
|
||||
"strings"
|
||||
"syscall"
|
||||
"testing"
|
||||
@@ -152,7 +151,7 @@ func createAllRuntimeConfigFiles(dir, hypervisor string) (config testRuntimeConf
|
||||
KernelParams: vc.DeserializeParams(strings.Fields(kernelParams)),
|
||||
HypervisorMachineType: machineType,
|
||||
NumVCPUs: defaultVCPUCount,
|
||||
- DefaultMaxVCPUs: uint32(goruntime.NumCPU()),
|
||||
+ DefaultMaxVCPUs: uint32(utils.GetPhysicalCPUNumber()),
|
||||
MemorySize: defaultMemSize,
|
||||
DisableBlockDeviceUse: disableBlockDevice,
|
||||
BlockDeviceDriver: defaultBlockDeviceDriver,
|
||||
@@ -727,6 +726,8 @@ func TestMinimalRuntimeConfigWithVsock(t *testing.T) {
|
||||
[hypervisor.qemu]
|
||||
use_vsock = true
|
||||
image = "` + imagePath + `"
|
||||
+ default_vcpus = 1
|
||||
+ default_memory = 1024
|
||||
|
||||
[proxy.kata]
|
||||
path = "` + proxyPath + `"
|
||||
@@ -786,6 +787,8 @@ func TestNewQemuHypervisorConfig(t *testing.T) {
|
||||
utils.VHostVSockDevicePath = orgVHostVSockDevicePath
|
||||
}()
|
||||
utils.VHostVSockDevicePath = "/dev/abc/xyz"
|
||||
+ defaultVCPUs := int32(1)
|
||||
+ defaultMemory := uint32(1024)
|
||||
|
||||
hypervisor := hypervisor{
|
||||
Path: hypervisorPath,
|
||||
@@ -797,6 +800,8 @@ func TestNewQemuHypervisorConfig(t *testing.T) {
|
||||
HotplugVFIOOnRootBus: hotplugVFIOOnRootBus,
|
||||
PCIeRootPort: pcieRootPort,
|
||||
UseVSock: true,
|
||||
+ NumVCPUs: defaultVCPUs,
|
||||
+ MemorySize: defaultMemory,
|
||||
}
|
||||
|
||||
files := []string{hypervisorPath, kernelPath, imagePath}
|
||||
@@ -996,7 +1001,7 @@ func TestNewShimConfig(t *testing.T) {
|
||||
func TestHypervisorDefaults(t *testing.T) {
|
||||
assert := assert.New(t)
|
||||
|
||||
- numCPUs := goruntime.NumCPU()
|
||||
+ numCPUs := utils.GetPhysicalCPUNumber()
|
||||
|
||||
h := hypervisor{}
|
||||
|
||||
diff --git a/virtcontainers/pkg/oci/utils.go b/virtcontainers/pkg/oci/utils.go
|
||||
index 05181efd..0a6f08c7 100644
|
||||
--- a/virtcontainers/pkg/oci/utils.go
|
||||
+++ b/virtcontainers/pkg/oci/utils.go
|
||||
@@ -10,22 +10,21 @@ import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"path/filepath"
|
||||
- goruntime "runtime"
|
||||
"strconv"
|
||||
"strings"
|
||||
"syscall"
|
||||
|
||||
criContainerdAnnotations "github.com/containerd/cri-containerd/pkg/annotations"
|
||||
crioAnnotations "github.com/cri-o/cri-o/pkg/annotations"
|
||||
- specs "github.com/opencontainers/runtime-spec/specs-go"
|
||||
- "github.com/sirupsen/logrus"
|
||||
-
|
||||
vc "github.com/kata-containers/runtime/virtcontainers"
|
||||
"github.com/kata-containers/runtime/virtcontainers/device/config"
|
||||
exp "github.com/kata-containers/runtime/virtcontainers/experimental"
|
||||
vcAnnotations "github.com/kata-containers/runtime/virtcontainers/pkg/annotations"
|
||||
dockershimAnnotations "github.com/kata-containers/runtime/virtcontainers/pkg/annotations/dockershim"
|
||||
"github.com/kata-containers/runtime/virtcontainers/types"
|
||||
+ "github.com/kata-containers/runtime/virtcontainers/utils"
|
||||
+ specs "github.com/opencontainers/runtime-spec/specs-go"
|
||||
+ "github.com/sirupsen/logrus"
|
||||
)
|
||||
|
||||
type annotationContainerType struct {
|
||||
@@ -560,7 +559,7 @@ func addHypervisorCPUOverrides(ocispec specs.Spec, sbConfig *vc.SandboxConfig) e
|
||||
return fmt.Errorf("Error encountered parsing annotation default_vcpus: %v, please specify numeric value", err)
|
||||
}
|
||||
|
||||
- numCPUs := goruntime.NumCPU()
|
||||
+ numCPUs := utils.GetPhysicalCPUNumber()
|
||||
|
||||
if uint32(vcpus) > uint32(numCPUs) {
|
||||
return fmt.Errorf("Number of cpus %d specified in annotation default_vcpus is greater than the number of CPUs %d on the system", vcpus, numCPUs)
|
||||
@@ -575,7 +574,7 @@ func addHypervisorCPUOverrides(ocispec specs.Spec, sbConfig *vc.SandboxConfig) e
|
||||
return fmt.Errorf("Error encountered parsing annotation for default_maxvcpus: %v, please specify positive numeric value", err)
|
||||
}
|
||||
|
||||
- numCPUs := goruntime.NumCPU()
|
||||
+ numCPUs := utils.GetPhysicalCPUNumber()
|
||||
max := uint32(maxVCPUs)
|
||||
|
||||
if max > uint32(numCPUs) {
|
||||
diff --git a/virtcontainers/qemu_arm64.go b/virtcontainers/qemu_arm64.go
|
||||
index 6d089cf0..40fb2a80 100644
|
||||
--- a/virtcontainers/qemu_arm64.go
|
||||
+++ b/virtcontainers/qemu_arm64.go
|
||||
@@ -8,11 +8,11 @@ package virtcontainers
|
||||
import (
|
||||
"context"
|
||||
"io/ioutil"
|
||||
- "runtime"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
govmmQemu "github.com/intel/govmm/qemu"
|
||||
+ "github.com/kata-containers/runtime/virtcontainers/utils"
|
||||
"github.com/sirupsen/logrus"
|
||||
)
|
||||
|
||||
@@ -121,7 +121,7 @@ func MaxQemuVCPUs() uint32 {
|
||||
if hostGICVersion != 0 {
|
||||
return gicList[hostGICVersion]
|
||||
}
|
||||
- return uint32(runtime.NumCPU())
|
||||
+ return uint32(utils.GetPhysicalCPUNumber())
|
||||
}
|
||||
|
||||
func newQemuArch(config HypervisorConfig) qemuArch {
|
||||
diff --git a/virtcontainers/qemu_arm64_test.go b/virtcontainers/qemu_arm64_test.go
|
||||
index 40351158..372fc4a9 100644
|
||||
--- a/virtcontainers/qemu_arm64_test.go
|
||||
+++ b/virtcontainers/qemu_arm64_test.go
|
||||
@@ -10,10 +10,10 @@ import (
|
||||
"io/ioutil"
|
||||
"os"
|
||||
"path/filepath"
|
||||
- "runtime"
|
||||
"testing"
|
||||
|
||||
govmmQemu "github.com/intel/govmm/qemu"
|
||||
+ "github.com/kata-containers/runtime/virtcontainers/utils"
|
||||
"github.com/stretchr/testify/assert"
|
||||
)
|
||||
|
||||
@@ -63,7 +63,7 @@ func TestMaxQemuVCPUs(t *testing.T) {
|
||||
}
|
||||
|
||||
data := []testData{
|
||||
- {"", uint32(runtime.NumCPU())},
|
||||
+ {"", uint32(utils.GetPhysicalCPUNumber())},
|
||||
{" 1: 0 0 GICv2 25 Level vgic \n", uint32(8)},
|
||||
{" 1: 0 0 GICv3 25 Level vgic \n", uint32(123)},
|
||||
{" 1: 0 0 GICv4 25 Level vgic \n", uint32(123)},
|
||||
diff --git a/virtcontainers/sandbox.go b/virtcontainers/sandbox.go
|
||||
index a318d677..a8522b96 100644
|
||||
--- a/virtcontainers/sandbox.go
|
||||
+++ b/virtcontainers/sandbox.go
|
||||
@@ -1875,6 +1875,11 @@ func (s *Sandbox) updateResources() error {
|
||||
reqMemMB := uint32(sandboxMemoryByte >> utils.MibToBytesShift)
|
||||
currentMemMB := s.hypervisor.getMemorySize()
|
||||
|
||||
+ // check request sandbox memory size larger than utils.MaxMemorySizInMB or not
|
||||
+ if reqMemMB > utils.MaxMemorySizeInMB {
|
||||
+ return fmt.Errorf("updateResources failed! Sandbox memory should <= %d MiB", utils.MaxMemorySizeInMB)
|
||||
+ }
|
||||
+
|
||||
// If request hotplug memory size larger than utils.MaxHotplugMemMBOnceTime,
|
||||
// inorder to avoid hotplug memory oom problem, we need to hotplug large memory
|
||||
// with two steps. First, hotplug utils.MaxHotplugMemMBOnceTime size memory into
|
||||
@@ -1898,7 +1903,7 @@ func (s *Sandbox) updateResources() error {
|
||||
return err
|
||||
}
|
||||
}
|
||||
- if err := s.agent.onlineCPUMem(0, false, false); err != nil {
|
||||
+ if err := s.agent.onlineCPUMem(0, false, true); err != nil {
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
diff --git a/virtcontainers/utils/utils.go b/virtcontainers/utils/utils.go
|
||||
index 3ae95aef..5d38e594 100644
|
||||
--- a/virtcontainers/utils/utils.go
|
||||
+++ b/virtcontainers/utils/utils.go
|
||||
@@ -6,6 +6,7 @@
|
||||
package utils
|
||||
|
||||
import (
|
||||
+ "bufio"
|
||||
"crypto/rand"
|
||||
"errors"
|
||||
"fmt"
|
||||
@@ -28,11 +29,26 @@ const MibToBytesShift = 20
|
||||
// Max Hotplug Memory size at once time, unit is MB
|
||||
const MaxHotplugMemMBOnceTime = 32 * 1024
|
||||
|
||||
+const (
|
||||
+ // Min needed memory size to start a Kata VM
|
||||
+ MinMemorySizeInMB = 300
|
||||
+ MinMemorySizeInByte = MinMemorySizeInMB << MibToBytesShift
|
||||
+
|
||||
+ // Max support memory size in the Kata VM
|
||||
+ MaxMemorySizeInMB = 512 * 1024
|
||||
+ MaxMemorySizeInByte = MaxMemorySizeInMB << MibToBytesShift
|
||||
+)
|
||||
+
|
||||
// MaxSocketPathLen is the effective maximum Unix domain socket length.
|
||||
//
|
||||
// See unix(7).
|
||||
const MaxSocketPathLen = 107
|
||||
|
||||
+const (
|
||||
+ procCPUInfoPath = "/proc/cpuinfo"
|
||||
+ processorIdentifier = "processor"
|
||||
+)
|
||||
+
|
||||
// VHostVSockDevicePath path to vhost-vsock device
|
||||
var VHostVSockDevicePath = "/dev/vhost-vsock"
|
||||
|
||||
@@ -324,3 +340,28 @@ func IsProcessRunning(pid int, processName string, sandboxID string) bool {
|
||||
|
||||
return false
|
||||
}
|
||||
+
|
||||
+// GetPhysicalCPUNumber return the number of the CPUs in the physical machine
|
||||
+func GetPhysicalCPUNumber() int {
|
||||
+ f, err := os.Open(procCPUInfoPath)
|
||||
+ if err != nil {
|
||||
+ return 0
|
||||
+ }
|
||||
+ defer f.Close()
|
||||
+
|
||||
+ cpuNum := 0
|
||||
+ s := bufio.NewScanner(f)
|
||||
+ for s.Scan() {
|
||||
+ if err := s.Err(); err != nil {
|
||||
+ return 0
|
||||
+ }
|
||||
+
|
||||
+ fields := strings.Fields(s.Text())
|
||||
+ if len(fields) > 0 {
|
||||
+ if fields[0] == processorIdentifier {
|
||||
+ cpuNum++
|
||||
+ }
|
||||
+ }
|
||||
+ }
|
||||
+ return cpuNum
|
||||
+}
|
||||
--
|
||||
2.14.3 (Apple Git-98)
|
||||
|
||||
@ -1,50 +0,0 @@
|
||||
From c785f8f744050155102664d56de5bfb55e91915d Mon Sep 17 00:00:00 2001
|
||||
From: Evan Foster <efoster@adobe.com>
|
||||
Date: Mon, 13 Jul 2020 12:53:40 -0600
|
||||
Subject: [PATCH 18/50] sandbox: Stop and clean up containers that fail to
|
||||
create
|
||||
|
||||
A container that is created and added to a sandbox can still fail
|
||||
the final creation steps. In this case, the container must be stopped
|
||||
and have its resources cleaned up to prevent leaking sandbox mounts.
|
||||
|
||||
Fixes #2816
|
||||
|
||||
cherry-pick from: https://github.com/kata-containers/runtime/pull/2826
|
||||
|
||||
Signed-off-by: Evan Foster <efoster@adobe.com>
|
||||
Signed-off-by: jiangpengfei <jiangpengfei9@huawei.com>
|
||||
---
|
||||
virtcontainers/sandbox.go | 11 +++++++++++
|
||||
1 file changed, 11 insertions(+)
|
||||
|
||||
diff --git a/virtcontainers/sandbox.go b/virtcontainers/sandbox.go
|
||||
index a8522b96..3dbf640e 100644
|
||||
--- a/virtcontainers/sandbox.go
|
||||
+++ b/virtcontainers/sandbox.go
|
||||
@@ -1,4 +1,5 @@
|
||||
// Copyright (c) 2016 Intel Corporation
|
||||
+// Copyright (c) 2020 Adobe Inc.
|
||||
//
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
//
|
||||
@@ -1172,6 +1173,16 @@ func (s *Sandbox) CreateContainer(contConfig ContainerConfig) (VCContainer, erro
|
||||
defer func() {
|
||||
// Rollback if error happens.
|
||||
if err != nil {
|
||||
+ logger := s.Logger().WithFields(logrus.Fields{"container-id": c.id, "sandox-id": s.id, "rollback": true})
|
||||
+
|
||||
+ logger.Warning("Cleaning up partially created container")
|
||||
+
|
||||
+ if err2 := c.stop(true); err2 != nil {
|
||||
+ logger.WithError(err2).Warning("Could not delete container")
|
||||
+ }
|
||||
+
|
||||
+ logger.Debug("Removing stopped container from sandbox store")
|
||||
+
|
||||
s.removeContainer(c.id)
|
||||
}
|
||||
}()
|
||||
--
|
||||
2.14.3 (Apple Git-98)
|
||||
|
||||
@ -1,33 +0,0 @@
|
||||
From d20cb25c8a145e1d3e64eefa69242d87b7a67f92 Mon Sep 17 00:00:00 2001
|
||||
From: jiangpengfei <jiangpengfei9@huawei.com>
|
||||
Date: Sat, 8 Aug 2020 16:00:45 -0400
|
||||
Subject: [PATCH 19/50] virtcontainers: fix delete sandbox failed problem
|
||||
|
||||
fixes: #2882
|
||||
|
||||
reason: If error happens after container create and before sandbox
|
||||
updateResouce in the `CreateContainer()`, then delete sandbox
|
||||
forcefully will return error because s.config.Containers config not
|
||||
flushed into persist store.
|
||||
|
||||
Signed-off-by: jiangpengfei <jiangpengfei9@huawei.com>
|
||||
---
|
||||
virtcontainers/sandbox.go | 2 ++
|
||||
1 file changed, 2 insertions(+)
|
||||
|
||||
diff --git a/virtcontainers/sandbox.go b/virtcontainers/sandbox.go
|
||||
index 3dbf640e..7322ef58 100644
|
||||
--- a/virtcontainers/sandbox.go
|
||||
+++ b/virtcontainers/sandbox.go
|
||||
@@ -1156,6 +1156,8 @@ func (s *Sandbox) CreateContainer(contConfig ContainerConfig) (VCContainer, erro
|
||||
if len(s.config.Containers) > 0 {
|
||||
// delete container config
|
||||
s.config.Containers = s.config.Containers[:len(s.config.Containers)-1]
|
||||
+ // need to flush change to persist storage
|
||||
+ _ = s.storeSandbox()
|
||||
}
|
||||
}
|
||||
}()
|
||||
--
|
||||
2.14.3 (Apple Git-98)
|
||||
|
||||
@ -1,188 +0,0 @@
|
||||
From 75141529674545a2f84b01c730f614901ad2265e Mon Sep 17 00:00:00 2001
|
||||
From: jiangpengfei <jiangpengfei9@huawei.com>
|
||||
Date: Mon, 10 Aug 2020 10:08:47 +0800
|
||||
Subject: [PATCH 20/50] virtcontainers: add enable_cpu_memory_hotplug config in
|
||||
the configuration.toml
|
||||
|
||||
reason: add enable_cpu_memory_hotplug config to control whether enable hotplug
|
||||
memory and cpu resource into the Kata VM. If we can calculate the whole sandbox
|
||||
resource usage already, we just need to pass the value of resouces by annotation
|
||||
without hotplugging.
|
||||
|
||||
Signed-off-by: jiangpengfei <jiangpengfei9@huawei.com>
|
||||
---
|
||||
cli/config/configuration-qemu.toml.in | 4 ++++
|
||||
pkg/katautils/config.go | 2 ++
|
||||
virtcontainers/container.go | 6 ++++--
|
||||
virtcontainers/hypervisor.go | 3 +++
|
||||
virtcontainers/persist.go | 2 ++
|
||||
virtcontainers/persist/api/config.go | 3 +++
|
||||
virtcontainers/sandbox.go | 38 +++++++++++++++++++++--------------
|
||||
7 files changed, 41 insertions(+), 17 deletions(-)
|
||||
|
||||
diff --git a/cli/config/configuration-qemu.toml.in b/cli/config/configuration-qemu.toml.in
|
||||
index 82461732..b44e84d8 100644
|
||||
--- a/cli/config/configuration-qemu.toml.in
|
||||
+++ b/cli/config/configuration-qemu.toml.in
|
||||
@@ -100,6 +100,10 @@ default_memory = @DEFMEMSZ@
|
||||
# Default false
|
||||
#enable_reuse_cpu_memory = false
|
||||
|
||||
+# If enabled, the runtime will support cpu and memory hot plug
|
||||
+# (default: disabled)
|
||||
+#enable_cpu_memory_hotplug = true
|
||||
+
|
||||
# Disable block device from being used for a container's rootfs.
|
||||
# In case of a storage driver like devicemapper where a container's
|
||||
# root file system is backed by a block device, the block device is passed
|
||||
diff --git a/pkg/katautils/config.go b/pkg/katautils/config.go
|
||||
index b9b08469..9a99b9d4 100644
|
||||
--- a/pkg/katautils/config.go
|
||||
+++ b/pkg/katautils/config.go
|
||||
@@ -112,6 +112,7 @@ type hypervisor struct {
|
||||
MemSlots uint32 `toml:"memory_slots"`
|
||||
MemOffset uint32 `toml:"memory_offset"`
|
||||
EnableCPUMemoryReuse bool `toml:"enable_reuse_cpu_memory"`
|
||||
+ EnableCPUMemoryHotPlug bool `toml:"enable_cpu_memory_hotplug"`
|
||||
DefaultBridges uint32 `toml:"default_bridges"`
|
||||
Msize9p uint32 `toml:"msize_9p"`
|
||||
PCIeRootPort uint32 `toml:"pcie_root_port"`
|
||||
@@ -675,6 +676,7 @@ func newQemuHypervisorConfig(h hypervisor) (vc.HypervisorConfig, error) {
|
||||
MemSlots: h.defaultMemSlots(),
|
||||
MemOffset: h.defaultMemOffset(),
|
||||
EnableCPUMemoryReuse: h.EnableCPUMemoryReuse,
|
||||
+ EnableCPUMemoryHotPlug: h.EnableCPUMemoryHotPlug,
|
||||
VirtioMem: h.VirtioMem,
|
||||
EntropySource: h.GetEntropySource(),
|
||||
DefaultBridges: h.defaultBridges(),
|
||||
diff --git a/virtcontainers/container.go b/virtcontainers/container.go
|
||||
index 75f590eb..1b89f6ac 100644
|
||||
--- a/virtcontainers/container.go
|
||||
+++ b/virtcontainers/container.go
|
||||
@@ -1273,8 +1273,10 @@ func (c *Container) update(resources specs.LinuxResources) error {
|
||||
c.config.Resources.Memory.Limit = mem.Limit
|
||||
}
|
||||
|
||||
- if err := c.sandbox.updateResources(); err != nil {
|
||||
- return err
|
||||
+ if c.sandbox.config.HypervisorConfig.EnableCPUMemoryHotPlug {
|
||||
+ if err := c.sandbox.updateResources(); err != nil {
|
||||
+ return err
|
||||
+ }
|
||||
}
|
||||
|
||||
if !c.sandbox.config.SandboxCgroupOnly {
|
||||
diff --git a/virtcontainers/hypervisor.go b/virtcontainers/hypervisor.go
|
||||
index 9cd685ad..c0723daa 100644
|
||||
--- a/virtcontainers/hypervisor.go
|
||||
+++ b/virtcontainers/hypervisor.go
|
||||
@@ -253,6 +253,9 @@ type HypervisorConfig struct {
|
||||
// Enable hypervisor cpu and memory resource share with container
|
||||
EnableCPUMemoryReuse bool
|
||||
|
||||
+ // Enable hotplug cpu and memory resource into container
|
||||
+ EnableCPUMemoryHotPlug bool
|
||||
+
|
||||
// VirtioFSCacheSize is the DAX cache size in MiB
|
||||
VirtioFSCacheSize uint32
|
||||
|
||||
diff --git a/virtcontainers/persist.go b/virtcontainers/persist.go
|
||||
index aef4d18d..6bd09a0b 100644
|
||||
--- a/virtcontainers/persist.go
|
||||
+++ b/virtcontainers/persist.go
|
||||
@@ -215,6 +215,7 @@ func (s *Sandbox) dumpConfig(ss *persistapi.SandboxState) {
|
||||
MemSlots: sconfig.HypervisorConfig.MemSlots,
|
||||
MemOffset: sconfig.HypervisorConfig.MemOffset,
|
||||
EnableCPUMemoryReuse: sconfig.HypervisorConfig.EnableCPUMemoryReuse,
|
||||
+ EnableCPUMemoryHotPlug: sconfig.HypervisorConfig.EnableCPUMemoryHotPlug,
|
||||
VirtioMem: sconfig.HypervisorConfig.VirtioMem,
|
||||
VirtioFSCacheSize: sconfig.HypervisorConfig.VirtioFSCacheSize,
|
||||
KernelPath: sconfig.HypervisorConfig.KernelPath,
|
||||
@@ -505,6 +506,7 @@ func loadSandboxConfig(id string) (*SandboxConfig, error) {
|
||||
MemSlots: hconf.MemSlots,
|
||||
MemOffset: hconf.MemOffset,
|
||||
EnableCPUMemoryReuse: hconf.EnableCPUMemoryReuse,
|
||||
+ EnableCPUMemoryHotPlug: hconf.EnableCPUMemoryHotPlug,
|
||||
VirtioMem: hconf.VirtioMem,
|
||||
VirtioFSCacheSize: hconf.VirtioFSCacheSize,
|
||||
KernelPath: hconf.KernelPath,
|
||||
diff --git a/virtcontainers/persist/api/config.go b/virtcontainers/persist/api/config.go
|
||||
index a3c6ec91..cfbee849 100644
|
||||
--- a/virtcontainers/persist/api/config.go
|
||||
+++ b/virtcontainers/persist/api/config.go
|
||||
@@ -38,6 +38,9 @@ type HypervisorConfig struct {
|
||||
// Enable hypervisor cpu and memory resource share with container
|
||||
EnableCPUMemoryReuse bool
|
||||
|
||||
+ // Enable hotplug cpu and memory resource into container
|
||||
+ EnableCPUMemoryHotPlug bool
|
||||
+
|
||||
// VirtioFSCacheSize is the DAX cache size in MiB
|
||||
VirtioFSCacheSize uint32
|
||||
|
||||
diff --git a/virtcontainers/sandbox.go b/virtcontainers/sandbox.go
|
||||
index 7322ef58..8fcd92d4 100644
|
||||
--- a/virtcontainers/sandbox.go
|
||||
+++ b/virtcontainers/sandbox.go
|
||||
@@ -1189,12 +1189,16 @@ func (s *Sandbox) CreateContainer(contConfig ContainerConfig) (VCContainer, erro
|
||||
}
|
||||
}()
|
||||
|
||||
- // Sandbox is reponsable to update VM resources needed by Containers
|
||||
- // Update resources after having added containers to the sandbox, since
|
||||
- // container status is requiered to know if more resources should be added.
|
||||
- err = s.updateResources()
|
||||
- if err != nil {
|
||||
- return nil, err
|
||||
+ // update sandbox resource only when enable_cpu_memory_hotplug config set true
|
||||
+ // in the config.toml file
|
||||
+ if s.config.HypervisorConfig.EnableCPUMemoryHotPlug {
|
||||
+ // Sandbox is reponsable to update VM resources needed by Containers
|
||||
+ // Update resources after having added containers to the sandbox, since
|
||||
+ // container status is requiered to know if more resources should be added.
|
||||
+ err = s.updateResources()
|
||||
+ if err != nil {
|
||||
+ return nil, err
|
||||
+ }
|
||||
}
|
||||
|
||||
if err = s.cgroupsUpdate(); err != nil {
|
||||
@@ -1228,11 +1232,13 @@ func (s *Sandbox) StartContainer(containerID string) (VCContainer, error) {
|
||||
|
||||
s.Logger().Info("Container is started")
|
||||
|
||||
- // Update sandbox resources in case a stopped container
|
||||
- // is started
|
||||
- err = s.updateResources()
|
||||
- if err != nil {
|
||||
- return nil, err
|
||||
+ if s.config.HypervisorConfig.EnableCPUMemoryHotPlug {
|
||||
+ // Update sandbox resources in case a stopped container
|
||||
+ // is started
|
||||
+ err = s.updateResources()
|
||||
+ if err != nil {
|
||||
+ return nil, err
|
||||
+ }
|
||||
}
|
||||
|
||||
return c, nil
|
||||
@@ -1503,10 +1509,12 @@ func (s *Sandbox) createContainers() error {
|
||||
}
|
||||
}
|
||||
|
||||
- // Update resources after having added containers to the sandbox, since
|
||||
- // container status is requiered to know if more resources should be added.
|
||||
- if err := s.updateResources(); err != nil {
|
||||
- return err
|
||||
+ if s.config.HypervisorConfig.EnableCPUMemoryHotPlug {
|
||||
+ // Update resources after having added containers to the sandbox, since
|
||||
+ // container status is requiered to know if more resources should be added.
|
||||
+ if err := s.updateResources(); err != nil {
|
||||
+ return err
|
||||
+ }
|
||||
}
|
||||
|
||||
if err := s.cgroupsUpdate(); err != nil {
|
||||
--
|
||||
2.14.3 (Apple Git-98)
|
||||
|
||||
@ -1,195 +0,0 @@
|
||||
From 79cf2f5a52af51d8a62353a99e894808281769e2 Mon Sep 17 00:00:00 2001
|
||||
From: jiangpengfei <jiangpengfei9@huawei.com>
|
||||
Date: Mon, 10 Aug 2020 11:25:02 +0800
|
||||
Subject: [PATCH 21/50] kata-runtime: add sandbox_cpu and sandbox_mem
|
||||
annotations
|
||||
|
||||
reason: add sandbox_cpu and sandbox_men annotations to set
|
||||
Kata VM vCPU number and memory size.
|
||||
|
||||
Signed-off-by: jiangpengfei <jiangpengfei9@huawei.com>
|
||||
---
|
||||
virtcontainers/pkg/annotations/annotations.go | 6 +++++
|
||||
virtcontainers/pkg/oci/utils.go | 39 +++++++++++++++++++++++++--
|
||||
virtcontainers/sandbox.go | 32 ++++++++++++++++++++++
|
||||
virtcontainers/utils/utils.go | 14 ++++++++++
|
||||
4 files changed, 89 insertions(+), 2 deletions(-)
|
||||
|
||||
diff --git a/virtcontainers/pkg/annotations/annotations.go b/virtcontainers/pkg/annotations/annotations.go
|
||||
index 10ce7833..903c7f03 100644
|
||||
--- a/virtcontainers/pkg/annotations/annotations.go
|
||||
+++ b/virtcontainers/pkg/annotations/annotations.go
|
||||
@@ -251,6 +251,12 @@ const (
|
||||
ContainerPipeSizeKernelParam = "agent." + ContainerPipeSizeOption
|
||||
)
|
||||
|
||||
+// iSula self defined annotations
|
||||
+const (
|
||||
+ StaticCPUTypeKey = kataAnnotationsPrefix + "sandbox_cpu"
|
||||
+ StaticMemTypeKey = kataAnnotationsPrefix + "sandbox_mem"
|
||||
+)
|
||||
+
|
||||
const (
|
||||
// SHA512 is the SHA-512 (64) hash algorithm
|
||||
SHA512 string = "sha512"
|
||||
diff --git a/virtcontainers/pkg/oci/utils.go b/virtcontainers/pkg/oci/utils.go
|
||||
index 0a6f08c7..36c730b7 100644
|
||||
--- a/virtcontainers/pkg/oci/utils.go
|
||||
+++ b/virtcontainers/pkg/oci/utils.go
|
||||
@@ -16,6 +16,7 @@ import (
|
||||
|
||||
criContainerdAnnotations "github.com/containerd/cri-containerd/pkg/annotations"
|
||||
crioAnnotations "github.com/cri-o/cri-o/pkg/annotations"
|
||||
+ "github.com/docker/go-units"
|
||||
vc "github.com/kata-containers/runtime/virtcontainers"
|
||||
"github.com/kata-containers/runtime/virtcontainers/device/config"
|
||||
exp "github.com/kata-containers/runtime/virtcontainers/experimental"
|
||||
@@ -34,7 +35,10 @@ type annotationContainerType struct {
|
||||
|
||||
type annotationHandler func(value string) error
|
||||
|
||||
-var annotationHandlerList = map[string]annotationHandler{}
|
||||
+var annotationHandlerList = map[string]annotationHandler{
|
||||
+ vcAnnotations.StaticCPUTypeKey: validateSandboxCPU,
|
||||
+ vcAnnotations.StaticMemTypeKey: validateSandboxMem,
|
||||
+}
|
||||
|
||||
var (
|
||||
// ErrNoLinux is an error for missing Linux sections in the OCI configuration file.
|
||||
@@ -1036,7 +1040,10 @@ func validateOtherSandboxAnnotations(annotation, value string) error {
|
||||
|
||||
// addOtherSandboxAnnotation add self defined annotation for sandbox
|
||||
func addOtherSandboxAnnotation(ocispec specs.Spec, sbConfig *vc.SandboxConfig) error {
|
||||
- otherSandboxAnnotationsKey := []string{}
|
||||
+ otherSandboxAnnotationsKey := []string{
|
||||
+ vcAnnotations.StaticCPUTypeKey,
|
||||
+ vcAnnotations.StaticMemTypeKey,
|
||||
+ }
|
||||
|
||||
for _, a := range otherSandboxAnnotationsKey {
|
||||
value, ok := ocispec.Annotations[a]
|
||||
@@ -1053,3 +1060,31 @@ func addOtherSandboxAnnotation(ocispec specs.Spec, sbConfig *vc.SandboxConfig) e
|
||||
|
||||
return nil
|
||||
}
|
||||
+
|
||||
+func validateSandboxCPU(value string) error {
|
||||
+ // check min cpu value
|
||||
+ cpus, err := utils.RoundVCPUNumber(value)
|
||||
+ if err != nil {
|
||||
+ return fmt.Errorf("valiate sandbox_cpu annotation fail: %v", err)
|
||||
+ }
|
||||
+
|
||||
+ maxPhysicalCPUs := utils.GetPhysicalCPUNumber()
|
||||
+ if cpus > maxPhysicalCPUs {
|
||||
+ return fmt.Errorf("sandbox_cpu annotation value exceed the machine max CPU number: %d", cpus)
|
||||
+ }
|
||||
+
|
||||
+ return nil
|
||||
+}
|
||||
+
|
||||
+func validateSandboxMem(value string) error {
|
||||
+ memSizeInBytes, err := units.RAMInBytes(value)
|
||||
+ if err != nil {
|
||||
+ return fmt.Errorf("parse sandbox_mem value: %d fail: %v", memSizeInBytes, err)
|
||||
+ }
|
||||
+
|
||||
+ if memSizeInBytes < utils.MinMemorySizeInByte || memSizeInBytes > utils.MaxMemorySizeInByte {
|
||||
+ return fmt.Errorf("invalid sandbox_mem value size in bytes: %v", memSizeInBytes)
|
||||
+ }
|
||||
+
|
||||
+ return nil
|
||||
+}
|
||||
diff --git a/virtcontainers/sandbox.go b/virtcontainers/sandbox.go
|
||||
index 8fcd92d4..ba704249 100644
|
||||
--- a/virtcontainers/sandbox.go
|
||||
+++ b/virtcontainers/sandbox.go
|
||||
@@ -20,6 +20,7 @@ import (
|
||||
|
||||
"github.com/containerd/cgroups"
|
||||
"github.com/containernetworking/plugins/pkg/ns"
|
||||
+ "github.com/docker/go-units"
|
||||
"github.com/kata-containers/agent/protocols/grpc"
|
||||
"github.com/kata-containers/runtime/virtcontainers/device/api"
|
||||
"github.com/kata-containers/runtime/virtcontainers/device/config"
|
||||
@@ -479,6 +480,10 @@ func createSandbox(ctx context.Context, sandboxConfig SandboxConfig, factory Fac
|
||||
return nil, err
|
||||
}
|
||||
|
||||
+ if err := updateStaticSandboxResources(&sandboxConfig); err != nil {
|
||||
+ return nil, err
|
||||
+ }
|
||||
+
|
||||
s, err := newSandbox(ctx, sandboxConfig, factory)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
@@ -2359,3 +2364,30 @@ func (s *Sandbox) setContainersState(state types.StateString) error {
|
||||
|
||||
return nil
|
||||
}
|
||||
+
|
||||
+// updateStaticSandboxResources update sandbox's cpu and memory resource passed by
|
||||
+// sandbox_cpu and sandbox_mem annotations
|
||||
+func updateStaticSandboxResources(sandboxConfig *SandboxConfig) error {
|
||||
+ // update cpu resource
|
||||
+ if cpuNumVal, ok := sandboxConfig.Annotations[annotations.StaticCPUTypeKey]; ok {
|
||||
+ cpuNum, err := utils.RoundVCPUNumber(cpuNumVal)
|
||||
+ if err != nil {
|
||||
+ return err
|
||||
+ }
|
||||
+
|
||||
+ sandboxConfig.HypervisorConfig.NumVCPUs = (uint32)(cpuNum)
|
||||
+ }
|
||||
+
|
||||
+ // update mem resource
|
||||
+ if memVal, ok := sandboxConfig.Annotations[annotations.StaticMemTypeKey]; ok {
|
||||
+ memSizeInBytes, err := units.RAMInBytes(memVal)
|
||||
+ if err != nil {
|
||||
+ return err
|
||||
+ }
|
||||
+
|
||||
+ memSizeInMB := memSizeInBytes >> utils.MibToBytesShift
|
||||
+ sandboxConfig.HypervisorConfig.MemorySize = (uint32)(memSizeInMB)
|
||||
+ }
|
||||
+
|
||||
+ return nil
|
||||
+}
|
||||
diff --git a/virtcontainers/utils/utils.go b/virtcontainers/utils/utils.go
|
||||
index 5d38e594..9490faa1 100644
|
||||
--- a/virtcontainers/utils/utils.go
|
||||
+++ b/virtcontainers/utils/utils.go
|
||||
@@ -11,6 +11,7 @@ import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"io/ioutil"
|
||||
+ "math"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
@@ -30,6 +31,9 @@ const MibToBytesShift = 20
|
||||
const MaxHotplugMemMBOnceTime = 32 * 1024
|
||||
|
||||
const (
|
||||
+ // minCPUs is allowed minimum CPU
|
||||
+ minCPUs = 0.25
|
||||
+
|
||||
// Min needed memory size to start a Kata VM
|
||||
MinMemorySizeInMB = 300
|
||||
MinMemorySizeInByte = MinMemorySizeInMB << MibToBytesShift
|
||||
@@ -365,3 +369,13 @@ func GetPhysicalCPUNumber() int {
|
||||
}
|
||||
return cpuNum
|
||||
}
|
||||
+
|
||||
+func RoundVCPUNumber(value string) (int, error) {
|
||||
+ cpuNum, err := strconv.ParseFloat(value, 64)
|
||||
+ if err != nil || cpuNum < minCPUs {
|
||||
+ return 0, fmt.Errorf("invalid sandbox cpu number: %v", cpuNum)
|
||||
+ }
|
||||
+
|
||||
+ cpus := int(math.Ceil(cpuNum))
|
||||
+ return cpus, nil
|
||||
+}
|
||||
--
|
||||
2.14.3 (Apple Git-98)
|
||||
|
||||
@ -1,39 +0,0 @@
|
||||
From e5e3232f7268110f7e3e3c4814eab31a6704b672 Mon Sep 17 00:00:00 2001
|
||||
From: jiangpengfei <jiangpengfei9@huawei.com>
|
||||
Date: Mon, 10 Aug 2020 20:11:07 +0800
|
||||
Subject: [PATCH 22/50] kata-runtime: skip go version check and do not build
|
||||
containerd-shim-v2
|
||||
|
||||
reason: skip go version check and do not build containerd-shim-v2
|
||||
because iSulad current not support shimV2
|
||||
|
||||
Signed-off-by: jiangpengfei <jiangpengfei9@huawei.com>
|
||||
---
|
||||
Makefile | 4 ++--
|
||||
1 file changed, 2 insertions(+), 2 deletions(-)
|
||||
|
||||
diff --git a/Makefile b/Makefile
|
||||
index 14a0ea47..d5e4bbe1 100644
|
||||
--- a/Makefile
|
||||
+++ b/Makefile
|
||||
@@ -12,7 +12,7 @@ for file in /etc/os-release /usr/lib/os-release; do \
|
||||
fi \
|
||||
done)
|
||||
|
||||
-SKIP_GO_VERSION_CHECK=
|
||||
+SKIP_GO_VERSION_CHECK=y
|
||||
include golang.mk
|
||||
|
||||
#Get ARCH.
|
||||
@@ -503,7 +503,7 @@ define SHOW_ARCH
|
||||
$(shell printf "\\t%s%s\\\n" "$(1)" $(if $(filter $(ARCH),$(1))," (default)",""))
|
||||
endef
|
||||
|
||||
-all: runtime containerd-shim-v2 netmon
|
||||
+all: runtime netmon
|
||||
|
||||
# Targets that depend on .git-commit can use $(shell cat .git-commit) to get a
|
||||
# git revision string. They will only be rebuilt if the revision string
|
||||
--
|
||||
2.14.3 (Apple Git-98)
|
||||
|
||||
@ -1,42 +0,0 @@
|
||||
From 226c3336dcc70bd17e3471ff98106a2f8dee9ac5 Mon Sep 17 00:00:00 2001
|
||||
From: jiangpengfei <jiangpengfei9@huawei.com>
|
||||
Date: Tue, 11 Aug 2020 22:32:49 +0800
|
||||
Subject: [PATCH 23/50] kata-runtime: set PCIBridgeMaxCapacity limit to 25
|
||||
|
||||
reason: set PCIBridgeMaxCapacity limit to 25.
|
||||
|
||||
Signed-off-by: jiangpengfei <jiangpengfei9@huawei.com>
|
||||
---
|
||||
virtcontainers/qemu_arch_base_test.go | 2 +-
|
||||
virtcontainers/types/bridges.go | 2 +-
|
||||
2 files changed, 2 insertions(+), 2 deletions(-)
|
||||
|
||||
diff --git a/virtcontainers/qemu_arch_base_test.go b/virtcontainers/qemu_arch_base_test.go
|
||||
index 169e002e..8219f3c5 100644
|
||||
--- a/virtcontainers/qemu_arch_base_test.go
|
||||
+++ b/virtcontainers/qemu_arch_base_test.go
|
||||
@@ -175,7 +175,7 @@ func TestQemuAddDeviceToBridge(t *testing.T) {
|
||||
}
|
||||
|
||||
// fail to add device to bridge cause no more available bridge slot
|
||||
- _, _, err := q.addDeviceToBridge("qemu-bridge-31", types.PCI)
|
||||
+ _, _, err := q.addDeviceToBridge("qemu-bridge-26", types.PCI)
|
||||
exceptErr := errors.New("no more bridge slots available")
|
||||
assert.Equal(exceptErr.Error(), err.Error())
|
||||
|
||||
diff --git a/virtcontainers/types/bridges.go b/virtcontainers/types/bridges.go
|
||||
index cb15a88f..c3538ce4 100644
|
||||
--- a/virtcontainers/types/bridges.go
|
||||
+++ b/virtcontainers/types/bridges.go
|
||||
@@ -10,7 +10,7 @@ import "fmt"
|
||||
// Type represents a type of bus and bridge.
|
||||
type Type string
|
||||
|
||||
-const PCIBridgeMaxCapacity = 30
|
||||
+const PCIBridgeMaxCapacity = 25
|
||||
|
||||
const (
|
||||
// PCI represents a PCI bus and bridge
|
||||
--
|
||||
2.14.3 (Apple Git-98)
|
||||
|
||||
@ -1,462 +0,0 @@
|
||||
From e861f426c9e6702e820348ddc61b18013c853402 Mon Sep 17 00:00:00 2001
|
||||
From: jiangpengfei <jiangpengfei9@huawei.com>
|
||||
Date: Thu, 13 Aug 2020 11:24:58 +0800
|
||||
Subject: [PATCH 24/50] kata-runtime: support hotplug tap interface into kata
|
||||
VM
|
||||
|
||||
reason: support hotplug exist tap interface or a new created tap
|
||||
interface into kata VM.
|
||||
|
||||
Signed-off-by: jiangpengfei <jiangpengfei9@huawei.com>
|
||||
---
|
||||
cli/network.go | 45 ++++++++++++++---
|
||||
virtcontainers/kata_agent.go | 12 ++++-
|
||||
virtcontainers/network.go | 100 +++++++++++++++++++++++---------------
|
||||
virtcontainers/pkg/types/types.go | 16 +++---
|
||||
virtcontainers/qemu.go | 11 +++--
|
||||
virtcontainers/sandbox.go | 24 +++++++--
|
||||
virtcontainers/tap_endpoint.go | 23 +++++++--
|
||||
7 files changed, 161 insertions(+), 70 deletions(-)
|
||||
|
||||
diff --git a/cli/network.go b/cli/network.go
|
||||
index 881a2358..7e7791f1 100644
|
||||
--- a/cli/network.go
|
||||
+++ b/cli/network.go
|
||||
@@ -26,6 +26,8 @@ const (
|
||||
routeType
|
||||
)
|
||||
|
||||
+const defaultLinkType = "tap"
|
||||
+
|
||||
var kataNetworkCLICommand = cli.Command{
|
||||
Name: "kata-network",
|
||||
Usage: "manage interfaces and routes for container",
|
||||
@@ -42,10 +44,22 @@ var kataNetworkCLICommand = cli.Command{
|
||||
}
|
||||
|
||||
var addIfaceCommand = cli.Command{
|
||||
- Name: "add-iface",
|
||||
- Usage: "add an interface to a container",
|
||||
- ArgsUsage: `add-iface <container-id> file or - for stdin`,
|
||||
- Flags: []cli.Flag{},
|
||||
+ Name: "add-iface",
|
||||
+ Usage: "add an interface to a container",
|
||||
+ ArgsUsage: `add-iface <container-id> file or - for stdin
|
||||
+ file or stdin for example:
|
||||
+ {
|
||||
+ "device":"<device-name>",
|
||||
+ "name":"<interface-name>",
|
||||
+ "IPAddresses":[{"address":"<ip>","mask":"<mask>"}],
|
||||
+ "mtu":<mtu>,
|
||||
+ "hwAddr":"<mac>",
|
||||
+ "linkType":"tap",
|
||||
+ "vhostUserSocket":"<path>"
|
||||
+ }
|
||||
+ device,name,mtu,hwAddr are required, IPAddresses and vhostUserSocket are optional.
|
||||
+ `,
|
||||
+ Flags: []cli.Flag{},
|
||||
Action: func(context *cli.Context) error {
|
||||
ctx, err := cliContextToContext(context)
|
||||
if err != nil {
|
||||
@@ -57,10 +71,20 @@ var addIfaceCommand = cli.Command{
|
||||
}
|
||||
|
||||
var delIfaceCommand = cli.Command{
|
||||
- Name: "del-iface",
|
||||
- Usage: "delete an interface from a container",
|
||||
- ArgsUsage: `del-iface <container-id> file or - for stdin`,
|
||||
- Flags: []cli.Flag{},
|
||||
+ Name: "del-iface",
|
||||
+ Usage: "delete an interface from a container",
|
||||
+ ArgsUsage: `del-iface <container-id> file or - for stdin
|
||||
+ file or stdin for example:
|
||||
+ {
|
||||
+ "device":"",
|
||||
+ "name":"<interface-name>",
|
||||
+ "IPAddresses":[],
|
||||
+ "mtu":0,
|
||||
+ "hwAddr":""
|
||||
+ }
|
||||
+ Only the "name" field is required.
|
||||
+ `,
|
||||
+ Flags: []cli.Flag{},
|
||||
Action: func(context *cli.Context) error {
|
||||
ctx, err := cliContextToContext(context)
|
||||
if err != nil {
|
||||
@@ -156,6 +180,11 @@ func networkModifyCommand(ctx context.Context, containerID, input string, opType
|
||||
if err = json.NewDecoder(f).Decode(&inf); err != nil {
|
||||
return err
|
||||
}
|
||||
+
|
||||
+ if len(inf.LinkType) == 0 {
|
||||
+ inf.LinkType = defaultLinkType
|
||||
+ }
|
||||
+
|
||||
if add {
|
||||
resultingInf, err = vci.AddInterface(ctx, sandboxID, inf)
|
||||
if err != nil {
|
||||
diff --git a/virtcontainers/kata_agent.go b/virtcontainers/kata_agent.go
|
||||
index 8e073339..dfdd263b 100644
|
||||
--- a/virtcontainers/kata_agent.go
|
||||
+++ b/virtcontainers/kata_agent.go
|
||||
@@ -600,8 +600,16 @@ func (k *kataAgent) updateInterface(ifc *vcTypes.Interface) (*vcTypes.Interface,
|
||||
"resulting-interface": fmt.Sprintf("%+v", resultingInterface),
|
||||
}).WithError(err).Error("update interface request failed")
|
||||
}
|
||||
- if resultInterface, ok := resultingInterface.(*vcTypes.Interface); ok {
|
||||
- return resultInterface, err
|
||||
+ if resultInterface, ok := resultingInterface.(*aTypes.Interface); ok {
|
||||
+ iface := &vcTypes.Interface{
|
||||
+ Device: resultInterface.Device,
|
||||
+ Name: resultInterface.Name,
|
||||
+ IPAddresses: k.convertToIPAddresses(resultInterface.IPAddresses),
|
||||
+ Mtu: resultInterface.Mtu,
|
||||
+ HwAddr: resultInterface.HwAddr,
|
||||
+ PciAddr: resultInterface.PciAddr,
|
||||
+ }
|
||||
+ return iface, err
|
||||
}
|
||||
return nil, err
|
||||
}
|
||||
diff --git a/virtcontainers/network.go b/virtcontainers/network.go
|
||||
index d70c5360..e909a822 100644
|
||||
--- a/virtcontainers/network.go
|
||||
+++ b/virtcontainers/network.go
|
||||
@@ -117,6 +117,7 @@ type NetlinkIface struct {
|
||||
// NetworkInfo gathers all information related to a network interface.
|
||||
// It can be used to store the description of the underlying network.
|
||||
type NetworkInfo struct {
|
||||
+ Device string
|
||||
Iface NetlinkIface
|
||||
Addrs []netlink.Addr
|
||||
Routes []netlink.Route
|
||||
@@ -303,6 +304,20 @@ func createLink(netHandle *netlink.Handle, name string, expectedLink netlink.Lin
|
||||
var newLink netlink.Link
|
||||
var fds []*os.File
|
||||
|
||||
+ // check if tapname exists, if true, use existing one instead of create new one
|
||||
+ if name != "" {
|
||||
+ retLink, err := netlink.LinkByName(name)
|
||||
+ // link exist, use it
|
||||
+ if err == nil {
|
||||
+ networkLogger().Debugf("exist tap device is found, instead of creating new one")
|
||||
+ tuntapLink, ok := retLink.(*netlink.Tuntap)
|
||||
+ if ok {
|
||||
+ fds = tuntapLink.Fds
|
||||
+ }
|
||||
+ return retLink, nil, nil
|
||||
+ }
|
||||
+ }
|
||||
+
|
||||
switch expectedLink.Type() {
|
||||
case (&netlink.Tuntap{}).Type():
|
||||
flags := netlink.TUNTAP_VNET_HDR
|
||||
@@ -1156,7 +1171,7 @@ func createEndpoint(netInfo NetworkInfo, idx int, model NetInterworkingModel, li
|
||||
|
||||
// Check if interface is a physical interface. Do not create
|
||||
// tap interface/bridge if it is.
|
||||
- isPhysical, err := isPhysicalIface(netInfo.Iface.Name)
|
||||
+ isPhysical, err := isPhysicalIface(netInfo.Device)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@@ -1164,49 +1179,54 @@ func createEndpoint(netInfo NetworkInfo, idx int, model NetInterworkingModel, li
|
||||
if isPhysical {
|
||||
networkLogger().WithField("interface", netInfo.Iface.Name).Info("Physical network interface found")
|
||||
endpoint, err = createPhysicalEndpoint(netInfo)
|
||||
- } else {
|
||||
- var socketPath string
|
||||
+ return endpoint, err
|
||||
+ }
|
||||
|
||||
- // Check if this is a dummy interface which has a vhost-user socket associated with it
|
||||
- socketPath, err = vhostUserSocketPath(netInfo)
|
||||
- if err != nil {
|
||||
- return nil, err
|
||||
- }
|
||||
+ // Check if this is a dummy interface which has a vhost-user socket associated with it
|
||||
+ socketPath, err := vhostUserSocketPath(netInfo)
|
||||
+ if err != nil {
|
||||
+ return nil, err
|
||||
+ }
|
||||
|
||||
- if socketPath != "" {
|
||||
- networkLogger().WithField("interface", netInfo.Iface.Name).Info("VhostUser network interface found")
|
||||
- endpoint, err = createVhostUserEndpoint(netInfo, socketPath)
|
||||
- } else if netInfo.Iface.Type == "macvlan" {
|
||||
- networkLogger().Infof("macvlan interface found")
|
||||
- endpoint, err = createBridgedMacvlanNetworkEndpoint(idx, netInfo.Iface.Name, model)
|
||||
- } else if netInfo.Iface.Type == "macvtap" {
|
||||
- networkLogger().Infof("macvtap interface found")
|
||||
- endpoint, err = createMacvtapNetworkEndpoint(netInfo)
|
||||
- } else if netInfo.Iface.Type == "tap" {
|
||||
- networkLogger().Info("tap interface found")
|
||||
- endpoint, err = createTapNetworkEndpoint(idx, netInfo.Iface.Name)
|
||||
- } else if netInfo.Iface.Type == "tuntap" {
|
||||
- if link != nil {
|
||||
- switch link.(*netlink.Tuntap).Mode {
|
||||
- case 0:
|
||||
- // mount /sys/class/net to get links
|
||||
- return nil, fmt.Errorf("Network device mode not determined correctly. Mount sysfs in caller")
|
||||
- case 1:
|
||||
- return nil, fmt.Errorf("tun networking device not yet supported")
|
||||
- case 2:
|
||||
- networkLogger().Info("tuntap tap interface found")
|
||||
- endpoint, err = createTuntapNetworkEndpoint(idx, netInfo.Iface.Name, netInfo.Iface.HardwareAddr, model)
|
||||
- default:
|
||||
- return nil, fmt.Errorf("tuntap network %v mode unsupported", link.(*netlink.Tuntap).Mode)
|
||||
- }
|
||||
+ if socketPath != "" {
|
||||
+ networkLogger().WithField("interface", netInfo.Iface.Name).Info("VhostUser network interface found")
|
||||
+ endpoint, err = createVhostUserEndpoint(netInfo, socketPath)
|
||||
+ return endpoint, err
|
||||
+ }
|
||||
+
|
||||
+ // We should create tap interface/bridge of other interface type.
|
||||
+ networkLogger().Infof("%s interface found", netInfo.Iface.Type)
|
||||
+ switch netInfo.Iface.Type {
|
||||
+ case "macvlan":
|
||||
+ networkLogger().Infof("macvlan interface found")
|
||||
+ endpoint, err = createBridgedMacvlanNetworkEndpoint(idx, netInfo.Iface.Name, model)
|
||||
+ case "macvtap":
|
||||
+ networkLogger().Infof("macvtap interface found")
|
||||
+ endpoint, err = createMacvtapNetworkEndpoint(netInfo)
|
||||
+ case "tap":
|
||||
+ networkLogger().Info("tap interface found")
|
||||
+ endpoint, err = createTapNetworkEndpoint(idx, netInfo.Iface.Name, netInfo.Device)
|
||||
+ case "tuntap":
|
||||
+ if link != nil {
|
||||
+ switch link.(*netlink.Tuntap).Mode {
|
||||
+ case 0:
|
||||
+ // mount /sys/class/net to get links
|
||||
+ return nil, fmt.Errorf("Network device mode not determined correctly. Mount sysfs in caller")
|
||||
+ case 1:
|
||||
+ return nil, fmt.Errorf("tun networking device not yet supported")
|
||||
+ case 2:
|
||||
+ networkLogger().Info("tuntap tap interface found")
|
||||
+ endpoint, err = createTuntapNetworkEndpoint(idx, netInfo.Iface.Name, netInfo.Iface.HardwareAddr, model)
|
||||
+ default:
|
||||
+ return nil, fmt.Errorf("tuntap network %v mode unsupported", link.(*netlink.Tuntap).Mode)
|
||||
}
|
||||
- } else if netInfo.Iface.Type == "veth" {
|
||||
- endpoint, err = createVethNetworkEndpoint(idx, netInfo.Iface.Name, model)
|
||||
- } else if netInfo.Iface.Type == "ipvlan" {
|
||||
- endpoint, err = createIPVlanNetworkEndpoint(idx, netInfo.Iface.Name)
|
||||
- } else {
|
||||
- return nil, fmt.Errorf("Unsupported network interface: %s", netInfo.Iface.Type)
|
||||
}
|
||||
+ case "veth":
|
||||
+ endpoint, err = createVethNetworkEndpoint(idx, netInfo.Iface.Name, model)
|
||||
+ case "ipvlan":
|
||||
+ endpoint, err = createIPVlanNetworkEndpoint(idx, netInfo.Iface.Name)
|
||||
+ default:
|
||||
+ err = fmt.Errorf("Unsupported network interface, %s", netInfo.Iface.Type)
|
||||
}
|
||||
|
||||
return endpoint, err
|
||||
diff --git a/virtcontainers/pkg/types/types.go b/virtcontainers/pkg/types/types.go
|
||||
index 0d4a9cfa..fcc63d84 100644
|
||||
--- a/virtcontainers/pkg/types/types.go
|
||||
+++ b/virtcontainers/pkg/types/types.go
|
||||
@@ -14,21 +14,21 @@ type IPAddress struct {
|
||||
|
||||
// Interface describes a network interface.
|
||||
type Interface struct {
|
||||
- Device string
|
||||
- Name string
|
||||
- IPAddresses []*IPAddress
|
||||
- Mtu uint64
|
||||
- RawFlags uint32
|
||||
- HwAddr string
|
||||
+ Device string `json:"device,omitempty"`
|
||||
+ Name string `json:"name,omitempty"`
|
||||
+ IPAddresses []*IPAddress `json:"IPAddresses,omitempty"`
|
||||
+ Mtu uint64 `json:"mtu,omitempty"`
|
||||
+ RawFlags uint32 `json:"rawFlags,omitempty"`
|
||||
+ HwAddr string `json:"hwAddr,omitempty"`
|
||||
// pciAddr is the PCI address in the format "bridgeAddr/deviceAddr".
|
||||
// Here, bridgeAddr is the address at which the bridge is attached on the root bus,
|
||||
// while deviceAddr is the address at which the network device is attached on the bridge.
|
||||
- PciAddr string
|
||||
+ PciAddr string `json:"pciAddr,omitempty"`
|
||||
// LinkType defines the type of interface described by this structure.
|
||||
// The expected values are the one that are defined by the netlink
|
||||
// library, regarding each type of link. Here is a non exhaustive
|
||||
// list: "veth", "macvtap", "vlan", "macvlan", "tap", ...
|
||||
- LinkType string
|
||||
+ LinkType string `json:"linkType,omitempty"`
|
||||
}
|
||||
|
||||
// Route describes a network route.
|
||||
diff --git a/virtcontainers/qemu.go b/virtcontainers/qemu.go
|
||||
index 7bae3278..bb83b1bb 100644
|
||||
--- a/virtcontainers/qemu.go
|
||||
+++ b/virtcontainers/qemu.go
|
||||
@@ -1361,7 +1361,7 @@ func (q *qemu) hotplugVFIODevice(device *config.VFIODev, op operation) (err erro
|
||||
return nil
|
||||
}
|
||||
|
||||
-func (q *qemu) hotAddNetDevice(name, hardAddr string, VMFds, VhostFds []*os.File) error {
|
||||
+func (q *qemu) hotAddNetDevice(deviceName, name, hardAddr string, VMFds, VhostFds []*os.File) error {
|
||||
var (
|
||||
VMFdNames []string
|
||||
VhostFdNames []string
|
||||
@@ -1381,7 +1381,12 @@ func (q *qemu) hotAddNetDevice(name, hardAddr string, VMFds, VhostFds []*os.File
|
||||
VhostFd.Close()
|
||||
VhostFdNames = append(VhostFdNames, fdName)
|
||||
}
|
||||
- return q.qmpMonitorCh.qmp.ExecuteNetdevAddByFds(q.qmpMonitorCh.ctx, "tap", name, VMFdNames, VhostFdNames)
|
||||
+
|
||||
+ if len(VMFdNames) != 0 || len(VhostFdNames) != 0 {
|
||||
+ return q.qmpMonitorCh.qmp.ExecuteNetdevAddByFds(q.qmpMonitorCh.ctx, "tap", name, VMFdNames, VhostFdNames)
|
||||
+ }
|
||||
+
|
||||
+ return q.qmpMonitorCh.qmp.ExecuteNetdevAdd(q.qmpMonitorCh.ctx, "tap", name, deviceName, "no", "no", 0)
|
||||
}
|
||||
|
||||
func (q *qemu) hotplugNetDevice(endpoint Endpoint, op operation) (err error) {
|
||||
@@ -1404,7 +1409,7 @@ func (q *qemu) hotplugNetDevice(endpoint Endpoint, op operation) (err error) {
|
||||
|
||||
devID := "virtio-" + tap.ID
|
||||
if op == addDevice {
|
||||
- if err = q.hotAddNetDevice(tap.Name, endpoint.HardwareAddr(), tap.VMFds, tap.VhostFds); err != nil {
|
||||
+ if err = q.hotAddNetDevice(tap.TAPIface.Name, tap.Name, endpoint.HardwareAddr(), tap.VMFds, tap.VhostFds); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
diff --git a/virtcontainers/sandbox.go b/virtcontainers/sandbox.go
|
||||
index ba704249..c8981a41 100644
|
||||
--- a/virtcontainers/sandbox.go
|
||||
+++ b/virtcontainers/sandbox.go
|
||||
@@ -937,6 +937,7 @@ func (s *Sandbox) generateNetInfo(inf *vcTypes.Interface) (NetworkInfo, error) {
|
||||
}
|
||||
|
||||
return NetworkInfo{
|
||||
+ Device: inf.Device,
|
||||
Iface: NetlinkIface{
|
||||
LinkAttrs: netlink.LinkAttrs{
|
||||
Name: inf.Name,
|
||||
@@ -950,7 +951,7 @@ func (s *Sandbox) generateNetInfo(inf *vcTypes.Interface) (NetworkInfo, error) {
|
||||
}
|
||||
|
||||
// AddInterface adds new nic to the sandbox.
|
||||
-func (s *Sandbox) AddInterface(inf *vcTypes.Interface) (*vcTypes.Interface, error) {
|
||||
+func (s *Sandbox) AddInterface(inf *vcTypes.Interface) (grpcIf *vcTypes.Interface, err error) {
|
||||
netInfo, err := s.generateNetInfo(inf)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
@@ -962,28 +963,41 @@ func (s *Sandbox) AddInterface(inf *vcTypes.Interface) (*vcTypes.Interface, erro
|
||||
}
|
||||
|
||||
endpoint.SetProperties(netInfo)
|
||||
- if err := doNetNS(s.networkNS.NetNsPath, func(_ ns.NetNS) error {
|
||||
+ if err = doNetNS(s.networkNS.NetNsPath, func(_ ns.NetNS) error {
|
||||
s.Logger().WithField("endpoint-type", endpoint.Type()).Info("Hot attaching endpoint")
|
||||
return endpoint.HotAttach(s.hypervisor)
|
||||
}); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
+ defer func() {
|
||||
+ if err != nil {
|
||||
+ if errDetach := endpoint.HotDetach(s.hypervisor, s.networkNS.NetNsCreated, s.networkNS.NetNsPath); errDetach != nil {
|
||||
+ s.Logger().WithField("endpoint-type", endpoint.Type()).Errorf("rollback hot attach endpoint failed")
|
||||
+ }
|
||||
+ }
|
||||
+ }()
|
||||
+
|
||||
// Update the sandbox storage
|
||||
s.networkNS.Endpoints = append(s.networkNS.Endpoints, endpoint)
|
||||
- if err := s.Save(); err != nil {
|
||||
+ if err = s.Save(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// Add network for vm
|
||||
inf.PciAddr = endpoint.PciAddr()
|
||||
- return s.agent.updateInterface(inf)
|
||||
+ grpcIf, err = s.agent.updateInterface(inf)
|
||||
+ if err != nil {
|
||||
+ return nil, err
|
||||
+ }
|
||||
+
|
||||
+ return
|
||||
}
|
||||
|
||||
// RemoveInterface removes a nic of the sandbox.
|
||||
func (s *Sandbox) RemoveInterface(inf *vcTypes.Interface) (*vcTypes.Interface, error) {
|
||||
for i, endpoint := range s.networkNS.Endpoints {
|
||||
- if endpoint.HardwareAddr() == inf.HwAddr {
|
||||
+ if endpoint.HardwareAddr() == inf.HwAddr || endpoint.Name() == inf.Name {
|
||||
s.Logger().WithField("endpoint-type", endpoint.Type()).Info("Hot detaching endpoint")
|
||||
if err := endpoint.HotDetach(s.hypervisor, s.networkNS.NetNsCreated, s.networkNS.NetNsPath); err != nil {
|
||||
return inf, err
|
||||
diff --git a/virtcontainers/tap_endpoint.go b/virtcontainers/tap_endpoint.go
|
||||
index cb441b87..7d33d5a2 100644
|
||||
--- a/virtcontainers/tap_endpoint.go
|
||||
+++ b/virtcontainers/tap_endpoint.go
|
||||
@@ -7,6 +7,7 @@ package virtcontainers
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
+ "os"
|
||||
|
||||
"github.com/containernetworking/plugins/pkg/ns"
|
||||
"github.com/vishvananda/netlink"
|
||||
@@ -111,7 +112,7 @@ func (endpoint *TapEndpoint) HotDetach(h hypervisor, netNsCreated bool, netNsPat
|
||||
return nil
|
||||
}
|
||||
|
||||
-func createTapNetworkEndpoint(idx int, ifName string) (*TapEndpoint, error) {
|
||||
+func createTapNetworkEndpoint(idx int, ifName string, tapIfName string) (*TapEndpoint, error) {
|
||||
if idx < 0 {
|
||||
return &TapEndpoint{}, fmt.Errorf("invalid network endpoint index: %d", idx)
|
||||
}
|
||||
@@ -131,6 +132,10 @@ func createTapNetworkEndpoint(idx int, ifName string) (*TapEndpoint, error) {
|
||||
endpoint.TapInterface.Name = ifName
|
||||
}
|
||||
|
||||
+ if tapIfName != "" {
|
||||
+ endpoint.TapInterface.TAPIface.Name = tapIfName
|
||||
+ }
|
||||
+
|
||||
return endpoint, nil
|
||||
}
|
||||
|
||||
@@ -145,9 +150,19 @@ func tapNetwork(endpoint *TapEndpoint, numCPUs uint32, disableVhostNet bool) err
|
||||
if err != nil {
|
||||
return fmt.Errorf("Could not create TAP interface: %s", err)
|
||||
}
|
||||
+
|
||||
+ defer func() {
|
||||
+ if err != nil {
|
||||
+ if errDel := netHandle.LinkDel(tapLink); errDel != nil {
|
||||
+ networkLogger().WithError(errDel).Error("tapNetwork fail to rollback del link")
|
||||
+ }
|
||||
+ }
|
||||
+ }()
|
||||
+
|
||||
endpoint.TapInterface.VMFds = fds
|
||||
if !disableVhostNet {
|
||||
- vhostFds, err := createVhostFds(int(numCPUs))
|
||||
+ var vhostFds []*os.File
|
||||
+ vhostFds, err = createVhostFds(int(numCPUs))
|
||||
if err != nil {
|
||||
return fmt.Errorf("Could not setup vhost fds %s : %s", endpoint.TapInterface.Name, err)
|
||||
}
|
||||
@@ -161,10 +176,10 @@ func tapNetwork(endpoint *TapEndpoint, numCPUs uint32, disableVhostNet bool) err
|
||||
// bridge created by the network plugin on the host actually expects
|
||||
// to see traffic from this MAC address and not another one.
|
||||
endpoint.TapInterface.TAPIface.HardAddr = linkAttrs.HardwareAddr.String()
|
||||
- if err := netHandle.LinkSetMTU(tapLink, linkAttrs.MTU); err != nil {
|
||||
+ if err = netHandle.LinkSetMTU(tapLink, linkAttrs.MTU); err != nil {
|
||||
return fmt.Errorf("Could not set TAP MTU %d: %s", linkAttrs.MTU, err)
|
||||
}
|
||||
- if err := netHandle.LinkSetUp(tapLink); err != nil {
|
||||
+ if err = netHandle.LinkSetUp(tapLink); err != nil {
|
||||
return fmt.Errorf("Could not enable TAP %s: %s", endpoint.TapInterface.Name, err)
|
||||
}
|
||||
return nil
|
||||
--
|
||||
2.14.3 (Apple Git-98)
|
||||
|
||||
@ -1,260 +0,0 @@
|
||||
From be8153f21c0b81d2b194075ecd654501bc708577 Mon Sep 17 00:00:00 2001
|
||||
From: jiangpengfei <jiangpengfei9@huawei.com>
|
||||
Date: Thu, 13 Aug 2020 18:54:49 +0800
|
||||
Subject: [PATCH 25/50] network: keep list-ifaces result compatible with cni
|
||||
|
||||
reason: community list-ifaces command will return the all
|
||||
interfaces info in the Kata VM, however we may just want
|
||||
to get the interfaces that we hotplug, so just return the
|
||||
hotplugged interfaces and convert the interface info to
|
||||
be compatible with cni.
|
||||
|
||||
Signed-off-by: jiangpengfei <jiangpengfei9@huawei.com>
|
||||
---
|
||||
cli/network.go | 29 ++++++++++++++++++++++-
|
||||
virtcontainers/api.go | 4 +++-
|
||||
virtcontainers/endpoint.go | 44 +++++++++++++++++++++++++++++++++++
|
||||
virtcontainers/network.go | 27 +++++++++++++++++++++
|
||||
virtcontainers/persist/api/network.go | 24 +++++++++++++++++++
|
||||
virtcontainers/tap_endpoint.go | 9 +++++++
|
||||
6 files changed, 135 insertions(+), 2 deletions(-)
|
||||
|
||||
diff --git a/cli/network.go b/cli/network.go
|
||||
index 7e7791f1..66955725 100644
|
||||
--- a/cli/network.go
|
||||
+++ b/cli/network.go
|
||||
@@ -28,6 +28,13 @@ const (
|
||||
|
||||
const defaultLinkType = "tap"
|
||||
|
||||
+type compatInterface struct {
|
||||
+ Name string `json:"name,omitempty"`
|
||||
+ Mac string `json:"mac,omitempty"`
|
||||
+ IP []string `json:"ip,omitempty"`
|
||||
+ Mtu int `json:"mtu,omitempty"`
|
||||
+}
|
||||
+
|
||||
var kataNetworkCLICommand = cli.Command{
|
||||
Name: "kata-network",
|
||||
Usage: "manage interfaces and routes for container",
|
||||
@@ -244,7 +251,8 @@ func networkListCommand(ctx context.Context, containerID string, opType networkT
|
||||
kataLog.WithField("existing-interfaces", fmt.Sprintf("%+v", interfaces)).
|
||||
WithError(err).Error("list interfaces failed")
|
||||
}
|
||||
- json.NewEncoder(file).Encode(interfaces)
|
||||
+ compatInfs := convertCompatInterfaces(interfaces)
|
||||
+ json.NewEncoder(file).Encode(compatInfs)
|
||||
case routeType:
|
||||
var routes []*vcTypes.Route
|
||||
routes, err = vci.ListRoutes(ctx, sandboxID)
|
||||
@@ -256,3 +264,22 @@ func networkListCommand(ctx context.Context, containerID string, opType networkT
|
||||
}
|
||||
return err
|
||||
}
|
||||
+
|
||||
+func convertCompatInterfaces(interfaces []*vcTypes.Interface) []compatInterface {
|
||||
+ var infs []compatInterface
|
||||
+ for _, i := range interfaces {
|
||||
+ var addrs []string
|
||||
+ for _, a := range i.IPAddresses {
|
||||
+ addrs = append(addrs, fmt.Sprintf("%s/%s", a.Address, a.Mask))
|
||||
+ }
|
||||
+
|
||||
+ infs = append(infs, compatInterface{
|
||||
+ Name: i.Name,
|
||||
+ Mac: i.HwAddr,
|
||||
+ IP: addrs,
|
||||
+ Mtu: int(i.Mtu),
|
||||
+ })
|
||||
+ }
|
||||
+
|
||||
+ return infs
|
||||
+}
|
||||
diff --git a/virtcontainers/api.go b/virtcontainers/api.go
|
||||
index 5e8c9c9e..eb5b4995 100644
|
||||
--- a/virtcontainers/api.go
|
||||
+++ b/virtcontainers/api.go
|
||||
@@ -949,7 +949,9 @@ func ListInterfaces(ctx context.Context, sandboxID string) ([]*vcTypes.Interface
|
||||
}
|
||||
defer s.releaseStatelessSandbox()
|
||||
|
||||
- return s.ListInterfaces()
|
||||
+ // get interfaces info from persist.json file
|
||||
+ // instead of by s.ListInterfaces()
|
||||
+ return convertToCompatInterfaces(&s.networkNS.Endpoints), nil
|
||||
}
|
||||
|
||||
// UpdateRoutes is the virtcontainers update routes entry point.
|
||||
diff --git a/virtcontainers/endpoint.go b/virtcontainers/endpoint.go
|
||||
index 01b5e77f..7efcf49c 100644
|
||||
--- a/virtcontainers/endpoint.go
|
||||
+++ b/virtcontainers/endpoint.go
|
||||
@@ -132,6 +132,28 @@ func saveTapIf(tapif *TapInterface) *persistapi.TapInterface {
|
||||
}
|
||||
}
|
||||
|
||||
+func saveTapEndpointProperties(networkInfo *NetworkInfo) *persistapi.NetworkProperties {
|
||||
+ if networkInfo == nil {
|
||||
+ return nil
|
||||
+ }
|
||||
+
|
||||
+ return &persistapi.NetworkProperties{
|
||||
+ Device: networkInfo.Device,
|
||||
+ Iface: persistapi.NetlinkIface{
|
||||
+ LinkAttrs: networkInfo.Iface.LinkAttrs,
|
||||
+ Type: networkInfo.Iface.Type,
|
||||
+ },
|
||||
+ Addrs: networkInfo.Addrs,
|
||||
+ Routes: networkInfo.Routes,
|
||||
+ DNS: persistapi.DNSInfo{
|
||||
+ Servers: networkInfo.DNS.Servers,
|
||||
+ Domain: networkInfo.DNS.Domain,
|
||||
+ Searches: networkInfo.DNS.Searches,
|
||||
+ Options: networkInfo.DNS.Options,
|
||||
+ },
|
||||
+ }
|
||||
+}
|
||||
+
|
||||
func loadTapIf(tapif *persistapi.TapInterface) *TapInterface {
|
||||
if tapif == nil {
|
||||
return nil
|
||||
@@ -148,6 +170,28 @@ func loadTapIf(tapif *persistapi.TapInterface) *TapInterface {
|
||||
}
|
||||
}
|
||||
|
||||
+func loadTapEndpointProperties(endpointProperties *persistapi.NetworkProperties) *NetworkInfo {
|
||||
+ if endpointProperties == nil {
|
||||
+ return nil
|
||||
+ }
|
||||
+
|
||||
+ return &NetworkInfo{
|
||||
+ Device: endpointProperties.Device,
|
||||
+ Iface: NetlinkIface{
|
||||
+ LinkAttrs: endpointProperties.Iface.LinkAttrs,
|
||||
+ Type: endpointProperties.Iface.Type,
|
||||
+ },
|
||||
+ Addrs: endpointProperties.Addrs,
|
||||
+ Routes: endpointProperties.Routes,
|
||||
+ DNS: DNSInfo{
|
||||
+ Servers: endpointProperties.DNS.Servers,
|
||||
+ Domain: endpointProperties.DNS.Domain,
|
||||
+ Searches: endpointProperties.DNS.Searches,
|
||||
+ Options: endpointProperties.DNS.Options,
|
||||
+ },
|
||||
+ }
|
||||
+}
|
||||
+
|
||||
func saveNetIfPair(pair *NetworkInterfacePair) *persistapi.NetworkInterfacePair {
|
||||
if pair == nil {
|
||||
return nil
|
||||
diff --git a/virtcontainers/network.go b/virtcontainers/network.go
|
||||
index e909a822..bf7f9336 100644
|
||||
--- a/virtcontainers/network.go
|
||||
+++ b/virtcontainers/network.go
|
||||
@@ -1340,3 +1340,30 @@ func (n *Network) Remove(ctx context.Context, ns *NetworkNamespace, hypervisor h
|
||||
|
||||
return nil
|
||||
}
|
||||
+
|
||||
+// convertCompatInterfaces convert Endpoint info to vcTypes.Interface
|
||||
+func convertToCompatInterfaces(es *[]Endpoint) []*vcTypes.Interface {
|
||||
+ var infs []*vcTypes.Interface
|
||||
+ for _, e := range *es {
|
||||
+ var addrs []*vcTypes.IPAddress
|
||||
+ for _, a := range e.Properties().Addrs {
|
||||
+ m, _ := a.Mask.Size()
|
||||
+ addr := &vcTypes.IPAddress{
|
||||
+ Address: fmt.Sprintf("%s", a.IP),
|
||||
+ Mask: fmt.Sprintf("%d", m),
|
||||
+ }
|
||||
+ addrs = append(addrs, addr)
|
||||
+ }
|
||||
+ inf := &vcTypes.Interface{
|
||||
+ LinkType: string(e.Type()),
|
||||
+ Name: e.Name(),
|
||||
+ Mtu: uint64(e.Properties().Iface.MTU),
|
||||
+ HwAddr: e.HardwareAddr(),
|
||||
+ IPAddresses: addrs,
|
||||
+ }
|
||||
+
|
||||
+ infs = append(infs, inf)
|
||||
+ }
|
||||
+
|
||||
+ return infs
|
||||
+}
|
||||
diff --git a/virtcontainers/persist/api/network.go b/virtcontainers/persist/api/network.go
|
||||
index 69610c67..53c6de44 100644
|
||||
--- a/virtcontainers/persist/api/network.go
|
||||
+++ b/virtcontainers/persist/api/network.go
|
||||
@@ -11,6 +11,27 @@ import (
|
||||
)
|
||||
|
||||
// ============= sandbox level resources =============
|
||||
+// DNSInfo describes the DNS setup related to a network interface.
|
||||
+type DNSInfo struct {
|
||||
+ Servers []string
|
||||
+ Domain string
|
||||
+ Searches []string
|
||||
+ Options []string
|
||||
+}
|
||||
+
|
||||
+// NetlinkIface describes fully a network interface.
|
||||
+type NetlinkIface struct {
|
||||
+ netlink.LinkAttrs
|
||||
+ Type string
|
||||
+}
|
||||
+
|
||||
+type NetworkProperties struct {
|
||||
+ Device string
|
||||
+ Iface NetlinkIface
|
||||
+ Addrs []netlink.Addr
|
||||
+ Routes []netlink.Route
|
||||
+ DNS DNSInfo
|
||||
+}
|
||||
|
||||
type NetworkInterface struct {
|
||||
Name string
|
||||
@@ -91,6 +112,9 @@ type NetworkEndpoint struct {
|
||||
Tap *TapEndpoint `json:",omitempty"`
|
||||
IPVlan *IPVlanEndpoint `json:",omitempty"`
|
||||
Tuntap *TuntapEndpoint `json:",omitempty"`
|
||||
+
|
||||
+ // store the endpoint properties info
|
||||
+ EndPointProperties *NetworkProperties `json:",omitempty"`
|
||||
}
|
||||
|
||||
// NetworkInfo contains network information of sandbox
|
||||
diff --git a/virtcontainers/tap_endpoint.go b/virtcontainers/tap_endpoint.go
|
||||
index 7d33d5a2..c897670e 100644
|
||||
--- a/virtcontainers/tap_endpoint.go
|
||||
+++ b/virtcontainers/tap_endpoint.go
|
||||
@@ -206,12 +206,15 @@ func unTapNetwork(name string) error {
|
||||
|
||||
func (endpoint *TapEndpoint) save() persistapi.NetworkEndpoint {
|
||||
tapif := saveTapIf(&endpoint.TapInterface)
|
||||
+ // save tap endpoint network properties into persist storage
|
||||
+ properties := saveTapEndpointProperties(&endpoint.EndpointProperties)
|
||||
|
||||
return persistapi.NetworkEndpoint{
|
||||
Type: string(endpoint.Type()),
|
||||
Tap: &persistapi.TapEndpoint{
|
||||
TapInterface: *tapif,
|
||||
},
|
||||
+ EndPointProperties: properties,
|
||||
}
|
||||
}
|
||||
func (endpoint *TapEndpoint) load(s persistapi.NetworkEndpoint) {
|
||||
@@ -221,4 +224,10 @@ func (endpoint *TapEndpoint) load(s persistapi.NetworkEndpoint) {
|
||||
tapif := loadTapIf(&s.Tap.TapInterface)
|
||||
endpoint.TapInterface = *tapif
|
||||
}
|
||||
+
|
||||
+ if s.EndPointProperties != nil {
|
||||
+ // restore tap endpoint network properties from persist storage
|
||||
+ properties := loadTapEndpointProperties(s.EndPointProperties)
|
||||
+ endpoint.EndpointProperties = *properties
|
||||
+ }
|
||||
}
|
||||
--
|
||||
2.14.3 (Apple Git-98)
|
||||
|
||||
@ -1,285 +0,0 @@
|
||||
From eeca1e47e9a6422d89d08275864f2c1b15e54941 Mon Sep 17 00:00:00 2001
|
||||
From: jiangpengfei <jiangpengfei9@huawei.com>
|
||||
Date: Fri, 14 Aug 2020 19:14:35 +0800
|
||||
Subject: [PATCH 26/50] network: add enable_compat_old_cni config
|
||||
|
||||
reason: old version kata-network list-ifaces output result different
|
||||
from the community version, inorder to compatible with the old version
|
||||
list-ifaces command output format, add enable_compat_old_cni to control
|
||||
the list-ifaces command output format.
|
||||
|
||||
Signed-off-by: jiangpengfei <jiangpengfei9@huawei.com>
|
||||
---
|
||||
cli/config/configuration-qemu.toml.in | 4 ++++
|
||||
cli/network.go | 17 +++++++++++++++--
|
||||
cli/network_test.go | 6 ++++++
|
||||
pkg/katautils/config.go | 2 ++
|
||||
virtcontainers/api.go | 10 +++++++---
|
||||
virtcontainers/interfaces.go | 1 +
|
||||
virtcontainers/network.go | 11 ++++++-----
|
||||
virtcontainers/persist.go | 18 ++++++++++--------
|
||||
virtcontainers/persist/api/config.go | 9 +++++----
|
||||
virtcontainers/pkg/oci/utils.go | 4 ++++
|
||||
virtcontainers/pkg/vcmock/sandbox.go | 5 +++++
|
||||
virtcontainers/sandbox.go | 5 +++++
|
||||
12 files changed, 70 insertions(+), 22 deletions(-)
|
||||
|
||||
diff --git a/cli/config/configuration-qemu.toml.in b/cli/config/configuration-qemu.toml.in
|
||||
index b44e84d8..46f8b632 100644
|
||||
--- a/cli/config/configuration-qemu.toml.in
|
||||
+++ b/cli/config/configuration-qemu.toml.in
|
||||
@@ -453,6 +453,10 @@ disable_guest_seccomp=@DEFDISABLEGUESTSECCOMP@
|
||||
# (default: false)
|
||||
#disable_new_netns = true
|
||||
|
||||
+# If enabled, the kata-network will return the old interface format info to be compatible with
|
||||
+# old version CNI plugin
|
||||
+enable_compat_old_cni = true
|
||||
+
|
||||
# if enabled, the runtime will add all the kata processes inside one dedicated cgroup.
|
||||
# The container cgroups in the host are not created, just one single cgroup per sandbox.
|
||||
# The runtime caller is free to restrict or collect cgroup stats of the overall Kata sandbox.
|
||||
diff --git a/cli/network.go b/cli/network.go
|
||||
index 66955725..a1a24425 100644
|
||||
--- a/cli/network.go
|
||||
+++ b/cli/network.go
|
||||
@@ -251,8 +251,21 @@ func networkListCommand(ctx context.Context, containerID string, opType networkT
|
||||
kataLog.WithField("existing-interfaces", fmt.Sprintf("%+v", interfaces)).
|
||||
WithError(err).Error("list interfaces failed")
|
||||
}
|
||||
- compatInfs := convertCompatInterfaces(interfaces)
|
||||
- json.NewEncoder(file).Encode(compatInfs)
|
||||
+
|
||||
+ sandbox, err := vci.FetchSandbox(ctx, sandboxID)
|
||||
+ if err != nil {
|
||||
+ kataLog.WithField("existing-interfaces", fmt.Sprintf("%+v", interfaces)).
|
||||
+ WithError(err).Error("fetch sandbox failed")
|
||||
+ }
|
||||
+
|
||||
+ // If sandbox network config need to be compatible with old CNI,
|
||||
+ // convert the interface format to old version format.
|
||||
+ if sandbox.IsCompatOldCNI() {
|
||||
+ compatInfs := convertCompatInterfaces(interfaces)
|
||||
+ json.NewEncoder(file).Encode(compatInfs)
|
||||
+ } else {
|
||||
+ json.NewEncoder(file).Encode(interfaces)
|
||||
+ }
|
||||
case routeType:
|
||||
var routes []*vcTypes.Route
|
||||
routes, err = vci.ListRoutes(ctx, sandboxID)
|
||||
diff --git a/cli/network_test.go b/cli/network_test.go
|
||||
index 4e3d943d..95fd2749 100644
|
||||
--- a/cli/network_test.go
|
||||
+++ b/cli/network_test.go
|
||||
@@ -17,6 +17,7 @@ import (
|
||||
|
||||
vc "github.com/kata-containers/runtime/virtcontainers"
|
||||
vcTypes "github.com/kata-containers/runtime/virtcontainers/pkg/types"
|
||||
+ "github.com/kata-containers/runtime/virtcontainers/pkg/vcmock"
|
||||
"github.com/kata-containers/runtime/virtcontainers/types"
|
||||
)
|
||||
|
||||
@@ -59,6 +60,10 @@ func TestNetworkCliFunction(t *testing.T) {
|
||||
return newSingleContainerStatus(testContainerID, state, map[string]string{}, &specs.Spec{}), nil
|
||||
}
|
||||
|
||||
+ testingImpl.FetchSandboxFunc = func(ctx context.Context, id string) (vc.VCSandbox, error) {
|
||||
+ return &vcmock.Sandbox{}, nil
|
||||
+ }
|
||||
+
|
||||
defer func() {
|
||||
testingImpl.AddInterfaceFunc = nil
|
||||
testingImpl.RemoveInterfaceFunc = nil
|
||||
@@ -66,6 +71,7 @@ func TestNetworkCliFunction(t *testing.T) {
|
||||
testingImpl.UpdateRoutesFunc = nil
|
||||
testingImpl.ListRoutesFunc = nil
|
||||
testingImpl.StatusContainerFunc = nil
|
||||
+ testingImpl.FetchSandboxFunc = nil
|
||||
}()
|
||||
|
||||
set := flag.NewFlagSet("", 0)
|
||||
diff --git a/pkg/katautils/config.go b/pkg/katautils/config.go
|
||||
index 9a99b9d4..94c916a0 100644
|
||||
--- a/pkg/katautils/config.go
|
||||
+++ b/pkg/katautils/config.go
|
||||
@@ -141,6 +141,7 @@ type runtime struct {
|
||||
Debug bool `toml:"enable_debug"`
|
||||
Tracing bool `toml:"enable_tracing"`
|
||||
DisableNewNetNs bool `toml:"disable_new_netns"`
|
||||
+ EnableCompatOldCNI bool `toml:"enable_compat_old_cni"`
|
||||
DisableGuestSeccomp bool `toml:"disable_guest_seccomp"`
|
||||
SandboxCgroupOnly bool `toml:"sandbox_cgroup_only"`
|
||||
Experimental []string `toml:"experimental"`
|
||||
@@ -1235,6 +1236,7 @@ func LoadConfiguration(configPath string, ignoreLogging, builtIn bool, debugFlag
|
||||
|
||||
config.SandboxCgroupOnly = tomlConf.Runtime.SandboxCgroupOnly
|
||||
config.DisableNewNetNs = tomlConf.Runtime.DisableNewNetNs
|
||||
+ config.EnableCompatOldCNI = tomlConf.Runtime.EnableCompatOldCNI
|
||||
for _, f := range tomlConf.Runtime.Experimental {
|
||||
feature := exp.Get(f)
|
||||
if feature == nil {
|
||||
diff --git a/virtcontainers/api.go b/virtcontainers/api.go
|
||||
index eb5b4995..fb044fe1 100644
|
||||
--- a/virtcontainers/api.go
|
||||
+++ b/virtcontainers/api.go
|
||||
@@ -949,9 +949,13 @@ func ListInterfaces(ctx context.Context, sandboxID string) ([]*vcTypes.Interface
|
||||
}
|
||||
defer s.releaseStatelessSandbox()
|
||||
|
||||
- // get interfaces info from persist.json file
|
||||
- // instead of by s.ListInterfaces()
|
||||
- return convertToCompatInterfaces(&s.networkNS.Endpoints), nil
|
||||
+ // If enable_compat_old_cni is enabled, get interfaces info from
|
||||
+ // persist.json file instead of by s.ListInterfaces()
|
||||
+ if s.config.NetworkConfig.EnableCompatOldCNI {
|
||||
+ return convertToCompatInterfaces(&s.networkNS.Endpoints), nil
|
||||
+ }
|
||||
+
|
||||
+ return s.ListInterfaces()
|
||||
}
|
||||
|
||||
// UpdateRoutes is the virtcontainers update routes entry point.
|
||||
diff --git a/virtcontainers/interfaces.go b/virtcontainers/interfaces.go
|
||||
index fa6b584e..499b386e 100644
|
||||
--- a/virtcontainers/interfaces.go
|
||||
+++ b/virtcontainers/interfaces.go
|
||||
@@ -98,6 +98,7 @@ type VCSandbox interface {
|
||||
ListInterfaces() ([]*vcTypes.Interface, error)
|
||||
UpdateRoutes(routes []*vcTypes.Route) ([]*vcTypes.Route, error)
|
||||
ListRoutes() ([]*vcTypes.Route, error)
|
||||
+ IsCompatOldCNI() bool
|
||||
}
|
||||
|
||||
// VCContainer is the Container interface
|
||||
diff --git a/virtcontainers/network.go b/virtcontainers/network.go
|
||||
index bf7f9336..db235cf6 100644
|
||||
--- a/virtcontainers/network.go
|
||||
+++ b/virtcontainers/network.go
|
||||
@@ -155,11 +155,12 @@ type NetworkInterfacePair struct {
|
||||
|
||||
// NetworkConfig is the network configuration related to a network.
|
||||
type NetworkConfig struct {
|
||||
- NetNSPath string
|
||||
- NetNsCreated bool
|
||||
- DisableNewNetNs bool
|
||||
- NetmonConfig NetmonConfig
|
||||
- InterworkingModel NetInterworkingModel
|
||||
+ NetNSPath string
|
||||
+ NetNsCreated bool
|
||||
+ DisableNewNetNs bool
|
||||
+ EnableCompatOldCNI bool
|
||||
+ NetmonConfig NetmonConfig
|
||||
+ InterworkingModel NetInterworkingModel
|
||||
}
|
||||
|
||||
func networkLogger() *logrus.Entry {
|
||||
diff --git a/virtcontainers/persist.go b/virtcontainers/persist.go
|
||||
index 6bd09a0b..fe00bf9a 100644
|
||||
--- a/virtcontainers/persist.go
|
||||
+++ b/virtcontainers/persist.go
|
||||
@@ -187,10 +187,11 @@ func (s *Sandbox) dumpConfig(ss *persistapi.SandboxState) {
|
||||
},
|
||||
ShimType: string(sconfig.ShimType),
|
||||
NetworkConfig: persistapi.NetworkConfig{
|
||||
- NetNSPath: sconfig.NetworkConfig.NetNSPath,
|
||||
- NetNsCreated: sconfig.NetworkConfig.NetNsCreated,
|
||||
- DisableNewNetNs: sconfig.NetworkConfig.DisableNewNetNs,
|
||||
- InterworkingModel: int(sconfig.NetworkConfig.InterworkingModel),
|
||||
+ NetNSPath: sconfig.NetworkConfig.NetNSPath,
|
||||
+ NetNsCreated: sconfig.NetworkConfig.NetNsCreated,
|
||||
+ DisableNewNetNs: sconfig.NetworkConfig.DisableNewNetNs,
|
||||
+ EnableCompatOldCNI: sconfig.NetworkConfig.EnableCompatOldCNI,
|
||||
+ InterworkingModel: int(sconfig.NetworkConfig.InterworkingModel),
|
||||
},
|
||||
|
||||
ShmSize: sconfig.ShmSize,
|
||||
@@ -477,10 +478,11 @@ func loadSandboxConfig(id string) (*SandboxConfig, error) {
|
||||
},
|
||||
ShimType: ShimType(savedConf.ShimType),
|
||||
NetworkConfig: NetworkConfig{
|
||||
- NetNSPath: savedConf.NetworkConfig.NetNSPath,
|
||||
- NetNsCreated: savedConf.NetworkConfig.NetNsCreated,
|
||||
- DisableNewNetNs: savedConf.NetworkConfig.DisableNewNetNs,
|
||||
- InterworkingModel: NetInterworkingModel(savedConf.NetworkConfig.InterworkingModel),
|
||||
+ NetNSPath: savedConf.NetworkConfig.NetNSPath,
|
||||
+ NetNsCreated: savedConf.NetworkConfig.NetNsCreated,
|
||||
+ DisableNewNetNs: savedConf.NetworkConfig.DisableNewNetNs,
|
||||
+ EnableCompatOldCNI: savedConf.NetworkConfig.EnableCompatOldCNI,
|
||||
+ InterworkingModel: NetInterworkingModel(savedConf.NetworkConfig.InterworkingModel),
|
||||
},
|
||||
|
||||
ShmSize: savedConf.ShmSize,
|
||||
diff --git a/virtcontainers/persist/api/config.go b/virtcontainers/persist/api/config.go
|
||||
index cfbee849..3a2df32b 100644
|
||||
--- a/virtcontainers/persist/api/config.go
|
||||
+++ b/virtcontainers/persist/api/config.go
|
||||
@@ -210,10 +210,11 @@ type ShimConfig struct {
|
||||
|
||||
// NetworkConfig is the network configuration related to a network.
|
||||
type NetworkConfig struct {
|
||||
- NetNSPath string
|
||||
- NetNsCreated bool
|
||||
- DisableNewNetNs bool
|
||||
- InterworkingModel int
|
||||
+ NetNSPath string
|
||||
+ NetNsCreated bool
|
||||
+ DisableNewNetNs bool
|
||||
+ EnableCompatOldCNI bool
|
||||
+ InterworkingModel int
|
||||
}
|
||||
|
||||
type ContainerConfig struct {
|
||||
diff --git a/virtcontainers/pkg/oci/utils.go b/virtcontainers/pkg/oci/utils.go
|
||||
index 36c730b7..948bd3cb 100644
|
||||
--- a/virtcontainers/pkg/oci/utils.go
|
||||
+++ b/virtcontainers/pkg/oci/utils.go
|
||||
@@ -130,6 +130,9 @@ type RuntimeConfig struct {
|
||||
//Determines if create a netns for hypervisor process
|
||||
DisableNewNetNs bool
|
||||
|
||||
+ // Determines if compatible with old CNI plugin
|
||||
+ EnableCompatOldCNI bool
|
||||
+
|
||||
//Determines kata processes are managed only in sandbox cgroup
|
||||
SandboxCgroupOnly bool
|
||||
|
||||
@@ -275,6 +278,7 @@ func networkConfig(ocispec specs.Spec, config RuntimeConfig) (vc.NetworkConfig,
|
||||
}
|
||||
netConf.InterworkingModel = config.InterNetworkModel
|
||||
netConf.DisableNewNetNs = config.DisableNewNetNs
|
||||
+ netConf.EnableCompatOldCNI = config.EnableCompatOldCNI
|
||||
|
||||
netConf.NetmonConfig = vc.NetmonConfig{
|
||||
Path: config.NetmonConfig.Path,
|
||||
diff --git a/virtcontainers/pkg/vcmock/sandbox.go b/virtcontainers/pkg/vcmock/sandbox.go
|
||||
index 677457ef..11b83ccd 100644
|
||||
--- a/virtcontainers/pkg/vcmock/sandbox.go
|
||||
+++ b/virtcontainers/pkg/vcmock/sandbox.go
|
||||
@@ -212,3 +212,8 @@ func (s *Sandbox) UpdateRoutes(routes []*vcTypes.Route) ([]*vcTypes.Route, error
|
||||
func (s *Sandbox) ListRoutes() ([]*vcTypes.Route, error) {
|
||||
return nil, nil
|
||||
}
|
||||
+
|
||||
+// IsCompatOldCNI return the whether enable compatible with old CNI
|
||||
+func (s *Sandbox) IsCompatOldCNI() bool {
|
||||
+ return false
|
||||
+}
|
||||
diff --git a/virtcontainers/sandbox.go b/virtcontainers/sandbox.go
|
||||
index c8981a41..6a643a12 100644
|
||||
--- a/virtcontainers/sandbox.go
|
||||
+++ b/virtcontainers/sandbox.go
|
||||
@@ -2379,6 +2379,11 @@ func (s *Sandbox) setContainersState(state types.StateString) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
+// IsCompatOldCNI return the whether enable compatible with old CNI
|
||||
+func (s *Sandbox) IsCompatOldCNI() bool {
|
||||
+ return s.config.NetworkConfig.EnableCompatOldCNI
|
||||
+}
|
||||
+
|
||||
// updateStaticSandboxResources update sandbox's cpu and memory resource passed by
|
||||
// sandbox_cpu and sandbox_mem annotations
|
||||
func updateStaticSandboxResources(sandboxConfig *SandboxConfig) error {
|
||||
--
|
||||
2.14.3 (Apple Git-98)
|
||||
|
||||
@ -1,182 +0,0 @@
|
||||
From ec15337fc816767ca0e8183576405499080b9b1e Mon Sep 17 00:00:00 2001
|
||||
From: jiangpengfei <jiangpengfei9@huawei.com>
|
||||
Date: Sun, 16 Aug 2020 16:41:18 +0800
|
||||
Subject: [PATCH 27/50] network: add more strict check for input network
|
||||
interface
|
||||
|
||||
reason: kata-network add-iface command will receive the network
|
||||
interface info from untrust user, so we need to add more strict
|
||||
check for input network interface info.
|
||||
|
||||
Signed-off-by: jiangpengfei <jiangpengfei9@huawei.com>
|
||||
---
|
||||
virtcontainers/network.go | 117 ++++++++++++++++++++++++++++++++++++++++++++++
|
||||
virtcontainers/sandbox.go | 11 +++++
|
||||
2 files changed, 128 insertions(+)
|
||||
|
||||
diff --git a/virtcontainers/network.go b/virtcontainers/network.go
|
||||
index db235cf6..a1676ccd 100644
|
||||
--- a/virtcontainers/network.go
|
||||
+++ b/virtcontainers/network.go
|
||||
@@ -13,8 +13,10 @@ import (
|
||||
"math/rand"
|
||||
"net"
|
||||
"os"
|
||||
+ "regexp"
|
||||
"runtime"
|
||||
"sort"
|
||||
+ "strings"
|
||||
"time"
|
||||
|
||||
"github.com/containernetworking/plugins/pkg/ns"
|
||||
@@ -55,6 +57,17 @@ const (
|
||||
NetXConnectInvalidModel
|
||||
)
|
||||
|
||||
+const (
|
||||
+ maxIPAddrLen = 18
|
||||
+ maxInterfaceLen = 15
|
||||
+ minMTUVal = 46
|
||||
+ maxMTUVal = 9600
|
||||
+)
|
||||
+
|
||||
+var (
|
||||
+ regInfName = regexp.MustCompile(`^[A-Za-z][A-Za-z0-9_\-.]*$`)
|
||||
+)
|
||||
+
|
||||
//IsValid checks if a model is valid
|
||||
func (n NetInterworkingModel) IsValid() bool {
|
||||
return 0 <= int(n) && int(n) < int(NetXConnectInvalidModel)
|
||||
@@ -1368,3 +1381,107 @@ func convertToCompatInterfaces(es *[]Endpoint) []*vcTypes.Interface {
|
||||
|
||||
return infs
|
||||
}
|
||||
+
|
||||
+// verifyInterfaceName verifies the interface name valid or not
|
||||
+func verifyInterfaceName(name string) error {
|
||||
+ // verify `Name` before `Tapname` because of the strict rules
|
||||
+ name = strings.TrimSpace(name)
|
||||
+ if len(name) > maxInterfaceLen {
|
||||
+ return fmt.Errorf("interface name can't be longer than 15")
|
||||
+ } else if len(name) == 0 || name == "\n" {
|
||||
+ return fmt.Errorf("interface name can't be empty")
|
||||
+ }
|
||||
+
|
||||
+ // QMP rules of `Name`
|
||||
+ chk := regInfName.FindAllString(name, -1)
|
||||
+ if chk == nil {
|
||||
+ return fmt.Errorf("invalid input of interface name, please check the rules")
|
||||
+ }
|
||||
+
|
||||
+ return nil
|
||||
+}
|
||||
+
|
||||
+// verifyIPAndMask verifies CIDR notation IP address and mask like "192.0.2.0/24"
|
||||
+func verifyIPAndMask(ip string) (nlIpMask *net.IPNet, err error) {
|
||||
+ ip = strings.TrimSpace(ip)
|
||||
+
|
||||
+ if len(ip) > maxIPAddrLen {
|
||||
+ return nil, fmt.Errorf("the length of IP address is too long, max ip len :%d", maxIPAddrLen)
|
||||
+ }
|
||||
+
|
||||
+ if nlIpMask, err = netlink.ParseIPNet(ip); err != nil {
|
||||
+ return nil, fmt.Errorf("invalid input of IP : %v", err)
|
||||
+ }
|
||||
+
|
||||
+ return nlIpMask, nil
|
||||
+}
|
||||
+
|
||||
+// verifyMac verifies MAC address
|
||||
+func verifyMac(mac string) error {
|
||||
+ mac = strings.TrimSpace(mac)
|
||||
+ if _, err := net.ParseMAC(mac); err != nil {
|
||||
+ return fmt.Errorf("invalid input of Mac : %v", err)
|
||||
+ }
|
||||
+
|
||||
+ return nil
|
||||
+}
|
||||
+
|
||||
+// verifyMtu verifies MTU value of interface
|
||||
+func verifyMtu(mtu uint64) error {
|
||||
+ if mtu < minMTUVal || mtu > maxMTUVal {
|
||||
+ return fmt.Errorf("invalid input of MTU : %v", mtu)
|
||||
+ }
|
||||
+ return nil
|
||||
+}
|
||||
+
|
||||
+// verifyIP verifies the IP address
|
||||
+func verifyIP(ip string) (*net.IP, error) {
|
||||
+ ip = strings.TrimSpace(ip)
|
||||
+
|
||||
+ if len(ip) > maxIPAddrLen {
|
||||
+ return nil, fmt.Errorf("the length of IP address is too long, max ip len :%d", maxIPAddrLen)
|
||||
+ }
|
||||
+
|
||||
+ var netIP net.IP
|
||||
+ if netIP = net.ParseIP(ip); netIP == nil {
|
||||
+ return nil, fmt.Errorf("invalid IP: %s", ip)
|
||||
+ }
|
||||
+
|
||||
+ return &netIP, nil
|
||||
+}
|
||||
+
|
||||
+// validInterface check the input interface valid or not
|
||||
+func validInterface(inf *vcTypes.Interface, enableCompatOldCNI bool) error {
|
||||
+ if enableCompatOldCNI && verifyInterfaceName(inf.Device) != nil {
|
||||
+ return fmt.Errorf("device name should not be empty when enable_compat_old_cni config enabled")
|
||||
+ }
|
||||
+
|
||||
+ if inf.Name == "" || inf.Mtu == 0 || inf.HwAddr == "" {
|
||||
+ return fmt.Errorf("name/mtu/hwaddr of interface must be specified")
|
||||
+ }
|
||||
+
|
||||
+ if err := verifyInterfaceName(inf.Name); err != nil {
|
||||
+ return err
|
||||
+ }
|
||||
+
|
||||
+ if err := verifyMac(inf.HwAddr); err != nil {
|
||||
+ return err
|
||||
+ }
|
||||
+
|
||||
+ if err := verifyMtu(inf.Mtu); err != nil {
|
||||
+ return err
|
||||
+ }
|
||||
+
|
||||
+ // Currently, only one IP address can be passed, which reduces the test entry and fault injection.
|
||||
+ if len(inf.IPAddresses) > 0 {
|
||||
+ if len(inf.IPAddresses) != 1 {
|
||||
+ return fmt.Errorf("only one IP address is supported currently")
|
||||
+ }
|
||||
+ _, err := verifyIP(inf.IPAddresses[0].Address)
|
||||
+ if err != nil {
|
||||
+ return err
|
||||
+ }
|
||||
+ }
|
||||
+
|
||||
+ return nil
|
||||
+}
|
||||
diff --git a/virtcontainers/sandbox.go b/virtcontainers/sandbox.go
|
||||
index 6a643a12..f6826812 100644
|
||||
--- a/virtcontainers/sandbox.go
|
||||
+++ b/virtcontainers/sandbox.go
|
||||
@@ -952,6 +952,17 @@ func (s *Sandbox) generateNetInfo(inf *vcTypes.Interface) (NetworkInfo, error) {
|
||||
|
||||
// AddInterface adds new nic to the sandbox.
|
||||
func (s *Sandbox) AddInterface(inf *vcTypes.Interface) (grpcIf *vcTypes.Interface, err error) {
|
||||
+ err = validInterface(inf, s.config.NetworkConfig.EnableCompatOldCNI)
|
||||
+ if err != nil {
|
||||
+ return nil, err
|
||||
+ }
|
||||
+
|
||||
+ for _, ep := range s.networkNS.Endpoints {
|
||||
+ if ep.Name() == inf.Name {
|
||||
+ return nil, fmt.Errorf("interface %s is already exist", inf.Name)
|
||||
+ }
|
||||
+ }
|
||||
+
|
||||
netInfo, err := s.generateNetInfo(inf)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
--
|
||||
2.14.3 (Apple Git-98)
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@ -1,192 +0,0 @@
|
||||
From 51a7270987557ab12ea735fc9781725d1ce1b0a6 Mon Sep 17 00:00:00 2001
|
||||
From: jiangpengfei <jiangpengfei9@huawei.com>
|
||||
Date: Mon, 17 Aug 2020 15:36:32 +0800
|
||||
Subject: [PATCH 29/50] network: add kata-network del-route subcommand
|
||||
|
||||
reason: add kata-network del-route subcommand to delete the
|
||||
specified route in the Kata VM, del-route is more efficient
|
||||
than the update-routes subcommand.
|
||||
|
||||
Signed-off-by: jiangpengfei <jiangpengfei9@huawei.com>
|
||||
---
|
||||
cli/network.go | 24 ++++++++++
|
||||
virtcontainers/network.go | 115 ++++++++++++++++++++++++++++++++++++++++++++++
|
||||
2 files changed, 139 insertions(+)
|
||||
|
||||
diff --git a/cli/network.go b/cli/network.go
|
||||
index 2265f54b..046d0ee9 100644
|
||||
--- a/cli/network.go
|
||||
+++ b/cli/network.go
|
||||
@@ -45,6 +45,7 @@ var kataNetworkCLICommand = cli.Command{
|
||||
updateRoutesCommand,
|
||||
listRoutesCommand,
|
||||
addRoutesCommand,
|
||||
+ deleteRoutesCommand,
|
||||
},
|
||||
Action: func(context *cli.Context) error {
|
||||
return cli.ShowSubcommandHelp(context)
|
||||
@@ -155,6 +156,29 @@ var addRoutesCommand = cli.Command{
|
||||
},
|
||||
}
|
||||
|
||||
+var deleteRoutesCommand = cli.Command{
|
||||
+ Name: "del-route",
|
||||
+ Usage: "delete one route for a container",
|
||||
+ ArgsUsage: `del-route <container-id> file or - for stdin
|
||||
+ file or stdin for example:
|
||||
+ {
|
||||
+ "dest":"<[<ip>[/mask] | "default" ]>",
|
||||
+ "gateway":"[ip]",
|
||||
+ "device":"[tap-name]",
|
||||
+ }
|
||||
+ Only destination is required and others are optional,
|
||||
+ if device is empty, means search every device`,
|
||||
+ Flags: []cli.Flag{},
|
||||
+ Action: func(context *cli.Context) error {
|
||||
+ ctx, err := cliContextToContext(context)
|
||||
+ if err != nil {
|
||||
+ return err
|
||||
+ }
|
||||
+
|
||||
+ return networkModifyCommand(ctx, context.Args().First(), context.Args().Get(1), routeType, vcTypes.NetworkOpRemove)
|
||||
+ },
|
||||
+}
|
||||
+
|
||||
var listRoutesCommand = cli.Command{
|
||||
Name: "list-routes",
|
||||
Usage: "list network routes in a container",
|
||||
diff --git a/virtcontainers/network.go b/virtcontainers/network.go
|
||||
index f3757f84..c7066a11 100644
|
||||
--- a/virtcontainers/network.go
|
||||
+++ b/virtcontainers/network.go
|
||||
@@ -1631,6 +1631,110 @@ func addOneRoute(ns *NetworkNamespace, route *vcTypes.Route) (added *vcTypes.Rou
|
||||
return added, nil
|
||||
}
|
||||
|
||||
+func generateRmRoute(route *vcTypes.Route) (r *netlink.Route, err error) {
|
||||
+ // destination is required and others are optional
|
||||
+ if route.Dest == "" {
|
||||
+ return nil, fmt.Errorf("destination must be specified when remove route.")
|
||||
+ }
|
||||
+
|
||||
+ if route.Device != "" {
|
||||
+ err = verifyInterfaceName(route.Device)
|
||||
+ if err != nil {
|
||||
+ return nil, err
|
||||
+ }
|
||||
+ }
|
||||
+
|
||||
+ r = &netlink.Route{}
|
||||
+ if route.Dest != "default" {
|
||||
+ nlIpNet, err := verifyRouteDest(&route.Dest)
|
||||
+ if err != nil {
|
||||
+ return nil, err
|
||||
+ }
|
||||
+ r.Dst = nlIpNet
|
||||
+ }
|
||||
+
|
||||
+ if route.Gateway != "" {
|
||||
+ nIP, err := verifyIP(route.Gateway)
|
||||
+ if err != nil {
|
||||
+ return nil, err
|
||||
+ }
|
||||
+ r.Gw = *nIP
|
||||
+ }
|
||||
+
|
||||
+ if route.Source != "" {
|
||||
+ nIP, err := verifyIP(route.Source)
|
||||
+ if err != nil {
|
||||
+ return nil, err
|
||||
+ }
|
||||
+ r.Src = *nIP
|
||||
+ }
|
||||
+
|
||||
+ r.Scope = netlink.Scope(route.Scope)
|
||||
+
|
||||
+ return r, nil
|
||||
+}
|
||||
+
|
||||
+// parseToGrpcRoute convert the netlink.Route to vcTypes.Route and deleted route.Dest will
|
||||
+// be added a prefix "-"
|
||||
+func parseToGrpcRoute(device string, route *netlink.Route, add bool) (r *vcTypes.Route) {
|
||||
+ r = &vcTypes.Route{
|
||||
+ Device: device,
|
||||
+ }
|
||||
+ if route.Dst != nil && route.Dst.String() != "<nil>" {
|
||||
+ r.Dest = route.Dst.String()
|
||||
+ if add == false {
|
||||
+ r.Dest = "-" + route.Dst.String()
|
||||
+ }
|
||||
+ } else if route.Dst == nil {
|
||||
+ r.Dest = "default"
|
||||
+ if add == false {
|
||||
+ r.Dest = "-default"
|
||||
+ }
|
||||
+ }
|
||||
+ if route.Gw != nil && route.Gw.String() != "<nil>" {
|
||||
+ r.Gateway = route.Gw.String()
|
||||
+ }
|
||||
+
|
||||
+ return r
|
||||
+}
|
||||
+
|
||||
+func removeRoutes(ns *NetworkNamespace, route *vcTypes.Route) (removed []*vcTypes.Route, err error) {
|
||||
+ del, err := generateRmRoute(route)
|
||||
+ if err != nil {
|
||||
+ return nil, err
|
||||
+ }
|
||||
+
|
||||
+ // remove the lo device related routes
|
||||
+ if route.Device == localHostDeviceName {
|
||||
+ removed = append(removed, parseToGrpcRoute(localHostDeviceName, del, false))
|
||||
+
|
||||
+ return removed, nil
|
||||
+ }
|
||||
+
|
||||
+ for _, ep := range ns.Endpoints {
|
||||
+ // if device is empty, means search every device
|
||||
+ if route.Device != "" && ep.Name() != route.Device {
|
||||
+ continue
|
||||
+ }
|
||||
+
|
||||
+ netInfo := ep.Properties()
|
||||
+ for i, exist := range ep.Properties().Routes {
|
||||
+ if isSameRoute(&exist, del, true) {
|
||||
+ // need remove
|
||||
+ netInfo.Routes = append(netInfo.Routes[:i], netInfo.Routes[i+1:]...)
|
||||
+ ep.SetProperties(netInfo)
|
||||
+ dev := route.Device
|
||||
+ if route.Device == "" {
|
||||
+ dev = netInfo.Iface.Name
|
||||
+ }
|
||||
+ removed = append(removed, parseToGrpcRoute(dev, del, false))
|
||||
+ }
|
||||
+ }
|
||||
+ }
|
||||
+
|
||||
+ return removed, nil
|
||||
+}
|
||||
+
|
||||
func updateRoute(ns *NetworkNamespace, route *vcTypes.Route, op vcTypes.NetworkOp) ([]*vcTypes.Route, error) {
|
||||
var updRoutes []*vcTypes.Route
|
||||
|
||||
@@ -1645,5 +1749,16 @@ func updateRoute(ns *NetworkNamespace, route *vcTypes.Route, op vcTypes.NetworkO
|
||||
updRoutes = append(updRoutes, added)
|
||||
}
|
||||
|
||||
+ if op == vcTypes.NetworkOpRemove {
|
||||
+ removed, err := removeRoutes(ns, route)
|
||||
+ if err != nil {
|
||||
+ return nil, err
|
||||
+ }
|
||||
+ if len(removed) <= 0 {
|
||||
+ return nil, fmt.Errorf("route of device %s with destination %s is not found", route.Device, route.Dest)
|
||||
+ }
|
||||
+ updRoutes = removed
|
||||
+ }
|
||||
+
|
||||
return updRoutes, nil
|
||||
}
|
||||
--
|
||||
2.14.3 (Apple Git-98)
|
||||
|
||||
@ -1,104 +0,0 @@
|
||||
From 9cf769178b8f73c7fd624895589e918d6c7b0645 Mon Sep 17 00:00:00 2001
|
||||
From: jiangpengfei <jiangpengfei9@huawei.com>
|
||||
Date: Mon, 17 Aug 2020 16:29:17 +0800
|
||||
Subject: [PATCH 30/50] network: kata-network list-routes support display
|
||||
compatible format
|
||||
|
||||
reason: kata-network list-routes subcommand support display compatible
|
||||
format when enable_compat_old_cni config is enabled.
|
||||
|
||||
Signed-off-by: jiangpengfei <jiangpengfei9@huawei.com>
|
||||
---
|
||||
virtcontainers/api.go | 13 ++++++++++++-
|
||||
virtcontainers/network.go | 29 +++++++++++++++++++++++++++++
|
||||
virtcontainers/pkg/types/types.go | 10 +++++-----
|
||||
3 files changed, 46 insertions(+), 6 deletions(-)
|
||||
|
||||
diff --git a/virtcontainers/api.go b/virtcontainers/api.go
|
||||
index 2331eb94..a4bf41bb 100644
|
||||
--- a/virtcontainers/api.go
|
||||
+++ b/virtcontainers/api.go
|
||||
@@ -1003,7 +1003,18 @@ func ListRoutes(ctx context.Context, sandboxID string) ([]*vcTypes.Route, error)
|
||||
}
|
||||
defer s.releaseStatelessSandbox()
|
||||
|
||||
- return s.ListRoutes()
|
||||
+ routes, err := s.ListRoutes()
|
||||
+ if err != nil {
|
||||
+ return nil, err
|
||||
+ }
|
||||
+
|
||||
+ // If enable_compat_old_cni is enabled, convert routes info to
|
||||
+ // compatible display format
|
||||
+ if s.config.NetworkConfig.EnableCompatOldCNI {
|
||||
+ return convertToDisplayRoutes(&routes), nil
|
||||
+ }
|
||||
+
|
||||
+ return routes, nil
|
||||
}
|
||||
|
||||
// CleanupContaienr is used by shimv2 to stop and delete a container exclusively, once there is no container
|
||||
diff --git a/virtcontainers/network.go b/virtcontainers/network.go
|
||||
index c7066a11..488bd00c 100644
|
||||
--- a/virtcontainers/network.go
|
||||
+++ b/virtcontainers/network.go
|
||||
@@ -1384,6 +1384,35 @@ func convertToCompatInterfaces(es *[]Endpoint) []*vcTypes.Interface {
|
||||
return infs
|
||||
}
|
||||
|
||||
+// convertToDisplayRoutes convert the default route format to more simple
|
||||
+// route display format, it is called when enable_compat_old_cni config
|
||||
+// is enabled.
|
||||
+func convertToDisplayRoutes(routes *[]*vcTypes.Route) []*vcTypes.Route {
|
||||
+ var displayRoutes []*vcTypes.Route
|
||||
+ if routes == nil {
|
||||
+ return displayRoutes
|
||||
+ }
|
||||
+ for _, r := range *routes {
|
||||
+ // we don't support IPV6 temporarily, so we need to filter ":" which
|
||||
+ // is the characteristics of IPV6 for those default routes
|
||||
+ if strings.Contains(r.Dest, ":") {
|
||||
+ continue
|
||||
+ }
|
||||
+
|
||||
+ defaultDest := r.Dest
|
||||
+ if r.Dest == "" {
|
||||
+ defaultDest = "default"
|
||||
+ }
|
||||
+ displayRoutes = append(displayRoutes, &vcTypes.Route{
|
||||
+ Dest: defaultDest,
|
||||
+ Gateway: r.Gateway,
|
||||
+ Device: r.Device,
|
||||
+ })
|
||||
+ }
|
||||
+
|
||||
+ return displayRoutes
|
||||
+}
|
||||
+
|
||||
// verifyInterfaceName verifies the interface name valid or not
|
||||
func verifyInterfaceName(name string) error {
|
||||
// verify `Name` before `Tapname` because of the strict rules
|
||||
diff --git a/virtcontainers/pkg/types/types.go b/virtcontainers/pkg/types/types.go
|
||||
index 71fe7fbb..b41b0c75 100644
|
||||
--- a/virtcontainers/pkg/types/types.go
|
||||
+++ b/virtcontainers/pkg/types/types.go
|
||||
@@ -33,11 +33,11 @@ type Interface struct {
|
||||
|
||||
// Route describes a network route.
|
||||
type Route struct {
|
||||
- Dest string
|
||||
- Gateway string
|
||||
- Device string
|
||||
- Source string
|
||||
- Scope uint32
|
||||
+ Dest string `json:"dest,omitempty"`
|
||||
+ Gateway string `json:"gateway,omitempty"`
|
||||
+ Device string `json:"device,omitempty"`
|
||||
+ Source string `json:"source,omitempty"`
|
||||
+ Scope uint32 `json:"scope,omitempty"`
|
||||
}
|
||||
|
||||
//NetworkOp describes network operation
|
||||
--
|
||||
2.14.3 (Apple Git-98)
|
||||
|
||||
@ -1,41 +0,0 @@
|
||||
From 326e90f97cf5ace73dff95257f7b4faa43c56f99 Mon Sep 17 00:00:00 2001
|
||||
From: holyfei <yangfeiyu20092010@163.com>
|
||||
Date: Mon, 17 Aug 2020 16:11:35 +0800
|
||||
Subject: [PATCH 31/50] device_mangaer: check VFIO when create device
|
||||
|
||||
reason: check VFIO when create device, block device can be
|
||||
reused and VFIO device can not
|
||||
|
||||
Signed-off-by: yangfeiyu <yangfeiyu2@huawei.com>
|
||||
---
|
||||
virtcontainers/device/manager/manager.go | 7 ++++++-
|
||||
1 file changed, 6 insertions(+), 1 deletion(-)
|
||||
|
||||
diff --git a/virtcontainers/device/manager/manager.go b/virtcontainers/device/manager/manager.go
|
||||
index 6a2b665a..d24a29ab 100644
|
||||
--- a/virtcontainers/device/manager/manager.go
|
||||
+++ b/virtcontainers/device/manager/manager.go
|
||||
@@ -9,6 +9,7 @@ package manager
|
||||
import (
|
||||
"encoding/hex"
|
||||
"errors"
|
||||
+ "fmt"
|
||||
"sync"
|
||||
|
||||
"github.com/sirupsen/logrus"
|
||||
@@ -115,7 +116,11 @@ func (dm *deviceManager) createDevice(devInfo config.DeviceInfo) (dev api.Device
|
||||
}()
|
||||
|
||||
if existingDev := dm.findDeviceByMajorMinor(devInfo.Major, devInfo.Minor); existingDev != nil {
|
||||
- return existingDev, nil
|
||||
+ if isVFIO(devInfo.HostPath) {
|
||||
+ return nil, fmt.Errorf("device %s is replicated in the same Pod!", devInfo.ContainerPath)
|
||||
+ } else {
|
||||
+ return existingDev, nil
|
||||
+ }
|
||||
}
|
||||
|
||||
// device ID must be generated by manager instead of device itself
|
||||
--
|
||||
2.14.3 (Apple Git-98)
|
||||
|
||||
@ -1,46 +0,0 @@
|
||||
From 77711b531867be899df5d2c59a525ea1b7f0e6ac Mon Sep 17 00:00:00 2001
|
||||
From: jiangpengfei <jiangpengfei9@huawei.com>
|
||||
Date: Mon, 17 Aug 2020 17:14:35 +0800
|
||||
Subject: [PATCH 32/50] network: add more detail usage for update-routes
|
||||
subcommand
|
||||
|
||||
reason: add more detail usage for update-routes subcommand
|
||||
to explain how to use update-routes subcommand
|
||||
|
||||
Signed-off-by: jiangpengfei <jiangpengfei9@huawei.com>
|
||||
---
|
||||
cli/network.go | 18 ++++++++++++++----
|
||||
1 file changed, 14 insertions(+), 4 deletions(-)
|
||||
|
||||
diff --git a/cli/network.go b/cli/network.go
|
||||
index 046d0ee9..3dd0971e 100644
|
||||
--- a/cli/network.go
|
||||
+++ b/cli/network.go
|
||||
@@ -120,10 +120,20 @@ var listIfacesCommand = cli.Command{
|
||||
}
|
||||
|
||||
var updateRoutesCommand = cli.Command{
|
||||
- Name: "update-routes",
|
||||
- Usage: "update routes of a container",
|
||||
- ArgsUsage: `update-routes <container-id> file or - for stdin`,
|
||||
- Flags: []cli.Flag{},
|
||||
+ Name: "update-routes",
|
||||
+ Usage: "update routes of a container",
|
||||
+ ArgsUsage: `update-routes <container-id> file or - for stdin
|
||||
+ file or stdin for example:
|
||||
+ [
|
||||
+ {
|
||||
+ "dest":"<[<ip>[/mask] | "default" ]>",
|
||||
+ "gateway":"[ip]",
|
||||
+ "device":"[tap-name]",
|
||||
+ "source": "[source]",
|
||||
+ "scope":[scope]
|
||||
+ }
|
||||
+ ]`,
|
||||
+ Flags: []cli.Flag{},
|
||||
Action: func(context *cli.Context) error {
|
||||
ctx, err := cliContextToContext(context)
|
||||
if err != nil {
|
||||
--
|
||||
2.14.3 (Apple Git-98)
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@ -1,234 +0,0 @@
|
||||
From cf2b34f477cba88641de3719bf5c8f933b919bcc Mon Sep 17 00:00:00 2001
|
||||
From: holyfei <yangfeiyu20092010@163.com>
|
||||
Date: Mon, 17 Aug 2020 21:44:57 +0800
|
||||
Subject: [PATCH 35/50] device: mount blockdevices in the guest VM
|
||||
|
||||
reason: support mount blockdevices in the guest VM like
|
||||
"-v /dev/blockdevice:/home/test"
|
||||
|
||||
Signed-off-by: yangfeiyu <yangfeiyu2@huawei.com>
|
||||
---
|
||||
cli/config/configuration-qemu.toml.in | 5 ++++
|
||||
pkg/katautils/config.go | 6 +++++
|
||||
virtcontainers/kata_agent.go | 27 ++++++++++++++++---
|
||||
virtcontainers/kata_agent_test.go | 2 +-
|
||||
virtcontainers/utils/utils.go | 50 +++++++++++++++++++++++++++++++++++
|
||||
virtcontainers/vm_test.go | 2 +-
|
||||
6 files changed, 86 insertions(+), 6 deletions(-)
|
||||
|
||||
diff --git a/cli/config/configuration-qemu.toml.in b/cli/config/configuration-qemu.toml.in
|
||||
index 46f8b632..aa11b38f 100644
|
||||
--- a/cli/config/configuration-qemu.toml.in
|
||||
+++ b/cli/config/configuration-qemu.toml.in
|
||||
@@ -391,6 +391,11 @@ path = "@SHIMPATH@"
|
||||
#
|
||||
kernel_modules=[]
|
||||
|
||||
+# If enabled, when we pass a block device to the guest VM
|
||||
+# through -v, such as `docker run -v /dev/loop100:/foo/bar`,
|
||||
+# agent will create a directory in guest VM and mount the
|
||||
+# file system on `/dev/loop100` inside the container
|
||||
+enable_blk_mount = true
|
||||
|
||||
[netmon]
|
||||
# If enabled, the network monitoring process gets started when the
|
||||
diff --git a/pkg/katautils/config.go b/pkg/katautils/config.go
|
||||
index 94c916a0..51120311 100644
|
||||
--- a/pkg/katautils/config.go
|
||||
+++ b/pkg/katautils/config.go
|
||||
@@ -160,6 +160,7 @@ type agent struct {
|
||||
TraceMode string `toml:"trace_mode"`
|
||||
TraceType string `toml:"trace_type"`
|
||||
KernelModules []string `toml:"kernel_modules"`
|
||||
+ MountBlkInVM bool `toml:"enable_blk_mount"`
|
||||
}
|
||||
|
||||
type netmon struct {
|
||||
@@ -463,6 +464,10 @@ func (h hypervisor) getInitrdAndImage() (initrd string, image string, err error)
|
||||
return
|
||||
}
|
||||
|
||||
+func (a agent) mountBlkDevInVM() bool {
|
||||
+ return a.MountBlkInVM
|
||||
+}
|
||||
+
|
||||
func (p proxy) path() (string, error) {
|
||||
path := p.Path
|
||||
if path == "" {
|
||||
@@ -978,6 +983,7 @@ func updateRuntimeConfigAgent(configPath string, tomlConf tomlConfig, config *oc
|
||||
TraceMode: agent.traceMode(),
|
||||
TraceType: agent.traceType(),
|
||||
KernelModules: agent.kernelModules(),
|
||||
+ MountBlkInVM: agent.mountBlkDevInVM(),
|
||||
}
|
||||
default:
|
||||
return fmt.Errorf("%s agent type is not supported", k)
|
||||
diff --git a/virtcontainers/kata_agent.go b/virtcontainers/kata_agent.go
|
||||
index 16662949..b0f88c15 100644
|
||||
--- a/virtcontainers/kata_agent.go
|
||||
+++ b/virtcontainers/kata_agent.go
|
||||
@@ -30,6 +30,7 @@ import (
|
||||
ns "github.com/kata-containers/runtime/virtcontainers/pkg/nsenter"
|
||||
"github.com/kata-containers/runtime/virtcontainers/pkg/rootless"
|
||||
vcTypes "github.com/kata-containers/runtime/virtcontainers/pkg/types"
|
||||
+ "github.com/kata-containers/runtime/virtcontainers/utils"
|
||||
"github.com/kata-containers/runtime/virtcontainers/pkg/uuid"
|
||||
"github.com/kata-containers/runtime/virtcontainers/store"
|
||||
"github.com/kata-containers/runtime/virtcontainers/types"
|
||||
@@ -64,6 +65,7 @@ var (
|
||||
errorMissingOCISpec = errors.New("Missing OCI specification")
|
||||
defaultKataHostSharedDir = "/run/kata-containers/shared/sandboxes/"
|
||||
defaultKataGuestSharedDir = "/run/kata-containers/shared/containers/"
|
||||
+ kataGuestStorageDir = "/run/kata-containers/storage/containers/"
|
||||
mountGuestTag = "kataShared"
|
||||
defaultKataGuestSandboxDir = "/run/kata-containers/sandbox/"
|
||||
type9pFs = "9p"
|
||||
@@ -199,6 +201,7 @@ type KataAgentConfig struct {
|
||||
TraceMode string
|
||||
TraceType string
|
||||
KernelModules []string
|
||||
+ MountBlkInVM bool
|
||||
}
|
||||
|
||||
// KataAgentState is the structure describing the data stored from this
|
||||
@@ -1061,7 +1064,12 @@ func (k *kataAgent) replaceOCIMountsForStorages(spec *specs.Spec, volumeStorages
|
||||
// Create a temporary location to mount the Storage. Mounting to the correct location
|
||||
// will be handled by the OCI mount structure.
|
||||
filename := fmt.Sprintf("%s-%s", uuid.Generate().String(), filepath.Base(m.Destination))
|
||||
- path := filepath.Join(kataGuestSandboxStorageDir(), filename)
|
||||
+ var path string
|
||||
+ if v.Driver == kataBlkDevType || v.Driver == kataSCSIDevType {
|
||||
+ path = filepath.Join(kataGuestStorageDir, filename)
|
||||
+ } else {
|
||||
+ path = filepath.Join(kataGuestSandboxStorageDir(), filename)
|
||||
+ }
|
||||
|
||||
k.Logger().Debugf("Replacing OCI mount source (%s) with %s", m.Source, path)
|
||||
ociMounts[index].Source = path
|
||||
@@ -1436,7 +1444,7 @@ func (k *kataAgent) createContainer(sandbox *Sandbox, c *Container) (p *Process,
|
||||
// Note this call modifies the list of container devices to make sure
|
||||
// all hotplugged devices are unplugged, so this needs be done
|
||||
// after devices passed with --device are handled.
|
||||
- volumeStorages, err := k.handleBlockVolumes(c)
|
||||
+ volumeStorages, err := k.handleBlockVolumes(sandbox, c)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@@ -1609,7 +1617,7 @@ func (k *kataAgent) handleVhostUserBlkVolume(c *Container, device api.Device) (*
|
||||
|
||||
// handleBlockVolumes handles volumes that are block devices files
|
||||
// by passing the block devices as Storage to the agent.
|
||||
-func (k *kataAgent) handleBlockVolumes(c *Container) ([]*grpc.Storage, error) {
|
||||
+func (k *kataAgent) handleBlockVolumes(sandbox *Sandbox, c *Container) ([]*grpc.Storage, error) {
|
||||
|
||||
var volumeStorages []*grpc.Storage
|
||||
|
||||
@@ -1648,9 +1656,20 @@ func (k *kataAgent) handleBlockVolumes(c *Container) ([]*grpc.Storage, error) {
|
||||
}
|
||||
|
||||
vol.MountPoint = m.Destination
|
||||
- if vol.Fstype == "" {
|
||||
+
|
||||
+ ac, _ := sandbox.config.AgentConfig.(KataAgentConfig)
|
||||
+ if ac.MountBlkInVM {
|
||||
+ // Ensure the block device is formatted, for the devices here are specified as volumes
|
||||
+ fsType, gerr := utils.GetDevFormat(m.Source)
|
||||
+ if gerr != nil || fsType == "" {
|
||||
+ k.Logger().WithField("device", id).WithError(gerr).Error("get device format failed")
|
||||
+ return nil, gerr
|
||||
+ }
|
||||
+ vol.Fstype = fmt.Sprintf("bind-%s", fsType)
|
||||
+ } else {
|
||||
vol.Fstype = "bind"
|
||||
}
|
||||
+
|
||||
if len(vol.Options) == 0 {
|
||||
vol.Options = []string{"bind"}
|
||||
}
|
||||
diff --git a/virtcontainers/kata_agent_test.go b/virtcontainers/kata_agent_test.go
|
||||
index 18a5a0a6..68caaab2 100644
|
||||
--- a/virtcontainers/kata_agent_test.go
|
||||
+++ b/virtcontainers/kata_agent_test.go
|
||||
@@ -446,7 +446,7 @@ func TestHandleBlockVolume(t *testing.T) {
|
||||
containers[c.id].sandbox = &sandbox
|
||||
containers[c.id].mounts = mounts
|
||||
|
||||
- volumeStorages, err := k.handleBlockVolumes(c)
|
||||
+ volumeStorages, err := k.handleBlockVolumes(&sandbox, c)
|
||||
assert.Nil(t, err, "Error while handling block volumes")
|
||||
|
||||
vStorage := &pb.Storage{
|
||||
diff --git a/virtcontainers/utils/utils.go b/virtcontainers/utils/utils.go
|
||||
index 9490faa1..36ac67a7 100644
|
||||
--- a/virtcontainers/utils/utils.go
|
||||
+++ b/virtcontainers/utils/utils.go
|
||||
@@ -379,3 +379,53 @@ func RoundVCPUNumber(value string) (int, error) {
|
||||
cpus := int(math.Ceil(cpuNum))
|
||||
return cpus, nil
|
||||
}
|
||||
+
|
||||
+// GetDevFormat get the formated filesystem of input disk use `blkid`, Such as `ext4`,`xfs`,etc.
|
||||
+func GetDevFormat(disk string) (string, error) {
|
||||
+ // refer to https://github.com/kubernetes/kubernetes/blob/v1.12.2/pkg/util/mount/mount_linux.go#L512
|
||||
+ args := []string{"-p", "-s", "TYPE", "-s", "PTTYPE", "-o", "export", disk}
|
||||
+ dataOut, err := exec.Command("blkid", args...).Output()
|
||||
+ output := string(dataOut)
|
||||
+
|
||||
+ if err != nil {
|
||||
+ if strings.Contains(err.Error(), "exit status 2") {
|
||||
+ // Disk device is unformatted.
|
||||
+ // For `blkid`, if the specified token (TYPE/PTTYPE, etc) was
|
||||
+ // not found, or no (specified) devices could be identified, an
|
||||
+ // exit code of 2 is returned.
|
||||
+ return "", nil
|
||||
+ }
|
||||
+ return "", err
|
||||
+ }
|
||||
+
|
||||
+ var fstype string
|
||||
+
|
||||
+ lines := strings.Split(output, "\n")
|
||||
+ for _, l := range lines {
|
||||
+ if len(l) <= 0 {
|
||||
+ // Ignore empty line.
|
||||
+ continue
|
||||
+ }
|
||||
+ // if we use busybox as rootfs,the output of command
|
||||
+ // `busybox blkid` is different with original`blkid`,
|
||||
+ // so we should make a compatible parse
|
||||
+ subLine := strings.Split(l, " ")
|
||||
+ for _, sl := range subLine {
|
||||
+ if len(subLine) <= 0 {
|
||||
+ continue
|
||||
+ }
|
||||
+ cs := strings.Split(sl, "=")
|
||||
+ if len(cs) != 2 {
|
||||
+ continue
|
||||
+ }
|
||||
+ if cs[0] == "TYPE" {
|
||||
+ fstype = cs[1]
|
||||
+ if strings.Contains(fstype, `"`) {
|
||||
+ fstype = strings.Replace(fstype, `"`, "", -1)
|
||||
+ }
|
||||
+ }
|
||||
+ }
|
||||
+ }
|
||||
+
|
||||
+ return fstype, nil
|
||||
+}
|
||||
diff --git a/virtcontainers/vm_test.go b/virtcontainers/vm_test.go
|
||||
index 36fd5c2f..d2414232 100644
|
||||
--- a/virtcontainers/vm_test.go
|
||||
+++ b/virtcontainers/vm_test.go
|
||||
@@ -125,7 +125,7 @@ func TestVMConfigGrpc(t *testing.T) {
|
||||
HypervisorType: QemuHypervisor,
|
||||
HypervisorConfig: newQemuConfig(),
|
||||
AgentType: KataContainersAgent,
|
||||
- AgentConfig: KataAgentConfig{false, true, false, false, 0, "", "", []string{}},
|
||||
+ AgentConfig: KataAgentConfig{false, true, false, false, 0, "", "", []string{}, false },
|
||||
ProxyType: NoopProxyType,
|
||||
}
|
||||
|
||||
--
|
||||
2.14.3 (Apple Git-98)
|
||||
|
||||
@ -1,131 +0,0 @@
|
||||
From 448bb661d6759c1e20c18084604b150bff08ada4 Mon Sep 17 00:00:00 2001
|
||||
From: holyfei <yangfeiyu20092010@163.com>
|
||||
Date: Tue, 18 Aug 2020 11:49:26 +0800
|
||||
Subject: [PATCH 36/50] mount: limit the maximum number of virtio-scsi bus
|
||||
slots
|
||||
|
||||
reason:
|
||||
1. add SCSIBus functions for add or remove device about ScsiBus
|
||||
2. limit the maximum number of virtio-scsi bus slots to 25
|
||||
|
||||
Signed-off-by: yangfeiyu <yangfeiyu2@huawei.com>
|
||||
---
|
||||
virtcontainers/qemu.go | 20 +++++++++++++++++
|
||||
virtcontainers/types/scsi.go | 53 ++++++++++++++++++++++++++++++++++++++++++++
|
||||
2 files changed, 73 insertions(+)
|
||||
create mode 100644 virtcontainers/types/scsi.go
|
||||
|
||||
diff --git a/virtcontainers/qemu.go b/virtcontainers/qemu.go
|
||||
index bb83b1bb..c2b65376 100644
|
||||
--- a/virtcontainers/qemu.go
|
||||
+++ b/virtcontainers/qemu.go
|
||||
@@ -65,6 +65,7 @@ type CPUDevice struct {
|
||||
// QemuState keeps Qemu's state
|
||||
type QemuState struct {
|
||||
Bridges []types.Bridge
|
||||
+ ScsiBus *types.SCSIBus
|
||||
// HotpluggedCPUs is the list of CPUs that were hot-added
|
||||
HotpluggedVCPUs []CPUDevice
|
||||
HotpluggedMemory int
|
||||
@@ -411,6 +412,13 @@ func (q *qemu) buildDevices(initrdPath string) ([]govmmQemu.Device, *govmmQemu.I
|
||||
|
||||
var ioThread *govmmQemu.IOThread
|
||||
if q.config.BlockDeviceDriver == config.VirtioSCSI {
|
||||
+ if q.state.ScsiBus == nil {
|
||||
+ // only use `scsi0.0`
|
||||
+ q.state.ScsiBus = &types.SCSIBus{
|
||||
+ Address: make(map[uint32]int),
|
||||
+ ID: fmt.Sprintf("%s.0", scsiControllerID),
|
||||
+ }
|
||||
+ }
|
||||
return q.arch.appendSCSIController(devices, q.config.EnableIOThreads)
|
||||
}
|
||||
|
||||
@@ -1168,6 +1176,16 @@ func (q *qemu) hotplugAddBlockDevice(drive *config.BlockDrive, op operation, dev
|
||||
|
||||
// Bus exposed by the SCSI Controller
|
||||
bus := scsiControllerID + ".0"
|
||||
+ var err error
|
||||
+ if err = q.state.ScsiBus.AddDevToScsiBus(drive.Index); err != nil {
|
||||
+ return err
|
||||
+ }
|
||||
+
|
||||
+ defer func() {
|
||||
+ if err != nil {
|
||||
+ q.state.ScsiBus.RemoveDevFromScsiBus(drive.Index)
|
||||
+ }
|
||||
+ }()
|
||||
|
||||
// Get SCSI-id and LUN based on the order of attaching drives.
|
||||
scsiID, lun, err := utils.GetSCSIIdLun(drive.Index)
|
||||
@@ -1234,6 +1252,8 @@ func (q *qemu) hotplugBlockDevice(drive *config.BlockDrive, op operation) error
|
||||
if err := q.arch.removeDeviceFromBridge(drive.ID); err != nil {
|
||||
return err
|
||||
}
|
||||
+ } else if q.config.BlockDeviceDriver == config.VirtioSCSI {
|
||||
+ q.state.ScsiBus.RemoveDevFromScsiBus(drive.Index)
|
||||
}
|
||||
|
||||
if err := q.qmpMonitorCh.qmp.ExecuteDeviceDel(q.qmpMonitorCh.ctx, devID); err != nil {
|
||||
diff --git a/virtcontainers/types/scsi.go b/virtcontainers/types/scsi.go
|
||||
new file mode 100644
|
||||
index 00000000..f4d489f0
|
||||
--- /dev/null
|
||||
+++ b/virtcontainers/types/scsi.go
|
||||
@@ -0,0 +1,53 @@
|
||||
+// Copyright (c) Huawei Technologies Co., Ltd. 2019. All rights reserved.
|
||||
+// SPDX-License-Identifier: Apache-2.0
|
||||
+// Description: common functions
|
||||
+// Author: leizhongkai
|
||||
+// Create: 2019-06-30
|
||||
+
|
||||
+package types
|
||||
+
|
||||
+import (
|
||||
+ "fmt"
|
||||
+)
|
||||
+
|
||||
+const (
|
||||
+ scsiMaxCapacity = 25
|
||||
+)
|
||||
+
|
||||
+type SCSIBus struct {
|
||||
+ // Address contains information about devices plugged and number limit
|
||||
+ Address map[uint32]int
|
||||
+
|
||||
+ // ID is used to identify the bus exposed by the SCSI Controller
|
||||
+ ID string
|
||||
+}
|
||||
+
|
||||
+func (sb *SCSIBus) AddDevToScsiBus(index int) error {
|
||||
+ var addr uint32
|
||||
+
|
||||
+ // looking for the first available address
|
||||
+ for i := uint32(1); i <= scsiMaxCapacity; i++ {
|
||||
+ if _, ok := sb.Address[i]; !ok {
|
||||
+ addr = i
|
||||
+ break
|
||||
+ }
|
||||
+ }
|
||||
+
|
||||
+ if addr == 0 {
|
||||
+ return fmt.Errorf("Scsi bus capacity limited.")
|
||||
+ }
|
||||
+
|
||||
+ // save address and device
|
||||
+ sb.Address[addr] = index
|
||||
+
|
||||
+ return nil
|
||||
+}
|
||||
+
|
||||
+func (sb *SCSIBus) RemoveDevFromScsiBus(index int) {
|
||||
+ for addr, i := range sb.Address {
|
||||
+ if i == index {
|
||||
+ delete(sb.Address, addr)
|
||||
+ return
|
||||
+ }
|
||||
+ }
|
||||
+}
|
||||
--
|
||||
2.14.3 (Apple Git-98)
|
||||
|
||||
@ -1,875 +0,0 @@
|
||||
From d083f0e0247fbded92a0ae2a0e71da4176baed95 Mon Sep 17 00:00:00 2001
|
||||
From: xiadanni <xiadanni1@huawei.com>
|
||||
Date: Tue, 18 Aug 2020 17:08:23 +0800
|
||||
Subject: [PATCH 37/50] runtime: add IPVS test
|
||||
|
||||
Signed-off-by: xiadanni <xiadanni1@huawei.com>
|
||||
---
|
||||
cli/ipvsadm_test.go | 775 +++++++++++++++++++++++++++++++++++++
|
||||
virtcontainers/api_test.go | 36 ++
|
||||
virtcontainers/kata_agent_test.go | 4 +
|
||||
virtcontainers/pkg/vcmock/types.go | 2 +-
|
||||
4 files changed, 816 insertions(+), 1 deletion(-)
|
||||
create mode 100644 cli/ipvsadm_test.go
|
||||
|
||||
diff --git a/cli/ipvsadm_test.go b/cli/ipvsadm_test.go
|
||||
new file mode 100644
|
||||
index 00000000..92958aee
|
||||
--- /dev/null
|
||||
+++ b/cli/ipvsadm_test.go
|
||||
@@ -0,0 +1,775 @@
|
||||
+// Copyright (c) Huawei Technologies Co., Ltd. 2020. All rights reserved.
|
||||
+// SPDX-License-Identifier: Apache-2.0
|
||||
+// Description: IPVS common functions test
|
||||
+// Author: xiadanni
|
||||
+// Create: 2020-08-01
|
||||
+
|
||||
+package main
|
||||
+
|
||||
+import (
|
||||
+ "context"
|
||||
+ "flag"
|
||||
+ "fmt"
|
||||
+ "os"
|
||||
+ "strings"
|
||||
+ "testing"
|
||||
+
|
||||
+ "github.com/kata-containers/agent/protocols/grpc"
|
||||
+ vc "github.com/kata-containers/runtime/virtcontainers"
|
||||
+ vcAnnotations "github.com/kata-containers/runtime/virtcontainers/pkg/annotations"
|
||||
+ "github.com/kata-containers/runtime/virtcontainers/pkg/vcmock"
|
||||
+ "github.com/kata-containers/runtime/virtcontainers/types"
|
||||
+ "github.com/stretchr/testify/assert"
|
||||
+ "github.com/urfave/cli"
|
||||
+)
|
||||
+
|
||||
+func TestKataIPVSCliAction(t *testing.T) {
|
||||
+ assert := assert.New(t)
|
||||
+
|
||||
+ actionFunc, ok := kataIPVSCLICommand.Action.(func(ctx *cli.Context) error)
|
||||
+ assert.True(ok)
|
||||
+
|
||||
+ flagSet := flag.NewFlagSet("kata-IPVS", flag.ContinueOnError)
|
||||
+ ctx := createCLIContext(flagSet)
|
||||
+
|
||||
+ err := actionFunc(ctx)
|
||||
+ assert.NoError(err)
|
||||
+}
|
||||
+
|
||||
+func TestIPVSadmCliAction(t *testing.T) {
|
||||
+ assert := assert.New(t)
|
||||
+
|
||||
+ actionFunc, ok := IPVSadmCommand.Action.(func(ctx *cli.Context) error)
|
||||
+ assert.True(ok)
|
||||
+
|
||||
+ flagSet := flag.NewFlagSet("IPVSadm", flag.ContinueOnError)
|
||||
+ ctx := createCLIContext(flagSet)
|
||||
+
|
||||
+ err := actionFunc(ctx)
|
||||
+ assert.Error(err, "Missing container ID")
|
||||
+}
|
||||
+
|
||||
+func TestIPVSadmCLISuccessful(t *testing.T) {
|
||||
+ assert := assert.New(t)
|
||||
+
|
||||
+ sandbox := &vcmock.Sandbox{
|
||||
+ MockID: testContainerID,
|
||||
+ }
|
||||
+
|
||||
+ sandbox.MockContainers = []*vcmock.Container{
|
||||
+ {
|
||||
+ MockID: sandbox.ID(),
|
||||
+ MockSandbox: sandbox,
|
||||
+ },
|
||||
+ }
|
||||
+
|
||||
+ testingImpl.StatusContainerFunc = func(ctx context.Context, sandboxID, containerID string) (vc.ContainerStatus, error) {
|
||||
+ return vc.ContainerStatus{
|
||||
+ ID: sandbox.ID(),
|
||||
+ Annotations: map[string]string{
|
||||
+ vcAnnotations.ContainerTypeKey: string(vc.PodContainer),
|
||||
+ },
|
||||
+ State: types.ContainerState{
|
||||
+ State: types.StateRunning,
|
||||
+ },
|
||||
+ }, nil
|
||||
+ }
|
||||
+
|
||||
+ testingImpl.UpdateIPVSRuleFunc = func(ctx context.Context, sandboxID string, IPVS *grpc.UpdateIPVSRequest) (*grpc.IPVSResponse, error) {
|
||||
+ return &grpc.IPVSResponse{}, nil
|
||||
+ }
|
||||
+
|
||||
+ defer func() {
|
||||
+ testingImpl.StatusContainerFunc = nil
|
||||
+ testingImpl.UpdateIPVSRuleFunc = nil
|
||||
+ }()
|
||||
+
|
||||
+ path, err := createTempContainerIDMapping(sandbox.ID(), sandbox.ID())
|
||||
+ assert.NoError(err)
|
||||
+ defer os.RemoveAll(path)
|
||||
+
|
||||
+ actionFunc, ok := IPVSadmCommand.Action.(func(ctx *cli.Context) error)
|
||||
+ assert.True(ok)
|
||||
+
|
||||
+ flagSet := flag.NewFlagSet("IPVSadm", flag.ContinueOnError)
|
||||
+ flagSet.Parse([]string{testContainerID})
|
||||
+ flagSet.String("parameters", "-a -t 192.168.0.7:80 -r 192.168.0.4:80", "")
|
||||
+ ctx := createCLIContext(flagSet)
|
||||
+ err = actionFunc(ctx)
|
||||
+ assert.NoError(err)
|
||||
+
|
||||
+ // result not nil
|
||||
+ testingImpl.UpdateIPVSRuleFunc = func(ctx context.Context, sandboxID string, IPVS *grpc.UpdateIPVSRequest) (*grpc.IPVSResponse, error) {
|
||||
+ return &grpc.IPVSResponse{IPVSRes: "IPVS rule updating success"}, nil
|
||||
+ }
|
||||
+
|
||||
+ err = actionFunc(ctx)
|
||||
+ assert.NoError(err)
|
||||
+}
|
||||
+
|
||||
+func TestIPVSadmCLIError(t *testing.T) {
|
||||
+ assert := assert.New(t)
|
||||
+
|
||||
+ sandbox := &vcmock.Sandbox{
|
||||
+ MockID: testContainerID,
|
||||
+ }
|
||||
+
|
||||
+ sandbox.MockContainers = []*vcmock.Container{
|
||||
+ {
|
||||
+ MockID: sandbox.ID(),
|
||||
+ MockSandbox: sandbox,
|
||||
+ },
|
||||
+ }
|
||||
+
|
||||
+ testingImpl.StatusContainerFunc = func(ctx context.Context, sandboxID, containerID string) (vc.ContainerStatus, error) {
|
||||
+ return vc.ContainerStatus{
|
||||
+ ID: sandbox.ID(),
|
||||
+ Annotations: map[string]string{
|
||||
+ vcAnnotations.ContainerTypeKey: string(vc.PodContainer),
|
||||
+ },
|
||||
+ State: types.ContainerState{
|
||||
+ State: types.StateRunning,
|
||||
+ },
|
||||
+ }, nil
|
||||
+ }
|
||||
+
|
||||
+ testingImpl.UpdateIPVSRuleFunc = func(ctx context.Context, sandboxID string, IPVS *grpc.UpdateIPVSRequest) (*grpc.IPVSResponse, error) {
|
||||
+ return &grpc.IPVSResponse{}, nil
|
||||
+ }
|
||||
+
|
||||
+ defer func() {
|
||||
+ testingImpl.StatusContainerFunc = nil
|
||||
+ testingImpl.UpdateIPVSRuleFunc = nil
|
||||
+ }()
|
||||
+
|
||||
+ path, err := createTempContainerIDMapping(sandbox.ID(), sandbox.ID())
|
||||
+ assert.NoError(err)
|
||||
+ defer os.RemoveAll(path)
|
||||
+
|
||||
+ actionFunc, ok := IPVSadmCommand.Action.(func(ctx *cli.Context) error)
|
||||
+ assert.True(ok)
|
||||
+
|
||||
+ // no stdin rule file
|
||||
+ flagSet := flag.NewFlagSet("IPVSadm", flag.ContinueOnError)
|
||||
+ flagSet.Parse([]string{testContainerID})
|
||||
+ flagSet.String("restore", "-", "")
|
||||
+ ctx := createCLIContext(flagSet)
|
||||
+ err = actionFunc(ctx)
|
||||
+ assert.Error(err)
|
||||
+
|
||||
+ // restore parameter error
|
||||
+ flagSet = flag.NewFlagSet("IPVSadm", flag.ContinueOnError)
|
||||
+ flagSet.Parse([]string{testContainerID})
|
||||
+ flagSet.String("restore", "abc", "")
|
||||
+ ctx = createCLIContext(flagSet)
|
||||
+ err = actionFunc(ctx)
|
||||
+ assert.Error(err)
|
||||
+
|
||||
+ // checkrule returns error
|
||||
+ flagSet = flag.NewFlagSet("IPVSadm", flag.ContinueOnError)
|
||||
+ flagSet.Parse([]string{testContainerID})
|
||||
+ flagSet.String("parameters", "-A -t 192.168.0.7:80 -s rr -p -3000", "")
|
||||
+ ctx = createCLIContext(flagSet)
|
||||
+ err = actionFunc(ctx)
|
||||
+ assert.Error(err)
|
||||
+
|
||||
+ // updatefunction returns error
|
||||
+ testingImpl.UpdateIPVSRuleFunc = func(ctx context.Context, sandboxID string, IPVS *grpc.UpdateIPVSRequest) (*grpc.IPVSResponse, error) {
|
||||
+ return &grpc.IPVSResponse{}, fmt.Errorf("IPVSadm test error")
|
||||
+ }
|
||||
+
|
||||
+ flagSet = flag.NewFlagSet("IPVSadm", flag.ContinueOnError)
|
||||
+ flagSet.Parse([]string{testContainerID})
|
||||
+ flagSet.String("parameters", "-A -t 192.168.0.7:80 -s rr -p 3000", "")
|
||||
+ ctx = createCLIContext(flagSet)
|
||||
+ err = actionFunc(ctx)
|
||||
+ assert.Error(err)
|
||||
+}
|
||||
+
|
||||
+func TestCleanupCliAction(t *testing.T) {
|
||||
+ assert := assert.New(t)
|
||||
+
|
||||
+ actionFunc, ok := CleanupCommand.Action.(func(ctx *cli.Context) error)
|
||||
+ assert.True(ok)
|
||||
+
|
||||
+ flagSet := flag.NewFlagSet("cleanup", flag.ContinueOnError)
|
||||
+ ctx := createCLIContext(flagSet)
|
||||
+
|
||||
+ err := actionFunc(ctx)
|
||||
+ assert.Error(err, "Missing container ID")
|
||||
+}
|
||||
+
|
||||
+func TestCleanupCLISuccessful(t *testing.T) {
|
||||
+ assert := assert.New(t)
|
||||
+
|
||||
+ sandbox := &vcmock.Sandbox{
|
||||
+ MockID: testContainerID,
|
||||
+ }
|
||||
+
|
||||
+ sandbox.MockContainers = []*vcmock.Container{
|
||||
+ {
|
||||
+ MockID: sandbox.ID(),
|
||||
+ MockSandbox: sandbox,
|
||||
+ },
|
||||
+ }
|
||||
+
|
||||
+ testingImpl.StatusContainerFunc = func(ctx context.Context, sandboxID, containerID string) (vc.ContainerStatus, error) {
|
||||
+ return vc.ContainerStatus{
|
||||
+ ID: sandbox.ID(),
|
||||
+ Annotations: map[string]string{
|
||||
+ vcAnnotations.ContainerTypeKey: string(vc.PodContainer),
|
||||
+ },
|
||||
+ State: types.ContainerState{
|
||||
+ State: types.StateRunning,
|
||||
+ },
|
||||
+ }, nil
|
||||
+ }
|
||||
+
|
||||
+ testingImpl.UpdateIPVSRuleFunc = func(ctx context.Context, sandboxID string, IPVS *grpc.UpdateIPVSRequest) (*grpc.IPVSResponse, error) {
|
||||
+ return &grpc.IPVSResponse{}, nil
|
||||
+ }
|
||||
+
|
||||
+ defer func() {
|
||||
+ testingImpl.StatusContainerFunc = nil
|
||||
+ testingImpl.UpdateIPVSRuleFunc = nil
|
||||
+ }()
|
||||
+
|
||||
+ path, err := createTempContainerIDMapping(sandbox.ID(), sandbox.ID())
|
||||
+ assert.NoError(err)
|
||||
+ defer os.RemoveAll(path)
|
||||
+
|
||||
+ actionFunc, ok := CleanupCommand.Action.(func(ctx *cli.Context) error)
|
||||
+ assert.True(ok)
|
||||
+
|
||||
+ flagSet := flag.NewFlagSet("cleanup", flag.ContinueOnError)
|
||||
+ flagSet.Parse([]string{testContainerID})
|
||||
+ flagSet.String("parameters", "-d 192.168.0.4 -p tcp", "")
|
||||
+ ctx := createCLIContext(flagSet)
|
||||
+ err = actionFunc(ctx)
|
||||
+ assert.NoError(err)
|
||||
+}
|
||||
+
|
||||
+func TestCleanupCLIError(t *testing.T) {
|
||||
+ assert := assert.New(t)
|
||||
+
|
||||
+ sandbox := &vcmock.Sandbox{
|
||||
+ MockID: testContainerID,
|
||||
+ }
|
||||
+
|
||||
+ sandbox.MockContainers = []*vcmock.Container{
|
||||
+ {
|
||||
+ MockID: sandbox.ID(),
|
||||
+ MockSandbox: sandbox,
|
||||
+ },
|
||||
+ }
|
||||
+
|
||||
+ testingImpl.StatusContainerFunc = func(ctx context.Context, sandboxID, containerID string) (vc.ContainerStatus, error) {
|
||||
+ return vc.ContainerStatus{
|
||||
+ ID: sandbox.ID(),
|
||||
+ Annotations: map[string]string{
|
||||
+ vcAnnotations.ContainerTypeKey: string(vc.PodContainer),
|
||||
+ },
|
||||
+ State: types.ContainerState{
|
||||
+ State: types.StateRunning,
|
||||
+ },
|
||||
+ }, nil
|
||||
+ }
|
||||
+
|
||||
+ testingImpl.UpdateIPVSRuleFunc = func(ctx context.Context, sandboxID string, IPVS *grpc.UpdateIPVSRequest) (*grpc.IPVSResponse, error) {
|
||||
+ return &grpc.IPVSResponse{}, nil
|
||||
+ }
|
||||
+
|
||||
+ defer func() {
|
||||
+ testingImpl.StatusContainerFunc = nil
|
||||
+ testingImpl.UpdateIPVSRuleFunc = nil
|
||||
+ }()
|
||||
+
|
||||
+ path, err := createTempContainerIDMapping(sandbox.ID(), sandbox.ID())
|
||||
+ assert.NoError(err)
|
||||
+ defer os.RemoveAll(path)
|
||||
+
|
||||
+ actionFunc, ok := CleanupCommand.Action.(func(ctx *cli.Context) error)
|
||||
+ assert.True(ok)
|
||||
+
|
||||
+ // checkrule returns error
|
||||
+ flagSet := flag.NewFlagSet("cleanup", flag.ContinueOnError)
|
||||
+ flagSet.Parse([]string{testContainerID})
|
||||
+ flagSet.String("parameters", "-d 192.168.0.4", "")
|
||||
+ ctx := createCLIContext(flagSet)
|
||||
+ err = actionFunc(ctx)
|
||||
+ assert.Error(err)
|
||||
+
|
||||
+ // updatefunction returns error
|
||||
+ testingImpl.UpdateIPVSRuleFunc = func(ctx context.Context, sandboxID string, IPVS *grpc.UpdateIPVSRequest) (*grpc.IPVSResponse, error) {
|
||||
+ return &grpc.IPVSResponse{}, fmt.Errorf("IPVSadm cleanup test error")
|
||||
+ }
|
||||
+
|
||||
+ flagSet = flag.NewFlagSet("cleanup", flag.ContinueOnError)
|
||||
+ flagSet.Parse([]string{testContainerID})
|
||||
+ flagSet.String("parameters", "-d 192.168.0.4 -p tcp", "")
|
||||
+ ctx = createCLIContext(flagSet)
|
||||
+ err = actionFunc(ctx)
|
||||
+ assert.Error(err)
|
||||
+}
|
||||
+
|
||||
+func TestAddServiceSuccessfully(t *testing.T) {
|
||||
+ assert := assert.New(t)
|
||||
+ addServiceCommandTCP := "IPVSadm --add-service --tcp-service 192.168.0.7:80 --scheduler rr --persistent 3000"
|
||||
+ _, err := checkIPVSRule(addServiceCommandTCP)
|
||||
+ assert.NoError(err)
|
||||
+ addServiceCommandUDP := "IPVSadm --add-service --udp-service 192.168.0.7:80 --scheduler rr --persistent 3000"
|
||||
+ _, err = checkIPVSRule(addServiceCommandUDP)
|
||||
+ assert.NoError(err)
|
||||
+ addServiceCmd := "IPVSadm -A -t 192.168.0.7:80 -s rr -p 3000"
|
||||
+ _, err = checkIPVSRule(addServiceCmd)
|
||||
+ assert.NoError(err)
|
||||
+}
|
||||
+
|
||||
+func TestAddServiceNoService(t *testing.T) {
|
||||
+ assert := assert.New(t)
|
||||
+ addServiceCommand := "IPVSadm --add-service --scheduler rr --persistent 3000"
|
||||
+ _, err := checkIPVSRule(addServiceCommand)
|
||||
+ assert.Error(err)
|
||||
+ addServiceCmd := "IPVSadm -A -s rr -p 3000"
|
||||
+ _, err = checkIPVSRule(addServiceCmd)
|
||||
+ assert.Error(err)
|
||||
+}
|
||||
+
|
||||
+func TestAddServiceNoScheduler(t *testing.T) {
|
||||
+ assert := assert.New(t)
|
||||
+ addServiceCommand := "IPVSadm --add-service --tcp-service 192.168.0.7:80 --persistent 3000"
|
||||
+ _, err := checkIPVSRule(addServiceCommand)
|
||||
+ assert.Error(err)
|
||||
+ addServiceCmd := "IPVSadm -A -t 192.168.0.7:80 -p 3000"
|
||||
+ _, err = checkIPVSRule(addServiceCmd)
|
||||
+ assert.Error(err)
|
||||
+}
|
||||
+
|
||||
+func TestAddServiceNoPersistent(t *testing.T) {
|
||||
+ assert := assert.New(t)
|
||||
+ addServiceCommand := "IPVSadm --add-service --tcp-service 192.168.0.7:80 --scheduler rr"
|
||||
+ _, err := checkIPVSRule(addServiceCommand)
|
||||
+ assert.Error(err)
|
||||
+ addServiceCmd := "IPVSadm -A -t 192.168.0.7:80 -s rr"
|
||||
+ _, err = checkIPVSRule(addServiceCmd)
|
||||
+ assert.Error(err)
|
||||
+}
|
||||
+
|
||||
+func TestAddServiceIpError(t *testing.T) {
|
||||
+ assert := assert.New(t)
|
||||
+ addServiceCommand := "IPVSadm --add-service --tcp-service 192.168.2.2.7:80 --scheduler rr --persistent 3000"
|
||||
+ _, err := checkIPVSRule(addServiceCommand)
|
||||
+ assert.Error(err)
|
||||
+ addServiceCmd := "IPVSadm -A -t 192.168.2.2.7:80 -s rr -p 3000"
|
||||
+ _, err = checkIPVSRule(addServiceCmd)
|
||||
+ assert.Error(err)
|
||||
+}
|
||||
+
|
||||
+func TestAddServicePortError(t *testing.T) {
|
||||
+ assert := assert.New(t)
|
||||
+ addServiceCommand := "IPVSadm --add-service --tcp-service 192.168.0.7:9999999 --scheduler rr --persistent 3000"
|
||||
+ _, err := checkIPVSRule(addServiceCommand)
|
||||
+ assert.Error(err)
|
||||
+ addServiceCmd := "IPVSadm -A -t 192.168.0.7:9999999 -s rr -p 3000"
|
||||
+ _, err = checkIPVSRule(addServiceCmd)
|
||||
+ assert.Error(err)
|
||||
+}
|
||||
+
|
||||
+func TestAddServiceSchedulerError(t *testing.T) {
|
||||
+ assert := assert.New(t)
|
||||
+ addServiceCommand := "IPVSadm --add-service --tcp-service 192.168.0.7:80 --scheduler rrr --persistent 3000"
|
||||
+ _, err := checkIPVSRule(addServiceCommand)
|
||||
+ assert.Error(err)
|
||||
+ addServiceCmd := "IPVSadm -A -t 192.168.0.7:80 -s rrr -p 3000"
|
||||
+ _, err = checkIPVSRule(addServiceCmd)
|
||||
+ assert.Error(err)
|
||||
+}
|
||||
+
|
||||
+func TestAddServicePersistentError(t *testing.T) {
|
||||
+ assert := assert.New(t)
|
||||
+ addServiceCommand := "IPVSadm --add-service --tcp-service 192.168.0.7:80 --scheduler rr --persistent 99999999999"
|
||||
+ _, err := checkIPVSRule(addServiceCommand)
|
||||
+ assert.Error(err)
|
||||
+ addServiceCmd := "IPVSadm -A -t 192.168.0.7:80 -s rr -p 99999999999"
|
||||
+ _, err = checkIPVSRule(addServiceCmd)
|
||||
+ assert.Error(err)
|
||||
+}
|
||||
+
|
||||
+func TestEditServiceSuccessfully(t *testing.T) {
|
||||
+ assert := assert.New(t)
|
||||
+ editServiceCommandTCP := "IPVSadm --edit-service --tcp-service 192.168.0.7:80 --scheduler rr --persistent 3000"
|
||||
+ _, err := checkIPVSRule(editServiceCommandTCP)
|
||||
+ assert.NoError(err)
|
||||
+ editServiceCommandUcp := "IPVSadm --edit-service --udp-service 192.168.0.7:80 --scheduler rr --persistent 3000"
|
||||
+ _, err = checkIPVSRule(editServiceCommandUcp)
|
||||
+ assert.NoError(err)
|
||||
+ editServiceCmd := "IPVSadm -A -t 192.168.0.7:80 -s rr -p 3000"
|
||||
+ _, err = checkIPVSRule(editServiceCmd)
|
||||
+ assert.NoError(err)
|
||||
+}
|
||||
+
|
||||
+func TestEditServiceNoService(t *testing.T) {
|
||||
+ assert := assert.New(t)
|
||||
+ editServiceCommand := "IPVSadm --edit-service --scheduler rr --persistent 3000"
|
||||
+ _, err := checkIPVSRule(editServiceCommand)
|
||||
+ assert.Error(err)
|
||||
+ editServiceCmd := "IPVSadm -E -s rr -p 3000"
|
||||
+ _, err = checkIPVSRule(editServiceCmd)
|
||||
+ assert.Error(err)
|
||||
+}
|
||||
+
|
||||
+func TestEditServiceNoScheduler(t *testing.T) {
|
||||
+ assert := assert.New(t)
|
||||
+ editServiceCommand := "IPVSadm --edit-service --tcp-service 192.168.0.7:80 --persistent 3000"
|
||||
+ _, err := checkIPVSRule(editServiceCommand)
|
||||
+ assert.Error(err)
|
||||
+ editServiceCmd := "IPVSadm -E -t 192.168.0.7:80 -p 3000"
|
||||
+ _, err = checkIPVSRule(editServiceCmd)
|
||||
+ assert.Error(err)
|
||||
+}
|
||||
+
|
||||
+func TestEditServiceNoPersistent(t *testing.T) {
|
||||
+ assert := assert.New(t)
|
||||
+ editServiceCommand := "IPVSadm --edit-service --tcp-service 192.168.0.7:80 --scheduler rr"
|
||||
+ _, err := checkIPVSRule(editServiceCommand)
|
||||
+ assert.Error(err)
|
||||
+ editServiceCmd := "IPVSadm -E -t 192.168.0.7:80 -s rr"
|
||||
+ _, err = checkIPVSRule(editServiceCmd)
|
||||
+ assert.Error(err)
|
||||
+}
|
||||
+
|
||||
+func TestEditServiceIpError(t *testing.T) {
|
||||
+ assert := assert.New(t)
|
||||
+ editServiceCommand := "IPVSadm --edit-service --tcp-service 192.168.2.2.7:80 --scheduler rr --persistent 3000"
|
||||
+ _, err := checkIPVSRule(editServiceCommand)
|
||||
+ assert.Error(err)
|
||||
+ editServiceCmd := "IPVSadm -E -t 192.168.2.2.7:80 -s rr -p 3000"
|
||||
+ _, err = checkIPVSRule(editServiceCmd)
|
||||
+ assert.Error(err)
|
||||
+}
|
||||
+
|
||||
+func TestEditServicePortError(t *testing.T) {
|
||||
+ assert := assert.New(t)
|
||||
+ editServiceCommand := "IPVSadm --edit-service --tcp-service 192.168.0.7:9999999 --scheduler rr --persistent 3000"
|
||||
+ _, err := checkIPVSRule(editServiceCommand)
|
||||
+ assert.Error(err)
|
||||
+ editServiceCmd := "IPVSadm -E -t 192.168.0.7:9999999 -s rr -p 3000"
|
||||
+ _, err = checkIPVSRule(editServiceCmd)
|
||||
+ assert.Error(err)
|
||||
+}
|
||||
+
|
||||
+func TestEditServiceSchedulerError(t *testing.T) {
|
||||
+ assert := assert.New(t)
|
||||
+ editServiceCommand := "IPVSadm --edit-service --tcp-service 192.168.0.7:80 --scheduler rrr --persistent 3000"
|
||||
+ _, err := checkIPVSRule(editServiceCommand)
|
||||
+ assert.Error(err)
|
||||
+ editServiceCmd := "IPVSadm -E -t 192.168.0.7:80 -s rrr -p 3000"
|
||||
+ _, err = checkIPVSRule(editServiceCmd)
|
||||
+ assert.Error(err)
|
||||
+}
|
||||
+
|
||||
+func TestEditServicePersistentError(t *testing.T) {
|
||||
+ assert := assert.New(t)
|
||||
+ editServiceCommand := "IPVSadm --edit-service --tcp-service 192.168.0.7:80 --scheduler rr --persistent 99999999999"
|
||||
+ _, err := checkIPVSRule(editServiceCommand)
|
||||
+ assert.Error(err)
|
||||
+ editServiceCmd := "IPVSadm -E -t 192.168.0.7:80 -s rr -p 99999999999"
|
||||
+ _, err = checkIPVSRule(editServiceCmd)
|
||||
+ assert.Error(err)
|
||||
+}
|
||||
+
|
||||
+func TestDeleteServiceSuccessfully(t *testing.T) {
|
||||
+ assert := assert.New(t)
|
||||
+ deleteServiceCommandTCP := "IPVSadm --delete-service --tcp-service 192.168.0.7:80"
|
||||
+ _, err := checkIPVSRule(deleteServiceCommandTCP)
|
||||
+ assert.NoError(err)
|
||||
+ deleteServiceCommandUDP := "IPVSadm --delete-service --udp-service 192.168.0.7:80"
|
||||
+ _, err = checkIPVSRule(deleteServiceCommandUDP)
|
||||
+ assert.NoError(err)
|
||||
+ deleteServiceCmd := "IPVSadm -D -t 192.168.0.7:80"
|
||||
+ _, err = checkIPVSRule(deleteServiceCmd)
|
||||
+ assert.NoError(err)
|
||||
+}
|
||||
+
|
||||
+func TestDeleteServiceNoService(t *testing.T) {
|
||||
+ assert := assert.New(t)
|
||||
+ deleteServiceCommand := "IPVSadm --delete-service"
|
||||
+ _, err := checkIPVSRule(deleteServiceCommand)
|
||||
+ assert.Error(err)
|
||||
+ deleteServiceCmd := "IPVSadm -D"
|
||||
+ _, err = checkIPVSRule(deleteServiceCmd)
|
||||
+ assert.Error(err)
|
||||
+}
|
||||
+
|
||||
+func TestDeleteServiceIpError(t *testing.T) {
|
||||
+ assert := assert.New(t)
|
||||
+ deleteServiceCommand := "IPVSadm --delete-service --tcp-service 192.168.2.2.7:80"
|
||||
+ _, err := checkIPVSRule(deleteServiceCommand)
|
||||
+ assert.Error(err)
|
||||
+ deleteServiceCmd := "IPVSadm -D -t 192.168.2.2.7:80"
|
||||
+ _, err = checkIPVSRule(deleteServiceCmd)
|
||||
+ assert.Error(err)
|
||||
+}
|
||||
+
|
||||
+func TestDeleteServicePortError(t *testing.T) {
|
||||
+ assert := assert.New(t)
|
||||
+ deleteServiceCommand := "IPVSadm --delete-service --tcp-service 192.168.0.7:9999999"
|
||||
+ _, err := checkIPVSRule(deleteServiceCommand)
|
||||
+ assert.Error(err)
|
||||
+ deleteServiceCmd := "IPVSadm -D -t 192.168.0.7:9999999"
|
||||
+ _, err = checkIPVSRule(deleteServiceCmd)
|
||||
+ assert.Error(err)
|
||||
+}
|
||||
+
|
||||
+func TestAddServerSuccessfully(t *testing.T) {
|
||||
+ assert := assert.New(t)
|
||||
+ addServerCommandTCP := "IPVSadm --add-server --tcp-service 192.168.0.7:80 --real-server 192.168.0.4:80 --weight 100"
|
||||
+ _, err := checkIPVSRule(addServerCommandTCP)
|
||||
+ assert.NoError(err)
|
||||
+ addServerCommandUDP := "IPVSadm --add-server --udp-service 192.168.0.7:80 --real-server 192.168.0.4:80 --weight 100"
|
||||
+ _, err = checkIPVSRule(addServerCommandUDP)
|
||||
+ assert.NoError(err)
|
||||
+ addServerCmd := "IPVSadm -a -t 192.168.0.7:80 -r 192.168.0.4:80 -w 100"
|
||||
+ _, err = checkIPVSRule(addServerCmd)
|
||||
+ assert.NoError(err)
|
||||
+}
|
||||
+
|
||||
+func TestAddServerNoService(t *testing.T) {
|
||||
+ assert := assert.New(t)
|
||||
+ addServerCommand := "IPVSadm --add-server --real-server 192.168.0.4:80"
|
||||
+ _, err := checkIPVSRule(addServerCommand)
|
||||
+ assert.Error(err)
|
||||
+ addServerCmd := "IPVSadm -a -r 192.168.0.4:80"
|
||||
+ _, err = checkIPVSRule(addServerCmd)
|
||||
+ assert.Error(err)
|
||||
+}
|
||||
+
|
||||
+func TestAddServerNoServer(t *testing.T) {
|
||||
+ assert := assert.New(t)
|
||||
+ addServerCommand := "IPVSadm --add-server --tcp-service 192.168.0.7:80"
|
||||
+ _, err := checkIPVSRule(addServerCommand)
|
||||
+ assert.Error(err)
|
||||
+ addServerCmd := "IPVSadm -a -t 192.168.0.7:80"
|
||||
+ _, err = checkIPVSRule(addServerCmd)
|
||||
+ assert.Error(err)
|
||||
+}
|
||||
+
|
||||
+func TestAddServerIpError(t *testing.T) {
|
||||
+ assert := assert.New(t)
|
||||
+ addServerCommand := "IPVSadm --add-server --tcp-service 192.168.2.0.7:80 --real-server 192.168.0.4:80"
|
||||
+ _, err := checkIPVSRule(addServerCommand)
|
||||
+ assert.Error(err)
|
||||
+ addServerCmd := "IPVSadm -a -t 192.168.2.0.7:80 -r 192.168.0.4:80"
|
||||
+ _, err = checkIPVSRule(addServerCmd)
|
||||
+ assert.Error(err)
|
||||
+}
|
||||
+
|
||||
+func TestAddServerPortError(t *testing.T) {
|
||||
+ assert := assert.New(t)
|
||||
+ addServerCommand := "IPVSadm --add-server --tcp-service 192.168.0.7:99999 --real-server 192.168.0.4:80"
|
||||
+ _, err := checkIPVSRule(addServerCommand)
|
||||
+ assert.Error(err)
|
||||
+ addServerCmd := "IPVSadm -a -t 192.168.0.7:99999 -r 192.168.0.4:80"
|
||||
+ _, err = checkIPVSRule(addServerCmd)
|
||||
+ assert.Error(err)
|
||||
+}
|
||||
+
|
||||
+func TestEditServerSuccessfully(t *testing.T) {
|
||||
+ assert := assert.New(t)
|
||||
+ editServerCommandTCP := "IPVSadm --edit-server --tcp-service 192.168.0.7:80 --real-server 192.168.0.4:80 --weight 100"
|
||||
+ _, err := checkIPVSRule(editServerCommandTCP)
|
||||
+ assert.NoError(err)
|
||||
+ editServerCommandUDP := "IPVSadm --edit-server --udp-service 192.168.0.7:80 --real-server 192.168.0.4:80 --weight 100"
|
||||
+ _, err = checkIPVSRule(editServerCommandUDP)
|
||||
+ assert.NoError(err)
|
||||
+ editServerCmd := "IPVSadm -e -t 192.168.0.7:80 -r 192.168.0.4:80 -w 100"
|
||||
+ _, err = checkIPVSRule(editServerCmd)
|
||||
+ assert.NoError(err)
|
||||
+}
|
||||
+
|
||||
+func TestEditServerNoService(t *testing.T) {
|
||||
+ assert := assert.New(t)
|
||||
+ editServerCommand := "IPVSadm --edit-server --real-server 192.168.0.4:80"
|
||||
+ _, err := checkIPVSRule(editServerCommand)
|
||||
+ assert.Error(err)
|
||||
+ editServerCmd := "IPVSadm -e -r 192.168.0.4:80"
|
||||
+ _, err = checkIPVSRule(editServerCmd)
|
||||
+ assert.Error(err)
|
||||
+}
|
||||
+
|
||||
+func TestEditServerNoServer(t *testing.T) {
|
||||
+ assert := assert.New(t)
|
||||
+ editServerCommand := "IPVSadm --edit-server --tcp-service 192.168.0.7:80"
|
||||
+ _, err := checkIPVSRule(editServerCommand)
|
||||
+ assert.Error(err)
|
||||
+ editServerCmd := "IPVSadm -e -t 192.168.0.7:80"
|
||||
+ _, err = checkIPVSRule(editServerCmd)
|
||||
+ assert.Error(err)
|
||||
+}
|
||||
+
|
||||
+func TestEditServerIpError(t *testing.T) {
|
||||
+ assert := assert.New(t)
|
||||
+ editServerCommand := "IPVSadm --edit-server --tcp-service 192.168.2.0.7:80 --real-server 192.168.0.4:80"
|
||||
+ _, err := checkIPVSRule(editServerCommand)
|
||||
+ assert.Error(err)
|
||||
+ editServerCmd := "IPVSadm -e -t 192.168.2.0.7:80 -r 192.168.0.4:80"
|
||||
+ _, err = checkIPVSRule(editServerCmd)
|
||||
+ assert.Error(err)
|
||||
+}
|
||||
+
|
||||
+func TestEditServerPortError(t *testing.T) {
|
||||
+ assert := assert.New(t)
|
||||
+ editServerCommand := "IPVSadm --edit-server --tcp-service 192.168.0.7:99999 --real-server 192.168.0.4:80"
|
||||
+ _, err := checkIPVSRule(editServerCommand)
|
||||
+ assert.Error(err)
|
||||
+ editServerCmd := "IPVSadm -e -t 192.168.0.7:99999 -r 192.168.0.4:80"
|
||||
+ _, err = checkIPVSRule(editServerCmd)
|
||||
+ assert.Error(err)
|
||||
+}
|
||||
+
|
||||
+func TestDeleteServerSuccessfully(t *testing.T) {
|
||||
+ assert := assert.New(t)
|
||||
+ deleteServerCommandTCP := "IPVSadm --delete-server --tcp-service 192.168.0.7:80 --real-server 192.168.0.4:80"
|
||||
+ _, err := checkIPVSRule(deleteServerCommandTCP)
|
||||
+ assert.NoError(err)
|
||||
+ deleteServerCommandUDP := "IPVSadm --delete-server --udp-service 192.168.0.7:80 --real-server 192.168.0.4:80"
|
||||
+ _, err = checkIPVSRule(deleteServerCommandUDP)
|
||||
+ assert.NoError(err)
|
||||
+ deleteServerCmd := "IPVSadm -d -t 192.168.0.7:80 -r 192.168.0.4:80"
|
||||
+ _, err = checkIPVSRule(deleteServerCmd)
|
||||
+ assert.NoError(err)
|
||||
+}
|
||||
+
|
||||
+func TestDeleteServerNoService(t *testing.T) {
|
||||
+ assert := assert.New(t)
|
||||
+ deleteServerCommand := "IPVSadm --delete-server --real-server 192.168.0.4:80"
|
||||
+ _, err := checkIPVSRule(deleteServerCommand)
|
||||
+ assert.Error(err)
|
||||
+ deleteServerCmd := "IPVSadm -d -r 192.168.0.4:80"
|
||||
+ _, err = checkIPVSRule(deleteServerCmd)
|
||||
+ assert.Error(err)
|
||||
+}
|
||||
+
|
||||
+func TestDeleteServerNoServer(t *testing.T) {
|
||||
+ assert := assert.New(t)
|
||||
+ deleteServerCommand := "IPVSadm --delete-server --tcp-service 192.168.0.7:80"
|
||||
+ _, err := checkIPVSRule(deleteServerCommand)
|
||||
+ assert.Error(err)
|
||||
+ deleteServerCmd := "IPVSadm -d -t 192.168.0.7:80"
|
||||
+ _, err = checkIPVSRule(deleteServerCmd)
|
||||
+ assert.Error(err)
|
||||
+}
|
||||
+
|
||||
+func TestDeleteServerIpError(t *testing.T) {
|
||||
+ assert := assert.New(t)
|
||||
+ deleteServerCommand := "IPVSadm --delete-server --tcp-service 192.168.2.0.7:80 --real-server 192.168.0.4:80"
|
||||
+ _, err := checkIPVSRule(deleteServerCommand)
|
||||
+ assert.Error(err)
|
||||
+ deleteServerCmd := "IPVSadm -d -t 192.168.2.0.7:80 -r 192.168.0.4:80"
|
||||
+ _, err = checkIPVSRule(deleteServerCmd)
|
||||
+ assert.Error(err)
|
||||
+}
|
||||
+
|
||||
+func TestDeleteServerPortError(t *testing.T) {
|
||||
+ assert := assert.New(t)
|
||||
+ deleteServerCommand := "IPVSadm --delete-server --tcp-service 192.168.0.7:99999 --real-server 192.168.0.4:80"
|
||||
+ _, err := checkIPVSRule(deleteServerCommand)
|
||||
+ assert.Error(err)
|
||||
+ deleteServerCmd := "IPVSadm -d -t 192.168.0.7:99999 -r 192.168.0.4:80"
|
||||
+ _, err = checkIPVSRule(deleteServerCmd)
|
||||
+ assert.Error(err)
|
||||
+}
|
||||
+
|
||||
+func TestListSuccessfully(t *testing.T) {
|
||||
+ assert := assert.New(t)
|
||||
+ listCommand := "IPVSadm --list"
|
||||
+ _, err := checkIPVSRule(listCommand)
|
||||
+ assert.NoError(err)
|
||||
+ listCmd := "IPVSadm -L"
|
||||
+ _, err = checkIPVSRule(listCmd)
|
||||
+ assert.NoError(err)
|
||||
+}
|
||||
+
|
||||
+func TestSetSuccessfully(t *testing.T) {
|
||||
+ assert := assert.New(t)
|
||||
+ setCommand := "IPVSadm --set"
|
||||
+ _, err := checkIPVSRule(setCommand)
|
||||
+ assert.NoError(err)
|
||||
+}
|
||||
+
|
||||
+func TestCleanupSuccessfully(t *testing.T) {
|
||||
+ assert := assert.New(t)
|
||||
+ cleanupCommand := "conntrack -D --orig-dst 192.168.0.4 --protonum tcp"
|
||||
+ _, err := checkIPVSRule(cleanupCommand)
|
||||
+ assert.NoError(err)
|
||||
+ cleanupCmd := "conntrack -D -d 192.168.0.4 -p tcp"
|
||||
+ _, err = checkIPVSRule(cleanupCmd)
|
||||
+ assert.NoError(err)
|
||||
+}
|
||||
+
|
||||
+func TestCleanupNoDestination(t *testing.T) {
|
||||
+ assert := assert.New(t)
|
||||
+ cleanupCommand := "conntrack -D --protonum tcp"
|
||||
+ _, err := checkIPVSRule(cleanupCommand)
|
||||
+ assert.Error(err)
|
||||
+ cleanupCmd := "conntrack -D -p tcp"
|
||||
+ _, err = checkIPVSRule(cleanupCmd)
|
||||
+ assert.Error(err)
|
||||
+}
|
||||
+
|
||||
+func TestCleanupNoProtocol(t *testing.T) {
|
||||
+ assert := assert.New(t)
|
||||
+ cleanupCommand := "conntrack -D --orig-dst 192.168.0.4"
|
||||
+ _, err := checkIPVSRule(cleanupCommand)
|
||||
+ assert.Error(err)
|
||||
+ cleanupCmd := "conntrack -D -d 192.168.0.4"
|
||||
+ _, err = checkIPVSRule(cleanupCmd)
|
||||
+ assert.Error(err)
|
||||
+}
|
||||
+
|
||||
+func TestCleanupIpError(t *testing.T) {
|
||||
+ assert := assert.New(t)
|
||||
+ cleanupCommand := "conntrack -D --orig-dst 192.168.0.2.4 --protonum tcp"
|
||||
+ _, err := checkIPVSRule(cleanupCommand)
|
||||
+ assert.Error(err)
|
||||
+ cleanupCmd := "conntrack -D -d 192.168.0.2.4 -p tcp"
|
||||
+ _, err = checkIPVSRule(cleanupCmd)
|
||||
+ assert.Error(err)
|
||||
+}
|
||||
+
|
||||
+func TestCleanupProtocolError(t *testing.T) {
|
||||
+ assert := assert.New(t)
|
||||
+ cleanupCommand := "conntrack -D --orig-dst 192.168.0.4 --protonum ttcp"
|
||||
+ _, err := checkIPVSRule(cleanupCommand)
|
||||
+ assert.Error(err)
|
||||
+ cleanupCmd := "conntrack -D -d 192.168.0.4 -p ttcp"
|
||||
+ _, err = checkIPVSRule(cleanupCmd)
|
||||
+ assert.Error(err)
|
||||
+}
|
||||
+
|
||||
+func TestRestoreFileNil(t *testing.T) {
|
||||
+ assert := assert.New(t)
|
||||
+ stdin := strings.NewReader("")
|
||||
+ fileContext, cnt, err := getRestoreFile(stdin)
|
||||
+ assert.Equal(cnt, 0)
|
||||
+ assert.Equal(fileContext, "")
|
||||
+ assert.Error(err)
|
||||
+}
|
||||
+
|
||||
+func TestRestoreFileError(t *testing.T) {
|
||||
+ assert := assert.New(t)
|
||||
+ stdin := strings.NewReader("aaa")
|
||||
+ fileContext, cnt, err := getRestoreFile(stdin)
|
||||
+ assert.Equal(cnt, 0)
|
||||
+ assert.Equal(fileContext, "")
|
||||
+ assert.Error(err)
|
||||
+}
|
||||
+
|
||||
+func TestRestoreFile(t *testing.T) {
|
||||
+ assert := assert.New(t)
|
||||
+ stdin := strings.NewReader("-A -t 10.10.11.12:100 -s rr -p 3000\n-a -t 10.10.11.12:100 -r 172.16.0.1:80 -m")
|
||||
+ fileContext, cnt, err := getRestoreFile(stdin)
|
||||
+ assert.Equal(cnt, 2)
|
||||
+ assert.Equal(fileContext, "-A -t 10.10.11.12:100 -s rr -p 3000\n-a -t 10.10.11.12:100 -r 172.16.0.1:80 -m")
|
||||
+ assert.NoError(err)
|
||||
+}
|
||||
diff --git a/virtcontainers/api_test.go b/virtcontainers/api_test.go
|
||||
index c01d47b8..2a5488f9 100644
|
||||
--- a/virtcontainers/api_test.go
|
||||
+++ b/virtcontainers/api_test.go
|
||||
@@ -15,6 +15,7 @@ import (
|
||||
"syscall"
|
||||
"testing"
|
||||
|
||||
+ "github.com/kata-containers/agent/protocols/grpc"
|
||||
ktu "github.com/kata-containers/runtime/pkg/katatestutils"
|
||||
"github.com/kata-containers/runtime/virtcontainers/persist"
|
||||
"github.com/kata-containers/runtime/virtcontainers/pkg/annotations"
|
||||
@@ -1745,3 +1746,38 @@ func TestCleanupContainer(t *testing.T) {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
+
|
||||
+func TestUpdateIPVSRule(t *testing.T) {
|
||||
+ defer cleanUp()
|
||||
+ assert := assert.New(t)
|
||||
+
|
||||
+ contID := "abc"
|
||||
+ ctx := context.Background()
|
||||
+ var ipvs = &grpc.UpdateIPVSRequest{
|
||||
+ IPVSReq: "ipvsadm -A -t 192.168.0.7:80 -s rr -p -3000",
|
||||
+ }
|
||||
+
|
||||
+ config := newTestSandboxConfigNoop()
|
||||
+ p, _, err := createAndStartSandbox(ctx, config)
|
||||
+ if p == nil || err != nil {
|
||||
+ t.Fatal(err)
|
||||
+ }
|
||||
+
|
||||
+ s, ok := p.(*Sandbox)
|
||||
+ assert.True(ok)
|
||||
+
|
||||
+ contConfig := newTestContainerConfigNoop(contID)
|
||||
+ c, err := p.CreateContainer(contConfig)
|
||||
+ if c == nil || err != nil {
|
||||
+ t.Fatal(err)
|
||||
+ }
|
||||
+
|
||||
+ _, err = UpdateIPVSRule(ctx, s.id, ipvs)
|
||||
+ assert.NoError(err)
|
||||
+
|
||||
+ _, err = UpdateIPVSRule(ctx, "aaa", ipvs)
|
||||
+ assert.Error(err)
|
||||
+
|
||||
+ _, err = UpdateIPVSRule(ctx, "", ipvs)
|
||||
+ assert.Error(err)
|
||||
+}
|
||||
diff --git a/virtcontainers/kata_agent_test.go b/virtcontainers/kata_agent_test.go
|
||||
index 18a5a0a6..4f9409a0 100644
|
||||
--- a/virtcontainers/kata_agent_test.go
|
||||
+++ b/virtcontainers/kata_agent_test.go
|
||||
@@ -246,6 +246,10 @@ func (p *gRPCProxy) MemHotplugByProbe(ctx context.Context, req *pb.MemHotplugByP
|
||||
return &gpb.Empty{}, nil
|
||||
}
|
||||
|
||||
+func (p *gRPCProxy) UpdateIPVSRule(ctx context.Context, req *pb.UpdateIPVSRequest) (*pb.IPVSResponse, error) {
|
||||
+ return &pb.IPVSResponse{}, nil
|
||||
+}
|
||||
+
|
||||
func gRPCRegister(s *grpc.Server, srv interface{}) {
|
||||
switch g := srv.(type) {
|
||||
case *gRPCProxy:
|
||||
diff --git a/virtcontainers/pkg/vcmock/types.go b/virtcontainers/pkg/vcmock/types.go
|
||||
index 43247ef9..610b4602 100644
|
||||
--- a/virtcontainers/pkg/vcmock/types.go
|
||||
+++ b/virtcontainers/pkg/vcmock/types.go
|
||||
@@ -76,5 +76,5 @@ type VCMock struct {
|
||||
UpdateRoutesFunc func(ctx context.Context, sandboxID string, routes []*vcTypes.Route) ([]*vcTypes.Route, error)
|
||||
ListRoutesFunc func(ctx context.Context, sandboxID string) ([]*vcTypes.Route, error)
|
||||
CleanupContainerFunc func(ctx context.Context, sandboxID, containerID string, force bool) error
|
||||
- UpdateIPVSRuleFunc func(ctx context.Context, sandboxID string, ipvs *grpc.UpdateIpvsRequest) (*grpc.IpvsResponse, error)
|
||||
+ UpdateIPVSRuleFunc func(ctx context.Context, sandboxID string, ipvs *grpc.UpdateIPVSRequest) (*grpc.IPVSResponse, error)
|
||||
}
|
||||
--
|
||||
2.14.3 (Apple Git-98)
|
||||
|
||||
@ -1,501 +0,0 @@
|
||||
From 5a220e9be1cfb03316a62aa00d2040638ba1a855 Mon Sep 17 00:00:00 2001
|
||||
From: jiangpengfei <jiangpengfei9@huawei.com>
|
||||
Date: Tue, 18 Aug 2020 15:15:52 +0800
|
||||
Subject: [PATCH 38/50] pcie: using pcie-root-port driver to hotplug device in
|
||||
aarch64
|
||||
|
||||
reason: Since qemu with "virt" machine type doesn't support hotplug
|
||||
device in the pcie.0 root bus, so need to use add root port devices
|
||||
to support hotplug pci device in aarch64.
|
||||
|
||||
we reuse the pcie_root_port config in the configuration.toml file to
|
||||
set pcie_root_port device number when qemu process start.
|
||||
|
||||
Signed-off-by: jiangpengfei <jiangpengfei9@huawei.com>
|
||||
---
|
||||
vendor/github.com/intel/govmm/qemu/qemu.go | 22 ++---
|
||||
virtcontainers/persist/api/hypervisor.go | 5 +-
|
||||
virtcontainers/qemu.go | 130 +++++++++++++++++++++++------
|
||||
virtcontainers/qemu_arch_base.go | 9 ++
|
||||
virtcontainers/types/pcie.go | 103 +++++++++++++++++++++++
|
||||
5 files changed, 230 insertions(+), 39 deletions(-)
|
||||
create mode 100644 virtcontainers/types/pcie.go
|
||||
|
||||
diff --git a/vendor/github.com/intel/govmm/qemu/qemu.go b/vendor/github.com/intel/govmm/qemu/qemu.go
|
||||
index a5e5dfaf..3e7720b4 100644
|
||||
--- a/vendor/github.com/intel/govmm/qemu/qemu.go
|
||||
+++ b/vendor/github.com/intel/govmm/qemu/qemu.go
|
||||
@@ -1252,6 +1252,8 @@ type PCIeRootPortDevice struct {
|
||||
Chassis string // (slot, chassis) pair is mandatory and must be unique for each pcie-root-port, >=0, default is 0x00
|
||||
Slot string // >=0, default is 0x00
|
||||
|
||||
+ Port string // Port number is the device index
|
||||
+
|
||||
Multifunction bool // true => "on", false => "off", default is off
|
||||
Addr string // >=0, default is 0x00
|
||||
|
||||
@@ -1277,6 +1279,10 @@ func (b PCIeRootPortDevice) QemuParams(config *Config) []string {
|
||||
|
||||
deviceParams = append(deviceParams, fmt.Sprintf("%s,id=%s", driver, b.ID))
|
||||
|
||||
+ if b.Port != "" {
|
||||
+ deviceParams = append(deviceParams, fmt.Sprintf("port=%s", b.Port))
|
||||
+ }
|
||||
+
|
||||
if b.Bus == "" {
|
||||
b.Bus = "pcie.0"
|
||||
}
|
||||
@@ -1287,20 +1293,10 @@ func (b PCIeRootPortDevice) QemuParams(config *Config) []string {
|
||||
}
|
||||
deviceParams = append(deviceParams, fmt.Sprintf("chassis=%s", b.Chassis))
|
||||
|
||||
- if b.Slot == "" {
|
||||
- b.Slot = "0x00"
|
||||
- }
|
||||
- deviceParams = append(deviceParams, fmt.Sprintf("slot=%s", b.Slot))
|
||||
-
|
||||
- multifunction := "off"
|
||||
if b.Multifunction {
|
||||
- multifunction = "on"
|
||||
- if b.Addr == "" {
|
||||
- b.Addr = "0x00"
|
||||
- }
|
||||
- deviceParams = append(deviceParams, fmt.Sprintf("addr=%s", b.Addr))
|
||||
+ deviceParams = append(deviceParams, "multifunction=on")
|
||||
}
|
||||
- deviceParams = append(deviceParams, fmt.Sprintf("multifunction=%v", multifunction))
|
||||
+ deviceParams = append(deviceParams, fmt.Sprintf("addr=%s", b.Addr))
|
||||
|
||||
if b.BusReserve != "" {
|
||||
deviceParams = append(deviceParams, fmt.Sprintf("bus-reserve=%s", b.BusReserve))
|
||||
@@ -1337,7 +1333,7 @@ func (b PCIeRootPortDevice) Valid() bool {
|
||||
if b.Pref64Reserve != "" && b.Pref32Reserve != "" {
|
||||
return false
|
||||
}
|
||||
- if b.ID == "" {
|
||||
+ if b.ID == "" || b.Port == "" || b.Bus == "" || b.Addr == ""{
|
||||
return false
|
||||
}
|
||||
return true
|
||||
diff --git a/virtcontainers/persist/api/hypervisor.go b/virtcontainers/persist/api/hypervisor.go
|
||||
index 375fd56b..fd61b3c2 100644
|
||||
--- a/virtcontainers/persist/api/hypervisor.go
|
||||
+++ b/virtcontainers/persist/api/hypervisor.go
|
||||
@@ -5,6 +5,8 @@
|
||||
|
||||
package persistapi
|
||||
|
||||
+import "github.com/kata-containers/runtime/virtcontainers/types"
|
||||
+
|
||||
// Bridge is a bridge where devices can be hot plugged
|
||||
type Bridge struct {
|
||||
// DeviceAddr contains information about devices plugged and its address in the bridge
|
||||
@@ -35,7 +37,8 @@ type HypervisorState struct {
|
||||
|
||||
// Belows are qemu specific
|
||||
// Refs: virtcontainers/qemu.go:QemuState
|
||||
- Bridges []Bridge
|
||||
+ Bridges []Bridge
|
||||
+ PCIeRootPortsPool *types.PCIeRootPortPool
|
||||
// HotpluggedCPUs is the list of CPUs that were hot-added
|
||||
HotpluggedVCPUs []CPUDevice
|
||||
HotpluggedMemory int
|
||||
diff --git a/virtcontainers/qemu.go b/virtcontainers/qemu.go
|
||||
index c2b65376..a10c66fb 100644
|
||||
--- a/virtcontainers/qemu.go
|
||||
+++ b/virtcontainers/qemu.go
|
||||
@@ -64,8 +64,9 @@ type CPUDevice struct {
|
||||
|
||||
// QemuState keeps Qemu's state
|
||||
type QemuState struct {
|
||||
- Bridges []types.Bridge
|
||||
- ScsiBus *types.SCSIBus
|
||||
+ Bridges []types.Bridge
|
||||
+ ScsiBus *types.SCSIBus
|
||||
+ PCIeRootPortsPool *types.PCIeRootPortPool
|
||||
// HotpluggedCPUs is the list of CPUs that were hot-added
|
||||
HotpluggedVCPUs []CPUDevice
|
||||
HotpluggedMemory int
|
||||
@@ -271,6 +272,9 @@ func (q *qemu) setup(id string, hypervisorConfig *HypervisorConfig) error {
|
||||
|
||||
q.state.HotplugVFIOOnRootBus = q.config.HotplugVFIOOnRootBus
|
||||
q.state.PCIeRootPort = int(q.config.PCIeRootPort)
|
||||
+ // init the PCIeRootPortsPool with pcie_root_port config value
|
||||
+ q.state.PCIeRootPortsPool = &types.PCIeRootPortPool{}
|
||||
+ q.state.PCIeRootPortsPool.Init(q.state.PCIeRootPort)
|
||||
|
||||
// The path might already exist, but in case of VM templating,
|
||||
// we have to create it since the sandbox has not created it yet.
|
||||
@@ -394,9 +398,18 @@ func (q *qemu) buildDevices(initrdPath string) ([]govmmQemu.Device, *govmmQemu.I
|
||||
return nil, nil, err
|
||||
}
|
||||
|
||||
- // Add bridges before any other devices. This way we make sure that
|
||||
- // bridge gets the first available PCI address i.e bridgePCIStartAddr
|
||||
- devices = q.arch.appendBridges(devices)
|
||||
+ machine, err := q.getQemuMachine()
|
||||
+ if err != nil {
|
||||
+ return nil, nil, err
|
||||
+ }
|
||||
+ switch machine.Type {
|
||||
+ case QemuVirt:
|
||||
+ devices = q.arch.appendRootPorts(devices, q.state.PCIeRootPortsPool)
|
||||
+ default:
|
||||
+ // Add bridges before any other devices. This way we make sure that
|
||||
+ // bridge gets the first available PCI address i.e bridgePCIStartAddr
|
||||
+ devices = q.arch.appendBridges(devices)
|
||||
+ }
|
||||
|
||||
devices, err = q.arch.appendConsole(devices, console)
|
||||
if err != nil {
|
||||
@@ -608,7 +621,7 @@ func (q *qemu) createSandbox(ctx context.Context, id string, networkNS NetworkNa
|
||||
// Add PCIe Root Port devices to hypervisor
|
||||
// The pcie.0 bus do not support hot-plug, but PCIe device can be hot-plugged into PCIe Root Port.
|
||||
// For more details, please see https://github.com/qemu/qemu/blob/master/docs/pcie.txt
|
||||
- if hypervisorConfig.PCIeRootPort > 0 {
|
||||
+ if hypervisorConfig.PCIeRootPort > 0 && hypervisorConfig.HypervisorMachineType == QemuQ35 {
|
||||
qemuConfig.Devices = q.arch.appendPCIeRootPortDevice(qemuConfig.Devices, hypervisorConfig.PCIeRootPort)
|
||||
}
|
||||
|
||||
@@ -1154,21 +1167,19 @@ func (q *qemu) hotplugAddBlockDevice(drive *config.BlockDrive, op operation, dev
|
||||
}
|
||||
case q.config.BlockDeviceDriver == config.VirtioBlock:
|
||||
driver := "virtio-blk-pci"
|
||||
- addr, bridge, err := q.arch.addDeviceToBridge(drive.ID, types.PCI)
|
||||
+
|
||||
+ addr, bus, pciAddr, err := q.getPciAddress(drive.ID, types.PCI)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
-
|
||||
defer func() {
|
||||
if err != nil {
|
||||
- q.arch.removeDeviceFromBridge(drive.ID)
|
||||
+ q.putPciAddress(drive.ID)
|
||||
}
|
||||
}()
|
||||
+ drive.PCIAddr = pciAddr
|
||||
|
||||
- // PCI address is in the format bridge-addr/device-addr eg. "03/02"
|
||||
- drive.PCIAddr = fmt.Sprintf("%02x", bridge.Addr) + "/" + addr
|
||||
-
|
||||
- if err = q.qmpMonitorCh.qmp.ExecutePCIDeviceAdd(q.qmpMonitorCh.ctx, drive.ID, devID, driver, addr, bridge.ID, romFile, 0, true, defaultDisableModern); err != nil {
|
||||
+ if err = q.qmpMonitorCh.qmp.ExecutePCIDeviceAdd(q.qmpMonitorCh.ctx, drive.ID, devID, driver, addr, bus, romFile, 0, true, defaultDisableModern); err != nil {
|
||||
return err
|
||||
}
|
||||
case q.config.BlockDeviceDriver == config.VirtioSCSI:
|
||||
@@ -1249,7 +1260,7 @@ func (q *qemu) hotplugBlockDevice(drive *config.BlockDrive, op operation) error
|
||||
err = q.hotplugAddBlockDevice(drive, op, devID)
|
||||
} else {
|
||||
if q.config.BlockDeviceDriver == config.VirtioBlock {
|
||||
- if err := q.arch.removeDeviceFromBridge(drive.ID); err != nil {
|
||||
+ if err := q.putPciAddress(drive.ID); err != nil {
|
||||
return err
|
||||
}
|
||||
} else if q.config.BlockDeviceDriver == config.VirtioSCSI {
|
||||
@@ -1345,22 +1356,22 @@ func (q *qemu) hotplugVFIODevice(device *config.VFIODev, op operation) (err erro
|
||||
}
|
||||
}
|
||||
|
||||
- addr, bridge, err := q.arch.addDeviceToBridge(devID, types.PCI)
|
||||
+ addr, bus, _, err := q.getPciAddress(devID, types.PCI)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
defer func() {
|
||||
if err != nil {
|
||||
- q.arch.removeDeviceFromBridge(devID)
|
||||
+ q.putPciAddress(devID)
|
||||
}
|
||||
}()
|
||||
|
||||
switch device.Type {
|
||||
case config.VFIODeviceNormalType:
|
||||
- return q.qmpMonitorCh.qmp.ExecutePCIVFIODeviceAdd(q.qmpMonitorCh.ctx, devID, device.BDF, addr, bridge.ID, romFile)
|
||||
+ return q.qmpMonitorCh.qmp.ExecutePCIVFIODeviceAdd(q.qmpMonitorCh.ctx, devID, device.BDF, addr, bus, romFile)
|
||||
case config.VFIODeviceMediatedType:
|
||||
- return q.qmpMonitorCh.qmp.ExecutePCIVFIOMediatedDeviceAdd(q.qmpMonitorCh.ctx, devID, device.SysfsDev, addr, bridge.ID, romFile)
|
||||
+ return q.qmpMonitorCh.qmp.ExecutePCIVFIOMediatedDeviceAdd(q.qmpMonitorCh.ctx, devID, device.SysfsDev, addr, bus, romFile)
|
||||
default:
|
||||
return fmt.Errorf("Incorrect VFIO device type found")
|
||||
}
|
||||
@@ -1368,7 +1379,7 @@ func (q *qemu) hotplugVFIODevice(device *config.VFIODev, op operation) (err erro
|
||||
q.Logger().WithField("dev-id", devID).Info("Start hot-unplug VFIO device")
|
||||
|
||||
if !q.state.HotplugVFIOOnRootBus {
|
||||
- if err := q.arch.removeDeviceFromBridge(devID); err != nil {
|
||||
+ if err := q.putPciAddress(devID); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
@@ -1439,18 +1450,17 @@ func (q *qemu) hotplugNetDevice(endpoint Endpoint, op operation) (err error) {
|
||||
}
|
||||
}()
|
||||
|
||||
- addr, bridge, err := q.arch.addDeviceToBridge(tap.ID, types.PCI)
|
||||
+ addr, bus, pciAddr, err := q.getPciAddress(tap.ID, types.PCI)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
defer func() {
|
||||
if err != nil {
|
||||
- q.arch.removeDeviceFromBridge(tap.ID)
|
||||
+ q.putPciAddress(tap.ID)
|
||||
}
|
||||
}()
|
||||
|
||||
- pciAddr := fmt.Sprintf("%02x/%s", bridge.Addr, addr)
|
||||
endpoint.SetPciAddr(pciAddr)
|
||||
|
||||
var machine govmmQemu.Machine
|
||||
@@ -1459,14 +1469,14 @@ func (q *qemu) hotplugNetDevice(endpoint Endpoint, op operation) (err error) {
|
||||
return err
|
||||
}
|
||||
if machine.Type == QemuCCWVirtio {
|
||||
- devNoHotplug := fmt.Sprintf("fe.%x.%x", bridge.Addr, addr)
|
||||
+ devNoHotplug := fmt.Sprintf("fe.%x.%x", bus, addr)
|
||||
return q.qmpMonitorCh.qmp.ExecuteNetCCWDeviceAdd(q.qmpMonitorCh.ctx, tap.Name, devID, endpoint.HardwareAddr(), devNoHotplug, int(q.config.NumVCPUs))
|
||||
}
|
||||
- return q.qmpMonitorCh.qmp.ExecuteNetPCIDeviceAdd(q.qmpMonitorCh.ctx, tap.Name, devID, endpoint.HardwareAddr(), addr, bridge.ID, romFile, int(q.config.NumVCPUs), defaultDisableModern)
|
||||
+ return q.qmpMonitorCh.qmp.ExecuteNetPCIDeviceAdd(q.qmpMonitorCh.ctx, tap.Name, devID, endpoint.HardwareAddr(), addr, bus, romFile, int(q.config.NumVCPUs), defaultDisableModern)
|
||||
|
||||
}
|
||||
|
||||
- if err := q.arch.removeDeviceFromBridge(tap.ID); err != nil {
|
||||
+ if err := q.putPciAddress(tap.ID); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
@@ -2042,6 +2052,21 @@ func genericMemoryTopology(memoryMb, hostMemoryMb uint64, slots uint8, memoryOff
|
||||
return memory
|
||||
}
|
||||
|
||||
+func genericAppendRootPorts(devices []govmmQemu.Device, rootPorts *types.PCIeRootPortPool) []govmmQemu.Device {
|
||||
+ for _, rp := range rootPorts.Items {
|
||||
+ devices = append(devices, govmmQemu.PCIeRootPortDevice{
|
||||
+ Port: rp.Port,
|
||||
+ Bus: rp.Bus,
|
||||
+ ID: rp.ID,
|
||||
+ Chassis: strconv.Itoa(int(rp.Chassis)),
|
||||
+ Multifunction: rp.Multifunction,
|
||||
+ Addr: fmt.Sprintf("0x%s.0x%s", rp.Slot, rp.Function),
|
||||
+ })
|
||||
+ }
|
||||
+
|
||||
+ return devices
|
||||
+}
|
||||
+
|
||||
// genericAppendPCIeRootPort appends to devices the given pcie-root-port
|
||||
func genericAppendPCIeRootPort(devices []govmmQemu.Device, number uint32, machineType string) []govmmQemu.Device {
|
||||
var (
|
||||
@@ -2241,6 +2266,7 @@ func (q *qemu) save() (s persistapi.HypervisorState) {
|
||||
s.HotpluggedMemory = q.state.HotpluggedMemory
|
||||
s.HotplugVFIOOnRootBus = q.state.HotplugVFIOOnRootBus
|
||||
s.PCIeRootPort = q.state.PCIeRootPort
|
||||
+ s.PCIeRootPortsPool = q.state.PCIeRootPortsPool
|
||||
|
||||
for _, bridge := range q.arch.getBridges() {
|
||||
s.Bridges = append(s.Bridges, persistapi.Bridge{
|
||||
@@ -2265,6 +2291,7 @@ func (q *qemu) load(s persistapi.HypervisorState) {
|
||||
q.state.HotplugVFIOOnRootBus = s.HotplugVFIOOnRootBus
|
||||
q.state.VirtiofsdPid = s.VirtiofsdPid
|
||||
q.state.PCIeRootPort = s.PCIeRootPort
|
||||
+ q.state.PCIeRootPortsPool = s.PCIeRootPortsPool
|
||||
|
||||
for _, bridge := range s.Bridges {
|
||||
q.state.Bridges = append(q.state.Bridges, types.NewBridge(types.Type(bridge.Type), bridge.ID, bridge.DeviceAddr, bridge.Addr))
|
||||
@@ -2302,3 +2329,56 @@ func (q *qemu) generateSocket(id string, useVsock bool) (interface{}, error) {
|
||||
func (q *qemu) getMemorySize() uint32 {
|
||||
return q.config.MemorySize + uint32(q.state.HotpluggedMemory)
|
||||
}
|
||||
+
|
||||
+// getPciAddress allocate the pci slot to hotplugged device and
|
||||
+// return the pci slot address
|
||||
+func (q *qemu) getPciAddress(devID string, t types.Type) (slot, bus, pciAddr string, err error) {
|
||||
+ machine, err := q.getQemuMachine()
|
||||
+ if err != nil {
|
||||
+ return "", "", "", err
|
||||
+ }
|
||||
+
|
||||
+ switch machine.Type {
|
||||
+ case QemuVirt:
|
||||
+ rp, err := q.state.PCIeRootPortsPool.AddDevice(devID)
|
||||
+ if err != nil {
|
||||
+ return "", "", "", err
|
||||
+ }
|
||||
+ // PCIe Root Port only have one slot
|
||||
+ slot = "0x0"
|
||||
+ // pciAddr specifies the slot and function of the Root Port and the slot of the device
|
||||
+ pciAddr = fmt.Sprintf("%s.%s/00", rp.Slot, rp.Function)
|
||||
+ bus = rp.ID
|
||||
+ default:
|
||||
+ var bridge types.Bridge
|
||||
+ slot, bridge, err = q.arch.addDeviceToBridge(devID, t)
|
||||
+ if err != nil {
|
||||
+ return "", "", "", err
|
||||
+ }
|
||||
+ bus = bridge.ID
|
||||
+ // PCI address is in the format bridge-addr.0/device-addr eg. "03.0/02"
|
||||
+ pciAddr = fmt.Sprintf("%02x.0", bridge.Addr) + "/" + slot
|
||||
+ }
|
||||
+ return slot, bus, pciAddr, nil
|
||||
+}
|
||||
+
|
||||
+func (q *qemu) putPciAddress(devID string) error {
|
||||
+ machine, err := q.getQemuMachine()
|
||||
+ if err != nil {
|
||||
+ return err
|
||||
+ }
|
||||
+
|
||||
+ switch machine.Type {
|
||||
+ case QemuVirt:
|
||||
+ err := q.state.PCIeRootPortsPool.RemoveDevice(devID)
|
||||
+ if err != nil {
|
||||
+ return err
|
||||
+ }
|
||||
+ default:
|
||||
+ if err := q.arch.removeDeviceFromBridge(devID); err != nil {
|
||||
+ return err
|
||||
+ }
|
||||
+ }
|
||||
+
|
||||
+ return nil
|
||||
+}
|
||||
diff --git a/virtcontainers/qemu_arch_base.go b/virtcontainers/qemu_arch_base.go
|
||||
index 9d72dd09..cb045530 100644
|
||||
--- a/virtcontainers/qemu_arch_base.go
|
||||
+++ b/virtcontainers/qemu_arch_base.go
|
||||
@@ -130,6 +130,9 @@ type qemuArch interface {
|
||||
|
||||
// appendPCIeRootPortDevice appends a pcie-root-port device to pcie.0 bus
|
||||
appendPCIeRootPortDevice(devices []govmmQemu.Device, number uint32) []govmmQemu.Device
|
||||
+
|
||||
+ // appendRootPorts appends a pcie-root-port device to devices when qemu machine type is "virt"
|
||||
+ appendRootPorts(devices []govmmQemu.Device, rootPorts *types.PCIeRootPortPool) []govmmQemu.Device
|
||||
}
|
||||
|
||||
type qemuArchBase struct {
|
||||
@@ -766,3 +769,9 @@ func (q *qemuArchBase) addBridge(b types.Bridge) {
|
||||
func (q *qemuArchBase) appendPCIeRootPortDevice(devices []govmmQemu.Device, number uint32) []govmmQemu.Device {
|
||||
return genericAppendPCIeRootPort(devices, number, q.machineType)
|
||||
}
|
||||
+
|
||||
+// appendRootPorts appends a pcie-root-port device to devices when qemu machine type is "virt"
|
||||
+// which is different appendPCIeRootPortDevice function
|
||||
+func (q *qemuArchBase) appendRootPorts(devices []govmmQemu.Device, rootPorts *types.PCIeRootPortPool) []govmmQemu.Device {
|
||||
+ return genericAppendRootPorts(devices, rootPorts)
|
||||
+}
|
||||
diff --git a/virtcontainers/types/pcie.go b/virtcontainers/types/pcie.go
|
||||
new file mode 100644
|
||||
index 00000000..83eb6944
|
||||
--- /dev/null
|
||||
+++ b/virtcontainers/types/pcie.go
|
||||
@@ -0,0 +1,103 @@
|
||||
+package types
|
||||
+
|
||||
+import (
|
||||
+ "fmt"
|
||||
+)
|
||||
+
|
||||
+const (
|
||||
+ maxRootPortsCapacity = 25
|
||||
+ slotPerDevice = 7
|
||||
+
|
||||
+ // PCIeRootBus is "pcie.0"
|
||||
+ PCIeRootBus = "pcie.0"
|
||||
+
|
||||
+ // startPort specifies the start port of pcie-root-port
|
||||
+ // for the first slot of "pcie.0" is reserved, so the 1~7 ports are also reserved
|
||||
+ startPort = 8
|
||||
+ funcNumPerSlot = 8
|
||||
+)
|
||||
+
|
||||
+// PCIeRootPort describe the PCIe Root Port
|
||||
+type PCIeRootPort struct {
|
||||
+ // DeviceID specify the device hotplug on the Root Port
|
||||
+ DeviceID string
|
||||
+
|
||||
+ // Port number is the Root Port index
|
||||
+ Port string
|
||||
+
|
||||
+ // Bus number where the Root Port is plugged, typically pcie.0
|
||||
+ Bus string
|
||||
+
|
||||
+ // ID is used to identify the pcie-root-port in qemu
|
||||
+ ID string
|
||||
+
|
||||
+ // Slot specifies slot address of Root Port
|
||||
+ Slot string
|
||||
+
|
||||
+ // Function specifies function of Root Port
|
||||
+ Function string
|
||||
+
|
||||
+ // Chassis number
|
||||
+ Chassis uint32
|
||||
+
|
||||
+ // Multifunction is used to specify the pcie-root-port is multifunction supported
|
||||
+ Multifunction bool
|
||||
+}
|
||||
+
|
||||
+// PCIeRootPortPool describe a set of PCIe Root Ports
|
||||
+type PCIeRootPortPool struct {
|
||||
+ // Items contains information about devices plugged and number limit
|
||||
+ Items []*PCIeRootPort
|
||||
+}
|
||||
+
|
||||
+// Init Initialized the PCIeRootPortPool instance
|
||||
+func (rp *PCIeRootPortPool) Init(number int) {
|
||||
+ if number == 0 || number > maxRootPortsCapacity {
|
||||
+ number = maxRootPortsCapacity
|
||||
+ }
|
||||
+
|
||||
+ for i := 0; i < number; i++ {
|
||||
+ dev := &PCIeRootPort{
|
||||
+ DeviceID: "",
|
||||
+ Port: fmt.Sprintf("0x%x", startPort+i),
|
||||
+ Bus: PCIeRootBus,
|
||||
+ ID: fmt.Sprintf("pci.%d", i+1),
|
||||
+ Chassis: uint32(i + 1),
|
||||
+ }
|
||||
+
|
||||
+ major := i / funcNumPerSlot
|
||||
+ minor := i % funcNumPerSlot
|
||||
+ dev.Multifunction = false
|
||||
+ if minor == 0 {
|
||||
+ dev.Multifunction = true
|
||||
+ }
|
||||
+ dev.Slot = fmt.Sprintf("%02x", major+1)
|
||||
+ dev.Function = fmt.Sprintf("%x", minor)
|
||||
+
|
||||
+ rp.Items = append(rp.Items, dev)
|
||||
+ }
|
||||
+}
|
||||
+
|
||||
+// AddDevice add a device to the PCIeRootPortPool
|
||||
+func (rp *PCIeRootPortPool) AddDevice(devID string) (*PCIeRootPort, error) {
|
||||
+ for _, it := range rp.Items {
|
||||
+ if it.DeviceID == "" {
|
||||
+ it.DeviceID = devID
|
||||
+ return it, nil
|
||||
+ }
|
||||
+ }
|
||||
+ return nil, fmt.Errorf("Unable to hot plug device on Root Ports: there are not empty slots")
|
||||
+}
|
||||
+
|
||||
+// RemoveDevice remove a device from the PCIeRootPortPool
|
||||
+func (rp *PCIeRootPortPool) RemoveDevice(devID string) error {
|
||||
+ for _, it := range rp.Items {
|
||||
+ if it.DeviceID == devID {
|
||||
+ // free address to re-use the same slot with other devices
|
||||
+ it.DeviceID = ""
|
||||
+ return nil
|
||||
+ }
|
||||
+ }
|
||||
+
|
||||
+ return fmt.Errorf("Unable to hot unplug device %s: not present on Root Port", devID)
|
||||
+}
|
||||
--
|
||||
2.14.3 (Apple Git-98)
|
||||
|
||||
@ -1,245 +0,0 @@
|
||||
From a36c9857447aaf22628af1ef01406a916436133b Mon Sep 17 00:00:00 2001
|
||||
From: jiangpengfei <jiangpengfei9@huawei.com>
|
||||
Date: Tue, 18 Aug 2020 22:00:04 +0800
|
||||
Subject: [PATCH 39/50] storage: add storage common functions and structs
|
||||
|
||||
reason:
|
||||
1. add storage.go and storage_spec.go
|
||||
2. provide funcs for mount nfs and gpath
|
||||
|
||||
Signed-off-by: jiangpengfei <jiangpengfei9@huawei.com>
|
||||
---
|
||||
virtcontainers/storage/storage.go | 115 +++++++++++++++++++++++++++++++++
|
||||
virtcontainers/storage/storage_spec.go | 98 ++++++++++++++++++++++++++++
|
||||
2 files changed, 213 insertions(+)
|
||||
create mode 100644 virtcontainers/storage/storage.go
|
||||
create mode 100644 virtcontainers/storage/storage_spec.go
|
||||
|
||||
diff --git a/virtcontainers/storage/storage.go b/virtcontainers/storage/storage.go
|
||||
new file mode 100644
|
||||
index 00000000..77ef994f
|
||||
--- /dev/null
|
||||
+++ b/virtcontainers/storage/storage.go
|
||||
@@ -0,0 +1,115 @@
|
||||
+// Copyright (c) Huawei Technologies Co., Ltd. 2019. All rights reserved.
|
||||
+// SPDX-License-Identifier: Apache-2.0
|
||||
+// Description: common functions
|
||||
+// Author: licuifang
|
||||
+// Create: 2019-07-14
|
||||
+
|
||||
+// Package storage provides functions for mounting
|
||||
+package storage
|
||||
+
|
||||
+import (
|
||||
+ "fmt"
|
||||
+ "path/filepath"
|
||||
+ "regexp"
|
||||
+ "strings"
|
||||
+
|
||||
+ "github.com/kata-containers/agent/protocols/grpc"
|
||||
+ "github.com/opencontainers/runtime-spec/specs-go"
|
||||
+)
|
||||
+
|
||||
+var (
|
||||
+ // simple regular expressions for "nfs-server.com:/aaa/bbb/ccc/" or "192.168.1.1:/remote/path"
|
||||
+ regUrl = regexp.MustCompile(`^((([a-zA-Z0-9]([a-zA-Z0-9\-]{0,61}[a-zA-Z0-9])?\.)+[a-zA-Z]{2,6})|((?:(?:25[0-5]|2[0-4]\d|[01]?\d?\d)\.){3}(?:25[0-5]|2[0-4]\d|[01]?\d?\d))):/[A-Za-z0-9_./-]{0,1000}$`)
|
||||
+
|
||||
+ defaultMountOption = "nolock"
|
||||
+ storageMountOptionKey = "mount_op"
|
||||
+ storageCustomOptionKey = "custom_op"
|
||||
+)
|
||||
+
|
||||
+type RemoteStorage struct {
|
||||
+ Source string
|
||||
+ Dest string
|
||||
+ Options map[string][]string
|
||||
+}
|
||||
+
|
||||
+func (n *RemoteStorage) Validate(volumeType string) error {
|
||||
+ if n.Source == "" || n.Dest == "" {
|
||||
+ return fmt.Errorf("the source and dest of storage cannot be empty")
|
||||
+ }
|
||||
+ switch volumeType {
|
||||
+ case NFS:
|
||||
+ chk := regUrl.FindAllString(n.Source, -1)
|
||||
+ if chk == nil {
|
||||
+ return fmt.Errorf("invalid url for nfs")
|
||||
+ }
|
||||
+ case GPATHFS:
|
||||
+ if !filepath.IsAbs(n.Source) {
|
||||
+ return fmt.Errorf("invalid gpath")
|
||||
+ }
|
||||
+ }
|
||||
+
|
||||
+ return nil
|
||||
+}
|
||||
+
|
||||
+func (n *RemoteStorage) GetGrpcStorageAndAppendMount(volumeType string, vmBasePath string, spec *specs.Spec, sandboxId string) *grpc.Storage {
|
||||
+ var (
|
||||
+ grpcStorag *grpc.Storage
|
||||
+ vmPath string
|
||||
+ )
|
||||
+ switch volumeType {
|
||||
+ case NFS:
|
||||
+ // source of sfs like remote-nfs.com:/share-53ee51a1
|
||||
+ // source of sfs with subpath like remote-nfs.com:/share-53ee51a1/aaa/bbb
|
||||
+ // source of sfs-turbo like ip:/
|
||||
+ // source of sfs-turbo with subpath like ip://aaa or ip://aaa/bbb
|
||||
+ // here we only get the origin source such as remote-nfs.com:/share-53ee51a1 or ip:/ as the Source of grpcStorage
|
||||
+ item := strings.Split(n.Source, ":")
|
||||
+ if len(item) != 2 {
|
||||
+ return nil
|
||||
+ }
|
||||
+ vmPath = filepath.Join(vmBasePath, sandboxId, item[0], item[1])
|
||||
+ parts := strings.SplitAfter(item[1], "/")
|
||||
+ if len(parts) > 2 {
|
||||
+ n.Source = item[0] + ":" + parts[0] + parts[1]
|
||||
+ }
|
||||
+
|
||||
+ grpcStorag = &grpc.Storage{
|
||||
+ Driver: NFS,
|
||||
+ Source: n.Source,
|
||||
+ MountPoint: vmPath,
|
||||
+ }
|
||||
+ grpcStorag.Options = append(grpcStorag.Options, defaultMountOption)
|
||||
+ case GPATHFS:
|
||||
+ vmPath = n.Source
|
||||
+ grpcStorag = &grpc.Storage{
|
||||
+ Driver: GPATHFS,
|
||||
+ Source: vmPath,
|
||||
+ MountPoint: n.Dest,
|
||||
+ }
|
||||
+ }
|
||||
+ if custumOpts, ok := n.Options[storageCustomOptionKey]; ok {
|
||||
+ for _, custumOpt := range custumOpts {
|
||||
+ grpcStorag.Options = append(grpcStorag.Options, custumOpt)
|
||||
+ }
|
||||
+ }
|
||||
+
|
||||
+ mnt := specs.Mount{
|
||||
+ Source: vmPath,
|
||||
+ Destination: n.Dest,
|
||||
+ }
|
||||
+ if mountOpts, ok := n.Options[storageMountOptionKey]; ok {
|
||||
+ for _, mountOpt := range mountOpts {
|
||||
+ if mountOpt == "shared" {
|
||||
+ // in order to support "shared" propagation for guestpath mounting to container directory
|
||||
+ // we should set rootfspropagation shared
|
||||
+ spec.Linux.RootfsPropagation = "shared"
|
||||
+ }
|
||||
+ mnt.Options = append(mnt.Options, mountOpt)
|
||||
+ }
|
||||
+ }
|
||||
+ mnt.Options = append(mnt.Options, "rbind")
|
||||
+ mnt.Type = "bind"
|
||||
+ spec.Mounts = append(spec.Mounts, mnt)
|
||||
+
|
||||
+ return grpcStorag
|
||||
+}
|
||||
diff --git a/virtcontainers/storage/storage_spec.go b/virtcontainers/storage/storage_spec.go
|
||||
new file mode 100644
|
||||
index 00000000..8e866b8d
|
||||
--- /dev/null
|
||||
+++ b/virtcontainers/storage/storage_spec.go
|
||||
@@ -0,0 +1,98 @@
|
||||
+// Copyright (c) Huawei Technologies Co., Ltd. 2019. All rights reserved.
|
||||
+// Description: common functions
|
||||
+// Author: caihaomin c00416947
|
||||
+// Create: 2019-05-05
|
||||
+
|
||||
+package storage
|
||||
+
|
||||
+import (
|
||||
+ "encoding/json"
|
||||
+ "fmt"
|
||||
+
|
||||
+ "github.com/kata-containers/agent/protocols/grpc"
|
||||
+ "github.com/opencontainers/runtime-spec/specs-go"
|
||||
+)
|
||||
+
|
||||
+const (
|
||||
+ NFS = "nfs"
|
||||
+ GPATHFS = "gpath"
|
||||
+)
|
||||
+
|
||||
+// STORAGES specifies the storage type and is supported
|
||||
+var STORAGES = map[string]bool{
|
||||
+ NFS: true,
|
||||
+ GPATHFS: true,
|
||||
+}
|
||||
+
|
||||
+type StorageSpec struct {
|
||||
+ // StorageType specifies the type of storage passing down
|
||||
+ StorageType string `json:"storage_type,omitempty"`
|
||||
+
|
||||
+ Source string `json:"source,omitempty"`
|
||||
+ Destination string `json:"dest,omitempty"`
|
||||
+
|
||||
+ // Options specifies the options of storage type
|
||||
+ Options map[string][]string `json:"options,omitempty"`
|
||||
+}
|
||||
+
|
||||
+func (s *StorageSpec) generateInstances() StorageOperation {
|
||||
+ var storageOpt StorageOperation
|
||||
+ storageOpt = &RemoteStorage{
|
||||
+ Source: s.Source,
|
||||
+ Dest: s.Destination,
|
||||
+ Options: s.Options,
|
||||
+ }
|
||||
+
|
||||
+ return storageOpt
|
||||
+}
|
||||
+
|
||||
+func ValidateStorageValue(value string) error {
|
||||
+ var storageSpecs []StorageSpec
|
||||
+ if err := json.Unmarshal([]byte(value), &storageSpecs); err != nil {
|
||||
+ return err
|
||||
+ }
|
||||
+
|
||||
+ var storageOpt []StorageOperation
|
||||
+ for _, item := range storageSpecs {
|
||||
+ if supported, ok := STORAGES[item.StorageType]; !ok || !supported {
|
||||
+ return fmt.Errorf("type %s of storage is not supported", item.StorageType)
|
||||
+ }
|
||||
+ storageItem := item.generateInstances()
|
||||
+ if err := storageItem.Validate(item.StorageType); err != nil {
|
||||
+ return err
|
||||
+ }
|
||||
+ storageOpt = append(storageOpt, storageItem)
|
||||
+ }
|
||||
+
|
||||
+ return nil
|
||||
+}
|
||||
+
|
||||
+func GetGrpcStorageAndAppendMount(vmBasePath, value string, spec *specs.Spec, sandboxId string) []*grpc.Storage {
|
||||
+ var (
|
||||
+ grpcStorages []*grpc.Storage
|
||||
+ storageSpec []StorageSpec
|
||||
+ )
|
||||
+
|
||||
+ // the value has been validated before, and there is no need to judge the return value here
|
||||
+ json.Unmarshal([]byte(value), &storageSpec)
|
||||
+ for _, item := range storageSpec {
|
||||
+ storageItem := item.generateInstances()
|
||||
+ grpcStorages = append(grpcStorages, storageItem.GetGrpcStorageAndAppendMount(item.StorageType, vmBasePath, spec, sandboxId))
|
||||
+ }
|
||||
+ return grpcStorages
|
||||
+}
|
||||
+
|
||||
+type StorageOperation interface {
|
||||
+ Validate(volumeType string) error
|
||||
+ GetGrpcStorageAndAppendMount(volumeType string, vmBasePath string, spec *specs.Spec, sandboxId string) *grpc.Storage
|
||||
+}
|
||||
+
|
||||
+type FakeStorage struct{}
|
||||
+
|
||||
+func (f *FakeStorage) Validate() error {
|
||||
+ return nil
|
||||
+}
|
||||
+
|
||||
+func (f *FakeStorage) GetGrpcStorageAndAppendMount(vmBasePath string, spec *specs.Spec) *grpc.Storage {
|
||||
+ return &grpc.Storage{}
|
||||
+}
|
||||
--
|
||||
2.14.3 (Apple Git-98)
|
||||
|
||||
@ -1,221 +0,0 @@
|
||||
From 2e32e2c156a134605de42c53ef77366ac73f2614 Mon Sep 17 00:00:00 2001
|
||||
From: jiangpengfei <jiangpengfei9@huawei.com>
|
||||
Date: Tue, 18 Aug 2020 22:01:58 +0800
|
||||
Subject: [PATCH 40/50] storage: add go tests for storage
|
||||
|
||||
reason: add go tests file storage_test.go and storage_spec_test.go
|
||||
|
||||
Signed-off-by: jiangpengfei <jiangpengfei9@huawei.com>
|
||||
---
|
||||
virtcontainers/storage/storage_spec_test.go | 151 ++++++++++++++++++++++++++++
|
||||
virtcontainers/storage/storage_test.go | 40 ++++++++
|
||||
2 files changed, 191 insertions(+)
|
||||
create mode 100644 virtcontainers/storage/storage_spec_test.go
|
||||
create mode 100644 virtcontainers/storage/storage_test.go
|
||||
|
||||
diff --git a/virtcontainers/storage/storage_spec_test.go b/virtcontainers/storage/storage_spec_test.go
|
||||
new file mode 100644
|
||||
index 00000000..f638245a
|
||||
--- /dev/null
|
||||
+++ b/virtcontainers/storage/storage_spec_test.go
|
||||
@@ -0,0 +1,151 @@
|
||||
+/*
|
||||
+Copyright (c) Huawei Technologies Co., Ltd. 2019. All rights reserved.
|
||||
+Description: common functions test
|
||||
+Author: yangfeiyu
|
||||
+Create: 2019-07-24
|
||||
+*/
|
||||
+
|
||||
+package storage
|
||||
+
|
||||
+import (
|
||||
+ "encoding/json"
|
||||
+ "testing"
|
||||
+
|
||||
+ "github.com/stretchr/testify/assert"
|
||||
+ "github.com/opencontainers/runtime-spec/specs-go"
|
||||
+)
|
||||
+
|
||||
+func TestValidateStorageValue(t *testing.T) {
|
||||
+ assert := assert.New(t)
|
||||
+ //invalid storage type,not NFS or GPATHS
|
||||
+ spec := &StorageSpec{
|
||||
+ StorageType: "invalidStorage",
|
||||
+ Source: "sfs.com:/remote/path",
|
||||
+ Destination: "/opt/",
|
||||
+ }
|
||||
+ data, err := json.Marshal(spec)
|
||||
+ assert.NoError(err)
|
||||
+
|
||||
+ err = ValidateStorageValue(string(data))
|
||||
+ assert.Error(err, "StorageType is not NFS or GPATHS,it should return error")
|
||||
+
|
||||
+ //case 1: NFS
|
||||
+ specNFS := []StorageSpec{
|
||||
+ {
|
||||
+ StorageType: NFS,
|
||||
+ Source: "sfs.com:/remote/path",
|
||||
+ Destination: "/opt/"},
|
||||
+ }
|
||||
+ data, err = json.Marshal(specNFS)
|
||||
+ assert.NoError(err)
|
||||
+
|
||||
+ err = ValidateStorageValue(string(data))
|
||||
+ assert.NoError(err)
|
||||
+
|
||||
+ // case2: invalid source domain address
|
||||
+ specNFS = []StorageSpec{
|
||||
+ {
|
||||
+ StorageType: NFS,
|
||||
+ Source: "sfs..com:/remote/path",
|
||||
+ Destination: "/opt/"},
|
||||
+ }
|
||||
+ data, err = json.Marshal(specNFS)
|
||||
+ assert.NoError(err)
|
||||
+
|
||||
+ err = ValidateStorageValue(string(data))
|
||||
+ assert.Error(err)
|
||||
+
|
||||
+ // case 3: nfs source is valid ip address
|
||||
+ ipSpecNFS := []StorageSpec{
|
||||
+ {
|
||||
+ StorageType: NFS,
|
||||
+ Source: "192.168.18.147:/remote/path",
|
||||
+ Destination: "/tmp/sfsturbo0",
|
||||
+ },
|
||||
+ }
|
||||
+
|
||||
+ data, err = json.Marshal(ipSpecNFS)
|
||||
+ assert.NoError(err)
|
||||
+
|
||||
+ err = ValidateStorageValue(string(data))
|
||||
+ assert.NoError(err)
|
||||
+
|
||||
+ // case 4: invalid ip address for nfs source
|
||||
+ ipSpecNFS = []StorageSpec{
|
||||
+ {
|
||||
+ StorageType: NFS,
|
||||
+ Source: "192.168.18.300:/remote/path",
|
||||
+ Destination: "/tmp/sfsturbo0",
|
||||
+ },
|
||||
+ }
|
||||
+
|
||||
+ data, err = json.Marshal(ipSpecNFS)
|
||||
+ assert.NoError(err)
|
||||
+
|
||||
+ err = ValidateStorageValue(string(data))
|
||||
+ assert.Error(err)
|
||||
+
|
||||
+ // case 5: validate ip address and source remote path is /
|
||||
+ ipSpecNFS = []StorageSpec{
|
||||
+ {
|
||||
+ StorageType: NFS,
|
||||
+ Source: "192.168.18.3:/",
|
||||
+ Destination: "/tmp/sfsturbo0",
|
||||
+ },
|
||||
+ }
|
||||
+
|
||||
+ data, err = json.Marshal(ipSpecNFS)
|
||||
+ assert.NoError(err)
|
||||
+
|
||||
+ err = ValidateStorageValue(string(data))
|
||||
+ assert.NoError(err)
|
||||
+
|
||||
+ // case 6: invalid ip address 192.168.18.147.11
|
||||
+ ipSpecNFS = []StorageSpec{
|
||||
+ {
|
||||
+ StorageType: NFS,
|
||||
+ Source: "192.168.18.147.11:/",
|
||||
+ Destination: "/tmp/sfsturbo0",
|
||||
+ },
|
||||
+ }
|
||||
+
|
||||
+ data, err = json.Marshal(ipSpecNFS)
|
||||
+ assert.NoError(err)
|
||||
+
|
||||
+ err = ValidateStorageValue(string(data))
|
||||
+ assert.Error(err)
|
||||
+
|
||||
+ //GPATHS
|
||||
+ specGPATHFS := []StorageSpec{
|
||||
+ {
|
||||
+ StorageType: GPATHFS,
|
||||
+ Source: "/remote/path",
|
||||
+ Destination: "/opt/"},
|
||||
+ }
|
||||
+ data, err = json.Marshal(specGPATHFS)
|
||||
+ assert.NoError(err)
|
||||
+
|
||||
+ err = ValidateStorageValue(string(data))
|
||||
+ if STORAGES[GPATHFS] == true {
|
||||
+ assert.NoError(err, "StorageType GPATHFS is valid,it should return no error")
|
||||
+ } else {
|
||||
+ assert.Error(err, "StorageType GPATHFS is invalid,it should return error")
|
||||
+ }
|
||||
+}
|
||||
+
|
||||
+func TestGetGrpcStorageAndAppendMount(t *testing.T) {
|
||||
+ assert := assert.New(t)
|
||||
+ var spec specs.Spec
|
||||
+ sandboxId := "7f3b5e32cc7fe757e1b69f7a005ca1971def5011e730c82535783630fe24b318"
|
||||
+ vmBasePath := "/tmp"
|
||||
+
|
||||
+ // storageType nfs with legal source and dest will succeed
|
||||
+ storageSpec := "[{\"storage_type\":\"nfs\",\"source\":\"sfs.com:/remote/path\",\"dest\":\"/opt/nfs\"}]"
|
||||
+ grpcStorages := GetGrpcStorageAndAppendMount(vmBasePath,storageSpec,&spec,sandboxId)
|
||||
+ assert.NotNil(grpcStorages[0])
|
||||
+
|
||||
+ // storageType nfs with illegal source and dest will fail
|
||||
+ storageSpec = "[{\"storage_type\":\"nfs\",\"source\":\"sfs.com\",\"dest\":\"/opt/nfs\"}]"
|
||||
+ grpcStorages = GetGrpcStorageAndAppendMount(vmBasePath,storageSpec,&spec,sandboxId)
|
||||
+ assert.Nil(grpcStorages[0])
|
||||
+}
|
||||
diff --git a/virtcontainers/storage/storage_test.go b/virtcontainers/storage/storage_test.go
|
||||
new file mode 100644
|
||||
index 00000000..c9ca4926
|
||||
--- /dev/null
|
||||
+++ b/virtcontainers/storage/storage_test.go
|
||||
@@ -0,0 +1,40 @@
|
||||
+/*
|
||||
+Copyright (c) Huawei Technologies Co., Ltd. 2019. All rights reserved.
|
||||
+Description: common functions test
|
||||
+Author: yangfeiyu
|
||||
+Create: 2019-07-24
|
||||
+*/
|
||||
+
|
||||
+package storage
|
||||
+
|
||||
+import (
|
||||
+ "testing"
|
||||
+
|
||||
+ "github.com/stretchr/testify/assert"
|
||||
+)
|
||||
+
|
||||
+func TestValidate(t *testing.T) {
|
||||
+ //NFS valid
|
||||
+ assert := assert.New(t)
|
||||
+ remoteStorage := &RemoteStorage{
|
||||
+ Source: "sfs.com:/remote/path",
|
||||
+ Dest: "/opt/",
|
||||
+ }
|
||||
+ err := remoteStorage.Validate(NFS)
|
||||
+ assert.NoError(err, "NFS url is valid,it should return no error")
|
||||
+
|
||||
+ //NFS invalid
|
||||
+ remoteStorage.Source = "sfs.com/../remote/path"
|
||||
+ err = remoteStorage.Validate(NFS)
|
||||
+ assert.Error(err, "NFS url is invalid,it should return error")
|
||||
+
|
||||
+ //GPATHS valid
|
||||
+ remoteStorage.Source = "/path/in/vm"
|
||||
+ err = remoteStorage.Validate(GPATHFS)
|
||||
+ assert.NoError(err, "GPATHFS is valid,it should return no error")
|
||||
+
|
||||
+ //GPATHS invalid
|
||||
+ remoteStorage.Source = "./../path/in/../vm"
|
||||
+ err = remoteStorage.Validate(GPATHFS)
|
||||
+ assert.Error(err, "GPATHFS is invalid,it should return error")
|
||||
+}
|
||||
--
|
||||
2.14.3 (Apple Git-98)
|
||||
|
||||
@ -1,115 +0,0 @@
|
||||
From 31e07f1b6cbf361783c4d7adf9e4b8da30c67384 Mon Sep 17 00:00:00 2001
|
||||
From: jiangpengfei <jiangpengfei9@huawei.com>
|
||||
Date: Tue, 18 Aug 2020 22:05:25 +0800
|
||||
Subject: [PATCH 41/50] storage: mount nfs and gpath with given annotation
|
||||
|
||||
reason: when run container with annotation about storage spec,
|
||||
prepare basic info in kata-runtime
|
||||
|
||||
Signed-off-by: jiangpengfei <jiangpengfei9@huawei.com>
|
||||
---
|
||||
virtcontainers/kata_agent.go | 14 +++++++++++++-
|
||||
virtcontainers/pkg/annotations/annotations.go | 3 +++
|
||||
virtcontainers/pkg/oci/utils.go | 16 ++++++++++++++++
|
||||
3 files changed, 32 insertions(+), 1 deletion(-)
|
||||
|
||||
diff --git a/virtcontainers/kata_agent.go b/virtcontainers/kata_agent.go
|
||||
index d82a7f2d..ac64817a 100644
|
||||
--- a/virtcontainers/kata_agent.go
|
||||
+++ b/virtcontainers/kata_agent.go
|
||||
@@ -30,10 +30,11 @@ import (
|
||||
ns "github.com/kata-containers/runtime/virtcontainers/pkg/nsenter"
|
||||
"github.com/kata-containers/runtime/virtcontainers/pkg/rootless"
|
||||
vcTypes "github.com/kata-containers/runtime/virtcontainers/pkg/types"
|
||||
- "github.com/kata-containers/runtime/virtcontainers/utils"
|
||||
"github.com/kata-containers/runtime/virtcontainers/pkg/uuid"
|
||||
+ "github.com/kata-containers/runtime/virtcontainers/storage"
|
||||
"github.com/kata-containers/runtime/virtcontainers/store"
|
||||
"github.com/kata-containers/runtime/virtcontainers/types"
|
||||
+ "github.com/kata-containers/runtime/virtcontainers/utils"
|
||||
"github.com/opencontainers/runtime-spec/specs-go"
|
||||
opentracing "github.com/opentracing/opentracing-go"
|
||||
"github.com/sirupsen/logrus"
|
||||
@@ -1427,6 +1428,9 @@ func (k *kataAgent) createContainer(sandbox *Sandbox, c *Container) (p *Process,
|
||||
localStorages := k.handleLocalStorage(ociSpec.Mounts, sandbox.id, c.rootfsSuffix)
|
||||
ctrStorages = append(ctrStorages, localStorages...)
|
||||
|
||||
+ remoteStoragtes := k.handleRemoteStorage(ociSpec, sandbox.id)
|
||||
+ ctrStorages = append(ctrStorages, remoteStoragtes...)
|
||||
+
|
||||
// We replace all OCI mount sources that match our container mount
|
||||
// with the right source path (The guest one).
|
||||
if err = k.replaceOCIMountSource(ociSpec, newMounts); err != nil {
|
||||
@@ -1510,6 +1514,14 @@ func (k *kataAgent) createContainer(sandbox *Sandbox, c *Container) (p *Process,
|
||||
k.state.URL, consoleURL, c.config.Cmd, createNSList, enterNSList)
|
||||
}
|
||||
|
||||
+func (k *kataAgent) handleRemoteStorage(spec *specs.Spec, sandboxId string) []*grpc.Storage {
|
||||
+ if value, ok := spec.Annotations[vcAnnotations.StorageSpecTypeKey]; ok {
|
||||
+ return storage.GetGrpcStorageAndAppendMount(kataGuestStorageDir, value, spec, sandboxId)
|
||||
+ }
|
||||
+
|
||||
+ return []*grpc.Storage{}
|
||||
+}
|
||||
+
|
||||
// handleEphemeralStorage handles ephemeral storages by
|
||||
// creating a Storage from corresponding source of the mount point
|
||||
func (k *kataAgent) handleEphemeralStorage(mounts []specs.Mount) []*grpc.Storage {
|
||||
diff --git a/virtcontainers/pkg/annotations/annotations.go b/virtcontainers/pkg/annotations/annotations.go
|
||||
index 903c7f03..e50a697c 100644
|
||||
--- a/virtcontainers/pkg/annotations/annotations.go
|
||||
+++ b/virtcontainers/pkg/annotations/annotations.go
|
||||
@@ -68,6 +68,9 @@ const (
|
||||
// AssetHashType is the hash type used for assets verification
|
||||
AssetHashType = kataAnnotationsPrefix + "asset_hash_type"
|
||||
|
||||
+ // StorageSpecTypeKey is the annotation key to fetch storage_spec
|
||||
+ StorageSpecTypeKey = kataAnnotationsPrefix + "storage_spec"
|
||||
+
|
||||
//
|
||||
// Generic annotations
|
||||
//
|
||||
diff --git a/virtcontainers/pkg/oci/utils.go b/virtcontainers/pkg/oci/utils.go
|
||||
index 948bd3cb..d032227e 100644
|
||||
--- a/virtcontainers/pkg/oci/utils.go
|
||||
+++ b/virtcontainers/pkg/oci/utils.go
|
||||
@@ -22,6 +22,7 @@ import (
|
||||
exp "github.com/kata-containers/runtime/virtcontainers/experimental"
|
||||
vcAnnotations "github.com/kata-containers/runtime/virtcontainers/pkg/annotations"
|
||||
dockershimAnnotations "github.com/kata-containers/runtime/virtcontainers/pkg/annotations/dockershim"
|
||||
+ "github.com/kata-containers/runtime/virtcontainers/storage"
|
||||
"github.com/kata-containers/runtime/virtcontainers/types"
|
||||
"github.com/kata-containers/runtime/virtcontainers/utils"
|
||||
specs "github.com/opencontainers/runtime-spec/specs-go"
|
||||
@@ -340,6 +341,17 @@ func SandboxID(spec specs.Spec) (string, error) {
|
||||
return "", fmt.Errorf("Could not find sandbox ID")
|
||||
}
|
||||
|
||||
+func validateStorageSpec(spec specs.Spec) error {
|
||||
+ if storageSpec, ok := spec.Annotations[vcAnnotations.StorageSpecTypeKey]; ok {
|
||||
+ err := storage.ValidateStorageValue(storageSpec)
|
||||
+ if err != nil {
|
||||
+ return err
|
||||
+ }
|
||||
+ }
|
||||
+
|
||||
+ return nil
|
||||
+}
|
||||
+
|
||||
func addAnnotations(ocispec specs.Spec, config *vc.SandboxConfig) error {
|
||||
addAssetAnnotations(ocispec, config)
|
||||
if err := addHypervisorConfigOverrides(ocispec, config); err != nil {
|
||||
@@ -873,6 +885,10 @@ func SandboxConfig(ocispec specs.Spec, runtime RuntimeConfig, bundlePath, cid, c
|
||||
// ContainerConfig converts an OCI compatible runtime configuration
|
||||
// file to a virtcontainers container configuration structure.
|
||||
func ContainerConfig(ocispec specs.Spec, bundlePath, cid, console string, detach bool) (vc.ContainerConfig, error) {
|
||||
+ err := validateStorageSpec(ocispec)
|
||||
+ if err != nil {
|
||||
+ return vc.ContainerConfig{}, err
|
||||
+ }
|
||||
rootfs := vc.RootFs{Target: ocispec.Root.Path, Mounted: true}
|
||||
if !filepath.IsAbs(rootfs.Target) {
|
||||
rootfs.Target = filepath.Join(bundlePath, ocispec.Root.Path)
|
||||
--
|
||||
2.14.3 (Apple Git-98)
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Loading…
x
Reference in New Issue
Block a user