From 8c0dd0f91366910b3029cb982a3cb807bdbb34cf Mon Sep 17 00:00:00 2001
From: "Xinle.Guo" <guoxinle1@huawei.com>
Date: Fri, 14 Jan 2022 17:07:37 +0800
Subject: [PATCH] stratovirt: add a standard virtual machine sandbox type to
 kata container

Because stratovirt supports both microVM and standardVM
architectures, adapts two sandbox types for kata container.
Besides basic features, it also support virtio-fs, hotplug
VFIO, support more devices.

Signed-off-by: Xinle.Guo <guoxinle1@huawei.com>
---
 src/runtime/pkg/katautils/config.go      |   1 +
 src/runtime/virtcontainers/stratovirt.go | 482 ++++++++++++++++++++---
 2 files changed, 438 insertions(+), 45 deletions(-)

diff --git a/src/runtime/pkg/katautils/config.go b/src/runtime/pkg/katautils/config.go
index e523ed3..b04cdee 100644
--- a/src/runtime/pkg/katautils/config.go
+++ b/src/runtime/pkg/katautils/config.go
@@ -1007,6 +1007,7 @@ func newStratovirtHypervisorConfig(h hypervisor) (vc.HypervisorConfig, error) {
 		BlockDeviceCacheDirect:  h.BlockDeviceCacheDirect,
 		BlockDeviceCacheNoflush: h.BlockDeviceCacheNoflush,
 		EnableIOThreads:         h.EnableIOThreads,
+		PCIeRootPort:            h.PCIeRootPort,
 		DisableVhostNet:         h.DisableVhostNet,
 		EnableVhostUserStore:    h.EnableVhostUserStore,
 		VhostUserStorePath:      h.vhostUserStorePath(),
diff --git a/src/runtime/virtcontainers/stratovirt.go b/src/runtime/virtcontainers/stratovirt.go
index 4fcfb94..ffe8965 100644
--- a/src/runtime/virtcontainers/stratovirt.go
+++ b/src/runtime/virtcontainers/stratovirt.go
@@ -14,14 +14,14 @@ import (
 	"time"
 
 	govmmQemu "github.com/kata-containers/govmm/qemu"
-	"github.com/pkg/errors"
-	"github.com/sirupsen/logrus"
-
 	"github.com/kata-containers/kata-containers/src/runtime/virtcontainers/device/config"
 	persistapi "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/persist/api"
+	vcTypes "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/pkg/types"
 	"github.com/kata-containers/kata-containers/src/runtime/virtcontainers/pkg/uuid"
 	"github.com/kata-containers/kata-containers/src/runtime/virtcontainers/types"
 	"github.com/kata-containers/kata-containers/src/runtime/virtcontainers/utils"
+	"github.com/pkg/errors"
+	"github.com/sirupsen/logrus"
 	"go.opentelemetry.io/otel"
 	otelLabel "go.opentelemetry.io/otel/label"
 	otelTrace "go.opentelemetry.io/otel/trace"
@@ -44,6 +44,11 @@ const (
 const (
 	WaitSandboxTimeoutSecs              = 15
 	MachineTypeMicrovm                  = "microvm"
+	MachineTypeQ35                      = "q35"
+	MachineTypeVirt                     = "virt"
+	RootPortPrefix                      = "pcie"
+	Q35PFlashCode                       = "/usr/share/edk2/ovmf/OVMF_CODE.fd"
+	VirtPFlashCode                      = "/usr/share/edk2/aarch64/QEMU_EFI-pflash.raw"
 	MmioBus                VirtioDriver = "mmio"
 	PciBus                 VirtioDriver = "pci"
 )
@@ -67,6 +72,42 @@ func (d VirtioDriver) getDriver(config *vmConfig) VirtioDriver {
 	}
 }
 
+type rootPortDevice struct {
+	id       string
+	port     string
+	bus      string
+	slot     int
+	plugged  bool
+	addedDev string
+}
+
+func (r rootPortDevice) isVaild() bool {
+	if r.id == "" || r.port == "" {
+		return false
+	}
+	return true
+}
+
+func (r rootPortDevice) getParams(config *vmConfig) []string {
+	if !r.isVaild() {
+		return nil
+	}
+
+	var params []string
+	var devParams []Param
+	devParams = append(devParams, Param{"id", r.id})
+	devParams = append(devParams, Param{"port", r.port})
+	if r.bus == "" {
+		r.bus = "pcie.0"
+	}
+	devParams = append(devParams, Param{"bus", r.bus})
+	devParams = append(devParams, Param{"addr", fmt.Sprintf("%d", r.slot)})
+
+	driver := "pcie-root-port"
+	params = append(params, "-device", fmt.Sprintf("%s,%s", driver, strings.Join(SerializeParams(devParams, "="), ",")))
+	return params
+}
+
 type blkDevice struct {
 	id       string
 	filePath string
@@ -179,6 +220,56 @@ func (n netDevice) getParams(config *vmConfig) []string {
 	return params
 }
 
+type virtioFs struct {
+	driver   VirtioDriver
+	backend  string
+	charID   string
+	charDev  string
+	tag      string
+	deviceID string
+	bus      string
+	addr     string
+}
+
+var virtiofsDriver = map[VirtioDriver]string{
+	MmioBus: "vhost-user-fs-device",
+	PciBus:  "vhost-user-fs-pci",
+}
+
+func (v virtioFs) isVaild() bool {
+	if v.charID == "" || v.charDev == "" || v.deviceID == "" {
+		return false
+	}
+	return true
+}
+
+func (v virtioFs) getParams(config *vmConfig) []string {
+	if !v.isVaild() {
+		return nil
+	}
+
+	var params []string
+	var charParams []Param
+	var fsParams []Param
+
+	charParams = append(charParams, Param{"id", v.charID})
+	charParams = append(charParams, Param{"path", config.fsSockPath})
+
+	v.driver = v.driver.getDriver(config)
+	driver := virtiofsDriver[v.driver]
+	fsParams = append(fsParams, Param{"chardev", v.charDev})
+	fsParams = append(fsParams, Param{"tag", v.tag})
+	fsParams = append(fsParams, Param{"id", v.deviceID})
+	if v.bus != "" {
+		fsParams = append(fsParams, Param{"bus", v.bus})
+		fsParams = append(fsParams, Param{"addr", v.addr})
+	}
+
+	params = append(params, "-chardev", fmt.Sprintf("%s,%s,server,nowait", v.backend, strings.Join(SerializeParams(charParams, "="), ",")))
+	params = append(params, "-device", fmt.Sprintf("%s,%s", driver, strings.Join(SerializeParams(fsParams, "="), ",")))
+	return params
+}
+
 type vhostVsock struct {
 	driver  VirtioDriver
 	id      string
@@ -442,6 +533,12 @@ func (c *vmConfig) appendDevices(params *[]string) {
 	for _, d := range c.devices {
 		*params = append(*params, d.getParams(c)...)
 	}
+
+	if c.machineType == MachineTypeMicrovm {
+		return
+	}
+	// Add flag to unplug devices from their root port faster.
+	*params = append(*params, "-global", "pcie-root-port.fast-unplug=1")
 }
 
 func (c *vmConfig) appendPFlash(params *[]string) {
@@ -499,6 +596,8 @@ func (c *vmConfig) appendIncoming(params *[]string) {
 type State struct {
 	mmioBlkSlots [mmioBlkCount]bool
 	mmioNetSlots [mmioNetCount]bool
+	// The list of RootPorts that can be hot-added.
+	rootPort     []rootPortDevice
 	pid          int
 	virtiofsPid  int
 }
@@ -558,6 +657,26 @@ func (s *stratovirt) getKernelParams(machineType string, initrdPath string) (str
 	return strings.Join(params, " "), nil
 }
 
+func (s *stratovirt) getPFlash(machineType string) ([]string, error) {
+	if s.config.PFlash != nil {
+		return s.config.PFlash, nil
+	}
+
+	var PFlash []string
+	switch machineType {
+	case MachineTypeQ35:
+		PFlash = append(PFlash, fmt.Sprintf("file=%s,if=pflash,unit=0", Q35PFlashCode))
+	case MachineTypeVirt:
+		PFlash = append(PFlash, fmt.Sprintf("file=%s,if=pflash,unit=0", VirtPFlashCode))
+	case MachineTypeMicrovm:
+		return nil, nil
+	default:
+		return nil, fmt.Errorf("failed to match machine type %s", machineType)
+	}
+
+	return PFlash, nil
+}
+
 func (s *stratovirt) createQMPSocket(vmPath string) govmmQemu.QMPSocket {
 	socketPath := filepath.Join(vmPath, apiSocket)
 
@@ -590,6 +709,34 @@ func (s *stratovirt) createDevices() []VirtioDev {
 		}
 	}
 
+	// Create root port for all devices that need to be hot-added.
+	if s.vmConfig.machineType != MachineTypeMicrovm && s.config.PCIeRootPort > 0 {
+		devices = s.appendRootPort(ctx, devices)
+	}
+
+	return devices
+}
+
+func (s *stratovirt) appendRootPort(ctx context.Context, devices []VirtioDev) []VirtioDev {
+	number := s.config.PCIeRootPort
+
+	for i := uint32(1); i < number+1; i++ {
+		addr, err := s.vmConfig.rootBus.AddDevice(ctx, fmt.Sprintf("%s.%d", RootPortPrefix, i))
+		if err != nil {
+			return devices
+		}
+
+		rp := rootPortDevice{
+			id:       fmt.Sprintf("%s.%d", RootPortPrefix, i),
+			port:     fmt.Sprintf("%d", i),
+			bus:      defaultBridgeBus,
+			slot:     int(addr),
+			addedDev: "",
+		}
+		s.state.rootPort = append(s.state.rootPort, rp)
+		devices = append(devices, rp)
+	}
+
 	return devices
 }
 
@@ -729,6 +876,38 @@ func (s *stratovirt) appendNetwork(ctx context.Context, devices []VirtioDev, end
 	return devices
 }
 
+func (s *stratovirt) appendVirtioFs(ctx context.Context, devices []VirtioDev, volume types.Volume) []VirtioDev {
+	if s.config.SharedFS != config.VirtioFS {
+		return devices
+	}
+
+	var bus string
+	var addr uint32
+	var err error
+	name := "virtio_fs"
+
+	if s.vmConfig.machineType != MachineTypeMicrovm {
+		bus = "pcie.0"
+		addr, err = s.vmConfig.rootBus.AddDevice(ctx, name)
+		if err != nil {
+			return devices
+		}
+	}
+
+	devices = append(devices, virtioFs{
+		backend:  "socket",
+		// Virtio-fs must be bound to unique charDev, it uses the same name.
+		charID:   name,
+		charDev:  name,
+		tag:      volume.MountTag,
+		deviceID: "virtio-fs0",
+		bus:      bus,
+		addr:     fmt.Sprintf("%d", addr),
+	})
+
+	return devices
+}
+
 func (s *stratovirt) setVMConfig(id string, hypervisorConfig *HypervisorConfig) error {
 	span, _ := s.trace(s.ctx, "setStratoVirtUp")
 	defer span.End()
@@ -765,7 +944,10 @@ func (s *stratovirt) setVMConfig(id string, hypervisorConfig *HypervisorConfig)
 		return err
 	}
 
-	var PFlash []string
+	PFlash, err := s.getPFlash(machineType)
+	if err != nil {
+		return err
+	}
 
 	vmPath := filepath.Join(s.store.RunVMStoragePath(), s.id)
 	qmpSocket := s.createQMPSocket(vmPath)
@@ -852,8 +1034,33 @@ func (s *stratovirt) setOzone() error {
 	return nil
 }
 
-func (s *stratovirt) hypervisorConfig() HypervisorConfig {
-	return s.config
+// Virtio fs daemon is a shared file system that lets VM access a directory
+// tree on the host.
+func (s *stratovirt) setupVirtioFs() error {
+	if !s.config.DisableBlockDeviceUse || s.config.SharedFS != config.VirtioFS {
+		return nil
+	}
+
+	if _, err := os.Stat(s.config.VirtioFSDaemon); os.IsNotExist(err) {
+		return fmt.Errorf("virtiofsd path (%s) does not exist", s.config.VirtioFSDaemon)
+	}
+
+	args := []string{
+		"-socket-path", filepath.Join(s.vmConfig.vmPath, "virtiofs_kata.sock"),
+		"-source", getSharePath(s.id)}
+	if len(s.config.VirtioFSExtraArgs) != 0 {
+		args = append(args, s.config.VirtioFSExtraArgs...)
+	}
+
+	cmd := exec.Command(s.config.VirtioFSDaemon, args...)
+	s.Logger().Info("Virtiofsd start with cmd: ", cmd)
+
+	if err := cmd.Start(); err != nil {
+		return fmt.Errorf("failed to strat virtiofsd: %v", cmd)
+	}
+
+	s.state.virtiofsPid = cmd.Process.Pid
+	return nil
 }
 
 // Get StratoVirt binary path.
@@ -873,6 +1080,10 @@ func (s *stratovirt) binPath() (string, error) {
 	return path, nil
 }
 
+func (s *stratovirt) hypervisorConfig() HypervisorConfig {
+	return s.config
+}
+
 func (s *stratovirt) createSandbox(ctx context.Context, id string, networkNS NetworkNamespace, hypervisorConfig *HypervisorConfig) error {
 	var span otelTrace.Span
 	span, _ = s.trace(ctx, "createSandbox")
@@ -1000,6 +1211,10 @@ func (s *stratovirt) startSandbox(ctx context.Context, timeout int) error {
 		}
 	}()
 
+	if err = s.setupVirtioFs(); err != nil {
+		return err
+	}
+
 	var params []string
 	s.createBaseParams(s.vmConfig, &params)
 
@@ -1134,28 +1349,29 @@ func (s *stratovirt) addDevice(ctx context.Context, devInfo interface{}, devType
 		s.vmConfig.devices = s.appendNetwork(ctx, s.vmConfig.devices, v)
 	case config.BlockDrive:
 		s.vmConfig.devices = s.appendBlock(ctx, s.vmConfig.devices)
+	case types.Volume:
+		s.vmConfig.devices = s.appendVirtioFs(ctx, s.vmConfig.devices, v)
 	default:
 		s.Logger().WithField("dev-type", v).Warn("Could not append device: unsupported device type")
 	}
 	return nil
 }
 
-func (s *stratovirt) getDevSlot(Name string, isPut bool) (slot int, err error) {
+func (s *stratovirt) setupMmioSlot(Name string, isPut bool) (int, error) {
 	Name = filepath.Base(strings.ToLower(Name))
-
 	if strings.HasPrefix(Name, "eth") {
 		idxStr := strings.TrimPrefix(Name, "eth")
 		if idxStr == Name {
-			return 0, fmt.Errorf("Could not parse idx from Name %q", Name)
+			return 0, fmt.Errorf("could not parse idx from name %q", Name)
 		}
 
 		idx, err := strconv.Atoi(idxStr)
 		if err != nil {
-			return 0, fmt.Errorf("Could not convert to int from Str %q", idxStr)
+			return 0, fmt.Errorf("could not convert to int from str %q", idxStr)
 		}
 
 		if !isPut && s.state.mmioNetSlots[idx] {
-			return 0, fmt.Errorf("GetDevSlot failed, slot is being used %q", idxStr)
+			return 0, fmt.Errorf("failed to setup mmio slot, slot is being used %q", idxStr)
 		}
 		s.state.mmioNetSlots[idx] = !isPut
 
@@ -1163,25 +1379,80 @@ func (s *stratovirt) getDevSlot(Name string, isPut bool) (slot int, err error) {
 	} else if strings.HasPrefix(Name, "vd") {
 		charStr := strings.TrimPrefix(Name, "vd")
 		if charStr == Name {
-			return 0, fmt.Errorf("Could not parse idx from Name %q", Name)
+			return 0, fmt.Errorf("could not parse idx from name %q", Name)
 		}
 
 		char := []rune(charStr)
 		idx := int(char[0] - 'a')
 
 		if !isPut && s.state.mmioBlkSlots[idx] {
-			return 0, fmt.Errorf("GetDevSlot failed, slot is being used %q", charStr)
+			return 0, fmt.Errorf("failed to setup mmio slot, slot is being used %q", charStr)
 		}
 		s.state.mmioBlkSlots[idx] = !isPut
 
 		return idx, nil
 	}
 
-	return 0, fmt.Errorf("GetDevSlot failed, Name is invalid %q", Name)
+	return 0, fmt.Errorf("failed to setup mmio slot , Name is invalid %q", Name)
+}
+
+func (s *stratovirt) setupPciSlot(Name string, isPut bool) (string, int, error) {
+	rootports := &s.state.rootPort
+	if len(*rootports) == 0 {
+		return "", 0, fmt.Errorf("failed to get available address from bridges")
+	}
+
+	for i, rootport := range *rootports {
+		if !isPut && !rootport.plugged && rootport.addedDev == "" {
+			(*rootports)[i].plugged = true
+			(*rootports)[i].addedDev = Name
+			return rootport.id, rootport.slot, nil
+		} else if isPut && rootport.plugged && rootport.addedDev == Name {
+			(*rootports)[i].plugged = false
+			(*rootports)[i].addedDev = ""
+			return rootport.id, rootport.slot, nil
+		}
+	}
+
+	return "", 0, fmt.Errorf("no more bridge slots available")
+}
+
+func (s *stratovirt) getDevSlot(Name string) (string, int, error) {
+	if s.config.HypervisorMachineType == MachineTypeMicrovm {
+		slot, err := s.setupMmioSlot(Name, false)
+		if err != nil {
+			return "", 0, err
+		}
+
+		return "", slot, nil
+	}
+
+	bus, slot, err := s.setupPciSlot(Name, false)
+	if err != nil {
+		return "pcie.0", 0, err
+	}
+
+	return bus, slot, nil
+}
+
+func (s *stratovirt) delDevSlot(Name string) error {
+	if s.vmConfig.machineType == MachineTypeMicrovm {
+		if _, err := s.setupMmioSlot(Name, true); err != nil {
+			return err
+		}
+
+		return nil
+	}
+
+	if _, _, err := s.setupPciSlot(Name, true); err != nil {
+		return err
+	}
+
+	return nil
 }
 
-func (s *stratovirt) hotplugNet(ctx context.Context, endpoint Endpoint, op operation) (err error) {
-	err = s.qmpSetup()
+func (s *stratovirt) hotplugNet(ctx context.Context, endpoint Endpoint, op operation) error {
+	err := s.qmpSetup()
 	if err != nil {
 		return err
 	}
@@ -1198,6 +1469,14 @@ func (s *stratovirt) hotplugNet(ctx context.Context, endpoint Endpoint, op opera
 		return fmt.Errorf("Endpoint is not supported")
 	}
 
+	defer func() {
+		if err != nil {
+			if errDel := s.delDevSlot(endpoint.Name()); errDel != nil {
+				s.Logger().WithError(errDel).Warnf("Failed to delete device slot.")
+			}
+		}
+	}()
+
 	switch op {
 	case addDevice:
 		var (
@@ -1220,27 +1499,51 @@ func (s *stratovirt) hotplugNet(ctx context.Context, endpoint Endpoint, op opera
 			VhostFdNames = append(VhostFdNames, fdName)
 		}
 
-		slot, err := s.getDevSlot(endpoint.Name(), false)
+		bus, slot, err := s.getDevSlot(endpoint.Name())
 		if err != nil {
-			return fmt.Errorf("Could not get unused slot for %q", endpoint.Name())
+			return fmt.Errorf("could not get unused slot for %q", endpoint.Name())
 		}
 
 		if len(VMFdNames) != 0 || len(VhostFdNames) != 0 {
 			if err := s.qmpMonitorCh.qmp.ExecuteNetdevAddByFds(s.qmpMonitorCh.ctx, "tap", tap.ID, VMFdNames, VhostFdNames); err != nil {
-				s.getDevSlot(endpoint.Name(), true)
 				return err
 			}
 		} else {
 			if err := s.qmpMonitorCh.qmp.ExecuteNetdevAdd(s.qmpMonitorCh.ctx, "tap", tap.ID, tap.TAPIface.Name, "no", "no", 0); err != nil {
-				s.getDevSlot(endpoint.Name(), true)
 				return err
 			}
 		}
-		if err := s.qmpMonitorCh.qmp.ExecuteNetPCIDeviceAdd(s.qmpMonitorCh.ctx, tap.Name, tap.ID, endpoint.HardwareAddr(), fmt.Sprintf("%d", slot), "", "", 0, false); err != nil {
-			s.getDevSlot(endpoint.Name(), true)
+
+		// The slot of net device that hotplugged to the root port 
+		// must be zero.
+		devAddr := "0x0.0x0"
+		if s.vmConfig.machineType == MachineTypeMicrovm {
+			devAddr = fmt.Sprintf("%d", slot)
+		} else { 
+			bridgeSlot, err := vcTypes.PciSlotFromInt(slot)
+			if err != nil {
+				return err
+			}
+
+			devSlot, err := vcTypes.PciSlotFromString("0")
+			if err != nil {
+				return err
+			}
+
+			pciPath, err := vcTypes.PciPathFromSlots(bridgeSlot, devSlot)
+			if err != nil {
+				return err
+			}
+			endpoint.SetPciPath(pciPath)
+		}
+
+		if err := s.qmpMonitorCh.qmp.ExecuteNetPCIDeviceAdd(s.qmpMonitorCh.ctx, tap.ID, tap.ID, endpoint.HardwareAddr(), devAddr, bus, "", 0, false); err != nil {
 			return err
 		}
 	case removeDevice:
+		if errDel := s.delDevSlot(endpoint.Name()); errDel != nil {
+			s.Logger().WithError(errDel).Warnf("Failed to delete device slot.")
+		}
 		if err := s.qmpMonitorCh.qmp.ExecuteDeviceDel(s.qmpMonitorCh.ctx, tap.ID); err != nil {
 			return err
 		}
@@ -1248,58 +1551,134 @@ func (s *stratovirt) hotplugNet(ctx context.Context, endpoint Endpoint, op opera
 			return err
 		}
 	default:
-		return fmt.Errorf("Operation is not supported")
+		return fmt.Errorf("operation is not supported")
 	}
 
 	return nil
 }
 
-func (s *stratovirt) hotplugBlk(drive *config.BlockDrive, op operation) (err error) {
-	var filePath string
-	err = s.qmpSetup()
+func (s *stratovirt) hotplugBlk(ctx context.Context, drive *config.BlockDrive, op operation) error {
+	err := s.qmpSetup()
 	if err != nil {
 		return err
 	}
 
+	driver := "virtio-blk-pci"
+	if s.vmConfig.machineType == MachineTypeMicrovm {
+		driver = "virtio-blk-mmio"
+	}
+
+	defer func() {
+		if err != nil {
+			s.qmpMonitorCh.qmp.ExecuteBlockdevDel(s.qmpMonitorCh.ctx, drive.ID)
+			if errDel := s.delDevSlot(drive.VirtPath); errDel != nil {
+				s.Logger().WithError(errDel).Warnf("Failed to delete device slot.")
+			}
+		}
+	}()
+
 	switch op {
 	case addDevice:
-		driver := "virtio-blk-mmio"
+		filePath := drive.File
 		if s.vmConfig.useOzone {
 			filePath, err = s.updateOzoneRes(drive.File, true)
-			if err != nil {
-				return fmt.Errorf("Failed to update ozone resources")
-			}
-		} else {
-			filePath = drive.File
-		}
-		slot, err := s.getDevSlot(drive.VirtPath, false)
-		if err != nil {
-			return fmt.Errorf("Could not get unused slot for %q", drive.VirtPath)
 		}
 
 		if err := s.qmpMonitorCh.qmp.ExecuteBlockdevAdd(s.qmpMonitorCh.ctx, filePath, drive.ID, false); err != nil {
-			s.getDevSlot(drive.VirtPath, true)
 			return err
 		}
 
-		if err := s.qmpMonitorCh.qmp.ExecutePCIDeviceAdd(s.qmpMonitorCh.ctx, drive.ID, drive.ID, driver, fmt.Sprintf("%d", slot), "", "", 0, true, false); err != nil {
-			s.getDevSlot(drive.VirtPath, true)
+		bus, slot, err := s.getDevSlot(drive.VirtPath)
+		if err != nil {
+			return err
+		}
+
+		// The slot of block device that hotplugged to the root port 
+		// must be zero.
+		devAddr := "0x0.0x0"
+		if s.vmConfig.machineType == MachineTypeMicrovm {
+			devAddr = fmt.Sprintf("%d", slot)
+		} else {
+			bridgeSlot, err := vcTypes.PciSlotFromInt(slot)
+			if err != nil {
+				return err
+			}
+
+			devSlot, err := vcTypes.PciSlotFromString("0")
+			if err != nil {
+				return err
+			}
+
+			drive.PCIPath, err = vcTypes.PciPathFromSlots(bridgeSlot, devSlot)
+			if err != nil {
+				return err
+			}
+		}
+
+		if err := s.qmpMonitorCh.qmp.ExecutePCIDeviceAdd(s.qmpMonitorCh.ctx, drive.ID, drive.ID, driver, devAddr, bus, "", 0, false, false); err != nil {
 			return err
 		}
 	case removeDevice:
 		if s.vmConfig.useOzone {
 			s.updateOzoneRes(drive.File, false)
 		}
+
+		if errDel := s.delDevSlot(drive.VirtPath); errDel != nil {
+			s.Logger().WithError(errDel).Warnf("Failed to delete device slot.")
+		}
 		if err := s.qmpMonitorCh.qmp.ExecuteDeviceDel(s.qmpMonitorCh.ctx, drive.ID); err != nil {
 			return err
 		}
 		if err := s.qmpMonitorCh.qmp.ExecuteBlockdevDel(s.qmpMonitorCh.ctx, drive.ID); err != nil {
 			return err
 		}
+	default:
+		return fmt.Errorf("operation is not supported %d", op)
+	}
+
+	return nil
+}
+
+func (s *stratovirt) hotplugVFIO(ctx context.Context, device *config.VFIODev, op operation) error {
+	err := s.qmpSetup()
+	if err != nil {
+		return err
+	}
+
+	defer func() {
+		if err != nil {
+			if errDel := s.delDevSlot(device.ID); errDel != nil {
+				s.Logger().WithError(errDel).Warnf("Failed to delete device slot.")
+			}
+		}
+	}()
 
-		s.getDevSlot(drive.VirtPath, true)
+	switch op {
+	case addDevice:
+		var bus string
+		// The slot of block device that hotplugged to the root port 
+		// must be zero.
+		devAddr := "0x0.0x0"
+		// The vfio device BDF format is 0000:1a:00.3
+		device.BDF = "0000:" + device.BDF
+
+		bus, _, err = s.getDevSlot(device.ID)
+		if err != nil {
+			return err
+		}
+
+		if err = s.qmpMonitorCh.qmp.ExecutePCIVFIODeviceAdd(s.qmpMonitorCh.ctx, device.ID, device.BDF, devAddr, bus, ""); err != nil {
+			return err
+		}
+	case removeDevice:
+		if errDel := s.delDevSlot(device.ID); errDel != nil {
+			s.Logger().WithError(errDel).Warnf("Failed to delete device slot.")
+		}
+		if err := s.qmpMonitorCh.qmp.ExecuteDeviceDel(s.qmpMonitorCh.ctx, device.ID); err != nil {
+			return err
+		}
 	default:
-		return fmt.Errorf("Operation is not supported")
+		return fmt.Errorf("operation is not supported %d", op)
 	}
 
 	return nil
@@ -1313,9 +1692,11 @@ func (s *stratovirt) hotplugAddDevice(ctx context.Context, devInfo interface{},
 	case netDev:
 		return nil, s.hotplugNet(ctx, devInfo.(Endpoint), addDevice)
 	case blockDev:
-		return nil, s.hotplugBlk(devInfo.(*config.BlockDrive), addDevice)
+		return nil, s.hotplugBlk(ctx, devInfo.(*config.BlockDrive), addDevice)
+	case vfioDev:
+		return nil, s.hotplugVFIO(ctx, devInfo.(*config.VFIODev), addDevice)
 	default:
-		return nil, fmt.Errorf("Hotplug add device failed: unsupported device type '%v'", devType)
+		return nil, fmt.Errorf("hotplug add device failed: unsupported device type '%v'", devType)
 	}
 }
 
@@ -1327,9 +1708,11 @@ func (s *stratovirt) hotplugRemoveDevice(ctx context.Context, devInfo interface{
 	case netDev:
 		return nil, s.hotplugNet(ctx, devInfo.(Endpoint), removeDevice)
 	case blockDev:
-		return nil, s.hotplugBlk(devInfo.(*config.BlockDrive), removeDevice)
+		return nil, s.hotplugBlk(ctx, devInfo.(*config.BlockDrive), removeDevice)
+	case vfioDev:
+		return nil, s.hotplugVFIO(ctx, devInfo.(*config.VFIODev), removeDevice)
 	default:
-		return nil, fmt.Errorf("Hotplug remove device: unsupported device type '%v'", devType)
+		return nil, fmt.Errorf("hotplug remove device: unsupported device type '%v'", devType)
 	}
 }
 
@@ -1371,6 +1754,10 @@ func (s *stratovirt) capabilities(ctx context.Context) types.Capabilities {
 	var caps types.Capabilities
 	caps.SetBlockDeviceHotplugSupport()
 
+	if s.config.DisableBlockDeviceUse && s.config.SharedFS == config.VirtioFS {
+		caps.SetFsSharingSupport()
+	}
+
 	return caps
 }
 
@@ -1510,6 +1897,9 @@ func (s *stratovirt) getPids() []int {
 
 	pids = append(pids, s.state.pid)
 
+	if s.state.virtiofsPid != 0 {
+		pids = append(pids, s.state.virtiofsPid)
+	}
 	return pids
 }
 
@@ -1544,12 +1934,14 @@ func (s *stratovirt) isRateLimiterBuiltin() bool {
 func (s *stratovirt) save() (p persistapi.HypervisorState) {
 	pids := s.getPids()
 	p.Pid = pids[0]
+	p.VirtiofsdPid = s.state.virtiofsPid
 	p.Type = string(StratovirtHypervisor)
 	return
 }
 
 func (s *stratovirt) load(p persistapi.HypervisorState) {
 	s.state.pid = p.Pid
+	s.state.virtiofsPid = p.VirtiofsdPid
 }
 
 func (s *stratovirt) setSandbox(sandbox *Sandbox) {
-- 
2.20.1.windows.1

