platform: add SlimVM platform

SlimVM is a lightweight VMX-based platform for gVisor that accelerates sentry system calls by eliminating unnecessary user/kernel context switches.

In a traditional gVisor + KVM setup, every guest syscall triggers a full VM exit to userspace, then re-enters the kernel via a regular syscall:

  gr0 -> hr0 (vm_exit) -> hr3 (ioctl return) -> hr0 (re-invoke syscall)

SlimVM eliminates this overhead. When the guest executes a VMCALL, the VM exits into VMX root mode where SlimVM directly invokes the host kernel's syscall handler, then immediately resumes the guest:

  gr0 -> hr0 (vmcall + direct function call)

SlimVM requires a companion kernel module that provides the /dev/slimvm device interface. See https://github.com/antgroup/slimvm for the kernel module and full documentation.

A sentry->host getpid microbenchmark (Xeon 6982P-C, host kernel 5.10.134)
shows the effect:

| path   | ns/op (mean of 10) | vs base |
| ------ | -----------------: | ------: |
| base   |               82.5 |   1.0x |
| slimvm |              420.9 |   5.1x |
| kvm    |             3566.1 |  43.2x  |

slimvm is ~8.5x faster than the kvm platform on this path. See the issue
for details.

Updates: #13330

Co-developed-by: Chenggang <chenggang.qcg@antfin.com>
Co-developed-by: Robin Luk <lubin.lu@antgroup.com>
Co-developed-by: Tiwei Bie <tiwei.btw@antgroup.com>
Co-developed-by: Jianfeng Tan <henry.tjf@antgroup.com>
Co-developed-by: liushi.ls <liushi.ls@antfin.com>
Co-developed-by: Yong He <chenglang.hy@antgroup.com>
Co-developed-by: Aaron Lu <ziqian.lzq@antfin.com>
Co-developed-by: chris.zn <chris.zn@alibaba-inc.com>
Co-developed-by: Lai Jiangshan <jiangshan.ljs@antfin.com>
Co-developed-by: Lingfu <yupeng.chenyp@alibaba-inc.com>
Co-developed-by: Min Le <lemin.lm@antgroup.com>
Co-developed-by: Quan Xu <wutu.xq@alibaba-inc.com>
Co-developed-by: Zhang Haoyu <zhanghaoyu.zhy@alibaba-inc.com>
Co-developed-by: Dawei Shen <shendawei.sdw@antgroup.com>
FUTURE_COPYBARA_INTEGRATE_REVIEW=#13331 from tianyuzhou95:albert/slimvm-pr 86eddc2
PiperOrigin-RevId: 940094281

Author: tianyuzhou95

.buildkite/pipeline.yaml

@@ -310,7 +310,7 @@ steps:
   # All system call tests.
   - <<: *common
     label: ":toolbox: System call tests (AMD64)"
-    command: make BAZEL_OPTIONS=--test_tag_filters=-allsave syscall-tests
+    command: make BAZEL_OPTIONS=--test_tag_filters=-allsave,-runsc_slimvm syscall-tests
     parallelism: 20
     agents:
       <<: *platform_specific_agents

pkg/cpuid/cpuid_amd64.go

@@ -192,7 +192,24 @@ func (fs FeatureSet) VirtualAddressBits() uint32 {
 //go:nosplit
 func (fs FeatureSet) PhysicalAddressBits() uint32 {
 	ax, _, _, _ := fs.query(addressSizes)
-	return ax & 0xff
+	physBits := ax & 0xff
+	if !fs.AMD() {
+		return physBits
+	}
+
+	maxExtended, _, _, _ := fs.query(extendedFunctionInfo)
+	if maxExtended < uint32(amdMemoryEncryptionInfo) {
+		return physBits
+	}
+
+	memEncAX, memEncBX, _, _ := fs.query(amdMemoryEncryptionInfo)
+	if memEncAX&amdMemoryEncryptionFeatureMask == 0 {
+		return physBits
+	}
+	// AMD memory encryption reduces usable physical address width by the
+	// CPUID-reported amount. Match Linux's
+	// arch/x86/kernel/cpu/amd.c:early_detect_mem_encrypt().
+	return physBits - ((memEncBX >> amdPhysAddrReductionShift) & amdPhysAddrReductionMask)
 }
 
 // CacheType describes the type of a cache, as returned in eax[4:0] for eax=4.

pkg/cpuid/native_amd64.go

@@ -63,15 +63,25 @@ const xSaveInfoNumLeaves = 64 // Maximum number of xSaveInfo leaves.
 
 // The "extended" functions.
 const (
-	extendedStart         cpuidFunction = 0x80000000
-	extendedFunctionInfo  cpuidFunction = extendedStart + 0 // Returns highest available extended function in eax.
-	extendedFeatures                    = extendedStart + 1 // Returns some extended feature bits in edx and ecx.
-	processorBrandString2               = extendedStart + 2 // Processor Name String Identifier.
-	processorBrandString3               = extendedStart + 3 // Processor Name String Identifier.
-	processorBrandString4               = extendedStart + 4 // Processor Name String Identifier.
-	l1CacheAndTLBInfo                   = extendedStart + 5 // Returns L2 cache information.
-	l2CacheInfo                         = extendedStart + 6 // Returns L2 cache information.
-	addressSizes                        = extendedStart + 8 // Physical and virtual address sizes.
+	extendedStart           cpuidFunction = 0x80000000
+	extendedFunctionInfo    cpuidFunction = extendedStart + 0  // Returns highest available extended function in eax.
+	extendedFeatures                      = extendedStart + 1  // Returns some extended feature bits in edx and ecx.
+	processorBrandString2                 = extendedStart + 2  // Processor Name String Identifier.
+	processorBrandString3                 = extendedStart + 3  // Processor Name String Identifier.
+	processorBrandString4                 = extendedStart + 4  // Processor Name String Identifier.
+	l1CacheAndTLBInfo                     = extendedStart + 5  // Returns L2 cache information.
+	l2CacheInfo                           = extendedStart + 6  // Returns L2 cache information.
+	addressSizes                          = extendedStart + 8  // Physical and virtual address sizes.
+	amdMemoryEncryptionInfo               = extendedStart + 31 // AMD memory encryption information.
+)
+
+// AMD-defined memory encryption feature bits and fields.
+const (
+	amdSecureMemoryEncryption        = 1 << 0
+	amdSecureEncryptedVirtualization = 1 << 1
+	amdMemoryEncryptionFeatureMask   = amdSecureMemoryEncryption | amdSecureEncryptedVirtualization
+	amdPhysAddrReductionShift        = 6
+	amdPhysAddrReductionMask         = 0x3f
 )
 
 var allowedBasicFunctions = [...]bool{
@@ -84,14 +94,15 @@ var allowedBasicFunctions = [...]bool{
 }
 
 var allowedExtendedFunctions = [...]bool{
-	extendedFunctionInfo - extendedStart:  true,
-	extendedFeatures - extendedStart:      true,
-	addressSizes - extendedStart:          true,
-	processorBrandString2 - extendedStart: true,
-	processorBrandString3 - extendedStart: true,
-	processorBrandString4 - extendedStart: true,
-	l1CacheAndTLBInfo - extendedStart:     true,
-	l2CacheInfo - extendedStart:           true,
+	extendedFunctionInfo - extendedStart:    true,
+	extendedFeatures - extendedStart:        true,
+	addressSizes - extendedStart:            true,
+	processorBrandString2 - extendedStart:   true,
+	processorBrandString3 - extendedStart:   true,
+	processorBrandString4 - extendedStart:   true,
+	l1CacheAndTLBInfo - extendedStart:       true,
+	l2CacheInfo - extendedStart:             true,
+	amdMemoryEncryptionInfo - extendedStart: true,
 }
 
 // Function executes a CPUID function.
@@ -119,7 +130,7 @@ func (i *In) normalize() {
 	switch cpuidFunction(i.Eax) {
 	case vendorID, featureInfo, intelCacheDescriptors, extendedFunctionInfo, extendedFeatures:
 		i.Ecx = 0 // Ignore.
-	case processorBrandString2, processorBrandString3, processorBrandString4, l1CacheAndTLBInfo, l2CacheInfo:
+	case processorBrandString2, processorBrandString3, processorBrandString4, l1CacheAndTLBInfo, l2CacheInfo, amdMemoryEncryptionInfo:
 		i.Ecx = 0 // Ignore.
 	case intelDeterministicCacheParams, extendedFeatureInfo:
 		// Preserve i.Ecx.

pkg/hostarch/hostarch_arm64.go

@@ -29,9 +29,15 @@ const (
 	// HugePageSize is the system huge page size.
 	HugePageSize = 1 << HugePageShift
 
+	// JumboPageSize is the 1GB jumbo page size.
+	JumboPageSize = 1 << JumboPageShift
+
 	// CacheLineSize is the size of the cache line.
 	CacheLineSize = 1 << CacheLineShift
 
+	// JumboPageShift is the binary log of jumbo page whose size is 1GB.
+	JumboPageShift = 30
+
 	// CacheLineShift is the binary log of the cache line size.
 	CacheLineShift = 6
 )

pkg/hostarch/hostarch_x86.go

@@ -26,6 +26,9 @@ const (
 	// HugePageSize is the system huge page size.
 	HugePageSize = 1 << HugePageShift
 
+	// JumboPageSize is the 1GB jumbo page size.
+	JumboPageSize = 1 << JumboPageShift
+
 	// CacheLineSize is the size of the cache line.
 	CacheLineSize = 1 << CacheLineShift
 
@@ -35,6 +38,9 @@ const (
 	// HugePageShift is the binary log of the system huge page size.
 	HugePageShift = 21
 
+	// JumboPageShift is the binary log of jumbo page whose size is 1GB.
+	JumboPageShift = 30
+
 	// CacheLineShift is the binary log of the cache line size.
 	CacheLineShift = 6
 )

pkg/hostarch/sizes_util.go

@@ -11,6 +11,7 @@ const (
 	PageMask      = PageSize - 1
 	HugePageMask  = HugePageSize - 1
 	CacheLineMask = CacheLineSize - 1
+	JumboPageMask = ^uintptr(JumboPageSize - 1)
 )
 
 type bytecount interface {

pkg/ring0/defs.go

@@ -20,6 +20,18 @@ import (
 	"gvisor.dev/gvisor/pkg/sentry/arch/fpu"
 )
 
+const (
+	// CPUIntel is Intel CPU.
+	CPUIntel uint64 = iota
+
+	// CPUAMD is AMD (and compatible) CPU.
+	CPUAMD
+)
+
+var (
+	CPUVendor uint64
+)
+
 // Kernel is a global kernel object.
 //
 // This contains global state, shared by multiple CPUs.

pkg/ring0/defs_amd64.go

@@ -108,6 +108,9 @@ type kernelEntry struct {
 	// kernelCR3 is the cr3 used for sentry kernel.
 	kernelCR3 uintptr
 
+	// whether enable VMCALL
+	enableVMCALL uint64
+
 	// gdt is the CPU's descriptor table.
 	gdt descriptorTable
 
@@ -180,6 +183,14 @@ func (c *CPU) FaultAddr() uintptr {
 	return c.faultAddr
 }
 
+func (c *CPU) EnableVMCALL() {
+	c.enableVMCALL = 1
+}
+
+func (c *CPU) DisableVMCALL() {
+	c.enableVMCALL = 0
+}
+
 // SwitchArchOpts are embedded in SwitchOpts.
 type SwitchArchOpts struct {
 	// UserPCID indicates that the application PCID to be used on switch,

pkg/ring0/entry_amd64.s

@@ -33,6 +33,7 @@
 #define ENTRY_STACK_TOP  264 // +checkoffset . kernelEntry.stackTop
 #define ENTRY_CPU_SELF   272 // +checkoffset . kernelEntry.cpuSelf
 #define ENTRY_KERNEL_CR3 280 // +checkoffset . kernelEntry.kernelCR3
+#define ENTRY_ENABLE_VMCALL 288 // +checkoffset . kernelEntry.enableVMCALL
 
 // Bits.
 #define _RFLAGS_IF    512  // +checkconst . _RFLAGS_IF
@@ -64,6 +65,12 @@
 #define SyscallInt80               128 // +checkconst . SyscallInt80
 #define Syscall                    256 // +checkconst . Syscall
 
+#define SyscallExit       60         // +checkconst . SyscallExit
+#define SyscallExitGroup  231        // +checkconst . SyscallExitGroup
+#define SyscallRedPill    4294967295 // +checkconst . SyscallRedPill
+
+#define CPUIntel        0 // +checkconst . CPUIntel
+
 #define PTRACE_R15      0   // +checkoffset linux PtraceRegs.R15
 #define PTRACE_R14      8   // +checkoffset linux PtraceRegs.R14
 #define PTRACE_R13      16  // +checkoffset linux PtraceRegs.R13
@@ -160,6 +167,15 @@
 #define LOAD_KERNEL_STACK(entry) \
 	MOVQ ENTRY_STACK_TOP(entry), SP;
 
+// VMCALL do vmcall/vmmcal instruction
+#define VMCALL() \
+	CMPQ ·CPUVendor(SB), $CPUIntel; \
+	JE 2(PC); \
+	JMP 5(PC); \ // vmmcall and vmcall will be treated as 3 independent instructions
+	BYTE $0x0F; BYTE $0x01; BYTE $0xC1; \
+	JMP 4(PC); \ // vmmcall and vmcall will be treated as 3 independent instructions
+	BYTE $0x0F; BYTE $0x01; BYTE $0xD9;
+
 // ADDR_OF_FUNC defines a function named 'name' that returns the address of
 // 'symbol'.
 #define ADDR_OF_FUNC(name, symbol) \
@@ -488,6 +504,45 @@ sysenter_skip_gs:
 	RET
 
 kernel:
+	// Handle any syscalls from GR0 in HR3 when EnableVMCALL is false.
+	// Currently there are 2 use cases:
+	// 1. Using KVM platform.
+	// 2. Upgrading SlimVM platform. This is one such method to return M to
+	//    user mode (HR3) for upgrading platform.
+	CMPQ ENTRY_ENABLE_VMCALL(GS), $0
+	JE hr3_do_syscall
+
+	CMPQ AX, $SyscallRedPill
+	JE hr3_do_syscall
+
+	CMPQ AX, $SyscallExit
+	JE hr3_do_syscall
+
+	CMPQ AX, $SyscallExitGroup
+	JE hr3_do_syscall
+
+vmcall:
+	// handle syscall from GR0 in host kernel
+	// copy from "handle system calls from G0" part of __dune_syscall in libdune/dune.S
+	PUSHQ R11
+	POPFQ
+
+	CMPQ AX, $158 // arch_prctl syscall
+	JNE 3(PC)
+	CMPQ DI, $0x1002 //ARCH_SET_FS
+	JE arch_prctl_vmcall
+
+	VMCALL()
+	JMP *CX
+
+arch_prctl_vmcall:
+	VMCALL()
+	CMPQ AX, $0
+	JNE 2(PC)
+	MOVQ SI, CPU_REGISTERS+PTRACE_FS_BASE(GS)
+	JMP *CX
+
+hr3_do_syscall:
 	// We can't restore the original stack, but we can access the registers
 	// in the CPU state directly. No need for temporary juggling.
 	MOVQ AX,  ENTRY_SCRATCH0(GS)

pkg/ring0/kernel_amd64.go

@@ -324,3 +324,30 @@ func startGo(c *CPU) {
 func ReadCR2() uintptr {
 	return readCR2()
 }
+
+//go:noinline
+//go:nosplit
+func (c *CPU) PrefaultIDT() uint32 {
+	return c.kernel.globalIDT[0].bits[0] + c.kernel.globalIDT[_NR_INTERRUPTS-1].bits[3]
+}
+
+// SetCPUIDFaulting sets CPUID faulting per the boolean value.
+//
+// True is returned if faulting could be set.
+//
+//go:nosplit
+func SetCPUIDFaulting(on bool) bool {
+	// Per the SDM (Vol 3, Table 2-43), PLATFORM_INFO bit 31 denotes support
+	// for CPUID faulting, and we enable and disable via the MISC_FEATURES MSR.
+	if rdmsr(_MSR_PLATFORM_INFO)&_PLATFORM_INFO_CPUID_FAULT != 0 {
+		features := rdmsr(_MSR_MISC_FEATURES)
+		if on {
+			features |= _MISC_FEATURE_CPUID_TRAP
+		} else {
+			features &^= _MISC_FEATURE_CPUID_TRAP
+		}
+		wrmsr(_MSR_MISC_FEATURES, features)
+		return true // Setting successful.
+	}
+	return false
+}