Add encrypted scratch space for container sandbox

.github/workflows/podvm_mkosi.yaml

@@ -153,12 +153,12 @@ jobs:
           ARCH: ${{ inputs.arch }}
 
       - name: Build mkosi debug image
-        if: ${{ inputs.debug == 'true' }}
+        if: ${{ inputs.debug == true }}
         working-directory: src/cloud-api-adaptor/podvm-mkosi
         run: make image-debug
 
       - name: Build mkosi image
-        if: ${{ inputs.debug != 'true' }}
+        if: ${{ inputs.debug != true }}
         working-directory: src/cloud-api-adaptor/podvm-mkosi
         run: make image
 

src/cloud-api-adaptor/cmd/cloud-api-adaptor/main.go

@@ -125,6 +125,7 @@ func (cfg *daemonConfig) Setup() (cmd.Starter, error) {
 		flags.StringVar(&cfg.serverConfig.Initdata, "initdata", "", "Default initdata for all Pods")
 		flags.BoolVar(&cfg.serverConfig.EnableCloudConfigVerify, "cloud-config-verify", false, "Enable cloud config verify - should use it for production")
 		flags.IntVar(&cfg.serverConfig.PeerPodsLimitPerNode, "peerpods-limit-per-node", 10, "peer pods limit per node (default=10)")
+		flags.Uint64Var(&cfg.serverConfig.RootVolumeSize, "root-volume-size", 0, "Root volume size in GB. Default is 0, which implies the default image disk size")
 
 		cloud.ParseCmd(flags)
 	})

src/cloud-api-adaptor/entrypoint.sh

@@ -77,6 +77,7 @@ azure() {
     [[ "${TAGS}" ]] && optionals+="-tags ${TAGS} " # Custom tags applied to pod vm
     [[ "${ENABLE_SECURE_BOOT}" == "true" ]] && optionals+="-enable-secure-boot "
     [[ "${USE_PUBLIC_IP}" == "true" ]] && optionals+="-use-public-ip "
+    [[ "${ROOT_VOLUME_SIZE}" ]] && optionals+="-root-volume-size ${ROOT_VOLUME_SIZE} " # OS disk size in GB
 
     set -x
     exec cloud-api-adaptor azure \

src/cloud-api-adaptor/pkg/adaptor/cloud/cloud.go

@@ -63,6 +63,7 @@ type ServerConfig struct {
 	SecureCommsPpOutbounds  string
 	SecureCommsKbsAddress   string
 	PeerPodsLimitPerNode    int
+	RootVolumeSize          uint64
 }
 
 var logger = log.New(log.Writer(), "[adaptor/cloud] ", log.LstdFlags|log.Lmsgprefix)
@@ -221,17 +222,20 @@ func (s *cloudService) CreateVM(ctx context.Context, req *pb.CreateVMRequest) (r
 	instanceType := util.GetInstanceTypeFromAnnotation(req.Annotations)
 
 	// Get Pod VM cpu and memory from annotations
-	vcpus, memory, gpus := util.GetPodvmResourcesFromAnnotation(req.Annotations)
+	resources := util.GetPodVMResourcesFromAnnotation(req.Annotations)
+
+	// Set caa-wide root volume size to resources if the storage has not been configured yet
+	if resources.Storage == 0 && s.serverConfig.RootVolumeSize > 0 {
+		resources.Storage = int64(s.serverConfig.RootVolumeSize)
+	}
 
 	// Get Pod VM image from annotations
 	image := util.GetImageFromAnnotation(req.Annotations)
 
 	// Pod VM spec
 	vmSpec := provider.InstanceTypeSpec{
 		InstanceType: instanceType,
-		VCPUs:        vcpus,
-		Memory:       memory,
-		GPUs:         gpus,
+		Resources:    resources,
 		Image:        image,
 	}
 
@@ -320,6 +324,14 @@ func (s *cloudService) CreateVM(ctx context.Context, req *pb.CreateVMRequest) (r
 		},
 	}
 
+	if s.serverConfig.RootVolumeSize > 0 {
+		// Write an empty file to indicate that we want to use available space as sandbox storage
+		cloudConfig.WriteFiles = append(cloudConfig.WriteFiles, cloudinit.WriteFile{
+			Path:    UseScratchPath,
+			Content: "",
+		})
+	}
+
 	if authJSON != nil {
 		if len(authJSON) > cloudinit.DefaultAuthfileLimit {
 			logger.Printf("Credentials file is too large to be included in cloud-config")

src/cloud-api-adaptor/pkg/paths/paths.go

@@ -8,4 +8,5 @@ const (
 	AgentCfgPath     = "/run/peerpod/agent-config.toml"
 	ForwarderCfgPath = "/run/peerpod/daemon.json"
 	UserDataPath     = "/media/cidata/user-data"
+	UseScratchPath   = "/run/peerpod/mount-scratch"
 )

src/cloud-api-adaptor/pkg/userdata/provision.go

@@ -37,7 +37,7 @@ const (
 )
 
 var logger = log.New(log.Writer(), "[userdata/provision] ", log.LstdFlags|log.Lmsgprefix)
-var WriteFilesList = []string{AACfgPath, CDHCfgPath, ForwarderCfgPath, AuthFilePath, InitDataPath}
+var WriteFilesList = []string{AACfgPath, CDHCfgPath, ForwarderCfgPath, AuthFilePath, InitDataPath, UseScratchPath}
 var InitdDataFilesList = []string{AACfgPath, CDHCfgPath, PolicyPath}
 
 type Config struct {

src/cloud-api-adaptor/pkg/util/cloud.go

@@ -5,6 +5,7 @@ import (
 	"strconv"
 	"strings"
 
+	provider "github.com/confidential-containers/cloud-api-adaptor/src/cloud-providers"
 	cri "github.com/containerd/containerd/pkg/cri/annotations"
 	hypannotations "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/pkg/annotations"
 )
@@ -44,48 +45,48 @@ func GetImageFromAnnotation(annotations map[string]string) string {
 }
 
 // Method to get vCPU, memory and gpus from annotations
-func GetPodvmResourcesFromAnnotation(annotations map[string]string) (int64, int64, int64) {
-
-	var vcpuInt, memoryInt, gpuInt int64
+func GetPodVMResourcesFromAnnotation(annotations map[string]string) provider.PodVMResources {
+	var vcpus, gpus, memory int64
 	var err error
 
 	vcpu, ok := annotations[hypannotations.DefaultVCPUs]
 	if ok {
-		vcpuInt, err = strconv.ParseInt(vcpu, 10, 64)
+		vcpus, err = strconv.ParseInt(vcpu, 10, 64)
 		if err != nil {
 			fmt.Printf("Error converting vcpu to int64. Defaulting to 0: %v\n", err)
-			vcpuInt = 0
+			vcpus = 0
 		}
 	} else {
-		vcpuInt = 0
+		vcpus = 0
 	}
 
-	memory, ok := annotations[hypannotations.DefaultMemory]
+	mem, ok := annotations[hypannotations.DefaultMemory]
 	if ok {
 		// Use strconv.ParseInt to convert string to int64
-		memoryInt, err = strconv.ParseInt(memory, 10, 64)
+		memory, err = strconv.ParseInt(mem, 10, 64)
 		if err != nil {
 			fmt.Printf("Error converting memory to int64. Defaulting to 0: %v\n", err)
-			memoryInt = 0
+			memory = 0
 		}
 
 	} else {
-		memoryInt = 0
+		memory = 0
 	}
 
 	gpu, ok := annotations[hypannotations.DefaultGPUs]
 	if ok {
-		gpuInt, err = strconv.ParseInt(gpu, 10, 64)
+		gpus, err = strconv.ParseInt(gpu, 10, 64)
 		if err != nil {
 			fmt.Printf("Error converting gpu to int64. Defaulting to 0: %v\n", err)
-			gpuInt = 0
+			gpus = 0
 		}
 	} else {
-		gpuInt = 0
+		gpus = 0
 	}
 
-	// Return vCPU, memory and GPU
-	return vcpuInt, memoryInt, gpuInt
+	storage := int64(0)
+
+	return provider.PodVMResources{VCPUs: vcpus, Memory: memory, GPUs: gpus, Storage: storage}
 }
 
 // Method to get initdata from annotation

src/cloud-api-adaptor/pkg/util/cloud_test.go

@@ -98,7 +98,8 @@ func TestGetPodvmResourcesFromAnnotation(t *testing.T) {
 	}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
-			got, got1, got2 := GetPodvmResourcesFromAnnotation(tt.args.annotations)
+			r := GetPodVMResourcesFromAnnotation(tt.args.annotations)
+			got, got1, got2 := r.VCPUs, r.Memory, r.GPUs
 			if got != tt.want {
 				t.Errorf("GetPodvmResourcesFromAnnotation() got = %v, want %v", got, tt.want)
 			}

src/cloud-api-adaptor/podvm-mkosi/Dockerfile.mkosi

@@ -21,6 +21,7 @@ COPY mkosi.presets /image/mkosi.presets
 COPY mkosi.profiles /image/mkosi.profiles
 COPY mkosi.skeleton /image/mkosi.skeleton
 COPY mkosi.skeleton-debug /image/mkosi.skeleton-debug
+COPY mkosi.skeleton-rootfs /image/mkosi.skeleton-rootfs
 COPY mkosi.workspace /image/mkosi.workspace
 COPY resources /image/resources
 COPY mkosi.conf /image/mkosi.conf

src/cloud-api-adaptor/podvm-mkosi/Makefile

@@ -88,6 +88,7 @@ else
 		--allow security.insecure \
 		.
 	qemu-img convert -f raw -O qcow2 build/system.raw build/podvm-$(PODVM_DISTRO)-$(ARCH).qcow2
+	qemu-img resize build/podvm-$(PODVM_DISTRO)-$(ARCH).qcow2 +100M
 endif
 
 PHONY: image-debug
@@ -112,6 +113,7 @@ else
 		--allow security.insecure \
 		.
 	qemu-img convert -f raw -O qcow2 build/system.raw build/podvm-$(PODVM_DISTRO)-$(ARCH).qcow2
+	qemu-img resize build/podvm-$(PODVM_DISTRO)-$(ARCH).qcow2 +100M
 endif
 
 PHONY: image-container

src/cloud-api-adaptor/podvm-mkosi/mkosi.presets/system/mkosi.conf.d/fedora.conf

@@ -7,7 +7,7 @@ Release=39
 
 [Content]
 CleanPackageMetadata=true
-SkeletonTrees=../../resources/binaries-tree
+SkeletonTrees=../../mkosi.skeleton-rootfs,../../resources/binaries-tree,
 Packages=
     kernel
     kernel-core
@@ -23,6 +23,7 @@ Packages=
     iptables
     afterburn
     neofetch
+    e2fsprogs
 
 RemoveFiles=/etc/issue
 RemoveFiles=/etc/issue.net

src/cloud-api-adaptor/podvm-mkosi/mkosi.skeleton-rootfs/etc/crypttab

@@ -0,0 +1 @@
+scratch /dev/disk/by-label/scratch - try-empty-password

src/cloud-api-adaptor/podvm-mkosi/mkosi.skeleton-rootfs/usr/lib/systemd/system/kata-agent.service.d/10-override.conf

@@ -0,0 +1,2 @@
+[Service]
+ExecStartPre=sh -c '[[ ! -f /run/peerpod/mount-scratch ]] || mount /dev/mapper/scratch /run/kata-containers'

src/cloud-api-adaptor/podvm-mkosi/mkosi.skeleton/usr/lib/repart.d/30-scratch.conf

@@ -0,0 +1,5 @@
+[Partition]
+Type=linux-generic
+Label=scratch
+Encrypt=key-file
+Format=ext4

src/cloud-providers/aws/provider.go

@@ -370,8 +370,10 @@ func (p *awsProvider) updateInstanceTypeSpecList() error {
 		if err != nil {
 			return err
 		}
+		resources := provider.NewPodVMResources(vcpus, memory)
+		resources.GPUs = gpuCount
 		instanceTypeSpecList = append(instanceTypeSpecList,
-			provider.InstanceTypeSpec{InstanceType: instanceType, VCPUs: vcpus, Memory: memory, GPUs: gpuCount})
+			provider.InstanceTypeSpec{InstanceType: instanceType, Resources: resources})
 	}
 
 	// Sort the instanceTypeSpecList and update the serviceConfig

src/cloud-providers/azure/provider.go

@@ -11,7 +11,6 @@ import (
 	"log"
 	"net/netip"
 	"os"
-	"path/filepath"
 	"regexp"
 	"strings"
 
@@ -35,24 +34,44 @@ const (
 type azureProvider struct {
 	azureClient   azcore.TokenCredential
 	serviceConfig *Config
+	sshKey        armcompute.SSHPublicKey
 }
 
 func NewProvider(config *Config) (provider.Provider, error) {
 
 	logger.Printf("azure config %+v", config.Redact())
 
-	// Clean the config.SSHKeyPath to avoid bad paths
-	config.SSHKeyPath = filepath.Clean(config.SSHKeyPath)
-
 	azureClient, err := NewAzureClient(*config)
 	if err != nil {
 		logger.Printf("creating azure client: %v", err)
 		return nil, err
 	}
 
+	// The podvm disk doesn't support sshd logins. keys can be baked
+	// into a debug-image. The ARM api mandates a pubkey, though.
+	sshPublicKeyPath := os.ExpandEnv(config.SSHKeyPath)
+	var pubkeyBytes []byte
+	if _, err := os.Stat(sshPublicKeyPath); err == nil {
+		pubkeyBytes, err = os.ReadFile(sshPublicKeyPath)
+		if err != nil {
+			err = fmt.Errorf("reading ssh public key file: %w", err)
+			logger.Printf("%v", err)
+			return nil, err
+		}
+	} else {
+		err = fmt.Errorf("ssh public key: %w", err)
+		logger.Printf("%v", err)
+		return nil, err
+	}
+	dummySSHKey := armcompute.SSHPublicKey{
+		Path:    to.Ptr(fmt.Sprintf("/home/%s/.ssh/authorized_keys", config.SSHUserName)),
+		KeyData: to.Ptr(string(pubkeyBytes)),
+	}
+
 	provider := &azureProvider{
 		azureClient:   azureClient,
 		serviceConfig: config,
+		sshKey:        dummySSHKey,
 	}
 
 	if err = provider.updateInstanceSizeSpecList(); err != nil {
@@ -218,28 +237,12 @@ func (p *azureProvider) CreateInstance(ctx context.Context, podName, sandboxID s
 	diskName := fmt.Sprintf("%s-disk", instanceName)
 	nicName := fmt.Sprintf("%s-net", instanceName)
 
-	// require ssh key for authentication on linux
-	sshPublicKeyPath := os.ExpandEnv(p.serviceConfig.SSHKeyPath)
-	var sshBytes []byte
-	if _, err := os.Stat(sshPublicKeyPath); err == nil {
-		sshBytes, err = os.ReadFile(sshPublicKeyPath)
-		if err != nil {
-			err = fmt.Errorf("reading ssh public key file: %w", err)
-			logger.Printf("%v", err)
-			return nil, err
-		}
-	} else {
-		err = fmt.Errorf("ssh public key: %w", err)
-		logger.Printf("%v", err)
-		return nil, err
-	}
-
 	if spec.Image != "" {
 		logger.Printf("Choosing %s from annotation as the Azure Image for the PodVM image", spec.Image)
 		p.serviceConfig.ImageId = spec.Image
 	}
 
-	vmParameters, err := p.getVMParameters(instanceSize, diskName, cloudConfigData, sshBytes, instanceName, nicName)
+	vmParameters, err := p.getVMParameters(instanceSize, diskName, cloudConfigData, instanceName, nicName, spec.Resources.Storage)
 	if err != nil {
 		return nil, err
 	}
@@ -350,7 +353,10 @@ func (p *azureProvider) updateInstanceSizeSpecList() error {
 		}
 		for _, vmSize := range nextResult.VirtualMachineSizeListResult.Value {
 			if util.Contains(instanceSizes, *vmSize.Name) {
-				instanceSizeSpecList = append(instanceSizeSpecList, provider.InstanceTypeSpec{InstanceType: *vmSize.Name, VCPUs: int64(*vmSize.NumberOfCores), Memory: int64(*vmSize.MemoryInMB)})
+				vcpus, memory := int64(*vmSize.NumberOfCores), int64(*vmSize.MemoryInMB)
+				resources := provider.NewPodVMResources(vcpus, memory)
+				instanceSizeSpec := provider.InstanceTypeSpec{InstanceType: *vmSize.Name, Resources: resources}
+				instanceSizeSpecList = append(instanceSizeSpecList, instanceSizeSpec)
 			}
 		}
 	}
@@ -371,7 +377,7 @@ func (p *azureProvider) getResourceTags() map[string]*string {
 	return tags
 }
 
-func (p *azureProvider) getVMParameters(instanceSize, diskName, cloudConfig string, sshBytes []byte, instanceName, nicName string) (*armcompute.VirtualMachine, error) {
+func (p *azureProvider) getVMParameters(instanceSize, diskName, cloudConfig, instanceName, nicName string, diskSize int64) (*armcompute.VirtualMachine, error) {
 	userDataB64 := base64.StdEncoding.EncodeToString([]byte(cloudConfig))
 
 	// Azure limits the base64 encrypted userData to 64KB.
@@ -416,6 +422,18 @@ func (p *azureProvider) getVMParameters(instanceSize, diskName, cloudConfig stri
 
 	networkConfig := p.buildNetworkConfig(nicName)
 
+	osDisk := armcompute.OSDisk{
+		Name:         to.Ptr(diskName),
+		CreateOption: to.Ptr(armcompute.DiskCreateOptionTypesFromImage),
+		Caching:      to.Ptr(armcompute.CachingTypesReadWrite),
+		DeleteOption: to.Ptr(armcompute.DiskDeleteOptionTypesDelete),
+		ManagedDisk:  managedDiskParams,
+	}
+
+	if diskSize > 0 {
+		osDisk.DiskSizeGB = to.Ptr(int32(diskSize))
+	}
+
 	vmParameters := armcompute.VirtualMachine{
 		Location: to.Ptr(p.serviceConfig.Region),
 		Properties: &armcompute.VirtualMachineProperties{
@@ -424,25 +442,15 @@ func (p *azureProvider) getVMParameters(instanceSize, diskName, cloudConfig stri
 			},
 			StorageProfile: &armcompute.StorageProfile{
 				ImageReference: imgRef,
-				OSDisk: &armcompute.OSDisk{
-					Name:         to.Ptr(diskName),
-					CreateOption: to.Ptr(armcompute.DiskCreateOptionTypesFromImage),
-					Caching:      to.Ptr(armcompute.CachingTypesReadWrite),
-					DeleteOption: to.Ptr(armcompute.DiskDeleteOptionTypesDelete),
-					ManagedDisk:  managedDiskParams,
-				},
+				OSDisk:         &osDisk,
 			},
 			OSProfile: &armcompute.OSProfile{
 				AdminUsername: to.Ptr(p.serviceConfig.SSHUserName),
 				ComputerName:  to.Ptr(instanceName),
 				LinuxConfiguration: &armcompute.LinuxConfiguration{
 					DisablePasswordAuthentication: to.Ptr(true),
-					//TBD: replace with a suitable mechanism to use precreated SSH key
 					SSH: &armcompute.SSHConfiguration{
-						PublicKeys: []*armcompute.SSHPublicKey{{
-							Path:    to.Ptr(fmt.Sprintf("/home/%s/.ssh/authorized_keys", p.serviceConfig.SSHUserName)),
-							KeyData: to.Ptr(string(sshBytes)),
-						}},
+						PublicKeys: []*armcompute.SSHPublicKey{&p.sshKey},
 					},
 				},
 			},