Add a per-disk direct I/O option for the cloud-hypervisor backend

Sources/Containerization/CHHotplugProvider.swift

@@ -98,15 +98,14 @@ final class CHHotplugProvider: HotplugProvider {
         case .virtioblk:
             let letter = try allocator.allocate()
             let chId = "blk-\(id)-\(letter)"
-            let disk = CloudHypervisor.DiskConfig(
-                path: rootfs.source,
-                readonly: rootfs.options.contains("ro"),
-                id: chId,
-                imageType: .raw
-            )
 
+            // chDiskConfig applies the per-device runtime options of the mount, the
+            // same as for boot disks, and throws on an invalid value.
             let pci: CloudHypervisor.PciDeviceInfo
             do {
+                guard let disk = try rootfs.chDiskConfig(id: chId) else {
+                    throw ContainerizationError(.invalidArgument, message: "mount \(rootfs.source) is not a virtio-blk device")
+                }
                 pci = try await chCall { try await self.client.vmAddDisk(disk) }
             } catch {
                 try? allocator.release(letter)

Sources/Containerization/CHVirtualMachineInstance.swift

@@ -511,10 +511,12 @@ extension CHVirtualMachineInstance {
 
         // Disks: rootfs forced read-only at the device level; container disks
         // honor their `ro` option through chDiskConfig.
+        // Every disk also gets the per-device runtime options of its mount, such
+        // as `chDiskDirect`, through chDiskConfig.
         var disks: [CloudHypervisor.DiskConfig] = []
         for bd in bootDisks {
             let chId = bd.containerId.map { "blk-\($0)-\(bd.letter)" } ?? "rootfs"
-            if var disk = bd.mount.chDiskConfig(id: chId) {
+            if var disk = try bd.mount.chDiskConfig(id: chId) {
                 if bd.containerId == nil {
                     disk.readonly = true
                 }

Sources/Containerization/Mount+CH.swift

@@ -15,6 +15,7 @@
 //===----------------------------------------------------------------------===//
 
 import CloudHypervisor
+import ContainerizationError
 import Foundation
 
 extension Mount {
@@ -31,14 +32,59 @@ extension Mount {
     /// back to `Unknown` and silently rejects all writes — see CH's
     /// `virtio-devices/src/block.rs` "Attempting to write to sector 0 on a
     /// disk without specifying image_type" warning.
-    public func chDiskConfig(id: String) -> CloudHypervisor.DiskConfig? {
-        guard case .virtioblk = self.runtimeOptions else {
+    ///
+    /// The runtime options of the mount (`Mount.block(runtimeOptions:)`) can carry
+    /// `chDiskDirect=true` or `chDiskDirect=false`, which sets `direct` on this disk.
+    /// With `true`, cloud-hypervisor opens the image with `O_DIRECT`, so guest I/O
+    /// to the disk bypasses the host page cache.
+    /// Without the key, `direct` is not set and cloud-hypervisor keeps its default.
+    /// The option is per device because the right choice depends on the disk.
+    /// For example, many VMs can boot from the same read-only rootfs image without
+    /// the key, so the host page cache holds one copy of it for all of them.
+    /// Each VM can then attach its own read-write volume with `chDiskDirect=true`,
+    /// so the writes of the guest do not also fill the host page cache.
+    ///
+    /// Other runtime options, such as the `vz` keys of the Virtualization backend,
+    /// are ignored.
+    /// The Virtualization backend rejects keys that it does not know, so do not
+    /// set `chDiskDirect` on a mount that is also given to that backend.
+    ///
+    /// Throws if `chDiskDirect` has a value other than `true` or `false`.
+    public func chDiskConfig(id: String) throws -> CloudHypervisor.DiskConfig? {
+        guard case .virtioblk(let runtimeOptions) = self.runtimeOptions else {
             return nil
         }
+        var direct: Bool?
+        for option in runtimeOptions {
+            let split = option.split(separator: "=")
+            if split.count != 2 {
+                continue
+            }
+
+            let key = String(split[0])
+            let value = String(split[1])
+
+            switch key {
+            case "chDiskDirect":
+                switch value {
+                case "true":
+                    direct = true
+                case "false":
+                    direct = false
+                default:
+                    throw ContainerizationError(
+                        .invalidArgument,
+                        message: "unknown chDiskDirect value for virtio block device: \(value)"
+                    )
+                }
+            default:
+                continue
+            }
+        }
         return CloudHypervisor.DiskConfig(
             path: self.source,
             readonly: self.options.contains("ro"),
-            direct: nil,
+            direct: direct,
             iommu: nil,
             id: id,
             pciSegment: nil,