Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 2 additions & 1 deletion crates/bcvk-qemu/src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -54,7 +54,8 @@ pub use credentials::{

pub use qemu::{
qemu_version, BootMode, DiskFormat, DisplayMode, MachineType, NetworkMode, QemuConfig,
ResourceLimits, RunningQemu, VirtioBlkDevice, VirtioSerialOut, VirtiofsMount, VHOST_VSOCK,
ResourceLimits, RunningQemu, TpmConfig, VirtioBlkDevice, VirtioSerialOut, VirtiofsMount,
VHOST_VSOCK,
};

pub use virtiofsd::{spawn_virtiofsd_async, validate_virtiofsd_config, VirtiofsConfig};
154 changes: 123 additions & 31 deletions crates/bcvk-qemu/src/qemu.rs
Original file line number Diff line number Diff line change
Expand Up @@ -186,6 +186,15 @@ pub enum BootMode {
},
}

/// A TPM 2.0 emulated by swtpm.
#[derive(Debug, Clone)]
pub struct TpmConfig {
/// Directory holding the TPM's persistent state.
pub state_dir: Utf8PathBuf,
/// Socket QEMU connects to.
pub socket_path: Utf8PathBuf,
}

/// Complete QEMU VM configuration with builder pattern.
#[derive(Default, Debug)]
pub struct QemuConfig {
Expand Down Expand Up @@ -231,6 +240,9 @@ pub struct QemuConfig {

/// fw_cfg entries for passing config files to the guest
fw_cfg_entries: Vec<(String, Utf8PathBuf)>,

/// TPM 2.0 to attach.
tpm: Option<TpmConfig>,
}

impl QemuConfig {
Expand Down Expand Up @@ -306,6 +318,10 @@ impl QemuConfig {
return Err(eyre!("Memory too high: {}MB (maximum 1TB)", self.memory_mb));
}

if self.tpm.is_some() {
tpm_device()?;
}

// CPU validation
if self.vcpus == 0 {
return Err(eyre!("vCPU count must be at least 1"));
Expand Down Expand Up @@ -393,6 +409,12 @@ impl QemuConfig {
self
}

/// Attach a TPM 2.0 emulated by swtpm.
pub fn set_tpm(&mut self, tpm: TpmConfig) -> &mut Self {
self.tpm = Some(tpm);
self
}

/// Set the main virtiofs configuration for the root filesystem.
pub fn set_main_virtiofs(&mut self, config: VirtiofsConfig) -> &mut Self {
self.main_virtiofs_config = Some(config);
Expand Down Expand Up @@ -815,6 +837,17 @@ fn spawn(
cmd.args(["-fw_cfg", &format!("name={},file={}", name, file_path)]);
}

if let Some(tpm) = &config.tpm {
cmd.args([
"-chardev",
&format!("socket,id=chrtpm,path={}", tpm.socket_path),
"-tpmdev",
"emulator,id=tpm0,chardev=chrtpm",
"-device",
&format!("{},tpmdev=tpm0", tpm_device()?),
]);
}

// Configure stdio based on display mode
match &config.display_mode {
DisplayMode::Console => {
Expand All @@ -837,6 +870,79 @@ fn spawn(
cmd.spawn().context("Failed to spawn QEMU")
}

/// Wait until the process `name` creates `socket_path`. Fails if
/// `process` exits first or `timeout` passes.
async fn wait_for_socket(

Copy link
Copy Markdown
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

For virtiofsd at least we should have always been using --fd to avoid any polling at all.

Something to improve in a followup

name: &str,
socket_path: &Utf8Path,
process: &mut Pin<Box<dyn Future<Output = std::io::Result<Output>>>>,
timeout: Duration,
) -> Result<()> {
let query_exists = async {
while !socket_path.exists() {
tokio::time::sleep(Duration::from_millis(100)).await;
}
};
debug!("Waiting for socket at {socket_path}");
tokio::select! {
output = process => {
tracing::trace!("{name} exited");
let output = output?;
let status = output.status;
let stderr = String::from_utf8_lossy(&output.stderr);
Err(eyre!(
"{name} failed to start for socket {socket_path}\nExit status: {status:?}\nOutput: {stderr}"
))
}
_ = tokio::time::sleep(timeout) => {
Err(eyre!("timed out waiting for {name} socket {socket_path} to be created (waited {timeout:?})"))
}
_ = query_exists => {
debug!("{name} socket created: {socket_path}");
Ok(())
}
}
}

/// The QEMU TPM device model for the host architecture.
fn tpm_device() -> Result<&'static str> {
match std::env::consts::ARCH {
"x86_64" => Ok("tpm-tis"),
"aarch64" => Ok("tpm-tis-device"),
arch => Err(eyre!("TPM emulation is not supported on {arch}")),
}
}

/// Start swtpm for `tpm` and wait until it listens on its socket.
async fn spawn_swtpm(
tpm: &TpmConfig,
) -> Result<Pin<Box<dyn Future<Output = std::io::Result<Output>>>>> {
std::fs::create_dir_all(&tpm.state_dir)
.with_context(|| format!("Creating TPM state directory {}", tpm.state_dir))?;
let mut cmd = Command::new("swtpm");
cmd.args(["socket", "--tpm2", "--tpmstate"])
.arg(format!("dir={}", tpm.state_dir))
.arg("--ctrl")
.arg(format!("type=unixio,path={}", tpm.socket_path))
.stdout(Stdio::null())
.stderr(Stdio::piped())
.lifecycle_bind_to_parent_thread();
let mut cmd = tokio::process::Command::from(cmd);
cmd.kill_on_drop(true);
debug!("{cmd:?}");
let child = cmd.spawn().context("Failed to spawn swtpm")?;
let mut process: Pin<Box<dyn Future<Output = std::io::Result<Output>>>> =
Box::pin(child.wait_with_output());
wait_for_socket(
"swtpm",
&tpm.socket_path,
&mut process,
Duration::from_secs(10),
)
.await?;
Ok(process)
}

struct VsockCopier {
port: VsockAddr,
#[allow(dead_code)]
Expand All @@ -860,6 +966,9 @@ pub struct RunningQemu {
pub virtiofsd_processes: Vec<Pin<Box<dyn Future<Output = std::io::Result<Output>>>>>,
#[allow(dead_code)]
sd_notification: Option<VsockCopier>,
/// The swtpm process backing the VM's TPM.
#[allow(dead_code)]
swtpm_process: Option<Pin<Box<dyn Future<Output = std::io::Result<Output>>>>>,
}

impl std::fmt::Debug for RunningQemu {
Expand All @@ -877,6 +986,12 @@ impl std::fmt::Debug for RunningQemu {
impl RunningQemu {
/// Spawn QEMU with the given configuration.
pub async fn spawn(mut config: QemuConfig) -> Result<Self> {
config.validate()?;
let swtpm_process = match &config.tpm {
Some(tpm) => Some(spawn_swtpm(tpm).await?),
None => None,
};

// Spawn all virtiofsd processes first
let mut awaiting_virtiofsd = Vec::new();
let virtiofsd_configs = config
Expand All @@ -891,40 +1006,16 @@ impl RunningQemu {
// Wait for all virtiofsd to be ready
let mut virtiofsd_processes = Vec::new();
while let Some((proc, socket_path)) = awaiting_virtiofsd.pop() {
let socket_path = &socket_path;
let query_exists = async move {
loop {
if socket_path.exists() {
break;
}
tokio::time::sleep(Duration::from_millis(100)).await;
}
};
tokio::pin!(query_exists);
let timeout_val = Duration::from_secs(60);
let timeout = tokio::time::sleep(timeout_val);
tokio::pin!(timeout);
debug!("Waiting for socket at {socket_path}");
let mut output: Pin<Box<dyn Future<Output = std::io::Result<Output>>>> =
Box::pin(proc.wait_with_output());
tokio::select! {
output = &mut output => {
tracing::trace!("virtiofsd exited");
let output = output?;
let status = output.status;
let stderr = String::from_utf8_lossy(&output.stderr);
return Err(eyre!(
"virtiofsd failed to start for socket {socket_path}\nExit status: {status:?}\nOutput: {stderr}"
));
}
_ = timeout => {
return Err(eyre!("timed out waiting for virtiofsd socket {} to be created (waited {timeout_val:?})", socket_path));
}
_ = query_exists => {
}
}
wait_for_socket(
"virtiofsd",
&socket_path,
&mut output,
Duration::from_secs(60),
)
.await?;
virtiofsd_processes.push(output);
tracing::debug!("virtiofsd socket created: {socket_path}");
}

let vsockdata = if let Some(vhost_fd) = config.vhost_fd.take() {
Expand Down Expand Up @@ -1034,6 +1125,7 @@ impl RunningQemu {
qemu_process,
virtiofsd_processes,
sd_notification,
swtpm_process,
})
}

Expand Down
16 changes: 16 additions & 0 deletions crates/integration-tests/src/tests/run_ephemeral_ssh.rs
Original file line number Diff line number Diff line change
Expand Up @@ -286,6 +286,22 @@ echo "All checks passed!"
}
integration_test!(test_run_tmpfs);

/// Test that --tpm attaches a TPM 2.0 to the VM
fn test_run_ephemeral_tpm() -> TestResult {
let sh = shell()?;
let bck = get_bck_command()?;
let image = get_test_image();
let label = INTEGRATION_TEST_LABEL;

cmd!(
sh,
"{bck} ephemeral run-ssh --label {label} --tpm {image} -- test -c /dev/tpmrm0"
)
.run()?;
Ok(())
}
integration_test!(test_run_ephemeral_tpm);

/// Test that containers are properly cleaned up even when the image is broken
///
/// This test verifies that the drop handler for ContainerCleanup works correctly
Expand Down
19 changes: 19 additions & 0 deletions crates/kit/src/run_ephemeral.rs
Original file line number Diff line number Diff line change
Expand Up @@ -133,6 +133,12 @@ const IGNITION_SERIAL_NAME: &str = "ignition";
/// Mount path for Ignition config inside the container
const IGNITION_CONFIG_MOUNT_PATH: &str = "/run/ignition-config.json";

/// The TPM state directory inside the container.
const SWTPM_STATE_DIR: &str = "/run/swtpm";

/// The swtpm control socket QEMU connects to.
const SWTPM_SOCKET: &str = "/run/inner-shared/swtpm.sock";

/// Kernel initramfs magic marker for bootconfig trailer.
const BOOTCONFIG_MAGIC: &[u8; 12] = b"#BOOTCONFIG\n";

Expand Down Expand Up @@ -546,6 +552,12 @@ pub struct RunEphemeralOpts {
)]
pub mount_disk_files: Vec<String>,

#[clap(
long = "tpm",
help = "Enable TPM 2.0 support using swtpm, with state discarded on exit"
)]
pub tpm: bool,

#[clap(long = "karg", help = "Additional kernel command line arguments")]
pub kernel_args: Vec<String>,

Expand Down Expand Up @@ -2132,6 +2144,13 @@ Options=
None
};

if opts.tpm {
qemu_config.set_tpm(qemu::TpmConfig {
state_dir: SWTPM_STATE_DIR.into(),
socket_path: SWTPM_SOCKET.into(),
});
}

// Add virtio-blk devices
for blk_device in virtio_blk_devices {
qemu_config.add_virtio_blk_device_with_format(
Expand Down
1 change: 1 addition & 0 deletions crates/kit/src/to_disk.rs
Original file line number Diff line number Diff line change
Expand Up @@ -573,6 +573,7 @@ pub fn run(mut opts: ToDiskOpts) -> Result<RunOutcome> {
opts.target_disk,
opts.additional.format.as_str()
)], // Attach target disk
tpm: false,
kernel_args: vec!["systemd.unit=bcvk-to-disk.target".to_string()],
ignition_config: None,
debug_entrypoint: None,
Expand Down
1 change: 1 addition & 0 deletions crates/kit/src/varlink_ipc.rs
Original file line number Diff line number Diff line change
Expand Up @@ -274,6 +274,7 @@ impl BcvkService {
bind_storage_ro: false,
add_swap: opts.add_swap,
mount_disk_files: opts.mount_disk_files.unwrap_or_default(),
tpm: false,
kernel_args: opts.kargs.unwrap_or_default(),
ignition_config: None,
host_dns_servers: None,
Expand Down
4 changes: 4 additions & 0 deletions docs/src/man/bcvk-ephemeral-run-ssh.md
Original file line number Diff line number Diff line change
Expand Up @@ -160,6 +160,10 @@ For longer-running VMs where you need to reconnect multiple times, use

Mount disk file as virtio-blk device at /dev/disk/by-id/virtio-<name>

**--tpm**

Copy link
Copy Markdown
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

While this is the conservative default, I would argue to turn this on by default on supported architectures in the future

Copy link
Copy Markdown
Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yeah it'll depend on swtpm being installed on the host but that should be easy to check.


Enable TPM 2.0 support using swtpm, with state discarded on exit

**--karg**=*KERNEL_ARGS*

Additional kernel command line arguments
Expand Down
4 changes: 4 additions & 0 deletions docs/src/man/bcvk-ephemeral-run.md
Original file line number Diff line number Diff line change
Expand Up @@ -163,6 +163,10 @@ This design allows bcvk to provide VM-like isolation and boot behavior while lev

Mount disk file as virtio-blk device at /dev/disk/by-id/virtio-<name>

**--tpm**

Enable TPM 2.0 support using swtpm, with state discarded on exit

**--karg**=*KERNEL_ARGS*

Additional kernel command line arguments
Expand Down
4 changes: 4 additions & 0 deletions docs/src/man/bcvk-ephemeral-test-basic.md
Original file line number Diff line number Diff line change
Expand Up @@ -161,6 +161,10 @@ and print an error on stderr.

Mount disk file as virtio-blk device at /dev/disk/by-id/virtio-<name>

**--tpm**

Enable TPM 2.0 support using swtpm, with state discarded on exit

**--karg**=*KERNEL_ARGS*

Additional kernel command line arguments
Expand Down
Loading