From 3edf55f5e45119f1c33cf799a2bc8a079357df12 Mon Sep 17 00:00:00 2001 From: greatbridf Date: Sat, 24 Oct 2026 11:38:28 +0800 Subject: [PATCH 01/36] build: bump rust toolchain version to 2026-01-09 Update rust toolchain version. The latest nightly doesn't support json targets without passing some argument to rustc. So don't use the latest. concat_ident! is removed in favor of concat!, and we are actually already using that. So remove the unstable feature declaration in src/lib.rs as well. Signed-off-by: greatbridf --- rust-toolchain | 2 +- src/lib.rs | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/rust-toolchain b/rust-toolchain index 8adb8e58..11ad5efd 100644 --- a/rust-toolchain +++ b/rust-toolchain @@ -1 +1 @@ -nightly-2025-05-16 +nightly-2026-01-09 diff --git a/src/lib.rs b/src/lib.rs index 80d24c28..5df6a1e1 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -2,7 +2,6 @@ #![no_main] #![feature(allocator_api)] #![feature(c_size_t)] -#![feature(concat_idents)] #![feature(arbitrary_self_types)] #![feature(get_mut_unchecked)] #![feature(macro_metavar_expr)] From e49012a1de610fd689c00d354346da9700e044f7 Mon Sep 17 00:00:00 2001 From: greatbridf Date: Mon, 26 Jan 2026 02:05:28 +0800 Subject: [PATCH 02/36] target: update x86 json to support new rust nightly Change the type of target width items to numbers according to the specifications. Signed-off-by: greatbridf (cherry picked from commit a5f6349c0dd450d23319cc95db321aed8e2d4207) Signed-off-by: greatbridf --- x86_64-unknown-none.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/x86_64-unknown-none.json b/x86_64-unknown-none.json index 77e4d1dd..a66b07ba 100644 --- a/x86_64-unknown-none.json +++ b/x86_64-unknown-none.json @@ -3,8 +3,8 @@ "data-layout": "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128", "arch": "x86_64", "target-endian": "little", - "target-pointer-width": "64", - "target-c-int-width": "32", + "target-pointer-width": 64, + "target-c-int-width": 32, "os": "none", "executables": true, "linker-flavor": "ld.lld", From 5071f129c52b1c4ced142e3093a40381f784a3ec Mon Sep 17 00:00:00 2001 From: greatbridf Date: Mon, 26 Oct 2026 23:17:01 +0800 Subject: [PATCH 03/36] loongarch64, interrupt: implement external interrupt delivery External interrupts were unimplemented on loongarch64: the trap context mapped HWI0..HWI5 to a `todo!()`, so no driver could ever be woken by hardware. This made the serial port unusable for input -- the read worker armed the UART's receive interrupt and then waited forever. On the QEMU virt machine the path from a device to the CPU is device -> PCH-PIC line -> EIOINTC vector -> CPU HWI line The PCH-PIC is an ordinary MMIO block at 0x1000_0000; the EIOINTC lives in the IOCSR address space and is accessed with iocsrrd/iocsrwr. Add a small interrupt module that programs both: every EIOINTC vector is routed to HWI1 (matching the interrupt controller's `interrupts = <3>` device tree entry) and to CPU 0, and PCH-PIC lines are additionally routed to the vector of the same number through HTMSI_VEC. Lines stay masked until a handler exists. register_irq_handler() now unmasks the corresponding PCH-PIC line through the HAL on loongarch64, so an interrupt cannot be delivered before the kernel can service it, and enabling a line clears any stale EIOINTC pending bit first. dispatch_irq() acknowledges the pending bit in COREISR before invoking the handler, so a peripheral that keeps its line asserted simply re-raises the interrupt until it is drained. Only vectors 0..32 are dispatched; the remaining vectors belong to the PCI MSI controller, which is not supported yet, so they are acknowledged and dropped. With this in place UART0 (PCH-PIC line 2) delivers receive interrupts and serial input works under QEMU: typed commands are echoed and executed, and the system stays idle at the prompt instead of spinning on a level-triggered interrupt. Assisted-by: deepseek-flash Signed-off-by: greatbridf --- crates/eonix_hal/src/arch/loongarch64/cpu.rs | 1 + .../src/arch/loongarch64/interrupt/mod.rs | 163 ++++++++++++++++++ crates/eonix_hal/src/arch/loongarch64/mod.rs | 1 + .../src/arch/loongarch64/trap/trap_context.rs | 16 +- src/driver/serial.rs | 4 +- src/kernel/interrupt.rs | 7 + 6 files changed, 178 insertions(+), 14 deletions(-) create mode 100644 crates/eonix_hal/src/arch/loongarch64/interrupt/mod.rs diff --git a/crates/eonix_hal/src/arch/loongarch64/cpu.rs b/crates/eonix_hal/src/arch/loongarch64/cpu.rs index 612fbc3e..38e84fad 100644 --- a/crates/eonix_hal/src/arch/loongarch64/cpu.rs +++ b/crates/eonix_hal/src/arch/loongarch64/cpu.rs @@ -38,6 +38,7 @@ impl CPU { pub unsafe fn init(mut self: Pin<&mut Self>) { let me = self.as_mut().get_unchecked_mut(); setup_trap(); + super::interrupt::init(); } /// Boot all other hart. diff --git a/crates/eonix_hal/src/arch/loongarch64/interrupt/mod.rs b/crates/eonix_hal/src/arch/loongarch64/interrupt/mod.rs new file mode 100644 index 00000000..8409c4d2 --- /dev/null +++ b/crates/eonix_hal/src/arch/loongarch64/interrupt/mod.rs @@ -0,0 +1,163 @@ +//! External interrupt support for LoongArch64. +//! +//! On the QEMU `virt` machine the external interrupt path is: +//! +//! ```text +//! device (UART, ...) -> PCH-PIC line -> EIOINTC vector -> CPU HWI line +//! ``` +//! +//! The PCH-PIC is a normal MMIO device, while the EIOINTC (EXTIOI) lives in +//! the IOCSR address space and must be accessed with `iocsrrd`/`iocsrwr`. +//! +//! EIOINTC vectors are grouped: each group of 32 vectors is routed to one of +//! the four CPU hardware interrupt lines (HWI0..HWI3, i.e. CPU interrupt +//! numbers 2..5). We route every group to HWI1, which is the line the device +//! tree advertises for the interrupt controller (`interrupts = <3>`), and +//! every vector to CPU 0. + +use super::mm::ArchPhysAccess; +use eonix_mm::address::{PAddr, PhysAccess}; +use loongArch64::iocsr; + +/// Base address of the PCH-PIC (normal MMIO). +const PCH_PIC_BASE: usize = 0x1000_0000; + +const PCH_PIC_INT_MASK: usize = 0x020; +const PCH_PIC_HTMSI_VEC: usize = 0x200; + +/// Number of interrupt lines handled by the PCH-PIC. +const PCH_PIC_LINES: usize = 64; + +/// Number of EIOINTC vectors wired to PCH-PIC lines. +const PCH_PIC_VECTORS: usize = 32; + +/// EIOINTC registers, in the IOCSR address space. +const EIOINTC_IPMAP: usize = 0x14c0; +const EIOINTC_ENABLE: usize = 0x1600; +const EIOINTC_COREISR: usize = 0x1800; +const EIOINTC_COREMAP: usize = 0x1c00; + +const EIOINTC_VECTORS: usize = 256; +const EIOINTC_GROUP_SIZE: usize = 32; +const EIOINTC_GROUPS: usize = EIOINTC_VECTORS / EIOINTC_GROUP_SIZE; + +/// CPU interrupt line all EIOINTC vectors are routed to. +/// +/// This is the `ipnum` passed to the EIOINTC, whose outputs are connected to +/// CPU interrupt inputs `ipnum + 2` (HWI0 = 2). +const EIOINTC_IPNUM: usize = 1; + +fn pch_read32(offset: usize) -> u32 { + unsafe { + // SAFETY: The PCH-PIC registers are memory mapped at a well-known + // physical address on the QEMU virt platform. + ArchPhysAccess::as_ptr::(PAddr::from(PCH_PIC_BASE + offset)) + .as_ptr() + .read_volatile() + } +} + +fn pch_write32(offset: usize, value: u32) { + unsafe { + // SAFETY: See `pch_read32`. + ArchPhysAccess::as_ptr::(PAddr::from(PCH_PIC_BASE + offset)) + .as_ptr() + .write_volatile(value) + } +} + +fn pch_write8(offset: usize, value: u8) { + unsafe { + // SAFETY: See `pch_read32`. + ArchPhysAccess::as_ptr::(PAddr::from(PCH_PIC_BASE + offset)) + .as_ptr() + .write_volatile(value) + } +} + +fn eiointc_enable_vector(vector: usize) { + let offset = EIOINTC_ENABLE + (vector / EIOINTC_GROUP_SIZE) * size_of::(); + let bit = 1u32 << (vector % EIOINTC_GROUP_SIZE); + + iocsr::iocsr_write_w(offset, iocsr::iocsr_read_w(offset) | bit); +} + +fn eiointc_clear_vector(vector: usize) { + let offset = EIOINTC_COREISR + (vector / EIOINTC_GROUP_SIZE) * size_of::(); + let bit = 1u32 << (vector % EIOINTC_GROUP_SIZE); + + // Writing one acknowledges (clears) the pending bit. + iocsr::iocsr_write_w(offset, bit); +} + +/// Initialize the external interrupt controllers. +/// +/// Must be called once per CPU before interrupts are enabled. +pub fn init() { + // Route all 256 vectors to HWI1. The IPMAP register contains one byte + // per 32-vector group, holding a bitmask of the target `ipnum`. + let ipmap = 0x01u32 << EIOINTC_IPNUM; + let ipmap = ipmap * 0x0101_0101; + for group in 0..(EIOINTC_VECTORS / EIOINTC_GROUP_SIZE / 4) { + iocsr::iocsr_write_w(EIOINTC_IPMAP + group * size_of::(), ipmap); + } + + // Route all vectors to CPU 0. COREMAP stores four bytes (one vector + // each) per 32-bit register; a byte holds a bitmask of target CPUs. + for word in 0..(EIOINTC_VECTORS / 4) { + iocsr::iocsr_write_w(EIOINTC_COREMAP + word * size_of::(), 0x0101_0101); + } +} + +/// Enable an external interrupt line. +/// +/// `irq` is the PCH-PIC line number, which is also the EIOINTC vector number +/// for the lower 32 vectors. +pub fn enable_irq(irq: usize) { + assert!(irq < PCH_PIC_LINES, "PCH-PIC line out of range"); + + // Route this PCH-PIC line to the EIOINTC vector with the same number + // before unmasking it, so the first interrupt already lands on the + // correct vector. + pch_write8(PCH_PIC_HTMSI_VEC + irq, irq as u8); + + if irq < PCH_PIC_VECTORS { + eiointc_clear_vector(irq); + eiointc_enable_vector(irq); + } + + // Unmask the line in the PCH-PIC. + let offset = PCH_PIC_INT_MASK + (irq / 32) * size_of::(); + let bit = 1u32 << (irq % 32); + pch_write32(offset, pch_read32(offset) & !bit); +} + +/// Handle pending external interrupts by invoking `handler` for each of them. +pub fn dispatch_irq(handler: fn(usize)) { + for group in 0..EIOINTC_GROUPS { + let offset = EIOINTC_COREISR + group * size_of::(); + let mut pending = iocsr::iocsr_read_w(offset); + + if pending == 0 { + continue; + } + + // Acknowledge before dispatching, like Linux does. The peripheral + // keeps the line asserted until its handler has serviced it, and any + // new event after that re-raises the interrupt. + iocsr::iocsr_write_w(offset, pending); + + while pending != 0 { + let bit = pending.trailing_zeros() as usize; + pending &= pending - 1; + + let vector = group * EIOINTC_GROUP_SIZE + bit; + + // Vectors 0..32 are wired to the PCH-PIC. Higher vectors belong + // to the PCI MSI controller, which we do not support yet. + if vector < PCH_PIC_VECTORS { + handler(vector); + } + } + } +} diff --git a/crates/eonix_hal/src/arch/loongarch64/mod.rs b/crates/eonix_hal/src/arch/loongarch64/mod.rs index 46358d8d..da406a47 100644 --- a/crates/eonix_hal/src/arch/loongarch64/mod.rs +++ b/crates/eonix_hal/src/arch/loongarch64/mod.rs @@ -4,5 +4,6 @@ pub mod cpu; pub mod fdt; pub mod fence; pub mod fpu; +pub mod interrupt; pub mod mm; pub mod trap; diff --git a/crates/eonix_hal/src/arch/loongarch64/trap/trap_context.rs b/crates/eonix_hal/src/arch/loongarch64/trap/trap_context.rs index 56bf59b6..cc3da77f 100644 --- a/crates/eonix_hal/src/arch/loongarch64/trap/trap_context.rs +++ b/crates/eonix_hal/src/arch/loongarch64/trap/trap_context.rs @@ -1,4 +1,4 @@ -use crate::{arch::trap::CSR_KERNEL_TP, processor::CPU}; +use crate::arch::trap::CSR_KERNEL_TP; use core::{arch::asm, mem::offset_of}; use eonix_hal_traits::{ fault::{Fault, PageFaultErrorCode}, @@ -151,18 +151,8 @@ impl RawTrapContext for TrapContext { }, }, Trap::Interrupt(interrupt) => match interrupt as usize { - 2..=7 => TrapType::Irq { - callback: |handler| { - todo!("handle IRQs"); - // let mut cpu = CPU::local(); - // match cpu.as_mut().interrupt.plic.claim_interrupt() { - // None => {} - // Some(irqno) => { - // cpu.interrupt.plic.complete_interrupt(irqno); - // handler(irqno); - // } - // } - }, + 2..=9 => TrapType::Irq { + callback: |handler| crate::arch::interrupt::dispatch_irq(handler), }, interrupt => TrapType::Fault(Fault::Unknown(interrupt)), }, diff --git a/src/driver/serial.rs b/src/driver/serial.rs index d69965f4..7561828a 100644 --- a/src/driver/serial.rs +++ b/src/driver/serial.rs @@ -249,7 +249,9 @@ pub fn init() -> KResult<()> { let serial = Serial::new(0, port)?; serial.register_as_char_device( - // 2 or 4 here, let's try 2 first! + // UART0 is wired to PCH-PIC line 2 on the QEMU virt machine + // (`interrupts = <2 4>` in the device tree). The HAL routes it + // to an EIOINTC vector and delivers it as an external IRQ. 2, )?; } diff --git a/src/kernel/interrupt.rs b/src/kernel/interrupt.rs index 742727cb..2f07758a 100644 --- a/src/kernel/interrupt.rs +++ b/src/kernel/interrupt.rs @@ -65,5 +65,12 @@ where } IRQ_HANDLERS.lock_irq()[irqno as usize].push(Arc::new(handler)); + + // On LoongArch the IRQ lines are masked by default and must be unmasked + // once a handler is available, so that an interrupt cannot be delivered + // before anyone can service the device. + #[cfg(target_arch = "loongarch64")] + eonix_hal::arch_exported::interrupt::enable_irq(irqno as usize); + Ok(()) } From 2202f1838360e6a9aafd59c575dc34f508ee63f4 Mon Sep 17 00:00:00 2001 From: greatbridf Date: Sat, 24 Jan 2026 14:01:16 +0800 Subject: [PATCH 04/36] script: refactor the build image script Rearrange code to pass args instead of by envs, no functionality changes. Signed-off-by: greatbridf --- Makefile.src | 4 +- script/build-img.sh | 58 ----------------- script/build-img/build-img | 125 +++++++++++++++++++++++++++++++++++++ 3 files changed, 127 insertions(+), 60 deletions(-) delete mode 100644 script/build-img.sh create mode 100755 script/build-img/build-img diff --git a/Makefile.src b/Makefile.src index ab13f5b8..1c8ecb2a 100644 --- a/Makefile.src +++ b/Makefile.src @@ -173,8 +173,8 @@ $(BINARY_DIR)/eonix_kernel: $(KERNEL_DEPS) build/kernel.sym: $(BINARY_DIR)/eonix_kernel CARGO_TARGET_DIR=build cargo objcopy -q $(CARGO_FLAGS) -- --only-keep-debug build/kernel.sym -build/fs-%.img: user-programs/init_script_%.sh script/build-img.sh $(USER_PROGRAMS) - ARCH=$* OUTPUT=$@ sh script/build-img.sh +build/fs-%.img: user-programs/init_script_%.sh script/build-img/build-img $(USER_PROGRAMS) + script/build-img/build-img -a $* -o $@ build/mbr.bin: $(BINARY_DIR)/eonix_kernel CARGO_TARGET_DIR=build cargo objcopy -q $(CARGO_FLAGS) -- -O binary -j .mbr build/mbr.bin diff --git a/script/build-img.sh b/script/build-img.sh deleted file mode 100644 index 6a83638f..00000000 --- a/script/build-img.sh +++ /dev/null @@ -1,58 +0,0 @@ -#!/bin/sh - -OS=`uname -s` - -if sudo --version > /dev/null 2>&1; then - SUDO=sudo -fi - -if [ "$OUTPUT" = "" ]; then - OUTPUT="build/fs-$ARCH.img" -fi - -if [ "$ARCH" = "" ]; then - echo "ARCH is not set, exiting..." >&2 - exit 1 -fi - -dd if=/dev/zero of="$OUTPUT" bs=`expr 1024 \* 1024` count=1020 -mkfs.fat -n SYSTEM "$OUTPUT" - -if [ "$OS" = "Darwin" ]; then - SUDO='' - hdiutil detach build/mnt > /dev/null 2>&1 || true - hdiutil attach "$OUTPUT" -mountpoint build/mnt -else - mkdir -p build/mnt - $SUDO losetup -P /dev/loop2 "$OUTPUT" - $SUDO mount /dev/loop2 build/mnt -fi - -if [ "$ARCH" = "x86_64" ]; then - $SUDO cp ./user-programs/init.out build/mnt/init - $SUDO cp ./user-programs/int.out build/mnt/int - $SUDO cp ./user-programs/dynamic_test build/mnt/dynamic_test - $SUDO cp ./user-programs/busybox build/mnt/busybox - $SUDO cp ./user-programs/busybox-minimal build/mnt/busybox_ - $SUDO cp ./user-programs/ld-musl-i386.so.1 build/mnt/ld-musl-i386.so.1 - $SUDO cp ./user-programs/pthread_test build/mnt/pthread_test - $SUDO cp ./user-programs/init_script_x86_64.sh build/mnt/initsh -elif [ "$ARCH" = "riscv64" ]; then - $SUDO cp ./user-programs/busybox.static build/mnt/busybox - $SUDO cp ./user-programs/init_script_riscv64.sh build/mnt/initsh -elif [ "$ARCH" = "loongarch64" ]; then - $SUDO cp ./user-programs/busybox.la64 build/mnt/busybox - $SUDO cp ./user-programs/init_script_loongarch64.sh build/mnt/initsh -fi - -# Add your custom files here - - -# End of custom files - -if [ "$OS" = "Darwin" ]; then - hdiutil detach build/mnt -else - $SUDO losetup -d /dev/loop2 - $SUDO umount build/mnt -fi diff --git a/script/build-img/build-img b/script/build-img/build-img new file mode 100755 index 00000000..4b203968 --- /dev/null +++ b/script/build-img/build-img @@ -0,0 +1,125 @@ +#!/bin/sh + +KB=1024 +MB=$((1024 * KB)) + +OS=$(uname -s) +SUDO=$(which sudo || :) + +ARCH= +IMAGE= +MOUNTPOINT="$(mktemp -d)" + +die() { + echo "error: $1" >&2 + exit 1 +} + +usage() { + cat >&2 < -o [-h] + +Options + -a The arch building image for [available: riscv64 x86 loongarch64] + -o Output image path + -h Show help message +EOF + + exit "$1" +} + +sudo() { + "$SUDO" "$@" +} + +ensure_mount_macos() { + hdiutil detach "$MOUNTPOINT" > /dev/null 2>&1 || : + hdiutil attach "$IMAGE" -mountpoint "$MOUNTPOINT" +} + +ensure_mount_linux() { + DEV=$(sudo losetup -f "$IMAGE" --show) + sudo mount "$DEV" "$MOUNTPOINT" +} + +ensure_mount() { + if [ "$OS" = Darwin ]; then + ensure_mount_macos + return + fi + + ensure_mount_linux +} + +copy_to_image() { + _prefix=sudo + [ "$OS" = Darwin ] && _prefix= + + $_prefix cp "$1" "$MOUNTPOINT/$2" +} + +unmount_macos() { + hdiutil detach "$MOUNTPOINT" +} + +unmount_linux() { + sudo umount "$MOUNTPOINT" + sudo losetup -d "$DEV" +} + +cleanup() { + case "$OS" in + Darwin) + unmount_macos + ;; + *) + unmount_linux + ;; + esac +} + +set -eu + +while getopts "a:o:h" opt; do + case "$opt" in + a) ARCH="$OPTARG";; + o) IMAGE="$OPTARG";; + h) usage 0;; + ?) usage 1;; + esac +done + +shift $((OPTIND - 1)) + +[ -z "$ARCH" ] && die "ARCH is not set" +[ -z "$IMAGE" ] && die "output image path is not set" + +echo "Build image with ARCH=$ARCH OUTPUT=$IMAGE MOUNTPOINT=$MOUNTPOINT" + +dd if=/dev/zero of="$IMAGE" bs=$((1 * MB)) count=$((1024 - 4)) +mkfs.fat -n SYSTEM "$IMAGE" + +ensure_mount +trap cleanup EXIT + +if [ "$ARCH" = "x86_64" ]; then + copy_to_image ./user-programs/init.out init + copy_to_image ./user-programs/int.out int + copy_to_image ./user-programs/dynamic_test dynamic_test + copy_to_image ./user-programs/busybox busybox + copy_to_image ./user-programs/busybox-minimal busybox_ + copy_to_image ./user-programs/ld-musl-i386.so.1 ld-musl-i386.so.1 + copy_to_image ./user-programs/pthread_test pthread_test + copy_to_image ./user-programs/init_script_x86_64.sh initsh +elif [ "$ARCH" = "riscv64" ]; then + copy_to_image ./user-programs/busybox.static busybox + copy_to_image ./user-programs/init_script_riscv64.sh initsh +elif [ "$ARCH" = "loongarch64" ]; then + copy_to_image ./user-programs/busybox.la64 busybox + copy_to_image ./user-programs/init_script_loongarch64.sh initsh +fi + +# Add your custom files here + + +# End of custom files From 2e20ee7faf34c004048a064205b925576cb27479 Mon Sep 17 00:00:00 2001 From: greatbridf Date: Sat, 24 Jan 2026 16:02:25 +0800 Subject: [PATCH 05/36] test: rework test script and github workflow Signed-off-by: greatbridf --- .github/workflows/test-build.yaml | 1 + script/test.sh | 74 +++++++++++++++++++++++++++---- 2 files changed, 67 insertions(+), 8 deletions(-) diff --git a/.github/workflows/test-build.yaml b/.github/workflows/test-build.yaml index f6063ab3..23f2d7ed 100644 --- a/.github/workflows/test-build.yaml +++ b/.github/workflows/test-build.yaml @@ -30,6 +30,7 @@ jobs: - name: Setup QEMU run: | + sudo apt-get update sudo apt-get install -y qemu-system-${{ matrix.arch }} qemu-kvm - name: Configure diff --git a/script/test.sh b/script/test.sh index 01097bc3..81cdb9c8 100644 --- a/script/test.sh +++ b/script/test.sh @@ -1,13 +1,71 @@ -#!/bin/sh +#!/bin/bash +LOGFILE="build/test-$$.log" SUCCESS_MSG="###$RANDOM :SuCCeSS: $RANDOM###" -if [ "$ARCH" = "" ]; then - echo "Error: ARCH environment variable is not set." >&2 +die() { + echo "error: $1" >&2 exit 1 -fi +} + +runcmd() { + printf -v _cmd "%q" "$*" + + cat < /dev/null && echo TEST\ $$\ WITH\ ARCH=$ARCH\ PASSED +$(runcmd ls -la) + +$(wait_str proc) + +$(runcmd echo \""$SUCCESS_MSG"\") + +$(wait_str "\n$SUCCESS_MSG") + +$(runcmd poweroff) + +$(wait_init_exit) + +EOF + +status=$? + +echo + +# shellcheck disable=SC2181 +if [ $status -ne 0 ]; then + echo "=== Test $$ with ARCH=$ARCH failed" +else + echo "=== Test $$ with ARCH=$ARCH passed" +fi From e387f7ef2ef0289b0bee169fe5552aaa196e9890 Mon Sep 17 00:00:00 2001 From: greatbridf Date: Sat, 24 Jan 2026 16:10:21 +0800 Subject: [PATCH 06/36] ci: install expect and update printf usage - printf -v is non-standard, use var=$(printf) instead. - expect is not installed in the runner. install it manually Signed-off-by: greatbridf --- .github/workflows/test-build.yaml | 2 +- script/test.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test-build.yaml b/.github/workflows/test-build.yaml index 23f2d7ed..0cd9ae87 100644 --- a/.github/workflows/test-build.yaml +++ b/.github/workflows/test-build.yaml @@ -31,7 +31,7 @@ jobs: - name: Setup QEMU run: | sudo apt-get update - sudo apt-get install -y qemu-system-${{ matrix.arch }} qemu-kvm + sudo apt-get install -y qemu-system-${{ matrix.arch }} qemu-kvm expect - name: Configure run: ./configure diff --git a/script/test.sh b/script/test.sh index 81cdb9c8..cc5bde7e 100644 --- a/script/test.sh +++ b/script/test.sh @@ -9,7 +9,7 @@ die() { } runcmd() { - printf -v _cmd "%q" "$*" + _cmd=$(printf "%q" "$*") cat < Date: Sat, 24 Jan 2026 16:46:38 +0800 Subject: [PATCH 07/36] script, build-img: add script folder for each platforms Add PLATFORM in Makefile to indicate the actual system setup. Put build image script of each platforms to dedicated folders. This makes creating test setups easier. Signed-off-by: greatbridf --- .github/workflows/test-build.yaml | 6 +- Makefile.src | 34 +++++----- script/build-img/build-img | 63 +++++-------------- script/build-img/lib/lib.sh | 39 ++++++++++++ .../platform/loongarch64-virt/10-base.sh | 5 ++ .../platform/riscv64-virt/10-base.sh | 5 ++ .../platform/riscv64-virt/20-tests.sh | 21 +++++++ .../platform/x86_64-acpi_compat/10-base.sh | 7 +++ .../platform/x86_64-acpi_compat/20-tests.sh | 7 +++ ...h64.sh => init_script_loongarch64-virt.sh} | 0 ...riscv64.sh => init_script_riscv64-virt.sh} | 0 ...4.sh => init_script_x86_64-acpi_compat.sh} | 0 12 files changed, 122 insertions(+), 65 deletions(-) create mode 100644 script/build-img/lib/lib.sh create mode 100644 script/build-img/platform/loongarch64-virt/10-base.sh create mode 100644 script/build-img/platform/riscv64-virt/10-base.sh create mode 100644 script/build-img/platform/riscv64-virt/20-tests.sh create mode 100644 script/build-img/platform/x86_64-acpi_compat/10-base.sh create mode 100644 script/build-img/platform/x86_64-acpi_compat/20-tests.sh rename user-programs/{init_script_loongarch64.sh => init_script_loongarch64-virt.sh} (100%) rename user-programs/{init_script_riscv64.sh => init_script_riscv64-virt.sh} (100%) rename user-programs/{init_script_x86_64.sh => init_script_x86_64-acpi_compat.sh} (100%) diff --git a/.github/workflows/test-build.yaml b/.github/workflows/test-build.yaml index 0cd9ae87..61289e57 100644 --- a/.github/workflows/test-build.yaml +++ b/.github/workflows/test-build.yaml @@ -19,8 +19,10 @@ jobs: include: - arch: riscv64 target: riscv64gc-unknown-none-elf + platform: virt - arch: loongarch64 target: loongarch64-unknown-none-softfloat + platform: virt steps: - name: Checkout code uses: actions/checkout@v4 @@ -39,7 +41,7 @@ jobs: - name: Run build for ${{ matrix.arch }} targets run: | make build ARCH=${{ matrix.arch }} MODE=release - zstd -k build/boot-${{ matrix.arch }}.img + zstd -k build/boot-${{ matrix.arch }}-${{ matrix.platform }}.img - name: Upload build artifacts uses: actions/upload-artifact@v4 @@ -47,7 +49,7 @@ jobs: name: eonix-kernel-and-image-${{ matrix.arch }} path: | build/${{ matrix.target }}/release/eonix_kernel - build/boot-${{ matrix.arch }}.img.zst + build/boot-${{ matrix.arch }}-${{ matrix.platform }}.img.zst - name: Test run for ${{ matrix.arch }} run: | diff --git a/Makefile.src b/Makefile.src index 1c8ecb2a..eca1eda4 100644 --- a/Makefile.src +++ b/Makefile.src @@ -54,6 +54,8 @@ endif ifeq ($(ARCH),riscv64) +PLATFORM ?= virt + BINARY_DIR_BASE := build/riscv64gc-unknown-none-elf BINARY_DIR := $(BINARY_DIR_BASE)/$(MODE) @@ -61,7 +63,7 @@ QEMU_ARGS += \ -machine virt -kernel $(BINARY_DIR)/eonix_kernel \ -device virtio-blk-device,drive=disk0,bus=virtio-mmio-bus.0 \ -device virtio-net-device,netdev=mynet0 \ - -drive id=disk0,file=build/boot-riscv64.img,format=raw,if=none \ + -drive id=disk0,file=build/boot-riscv64-virt.img,format=raw,if=none \ -netdev user,id=mynet0 \ -rtc base=utc @@ -73,11 +75,10 @@ endif CARGO_FLAGS += --target riscv64gc-unknown-none-elf -.PHONY: build -build: $(BINARY_DIR)/eonix_kernel build/boot-riscv64.img - else ifeq ($(ARCH),loongarch64) +PLATFORM ?= virt + BINARY_DIR_BASE := build/loongarch64-unknown-none-softfloat BINARY_DIR := $(BINARY_DIR_BASE)/$(MODE) @@ -85,7 +86,7 @@ QEMU_ARGS += \ -machine virt -kernel $(BINARY_DIR)/eonix_kernel -m 1G \ -device virtio-blk-pci,drive=disk0 \ -device virtio-net-pci,netdev=mynet0 \ - -drive id=disk0,file=build/boot-loongarch64.img,format=raw,if=none \ + -drive id=disk0,file=build/boot-loongarch64-virt.img,format=raw,if=none \ -netdev user,id=mynet0,hostfwd=tcp::5555-:5555,hostfwd=udp::5555-:5555 \ -rtc base=utc @@ -97,11 +98,10 @@ endif CARGO_FLAGS += --target loongarch64-unknown-none-softfloat -.PHONY: build -build: $(BINARY_DIR)/eonix_kernel build/boot-loongarch64.img - else ifeq ($(ARCH),x86_64) +PLATFORM ?= acpi_compat + BINARY_DIR_BASE := build/x86_64-unknown-none BINARY_DIR := $(BINARY_DIR_BASE)/$(MODE) @@ -110,7 +110,7 @@ QEMU_ARGS += \ -device ahci,id=ahci \ -device ide-hd,drive=disk0,bus=ahci.0 \ -device e1000e,netdev=mynet0 \ - -drive id=disk0,file=build/boot-x86_64.img,format=raw,if=none \ + -drive id=disk0,file=build/boot-x86_64-acpi_compat.img,format=raw,if=none \ -netdev user,id=mynet0 ifneq ($(IMG),) @@ -121,11 +121,13 @@ endif CARGO_FLAGS += --target x86_64-unknown-none.json -.PHONY: build -build: $(BINARY_DIR)/eonix_kernel build/boot-x86_64.img - endif +TARGET := $(ARCH)-$(PLATFORM) + +.PHONY: build +build: $(BINARY_DIR)/eonix_kernel build/boot-$(TARGET).img + .PHONY: run run: build build/kernel.sym $(QEMU) $(QEMU_ARGS) -display none -serial mon:stdio @@ -174,7 +176,7 @@ build/kernel.sym: $(BINARY_DIR)/eonix_kernel CARGO_TARGET_DIR=build cargo objcopy -q $(CARGO_FLAGS) -- --only-keep-debug build/kernel.sym build/fs-%.img: user-programs/init_script_%.sh script/build-img/build-img $(USER_PROGRAMS) - script/build-img/build-img -a $* -o $@ + script/build-img/build-img -t $* -o $@ build/mbr.bin: $(BINARY_DIR)/eonix_kernel CARGO_TARGET_DIR=build cargo objcopy -q $(CARGO_FLAGS) -- -O binary -j .mbr build/mbr.bin @@ -186,7 +188,7 @@ build/kernel.bin: $(BINARY_DIR)/eonix_kernel CARGO_TARGET_DIR=build cargo objcopy -q $(CARGO_FLAGS) -- -O binary --strip-debug \ -R .mbr -R .stage1 build/kernel.bin -build/boot-x86_64.img: build/fs-x86_64.img build/mbr.bin build/stage1.bin build/kernel.bin +build/boot-x86_64-acpi_compat.img: build/fs-x86_64-acpi_compat.img build/mbr.bin build/stage1.bin build/kernel.bin dd if=build/mbr.bin of=$@ bs=512 count=1 conv=notrunc 2> /dev/null dd if=build/stage1.bin of=$@ bs=512 seek=1 conv=notrunc 2> /dev/null dd if=build/kernel.bin of=$@ bs=4096 seek=1 conv=notrunc 2> /dev/null @@ -195,13 +197,13 @@ build/boot-x86_64.img: build/fs-x86_64.img build/mbr.bin build/stage1.bin build/ sh -c 'echo n; echo; echo; echo 8192; echo; echo a; echo w' \ | $(FDISK) $@ 2> /dev/null > /dev/null -build/boot-riscv64.img: build/fs-riscv64.img +build/boot-riscv64-virt.img: build/fs-riscv64-virt.img dd if=$< of=$@ bs=$(shell expr 4 \* 1024 \* 1024) \ seek=1 conv=notrunc 2> /dev/null sh -c 'echo n; echo; echo; echo 8192; echo; echo a; echo w' \ | $(FDISK) $@ 2> /dev/null > /dev/null -build/boot-loongarch64.img: build/fs-loongarch64.img +build/boot-loongarch64-virt.img: build/fs-loongarch64-virt.img dd if=$< of=$@ bs=$(shell expr 4 \* 1024 \* 1024) \ seek=1 conv=notrunc 2> /dev/null sh -c 'echo n; echo; echo; echo 8192; echo; echo a; echo w' \ diff --git a/script/build-img/build-img b/script/build-img/build-img index 4b203968..638866af 100755 --- a/script/build-img/build-img +++ b/script/build-img/build-img @@ -1,26 +1,24 @@ #!/bin/sh +# shellcheck source=./lib/lib.sh + +. "$(dirname "$(realpath "$0")")/lib/lib.sh" + +BASE_DIR="$(dirname "$(realpath "$0")")" KB=1024 MB=$((1024 * KB)) -OS=$(uname -s) -SUDO=$(which sudo || :) - -ARCH= +TARGET= IMAGE= MOUNTPOINT="$(mktemp -d)" -die() { - echo "error: $1" >&2 - exit 1 -} - usage() { cat >&2 < -o [-h] +Usage: $0 -t -o [-h] Options - -a The arch building image for [available: riscv64 x86 loongarch64] + -t The target building image for + [available: riscv64 x86 loongarch64] -o Output image path -h Show help message EOF @@ -28,10 +26,6 @@ EOF exit "$1" } -sudo() { - "$SUDO" "$@" -} - ensure_mount_macos() { hdiutil detach "$MOUNTPOINT" > /dev/null 2>&1 || : hdiutil attach "$IMAGE" -mountpoint "$MOUNTPOINT" @@ -51,13 +45,6 @@ ensure_mount() { ensure_mount_linux } -copy_to_image() { - _prefix=sudo - [ "$OS" = Darwin ] && _prefix= - - $_prefix cp "$1" "$MOUNTPOINT/$2" -} - unmount_macos() { hdiutil detach "$MOUNTPOINT" } @@ -80,9 +67,9 @@ cleanup() { set -eu -while getopts "a:o:h" opt; do +while getopts "t:o:h" opt; do case "$opt" in - a) ARCH="$OPTARG";; + t) TARGET="$OPTARG";; o) IMAGE="$OPTARG";; h) usage 0;; ?) usage 1;; @@ -91,10 +78,10 @@ done shift $((OPTIND - 1)) -[ -z "$ARCH" ] && die "ARCH is not set" +[ -z "$TARGET" ] && die "ARCH is not set" [ -z "$IMAGE" ] && die "output image path is not set" -echo "Build image with ARCH=$ARCH OUTPUT=$IMAGE MOUNTPOINT=$MOUNTPOINT" +echo "Build image with ARCH=$TARGET OUTPUT=$IMAGE MOUNTPOINT=$MOUNTPOINT" dd if=/dev/zero of="$IMAGE" bs=$((1 * MB)) count=$((1024 - 4)) mkfs.fat -n SYSTEM "$IMAGE" @@ -102,24 +89,6 @@ mkfs.fat -n SYSTEM "$IMAGE" ensure_mount trap cleanup EXIT -if [ "$ARCH" = "x86_64" ]; then - copy_to_image ./user-programs/init.out init - copy_to_image ./user-programs/int.out int - copy_to_image ./user-programs/dynamic_test dynamic_test - copy_to_image ./user-programs/busybox busybox - copy_to_image ./user-programs/busybox-minimal busybox_ - copy_to_image ./user-programs/ld-musl-i386.so.1 ld-musl-i386.so.1 - copy_to_image ./user-programs/pthread_test pthread_test - copy_to_image ./user-programs/init_script_x86_64.sh initsh -elif [ "$ARCH" = "riscv64" ]; then - copy_to_image ./user-programs/busybox.static busybox - copy_to_image ./user-programs/init_script_riscv64.sh initsh -elif [ "$ARCH" = "loongarch64" ]; then - copy_to_image ./user-programs/busybox.la64 busybox - copy_to_image ./user-programs/init_script_loongarch64.sh initsh -fi - -# Add your custom files here - - -# End of custom files +for name in $(iter_files_sorted "$BASE_DIR/platform/$TARGET"); do + ( . "$name" ) +done diff --git a/script/build-img/lib/lib.sh b/script/build-img/lib/lib.sh new file mode 100644 index 00000000..5cc6c036 --- /dev/null +++ b/script/build-img/lib/lib.sh @@ -0,0 +1,39 @@ +#!/bin/sh + +OS=$(uname -s) +SUDO=$(which sudo || :) + +info() { + echo "info : $1" +} + +warn() { + echo "warn : $1" >&2 +} + +error() { + echo "fatal: $1" >&2 +} + +die() { + error "$1" && exit 1 +} + +sudo() { + "$SUDO" "$@" +} + +copy_to_image() { + _prefix=sudo + [ "$OS" = Darwin ] && _prefix= + + $_prefix cp "$1" "$MOUNTPOINT/$2" +} + +iter_files() { + find -L "$1" -maxdepth 1 -type f +} + +iter_files_sorted() { + iter_files "$@" | sort +} diff --git a/script/build-img/platform/loongarch64-virt/10-base.sh b/script/build-img/platform/loongarch64-virt/10-base.sh new file mode 100644 index 00000000..d0f21dbd --- /dev/null +++ b/script/build-img/platform/loongarch64-virt/10-base.sh @@ -0,0 +1,5 @@ +#!/bin/sh +# shellcheck source=./../../lib/lib.sh + +copy_to_image user-programs/busybox.la64 busybox +copy_to_image user-programs/init_script_loongarch64-virt.sh initsh diff --git a/script/build-img/platform/riscv64-virt/10-base.sh b/script/build-img/platform/riscv64-virt/10-base.sh new file mode 100644 index 00000000..efb32cf7 --- /dev/null +++ b/script/build-img/platform/riscv64-virt/10-base.sh @@ -0,0 +1,5 @@ +#!/bin/sh +# shellcheck source=./../../lib/lib.sh + +copy_to_image user-programs/busybox.static busybox +copy_to_image user-programs/init_script_riscv64-virt.sh initsh diff --git a/script/build-img/platform/riscv64-virt/20-tests.sh b/script/build-img/platform/riscv64-virt/20-tests.sh new file mode 100644 index 00000000..1d9e0d71 --- /dev/null +++ b/script/build-img/platform/riscv64-virt/20-tests.sh @@ -0,0 +1,21 @@ +#!/bin/sh + +name="$(mktemp)" + +cat > "$name" < Date: Sun, 25 Jan 2026 01:18:05 +0800 Subject: [PATCH 08/36] style, hal: rename x86 trap.rs to trap/mod.rs No functional change. Signed-off-by: greatbridf --- .../src/arch/x86_64/{trap.rs => trap/mod.rs} | 33 +++++++++---------- 1 file changed, 16 insertions(+), 17 deletions(-) rename crates/eonix_hal/src/arch/x86_64/{trap.rs => trap/mod.rs} (97%) diff --git a/crates/eonix_hal/src/arch/x86_64/trap.rs b/crates/eonix_hal/src/arch/x86_64/trap/mod.rs similarity index 97% rename from crates/eonix_hal/src/arch/x86_64/trap.rs rename to crates/eonix_hal/src/arch/x86_64/trap/mod.rs index 6a47a753..20f1472f 100644 --- a/crates/eonix_hal/src/arch/x86_64/trap.rs +++ b/crates/eonix_hal/src/arch/x86_64/trap/mod.rs @@ -1,14 +1,13 @@ mod trap_context; -use super::context::TaskContext; use core::arch::{asm, global_asm, naked_asm}; -use eonix_hal_traits::{ - context::RawTaskContext, - trap::{IrqState as IrqStateTrait, TrapReturn}, -}; +use eonix_hal_traits::context::RawTaskContext; +use eonix_hal_traits::trap::{IrqState as IrqStateTrait, TrapReturn}; pub use trap_context::TrapContext; +use super::context::TaskContext; + unsafe extern "C" { fn _default_trap_handler(trap_context: &mut TrapContext); pub fn trap_stubs_start(); @@ -147,7 +146,7 @@ global_asm!( build_isr_no_err %i .set i, i+1 .endr - + .globl _raw_trap_entry .type _raw_trap_entry @function _raw_trap_entry: @@ -157,10 +156,10 @@ global_asm!( .cfi_offset %rsp, 0x10 cfi_all_same_value - + sub $0x78, %rsp .cfi_def_cfa_offset CS - + mov %rax, RAX(%rsp) .cfi_rel_offset %rax, RAX mov %rbx, RBX(%rsp) @@ -191,24 +190,24 @@ global_asm!( .cfi_rel_offset %r15, R15 mov %rbp, RBP(%rsp) .cfi_rel_offset %rbp, RBP - + mov INT_NO(%rsp), %rcx sub ${trap_stubs_start}, %rcx shr $3, %rcx mov %rcx, INT_NO(%rsp) - + cmpq $0x08, CS(%rsp) je 2f swapgs - + 2: mov %gs:0, %rcx add ${handler}, %rcx mov (%rcx), %rcx - + jmp *%rcx .cfi_endproc - + _raw_trap_return: .cfi_startproc .cfi_def_cfa %rsp, CS @@ -228,7 +227,7 @@ global_asm!( .cfi_rel_offset %r15, R15 .cfi_rel_offset %rbp, RBP .cfi_rel_offset %rsp, RSP - + mov RAX(%rsp), %rax .cfi_restore %rax mov RBX(%rsp), %rbx @@ -259,16 +258,16 @@ global_asm!( .cfi_restore %r15 mov RBP(%rsp), %rbp .cfi_restore %rbp - + cmpq $0x08, CS(%rsp) je 2f swapgs - + 2: lea RIP(%rsp), %rsp .cfi_def_cfa %rsp, 0x08 .cfi_offset %rsp, 0x10 - + iretq .cfi_endproc ", From f870a560f5b10c890163d5aa191c65f0d5d1eb9e Mon Sep 17 00:00:00 2001 From: greatbridf Date: Mon, 26 Jan 2026 02:00:49 +0800 Subject: [PATCH 09/36] chore: remove the specific target from config.toml Instead, add an example vscode settings file for IDE use. Signed-off-by: greatbridf --- .cargo/config.toml | 1 - .vscode/settings.example.json | 10 ++++++++++ 2 files changed, 10 insertions(+), 1 deletion(-) create mode 100644 .vscode/settings.example.json diff --git a/.cargo/config.toml b/.cargo/config.toml index 9c7ba798..12e65525 100644 --- a/.cargo/config.toml +++ b/.cargo/config.toml @@ -1,5 +1,4 @@ [build] -target = "riscv64gc-unknown-none-elf" target-dir = 'build' rustflags = ["-C", "force-unwind-tables"] diff --git a/.vscode/settings.example.json b/.vscode/settings.example.json new file mode 100644 index 00000000..fbafa867 --- /dev/null +++ b/.vscode/settings.example.json @@ -0,0 +1,10 @@ +{ + "editor.formatOnSave": true, + // Uncomment and set the target you'd like to use for analysis. + // + // Available targets: + // - x86_64-unknown-none.json + // - riscv64gc-unknown-none-elf + // - loongarch64-unknown-none-softfloat + // "rust-analyzer.cargo.target": "x86_64-unknown-none.json", +} From 2779cf8caac9751e0f1998b2c8c6ef0344e73852 Mon Sep 17 00:00:00 2001 From: greatbridf Date: Sun, 18 Jan 2026 14:33:38 +0800 Subject: [PATCH 10/36] chore: remove and ignore vscode settings from svc Signed-off-by: greatbridf --- .gitignore | 1 + .vscode/settings.json | 3 --- 2 files changed, 1 insertion(+), 3 deletions(-) delete mode 100644 .vscode/settings.json diff --git a/.gitignore b/.gitignore index fbc2a9b1..4684b698 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,7 @@ build/ .idea/ +.vscode/settings.json test/ diff --git a/.vscode/settings.json b/.vscode/settings.json deleted file mode 100644 index 634d16af..00000000 --- a/.vscode/settings.json +++ /dev/null @@ -1,3 +0,0 @@ -{ - "makefile.configureOnOpen": false, -} From 916354bd5343b3d72b04197706656494a0860f81 Mon Sep 17 00:00:00 2001 From: greatbridf Date: Tue, 20 Oct 2026 00:12:13 +0800 Subject: [PATCH 11/36] agents: add fix-and-pr workflow as a skill Add a project-scoped Pi skill that guides kernel issue reproduction, root-cause analysis, fix verification, and PR submission. The skill requires GH_TOKEN before doing any work and uses phase-gated subagent handoffs. Split the detailed protocol into phase-specific reference documents so only the relevant guidance is loaded for each phase. Signed-off-by: greatbridf --- .agents/skills/fix-and-pr/SKILL.md | 190 +++++++++++++++ .../references/00-operating-contract.md | 121 ++++++++++ .../references/01-environment-and-intake.md | 152 ++++++++++++ .../fix-and-pr/references/02-reproduction.md | 219 ++++++++++++++++++ .../fix-and-pr/references/03-analysis.md | 148 ++++++++++++ .../references/04-fix-and-commits.md | 153 ++++++++++++ .../references/05-commit-conventions.md | 70 ++++++ .../fix-and-pr/references/06-verification.md | 105 +++++++++ .../fix-and-pr/references/07-pr-submission.md | 180 ++++++++++++++ .../references/08-failure-handling.md | 25 ++ .../references/09-analysis-template.md | 55 +++++ .../references/10-command-reference.md | 46 ++++ .../references/11-prompt-hygiene.md | 23 ++ .../skills/fix-and-pr/references/README.md | 42 ++++ 14 files changed, 1529 insertions(+) create mode 100644 .agents/skills/fix-and-pr/SKILL.md create mode 100644 .agents/skills/fix-and-pr/references/00-operating-contract.md create mode 100644 .agents/skills/fix-and-pr/references/01-environment-and-intake.md create mode 100644 .agents/skills/fix-and-pr/references/02-reproduction.md create mode 100644 .agents/skills/fix-and-pr/references/03-analysis.md create mode 100644 .agents/skills/fix-and-pr/references/04-fix-and-commits.md create mode 100644 .agents/skills/fix-and-pr/references/05-commit-conventions.md create mode 100644 .agents/skills/fix-and-pr/references/06-verification.md create mode 100644 .agents/skills/fix-and-pr/references/07-pr-submission.md create mode 100644 .agents/skills/fix-and-pr/references/08-failure-handling.md create mode 100644 .agents/skills/fix-and-pr/references/09-analysis-template.md create mode 100644 .agents/skills/fix-and-pr/references/10-command-reference.md create mode 100644 .agents/skills/fix-and-pr/references/11-prompt-hygiene.md create mode 100644 .agents/skills/fix-and-pr/references/README.md diff --git a/.agents/skills/fix-and-pr/SKILL.md b/.agents/skills/fix-and-pr/SKILL.md new file mode 100644 index 00000000..5122e5d2 --- /dev/null +++ b/.agents/skills/fix-and-pr/SKILL.md @@ -0,0 +1,190 @@ +--- +name: fix-and-pr +description: Reproduce, debug, fix, verify, and submit GBOS/Eonix kernel issues as pull requests. Use for kernel bug fixing, headless QEMU reproduction scripts, subagent-driven root-cause analysis, commit preparation, and credential-safe PR creation. +--- + +# GBOS / Eonix Fix and PR + +## Mandatory credential gate + +Run this before **any** other action: preflight, branch creation, issue +fetching, builds, analysis, or subagent launches. + +1. Check `GH_TOKEN` without printing it: + + ```bash + test -n "${GH_TOKEN:-}" + ``` + +2. If the check succeeds, continue. +3. If `GH_TOKEN` is unset or empty, stop immediately. Ask the user for a GitHub + token, preferably by exporting it: + + ```text + GH_TOKEN is not set. Please provide a GitHub token by exporting it in your + shell, for example: + export GH_TOKEN='...' + Tell me when it is set; I will not continue until then. + ``` + +4. Do not run tools, inspect the repository, spawn subagents, or begin the + workflow until the user confirms it is available and the check succeeds. +5. Never echo, log, commit, or put `GH_TOKEN` in a subagent prompt. Do not + write it to a file. + +The detailed protocol is split by phase under `references/`. Read +`references/README.md`, then `references/00-operating-contract.md`, then the +document for the current phase. Load only the documents needed for that phase. + +If the task is not a GBOS/Eonix kernel issue, keep the same phase gates but +replace repository-specific paths, build commands, and remotes. + +## When this skill applies + +- A GitHub issue, bug report, stack trace, or failing test must be reproduced + and fixed. +- A QEMU-based kernel reproduction script is needed. +- The work needs subagent orchestration for analysis, implementation, review, + or PR publication. +- The user asks to prepare commits or open a PR for a verified fix. + +## Entry point + +```bash +ISSUE= +SLUG= +BASE= +ART="build/work/issue-${ISSUE}" +mkdir -p "$ART" +``` + +Then: + +1. Confirm the credential gate above passes. +2. Read `references/README.md` and `references/00-operating-contract.md`. +3. Run the repository preflight in `references/01-environment-and-intake.md`. +4. Capture the issue in `$ART/issue.json`. +5. Follow the phase gates below in order, loading the matching reference + document when entering each phase. + +## Phase gates + +| Phase | Required evidence | Reference document | +| --- | --- | --- | +| 0. Intake and environment | `GH_TOKEN` present; clean checkout, configured toolchain, successful build, target QEMU boot, issue details captured. | [01 — Environment and issue intake](references/01-environment-and-intake.md) | +| 1. Reproduction | Headless script independently verified to distinguish bug from fixed behavior. | [02 — Automated reproduction](references/02-reproduction.md) | +| 2. Analysis | Root cause stated with evidence; fix boundary and invariants recorded. | [03 — Root-cause analysis](references/03-analysis.md) | +| 3. Fix and commits | Minimal scoped fix; clean, independently buildable commits with repository-style messages. | [04 — Fix planning and implementation](references/04-fix-and-commits.md), [05 — Commit conventions](references/05-commit-conventions.md) | +| 4. Verification and review | Baseline `--expect bug` passes, fixed `--expect fixed` passes, builds/tests pass, independent review has no blockers. | [06 — Verification and review](references/06-verification.md) | +| 5. PR | Correct base/head, pushed branch, PR body with verification evidence, local and PR diffs match, no secrets. | [07 — PR submission](references/07-pr-submission.md) | + +Do not advance because a plausible fix exists. Advance only when the gate +evidence exists. + +## Critical rules + +1. **One writer per worktree.** Never run concurrent writers in the same + checkout. +2. **No reproduction, no fix.** Do not implement a speculative fix. +3. **No post-fix verification, no PR.** The same reproducer must pass in fixed + mode after the fix. +4. **Never use `git add -A` or `git add .`.** Add explicit paths only. +5. **Never leak `GH_TOKEN`.** Do not print it, pass it in prompts, write it to + disk, embed it in URLs, or commit it. +6. **Keep the fix minimal.** No unrelated refactors, formatting sweeps, or + dependency updates. +7. **Separate authoring from review.** Use a fresh-context `reviewer` for the + independent diff review. +8. **Stop on ambiguity.** Ask before changing unspecified behavior or choosing + a PR base branch. + +## Reproduction contract + +Save the script at: + +```text +script/repro/issue--.exp +``` + +It must support: + +```text +--expect bug|fixed +--arch +--mode +--smp +--timeout +--log +--help +``` + +Exit codes: + +| Code | Meaning | +| --- | --- | +| `0` | Observed behavior matched `--expect` | +| `1` | Observed behavior contradicted `--expect` | +| `2` | Infrastructure failure | +| `3` | Usage error | +| `130` | Interrupted | + +Use `make test-run` for headless execution. Give QEMU a unique name and kill +only that instance; never use global `killall qemu-system-*`. + +Before trusting the script, have a fresh verifier confirm: + +- `--expect bug` exits 0 on the unmodified tree. +- `--expect fixed` exits 1 on the unmodified tree (negative control). +- No QEMU process remains after any exit path. +- Logs contain no secrets and are bounded. + +## Supporting references + +- [08 — Failure handling](references/08-failure-handling.md) +- [09 — Issue analysis template](references/09-analysis-template.md) +- [10 — Quick command reference](references/10-command-reference.md) +- [11 — Prompt hygiene](references/11-prompt-hygiene.md) + +## Artifacts + +Keep issue state outside chat history: + +```text +build/work/issue-/ +├── issue.json +├── analysis.md +├── plan.md +├── script-review.md +├── before.log +├── after.log +└── pr.md +``` + +`build/` is ignored by Git, but `make clean` deletes it. Preserve important logs +elsewhere before a clean rebuild. + +## Subagent roles + +| Role | Use | +| --- | --- | +| `scout` | Read-only issue intake, code mapping, log analysis, hypothesis generation. | +| `worker` | One bounded implementation or script-generation task in one worktree. | +| `reviewer` | Fresh-context challenge of the reproduction script, root cause, plan, or final diff. | +| Publisher `worker` | Push the verified branch and create the PR; no code changes. | + +The supervisor retains phase-gate decisions, final diff ownership, and +publication authority. + +## Output expectations + +When reporting progress or completion, include: + +- Issue id and chosen base/head branches. +- Root cause and changed files. +- Baseline and fixed reproduction results with log paths. +- Build/test commands and exit codes. +- Independent review verdict. +- Commit hashes and PR URL, if created. +- Remaining risks or blockers. + +READ ALL REFERENCE DOCUMENTATIONS UNDER THIS SKILL'S references/ DIRECTORY BEFORE CONTINUING! diff --git a/.agents/skills/fix-and-pr/references/00-operating-contract.md b/.agents/skills/fix-and-pr/references/00-operating-contract.md new file mode 100644 index 00000000..636e9f48 --- /dev/null +++ b/.agents/skills/fix-and-pr/references/00-operating-contract.md @@ -0,0 +1,121 @@ +# 00 — Operating contract + +Part of the GBOS/Eonix `fix-and-pr` skill. + +A standardized, end-to-end workflow for reproducing kernel issues, finding the +root cause, implementing and verifying a fix, and opening a pull request with +subagents. + +The workflow is deliberately **evidence-gated**. Do not advance to the next +phase because a plausible fix exists; advance only when the previous phase has +produced verifiable evidence. + +--- + +## 0. Operating contract + +### 0.1 Roles + +| Role | Owner | Responsibility | +| --- | --- | --- | +| Supervisor | Parent agent | Owns issue interpretation, phase gates, final diff, commits, and PR. Resolves ambiguity and approves publication. | +| Scout | `scout` subagent (read-only) | Fetches issue context, maps code paths, collects logs, proposes hypotheses. | +| Worker | `worker` subagent (writer) | Implements one bounded change in one worktree. | +| Reviewer | `reviewer` subagent (read-only, fresh context) | Independently challenges the reproduction script, diff, or root cause. | +| Verifier | `worker` or `reviewer` with a test-only mandate | Runs the reproduction and builds; does not edit kernel code. | +| Publisher | `worker` subagent | Pushes the branch and creates the PR. Does not change code. | + +The supervisor may perform any step directly when that is cheaper than +delegation. Subagents are used to obtain isolation, independent verification, +and parallelism, not as ceremony. + +### 0.2 Mandatory credential gate + +Before any other workflow action, check `GH_TOKEN` without printing it: + +```bash +test -n "${GH_TOKEN:-}" +``` + +- If it is set and non-empty, continue. +- If it is unset or empty, stop and ask the user for the token. Prefer asking + them to export it in their shell: + + ```text + GH_TOKEN is not set. Please provide a GitHub token by exporting it: + export GH_TOKEN='...' + Tell me when it is set; I will not continue until then. + ``` + +- Do not inspect the repository, fetch issues, build, spawn subagents, or begin + any phase until the user confirms it is available and the check succeeds. +- Never echo, log, commit, or put `GH_TOKEN` in a prompt. Do not write it to a + file. + +### 0.3 Non-negotiable rules + +1. **One writer per worktree.** Never let two subagents edit the same checkout + concurrently. +2. **No fix without reproduction.** If the issue cannot be reproduced, record + why and stop; do not implement a speculative fix. +3. **No PR without post-fix verification.** The same reproduction that failed + before the fix must pass after it. +4. **Separate evidence from authority.** A subagent report is evidence. The + supervisor decides whether it is sufficient. +5. **Never leak credentials.** `GH_TOKEN` is read from the environment only. + Never print it, put it in a URL, a prompt, a commit, a log, or a file. +6. **Never use `git add -A` or `git add .`.** Explicitly add intended files; + otherwise an untracked secret or log can enter the commit. +7. **Minimal scope.** No unrelated refactors, formatting sweeps, or speculative + improvements in a bug-fix PR. +8. **Stop on ambiguity.** Ask before changing behavior that is not clearly + required by the issue, or when the target base branch is unclear. + +### 0.4 Phase gates + +| Phase | Gate to leave the phase | +| --- | --- | +| 0. Intake and environment | `GH_TOKEN` is present; repository builds, target architecture runs, issue details captured. | +| 1. Reproduction | A headless script exists and has been independently verified to distinguish bug from fixed behavior. | +| 2. Analysis | Root cause is stated with evidence; the fix boundary is identified. | +| 3. Fix and commits | Minimal fix is implemented; commits are clean and build independently. | +| 4. Verification and review | Reproduction passes in fixed mode, baseline passes in bug mode, builds/tests pass, independent review has no blocking findings. | +| 5. PR | Branch is pushed, PR is created with the correct base/head and verification evidence, and the local diff matches the PR diff. | + +### 0.5 Persistent artifacts + +Use issue-specific paths so later subagents can resume without relying on chat +history: + +```bash +ISSUE=123 +SLUG=short-issue-slug +ART="build/work/issue-${ISSUE}" +mkdir -p "$ART" +``` + +```text +build/work/issue-/ +├── issue.json # issue metadata and comments, if fetched +├── analysis.md # observations, hypotheses, root cause +├── plan.md # fix plan and test matrix +├── script-review.md # independent reproduction-script verdict +├── before.log # --expect bug run on the unmodified tree +├── after.log # --expect fixed run on the fixed tree +└── pr.md # final PR title/body and URL +``` + +Reproduction scripts belong in `script/repro/`, because they may be reused by +later subagents and optionally committed as regression tests: + +```text +script/repro/issue--.exp +``` + +`build/` is ignored by Git, but `make clean` deletes it. Copy important logs +out before a clean rebuild, or keep the canonical logs in a temporary directory +such as `/tmp/gbos-issue-/`. + +--- + +Next: [01 — Environment and issue intake](01-environment-and-intake.md) diff --git a/.agents/skills/fix-and-pr/references/01-environment-and-intake.md b/.agents/skills/fix-and-pr/references/01-environment-and-intake.md new file mode 100644 index 00000000..d9bbac5a --- /dev/null +++ b/.agents/skills/fix-and-pr/references/01-environment-and-intake.md @@ -0,0 +1,152 @@ +# 01 — Environment and issue intake + +Part of the GBOS/Eonix `fix-and-pr` skill. Read [00-operating-contract.md](00-operating-contract.md) for the phase gates and hard rules before using this document. + +--- + +## 1. Phase 0: Intake, environment, and target architecture + +### 1.1 Credential gate and repository preflight + +Before running any repository command, confirm the mandatory credential gate +from [00 — Operating contract](00-operating-contract.md) has passed: + +```bash +test -n "${GH_TOKEN:-}" +``` + +If it fails, stop and ask the user to provide/export `GH_TOKEN`. Do not run the +commands below, spawn subagents, or start the workflow until it succeeds. + +After the gate passes, run this repository preflight: + +```bash +pwd +git status --short --branch +git remote -v +git branch --show-current +git log --oneline -10 +``` + +Requirements: + +- The checkout is on the intended base branch or a dedicated fix branch. +- There are no unintended local changes. Untracked workflow artifacts are fine + only when they are known and will not be committed. +- `GH_TOKEN` is present in the environment and has not been written to the + repository or any workflow artifact. +- The remote used for the PR is known: + - `github` often points at the upstream mirror or fork. + - `origin` may point at a private/self-hosted remote. + - Do not assume which remote is the PR target; confirm it in Phase 5. + +### 1.2 Build and runtime environment + +The kernel primarily targets **x86_64** (default target +`x86_64-acpi_compat`), with cross-support for `riscv64` and `loongarch64`. + +Key build parameters: + +| Variable | Meaning | Default | +| --- | --- | --- | +| `ARCH` | Target architecture: `x86_64`, `riscv64`, `loongarch64` | `x86_64` | +| `MODE` | Build profile: `release` or `debug` | `debug` | +| `SMP` | Number of virtual CPUs; empty disables the `smp` feature | `4` | +| `FEATURES` | Comma-separated Cargo features, e.g. `log_debug,trace_syscall` | empty | +| `QEMU_ACCEL` | QEMU accelerator; `-accel tcg` on macOS, `-accel kvm` on Linux when available | auto | +| `DEBUG_TRAPS` | Enables QEMU trap logging to `build/qemu.log` | unset | + +Configure and build: + +```bash +./configure +make build MODE=release +``` + +If `./configure` cannot find a required tool, fix the environment before +continuing. `./configure` generates the `Makefile`; the generated file is +ignored by Git. + +Interactive run: + +```bash +make run MODE=release +``` + +Wait for the shell prompt (`/ # `), run a harmless command such as +`echo READY`, then exit QEMU with `Ctrl-A`, then `x`. + +Headless/test run: + +```bash +make test-run MODE=release ARCH=x86_64 +``` + +`make test-run` uses `-serial stdio` and is the preferred target for automated +reproduction. Unlike `make run`, it does not multiplex the QEMU monitor into +the serial stream, so exit by terminating the QEMU process rather than +`Ctrl-A x`. + +Debug run: + +```bash +make srun MODE=release # starts QEMU paused with GDB stub on :1234 +make debug # attaches GDB and loads build/kernel.sym +# or, when tmux is available: +make tmux-debug +``` + +### 1.3 Create a fix branch + +Use an isolated worktree when other work may continue in the main checkout: + +```bash +git fetch github +git switch -c "fix/${ISSUE}-${SLUG}" "github/master" +# or: +git worktree add -b "fix/${ISSUE}-${SLUG}" "../eonix-${ISSUE}" "github/master" +``` + +If the issue targets a feature branch, branch from that branch instead of +`master`. Record the chosen base in `build/work/issue-/plan.md`. + +### 1.4 Capture the issue + +Preferred, if `gh` is installed: + +```bash +gh issue view "$ISSUE" \ + --repo greatbridf/osdev \ + --json number,title,body,labels,comments \ + > "$ART/issue.json" +``` + +`GH_TOKEN` is the standard environment variable read by the `gh` CLI and by the +REST fallback. Export it once for the workflow; do not write it to a file. + +Fallback with the GitHub REST API: + +```bash +curl -sS \ + -H "Authorization: Bearer ${GH_TOKEN}" \ + -H "Accept: application/vnd.github+json" \ + "https://api.github.com/repos/greatbridf/osdev/issues/${ISSUE}" \ + > "$ART/issue.json" +``` + +Do not put the token in shell history or command output. If a command would +print it, redirect the output or redact the value. + +Capture in `analysis.md`: + +- Expected behavior vs. actual behavior. +- Reproducer commands, stack traces, logs, and screenshots referenced by the + issue. +- Target architecture, `MODE`, `SMP`, and whether the bug is SMP-only. +- Whether the issue is a hang, panic, wrong result, memory corruption, race, or + build failure. +- The commit or branch the reporter used, if known. + +--- + +Next: [02 — Automated reproduction](02-reproduction.md) diff --git a/.agents/skills/fix-and-pr/references/02-reproduction.md b/.agents/skills/fix-and-pr/references/02-reproduction.md new file mode 100644 index 00000000..ec2119f8 --- /dev/null +++ b/.agents/skills/fix-and-pr/references/02-reproduction.md @@ -0,0 +1,219 @@ +# 02 — Automated reproduction + +Part of the GBOS/Eonix `fix-and-pr` skill. Read [00-operating-contract.md](00-operating-contract.md) for the phase gates and hard rules before using this document. + +--- + +## 2. Phase 1: Automated reproduction + +Manual interactive QEMU sessions easily hang terminals or leave orphaned +processes. Reproduce with a script, and verify the script itself before trusting +its result. + +### 2.1 Spawn a subagent to generate the reproduction script + +Spawn a `scout` or `worker` with a bounded, script-only mandate. The subagent +may inspect the repository, but must not edit kernel source. + +#### Generator task template + +```text +Objective +Create an automated, headless reproduction for GBOS/Eonix issue #: +. + +Repository and target +- cwd: +- ref/branch: +- target: ARCH=, MODE=release, SMP= +- issue evidence: + +Authority and boundaries +- You may create or edit only script/repro/issue--.exp. +- Do not edit kernel code, user programs, the Makefile, or repository docs. +- Do not commit. +- Do not print, copy, or commit any credential. + +Required script interface +The script must accept: + --expect bug|fixed required; a default of "bug" is acceptable if documented + --arch default x86_64 + --mode default release + --smp default 4 + --timeout overall budget, default 180 + --log default build/work/issue-/.log + --help print usage and exit without launching QEMU + +Required behavior +1. Validate arguments before launching anything. +2. Launch the kernel with `make test-run` (not `make run`), using + `ARCH`, `MODE`, and `SMP` from the arguments. +3. Give QEMU a unique name, e.g. `-name gbos-repro-`, via QEMU_ARGS, so + cleanup can target this instance only. +4. Wait for the busybox shell prompt `/ # ` and fail as infrastructure error + if it never appears. +5. Send the exact command(s) that trigger the issue. +6. Detect success, wrong output, panic, hang, or QEMU exit with explicit + timeouts. Treat a kernel panic or a timeout after the trigger as "bug + observed", not as infrastructure failure. +7. On completion, terminate only the QEMU instance started by this script — + never `killall qemu-system-*`. Kill the process group or use the unique + QEMU name. Ensure no orphan remains. +8. Write a bounded log; do not let it grow without limit. +9. Exit with the contract below. + +Exit-code contract +0 observed behavior matches --expect +1 observed behavior contradicts --expect +2 infrastructure failure: build failure, no prompt, missing tool, cleanup + failure, or timeout before the trigger could be sent +3 usage/argument error +130 interrupted by the user + +Required log markers +REPRO issue= expect= +REPRO arch= mode= smp= +REPRO booted +REPRO trigger-sent +REPRO bug-observed | REPRO fixed-observed | REPRO timeout +REPRO result=pass|fail exit= log= + +Acceptance +- `--help` exits 0 without starting QEMU. +- On the unmodified tree, `--expect bug` exits 0 and the log shows the issue. +- On the unmodified tree, `--expect fixed` exits non-zero with a mismatch, not + a false success. +- No QEMU process started by the script remains after any exit path. +- The script contains no credentials and no absolute paths tied to your home + directory. + +Deliverable +- The script path. +- The exact command used for a baseline run. +- The baseline log path and exit code. +- Any assumptions or flakiness observed. +``` + +Use `expect` or Python `pexpect`. Existing `script/test.sh` is a useful +reference for prompt matching, timeouts, and `poweroff`, but its global +`killall` cleanup is not acceptable for a reusable reproducer. + +### 2.2 Spawn a subagent to verify the reproduction script + +Do **not** immediately trust the generator. A broken script can "pass" for the +wrong reason or fail on infrastructure. The verifier must use a fresh context +and must not modify kernel code. + +#### Verifier task template + +```text +Objective +Independently verify that script/repro/issue--.exp is a trustworthy +reproduction for issue #. Do not fix the kernel. + +Repository and ref +- cwd: +- ref: +- artifacts: build/work/issue-/ + +Authority and boundaries +- Read-only for kernel code. +- You may edit only the reproduction script if it is defective, and you must + report every change. Do not edit tests, Makefile, or kernel sources. +- Do not commit, push, or create a PR. +- Never print credentials. + +Verification steps +1. Read the issue and the script. Confirm the script tests the reported behavior + rather than an unrelated proxy. +2. Run `script/repro/issue--.exp --help`; confirm it exits 0 and starts + no QEMU. +3. On the unmodified tree, run `--expect bug` in the documented configuration. + Confirm exit 0 and that the log contains the trigger and a bug/timeout/panic + marker. +4. Run the negative control on the same unmodified tree: + `--expect fixed`. It must exit 1 (mismatch) and must not report a false + success. If it exits 0, the script cannot distinguish bug from fixed. +5. Check the timeout behavior: the script must fail as infrastructure (2) if the + prompt never appears, and report "bug observed" (0 for --expect bug) if the + trigger hangs. +6. Check cleanup after every run: + `pgrep -fl 'qemu-system-.*gbos-repro-'` must return nothing. +7. Check that logs do not contain credentials, home-directory paths, or unbounded + output. +8. If the bug is timing-dependent, repeat the baseline at least 3 times and + report the pass rate. Do not hide flakiness. + +Report +Verdict: PASS | FAIL +- Script path and exact commands run. +- Baseline --expect bug: exit code, log path. +- Negative control --expect fixed: exit code, log path. +- Cleanup check: command and result. +- Flakiness: number of runs and pass rate. +- Defects found; if you edited the script, show the diff and explain why. +- Remaining uncertainty. +``` + +If the verifier cannot make the script distinguish bug from fixed behavior, +return to the generator. Do not proceed to analysis with an ambiguous test. + +### 2.3 Running and reusing the script + +Use the same script and configuration throughout the workflow. + +```bash +# On the unmodified tree: confirm the bug is present. +script/repro/issue-${ISSUE}-${SLUG}.exp \ + --expect bug \ + --arch x86_64 --mode release --smp 4 \ + --log "$ART/before.log" + +# After the fix and rebuild: confirm the bug is gone. +script/repro/issue-${ISSUE}-${SLUG}.exp \ + --expect fixed \ + --arch x86_64 --mode release --smp 4 \ + --log "$ART/after.log" +``` + +Interpretation: + +| Exit | Meaning | +| --- | --- | +| `0` | Observed behavior matched `--expect`; this is a successful run. | +| `1` | Behavior contradicted `--expect`; the fix is incomplete or the baseline did not reproduce. | +| `2` | Infrastructure failure; fix the environment before drawing conclusions. | +| `3` | Invalid invocation; fix the command. | + +The exact same script and arguments must be used before and after the fix. If +the script must change, re-run both baseline and fixed cases with the new +version and record why it changed. + +For SMP or race-related issues, run a matrix and keep all logs: + +```bash +for smp in 1 4; do + for i in 1 2 3; do + script/repro/issue-${ISSUE}-${SLUG}.exp \ + --expect bug --smp "$smp" \ + --log "$ART/before-smp${smp}-run${i}.log" + done +done +``` + +### 2.4 If the issue cannot be reproduced + +Stop and report: + +- The exact commands and environment tried. +- The arch, `MODE`, `SMP`, and QEMU accelerator used. +- Whether the issue depends on a feature branch, a user program, an additional + disk image (`IMG=`), or a particular init script. +- The closest observed behavior and its log. + +Ask the issue reporter or supervisor for missing details. Do not invent a +reproduction by weakening the assertion or by testing a different behavior. + +--- + +Next: [03 — Root-cause analysis and debugging](03-analysis.md) diff --git a/.agents/skills/fix-and-pr/references/03-analysis.md b/.agents/skills/fix-and-pr/references/03-analysis.md new file mode 100644 index 00000000..e279bfa5 --- /dev/null +++ b/.agents/skills/fix-and-pr/references/03-analysis.md @@ -0,0 +1,148 @@ +# 03 — Root-cause analysis and debugging + +Part of the GBOS/Eonix `fix-and-pr` skill. Read [00-operating-contract.md](00-operating-contract.md) for the phase gates and hard rules before using this document. + +--- + +## 3. Phase 2: Root-cause analysis and debugging + +The goal is a **causal explanation**, not a patch that happens to make the test +green. + +### 3.1 Freeze the evidence + +Before editing anything: + +```bash +git rev-parse HEAD > "$ART/baseline-commit.txt" +git status --short > "$ART/baseline-status.txt" +cp "$ART/before.log" "$ART/before.keep.log" +``` + +Preserve: + +- The baseline commit and branch. +- The exact reproduction command and configuration. +- The complete panic/backtrace text. +- The QEMU log, if trap logging was enabled. +- Any kernel symbols or disassembly used. + +For trap logging: + +```bash +DEBUG_TRAPS=1 make test-run MODE=release ARCH=x86_64 +# QEMU writes detailed logs to build/qemu.log +``` + +### 3.2 Static reconnaissance + +Map the code path before changing it: + +```bash +rg -n "" src crates macros +rg -n "" src crates +git log --oneline --all -- +git blame -L , -- +``` + +Inspect: + +- Callers and callees of the failing function. +- Lock ordering, reference counting, and lifetime assumptions. +- Error paths: allocation failure, invalid user pointers, partial initialization. +- Architecture-specific implementations under `src/` and `crates/eonix_hal`. +- Related filesystem, scheduler, VFS, MM, or syscall code. +- Existing tests and documentation under `doc/` and `script/`. + +For GBOS/Eonix, explicitly consider: + +- **SMP:** Is the code correct with concurrent access? Does it assume a single + CPU? Is the lock held across an await point or a blocking operation? +- **Interrupts/preemption:** Can the state change between check and use? +- **Refcounts:** Who owns each reference, and which path drops it? +- **Page tables/folios:** Are mappings, CoW, and page flags updated atomically? +- **User memory:** Is the access validated, and can it fault mid-copy? +- **Architecture:** Does the fix need a RISC-V or LoongArch counterpart? +- **ABI:** Does the change alter syscall return values, errno, or struct layout? + +### 3.3 Dynamic analysis + +Use the smallest tool that answers the next question. + +- **QEMU trap log:** `DEBUG_TRAPS=1` and inspect `build/qemu.log`. +- **GDB:** `make srun` then `make debug`, or `make tmux-debug`. +- **Backtrace:** `bt full`, `thread apply all bt`, `info registers`. +- **Memory/state:** `x/`, `p`, `p/x`, `info threads`, `monitor info registers`. +- **Tracing:** enable `trace_syscall`, `trace_scheduler`, `trace_pci`, or + `log_debug` only as needed. Remove or feature-gate temporary prints before + committing. +- **SMP comparison:** run `SMP=1` and `SMP=4`; a difference is strong evidence + of a race or missing synchronization. +- **Architecture comparison:** if x86_64 fails, check whether the shared code + or the HAL is responsible; test a second architecture when feasible. + +Do not leave GDB or QEMU running between phases. Record the exact commands and +observations in `analysis.md`. + +### 3.4 Form and disprove hypotheses + +Write down candidates before testing: + +```text +Hypothesis: +Expected behavior: +Actual behavior: +Evidence for: +Evidence against: +Falsification test: +``` + +Rules: + +- Prefer the hypothesis that explains the **first** wrong state, not the last + visible symptom. +- Distinguish the bug from the mechanism that exposes it. +- If a hypothesis predicts a log line, register value, or code path, verify it + directly. +- If two independent hypotheses remain, run one read-only `scout` per + hypothesis; do not run duplicate scouts with the same prompt. +- Keep failed hypotheses in `analysis.md`; they prevent the next agent from + repeating the work. + +### 3.5 Write the root-cause statement + +Before planning the fix, complete this sentence with evidence: + +```text +Root cause: causes because , as shown by +. +``` + +Also record: + +- The smallest code boundary that must change. +- What must **not** change. +- Which invariants the fix must preserve. +- Which tests or configurations could regress. + +If the root cause is not established, do not proceed to Phase 3. + +### 3.6 Optional independent analysis + +For a high-risk or surprising diagnosis, spawn a fresh `reviewer` with the +evidence and ask it to challenge the conclusion: + +```text +Challenge this root-cause claim using the supplied issue, logs, and code paths. +Do not edit files. Identify: +1. evidence that contradicts the claim; +2. a simpler competing explanation; +3. a falsification test the supervisor can run; +4. any lifecycle/locking/refcount assumption that is not proven. +Return a concise verdict: supported, uncertain, or contradicted. +``` + +--- + +Next: [04 — Fix planning and implementation](04-fix-and-commits.md) diff --git a/.agents/skills/fix-and-pr/references/04-fix-and-commits.md b/.agents/skills/fix-and-pr/references/04-fix-and-commits.md new file mode 100644 index 00000000..9d8f062f --- /dev/null +++ b/.agents/skills/fix-and-pr/references/04-fix-and-commits.md @@ -0,0 +1,153 @@ +# 04 — Fix planning and implementation + +Part of the GBOS/Eonix `fix-and-pr` skill. Read [00-operating-contract.md](00-operating-contract.md) for the phase gates and hard rules before using this document. + +--- + +## 4. Phase 3: Implement the fix + +### 4.1 Plan the fix + +Write `build/work/issue-/plan.md`: + +```markdown +# Fix plan: issue + +## Root cause +... + +## Change boundary +Files/functions to change: +- ... + +Explicitly out of scope: +- ... + +## Invariants +- ... + +## Failure paths to preserve +- ... + +## Commit split +1. +2. +3. + +## Verification matrix +| Case | Arch | MODE | SMP | Command | Expected | +| --- | --- | --- | --- | --- | --- | +| Baseline repro | x86_64 | release | 4 | `--expect bug` | 0 | +| Fixed repro | x86_64 | release | 4 | `--expect fixed` | 0 | +| Build | x86_64 | release | 4 | `make build` | 0 | +| Existing smoke | x86_64 | release | 4 | `script/test.sh` | 0 | + +## Risks / open questions +- ... +``` + +For a complex or cross-subsystem fix, have a fresh `reviewer` critique the plan +before any edit: + +```text +Review this fix plan against the established root cause and repository +conventions. Do not edit files. Identify missing lock/lifetime/refcount/error +paths, missing architecture variants, and any step that is broader than the +root cause requires. Return blocking issues first, then non-blocking suggestions. +``` + +Approve the plan only when the change boundary and verification matrix are +explicit. + +### 4.2 Write the fix with one worker + +Use one writer per worktree. The task must state the exact files or subsystem +the worker may touch. + +```text +Objective +Implement the approved fix for issue # according to plan.md. + +Repository and ref +- cwd: +- branch: fix/- +- base commit: + +Authority +- You may edit only: . +- Do not edit unrelated modules, do not reformat files, do not update + dependencies, and do not add debug prints that survive the commit. +- Do not commit unless the supervisor asks you to. +- Stop and report if the plan is wrong or if a second, independent bug is found. + +Required work +1. Read the relevant code and the root-cause analysis before editing. +2. Apply the minimal change that restores the violated invariant. +3. Handle error paths and partial-failure cases. +4. Preserve SMP, interrupt, and preemption safety where applicable. +5. Run `cargo fmt --all -- --check` if the repository expects rustfmt. +6. Build: `make build MODE=release` and `make build MODE=debug` for the target + architecture; add a second architecture when the change is shared. +7. Run the reproduction: + - `--expect bug` must still exit 0 on the baseline commit. + - `--expect fixed` must exit 0 on the fixed tree. +8. Report changed files, commands, exit codes, and log paths. + +Acceptance +- The reproduction passes in fixed mode. +- The baseline still reproduces. +- No unrelated diff. +- No secrets, logs, or build artifacts staged. +``` + +Implementation guardrails: + +- Make the smallest change that fixes the root cause. Do not combine a bug fix + with a broad cleanup. +- If a refactor is required to make the fix safe, keep it in a separate commit. +- Remove temporary instrumentation or put it behind an existing feature flag. +- If the build or reproduction fails, collect evidence and return to the + supervisor. Do not keep editing blindly. +- If a second independent bug appears, record it and ask whether to expand the + scope; do not silently fix it. + +### 4.3 Refine and split the fix + +After the first working version, review the diff as if it were hostile: + +```bash +git diff --stat +git diff +git diff --check +``` + +Check: + +- Is every changed line necessary for the root cause? +- Are lock order, reference ownership, and error paths still correct? +- Is the fix race-free under the reported `SMP` configuration? +- Are architecture-specific assumptions handled? +- Is user input validated on all paths? +- Are there any behavior changes that the issue does not require? +- Are debug prints, temporary hacks, and unrelated formatting removed? + +If the fix is clearer as several commits, split it while preserving +bisectability. A typical sequence is: + +1. `refactor:` or `prep:` — a behavior-preserving preparation, if required. +2. `fix:` — the actual bug fix. +3. `test:` — an optional regression test or reproducer. + +Every commit must build on its own. Use an interactive rebase only on a private +branch: + +```bash +git rebase -i "github/${BASE}" +git diff "github/${BASE}"...HEAD +``` + +Do not rewrite a shared branch that others may have based work on. + +--- + +Next: [05 — Commit conventions](05-commit-conventions.md) diff --git a/.agents/skills/fix-and-pr/references/05-commit-conventions.md b/.agents/skills/fix-and-pr/references/05-commit-conventions.md new file mode 100644 index 00000000..e453aa3e --- /dev/null +++ b/.agents/skills/fix-and-pr/references/05-commit-conventions.md @@ -0,0 +1,70 @@ +# 05 — Commit conventions + +Part of the GBOS/Eonix `fix-and-pr` skill. Read [00-operating-contract.md](00-operating-contract.md) for the phase gates and hard rules before using this document. + +--- + +## 4.4 Commit conventions + +Follow the repository's existing style. Commit subjects use a subsystem prefix: + +```text +: +``` + +Examples from this repository: + +```text +fix/futex: forget to .await in `futex_exit` +mm: introduce MapFolio for folios mapped to page tables +vfs, inode: remove {read,write}_direct +style, vfs: reformat fat32.rs, file.rs and inode.rs +``` + +Commit body requirements: + +- What failed and under which conditions. +- Why it failed (root cause), not just what changed. +- How the fix restores the invariant. +- Reference the fixed commit or issue: + - `Fixes: ("")` + - `Fixes #` +- Use `Signed-off-by` if the repository expects it (`git commit -s`). +- Keep lines readable (typically 72 columns). + +Example: + +```text +fix/futex: wake waiters in futex_exit + +Testcase `pthread_test` freezes while waiting for clear-tid futex wakeups. +`futex_exit()` forgets to await `do_futex_exit()`, so the exit path returns +before the wakeup is performed. + +Await the exit operation so waiters are woken before the task is reaped. + +Fixes: ("") +Fixes # +Signed-off-by: +``` + +Before committing: + +```bash +git status --short +git diff --cached --name-only +``` + +Commit explicitly, never with `-A`: + +```bash +git add path/to/file.rs path/to/other.rs +git commit -s +``` + +If the commit must be amended after review, re-run the build and reproduction +before force-pushing. + +--- + +Next: [06 — Verification and independent review](06-verification.md) diff --git a/.agents/skills/fix-and-pr/references/06-verification.md b/.agents/skills/fix-and-pr/references/06-verification.md new file mode 100644 index 00000000..afea115b --- /dev/null +++ b/.agents/skills/fix-and-pr/references/06-verification.md @@ -0,0 +1,105 @@ +# 06 — Verification and independent review + +Part of the GBOS/Eonix `fix-and-pr` skill. Read [00-operating-contract.md](00-operating-contract.md) for the phase gates and hard rules before using this document. + +--- + +## 5. Phase 4: Verification and independent review + +Verification is a separate phase. The author's successful run is not enough. + +### 5.1 Verification matrix + +Run all applicable rows from `plan.md`. At minimum: + +```bash +# Rebuild from the fixed tree. +make build MODE=release ARCH=x86_64 +make build MODE=debug ARCH=x86_64 + +# The exact same reproducer, now expecting the fix. +script/repro/issue-${ISSUE}-${SLUG}.exp \ + --expect fixed --arch x86_64 --mode release --smp 4 \ + --log "$ART/after.log" + +# Existing repository smoke test, if it is applicable and available. +ARCH=x86_64 sh script/test.sh + +# Diff hygiene. +git diff --check +git diff "github/${BASE}"...HEAD +``` + +For shared code, also build and, where feasible, run a second architecture: + +```bash +make build MODE=release ARCH=riscv64 +make build MODE=release ARCH=loongarch64 +``` + +For SMP/race issues: + +- Run the reproduction at least 5–10 times in the fixed configuration. +- Run with `SMP=1` and `SMP=4`. +- Record failures and flakiness, not just the final green run. + +Acceptance for this phase: + +- Baseline `--expect bug` still exits 0 on the base commit. +- Fixed `--expect fixed` exits 0 on the fixed commit. +- Builds pass for the required architectures and modes. +- Existing tests pass, or any failure is proven unrelated and documented. +- No orphan QEMU processes remain. +- Logs are saved and contain no secrets. + +### 5.2 Independent fix review + +Spawn a fresh `reviewer` that did not write the fix: + +```text +Review the diff for issue # against base . Do not edit files. + +Inputs +- diff: `git diff github/...HEAD` +- root cause: build/work/issue-/analysis.md +- plan: build/work/issue-/plan.md +- before/after logs: build/work/issue-/before.log, after.log + +Check, in order: +1. Correctness: does the change actually fix the stated root cause? +2. Lifecycle and ownership: refcounts, locks, lock ordering, awaits, + interrupts, and preemption. +3. Error paths: allocation failure, invalid user pointers, partial setup, + teardown, and double-free/use-after-free. +4. Concurrency: SMP races, lost wakeups, check-then-act windows. +5. Architecture: shared-code assumptions and missing HAL/arch variants. +6. Scope: unrelated changes, debug prints, formatting noise, new dependencies. +7. Verification: does the evidence actually prove the fix, or only a proxy? + +Report blocking findings first, with file:line and a concrete failure scenario. +Then report non-blocking suggestions. If there are no blocking findings, say so +explicitly. Do not approve merely because the reproducer passes. +``` + +If the reviewer finds a blocking issue, return to Phase 3 to correct it, then +re-run the full Phase 4 verification matrix. Do not open a PR with an unresolved +review finding. + +### 5.3 Definition of done + +- [ ] Issue details and expected behavior are recorded. +- [ ] Reproduction script exists and is independently verified. +- [ ] Baseline `--expect bug` exits 0. +- [ ] Fixed `--expect fixed` exits 0. +- [ ] Root cause is stated with evidence. +- [ ] Fix is minimal and scoped to the root cause. +- [ ] Required builds and tests pass. +- [ ] SMP/race behavior is tested when relevant. +- [ ] Independent review has no blocking findings. +- [ ] Commits are clean, build independently, and follow conventions. +- [ ] No secrets, logs, or build artifacts are staged. +- [ ] PR base and head branches are correct. + +--- + +Next: [07 — PR submission](07-pr-submission.md) diff --git a/.agents/skills/fix-and-pr/references/07-pr-submission.md b/.agents/skills/fix-and-pr/references/07-pr-submission.md new file mode 100644 index 00000000..c5afb85e --- /dev/null +++ b/.agents/skills/fix-and-pr/references/07-pr-submission.md @@ -0,0 +1,180 @@ +# 07 — PR submission + +Part of the GBOS/Eonix `fix-and-pr` skill. Read [00-operating-contract.md](00-operating-contract.md) for the phase gates and hard rules before using this document. + +--- + +## 6. Phase 5: PR submission + +Use a `worker` as publisher. It must not change code during this phase. + +### 6.1 Pre-flight + +```bash +git status --short --branch +git log --oneline "github/${BASE}..HEAD" +git diff --stat "github/${BASE}...HEAD" +git remote -v +``` + +Confirm: + +- The working tree is clean. +- The branch contains only intended commits. +- The base branch is correct. If the issue targets a feature branch, use that + branch as the PR base; do not assume `master`. +- The head branch has been pushed to the repository expected by the PR host. +- No credential material, `GH_TOKEN` value, or workflow logs are in the diff. + +If the base branch is ambiguous, ask the maintainer or supervisor. A PR against +the wrong base can create a misleading diff and break CI. + +### 6.2 Credential handling + +The mandatory gate in [00 — Operating contract](00-operating-contract.md) has +already confirmed that `GH_TOKEN` is present. Re-check it immediately before +publishing: + +```bash +test -n "${GH_TOKEN:-}" || { + echo "GH_TOKEN is not set; ask the user before continuing." >&2 + return 1 +} +``` + +Preferred: use the existing SSH remote for the push and the token only for +GitHub API calls. `GH_TOKEN` is the standard environment variable read by `gh`. +Export it before publishing: + +```bash +gh pr create ... +``` + +Rules: + +- Never `echo "$GH_TOKEN"`, enable `set -x` around credential handling, or + write the value to a file. +- Never embed the token in a remote URL such as + `https://@github.com/...`. +- Never put the token in a subagent prompt, PR body, commit message, or log. +- If an HTTPS push is unavoidable, use a temporary `GIT_ASKPASS` helper that + reads `GH_TOKEN` from the environment, set it to mode `0700`, and remove it + afterward. Prefer `gh`/SSH when available. +- Redact token-bearing URLs from any command output returned to the supervisor. + +### 6.3 Publisher subagent task + +```text +Objective +Publish the verified fix for issue # as a pull request. Do not modify code. + +Parameters +- Local cwd: +- Target repository: greatbridf/osdev +- Base branch: +- Head branch: +- Remote to push to: +- PR title: : +- PR body file: build/work/issue-/pr.md + +Authority +- You may run git push and the GitHub API/CLI. +- Do not edit source files, amend commits, rebase, force-push, or merge. +- Do not print or persist `GH_TOKEN`. +- If authentication fails or the base/head is ambiguous, stop and report. + +Steps +1. Verify the local branch, HEAD sha, clean worktree, and intended diff. +2. Confirm `GH_TOKEN` is present in the environment. Do not read it from a + file. Ensure it is exported; `gh` reads it directly. +3. Push the head branch to its remote. Use SSH if configured; otherwise use a + temporary askpass mechanism that reads `GH_TOKEN`, never a tokenized URL. +4. Create a PR against the target repository with the given base and head. + Include the verification evidence and `Fixes #` in the body. +5. Read back the PR: URL, number, base, head, head SHA, mergeable state, body. +6. Confirm the PR diff matches the local diff and contains no secrets or + unintended files. +7. Report the PR URL, head SHA, checks observed, and any failure. + +Stop conditions +- Push rejected and would require force. +- PR base/head mismatch. +- Token invalid or insufficient scope. +- PR diff contains files outside the intended change. +``` + +Recommended `gh` command: + +```bash +gh pr create \ + --repo greatbridf/osdev \ + --base "$BASE" \ + --head "$HEAD" \ + --title "$TITLE" \ + --body-file "$ART/pr.md" +``` + +If the head branch lives on a fork, use the fully qualified head form required +by the host, for example `--head :`. + +If `gh` is unavailable, use the GitHub REST API with `GH_TOKEN`. Never place the +token in the JSON body or URL. + +### 6.4 PR content + +Use a concise, evidence-based body: + +```markdown +## Summary + + +## Root cause + + +## Fix + + +## Verification +- Baseline reproduction: `script/repro/issue--.exp --expect bug` — exit 0, `before.log` +- Fixed reproduction: `... --expect fixed` — exit 0, `after.log` +- Build: `make build MODE=release ARCH=x86_64` — pass +- Build: `make build MODE=release ARCH=riscv64` — pass (if applicable) +- Existing smoke test: `ARCH=x86_64 sh script/test.sh` — pass (if applicable) +- SMP stress: 10/10 runs passed at `SMP=4` (if applicable) + +## Risks / notes + + +Fixes # +``` + +Do not paste full logs into the PR. Link or summarize them, and attach only what +is necessary. Do not include machine-local absolute paths. + +### 6.5 Post-create validation + +- Open the PR URL and confirm the title, base, head, and body. +- Confirm `Fixes #` is present so the issue closes when merged. +- Confirm CI starts and record the check names. +- Confirm the PR diff matches `git diff "github/${BASE}"...HEAD`. +- Confirm no credential value, `build/`, or workflow logs appear in the diff. +- If CI fails, return to Phase 4 with the failing log; do not "fix" CI by + weakening the reproduction. + +Report to the user: + +```text +PR: +Head: @ +Base: +Reproduction: before=pass (bug), after=pass (fixed) +Builds: +Review: +CI: +``` + +Do not merge the PR unless the user explicitly asks. + +--- + +Next: [08 — Failure handling and escalation](08-failure-handling.md) diff --git a/.agents/skills/fix-and-pr/references/08-failure-handling.md b/.agents/skills/fix-and-pr/references/08-failure-handling.md new file mode 100644 index 00000000..ee69dd40 --- /dev/null +++ b/.agents/skills/fix-and-pr/references/08-failure-handling.md @@ -0,0 +1,25 @@ +# 08 — Failure handling and escalation + +Part of the GBOS/Eonix `fix-and-pr` skill. Read [00-operating-contract.md](00-operating-contract.md) for the phase gates and hard rules before using this document. + +--- + +## 7. Failure handling and escalation + +| Situation | Action | +| --- | --- | +| `GH_TOKEN` missing | Stop immediately. Ask the user to provide/export it. Do not start any phase or spawn subagents until it is set. | +| Cannot reproduce | Return to issue intake. Ask for arch, mode, SMP, image, and exact commands. Do not guess a fix. | +| Reproduction is flaky | Run multiple times, record pass rate, distinguish race from infrastructure, and test `SMP=1` vs `SMP=4`. | +| Build fails after a fix | Fix the build first; do not run the reproducer on a stale binary. | +| Fix makes the reproducer pass but root cause is unclear | Stop. A passing proxy is not a root cause. Return to Phase 2. | +| Reviewer finds a blocking issue | Return to Phase 3, correct it, and re-run the full Phase 4 matrix. | +| Fix requires a broad refactor | Split the work or ask for scope approval. Keep the bug-fix PR minimal. | +| Second independent bug appears | Record it, ask whether to expand scope, and do not silently include it. | +| Auth/push/PR failure | Stop and report the exact failure. Never print `GH_TOKEN` or switch to a tokenized URL. | +| Subagent infrastructure failure | Report the run state and partial diff. Do not silently switch execution modes. | +| Base branch ambiguous | Ask. Do not default to `master` when the issue targets a feature branch. | + +--- + +Next: [09 — Issue analysis template](09-analysis-template.md) diff --git a/.agents/skills/fix-and-pr/references/09-analysis-template.md b/.agents/skills/fix-and-pr/references/09-analysis-template.md new file mode 100644 index 00000000..eac2990c --- /dev/null +++ b/.agents/skills/fix-and-pr/references/09-analysis-template.md @@ -0,0 +1,55 @@ +# 09 — Issue analysis template + +Part of the GBOS/Eonix `fix-and-pr` skill. Read [00-operating-contract.md](00-operating-contract.md) for the phase gates and hard rules before using this document. + +--- + +## Appendix A: Issue analysis record + +Keep this in `build/work/issue-/analysis.md`: + +```markdown +# Issue : + +## Reported behavior +Expected: +Actual: + +## Environment +- Credential gate: `GH_TOKEN` present (never record the value) +- Commit/branch: +- ARCH: +- MODE: +- SMP: +- QEMU / accelerator: +- Extra image or user program: + +## Reproduction +- Script: +- Command: +- Baseline result: +- Log: + +## Timeline / observations +1. ... +2. ... + +## Candidate hypotheses +| Hypothesis | Evidence for | Evidence against | Test | Result | +| --- | --- | --- | --- | --- | + +## Root cause +<Specific mechanism and evidence.> + +## Fix boundary +Must change: +Must not change: +Invariants: + +## Open questions +- ... +``` + +--- + +Next: [10 — Quick command reference](10-command-reference.md) diff --git a/.agents/skills/fix-and-pr/references/10-command-reference.md b/.agents/skills/fix-and-pr/references/10-command-reference.md new file mode 100644 index 00000000..da40139d --- /dev/null +++ b/.agents/skills/fix-and-pr/references/10-command-reference.md @@ -0,0 +1,46 @@ +# 10 — Quick command reference + +Part of the GBOS/Eonix `fix-and-pr` skill. Read [00-operating-contract.md](00-operating-contract.md) for the phase gates and hard rules before using this document. + +--- + +## Appendix B: Quick command reference + +```bash +# Mandatory credential gate +test -n "${GH_TOKEN:-}" || echo "Set GH_TOKEN before doing any work" + +# Environment +./configure +make build MODE=release +make run MODE=release # interactive; exit with Ctrl-A then x +make test-run MODE=release # headless, serial stdio +make srun MODE=release # paused QEMU + GDB stub +make debug # attach GDB +make tmux-debug # both, in tmux +DEBUG_TRAPS=1 make test-run # writes build/qemu.log + +# Reproducer +script/repro/issue-<id>-<slug>.exp --expect bug --log "$ART/before.log" +script/repro/issue-<id>-<slug>.exp --expect fixed --log "$ART/after.log" + +# Diff and commit hygiene +git diff --check +git status --short +git diff --cached --name-only +git log --oneline "github/${BASE}..HEAD" + +# Verification +ARCH=x86_64 sh script/test.sh +make build MODE=release ARCH=riscv64 +make build MODE=release ARCH=loongarch64 + +# PR +gh pr create \ + --repo greatbridf/osdev --base "$BASE" --head "$HEAD" \ + --title "$TITLE" --body-file "$ART/pr.md" +``` + +--- + +Next: [11 — Prompt hygiene](11-prompt-hygiene.md) diff --git a/.agents/skills/fix-and-pr/references/11-prompt-hygiene.md b/.agents/skills/fix-and-pr/references/11-prompt-hygiene.md new file mode 100644 index 00000000..c8a62207 --- /dev/null +++ b/.agents/skills/fix-and-pr/references/11-prompt-hygiene.md @@ -0,0 +1,23 @@ +# 11 — Prompt hygiene + +Part of the GBOS/Eonix `fix-and-pr` skill. Read [00-operating-contract.md](00-operating-contract.md) for the phase gates and hard rules before using this document. + +--- + +## Appendix C: Prompt hygiene + +Every subagent prompt should contain: + +1. **Objective** — one concrete deliverable. +2. **Repository, cwd, and ref** — exact paths and commit/branch. +3. **Authority boundary** — what the subagent may read, edit, run, or publish. +4. **Relevant evidence** — issue, logs, root cause, plan, or diff paths. +5. **Acceptance criteria** — commands and expected exit codes. +6. **Output contract** — changed files, commands run, exit codes, log paths, + verdict, and open questions. +7. **Stop conditions** — ambiguity, missing credentials, infrastructure + failure, or a second independent bug. + +The supervisor checks `GH_TOKEN`; never ask a subagent to discover, retrieve, +or validate the token, and never include the token in a prompt. Prompts must not +contain credentials or unredacted credential-bearing URLs. \ No newline at end of file diff --git a/.agents/skills/fix-and-pr/references/README.md b/.agents/skills/fix-and-pr/references/README.md new file mode 100644 index 00000000..77cfc928 --- /dev/null +++ b/.agents/skills/fix-and-pr/references/README.md @@ -0,0 +1,42 @@ +# Fix-and-PR reference library + +Detailed reference for the GBOS/Eonix `fix-and-pr` skill. The workflow is split +by phase so an agent can load only what it needs. + +Start with [00 — Operating contract](00-operating-contract.md), then read the +document for the phase you are entering. Read the next phase's document before +crossing its gate. + +## Phase documents + +| Phase | Document | Read it when | +| --- | --- | --- | +| 0. Intake and environment | [01 — Environment and issue intake](01-environment-and-intake.md) | Setting up the checkout, build, QEMU, branch, and issue record. | +| 1. Reproduction | [02 — Automated reproduction](02-reproduction.md) | Writing or verifying a headless QEMU reproducer. | +| 2. Analysis | [03 — Root-cause analysis](03-analysis.md) | Collecting evidence, mapping code, and proving the root cause. | +| 3. Fix | [04 — Fix planning and implementation](04-fix-and-commits.md) | Planning the change, writing it, and refining the diff. | +| 3. Commits | [05 — Commit conventions](05-commit-conventions.md) | Splitting and writing commits. | +| 4. Verification | [06 — Verification and review](06-verification.md) | Running the matrix, independent review, and final acceptance. | +| 5. PR | [07 — PR submission](07-pr-submission.md) | Pushing the branch, creating the PR, and validating it. | + +## Supporting documents + +| Document | Purpose | +| --- | --- | +| [08 — Failure handling](08-failure-handling.md) | Stop conditions, escalation, and recovery choices. | +| [09 — Issue analysis template](09-analysis-template.md) | Recording observations, hypotheses, and root cause. | +| [10 — Quick command reference](10-command-reference.md) | Copy-paste commands for build, debug, reproduction, and PR. | +| [11 — Prompt hygiene](11-prompt-hygiene.md) | Required structure for subagent handoffs. | + +## Reading rules + +1. Confirm `GH_TOKEN` is set before doing anything else. If it is missing, stop + and ask the user; do not read further or start a phase. +2. Read [00 — Operating contract](00-operating-contract.md) first. It contains + the roles, hard rules, phase gates, and artifact layout. +3. Load only the current phase document and any supporting document it points + to. +4. Before leaving a phase, read the next phase document and confirm its gate + evidence exists. +5. Do not skip the reproduction or verification documents; they define the + evidence that makes the fix and PR trustworthy. From d6916ba1451a990c09237e3ac61df1d6814e8eeb Mon Sep 17 00:00:00 2001 From: greatbridf <greatbridf@icloud.com> Date: Sun, 18 Jan 2026 01:09:33 +0800 Subject: [PATCH 12/36] script: add a script to help translate stacktraces - Add script/backtrace to translate backtraces. - Add a cut sign in the kernel panic routine to indicate the start of stack backtrace. Signed-off-by: greatbridf <greatbridf@icloud.com> --- script/backtrace | 100 +++++++++++++++++++++++++++++++++++++++++++++++ src/panic.rs | 6 +++ 2 files changed, 106 insertions(+) create mode 100755 script/backtrace diff --git a/script/backtrace b/script/backtrace new file mode 100755 index 00000000..8a60c829 --- /dev/null +++ b/script/backtrace @@ -0,0 +1,100 @@ +#!/bin/bash + +ADDR2LINE=${ADDR2LINE:-riscv64-unknown-elf-addr2line} + +ksym=build/riscv64gc-unknown-none-elf/debug/eonix_kernel + +usage() { + cat <<EOF +Usage: $0 [options] [--] [addr2line options] + +Filter and translate backtrace addresses into human readable line numbers. + +Options: + -s, --ksym <symbol file> Use the given kernel symbol file + -o, --only-gbos Show kernel function calls only + -h, --help Show this message +EOF + exit "$1" +} + +# $1: instruction address +parse_pos() { + addr="$1" + shift + + "$ADDR2LINE" -e "$ksym" -i "$addr" "$@" 2>/dev/null +} + +filter_col() { + [ "$1" -eq 0 ] || awk "{ print \$$1; }" +} + +str_contains() { + grep -E "$1" >/dev/null 2>&1 +} + +filter_stacktrace() { + NL=$'\n' + _state=nonstart + _out= + while [ $_state != "end" ]; do + read -r _line + case $_state in + nonstart) + str_contains "8< CUT HERE" <<< "$_line" && _state=save + ;; + save) + if str_contains "8< CUT HERE" <<< "$_line"; then + _state=end + else + _out="$_out$_line$NL" + fi + ;; + esac + done + + echo "$_out" +} + +while [ "$#" -gt 0 ]; do + case "$1" in + -s|--ksym) + shift + ksym="$1" + ;; + -o|--only-gbos) + only_gb=y + ;; + --) + shift + break + ;; + -h|--help) + usage 0 + ;; + *) + usage 1 + ;; + esac + shift +done + +stacktrace="$(filter_stacktrace)" + +i=1 +for addr in $(filter_col 3 <<< "$stacktrace"); do + pos="$(parse_pos "$addr" "$@")" + + if [ -n "$only_gb" ]; then + if ! str_contains "greatbridf_os" <<< "$pos"; then + continue + fi + fi + + printf "========== %4d ==========\n" "$i" + + parse_pos "$addr" "$@" + + i=$((i + 1)) +done diff --git a/src/panic.rs b/src/panic.rs index 3c9c5f34..4a9ef92c 100644 --- a/src/panic.rs +++ b/src/panic.rs @@ -24,6 +24,12 @@ pub fn stack_trace() { UnwindReasonCode::NO_REASON } + println_fatal!("--------------8< CUT HERE 8<--------------"); + println_fatal!("Stacktrace:"); + println_fatal!(); + let mut data = CallbackData { counter: 0 }; _Unwind_Backtrace(callback, &raw mut data as *mut c_void); + + println_fatal!("--------------8< CUT HERE 8<--------------"); } From 675c9f11aaa95b4519fa0cdf64b2ac7f4e970734 Mon Sep 17 00:00:00 2001 From: greatbridf <greatbridf@icloud.com> Date: Tue, 2 Sep 2025 22:45:55 +0800 Subject: [PATCH 13/36] vfs: rework the vfs subsystem with async Inode and superblock rework: Remove old Inode trait as it used to undertake too much responsibility. The new method use three new traits: InodeOps is used to acquire generic inode attributes. InodeFileOps and InodeDirOps handle file and directory requests respectively. All the three have async fn trait methods and don't need to be ?Sized. Then, we implement Inode, InodeFile and InodeDir for the implementors of the three "Ops" traits, erasing their actual type and provide generic dyn interface by wrapping the futures in boxes. We should provide an io worker? or some io context with an allocator for futures to reduce the overhead of io requests, or come up with some better ideas? For inode usage, we introduce InodeRef and InodeUse. InodeRef is a simple wrapper for Weak<impl Inode> and InodeUse for Arc<impl Inode>. This helps us use them better as we can't define impls for Arc<dyn Inode>'s as they are foreign types. We also provide some more helper methods for them. After the change, we don't impose ANY structural restriction except for the spinlock wrapped InodeInfo. The InodeInfo struct design might need rethinking but the current implementation seems to be fine aside from unnecessary locks when we try to access some of its fields but this shouldn't be a VERY big problem and very urgent... Similar changes are also made to superblock traits and types. But for the superblock objects, we use a SuperBlockComplex struct to store common fields such as whether the superblock is read only, their device id and so on. Also the structs have a superblock rwsem inside. But we haven't decided how to use that (such as whether we should acquire the lock and pass it to the inode methods) and even whether they should exist and be there. This will need further thinking so we put this off for now... Filesystem rework: Rework tmpfs, fatfs and procfs with the new technology mentioned above, leaving the old ext4 unchanged. The current implementation of ext4 uses some "random" library from the "camp". Its code hasn't been fully reviewed for time reasons but seems to be rather "problematic"... We might rewrite the whole module later and the page cache subsystem requires fully reworking as well. So we put this work off as well. Block device and other parts rework: Wraps PageCacheBackend, MountCreator and BlockRequestQueue with async_trait to provide dyn compatible async functions. Dentry walking functions are also put to the heap since they are recursive functions... This has similar problems to the inode traits, ugly solution. Further optimization is required. Signed-off-by: greatbridf <greatbridf@icloud.com> --- .vscode/tasks.json | 2 +- Cargo.lock | 98 +++++ Cargo.toml | 2 + src/driver/ahci/mod.rs | 63 +-- src/driver/ahci/port.rs | 48 ++- src/driver/ahci/slot.rs | 19 +- src/driver/e1000e.rs | 8 +- src/driver/serial.rs | 6 +- src/driver/virtio/riscv64.rs | 7 +- src/driver/virtio/virtio_blk.rs | 5 +- src/fs/ext4.rs | 35 +- src/fs/fat32.rs | 567 ++++++++++++++++----------- src/fs/fat32/dir.rs | 214 +++++------ src/fs/fat32/file.rs | 46 +-- src/fs/mod.rs | 3 +- src/fs/procfs.rs | 437 +++++++++------------ src/fs/shm.rs | 146 ------- src/fs/tmpfs.rs | 613 ------------------------------ src/fs/tmpfs/dir.rs | 415 ++++++++++++++++++++ src/fs/tmpfs/file.rs | 298 +++++++++++++++ src/fs/tmpfs/mod.rs | 73 ++++ src/kernel/block.rs | 38 +- src/kernel/block/mbr.rs | 2 +- src/kernel/chardev.rs | 15 +- src/kernel/mem.rs | 2 +- src/kernel/mem/mm_list/mapping.rs | 15 +- src/kernel/mem/page_cache.rs | 55 ++- src/kernel/pcie/driver.rs | 9 +- src/kernel/syscall/file_rw.rs | 94 +++-- src/kernel/syscall/mm.rs | 122 +----- src/kernel/syscall/procops.rs | 20 +- src/kernel/task/loader/elf.rs | 34 +- src/kernel/task/loader/mod.rs | 13 +- src/kernel/timer.rs | 2 + src/kernel/vfs/dentry.rs | 295 +++++++------- src/kernel/vfs/dentry/dcache.rs | 29 +- src/kernel/vfs/file/inode_file.rs | 139 ++++--- src/kernel/vfs/filearray.rs | 23 +- src/kernel/vfs/inode.rs | 494 ------------------------ src/kernel/vfs/inode/ino.rs | 31 ++ src/kernel/vfs/inode/inode.rs | 389 +++++++++++++++++++ src/kernel/vfs/inode/mod.rs | 10 + src/kernel/vfs/inode/ops.rs | 18 + src/kernel/vfs/inode/statx.rs | 97 +++++ src/kernel/vfs/mod.rs | 22 +- src/kernel/vfs/mount.rs | 32 +- src/kernel/vfs/superblock.rs | 127 +++++++ src/kernel/vfs/types/device_id.rs | 36 ++ src/kernel/vfs/types/mod.rs | 5 + src/kernel/vfs/types/mode.rs | 169 ++++++++ src/kernel/vfs/vfs.rs | 10 - src/lib.rs | 34 +- src/prelude.rs | 30 +- 53 files changed, 2987 insertions(+), 2529 deletions(-) delete mode 100644 src/fs/shm.rs delete mode 100644 src/fs/tmpfs.rs create mode 100644 src/fs/tmpfs/dir.rs create mode 100644 src/fs/tmpfs/file.rs create mode 100644 src/fs/tmpfs/mod.rs delete mode 100644 src/kernel/vfs/inode.rs create mode 100644 src/kernel/vfs/inode/ino.rs create mode 100644 src/kernel/vfs/inode/inode.rs create mode 100644 src/kernel/vfs/inode/mod.rs create mode 100644 src/kernel/vfs/inode/ops.rs create mode 100644 src/kernel/vfs/inode/statx.rs create mode 100644 src/kernel/vfs/superblock.rs create mode 100644 src/kernel/vfs/types/device_id.rs create mode 100644 src/kernel/vfs/types/mod.rs create mode 100644 src/kernel/vfs/types/mode.rs delete mode 100644 src/kernel/vfs/vfs.rs diff --git a/.vscode/tasks.json b/.vscode/tasks.json index e7a54791..a85ea0cf 100644 --- a/.vscode/tasks.json +++ b/.vscode/tasks.json @@ -6,7 +6,7 @@ { "label": "debug run riscv64", "type": "shell", - "command": "make srun ARCH=riscv64", + "command": "make srun ARCH=riscv64 IMG=/Volumes/oscomp/sdcard-rv.img", "isBackground": true, "problemMatcher": [ { diff --git a/Cargo.lock b/Cargo.lock index 59242bbc..32868677 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -28,6 +28,17 @@ dependencies = [ "log", ] +[[package]] +name = "async-trait" +version = "0.1.89" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9035ad2d096bed7955a320ee7e2230574d28fd3c3a0f186cbea1ff3c7eed5dbb" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "atomic_unique_refcell" version = "0.1.0" @@ -144,6 +155,7 @@ dependencies = [ "acpi", "align_ext", "another_ext4", + "async-trait", "atomic_unique_refcell", "bitflags", "buddy_allocator", @@ -155,6 +167,7 @@ dependencies = [ "eonix_preempt", "eonix_runtime", "eonix_sync", + "futures", "intrusive-collections 0.9.8", "intrusive_list", "itertools", @@ -265,6 +278,79 @@ version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "784a4df722dc6267a04af36895398f59d21d07dce47232adf31ec0ff2fa45e67" +[[package]] +name = "futures" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "65bc07b1a8bc7c85c5f2e110c476c7389b4554ba72af57d8445ea63a576b0876" +dependencies = [ + "futures-channel", + "futures-core", + "futures-io", + "futures-sink", + "futures-task", + "futures-util", +] + +[[package]] +name = "futures-channel" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2dff15bf788c671c1934e366d07e30c1814a8ef514e1af724a602e8a2fbe1b10" +dependencies = [ + "futures-core", + "futures-sink", +] + +[[package]] +name = "futures-core" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05f29059c0c2090612e8d742178b0580d2dc940c837851ad723096f87af6663e" + +[[package]] +name = "futures-io" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9e5c1b78ca4aae1ac06c48a526a655760685149f0d465d21f37abfe57ce075c6" + +[[package]] +name = "futures-macro" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "162ee34ebcb7c64a8abebc059ce0fee27c2262618d7b60ed8faf72fef13c3650" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "futures-sink" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e575fab7d1e0dcb8d0c7bcf9a63ee213816ab51902e6d244a95819acacf1d4f7" + +[[package]] +name = "futures-task" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f90f7dce0722e95104fcb095585910c0977252f286e354b5e3bd38902cd99988" + +[[package]] +name = "futures-util" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9fa08315bb612088cc391249efdc3bc77536f16c91f6cf495e6fbe85b20a4a81" +dependencies = [ + "futures-core", + "futures-macro", + "futures-sink", + "futures-task", + "pin-project-lite", + "pin-utils", +] + [[package]] name = "gimli" version = "0.32.0" @@ -332,6 +418,18 @@ version = "1.0.15" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "57c0d7b74b563b49d38dae00a0c37d4d6de9b432382b2892f0574ddcae73fd0a" +[[package]] +name = "pin-project-lite" +version = "0.2.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3b3cff922bd51709b605d9ead9aa71031d81447142d828eb4a6eba76fe619f9b" + +[[package]] +name = "pin-utils" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" + [[package]] name = "pointers" version = "0.1.0" diff --git a/Cargo.toml b/Cargo.toml index e70d8c65..bc7e7b0c 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -34,6 +34,8 @@ another_ext4 = { git = "https://github.com/SMS-Derfflinger/another_ext4", branch stalloc = { version = "0.6.1", default-features = false, features = [ "allocator-api", ] } +async-trait = "0.1.89" +futures = { version = "0.3.31", features = ["alloc", "async-await"], default-features = false } [target.'cfg(any(target_arch = "riscv64", target_arch = "loongarch64"))'.dependencies] virtio-drivers = { version = "0.11.0" } diff --git a/src/driver/ahci/mod.rs b/src/driver/ahci/mod.rs index c3b1cfa0..ab405829 100644 --- a/src/driver/ahci/mod.rs +++ b/src/driver/ahci/mod.rs @@ -2,15 +2,16 @@ use crate::{ fs::procfs, io::Buffer as _, kernel::{ - block::{make_device, BlockDevice}, + block::BlockDevice, constants::{EINVAL, EIO}, interrupt::register_irq_handler, pcie::{self, Header, PCIDevice, PCIDriver, PciError}, - task::block_on, + vfs::types::DeviceId, }, prelude::*, }; use alloc::{format, sync::Arc}; +use async_trait::async_trait; use control::AdapterControl; use defs::*; use eonix_mm::address::{AddrOps as _, PAddr}; @@ -108,7 +109,30 @@ impl Device<'_> { } impl Device<'static> { - fn probe_ports(&self) -> KResult<()> { + async fn probe_port(&self, port: Arc<AdapterPort<'static>>) -> KResult<()> { + port.init().await?; + + { + let port = port.clone(); + let name = format!("ahci-p{}-stats", port.nport); + procfs::populate_root(name.into_bytes().into(), move |buffer| { + port.print_stats(&mut buffer.get_writer()) + }) + .await; + } + + let port = BlockDevice::register_disk( + DeviceId::new(8, port.nport as u16 * 16), + 2147483647, // TODO: get size from device + port, + )?; + + port.partprobe().await?; + + Ok(()) + } + + async fn probe_ports(&self) -> KResult<()> { for nport in self.control.implemented_ports() { let port = Arc::new(AdapterPort::new(self.control_base, nport)); if !port.status_ok() { @@ -116,27 +140,7 @@ impl Device<'static> { } self.ports.lock_irq()[nport as usize] = Some(port.clone()); - if let Err(e) = (|| -> KResult<()> { - port.init()?; - - { - let port = port.clone(); - let name = format!("ahci-p{}-stats", port.nport); - procfs::populate_root(name.into_bytes().into(), move |buffer| { - port.print_stats(&mut buffer.get_writer()) - })?; - } - - let port = BlockDevice::register_disk( - make_device(8, nport * 16), - 2147483647, // TODO: get size from device - port, - )?; - - block_on(port.partprobe())?; - - Ok(()) - })() { + if let Err(e) = self.probe_port(port).await { self.ports.lock_irq()[nport as usize] = None; println_warn!("probe port {nport} failed with {e}"); } @@ -154,6 +158,7 @@ impl AHCIDriver { } } +#[async_trait] impl PCIDriver for AHCIDriver { fn vendor_id(&self) -> u16 { VENDOR_INTEL @@ -163,7 +168,7 @@ impl PCIDriver for AHCIDriver { DEVICE_AHCI } - fn handle_device(&self, pcidev: Arc<PCIDevice<'static>>) -> Result<(), PciError> { + async fn handle_device(&self, pcidev: Arc<PCIDevice<'static>>) -> Result<(), PciError> { let Header::Endpoint(header) = pcidev.header else { Err(EINVAL)? }; @@ -200,7 +205,7 @@ impl PCIDriver for AHCIDriver { let device_irq = device.clone(); register_irq_handler(irqno as i32, move || device_irq.handle_interrupt())?; - device.probe_ports()?; + device.probe_ports().await?; self.devices.lock().push(device); @@ -208,6 +213,8 @@ impl PCIDriver for AHCIDriver { } } -pub fn register_ahci_driver() { - pcie::register_driver(AHCIDriver::new()).expect("Register ahci driver failed"); +pub async fn register_ahci_driver() { + pcie::register_driver(AHCIDriver::new()) + .await + .expect("Register ahci driver failed"); } diff --git a/src/driver/ahci/port.rs b/src/driver/ahci/port.rs index f558f6e1..77286ec5 100644 --- a/src/driver/ahci/port.rs +++ b/src/driver/ahci/port.rs @@ -9,9 +9,9 @@ use crate::kernel::block::{BlockDeviceRequest, BlockRequestQueue}; use crate::kernel::constants::{EINVAL, EIO}; use crate::kernel::mem::paging::Page; use crate::kernel::mem::AsMemoryBlock as _; -use crate::kernel::task::block_on; use crate::prelude::*; use alloc::collections::vec_deque::VecDeque; +use async_trait::async_trait; use core::pin::pin; use eonix_mm::address::{Addr as _, PAddr}; use eonix_sync::{SpinIrq as _, WaitList}; @@ -145,18 +145,25 @@ impl AdapterPort<'_> { self.sata_status().read_once() & 0xf == 0x3 } - fn get_free_slot(&self) -> u32 { + async fn get_free_slot(&self) -> u32 { loop { - let mut free_list = self.free_list.lock_irq(); - let free_slot = free_list.free.pop_front(); - if let Some(slot) = free_slot { - return slot; - } let mut wait = pin!(self.free_list_wait.prepare_to_wait()); - wait.as_mut().add_to_wait_list(); - drop(free_list); - block_on(wait); + { + let mut free_list = self.free_list.lock_irq(); + + if let Some(slot) = free_list.free.pop_front() { + return slot; + } + + wait.as_mut().add_to_wait_list(); + + if let Some(slot) = free_list.free.pop_front() { + return slot; + } + } + + wait.await; } } @@ -204,11 +211,11 @@ impl AdapterPort<'_> { Ok(()) } - fn send_command(&self, cmd: &impl Command) -> KResult<()> { + async fn send_command(&self, cmd: &impl Command) -> KResult<()> { let mut cmdtable = CommandTable::new(); cmdtable.setup(cmd); - let slot_index = self.get_free_slot(); + let slot_index = self.get_free_slot().await; let slot = &self.slots[slot_index as usize]; slot.prepare_command(&cmdtable, cmd.write()); @@ -222,7 +229,7 @@ impl AdapterPort<'_> { self.stats.inc_cmd_sent(); - if let Err(_) = block_on(slot.wait_finish()) { + if let Err(_) = slot.wait_finish().await { self.stats.inc_cmd_error(); return Err(EIO); }; @@ -231,16 +238,16 @@ impl AdapterPort<'_> { Ok(()) } - fn identify(&self) -> KResult<()> { + async fn identify(&self) -> KResult<()> { let cmd = IdentifyCommand::new(); // TODO: check returned data - self.send_command(&cmd)?; + self.send_command(&cmd).await?; Ok(()) } - pub fn init(&self) -> KResult<()> { + pub async fn init(&self) -> KResult<()> { self.stop_command()?; self.command_list_base() @@ -251,7 +258,7 @@ impl AdapterPort<'_> { self.start_command()?; - match self.identify() { + match self.identify().await { Err(err) => { self.stop_command()?; Err(err) @@ -269,12 +276,13 @@ impl AdapterPort<'_> { } } +#[async_trait] impl BlockRequestQueue for AdapterPort<'_> { fn max_request_pages(&self) -> u64 { 1024 } - fn submit(&self, req: BlockDeviceRequest) -> KResult<()> { + async fn submit<'a>(&'a self, req: BlockDeviceRequest<'a>) -> KResult<()> { match req { BlockDeviceRequest::Read { sector, @@ -287,7 +295,7 @@ impl BlockRequestQueue for AdapterPort<'_> { let command = ReadLBACommand::new(buffer, sector, count as u16)?; - self.send_command(&command) + self.send_command(&command).await } BlockDeviceRequest::Write { sector, @@ -300,7 +308,7 @@ impl BlockRequestQueue for AdapterPort<'_> { let command = WriteLBACommand::new(buffer, sector, count as u16)?; - self.send_command(&command) + self.send_command(&command).await } } } diff --git a/src/driver/ahci/slot.rs b/src/driver/ahci/slot.rs index 2198c457..60a66de3 100644 --- a/src/driver/ahci/slot.rs +++ b/src/driver/ahci/slot.rs @@ -71,19 +71,20 @@ impl<'a> CommandSlot<'a> { pub async fn wait_finish(&self) -> KResult<()> { let mut inner = loop { - let inner = self.inner.lock_irq(); - if inner.state != SlotState::Working { - break inner; - } - let mut wait = pin!(self.wait_list.prepare_to_wait()); - wait.as_mut().add_to_wait_list(); - if inner.state != SlotState::Working { - break inner; + { + let inner = self.inner.lock_irq(); + if inner.state != SlotState::Working { + break inner; + } + wait.as_mut().add_to_wait_list(); + + if inner.state != SlotState::Working { + break inner; + } } - drop(inner); wait.await; }; diff --git a/src/driver/e1000e.rs b/src/driver/e1000e.rs index ba31b8b1..f362f477 100644 --- a/src/driver/e1000e.rs +++ b/src/driver/e1000e.rs @@ -8,6 +8,7 @@ use crate::prelude::*; use alloc::boxed::Box; use alloc::sync::Arc; use alloc::vec::Vec; +use async_trait::async_trait; use core::ptr::NonNull; use eonix_hal::fence::memory_barrier; use eonix_mm::address::{Addr, PAddr}; @@ -437,6 +438,7 @@ struct Driver { dev_id: u16, } +#[async_trait] impl PCIDriver for Driver { fn vendor_id(&self) -> u16 { 0x8086 @@ -446,7 +448,7 @@ impl PCIDriver for Driver { self.dev_id } - fn handle_device(&self, device: Arc<PCIDevice<'static>>) -> Result<(), PciError> { + async fn handle_device(&self, device: Arc<PCIDevice<'static>>) -> Result<(), PciError> { let Header::Endpoint(header) = device.header else { Err(EINVAL)? }; @@ -473,10 +475,10 @@ impl PCIDriver for Driver { } } -pub fn register_e1000e_driver() { +pub async fn register_e1000e_driver() { let dev_ids = [0x100e, 0x10d3, 0x10ea, 0x153a]; for id in dev_ids.into_iter() { - pcie::register_driver(Driver { dev_id: id }).unwrap(); + pcie::register_driver(Driver { dev_id: id }).await.unwrap(); } } diff --git a/src/driver/serial.rs b/src/driver/serial.rs index 7561828a..b84a1c5f 100644 --- a/src/driver/serial.rs +++ b/src/driver/serial.rs @@ -2,8 +2,8 @@ mod io; use crate::{ kernel::{ - block::make_device, console::set_console, constants::EIO, interrupt::register_irq_handler, - CharDevice, CharDeviceType, Terminal, TerminalDevice, + console::set_console, constants::EIO, interrupt::register_irq_handler, + vfs::types::DeviceId, CharDevice, CharDeviceType, Terminal, TerminalDevice, }, prelude::*, }; @@ -167,7 +167,7 @@ impl Serial { eonix_log::set_console(terminal.clone()); CharDevice::register( - make_device(4, 64 + port.id), + DeviceId::new(4, 64 + port.id as u16), port.name.clone(), CharDeviceType::Terminal(terminal), )?; diff --git a/src/driver/virtio/riscv64.rs b/src/driver/virtio/riscv64.rs index 9bdbf6ce..b33e16ac 100644 --- a/src/driver/virtio/riscv64.rs +++ b/src/driver/virtio/riscv64.rs @@ -1,8 +1,5 @@ use super::virtio_blk::HAL; -use crate::kernel::{ - block::{make_device, BlockDevice}, - task::block_on, -}; +use crate::kernel::{block::BlockDevice, task::block_on, vfs::types::DeviceId}; use alloc::{sync::Arc, vec::Vec}; use eonix_hal::arch_exported::fdt::FDT; use eonix_hal::mm::ArchPhysAccess; @@ -43,7 +40,7 @@ pub fn init() { .expect("Failed to initialize VirtIO Block device"); let block_device = BlockDevice::register_disk( - make_device(8, 16 * disk_id), + DeviceId::new(8, 16 * disk_id), 2147483647, Arc::new(Spin::new(block_device)), ) diff --git a/src/driver/virtio/virtio_blk.rs b/src/driver/virtio/virtio_blk.rs index 37e4fe77..86b500b6 100644 --- a/src/driver/virtio/virtio_blk.rs +++ b/src/driver/virtio/virtio_blk.rs @@ -7,6 +7,8 @@ use crate::{ }, prelude::KResult, }; +use alloc::boxed::Box; +use async_trait::async_trait; use eonix_hal::mm::ArchPhysAccess; use eonix_mm::{ address::{Addr, PAddr, PhysAccess}, @@ -74,6 +76,7 @@ unsafe impl Hal for HAL { } } +#[async_trait] impl<T> BlockRequestQueue for Spin<VirtIOBlk<HAL, T>> where T: Transport + Send, @@ -82,7 +85,7 @@ where 1024 } - fn submit(&self, req: BlockDeviceRequest) -> KResult<()> { + async fn submit<'a>(&'a self, req: BlockDeviceRequest<'a>) -> KResult<()> { match req { BlockDeviceRequest::Write { sector, diff --git a/src/fs/ext4.rs b/src/fs/ext4.rs index 76ca4a34..121339d3 100644 --- a/src/fs/ext4.rs +++ b/src/fs/ext4.rs @@ -1,5 +1,3 @@ -use core::sync::atomic::{AtomicU32, AtomicU64, Ordering}; - use crate::kernel::mem::{CachePage, CachePageStream, PageCache, PageCacheBackend}; use crate::kernel::task::block_on; use crate::kernel::timer::Ticks; @@ -31,6 +29,8 @@ use alloc::{ use another_ext4::{ Block, BlockDevice as Ext4BlockDeviceTrait, Ext4, FileType, InodeMode, PBlockId, }; +use async_trait::async_trait; +use core::sync::atomic::{AtomicU32, AtomicU64, Ordering}; use eonix_sync::RwLock; pub struct Ext4BlockDevice { @@ -194,7 +194,7 @@ impl Ext4Fs { root_inode.inode.mtime_extra() as _, )), rwsem: RwLock::new(()), - vfs: Arc::downgrade(&ext4fs) as _, + sb: Arc::downgrade(&ext4fs) as _, }, ) }; @@ -290,7 +290,7 @@ impl Inode for FileInode { } fn read_direct(&self, buffer: &mut dyn Buffer, offset: usize) -> KResult<usize> { - let vfs = self.vfs.upgrade().ok_or(EIO)?; + let vfs = self.sb.upgrade().ok_or(EIO)?; let ext4fs = vfs.as_any().downcast_ref::<Ext4Fs>().unwrap(); let mut temp_buf = vec![0u8; buffer.total()]; @@ -334,7 +334,7 @@ impl Inode for FileInode { fn write_direct(&self, stream: &mut dyn Stream, offset: usize) -> KResult<usize> { //let _lock = Task::block_on(self.rwsem.write()); - let vfs = self.vfs.upgrade().ok_or(EIO)?; + let vfs = self.sb.upgrade().ok_or(EIO)?; let ext4fs = vfs.as_any().downcast_ref::<Ext4Fs>().unwrap(); let mut temp_buf = vec![0u8; 4096]; @@ -363,7 +363,7 @@ impl Inode for FileInode { fn chmod(&self, mode: Mode) -> KResult<()> { let _lock = block_on(self.rwsem.write()); - let vfs = self.vfs.upgrade().ok_or(EIO)?; + let vfs = self.sb.upgrade().ok_or(EIO)?; let ext4fs = vfs.as_any().downcast_ref::<Ext4Fs>().unwrap(); let old_mode = self.mode.load(); let new_mode = old_mode.perm(mode.bits()); @@ -428,7 +428,7 @@ impl DirInode { impl Inode for DirInode { fn lookup(&self, dentry: &Arc<Dentry>) -> KResult<Option<Arc<dyn Inode>>> { - let vfs = self.vfs.upgrade().ok_or(EIO)?; + let vfs = self.sb.upgrade().ok_or(EIO)?; let ext4fs = vfs.as_any().downcast_ref::<Ext4Fs>().unwrap(); let name = dentry.get_name(); @@ -477,7 +477,7 @@ impl Inode for DirInode { ctime: Spin::new(Instant::new(attr.ctime as _, 0)), mtime: Spin::new(Instant::new(attr.mtime as _, 0)), rwsem: RwLock::new(()), - vfs: self.vfs.clone(), + sb: self.sb.clone(), }, ); @@ -489,7 +489,7 @@ impl Inode for DirInode { offset: usize, callback: &mut dyn FnMut(&[u8], Ino) -> KResult<core::ops::ControlFlow<(), ()>>, ) -> KResult<usize> { - let vfs = self.vfs.upgrade().ok_or(EIO)?; + let vfs = self.sb.upgrade().ok_or(EIO)?; let ext4fs = vfs.as_any().downcast_ref::<Ext4Fs>().unwrap(); let entries = ext4fs @@ -519,7 +519,7 @@ impl Inode for DirInode { fn creat(&self, at: &Arc<Dentry>, mode: Mode) -> KResult<()> { let _lock = block_on(self.rwsem.write()); - let vfs = self.vfs.upgrade().ok_or(EIO)?; + let vfs = self.sb.upgrade().ok_or(EIO)?; let ext4fs = vfs.as_any().downcast_ref::<Ext4Fs>().unwrap(); let name = at.get_name(); @@ -534,7 +534,7 @@ impl Inode for DirInode { ) .unwrap(); - let file = FileInode::new(new_ino as u64, self.vfs.clone(), mode); + let file = FileInode::new(new_ino as u64, self.sb.clone(), mode); let now = Instant::now(); self.update_child_time(file.as_ref(), now); self.link_file(); @@ -547,7 +547,7 @@ impl Inode for DirInode { fn mkdir(&self, at: &Dentry, mode: Mode) -> KResult<()> { let _lock = block_on(self.rwsem.write()); - let vfs = self.vfs.upgrade().ok_or(EIO)?; + let vfs = self.sb.upgrade().ok_or(EIO)?; let ext4fs = vfs.as_any().downcast_ref::<Ext4Fs>().unwrap(); let name = at.get_name(); @@ -562,7 +562,7 @@ impl Inode for DirInode { ) .unwrap(); - let new_dir = DirInode::new(new_ino as u64, self.vfs.clone(), mode); + let new_dir = DirInode::new(new_ino as u64, self.sb.clone(), mode); let now = Instant::now(); self.update_child_time(new_dir.as_ref(), now); self.link_dir(); @@ -575,7 +575,7 @@ impl Inode for DirInode { fn unlink(&self, at: &Arc<Dentry>) -> KResult<()> { let _dir_lock = block_on(self.rwsem.write()); - let vfs = self.vfs.upgrade().ok_or(EIO)?; + let vfs = self.sb.upgrade().ok_or(EIO)?; let ext4fs = vfs.as_any().downcast_ref::<Ext4Fs>().unwrap(); let file = at.get_inode()?; @@ -602,7 +602,7 @@ impl Inode for DirInode { fn chmod(&self, mode: Mode) -> KResult<()> { let _lock = block_on(self.rwsem.write()); - let vfs = self.vfs.upgrade().ok_or(EIO)?; + let vfs = self.sb.upgrade().ok_or(EIO)?; let ext4fs = vfs.as_any().downcast_ref::<Ext4Fs>().unwrap(); let old_mode = self.mode.load(); let new_mode = old_mode.perm(mode.bits()); @@ -638,7 +638,7 @@ impl Inode for DirInode { // TODO: may need another lock let _lock = block_on(self.rwsem.write()); - let vfs = self.vfs.upgrade().ok_or(EIO)?; + let vfs = self.sb.upgrade().ok_or(EIO)?; let ext4fs = vfs.as_any().downcast_ref::<Ext4Fs>().unwrap(); let old_file = old_dentry.get_inode()?; @@ -698,6 +698,7 @@ impl From<InodeMode> for Mode { struct Ext4MountCreator; +#[async_trait] impl MountCreator for Ext4MountCreator { fn check_signature(&self, mut first_block: &[u8]) -> KResult<bool> { match first_block.split_off(1080..) { @@ -707,7 +708,7 @@ impl MountCreator for Ext4MountCreator { } } - fn create_mount(&self, source: &str, _flags: u64, mp: &Arc<Dentry>) -> KResult<Mount> { + async fn create_mount(&self, source: &str, _flags: u64, mp: &Arc<Dentry>) -> KResult<Mount> { let source = source.as_bytes(); let path = Path::new(source)?; diff --git a/src/fs/fat32.rs b/src/fs/fat32.rs index 12eabdd5..1104337d 100644 --- a/src/fs/fat32.rs +++ b/src/fs/fat32.rs @@ -1,41 +1,72 @@ mod dir; mod file; -use crate::io::Stream; -use crate::kernel::constants::EIO; +use core::future::Future; +use core::ops::Deref; + +use alloc::sync::{Arc, Weak}; +use async_trait::async_trait; +use dir::{as_raw_dirents, ParseDirent}; +use eonix_sync::RwLock; +use itertools::Itertools; + +use crate::kernel::constants::{EINVAL, EIO}; use crate::kernel::mem::{AsMemoryBlock, CachePageStream}; -use crate::kernel::task::block_on; -use crate::kernel::vfs::inode::{Mode, WriteOffset}; +use crate::kernel::timer::Instant; +use crate::kernel::vfs::inode::{InodeDirOps, InodeFileOps, InodeInfo, InodeOps, InodeUse}; +use crate::kernel::vfs::types::{DeviceId, Format, Permission}; +use crate::kernel::vfs::{SbRef, SbUse, SuperBlock, SuperBlockInfo}; +use crate::prelude::*; use crate::{ io::{Buffer, ByteBuffer, UninitBuffer}, kernel::{ - block::{make_device, BlockDevice, BlockDeviceRequest}, + block::{BlockDevice, BlockDeviceRequest}, mem::{ paging::Page, - {CachePage, PageCache, PageCacheBackend}, + {CachePage, PageCache, PageCacheBackendOps}, }, vfs::{ dentry::Dentry, - inode::{define_struct_inode, Ino, Inode, InodeData}, + inode::{Ino, Inode}, mount::{register_filesystem, Mount, MountCreator}, - vfs::Vfs, - DevId, }, }, - prelude::*, KResult, }; -use alloc::{ - collections::btree_map::BTreeMap, - sync::{Arc, Weak}, - vec::Vec, -}; -use core::{ops::ControlFlow, sync::atomic::Ordering}; -use dir::Dirs as _; -use eonix_sync::RwLock; -use file::ClusterRead; -type ClusterNo = u32; +#[repr(transparent)] +#[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord)] +struct Cluster(u32); + +#[repr(transparent)] +#[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord)] +struct RawCluster(pub u32); + +impl RawCluster { + const START: u32 = 2; + const EOC: u32 = 0x0FFF_FFF8; + const INVL: u32 = 0xF000_0000; + + fn parse(self) -> Option<Cluster> { + match self.0 { + ..Self::START | Self::EOC..Self::INVL => None, + Self::INVL.. => { + unreachable!("invalid cluster number: RawCluster({:#08x})", self.0) + } + no => Some(Cluster(no)), + } + } +} + +impl Cluster { + pub fn as_ino(self) -> Ino { + Ino::new(self.0 as _) + } + + fn normalized(self) -> Self { + Self(self.0 - 2) + } +} const SECTOR_SIZE: usize = 512; @@ -59,7 +90,7 @@ struct Bootsector { sectors_per_fat: u32, flags: u16, fat_version: u16, - root_cluster: ClusterNo, + root_cluster: RawCluster, fsinfo_sector: u16, backup_bootsector: u16, _reserved: [u8; 12], @@ -73,222 +104,203 @@ struct Bootsector { mbr_signature: u16, } -impl_any!(FatFs); /// # Lock order /// 2. FatTable /// 3. Inodes /// struct FatFs { sectors_per_cluster: u8, - rootdir_cluster: ClusterNo, - data_start: u64, - volume_label: [u8; 11], + data_start_sector: u64, + _rootdir_cluster: Cluster, + _volume_label: Box<str>, device: Arc<BlockDevice>, - fat: RwLock<Vec<ClusterNo>>, - weak: Weak<FatFs>, - icache: BTreeMap<Ino, FatInode>, + fat: RwLock<Box<[RawCluster]>>, } -impl Vfs for FatFs { - fn io_blksize(&self) -> usize { - 4096 - } - - fn fs_devid(&self) -> DevId { - self.device.devid() - } - - fn is_read_only(&self) -> bool { - true - } -} +impl SuperBlock for FatFs {} impl FatFs { - fn read_cluster(&self, cluster: ClusterNo, buf: &Page) -> KResult<()> { - let cluster = cluster - 2; + async fn read_cluster(&self, mut cluster: Cluster, buf: &Page) -> KResult<()> { + cluster = cluster.normalized(); let rq = BlockDeviceRequest::Read { - sector: self.data_start as u64 + cluster as u64 * self.sectors_per_cluster as u64, + sector: self.data_start_sector as u64 + + cluster.0 as u64 * self.sectors_per_cluster as u64, count: self.sectors_per_cluster as u64, buffer: core::slice::from_ref(buf), }; - self.device.commit_request(rq)?; + self.device.commit_request(rq).await?; Ok(()) } - - fn get_or_alloc_inode(&self, ino: Ino, is_directory: bool, size: u32) -> Arc<dyn Inode> { - self.icache - .get(&ino) - .cloned() - .map(FatInode::unwrap) - .unwrap_or_else(|| { - if is_directory { - DirInode::new(ino, self.weak.clone(), size) - } else { - FileInode::new(ino, self.weak.clone(), size) - } - }) - } } impl FatFs { - pub fn create(device: DevId) -> KResult<(Arc<Self>, Arc<dyn Inode>)> { + pub async fn create(device: DeviceId) -> KResult<(SbUse<Self>, InodeUse<dyn Inode>)> { let device = BlockDevice::get(device)?; - let mut fatfs_arc = Arc::new_cyclic(|weak: &Weak<FatFs>| Self { - device, - sectors_per_cluster: 0, - rootdir_cluster: 0, - data_start: 0, - fat: RwLock::new(Vec::new()), - weak: weak.clone(), - icache: BTreeMap::new(), - volume_label: [0; 11], - }); - let fatfs = unsafe { Arc::get_mut_unchecked(&mut fatfs_arc) }; - - let mut info: UninitBuffer<Bootsector> = UninitBuffer::new(); - fatfs.device.read_some(0, &mut info)?.ok_or(EIO)?; + let mut info = UninitBuffer::<Bootsector>::new(); + device.read_some(0, &mut info).await?.ok_or(EIO)?; let info = info.assume_filled_ref()?; - fatfs.sectors_per_cluster = info.sectors_per_cluster; - fatfs.rootdir_cluster = info.root_cluster; - fatfs.data_start = - info.reserved_sectors as u64 + info.fat_copies as u64 * info.sectors_per_fat as u64; - - let fat = fatfs.fat.get_mut(); - - fat.resize( - 512 * info.sectors_per_fat as usize / core::mem::size_of::<ClusterNo>(), - 0, + let mut fat = Box::new_uninit_slice( + 512 * info.sectors_per_fat as usize / core::mem::size_of::<Cluster>(), ); - let mut buffer = ByteBuffer::from(fat.as_mut_slice()); - - fatfs - .device - .read_some(info.reserved_sectors as usize * 512, &mut buffer)? + device + .read_some( + info.reserved_sectors as usize * 512, + &mut ByteBuffer::from(fat.as_mut()), + ) + .await? .ok_or(EIO)?; - info.volume_label - .iter() - .take_while(|&&c| c != ' ' as u8) - .take(11) - .enumerate() - .for_each(|(idx, c)| fatfs.volume_label[idx] = *c); + let sectors_per_cluster = info.sectors_per_cluster; + let rootdir_cluster = info.root_cluster.parse().ok_or(EINVAL)?; - let root_dir_cluster_count = ClusterIterator::new(fat, fatfs.rootdir_cluster).count(); - let root_dir_size = root_dir_cluster_count as u32 * info.sectors_per_cluster as u32 * 512; + let data_start_sector = + info.reserved_sectors as u64 + info.fat_copies as u64 * info.sectors_per_fat as u64; + + let volume_label = { + let end = info + .volume_label + .iter() + .position(|&c| c == b' ') + .unwrap_or(info.volume_label.len()); + + String::from_utf8_lossy(&info.volume_label[..end]) + .into_owned() + .into_boxed_str() + }; - let root_inode = DirInode::new( - (info.root_cluster & !0xF000_0000) as Ino, - fatfs.weak.clone(), - root_dir_size, + let fat = unsafe { fat.assume_init() }; + + let rootdir_cluster_count = ClusterIterator::new(fat.as_ref(), rootdir_cluster).count(); + let rootdir_size = rootdir_cluster_count as u32 * sectors_per_cluster as u32 * 512; + + let fatfs = SbUse::new( + SuperBlockInfo { + io_blksize: 4096, + device_id: device.devid(), + read_only: true, + }, + Self { + device, + sectors_per_cluster, + _rootdir_cluster: rootdir_cluster, + data_start_sector, + fat: RwLock::new(fat), + _volume_label: volume_label, + }, ); - Ok((fatfs_arc, root_inode)) + let sbref = SbRef::from(&fatfs); + Ok((fatfs, DirInode::new(rootdir_cluster, sbref, rootdir_size))) } } -struct ClusterIterator<'fat> { - fat: &'fat [ClusterNo], - cur: ClusterNo, +struct ClusterIterator<'a> { + fat: &'a [RawCluster], + cur: Option<Cluster>, } -impl<'fat> ClusterIterator<'fat> { - fn new(fat: &'fat [ClusterNo], start: ClusterNo) -> Self { - Self { fat, cur: start } +impl<'a> ClusterIterator<'a> { + fn new(fat: &'a [RawCluster], start: Cluster) -> Self { + Self { + fat, + cur: Some(start), + } } } impl<'fat> Iterator for ClusterIterator<'fat> { - type Item = ClusterNo; + type Item = Cluster; fn next(&mut self) -> Option<Self::Item> { - const EOC: ClusterNo = 0x0FFF_FFF8; - const INVL: ClusterNo = 0xF000_0000; - - match self.cur { - ..2 | EOC..INVL => None, - INVL.. => unreachable!("Invalid cluster number: {}", self.cur), - next => { - self.cur = self.fat[next as usize] & !INVL; - Some(next) - } - } + self.cur.inspect(|&Cluster(no)| { + self.cur = self.fat[no as usize].parse(); + }) } } -#[allow(dead_code)] -#[derive(Clone)] -enum FatInode { - File(Arc<FileInode>), - Dir(Arc<DirInode>), +struct FileInode { + cluster: Cluster, + info: Spin<InodeInfo>, + sb: SbRef<FatFs>, + page_cache: PageCache, } -impl FatInode { - fn unwrap(self) -> Arc<dyn Inode> { - match self { - FatInode::File(inode) => inode, - FatInode::Dir(inode) => inode, - } +impl FileInode { + fn new(cluster: Cluster, sb: SbRef<FatFs>, size: u32) -> InodeUse<FileInode> { + InodeUse::new_cyclic(|weak: &Weak<FileInode>| Self { + cluster, + info: Spin::new(InodeInfo { + size: size as u64, + nlink: 1, + uid: 0, + gid: 0, + perm: Permission::new(0o777), + atime: Instant::UNIX_EPOCH, + ctime: Instant::UNIX_EPOCH, + mtime: Instant::UNIX_EPOCH, + }), + sb, + page_cache: PageCache::new(weak.clone()), + }) } } -define_struct_inode! { - struct FileInode { - page_cache: PageCache, +impl InodeOps for FileInode { + type SuperBlock = FatFs; + + fn ino(&self) -> Ino { + self.cluster.as_ino() } -} -impl FileInode { - fn new(ino: Ino, weak: Weak<FatFs>, size: u32) -> Arc<Self> { - let inode = Arc::new_cyclic(|weak_self: &Weak<FileInode>| Self { - idata: InodeData::new(ino, weak), - page_cache: PageCache::new(weak_self.clone()), - }); + fn format(&self) -> Format { + Format::REG + } - // Safety: We are initializing the inode - inode.nlink.store(1, Ordering::Relaxed); - inode.mode.store(Mode::REG.perm(0o777)); - inode.size.store(size as u64, Ordering::Relaxed); + fn info(&self) -> &Spin<InodeInfo> { + &self.info + } - inode + fn super_block(&self) -> &SbRef<Self::SuperBlock> { + &self.sb } -} -impl Inode for FileInode { fn page_cache(&self) -> Option<&PageCache> { Some(&self.page_cache) } +} - fn read(&self, buffer: &mut dyn Buffer, offset: usize) -> KResult<usize> { - block_on(self.page_cache.read(buffer, offset)) +impl InodeDirOps for FileInode {} +impl InodeFileOps for FileInode { + async fn read(&self, buffer: &mut dyn Buffer, offset: usize) -> KResult<usize> { + self.page_cache.read(buffer, offset).await } - fn read_direct(&self, buffer: &mut dyn Buffer, offset: usize) -> KResult<usize> { - let vfs = self.vfs.upgrade().ok_or(EIO)?; - let vfs = vfs.as_any().downcast_ref::<FatFs>().unwrap(); - let fat = block_on(vfs.fat.read()); + async fn read_direct(&self, buffer: &mut dyn Buffer, offset: usize) -> KResult<usize> { + let sb = self.sb.get()?; + let fs = &sb.backend; + let fat = sb.backend.fat.read().await; - if self.size.load(Ordering::Relaxed) as usize == 0 { + if offset >= self.info.lock().size as usize { return Ok(0); } - let cluster_size = vfs.sectors_per_cluster as usize * SECTOR_SIZE; + let cluster_size = fs.sectors_per_cluster as usize * SECTOR_SIZE; assert!(cluster_size <= 0x1000, "Cluster size is too large"); let skip_clusters = offset / cluster_size; let inner_offset = offset % cluster_size; - let cluster_iter = - ClusterIterator::new(fat.as_ref(), self.ino as ClusterNo).skip(skip_clusters); + let cluster_iter = ClusterIterator::new(fat.as_ref(), self.cluster).skip(skip_clusters); let buffer_page = Page::alloc(); for cluster in cluster_iter { - vfs.read_cluster(cluster, &buffer_page)?; + fs.read_cluster(cluster, &buffer_page).await?; let data = unsafe { // SAFETY: We are the only one holding this page. @@ -296,7 +308,7 @@ impl Inode for FileInode { }; let end = offset + data.len(); - let real_end = core::cmp::min(end, self.size.load(Ordering::Relaxed) as usize); + let real_end = end.min(self.info.lock().size as usize); let real_size = real_end - offset; if buffer.fill(&data[..real_size])?.should_stop() { @@ -306,108 +318,203 @@ impl Inode for FileInode { Ok(buffer.wrote()) } +} - fn write(&self, _stream: &mut dyn Stream, _offset: WriteOffset) -> KResult<usize> { - todo!() +impl PageCacheBackendOps for FileInode { + async fn read_page(&self, page: &mut CachePage, offset: usize) -> KResult<usize> { + self.read_direct(page, offset).await } - fn write_direct(&self, _stream: &mut dyn Stream, _offset: usize) -> KResult<usize> { + async fn write_page(&self, _page: &mut CachePageStream, _offset: usize) -> KResult<usize> { todo!() } + + fn size(&self) -> usize { + self.info.lock().size as usize + } } -impl PageCacheBackend for FileInode { - fn read_page(&self, page: &mut CachePage, offset: usize) -> KResult<usize> { - self.read_direct(page, offset) +struct DirInode { + cluster: Cluster, + info: Spin<InodeInfo>, + sb: SbRef<FatFs>, + + // TODO: Use the new PageCache... + dir_pages: RwLock<Vec<Page>>, +} + +impl DirInode { + fn new(cluster: Cluster, sb: SbRef<FatFs>, size: u32) -> InodeUse<Self> { + InodeUse::new(Self { + cluster, + info: Spin::new(InodeInfo { + size: size as u64, + nlink: 2, // '.' and '..' + uid: 0, + gid: 0, + perm: Permission::new(0o777), + atime: Instant::UNIX_EPOCH, + ctime: Instant::UNIX_EPOCH, + mtime: Instant::UNIX_EPOCH, + }), + sb, + dir_pages: RwLock::new(Vec::new()), + }) } - fn write_page(&self, _page: &mut CachePageStream, _offset: usize) -> KResult<usize> { - todo!() + async fn read_dir_pages(&self) -> KResult<()> { + let mut dir_pages = self.dir_pages.write().await; + if !dir_pages.is_empty() { + return Ok(()); + } + + let sb = self.sb.get()?; + let fs = &sb.backend; + let fat = fs.fat.read().await; + + let clusters = ClusterIterator::new(fat.as_ref(), self.cluster); + + for cluster in clusters { + let page = Page::alloc(); + fs.read_cluster(cluster, &page).await?; + + dir_pages.push(page); + } + + Ok(()) } - fn size(&self) -> usize { - self.size.load(Ordering::Relaxed) as usize + async fn get_dir_pages(&self) -> KResult<impl Deref<Target = Vec<Page>> + use<'_>> { + { + let dir_pages = self.dir_pages.read().await; + if !dir_pages.is_empty() { + return Ok(dir_pages); + } + } + + self.read_dir_pages().await?; + + if let Some(dir_pages) = self.dir_pages.try_read() { + return Ok(dir_pages); + } + + Ok(self.dir_pages.read().await) } } -define_struct_inode! { - struct DirInode; -} +impl InodeOps for DirInode { + type SuperBlock = FatFs; -impl DirInode { - fn new(ino: Ino, weak: Weak<FatFs>, size: u32) -> Arc<Self> { - let inode = Arc::new(Self { - idata: InodeData::new(ino, weak), - }); + fn ino(&self) -> Ino { + self.cluster.as_ino() + } + + fn format(&self) -> Format { + Format::DIR + } - // Safety: We are initializing the inode - inode.nlink.store(2, Ordering::Relaxed); - inode.mode.store(Mode::DIR.perm(0o777)); - inode.size.store(size as u64, Ordering::Relaxed); + fn info(&self) -> &Spin<InodeInfo> { + &self.info + } - inode + fn super_block(&self) -> &SbRef<Self::SuperBlock> { + &self.sb + } + + fn page_cache(&self) -> Option<&PageCache> { + None } } -impl Inode for DirInode { - fn lookup(&self, dentry: &Arc<Dentry>) -> KResult<Option<Arc<dyn Inode>>> { - let vfs = self.vfs.upgrade().ok_or(EIO)?; - let vfs = vfs.as_any().downcast_ref::<FatFs>().unwrap(); - let fat = block_on(vfs.fat.read()); - - let mut entries = ClusterIterator::new(fat.as_ref(), self.ino as ClusterNo) - .read(vfs, 0) - .dirs(); - - let entry = entries.find(|entry| { - entry - .as_ref() - .map(|entry| &entry.filename == &***dentry.name()) - .unwrap_or(true) +impl InodeFileOps for DirInode {} +impl InodeDirOps for DirInode { + async fn lookup(&self, dentry: &Arc<Dentry>) -> KResult<Option<InodeUse<dyn Inode>>> { + let sb = self.sb.get()?; + let dir_pages = self.get_dir_pages().await?; + + let dir_data = dir_pages.iter().map(|page| { + unsafe { + // SAFETY: No one could be writing to it. + page.as_memblk().as_bytes() + } }); - match entry { - None => Ok(None), - Some(Err(err)) => Err(err), - Some(Ok(entry)) => Ok(Some(vfs.get_or_alloc_inode( - entry.cluster as Ino, - entry.is_directory, - entry.size, - ))), + let raw_dirents = dir_data + .map(as_raw_dirents) + .take_while_inclusive(Result::is_ok) + .flatten_ok(); + + let mut dirents = futures::stream::iter(raw_dirents); + + while let Some(result) = dirents.next_dirent().await { + let entry = result?; + + if *entry.filename != ****dentry.name() { + continue; + } + + let sbref = SbRef::from(&sb); + + if entry.is_directory { + return Ok(Some(DirInode::new(entry.cluster, sbref, entry.size) as _)); + } else { + return Ok(Some(FileInode::new(entry.cluster, sbref, entry.size) as _)); + } } + + Ok(None) } - fn do_readdir( - &self, + fn readdir<'r, 'a: 'r, 'b: 'r>( + &'a self, offset: usize, - callback: &mut dyn FnMut(&[u8], Ino) -> KResult<ControlFlow<(), ()>>, - ) -> KResult<usize> { - let vfs = self.vfs.upgrade().ok_or(EIO)?; - let vfs = vfs.as_any().downcast_ref::<FatFs>().unwrap(); - let fat = block_on(vfs.fat.read()); - - let cluster_iter = ClusterIterator::new(fat.as_ref(), self.ino as ClusterNo) - .read(vfs, offset) - .dirs(); - - let mut nread = 0usize; - for entry in cluster_iter { - let entry = entry?; - - vfs.get_or_alloc_inode(entry.cluster as Ino, entry.is_directory, entry.size); - if callback(&entry.filename, entry.cluster as Ino)?.is_break() { - break; + callback: &'b mut (dyn FnMut(&[u8], Ino) -> KResult<bool> + Send), + ) -> impl Future<Output = KResult<KResult<usize>>> + Send + 'r { + async move { + let sb = self.sb.get()?; + let fs = &sb.backend; + let dir_pages = self.get_dir_pages().await?; + + let cluster_size = fs.sectors_per_cluster as usize * SECTOR_SIZE; + + let cluster_offset = offset / cluster_size; + let inner_offset = offset % cluster_size; + let inner_raw_dirent_offset = inner_offset / core::mem::size_of::<dir::RawDirEntry>(); + + let dir_data = dir_pages.iter().skip(cluster_offset).map(|page| { + unsafe { + // SAFETY: No one could be writing to it. + page.as_memblk().as_bytes() + } + }); + + let raw_dirents = dir_data + .map(as_raw_dirents) + .take_while_inclusive(Result::is_ok) + .flatten_ok() + .skip(inner_raw_dirent_offset); + + let mut dirents = futures::stream::iter(raw_dirents); + + let mut nread = 0; + while let Some(result) = dirents.next_dirent().await { + let entry = result?; + + match callback(&entry.filename, entry.cluster.as_ino()) { + Err(err) => return Ok(Err(err)), + Ok(true) => nread += entry.entry_offset as usize, + Ok(false) => break, + } } - nread += entry.entry_offset as usize; + Ok(Ok(nread)) } - - Ok(nread) } } struct FatMountCreator; +#[async_trait] impl MountCreator for FatMountCreator { fn check_signature(&self, mut first_block: &[u8]) -> KResult<bool> { match first_block.split_off(82..) { @@ -417,8 +524,8 @@ impl MountCreator for FatMountCreator { } } - fn create_mount(&self, _source: &str, _flags: u64, mp: &Arc<Dentry>) -> KResult<Mount> { - let (fatfs, root_inode) = FatFs::create(make_device(8, 1))?; + async fn create_mount(&self, _source: &str, _flags: u64, mp: &Arc<Dentry>) -> KResult<Mount> { + let (fatfs, root_inode) = FatFs::create(DeviceId::new(8, 1)).await?; Mount::new(mp, fatfs, root_inode) } diff --git a/src/fs/fat32/dir.rs b/src/fs/fat32/dir.rs index c4ac6c0d..8a5b6f40 100644 --- a/src/fs/fat32/dir.rs +++ b/src/fs/fat32/dir.rs @@ -1,11 +1,16 @@ -use super::file::ClusterReadIterator; +use core::pin::Pin; + +use alloc::{boxed::Box, string::String}; +use futures::{Stream, StreamExt}; +use posix_types::result::PosixError; + use crate::kernel::constants::EINVAL; use crate::prelude::*; -use alloc::{string::String, sync::Arc}; -use itertools::Itertools; + +use super::{Cluster, RawCluster}; #[repr(C, packed)] -pub(super) struct RawDirEntry { +pub struct RawDirEntry { name: [u8; 8], extension: [u8; 3], attr: u8, @@ -21,9 +26,9 @@ pub(super) struct RawDirEntry { size: u32, } -pub(super) struct FatDirectoryEntry { - pub filename: Arc<[u8]>, - pub cluster: u32, +pub struct FatDirectoryEntry { + pub filename: Box<[u8]>, + pub cluster: Cluster, pub size: u32, pub entry_offset: u32, pub is_directory: bool, @@ -79,7 +84,7 @@ impl RawDirEntry { self.attr & Self::ATTR_DIRECTORY != 0 } - fn long_filename(&self) -> Option<[u16; 13]> { + fn as_raw_long_filename(&self) -> Option<[u16; 13]> { if !self.is_long_filename() { return None; } @@ -103,137 +108,114 @@ impl RawDirEntry { } } -impl<'data, I> RawDirs<'data> for I where I: ClusterReadIterator<'data> {} -trait RawDirs<'data>: ClusterReadIterator<'data> { - fn raw_dirs(self) -> impl Iterator<Item = KResult<&'data RawDirEntry>> + 'data - where - Self: Sized, - { - const ENTRY_SIZE: usize = size_of::<RawDirEntry>(); - - self.map(|result| { - let data = result?; - if data.len() % ENTRY_SIZE != 0 { - return Err(EINVAL); - } - - Ok(unsafe { - core::slice::from_raw_parts( - data.as_ptr() as *const RawDirEntry, - data.len() / ENTRY_SIZE, - ) - }) - }) - .flatten_ok() +pub fn as_raw_dirents(data: &[u8]) -> KResult<&[RawDirEntry]> { + let len = data.len(); + if len % size_of::<RawDirEntry>() != 0 { + return Err(EINVAL); } -} - -pub(super) trait Dirs<'data>: ClusterReadIterator<'data> { - fn dirs(self) -> impl Iterator<Item = KResult<FatDirectoryEntry>> + 'data - where - Self: Sized; -} -impl<'data, I> Dirs<'data> for I -where - I: ClusterReadIterator<'data>, -{ - fn dirs(self) -> impl Iterator<Item = KResult<FatDirectoryEntry>> + 'data - where - Self: Sized, - { - self.raw_dirs().real_dirs() + unsafe { + Ok(core::slice::from_raw_parts( + data.as_ptr() as *const RawDirEntry, + len / size_of::<RawDirEntry>(), + )) } } -trait RealDirs<'data>: Iterator<Item = KResult<&'data RawDirEntry>> + 'data { - fn real_dirs(self) -> DirsIter<'data, Self> - where - Self: Sized; +pub trait ParseDirent { + async fn next_dirent(&mut self) -> Option<KResult<FatDirectoryEntry>>; } -impl<'data, I> RealDirs<'data> for I +impl<'a, T> ParseDirent for T where - I: Iterator<Item = KResult<&'data RawDirEntry>> + 'data, + T: Stream<Item = KResult<&'a RawDirEntry>>, { - fn real_dirs(self) -> DirsIter<'data, Self> - where - Self: Sized, - { - DirsIter { iter: self } - } -} + async fn next_dirent(&mut self) -> Option<KResult<FatDirectoryEntry>> { + let mut me = unsafe { Pin::new_unchecked(self) }; + + // The long filename entries are stored in reverse order. + // So we reverse all filename segments and then reverse the whole string at the end. + let mut filename_rev = String::new(); + + let mut is_lfn = false; + let mut nr_entry_scanned = 0; + let mut cur_entry; + + loop { + match me.as_mut().next().await { + Some(Err(err)) => return Some(Err(err)), + Some(Ok(ent)) => { + cur_entry = ent; + nr_entry_scanned += 1; + } + None => { + if is_lfn { + // Unterminated long filename entries are invalid. + return Some(Err(PosixError::EINVAL.into())); + } else { + return None; + } + } + }; -pub(super) struct DirsIter<'data, I> -where - I: Iterator<Item = KResult<&'data RawDirEntry>> + 'data, -{ - iter: I, -} + if !cur_entry.is_invalid() { + break; + } -impl<'data, I> Iterator for DirsIter<'data, I> -where - I: Iterator<Item = KResult<&'data RawDirEntry>> + 'data, -{ - type Item = KResult<FatDirectoryEntry>; - - fn next(&mut self) -> Option<Self::Item> { - let mut filename = String::new(); - let mut entry_offset = 0; - let entry = loop { - let entry = match self.iter.next()? { - Ok(entry) => entry, - Err(err) => return Some(Err(err)), - }; - entry_offset += 1; - - let long_filename = entry.long_filename(); - if entry.is_invalid() { - if let Some(long_filename) = long_filename { - let long_filename = long_filename - .iter() - .position(|&ch| ch == 0) - .map(|pos| &long_filename[..pos]) - .unwrap_or(&long_filename); - - filename.extend( - long_filename - .into_iter() - .map(|&ch| char::from_u32(ch as u32).unwrap_or('?')) - .rev(), - ); - } + let Some(raw_long_filename) = cur_entry.as_raw_long_filename() else { continue; - } - break entry; + }; + + // We are processing a long filename entry. + is_lfn = true; + + let real_len = raw_long_filename + .iter() + .position(|&ch| ch == 0) + .unwrap_or(raw_long_filename.len()); + + let name_codes_rev = raw_long_filename.into_iter().take(real_len).rev(); + let name_chars_rev = char::decode_utf16(name_codes_rev).map(|r| r.unwrap_or('?')); + + filename_rev.extend(name_chars_rev); + } + + // From now on, `entry` represents a valid directory entry. + + let raw_cluster = + RawCluster(cur_entry.cluster_low as u32 | ((cur_entry.cluster_high as u32) << 16)); + + let Some(cluster) = raw_cluster.parse() else { + return Some(Err(PosixError::EINVAL.into())); }; - let filename: Arc<[u8]> = if filename.is_empty() { - let mut filename = entry.filename().to_vec(); - let extension = entry.extension(); + let filename; + + if filename_rev.is_empty() { + let mut name = cur_entry.filename().to_vec(); + let extension = cur_entry.extension(); if !extension.is_empty() { - filename.push(b'.'); - filename.extend_from_slice(extension); + name.push(b'.'); + name.extend_from_slice(extension); } - if entry.is_filename_lowercase() { - filename.make_ascii_lowercase(); + if cur_entry.is_filename_lowercase() { + name.make_ascii_lowercase(); } - filename.into() + filename = name.into_boxed_slice(); } else { - let mut bytes = filename.into_bytes(); - bytes.reverse(); - - bytes.into() - }; + let mut name = filename_rev.into_bytes(); + name.reverse(); + filename = name.into_boxed_slice(); + } Some(Ok(FatDirectoryEntry { - size: entry.size, - entry_offset, + size: cur_entry.size, + entry_offset: nr_entry_scanned * size_of::<RawDirEntry>() as u32, filename, - cluster: entry.cluster_low as u32 | (((entry.cluster_high & !0xF000) as u32) << 16), - is_directory: entry.is_directory(), + cluster, + is_directory: cur_entry.is_directory(), })) } } diff --git a/src/fs/fat32/file.rs b/src/fs/fat32/file.rs index db16df50..2df69728 100644 --- a/src/fs/fat32/file.rs +++ b/src/fs/fat32/file.rs @@ -1,40 +1,24 @@ -use super::{ClusterIterator, FatFs}; -use crate::{ - kernel::mem::{AsMemoryBlock as _, Page}, - KResult, -}; - -pub trait ClusterReadIterator<'data>: Iterator<Item = KResult<&'data [u8]>> + 'data {} -impl<'a, I> ClusterReadIterator<'a> for I where I: Iterator<Item = KResult<&'a [u8]>> + 'a {} +use futures::Stream; -pub(super) trait ClusterRead<'data> { - fn read<'vfs>(self, vfs: &'vfs FatFs, offset: usize) -> impl ClusterReadIterator<'data> - where - Self: Sized, - 'vfs: 'data; -} +use crate::{kernel::mem::Page, prelude::KResult}; -impl<'data, 'fat: 'data> ClusterRead<'data> for ClusterIterator<'fat> { - fn read<'vfs: 'data>(self, vfs: &'vfs FatFs, offset: usize) -> impl ClusterReadIterator<'data> { - const SECTOR_SIZE: usize = 512; +use super::{ClusterIterator, FatFs}; - let cluster_size = vfs.sectors_per_cluster as usize * SECTOR_SIZE; - assert!(cluster_size <= 0x1000, "Cluster size is too large"); +pub trait ReadClusters { + fn read_clusters(self, fs: &FatFs) -> impl Stream<Item = KResult<Page>> + Send; +} - let skip_clusters = offset / cluster_size; - let mut inner_offset = offset % cluster_size; +impl ReadClusters for ClusterIterator<'_> { + fn read_clusters(self, fs: &FatFs) -> impl Stream<Item = KResult<Page>> + Send { + futures::stream::unfold(self, move |mut me| async { + let cluster = me.next()?; + let page = Page::alloc(); - // TODO: Use block cache. - let buffer_page = Page::alloc(); + if let Err(err) = fs.read_cluster(cluster, &page).await { + return Some((Err(err), me)); + } - self.skip(skip_clusters).map(move |cluster| { - vfs.read_cluster(cluster, &buffer_page)?; - let data = unsafe { - // SAFETY: No one could be writing to it. - &buffer_page.as_memblk().as_bytes()[inner_offset..] - }; - inner_offset = 0; - Ok(data) + Some((Ok(page), me)) }) } } diff --git a/src/fs/mod.rs b/src/fs/mod.rs index 5d9285ec..c59ee801 100644 --- a/src/fs/mod.rs +++ b/src/fs/mod.rs @@ -1,5 +1,4 @@ +// pub mod ext4; pub mod fat32; pub mod procfs; -pub mod shm; pub mod tmpfs; -pub mod ext4; diff --git a/src/fs/procfs.rs b/src/fs/procfs.rs index 2ed24613..57b881df 100644 --- a/src/fs/procfs.rs +++ b/src/fs/procfs.rs @@ -1,325 +1,264 @@ -use crate::kernel::constants::{EACCES, ENOTDIR}; -use crate::kernel::task::block_on; +use crate::kernel::constants::{EACCES, EISDIR, ENOTDIR}; use crate::kernel::timer::Instant; -use crate::kernel::vfs::inode::{AtomicMode, Mode}; +use crate::kernel::vfs::inode::{InodeDirOps, InodeFileOps, InodeInfo, InodeOps, InodeUse}; +use crate::kernel::vfs::types::{DeviceId, Format, Permission}; +use crate::kernel::vfs::{SbRef, SbUse, SuperBlock, SuperBlockInfo}; use crate::{ io::Buffer, kernel::{ mem::paging::PageBuffer, vfs::{ dentry::Dentry, - inode::{define_struct_inode, AtomicIno, Ino, Inode, InodeData}, + inode::{Ino, Inode}, mount::{dump_mounts, register_filesystem, Mount, MountCreator}, - vfs::Vfs, - DevId, }, }, prelude::*, }; -use alloc::sync::{Arc, Weak}; -use core::{ops::ControlFlow, sync::atomic::Ordering}; -use eonix_sync::{AsProof as _, AsProofMut as _, LazyLock, Locked}; -use itertools::Itertools; - -#[allow(dead_code)] -pub trait ProcFsFile: Send + Sync { - fn can_read(&self) -> bool { - false - } - - fn can_write(&self) -> bool { - false - } +use alloc::sync::Arc; +use async_trait::async_trait; +use core::future::Future; +use core::sync::atomic::{AtomicU64, Ordering}; +use eonix_sync::{LazyLock, RwLock}; + +struct Node { + ino: Ino, + sb: SbRef<ProcFs>, + info: Spin<InodeInfo>, + kind: NodeKind, +} - fn read(&self, _buffer: &mut PageBuffer) -> KResult<usize> { - Err(EACCES) - } +enum NodeKind { + File(FileInode), + Dir(DirInode), +} - fn write(&self, _buffer: &[u8]) -> KResult<usize> { - Err(EACCES) - } +struct FileInode { + read: Option<Box<dyn Fn(&mut PageBuffer) -> KResult<()> + Send + Sync>>, + write: Option<()>, } -pub enum ProcFsNode { - File(Arc<FileInode>), - Dir(Arc<DirInode>), +struct DirInode { + entries: RwLock<Vec<(Arc<[u8]>, InodeUse<Node>)>>, } -impl ProcFsNode { - fn unwrap(&self) -> Arc<dyn Inode> { - match self { - ProcFsNode::File(inode) => inode.clone(), - ProcFsNode::Dir(inode) => inode.clone(), - } - } +impl InodeOps for Node { + type SuperBlock = ProcFs; fn ino(&self) -> Ino { - match self { - ProcFsNode::File(inode) => inode.ino, - ProcFsNode::Dir(inode) => inode.ino, - } + self.ino } -} -define_struct_inode! { - pub struct FileInode { - file: Box<dyn ProcFsFile>, - } -} - -impl FileInode { - pub fn new(ino: Ino, vfs: Weak<ProcFs>, file: Box<dyn ProcFsFile>) -> Arc<Self> { - let mut mode = Mode::REG; - if file.can_read() { - mode.set_perm(0o444); - } - if file.can_write() { - mode.set_perm(0o222); + fn format(&self) -> Format { + match &self.kind { + NodeKind::File(_) => Format::REG, + NodeKind::Dir(_) => Format::DIR, } + } - let mut inode = Self { - idata: InodeData::new(ino, vfs), - file, - }; + fn info(&self) -> &Spin<InodeInfo> { + &self.info + } - inode.idata.mode.store(mode); - inode.idata.nlink.store(1, Ordering::Relaxed); - *inode.ctime.get_mut() = Instant::now(); - *inode.mtime.get_mut() = Instant::now(); - *inode.atime.get_mut() = Instant::now(); + fn super_block(&self) -> &SbRef<Self::SuperBlock> { + &self.sb + } - Arc::new(inode) + fn page_cache(&self) -> Option<&crate::kernel::mem::PageCache> { + None } } -impl Inode for FileInode { - fn read(&self, buffer: &mut dyn Buffer, offset: usize) -> KResult<usize> { - if !self.file.can_read() { +impl InodeFileOps for Node { + async fn read(&self, buffer: &mut dyn Buffer, offset: usize) -> KResult<usize> { + let NodeKind::File(file_inode) = &self.kind else { + return Err(EISDIR); + }; + + let Some(read_fn) = &file_inode.read else { return Err(EACCES); - } + }; let mut page_buffer = PageBuffer::new(); - self.file.read(&mut page_buffer)?; + read_fn(&mut page_buffer)?; - let data = page_buffer - .data() - .split_at_checked(offset) - .map(|(_, data)| data); + let Some((_, data)) = page_buffer.data().split_at_checked(offset) else { + return Ok(0); + }; - match data { - None => Ok(0), - Some(data) => Ok(buffer.fill(data)?.allow_partial()), - } + Ok(buffer.fill(data)?.allow_partial()) } } -define_struct_inode! { - pub struct DirInode { - entries: Locked<Vec<(Arc<[u8]>, ProcFsNode)>, ()>, - } -} +impl InodeDirOps for Node { + async fn lookup(&self, dentry: &Arc<Dentry>) -> KResult<Option<InodeUse<dyn Inode>>> { + let NodeKind::Dir(dir) = &self.kind else { + return Err(ENOTDIR); + }; -impl DirInode { - pub fn new(ino: Ino, vfs: Weak<ProcFs>) -> Arc<Self> { - Self::new_locked(ino, vfs, |inode, rwsem| unsafe { - addr_of_mut_field!(inode, entries).write(Locked::new(vec![], rwsem)); - addr_of_mut_field!(&mut *inode, mode).write(AtomicMode::from(Mode::DIR.perm(0o755))); - addr_of_mut_field!(&mut *inode, nlink).write(1.into()); - addr_of_mut_field!(&mut *inode, ctime).write(Spin::new(Instant::now())); - addr_of_mut_field!(&mut *inode, mtime).write(Spin::new(Instant::now())); - addr_of_mut_field!(&mut *inode, atime).write(Spin::new(Instant::now())); - }) - } -} + let entries = dir.entries.read().await; + + let dent_name = dentry.name(); + for (name, node) in entries.iter() { + if *name == ***dent_name { + return Ok(Some(node.clone() as _)); + } + } -impl Inode for DirInode { - fn lookup(&self, dentry: &Arc<Dentry>) -> KResult<Option<Arc<dyn Inode>>> { - let lock = block_on(self.rwsem.read()); - Ok(self - .entries - .access(lock.prove()) - .iter() - .find_map(|(name, node)| (name == &***dentry.name()).then(|| node.unwrap()))) + Ok(None) } - fn do_readdir( - &self, + fn readdir<'r, 'a: 'r, 'b: 'r>( + &'a self, offset: usize, - callback: &mut dyn FnMut(&[u8], Ino) -> KResult<ControlFlow<(), ()>>, - ) -> KResult<usize> { - let lock = block_on(self.rwsem.read()); - self.entries - .access(lock.prove()) - .iter() - .skip(offset) - .map(|(name, node)| callback(name.as_ref(), node.ino())) - .take_while(|result| result.map_or(true, |flow| flow.is_continue())) - .take_while_inclusive(|result| result.is_ok()) - .fold_ok(0, |acc, _| acc + 1) + callback: &'b mut (dyn FnMut(&[u8], Ino) -> KResult<bool> + Send), + ) -> impl Future<Output = KResult<KResult<usize>>> + Send + 'r { + Box::pin(async move { + let NodeKind::Dir(dir) = &self.kind else { + return Err(ENOTDIR); + }; + + let entries = dir.entries.read().await; + + let mut count = 0; + for (name, node) in entries.iter().skip(offset) { + match callback(name.as_ref(), node.ino) { + Err(err) => return Ok(Err(err)), + Ok(true) => count += 1, + Ok(false) => break, + } + } + + Ok(Ok(count)) + }) } } -impl_any!(ProcFs); -pub struct ProcFs { - root_node: Arc<DirInode>, - next_ino: AtomicIno, -} - -impl Vfs for ProcFs { - fn io_blksize(&self) -> usize { - 4096 - } - - fn fs_devid(&self) -> DevId { - 10 +impl Node { + pub fn new_file( + ino: Ino, + sb: SbRef<ProcFs>, + read: impl Fn(&mut PageBuffer) -> KResult<()> + Send + Sync + 'static, + ) -> InodeUse<Self> { + InodeUse::new(Self { + ino, + sb, + info: Spin::new(InodeInfo { + size: 0, + nlink: 1, + uid: 0, + gid: 0, + perm: Permission::new(0o444), + atime: Instant::UNIX_EPOCH, + ctime: Instant::UNIX_EPOCH, + mtime: Instant::UNIX_EPOCH, + }), + kind: NodeKind::File(FileInode::new(Box::new(read))), + }) } - fn is_read_only(&self) -> bool { - false + fn new_dir(ino: Ino, sb: SbRef<ProcFs>) -> InodeUse<Self> { + InodeUse::new(Self { + ino, + sb, + info: Spin::new(InodeInfo { + size: 0, + nlink: 1, + uid: 0, + gid: 0, + perm: Permission::new(0o755), + atime: Instant::UNIX_EPOCH, + ctime: Instant::UNIX_EPOCH, + mtime: Instant::UNIX_EPOCH, + }), + kind: NodeKind::Dir(DirInode::new()), + }) } } -static GLOBAL_PROCFS: LazyLock<Arc<ProcFs>> = LazyLock::new(|| { - Arc::new_cyclic(|weak: &Weak<ProcFs>| ProcFs { - root_node: DirInode::new(0, weak.clone()), - next_ino: AtomicIno::new(1), - }) -}); - -struct ProcFsMountCreator; - -#[allow(dead_code)] -impl ProcFsMountCreator { - pub fn get() -> Arc<ProcFs> { - GLOBAL_PROCFS.clone() - } - - pub fn get_weak() -> Weak<ProcFs> { - Arc::downgrade(&GLOBAL_PROCFS) +impl FileInode { + fn new(read: Box<dyn Fn(&mut PageBuffer) -> KResult<()> + Send + Sync>) -> Self { + Self { + read: Some(read), + write: None, + } } } -impl MountCreator for ProcFsMountCreator { - fn create_mount(&self, _source: &str, _flags: u64, mp: &Arc<Dentry>) -> KResult<Mount> { - let vfs = ProcFsMountCreator::get(); - let root_inode = vfs.root_node.clone(); - Mount::new(mp, vfs, root_inode) - } - - fn check_signature(&self, _: &[u8]) -> KResult<bool> { - Ok(true) +impl DirInode { + pub fn new() -> Self { + Self { + entries: RwLock::new(vec![]), + } } } -pub fn root() -> ProcFsNode { - let vfs = ProcFsMountCreator::get(); - let root = vfs.root_node.clone(); - - ProcFsNode::Dir(root) +pub struct ProcFs { + root: InodeUse<Node>, + next_ino: AtomicU64, } -pub fn creat( - parent: &ProcFsNode, - name: Arc<[u8]>, - file: Box<dyn ProcFsFile>, -) -> KResult<ProcFsNode> { - let parent = match parent { - ProcFsNode::File(_) => return Err(ENOTDIR), - ProcFsNode::Dir(parent) => parent, - }; - - let fs = ProcFsMountCreator::get(); - let ino = fs.next_ino.fetch_add(1, Ordering::Relaxed); - - let inode = FileInode::new(ino, Arc::downgrade(&fs), file); - - { - let lock = block_on(parent.idata.rwsem.write()); - parent - .entries - .access_mut(lock.prove_mut()) - .push((name, ProcFsNode::File(inode.clone()))); +impl SuperBlock for ProcFs {} +impl ProcFs { + fn assign_ino(&self) -> Ino { + Ino::new(self.next_ino.fetch_add(1, Ordering::Relaxed)) } - - Ok(ProcFsNode::File(inode)) } -#[allow(dead_code)] -pub fn mkdir(parent: &ProcFsNode, name: &[u8]) -> KResult<ProcFsNode> { - let parent = match parent { - ProcFsNode::File(_) => return Err(ENOTDIR), - ProcFsNode::Dir(parent) => parent, - }; - - let fs = ProcFsMountCreator::get(); - let ino = fs.next_ino.fetch_add(1, Ordering::Relaxed); - - let inode = DirInode::new(ino, Arc::downgrade(&fs)); +static GLOBAL_PROCFS: LazyLock<SbUse<ProcFs>> = LazyLock::new(|| { + SbUse::new_cyclic( + SuperBlockInfo { + io_blksize: 4096, + device_id: DeviceId::new(0, 10), + read_only: false, + }, + |sbref| ProcFs { + root: Node::new_dir(Ino::new(0), sbref), + next_ino: AtomicU64::new(1), + }, + ) +}); - parent - .entries - .access_mut(block_on(inode.rwsem.write()).prove_mut()) - .push((Arc::from(name), ProcFsNode::Dir(inode.clone()))); +struct ProcFsMountCreator; - Ok(ProcFsNode::Dir(inode)) -} +#[async_trait] +impl MountCreator for ProcFsMountCreator { + async fn create_mount(&self, _source: &str, _flags: u64, mp: &Arc<Dentry>) -> KResult<Mount> { + let fs = GLOBAL_PROCFS.clone(); + let root_inode = fs.backend.root.clone(); -struct DumpMountsFile; -impl ProcFsFile for DumpMountsFile { - fn can_read(&self) -> bool { - true + Mount::new(mp, fs, root_inode) } - fn read(&self, buffer: &mut PageBuffer) -> KResult<usize> { - dump_mounts(&mut buffer.get_writer()); - - Ok(buffer.data().len()) + fn check_signature(&self, _: &[u8]) -> KResult<bool> { + Ok(true) } } -pub fn init() { - register_filesystem("procfs", Arc::new(ProcFsMountCreator)).unwrap(); - - creat( - &root(), - Arc::from(b"mounts".as_slice()), - Box::new(DumpMountsFile), - ) - .unwrap(); -} - -pub struct GenericProcFsFile<ReadFn> +pub async fn populate_root<F>(name: Arc<[u8]>, read_fn: F) where - ReadFn: Send + Sync + Fn(&mut PageBuffer) -> KResult<()>, + F: Send + Sync + Fn(&mut PageBuffer) -> KResult<()> + 'static, { - read_fn: Option<ReadFn>, -} + let procfs = &GLOBAL_PROCFS.backend; + let root = &procfs.root; -impl<ReadFn> ProcFsFile for GenericProcFsFile<ReadFn> -where - ReadFn: Send + Sync + Fn(&mut PageBuffer) -> KResult<()>, -{ - fn can_read(&self) -> bool { - self.read_fn.is_some() - } + let NodeKind::Dir(root) = &root.kind else { + unreachable!(); + }; - fn read(&self, buffer: &mut PageBuffer) -> KResult<usize> { - self.read_fn.as_ref().ok_or(EACCES)?(buffer).map(|_| buffer.data().len()) - } + let mut entries = root.entries.write().await; + entries.push(( + name.clone(), + Node::new_file(procfs.assign_ino(), SbRef::from(&GLOBAL_PROCFS), read_fn), + )); } -pub fn populate_root<F>(name: Arc<[u8]>, read_fn: F) -> KResult<()> -where - F: Send + Sync + Fn(&mut PageBuffer) -> KResult<()> + 'static, -{ - let root = root(); - - creat( - &root, - name, - Box::new(GenericProcFsFile { - read_fn: Some(read_fn), - }), - ) - .map(|_| ()) +pub async fn init() { + register_filesystem("procfs", Arc::new(ProcFsMountCreator)).unwrap(); + + populate_root(Arc::from(b"mounts".as_slice()), |buffer| { + dump_mounts(&mut buffer.get_writer()); + Ok(()) + }) + .await; } diff --git a/src/fs/shm.rs b/src/fs/shm.rs deleted file mode 100644 index 09d36141..00000000 --- a/src/fs/shm.rs +++ /dev/null @@ -1,146 +0,0 @@ -use core::sync::atomic::{AtomicU32, Ordering}; - -use alloc::{collections::btree_map::BTreeMap, sync::Arc}; -use bitflags::bitflags; -use eonix_sync::{LazyLock, Mutex}; - -use crate::{ - fs::tmpfs::{DirectoryInode, FileInode, TmpFs}, - kernel::{constants::ENOSPC, vfs::inode::Mode}, - prelude::KResult, -}; - -bitflags! { - #[derive(Debug, Clone, Copy)] - pub struct ShmFlags: u32 { - /// Create a new segment. If this flag is not used, then shmget() will - /// find the segment associated with key and check to see if the user - /// has permission to access the segment. - const IPC_CREAT = 0o1000; - /// This flag is used with IPC_CREAT to ensure that this call creates - /// the segment. If the segment already exists, the call fails. - const IPC_EXCL = 0o2000; - - /// Attach the segment for read-only access.If this flag is not specified, - /// the segment is attached for read and write access, and the process - /// must have read and write permission for the segment. - const SHM_RDONLY = 0o10000; - /// round attach address to SHMLBA boundary - const SHM_RND = 0o20000; - /// Allow the contents of the segment to be executed. - const SHM_EXEC = 0o100000; - } -} - -pub const IPC_PRIVATE: usize = 0; - -pub struct ShmManager { - tmpfs: Arc<TmpFs>, - root: Arc<DirectoryInode>, - areas: BTreeMap<u32, ShmArea>, -} - -#[repr(C)] -#[derive(Default, Clone, Copy, Debug)] -pub struct IpcPerm { - key: i32, - uid: u32, - gid: u32, - cuid: u32, - cgid: u32, - mode: u16, - seq: u16, -} - -#[repr(C)] -#[derive(Debug, Clone, Copy)] -pub struct ShmIdDs { - // Ownership and permissions - pub shm_perm: IpcPerm, - // Size of segment (bytes). In our system, this must be aligned - pub shm_segsz: usize, - // Last attach time - pub shm_atime: usize, - // Last detach time - pub shm_dtime: usize, - // Creation time/time of last modification via shmctl() - pub shm_ctime: usize, - // PID of creator - pub shm_cpid: usize, - // PID of last shmat(2)/shmdt(2) - pub shm_lpid: usize, - // No. of current attaches - pub shm_nattch: usize, -} - -impl ShmIdDs { - fn new(size: usize, pid: u32) -> Self { - Self { - shm_perm: IpcPerm::default(), - shm_segsz: size, - shm_atime: 0, - shm_dtime: 0, - shm_ctime: 0, // Should set instant now - shm_cpid: pid as usize, - shm_lpid: 0, - shm_nattch: 0, - } - } -} - -#[derive(Debug)] -pub struct ShmArea { - pub area: Arc<FileInode>, - pub shmid_ds: ShmIdDs, -} - -// A big lock here to protect the shared memory area. -// Can be improved with finer-grained locking? -pub static SHM_MANAGER: LazyLock<Mutex<ShmManager>> = - LazyLock::new(|| Mutex::new(ShmManager::new())); - -impl ShmManager { - fn new() -> Self { - let (tmpfs, root) = TmpFs::create(false).expect("should create shm_area successfully"); - Self { - tmpfs, - root, - areas: BTreeMap::new(), - } - } - - pub fn create_shared_area(&self, size: usize, pid: u32, mode: Mode) -> ShmArea { - let ino = self.tmpfs.assign_ino(); - let vfs = Arc::downgrade(&self.tmpfs); - ShmArea { - area: FileInode::new(ino, vfs, size, mode), - shmid_ds: ShmIdDs::new(size, pid), - } - } - - pub fn get(&self, shmid: u32) -> Option<&ShmArea> { - self.areas.get(&shmid) - } - - pub fn insert(&mut self, shmid: u32, area: ShmArea) { - self.areas.insert(shmid, area); - } -} - -pub fn gen_shm_id(key: usize) -> KResult<u32> { - const SHM_MAGIC: u32 = 114514000; - - static NEXT_SHMID: AtomicU32 = AtomicU32::new(0); - - if key == IPC_PRIVATE { - let shmid = NEXT_SHMID.fetch_add(1, Ordering::Relaxed); - - if shmid >= SHM_MAGIC { - return Err(ENOSPC); - } else { - return Ok(shmid); - } - } - - (key as u32).checked_add(SHM_MAGIC).ok_or(ENOSPC) -} diff --git a/src/fs/tmpfs.rs b/src/fs/tmpfs.rs deleted file mode 100644 index 7a5bd52b..00000000 --- a/src/fs/tmpfs.rs +++ /dev/null @@ -1,613 +0,0 @@ -use crate::io::Stream; -use crate::kernel::constants::{EEXIST, EINVAL, EIO, EISDIR, ENOENT, ENOSYS, ENOTDIR}; -use crate::kernel::mem::{CachePage, CachePageStream, PageCache, PageCacheBackend}; -use crate::kernel::task::block_on; -use crate::kernel::timer::Instant; -use crate::kernel::vfs::inode::RenameData; -use crate::kernel::vfs::inode::{AtomicMode, InodeData}; -use crate::{ - io::Buffer, - kernel::vfs::{ - dentry::{dcache, Dentry}, - inode::{define_struct_inode, AtomicIno, Ino, Inode, Mode, WriteOffset}, - mount::{register_filesystem, Mount, MountCreator, MS_RDONLY}, - vfs::Vfs, - DevId, - }, - prelude::*, -}; -use alloc::sync::{Arc, Weak}; -use core::fmt::Debug; -use core::{ops::ControlFlow, sync::atomic::Ordering}; -use eonix_mm::paging::PAGE_SIZE; -use eonix_sync::{AsProof as _, AsProofMut as _, Locked, Mutex, ProofMut}; -use itertools::Itertools; - -fn acquire(vfs: &Weak<dyn Vfs>) -> KResult<Arc<dyn Vfs>> { - vfs.upgrade().ok_or(EIO) -} - -fn astmp(vfs: &Arc<dyn Vfs>) -> &TmpFs { - vfs.as_any() - .downcast_ref::<TmpFs>() - .expect("corrupted tmpfs data structure") -} - -define_struct_inode! { - struct NodeInode { - devid: DevId, - } -} - -impl NodeInode { - fn new(ino: Ino, vfs: Weak<dyn Vfs>, mode: Mode, devid: DevId) -> Arc<Self> { - Self::new_locked(ino, vfs, |inode, _| unsafe { - addr_of_mut_field!(inode, devid).write(devid); - - addr_of_mut_field!(&mut *inode, mode).write(AtomicMode::from(mode)); - addr_of_mut_field!(&mut *inode, nlink).write(1.into()); - addr_of_mut_field!(&mut *inode, ctime).write(Spin::new(Instant::now())); - addr_of_mut_field!(&mut *inode, mtime).write(Spin::new(Instant::now())); - addr_of_mut_field!(&mut *inode, atime).write(Spin::new(Instant::now())); - }) - } -} - -impl Inode for NodeInode { - fn devid(&self) -> KResult<DevId> { - Ok(self.devid) - } -} - -define_struct_inode! { - pub(super) struct DirectoryInode { - entries: Locked<Vec<(Arc<[u8]>, Ino)>, ()>, - } -} - -impl DirectoryInode { - fn new(ino: Ino, vfs: Weak<dyn Vfs>, mode: Mode) -> Arc<Self> { - Self::new_locked(ino, vfs, |inode, rwsem| unsafe { - addr_of_mut_field!(inode, entries) - .write(Locked::new(vec![(Arc::from(b".".as_slice()), ino)], rwsem)); - - addr_of_mut_field!(&mut *inode, size).write(1.into()); - addr_of_mut_field!(&mut *inode, mode) - .write(AtomicMode::from(Mode::DIR.perm(mode.non_format_bits()))); - addr_of_mut_field!(&mut *inode, nlink).write(1.into()); // link from `.` to itself - addr_of_mut_field!(&mut *inode, ctime).write(Spin::new(Instant::now())); - addr_of_mut_field!(&mut *inode, mtime).write(Spin::new(Instant::now())); - addr_of_mut_field!(&mut *inode, atime).write(Spin::new(Instant::now())); - }) - } - - fn link(&self, name: Arc<[u8]>, file: &dyn Inode, dlock: ProofMut<'_, ()>) { - let now = Instant::now(); - - // SAFETY: Only `unlink` will do something based on `nlink` count - // No need to synchronize here - file.nlink.fetch_add(1, Ordering::Relaxed); - *self.ctime.lock() = now; - - // SAFETY: `rwsem` has done the synchronization - self.size.fetch_add(1, Ordering::Relaxed); - *self.mtime.lock() = now; - - self.entries.access_mut(dlock).push((name, file.ino)); - } - - fn do_unlink( - &self, - file: &Arc<dyn Inode>, - filename: &[u8], - entries: &mut Vec<(Arc<[u8]>, Ino)>, - now: Instant, - decrease_size: bool, - _dir_lock: ProofMut<()>, - _file_lock: ProofMut<()>, - ) -> KResult<()> { - // SAFETY: `file_lock` has done the synchronization - if file.mode.load().is_dir() { - return Err(EISDIR); - } - - entries.retain(|(name, ino)| *ino != file.ino || name.as_ref() != filename); - - if decrease_size { - // SAFETY: `dir_lock` has done the synchronization - self.size.fetch_sub(1, Ordering::Relaxed); - } - - *self.mtime.lock() = now; - - // The last reference to the inode is held by some dentry - // and will be released when the dentry is released - - // SAFETY: `file_lock` has done the synchronization - file.nlink.fetch_sub(1, Ordering::Relaxed); - *file.ctime.lock() = now; - - Ok(()) - } -} - -impl Inode for DirectoryInode { - fn do_readdir( - &self, - offset: usize, - callback: &mut dyn FnMut(&[u8], Ino) -> KResult<ControlFlow<(), ()>>, - ) -> KResult<usize> { - let lock = block_on(self.rwsem.read()); - self.entries - .access(lock.prove()) - .iter() - .skip(offset) - .map(|(name, ino)| callback(&name, *ino)) - .take_while(|result| result.map_or(true, |flow| flow.is_continue())) - .take_while_inclusive(|result| result.is_ok()) - .fold_ok(0, |acc, _| acc + 1) - } - - fn creat(&self, at: &Arc<Dentry>, mode: Mode) -> KResult<()> { - let vfs = acquire(&self.vfs)?; - let vfs = astmp(&vfs); - - let rwsem = block_on(self.rwsem.write()); - - let ino = vfs.assign_ino(); - let file = FileInode::new(ino, self.vfs.clone(), 0, mode); - - self.link(at.get_name(), file.as_ref(), rwsem.prove_mut()); - at.save_reg(file) - } - - fn mknod(&self, at: &Dentry, mode: Mode, dev: DevId) -> KResult<()> { - if !mode.is_chr() && !mode.is_blk() { - return Err(EINVAL); - } - - let vfs = acquire(&self.vfs)?; - let vfs = astmp(&vfs); - - let rwsem = block_on(self.rwsem.write()); - - let ino = vfs.assign_ino(); - let file = NodeInode::new(ino, self.vfs.clone(), mode, dev); - - self.link(at.get_name(), file.as_ref(), rwsem.prove_mut()); - at.save_reg(file) - } - - fn symlink(&self, at: &Arc<Dentry>, target: &[u8]) -> KResult<()> { - let vfs = acquire(&self.vfs)?; - let vfs = astmp(&vfs); - - let rwsem = block_on(self.rwsem.write()); - - let ino = vfs.assign_ino(); - let file = SymlinkInode::new(ino, self.vfs.clone(), target.into()); - - self.link(at.get_name(), file.as_ref(), rwsem.prove_mut()); - at.save_symlink(file) - } - - fn mkdir(&self, at: &Dentry, mode: Mode) -> KResult<()> { - let vfs = acquire(&self.vfs)?; - let vfs = astmp(&vfs); - - let rwsem = block_on(self.rwsem.write()); - - let ino = vfs.assign_ino(); - let newdir = DirectoryInode::new(ino, self.vfs.clone(), mode); - - self.link(at.get_name(), newdir.as_ref(), rwsem.prove_mut()); - at.save_dir(newdir) - } - - fn unlink(&self, at: &Arc<Dentry>) -> KResult<()> { - let _vfs = acquire(&self.vfs)?; - - let dir_lock = block_on(self.rwsem.write()); - - let file = at.get_inode()?; - let filename = at.get_name(); - let file_lock = block_on(file.rwsem.write()); - - let entries = self.entries.access_mut(dir_lock.prove_mut()); - - self.do_unlink( - &file, - &filename, - entries, - Instant::now(), - true, - dir_lock.prove_mut(), - file_lock.prove_mut(), - )?; - - // Remove the dentry from the dentry cache immediately - // so later lookup will fail with ENOENT - dcache::d_remove(at); - - Ok(()) - } - - fn chmod(&self, mode: Mode) -> KResult<()> { - let _vfs = acquire(&self.vfs)?; - let _lock = block_on(self.rwsem.write()); - - // SAFETY: `rwsem` has done the synchronization - let old = self.mode.load(); - self.mode.store(old.perm(mode.non_format_bits())); - *self.ctime.lock() = Instant::now(); - - Ok(()) - } - - fn rename(&self, rename_data: RenameData) -> KResult<()> { - let RenameData { - old_dentry, - new_dentry, - new_parent, - is_exchange, - no_replace, - vfs, - } = rename_data; - - if is_exchange { - println_warn!("TmpFs does not support exchange rename for now"); - return Err(ENOSYS); - } - - let vfs = vfs - .as_any() - .downcast_ref::<TmpFs>() - .expect("vfs must be a TmpFs"); - - let _rename_lock = block_on(vfs.rename_lock.lock()); - - let old_file = old_dentry.get_inode()?; - let new_file = new_dentry.get_inode(); - - if no_replace && new_file.is_ok() { - return Err(EEXIST); - } - - let same_parent = Arc::as_ptr(&new_parent) == &raw const *self; - if same_parent { - // Same directory rename - // Remove from old location and add to new location - let parent_lock = block_on(self.rwsem.write()); - let entries = self.entries.access_mut(parent_lock.prove_mut()); - - fn rename_old( - old_entry: &mut (Arc<[u8]>, Ino), - old_file: &Arc<dyn Inode + 'static>, - new_dentry: &Arc<Dentry>, - now: Instant, - ) { - let (name, _) = old_entry; - *name = new_dentry.get_name(); - *old_file.ctime.lock() = now; - } - - let old_ino = old_file.ino; - let new_ino = new_file.as_ref().ok().map(|f| f.ino); - let old_name = old_dentry.get_name(); - let new_name = new_dentry.get_name(); - - // Find the old and new entries in the directory after we've locked the directory. - let indices = - entries - .iter() - .enumerate() - .fold([None, None], |[old, new], (idx, (name, ino))| { - if Some(*ino) == new_ino && *name == new_name { - [old, Some(idx)] - } else if *ino == old_ino && *name == old_name { - [Some(idx), new] - } else { - [old, new] - } - }); - - let (old_entry_idx, new_entry_idx) = match indices { - [None, ..] => return Err(ENOENT), - [Some(old_idx), new_idx] => (old_idx, new_idx), - }; - - let now = Instant::now(); - - if let Some(new_idx) = new_entry_idx { - // Replace existing file (i.e. rename the old and unlink the new) - let new_file = new_file.unwrap(); - let _new_file_lock = block_on(new_file.rwsem.write()); - - // SAFETY: `new_file_lock` has done the synchronization - match (new_file.mode.load(), old_file.mode.load()) { - (Mode::DIR, _) => return Err(EISDIR), - (_, Mode::DIR) => return Err(ENOTDIR), - _ => {} - } - - entries.remove(new_idx); - - // SAFETY: `parent_lock` has done the synchronization - self.size.fetch_sub(1, Ordering::Relaxed); - - // The last reference to the inode is held by some dentry - // and will be released when the dentry is released - - // SAFETY: `new_file_lock` has done the synchronization - new_file.nlink.fetch_sub(1, Ordering::Relaxed); - *new_file.ctime.lock() = now; - } - - rename_old(&mut entries[old_entry_idx], &old_file, new_dentry, now); - *self.mtime.lock() = now; - } else { - // Cross-directory rename - handle similar to same directory case - - // Get new parent directory - let new_parent_inode = new_dentry.parent().get_inode()?; - assert!(new_parent_inode.is_dir()); - let new_parent = (new_parent_inode.as_ref() as &dyn Any) - .downcast_ref::<DirectoryInode>() - .expect("new parent must be a DirectoryInode"); - - let old_parent_lock = block_on(self.rwsem.write()); - let new_parent_lock = block_on(new_parent_inode.rwsem.write()); - - let old_ino = old_file.ino; - let new_ino = new_file.as_ref().ok().map(|f| f.ino); - let old_name = old_dentry.get_name(); - let new_name = new_dentry.get_name(); - - // Find the old entry in the old directory - let old_entries = self.entries.access_mut(old_parent_lock.prove_mut()); - let old_pos = old_entries - .iter() - .position(|(name, ino)| *ino == old_ino && *name == old_name) - .ok_or(ENOENT)?; - - // Find the new entry in the new directory (if it exists) - let new_entries = new_parent.entries.access_mut(new_parent_lock.prove_mut()); - let has_new = new_entries - .iter() - .position(|(name, ino)| Some(*ino) == new_ino && *name == new_name) - .is_some(); - - let now = Instant::now(); - - if has_new { - // Replace existing file (i.e. move the old and unlink the new) - let new_file = new_file.unwrap(); - let new_file_lock = block_on(new_file.rwsem.write()); - - match (old_file.mode.load(), new_file.mode.load()) { - (Mode::DIR, Mode::DIR) => {} - (Mode::DIR, _) => return Err(ENOTDIR), - (_, _) => {} - } - - // Unlink the old file that was replaced - new_parent.do_unlink( - &new_file, - &new_name, - new_entries, - now, - false, - new_parent_lock.prove_mut(), - new_file_lock.prove_mut(), - )?; - } else { - new_parent.size.fetch_add(1, Ordering::Relaxed); - } - - // Remove from old directory - old_entries.remove(old_pos); - - // Add new entry - new_entries.push((new_name, old_ino)); - - self.size.fetch_sub(1, Ordering::Relaxed); - *self.mtime.lock() = now; - *old_file.ctime.lock() = now; - } - - block_on(dcache::d_exchange(old_dentry, new_dentry)); - - Ok(()) - } -} - -define_struct_inode! { - struct SymlinkInode { - target: Arc<[u8]>, - } -} - -impl SymlinkInode { - fn new(ino: Ino, vfs: Weak<dyn Vfs>, target: Arc<[u8]>) -> Arc<Self> { - Self::new_locked(ino, vfs, |inode, _| unsafe { - let len = target.len(); - addr_of_mut_field!(inode, target).write(target); - - addr_of_mut_field!(&mut *inode, mode).write(AtomicMode::from(Mode::LNK.perm(0o777))); - addr_of_mut_field!(&mut *inode, size).write((len as u64).into()); - addr_of_mut_field!(&mut *inode, ctime).write(Spin::new(Instant::now())); - addr_of_mut_field!(&mut *inode, mtime).write(Spin::new(Instant::now())); - addr_of_mut_field!(&mut *inode, atime).write(Spin::new(Instant::now())); - }) - } -} - -impl Inode for SymlinkInode { - fn readlink(&self, buffer: &mut dyn Buffer) -> KResult<usize> { - buffer - .fill(self.target.as_ref()) - .map(|result| result.allow_partial()) - } - - fn chmod(&self, _: Mode) -> KResult<()> { - Ok(()) - } -} - -define_struct_inode! { - pub struct FileInode { - pages: PageCache, - } -} - -impl Debug for FileInode { - fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { - write!(f, "FileInode({:?})", self.idata) - } -} - -impl FileInode { - pub fn new(ino: Ino, vfs: Weak<dyn Vfs>, size: usize, mode: Mode) -> Arc<Self> { - let inode = Arc::new_cyclic(|weak_self: &Weak<FileInode>| FileInode { - idata: InodeData::new(ino, vfs), - pages: PageCache::new(weak_self.clone()), - }); - - inode.mode.store(Mode::REG.perm(mode.non_format_bits())); - inode.nlink.store(1, Ordering::Relaxed); - inode.size.store(size as u64, Ordering::Relaxed); - inode - } -} - -impl PageCacheBackend for FileInode { - fn read_page(&self, _cache_page: &mut CachePage, _offset: usize) -> KResult<usize> { - Ok(PAGE_SIZE) - } - - fn write_page(&self, _page: &mut CachePageStream, _offset: usize) -> KResult<usize> { - Ok(PAGE_SIZE) - } - - fn size(&self) -> usize { - self.size.load(Ordering::Relaxed) as usize - } -} - -impl Inode for FileInode { - fn page_cache(&self) -> Option<&PageCache> { - Some(&self.pages) - } - - fn read(&self, buffer: &mut dyn Buffer, offset: usize) -> KResult<usize> { - let _lock = block_on(self.rwsem.write()); - block_on(self.pages.read(buffer, offset)) - } - - fn write(&self, stream: &mut dyn Stream, offset: WriteOffset) -> KResult<usize> { - // TODO: We don't need that strong guarantee, find some way to avoid locks - let _lock = block_on(self.rwsem.write()); - - let mut store_new_end = None; - let offset = match offset { - WriteOffset::Position(offset) => offset, - WriteOffset::End(end) => { - store_new_end = Some(end); - - // SAFETY: `lock` has done the synchronization - self.size.load(Ordering::Relaxed) as usize - } - }; - - let wrote = block_on(self.pages.write(stream, offset))?; - let cursor_end = offset + wrote; - - if let Some(store_end) = store_new_end { - *store_end = cursor_end; - } - - // SAFETY: `lock` has done the synchronization - *self.mtime.lock() = Instant::now(); - self.size.store(cursor_end as u64, Ordering::Relaxed); - - Ok(wrote) - } - - fn truncate(&self, length: usize) -> KResult<()> { - let _lock = block_on(self.rwsem.write()); - block_on(self.pages.resize(length))?; - self.size.store(length as u64, Ordering::Relaxed); - *self.mtime.lock() = Instant::now(); - Ok(()) - } - - fn chmod(&self, mode: Mode) -> KResult<()> { - let _vfs = acquire(&self.vfs)?; - let _lock = block_on(self.rwsem.write()); - - // SAFETY: `rwsem` has done the synchronization - let old = self.mode.load(); - self.mode.store(old.perm(mode.non_format_bits())); - *self.ctime.lock() = Instant::now(); - - Ok(()) - } -} - -impl_any!(TmpFs); -pub(super) struct TmpFs { - next_ino: AtomicIno, - readonly: bool, - rename_lock: Mutex<()>, -} - -impl Vfs for TmpFs { - fn io_blksize(&self) -> usize { - 4096 - } - - fn fs_devid(&self) -> DevId { - 2 - } - - fn is_read_only(&self) -> bool { - self.readonly - } -} - -impl TmpFs { - pub(super) fn assign_ino(&self) -> Ino { - self.next_ino.fetch_add(1, Ordering::AcqRel) - } - - pub fn create(readonly: bool) -> KResult<(Arc<TmpFs>, Arc<DirectoryInode>)> { - let tmpfs = Arc::new(Self { - next_ino: AtomicIno::new(1), - readonly, - rename_lock: Mutex::new(()), - }); - - let weak = Arc::downgrade(&tmpfs); - let root_dir = DirectoryInode::new(0, weak, Mode::new(0o755)); - - Ok((tmpfs, root_dir)) - } -} - -struct TmpFsMountCreator; - -impl MountCreator for TmpFsMountCreator { - fn create_mount(&self, _source: &str, flags: u64, mp: &Arc<Dentry>) -> KResult<Mount> { - let (fs, root_inode) = TmpFs::create(flags & MS_RDONLY != 0)?; - - Mount::new(mp, fs, root_inode) - } - - fn check_signature(&self, _: &[u8]) -> KResult<bool> { - Ok(true) - } -} - -pub fn init() { - register_filesystem("tmpfs", Arc::new(TmpFsMountCreator)).unwrap(); -} diff --git a/src/fs/tmpfs/dir.rs b/src/fs/tmpfs/dir.rs new file mode 100644 index 00000000..e2be1d12 --- /dev/null +++ b/src/fs/tmpfs/dir.rs @@ -0,0 +1,415 @@ +use core::{any::Any, future::Future}; + +use alloc::{boxed::Box, sync::Arc, vec, vec::Vec}; +use eonix_log::println_warn; +use eonix_sync::{LazyLock, RwLock, Spin}; + +use crate::{ + kernel::{ + constants::{EEXIST, EINVAL, EISDIR, ENOENT, ENOSYS, ENOTDIR}, + mem::PageCache, + timer::Instant, + vfs::{ + dentry::{dcache, Dentry}, + inode::{ + Ino, Inode, InodeDirOps, InodeFileOps, InodeInfo, InodeOps, InodeUse, RenameData, + }, + types::{DeviceId, Format, Mode, Permission}, + SbRef, + }, + }, + prelude::KResult, +}; + +use super::{ + file::{DeviceInode, FileInode, SymlinkInode}, + TmpFs, +}; + +pub struct DirectoryInode { + sb: SbRef<TmpFs>, + ino: Ino, + info: Spin<InodeInfo>, + entries: RwLock<Vec<(Arc<[u8]>, Ino)>>, +} + +impl InodeOps for DirectoryInode { + type SuperBlock = TmpFs; + + fn ino(&self) -> Ino { + self.ino + } + + fn format(&self) -> Format { + Format::DIR + } + + fn info(&self) -> &Spin<InodeInfo> { + &self.info + } + + fn super_block(&self) -> &SbRef<Self::SuperBlock> { + &self.sb + } + + fn page_cache(&self) -> Option<&PageCache> { + None + } +} + +impl DirectoryInode { + pub fn new(ino: Ino, sb: SbRef<TmpFs>, perm: Permission) -> InodeUse<Self> { + static DOT: LazyLock<Arc<[u8]>> = LazyLock::new(|| Arc::from(b".".as_slice())); + + let now = Instant::now(); + + InodeUse::new(Self { + sb, + ino, + info: Spin::new(InodeInfo { + size: 1, + nlink: 1, // link from `.` to itself + perm, + ctime: now, + mtime: now, + atime: now, + uid: 0, + gid: 0, + }), + entries: RwLock::new(vec![(DOT.clone(), ino)]), + }) + } + + fn link( + &self, + entries: &mut Vec<(Arc<[u8]>, Ino)>, + name: Arc<[u8]>, + file: &InodeUse<dyn Inode>, + ) { + let mut self_info = self.info.lock(); + let mut file_info = file.info().lock(); + + let now = Instant::now(); + + file_info.nlink += 1; + file_info.ctime = now; + + self_info.size += 1; + self_info.mtime = now; + self_info.ctime = now; + + entries.push((name, file.ino())); + } + + fn do_unlink( + &self, + file: &InodeUse<dyn Inode>, + filename: &[u8], + entries: &mut Vec<(Arc<[u8]>, Ino)>, + now: Instant, + decrease_size: bool, + self_info: &mut InodeInfo, + file_info: &mut InodeInfo, + ) -> KResult<()> { + // SAFETY: `file_lock` has done the synchronization + if file.format() == Format::DIR { + return Err(EISDIR); + } + + let file_ino = file.ino(); + entries.retain(|(name, ino)| *ino != file_ino || name.as_ref() != filename); + + if decrease_size { + self_info.size -= 1; + } + + self_info.mtime = now; + self_info.ctime = now; + + // The last reference to the inode is held by some dentry + // and will be released when the dentry is released + + file_info.nlink -= 1; + file_info.ctime = now; + + // TODO!!!: Remove the file if nlink == 1 + + Ok(()) + } +} + +impl InodeDirOps for DirectoryInode { + fn readdir<'r, 'a: 'r, 'b: 'r>( + &'a self, + offset: usize, + for_each_entry: &'b mut (dyn FnMut(&[u8], Ino) -> KResult<bool> + Send), + ) -> impl Future<Output = KResult<KResult<usize>>> + Send + 'r { + Box::pin(async move { + let _sb = self.sb.get()?; + let entries = self.entries.read().await; + + let mut count = 0; + for entry in entries.iter().skip(offset) { + match for_each_entry(&entry.0, entry.1) { + Err(err) => return Ok(Err(err)), + Ok(false) => break, + Ok(true) => count += 1, + } + } + + Ok(Ok(count)) + }) + } + + async fn create(&self, at: &Arc<Dentry>, perm: Permission) -> KResult<()> { + let sb = self.sb.get()?; + let mut entries = self.entries.write().await; + + let ino = sb.backend.assign_ino(); + let file: InodeUse<dyn Inode> = FileInode::new(ino, self.sb.clone(), 0, perm); + + self.link(&mut entries, at.get_name(), &file); + at.fill(file); + + Ok(()) + } + + async fn mknod(&self, at: &Dentry, mode: Mode, dev: DeviceId) -> KResult<()> { + if !mode.is_chr() && !mode.is_blk() { + return Err(EINVAL); + } + + let sb = self.sb.get()?; + let mut entries = self.entries.write().await; + + let ino = sb.backend.assign_ino(); + let file: InodeUse<dyn Inode> = DeviceInode::new(ino, self.sb.clone(), mode, dev); + + self.link(&mut entries, at.get_name(), &file); + at.fill(file); + + Ok(()) + } + + async fn symlink(&self, at: &Arc<Dentry>, target: &[u8]) -> KResult<()> { + let sb = self.sb.get()?; + let mut entries = self.entries.write().await; + + let ino = sb.backend.assign_ino(); + let file: InodeUse<dyn Inode> = SymlinkInode::new(ino, self.sb.clone(), target.into()); + + self.link(&mut entries, at.get_name(), &file); + at.fill(file); + + Ok(()) + } + + async fn mkdir(&self, at: &Dentry, perm: Permission) -> KResult<()> { + let sb = self.sb.get()?; + let mut entries = self.entries.write().await; + + let ino = sb.backend.assign_ino(); + let new_dir: InodeUse<dyn Inode> = DirectoryInode::new(ino, self.sb.clone(), perm); + + self.link(&mut entries, at.get_name(), &new_dir); + at.fill(new_dir); + + Ok(()) + } + + async fn unlink(&self, at: &Arc<Dentry>) -> KResult<()> { + let _sb = self.sb.get()?; + let mut entries = self.entries.write().await; + + let file = at.get_inode()?; + let filename = at.get_name(); + + self.do_unlink( + &file, + &filename, + &mut entries, + Instant::now(), + true, + &mut self.info.lock(), + &mut file.info().lock(), + )?; + + // Remove the dentry from the dentry cache immediately + // so later lookup will fail with ENOENT + dcache::d_remove(at); + + Ok(()) + } + + async fn rename(&self, rename_data: RenameData<'_, '_>) -> KResult<()> { + let sb = self.sb.get()?; + let _rename_lock = sb.backend.rename_lock.lock().await; + let mut self_entries = self.entries.write().await; + + let RenameData { + old_dentry, + new_dentry, + new_parent, + is_exchange, + no_replace, + } = rename_data; + + if is_exchange { + println_warn!("TmpFs does not support exchange rename for now"); + return Err(ENOSYS); + } + + let old_file = old_dentry.get_inode()?; + let new_file = new_dentry.inode(); + + if no_replace && new_file.is_some() { + return Err(EEXIST); + } + + if new_parent.as_raw() == &raw const *self { + // Same directory rename + // Remove from old location and add to new location + let old_ino = old_file.ino(); + let new_ino = new_file.as_ref().map(|f| f.ino()); + let old_name = old_dentry.get_name(); + let new_name = new_dentry.get_name(); + + // Find the old and new entries in the directory after we've locked the directory. + let (mut old_ent_idx, mut new_ent_idx) = (None, None); + for (idx, (name, ino)) in self_entries.iter().enumerate() { + if *ino == old_ino && *name == old_name { + old_ent_idx = Some(idx); + } + + if Some(*ino) == new_ino && *name == new_name { + new_ent_idx = Some(idx); + } + } + + let Some(old_ent_idx) = old_ent_idx else { + return Err(ENOENT); + }; + + if Some(old_ent_idx) == new_ent_idx { + return Ok(()); + } + + let now = Instant::now(); + if let Some(new_idx) = new_ent_idx { + // Replace existing file (i.e. rename the old and unlink the new) + let new_file = new_file.unwrap(); + + match (new_file.format(), old_file.format()) { + (Format::DIR, _) => return Err(EISDIR), + (_, Format::DIR) => return Err(ENOTDIR), + _ => {} + } + + self_entries.remove(new_idx); + + self.info.lock().size -= 1; + + // The last reference to the inode is held by some dentry + // and will be released when the dentry is released + + let mut new_info = new_file.info().lock(); + + new_info.nlink -= 1; + new_info.mtime = now; + new_info.ctime = now; + } + + let (name, _) = &mut self_entries[old_ent_idx]; + *name = new_dentry.get_name(); + + let mut self_info = self.info.lock(); + self_info.mtime = now; + self_info.ctime = now; + } else { + // Cross-directory rename - handle similar to same directory case + + // Get new parent directory + let new_parent_inode = new_dentry.parent().get_inode()?; + assert_eq!(new_parent_inode.format(), Format::DIR); + + let new_parent = (&new_parent_inode as &dyn Any) + .downcast_ref::<DirectoryInode>() + .expect("new parent must be a DirectoryInode"); + + let mut new_entries = new_parent.entries.write().await; + + let old_ino = old_file.ino(); + let new_ino = new_file.as_ref().map(|f| f.ino()); + let old_name = old_dentry.get_name(); + let new_name = new_dentry.get_name(); + + // Find the old entry in the old directory + let old_pos = self_entries + .iter() + .position(|(name, ino)| *ino == old_ino && *name == old_name) + .ok_or(ENOENT)?; + + // Find the new entry in the new directory (if it exists) + let has_new = new_entries + .iter() + .position(|(name, ino)| Some(*ino) == new_ino && *name == new_name) + .is_some(); + + let now = Instant::now(); + + if has_new { + // Replace existing file (i.e. move the old and unlink the new) + let new_file = new_file.unwrap(); + + match (old_file.format(), new_file.format()) { + (Format::DIR, Format::DIR) => {} + (Format::DIR, _) => return Err(ENOTDIR), + (_, _) => {} + } + + // Unlink the old file that was replaced + new_parent.do_unlink( + &new_file, + &new_name, + &mut new_entries, + now, + false, + &mut new_parent.info.lock(), + &mut new_file.info().lock(), + )?; + } else { + new_parent.info.lock().size += 1; + new_parent.info.lock().mtime = now; + new_parent.info.lock().ctime = now; + } + + // Remove from old directory + self_entries.remove(old_pos); + + // Add new entry + new_entries.push((new_name, old_ino)); + + let mut self_info = self.info.lock(); + self_info.size -= 1; + self_info.mtime = now; + self_info.ctime = now; + } + + dcache::d_exchange(old_dentry, new_dentry).await; + Ok(()) + } +} + +impl InodeFileOps for DirectoryInode { + async fn chmod(&self, perm: Permission) -> KResult<()> { + let _sb = self.sb.get()?; + + { + let mut info = self.info.lock(); + info.perm = perm; + info.ctime = Instant::now(); + } + + Ok(()) + } +} diff --git a/src/fs/tmpfs/file.rs b/src/fs/tmpfs/file.rs new file mode 100644 index 00000000..624112e0 --- /dev/null +++ b/src/fs/tmpfs/file.rs @@ -0,0 +1,298 @@ +use alloc::sync::Arc; +use eonix_mm::paging::PAGE_SIZE; +use eonix_sync::{RwLock, Spin}; + +use crate::{ + io::{Buffer, Stream}, + kernel::{ + mem::{CachePage, CachePageStream, PageCache, PageCacheBackendOps}, + timer::Instant, + vfs::{ + inode::{Ino, InodeDirOps, InodeFileOps, InodeInfo, InodeOps, InodeUse, WriteOffset}, + types::{DeviceId, Format, Mode, Permission}, + SbRef, + }, + }, + prelude::KResult, +}; + +use super::TmpFs; + +pub struct FileInode { + sb: SbRef<TmpFs>, + ino: Ino, + info: Spin<InodeInfo>, + rwsem: RwLock<()>, + pages: PageCache, +} + +impl FileInode { + pub fn new(ino: Ino, sb: SbRef<TmpFs>, size: usize, perm: Permission) -> InodeUse<Self> { + let now = Instant::now(); + + InodeUse::new_cyclic(|weak| Self { + sb, + ino, + info: Spin::new(InodeInfo { + size: size as _, + nlink: 1, + uid: 0, + gid: 0, + perm, + atime: now, + ctime: now, + mtime: now, + }), + rwsem: RwLock::new(()), + pages: PageCache::new(weak.clone() as _), + }) + } +} + +impl PageCacheBackendOps for FileInode { + async fn read_page(&self, _cache_page: &mut CachePage, _offset: usize) -> KResult<usize> { + Ok(PAGE_SIZE) + } + + async fn write_page(&self, _page: &mut CachePageStream, _offset: usize) -> KResult<usize> { + Ok(PAGE_SIZE) + } + + fn size(&self) -> usize { + self.info.lock().size as usize + } +} + +impl InodeOps for FileInode { + type SuperBlock = TmpFs; + + fn ino(&self) -> Ino { + self.ino + } + + fn format(&self) -> Format { + Format::REG + } + + fn info(&self) -> &Spin<InodeInfo> { + &self.info + } + + fn super_block(&self) -> &SbRef<Self::SuperBlock> { + &self.sb + } + + fn page_cache(&self) -> Option<&PageCache> { + Some(&self.pages) + } +} + +impl InodeDirOps for FileInode {} +impl InodeFileOps for FileInode { + async fn read(&self, buffer: &mut dyn Buffer, offset: usize) -> KResult<usize> { + let _lock = self.rwsem.read().await; + self.pages.read(buffer, offset).await + } + + async fn write(&self, stream: &mut dyn Stream, offset: WriteOffset<'_>) -> KResult<usize> { + let _lock = self.rwsem.write().await; + + let mut store_new_end = None; + let offset = match offset { + WriteOffset::Position(offset) => offset, + WriteOffset::End(end) => { + store_new_end = Some(end); + + // `info.size` won't change since we are holding the write lock. + self.info.lock().size as usize + } + }; + + let wrote = self.pages.write(stream, offset).await?; + let cursor_end = offset + wrote; + + if let Some(store_end) = store_new_end { + *store_end = cursor_end; + } + + { + let now = Instant::now(); + let mut info = self.info.lock(); + info.mtime = now; + info.ctime = now; + info.size = info.size.max(cursor_end as u64); + } + + Ok(wrote) + } + + async fn truncate(&self, length: usize) -> KResult<()> { + let _lock = self.rwsem.write().await; + + self.pages.resize(length).await?; + + { + let now = Instant::now(); + let mut info = self.info.lock(); + info.mtime = now; + info.ctime = now; + info.size = length as u64; + } + + Ok(()) + } + + async fn chmod(&self, perm: Permission) -> KResult<()> { + let _sb = self.sb.get()?; + + { + let mut info = self.info.lock(); + + info.perm = perm; + info.ctime = Instant::now(); + } + + Ok(()) + } +} + +pub struct DeviceInode { + sb: SbRef<TmpFs>, + ino: Ino, + info: Spin<InodeInfo>, + is_block: bool, + devid: DeviceId, +} + +impl DeviceInode { + pub fn new(ino: Ino, sb: SbRef<TmpFs>, mode: Mode, devid: DeviceId) -> InodeUse<Self> { + let now = Instant::now(); + + InodeUse::new(Self { + sb, + ino, + info: Spin::new(InodeInfo { + size: 0, + nlink: 1, + uid: 0, + gid: 0, + perm: Permission::new(mode.non_format_bits()), + atime: now, + ctime: now, + mtime: now, + }), + is_block: mode.format() == Format::BLK, + devid, + }) + } +} + +impl InodeOps for DeviceInode { + type SuperBlock = TmpFs; + + fn ino(&self) -> Ino { + self.ino + } + + fn format(&self) -> Format { + if self.is_block { + Format::BLK + } else { + Format::CHR + } + } + + fn info(&self) -> &Spin<InodeInfo> { + &self.info + } + + fn super_block(&self) -> &SbRef<Self::SuperBlock> { + &self.sb + } + + fn page_cache(&self) -> Option<&PageCache> { + None + } +} + +impl InodeDirOps for DeviceInode {} +impl InodeFileOps for DeviceInode { + async fn chmod(&self, perm: Permission) -> KResult<()> { + let _sb = self.sb.get()?; + + { + let mut info = self.info.lock(); + + info.perm = perm; + info.ctime = Instant::now(); + } + + Ok(()) + } + + fn devid(&self) -> KResult<DeviceId> { + Ok(self.devid) + } +} + +pub struct SymlinkInode { + sb: SbRef<TmpFs>, + ino: Ino, + info: Spin<InodeInfo>, + target: Arc<[u8]>, +} + +impl SymlinkInode { + pub fn new(ino: Ino, sb: SbRef<TmpFs>, target: Arc<[u8]>) -> InodeUse<Self> { + let now = Instant::now(); + + InodeUse::new(Self { + sb, + ino, + info: Spin::new(InodeInfo { + size: target.len() as _, + nlink: 1, + uid: 0, + gid: 0, + perm: Permission::new(0o777), + atime: now, + ctime: now, + mtime: now, + }), + target, + }) + } +} + +impl InodeDirOps for SymlinkInode {} +impl InodeOps for SymlinkInode { + type SuperBlock = TmpFs; + + fn ino(&self) -> Ino { + self.ino + } + + fn format(&self) -> Format { + Format::LNK + } + + fn info(&self) -> &Spin<InodeInfo> { + &self.info + } + + fn super_block(&self) -> &SbRef<Self::SuperBlock> { + &self.sb + } + + fn page_cache(&self) -> Option<&PageCache> { + None + } +} + +impl InodeFileOps for SymlinkInode { + async fn readlink(&self, buffer: &mut dyn Buffer) -> KResult<usize> { + buffer + .fill(self.target.as_ref()) + .map(|result| result.allow_partial()) + } +} diff --git a/src/fs/tmpfs/mod.rs b/src/fs/tmpfs/mod.rs new file mode 100644 index 00000000..2bef67b6 --- /dev/null +++ b/src/fs/tmpfs/mod.rs @@ -0,0 +1,73 @@ +mod dir; +mod file; + +use crate::kernel::vfs::inode::{Ino, InodeUse}; +use crate::kernel::vfs::types::{DeviceId, Permission}; +use crate::kernel::vfs::{SbRef, SbUse, SuperBlock, SuperBlockInfo}; +use crate::{ + kernel::vfs::{ + dentry::Dentry, + mount::{register_filesystem, Mount, MountCreator}, + }, + prelude::*, +}; +use alloc::sync::Arc; +use async_trait::async_trait; +use core::sync::atomic::AtomicU64; +use core::sync::atomic::Ordering; +use dir::DirectoryInode; +use eonix_sync::Mutex; + +pub struct TmpFs { + next_ino: AtomicU64, + rename_lock: Mutex<()>, +} + +impl SuperBlock for TmpFs {} + +impl TmpFs { + fn assign_ino(&self) -> Ino { + Ino::new(self.next_ino.fetch_add(1, Ordering::Relaxed)) + } + + fn create() -> KResult<(SbUse<TmpFs>, InodeUse<DirectoryInode>)> { + let tmpfs = SbUse::new( + SuperBlockInfo { + io_blksize: 4096, + device_id: DeviceId::new(0, 2), + read_only: false, + }, + Self { + next_ino: AtomicU64::new(1), + rename_lock: Mutex::new(()), + }, + ); + + let root_dir = DirectoryInode::new( + tmpfs.backend.assign_ino(), + SbRef::from(&tmpfs), + Permission::new(0o755), + ); + + Ok((tmpfs, root_dir)) + } +} + +struct TmpFsMountCreator; + +#[async_trait] +impl MountCreator for TmpFsMountCreator { + async fn create_mount(&self, _source: &str, _flags: u64, mp: &Arc<Dentry>) -> KResult<Mount> { + let (fs, root_inode) = TmpFs::create()?; + + Mount::new(mp, fs, root_inode) + } + + fn check_signature(&self, _: &[u8]) -> KResult<bool> { + Ok(true) + } +} + +pub fn init() { + register_filesystem("tmpfs", Arc::new(TmpFsMountCreator)).unwrap(); +} diff --git a/src/kernel/block.rs b/src/kernel/block.rs index 349e3656..3e4b65d1 100644 --- a/src/kernel/block.rs +++ b/src/kernel/block.rs @@ -3,7 +3,7 @@ mod mbr; use super::{ constants::ENOENT, mem::{paging::Page, AsMemoryBlock as _}, - vfs::DevId, + vfs::types::DeviceId, }; use crate::kernel::constants::{EEXIST, EINVAL}; use crate::{ @@ -14,13 +14,10 @@ use alloc::{ collections::btree_map::{BTreeMap, Entry}, sync::Arc, }; +use async_trait::async_trait; use core::cmp::Ordering; use mbr::MBRPartTable; -pub fn make_device(major: u32, minor: u32) -> DevId { - (major << 8) & 0xff00u32 | minor & 0xffu32 -} - pub struct Partition { pub lba_offset: u64, pub sector_count: u64, @@ -30,11 +27,12 @@ pub trait PartTable { fn partitions(&self) -> impl Iterator<Item = Partition> + use<'_, Self>; } +#[async_trait] pub trait BlockRequestQueue: Send + Sync { /// Maximum number of sectors that can be read in one request fn max_request_pages(&self) -> u64; - fn submit(&self, req: BlockDeviceRequest) -> KResult<()>; + async fn submit<'a>(&'a self, req: BlockDeviceRequest<'a>) -> KResult<()>; } enum BlockDeviceType { @@ -42,7 +40,7 @@ enum BlockDeviceType { queue: Arc<dyn BlockRequestQueue>, }, Partition { - disk_dev: DevId, + disk_dev: DeviceId, lba_offset: u64, queue: Arc<dyn BlockRequestQueue>, }, @@ -50,7 +48,7 @@ enum BlockDeviceType { pub struct BlockDevice { /// Unique device identifier, major and minor numbers - devid: DevId, + devid: DeviceId, /// Total size of the device in sectors (512 bytes each) sector_count: u64, @@ -77,11 +75,11 @@ impl Ord for BlockDevice { } } -static BLOCK_DEVICE_LIST: Spin<BTreeMap<DevId, Arc<BlockDevice>>> = Spin::new(BTreeMap::new()); +static BLOCK_DEVICE_LIST: Spin<BTreeMap<DeviceId, Arc<BlockDevice>>> = Spin::new(BTreeMap::new()); impl BlockDevice { pub fn register_disk( - devid: DevId, + devid: DeviceId, size: u64, queue: Arc<dyn BlockRequestQueue>, ) -> KResult<Arc<Self>> { @@ -97,13 +95,13 @@ impl BlockDevice { } } - pub fn get(devid: DevId) -> KResult<Arc<Self>> { + pub fn get(devid: DeviceId) -> KResult<Arc<Self>> { BLOCK_DEVICE_LIST.lock().get(&devid).cloned().ok_or(ENOENT) } } impl BlockDevice { - pub fn devid(&self) -> DevId { + pub fn devid(&self) -> DeviceId { self.devid } @@ -121,7 +119,7 @@ impl BlockDevice { }; let device = Arc::new(BlockDevice { - devid: make_device(self.devid >> 8, (self.devid & 0xff) + idx as u32 + 1), + devid: DeviceId::new(self.devid.major, self.devid.minor + idx as u16 + 1), sector_count: size, dev_type: BlockDeviceType::Partition { disk_dev: self.devid, @@ -159,7 +157,7 @@ impl BlockDevice { /// - `req.sector` must be within the disk size /// - `req.buffer` must be enough to hold the data /// - pub fn commit_request(&self, mut req: BlockDeviceRequest) -> KResult<()> { + pub async fn commit_request(&self, mut req: BlockDeviceRequest<'_>) -> KResult<()> { // Verify the request parameters. match &mut req { BlockDeviceRequest::Read { sector, count, .. } => { @@ -184,7 +182,7 @@ impl BlockDevice { } } - self.queue().submit(req) + self.queue().submit(req).await } /// Read some from the block device, may involve some copy and fragmentation @@ -194,7 +192,7 @@ impl BlockDevice { /// # Arguments /// `offset` - offset in bytes /// - pub fn read_some(&self, offset: usize, buffer: &mut dyn Buffer) -> KResult<FillResult> { + pub async fn read_some(&self, offset: usize, buffer: &mut dyn Buffer) -> KResult<FillResult> { let mut sector_start = offset as u64 / 512; let mut first_sector_offset = offset as u64 % 512; let mut sector_count = (first_sector_offset + buffer.total() as u64 + 511) / 512; @@ -241,7 +239,7 @@ impl BlockDevice { buffer: &pages, }; - self.commit_request(req)?; + self.commit_request(req).await?; for page in pages.iter() { // SAFETY: We are the only owner of the page so no one could be mutating it. @@ -277,7 +275,7 @@ impl BlockDevice { /// `offset` - offset in bytes /// `data` - data to write /// - pub fn write_some(&self, offset: usize, data: &[u8]) -> KResult<usize> { + pub async fn write_some(&self, offset: usize, data: &[u8]) -> KResult<usize> { let mut sector_start = offset as u64 / 512; let mut first_sector_offset = offset as u64 % 512; let mut remaining_data = data; @@ -320,7 +318,7 @@ impl BlockDevice { count: sector_count, buffer: pages, }; - self.commit_request(read_req)?; + self.commit_request(read_req).await?; } let mut data_offset = 0; @@ -356,7 +354,7 @@ impl BlockDevice { count: sector_count, buffer: pages, }; - self.commit_request(write_req)?; + self.commit_request(write_req).await?; let bytes_written = data_offset; nwritten += bytes_written; diff --git a/src/kernel/block/mbr.rs b/src/kernel/block/mbr.rs index 74cdc36e..c5820679 100644 --- a/src/kernel/block/mbr.rs +++ b/src/kernel/block/mbr.rs @@ -31,7 +31,7 @@ pub struct MBRPartTable { impl MBRPartTable { pub async fn from_disk(disk: &BlockDevice) -> KResult<Self> { let mut mbr: UninitBuffer<MBRData> = UninitBuffer::new(); - disk.read_some(0, &mut mbr)?.ok_or(EIO)?; + disk.read_some(0, &mut mbr).await?.ok_or(EIO)?; let mbr = mbr.assume_init()?; if mbr.magic != [0x55, 0xaa] { diff --git a/src/kernel/chardev.rs b/src/kernel/chardev.rs index aff3271e..4e01d83a 100644 --- a/src/kernel/chardev.rs +++ b/src/kernel/chardev.rs @@ -1,10 +1,9 @@ use super::{ - block::make_device, console::get_console, constants::{EEXIST, EIO}, task::{block_on, ProcessList, Thread}, terminal::Terminal, - vfs::{DevId, File, FileType, TerminalFile}, + vfs::{types::DeviceId, File, FileType, TerminalFile}, }; use crate::{ io::{Buffer, Stream, StreamRead}, @@ -34,7 +33,7 @@ pub struct CharDevice { device: CharDeviceType, } -static CHAR_DEVICES: Spin<BTreeMap<DevId, Arc<CharDevice>>> = Spin::new(BTreeMap::new()); +static CHAR_DEVICES: Spin<BTreeMap<DeviceId, Arc<CharDevice>>> = Spin::new(BTreeMap::new()); impl CharDevice { pub fn read(&self, buffer: &mut dyn Buffer) -> KResult<usize> { @@ -54,11 +53,11 @@ impl CharDevice { } } - pub fn get(devid: DevId) -> Option<Arc<CharDevice>> { + pub fn get(devid: DeviceId) -> Option<Arc<CharDevice>> { CHAR_DEVICES.lock().get(&devid).cloned() } - pub fn register(devid: DevId, name: Arc<str>, device: CharDeviceType) -> KResult<()> { + pub fn register(devid: DeviceId, name: Arc<str>, device: CharDeviceType) -> KResult<()> { match CHAR_DEVICES.lock().entry(devid) { Entry::Vacant(entry) => { entry.insert(Arc::new(CharDevice { name, device })); @@ -134,19 +133,19 @@ impl VirtualCharDevice for ConsoleDevice { impl CharDevice { pub fn init() -> KResult<()> { Self::register( - make_device(1, 3), + DeviceId::new(1, 3), Arc::from("null"), CharDeviceType::Virtual(Box::new(NullDevice)), )?; Self::register( - make_device(1, 5), + DeviceId::new(1, 5), Arc::from("zero"), CharDeviceType::Virtual(Box::new(ZeroDevice)), )?; Self::register( - make_device(5, 1), + DeviceId::new(5, 1), Arc::from("console"), CharDeviceType::Virtual(Box::new(ConsoleDevice)), )?; diff --git a/src/kernel/mem.rs b/src/kernel/mem.rs index efd06824..c147306e 100644 --- a/src/kernel/mem.rs +++ b/src/kernel/mem.rs @@ -12,5 +12,5 @@ pub use access::{AsMemoryBlock, MemoryBlock, PhysAccess}; pub(self) use mm_area::MMArea; pub use mm_list::{handle_kernel_page_fault, FileMapping, MMList, Mapping, Permission}; pub use page_alloc::{GlobalPageAlloc, RawPage}; -pub use page_cache::{CachePage, CachePageStream, PageCache, PageCacheBackend}; +pub use page_cache::{CachePage, CachePageStream, PageCache, PageCacheBackendOps}; pub use paging::{Page, PageBuffer}; diff --git a/src/kernel/mem/mm_list/mapping.rs b/src/kernel/mem/mm_list/mapping.rs index 662000ba..5446ae42 100644 --- a/src/kernel/mem/mm_list/mapping.rs +++ b/src/kernel/mem/mm_list/mapping.rs @@ -1,24 +1,15 @@ -use core::fmt::Debug; - -use crate::kernel::vfs::inode::Inode; -use alloc::sync::Arc; +use crate::kernel::vfs::inode::{Inode, InodeUse}; use eonix_mm::paging::PAGE_SIZE; #[derive(Debug, Clone)] pub struct FileMapping { - pub file: Arc<dyn Inode>, + pub file: InodeUse<dyn Inode>, /// Offset in the file, aligned to 4KB boundary. pub offset: usize, /// Length of the mapping. Exceeding part will be zeroed. pub length: usize, } -impl Debug for dyn Inode { - fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { - write!(f, "Inode()") - } -} - #[derive(Debug, Clone)] pub enum Mapping { // private anonymous memory @@ -28,7 +19,7 @@ pub enum Mapping { } impl FileMapping { - pub fn new(file: Arc<dyn Inode>, offset: usize, length: usize) -> Self { + pub fn new(file: InodeUse<dyn Inode>, offset: usize, length: usize) -> Self { assert_eq!(offset & (PAGE_SIZE - 1), 0); Self { file, diff --git a/src/kernel/mem/page_cache.rs b/src/kernel/mem/page_cache.rs index 3ccf3255..9deb50cf 100644 --- a/src/kernel/mem/page_cache.rs +++ b/src/kernel/mem/page_cache.rs @@ -6,8 +6,10 @@ use crate::{ GlobalPageAlloc, }; use align_ext::AlignExt; +use alloc::boxed::Box; use alloc::{collections::btree_map::BTreeMap, sync::Weak}; -use core::mem::ManuallyDrop; +use async_trait::async_trait; +use core::{future::Future, mem::ManuallyDrop}; use eonix_hal::mm::ArchPhysAccess; use eonix_mm::{ address::{PAddr, PhysAccess}, @@ -159,7 +161,8 @@ impl PageCache { self.backend .upgrade() .unwrap() - .read_page(&mut new_page, offset.align_down(PAGE_SIZE))?; + .read_page(&mut new_page, offset.align_down(PAGE_SIZE)) + .await?; pages.insert(page_id, new_page); } } @@ -205,7 +208,8 @@ impl PageCache { self.backend .upgrade() .unwrap() - .read_page(&mut new_page, offset.align_down(PAGE_SIZE))?; + .read_page(&mut new_page, offset.align_down(PAGE_SIZE)) + .await?; new_page }; @@ -224,7 +228,8 @@ impl PageCache { self.backend .upgrade() .unwrap() - .write_page(&mut CachePageStream::new(*page), page_id << PAGE_SIZE_BITS)?; + .write_page(&mut CachePageStream::new(*page), page_id << PAGE_SIZE_BITS) + .await?; page.clear_dirty(); } } @@ -286,7 +291,8 @@ impl PageCache { self.backend .upgrade() .unwrap() - .read_page(&mut new_page, offset_aligin)?; + .read_page(&mut new_page, offset_aligin) + .await?; pages.insert(page_id, new_page); new_page.0 } @@ -349,14 +355,47 @@ impl Stream for CachePageStream { // for fs, offset is file offset (floor algin to PAGE_SIZE) // for blkdev, offset is block idx (floor align to PAGE_SIZE / BLK_SIZE) // Oh no, this would make unnecessary cache -pub trait PageCacheBackend { - fn read_page(&self, page: &mut CachePage, offset: usize) -> KResult<usize>; +pub trait PageCacheBackendOps: Sized { + fn read_page( + &self, + page: &mut CachePage, + offset: usize, + ) -> impl Future<Output = KResult<usize>> + Send; + + fn write_page( + &self, + page: &mut CachePageStream, + offset: usize, + ) -> impl Future<Output = KResult<usize>> + Send; - fn write_page(&self, page: &mut CachePageStream, offset: usize) -> KResult<usize>; + fn size(&self) -> usize; +} +#[async_trait] +pub trait PageCacheBackend: Send + Sync { + async fn read_page(&self, page: &mut CachePage, offset: usize) -> KResult<usize>; + async fn write_page(&self, page: &mut CachePageStream, offset: usize) -> KResult<usize>; fn size(&self) -> usize; } +#[async_trait] +impl<T> PageCacheBackend for T +where + T: PageCacheBackendOps + Send + Sync + 'static, +{ + async fn read_page(&self, page: &mut CachePage, offset: usize) -> KResult<usize> { + self.read_page(page, offset).await + } + + async fn write_page(&self, page: &mut CachePageStream, offset: usize) -> KResult<usize> { + self.write_page(page, offset).await + } + + fn size(&self) -> usize { + self.size() + } +} + pub trait PageCacheRawPage: RawPage { fn valid_size(&self) -> &mut usize; diff --git a/src/kernel/pcie/driver.rs b/src/kernel/pcie/driver.rs index be88b7df..eebaa896 100644 --- a/src/kernel/pcie/driver.rs +++ b/src/kernel/pcie/driver.rs @@ -4,21 +4,24 @@ use super::{ }; use crate::{kernel::constants::EEXIST, KResult}; use alloc::{ + boxed::Box, collections::btree_map::{self, BTreeMap}, sync::Arc, }; +use async_trait::async_trait; use eonix_sync::Spin; static PCIE_DRIVERS: Spin<BTreeMap<u32, Arc<dyn PCIDriver>>> = Spin::new(BTreeMap::new()); +#[async_trait] pub trait PCIDriver: Send + Sync { fn vendor_id(&self) -> u16; fn device_id(&self) -> u16; - fn handle_device(&self, device: Arc<PCIDevice<'static>>) -> Result<(), PciError>; + async fn handle_device(&self, device: Arc<PCIDevice<'static>>) -> Result<(), PciError>; } -pub fn register_driver(driver: impl PCIDriver + 'static) -> KResult<()> { +pub async fn register_driver(driver: impl PCIDriver + 'static) -> KResult<()> { let index = (driver.vendor_id() as u32) << 16 | driver.device_id() as u32; let driver = Arc::new(driver); @@ -31,7 +34,7 @@ pub fn register_driver(driver: impl PCIDriver + 'static) -> KResult<()> { let devices = PCIE_DEVICES.lock().get(&index).cloned(); if let Some(devices) = devices { for device in devices { - driver.handle_device(device)?; + driver.handle_device(device).await?; } }; diff --git a/src/kernel/syscall/file_rw.rs b/src/kernel/syscall/file_rw.rs index 1a48b255..db32b0e5 100644 --- a/src/kernel/syscall/file_rw.rs +++ b/src/kernel/syscall/file_rw.rs @@ -7,7 +7,7 @@ use crate::kernel::syscall::UserMut; use crate::kernel::task::Thread; use crate::kernel::timer::sleep; use crate::kernel::vfs::filearray::FD; -use crate::kernel::vfs::inode::Mode; +use crate::kernel::vfs::types::{DeviceId, Mode}; use crate::kernel::vfs::{PollEvent, SeekOption}; use crate::{ io::{Buffer, BufferFill}, @@ -41,7 +41,7 @@ impl FromSyscallArg for AtFlags { } } -fn dentry_from( +async fn dentry_from( thread: &Thread, dirfd: FD, pathname: User<u8>, @@ -52,7 +52,7 @@ fn dentry_from( match (path.as_cstr().to_bytes_with_nul()[0], dirfd) { (b'/', _) | (_, FD::AT_FDCWD) => { let path = Path::new(path.as_cstr().to_bytes())?; - Dentry::open(&thread.fs_context, path, follow_symlink) + Dentry::open(&thread.fs_context, path, follow_symlink).await } (0, dirfd) => { let dir_file = thread.files.get(dirfd).ok_or(EBADF)?; @@ -63,7 +63,7 @@ fn dentry_from( let dir_file = thread.files.get(dirfd).ok_or(EBADF)?; let dir_dentry = dir_file.as_path().ok_or(ENOTDIR)?; - Dentry::open_at(&thread.fs_context, dir_dentry, path, follow_symlink) + Dentry::open_at(&thread.fs_context, dir_dentry, path, follow_symlink).await } } } @@ -119,13 +119,11 @@ async fn pwrite64(fd: FD, buffer: User<u8>, count: usize, offset: usize) -> KRes } #[eonix_macros::define_syscall(SYS_OPENAT)] -async fn openat(dirfd: FD, pathname: User<u8>, flags: OpenFlags, mut mode: Mode) -> KResult<FD> { - let dentry = dentry_from(thread, dirfd, pathname, flags.follow_symlink())?; +async fn openat(dirfd: FD, pathname: User<u8>, flags: OpenFlags, mode: Mode) -> KResult<FD> { + let dentry = dentry_from(thread, dirfd, pathname, flags.follow_symlink()).await?; + let perm = mode.perm().mask_with(*thread.fs_context.umask.lock()); - let umask = *thread.fs_context.umask.lock(); - mode.mask_perm(!umask.non_format_bits()); - - thread.files.open(&dentry, flags, mode) + thread.files.open(&dentry, flags, perm).await } #[cfg(target_arch = "x86_64")] @@ -206,7 +204,7 @@ async fn newfstatat( let file = thread.files.get(dirfd).ok_or(EBADF)?; file.as_path().ok_or(EBADF)?.clone() } else { - dentry_from(thread, dirfd, pathname, !flags.no_follow())? + dentry_from(thread, dirfd, pathname, !flags.no_follow()).await? }; let statbuf = UserPointerMut::new(statbuf)?; @@ -247,7 +245,7 @@ async fn statx( let file = thread.files.get(dirfd).ok_or(EBADF)?; file.as_path().ok_or(EBADF)?.clone() } else { - dentry_from(thread, dirfd, pathname, !flags.no_follow())? + dentry_from(thread, dirfd, pathname, !flags.no_follow()).await? }; dentry.statx(&mut statx, mask)?; @@ -257,12 +255,11 @@ async fn statx( } #[eonix_macros::define_syscall(SYS_MKDIRAT)] -async fn mkdirat(dirfd: FD, pathname: User<u8>, mut mode: Mode) -> KResult<()> { - let umask = *thread.fs_context.umask.lock(); - mode.mask_perm(!umask.non_format_bits()); +async fn mkdirat(dirfd: FD, pathname: User<u8>, mode: Mode) -> KResult<()> { + let dentry = dentry_from(thread, dirfd, pathname, true).await?; + let perm = mode.perm().mask_with(*thread.fs_context.umask.lock()); - let dentry = dentry_from(thread, dirfd, pathname, true)?; - dentry.mkdir(mode) + dentry.mkdir(perm).await } #[cfg(target_arch = "x86_64")] @@ -274,7 +271,7 @@ async fn mkdir(pathname: User<u8>, mode: u32) -> KResult<()> { #[eonix_macros::define_syscall(SYS_FTRUNCATE64)] async fn truncate64(fd: FD, length: usize) -> KResult<()> { let file = thread.files.get(fd).ok_or(EBADF)?; - file.as_path().ok_or(EBADF)?.truncate(length) + file.as_path().ok_or(EBADF)?.truncate(length).await } #[cfg(target_arch = "x86_64")] @@ -290,7 +287,10 @@ async fn truncate(pathname: User<u8>, length: usize) -> KResult<()> { #[eonix_macros::define_syscall(SYS_UNLINKAT)] async fn unlinkat(dirfd: FD, pathname: User<u8>) -> KResult<()> { - dentry_from(thread, dirfd, pathname, false)?.unlink() + dentry_from(thread, dirfd, pathname, false) + .await? + .unlink() + .await } #[cfg(target_arch = "x86_64")] @@ -302,9 +302,9 @@ async fn unlink(pathname: User<u8>) -> KResult<()> { #[eonix_macros::define_syscall(SYS_SYMLINKAT)] async fn symlinkat(target: User<u8>, dirfd: FD, linkpath: User<u8>) -> KResult<()> { let target = UserString::new(target)?; - let dentry = dentry_from(thread, dirfd, linkpath, false)?; + let dentry = dentry_from(thread, dirfd, linkpath, false).await?; - dentry.symlink(target.as_cstr().to_bytes()) + dentry.symlink(target.as_cstr().to_bytes()).await } #[cfg(target_arch = "x86_64")] @@ -313,18 +313,36 @@ async fn symlink(target: User<u8>, linkpath: User<u8>) -> KResult<()> { sys_symlinkat(thread, target, FD::AT_FDCWD, linkpath) } +#[derive(Clone, Copy, Debug)] +#[repr(transparent)] +struct UserDeviceId(u32); + +impl FromSyscallArg for UserDeviceId { + fn from_arg(value: usize) -> Self { + Self(value as u32) + } +} + +impl UserDeviceId { + pub fn into_devid(self) -> DeviceId { + let major = (self.0 >> 8) & 0xfff; + let minor = (self.0 & 0xff) | ((self.0 >> 12) & 0xfff00); + + // TODO: We strip off the high 4 bits of the minor ID for now... + DeviceId::new(major as u16, minor as u16) + } +} + #[eonix_macros::define_syscall(SYS_MKNODAT)] -async fn mknodat(dirfd: FD, pathname: User<u8>, mut mode: Mode, dev: u32) -> KResult<()> { +async fn mknodat(dirfd: FD, pathname: User<u8>, mut mode: Mode, dev: UserDeviceId) -> KResult<()> { if !mode.is_blk() && !mode.is_chr() { return Err(EINVAL); } - let dentry = dentry_from(thread, dirfd, pathname, true)?; - - let umask = *thread.fs_context.umask.lock(); - mode.mask_perm(!umask.non_format_bits()); + let dentry = dentry_from(thread, dirfd, pathname, true).await?; + mode.set_perm(mode.perm().mask_with(*thread.fs_context.umask.lock())); - dentry.mknod(mode, dev) + dentry.mknod(mode, dev.into_devid()).await } #[cfg(target_arch = "x86_64")] @@ -340,10 +358,10 @@ async fn readlinkat( buffer: UserMut<u8>, bufsize: usize, ) -> KResult<usize> { - let dentry = dentry_from(thread, dirfd, pathname, false)?; + let dentry = dentry_from(thread, dirfd, pathname, false).await?; let mut buffer = UserBuffer::new(buffer, bufsize)?; - dentry.readlink(&mut buffer) + dentry.readlink(&mut buffer).await } #[cfg(target_arch = "x86_64")] @@ -471,7 +489,7 @@ async fn faccessat(dirfd: FD, pathname: User<u8>, _mode: u32, flags: AtFlags) -> let file = thread.files.get(dirfd).ok_or(EBADF)?; file.as_path().ok_or(EBADF)?.clone() } else { - dentry_from(thread, dirfd, pathname, !flags.no_follow())? + dentry_from(thread, dirfd, pathname, !flags.no_follow()).await? }; if !dentry.is_valid() { @@ -614,12 +632,12 @@ async fn fchownat( gid: u32, flags: AtFlags, ) -> KResult<()> { - let dentry = dentry_from(thread, dirfd, pathname, !flags.no_follow())?; + let dentry = dentry_from(thread, dirfd, pathname, !flags.no_follow()).await?; if !dentry.is_valid() { return Err(ENOENT); } - dentry.chown(uid, gid) + dentry.chown(uid, gid).await } #[eonix_macros::define_syscall(SYS_FCHMODAT)] @@ -628,14 +646,14 @@ async fn fchmodat(dirfd: FD, pathname: User<u8>, mode: Mode, flags: AtFlags) -> let file = thread.files.get(dirfd).ok_or(EBADF)?; file.as_path().ok_or(EBADF)?.clone() } else { - dentry_from(thread, dirfd, pathname, !flags.no_follow())? + dentry_from(thread, dirfd, pathname, !flags.no_follow()).await? }; if !dentry.is_valid() { return Err(ENOENT); } - dentry.chmod(mode) + dentry.chmod(mode).await } #[eonix_macros::define_syscall(SYS_FCHMOD)] @@ -654,7 +672,7 @@ async fn utimensat( let file = thread.files.get(dirfd).ok_or(EBADF)?; file.as_path().ok_or(EBADF)?.clone() } else { - dentry_from(thread, dirfd, pathname, !flags.no_follow())? + dentry_from(thread, dirfd, pathname, !flags.no_follow()).await? }; if !dentry.is_valid() { @@ -688,10 +706,10 @@ async fn renameat2( Err(EINVAL)?; } - let old_dentry = dentry_from(thread, old_dirfd, old_pathname, false)?; - let new_dentry = dentry_from(thread, new_dirfd, new_pathname, false)?; + let old_dentry = dentry_from(thread, old_dirfd, old_pathname, false).await?; + let new_dentry = dentry_from(thread, new_dirfd, new_pathname, false).await?; - old_dentry.rename(&new_dentry, flags) + old_dentry.rename(&new_dentry, flags).await } #[cfg(target_arch = "x86_64")] diff --git a/src/kernel/syscall/mm.rs b/src/kernel/syscall/mm.rs index c6300ac7..4cb7908c 100644 --- a/src/kernel/syscall/mm.rs +++ b/src/kernel/syscall/mm.rs @@ -1,10 +1,8 @@ use super::FromSyscallArg; -use crate::fs::shm::{gen_shm_id, ShmFlags, IPC_PRIVATE, SHM_MANAGER}; -use crate::kernel::constants::{EBADF, EEXIST, EINVAL, ENOENT}; +use crate::kernel::constants::{EBADF, EINVAL}; use crate::kernel::mem::FileMapping; use crate::kernel::task::Thread; use crate::kernel::vfs::filearray::FD; -use crate::kernel::vfs::inode::Mode; use crate::{ kernel::{ constants::{UserMmapFlags, UserMmapProtocol}, @@ -66,13 +64,7 @@ async fn do_mmap2( if !is_shared { Mapping::Anonymous } else { - // The mode is unimportant here, since we are checking prot in mm_area. - let shared_area = SHM_MANAGER.lock().await.create_shared_area( - len, - thread.process.pid, - Mode::REG.perm(0o777), - ); - Mapping::File(FileMapping::new(shared_area.area.clone(), 0, len)) + unimplemented!("mmap MAP_ANONYMOUS | MAP_SHARED"); } } else { let file = thread @@ -179,114 +171,4 @@ async fn mprotect(addr: usize, len: usize, prot: UserMmapProtocol) -> KResult<() .await } -#[eonix_macros::define_syscall(SYS_SHMGET)] -async fn shmget(key: usize, size: usize, shmflg: u32) -> KResult<u32> { - let size = size.align_up(PAGE_SIZE); - - let mut shm_manager = SHM_MANAGER.lock().await; - let shmid = gen_shm_id(key)?; - - let mode = Mode::REG.perm(shmflg); - let shmflg = ShmFlags::from_bits_truncate(shmflg); - - if key == IPC_PRIVATE { - let new_shm = shm_manager.create_shared_area(size, thread.process.pid, mode); - shm_manager.insert(shmid, new_shm); - return Ok(shmid); - } - - if let Some(_) = shm_manager.get(shmid) { - if shmflg.contains(ShmFlags::IPC_CREAT | ShmFlags::IPC_EXCL) { - return Err(EEXIST); - } - - return Ok(shmid); - } - - if shmflg.contains(ShmFlags::IPC_CREAT) { - let new_shm = shm_manager.create_shared_area(size, thread.process.pid, mode); - shm_manager.insert(shmid, new_shm); - return Ok(shmid); - } - - Err(ENOENT) -} - -#[eonix_macros::define_syscall(SYS_SHMAT)] -async fn shmat(shmid: u32, addr: usize, shmflg: u32) -> KResult<usize> { - let mm_list = &thread.process.mm_list; - let shm_manager = SHM_MANAGER.lock().await; - let shm_area = shm_manager.get(shmid).ok_or(EINVAL)?; - - // Why is this not used? - let _mode = shmflg & 0o777; - let shmflg = ShmFlags::from_bits_truncate(shmflg); - - let mut permission = Permission { - read: true, - write: true, - execute: false, - }; - - if shmflg.contains(ShmFlags::SHM_EXEC) { - permission.execute = true; - } - if shmflg.contains(ShmFlags::SHM_RDONLY) { - permission.write = false; - } - - let size = shm_area.shmid_ds.shm_segsz; - - let mapping = Mapping::File(FileMapping { - file: shm_area.area.clone(), - offset: 0, - length: size, - }); - - let addr = if addr != 0 { - if addr % PAGE_SIZE != 0 && !shmflg.contains(ShmFlags::SHM_RND) { - return Err(EINVAL); - } - let addr = VAddr::from(addr.align_down(PAGE_SIZE)); - mm_list - .mmap_fixed(addr, size, mapping, permission, true) - .await - } else { - mm_list - .mmap_hint(VAddr::NULL, size, mapping, permission, true) - .await - }?; - - thread.process.shm_areas.lock().insert(addr, size); - - Ok(addr.addr()) -} - -#[eonix_macros::define_syscall(SYS_SHMDT)] -async fn shmdt(addr: usize) -> KResult<()> { - let addr = VAddr::from(addr); - - let size = { - let mut shm_areas = thread.process.shm_areas.lock(); - let size = *shm_areas.get(&addr).ok_or(EINVAL)?; - shm_areas.remove(&addr); - - size - }; - - thread.process.mm_list.unmap(addr, size).await -} - -#[eonix_macros::define_syscall(SYS_SHMCTL)] -async fn shmctl(_shmid: u32, _op: i32, _shmid_ds: usize) -> KResult<usize> { - // TODO - Ok(0) -} - -#[eonix_macros::define_syscall(SYS_MEMBARRIER)] -async fn membarrier(_cmd: usize, _flags: usize) -> KResult<()> { - // TODO - Ok(()) -} - pub fn keep_alive() {} diff --git a/src/kernel/syscall/procops.rs b/src/kernel/syscall/procops.rs index 7dd573cc..b4d3e449 100644 --- a/src/kernel/syscall/procops.rs +++ b/src/kernel/syscall/procops.rs @@ -16,7 +16,7 @@ use crate::kernel::task::{parse_futexop, CloneArgs}; use crate::kernel::timer::sleep; use crate::kernel::user::UserString; use crate::kernel::user::{UserPointer, UserPointerMut}; -use crate::kernel::vfs::inode::Mode; +use crate::kernel::vfs::types::Permission; use crate::kernel::vfs::{self, dentry::Dentry}; use crate::path::Path; use crate::{kernel::user::UserBuffer, prelude::*}; @@ -100,10 +100,11 @@ async fn clock_nanosleep( } #[eonix_macros::define_syscall(SYS_UMASK)] -async fn umask(mask: Mode) -> KResult<Mode> { - let mut umask = thread.fs_context.umask.lock(); +async fn umask(raw_new_mask: u32) -> KResult<u32> { + let new_mask = Permission::new(!raw_new_mask); + let old_mask = core::mem::replace(&mut *thread.fs_context.umask.lock(), new_mask); - Ok(core::mem::replace(&mut *umask, mask.non_format())) + Ok(!old_mask.bits()) } #[eonix_macros::define_syscall(SYS_GETCWD)] @@ -124,7 +125,7 @@ async fn chdir(path: User<u8>) -> KResult<()> { let path = UserString::new(path)?; let path = Path::new(path.as_cstr().to_bytes())?; - let dentry = Dentry::open(&thread.fs_context, path, true)?; + let dentry = Dentry::open(&thread.fs_context, path, true).await?; if !dentry.is_valid() { return Err(ENOENT); } @@ -159,7 +160,8 @@ async fn mount(source: User<u8>, target: User<u8>, fstype: User<u8>, flags: usiz &thread.fs_context, Path::new(target.as_cstr().to_bytes())?, true, - )?; + ) + .await?; if !mountpoint.is_valid() { return Err(ENOENT); @@ -172,6 +174,7 @@ async fn mount(source: User<u8>, target: User<u8>, fstype: User<u8>, flags: usiz fstype.as_cstr().to_str().map_err(|_| EINVAL)?, flags as u64, ) + .await } fn get_strings(mut ptr_strings: UserPointer<'_, PtrT>) -> KResult<Vec<CString>> { @@ -199,14 +202,15 @@ async fn execve(exec: User<u8>, argv: User<PtrT>, envp: User<PtrT>) -> KResult<S let argv = get_strings(UserPointer::new(argv)?)?; let envp = get_strings(UserPointer::new(envp)?)?; - let dentry = Dentry::open(&thread.fs_context, Path::new(exec.as_bytes())?, true)?; + let dentry = Dentry::open(&thread.fs_context, Path::new(exec.as_bytes())?, true).await?; if !dentry.is_valid() { Err(ENOENT)?; } // TODO: When `execve` is called by one of the threads in a process, the other threads // should be terminated and `execve` is performed in the thread group leader. - let load_info = ProgramLoader::parse(&thread.fs_context, exec, dentry.clone(), argv, envp)? + let load_info = ProgramLoader::parse(&thread.fs_context, exec, dentry.clone(), argv, envp) + .await? .load() .await?; diff --git a/src/kernel/task/loader/elf.rs b/src/kernel/task/loader/elf.rs index 859e0010..36d139a8 100644 --- a/src/kernel/task/loader/elf.rs +++ b/src/kernel/task/loader/elf.rs @@ -193,9 +193,9 @@ impl<E: ElfArch> Elf<E> { Err(ENOEXEC) } - fn parse(elf_file: Arc<Dentry>) -> KResult<Self> { + async fn parse(elf_file: Arc<Dentry>) -> KResult<Self> { let mut elf_header = UninitBuffer::<ElfHeader<E::Ea>>::new(); - elf_file.read(&mut elf_header, 0)?; + elf_file.read(&mut elf_header, 0).await?; let elf_header = elf_header.assume_init().map_err(|_| ENOEXEC)?; @@ -203,10 +203,12 @@ impl<E: ElfArch> Elf<E> { let ph_count = elf_header.pt2.ph_count; let mut program_headers = vec![E::Ph::default(); ph_count as usize]; - elf_file.read( - &mut ByteBuffer::from(program_headers.as_mut_slice()), - ph_offset.into_usize(), - )?; + elf_file + .read( + &mut ByteBuffer::from(program_headers.as_mut_slice()), + ph_offset.into_usize(), + ) + .await?; Ok(Self { file: elf_file, @@ -390,12 +392,13 @@ impl<E: ElfArch> Elf<E> { } async fn load_ldso(&self, mm_list: &MMList) -> KResult<Option<LdsoLoadInfo>> { - let ldso_path = self.ldso_path()?; + let ldso_path = self.ldso_path().await?; if let Some(ldso_path) = ldso_path { let fs_context = FsContext::global(); - let ldso_file = Dentry::open(fs_context, Path::new(ldso_path.as_bytes())?, true)?; - let ldso_elf = Elf::<E>::parse(ldso_file)?; + let ldso_file = + Dentry::open(fs_context, Path::new(ldso_path.as_bytes())?, true).await?; + let ldso_elf = Elf::<E>::parse(ldso_file).await?; let base = VAddr::from(E::LDSO_BASE_ADDR); @@ -420,7 +423,7 @@ impl<E: ElfArch> Elf<E> { mm_list.map_vdso().await } - fn ldso_path(&self) -> KResult<Option<String>> { + async fn ldso_path(&self) -> KResult<Option<String>> { for program_header in &self.program_headers { let type_ = program_header.type_().map_err(|_| ENOEXEC)?; @@ -430,7 +433,8 @@ impl<E: ElfArch> Elf<E> { let mut ldso_vec = vec![0u8; file_size - 1]; // -1 due to '\0' self.file - .read(&mut ByteBuffer::from(ldso_vec.as_mut_slice()), file_offset)?; + .read(&mut ByteBuffer::from(ldso_vec.as_mut_slice()), file_offset) + .await?; let ldso_path = String::from_utf8(ldso_vec).map_err(|_| ENOEXEC)?; return Ok(Some(ldso_path)); } @@ -445,16 +449,16 @@ pub enum ELF { } impl ELF { - pub fn parse(elf_file: Arc<Dentry>) -> KResult<Self> { + pub async fn parse(elf_file: Arc<Dentry>) -> KResult<Self> { let mut header_pt1 = UninitBuffer::<HeaderPt1>::new(); - elf_file.read(&mut header_pt1, 0)?; + elf_file.read(&mut header_pt1, 0).await?; let header_pt1 = header_pt1.assume_init().map_err(|_| ENOEXEC)?; assert_eq!(header_pt1.magic, ELF_MAGIC); match header_pt1.class() { - Class::ThirtyTwo => Ok(ELF::Elf32(Elf::parse(elf_file)?)), - Class::SixtyFour => Ok(ELF::Elf64(Elf::parse(elf_file)?)), + Class::ThirtyTwo => Ok(ELF::Elf32(Elf::parse(elf_file).await?)), + Class::SixtyFour => Ok(ELF::Elf64(Elf::parse(elf_file).await?)), _ => Err(ENOEXEC), } } diff --git a/src/kernel/task/loader/mod.rs b/src/kernel/task/loader/mod.rs index 4e3f4db1..7679aaf4 100644 --- a/src/kernel/task/loader/mod.rs +++ b/src/kernel/task/loader/mod.rs @@ -33,7 +33,7 @@ pub struct ProgramLoader { } impl ProgramLoader { - pub fn parse( + pub async fn parse( fs_context: &FsContext, mut exec_path: CString, mut file: Arc<Dentry>, @@ -49,12 +49,15 @@ impl ProgramLoader { } let mut magic = [0; 4]; - file.read(&mut ByteBuffer::new(magic.as_mut_slice()), 0)?; + file.read(&mut ByteBuffer::new(magic.as_mut_slice()), 0) + .await?; match magic { [b'#', b'!', ..] => { let mut interpreter_line = [0; 256]; - let nread = file.read(&mut ByteBuffer::new(&mut interpreter_line), 0)?; + let nread = file + .read(&mut ByteBuffer::new(&mut interpreter_line), 0) + .await?; // There is a tiny time gap between reading the magic number and // reading the interpreter line, so we need to check if the line @@ -77,7 +80,7 @@ impl ProgramLoader { } let path = Path::new(interpreter_name.as_bytes())?; - file = Dentry::open(fs_context, path, true)?; + file = Dentry::open(fs_context, path, true).await?; args.insert(0, interpreter_name.clone()); if let Some(arg) = interpreter_arg { @@ -92,7 +95,7 @@ impl ProgramLoader { exec_path = interpreter_name; } - ELF_MAGIC => break ELF::parse(file)?, + ELF_MAGIC => break ELF::parse(file).await?, _ => return Err(ENOEXEC), } diff --git a/src/kernel/timer.rs b/src/kernel/timer.rs index 9b6a3ff2..1dbb1382 100644 --- a/src/kernel/timer.rs +++ b/src/kernel/timer.rs @@ -76,6 +76,8 @@ impl Ticks { } impl Instant { + pub const UNIX_EPOCH: Self = Self::default(); + pub const fn default() -> Self { Instant { secs_since_epoch: 0, diff --git a/src/kernel/vfs/dentry.rs b/src/kernel/vfs/dentry.rs index 8bcd9f8a..5ac4e407 100644 --- a/src/kernel/vfs/dentry.rs +++ b/src/kernel/vfs/dentry.rs @@ -1,8 +1,9 @@ pub mod dcache; use super::{ - inode::{Ino, Inode, Mode, RenameData, WriteOffset}, - DevId, FsContext, + inode::{Ino, Inode, InodeUse, RenameData, WriteOffset}, + types::{DeviceId, Format, Mode, Permission}, + FsContext, }; use crate::{ hash::KernelHasher, @@ -14,22 +15,31 @@ use crate::{ }; use crate::{ io::Stream, - kernel::constants::{EEXIST, EINVAL, EIO, EISDIR, ELOOP, ENOENT, ENOTDIR, EPERM, ERANGE}, + kernel::constants::{EEXIST, EINVAL, EISDIR, ELOOP, ENOENT, ENOTDIR, EPERM, ERANGE}, }; -use alloc::sync::{Arc, Weak}; +use alloc::sync::Arc; use core::{ fmt, + future::Future, hash::{BuildHasher, BuildHasherDefault, Hasher}, - ops::ControlFlow, + pin::Pin, sync::atomic::{AtomicPtr, AtomicU64, Ordering}, }; use eonix_sync::LazyLock; use pointers::BorrowedArc; use posix_types::{namei::RenameFlags, open::OpenFlags, result::PosixError, stat::StatX}; +#[derive(PartialEq, Eq)] +enum DentryKind { + Regular, + Directory, + Symlink, + Mountpoint, +} + struct DentryData { - inode: Arc<dyn Inode>, - flags: u64, + inode: InodeUse<dyn Inode>, + kind: DentryKind, } /// # Safety @@ -79,12 +89,6 @@ impl fmt::Debug for Dentry { } } -const D_DIRECTORY: u64 = 1; -#[allow(dead_code)] -const D_MOUNTPOINT: u64 = 2; -const D_SYMLINK: u64 = 4; -const D_REGULAR: u64 = 8; - impl RCUNode<Dentry> for Dentry { fn rcu_prev(&self) -> &AtomicPtr<Self> { &self.prev @@ -116,11 +120,11 @@ impl Dentry { self.hash.store(hash, Ordering::Relaxed); } - fn find(self: &Arc<Self>, name: &[u8]) -> KResult<Arc<Self>> { + async fn find(self: &Arc<Self>, name: &[u8]) -> KResult<Arc<Self>> { let data = self.data.load(); let data = data.as_ref().ok_or(ENOENT)?; - if data.flags & D_DIRECTORY == 0 { + if data.kind != DentryKind::Directory { return Err(ENOTDIR); } @@ -141,7 +145,7 @@ impl Dentry { return Ok(found); } - dcache::d_try_revalidate(&dentry); + let _ = dcache::d_try_revalidate(&dentry).await; dcache::d_add(dentry.clone()); Ok(dentry) @@ -192,8 +196,8 @@ impl Dentry { .map_or(core::ptr::null(), |parent| Arc::as_ptr(&parent)) } - fn save_data(&self, inode: Arc<dyn Inode>, flags: u64) -> KResult<()> { - let new = DentryData { inode, flags }; + fn save(&self, inode: InodeUse<dyn Inode>, kind: DentryKind) { + let new = DentryData { inode, kind }; // TODO!!!: We don't actually need to use `RCUPointer` here // Safety: this function may only be called from `create`-like functions which requires the @@ -201,41 +205,35 @@ impl Dentry { // can't get a reference to the old data. let old = unsafe { self.data.swap(Some(Arc::new(new))) }; assert!(old.is_none()); - - Ok(()) } - pub fn save_reg(&self, file: Arc<dyn Inode>) -> KResult<()> { - self.save_data(file, D_REGULAR) - } - - pub fn save_symlink(&self, link: Arc<dyn Inode>) -> KResult<()> { - self.save_data(link, D_SYMLINK) + pub fn fill(&self, file: InodeUse<dyn Inode>) { + match file.format() { + Format::REG | Format::BLK | Format::CHR => self.save(file, DentryKind::Regular), + Format::DIR => self.save(file, DentryKind::Directory), + Format::LNK => self.save(file, DentryKind::Symlink), + } } - pub fn save_dir(&self, dir: Arc<dyn Inode>) -> KResult<()> { - self.save_data(dir, D_DIRECTORY) + pub fn inode(&self) -> Option<InodeUse<dyn Inode>> { + self.data.load().as_ref().map(|data| data.inode.clone()) } - pub fn get_inode(&self) -> KResult<Arc<dyn Inode>> { - self.data - .load() - .as_ref() - .ok_or(ENOENT) - .map(|data| data.inode.clone()) + pub fn get_inode(&self) -> KResult<InodeUse<dyn Inode>> { + self.inode().ok_or(ENOENT) } pub fn is_directory(&self) -> bool { let data = self.data.load(); data.as_ref() - .map_or(false, |data| data.flags & D_DIRECTORY != 0) + .map_or(false, |data| data.kind == DentryKind::Directory) } pub fn is_valid(&self) -> bool { self.data.load().is_some() } - pub fn open_check(self: &Arc<Self>, flags: OpenFlags, mode: Mode) -> KResult<()> { + pub async fn open_check(self: &Arc<Self>, flags: OpenFlags, perm: Permission) -> KResult<()> { let data = self.data.load(); if data.is_some() { @@ -250,7 +248,7 @@ impl Dentry { } let parent = self.parent().get_inode()?; - parent.creat(self, mode) + parent.create(self, perm).await } } } @@ -260,110 +258,120 @@ impl Dentry { context: &FsContext, dentry: Arc<Self>, nrecur: u32, - ) -> KResult<Arc<Self>> { - if nrecur >= 16 { - return Err(ELOOP); - } + ) -> Pin<Box<impl Future<Output = KResult<Arc<Self>>> + use<'_>>> { + Box::pin(async move { + if nrecur >= 16 { + return Err(ELOOP); + } - let data = dentry.data.load(); - let data = data.as_ref().ok_or(ENOENT)?; + let data = dentry.data.load(); + let data = data.as_ref().ok_or(ENOENT)?; - match data.flags { - flags if flags & D_REGULAR != 0 => Err(ENOTDIR), - flags if flags & D_DIRECTORY != 0 => Ok(dentry), - flags if flags & D_SYMLINK != 0 => { - let mut buffer = [0u8; 256]; - let mut buffer = ByteBuffer::new(&mut buffer); + match data.kind { + DentryKind::Regular => Err(ENOTDIR), + DentryKind::Directory => Ok(dentry), + DentryKind::Symlink => { + let mut buffer = [0u8; 256]; + let mut buffer = ByteBuffer::new(&mut buffer); - data.inode.readlink(&mut buffer)?; - let path = Path::new(buffer.data())?; + data.inode.readlink(&mut buffer).await?; + let path = Path::new(buffer.data())?; - let dentry = - Self::open_recursive(context, &dentry.parent(), path, true, nrecur + 1)?; + let dentry = + Self::open_recursive(context, &dentry.parent(), path, true, nrecur + 1) + .await?; - Self::resolve_directory(context, dentry, nrecur + 1) + Self::resolve_directory(context, dentry, nrecur + 1).await + } + _ => panic!("Invalid dentry flags"), } - _ => panic!("Invalid dentry flags"), - } + }) } - pub fn open_recursive( - context: &FsContext, - cwd: &Arc<Self>, - path: Path, + pub fn open_recursive<'r, 'a: 'r, 'b: 'r, 'c: 'r>( + context: &'a FsContext, + cwd: &'b Arc<Self>, + path: Path<'c>, follow: bool, nrecur: u32, - ) -> KResult<Arc<Self>> { - // too many recursive search layers will cause stack overflow - // so we use 16 for now - if nrecur >= 16 { - return Err(ELOOP); - } + ) -> Pin<Box<impl Future<Output = KResult<Arc<Self>>> + 'r>> { + Box::pin(async move { + // too many recursive search layers will cause stack overflow + // so we use 16 for now + if nrecur >= 16 { + return Err(ELOOP); + } - let mut cwd = if path.is_absolute() { - context.fsroot.clone() - } else { - cwd.clone() - }; + let mut cwd = if path.is_absolute() { + context.fsroot.clone() + } else { + cwd.clone() + }; - for item in path.iter() { - if let PathComponent::TrailingEmpty = item { - if cwd.data.load().as_ref().is_none() { - return Ok(cwd); + for item in path.iter() { + if let PathComponent::TrailingEmpty = item { + if cwd.data.load().as_ref().is_none() { + return Ok(cwd); + } } - } - cwd = Self::resolve_directory(context, cwd, nrecur)?; + cwd = Self::resolve_directory(context, cwd, nrecur).await?; - match item { - PathComponent::TrailingEmpty | PathComponent::Current => {} // pass - PathComponent::Parent => { - if !cwd.hash_eq(&context.fsroot) { - let parent = cwd.parent().clone(); - cwd = Self::resolve_directory(context, parent, nrecur)?; + match item { + PathComponent::TrailingEmpty | PathComponent::Current => {} // pass + PathComponent::Parent => { + if !cwd.hash_eq(&context.fsroot) { + let parent = cwd.parent().clone(); + cwd = Self::resolve_directory(context, parent, nrecur).await?; + } + continue; + } + PathComponent::Name(name) => { + cwd = cwd.find(name).await?; } - continue; - } - PathComponent::Name(name) => { - cwd = cwd.find(name)?; } } - } - if follow { - let data = cwd.data.load(); + if follow { + let data = cwd.data.load(); - if let Some(data) = data.as_ref() { - if data.flags & D_SYMLINK != 0 { - let data = cwd.data.load(); - let data = data.as_ref().unwrap(); - let mut buffer = [0u8; 256]; - let mut buffer = ByteBuffer::new(&mut buffer); + if let Some(data) = data.as_ref() { + if data.kind == DentryKind::Symlink { + let data = cwd.data.load(); + let data = data.as_ref().unwrap(); + let mut buffer = [0u8; 256]; + let mut buffer = ByteBuffer::new(&mut buffer); - data.inode.readlink(&mut buffer)?; - let path = Path::new(buffer.data())?; + data.inode.readlink(&mut buffer).await?; + let path = Path::new(buffer.data())?; - let parent = cwd.parent().clone(); - cwd = Self::open_recursive(context, &parent, path, true, nrecur + 1)?; + let parent = cwd.parent().clone(); + cwd = + Self::open_recursive(context, &parent, path, true, nrecur + 1).await?; + } } } - } - Ok(cwd) + Ok(cwd) + }) } - pub fn open(context: &FsContext, path: Path, follow_symlinks: bool) -> KResult<Arc<Self>> { + pub async fn open( + context: &FsContext, + path: Path<'_>, + follow_symlinks: bool, + ) -> KResult<Arc<Self>> { let cwd = context.cwd.lock().clone(); - Dentry::open_recursive(context, &cwd, path, follow_symlinks, 0) + Dentry::open_recursive(context, &cwd, path, follow_symlinks, 0).await } - pub fn open_at( + pub async fn open_at( context: &FsContext, at: &Arc<Self>, - path: Path, + path: Path<'_>, follow_symlinks: bool, ) -> KResult<Arc<Self>> { - Dentry::open_recursive(context, at, path, follow_symlinks, 0) + Dentry::open_recursive(context, at, path, follow_symlinks, 0).await } pub fn get_path( @@ -405,18 +413,18 @@ impl Dentry { } impl Dentry { - pub fn read(&self, buffer: &mut dyn Buffer, offset: usize) -> KResult<usize> { + pub async fn read(&self, buffer: &mut dyn Buffer, offset: usize) -> KResult<usize> { let inode = self.get_inode()?; // Safety: Changing mode alone will have no effect on the file's contents - match inode.mode.load().format() { - Mode::DIR => Err(EISDIR), - Mode::REG => inode.read(buffer, offset), - Mode::BLK => { + match inode.format() { + Format::DIR => Err(EISDIR), + Format::REG => inode.read(buffer, offset).await, + Format::BLK => { let device = BlockDevice::get(inode.devid()?)?; - Ok(device.read_some(offset, buffer)?.allow_partial()) + Ok(device.read_some(offset, buffer).await?.allow_partial()) } - Mode::CHR => { + Format::CHR => { let device = CharDevice::get(inode.devid()?).ok_or(EPERM)?; device.read(buffer) } @@ -424,32 +432,32 @@ impl Dentry { } } - pub fn write(&self, stream: &mut dyn Stream, offset: WriteOffset) -> KResult<usize> { + pub async fn write(&self, stream: &mut dyn Stream, offset: WriteOffset<'_>) -> KResult<usize> { let inode = self.get_inode()?; // Safety: Changing mode alone will have no effect on the file's contents - match inode.mode.load().format() { - Mode::DIR => Err(EISDIR), - Mode::REG => inode.write(stream, offset), - Mode::BLK => Err(EINVAL), // TODO - Mode::CHR => CharDevice::get(inode.devid()?).ok_or(EPERM)?.write(stream), + match inode.format() { + Format::DIR => Err(EISDIR), + Format::REG => inode.write(stream, offset).await, + Format::BLK => Err(EINVAL), // TODO + Format::CHR => CharDevice::get(inode.devid()?).ok_or(EPERM)?.write(stream), _ => Err(EINVAL), } } - pub fn readdir<F>(&self, offset: usize, mut callback: F) -> KResult<usize> + pub async fn readdir<F>(&self, offset: usize, mut for_each_entry: F) -> KResult<KResult<usize>> where - F: FnMut(&[u8], Ino) -> KResult<ControlFlow<(), ()>>, + F: FnMut(&[u8], Ino) -> KResult<bool> + Send, { let dir = self.get_inode()?; - dir.do_readdir(offset, &mut callback) + dir.readdir(offset, &mut for_each_entry).await } - pub fn mkdir(&self, mode: Mode) -> KResult<()> { + pub async fn mkdir(&self, perm: Permission) -> KResult<()> { if self.get_inode().is_ok() { Err(EEXIST) } else { let dir = self.parent().get_inode()?; - dir.mkdir(self, mode) + dir.mkdir(self, perm).await } } @@ -457,50 +465,50 @@ impl Dentry { self.get_inode()?.statx(stat, mask) } - pub fn truncate(&self, size: usize) -> KResult<()> { - self.get_inode()?.truncate(size) + pub async fn truncate(&self, size: usize) -> KResult<()> { + self.get_inode()?.truncate(size).await } - pub fn unlink(self: &Arc<Self>) -> KResult<()> { + pub async fn unlink(self: &Arc<Self>) -> KResult<()> { if self.get_inode().is_err() { Err(ENOENT) } else { let dir = self.parent().get_inode()?; - dir.unlink(self) + dir.unlink(self).await } } - pub fn symlink(self: &Arc<Self>, link: &[u8]) -> KResult<()> { + pub async fn symlink(self: &Arc<Self>, link: &[u8]) -> KResult<()> { if self.get_inode().is_ok() { Err(EEXIST) } else { let dir = self.parent().get_inode()?; - dir.symlink(self, link) + dir.symlink(self, link).await } } - pub fn readlink(&self, buffer: &mut dyn Buffer) -> KResult<usize> { - self.get_inode()?.readlink(buffer) + pub async fn readlink(&self, buffer: &mut dyn Buffer) -> KResult<usize> { + self.get_inode()?.readlink(buffer).await } - pub fn mknod(&self, mode: Mode, devid: DevId) -> KResult<()> { + pub async fn mknod(&self, mode: Mode, devid: DeviceId) -> KResult<()> { if self.get_inode().is_ok() { Err(EEXIST) } else { let dir = self.parent().get_inode()?; - dir.mknod(self, mode, devid) + dir.mknod(self, mode, devid).await } } - pub fn chmod(&self, mode: Mode) -> KResult<()> { - self.get_inode()?.chmod(mode) + pub async fn chmod(&self, mode: Mode) -> KResult<()> { + self.get_inode()?.chmod(mode).await } - pub fn chown(&self, uid: u32, gid: u32) -> KResult<()> { - self.get_inode()?.chown(uid, gid) + pub async fn chown(&self, uid: u32, gid: u32) -> KResult<()> { + self.get_inode()?.chown(uid, gid).await } - pub fn rename(self: &Arc<Self>, new: &Arc<Self>, flags: RenameFlags) -> KResult<()> { + pub async fn rename(self: &Arc<Self>, new: &Arc<Self>, flags: RenameFlags) -> KResult<()> { if Arc::ptr_eq(self, new) { return Ok(()); } @@ -509,22 +517,19 @@ impl Dentry { let new_parent = new.parent().get_inode()?; // If the two dentries are not in the same filesystem, return EXDEV. - if !Weak::ptr_eq(&old_parent.vfs, &new_parent.vfs) { + if old_parent.sbref().eq(&new_parent.sbref()) { Err(PosixError::EXDEV)?; } - let vfs = old_parent.vfs.upgrade().ok_or(EIO)?; - let rename_data = RenameData { old_dentry: self, new_dentry: new, new_parent, - vfs, is_exchange: flags.contains(RenameFlags::RENAME_EXCHANGE), no_replace: flags.contains(RenameFlags::RENAME_NOREPLACE), }; // Delegate to the parent directory's rename implementation - old_parent.rename(rename_data) + old_parent.rename(rename_data).await } } diff --git a/src/kernel/vfs/dentry/dcache.rs b/src/kernel/vfs/dentry/dcache.rs index 188a1cfc..e2491235 100644 --- a/src/kernel/vfs/dentry/dcache.rs +++ b/src/kernel/vfs/dentry/dcache.rs @@ -1,7 +1,5 @@ -use super::{Dentry, Inode}; +use super::Dentry; use crate::kernel::constants::ENOENT; -use crate::kernel::task::block_on; -use crate::kernel::vfs::inode::Mode; use crate::rcu::RCUPointer; use crate::{ prelude::*, @@ -41,27 +39,14 @@ pub fn d_find_fast(dentry: &Dentry) -> Option<Arc<Dentry>> { /// Call `lookup()` on the parent inode to try find if the dentry points to a valid inode /// /// Silently fail without any side effects -pub fn d_try_revalidate(dentry: &Arc<Dentry>) { - let _lock = block_on(D_EXCHANGE_LOCK.lock()); - - (|| -> KResult<()> { - let parent = dentry.parent().get_inode()?; - let inode = parent.lookup(dentry)?.ok_or(ENOENT)?; +pub async fn d_try_revalidate(dentry: &Arc<Dentry>) -> KResult<()> { + let _lock = D_EXCHANGE_LOCK.lock().await; - d_save(dentry, inode) - })() - .unwrap_or_default(); -} + let parent = dentry.parent().get_inode()?; + let inode = parent.lookup(dentry).await?.ok_or(ENOENT)?; -/// Save the inode to the dentry. -/// -/// Dentry flags will be determined by the inode's mode. -pub fn d_save(dentry: &Arc<Dentry>, inode: Arc<dyn Inode>) -> KResult<()> { - match inode.mode.load().format() { - Mode::DIR => dentry.save_dir(inode), - Mode::LNK => dentry.save_symlink(inode), - _ => dentry.save_reg(inode), - } + dentry.fill(inode); + Ok(()) } /// Replace the old dentry with the new one in the dcache diff --git a/src/kernel/vfs/file/inode_file.rs b/src/kernel/vfs/file/inode_file.rs index 6386ba92..96526ee9 100644 --- a/src/kernel/vfs/file/inode_file.rs +++ b/src/kernel/vfs/file/inode_file.rs @@ -5,13 +5,13 @@ use crate::{ constants::{EBADF, EFAULT, ENOTDIR, EOVERFLOW, ESPIPE}, vfs::{ dentry::Dentry, - inode::{Inode, Mode, WriteOffset}, + inode::{Inode, InodeUse, WriteOffset}, + types::Format, }, }, prelude::KResult, }; use alloc::sync::Arc; -use core::{ops::ControlFlow, sync::atomic::Ordering}; use eonix_sync::Mutex; use posix_types::{ getdent::{UserDirent, UserDirent64}, @@ -25,7 +25,7 @@ pub struct InodeFile { pub a: bool, /// Only a few modes those won't possibly change are cached here to speed up file operations. /// Specifically, `S_IFMT` masked bits. - pub mode: Mode, + pub format: Format, cursor: Mutex<usize>, dentry: Arc<Dentry>, } @@ -34,12 +34,7 @@ impl InodeFile { pub fn new(dentry: Arc<Dentry>, flags: OpenFlags) -> File { // SAFETY: `dentry` used to create `InodeFile` is valid. // SAFETY: `mode` should never change with respect to the `S_IFMT` fields. - let cached_mode = dentry - .get_inode() - .expect("`dentry` is invalid") - .mode - .load() - .format(); + let format = dentry.inode().expect("dentry should be invalid").format(); let (r, w, a) = flags.as_rwa(); @@ -50,15 +45,15 @@ impl InodeFile { r, w, a, - mode: cached_mode, + format, cursor: Mutex::new(0), }), ) } pub fn sendfile_check(&self) -> KResult<()> { - match self.mode { - Mode::REG | Mode::BLK => Ok(()), + match self.format { + Format::REG | Format::BLK => Ok(()), _ => Err(EBADF), } } @@ -70,21 +65,19 @@ impl InodeFile { let mut cursor = self.cursor.lock().await; - if self.a { - let nwrote = self.dentry.write(stream, WriteOffset::End(&mut cursor))?; + let (offset, update_offset) = match (self.a, offset) { + (true, _) => (WriteOffset::End(&mut cursor), None), + (false, Some(offset)) => (WriteOffset::Position(offset), None), + (false, None) => (WriteOffset::Position(*cursor), Some(&mut *cursor)), + }; - Ok(nwrote) - } else { - let nwrote = if let Some(offset) = offset { - self.dentry.write(stream, WriteOffset::Position(offset))? - } else { - let nwrote = self.dentry.write(stream, WriteOffset::Position(*cursor))?; - *cursor += nwrote; - nwrote - }; - - Ok(nwrote) + let nr_write = self.dentry.write(stream, offset).await?; + + if let Some(update_offset) = update_offset { + *update_offset += nr_write; } + + Ok(nr_write) } pub async fn read(&self, buffer: &mut dyn Buffer, offset: Option<usize>) -> KResult<usize> { @@ -92,24 +85,20 @@ impl InodeFile { return Err(EBADF); } - let nread = if let Some(offset) = offset { - let nread = self.dentry.read(buffer, offset)?; - nread - } else { - let mut cursor = self.cursor.lock().await; - - let nread = self.dentry.read(buffer, *cursor)?; + if let Some(offset) = offset { + return Ok(self.dentry.read(buffer, offset).await?); + } - *cursor += nread; - nread - }; + let mut cursor = self.cursor.lock().await; + let nread = self.dentry.read(buffer, *cursor).await?; + *cursor += nread; Ok(nread) } } impl File { - pub fn get_inode(&self) -> KResult<Option<Arc<dyn Inode>>> { + pub fn get_inode(&self) -> KResult<Option<InodeUse<dyn Inode>>> { if let FileType::Inode(inode_file) = &**self { Ok(Some(inode_file.dentry.get_inode()?)) } else { @@ -124,27 +113,30 @@ impl File { let mut cursor = inode_file.cursor.lock().await; - let nread = inode_file.dentry.readdir(*cursor, |filename, ino| { - // + 1 for filename length padding '\0', + 1 for d_type. - let real_record_len = core::mem::size_of::<UserDirent>() + filename.len() + 2; + let nread = inode_file + .dentry + .readdir(*cursor, |filename, ino| { + // + 1 for filename length padding '\0', + 1 for d_type. + let real_record_len = core::mem::size_of::<UserDirent>() + filename.len() + 2; - if buffer.available() < real_record_len { - return Ok(ControlFlow::Break(())); - } + if buffer.available() < real_record_len { + return Ok(false); + } - let record = UserDirent { - d_ino: ino as u32, - d_off: 0, - d_reclen: real_record_len as u16, - d_name: [0; 0], - }; + let record = UserDirent { + d_ino: ino.as_raw() as u32, + d_off: 0, + d_reclen: real_record_len as u16, + d_name: [0; 0], + }; - buffer.copy(&record)?.ok_or(EFAULT)?; - buffer.fill(filename)?.ok_or(EFAULT)?; - buffer.fill(&[0, 0])?.ok_or(EFAULT)?; + buffer.copy(&record)?.ok_or(EFAULT)?; + buffer.fill(filename)?.ok_or(EFAULT)?; + buffer.fill(&[0, 0])?.ok_or(EFAULT)?; - Ok(ControlFlow::Continue(())) - })?; + Ok(true) + }) + .await??; *cursor += nread; Ok(()) @@ -157,28 +149,31 @@ impl File { let mut cursor = inode_file.cursor.lock().await; - let nread = inode_file.dentry.readdir(*cursor, |filename, ino| { - // Filename length + 1 for padding '\0' - let real_record_len = core::mem::size_of::<UserDirent64>() + filename.len() + 1; + let nread = inode_file + .dentry + .readdir(*cursor, |filename, ino| { + // Filename length + 1 for padding '\0' + let real_record_len = core::mem::size_of::<UserDirent64>() + filename.len() + 1; - if buffer.available() < real_record_len { - return Ok(ControlFlow::Break(())); - } + if buffer.available() < real_record_len { + return Ok(false); + } - let record = UserDirent64 { - d_ino: ino, - d_off: 0, - d_reclen: real_record_len as u16, - d_type: 0, - d_name: [0; 0], - }; + let record = UserDirent64 { + d_ino: ino.as_raw(), + d_off: 0, + d_reclen: real_record_len as u16, + d_type: 0, + d_name: [0; 0], + }; - buffer.copy(&record)?.ok_or(EFAULT)?; - buffer.fill(filename)?.ok_or(EFAULT)?; - buffer.fill(&[0])?.ok_or(EFAULT)?; + buffer.copy(&record)?.ok_or(EFAULT)?; + buffer.fill(filename)?.ok_or(EFAULT)?; + buffer.fill(&[0])?.ok_or(EFAULT)?; - Ok(ControlFlow::Continue(())) - })?; + Ok(true) + }) + .await??; *cursor += nread; Ok(()) @@ -196,7 +191,7 @@ impl File { SeekOption::Set(n) => n, SeekOption::End(off) => { let inode = inode_file.dentry.get_inode()?; - let size = inode.size.load(Ordering::Relaxed) as usize; + let size = inode.info().lock().size as usize; size.checked_add_signed(off).ok_or(EOVERFLOW)? } }; diff --git a/src/kernel/vfs/filearray.rs b/src/kernel/vfs/filearray.rs index b457a425..1862a3e1 100644 --- a/src/kernel/vfs/filearray.rs +++ b/src/kernel/vfs/filearray.rs @@ -1,6 +1,6 @@ use super::{ file::{File, InodeFile, Pipe}, - inode::Mode, + types::{Format, Permission}, Spin, TerminalFile, }; use crate::kernel::{ @@ -280,26 +280,31 @@ impl FileArray { Ok((read_fd, write_fd)) } - pub fn open(&self, dentry: &Arc<Dentry>, flags: OpenFlags, mode: Mode) -> KResult<FD> { - dentry.open_check(flags, mode)?; + pub async fn open( + &self, + dentry: &Arc<Dentry>, + flags: OpenFlags, + perm: Permission, + ) -> KResult<FD> { + dentry.open_check(flags, perm).await?; let fdflag = flags.as_fd_flags(); let inode = dentry.get_inode()?; - let file_format = inode.mode.load().format(); + let file_format = inode.format(); match (flags.directory(), file_format, flags.write()) { - (true, Mode::DIR, _) => {} + (true, Format::DIR, _) => {} (true, _, _) => return Err(ENOTDIR), - (false, Mode::DIR, true) => return Err(EISDIR), + (false, Format::DIR, true) => return Err(EISDIR), _ => {} } - if flags.truncate() && flags.write() && file_format.is_reg() { - inode.truncate(0)?; + if flags.truncate() && flags.write() && file_format == Format::REG { + inode.truncate(0).await?; } - let file = if file_format.is_chr() { + let file = if file_format == Format::CHR { let device = CharDevice::get(inode.devid()?).ok_or(ENXIO)?; device.open(flags)? } else { diff --git a/src/kernel/vfs/inode.rs b/src/kernel/vfs/inode.rs deleted file mode 100644 index 52529f84..00000000 --- a/src/kernel/vfs/inode.rs +++ /dev/null @@ -1,494 +0,0 @@ -use super::{dentry::Dentry, vfs::Vfs, DevId}; -use crate::io::Stream; -use crate::kernel::constants::{ - EINVAL, EISDIR, ENOTDIR, EPERM, STATX_ATIME, STATX_BLOCKS, STATX_CTIME, STATX_GID, STATX_INO, - STATX_MODE, STATX_MTIME, STATX_NLINK, STATX_SIZE, STATX_TYPE, STATX_UID, S_IFBLK, S_IFCHR, - S_IFDIR, S_IFLNK, S_IFMT, S_IFREG, -}; -use crate::kernel::mem::PageCache; -use crate::kernel::syscall::{FromSyscallArg, SyscallRetVal}; -use crate::kernel::task::block_on; -use crate::kernel::timer::Instant; -use crate::{io::Buffer, prelude::*}; -use alloc::sync::{Arc, Weak}; -use core::{ - mem::MaybeUninit, - ops::ControlFlow, - ptr::addr_of_mut, - sync::atomic::{AtomicU32, AtomicU64, Ordering}, -}; -use eonix_sync::RwLock; -use posix_types::stat::StatX; - -pub type Ino = u64; -pub type AtomicIno = AtomicU64; -#[allow(dead_code)] -pub type ISize = u64; -pub type AtomicISize = AtomicU64; -#[allow(dead_code)] -pub type Nlink = u64; -pub type AtomicNlink = AtomicU64; -#[allow(dead_code)] -pub type Uid = u32; -pub type AtomicUid = AtomicU32; -#[allow(dead_code)] -pub type Gid = u32; -pub type AtomicGid = AtomicU32; - -#[derive(Clone, Copy, PartialEq, Eq)] -pub struct Mode(u32); - -pub struct AtomicMode(AtomicU32); - -#[derive(Debug)] -pub struct InodeData { - pub ino: Ino, - pub size: AtomicISize, - pub nlink: AtomicNlink, - - pub uid: AtomicUid, - pub gid: AtomicGid, - pub mode: AtomicMode, - - pub atime: Spin<Instant>, - pub ctime: Spin<Instant>, - pub mtime: Spin<Instant>, - - pub rwsem: RwLock<()>, - - pub vfs: Weak<dyn Vfs>, -} - -impl InodeData { - pub fn new(ino: Ino, vfs: Weak<dyn Vfs>) -> Self { - Self { - ino, - vfs, - atime: Spin::new(Instant::now()), - ctime: Spin::new(Instant::now()), - mtime: Spin::new(Instant::now()), - rwsem: RwLock::new(()), - size: AtomicU64::new(0), - nlink: AtomicNlink::new(0), - uid: AtomicUid::new(0), - gid: AtomicGid::new(0), - mode: AtomicMode::new(0), - } - } -} - -#[allow(dead_code)] -pub trait InodeInner: - Send + Sync + core::ops::Deref<Target = InodeData> + core::ops::DerefMut -{ - fn data(&self) -> &InodeData; - fn data_mut(&mut self) -> &mut InodeData; -} - -pub enum WriteOffset<'end> { - Position(usize), - End(&'end mut usize), -} - -pub struct RenameData<'a, 'b> { - pub old_dentry: &'a Arc<Dentry>, - pub new_dentry: &'b Arc<Dentry>, - pub new_parent: Arc<dyn Inode>, - pub vfs: Arc<dyn Vfs>, - pub is_exchange: bool, - pub no_replace: bool, -} - -#[allow(unused_variables)] -pub trait Inode: Send + Sync + InodeInner + Any { - fn is_dir(&self) -> bool { - self.mode.load().is_dir() - } - - fn lookup(&self, dentry: &Arc<Dentry>) -> KResult<Option<Arc<dyn Inode>>> { - Err(if !self.is_dir() { ENOTDIR } else { EPERM }) - } - - fn creat(&self, at: &Arc<Dentry>, mode: Mode) -> KResult<()> { - Err(if !self.is_dir() { ENOTDIR } else { EPERM }) - } - - fn mkdir(&self, at: &Dentry, mode: Mode) -> KResult<()> { - Err(if !self.is_dir() { ENOTDIR } else { EPERM }) - } - - fn mknod(&self, at: &Dentry, mode: Mode, dev: DevId) -> KResult<()> { - Err(if !self.is_dir() { ENOTDIR } else { EPERM }) - } - - fn unlink(&self, at: &Arc<Dentry>) -> KResult<()> { - Err(if !self.is_dir() { ENOTDIR } else { EPERM }) - } - - fn symlink(&self, at: &Arc<Dentry>, target: &[u8]) -> KResult<()> { - Err(if !self.is_dir() { ENOTDIR } else { EPERM }) - } - - fn read(&self, buffer: &mut dyn Buffer, offset: usize) -> KResult<usize> { - Err(if self.is_dir() { EISDIR } else { EINVAL }) - } - - fn read_direct(&self, buffer: &mut dyn Buffer, offset: usize) -> KResult<usize> { - Err(if self.is_dir() { EISDIR } else { EINVAL }) - } - - fn write(&self, stream: &mut dyn Stream, offset: WriteOffset) -> KResult<usize> { - Err(if self.is_dir() { EISDIR } else { EINVAL }) - } - - fn write_direct(&self, stream: &mut dyn Stream, offset: usize) -> KResult<usize> { - Err(if self.is_dir() { EISDIR } else { EINVAL }) - } - - fn devid(&self) -> KResult<DevId> { - Err(if self.is_dir() { EISDIR } else { EINVAL }) - } - - fn readlink(&self, buffer: &mut dyn Buffer) -> KResult<usize> { - Err(if self.is_dir() { EISDIR } else { EINVAL }) - } - - fn truncate(&self, length: usize) -> KResult<()> { - Err(if self.is_dir() { EISDIR } else { EPERM }) - } - - fn rename(&self, rename_data: RenameData) -> KResult<()> { - Err(if !self.is_dir() { ENOTDIR } else { EPERM }) - } - - fn do_readdir( - &self, - offset: usize, - callback: &mut dyn FnMut(&[u8], Ino) -> KResult<ControlFlow<(), ()>>, - ) -> KResult<usize> { - Err(if !self.is_dir() { ENOTDIR } else { EPERM }) - } - - fn chmod(&self, mode: Mode) -> KResult<()> { - Err(EPERM) - } - - fn chown(&self, uid: u32, gid: u32) -> KResult<()> { - Err(EPERM) - } - - fn page_cache(&self) -> Option<&PageCache> { - None - } - - fn statx(&self, stat: &mut StatX, mask: u32) -> KResult<()> { - // Safety: ffi should have checked reference - let vfs = self.vfs.upgrade().expect("Vfs is dropped"); - - let size = self.size.load(Ordering::Relaxed); - let mode = self.mode.load(); - - if mask & STATX_NLINK != 0 { - stat.stx_nlink = self.nlink.load(Ordering::Acquire) as _; - stat.stx_mask |= STATX_NLINK; - } - - if mask & STATX_ATIME != 0 { - let atime = *self.atime.lock(); - stat.stx_atime = atime.into(); - stat.stx_mask |= STATX_ATIME; - } - - if mask & STATX_MTIME != 0 { - let mtime = *self.mtime.lock(); - stat.stx_mtime = mtime.into(); - stat.stx_mask |= STATX_MTIME; - } - - if mask & STATX_CTIME != 0 { - let ctime = *self.ctime.lock(); - stat.stx_ctime = ctime.into(); - stat.stx_mask |= STATX_CTIME; - } - - if mask & STATX_SIZE != 0 { - stat.stx_size = self.size.load(Ordering::Relaxed) as _; - stat.stx_mask |= STATX_SIZE; - } - - stat.stx_mode = 0; - if mask & STATX_MODE != 0 { - stat.stx_mode |= mode.non_format_bits() as u16; - stat.stx_mask |= STATX_MODE; - } - - if mask & STATX_TYPE != 0 { - stat.stx_mode |= mode.format_bits() as u16; - if mode.is_blk() || mode.is_chr() { - let devid = self.devid(); - stat.stx_rdev_major = (devid? >> 8) & 0xff; - stat.stx_rdev_minor = devid? & 0xff; - } - stat.stx_mask |= STATX_TYPE; - } - - if mask & STATX_INO != 0 { - stat.stx_ino = self.ino as _; - stat.stx_mask |= STATX_INO; - } - - if mask & STATX_BLOCKS != 0 { - stat.stx_blocks = (size + 512 - 1) / 512; - stat.stx_blksize = vfs.io_blksize() as _; - stat.stx_mask |= STATX_BLOCKS; - } - - if mask & STATX_UID != 0 { - stat.stx_uid = self.uid.load(Ordering::Relaxed) as _; - stat.stx_mask |= STATX_UID; - } - - if mask & STATX_GID != 0 { - stat.stx_gid = self.gid.load(Ordering::Relaxed) as _; - stat.stx_mask |= STATX_GID; - } - - let fsdev = vfs.fs_devid(); - stat.stx_dev_major = (fsdev >> 8) & 0xff; - stat.stx_dev_minor = fsdev & 0xff; - - // TODO: support more attributes - stat.stx_attributes_mask = 0; - - Ok(()) - } - - fn new_locked<F>(ino: Ino, vfs: Weak<dyn Vfs>, f: F) -> Arc<Self> - where - Self: Sized, - F: FnOnce(*mut Self, &()), - { - let mut uninit = Arc::<Self>::new_uninit(); - - let uninit_mut = Arc::get_mut(&mut uninit).unwrap(); - - // Safety: `idata` is owned by `uninit` - let idata = unsafe { - addr_of_mut!(*(*uninit_mut.as_mut_ptr()).data_mut()) - .cast::<MaybeUninit<InodeData>>() - .as_mut() - .unwrap() - }; - - idata.write(InodeData::new(ino, vfs)); - - f( - uninit_mut.as_mut_ptr(), - // SAFETY: `idata` is initialized and we will never move the lock. - &block_on(unsafe { idata.assume_init_ref() }.rwsem.read()), - ); - - // Safety: `uninit` is initialized - unsafe { uninit.assume_init() } - } -} - -// TODO: define multiple inode structs a time -macro_rules! define_struct_inode { - ($v:vis struct $inode_t:ident;) => { - $v struct $inode_t { - /// Do not use this directly - idata: $crate::kernel::vfs::inode::InodeData, - } - - impl core::ops::Deref for $inode_t { - type Target = $crate::kernel::vfs::inode::InodeData; - - fn deref(&self) -> &Self::Target { - &self.idata - } - } - - impl core::ops::DerefMut for $inode_t { - fn deref_mut(&mut self) -> &mut Self::Target { - &mut self.idata - } - } - - impl $crate::kernel::vfs::inode::InodeInner for $inode_t { - fn data(&self) -> &$crate::kernel::vfs::inode::InodeData { - &self.idata - } - - fn data_mut(&mut self) -> &mut $crate::kernel::vfs::inode::InodeData { - &mut self.idata - } - } - }; - ($v:vis struct $inode_t:ident { $($vis:vis $name:ident: $type:ty,)* }) => { - $v struct $inode_t { - /// Do not use this directly - idata: $crate::kernel::vfs::inode::InodeData, - $($vis $name: $type,)* - } - - impl core::ops::Deref for $inode_t { - type Target = $crate::kernel::vfs::inode::InodeData; - - fn deref(&self) -> &Self::Target { - &self.idata - } - } - - impl core::ops::DerefMut for $inode_t { - fn deref_mut(&mut self) -> &mut Self::Target { - &mut self.idata - } - } - - impl $crate::kernel::vfs::inode::InodeInner for $inode_t { - fn data(&self) -> &$crate::kernel::vfs::inode::InodeData { - &self.idata - } - - fn data_mut(&mut self) -> &mut $crate::kernel::vfs::inode::InodeData { - &mut self.idata - } - } - }; -} - -pub(crate) use define_struct_inode; - -impl Mode { - pub const REG: Self = Self(S_IFREG); - pub const DIR: Self = Self(S_IFDIR); - pub const LNK: Self = Self(S_IFLNK); - pub const BLK: Self = Self(S_IFBLK); - pub const CHR: Self = Self(S_IFCHR); - - pub const fn new(bits: u32) -> Self { - Self(bits) - } - - pub const fn is_blk(&self) -> bool { - (self.0 & S_IFMT) == S_IFBLK - } - - pub const fn is_chr(&self) -> bool { - (self.0 & S_IFMT) == S_IFCHR - } - - pub const fn is_reg(&self) -> bool { - (self.0 & S_IFMT) == S_IFREG - } - - pub const fn is_dir(&self) -> bool { - (self.0 & S_IFMT) == S_IFDIR - } - - pub const fn is_lnk(&self) -> bool { - (self.0 & S_IFMT) == S_IFLNK - } - - pub const fn bits(&self) -> u32 { - self.0 - } - - pub const fn format_bits(&self) -> u32 { - self.0 & S_IFMT - } - - pub const fn format(&self) -> Self { - Self::new(self.format_bits()) - } - - pub const fn non_format_bits(&self) -> u32 { - self.0 & !S_IFMT - } - - pub const fn non_format(&self) -> Self { - Self::new(self.non_format_bits()) - } - - pub const fn perm(self, perm: u32) -> Self { - Self::new((self.0 & !0o777) | (perm & 0o777)) - } - - pub const fn set_perm(&mut self, perm: u32) { - *self = self.perm(perm); - } - - pub const fn mask_perm(&mut self, perm_mask: u32) { - let perm_mask = perm_mask & 0o777; - let self_perm = self.non_format_bits() & 0o777; - - *self = self.perm(self_perm & perm_mask); - } -} - -impl AtomicMode { - pub const fn new(bits: u32) -> Self { - Self(AtomicU32::new(bits)) - } - - pub const fn from(mode: Mode) -> Self { - Self::new(mode.0) - } - - pub fn load(&self) -> Mode { - Mode(self.0.load(Ordering::Relaxed)) - } - - pub fn store(&self, mode: Mode) { - self.0.store(mode.0, Ordering::Relaxed); - } -} - -impl core::fmt::Debug for AtomicMode { - fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { - f.debug_struct("AtomicMode") - .field("bits", &self.load().0) - .finish() - } -} - -impl core::fmt::Debug for Mode { - fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { - let format_name = match self.format() { - Mode::REG => "REG", - Mode::DIR => "DIR", - Mode::LNK => "LNK", - Mode::BLK => "BLK", - Mode::CHR => "CHR", - _ => "UNK", - }; - - match self.non_format_bits() & !0o777 { - 0 => write!( - f, - "Mode({format_name}, {perm:#o})", - perm = self.non_format_bits() - )?, - rem => write!( - f, - "Mode({format_name}, {perm:#o}, rem={rem:#x})", - perm = self.non_format_bits() & 0o777 - )?, - } - - Ok(()) - } -} - -impl FromSyscallArg for Mode { - fn from_arg(value: usize) -> Self { - Mode::new(value as u32) - } -} - -impl SyscallRetVal for Mode { - fn into_retval(self) -> Option<usize> { - Some(self.bits() as usize) - } -} diff --git a/src/kernel/vfs/inode/ino.rs b/src/kernel/vfs/inode/ino.rs new file mode 100644 index 00000000..b5ee7ac0 --- /dev/null +++ b/src/kernel/vfs/inode/ino.rs @@ -0,0 +1,31 @@ +use core::{ + fmt::{Debug, Display, Formatter}, + sync::atomic::AtomicU64, +}; + +#[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)] +pub struct Ino(u64); + +pub struct AtomicIno(AtomicU64); + +impl Ino { + pub const fn new(ino: u64) -> Self { + Self(ino) + } + + pub const fn as_raw(self) -> u64 { + self.0 + } +} + +impl Debug for Ino { + fn fmt(&self, f: &mut Formatter<'_>) -> core::fmt::Result { + write!(f, "Ino({})", self.0) + } +} + +impl Display for Ino { + fn fmt(&self, f: &mut Formatter<'_>) -> core::fmt::Result { + write!(f, "{:?}", self) + } +} diff --git a/src/kernel/vfs/inode/inode.rs b/src/kernel/vfs/inode/inode.rs new file mode 100644 index 00000000..786a31fe --- /dev/null +++ b/src/kernel/vfs/inode/inode.rs @@ -0,0 +1,389 @@ +use alloc::boxed::Box; +use core::{ + any::Any, + future::Future, + marker::Unsize, + ops::{CoerceUnsized, Deref}, + pin::Pin, +}; +use eonix_sync::Spin; + +use alloc::sync::{Arc, Weak}; +use async_trait::async_trait; + +use crate::{ + io::{Buffer, Stream}, + kernel::{ + constants::{EINVAL, EPERM}, + mem::PageCache, + timer::Instant, + vfs::{ + dentry::Dentry, + types::{DeviceId, Format, Mode, Permission}, + SbRef, SbUse, SuperBlock, + }, + }, + prelude::KResult, +}; + +use super::{Ino, RenameData, WriteOffset}; + +pub trait InodeOps: Sized + Send + Sync + 'static { + type SuperBlock: SuperBlock + Sized; + + fn ino(&self) -> Ino; + fn format(&self) -> Format; + fn info(&self) -> &Spin<InodeInfo>; + + fn super_block(&self) -> &SbRef<Self::SuperBlock>; + + fn page_cache(&self) -> Option<&PageCache>; +} + +#[allow(unused_variables)] +pub trait InodeDirOps: InodeOps { + fn lookup( + &self, + dentry: &Arc<Dentry>, + ) -> impl Future<Output = KResult<Option<InodeUse<dyn Inode>>>> + Send { + async { Err(EPERM) } + } + + /// Read directory entries and call the given closure for each entry. + /// + /// # Returns + /// - Ok(count): The number of entries read. + /// - Ok(Err(err)): Some error occurred while calling the given closure. + /// - Err(err): An error occurred while reading the directory. + fn readdir<'r, 'a: 'r, 'b: 'r>( + &'a self, + offset: usize, + for_each_entry: &'b mut (dyn FnMut(&[u8], Ino) -> KResult<bool> + Send), + ) -> impl Future<Output = KResult<KResult<usize>>> + Send + 'r { + async { Err(EPERM) } + } + + fn create( + &self, + at: &Arc<Dentry>, + mode: Permission, + ) -> impl Future<Output = KResult<()>> + Send { + async { Err(EPERM) } + } + + fn mkdir(&self, at: &Dentry, mode: Permission) -> impl Future<Output = KResult<()>> + Send { + async { Err(EPERM) } + } + + fn mknod( + &self, + at: &Dentry, + mode: Mode, + dev: DeviceId, + ) -> impl Future<Output = KResult<()>> + Send { + async { Err(EPERM) } + } + + fn unlink(&self, at: &Arc<Dentry>) -> impl Future<Output = KResult<()>> + Send { + async { Err(EPERM) } + } + + fn symlink(&self, at: &Arc<Dentry>, target: &[u8]) -> impl Future<Output = KResult<()>> + Send { + async { Err(EPERM) } + } + + fn rename(&self, rename_data: RenameData<'_, '_>) -> impl Future<Output = KResult<()>> + Send { + async { Err(EPERM) } + } +} + +#[allow(unused_variables)] +pub trait InodeFileOps: InodeOps { + fn read( + &self, + buffer: &mut dyn Buffer, + offset: usize, + ) -> impl Future<Output = KResult<usize>> + Send { + async { Err(EINVAL) } + } + + fn read_direct( + &self, + buffer: &mut dyn Buffer, + offset: usize, + ) -> impl Future<Output = KResult<usize>> + Send { + async { Err(EINVAL) } + } + + fn write( + &self, + stream: &mut dyn Stream, + offset: WriteOffset<'_>, + ) -> impl Future<Output = KResult<usize>> + Send { + async { Err(EINVAL) } + } + + fn write_direct( + &self, + stream: &mut dyn Stream, + offset: usize, + ) -> impl Future<Output = KResult<usize>> + Send { + async { Err(EINVAL) } + } + + fn devid(&self) -> KResult<DeviceId> { + Err(EINVAL) + } + + fn readlink(&self, buffer: &mut dyn Buffer) -> impl Future<Output = KResult<usize>> + Send { + async { Err(EINVAL) } + } + + fn truncate(&self, length: usize) -> impl Future<Output = KResult<()>> + Send { + async { Err(EPERM) } + } + + fn chmod(&self, perm: Permission) -> impl Future<Output = KResult<()>> + Send { + async { Err(EPERM) } + } + + fn chown(&self, uid: u32, gid: u32) -> impl Future<Output = KResult<()>> + Send { + async { Err(EPERM) } + } +} + +#[async_trait] +pub trait InodeDir { + async fn lookup(&self, dentry: &Arc<Dentry>) -> KResult<Option<InodeUse<dyn Inode>>>; + async fn create(&self, at: &Arc<Dentry>, perm: Permission) -> KResult<()>; + async fn mkdir(&self, at: &Dentry, perm: Permission) -> KResult<()>; + async fn mknod(&self, at: &Dentry, mode: Mode, dev: DeviceId) -> KResult<()>; + async fn unlink(&self, at: &Arc<Dentry>) -> KResult<()>; + async fn symlink(&self, at: &Arc<Dentry>, target: &[u8]) -> KResult<()>; + async fn rename(&self, rename_data: RenameData<'_, '_>) -> KResult<()>; + + fn readdir<'r, 'a: 'r, 'b: 'r>( + &'a self, + offset: usize, + callback: &'b mut (dyn FnMut(&[u8], Ino) -> KResult<bool> + Send), + ) -> Pin<Box<dyn Future<Output = KResult<KResult<usize>>> + Send + 'r>>; +} + +#[async_trait] +pub trait InodeFile { + async fn read(&self, buffer: &mut dyn Buffer, offset: usize) -> KResult<usize>; + async fn read_direct(&self, buffer: &mut dyn Buffer, offset: usize) -> KResult<usize>; + async fn write(&self, stream: &mut dyn Stream, offset: WriteOffset<'_>) -> KResult<usize>; + async fn write_direct(&self, stream: &mut dyn Stream, offset: usize) -> KResult<usize>; + fn devid(&self) -> KResult<DeviceId>; + async fn readlink(&self, buffer: &mut dyn Buffer) -> KResult<usize>; + async fn truncate(&self, length: usize) -> KResult<()>; + async fn chmod(&self, mode: Mode) -> KResult<()>; + async fn chown(&self, uid: u32, gid: u32) -> KResult<()>; +} + +pub trait Inode: InodeFile + InodeDir + Any + Send + Sync + 'static { + fn ino(&self) -> Ino; + fn format(&self) -> Format; + fn info(&self) -> &Spin<InodeInfo>; + + // TODO: This might should be removed... Temporary workaround for now. + fn page_cache(&self) -> Option<&PageCache>; + + fn sbref(&self) -> SbRef<dyn SuperBlock>; + fn sbget(&self) -> KResult<SbUse<dyn SuperBlock>>; +} + +#[async_trait] +impl<T> InodeFile for T +where + T: InodeFileOps, +{ + async fn read(&self, buffer: &mut dyn Buffer, offset: usize) -> KResult<usize> { + self.read(buffer, offset).await + } + + async fn read_direct(&self, buffer: &mut dyn Buffer, offset: usize) -> KResult<usize> { + self.read_direct(buffer, offset).await + } + + async fn write(&self, stream: &mut dyn Stream, offset: WriteOffset<'_>) -> KResult<usize> { + self.write(stream, offset).await + } + + async fn write_direct(&self, stream: &mut dyn Stream, offset: usize) -> KResult<usize> { + self.write_direct(stream, offset).await + } + + fn devid(&self) -> KResult<DeviceId> { + self.devid() + } + + async fn readlink(&self, buffer: &mut dyn Buffer) -> KResult<usize> { + self.readlink(buffer).await + } + + async fn truncate(&self, length: usize) -> KResult<()> { + self.truncate(length).await + } + + async fn chmod(&self, mode: Mode) -> KResult<()> { + self.chmod(Permission::new(mode.non_format_bits())).await + } + + async fn chown(&self, uid: u32, gid: u32) -> KResult<()> { + self.chown(uid, gid).await + } +} + +#[async_trait] +impl<T> InodeDir for T +where + T: InodeDirOps, +{ + async fn lookup(&self, dentry: &Arc<Dentry>) -> KResult<Option<InodeUse<dyn Inode>>> { + self.lookup(dentry).await + } + + async fn create(&self, at: &Arc<Dentry>, perm: Permission) -> KResult<()> { + self.create(at, perm).await + } + + async fn mkdir(&self, at: &Dentry, perm: Permission) -> KResult<()> { + self.mkdir(at, perm).await + } + + async fn mknod(&self, at: &Dentry, mode: Mode, dev: DeviceId) -> KResult<()> { + self.mknod(at, mode, dev).await + } + + async fn unlink(&self, at: &Arc<Dentry>) -> KResult<()> { + self.unlink(at).await + } + + async fn symlink(&self, at: &Arc<Dentry>, target: &[u8]) -> KResult<()> { + self.symlink(at, target).await + } + + async fn rename(&self, rename_data: RenameData<'_, '_>) -> KResult<()> { + self.rename(rename_data).await + } + + fn readdir<'r, 'a: 'r, 'b: 'r>( + &'a self, + offset: usize, + callback: &'b mut (dyn FnMut(&[u8], Ino) -> KResult<bool> + Send), + ) -> Pin<Box<dyn Future<Output = KResult<KResult<usize>>> + Send + 'r>> { + Box::pin(self.readdir(offset, callback)) + } +} + +impl<T> Inode for T +where + T: InodeOps + InodeFile + InodeDir, +{ + fn ino(&self) -> Ino { + self.ino() + } + + fn format(&self) -> Format { + self.format() + } + + fn info(&self) -> &Spin<InodeInfo> { + self.info() + } + + fn page_cache(&self) -> Option<&PageCache> { + self.page_cache() + } + + fn sbref(&self) -> SbRef<dyn SuperBlock> { + self.super_block().clone() + } + + fn sbget(&self) -> KResult<SbUse<dyn SuperBlock>> { + self.super_block().get().map(|sb| sb as _) + } +} + +#[derive(Debug, Clone)] +pub struct InodeInfo { + pub size: u64, + pub nlink: u64, + + pub uid: u32, + pub gid: u32, + pub perm: Permission, + + pub atime: Instant, + pub ctime: Instant, + pub mtime: Instant, +} + +#[derive(Clone)] +pub struct InodeRef<I>(Weak<I>) +where + I: Inode + ?Sized; + +pub struct InodeUse<I>(Arc<I>) +where + I: Inode + ?Sized; + +impl<I> InodeUse<I> +where + I: Inode, +{ + pub fn new(inode: I) -> Self { + Self(Arc::new(inode)) + } + + pub fn new_cyclic(inode_func: impl FnOnce(&Weak<I>) -> I) -> Self { + Self(Arc::new_cyclic(inode_func)) + } +} + +impl<I> InodeUse<I> +where + I: Inode + ?Sized, +{ + pub fn as_raw(&self) -> *const I { + Arc::as_ptr(&self.0) + } +} + +impl<T, U> CoerceUnsized<InodeUse<U>> for InodeUse<T> +where + T: Inode + Unsize<U> + ?Sized, + U: Inode + ?Sized, +{ +} + +impl<I> Clone for InodeUse<I> +where + I: Inode + ?Sized, +{ + fn clone(&self) -> Self { + Self(self.0.clone()) + } +} + +impl<I> core::fmt::Debug for InodeUse<I> +where + I: Inode + ?Sized, +{ + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + write!(f, "InodeUse(ino={})", self.ino()) + } +} + +impl<I> Deref for InodeUse<I> +where + I: Inode + ?Sized, +{ + type Target = I; + + fn deref(&self) -> &Self::Target { + self.0.deref() + } +} diff --git a/src/kernel/vfs/inode/mod.rs b/src/kernel/vfs/inode/mod.rs new file mode 100644 index 00000000..786d30fa --- /dev/null +++ b/src/kernel/vfs/inode/mod.rs @@ -0,0 +1,10 @@ +mod ino; +mod inode; +mod ops; +mod statx; + +pub use ino::Ino; +pub use inode::{ + Inode, InodeDir, InodeDirOps, InodeFile, InodeFileOps, InodeInfo, InodeOps, InodeRef, InodeUse, +}; +pub use ops::{RenameData, WriteOffset}; diff --git a/src/kernel/vfs/inode/ops.rs b/src/kernel/vfs/inode/ops.rs new file mode 100644 index 00000000..baab1a80 --- /dev/null +++ b/src/kernel/vfs/inode/ops.rs @@ -0,0 +1,18 @@ +use alloc::sync::Arc; + +use crate::kernel::vfs::dentry::Dentry; + +use super::{inode::InodeUse, Inode}; + +pub enum WriteOffset<'end> { + Position(usize), + End(&'end mut usize), +} + +pub struct RenameData<'a, 'b> { + pub old_dentry: &'a Arc<Dentry>, + pub new_dentry: &'b Arc<Dentry>, + pub new_parent: InodeUse<dyn Inode>, + pub is_exchange: bool, + pub no_replace: bool, +} diff --git a/src/kernel/vfs/inode/statx.rs b/src/kernel/vfs/inode/statx.rs new file mode 100644 index 00000000..a85ef3af --- /dev/null +++ b/src/kernel/vfs/inode/statx.rs @@ -0,0 +1,97 @@ +use posix_types::stat::StatX; + +use crate::{ + kernel::{ + constants::{ + STATX_ATIME, STATX_BLOCKS, STATX_CTIME, STATX_GID, STATX_INO, STATX_MODE, STATX_MTIME, + STATX_NLINK, STATX_SIZE, STATX_TYPE, STATX_UID, + }, + vfs::types::Format, + }, + prelude::KResult, +}; + +use super::{inode::InodeUse, Inode}; + +impl<I> InodeUse<I> +where + I: Inode + ?Sized, +{ + pub fn statx(&self, stat: &mut StatX, mask: u32) -> KResult<()> { + let sb = self.sbget()?; + let info = self.info().lock(); + + if mask & STATX_NLINK != 0 { + stat.stx_nlink = info.nlink as _; + stat.stx_mask |= STATX_NLINK; + } + + if mask & STATX_ATIME != 0 { + stat.stx_atime = info.atime.into(); + stat.stx_mask |= STATX_ATIME; + } + + if mask & STATX_MTIME != 0 { + stat.stx_mtime = info.mtime.into(); + stat.stx_mask |= STATX_MTIME; + } + + if mask & STATX_CTIME != 0 { + stat.stx_ctime = info.ctime.into(); + stat.stx_mask |= STATX_CTIME; + } + + if mask & STATX_SIZE != 0 { + stat.stx_size = info.size as _; + stat.stx_mask |= STATX_SIZE; + } + + stat.stx_mode = 0; + if mask & STATX_MODE != 0 { + stat.stx_mode |= info.perm.bits() as u16; + stat.stx_mask |= STATX_MODE; + } + + if mask & STATX_TYPE != 0 { + let format = self.format(); + + stat.stx_mode |= format.as_raw() as u16; + if let Format::BLK | Format::CHR = format { + let devid = self.devid()?; + stat.stx_rdev_major = devid.major as _; + stat.stx_rdev_minor = devid.minor as _; + } + stat.stx_mask |= STATX_TYPE; + } + + if mask & STATX_INO != 0 { + stat.stx_ino = self.ino().as_raw(); + stat.stx_mask |= STATX_INO; + } + + if mask & STATX_BLOCKS != 0 { + stat.stx_blocks = (info.size + 512 - 1) / 512; + stat.stx_blksize = sb.info.io_blksize as _; + stat.stx_mask |= STATX_BLOCKS; + } + + if mask & STATX_UID != 0 { + stat.stx_uid = info.uid; + stat.stx_mask |= STATX_UID; + } + + if mask & STATX_GID != 0 { + stat.stx_gid = info.gid; + stat.stx_mask |= STATX_GID; + } + + let fsdev = sb.info.device_id; + stat.stx_dev_major = fsdev.major as _; + stat.stx_dev_minor = fsdev.minor as _; + + // TODO: support more attributes + stat.stx_attributes_mask = 0; + + Ok(()) + } +} diff --git a/src/kernel/vfs/mod.rs b/src/kernel/vfs/mod.rs index f62cb9b9..5b8eca5a 100644 --- a/src/kernel/vfs/mod.rs +++ b/src/kernel/vfs/mod.rs @@ -1,31 +1,31 @@ -use crate::prelude::*; -use alloc::sync::Arc; -use dentry::Dentry; -use eonix_sync::LazyLock; -use inode::Mode; - pub mod dentry; mod file; pub mod filearray; pub mod inode; pub mod mount; -pub mod vfs; +mod superblock; +pub mod types; -pub use file::{File, FileType, PollEvent, SeekOption, TerminalFile}; +use crate::prelude::*; +use alloc::sync::Arc; +use dentry::Dentry; +use eonix_sync::LazyLock; +use types::Permission; -pub type DevId = u32; +pub use file::{File, FileType, PollEvent, SeekOption, TerminalFile}; +pub use superblock::{SbRef, SbUse, SuperBlock, SuperBlockInfo, SuperBlockLock}; pub struct FsContext { pub fsroot: Arc<Dentry>, pub cwd: Spin<Arc<Dentry>>, - pub umask: Spin<Mode>, + pub umask: Spin<Permission>, } static GLOBAL_FS_CONTEXT: LazyLock<Arc<FsContext>> = LazyLock::new(|| { Arc::new(FsContext { fsroot: Dentry::root().clone(), cwd: Spin::new(Dentry::root().clone()), - umask: Spin::new(Mode::new(0o022)), + umask: Spin::new(Permission::new(0o755)), }) }); diff --git a/src/kernel/vfs/mount.rs b/src/kernel/vfs/mount.rs index 0b38e0c0..213acae9 100644 --- a/src/kernel/vfs/mount.rs +++ b/src/kernel/vfs/mount.rs @@ -1,11 +1,15 @@ use super::{ dentry::{dcache, Dentry, DROOT}, - inode::Inode, - vfs::Vfs, + inode::{Inode, InodeUse}, + SbUse, SuperBlock, +}; +use crate::kernel::{ + constants::{EEXIST, ENODEV, ENOTDIR}, + task::block_on, }; -use crate::kernel::constants::{EEXIST, ENODEV, ENOTDIR}; use crate::prelude::*; use alloc::{collections::btree_map::BTreeMap, string::ToString as _, sync::Arc}; +use async_trait::async_trait; use eonix_sync::LazyLock; pub const MS_RDONLY: u64 = 1 << 0; @@ -30,17 +34,21 @@ static MOUNT_CREATORS: Spin<BTreeMap<String, Arc<dyn MountCreator>>> = Spin::new static MOUNTS: Spin<Vec<(Arc<Dentry>, MountPointData)>> = Spin::new(vec![]); pub struct Mount { - _vfs: Arc<dyn Vfs>, + sb: SbUse<dyn SuperBlock>, root: Arc<Dentry>, } impl Mount { - pub fn new(mp: &Dentry, vfs: Arc<dyn Vfs>, root_inode: Arc<dyn Inode>) -> KResult<Self> { + pub fn new( + mp: &Dentry, + sb: SbUse<dyn SuperBlock>, + root_inode: InodeUse<dyn Inode>, + ) -> KResult<Self> { let root_dentry = Dentry::create(mp.parent().clone(), &mp.get_name()); - root_dentry.save_dir(root_inode)?; + root_dentry.fill(root_inode); Ok(Self { - _vfs: vfs, + sb, root: root_dentry, }) } @@ -53,9 +61,10 @@ impl Mount { unsafe impl Send for Mount {} unsafe impl Sync for Mount {} +#[async_trait] pub trait MountCreator: Send + Sync { fn check_signature(&self, first_block: &[u8]) -> KResult<bool>; - fn create_mount(&self, source: &str, flags: u64, mp: &Arc<Dentry>) -> KResult<Mount>; + async fn create_mount(&self, source: &str, flags: u64, mp: &Arc<Dentry>) -> KResult<Mount>; } pub fn register_filesystem(fstype: &str, creator: Arc<dyn MountCreator>) -> KResult<()> { @@ -77,7 +86,7 @@ struct MountPointData { flags: u64, } -pub fn do_mount( +pub async fn do_mount( mountpoint: &Arc<Dentry>, source: &str, mountpoint_str: &str, @@ -101,7 +110,7 @@ pub fn do_mount( let creators = { MOUNT_CREATORS.lock() }; creators.get(fstype).ok_or(ENODEV)?.clone() }; - let mount = creator.create_mount(source, flags, mountpoint)?; + let mount = creator.create_mount(source, flags, mountpoint).await?; let root_dentry = mount.root().clone(); @@ -165,8 +174,7 @@ impl Dentry { .cloned() .expect("tmpfs not registered."); - let mount = creator - .create_mount(&source, mount_flags, &DROOT) + let mount = block_on(creator.create_mount(&source, mount_flags, &DROOT)) .expect("Failed to create root mount."); let root_dentry = mount.root().clone(); diff --git a/src/kernel/vfs/superblock.rs b/src/kernel/vfs/superblock.rs new file mode 100644 index 00000000..85b28c01 --- /dev/null +++ b/src/kernel/vfs/superblock.rs @@ -0,0 +1,127 @@ +use core::{ + marker::Unsize, + ops::{CoerceUnsized, Deref}, +}; + +use alloc::sync::{Arc, Weak}; +use eonix_sync::RwLock; + +use crate::{kernel::constants::EIO, prelude::KResult}; + +use super::types::DeviceId; + +pub trait SuperBlock: Send + Sync + 'static {} + +#[derive(Debug, Clone)] +pub struct SuperBlockInfo { + pub io_blksize: u32, + pub device_id: DeviceId, + pub read_only: bool, +} + +pub struct SuperBlockLock(()); + +pub struct SuperBlockComplex<Backend> +where + Backend: SuperBlock + ?Sized, +{ + pub info: SuperBlockInfo, + pub rwsem: RwLock<SuperBlockLock>, + pub backend: Backend, +} + +pub struct SbRef<S>(Weak<SuperBlockComplex<S>>) +where + S: SuperBlock + ?Sized; + +pub struct SbUse<S>(Arc<SuperBlockComplex<S>>) +where + S: SuperBlock + ?Sized; + +impl<S> SbRef<S> +where + S: SuperBlock + ?Sized, +{ + pub fn try_get(&self) -> Option<SbUse<S>> { + self.0.upgrade().map(|arc| SbUse(arc)) + } + + pub fn get(&self) -> KResult<SbUse<S>> { + self.try_get().ok_or(EIO) + } + + pub fn from(sb: &SbUse<S>) -> Self { + SbRef(Arc::downgrade(&sb.0)) + } + + pub fn eq<U>(&self, other: &SbRef<U>) -> bool + where + U: SuperBlock + ?Sized, + { + core::ptr::addr_eq(self.0.as_ptr(), other.0.as_ptr()) + } +} + +impl<S> SbUse<S> +where + S: SuperBlock, +{ + pub fn new(info: SuperBlockInfo, backend: S) -> Self { + Self(Arc::new(SuperBlockComplex { + info, + rwsem: RwLock::new(SuperBlockLock(())), + backend, + })) + } + + pub fn new_cyclic(info: SuperBlockInfo, backend_func: impl FnOnce(SbRef<S>) -> S) -> Self { + Self(Arc::new_cyclic(|weak| SuperBlockComplex { + info, + rwsem: RwLock::new(SuperBlockLock(())), + backend: backend_func(SbRef(weak.clone())), + })) + } +} + +impl<S> Clone for SbRef<S> +where + S: SuperBlock + ?Sized, +{ + fn clone(&self) -> Self { + SbRef(self.0.clone()) + } +} + +impl<S> Clone for SbUse<S> +where + S: SuperBlock + ?Sized, +{ + fn clone(&self) -> Self { + SbUse(self.0.clone()) + } +} + +impl<T, U> CoerceUnsized<SbRef<U>> for SbRef<T> +where + T: SuperBlock + Unsize<U> + ?Sized, + U: SuperBlock + ?Sized, +{ +} + +impl<T, U> CoerceUnsized<SbUse<U>> for SbUse<T> +where + T: SuperBlock + Unsize<U> + ?Sized, + U: SuperBlock + ?Sized, +{ +} + +impl<S> Deref for SbUse<S> +where + S: SuperBlock + ?Sized, +{ + type Target = SuperBlockComplex<S>; + + fn deref(&self) -> &Self::Target { + self.0.deref() + } +} diff --git a/src/kernel/vfs/types/device_id.rs b/src/kernel/vfs/types/device_id.rs new file mode 100644 index 00000000..cf3ea886 --- /dev/null +++ b/src/kernel/vfs/types/device_id.rs @@ -0,0 +1,36 @@ +use core::fmt::{Debug, Display, Formatter}; + +#[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)] +pub struct DeviceId { + pub major: u16, + pub minor: u16, +} + +impl DeviceId { + pub const fn new(major: u16, minor: u16) -> Self { + Self { major, minor } + } + + pub const fn from_raw(raw: u32) -> Self { + Self { + major: (raw >> 16) as u16, + minor: (raw & 0xFFFF) as u16, + } + } + + pub const fn to_raw(self) -> u32 { + ((self.major as u32) << 16) | (self.minor as u32) + } +} + +impl Debug for DeviceId { + fn fmt(&self, f: &mut Formatter<'_>) -> core::fmt::Result { + write!(f, "DeviceId({:04x}:{:04x})", self.major, self.minor) + } +} + +impl Display for DeviceId { + fn fmt(&self, f: &mut Formatter<'_>) -> core::fmt::Result { + write!(f, "{:04x}:{:04x}", self.major, self.minor) + } +} diff --git a/src/kernel/vfs/types/mod.rs b/src/kernel/vfs/types/mod.rs new file mode 100644 index 00000000..4a7505f7 --- /dev/null +++ b/src/kernel/vfs/types/mod.rs @@ -0,0 +1,5 @@ +mod device_id; +mod mode; + +pub use device_id::DeviceId; +pub use mode::{Format, Mode, Permission}; diff --git a/src/kernel/vfs/types/mode.rs b/src/kernel/vfs/types/mode.rs new file mode 100644 index 00000000..dc1b88ec --- /dev/null +++ b/src/kernel/vfs/types/mode.rs @@ -0,0 +1,169 @@ +use crate::kernel::{ + constants::{S_IFBLK, S_IFCHR, S_IFDIR, S_IFLNK, S_IFMT, S_IFREG}, + syscall::{FromSyscallArg, SyscallRetVal}, +}; + +#[derive(Clone, Copy, PartialEq, Eq)] +pub struct Mode(u32); + +#[derive(Clone, Copy, PartialEq, Eq)] +pub enum Format { + REG, + DIR, + LNK, + BLK, + CHR, +} + +#[derive(Clone, Copy, PartialEq, Eq)] +pub struct Permission(u32); + +impl Mode { + pub const fn new(bits: u32) -> Self { + Self(bits) + } + + pub const fn is_blk(&self) -> bool { + (self.0 & S_IFMT) == S_IFBLK + } + + pub const fn is_chr(&self) -> bool { + (self.0 & S_IFMT) == S_IFCHR + } + + pub const fn bits(&self) -> u32 { + self.0 + } + + pub const fn format_bits(&self) -> u32 { + self.0 & S_IFMT + } + + pub const fn non_format_bits(&self) -> u32 { + self.0 & !S_IFMT + } + + pub fn format(&self) -> Format { + match self.format_bits() { + S_IFREG => Format::REG, + S_IFDIR => Format::DIR, + S_IFLNK => Format::LNK, + S_IFBLK => Format::BLK, + S_IFCHR => Format::CHR, + _ => panic!("unknown format bits: {:#o}", self.format_bits()), + } + } + + pub fn perm(&self) -> Permission { + Permission::new(self.non_format_bits()) + } + + pub const fn non_format(&self) -> Self { + Self::new(self.non_format_bits()) + } + + pub const fn set_perm(&mut self, perm: Permission) { + self.0 = self.format_bits() | perm.bits(); + } +} + +impl Format { + pub const fn as_raw(&self) -> u32 { + match self { + Self::REG => S_IFREG, + Self::DIR => S_IFDIR, + Self::LNK => S_IFLNK, + Self::BLK => S_IFBLK, + Self::CHR => S_IFCHR, + } + } +} + +impl Permission { + const RWX: [&str; 8] = ["---", "--x", "-w-", "-wx", "r--", "r-x", "rw-", "rwx"]; + + pub const fn new(perm_bits: u32) -> Self { + Self(perm_bits & 0o7777) + } + + pub const fn bits(&self) -> u32 { + self.0 + } + + pub const fn mask_with(&self, mask: Self) -> Self { + Self(self.0 & mask.0) + } +} + +impl core::fmt::Debug for Mode { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + match self.non_format_bits() & !0o777 { + 0 => write!( + f, + "Mode({format:?}, {perm:#o})", + format = self.format(), + perm = self.non_format_bits() + )?, + rem => write!( + f, + "Mode({format:?}, {perm:#o}, rem={rem:#x})", + format = self.format(), + perm = self.non_format_bits() & 0o777 + )?, + } + + Ok(()) + } +} + +impl core::fmt::Debug for Format { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + match self { + Self::REG => write!(f, "REG"), + Self::DIR => write!(f, "DIR"), + Self::LNK => write!(f, "LNK"), + Self::BLK => write!(f, "BLK"), + Self::CHR => write!(f, "CHR"), + } + } +} + +impl core::fmt::Debug for Permission { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + let owner = self.0 >> 6 & 0o7; + let group = self.0 >> 3 & 0o7; + let other = self.0 & 0o7; + + write!( + f, + "{}{}{}", + Self::RWX[owner as usize], + Self::RWX[group as usize], + Self::RWX[other as usize] + ) + } +} + +impl FromSyscallArg for Mode { + fn from_arg(value: usize) -> Self { + Mode::new(value as u32) + } +} + +impl SyscallRetVal for Mode { + fn into_retval(self) -> Option<usize> { + Some(self.bits() as usize) + } +} + +impl FromSyscallArg for Permission { + fn from_arg(value: usize) -> Self { + Permission::new(value as u32) + } +} + +impl SyscallRetVal for Permission { + fn into_retval(self) -> Option<usize> { + Some(self.bits() as usize) + } +} diff --git a/src/kernel/vfs/vfs.rs b/src/kernel/vfs/vfs.rs deleted file mode 100644 index ee66f0b6..00000000 --- a/src/kernel/vfs/vfs.rs +++ /dev/null @@ -1,10 +0,0 @@ -use crate::prelude::*; - -use super::DevId; - -#[allow(dead_code)] -pub trait Vfs: Send + Sync + AsAny { - fn io_blksize(&self) -> usize; - fn fs_devid(&self) -> DevId; - fn is_read_only(&self) -> bool; -} diff --git a/src/lib.rs b/src/lib.rs index 5df6a1e1..98e196f8 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -2,9 +2,12 @@ #![no_main] #![feature(allocator_api)] #![feature(c_size_t)] +#![feature(coerce_unsized)] +#![feature(concat_idents)] #![feature(arbitrary_self_types)] #![feature(get_mut_unchecked)] #![feature(macro_metavar_expr)] +#![feature(unsize)] extern crate alloc; @@ -45,8 +48,8 @@ use kernel::{ task::{KernelStack, ProcessBuilder, ProcessList, ProgramLoader, ThreadBuilder}, vfs::{ dentry::Dentry, - inode::Mode, mount::{do_mount, MS_NOATIME, MS_NODEV, MS_NOSUID, MS_RDONLY}, + types::Permission, FsContext, }, CharDevice, @@ -191,16 +194,16 @@ async fn init_process(early_kstack: PRange) { { // We might want the serial initialized as soon as possible. driver::serial::init().unwrap(); - driver::e1000e::register_e1000e_driver(); - driver::ahci::register_ahci_driver(); + driver::e1000e::register_e1000e_driver().await; + driver::ahci::register_ahci_driver().await; } #[cfg(target_arch = "riscv64")] { driver::serial::init().unwrap(); driver::virtio::init_virtio_devices(); - driver::e1000e::register_e1000e_driver(); - driver::ahci::register_ahci_driver(); + driver::e1000e::register_e1000e_driver().await; + driver::ahci::register_ahci_driver().await; driver::goldfish_rtc::probe(); } @@ -208,21 +211,26 @@ async fn init_process(early_kstack: PRange) { { driver::serial::init().unwrap(); driver::virtio::init_virtio_devices(); - driver::e1000e::register_e1000e_driver(); - driver::ahci::register_ahci_driver(); + driver::e1000e::register_e1000e_driver().await; + driver::ahci::register_ahci_driver().await; } fs::tmpfs::init(); - fs::procfs::init(); + fs::procfs::init().await; fs::fat32::init(); - fs::ext4::init(); + // fs::ext4::init(); let load_info = { // mount fat32 /mnt directory let fs_context = FsContext::global(); - let mnt_dir = Dentry::open(fs_context, Path::new(b"/mnt/").unwrap(), true).unwrap(); + let mnt_dir = Dentry::open(fs_context, Path::new(b"/mnt/").unwrap(), true) + .await + .unwrap(); - mnt_dir.mkdir(Mode::new(0o755)).unwrap(); + mnt_dir + .mkdir(Permission::new(0o755)) + .await + .expect("Failed to create /mnt directory"); do_mount( &mnt_dir, @@ -231,6 +239,7 @@ async fn init_process(early_kstack: PRange) { "fat32", MS_RDONLY | MS_NOATIME | MS_NODEV | MS_NOSUID, ) + .await .unwrap(); let init_names = [&b"/init"[..], &b"/sbin/init"[..], &b"/mnt/initsh"[..]]; @@ -238,7 +247,7 @@ async fn init_process(early_kstack: PRange) { let mut init_name = None; let mut init = None; for name in init_names { - if let Ok(dentry) = Dentry::open(fs_context, Path::new(name).unwrap(), true) { + if let Ok(dentry) = Dentry::open(fs_context, Path::new(name).unwrap(), true).await { if dentry.is_valid() { init_name = Some(CString::new(name).unwrap()); init = Some(dentry); @@ -260,6 +269,7 @@ async fn init_process(early_kstack: PRange) { ]; ProgramLoader::parse(fs_context, init_name, init.clone(), argv, envp) + .await .expect("Failed to parse init program") .load() .await diff --git a/src/prelude.rs b/src/prelude.rs index b3dbe2ce..880489da 100644 --- a/src/prelude.rs +++ b/src/prelude.rs @@ -18,34 +18,6 @@ pub(crate) use crate::kernel::console::{ pub(crate) use alloc::{boxed::Box, string::String, vec, vec::Vec}; -pub(crate) use core::{any::Any, fmt::Write, marker::PhantomData, str}; +pub(crate) use core::{fmt::Write, marker::PhantomData, str}; pub use crate::sync::Spin; - -#[allow(dead_code)] -pub trait AsAny: Send + Sync { - fn as_any(&self) -> &dyn Any; - fn as_any_mut(&mut self) -> &mut dyn Any; -} - -macro_rules! impl_any { - ($t:ty) => { - impl AsAny for $t { - fn as_any(&self) -> &dyn Any { - self - } - - fn as_any_mut(&mut self) -> &mut dyn Any { - self - } - } - }; -} - -macro_rules! addr_of_mut_field { - ($pointer:expr, $field:ident) => { - core::ptr::addr_of_mut!((*$pointer).$field) - }; -} - -pub(crate) use {addr_of_mut_field, impl_any}; From 572333f978cc67adbbc1817358489046da1d5a29 Mon Sep 17 00:00:00 2001 From: greatbridf <greatbridf@icloud.com> Date: Sun, 14 Sep 2025 21:46:19 +0800 Subject: [PATCH 14/36] vfs, rcu: rework path walking with new rcu syntax The old path walking algorithm requires recursion, which is not supported in async rust. So we boxed them all as a temporary solution in previous commits. This would introduce mass overhead even for fast path walks just because we might sleep in `readlink()` and `lookup()` calls. The new proposed method is to break the walk into several phases similar to that in Linux: RCU walk and REF walk. The RCU walk will hold the RCU lock and never blocks so the function itself can be non-async. If we hit non-present dentries, we will fallback to REF walk. In REF walks, we clone the Arcs and consult to the VFS layer for an accurate answer. Note that in both the two methods mentioned above, symlinks are not handled and will be returned directly with all path components left untouched. We have a dedicated async function to follow the symlinks by recursive calling the walk function. This can be slow and won't be called frequently. So we wrapped the function with `Box::pin()` to break the recursion chain. After the symlink resolution is done, we return to the original position and continue the walk. We found that the association of an inode to a dentry is one way. So the `data` RCUPointer field is actually unnecessary and we can use the atomic dentry type to sync readers with the writer. This way we can eliminate `DentryData` allocations and improve performance. We also introduced a new RCU read lock syntax. In the RCU walk mentioned above, we need to store dentry references protected by some RCU read lock. With the old syntax, we can't express the lifetime associtated by the common RCU read lock. The new syntax provides a `rcu_read_lock()` method to acquire the RCU read lock. The lock returned has a associated lifetime so we can use it in the RCU session. Signed-off-by: greatbridf <greatbridf@icloud.com> --- Cargo.lock | 5 + Cargo.toml | 8 +- crates/arcref/Cargo.lock | 7 + crates/arcref/Cargo.toml | 11 + crates/arcref/src/arcref.rs | 216 +++++++++++++++++ crates/arcref/src/lib.rs | 8 + crates/posix_types/src/result.rs | 6 + src/kernel/vfs/dentry.rs | 394 +++++++++++++------------------ src/kernel/vfs/dentry/dcache.rs | 76 +++++- src/kernel/vfs/dentry/walk.rs | 370 +++++++++++++++++++++++++++++ src/path.rs | 40 ++-- src/rcu.rs | 99 ++++---- 12 files changed, 942 insertions(+), 298 deletions(-) create mode 100644 crates/arcref/Cargo.lock create mode 100644 crates/arcref/Cargo.toml create mode 100644 crates/arcref/src/arcref.rs create mode 100644 crates/arcref/src/lib.rs create mode 100644 src/kernel/vfs/dentry/walk.rs diff --git a/Cargo.lock b/Cargo.lock index 32868677..f85a9d2f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -28,6 +28,10 @@ dependencies = [ "log", ] +[[package]] +name = "arcref" +version = "0.1.0" + [[package]] name = "async-trait" version = "0.1.89" @@ -155,6 +159,7 @@ dependencies = [ "acpi", "align_ext", "another_ext4", + "arcref", "async-trait", "atomic_unique_refcell", "bitflags", diff --git a/Cargo.toml b/Cargo.toml index bc7e7b0c..5158025e 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -7,6 +7,9 @@ edition = "2021" crate-type = ["bin"] [dependencies] +arcref = { path = "./crates/arcref", default-features = false, features = [ + "alloc", +] } atomic_unique_refcell = { path = "./crates/atomic_unique_refcell", features = [ "no_std", ] } @@ -35,7 +38,10 @@ stalloc = { version = "0.6.1", default-features = false, features = [ "allocator-api", ] } async-trait = "0.1.89" -futures = { version = "0.3.31", features = ["alloc", "async-await"], default-features = false } +futures = { version = "0.3.31", features = [ + "alloc", + "async-await", +], default-features = false } [target.'cfg(any(target_arch = "riscv64", target_arch = "loongarch64"))'.dependencies] virtio-drivers = { version = "0.11.0" } diff --git a/crates/arcref/Cargo.lock b/crates/arcref/Cargo.lock new file mode 100644 index 00000000..3c4e1567 --- /dev/null +++ b/crates/arcref/Cargo.lock @@ -0,0 +1,7 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "arcref" +version = "0.1.0" diff --git a/crates/arcref/Cargo.toml b/crates/arcref/Cargo.toml new file mode 100644 index 00000000..a0af89f8 --- /dev/null +++ b/crates/arcref/Cargo.toml @@ -0,0 +1,11 @@ +[package] +name = "arcref" +version = "0.1.0" +edition = "2024" + +[dependencies] + +[features] +alloc = [] +std = ["alloc"] +default = ["std"] diff --git a/crates/arcref/src/arcref.rs b/crates/arcref/src/arcref.rs new file mode 100644 index 00000000..3d01852d --- /dev/null +++ b/crates/arcref/src/arcref.rs @@ -0,0 +1,216 @@ +#[cfg(not(feature = "std"))] +use core::{ + borrow::Borrow, + marker::{PhantomData, Unsize}, + mem::ManuallyDrop, + ops::{Deref, DispatchFromDyn}, +}; + +#[cfg(all(not(feature = "std"), feature = "alloc"))] +extern crate alloc; + +#[cfg(all(not(feature = "std"), feature = "alloc"))] +use alloc::sync::Arc; + +#[cfg(feature = "std")] +use std::{ + borrow::Borrow, + marker::{PhantomData, Unsize}, + mem::ManuallyDrop, + ops::{Deref, DispatchFromDyn}, + sync::Arc, +}; + +pub trait AsArcRef<T> +where + T: ?Sized, +{ + /// Borrow the [`Arc`] and convert the reference into [`ArcRef`]. + fn aref(&self) -> ArcRef<'_, T>; +} + +pub struct ArcRef<'a, T: ?Sized> { + ptr: *const T, + _phantom: PhantomData<&'a ()>, +} + +unsafe impl<T: ?Sized + Send + Sync> Send for ArcRef<'_, T> {} +unsafe impl<T: ?Sized + Send + Sync> Sync for ArcRef<'_, T> {} + +#[cfg(any(feature = "std", feature = "alloc"))] +impl<'a, T: ?Sized> ArcRef<'a, T> { + pub fn new(arc: &'a Arc<T>) -> Self { + Self { + ptr: Arc::as_ptr(arc), + _phantom: PhantomData, + } + } + + /// Create a new `ArcRef` from a raw pointer. + /// + /// # Safety + /// The given pointer MUST be created by `Arc::as_ptr` or `Arc::into_raw`. + /// The caller is responsible to ensure that the pointer is valid for the + /// lifetime of the `ArcRef`. + pub unsafe fn new_unchecked(arc_ptr: *const T) -> Self { + Self { + ptr: arc_ptr, + _phantom: PhantomData, + } + } + + pub fn with_arc<Func, Out>(self, func: Func) -> Out + where + Func: FnOnce(&Arc<T>) -> Out, + { + func(&ManuallyDrop::new(unsafe { Arc::from_raw(self.ptr) })) + } + + pub fn clone_arc(self) -> Arc<T> { + self.with_arc(|arc| arc.clone()) + } + + pub fn ptr_eq_arc(self, other: &Arc<T>) -> bool { + self.with_arc(|arc| Arc::ptr_eq(arc, other)) + } +} + +#[cfg(all(not(feature = "std"), feature = "alloc"))] +impl<T> AsArcRef<T> for Arc<T> +where + T: ?Sized, +{ + fn aref(&self) -> ArcRef<'_, T> { + ArcRef::new(self) + } +} + +impl<T> AsRef<T> for ArcRef<'_, T> +where + T: ?Sized, +{ + fn as_ref(&self) -> &T { + self.deref() + } +} + +impl<T> Borrow<T> for ArcRef<'_, T> +where + T: ?Sized, +{ + fn borrow(&self) -> &T { + self.deref() + } +} + +impl<'a, T> Clone for ArcRef<'a, T> +where + T: ?Sized, +{ + fn clone(&self) -> Self { + Self { + ptr: self.ptr, + _phantom: PhantomData, + } + } +} + +impl<T> Copy for ArcRef<'_, T> where T: ?Sized {} + +impl<T: ?Sized> Deref for ArcRef<'_, T> { + type Target = T; + + fn deref(&self) -> &T { + unsafe { + // SAFETY: `self.ptr` points to a valid `T` instance because it was + // created from a valid `Arc<T>`. + self.ptr.as_ref().unwrap_unchecked() + } + } +} + +impl<'a, T, U> DispatchFromDyn<ArcRef<'a, U>> for ArcRef<'a, T> +where + T: ?Sized + Unsize<U>, + U: ?Sized, +{ +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn create_from_arc() { + let data = Arc::new(42); + let _arc_ref = ArcRef::new(&data); + } + + #[test] + fn deref() { + let data = Arc::new(42); + let arc_ref = ArcRef::new(&data); + + assert_eq!(*arc_ref, 42); + } + + #[test] + fn clone_into_arc() { + let data = Arc::new(42); + let arc_ref = ArcRef::new(&data); + + let cloned = arc_ref.clone_arc(); + + assert_eq!(Arc::strong_count(&data), 2); + assert_eq!(*cloned, 42); + } + + #[test] + fn dyn_compatible_receiver() { + struct Data(u32); + + trait Trait { + fn foo(self: ArcRef<Self>) -> u32; + } + + impl Trait for Data { + fn foo(self: ArcRef<Self>) -> u32 { + self.0 + } + } + + let data = Arc::new(Data(42)); + let arc_ref = ArcRef::new(&data); + + assert_eq!(arc_ref.foo(), 42); + } + + #[test] + fn clone_from_train_methods() { + struct Data(u32); + + trait Trait { + fn foo(&self) -> u32; + + fn clone_self(self: ArcRef<Self>) -> Arc<dyn Trait>; + } + + impl Trait for Data { + fn foo(&self) -> u32 { + self.0 + } + + fn clone_self(self: ArcRef<Self>) -> Arc<dyn Trait> { + self.clone_arc() as _ + } + } + + let data = Arc::new(Data(42)); + let arc_ref = ArcRef::new(&data); + + let cloned = arc_ref.clone_self(); + + assert_eq!(arc_ref.foo(), 42); + assert_eq!(cloned.foo(), 42); + } +} diff --git a/crates/arcref/src/lib.rs b/crates/arcref/src/lib.rs new file mode 100644 index 00000000..83a61985 --- /dev/null +++ b/crates/arcref/src/lib.rs @@ -0,0 +1,8 @@ +#![cfg_attr(not(feature = "std"), no_std)] +#![feature(arbitrary_self_types)] +#![feature(dispatch_from_dyn)] +#![feature(unsize)] + +mod arcref; + +pub use arcref::{ArcRef, AsArcRef}; diff --git a/crates/posix_types/src/result.rs b/crates/posix_types/src/result.rs index a10ff0ad..1535c444 100644 --- a/crates/posix_types/src/result.rs +++ b/crates/posix_types/src/result.rs @@ -1,14 +1,18 @@ pub enum PosixError { + ENOENT = 2, EFAULT = 14, EXDEV = 18, + ENOTDIR = 20, EINVAL = 22, } impl From<PosixError> for u32 { fn from(error: PosixError) -> Self { match error { + PosixError::ENOENT => 2, PosixError::EFAULT => 14, PosixError::EXDEV => 18, + PosixError::ENOTDIR => 20, PosixError::EINVAL => 22, } } @@ -17,8 +21,10 @@ impl From<PosixError> for u32 { impl core::fmt::Debug for PosixError { fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { match self { + Self::ENOENT => write!(f, "ENOENT"), Self::EFAULT => write!(f, "EFAULT"), Self::EXDEV => write!(f, "EXDEV"), + Self::ENOTDIR => write!(f, "ENOTDIR"), Self::EINVAL => write!(f, "EINVAL"), } } diff --git a/src/kernel/vfs/dentry.rs b/src/kernel/vfs/dentry.rs index 5ac4e407..c1eb8cb8 100644 --- a/src/kernel/vfs/dentry.rs +++ b/src/kernel/vfs/dentry.rs @@ -1,45 +1,62 @@ pub mod dcache; +mod walk; -use super::{ - inode::{Ino, Inode, InodeUse, RenameData, WriteOffset}, - types::{DeviceId, Format, Mode, Permission}, - FsContext, -}; -use crate::{ - hash::KernelHasher, - io::{Buffer, ByteBuffer}, - kernel::{block::BlockDevice, CharDevice}, - path::{Path, PathComponent}, - prelude::*, - rcu::{RCUNode, RCUPointer, RCUReadGuard}, -}; -use crate::{ - io::Stream, - kernel::constants::{EEXIST, EINVAL, EISDIR, ELOOP, ENOENT, ENOTDIR, EPERM, ERANGE}, -}; -use alloc::sync::Arc; use core::{ + cell::UnsafeCell, fmt, - future::Future, hash::{BuildHasher, BuildHasherDefault, Hasher}, - pin::Pin, - sync::atomic::{AtomicPtr, AtomicU64, Ordering}, + sync::atomic::{AtomicPtr, AtomicU64, AtomicU8, Ordering}, }; + +use alloc::sync::Arc; +use arcref::AsArcRef; use eonix_sync::LazyLock; use pointers::BorrowedArc; use posix_types::{namei::RenameFlags, open::OpenFlags, result::PosixError, stat::StatX}; -#[derive(PartialEq, Eq)] +use crate::{ + hash::KernelHasher, + io::Buffer, + io::Stream, + kernel::constants::{EEXIST, EINVAL, EISDIR, ELOOP, ENOENT, EPERM, ERANGE}, + kernel::{block::BlockDevice, CharDevice}, + path::Path, + prelude::*, + rcu::{rcu_read_lock, RCUNode, RCUPointer, RCUReadGuard}, +}; + +use super::{ + inode::{Ino, Inode, InodeUse, RenameData, WriteOffset}, + types::{DeviceId, Format, Mode, Permission}, + FsContext, +}; + +const D_INVALID: u8 = 0; +const D_REGULAR: u8 = 1; +const D_DIRECTORY: u8 = 2; +const D_SYMLINK: u8 = 3; + +#[derive(Debug, PartialEq, Eq)] enum DentryKind { - Regular, - Directory, - Symlink, - Mountpoint, + Regular = D_REGULAR as isize, + Directory = D_DIRECTORY as isize, + Symlink = D_SYMLINK as isize, } -struct DentryData { - inode: InodeUse<dyn Inode>, - kind: DentryKind, +/// The [`Inode`] associated with a [`Dentry`]. +/// +/// We could assign an inode to a negative dentry exactly once when the dentry +/// is invalid and we create a file or directory on it, or the dentry is brought +/// to the dcache by [lookup()]. +/// +/// This guarantees that as long as we acquire a non-invalid from [`Self::kind`], +/// we are synced with the writer and can safely read the [`Self::inode`] field +/// without reading torn data. +/// +/// [lookup()]: crate::kernel::vfs::inode::InodeDirOps::lookup +struct AssociatedInode { + kind: UnsafeCell<Option<DentryKind>>, + inode: UnsafeCell<Option<InodeUse<dyn Inode>>>, } /// # Safety @@ -58,8 +75,7 @@ pub struct Dentry { prev: AtomicPtr<Dentry>, next: AtomicPtr<Dentry>, - // RCU Mutable - data: RCUPointer<DentryData>, + inode: AssociatedInode, } pub(super) static DROOT: LazyLock<Arc<Dentry>> = LazyLock::new(|| { @@ -69,7 +85,7 @@ pub(super) static DROOT: LazyLock<Arc<Dentry>> = LazyLock::new(|| { hash: AtomicU64::new(0), prev: AtomicPtr::default(), next: AtomicPtr::default(), - data: RCUPointer::empty(), + inode: AssociatedInode::new(), }); unsafe { @@ -119,50 +135,19 @@ impl Dentry { self.hash.store(hash, Ordering::Relaxed); } - - async fn find(self: &Arc<Self>, name: &[u8]) -> KResult<Arc<Self>> { - let data = self.data.load(); - let data = data.as_ref().ok_or(ENOENT)?; - - if data.kind != DentryKind::Directory { - return Err(ENOTDIR); - } - - match name { - b"." => Ok(self.clone()), - b".." => Ok(self.parent().clone()), - _ => { - let dentry = Dentry::create(self.clone(), name); - - if let Some(found) = dcache::d_find_fast(&dentry) { - unsafe { - // SAFETY: This is safe because the dentry is never shared with - // others so we can drop them safely. - let _ = dentry.name.swap(None); - let _ = dentry.parent.swap(None); - } - - return Ok(found); - } - - let _ = dcache::d_try_revalidate(&dentry).await; - dcache::d_add(dentry.clone()); - - Ok(dentry) - } - } - } } impl Dentry { pub fn create(parent: Arc<Dentry>, name: &[u8]) -> Arc<Self> { + // TODO!!!: don't acquire our parent's refcount here... + let val = Arc::new(Self { parent: RCUPointer::new(parent), name: RCUPointer::new(Arc::new(Arc::from(name))), hash: AtomicU64::new(0), prev: AtomicPtr::default(), next: AtomicPtr::default(), - data: RCUPointer::empty(), + inode: AssociatedInode::new(), }); val.rehash(); @@ -196,27 +181,12 @@ impl Dentry { .map_or(core::ptr::null(), |parent| Arc::as_ptr(&parent)) } - fn save(&self, inode: InodeUse<dyn Inode>, kind: DentryKind) { - let new = DentryData { inode, kind }; - - // TODO!!!: We don't actually need to use `RCUPointer` here - // Safety: this function may only be called from `create`-like functions which requires the - // superblock's write locks to be held, so only one creation can happen at a time and we - // can't get a reference to the old data. - let old = unsafe { self.data.swap(Some(Arc::new(new))) }; - assert!(old.is_none()); - } - pub fn fill(&self, file: InodeUse<dyn Inode>) { - match file.format() { - Format::REG | Format::BLK | Format::CHR => self.save(file, DentryKind::Regular), - Format::DIR => self.save(file, DentryKind::Directory), - Format::LNK => self.save(file, DentryKind::Symlink), - } + self.inode.store(file); } pub fn inode(&self) -> Option<InodeUse<dyn Inode>> { - self.data.load().as_ref().map(|data| data.inode.clone()) + self.inode.load().map(|(_, inode)| inode.clone()) } pub fn get_inode(&self) -> KResult<InodeUse<dyn Inode>> { @@ -224,181 +194,85 @@ impl Dentry { } pub fn is_directory(&self) -> bool { - let data = self.data.load(); - data.as_ref() - .map_or(false, |data| data.kind == DentryKind::Directory) + self.inode + .load() + .map_or(false, |(kind, _)| kind == DentryKind::Directory) } pub fn is_valid(&self) -> bool { - self.data.load().is_some() + self.inode.load().is_some() } pub async fn open_check(self: &Arc<Self>, flags: OpenFlags, perm: Permission) -> KResult<()> { - let data = self.data.load(); - - if data.is_some() { - if flags.contains(OpenFlags::O_CREAT | OpenFlags::O_EXCL) { - Err(EEXIST) - } else { - Ok(()) - } - } else { - if !flags.contains(OpenFlags::O_CREAT) { - return Err(ENOENT); - } - - let parent = self.parent().get_inode()?; - parent.create(self, perm).await - } - } -} - -impl Dentry { - fn resolve_directory( - context: &FsContext, - dentry: Arc<Self>, - nrecur: u32, - ) -> Pin<Box<impl Future<Output = KResult<Arc<Self>>> + use<'_>>> { - Box::pin(async move { - if nrecur >= 16 { - return Err(ELOOP); - } - - let data = dentry.data.load(); - let data = data.as_ref().ok_or(ENOENT)?; - - match data.kind { - DentryKind::Regular => Err(ENOTDIR), - DentryKind::Directory => Ok(dentry), - DentryKind::Symlink => { - let mut buffer = [0u8; 256]; - let mut buffer = ByteBuffer::new(&mut buffer); - - data.inode.readlink(&mut buffer).await?; - let path = Path::new(buffer.data())?; - - let dentry = - Self::open_recursive(context, &dentry.parent(), path, true, nrecur + 1) - .await?; - - Self::resolve_directory(context, dentry, nrecur + 1).await + match self.inode.load() { + Some(_) => { + if flags.contains(OpenFlags::O_CREAT | OpenFlags::O_EXCL) { + Err(EEXIST) + } else { + Ok(()) } - _ => panic!("Invalid dentry flags"), - } - }) - } - - pub fn open_recursive<'r, 'a: 'r, 'b: 'r, 'c: 'r>( - context: &'a FsContext, - cwd: &'b Arc<Self>, - path: Path<'c>, - follow: bool, - nrecur: u32, - ) -> Pin<Box<impl Future<Output = KResult<Arc<Self>>> + 'r>> { - Box::pin(async move { - // too many recursive search layers will cause stack overflow - // so we use 16 for now - if nrecur >= 16 { - return Err(ELOOP); } - - let mut cwd = if path.is_absolute() { - context.fsroot.clone() - } else { - cwd.clone() - }; - - for item in path.iter() { - if let PathComponent::TrailingEmpty = item { - if cwd.data.load().as_ref().is_none() { - return Ok(cwd); - } + None => { + if !flags.contains(OpenFlags::O_CREAT) { + return Err(ENOENT); } - cwd = Self::resolve_directory(context, cwd, nrecur).await?; - - match item { - PathComponent::TrailingEmpty | PathComponent::Current => {} // pass - PathComponent::Parent => { - if !cwd.hash_eq(&context.fsroot) { - let parent = cwd.parent().clone(); - cwd = Self::resolve_directory(context, parent, nrecur).await?; - } - continue; - } - PathComponent::Name(name) => { - cwd = cwd.find(name).await?; - } - } + let parent = self.parent().get_inode()?; + parent.create(self, perm).await } - - if follow { - let data = cwd.data.load(); - - if let Some(data) = data.as_ref() { - if data.kind == DentryKind::Symlink { - let data = cwd.data.load(); - let data = data.as_ref().unwrap(); - let mut buffer = [0u8; 256]; - let mut buffer = ByteBuffer::new(&mut buffer); - - data.inode.readlink(&mut buffer).await?; - let path = Path::new(buffer.data())?; - - let parent = cwd.parent().clone(); - cwd = - Self::open_recursive(context, &parent, path, true, nrecur + 1).await?; - } - } - } - - Ok(cwd) - }) + } } +} +impl Dentry { pub async fn open( context: &FsContext, - path: Path<'_>, + path: &Path, follow_symlinks: bool, ) -> KResult<Arc<Self>> { let cwd = context.cwd.lock().clone(); - Dentry::open_recursive(context, &cwd, path, follow_symlinks, 0).await + Self::open_at(context, &cwd, path, follow_symlinks).await } pub async fn open_at( context: &FsContext, at: &Arc<Self>, - path: Path<'_>, + path: &Path, follow_symlinks: bool, ) -> KResult<Arc<Self>> { - Dentry::open_recursive(context, at, path, follow_symlinks, 0).await - } + let mut found = context.start_recursive_walk(at, path).await?; - pub fn get_path( - self: &Arc<Dentry>, - context: &FsContext, - buffer: &mut dyn Buffer, - ) -> KResult<()> { - let locked_parent = self.parent(); + if !follow_symlinks { + return Ok(found); + } - let path = { - let mut path = vec![]; + loop { + match found.inode.load() { + Some((DentryKind::Symlink, inode)) => { + found = context.follow_symlink(found.aref(), inode, 0).await?; + } + _ => return Ok(found), + } + } + } - let mut parent = locked_parent.borrow(); - let mut dentry = BorrowedArc::new(self); + pub fn get_path(self: &Arc<Self>, context: &FsContext, buffer: &mut dyn Buffer) -> KResult<()> { + let rcu_read = rcu_read_lock(); - while Arc::as_ptr(&dentry) != Arc::as_ptr(&context.fsroot) { - if path.len() > 32 { - return Err(ELOOP); - } + let mut path = vec![]; + + let mut current = self.aref(); + let mut parent = self.parent.dereference(&rcu_read).unwrap(); - path.push(dentry.name().clone()); - dentry = parent; - parent = dentry.parent.load_protected(&locked_parent).unwrap(); + while !current.ptr_eq_arc(&context.fsroot) { + if path.len() > 32 { + return Err(ELOOP); } - path - }; + path.push(current.name.dereference(&rcu_read).unwrap()); + current = parent; + parent = current.parent.dereference(&rcu_read).unwrap(); + } buffer.fill(b"/")?.ok_or(ERANGE)?; for item in path.iter().rev().map(|name| name.as_ref()) { @@ -533,3 +407,71 @@ impl Dentry { old_parent.rename(rename_data).await } } + +impl DentryKind { + fn into_raw(self) -> u8 { + unsafe { core::mem::transmute(self) } + } + + fn from_raw(raw: u8) -> Option<Self> { + unsafe { core::mem::transmute(raw) } + } + + fn as_atomic(me: &UnsafeCell<Option<Self>>) -> &AtomicU8 { + unsafe { AtomicU8::from_ptr(me.get().cast()) } + } + + fn atomic_acq(me: &UnsafeCell<Option<Self>>) -> Option<Self> { + Self::from_raw(Self::as_atomic(me).load(Ordering::Acquire)) + } + + fn atomic_swap_acqrel(me: &UnsafeCell<Option<Self>>, kind: Option<Self>) -> Option<Self> { + Self::from_raw(Self::as_atomic(me).swap(kind.map_or(0, Self::into_raw), Ordering::AcqRel)) + } +} + +impl AssociatedInode { + fn new() -> Self { + Self { + inode: UnsafeCell::new(None), + kind: UnsafeCell::new(None), + } + } + + fn store(&self, inode: InodeUse<dyn Inode>) { + let kind = match inode.format() { + Format::REG | Format::BLK | Format::CHR => DentryKind::Regular, + Format::DIR => DentryKind::Directory, + Format::LNK => DentryKind::Symlink, + }; + + unsafe { + // SAFETY: We should be the first and only one to store the inode as + // is checked below. All other readers reading non-invalid + // kind will see the fully written inode. + self.inode.get().write(Some(inode)); + } + + assert_eq!( + DentryKind::atomic_swap_acqrel(&self.kind, Some(kind)), + None, + "Dentry can only be stored once." + ); + } + + fn kind(&self) -> Option<DentryKind> { + DentryKind::atomic_acq(&self.kind) + } + + fn load(&self) -> Option<(DentryKind, &InodeUse<dyn Inode>)> { + self.kind().map(|kind| unsafe { + let inode = (&*self.inode.get()) + .as_ref() + .expect("Dentry with non-invalid kind has no inode"); + (kind, inode) + }) + } +} + +unsafe impl Send for AssociatedInode {} +unsafe impl Sync for AssociatedInode {} diff --git a/src/kernel/vfs/dentry/dcache.rs b/src/kernel/vfs/dentry/dcache.rs index e2491235..ee7503dc 100644 --- a/src/kernel/vfs/dentry/dcache.rs +++ b/src/kernel/vfs/dentry/dcache.rs @@ -1,11 +1,13 @@ use super::Dentry; use crate::kernel::constants::ENOENT; -use crate::rcu::RCUPointer; +use crate::rcu::{RCUPointer, RCUReadLock}; use crate::{ prelude::*, rcu::{RCUIterator, RCUList}, }; use alloc::sync::Arc; +use arcref::ArcRef; +use core::ops::Deref; use core::sync::atomic::Ordering; use eonix_sync::Mutex; @@ -16,26 +18,47 @@ static DCACHE: [RCUList<Dentry>; 1 << DCACHE_HASH_BITS] = static D_EXCHANGE_LOCK: Mutex<()> = Mutex::new(()); -pub fn d_hinted(dentry: &Dentry) -> &'static RCUList<Dentry> { - let hash = dentry.hash.load(Ordering::Relaxed) as usize & ((1 << DCACHE_HASH_BITS) - 1); +pub trait DCacheItem { + fn d_hash(&self) -> usize; + fn d_parent(&self) -> *const Dentry; + fn d_name<'r, 'a: 'r, 'b: 'a>( + &'a self, + rcu_read: &'b RCUReadLock, + ) -> impl Deref<Target = [u8]> + 'r; +} + +fn d_eq(lhs: &impl DCacheItem, rhs: &impl DCacheItem, rcu_read: &RCUReadLock) -> bool { + lhs.d_hash() == rhs.d_hash() + && lhs.d_parent() == rhs.d_parent() + && *lhs.d_name(rcu_read) == *rhs.d_name(rcu_read) +} + +fn d_hinted(item: &impl DCacheItem) -> &'static RCUList<Dentry> { + let hash = item.d_hash() & ((1 << DCACHE_HASH_BITS) - 1); &DCACHE[hash] } -pub fn d_iter_for(dentry: &Dentry) -> RCUIterator<'static, Dentry> { - d_hinted(dentry).iter() +fn d_iter_for<'rcu>( + item: &impl DCacheItem, + rcu_read: &'rcu RCUReadLock, +) -> RCUIterator<'static, 'rcu, Dentry> { + d_hinted(item).iter(rcu_read) +} + +pub fn d_find_rcu<'rcu>( + item: &impl DCacheItem, + rcu_read: &'rcu RCUReadLock, +) -> Option<ArcRef<'rcu, Dentry>> { + d_iter_for(item, rcu_read).find(|cur_ref| cur_ref.with_arc(|cur| d_eq(cur, item, rcu_read))) } /// Add the dentry to the dcache pub fn d_add(dentry: Arc<Dentry>) { + // TODO: Add `children` field to parent and lock parent dentry to avoid + // concurrent insertion causing duplication. d_hinted(&dentry).insert(dentry); } -pub fn d_find_fast(dentry: &Dentry) -> Option<Arc<Dentry>> { - d_iter_for(dentry) - .find(|cur| cur.hash_eq(dentry)) - .map(|dentry| dentry.clone()) -} - /// Call `lookup()` on the parent inode to try find if the dentry points to a valid inode /// /// Silently fail without any side effects @@ -80,3 +103,34 @@ pub async fn d_exchange(old: &Arc<Dentry>, new: &Arc<Dentry>) { d_add(old.clone()); d_add(new.clone()); } + +impl DCacheItem for Arc<Dentry> { + fn d_hash(&self) -> usize { + self.hash.load(Ordering::Relaxed) as usize + } + + fn d_parent(&self) -> *const Dentry { + self.parent_addr() + } + + fn d_name<'r, 'a: 'r, 'b: 'a>( + &'a self, + rcu_read: &'b RCUReadLock, + ) -> impl Deref<Target = [u8]> + 'r { + struct Name<'a>(ArcRef<'a, Arc<[u8]>>); + + impl Deref for Name<'_> { + type Target = [u8]; + + fn deref(&self) -> &Self::Target { + &self.0 + } + } + + Name( + self.name + .dereference(rcu_read) + .expect("Dentry should have a non-null name"), + ) + } +} diff --git a/src/kernel/vfs/dentry/walk.rs b/src/kernel/vfs/dentry/walk.rs new file mode 100644 index 00000000..3e401b4b --- /dev/null +++ b/src/kernel/vfs/dentry/walk.rs @@ -0,0 +1,370 @@ +use core::{ + future::Future, + hash::{BuildHasher, BuildHasherDefault, Hasher}, + ops::Deref, + pin::Pin, +}; + +use alloc::{boxed::Box, sync::Arc}; +use arcref::{ArcRef, AsArcRef}; +use posix_types::result::PosixError; + +use crate::{ + hash::KernelHasher, + io::ByteBuffer, + kernel::{ + constants::ELOOP, + vfs::{ + inode::{Inode, InodeUse}, + FsContext, + }, + }, + path::{Path, PathComponent, PathIterator}, + prelude::KResult, + rcu::{rcu_read_lock, RCUReadLock}, +}; + +use super::{ + dcache::{self, DCacheItem}, + Dentry, DentryKind, +}; + +struct DentryFind<'a, 'b> { + parent: &'a Dentry, + name: &'b [u8], + hash: usize, +} + +pub enum WalkResultRcu<'rcu, 'path> { + Err(PosixError), + Ok(ArcRef<'rcu, Dentry>), + Symlink { + symlink: ArcRef<'rcu, Dentry>, + inode: InodeUse<dyn Inode>, + }, + Miss { + parent: ArcRef<'rcu, Dentry>, + name: &'path [u8], + }, +} + +pub enum WalkResult { + Err(PosixError), + Ok(Arc<Dentry>), + Symlink { + symlink: Arc<Dentry>, + inode: InodeUse<dyn Inode>, + }, +} + +impl Dentry { + /// Quick path of the dentry find operation. + /// + /// Check invalid and non-directory dentries, return immediately on dot and + /// dotdot component, and do a quick rcu dcache lookup. + /// + /// Note that while `Some(dentry)` guarantees present and valid dentry, + /// returning `None` is acceptable if the actual file exists but is not in + /// the dentry cache. If so, we should check again with `lookup`. + fn find_rcu<'r, 's: 'r>( + self: ArcRef<'s, Self>, + name: &[u8], + rcu_read: &'r RCUReadLock, + ) -> Result<Option<ArcRef<'r, Self>>, PosixError> { + match self.inode.load() { + Some((DentryKind::Directory, _)) => {} + Some(_) => return Err(PosixError::ENOTDIR), + None => return Err(PosixError::ENOENT), + } + + match name { + b"." => Ok(Some(self)), + b".." => Ok(Some( + self.parent + .dereference(rcu_read) + .expect("The field `parent` should be non-null"), + )), + _ => { + let dentry_find = DentryFind::new(&self, name); + Ok(dcache::d_find_rcu(&dentry_find, rcu_read)) + } + } + } + + async fn find_slow(self: &Arc<Self>, name: &[u8]) -> Result<Arc<Self>, PosixError> { + let dentry = Dentry::create(self.clone(), name); + + let _ = dcache::d_try_revalidate(&dentry).await; + dcache::d_add(dentry.clone()); + + Ok(dentry) + } + + pub async fn find_full(self: &Arc<Self>, name: &[u8]) -> Result<Arc<Self>, PosixError> { + if let Some(dentry) = self.aref().find_rcu(name, &rcu_read_lock())? { + return Ok(dentry.clone_arc()); + } + + self.find_slow(name).await + } +} + +impl FsContext { + /// Walk the pathname and try to find the corresponding dentry FAST without + /// consulting the VFS for invalid dentries encountered. + fn walk_rcu<'rcu, 'path>( + &self, + mut current: ArcRef<'rcu, Dentry>, + iter: &mut PathIterator<'path>, + rcu_read: &'rcu RCUReadLock, + ) -> WalkResultRcu<'rcu, 'path> { + use PathComponent::*; + + loop { + let inode = current.inode.load(); + + if iter.is_empty() { + break; + } + + // Skip symlink resolution in rcu walk without consuming the iter. + if let Some((DentryKind::Symlink, inode)) = inode { + return WalkResultRcu::Symlink { + symlink: current, + inode: inode.clone(), + }; + } + + let Some(component) = iter.next() else { + break; + }; + + match (inode, component) { + // Skip trailing empty and dot for normal directories. + (Some((DentryKind::Directory, _)), TrailingEmpty | Current) => {} + // Walk to parent directory unless we are at the filesystem root. + (Some((DentryKind::Directory, _)), Parent) => { + if current.ptr_eq_arc(&self.fsroot) { + continue; + } + + current = current + .parent + .dereference(&rcu_read) + .expect("parent should exist"); + } + // Normal directory traversal + (Some((DentryKind::Directory, _)), Name(name)) => { + match current.find_rcu(name, &rcu_read) { + Err(err) => return WalkResultRcu::Err(err), + Ok(Some(found)) => { + current = found; + } + Ok(None) => { + return WalkResultRcu::Miss { + name, + parent: current, + }; + } + } + } + // Not a directory, fail and exit. + (Some(_), _) => return WalkResultRcu::Err(PosixError::ENOTDIR), + // Return invalid trailing entries directly. + (None, TrailingEmpty) => return WalkResultRcu::Ok(current), + // Invalid intermediate entries are not acceptable. + (None, _) => return WalkResultRcu::Err(PosixError::ENOENT), + } + } + + WalkResultRcu::Ok(current) + } + + /// Walk the pathname slowly with refcounts held and VFS lookups. + async fn walk_slow(&self, mut current: Arc<Dentry>, iter: &mut PathIterator<'_>) -> WalkResult { + use PathComponent::*; + + loop { + // `current` should be the parent directory and `component` is the + // next path component we are stepping into. + + if iter.is_empty() { + break; + } + + if let Some((DentryKind::Symlink, inode)) = current.inode.load() { + return WalkResult::Symlink { + inode: inode.clone(), + symlink: current, + }; + } + + let Some(component) = iter.next() else { + break; + }; + + match (current.inode.load(), &component) { + // Normal directory traversal + (Some((DentryKind::Directory, _)), _) => {} + // Not a directory, fail and exit. + (Some(_), _) => return WalkResult::Err(PosixError::ENOTDIR), + // Return invalid trailing entries directly. + (None, TrailingEmpty) => return WalkResult::Ok(current), + // Invalid intermediate entries are not acceptable. + (None, _) => return WalkResult::Err(PosixError::ENOENT), + } + + match component { + PathComponent::TrailingEmpty => {} + PathComponent::Current => {} + PathComponent::Parent => { + if current.hash_eq(&self.fsroot) { + continue; + } + + let parent = current.parent().clone(); + current = parent; + } + PathComponent::Name(name) => { + match current.find_full(name).await { + Ok(found) => current = found, + Err(err) => return WalkResult::Err(err), + }; + } + } + } + + WalkResult::Ok(current) + } + + /// Walk the pathname and get an accurate answer. Stop at symlinks. + async fn walk_full( + &self, + current: ArcRef<'_, Dentry>, + iter: &mut PathIterator<'_>, + ) -> WalkResult { + let (parent_slow, name_slow); + + match self.walk_rcu(current, iter, &rcu_read_lock()) { + WalkResultRcu::Err(error) => return WalkResult::Err(error.into()), + WalkResultRcu::Ok(dentry) => return WalkResult::Ok(dentry.clone_arc()), + WalkResultRcu::Symlink { symlink, inode } => { + return WalkResult::Symlink { + symlink: symlink.clone_arc(), + inode, + }; + } + WalkResultRcu::Miss { parent, name } => { + // Fallback to regular refcounted lookup + parent_slow = parent.clone_arc(); + name_slow = name; + } + } + + match parent_slow.find_slow(name_slow).await { + Ok(found) => self.walk_slow(found, iter).await, + Err(err) => return WalkResult::Err(err), + } + } + + pub async fn follow_symlink( + &self, + symlink: ArcRef<'_, Dentry>, + inode: &InodeUse<dyn Inode>, + nr_follows: u32, + ) -> KResult<Arc<Dentry>> { + let mut target = [0; 256]; + let mut target = ByteBuffer::new(&mut target); + inode.readlink(&mut target).await?; + + self.walk_recursive( + &symlink.parent().clone(), + Path::new(target.data()).unwrap(), + nr_follows + 1, + ) + .await + } + + fn follow_symlink_boxed<'r, 'a: 'r, 'b: 'r, 'c: 'r>( + &'a self, + symlink: ArcRef<'b, Dentry>, + inode: &'c InodeUse<dyn Inode>, + nr_follows: u32, + ) -> Pin<Box<dyn Future<Output = KResult<Arc<Dentry>>> + Send + 'r>> { + Box::pin(self.follow_symlink(symlink, inode, nr_follows)) + } + + async fn walk_recursive( + &self, + cwd: &Arc<Dentry>, + path: &Path, + nr_follows: u32, + ) -> KResult<Arc<Dentry>> { + const MAX_NR_FOLLOWS: u32 = 16; + + let mut current_owned; + let mut current; + if path.is_absolute() { + current = self.fsroot.aref(); + } else { + current = cwd.aref(); + } + + let mut path_iter = path.iter(); + + loop { + match self.walk_full(current, &mut path_iter).await { + WalkResult::Err(posix_error) => return Err(posix_error.into()), + WalkResult::Ok(dentry) => return Ok(dentry), + WalkResult::Symlink { symlink, inode } => { + if nr_follows >= MAX_NR_FOLLOWS { + return Err(ELOOP); + } + + current_owned = self + .follow_symlink_boxed(symlink.aref(), &inode, nr_follows) + .await?; + current = current_owned.aref(); + } + } + } + } + + pub async fn start_recursive_walk( + &self, + cwd: &Arc<Dentry>, + path: &Path, + ) -> KResult<Arc<Dentry>> { + self.walk_recursive(cwd, path, 0).await + } +} + +impl<'a, 'b> DentryFind<'a, 'b> { + fn new(parent: &'a Dentry, name: &'b [u8]) -> Self { + let builder: BuildHasherDefault<KernelHasher> = Default::default(); + let mut hasher = builder.build_hasher(); + + hasher.write_usize(parent as *const _ as usize); + hasher.write(name); + let hash = hasher.finish() as usize; + + Self { parent, name, hash } + } +} + +impl DCacheItem for DentryFind<'_, '_> { + fn d_hash(&self) -> usize { + self.hash + } + + fn d_parent(&self) -> *const Dentry { + self.parent as *const _ + } + + fn d_name<'r, 'a: 'r, 'b: 'a>( + &'a self, + _rcu_read: &'b RCUReadLock, + ) -> impl Deref<Target = [u8]> + 'r { + self.name + } +} diff --git a/src/path.rs b/src/path.rs index 8b740095..b342ef5f 100644 --- a/src/path.rs +++ b/src/path.rs @@ -1,34 +1,30 @@ use crate::{kernel::constants::ENOENT, prelude::*}; use core::fmt::{self, Debug, Formatter}; -pub struct Path<'lt> { - all: &'lt [u8], +#[repr(transparent)] +pub struct Path { + all: [u8], } pub struct PathIterator<'lt> { rem: &'lt [u8], } -#[allow(dead_code)] -impl<'lt> Path<'lt> { - pub fn new(all: &'lt [u8]) -> KResult<Self> { +impl Path { + pub fn new(all: &[u8]) -> KResult<&Self> { if all.is_empty() { Err(ENOENT) } else { - Ok(Self { all }) + Ok(unsafe { &*(all as *const [u8] as *const Path) }) } } - pub fn from_str(all: &'lt str) -> KResult<Self> { - Self::new(all.as_bytes()) - } - pub fn is_absolute(&self) -> bool { self.all.starts_with(&['/' as u8]) } - pub fn iter(&self) -> PathIterator<'lt> { - PathIterator::new(self.all) + pub fn iter(&self) -> PathIterator { + PathIterator::new(&self.all) } } @@ -46,11 +42,17 @@ pub enum PathComponent<'lt> { Parent, } +impl PathIterator<'_> { + pub fn is_empty(&self) -> bool { + self.rem.is_empty() + } +} + impl<'lt> Iterator for PathIterator<'lt> { type Item = PathComponent<'lt>; fn next(&mut self) -> Option<Self::Item> { - if self.rem.is_empty() { + if self.is_empty() { return None; } @@ -71,16 +73,16 @@ impl<'lt> Iterator for PathIterator<'lt> { self.rem = rem; match cur { - cur if cur.is_empty() => Some(PathComponent::TrailingEmpty), - cur if cur == b"." => Some(PathComponent::Current), - cur if cur == b".." => Some(PathComponent::Parent), - cur => Some(PathComponent::Name(cur)), + b"" => Some(PathComponent::TrailingEmpty), + b"." => Some(PathComponent::Current), + b".." => Some(PathComponent::Parent), + name => Some(PathComponent::Name(name)), } } } -impl Debug for Path<'_> { +impl Debug for Path { fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result { - write!(f, "Path({:?})", self.all) + write!(f, "Path({:?})", &self.all) } } diff --git a/src/rcu.rs b/src/rcu.rs index c1645d33..b06db9e2 100644 --- a/src/rcu.rs +++ b/src/rcu.rs @@ -1,21 +1,35 @@ use crate::{kernel::task::block_on, prelude::*}; use alloc::sync::Arc; +use arcref::ArcRef; use core::{ ops::Deref, ptr::NonNull, sync::atomic::{AtomicPtr, Ordering}, }; +use eonix_preempt::PreemptGuard; use eonix_runtime::scheduler::RUNTIME; -use eonix_sync::{Mutex, RwLock, RwLockReadGuard}; +use eonix_sync::{RwLock, RwLockReadGuard}; use pointers::BorrowedArc; +/// The RCU Read Lock. Holding a reference to an instance of the struct assures +/// you that any RCU protected data would not be dropped. +/// +/// The struct cannot be created directly. Instead, use [`rcu_read_lock()`]. +#[derive(Debug)] +pub struct RCUReadLock(); + +pub struct RCUReadGuardNew { + guard: RwLockReadGuard<'static, RCUReadLock>, + _disable_preempt: PreemptGuard<()>, +} + pub struct RCUReadGuard<'data, T: 'data> { value: T, - _guard: RwLockReadGuard<'data, ()>, + _guard: RwLockReadGuard<'static, RCUReadLock>, _phantom: PhantomData<&'data T>, } -static GLOBAL_RCU_SEM: RwLock<()> = RwLock::new(()); +static GLOBAL_RCU_SEM: RwLock<RCUReadLock> = RwLock::new(RCUReadLock()); impl<'data, T> RCUReadGuard<'data, BorrowedArc<'data, T>> { fn lock(value: BorrowedArc<'data, T>) -> Self { @@ -25,14 +39,6 @@ impl<'data, T> RCUReadGuard<'data, BorrowedArc<'data, T>> { _phantom: PhantomData, } } - - pub fn borrow(&self) -> BorrowedArc<'data, T> { - unsafe { - BorrowedArc::from_raw(NonNull::new_unchecked( - &raw const *self.value.borrow() as *mut T - )) - } - } } impl<'data, T: 'data> Deref for RCUReadGuard<'data, T> { @@ -63,17 +69,14 @@ pub trait RCUNode<MySelf> { pub struct RCUList<T: RCUNode<T>> { head: AtomicPtr<T>, - - reader_lock: RwLock<()>, - update_lock: Mutex<()>, + update_lock: Spin<()>, } impl<T: RCUNode<T>> RCUList<T> { pub const fn new() -> Self { Self { head: AtomicPtr::new(core::ptr::null_mut()), - reader_lock: RwLock::new(()), - update_lock: Mutex::new(()), + update_lock: Spin::new(()), } } @@ -117,7 +120,6 @@ impl<T: RCUNode<T>> RCUList<T> { unsafe { Arc::from_raw(me) }; } - let _lck = self.reader_lock.write(); node.rcu_prev() .store(core::ptr::null_mut(), Ordering::Release); node.rcu_next() @@ -152,7 +154,6 @@ impl<T: RCUNode<T>> RCUList<T> { unsafe { Arc::from_raw(old) }; } - let _lck = self.reader_lock.write(); old_node .rcu_prev() .store(core::ptr::null_mut(), Ordering::Release); @@ -161,36 +162,36 @@ impl<T: RCUNode<T>> RCUList<T> { .store(core::ptr::null_mut(), Ordering::Release); } - pub fn iter(&self) -> RCUIterator<T> { - let _lck = block_on(self.reader_lock.read()); - + pub fn iter<'a, 'r>(&'a self, _lock: &'r RCUReadLock) -> RCUIterator<'a, 'r, T> { RCUIterator { - // SAFETY: We have a read lock, so the node is still alive. - cur: NonNull::new(self.head.load(Ordering::SeqCst)), - _lock: _lck, + cur: NonNull::new(self.head.load(Ordering::Acquire)), + _phantom: PhantomData, } } } -pub struct RCUIterator<'lt, T: RCUNode<T>> { +pub struct RCUIterator<'list, 'rcu, T: RCUNode<T>> { cur: Option<NonNull<T>>, - _lock: RwLockReadGuard<'lt, ()>, + _phantom: PhantomData<(&'list (), &'rcu ())>, } -impl<'lt, T: RCUNode<T>> Iterator for RCUIterator<'lt, T> { - type Item = BorrowedArc<'lt, T>; +impl<'rcu, T: RCUNode<T>> Iterator for RCUIterator<'_, 'rcu, T> { + type Item = ArcRef<'rcu, T>; fn next(&mut self) -> Option<Self::Item> { - match self.cur { - None => None, - Some(pointer) => { - // SAFETY: We have a read lock, so the node is still alive. - let reference = unsafe { pointer.as_ref() }; + self.cur.map(|pointer| { + let reference = unsafe { + // SAFETY: We have the read lock so the node is still alive. + pointer.as_ref() + }; + + self.cur = NonNull::new(reference.rcu_next().load(Ordering::Acquire)); - self.cur = NonNull::new(reference.rcu_next().load(Ordering::SeqCst)); - Some(unsafe { BorrowedArc::from_raw(pointer) }) + unsafe { + // SAFETY: We have the read lock so the node is still alive. + ArcRef::new_unchecked(pointer.as_ptr()) } - } + }) } } @@ -228,15 +229,16 @@ where } pub fn load<'lt>(&self) -> Option<RCUReadGuard<'lt, BorrowedArc<'lt, T>>> { + // BUG: We should acquire the lock before loading the pointer NonNull::new(self.0.load(Ordering::Acquire)) .map(|p| RCUReadGuard::lock(unsafe { BorrowedArc::from_raw(p) })) } - pub fn load_protected<'a, U: 'a>( - &self, - _guard: &RCUReadGuard<'a, U>, - ) -> Option<BorrowedArc<'a, T>> { - NonNull::new(self.0.load(Ordering::Acquire)).map(|p| unsafe { BorrowedArc::from_raw(p) }) + pub fn dereference<'r, 'a: 'r>(&self, _lock: &'a RCUReadLock) -> Option<ArcRef<'r, T>> { + NonNull::new(self.0.load(Ordering::Acquire)).map(|p| unsafe { + // SAFETY: We have a read lock, so the node is still alive. + ArcRef::new_unchecked(p.as_ptr()) + }) } /// # Safety @@ -289,3 +291,18 @@ where } } } + +impl Deref for RCUReadGuardNew { + type Target = RCUReadLock; + + fn deref(&self) -> &Self::Target { + &self.guard + } +} + +pub fn rcu_read_lock() -> RCUReadGuardNew { + RCUReadGuardNew { + guard: block_on(GLOBAL_RCU_SEM.read()), + _disable_preempt: PreemptGuard::new(()), + } +} From 6c6e9f10c2517e71c02625491aa828d43cf17de8 Mon Sep 17 00:00:00 2001 From: greatbridf <greatbridf@icloud.com> Date: Sun, 14 Sep 2025 22:22:51 +0800 Subject: [PATCH 15/36] style, vfs: remove unused imports and InodeRef Signed-off-by: greatbridf <greatbridf@icloud.com> --- src/kernel/vfs/inode/inode.rs | 5 ----- src/kernel/vfs/inode/mod.rs | 4 +--- 2 files changed, 1 insertion(+), 8 deletions(-) diff --git a/src/kernel/vfs/inode/inode.rs b/src/kernel/vfs/inode/inode.rs index 786a31fe..870a045d 100644 --- a/src/kernel/vfs/inode/inode.rs +++ b/src/kernel/vfs/inode/inode.rs @@ -321,11 +321,6 @@ pub struct InodeInfo { pub mtime: Instant, } -#[derive(Clone)] -pub struct InodeRef<I>(Weak<I>) -where - I: Inode + ?Sized; - pub struct InodeUse<I>(Arc<I>) where I: Inode + ?Sized; diff --git a/src/kernel/vfs/inode/mod.rs b/src/kernel/vfs/inode/mod.rs index 786d30fa..08471ef3 100644 --- a/src/kernel/vfs/inode/mod.rs +++ b/src/kernel/vfs/inode/mod.rs @@ -4,7 +4,5 @@ mod ops; mod statx; pub use ino::Ino; -pub use inode::{ - Inode, InodeDir, InodeDirOps, InodeFile, InodeFileOps, InodeInfo, InodeOps, InodeRef, InodeUse, -}; +pub use inode::{Inode, InodeDirOps, InodeFileOps, InodeInfo, InodeOps, InodeUse}; pub use ops::{RenameData, WriteOffset}; From c3db2d5f2f3cf43aa94ba67bf938851af5e29506 Mon Sep 17 00:00:00 2001 From: greatbridf <greatbridf@icloud.com> Date: Sun, 14 Sep 2025 22:23:09 +0800 Subject: [PATCH 16/36] vfs: fix debug print of `Mode` structs If we have invalid format, we should print a None instead of panicking. Signed-off-by: greatbridf <greatbridf@icloud.com> --- src/kernel/vfs/types/mode.rs | 35 +++++++++++++++++++---------------- 1 file changed, 19 insertions(+), 16 deletions(-) diff --git a/src/kernel/vfs/types/mode.rs b/src/kernel/vfs/types/mode.rs index dc1b88ec..a58c8215 100644 --- a/src/kernel/vfs/types/mode.rs +++ b/src/kernel/vfs/types/mode.rs @@ -44,13 +44,20 @@ impl Mode { } pub fn format(&self) -> Format { + match self.try_format() { + None => panic!("unknown format bits: {:#o}", self.format_bits()), + Some(format) => format, + } + } + + pub fn try_format(&self) -> Option<Format> { match self.format_bits() { - S_IFREG => Format::REG, - S_IFDIR => Format::DIR, - S_IFLNK => Format::LNK, - S_IFBLK => Format::BLK, - S_IFCHR => Format::CHR, - _ => panic!("unknown format bits: {:#o}", self.format_bits()), + S_IFREG => Some(Format::REG), + S_IFDIR => Some(Format::DIR), + S_IFLNK => Some(Format::LNK), + S_IFBLK => Some(Format::BLK), + S_IFCHR => Some(Format::CHR), + _ => None, } } @@ -58,10 +65,6 @@ impl Mode { Permission::new(self.non_format_bits()) } - pub const fn non_format(&self) -> Self { - Self::new(self.non_format_bits()) - } - pub const fn set_perm(&mut self, perm: Permission) { self.0 = self.format_bits() | perm.bits(); } @@ -100,15 +103,15 @@ impl core::fmt::Debug for Mode { match self.non_format_bits() & !0o777 { 0 => write!( f, - "Mode({format:?}, {perm:#o})", - format = self.format(), - perm = self.non_format_bits() + "Mode({format:?}, {perm:?})", + format = self.try_format(), + perm = Permission::new(self.non_format_bits()), )?, rem => write!( f, - "Mode({format:?}, {perm:#o}, rem={rem:#x})", - format = self.format(), - perm = self.non_format_bits() & 0o777 + "Mode({format:?}, {perm:?}, rem={rem:#x})", + format = self.try_format(), + perm = Permission::new(self.non_format_bits()) )?, } From 9d270c72b09b09a7711d9252ed74e82627741b52 Mon Sep 17 00:00:00 2001 From: greatbridf <greatbridf@icloud.com> Date: Tue, 27 Oct 2026 00:28:55 +0800 Subject: [PATCH 17/36] style: change import style and reformat codes Signed-off-by: greatbridf <greatbridf@icloud.com> --- .rustfmt.toml | 4 +-- src/driver/ahci/command_table.rs | 6 ++-- src/driver/ahci/mod.rs | 26 ++++++++---------- src/driver/ahci/port.rs | 12 ++++---- src/driver/ahci/slot.rs | 7 +++-- src/driver/e1000e.rs | 19 +++++++------ src/driver/virtio/virtio_blk.rs | 26 ++++++++---------- src/fs/fat32.rs | 32 +++++++++------------- src/io.rs | 4 ++- src/kernel/block.rs | 27 ++++++++---------- src/kernel/mem/access.rs | 3 +- src/kernel/mem/allocator.rs | 6 ++-- src/kernel/mem/mm_area.rs | 12 ++++---- src/kernel/mem/mm_list.rs | 29 +++++++++----------- src/kernel/mem/page_cache.rs | 29 ++++++++++---------- src/kernel/mem/paging.rs | 7 +++-- src/kernel/vfs/file/mod.rs | 29 ++++++++------------ src/lib.rs | 47 +++++++++++++++----------------- 18 files changed, 160 insertions(+), 165 deletions(-) diff --git a/.rustfmt.toml b/.rustfmt.toml index d69872c6..17b2bbc5 100644 --- a/.rustfmt.toml +++ b/.rustfmt.toml @@ -29,8 +29,8 @@ fn_single_line = false where_single_line = false imports_indent = "Block" imports_layout = "Mixed" -imports_granularity = "Preserve" -group_imports = "Preserve" +imports_granularity = "Module" +group_imports = "StdExternalCrate" reorder_imports = true reorder_modules = true reorder_impl_items = false diff --git a/src/driver/ahci/command_table.rs b/src/driver/ahci/command_table.rs index c77b4abd..766150ef 100644 --- a/src/driver/ahci/command_table.rs +++ b/src/driver/ahci/command_table.rs @@ -1,7 +1,9 @@ -use super::{command::Command, PRDTEntry, FISH2D}; -use crate::kernel::mem::{AsMemoryBlock as _, Page}; use eonix_mm::address::PAddr; +use super::command::Command; +use super::{PRDTEntry, FISH2D}; +use crate::kernel::mem::{Page, PageExt}; + pub struct CommandTable<'a> { page: Page, command_fis: &'a mut FISH2D, diff --git a/src/driver/ahci/mod.rs b/src/driver/ahci/mod.rs index ab405829..d0853875 100644 --- a/src/driver/ahci/mod.rs +++ b/src/driver/ahci/mod.rs @@ -1,25 +1,23 @@ -use crate::{ - fs::procfs, - io::Buffer as _, - kernel::{ - block::BlockDevice, - constants::{EINVAL, EIO}, - interrupt::register_irq_handler, - pcie::{self, Header, PCIDevice, PCIDriver, PciError}, - vfs::types::DeviceId, - }, - prelude::*, -}; -use alloc::{format, sync::Arc}; +use alloc::format; +use alloc::sync::Arc; + use async_trait::async_trait; use control::AdapterControl; use defs::*; use eonix_mm::address::{AddrOps as _, PAddr}; use eonix_sync::SpinIrq as _; use port::AdapterPort; - pub(self) use register::Register; +use crate::fs::procfs; +use crate::io::Buffer as _; +use crate::kernel::block::BlockDevice; +use crate::kernel::constants::{EINVAL, EIO}; +use crate::kernel::interrupt::register_irq_handler; +use crate::kernel::pcie::{self, Header, PCIDevice, PCIDriver, PciError}; +use crate::kernel::vfs::types::DeviceId; +use crate::prelude::*; + mod command; mod command_table; mod control; diff --git a/src/driver/ahci/port.rs b/src/driver/ahci/port.rs index 77286ec5..1cb49eb9 100644 --- a/src/driver/ahci/port.rs +++ b/src/driver/ahci/port.rs @@ -1,3 +1,10 @@ +use alloc::collections::vec_deque::VecDeque; +use core::pin::pin; + +use async_trait::async_trait; +use eonix_mm::address::{Addr as _, PAddr}; +use eonix_sync::{SpinIrq as _, WaitList}; + use super::command::{Command, IdentifyCommand, ReadLBACommand, WriteLBACommand}; use super::slot::CommandSlot; use super::stats::AdapterPortStats; @@ -10,11 +17,6 @@ use crate::kernel::constants::{EINVAL, EIO}; use crate::kernel::mem::paging::Page; use crate::kernel::mem::AsMemoryBlock as _; use crate::prelude::*; -use alloc::collections::vec_deque::VecDeque; -use async_trait::async_trait; -use core::pin::pin; -use eonix_mm::address::{Addr as _, PAddr}; -use eonix_sync::{SpinIrq as _, WaitList}; /// An `AdapterPort` is an HBA device in AHCI mode. /// diff --git a/src/driver/ahci/slot.rs b/src/driver/ahci/slot.rs index 60a66de3..745a84b5 100644 --- a/src/driver/ahci/slot.rs +++ b/src/driver/ahci/slot.rs @@ -1,9 +1,12 @@ -use super::{command_table::CommandTable, CommandHeader}; -use crate::KResult; use core::pin::pin; + use eonix_mm::address::Addr as _; use eonix_sync::{Spin, SpinIrq as _, WaitList}; +use super::command_table::CommandTable; +use super::CommandHeader; +use crate::KResult; + pub struct CommandSlot<'a> { /// # Usage /// `inner.cmdheader` might be used in irq handler. So in order to wait for diff --git a/src/driver/e1000e.rs b/src/driver/e1000e.rs index f362f477..f026efcb 100644 --- a/src/driver/e1000e.rs +++ b/src/driver/e1000e.rs @@ -1,19 +1,20 @@ -use crate::kernel::constants::{EAGAIN, EFAULT, EINVAL, EIO}; -use crate::kernel::interrupt::register_irq_handler; -use crate::kernel::mem::paging::{self, AllocZeroed}; -use crate::kernel::mem::{AsMemoryBlock, PhysAccess}; -use crate::kernel::pcie::{self, Header, PCIDevice, PCIDriver, PciError}; -use crate::net::netdev; -use crate::prelude::*; use alloc::boxed::Box; use alloc::sync::Arc; use alloc::vec::Vec; -use async_trait::async_trait; use core::ptr::NonNull; + +use async_trait::async_trait; use eonix_hal::fence::memory_barrier; use eonix_mm::address::{Addr, PAddr}; use eonix_sync::SpinIrq; -use paging::Page; + +use crate::kernel::constants::{EAGAIN, EFAULT, EINVAL, EIO}; +use crate::kernel::interrupt::register_irq_handler; +use crate::kernel::mem::{AsMemoryBlock, Page, PhysAccess}; +use crate::kernel::mem::paging::{self, AllocZeroed}; +use crate::kernel::pcie::{self, Header, PCIDevice, PCIDriver, PciError}; +use crate::net::netdev; +use crate::prelude::*; mod defs; diff --git a/src/driver/virtio/virtio_blk.rs b/src/driver/virtio/virtio_blk.rs index 86b500b6..1aee679a 100644 --- a/src/driver/virtio/virtio_blk.rs +++ b/src/driver/virtio/virtio_blk.rs @@ -1,21 +1,19 @@ -use crate::{ - io::Chunks, - kernel::{ - block::{BlockDeviceRequest, BlockRequestQueue}, - constants::EIO, - mem::{AsMemoryBlock, Page}, - }, - prelude::KResult, -}; use alloc::boxed::Box; + use async_trait::async_trait; use eonix_hal::mm::ArchPhysAccess; -use eonix_mm::{ - address::{Addr, PAddr, PhysAccess}, - paging::PFN, -}; +use eonix_mm::address::{Addr, PAddr, PhysAccess}; +use eonix_mm::paging::PFN; use eonix_sync::Spin; -use virtio_drivers::{device::blk::VirtIOBlk, transport::Transport, Hal}; +use virtio_drivers::device::blk::VirtIOBlk; +use virtio_drivers::transport::Transport; +use virtio_drivers::Hal; + +use crate::io::Chunks; +use crate::kernel::block::{BlockDeviceRequest, BlockRequestQueue}; +use crate::kernel::constants::EIO; +use crate::kernel::mem::{AsMemoryBlock, Page}; +use crate::prelude::KResult; pub struct HAL; diff --git a/src/fs/fat32.rs b/src/fs/fat32.rs index 1104337d..c4e247c8 100644 --- a/src/fs/fat32.rs +++ b/src/fs/fat32.rs @@ -1,38 +1,32 @@ mod dir; mod file; +use alloc::sync::{Arc, Weak}; use core::future::Future; use core::ops::Deref; -use alloc::sync::{Arc, Weak}; use async_trait::async_trait; use dir::{as_raw_dirents, ParseDirent}; use eonix_sync::RwLock; use itertools::Itertools; +use crate::io::{Buffer, ByteBuffer, UninitBuffer}; +use crate::kernel::block::{BlockDevice, BlockDeviceRequest}; use crate::kernel::constants::{EINVAL, EIO}; -use crate::kernel::mem::{AsMemoryBlock, CachePageStream}; +use crate::kernel::mem::{ + AsMemoryBlock, CachePage, CachePageStream, Page, PageCache, + PageCacheBackendOps, +}; use crate::kernel::timer::Instant; -use crate::kernel::vfs::inode::{InodeDirOps, InodeFileOps, InodeInfo, InodeOps, InodeUse}; +use crate::kernel::vfs::dentry::Dentry; +use crate::kernel::vfs::inode::{ + Ino, Inode, InodeDirOps, InodeFileOps, InodeInfo, InodeOps, InodeUse, +}; +use crate::kernel::vfs::mount::{register_filesystem, Mount, MountCreator}; use crate::kernel::vfs::types::{DeviceId, Format, Permission}; use crate::kernel::vfs::{SbRef, SbUse, SuperBlock, SuperBlockInfo}; use crate::prelude::*; -use crate::{ - io::{Buffer, ByteBuffer, UninitBuffer}, - kernel::{ - block::{BlockDevice, BlockDeviceRequest}, - mem::{ - paging::Page, - {CachePage, PageCache, PageCacheBackendOps}, - }, - vfs::{ - dentry::Dentry, - inode::{Ino, Inode}, - mount::{register_filesystem, Mount, MountCreator}, - }, - }, - KResult, -}; +use crate::KResult; #[repr(transparent)] #[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord)] diff --git a/src/io.rs b/src/io.rs index 85675dea..81c61d1c 100644 --- a/src/io.rs +++ b/src/io.rs @@ -1,6 +1,8 @@ +use core::mem::MaybeUninit; +use core::cmp; + use crate::kernel::constants::EFAULT; use crate::prelude::*; -use core::{cmp, mem::MaybeUninit}; #[must_use] #[derive(Debug)] diff --git a/src/kernel/block.rs b/src/kernel/block.rs index 3e4b65d1..4c2750a0 100644 --- a/src/kernel/block.rs +++ b/src/kernel/block.rs @@ -1,23 +1,20 @@ mod mbr; -use super::{ - constants::ENOENT, - mem::{paging::Page, AsMemoryBlock as _}, - vfs::types::DeviceId, -}; -use crate::kernel::constants::{EEXIST, EINVAL}; -use crate::{ - io::{Buffer, FillResult}, - prelude::*, -}; -use alloc::{ - collections::btree_map::{BTreeMap, Entry}, - sync::Arc, -}; -use async_trait::async_trait; +use alloc::collections::btree_map::{BTreeMap, Entry}; +use alloc::sync::Arc; use core::cmp::Ordering; + +use async_trait::async_trait; use mbr::MBRPartTable; +use super::constants::ENOENT; +use super::mem::paging::Page; +use super::mem::AsMemoryBlock as _; +use super::vfs::types::DeviceId; +use crate::io::{Buffer, FillResult}; +use crate::kernel::constants::{EEXIST, EINVAL}; +use crate::prelude::*; + pub struct Partition { pub lba_offset: u64, pub sector_count: u64, diff --git a/src/kernel/mem/access.rs b/src/kernel/mem/access.rs index ce525a0a..c4e9042b 100644 --- a/src/kernel/mem/access.rs +++ b/src/kernel/mem/access.rs @@ -1,4 +1,5 @@ -use core::{num::NonZero, ptr::NonNull}; +use core::num::NonZero; +use core::ptr::NonNull; use eonix_hal::mm::ArchPhysAccess; use eonix_mm::address::{PAddr, PhysAccess as _PhysAccess}; diff --git a/src/kernel/mem/allocator.rs b/src/kernel/mem/allocator.rs index 36b19612..017180a6 100644 --- a/src/kernel/mem/allocator.rs +++ b/src/kernel/mem/allocator.rs @@ -1,13 +1,15 @@ -use super::page_alloc::RawPagePtr; -use super::{AsMemoryBlock, GlobalPageAlloc, Page}; use core::alloc::{GlobalAlloc, Layout}; use core::ptr::NonNull; + use eonix_hal::mm::ArchPhysAccess; use eonix_mm::address::PhysAccess; use eonix_mm::paging::{PAGE_SIZE_BITS, PFN}; use eonix_sync::LazyLock; use slab_allocator::SlabAllocator; +use super::page_alloc::RawPagePtr; +use super::{AsMemoryBlock, GlobalPageAlloc, Page}; + static SLAB_ALLOCATOR: LazyLock<SlabAllocator<RawPagePtr, GlobalPageAlloc, 9>> = LazyLock::new(|| SlabAllocator::new_in(GlobalPageAlloc)); diff --git a/src/kernel/mem/mm_area.rs b/src/kernel/mem/mm_area.rs index 731c5303..63043535 100644 --- a/src/kernel/mem/mm_area.rs +++ b/src/kernel/mem/mm_area.rs @@ -1,15 +1,17 @@ -use super::mm_list::EMPTY_PAGE; -use super::paging::AllocZeroed as _; -use super::{AsMemoryBlock, Mapping, Page, Permission}; -use crate::kernel::constants::EINVAL; -use crate::prelude::KResult; use core::borrow::Borrow; use core::cell::UnsafeCell; use core::cmp; + use eonix_mm::address::{AddrOps as _, VAddr, VRange}; use eonix_mm::page_table::{PageAttribute, RawAttribute, PTE}; use eonix_mm::paging::{PAGE_SIZE, PFN}; +use super::mm_list::EMPTY_PAGE; +use super::paging::AllocZeroed as _; +use super::{AsMemoryBlock, Mapping, Page, Permission}; +use crate::kernel::constants::EINVAL; +use crate::prelude::KResult; + #[derive(Debug)] pub struct MMArea { range: UnsafeCell<VRange>, diff --git a/src/kernel/mem/mm_list.rs b/src/kernel/mem/mm_list.rs index ad1e45c2..1a0e1530 100644 --- a/src/kernel/mem/mm_list.rs +++ b/src/kernel/mem/mm_list.rs @@ -1,33 +1,30 @@ mod mapping; mod page_fault; -use super::address::{VAddrExt as _, VRangeExt as _}; -use super::page_alloc::GlobalPageAlloc; -use super::paging::AllocZeroed as _; -use super::{AsMemoryBlock, MMArea, Page}; -use crate::kernel::constants::{EEXIST, EFAULT, EINVAL, ENOMEM}; -use crate::kernel::mem::page_alloc::RawPagePtr; -use crate::{prelude::*, sync::ArcSwap}; use alloc::collections::btree_set::BTreeSet; use core::fmt; use core::sync::atomic::{AtomicUsize, Ordering}; + use eonix_hal::mm::{ flush_tlb_all, get_root_page_table_pfn, set_root_page_table_pfn, ArchPagingMode, ArchPhysAccess, GLOBAL_PAGE_TABLE, }; -use eonix_mm::address::{Addr as _, PAddr}; -use eonix_mm::page_table::PageAttribute; -use eonix_mm::paging::PFN; -use eonix_mm::{ - address::{AddrOps as _, VAddr, VRange}, - page_table::{PageTable, RawAttribute, PTE}, - paging::PAGE_SIZE, -}; +use eonix_mm::address::{Addr as _, AddrOps as _, PAddr, VAddr, VRange}; +use eonix_mm::page_table::{PageAttribute, PageTable, RawAttribute, PTE}; +use eonix_mm::paging::{PAGE_SIZE, PFN}; use eonix_sync::{LazyLock, Mutex}; - pub use mapping::{FileMapping, Mapping}; pub use page_fault::handle_kernel_page_fault; +use super::address::{VAddrExt as _, VRangeExt as _}; +use super::page_alloc::GlobalPageAlloc; +use super::paging::AllocZeroed as _; +use super::{AsMemoryBlock, MMArea, Page}; +use crate::kernel::constants::{EEXIST, EFAULT, EINVAL, ENOMEM}; +use crate::kernel::mem::page_alloc::RawPagePtr; +use crate::prelude::*; +use crate::sync::ArcSwap; + pub static EMPTY_PAGE: LazyLock<Page> = LazyLock::new(|| Page::zeroed()); #[derive(Debug, Clone, Copy)] diff --git a/src/kernel/mem/page_cache.rs b/src/kernel/mem/page_cache.rs index 9deb50cf..43e4314a 100644 --- a/src/kernel/mem/page_cache.rs +++ b/src/kernel/mem/page_cache.rs @@ -1,22 +1,23 @@ -use super::{paging::AllocZeroed, Page}; -use crate::{ - io::{Buffer, FillResult, Stream}, - kernel::mem::page_alloc::RawPagePtr, - prelude::KResult, - GlobalPageAlloc, -}; -use align_ext::AlignExt; use alloc::boxed::Box; -use alloc::{collections::btree_map::BTreeMap, sync::Weak}; +use alloc::collections::btree_map::BTreeMap; +use alloc::sync::Weak; +use core::future::Future; +use core::mem::ManuallyDrop; + +use align_ext::AlignExt; use async_trait::async_trait; -use core::{future::Future, mem::ManuallyDrop}; use eonix_hal::mm::ArchPhysAccess; -use eonix_mm::{ - address::{PAddr, PhysAccess}, - paging::{PageAlloc, RawPage, PAGE_SIZE, PAGE_SIZE_BITS, PFN}, -}; +use eonix_mm::address::{PAddr, PhysAccess}; +use eonix_mm::paging::{PageAlloc, RawPage, PAGE_SIZE, PAGE_SIZE_BITS, PFN}; use eonix_sync::Mutex; +use super::paging::AllocZeroed; +use super::Page; +use crate::io::{Buffer, FillResult, Stream}; +use crate::kernel::mem::page_alloc::RawPagePtr; +use crate::prelude::KResult; +use crate::GlobalPageAlloc; + pub struct PageCache { pages: Mutex<BTreeMap<usize, CachePage>>, backend: Weak<dyn PageCacheBackend>, diff --git a/src/kernel/mem/paging.rs b/src/kernel/mem/paging.rs index 8c5f41f2..7e604b65 100644 --- a/src/kernel/mem/paging.rs +++ b/src/kernel/mem/paging.rs @@ -1,7 +1,10 @@ -use super::{access::AsMemoryBlock, page_alloc::GlobalPageAlloc, MemoryBlock, PhysAccess}; -use crate::io::{Buffer, FillResult}; use eonix_mm::paging::{Page as GenericPage, PageAlloc}; +use super::access::AsMemoryBlock; +use super::page_alloc::GlobalPageAlloc; +use super::{MemoryBlock, PhysAccess}; +use crate::io::{Buffer, FillResult}; + pub type Page = GenericPage<GlobalPageAlloc>; /// A buffer that wraps a page and provides a `Buffer` interface. diff --git a/src/kernel/vfs/file/mod.rs b/src/kernel/vfs/file/mod.rs index bb1c66ec..74abc9fb 100644 --- a/src/kernel/vfs/file/mod.rs +++ b/src/kernel/vfs/file/mod.rs @@ -2,29 +2,24 @@ mod inode_file; mod pipe; mod terminal_file; -use crate::{ - io::{Buffer, ByteBuffer, Chunks, IntoStream, Stream}, - kernel::{ - constants::{EBADF, EINTR, EINVAL, ENOTTY}, - mem::{AsMemoryBlock, Page}, - task::Thread, - CharDevice, - }, - prelude::KResult, -}; use alloc::sync::Arc; -use bitflags::bitflags; -use core::{ - ops::Deref, - sync::atomic::{AtomicI32, AtomicU32, Ordering}, -}; -use pipe::{PipeReadEnd, PipeWriteEnd}; -use posix_types::open::OpenFlags; +use core::ops::Deref; +use core::sync::atomic::{AtomicI32, AtomicU32, Ordering}; +use bitflags::bitflags; pub use inode_file::InodeFile; pub use pipe::Pipe; +use pipe::{PipeReadEnd, PipeWriteEnd}; +use posix_types::open::OpenFlags; pub use terminal_file::TerminalFile; +use crate::io::{Buffer, ByteBuffer, Chunks, IntoStream, Stream}; +use crate::kernel::constants::{EBADF, EINTR, EINVAL, ENOTTY}; +use crate::kernel::mem::{AsMemoryBlock, Page}; +use crate::kernel::task::Thread; +use crate::kernel::CharDevice; +use crate::prelude::KResult; + pub enum FileType { Inode(InodeFile), PipeRead(PipeReadEnd), diff --git a/src/lib.rs b/src/lib.rs index 98e196f8..a0774c9c 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -28,36 +28,33 @@ mod prelude; mod rcu; mod sync; -use crate::kernel::task::alloc_pid; -use alloc::{ffi::CString, sync::Arc}; -use core::{ - hint::spin_loop, - sync::atomic::{AtomicBool, AtomicUsize, Ordering}, -}; -use eonix_hal::{ - arch_exported::bootstrap::shutdown, - context::TaskContext, - processor::{halt, CPU, CPU_COUNT}, - traits::{context::RawTaskContext, trap::IrqState}, - trap::disable_irqs_save, -}; +use alloc::ffi::CString; +use alloc::sync::Arc; +use core::hint::spin_loop; +use core::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; + +use eonix_hal::arch_exported::bootstrap::shutdown; +use eonix_hal::context::TaskContext; +use eonix_hal::processor::{halt, CPU, CPU_COUNT}; +use eonix_hal::traits::context::RawTaskContext; +use eonix_hal::traits::trap::IrqState; +use eonix_hal::trap::disable_irqs_save; use eonix_mm::address::PRange; -use eonix_runtime::{executor::Stack, scheduler::RUNTIME}; -use kernel::{ - mem::GlobalPageAlloc, - task::{KernelStack, ProcessBuilder, ProcessList, ProgramLoader, ThreadBuilder}, - vfs::{ - dentry::Dentry, - mount::{do_mount, MS_NOATIME, MS_NODEV, MS_NOSUID, MS_RDONLY}, - types::Permission, - FsContext, - }, - CharDevice, -}; +use eonix_runtime::executor::Stack; +use eonix_runtime::scheduler::RUNTIME; +use kernel::mem::GlobalPageAlloc; +use kernel::task::{KernelStack, ProcessBuilder, ProcessList, ProgramLoader, ThreadBuilder}; +use kernel::vfs::dentry::Dentry; +use kernel::vfs::mount::{do_mount, MS_NOATIME, MS_NODEV, MS_NOSUID, MS_RDONLY}; +use kernel::vfs::types::Permission; +use kernel::vfs::FsContext; +use kernel::CharDevice; use kernel_init::setup_memory; use path::Path; use prelude::*; +use crate::kernel::task::alloc_pid; + #[cfg(any(target_arch = "riscv64", target_arch = "loongarch64"))] fn do_panic() -> ! { #[cfg(target_arch = "riscv64")] From 776837398d1f24f681347893110651b764c57563 Mon Sep 17 00:00:00 2001 From: greatbridf <greatbridf@icloud.com> Date: Wed, 8 Oct 2025 11:10:13 +0800 Subject: [PATCH 18/36] mem, paging: introduce page locks and exclusive pages Introduce the new page locking mechanism to ensure exclusiveness of pages when we access them. The underlying locks are not implemented yet for now because we will change the paging structs in the following few patches. Introduce a new `PageExcl` struct representing a page that conforms with Rust's ownership rules. The page owned exclusively can be accessed without taking page locks. Remove the `MemoryBlock` structs as they are not easy to use and represent barely no semantic meanings. Signed-off-by: greatbridf <greatbridf@icloud.com> --- Cargo.lock | 7 + Cargo.toml | 1 + src/driver/ahci/command_table.rs | 55 +++---- src/driver/ahci/mod.rs | 16 +-- src/driver/ahci/port.rs | 96 ++++--------- src/driver/ahci/slot.rs | 174 ++++++++++++++++------ src/driver/e1000e.rs | 120 ++++++---------- src/driver/virtio/virtio_blk.rs | 31 ++-- src/fs/fat32.rs | 36 ++--- src/io.rs | 28 ++-- src/kernel/block.rs | 199 ++++++-------------------- src/kernel/mem.rs | 4 +- src/kernel/mem/access.rs | 117 --------------- src/kernel/mem/allocator.rs | 4 +- src/kernel/mem/mm_area.rs | 44 +++--- src/kernel/mem/mm_list.rs | 12 +- src/kernel/mem/page_alloc/raw_page.rs | 9 +- src/kernel/mem/page_cache.rs | 8 -- src/kernel/mem/paging.rs | 128 ++++++++++++++--- src/kernel/vfs/file/mod.rs | 7 +- src/lib.rs | 3 + 21 files changed, 493 insertions(+), 606 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index f85a9d2f..c70190a0 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -180,6 +180,7 @@ dependencies = [ "posix_types", "slab_allocator", "stalloc", + "static_assertions", "unwinding", "virtio-drivers", "xmas-elf", @@ -525,6 +526,12 @@ version = "0.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a37f0ead4094eeb54c6893316aa139e48b252f1c07511e5124fa1f9414df5b6c" +[[package]] +name = "static_assertions" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f" + [[package]] name = "syn" version = "2.0.104" diff --git a/Cargo.toml b/Cargo.toml index 5158025e..dca5d34d 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -42,6 +42,7 @@ futures = { version = "0.3.31", features = [ "alloc", "async-await", ], default-features = false } +static_assertions = "1.1.0" [target.'cfg(any(target_arch = "riscv64", target_arch = "loongarch64"))'.dependencies] virtio-drivers = { version = "0.11.0" } diff --git a/src/driver/ahci/command_table.rs b/src/driver/ahci/command_table.rs index 766150ef..7b78d26f 100644 --- a/src/driver/ahci/command_table.rs +++ b/src/driver/ahci/command_table.rs @@ -1,47 +1,54 @@ +use core::ptr::NonNull; + use eonix_mm::address::PAddr; use super::command::Command; use super::{PRDTEntry, FISH2D}; use crate::kernel::mem::{Page, PageExt}; -pub struct CommandTable<'a> { +pub struct CommandTable { page: Page, - command_fis: &'a mut FISH2D, - - prdt: &'a mut [PRDTEntry; 248], - prdt_entries: Option<u16>, + cmd_fis: NonNull<FISH2D>, + prdt: NonNull<[PRDTEntry; 248]>, + prdt_entries: usize, } -impl CommandTable<'_> { +unsafe impl Send for CommandTable {} +unsafe impl Sync for CommandTable {} + +impl CommandTable { pub fn new() -> Self { let page = Page::alloc(); - let memory = page.as_memblk(); - - let (lhs, prdt) = memory.split_at(0x80); - - let (command_fis, _) = lhs.split_at(size_of::<FISH2D>()); - let command_fis = unsafe { command_fis.as_ptr().as_mut() }; - let prdt = unsafe { prdt.as_ptr().as_mut() }; - - Self { - page, - command_fis, - prdt, - prdt_entries: None, + let base = page.get_ptr(); + + unsafe { + Self { + page, + cmd_fis: base.cast(), + prdt: base.byte_add(0x80).cast(), + prdt_entries: 0, + } } } pub fn setup(&mut self, cmd: &impl Command) { - self.command_fis.setup(cmd.cmd(), cmd.lba(), cmd.count()); - self.prdt_entries = Some(cmd.pages().len() as u16); + unsafe { + self.cmd_fis + .as_mut() + .setup(cmd.cmd(), cmd.lba(), cmd.count()); + } + + self.prdt_entries = cmd.pages().len(); for (idx, page) in cmd.pages().iter().enumerate() { - self.prdt[idx].setup(page); + unsafe { + self.prdt.as_mut()[idx].setup(page); + } } } - pub fn prdt_len(&self) -> u16 { - self.prdt_entries.unwrap() + pub fn prdt_len(&self) -> usize { + self.prdt_entries } pub fn base(&self) -> PAddr { diff --git a/src/driver/ahci/mod.rs b/src/driver/ahci/mod.rs index d0853875..3ea44ed3 100644 --- a/src/driver/ahci/mod.rs +++ b/src/driver/ahci/mod.rs @@ -28,7 +28,7 @@ pub(self) mod slot; mod stats; pub struct AHCIDriver { - devices: Spin<Vec<Arc<Device<'static>>>>, + devices: Spin<Vec<Arc<Device>>>, } pub struct BitsIterator { @@ -62,22 +62,22 @@ impl Iterator for BitsIterator { } } -struct Device<'a> { +struct Device { control_base: PAddr, control: AdapterControl, _pcidev: Arc<PCIDevice<'static>>, /// # Lock /// Might be accessed from irq handler, use with `lock_irq()` - ports: Spin<[Option<Arc<AdapterPort<'a>>>; 32]>, + ports: Spin<[Option<Arc<AdapterPort>>; 32]>, } /// # Safety /// `pcidev` is never accessed from Rust code /// TODO!!!: place *mut pci_device in a safe wrapper -unsafe impl Send for Device<'_> {} -unsafe impl Sync for Device<'_> {} +unsafe impl Send for Device {} +unsafe impl Sync for Device {} -impl Device<'_> { +impl Device { fn handle_interrupt(&self) { // Safety // `self.ports` is accessed inside irq handler @@ -106,8 +106,8 @@ impl Device<'_> { } } -impl Device<'static> { - async fn probe_port(&self, port: Arc<AdapterPort<'static>>) -> KResult<()> { +impl Device { + async fn probe_port(&self, port: Arc<AdapterPort>) -> KResult<()> { port.init().await?; { diff --git a/src/driver/ahci/port.rs b/src/driver/ahci/port.rs index 1cb49eb9..a54bbbba 100644 --- a/src/driver/ahci/port.rs +++ b/src/driver/ahci/port.rs @@ -1,21 +1,17 @@ use alloc::collections::vec_deque::VecDeque; -use core::pin::pin; +use core::task::{Poll, Waker}; use async_trait::async_trait; use eonix_mm::address::{Addr as _, PAddr}; -use eonix_sync::{SpinIrq as _, WaitList}; +use eonix_sync::SpinIrq as _; use super::command::{Command, IdentifyCommand, ReadLBACommand, WriteLBACommand}; -use super::slot::CommandSlot; +use super::slot::CommandList; use super::stats::AdapterPortStats; -use super::{ - CommandHeader, Register, PORT_CMD_CR, PORT_CMD_FR, PORT_CMD_FRE, PORT_CMD_ST, PORT_IE_DEFAULT, -}; +use super::{Register, PORT_CMD_CR, PORT_CMD_FR, PORT_CMD_FRE, PORT_CMD_ST, PORT_IE_DEFAULT}; use crate::driver::ahci::command_table::CommandTable; use crate::kernel::block::{BlockDeviceRequest, BlockRequestQueue}; use crate::kernel::constants::{EINVAL, EIO}; -use crate::kernel::mem::paging::Page; -use crate::kernel::mem::AsMemoryBlock as _; use crate::prelude::*; /// An `AdapterPort` is an HBA device in AHCI mode. @@ -57,6 +53,8 @@ pub struct AdapterPortData { struct FreeList { free: VecDeque<u32>, working: VecDeque<u32>, + + wakers: VecDeque<Waker>, } impl FreeList { @@ -64,57 +62,32 @@ impl FreeList { Self { free: (0..32).collect(), working: VecDeque::new(), + wakers: VecDeque::new(), } } } -pub struct AdapterPort<'a> { +pub struct AdapterPort { pub nport: u32, regs_base: PAddr, - slots: [CommandSlot<'a>; 32], + cmdlist: CommandList, free_list: Spin<FreeList>, - free_list_wait: WaitList, - - /// Holds the command list. - /// **DO NOT USE IT DIRECTLY** - _page: Page, - - cmdlist_base: PAddr, - fis_base: PAddr, stats: AdapterPortStats, } -impl<'a> AdapterPort<'a> { +impl AdapterPort { pub fn new(base: PAddr, nport: u32) -> Self { - let page = Page::alloc(); - let cmdlist_base = page.start(); - let cmdlist_size = 32 * size_of::<CommandHeader>(); - let fis_base = cmdlist_base + cmdlist_size; - - let (mut cmdheaders, _) = page.as_memblk().split_at(cmdlist_size); - let slots = core::array::from_fn(move |_| { - let (cmdheader, next) = cmdheaders.split_at(size_of::<CommandHeader>()); - cmdheaders = next; - CommandSlot::new(unsafe { cmdheader.as_ptr().as_mut() }) - }); - Self { nport, regs_base: base + 0x100 + 0x80 * nport as usize, - slots, + cmdlist: CommandList::new(), free_list: Spin::new(FreeList::new()), - free_list_wait: WaitList::new(), - _page: page, stats: AdapterPortStats::new(), - cmdlist_base, - fis_base, } } -} -impl AdapterPort<'_> { fn command_list_base(&self) -> Register<u64> { Register::new(self.regs_base + 0x00) } @@ -148,25 +121,16 @@ impl AdapterPort<'_> { } async fn get_free_slot(&self) -> u32 { - loop { - let mut wait = pin!(self.free_list_wait.prepare_to_wait()); - - { - let mut free_list = self.free_list.lock_irq(); - - if let Some(slot) = free_list.free.pop_front() { - return slot; - } - - wait.as_mut().add_to_wait_list(); - - if let Some(slot) = free_list.free.pop_front() { - return slot; - } + core::future::poll_fn(|ctx| { + let mut free_list = self.free_list.lock_irq(); + if let Some(slot) = free_list.free.pop_front() { + return Poll::Ready(slot); } - wait.await; - } + free_list.wakers.push_back(ctx.waker().clone()); + Poll::Pending + }) + .await } fn save_working(&self, slot: u32) { @@ -174,8 +138,10 @@ impl AdapterPort<'_> { } fn release_free_slot(&self, slot: u32) { - self.free_list.lock_irq().free.push_back(slot); - self.free_list_wait.notify_one(); + let mut free_list = self.free_list.lock_irq(); + + free_list.free.push_back(slot); + free_list.wakers.drain(..).for_each(|waker| waker.wake()); } pub fn handle_interrupt(&self) { @@ -189,7 +155,7 @@ impl AdapterPort<'_> { return true; } - self.slots[n as usize].handle_irq(); + self.cmdlist.get(n as usize).handle_irq(); self.stats.inc_int_fired(); false @@ -218,7 +184,7 @@ impl AdapterPort<'_> { cmdtable.setup(cmd); let slot_index = self.get_free_slot().await; - let slot = &self.slots[slot_index as usize]; + let slot = self.cmdlist.get(slot_index as usize); slot.prepare_command(&cmdtable, cmd.write()); self.save_working(slot_index); @@ -231,10 +197,9 @@ impl AdapterPort<'_> { self.stats.inc_cmd_sent(); - if let Err(_) = slot.wait_finish().await { + slot.wait_finish().await.inspect_err(|_| { self.stats.inc_cmd_error(); - return Err(EIO); - }; + })?; self.release_free_slot(slot_index); Ok(()) @@ -253,8 +218,9 @@ impl AdapterPort<'_> { self.stop_command()?; self.command_list_base() - .write(self.cmdlist_base.addr() as u64); - self.fis_base().write(self.fis_base.addr() as u64); + .write(self.cmdlist.cmdlist_base().addr() as u64); + self.fis_base() + .write(self.cmdlist.recv_fis_base().addr() as u64); self.interrupt_enable().write_once(PORT_IE_DEFAULT); @@ -279,7 +245,7 @@ impl AdapterPort<'_> { } #[async_trait] -impl BlockRequestQueue for AdapterPort<'_> { +impl BlockRequestQueue for AdapterPort { fn max_request_pages(&self) -> u64 { 1024 } diff --git a/src/driver/ahci/slot.rs b/src/driver/ahci/slot.rs index 745a84b5..06c6f2ec 100644 --- a/src/driver/ahci/slot.rs +++ b/src/driver/ahci/slot.rs @@ -1,23 +1,37 @@ -use core::pin::pin; +use core::cell::UnsafeCell; +use core::ptr::NonNull; +use core::task::{Poll, Waker}; -use eonix_mm::address::Addr as _; -use eonix_sync::{Spin, SpinIrq as _, WaitList}; +use eonix_mm::address::{Addr as _, PAddr}; +use eonix_sync::{Spin, SpinIrq as _}; use super::command_table::CommandTable; use super::CommandHeader; +use crate::kernel::constants::EIO; +use crate::kernel::mem::paging::AllocZeroed; +use crate::kernel::mem::{Page, PageExt}; use crate::KResult; +pub struct CommandList { + base: NonNull<u8>, + _page: Page, +} + +unsafe impl Send for CommandList {} +unsafe impl Sync for CommandList {} + pub struct CommandSlot<'a> { - /// # Usage - /// `inner.cmdheader` might be used in irq handler. So in order to wait for - /// commands to finish, we should use `lock_irq` on `inner` - inner: Spin<CommandSlotInner<'a>>, - wait_list: WaitList, + cmdheader: &'a UnsafeCell<CommandHeader>, + /// [`Self::control`] might be used in irq handlers. + control: &'a Spin<SlotControl>, } -struct CommandSlotInner<'a> { +unsafe impl Send for CommandSlot<'_> {} +unsafe impl Sync for CommandSlot<'_> {} + +struct SlotControl { state: SlotState, - cmdheader: &'a mut CommandHeader, + waker: Option<Waker>, } #[derive(Debug, PartialEq, Eq, Clone, Copy)] @@ -28,32 +42,103 @@ enum SlotState { Error, } -impl<'a> CommandSlot<'a> { - pub fn new(cmdheader: &'a mut CommandHeader) -> Self { +impl CommandList { + fn cmdheaders(&self) -> &[UnsafeCell<CommandHeader>; 32] { + unsafe { self.base.cast().as_ref() } + } + + fn controls_ptr(base: NonNull<u8>) -> NonNull<Spin<SlotControl>> { + // 24 bytes for SlotControl and extra 8 bytes for Spin. + const_assert_eq!(size_of::<Spin<SlotControl>>(), 32); + + unsafe { base.add(size_of::<UnsafeCell<CommandHeader>>() * 32).cast() } + } + + fn controls(&self) -> &[Spin<SlotControl>; 32] { + unsafe { Self::controls_ptr(self.base).cast().as_ref() } + } + + pub fn cmdlist_base(&self) -> PAddr { + self._page.start() + } + + pub fn recv_fis_base(&self) -> PAddr { + self._page.start() + + (size_of::<UnsafeCell<CommandHeader>>() + size_of::<Spin<SlotControl>>()) * 32 + } + + pub fn get(&self, index: usize) -> CommandSlot { + CommandSlot { + cmdheader: &self.cmdheaders()[index], + control: &self.controls()[index], + } + } + + pub fn new() -> Self { + let page = Page::zeroed(); + let base = page.get_ptr(); + + let controls_ptr = Self::controls_ptr(base); + + for i in 0..32 { + unsafe { + controls_ptr.add(i).write(Spin::new(SlotControl { + state: SlotState::Idle, + waker: None, + })); + } + } + Self { - inner: Spin::new(CommandSlotInner { - state: SlotState::Idle, - cmdheader, - }), - wait_list: WaitList::new(), + base: page.get_ptr(), + _page: page, } } +} +impl Drop for CommandList { + fn drop(&mut self) { + let controls_ptr = Self::controls_ptr(self.base); + + for i in 0..32 { + unsafe { + controls_ptr.add(i).drop_in_place(); + } + } + } +} + +impl CommandSlot<'_> { pub fn handle_irq(&self) { - let mut inner = self.inner.lock(); - debug_assert_eq!(inner.state, SlotState::Working); + // We are already in the IRQ handler. + let mut control = self.control.lock(); + assert_eq!(control.state, SlotState::Working); + + let cmdheader = unsafe { + // SAFETY: The IRQ handler is only called after the command + // is finished. + &mut *self.cmdheader.get() + }; // TODO: Check errors. - inner.state = SlotState::Finished; - inner.cmdheader.bytes_transferred = 0; - inner.cmdheader.prdt_length = 0; + cmdheader.bytes_transferred = 0; + cmdheader.prdt_length = 0; - self.wait_list.notify_all(); + control.state = SlotState::Finished; + + if let Some(waker) = control.waker.take() { + waker.wake(); + } } pub fn prepare_command(&self, cmdtable: &CommandTable, write: bool) { - let mut inner = self.inner.lock_irq(); - let cmdheader = &mut inner.cmdheader; + let mut control = self.control.lock_irq(); + assert_eq!(control.state, SlotState::Idle); + + let cmdheader = unsafe { + // SAFETY: We are in the idle state. + &mut *self.cmdheader.get() + }; cmdheader.first = 0x05; // FIS type @@ -63,36 +148,37 @@ impl<'a> CommandSlot<'a> { cmdheader.second = 0x00; - cmdheader.prdt_length = cmdtable.prdt_len(); + cmdheader.prdt_length = cmdtable.prdt_len() as u16; cmdheader.bytes_transferred = 0; cmdheader.command_table_base = cmdtable.base().addr() as u64; cmdheader._reserved = [0; 4]; - inner.state = SlotState::Working; + control.state = SlotState::Working; } pub async fn wait_finish(&self) -> KResult<()> { - let mut inner = loop { - let mut wait = pin!(self.wait_list.prepare_to_wait()); - - { - let inner = self.inner.lock_irq(); - if inner.state != SlotState::Working { - break inner; + core::future::poll_fn(|ctx| { + let mut control = self.control.lock_irq(); + + match control.state { + SlotState::Idle => unreachable!("Poll called in idle state"), + SlotState::Working => { + control.waker = Some(ctx.waker().clone()); + Poll::Pending } - wait.as_mut().add_to_wait_list(); + SlotState::Finished => { + control.state = SlotState::Idle; + Poll::Ready(Ok(())) + } + SlotState::Error => { + control.state = SlotState::Idle; - if inner.state != SlotState::Working { - break inner; + // TODO: Report errors. + Poll::Ready(Err(EIO)) } } - - wait.await; - }; - - inner.state = SlotState::Idle; - - Ok(()) + }) + .await } } diff --git a/src/driver/e1000e.rs b/src/driver/e1000e.rs index f026efcb..73143c2c 100644 --- a/src/driver/e1000e.rs +++ b/src/driver/e1000e.rs @@ -1,6 +1,5 @@ use alloc::boxed::Box; use alloc::sync::Arc; -use alloc::vec::Vec; use core::ptr::NonNull; use async_trait::async_trait; @@ -10,8 +9,7 @@ use eonix_sync::SpinIrq; use crate::kernel::constants::{EAGAIN, EFAULT, EINVAL, EIO}; use crate::kernel::interrupt::register_irq_handler; -use crate::kernel::mem::{AsMemoryBlock, Page, PhysAccess}; -use crate::kernel::mem::paging::{self, AllocZeroed}; +use crate::kernel::mem::{PageExcl, PageExt, PhysAccess}; use crate::kernel::pcie::{self, Header, PCIDevice, PCIDriver, PciError}; use crate::net::netdev; use crate::prelude::*; @@ -56,13 +54,13 @@ struct E1000eDev { id: u32, regs: Registers, - rt_desc_page: Page, + rt_desc_page: PageExcl, rx_head: Option<u32>, rx_tail: Option<u32>, tx_tail: Option<u32>, - rx_buffers: Option<Box<Vec<Page>>>, - tx_buffers: Option<Box<Vec<Page>>>, + rx_buffers: Box<[PageExcl; RX_DESC_SIZE]>, + tx_buffers: Box<[Option<PageExcl>; TX_DESC_SIZE]>, } fn test(val: u32, bit: u32) -> bool { @@ -197,7 +195,7 @@ impl netdev::Netdev for E1000eDev { break; } - let ref mut desc = self.rx_desc_table()[next_tail as usize]; + let desc = unsafe { &mut self.rx_desc_table()[next_tail as usize] }; if !test(desc.status as u32, defs::RXD_STAT_DD as u32) { Err(EIO)?; } @@ -205,11 +203,8 @@ impl netdev::Netdev for E1000eDev { desc.status = 0; let len = desc.length as usize; - let buffers = self.rx_buffers.as_mut().ok_or(EIO)?; - let data = unsafe { - // SAFETY: No one could be writing to the buffer at this point. - &buffers[next_tail as usize].as_memblk().as_bytes()[..len] - }; + let buffer = &self.rx_buffers[next_tail as usize]; + let data = &buffer.as_bytes()[..len]; println_debug!("e1000e: received {len} bytes, {:?}", PrintableBytes(data)); self.rx_tail = Some(next_tail); @@ -227,20 +222,17 @@ impl netdev::Netdev for E1000eDev { return Err(EAGAIN); } - let ref mut desc = self.tx_desc_table()[tail as usize]; + let desc = unsafe { &mut self.tx_desc_table()[tail as usize] }; if !test(desc.status as u32, defs::TXD_STAT_DD as u32) { return Err(EIO); } - let buffer_page = Page::alloc(); + let mut buffer_page = PageExcl::alloc(); if buf.len() > buffer_page.len() { return Err(EFAULT); } - unsafe { - // SAFETY: We are the only one writing to this memory block. - buffer_page.as_memblk().as_bytes_mut()[..buf.len()].copy_from_slice(buf); - } + buffer_page.as_bytes_mut()[..buf.len()].copy_from_slice(buf); desc.buffer = PAddr::from(buffer_page.pfn()).addr() as u64; desc.length = buf.len() as u16; @@ -250,9 +242,8 @@ impl netdev::Netdev for E1000eDev { self.tx_tail = Some(next_tail); self.regs.write(defs::REG_TDT, next_tail); - // TODO: check if the packets are sent and update self.tx_head state - - Ok(()) + unimplemented!("Check if the packets are sent and update self.tx_head state"); + // Ok(()) } } @@ -325,26 +316,26 @@ impl E1000eDev { Ok(()) } - fn reset(&self) -> Result<(), u32> { + fn reset(regs: &Registers) -> Result<(), u32> { // disable interrupts so we won't mess things up - self.regs.write(defs::REG_IMC, 0xffffffff); + regs.write(defs::REG_IMC, 0xffffffff); - let ctrl = self.regs.read(defs::REG_CTRL); - self.regs.write(defs::REG_CTRL, ctrl | defs::CTRL_GIOD); + let ctrl = regs.read(defs::REG_CTRL); + regs.write(defs::REG_CTRL, ctrl | defs::CTRL_GIOD); - while self.regs.read(defs::REG_STAT) & defs::STAT_GIOE != 0 { + while regs.read(defs::REG_STAT) & defs::STAT_GIOE != 0 { // wait for link up } - let ctrl = self.regs.read(defs::REG_CTRL); - self.regs.write(defs::REG_CTRL, ctrl | defs::CTRL_RST); + let ctrl = regs.read(defs::REG_CTRL); + regs.write(defs::REG_CTRL, ctrl | defs::CTRL_RST); - while self.regs.read(defs::REG_CTRL) & defs::CTRL_RST != 0 { + while regs.read(defs::REG_CTRL) & defs::CTRL_RST != 0 { // wait for reset } // disable interrupts again - self.regs.write(defs::REG_IMC, 0xffffffff); + regs.write(defs::REG_IMC, 0xffffffff); Ok(()) } @@ -361,64 +352,45 @@ impl E1000eDev { Ok(()) } - pub fn new(base: PAddr, irq_no: usize) -> Result<Self, u32> { - let page = Page::zeroed(); + pub fn new(base: PAddr, irq_no: usize) -> KResult<Self> { + let regs = Registers::new(base); + Self::reset(®s)?; - let mut dev = Self { + let dev = Self { irq_no, - mac: [0; 6], + mac: regs.read_as(0x5400), status: netdev::LinkStatus::Down, speed: netdev::LinkSpeed::SpeedUnknown, id: netdev::alloc_id(), - regs: Registers::new(base), - rt_desc_page: page, + regs, + rt_desc_page: PageExcl::zeroed(), rx_head: None, rx_tail: None, tx_tail: None, - rx_buffers: None, - tx_buffers: None, + rx_buffers: Box::new(core::array::from_fn(|_| PageExcl::alloc_order(2))), + tx_buffers: Box::new([const { None }; 32]), }; - dev.reset()?; - - dev.mac = dev.regs.read_as(0x5400); - dev.tx_buffers = Some(Box::new(Vec::with_capacity(TX_DESC_SIZE))); - - let mut rx_buffers = Box::new(Vec::with_capacity(RX_DESC_SIZE)); - - for index in 0..RX_DESC_SIZE { - let page = Page::alloc_order(2); - - let ref mut desc = dev.rx_desc_table()[index]; - desc.buffer = PAddr::from(page.pfn()).addr() as u64; - desc.status = 0; - - rx_buffers.push(page); - } + unsafe { + for (desc, page) in dev.rx_desc_table().into_iter().zip(dev.rx_buffers.iter()) { + desc.buffer = page.start().addr() as u64; + desc.status = 0; + } - for index in 0..TX_DESC_SIZE { - let ref mut desc = dev.tx_desc_table()[index]; - desc.status = defs::TXD_STAT_DD; + for desc in dev.tx_desc_table() { + desc.status = defs::TXD_STAT_DD; + } } - dev.rx_buffers = Some(rx_buffers); - Ok(dev) } - fn rx_desc_table(&self) -> &mut [RxDescriptor; RX_DESC_SIZE] { - unsafe { - // SAFETY: TODO - self.rt_desc_page.as_memblk().as_ptr().as_mut() - } + unsafe fn rx_desc_table(&self) -> &mut [RxDescriptor; RX_DESC_SIZE] { + self.rt_desc_page.get_ptr().cast().as_mut() } - fn tx_desc_table(&self) -> &mut [TxDescriptor; TX_DESC_SIZE] { - let (_, right) = self.rt_desc_page.as_memblk().split_at(0x200); - unsafe { - // SAFETY: TODO - right.as_ptr().as_mut() - } + unsafe fn tx_desc_table(&self) -> &mut [TxDescriptor; TX_DESC_SIZE] { + self.rt_desc_page.get_ptr().add(0x200).cast().as_mut() } } @@ -426,12 +398,8 @@ impl Drop for E1000eDev { fn drop(&mut self) { assert_eq!(self.status, netdev::LinkStatus::Down); - if let Some(_) = self.rx_buffers.take() {} - - // TODO: we should wait until all packets are sent - if let Some(_) = self.tx_buffers.take() {} - - let _ = self.rt_desc_page; + // TODO: we should wait until all packets are sent before dropping + // tx buffers. } } diff --git a/src/driver/virtio/virtio_blk.rs b/src/driver/virtio/virtio_blk.rs index 1aee679a..c5a3c3d2 100644 --- a/src/driver/virtio/virtio_blk.rs +++ b/src/driver/virtio/virtio_blk.rs @@ -12,7 +12,7 @@ use virtio_drivers::Hal; use crate::io::Chunks; use crate::kernel::block::{BlockDeviceRequest, BlockRequestQueue}; use crate::kernel::constants::EIO; -use crate::kernel::mem::{AsMemoryBlock, Page}; +use crate::kernel::mem::{Page, PageExt}; use crate::prelude::KResult; pub struct HAL; @@ -24,11 +24,10 @@ unsafe impl Hal for HAL { ) -> (virtio_drivers::PhysAddr, core::ptr::NonNull<u8>) { let page = Page::alloc_at_least(pages); - let paddr = page.start().addr(); - let ptr = page.as_memblk().as_byte_ptr(); - page.into_raw(); + let ptr = page.get_ptr(); + let pfn = page.into_raw(); - (paddr, ptr) + (PAddr::from(pfn).addr(), ptr) } unsafe fn dma_dealloc( @@ -91,15 +90,14 @@ where buffer, } => { let mut dev = self.lock(); - for ((start, len), buffer_page) in + for ((start, sectors), buffer_page) in Chunks::new(sector as usize, count as usize, 8).zip(buffer.iter()) { - let buffer = unsafe { - // SAFETY: Pages in `req.buffer` are guaranteed to be exclusively owned by us. - &buffer_page.as_memblk().as_bytes()[..len as usize * 512] - }; + let len = sectors * 512; + let pg = buffer_page.lock(); - dev.write_blocks(start, buffer).map_err(|_| EIO)?; + dev.write_blocks(start, &pg.as_bytes()[..len]) + .map_err(|_| EIO)?; } } BlockDeviceRequest::Read { @@ -108,15 +106,14 @@ where buffer, } => { let mut dev = self.lock(); - for ((start, len), buffer_page) in + for ((start, sectors), buffer_page) in Chunks::new(sector as usize, count as usize, 8).zip(buffer.iter()) { - let buffer = unsafe { - // SAFETY: Pages in `req.buffer` are guaranteed to be exclusively owned by us. - &mut buffer_page.as_memblk().as_bytes_mut()[..len as usize * 512] - }; + let len = sectors * 512; + let mut pg = buffer_page.lock(); - dev.read_blocks(start, buffer).map_err(|_| EIO)?; + dev.read_blocks(start, &mut pg.as_bytes_mut()[..len]) + .map_err(|_| EIO)?; } } } diff --git a/src/fs/fat32.rs b/src/fs/fat32.rs index c4e247c8..9a4e03ec 100644 --- a/src/fs/fat32.rs +++ b/src/fs/fat32.rs @@ -14,8 +14,7 @@ use crate::io::{Buffer, ByteBuffer, UninitBuffer}; use crate::kernel::block::{BlockDevice, BlockDeviceRequest}; use crate::kernel::constants::{EINVAL, EIO}; use crate::kernel::mem::{ - AsMemoryBlock, CachePage, CachePageStream, Page, PageCache, - PageCacheBackendOps, + CachePage, CachePageStream, Page, PageCache, PageCacheBackendOps, PageExcl, PageExt, }; use crate::kernel::timer::Instant; use crate::kernel::vfs::dentry::Dentry; @@ -33,7 +32,7 @@ use crate::KResult; struct Cluster(u32); #[repr(transparent)] -#[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord)] +#[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Debug)] struct RawCluster(pub u32); impl RawCluster { @@ -64,7 +63,7 @@ impl Cluster { const SECTOR_SIZE: usize = 512; -#[derive(Clone, Copy)] +#[derive(Clone, Copy, Debug)] #[repr(C, packed)] struct Bootsector { jmp: [u8; 3], @@ -296,10 +295,8 @@ impl InodeFileOps for FileInode { for cluster in cluster_iter { fs.read_cluster(cluster, &buffer_page).await?; - let data = unsafe { - // SAFETY: We are the only one holding this page. - &buffer_page.as_memblk().as_bytes()[inner_offset..] - }; + let pg = buffer_page.lock(); + let data = &pg.as_bytes()[inner_offset..]; let end = offset + data.len(); let real_end = end.min(self.info.lock().size as usize); @@ -334,7 +331,7 @@ struct DirInode { sb: SbRef<FatFs>, // TODO: Use the new PageCache... - dir_pages: RwLock<Vec<Page>>, + dir_pages: RwLock<Vec<PageExcl>>, } impl DirInode { @@ -369,7 +366,7 @@ impl DirInode { let clusters = ClusterIterator::new(fat.as_ref(), self.cluster); for cluster in clusters { - let page = Page::alloc(); + let page = PageExcl::alloc(); fs.read_cluster(cluster, &page).await?; dir_pages.push(page); @@ -378,7 +375,7 @@ impl DirInode { Ok(()) } - async fn get_dir_pages(&self) -> KResult<impl Deref<Target = Vec<Page>> + use<'_>> { + async fn get_dir_pages(&self) -> KResult<impl Deref<Target = Vec<PageExcl>> + use<'_>> { { let dir_pages = self.dir_pages.read().await; if !dir_pages.is_empty() { @@ -426,12 +423,7 @@ impl InodeDirOps for DirInode { let sb = self.sb.get()?; let dir_pages = self.get_dir_pages().await?; - let dir_data = dir_pages.iter().map(|page| { - unsafe { - // SAFETY: No one could be writing to it. - page.as_memblk().as_bytes() - } - }); + let dir_data = dir_pages.iter().map(|pg| pg.as_bytes()); let raw_dirents = dir_data .map(as_raw_dirents) @@ -475,12 +467,10 @@ impl InodeDirOps for DirInode { let inner_offset = offset % cluster_size; let inner_raw_dirent_offset = inner_offset / core::mem::size_of::<dir::RawDirEntry>(); - let dir_data = dir_pages.iter().skip(cluster_offset).map(|page| { - unsafe { - // SAFETY: No one could be writing to it. - page.as_memblk().as_bytes() - } - }); + let dir_data = dir_pages + .iter() + .skip(cluster_offset) + .map(|pg| pg.as_bytes()); let raw_dirents = dir_data .map(as_raw_dirents) diff --git a/src/io.rs b/src/io.rs index 81c61d1c..d7094f6d 100644 --- a/src/io.rs +++ b/src/io.rs @@ -1,5 +1,5 @@ use core::mem::MaybeUninit; -use core::cmp; +use core::ops::{Add, AddAssign, Sub}; use crate::kernel::constants::EFAULT; use crate::prelude::*; @@ -238,18 +238,26 @@ impl Buffer for ByteBuffer<'_> { } } +pub trait Integer: + Add<Output = Self> + Sub<Output = Self> + AddAssign + Copy + PartialOrd + Ord +{ +} + +impl Integer for u64 {} +impl Integer for usize {} + /// Iterator that generates chunks of a given length from a start index /// until the end of the total length. /// /// The iterator returns a tuple of (start, len) for each chunk. -pub struct Chunks { - end: usize, - cur: usize, - chunk_len: usize, +pub struct Chunks<T: Integer> { + end: T, + cur: T, + chunk_len: T, } -impl Chunks { - pub const fn new(start: usize, total_len: usize, chunk_len: usize) -> Self { +impl<T: Integer> Chunks<T> { + pub fn new(start: T, total_len: T, chunk_len: T) -> Self { Self { end: start + total_len, cur: start, @@ -258,8 +266,8 @@ impl Chunks { } } -impl Iterator for Chunks { - type Item = (usize, usize); +impl<T: Integer> Iterator for Chunks<T> { + type Item = (T, T); fn next(&mut self) -> Option<Self::Item> { if self.cur >= self.end { @@ -267,7 +275,7 @@ impl Iterator for Chunks { } let start = self.cur; - let len = cmp::min(self.chunk_len, self.end - start); + let len = self.chunk_len.min(self.end - start); self.cur += self.chunk_len; Some((start, len)) diff --git a/src/kernel/block.rs b/src/kernel/block.rs index 4c2750a0..8e017336 100644 --- a/src/kernel/block.rs +++ b/src/kernel/block.rs @@ -9,9 +9,9 @@ use mbr::MBRPartTable; use super::constants::ENOENT; use super::mem::paging::Page; -use super::mem::AsMemoryBlock as _; +use super::mem::PageExt; use super::vfs::types::DeviceId; -use crate::io::{Buffer, FillResult}; +use crate::io::{Buffer, Chunks, FillResult}; use crate::kernel::constants::{EEXIST, EINVAL}; use crate::prelude::*; @@ -190,177 +190,72 @@ impl BlockDevice { /// `offset` - offset in bytes /// pub async fn read_some(&self, offset: usize, buffer: &mut dyn Buffer) -> KResult<FillResult> { - let mut sector_start = offset as u64 / 512; - let mut first_sector_offset = offset as u64 % 512; - let mut sector_count = (first_sector_offset + buffer.total() as u64 + 511) / 512; - - let mut nfilled = 0; - 'outer: while sector_count != 0 { - let pages: &[Page]; - let page: Option<Page>; - let page_vec: Option<Vec<Page>>; - - let nread; - - match sector_count { - count if count <= 8 => { - nread = count; - - let _page = Page::alloc(); - page = Some(_page); - pages = core::slice::from_ref(page.as_ref().unwrap()); + let sector_start = offset as u64 / 512; + let mut first_sector_offset = offset % 512; + let nr_sectors = (first_sector_offset + buffer.total() + 511) / 512; + + let nr_sectors_per_batch = self.queue().max_request_pages() / 2 * 2 * 8; + + let mut nr_filled = 0; + for (start, nr_batch) in Chunks::new(sector_start, nr_sectors as u64, nr_sectors_per_batch) + { + let (page_slice, page, mut page_vec); + match nr_batch { + ..=8 => { + page = Page::alloc(); + page_slice = core::slice::from_ref(&page); } - count if count <= 16 => { - nread = count; - - let _pages = Page::alloc_order(1); - page = Some(_pages); - pages = core::slice::from_ref(page.as_ref().unwrap()); + ..=16 => { + page = Page::alloc_order(1); + page_slice = core::slice::from_ref(&page); + } + ..=32 => { + page = Page::alloc_order(2); + page_slice = core::slice::from_ref(&page); } count => { - nread = count.min(self.queue().max_request_pages()); + let nr_huge_pages = count as usize / 32; + let nr_small_pages = ((count as usize % 32) + 7) / 8; - let npages = (nread + 15) / 16; - let mut _page_vec = Vec::with_capacity(npages as usize); - for _ in 0..npages { - _page_vec.push(Page::alloc_order(1)); - } - page_vec = Some(_page_vec); - pages = page_vec.as_ref().unwrap().as_slice(); + let nr_pages = nr_huge_pages + nr_small_pages; + page_vec = Vec::with_capacity(nr_pages); + + page_vec.resize_with(nr_huge_pages, || Page::alloc_order(2)); + page_vec.resize_with(nr_pages, || Page::alloc()); + page_slice = &page_vec; } } let req = BlockDeviceRequest::Read { - sector: sector_start, - count: nread, - buffer: &pages, + sector: start, + count: nr_batch, + buffer: page_slice, }; self.commit_request(req).await?; - for page in pages.iter() { - // SAFETY: We are the only owner of the page so no one could be mutating it. - let data = unsafe { &page.as_memblk().as_bytes()[first_sector_offset as usize..] }; + for page in page_slice { + let pg = page.lock(); + let data = &pg.as_bytes()[first_sector_offset..]; first_sector_offset = 0; - match buffer.fill(data)? { - FillResult::Done(n) => nfilled += n, - FillResult::Partial(n) => { - nfilled += n; - break 'outer; - } - FillResult::Full => { - break 'outer; - } - } - } - - sector_start += nread; - sector_count -= nread; - } - - if nfilled == buffer.total() { - Ok(FillResult::Done(nfilled)) - } else { - Ok(FillResult::Partial(nfilled)) - } - } - - /// Write some data to the block device, may involve some copy and fragmentation - /// - /// # Arguments - /// `offset` - offset in bytes - /// `data` - data to write - /// - pub async fn write_some(&self, offset: usize, data: &[u8]) -> KResult<usize> { - let mut sector_start = offset as u64 / 512; - let mut first_sector_offset = offset as u64 % 512; - let mut remaining_data = data; - let mut nwritten = 0; - - while !remaining_data.is_empty() { - let pages: &[Page]; - let page: Option<Page>; - let page_vec: Option<Vec<Page>>; - - // Calculate sectors needed for this write - let write_end = first_sector_offset + remaining_data.len() as u64; - let sector_count = ((write_end + 511) / 512).min(self.queue().max_request_pages()); - - match sector_count { - count if count <= 8 => { - let _page = Page::alloc(); - page = Some(_page); - pages = core::slice::from_ref(page.as_ref().unwrap()); - } - count if count <= 16 => { - let _pages = Page::alloc_order(1); - page = Some(_pages); - pages = core::slice::from_ref(page.as_ref().unwrap()); - } - count => { - let npages = (count + 15) / 16; - let mut _page_vec = Vec::with_capacity(npages as usize); - for _ in 0..npages { - _page_vec.push(Page::alloc_order(1)); - } - page_vec = Some(_page_vec); - pages = page_vec.as_ref().unwrap().as_slice(); - } - } - - if first_sector_offset != 0 || remaining_data.len() < (sector_count * 512) as usize { - let read_req = BlockDeviceRequest::Read { - sector: sector_start, - count: sector_count, - buffer: pages, - }; - self.commit_request(read_req).await?; - } - - let mut data_offset = 0; - let mut page_offset = first_sector_offset as usize; - - for page in pages.iter() { - // SAFETY: We own the page and can modify it - let page_data = unsafe { - let memblk = page.as_memblk(); - core::slice::from_raw_parts_mut(memblk.addr().get() as *mut u8, memblk.len()) - }; + nr_filled += buffer.fill(data)?.allow_partial(); - let copy_len = - (remaining_data.len() - data_offset).min(page_data.len() - page_offset); - - if copy_len == 0 { - break; - } - - page_data[page_offset..page_offset + copy_len] - .copy_from_slice(&remaining_data[data_offset..data_offset + copy_len]); - - data_offset += copy_len; - page_offset = 0; // Only first page has offset - - if data_offset >= remaining_data.len() { + if buffer.available() == 0 { break; } } - let write_req = BlockDeviceRequest::Write { - sector: sector_start, - count: sector_count, - buffer: pages, - }; - self.commit_request(write_req).await?; - - let bytes_written = data_offset; - nwritten += bytes_written; - remaining_data = &remaining_data[bytes_written..]; - sector_start += sector_count; - first_sector_offset = 0; + if buffer.available() == 0 { + break; + } } - Ok(nwritten) + if buffer.available() == 0 { + Ok(FillResult::Done(nr_filled)) + } else { + Ok(FillResult::Partial(nr_filled)) + } } } diff --git a/src/kernel/mem.rs b/src/kernel/mem.rs index c147306e..bfc826bf 100644 --- a/src/kernel/mem.rs +++ b/src/kernel/mem.rs @@ -8,9 +8,9 @@ mod mm_list; mod page_alloc; mod page_cache; -pub use access::{AsMemoryBlock, MemoryBlock, PhysAccess}; +pub use access::PhysAccess; pub(self) use mm_area::MMArea; pub use mm_list::{handle_kernel_page_fault, FileMapping, MMList, Mapping, Permission}; pub use page_alloc::{GlobalPageAlloc, RawPage}; pub use page_cache::{CachePage, CachePageStream, PageCache, PageCacheBackendOps}; -pub use paging::{Page, PageBuffer}; +pub use paging::{Page, PageBuffer, PageExcl, PageExt}; diff --git a/src/kernel/mem/access.rs b/src/kernel/mem/access.rs index c4e9042b..328dcfbd 100644 --- a/src/kernel/mem/access.rs +++ b/src/kernel/mem/access.rs @@ -1,23 +1,7 @@ -use core::num::NonZero; use core::ptr::NonNull; use eonix_hal::mm::ArchPhysAccess; use eonix_mm::address::{PAddr, PhysAccess as _PhysAccess}; -/// A block of memory starting at a non-zero address and having a specific length. -/// -/// This struct is used to represent a memory block that can be accessed -/// in the kernel space. -pub struct MemoryBlock { - addr: NonZero<usize>, - len: usize, -} - -pub trait AsMemoryBlock { - /// Translate the physical page the page object pointing to into kernel - /// accessible pointer. Use it with care. - fn as_memblk(&self) -> MemoryBlock; -} - pub trait PhysAccess { /// Translate the data that this address is pointing to into kernel /// accessible pointer. Use it with care. @@ -31,107 +15,6 @@ pub trait PhysAccess { unsafe fn as_ptr<T>(&self) -> NonNull<T>; } -impl MemoryBlock { - /// Create a new `MemoryBlock` with the given address and length. - /// - /// # Safety - /// The caller must ensure that the address is valid. - /// Otherwise, it may lead to undefined behavior. - pub unsafe fn new(addr: NonZero<usize>, len: usize) -> Self { - Self { addr, len } - } - - /// Get the start address of the memory block. - #[allow(dead_code)] - pub fn addr(&self) -> NonZero<usize> { - self.addr - } - - /// Get the length of the memory block. - #[allow(dead_code)] - pub fn len(&self) -> usize { - self.len - } - - /// Split the memory block into two parts at the given offset. - pub fn split_at(&self, at: usize) -> (Self, Self) { - if at > self.len { - panic!("Out of bounds"); - } - - let rhs_start = self.addr.checked_add(at).expect("Overflow"); - - let lhs = unsafe { Self::new(self.addr, at) }; - let rhs = unsafe { Self::new(rhs_start, self.len - at) }; - - (lhs, rhs) - } - - /// Provide a pointer to the data. - /// - /// # Safety - /// Using the returned pointer is undefined behavior if the address is not - /// properly aligned or the size is not equal to the size of `T`. - pub unsafe fn as_ptr_unchecked<T>(&self) -> NonNull<T> { - // SAFETY: `self.addr` is a non-zero value. - NonNull::new_unchecked(self.addr.get() as *mut T) - } - - /// Provide a pointer to the data. - /// - /// # Panic - /// Panic if the address is not properly aligned. - pub fn as_ptr<T>(&self) -> NonNull<T> { - let alignment = align_of::<T>(); - - if self.addr.get() % alignment != 0 { - panic!("Alignment error"); - } - - unsafe { - // SAFETY: We've checked that `self.addr` is properly aligned. - self.as_ptr_unchecked() - } - } - - /// Provide a pointer to the bytes. - pub fn as_byte_ptr(&self) -> NonNull<u8> { - unsafe { - // SAFETY: No alignment check is needed for bytes. - self.as_ptr_unchecked() - } - } - - /// Provide immutable access to the data it pointed to. - /// - /// # Safety - /// This function is unsafe because it returns an immutable reference with - /// a created lifetime. - /// - /// The caller must ensure that the data has no other mutable aliases while - /// the reference is in use. Otherwise, it may lead to undefined behavior. - pub unsafe fn as_bytes<'a>(&self) -> &'a [u8] { - core::slice::from_raw_parts(self.as_ptr_unchecked().as_ptr(), self.len) - } - - /// Provide mutable access to the data it pointed to. - /// - /// # Panic - /// Panic if the address is not properly aligned or the size is not - /// equal to the size of `T`. - /// - /// # Safety - /// This function is unsafe because it returns a mutable reference with a - /// created lifetime. - /// - /// The caller must ensure that the data has no other immutable or mutable - /// aliases while the reference is in use. - /// Otherwise, it may lead to undefined behavior. - pub unsafe fn as_bytes_mut<'a>(&mut self) -> &'a mut [u8] { - core::slice::from_raw_parts_mut(self.as_ptr_unchecked().as_ptr(), self.len) - } -} - impl PhysAccess for PAddr { unsafe fn as_ptr<T>(&self) -> NonNull<T> { ArchPhysAccess::as_ptr(*self) diff --git a/src/kernel/mem/allocator.rs b/src/kernel/mem/allocator.rs index 017180a6..9e5df69b 100644 --- a/src/kernel/mem/allocator.rs +++ b/src/kernel/mem/allocator.rs @@ -8,7 +8,7 @@ use eonix_sync::LazyLock; use slab_allocator::SlabAllocator; use super::page_alloc::RawPagePtr; -use super::{AsMemoryBlock, GlobalPageAlloc, Page}; +use super::{GlobalPageAlloc, Page, PageExt}; static SLAB_ALLOCATOR: LazyLock<SlabAllocator<RawPagePtr, GlobalPageAlloc, 9>> = LazyLock::new(|| SlabAllocator::new_in(GlobalPageAlloc)); @@ -25,7 +25,7 @@ unsafe impl GlobalAlloc for Allocator { let page_count = size >> PAGE_SIZE_BITS; let page = Page::alloc_at_least(page_count); - let ptr = page.as_memblk().as_ptr(); + let ptr = page.get_ptr(); page.into_raw(); ptr.as_ptr() diff --git a/src/kernel/mem/mm_area.rs b/src/kernel/mem/mm_area.rs index 63043535..dcbeeb63 100644 --- a/src/kernel/mem/mm_area.rs +++ b/src/kernel/mem/mm_area.rs @@ -7,9 +7,9 @@ use eonix_mm::page_table::{PageAttribute, RawAttribute, PTE}; use eonix_mm::paging::{PAGE_SIZE, PFN}; use super::mm_list::EMPTY_PAGE; -use super::paging::AllocZeroed as _; -use super::{AsMemoryBlock, Mapping, Page, Permission}; +use super::{Mapping, Page, Permission}; use crate::kernel::constants::EINVAL; +use crate::kernel::mem::{PageExcl, PageExt}; use crate::prelude::KResult; #[derive(Debug)] @@ -107,25 +107,23 @@ impl MMArea { return; } - let new_page; + let mut new_page; if *pfn == EMPTY_PAGE.pfn() { - new_page = Page::zeroed(); + new_page = PageExcl::zeroed(); } else { - new_page = Page::alloc(); + new_page = PageExcl::alloc(); unsafe { // SAFETY: `page` is CoW, which means that others won't write to it. - let old_page_data = page.as_memblk().as_bytes(); - - // SAFETY: `new_page` is exclusive owned by us. - let new_page_data = new_page.as_memblk().as_bytes_mut(); + let old_page_data = page.get_bytes_ptr().as_ref(); + let new_page_data = new_page.as_bytes_mut(); new_page_data.copy_from_slice(old_page_data); }; } attr.remove(PageAttribute::ACCESSED); - *pfn = new_page.into_raw(); + *pfn = new_page.into_page().into_raw(); } /// # Arguments @@ -158,13 +156,12 @@ impl MMArea { // Bss is embarrassing in pagecache! // We have to assume cnt_to_read < PAGE_SIZE all bss if cnt_to_read < PAGE_SIZE { - let new_page = Page::zeroed(); - unsafe { - let page_data = new_page.as_memblk().as_bytes_mut(); - page_data[..cnt_to_read] - .copy_from_slice(&page.as_memblk().as_bytes()[..cnt_to_read]); - } - *pfn = new_page.into_raw(); + let mut new_page = PageExcl::zeroed(); + + new_page.as_bytes_mut()[..cnt_to_read] + .copy_from_slice(&page.lock().as_bytes()[..cnt_to_read]); + + *pfn = new_page.into_page().into_raw(); } else { *pfn = page.clone().into_raw(); } @@ -184,13 +181,12 @@ impl MMArea { cache_page.set_dirty(); *pfn = page.clone().into_raw(); } else { - let new_page = Page::zeroed(); - unsafe { - let page_data = new_page.as_memblk().as_bytes_mut(); - page_data[..cnt_to_read] - .copy_from_slice(&page.as_memblk().as_bytes()[..cnt_to_read]); - } - *pfn = new_page.into_raw(); + let mut new_page = PageExcl::zeroed(); + + new_page.as_bytes_mut()[..cnt_to_read] + .copy_from_slice(&page.lock().as_bytes()[..cnt_to_read]); + + *pfn = new_page.into_page().into_raw(); } attr.insert(PageAttribute::WRITE); diff --git a/src/kernel/mem/mm_list.rs b/src/kernel/mem/mm_list.rs index 1a0e1530..17dc1b05 100644 --- a/src/kernel/mem/mm_list.rs +++ b/src/kernel/mem/mm_list.rs @@ -19,7 +19,7 @@ pub use page_fault::handle_kernel_page_fault; use super::address::{VAddrExt as _, VRangeExt as _}; use super::page_alloc::GlobalPageAlloc; use super::paging::AllocZeroed as _; -use super::{AsMemoryBlock, MMArea, Page}; +use super::{MMArea, Page, PageExt}; use crate::kernel::constants::{EEXIST, EFAULT, EINVAL, ENOMEM}; use crate::kernel::mem::page_alloc::RawPagePtr; use crate::prelude::*; @@ -694,12 +694,10 @@ impl MMList { unsafe { // SAFETY: We are sure that the page is valid and we have the right to access it. Page::with_raw(pte.get_pfn(), |page| { - // SAFETY: The caller guarantees that no one else is using the page. - let page_data = page.as_memblk().as_bytes_mut(); - func( - offset + idx * 0x1000, - &mut page_data[start_offset..end_offset], - ); + let mut pg = page.lock(); + let page_data = &mut pg.as_bytes_mut()[start_offset..end_offset]; + + func(offset + idx * 0x1000, page_data); }); } } diff --git a/src/kernel/mem/page_alloc/raw_page.rs b/src/kernel/mem/page_alloc/raw_page.rs index 54d4d590..08536693 100644 --- a/src/kernel/mem/page_alloc/raw_page.rs +++ b/src/kernel/mem/page_alloc/raw_page.rs @@ -81,7 +81,7 @@ pub struct RawPagePtr(NonNull<RawPage>); impl PageFlags { pub const PRESENT: u32 = 1 << 0; - // pub const LOCKED: u32 = 1 << 1; + pub const LOCKED: u32 = 1 << 1; pub const BUDDY: u32 = 1 << 2; pub const SLAB: u32 = 1 << 3; pub const DIRTY: u32 = 1 << 4; @@ -99,6 +99,13 @@ impl PageFlags { pub fn clear(&self, flag: u32) { self.0.fetch_and(!flag, Ordering::Relaxed); } + + /// Set the flag and return whether it was already set. + /// + /// If multiple flags are given, returns true if any of them were already set. + pub fn test_and_set(&self, flag: u32) -> bool { + (self.0.fetch_or(flag, Ordering::Relaxed) & flag) != 0 + } } impl RawPagePtr { diff --git a/src/kernel/mem/page_cache.rs b/src/kernel/mem/page_cache.rs index 43e4314a..6a1c04ca 100644 --- a/src/kernel/mem/page_cache.rs +++ b/src/kernel/mem/page_cache.rs @@ -316,14 +316,6 @@ impl CachePageStream { pub fn new(page: CachePage) -> Self { Self { page, cur: 0 } } - - pub fn remaining(&self) -> usize { - self.page.valid_size().saturating_sub(self.cur) - } - - pub fn is_drained(&self) -> bool { - self.cur >= self.page.valid_size() - } } impl Stream for CachePageStream { diff --git a/src/kernel/mem/paging.rs b/src/kernel/mem/paging.rs index 7e604b65..bca573fb 100644 --- a/src/kernel/mem/paging.rs +++ b/src/kernel/mem/paging.rs @@ -1,44 +1,54 @@ -use eonix_mm::paging::{Page as GenericPage, PageAlloc}; +use core::ops::Deref; +use core::ptr::NonNull; + +use eonix_mm::paging::Page as GenericPage; -use super::access::AsMemoryBlock; use super::page_alloc::GlobalPageAlloc; -use super::{MemoryBlock, PhysAccess}; +use super::PhysAccess; use crate::io::{Buffer, FillResult}; pub type Page = GenericPage<GlobalPageAlloc>; /// A buffer that wraps a page and provides a `Buffer` interface. pub struct PageBuffer { - page: Page, + page: PageExcl, offset: usize, } +pub struct PageLocked<'a> { + page: &'a Page, +} + +/// A page that is exclusively owned. +#[repr(transparent)] +pub struct PageExcl(Page); + pub trait AllocZeroed { fn zeroed() -> Self; } -impl<A: PageAlloc> AsMemoryBlock for GenericPage<A> { - fn as_memblk(&self) -> MemoryBlock { - unsafe { - // SAFETY: `self.start()` points to valid memory of length `self.len()`. - MemoryBlock::new(self.start().as_ptr::<()>().addr(), self.len()) - } +pub trait PageExt { + fn lock(&self) -> PageLocked; + + /// Get a vmem pointer to the page data as a byte slice. + fn get_bytes_ptr(&self) -> NonNull<[u8]>; + + /// Get a vmem pointer to the start of the page. + fn get_ptr(&self) -> NonNull<u8> { + self.get_bytes_ptr().cast() } } impl PageBuffer { pub fn new() -> Self { Self { - page: Page::alloc(), + page: PageExcl::alloc(), offset: 0, } } pub fn all(&self) -> &[u8] { - unsafe { - // SAFETY: The page is exclusivly owned by us. - self.page.as_memblk().as_bytes() - } + self.page.as_bytes() } pub fn data(&self) -> &[u8] { @@ -46,10 +56,7 @@ impl PageBuffer { } pub fn available_mut(&mut self) -> &mut [u8] { - unsafe { - // SAFETY: The page is exclusivly owned by us. - &mut self.page.as_memblk().as_bytes_mut()[self.offset..] - } + &mut self.page.as_bytes_mut()[self.offset..] } } @@ -83,10 +90,87 @@ impl Buffer for PageBuffer { impl AllocZeroed for Page { fn zeroed() -> Self { let page = Self::alloc(); + + page.lock().as_bytes_mut().fill(0); + + page + } +} + +impl PageExt for Page { + fn lock(&self) -> PageLocked { + // TODO: Actually perform the lock. + PageLocked { page: self } + } + + fn get_bytes_ptr(&self) -> NonNull<[u8]> { unsafe { - // SAFETY: The page is exclusivly owned by us. - page.as_memblk().as_bytes_mut().fill(0); + // SAFETY: `self.start()` can't be null. + NonNull::slice_from_raw_parts(self.start().as_ptr(), self.len()) } - page + } +} + +impl PageLocked<'_> { + pub fn as_bytes(&self) -> &[u8] { + unsafe { + // SAFETY: `self.start()` points to valid memory of length `self.len()`. + core::slice::from_raw_parts(self.start().as_ptr().as_ptr(), self.len()) + } + } + + pub fn as_bytes_mut(&mut self) -> &mut [u8] { + unsafe { + // SAFETY: `self.start()` points to valid memory of length `self.len()`. + core::slice::from_raw_parts_mut(self.start().as_ptr().as_ptr(), self.len()) + } + } +} + +impl Deref for PageLocked<'_> { + type Target = Page; + + fn deref(&self) -> &Self::Target { + self.page + } +} + +impl PageExcl { + pub fn alloc() -> Self { + Self(Page::alloc()) + } + + pub fn alloc_order(order: u32) -> Self { + Self(Page::alloc_order(order)) + } + + pub fn zeroed() -> Self { + Self(Page::zeroed()) + } + + pub fn as_bytes(&self) -> &[u8] { + unsafe { + // SAFETY: The page is exclusively owned by us. + self.get_bytes_ptr().as_ref() + } + } + + pub fn as_bytes_mut(&mut self) -> &mut [u8] { + unsafe { + // SAFETY: The page is exclusively owned by us. + self.get_bytes_ptr().as_mut() + } + } + + pub fn into_page(self) -> Page { + self.0 + } +} + +impl Deref for PageExcl { + type Target = Page; + + fn deref(&self) -> &Self::Target { + &self.0 } } diff --git a/src/kernel/vfs/file/mod.rs b/src/kernel/vfs/file/mod.rs index 74abc9fb..eb00cc4c 100644 --- a/src/kernel/vfs/file/mod.rs +++ b/src/kernel/vfs/file/mod.rs @@ -15,7 +15,7 @@ pub use terminal_file::TerminalFile; use crate::io::{Buffer, ByteBuffer, Chunks, IntoStream, Stream}; use crate::kernel::constants::{EBADF, EINTR, EINVAL, ENOTTY}; -use crate::kernel::mem::{AsMemoryBlock, Page}; +use crate::kernel::mem::PageExcl; use crate::kernel::task::Thread; use crate::kernel::CharDevice; use crate::prelude::KResult; @@ -94,9 +94,8 @@ impl FileType { } pub async fn sendfile(&self, dest_file: &Self, count: usize) -> KResult<usize> { - let buffer_page = Page::alloc(); - // SAFETY: We are the only owner of the page. - let buffer = unsafe { buffer_page.as_memblk().as_bytes_mut() }; + let mut buffer_page = PageExcl::alloc(); + let buffer = buffer_page.as_bytes_mut(); self.sendfile_check()?; diff --git a/src/lib.rs b/src/lib.rs index a0774c9c..959cb29f 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -11,6 +11,9 @@ extern crate alloc; +#[macro_use] +extern crate static_assertions; + #[cfg(any(target_arch = "riscv64", target_arch = "x86_64"))] extern crate unwinding; From df7c7300a7fea4880d76827d7531277eef1635ca Mon Sep 17 00:00:00 2001 From: greatbridf <greatbridf@icloud.com> Date: Tue, 23 Dec 2025 00:12:42 +0800 Subject: [PATCH 19/36] tls: rework of arch's UserTLS design Separate old UserTLS into UserTLS and UserTLSDescriptor. UserTLS is for threads to hold infomation about its storage. Descriptors are used in clone syscalls. Signed-off-by: greatbridf <greatbridf@icloud.com> --- Cargo.lock | 5 +- Cargo.toml | 1 + crates/eonix_hal/src/arch/riscv64/cpu.rs | 17 --- crates/eonix_hal/src/lib.rs | 2 +- .../posix_types/src/syscall_no/loongarch64.rs | 3 +- crates/posix_types/src/syscall_no/riscv64.rs | 2 +- src/kernel/interrupt.rs | 12 +- src/kernel/pcie/init.rs | 18 +-- src/kernel/syscall/file_rw.rs | 67 +++++---- src/kernel/syscall/procops.rs | 134 +++++++----------- src/kernel/task.rs | 21 ++- src/kernel/task/clone.rs | 42 +++--- src/kernel/task/thread.rs | 76 ++++------ src/kernel/task/user_tls/mod.rs | 34 +++++ src/kernel/task/user_tls/x86_64.rs | 83 +++++++++++ src/kernel/user/dataflow.rs | 21 ++- 16 files changed, 304 insertions(+), 234 deletions(-) create mode 100644 src/kernel/task/user_tls/mod.rs create mode 100644 src/kernel/task/user_tls/x86_64.rs diff --git a/Cargo.lock b/Cargo.lock index c70190a0..5487f284 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -75,9 +75,9 @@ dependencies = [ [[package]] name = "cfg-if" -version = "1.0.1" +version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9555578bc9e57714c812a1f84e4fc5b4d21fcb063490c624de019f7464c91268" +checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" [[package]] name = "critical-section" @@ -164,6 +164,7 @@ dependencies = [ "atomic_unique_refcell", "bitflags", "buddy_allocator", + "cfg-if", "eonix_hal", "eonix_log", "eonix_macros", diff --git a/Cargo.toml b/Cargo.toml index dca5d34d..4fcb6f70 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -43,6 +43,7 @@ futures = { version = "0.3.31", features = [ "async-await", ], default-features = false } static_assertions = "1.1.0" +cfg-if = "1.0.4" [target.'cfg(any(target_arch = "riscv64", target_arch = "loongarch64"))'.dependencies] virtio-drivers = { version = "0.11.0" } diff --git a/crates/eonix_hal/src/arch/riscv64/cpu.rs b/crates/eonix_hal/src/arch/riscv64/cpu.rs index 9c843eaf..3c58580e 100644 --- a/crates/eonix_hal/src/arch/riscv64/cpu.rs +++ b/crates/eonix_hal/src/arch/riscv64/cpu.rs @@ -27,22 +27,11 @@ static DEFAULT_TRAP_CONTEXT: MaybeUninit<TrapContext> = MaybeUninit::uninit(); #[eonix_percpu::define_percpu] static LOCAL_CPU: LazyLock<CPU> = LazyLock::new(|| CPU::new(CPUID.get())); -#[derive(Debug, Clone)] -pub enum UserTLS { - Base(u64), -} - /// RISC-V Hart pub struct CPU { pub(crate) interrupt: InterruptControl, } -impl UserTLS { - pub fn new(base: u64) -> Self { - Self::Base(base) - } -} - impl CPU { fn new(cpuid: usize) -> Self { Self { @@ -66,12 +55,6 @@ impl CPU { sscratch::write(DEFAULT_TRAP_CONTEXT.as_ptr() as usize); } - pub unsafe fn load_interrupt_stack(self: Pin<&mut Self>, sp: u64) {} - - pub fn set_tls32(self: Pin<&mut Self>, _user_tls: &UserTLS) { - // nothing - } - pub fn local() -> PreemptGuard<Pin<&'static mut Self>> { unsafe { // SAFETY: We pass the reference into a `PreemptGuard`, which ensures diff --git a/crates/eonix_hal/src/lib.rs b/crates/eonix_hal/src/lib.rs index e789ecbb..b9c7d053 100644 --- a/crates/eonix_hal/src/lib.rs +++ b/crates/eonix_hal/src/lib.rs @@ -19,7 +19,7 @@ pub mod fpu { } pub mod processor { - pub use crate::arch::cpu::{halt, UserTLS, CPU, CPU_COUNT}; + pub use crate::arch::cpu::{halt, CPU, CPU_COUNT}; } /// Re-export the arch module for use in other crates diff --git a/crates/posix_types/src/syscall_no/loongarch64.rs b/crates/posix_types/src/syscall_no/loongarch64.rs index 19776a65..b0d54689 100644 --- a/crates/posix_types/src/syscall_no/loongarch64.rs +++ b/crates/posix_types/src/syscall_no/loongarch64.rs @@ -136,7 +136,7 @@ pub const SYS_RT_SIGSUSPEND: usize = 133; pub const SYS_RT_SIGACTION: usize = 134; pub const SYS_RT_SIGPROCMASK: usize = 135; pub const SYS_RT_SIGPENDING: usize = 136; -pub const SYS_RT_SIGTIMEDWAIT_TIME32: usize = 137; +pub const SYS_RT_SIGTIMEDWAIT: usize = 137; pub const SYS_RT_SIGQUEUEINFO: usize = 138; pub const SYS_RT_SIGRETURN: usize = 139; pub const SYS_SETPRIORITY: usize = 140; @@ -295,7 +295,6 @@ pub const SYS_RECVMMSG: usize = 417; pub const SYS_MQ_TIMEDSEND: usize = 418; pub const SYS_MQ_TIMEDRECEIVE: usize = 419; pub const SYS_SEMTIMEDOP: usize = 420; -pub const SYS_RT_SIGTIMEDWAIT: usize = 421; pub const SYS_FUTEX: usize = 422; pub const SYS_SCHED_RR_GET_INTERVAL: usize = 423; pub const SYS_PIDFD_SEND_SIGNAL: usize = 424; diff --git a/crates/posix_types/src/syscall_no/riscv64.rs b/crates/posix_types/src/syscall_no/riscv64.rs index 4457c20e..076942e9 100644 --- a/crates/posix_types/src/syscall_no/riscv64.rs +++ b/crates/posix_types/src/syscall_no/riscv64.rs @@ -136,7 +136,7 @@ pub const SYS_RT_SIGSUSPEND: usize = 133; pub const SYS_RT_SIGACTION: usize = 134; pub const SYS_RT_SIGPROCMASK: usize = 135; pub const SYS_RT_SIGPENDING: usize = 136; -pub const SYS_RT_SIGTIMEDWAIT_TIME32: usize = 137; +pub const SYS_RT_SIGTIMEDWAIT: usize = 137; pub const SYS_RT_SIGQUEUEINFO: usize = 138; pub const SYS_RT_SIGRETURN: usize = 139; pub const SYS_SETPRIORITY: usize = 140; diff --git a/src/kernel/interrupt.rs b/src/kernel/interrupt.rs index 2f07758a..a79d934a 100644 --- a/src/kernel/interrupt.rs +++ b/src/kernel/interrupt.rs @@ -1,15 +1,17 @@ -use super::mem::handle_kernel_page_fault; -use super::task::block_on; -use super::timer::timer_interrupt; -use crate::kernel::constants::EINVAL; -use crate::prelude::*; use alloc::sync::Arc; + use eonix_hal::traits::fault::Fault; use eonix_hal::traits::trap::{RawTrapContext, TrapType}; use eonix_hal::trap::TrapContext; use eonix_mm::address::{Addr as _, VAddr}; use eonix_sync::SpinIrq as _; +use super::mem::handle_kernel_page_fault; +use super::task::block_on; +use super::timer::timer_interrupt; +use crate::kernel::constants::EINVAL; +use crate::prelude::*; + static IRQ_HANDLERS: Spin<[Vec<Arc<dyn Fn() + Send + Sync>>; 16]> = Spin::new([const { Vec::new() }; 16]); diff --git a/src/kernel/pcie/init.rs b/src/kernel/pcie/init.rs index c0253f4e..4c183bc5 100644 --- a/src/kernel/pcie/init.rs +++ b/src/kernel/pcie/init.rs @@ -1,13 +1,14 @@ -use super::{ - device::{PCIDevice, SegmentGroup, PCIE_DEVICES}, - error::PciError, -}; -use crate::kernel::{mem::PhysAccess as _, pcie::device::PciMemoryAllocator}; -use acpi::{AcpiHandler, PhysicalMapping}; use alloc::collections::btree_map::Entry; use alloc::vec; + +use acpi::{AcpiHandler, PhysicalMapping}; use eonix_log::println_trace; -use eonix_mm::address::PAddr; +use eonix_mm::address::{PAddr, PRange}; + +use super::device::{PCIDevice, SegmentGroup, PCIE_DEVICES}; +use super::error::PciError; +use crate::kernel::mem::PhysAccess as _; +use crate::kernel::pcie::device::PciMemoryAllocator; #[derive(Clone)] struct AcpiHandlerImpl; @@ -67,10 +68,11 @@ pub fn init_pcie() -> Result<(), PciError> { #[cfg(any(target_arch = "riscv64", target_arch = "loongarch64"))] { - use crate::kernel::constants::{EINVAL, EIO, ENOENT}; use eonix_hal::arch_exported::fdt::FDT; use eonix_mm::address::PRange; + use crate::kernel::constants::{EINVAL, EIO, ENOENT}; + let pcie_node = FDT .find_compatible(&["pci-host-ecam-generic"]) .ok_or(ENOENT)?; diff --git a/src/kernel/syscall/file_rw.rs b/src/kernel/syscall/file_rw.rs index db32b0e5..93a543d7 100644 --- a/src/kernel/syscall/file_rw.rs +++ b/src/kernel/syscall/file_rw.rs @@ -1,33 +1,31 @@ +use alloc::sync::Arc; +use core::time::Duration; + +use posix_types::ctypes::{Long, PtrT}; +use posix_types::namei::RenameFlags; +use posix_types::open::{AtFlags, OpenFlags}; +use posix_types::poll::FDSet; +use posix_types::signal::{SigSet, Signal}; +use posix_types::stat::{Stat, StatX, TimeSpec}; +use posix_types::syscall_no::*; + use super::{FromSyscallArg, User}; -use crate::io::IntoStream; +use crate::io::{Buffer, BufferFill, IntoStream}; use crate::kernel::constants::{ EBADF, EFAULT, EINVAL, ENOENT, ENOSYS, ENOTDIR, SEEK_CUR, SEEK_END, SEEK_SET, }; use crate::kernel::syscall::UserMut; use crate::kernel::task::Thread; use crate::kernel::timer::sleep; +use crate::kernel::user::{ + CheckedUserPointer, UserBuffer, UserPointer, UserPointerMut, UserString, +}; +use crate::kernel::vfs::dentry::Dentry; use crate::kernel::vfs::filearray::FD; use crate::kernel::vfs::types::{DeviceId, Mode}; use crate::kernel::vfs::{PollEvent, SeekOption}; -use crate::{ - io::{Buffer, BufferFill}, - kernel::{ - user::{CheckedUserPointer, UserBuffer, UserPointer, UserPointerMut, UserString}, - vfs::dentry::Dentry, - }, - path::Path, - prelude::*, -}; -use alloc::sync::Arc; -use core::time::Duration; -use posix_types::ctypes::{Long, PtrT}; -use posix_types::namei::RenameFlags; -use posix_types::open::{AtFlags, OpenFlags}; -use posix_types::poll::FDSet; -use posix_types::signal::{SigSet, Signal}; -use posix_types::stat::Stat; -use posix_types::stat::{StatX, TimeSpec}; -use posix_types::syscall_no::*; +use crate::path::Path; +use crate::prelude::*; impl FromSyscallArg for OpenFlags { fn from_arg(value: usize) -> Self { @@ -128,7 +126,7 @@ async fn openat(dirfd: FD, pathname: User<u8>, flags: OpenFlags, mode: Mode) -> #[cfg(target_arch = "x86_64")] #[eonix_macros::define_syscall(SYS_OPEN)] -async fn open(path: User<u8>, flags: OpenFlags, mode: u32) -> KResult<FD> { +async fn open(path: User<u8>, flags: OpenFlags, mode: Mode) -> KResult<FD> { sys_openat(thread, FD::AT_FDCWD, path, flags, mode).await } @@ -145,7 +143,10 @@ async fn dup(fd: FD) -> KResult<FD> { #[cfg(target_arch = "x86_64")] #[eonix_macros::define_syscall(SYS_DUP2)] async fn dup2(old_fd: FD, new_fd: FD) -> KResult<FD> { - thread.files.dup_to(old_fd, new_fd, OpenFlags::empty()) + thread + .files + .dup_to(old_fd, new_fd, OpenFlags::empty()) + .await } #[eonix_macros::define_syscall(SYS_DUP3)] @@ -172,7 +173,13 @@ async fn pipe(pipe_fd: UserMut<[FD; 2]>) -> KResult<()> { async fn getdents(fd: FD, buffer: UserMut<u8>, bufsize: usize) -> KResult<usize> { let mut buffer = UserBuffer::new(buffer, bufsize)?; - thread.files.get(fd).ok_or(EBADF)?.getdents(&mut buffer)?; + thread + .files + .get(fd) + .ok_or(EBADF)? + .getdents(&mut buffer) + .await?; + Ok(buffer.wrote()) } @@ -264,7 +271,7 @@ async fn mkdirat(dirfd: FD, pathname: User<u8>, mode: Mode) -> KResult<()> { #[cfg(target_arch = "x86_64")] #[eonix_macros::define_syscall(SYS_MKDIR)] -async fn mkdir(pathname: User<u8>, mode: u32) -> KResult<()> { +async fn mkdir(pathname: User<u8>, mode: Mode) -> KResult<()> { sys_mkdirat(thread, FD::AT_FDCWD, pathname, mode).await } @@ -280,9 +287,9 @@ async fn truncate(pathname: User<u8>, length: usize) -> KResult<()> { let path = UserString::new(pathname)?; let path = Path::new(path.as_cstr().to_bytes())?; - let dentry = Dentry::open(&thread.fs_context, path, true)?; + let dentry = Dentry::open(&thread.fs_context, path, true).await?; - dentry.truncate(length) + dentry.truncate(length).await } #[eonix_macros::define_syscall(SYS_UNLINKAT)] @@ -296,7 +303,7 @@ async fn unlinkat(dirfd: FD, pathname: User<u8>) -> KResult<()> { #[cfg(target_arch = "x86_64")] #[eonix_macros::define_syscall(SYS_UNLINK)] async fn unlink(pathname: User<u8>) -> KResult<()> { - sys_unlinkat(thread, FD::AT_FDCWD, pathname) + sys_unlinkat(thread, FD::AT_FDCWD, pathname).await } #[eonix_macros::define_syscall(SYS_SYMLINKAT)] @@ -310,7 +317,7 @@ async fn symlinkat(target: User<u8>, dirfd: FD, linkpath: User<u8>) -> KResult<( #[cfg(target_arch = "x86_64")] #[eonix_macros::define_syscall(SYS_SYMLINK)] async fn symlink(target: User<u8>, linkpath: User<u8>) -> KResult<()> { - sys_symlinkat(thread, target, FD::AT_FDCWD, linkpath) + sys_symlinkat(thread, target, FD::AT_FDCWD, linkpath).await } #[derive(Clone, Copy, Debug)] @@ -347,7 +354,7 @@ async fn mknodat(dirfd: FD, pathname: User<u8>, mut mode: Mode, dev: UserDeviceI #[cfg(target_arch = "x86_64")] #[eonix_macros::define_syscall(SYS_MKNOD)] -async fn mknod(pathname: User<u8>, mode: u32, dev: u32) -> KResult<()> { +async fn mknod(pathname: User<u8>, mode: Mode, dev: UserDeviceId) -> KResult<()> { sys_mknodat(thread, FD::AT_FDCWD, pathname, mode, dev).await } @@ -389,7 +396,7 @@ async fn lseek(fd: FD, offset: u64, whence: u32) -> KResult<u64> { #[cfg(target_arch = "x86_64")] #[eonix_macros::define_syscall(SYS_LLSEEK)] -fn llseek( +async fn llseek( fd: FD, offset_high: u32, offset_low: u32, diff --git a/src/kernel/syscall/procops.rs b/src/kernel/syscall/procops.rs index b4d3e449..1359d0ab 100644 --- a/src/kernel/syscall/procops.rs +++ b/src/kernel/syscall/procops.rs @@ -1,38 +1,37 @@ +use alloc::borrow::ToOwned; +use alloc::ffi::CString; +use core::time::Duration; + +use bitflags::bitflags; +use eonix_hal::traits::trap::RawTrapContext; +use eonix_hal::trap::TrapContext; +use eonix_mm::address::Addr as _; +use eonix_sync::AsProof as _; +use posix_types::ctypes::PtrT; +use posix_types::signal::{SigAction, SigInfo, SigSet, Signal}; +use posix_types::stat::{TimeSpec, TimeVal}; +use posix_types::syscall_no::*; +use posix_types::SIGNAL_NOW; + use super::SyscallNoReturn; use crate::io::Buffer; use crate::kernel::constants::{ - CLOCK_MONOTONIC, CLOCK_REALTIME, CLOCK_REALTIME_COARSE, EINVAL, ENOENT, ENOTDIR, ERANGE, ESRCH, -}; -use crate::kernel::constants::{ - ENOSYS, PR_GET_NAME, PR_SET_NAME, RLIMIT_STACK, SIG_BLOCK, SIG_SETMASK, SIG_UNBLOCK, + CLOCK_MONOTONIC, CLOCK_REALTIME, CLOCK_REALTIME_COARSE, EINVAL, ENOENT, ENOSYS, ENOTDIR, + ERANGE, ESRCH, PR_GET_NAME, PR_SET_NAME, RLIMIT_STACK, SIG_BLOCK, SIG_SETMASK, SIG_UNBLOCK, }; use crate::kernel::mem::PageBuffer; use crate::kernel::syscall::{User, UserMut}; use crate::kernel::task::{ - do_clone, futex_wait, futex_wake, yield_now, FutexFlags, FutexOp, ProcessList, ProgramLoader, - RobustListHead, SignalAction, Thread, WaitId, WaitType, + do_clone, futex_wait, futex_wake, parse_futexop, yield_now, CloneArgs, FutexFlags, FutexOp, + ProcessList, ProgramLoader, RobustListHead, SignalAction, Thread, WaitId, WaitType, }; -use crate::kernel::task::{parse_futexop, CloneArgs}; use crate::kernel::timer::sleep; -use crate::kernel::user::UserString; -use crate::kernel::user::{UserPointer, UserPointerMut}; +use crate::kernel::user::{UserBuffer, UserPointer, UserPointerMut, UserString}; +use crate::kernel::vfs::dentry::Dentry; use crate::kernel::vfs::types::Permission; -use crate::kernel::vfs::{self, dentry::Dentry}; +use crate::kernel::vfs::{self}; use crate::path::Path; -use crate::{kernel::user::UserBuffer, prelude::*}; -use alloc::borrow::ToOwned; -use alloc::ffi::CString; -use bitflags::bitflags; -use core::time::Duration; -use eonix_hal::processor::UserTLS; -use eonix_hal::traits::trap::RawTrapContext; -use eonix_hal::trap::TrapContext; -use eonix_mm::address::Addr as _; -use eonix_sync::AsProof as _; -use posix_types::ctypes::PtrT; -use posix_types::signal::{SigAction, SigInfo, SigSet, Signal}; -use posix_types::stat::TimeVal; -use posix_types::{syscall_no::*, SIGNAL_NOW}; +use crate::prelude::*; #[repr(C)] #[derive(Debug, Clone, Copy)] @@ -366,7 +365,7 @@ async fn wait4( #[cfg(target_arch = "x86_64")] #[eonix_macros::define_syscall(SYS_WAITPID)] async fn waitpid(waitpid: i32, arg1: UserMut<u32>, options: u32) -> KResult<u32> { - sys_wait4(thread, waitpid, arg1, options, core::ptr::null_mut()).await + sys_wait4(thread, waitpid, arg1, options, UserMut::null()).await } #[eonix_macros::define_syscall(SYS_SETSID)] @@ -493,51 +492,15 @@ async fn gettid() -> KResult<u32> { Ok(thread.tid) } -pub fn parse_user_tls(arch_tls: usize) -> KResult<UserTLS> { - #[cfg(target_arch = "x86_64")] - { - let desc = arch_tls as *mut posix_types::x86_64::UserDescriptor; - let desc_pointer = UserPointerMut::new(desc)?; - let mut desc = desc_pointer.read()?; - - // Clear the TLS area if it is not present. - if desc.flags.is_read_exec_only() && !desc.flags.is_present() { - if desc.limit != 0 && desc.base != 0 { - let len = if desc.flags.is_limit_in_pages() { - (desc.limit as usize) << 12 - } else { - desc.limit as usize - }; - - CheckedUserPointer::new(desc.base as _, len)?.zero()?; - } - } - - let (new_tls, entry) = - UserTLS::new32(desc.base, desc.limit, desc.flags.is_limit_in_pages()); - desc.entry = entry; - desc_pointer.write(desc)?; - - Ok(new_tls) - } - - #[cfg(any(target_arch = "riscv64", target_arch = "loongarch64"))] - { - Ok(UserTLS::new(arch_tls as u64)) - } -} - #[cfg(target_arch = "x86_64")] #[eonix_macros::define_syscall(SYS_SET_THREAD_AREA)] -async fn set_thread_area(arch_tls: usize) -> KResult<()> { - thread.set_user_tls(parse_user_tls(arch_tls)?)?; +async fn set_thread_area(tls: PtrT) -> KResult<()> { + use crate::kernel::task::UserTLSDescriptor; - // SAFETY: Preemption is disabled on calling `load_thread_area32()`. - unsafe { - eonix_preempt::disable(); - thread.load_thread_area32(); - eonix_preempt::enable(); - } + let tls = UserTLSDescriptor::new(tls)?.read()?; + + thread.set_user_tls(tls)?; + thread.activate_tls(); Ok(()) } @@ -651,18 +614,14 @@ async fn rt_sigprocmask( Ok(()) } -#[repr(C)] -#[derive(Clone, Copy)] -struct TimeSpec32 { - tv_sec: i32, - tv_nsec: i32, -} - -#[eonix_macros::define_syscall(SYS_RT_SIGTIMEDWAIT_TIME32)] -async fn rt_sigtimedwait_time32( +#[cfg_attr( + any(target_arch = "riscv64", target_arch = "loongarch64"), + eonix_macros::define_syscall(SYS_RT_SIGTIMEDWAIT) +)] +async fn rt_sigtimedwait( _uthese: User<SigSet>, _uinfo: UserMut<SigInfo>, - _uts: User<TimeSpec32>, + _uts: User<TimeSpec>, ) -> KResult<i32> { // TODO Ok(0) @@ -820,7 +779,7 @@ async fn clone( clone_flags: usize, new_sp: usize, parent_tidptr: UserMut<u32>, - tls: usize, + tls: PtrT, child_tidptr: UserMut<u32>, ) -> KResult<u32> { let clone_args = CloneArgs::for_clone(clone_flags, new_sp, child_tidptr, parent_tidptr, tls)?; @@ -925,8 +884,23 @@ async fn sigreturn() -> KResult<SyscallNoReturn> { #[cfg(target_arch = "x86_64")] #[eonix_macros::define_syscall(SYS_ARCH_PRCTL)] -async fn arch_prctl(option: u32, addr: u32) -> KResult<u32> { - sys_arch_prctl(thread, option, addr).await +async fn arch_prctl(option: u32, addr: PtrT) -> KResult<u32> { + match option { + PR_SET_NAME => { + let name = UserPointer::<[u8; 16]>::new(User::with_addr(addr.addr()))?.read()?; + let len = name.iter().position(|&c| c == 0).unwrap_or(15); + thread.set_name(name[..len].into()); + Ok(0) + } + PR_GET_NAME => { + let name = thread.get_name(); + let len = name.len().min(15); + let name: [u8; 16] = core::array::from_fn(|i| if i < len { name[i] } else { 0 }); + UserPointerMut::<[u8; 16]>::new(UserMut::with_addr(addr.addr()))?.write(name)?; + Ok(0) + } + _ => Err(EINVAL), + } } pub fn keep_alive() {} diff --git a/src/kernel/task.rs b/src/kernel/task.rs index 2ef58069..3fe6fe97 100644 --- a/src/kernel/task.rs +++ b/src/kernel/task.rs @@ -8,6 +8,7 @@ mod process_list; mod session; mod signal; mod thread; +mod user_tls; pub use clone::{do_clone, CloneArgs, CloneFlags}; pub use futex::{futex_wait, futex_wake, parse_futexop, FutexFlags, FutexOp, RobustListHead}; @@ -19,6 +20,7 @@ pub use process_list::ProcessList; pub use session::Session; pub use signal::SignalAction; pub use thread::{yield_now, Thread, ThreadAlloc, ThreadBuilder}; +pub use user_tls::{UserTLS, UserTLSDescriptor}; fn do_block_on<F>(mut future: core::pin::Pin<&mut F>) -> F::Output where @@ -79,30 +81,25 @@ pub async fn stackful<F>(mut future: F) -> F::Output where F: core::future::Future, { - use crate::kernel::{ - interrupt::{default_fault_handler, default_irq_handler}, - timer::{should_reschedule, timer_interrupt}, - }; use alloc::sync::Arc; use alloc::task::Wake; use core::cell::UnsafeCell; use core::future::Future; use core::pin::Pin; use core::ptr::NonNull; - use core::sync::atomic::AtomicBool; - use core::sync::atomic::Ordering; - use core::task::Context; - use core::task::Poll; - use core::task::Waker; - use eonix_hal::traits::trap::RawTrapContext; - use eonix_hal::traits::trap::TrapReturn; - use eonix_hal::traits::trap::TrapType; + use core::sync::atomic::{AtomicBool, Ordering}; + use core::task::{Context, Poll, Waker}; + + use eonix_hal::traits::trap::{RawTrapContext, TrapReturn, TrapType}; use eonix_hal::trap::TrapContext; use eonix_preempt::assert_preempt_enabled; use eonix_runtime::executor::Stack; use eonix_runtime::task::Task; use thread::wait_for_wakeups; + use crate::kernel::interrupt::{default_fault_handler, default_irq_handler}; + use crate::kernel::timer::{should_reschedule, timer_interrupt}; + let stack = KernelStack::new(); fn execute<F>(mut future: Pin<&mut F>, output_ptr: NonNull<Option<F::Output>>) -> ! diff --git a/src/kernel/task/clone.rs b/src/kernel/task/clone.rs index e0d578c1..dd6f538d 100644 --- a/src/kernel/task/clone.rs +++ b/src/kernel/task/clone.rs @@ -1,18 +1,17 @@ -use crate::{ - kernel::{ - syscall::{procops::parse_user_tls, UserMut}, - task::{alloc_pid, ProcessBuilder, ProcessList, Thread, ThreadBuilder}, - user::UserPointerMut, - }, - KResult, -}; -use bitflags::bitflags; use core::num::NonZero; -use eonix_hal::processor::UserTLS; + +use bitflags::bitflags; use eonix_runtime::scheduler::RUNTIME; use eonix_sync::AsProof; +use posix_types::ctypes::PtrT; use posix_types::signal::Signal; +use super::{UserTLS, UserTLSDescriptor}; +use crate::kernel::syscall::UserMut; +use crate::kernel::task::{alloc_pid, ProcessBuilder, ProcessList, Thread, ThreadBuilder}; +use crate::kernel::user::UserPointerMut; +use crate::KResult; + bitflags! { #[derive(Debug, Default)] pub struct CloneFlags: usize { @@ -46,12 +45,18 @@ bitflags! { #[derive(Debug)] pub struct CloneArgs { pub flags: CloneFlags, - pub sp: Option<NonZero<usize>>, // Stack pointer for the new thread. - pub exit_signal: Option<Signal>, // Signal to send to the parent on exit. - pub set_tid_ptr: Option<UserMut<u32>>, // Pointer to set child TID in user space. - pub clear_tid_ptr: Option<UserMut<u32>>, // Pointer to clear child TID in user space. - pub parent_tid_ptr: Option<UserMut<u32>>, // Pointer to parent TID in user space. - pub tls: Option<UserTLS>, // Pointer to TLS information. + /// Stack pointer for the new thread. + pub sp: Option<NonZero<usize>>, + /// Signal to send to the parent on exit. + pub exit_signal: Option<Signal>, + /// Pointer to set child TID in user space. + pub set_tid_ptr: Option<UserMut<u32>>, + /// Pointer to clear child TID in user space. + pub clear_tid_ptr: Option<UserMut<u32>>, + /// Pointer to parent TID in user space. + pub parent_tid_ptr: Option<UserMut<u32>>, + /// Pointer to TLS information. + pub tls: Option<UserTLS>, } impl CloneArgs { @@ -62,7 +67,7 @@ impl CloneArgs { sp: usize, child_tid_ptr: UserMut<u32>, parent_tid_ptr: UserMut<u32>, - tls: usize, + tls: PtrT, ) -> KResult<Self> { let clone_flags = CloneFlags::from_bits_truncate(flags & !Self::MASK); let exit_signal = flags & Self::MASK; @@ -87,7 +92,8 @@ impl CloneArgs { .then_some(parent_tid_ptr); let tls = if clone_flags.contains(CloneFlags::CLONE_SETTLS) { - Some(parse_user_tls(tls)?) + let tls_desc = UserTLSDescriptor::new(tls)?; + Some(tls_desc.read()?) } else { None }; diff --git a/src/kernel/task/thread.rs b/src/kernel/task/thread.rs index 11348e51..77e8e618 100644 --- a/src/kernel/task/thread.rs +++ b/src/kernel/task/thread.rs @@ -1,43 +1,37 @@ -use super::{ - signal::{RaiseResult, SignalList}, - stackful, Process, ProcessList, WaitType, -}; -use crate::{ - kernel::{ - interrupt::default_irq_handler, - syscall::{syscall_handlers, SyscallHandler, User, UserMut}, - task::{clone::CloneArgs, futex::RobustListHead, CloneFlags}, - timer::{should_reschedule, timer_interrupt}, - user::{UserPointer, UserPointerMut}, - vfs::{filearray::FileArray, FsContext}, - }, - prelude::*, -}; -use alloc::{alloc::Allocator, sync::Arc}; +use alloc::alloc::Allocator; +use alloc::sync::Arc; +use core::future::{poll_fn, Future}; +use core::pin::Pin; +use core::ptr::NonNull; +use core::sync::atomic::{AtomicBool, Ordering}; +use core::task::{Context, Poll}; + use atomic_unique_refcell::AtomicUniqueRefCell; -use core::{ - future::{poll_fn, Future}, - pin::Pin, - ptr::NonNull, - sync::atomic::{AtomicBool, Ordering}, - task::{Context, Poll}, -}; -use eonix_hal::{ - fpu::FpuState, - processor::{UserTLS, CPU}, - traits::{ - fault::Fault, - fpu::RawFpuState as _, - trap::{RawTrapContext, TrapReturn, TrapType}, - }, - trap::TrapContext, -}; +use eonix_hal::fpu::FpuState; +use eonix_hal::traits::fault::Fault; +use eonix_hal::traits::fpu::RawFpuState as _; +use eonix_hal::traits::trap::{RawTrapContext, TrapReturn, TrapType}; +use eonix_hal::trap::TrapContext; use eonix_mm::address::{Addr as _, VAddr}; use eonix_sync::AsProofMut as _; use pointers::BorrowedArc; use posix_types::signal::Signal; use stalloc::UnsafeStalloc; +use super::signal::{RaiseResult, SignalList}; +use super::user_tls::UserTLS; +use super::{stackful, Process, ProcessList, WaitType}; +use crate::kernel::interrupt::default_irq_handler; +use crate::kernel::syscall::{syscall_handlers, SyscallHandler, User, UserMut}; +use crate::kernel::task::clone::CloneArgs; +use crate::kernel::task::futex::RobustListHead; +use crate::kernel::task::CloneFlags; +use crate::kernel::timer::{should_reschedule, timer_interrupt}; +use crate::kernel::user::{UserPointer, UserPointerMut}; +use crate::kernel::vfs::filearray::FileArray; +use crate::kernel::vfs::FsContext; +use crate::prelude::*; + #[eonix_percpu::define_percpu] static CURRENT_THREAD: Option<NonNull<Thread>> = None; @@ -275,12 +269,9 @@ impl Thread { self.signal_list.raise(signal) } - /// # Safety - /// This function is unsafe because it accesses the `current_cpu()`, which needs - /// to be called in a preemption disabled context. - pub unsafe fn load_thread_area32(&self) { + pub fn activate_tls(&self) { if let Some(tls) = self.inner.lock().tls.as_ref() { - CPU::local().as_mut().set_tls32(tls); + tls.activate(); } } @@ -442,14 +433,7 @@ impl Thread { CURRENT_THREAD.set(NonNull::new(&raw const *self as *mut _)); - unsafe { - eonix_preempt::disable(); - - // SAFETY: Preemption is disabled. - self.load_thread_area32(); - - eonix_preempt::enable(); - } + self.activate_tls(); let result = future.as_mut().poll(cx); diff --git a/src/kernel/task/user_tls/mod.rs b/src/kernel/task/user_tls/mod.rs new file mode 100644 index 00000000..2583b580 --- /dev/null +++ b/src/kernel/task/user_tls/mod.rs @@ -0,0 +1,34 @@ +cfg_if::cfg_if! { + if #[cfg(target_arch = "x86_64")] { + mod x86_64; + pub use x86_64::*; + } else { + use eonix_mm::address::VAddr; + use posix_types::ctypes::PtrT; + + use crate::prelude::KResult; + + + #[derive(Debug, Clone)] + pub struct UserTLS(VAddr); + + #[derive(Debug, Clone)] + pub struct UserTLSDescriptor(VAddr); + + impl UserTLS { + pub fn activate(&self) { + self.0; + } + } + + impl UserTLSDescriptor { + pub fn new(tp: PtrT) -> KResult<Self> { + Ok(Self(VAddr::from(tp.addr()))) + } + + pub fn read(&self) -> KResult<UserTLS> { + Ok(UserTLS(self.0)) + } + } + } +} diff --git a/src/kernel/task/user_tls/x86_64.rs b/src/kernel/task/user_tls/x86_64.rs new file mode 100644 index 00000000..5bb33b97 --- /dev/null +++ b/src/kernel/task/user_tls/x86_64.rs @@ -0,0 +1,83 @@ +use core::fmt; + +use eonix_hal::arch_exported::gdt::{GDTEntry, GDT}; +use eonix_hal::processor::CPU; +use eonix_mm::address::VAddr; +use posix_types::ctypes::PtrT; +use posix_types::x86_64::UserDescriptor; + +use crate::kernel::syscall::{User, UserMut}; +use crate::kernel::user::{CheckedUserPointer, UserPointerMut}; +use crate::prelude::KResult; + +#[derive(Debug, Clone)] +pub struct UserTLS { + desc: GDTEntry, + base: u64, +} + +pub struct UserTLSDescriptor<'a> { + ptr: UserPointerMut<'a, UserDescriptor>, +} + +impl UserTLS { + fn new(base: u32, limit: u32) -> Self { + Self { + desc: GDTEntry::new_tls(base, limit), + base: base as u64, + } + } + + fn new_page_limit(base: u32, limit_in_pages: u32) -> Self { + Self { + desc: GDTEntry::new_tls_page_limit(base, limit_in_pages), + base: base as u64, + } + } + + pub fn activate(&self) { + CPU::local().as_mut().set_tls32(self.desc, self.base); + } +} + +impl UserTLSDescriptor<'_> { + pub fn new(raw_tls: PtrT) -> KResult<Self> { + Ok(Self { + ptr: UserPointerMut::new(UserMut::<UserDescriptor>::with_addr(raw_tls.addr()))?, + }) + } + + pub fn read(&self) -> KResult<UserTLS> { + let mut desc = self.ptr.read()?; + + let base = VAddr::from(desc.base as usize); + + // Clear the TLS area if it is not present. + if desc.flags.is_read_exec_only() && !desc.flags.is_present() { + if desc.limit != 0 && base != VAddr::NULL { + let len = if desc.flags.is_limit_in_pages() { + (desc.limit as usize) << 12 + } else { + desc.limit as usize + }; + + CheckedUserPointer::new(User::new(base), len)?.zero()?; + } + } + + desc.entry = GDT::TLS32_INDEX as u32; + self.ptr.write(desc)?; + + Ok(if desc.flags.is_limit_in_pages() { + UserTLS::new_page_limit(desc.base, desc.limit) + } else { + UserTLS::new(desc.base, desc.limit) + }) + } +} + +impl fmt::Debug for UserTLSDescriptor<'_> { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_tuple("UserTLSDescriptor").finish_non_exhaustive() + } +} diff --git a/src/kernel/user/dataflow.rs b/src/kernel/user/dataflow.rs index 02e7d791..5d8ac167 100644 --- a/src/kernel/user/dataflow.rs +++ b/src/kernel/user/dataflow.rs @@ -1,18 +1,15 @@ -use crate::{ - io::{Buffer, FillResult}, - prelude::*, -}; -use crate::{ - io::{IntoStream, Stream}, - kernel::{ - constants::{EFAULT, EINVAL}, - syscall::{User, UserMut}, - }, -}; -use core::{arch::asm, ffi::CStr, marker::PhantomData}; +use core::arch::asm; +use core::ffi::CStr; +use core::marker::PhantomData; + use eonix_mm::address::Addr; use eonix_preempt::assert_preempt_enabled; +use crate::io::{Buffer, FillResult, IntoStream, Stream}; +use crate::kernel::constants::{EFAULT, EINVAL}; +use crate::kernel::syscall::{User, UserMut}; +use crate::prelude::*; + pub struct CheckedUserPointer<'a> { ptr: User<u8>, len: usize, From 03811435967481c9e6d91c682d7eec992b8edfe2 Mon Sep 17 00:00:00 2001 From: greatbridf <greatbridf@icloud.com> Date: Tue, 6 Jan 2026 22:02:22 +0800 Subject: [PATCH 20/36] mem, slab: rework the slab system Signed-off-by: greatbridf <greatbridf@icloud.com> --- Cargo.lock | 1 - crates/intrusive_list/src/lib.rs | 2 + crates/slab_allocator/Cargo.toml | 2 - crates/slab_allocator/src/lib.rs | 291 +++++++++++++++++++++--- crates/slab_allocator/src/slab_cache.rs | 164 ------------- src/kernel/mem/allocator.rs | 21 +- src/kernel/mem/page_alloc.rs | 10 +- src/kernel/mem/page_alloc/raw_page.rs | 199 ++++++++++------ 8 files changed, 400 insertions(+), 290 deletions(-) delete mode 100644 crates/slab_allocator/src/slab_cache.rs diff --git a/Cargo.lock b/Cargo.lock index 5487f284..3e8a36bd 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -518,7 +518,6 @@ version = "0.1.0" dependencies = [ "eonix_mm", "eonix_sync", - "intrusive_list", ] [[package]] diff --git a/crates/intrusive_list/src/lib.rs b/crates/intrusive_list/src/lib.rs index af8c4f1a..440944d0 100644 --- a/crates/intrusive_list/src/lib.rs +++ b/crates/intrusive_list/src/lib.rs @@ -25,6 +25,8 @@ impl List { } pub fn insert(&mut self, node: &mut Link) { + // TODO: `node` above should be of 'static. + self.head.insert(node); self.count += 1; } diff --git a/crates/slab_allocator/Cargo.toml b/crates/slab_allocator/Cargo.toml index 067b6f53..926ac688 100644 --- a/crates/slab_allocator/Cargo.toml +++ b/crates/slab_allocator/Cargo.toml @@ -6,5 +6,3 @@ edition = "2024" [dependencies] eonix_mm = { path = "../eonix_mm" } eonix_sync = { path = "../eonix_sync" } -intrusive_list = { path = "../intrusive_list" } - diff --git a/crates/slab_allocator/src/lib.rs b/crates/slab_allocator/src/lib.rs index ce163183..8a684edd 100644 --- a/crates/slab_allocator/src/lib.rs +++ b/crates/slab_allocator/src/lib.rs @@ -1,69 +1,288 @@ #![no_std] -mod slab_cache; +use core::ptr::NonNull; -use core::{cmp::max, ptr::NonNull}; - -use eonix_mm::paging::{PageAlloc, RawPage}; use eonix_sync::Spin; -use intrusive_list::Link; -use slab_cache::SlabCache; -pub trait SlabRawPage: RawPage { - /// Get the container raw page struct of the list link. +#[repr(C)] +pub union SlabSlot { + slab_slot: Option<NonNull<SlabSlot>>, + data: u8, +} + +pub trait SlabPageList: Sized { + type Page: SlabPage; + + fn new() -> Self; + fn is_empty(&self) -> bool; + + fn peek_head(&mut self) -> Option<&mut Self::Page>; + + fn pop_head(&mut self) -> Option<&'static mut Self::Page>; + fn push_tail(&mut self, page: &'static mut Self::Page); + fn remove(&mut self, page: &mut Self::Page); +} + +pub trait SlabPage: Sized + 'static { + fn get_data_ptr(&self) -> NonNull<[u8]>; + + fn get_free_slot(&self) -> Option<NonNull<SlabSlot>>; + fn set_free_slot(&mut self, next: Option<NonNull<SlabSlot>>); + + fn get_alloc_count(&self) -> usize; + + /// Increase the allocation count by 1 and return the increased value. + fn inc_alloc_count(&mut self) -> usize; + + /// Decrease the allocation count by 1 and return the decreased value. + fn dec_alloc_count(&mut self) -> usize; + + /// Get the [`SlabPage`] that `ptr` is allocated from. /// /// # Safety - /// The caller MUST ensure that the link points to a `RawPage`. - unsafe fn from_link(link: &mut Link) -> Self; + /// The caller MUST ensure that no others could be calling this function and + /// getting the [`SlabPage`] at the same time. + unsafe fn from_allocated(ptr: NonNull<u8>) -> &'static mut Self; +} + +pub(crate) trait SlabPageExt { + fn alloc_slot(&mut self) -> Option<NonNull<u8>>; - /// Get the list link of the raw page. - /// /// # Safety - /// The caller MUST ensure that at any time, only one mutable reference - /// to the link exists. - unsafe fn get_link(&self) -> &mut Link; + /// The caller MUST ensure that `slot_data_ptr` points to some position + /// previously allocated by [`SlabPageExt::alloc_slot`]. + unsafe fn free_slot(&mut self, slot_data_ptr: NonNull<u8>); + + fn is_empty(&self) -> bool; + fn is_full(&self) -> bool; +} + +impl<T> SlabPageExt for T +where + T: SlabPage, +{ + fn alloc_slot(&mut self) -> Option<NonNull<u8>> { + let mut free_slot = self.get_free_slot()?; + + unsafe { + let free_slot = free_slot.as_mut(); + + let next_slot = free_slot.slab_slot; + // ===== `free_slot` is now safe to be overwritten - fn slab_init(&self, first_free: Option<NonNull<usize>>); + self.set_free_slot(next_slot); + self.inc_alloc_count(); - // which slab page the ptr belong - fn in_which(ptr: *mut u8) -> Self; + Some(NonNull::new_unchecked(&mut free_slot.data)) + } + } + + unsafe fn free_slot(&mut self, slot_data_ptr: NonNull<u8>) { + unsafe { + let mut free_slot: NonNull<SlabSlot> = slot_data_ptr.cast(); + free_slot.as_mut().slab_slot = self.get_free_slot(); - fn real_page_ptr(&self) -> *mut u8; + self.set_free_slot(Some(free_slot)); + self.dec_alloc_count(); + } + } - fn allocated_count(&self) -> &mut u32; + fn is_empty(&self) -> bool { + self.get_alloc_count() == 0 + } - fn next_free(&self) -> &mut Option<NonNull<usize>>; + fn is_full(&self) -> bool { + self.get_free_slot().is_none() + } } -pub struct SlabAllocator<T, A, const SLAB_CACHE_COUNT: usize> { - slabs: [Spin<SlabCache<T, A>>; SLAB_CACHE_COUNT], - alloc: A, +pub trait SlabPageAlloc { + type Page: SlabPage; + type PageList: SlabPageList<Page = Self::Page>; + + /// Allocate a page suitable for slab system use. The page MUST come with + /// its allocation count 0 and next free slot None. + /// + /// # Safety + /// The page returned MUST be properly initialized before its usage. + unsafe fn alloc_uninit(&self) -> &'static mut Self::Page; } -unsafe impl<T, A, const SLAB_CACHE_COUNT: usize> Send for SlabAllocator<T, A, SLAB_CACHE_COUNT> {} -unsafe impl<T, A, const SLAB_CACHE_COUNT: usize> Sync for SlabAllocator<T, A, SLAB_CACHE_COUNT> {} +pub(crate) struct SlabList<T> +where + T: SlabPageList, +{ + empty_list: T, + partial_list: T, + full_list: T, + object_size: usize, +} + +pub struct SlabAlloc<P, const COUNT: usize> +where + P: SlabPageAlloc, +{ + slabs: [Spin<SlabList<P::PageList>>; COUNT], + alloc: P, +} + +unsafe impl<P, const COUNT: usize> Send for SlabAlloc<P, COUNT> where P: SlabPageAlloc {} +unsafe impl<P, const COUNT: usize> Sync for SlabAlloc<P, COUNT> where P: SlabPageAlloc {} -impl<Raw, Allocator, const SLAB_CACHE_COUNT: usize> SlabAllocator<Raw, Allocator, SLAB_CACHE_COUNT> +impl<L, const COUNT: usize> SlabAlloc<L, COUNT> where - Raw: SlabRawPage, - Allocator: PageAlloc<RawPage = Raw>, + L: SlabPageAlloc, { - pub fn new_in(alloc: Allocator) -> Self { + pub fn new_in(alloc: L) -> Self { Self { - slabs: core::array::from_fn(|i| Spin::new(SlabCache::new_in(1 << (i + 3)))), + slabs: core::array::from_fn(|i| Spin::new(SlabList::new(1 << (i + 3)))), alloc, } } - pub fn alloc(&self, mut size: usize) -> *mut u8 { - size = max(8, size); + pub fn alloc(&self, mut size: usize) -> NonNull<u8> { + size = size.max(8); let idx = size.next_power_of_two().trailing_zeros() - 3; self.slabs[idx as usize].lock().alloc(&self.alloc) } - pub fn dealloc(&self, ptr: *mut u8, mut size: usize) { - size = max(8, size); + pub unsafe fn dealloc(&self, ptr: NonNull<u8>, mut size: usize) { + size = size.max(8); let idx = size.next_power_of_two().trailing_zeros() - 3; - self.slabs[idx as usize].lock().dealloc(ptr, &self.alloc); + + unsafe { + // SAFETY: + self.slabs[idx as usize].lock().dealloc(ptr, &self.alloc); + } } } + +impl<T> SlabList<T> +where + T: SlabPageList, +{ + fn new(object_size: usize) -> Self { + Self { + empty_list: T::new(), + partial_list: T::new(), + full_list: T::new(), + object_size, + } + } + + fn alloc_from_partial(&mut self) -> NonNull<u8> { + let head = self.partial_list.peek_head().unwrap(); + let slot = head.alloc_slot().unwrap(); + + if head.is_full() { + let head = self.partial_list.pop_head().unwrap(); + self.full_list.push_tail(head); + } + + slot + } + + fn alloc_from_empty(&mut self) -> NonNull<u8> { + let head = self.empty_list.pop_head().unwrap(); + let slot = head.alloc_slot().unwrap(); + + if head.is_full() { + self.full_list.push_tail(head); + } else { + self.partial_list.push_tail(head); + } + + slot + } + + fn charge(&mut self, alloc: &impl SlabPageAlloc<Page = T::Page>) { + unsafe { + let slab = alloc.alloc_uninit(); + let free_slot = make_slab_page(slab.get_data_ptr(), self.object_size); + + slab.set_free_slot(Some(free_slot)); + + self.empty_list.push_tail(slab); + } + } + + fn alloc(&mut self, alloc: &impl SlabPageAlloc<Page = T::Page>) -> NonNull<u8> { + if !self.partial_list.is_empty() { + return self.alloc_from_partial(); + } + + if self.empty_list.is_empty() { + self.charge(alloc); + } + + self.alloc_from_empty() + } + + unsafe fn dealloc(&mut self, ptr: NonNull<u8>, _alloc: &impl SlabPageAlloc) { + let slab_page = unsafe { + // SAFETY: + <T::Page>::from_allocated(ptr) + }; + + let (was_full, is_empty); + + was_full = slab_page.is_full(); + + unsafe { + // SAFETY: + slab_page.free_slot(ptr); + } + + is_empty = slab_page.is_empty(); + + match (was_full, is_empty) { + (false, false) => {} + (false, true) => { + self.partial_list.remove(slab_page); + self.empty_list.push_tail(slab_page); + } + (true, false) => { + self.full_list.remove(slab_page); + self.partial_list.push_tail(slab_page); + } + (true, true) => { + self.full_list.remove(slab_page); + self.empty_list.push_tail(slab_page); + } + } + + // TODO: Check whether we should place some pages back with `alloc` if + // the global free page count is below the watermark. + } +} + +pub fn make_slab_page(page_ptr: NonNull<[u8]>, slot_size: usize) -> NonNull<SlabSlot> { + assert!( + slot_size >= core::mem::size_of::<usize>(), + "The minimum slot size is of a pointer's width" + ); + + let page_size = page_ptr.len(); + let slot_count = page_size / slot_size; + let page_start: NonNull<u8> = page_ptr.cast(); + + // Quick checks + assert!( + page_size % slot_size == 0, + "The page's size should be a multiple of the slot size" + ); + + let mut prev_free_slot = None; + for i in (0..slot_count).rev() { + let offset = i * slot_size; + + unsafe { + let mut slot_ptr: NonNull<SlabSlot> = page_start.add(offset).cast(); + + slot_ptr.as_mut().slab_slot = prev_free_slot; + prev_free_slot = Some(slot_ptr); + } + } + + prev_free_slot.expect("There should be at least one slot.") +} diff --git a/crates/slab_allocator/src/slab_cache.rs b/crates/slab_allocator/src/slab_cache.rs deleted file mode 100644 index 98e27fc8..00000000 --- a/crates/slab_allocator/src/slab_cache.rs +++ /dev/null @@ -1,164 +0,0 @@ -use super::SlabRawPage; -use core::{marker::PhantomData, ptr::NonNull}; -use eonix_mm::paging::{PageAlloc, PAGE_SIZE}; -use intrusive_list::List; - -pub(crate) struct SlabCache<T, A> { - empty_list: List, - partial_list: List, - full_list: List, - object_size: u32, - _phantom: PhantomData<(T, A)>, -} - -trait SlabRawPageExt { - fn alloc_slot(&self) -> Option<NonNull<usize>>; - fn dealloc_slot(&self, slot_ptr: *mut u8); - fn is_full(&self) -> bool; - fn is_empty(&self) -> bool; - fn slab_page_init(&self, object_size: u32) -> Option<NonNull<usize>>; -} - -impl<T> SlabRawPageExt for T -where - T: SlabRawPage, -{ - fn alloc_slot(&self) -> Option<NonNull<usize>> { - let ptr = self.next_free().clone(); - - let next_free = match ptr { - Some(ptr) => unsafe { ptr.read() as *mut usize }, - None => unreachable!(), - }; - *self.allocated_count() += 1; - *self.next_free() = NonNull::new(next_free); - return ptr; - } - - fn dealloc_slot(&self, slot_ptr: *mut u8) { - let slot_ptr = slot_ptr as *mut usize; - - if let Some(last_free) = self.next_free().clone() { - unsafe { *slot_ptr = last_free.as_ptr() as usize } - } else { - unsafe { *slot_ptr = 0 } - } - - *self.allocated_count() -= 1; - *self.next_free() = NonNull::new(slot_ptr); - } - - fn slab_page_init(&self, object_size: u32) -> Option<NonNull<usize>> { - assert!(object_size >= core::mem::size_of::<usize>() as u32); - - let first_free = self.real_page_ptr() as *mut usize; - - let mut slot_ptr = first_free; - let mut slot_count = PAGE_SIZE / object_size as usize; - - // SAFETY: carefully ptr operate - unsafe { - loop { - if slot_count == 1 { - *slot_ptr = 0; - break; - } - - let next_ptr = slot_ptr.byte_add(object_size as usize); - *slot_ptr = next_ptr as usize; - slot_ptr = next_ptr; - slot_count -= 1; - } - } - - NonNull::new(first_free) - } - - fn is_empty(&self) -> bool { - self.allocated_count().clone() == 0 - } - - fn is_full(&self) -> bool { - self.next_free().is_none() - } -} - -impl<Raw, Allocator> SlabCache<Raw, Allocator> -where - Raw: SlabRawPage, - Allocator: PageAlloc<RawPage = Raw>, -{ - pub(crate) const fn new_in(object_size: u32) -> Self { - // avoid unnecessary branch in alloc and dealloc - assert!(object_size <= PAGE_SIZE as u32 / 2); - - Self { - empty_list: List::new(), - partial_list: List::new(), - full_list: List::new(), - object_size: object_size, - _phantom: PhantomData, - } - } - - pub(crate) fn alloc(&mut self, alloc: &Allocator) -> *mut u8 { - if !self.partial_list.is_empty() { - let page_ptr = unsafe { - Raw::from_link( - self.partial_list - .head() - .expect("partial pages should not be empty"), - ) - }; - - let ptr = page_ptr.alloc_slot().expect("should get slot"); - - if page_ptr.is_full() { - self.partial_list.remove(unsafe { page_ptr.get_link() }); - self.full_list.insert(unsafe { page_ptr.get_link() }); - } - return ptr.as_ptr() as *mut u8; - } - - if !self.empty_list.is_empty() { - let page_ptr = unsafe { - Raw::from_link( - self.empty_list - .head() - .expect("empty pages should not be empty"), - ) - }; - - let ptr = page_ptr.alloc_slot().expect("should get slot"); - self.empty_list.remove(unsafe { page_ptr.get_link() }); - self.partial_list.insert(unsafe { page_ptr.get_link() }); - return ptr.as_ptr() as *mut u8; - } - - let new_page_ptr = alloc.alloc().expect("slab_cache get page fail!"); - let first_free = new_page_ptr.slab_page_init(self.object_size); - new_page_ptr.slab_init(first_free); - let ptr = new_page_ptr.alloc_slot().expect("should get slot"); - self.partial_list.insert(unsafe { new_page_ptr.get_link() }); - ptr.as_ptr() as *mut u8 - } - - pub(crate) fn dealloc(&mut self, ptr: *mut u8, _alloc: &Allocator) { - let page_ptr = Raw::in_which(ptr); - - if page_ptr.is_full() { - self.full_list.remove(unsafe { page_ptr.get_link() }); - self.partial_list.insert(unsafe { page_ptr.get_link() }); - } - - page_ptr.dealloc_slot(ptr); - - if page_ptr.is_empty() { - self.partial_list.remove(unsafe { page_ptr.get_link() }); - self.empty_list.insert(unsafe { page_ptr.get_link() }); - } - - // TODO: Check whether we should place some pages back with `alloc` if the global - // free page count is below the watermark. - } -} diff --git a/src/kernel/mem/allocator.rs b/src/kernel/mem/allocator.rs index 9e5df69b..a3676ce0 100644 --- a/src/kernel/mem/allocator.rs +++ b/src/kernel/mem/allocator.rs @@ -5,13 +5,12 @@ use eonix_hal::mm::ArchPhysAccess; use eonix_mm::address::PhysAccess; use eonix_mm::paging::{PAGE_SIZE_BITS, PFN}; use eonix_sync::LazyLock; -use slab_allocator::SlabAllocator; +use slab_allocator::SlabAlloc; -use super::page_alloc::RawPagePtr; use super::{GlobalPageAlloc, Page, PageExt}; -static SLAB_ALLOCATOR: LazyLock<SlabAllocator<RawPagePtr, GlobalPageAlloc, 9>> = - LazyLock::new(|| SlabAllocator::new_in(GlobalPageAlloc)); +static SLAB_ALLOCATOR: LazyLock<SlabAlloc<GlobalPageAlloc, 9>> = + LazyLock::new(|| SlabAlloc::new_in(GlobalPageAlloc)); struct Allocator; @@ -28,23 +27,23 @@ unsafe impl GlobalAlloc for Allocator { let ptr = page.get_ptr(); page.into_raw(); - ptr.as_ptr() + ptr }; - if result.is_null() { - core::ptr::null_mut() - } else { - result as *mut u8 - } + result.as_ptr() } unsafe fn dealloc(&self, ptr: *mut u8, layout: Layout) { let size = layout.size().next_power_of_two(); + let ptr = unsafe { + // SAFETY: The memory we've allocated MUST be non-null. + NonNull::new_unchecked(ptr) + }; if size <= 2048 { SLAB_ALLOCATOR.dealloc(ptr, size) } else { - let paddr = ArchPhysAccess::from_ptr(NonNull::new_unchecked(ptr)); + let paddr = ArchPhysAccess::from_ptr(ptr); let pfn = PFN::from(paddr); Page::from_raw(pfn); }; diff --git a/src/kernel/mem/page_alloc.rs b/src/kernel/mem/page_alloc.rs index fcbe9bb3..1c018f37 100644 --- a/src/kernel/mem/page_alloc.rs +++ b/src/kernel/mem/page_alloc.rs @@ -1,15 +1,13 @@ mod raw_page; -use buddy_allocator::{BuddyAllocator, BuddyRawPage as _}; use core::sync::atomic::Ordering; -use eonix_mm::{ - address::{AddrOps as _, PRange}, - paging::{GlobalPageAlloc as GlobalPageAllocTrait, PageAlloc, PFN}, -}; + +use buddy_allocator::{BuddyAllocator, BuddyRawPage as _}; +use eonix_mm::address::{AddrOps as _, PRange}; +use eonix_mm::paging::{GlobalPageAlloc as GlobalPageAllocTrait, PageAlloc, PFN}; use eonix_sync::{NoContext, Spin}; use intrusive_list::List; use raw_page::PageFlags; - pub use raw_page::{RawPage, RawPagePtr}; const COSTLY_ORDER: u32 = 3; diff --git a/src/kernel/mem/page_alloc/raw_page.rs b/src/kernel/mem/page_alloc/raw_page.rs index 08536693..d793ccd7 100644 --- a/src/kernel/mem/page_alloc/raw_page.rs +++ b/src/kernel/mem/page_alloc/raw_page.rs @@ -1,65 +1,46 @@ -use crate::kernel::mem::page_cache::PageCacheRawPage; -use crate::kernel::mem::PhysAccess; +use core::ptr::NonNull; +use core::sync::atomic::{AtomicU32, AtomicUsize, Ordering}; + use buddy_allocator::BuddyRawPage; -use core::{ - ptr::NonNull, - sync::atomic::{AtomicU32, AtomicUsize, Ordering}, -}; use eonix_hal::mm::ArchPhysAccess; -use eonix_mm::{ - address::{PAddr, PhysAccess as _}, - paging::{RawPage as RawPageTrait, PFN}, -}; -use intrusive_list::{container_of, Link}; -use slab_allocator::SlabRawPage; +use eonix_mm::address::{PAddr, PhysAccess as _}; +use eonix_mm::paging::{PageAlloc, RawPage as RawPageTrait, PFN}; +use intrusive_list::{container_of, Link, List}; +use slab_allocator::{SlabPage, SlabPageAlloc, SlabPageList, SlabSlot}; + +use super::GlobalPageAlloc; +use crate::kernel::mem::page_cache::PageCacheRawPage; +use crate::kernel::mem::PhysAccess; const PAGE_ARRAY: NonNull<RawPage> = unsafe { NonNull::new_unchecked(0xffffff8040000000 as *mut _) }; pub struct PageFlags(AtomicU32); -struct SlabPageInner { - allocated_count: u32, - free_next: Option<NonNull<usize>>, +#[derive(Clone, Copy)] +struct SlabPageData { + allocated_count: usize, + free_next: Option<NonNull<SlabSlot>>, } -impl SlabPageInner { - fn new(free_next: Option<NonNull<usize>>) -> Self { +impl SlabPageData { + const fn new() -> Self { Self { allocated_count: 0, - free_next, + free_next: None, } } } -struct PageCacheInner { +#[derive(Clone, Copy)] +struct PageCacheData { valid_size: usize, } -pub struct BuddyPageInner {} - -enum PageType { - Buddy(BuddyPageInner), - Slab(SlabPageInner), - PageCache(PageCacheInner), -} - -impl PageType { - fn slab_data(&mut self) -> &mut SlabPageInner { - if let PageType::Slab(slab_data) = self { - return slab_data; - } else { - unreachable!() - } - } - - fn page_cache_data(&mut self) -> &mut PageCacheInner { - if let PageType::PageCache(cache_data) = self { - return cache_data; - } else { - unreachable!() - } - } +#[repr(C)] +union PageData { + slab: SlabPageData, + page_cache: PageCacheData, } pub struct RawPage { @@ -73,7 +54,7 @@ pub struct RawPage { flags: PageFlags, refcount: AtomicUsize, - shared_data: PageType, + shared_data: PageData, } #[derive(Copy, Clone, PartialEq, Eq, PartialOrd, Ord)] @@ -109,6 +90,13 @@ impl PageFlags { } impl RawPagePtr { + pub const fn from_ref(raw_page_ref: &RawPage) -> Self { + Self::new(unsafe { + // SAFETY: Rust references always points to non-null addresses. + NonNull::new_unchecked(&raw const *raw_page_ref as *mut _) + }) + } + pub const fn new(ptr: NonNull<RawPage>) -> Self { Self(ptr) } @@ -215,50 +203,68 @@ impl BuddyRawPage for RawPagePtr { } } -impl SlabRawPage for RawPagePtr { - unsafe fn from_link(link: &mut Link) -> Self { - let raw_page_ptr = container_of!(link, RawPage, link); - Self(raw_page_ptr) - } +impl SlabPage for RawPage { + fn get_data_ptr(&self) -> NonNull<[u8]> { + let raw_page_ptr = RawPagePtr::from_ref(self); + let paddr_start = PAddr::from(PFN::from(raw_page_ptr)); + let page_data_ptr = unsafe { paddr_start.as_ptr() }; - unsafe fn get_link(&self) -> &mut Link { - &mut self.as_mut().link + NonNull::slice_from_raw_parts(page_data_ptr, 1 << (self.order + 12)) } - fn in_which(ptr: *mut u8) -> RawPagePtr { + fn get_free_slot(&self) -> Option<NonNull<SlabSlot>> { unsafe { - // SAFETY: The pointer is allocated from the slab allocator, - // which can't be null. - let ptr = NonNull::new_unchecked(ptr); + // SAFETY: TODO + self.shared_data.slab.free_next + } + } - // SAFETY: The pointer is valid. - let paddr = ArchPhysAccess::from_ptr(ptr); - let pfn = PFN::from(paddr); + fn set_free_slot(&mut self, next: Option<NonNull<SlabSlot>>) { + self.shared_data.slab.free_next = next; + } - RawPagePtr::from(pfn) + fn get_alloc_count(&self) -> usize { + unsafe { + // SAFETY: TODO + self.shared_data.slab.allocated_count } } - fn allocated_count(&self) -> &mut u32 { - &mut self.as_mut().shared_data.slab_data().allocated_count - } + fn inc_alloc_count(&mut self) -> usize { + unsafe { + // SAFETY: TODO + self.shared_data.slab.allocated_count += 1; - fn next_free(&self) -> &mut Option<NonNull<usize>> { - &mut self.as_mut().shared_data.slab_data().free_next + self.shared_data.slab.allocated_count + } } - fn real_page_ptr(&self) -> *mut u8 { - self.real_ptr().as_ptr() + fn dec_alloc_count(&mut self) -> usize { + unsafe { + // SAFETY: TODO + self.shared_data.slab.allocated_count -= 1; + + self.shared_data.slab.allocated_count + } } - fn slab_init(&self, first_free: Option<NonNull<usize>>) { - self.as_mut().shared_data = PageType::Slab(SlabPageInner::new(first_free)); + unsafe fn from_allocated(ptr: NonNull<u8>) -> &'static mut Self { + unsafe { + // SAFETY: The caller ensures that `ptr` is valid. + let paddr = ArchPhysAccess::from_ptr(ptr); + let pfn = PFN::from(paddr); + + RawPagePtr::from(pfn).as_mut() + } } } impl PageCacheRawPage for RawPagePtr { fn valid_size(&self) -> &mut usize { - &mut self.as_mut().shared_data.page_cache_data().valid_size + unsafe { + // SAFETY: The caller ensures that the page is in some page cache. + &mut self.as_mut().shared_data.page_cache.valid_size + } } fn is_dirty(&self) -> bool { @@ -274,6 +280,59 @@ impl PageCacheRawPage for RawPagePtr { } fn cache_init(&self) { - self.as_mut().shared_data = PageType::PageCache(PageCacheInner { valid_size: 0 }); + self.as_mut().shared_data.page_cache = PageCacheData { valid_size: 0 }; + } +} + +pub struct RawSlabPageList(List); + +impl SlabPageList for RawSlabPageList { + type Page = RawPage; + + fn new() -> Self { + Self(List::new()) + } + + fn is_empty(&self) -> bool { + self.0.is_empty() + } + + fn peek_head(&mut self) -> Option<&mut Self::Page> { + unsafe { + let link = self.0.head()?; + let mut raw_page_ptr = container_of!(link, RawPage, link); + + Some(raw_page_ptr.as_mut()) + } + } + + fn pop_head(&mut self) -> Option<&'static mut Self::Page> { + unsafe { + let link = self.0.pop()?; + let mut raw_page_ptr = container_of!(link, RawPage, link); + + Some(raw_page_ptr.as_mut()) + } + } + + fn push_tail(&mut self, page: &'static mut Self::Page) { + self.0.insert(&mut page.link); + } + + fn remove(&mut self, page: &mut Self::Page) { + self.0.remove(&mut page.link) + } +} + +impl SlabPageAlloc for GlobalPageAlloc { + type Page = RawPage; + type PageList = RawSlabPageList; + + unsafe fn alloc_uninit(&self) -> &'static mut RawPage { + let raw_page = self.alloc().expect("Out of memory").as_mut(); + raw_page.flags.set(PageFlags::SLAB); + raw_page.shared_data.slab = SlabPageData::new(); + + raw_page } } From 7792a5e8f2dffc15a90042839b0fe3e6329157f4 Mon Sep 17 00:00:00 2001 From: greatbridf <greatbridf@icloud.com> Date: Wed, 7 Jan 2026 02:16:47 +0800 Subject: [PATCH 21/36] mem, buddy: rework the buddy system Introduce `Zone`s: a Zone is a region of physical memory that all share the same NUMA node. The Zone will hold all RawPage structs. Buddy allocator will now store a reference to the zone that hold all its pages. Thus, we make the buddy allocator independent of underlying physical page frame management framework. Remove unnecessary page flags and structs. Signed-off-by: greatbridf <greatbridf@icloud.com> --- Cargo.lock | 1 - crates/buddy_allocator/Cargo.toml | 1 - crates/buddy_allocator/src/free_area.rs | 59 ------ crates/buddy_allocator/src/lib.rs | 265 +++++++++++++++++++----- crates/buddy_allocator/src/zone.rs | 146 ------------- crates/eonix_mm/src/paging.rs | 4 + crates/eonix_mm/src/paging/list.rs | 19 ++ crates/eonix_mm/src/paging/raw_page.rs | 9 +- crates/eonix_mm/src/paging/zone.rs | 20 ++ crates/slab_allocator/src/lib.rs | 35 ++-- src/kernel/mem/page_alloc.rs | 116 +++++------ src/kernel/mem/page_alloc/raw_page.rs | 87 ++++---- src/kernel/mem/page_alloc/zones.rs | 25 +++ src/kernel_init.rs | 24 +-- 14 files changed, 407 insertions(+), 404 deletions(-) delete mode 100644 crates/buddy_allocator/src/free_area.rs delete mode 100644 crates/buddy_allocator/src/zone.rs create mode 100644 crates/eonix_mm/src/paging/list.rs create mode 100644 crates/eonix_mm/src/paging/zone.rs create mode 100644 src/kernel/mem/page_alloc/zones.rs diff --git a/Cargo.lock b/Cargo.lock index 3e8a36bd..896ec493 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -70,7 +70,6 @@ name = "buddy_allocator" version = "0.1.0" dependencies = [ "eonix_mm", - "intrusive_list", ] [[package]] diff --git a/crates/buddy_allocator/Cargo.toml b/crates/buddy_allocator/Cargo.toml index 51f02295..bdb0a28c 100644 --- a/crates/buddy_allocator/Cargo.toml +++ b/crates/buddy_allocator/Cargo.toml @@ -5,4 +5,3 @@ edition = "2024" [dependencies] eonix_mm = { path = "../eonix_mm" } -intrusive_list = { path = "../intrusive_list" } diff --git a/crates/buddy_allocator/src/free_area.rs b/crates/buddy_allocator/src/free_area.rs deleted file mode 100644 index 837f733f..00000000 --- a/crates/buddy_allocator/src/free_area.rs +++ /dev/null @@ -1,59 +0,0 @@ -use crate::BuddyRawPage; -use core::marker::{PhantomData, Send, Sync}; -use intrusive_list::Link; - -pub struct FreeArea<T> { - free_list: Link, - count: usize, - _phantom: PhantomData<T>, -} - -unsafe impl<T> Send for FreeArea<T> {} -unsafe impl<T> Sync for FreeArea<T> {} - -impl<Raw> FreeArea<Raw> -where - Raw: BuddyRawPage, -{ - pub const fn new() -> Self { - Self { - free_list: Link::new(), - count: 0, - _phantom: PhantomData, - } - } - - pub fn get_free_pages(&mut self) -> Option<Raw> { - self.free_list.next_mut().map(|pages_link| { - assert_ne!(self.count, 0); - - let pages_ptr = unsafe { - // SAFETY: Items in `self.free_list` are guaranteed to be of type `Raw`. - Raw::from_link(pages_link) - }; - - self.count -= 1; - pages_link.remove(); - - pages_ptr - }) - } - - pub fn add_pages(&mut self, pages_ptr: Raw) { - self.count += 1; - pages_ptr.set_free(); - - unsafe { - self.free_list.insert(pages_ptr.get_link()); - } - } - - pub fn del_pages(&mut self, pages_ptr: Raw) { - assert!(self.count >= 1 && pages_ptr.is_free()); - self.count -= 1; - pages_ptr.clear_free(); - unsafe { - pages_ptr.get_link().remove(); - } - } -} diff --git a/crates/buddy_allocator/src/lib.rs b/crates/buddy_allocator/src/lib.rs index f8c8eeda..abe1ef7b 100644 --- a/crates/buddy_allocator/src/lib.rs +++ b/crates/buddy_allocator/src/lib.rs @@ -1,87 +1,250 @@ #![no_std] -mod free_area; -mod zone; +use core::hint::unreachable_unchecked; -use core::sync::atomic::Ordering; -use eonix_mm::{ - address::PAddr, - paging::{RawPage, PFN}, -}; -use intrusive_list::Link; -use zone::Zone; +use eonix_mm::address::{AddrOps as _, PAddr, PRange}; +use eonix_mm::paging::{PageList, PageListSized, Zone, PFN}; const MAX_ORDER: u32 = 10; -const ZONE_AREAS: usize = const { MAX_ORDER as usize + 1 }; +const AREAS: usize = const { MAX_ORDER as usize + 1 }; -pub trait BuddyRawPage: RawPage { - /// Get the container raw page struct of the list link. - /// - /// # Safety - /// The caller MUST ensure that the link points to a `RawPage`. - unsafe fn from_link(link: &mut Link) -> Self; - - /// Get the list link of the raw page. - /// - /// # Safety - /// The caller MUST ensure that at any time, only one mutable reference - /// to the link exists. - unsafe fn get_link(&self) -> &mut Link; - - fn set_order(&self, order: u32); +pub trait BuddyPage: Sized + 'static { + fn pfn(&self) -> PFN; + fn get_order(&self) -> u32; fn is_buddy(&self) -> bool; - fn is_free(&self) -> bool; - fn set_buddy(&self); - fn set_free(&self); + fn set_order(&mut self, order: u32); + fn set_buddy(&mut self, value: bool); +} - fn clear_buddy(&self); - fn clear_free(&self); +struct FreeArea<L> +where + L: PageList, +{ + free_list: L, + count: usize, } -pub struct BuddyAllocator<T> +unsafe impl<L> Send for FreeArea<L> where L: PageList {} +unsafe impl<L> Sync for FreeArea<L> where L: PageList {} + +pub struct BuddyAllocator<Z, L> where - T: BuddyRawPage, + Z: Zone + 'static, + L: PageList, { - zone: Zone<T, ZONE_AREAS>, + zone: &'static Z, + free_areas: [FreeArea<L>; AREAS], } -impl<T> BuddyAllocator<T> +impl<Z, L> BuddyAllocator<Z, L> where - T: BuddyRawPage, + Z: Zone + 'static, + Z::Page: BuddyPage, + L: PageListSized, { - pub const fn new() -> Self { - Self { zone: Zone::new() } + pub const fn new(zone: &'static Z) -> Self { + Self { + zone, + free_areas: [const { FreeArea::new() }; AREAS], + } } +} +impl<Z, L, P> BuddyAllocator<Z, L> +where + Z: Zone<Page = P>, + L: PageList<Page = P>, + P: BuddyPage + 'static, +{ pub fn create_pages(&mut self, start: PAddr, end: PAddr) { - self.zone.create_pages(start, end); + assert!( + self.zone + .contains_prange(PRange::new(start.ceil(), end.floor())), + "The given address range is not within the zone." + ); + + let mut pfn = PFN::from(start.ceil()); + let end_pfn = PFN::from(end.floor()); + + while pfn < end_pfn { + let mut order = usize::from(pfn).trailing_zeros().min(MAX_ORDER); + let new_end_pfn = loop { + let new_end = pfn + (1 << order); + + if new_end <= end_pfn { + break new_end; + } + + order -= 1; + }; + + unsafe { + // SAFETY: We've checked that the range is within the zone above. + self.add_page_unchecked(pfn, order) + }; + + pfn = new_end_pfn; + } + } + + fn add_page(&mut self, pfn: PFN, order: u32) { + let prange = PRange::from(PAddr::from(pfn)).grow(1 << (order + 12)); + assert!( + self.zone.contains_prange(prange), + "The given page is not within the zone." + ); + + unsafe { + // SAFETY: Checks above. + self.add_page_unchecked(pfn, order); + } + } + + unsafe fn add_page_unchecked(&mut self, pfn: PFN, order: u32) { + let Some(page) = self.zone.get_page(pfn) else { + unsafe { unreachable_unchecked() } + }; + + unsafe { + // SAFETY: The caller ensures that the page is unused. + let page_mut = &mut *page.get(); + self.free_areas[order as usize].add_page(page_mut, order); + } + } + + fn break_page(&mut self, page: &mut P, order: u32, target_order: u32) { + let pfn = page.pfn(); + + for order in (target_order..order).rev() { + let buddy_pfn = pfn + (1 << order); + + unsafe { + // SAFETY: We got the page from `self.free_areas`. Checks are + // done when we've put the page into the buddy system. + self.add_page_unchecked(buddy_pfn, order); + } + } + + page.set_order(target_order); + } + + pub fn alloc_order(&mut self, order: u32) -> Option<&'static mut Z::Page> { + for current_order in order..AREAS as u32 { + let Some(page) = self.free_areas[current_order as usize].get_free_page() else { + continue; + }; + + if current_order > order { + self.break_page(page, current_order, order); + } + + return Some(page); + } + + None + } + + pub unsafe fn dealloc(&mut self, page: &'static mut Z::Page) { + let mut pfn = page.pfn(); + let mut order = page.get_order(); + + assert!( + !page.is_buddy(), + "Trying to free a page that is already in the buddy system: {pfn:?}", + ); + + while order < MAX_ORDER { + let buddy_pfn = pfn.buddy_pfn(order); + let Some(buddy_page) = self.try_get_buddy(buddy_pfn, order) else { + break; + }; + + self.free_areas[order as usize].remove_page(buddy_page); + pfn = pfn.combined_pfn(buddy_pfn); + order += 1; + } + + self.add_page(pfn, order); } - pub fn alloc_order(&mut self, order: u32) -> Option<T> { - let pages_ptr = self.zone.get_free_pages(order); + /// This function checks whether the given page is within our [`Zone`] and + /// is a free buddy page with the specified order. + /// + /// We can assure exclusive access to a buddy page of [`order`] if + /// - the buddy is within the same [`Zone`] as us. + /// - the buddy is a free buddy (in some [`FreeArea`]) + /// - the buddy has order [`order`] + fn try_get_buddy<'a>(&mut self, buddy_pfn: PFN, order: u32) -> Option<&'a mut P> { + let buddy_page = self.zone.get_page(buddy_pfn)?; + + unsafe { + // SAFETY: We just test whether the page is a buddy. + let buddy_page_ref = &*buddy_page.get(); - if let Some(pages_ptr) = pages_ptr { - // SAFETY: Memory order here can be Relaxed is for the same reason as that - // in the copy constructor of `std::shared_ptr`. - pages_ptr.refcount().fetch_add(1, Ordering::Relaxed); - pages_ptr.clear_free(); + if !buddy_page_ref.is_buddy() { + return None; + } + + // Sad... + if buddy_page_ref.get_order() != order { + return None; + } + + // SAFETY: We have the mutable reference to the buddy allocator. + // So all the pages within are exclusively accessible to us. + Some(&mut *buddy_page.get()) + } + } +} + +impl<L> FreeArea<L> +where + L: PageListSized, +{ + const fn new() -> Self { + Self { + free_list: L::NEW, + count: 0, } + } +} + +impl<L> FreeArea<L> +where + L: PageList, + L::Page: BuddyPage + 'static, +{ + pub fn get_free_page(&mut self) -> Option<&'static mut L::Page> { + self.free_list.pop_head().map(|page| { + assert_ne!(self.count, 0, "Oops"); + + page.set_buddy(false); + self.count -= 1; - pages_ptr + page + }) } - pub unsafe fn dealloc(&mut self, page_ptr: T) { - self.zone.free_pages(page_ptr); + pub fn add_page(&mut self, page: &'static mut L::Page, order: u32) { + page.set_order(order); + page.set_buddy(true); + + self.count += 1; + self.free_list.push_tail(page); } - pub fn has_management_over(page_ptr: T) -> bool { - !page_ptr.is_free() && page_ptr.is_buddy() + pub fn remove_page(&mut self, page: &mut L::Page) { + assert_ne!(self.count, 0, "Oops"); + page.set_buddy(false); + + self.count -= 1; + self.free_list.remove(page); } } -pub(self) trait BuddyPFNOps { +trait BuddyPFNOps { fn buddy_pfn(self, order: u32) -> PFN; fn combined_pfn(self, buddy_pfn: PFN) -> PFN; } diff --git a/crates/buddy_allocator/src/zone.rs b/crates/buddy_allocator/src/zone.rs deleted file mode 100644 index 2c850ef9..00000000 --- a/crates/buddy_allocator/src/zone.rs +++ /dev/null @@ -1,146 +0,0 @@ -use super::free_area::FreeArea; -use crate::{BuddyPFNOps as _, BuddyRawPage}; -use core::sync::atomic::Ordering; -use eonix_mm::{ - address::{AddrOps as _, PAddr}, - paging::PFN, -}; - -pub(super) struct Zone<T, const AREAS: usize> { - free_areas: [FreeArea<T>; AREAS], -} - -impl<Raw, const AREAS: usize> Zone<Raw, AREAS> -where - Raw: BuddyRawPage, -{ - pub const fn new() -> Self { - Self { - free_areas: [const { FreeArea::new() }; AREAS], - } - } - - pub fn get_free_pages(&mut self, order: u32) -> Option<Raw> { - for current_order in order..AREAS as u32 { - let pages_ptr = self.free_areas[current_order as usize].get_free_pages(); - let Some(pages_ptr) = pages_ptr else { continue }; - - pages_ptr.set_order(order); - - if current_order > order { - self.expand(pages_ptr, current_order, order); - } - - assert!( - pages_ptr.is_present(), - "Page {:?} is not present", - pages_ptr.into(), - ); - - assert!( - pages_ptr.is_free(), - "Page {:?} is not free", - pages_ptr.into(), - ); - - return Some(pages_ptr); - } - None - } - - fn expand(&mut self, pages_ptr: Raw, order: u32, target_order: u32) { - let mut offset = 1 << order; - let pages_pfn = Into::<PFN>::into(pages_ptr); - - for order in (target_order..order).rev() { - offset >>= 1; - - let split_pages_ptr = Raw::from(pages_pfn + offset); - split_pages_ptr.set_order(order); - split_pages_ptr.set_buddy(); - self.free_areas[order as usize].add_pages(split_pages_ptr); - } - } - - pub fn free_pages(&mut self, mut pages_ptr: Raw) { - assert_eq!(pages_ptr.refcount().load(Ordering::Relaxed), 0); - - let mut pfn = Into::<PFN>::into(pages_ptr); - let mut current_order = pages_ptr.order(); - - assert!( - pages_ptr.is_present(), - "Freeing a page that is not present: {:?}", - pages_ptr.into(), - ); - - assert!( - !pages_ptr.is_free(), - "Freeing a page that is free: {:?}", - pages_ptr.into(), - ); - - while current_order < (AREAS - 1) as u32 { - let buddy_pfn = pfn.buddy_pfn(current_order); - let buddy_pages_ptr = Raw::from(buddy_pfn); - - if !self.buddy_check(buddy_pages_ptr, current_order) { - break; - } - - pages_ptr.clear_buddy(); - buddy_pages_ptr.clear_buddy(); - self.free_areas[current_order as usize].del_pages(buddy_pages_ptr); - - pages_ptr = Raw::from(pfn.combined_pfn(buddy_pfn)); - pfn = pfn.combined_pfn(buddy_pfn); - - pages_ptr.set_buddy(); - current_order += 1; - } - - pages_ptr.set_order(current_order); - self.free_areas[current_order as usize].add_pages(pages_ptr); - } - - /// This function checks whether a page is free && is a buddy - /// we can coalesce a page and its buddy if - /// - the buddy is valid(present) && - /// - the buddy is right now in free_areas && - /// - a page and its buddy have the same order && - /// - a page and its buddy are in the same zone (on smp systems). - fn buddy_check(&self, pages_ptr: Raw, order: u32) -> bool { - if !pages_ptr.is_present() { - return false; - } - if !pages_ptr.is_free() { - return false; - } - if pages_ptr.order() != order { - return false; - } - - assert_eq!(pages_ptr.refcount().load(Ordering::Relaxed), 0); - true - } - - /// Only used on buddy initialization - pub fn create_pages(&mut self, start: PAddr, end: PAddr) { - let mut start_pfn = PFN::from(start.ceil()); - let end_pfn = PFN::from(end.floor()); - - while start_pfn < end_pfn { - let mut order = usize::from(start_pfn) - .trailing_zeros() - .min((AREAS - 1) as u32); - - while start_pfn + (1 << order) as usize > end_pfn { - order -= 1; - } - let page_ptr = Raw::from(start_pfn); - page_ptr.set_buddy(); - self.free_areas[order as usize].add_pages(page_ptr); - start_pfn = start_pfn + (1 << order) as usize; - } - } -} diff --git a/crates/eonix_mm/src/paging.rs b/crates/eonix_mm/src/paging.rs index 88da902e..0c4811f2 100644 --- a/crates/eonix_mm/src/paging.rs +++ b/crates/eonix_mm/src/paging.rs @@ -1,9 +1,13 @@ +mod list; mod page; mod page_alloc; mod pfn; mod raw_page; +mod zone; +pub use list::{PageList, PageListSized}; pub use page::{Page, PageAccess, PageBlock, PAGE_SIZE, PAGE_SIZE_BITS}; pub use page_alloc::{GlobalPageAlloc, NoAlloc, PageAlloc}; pub use pfn::PFN; pub use raw_page::{RawPage, UnmanagedRawPage}; +pub use zone::Zone; diff --git a/crates/eonix_mm/src/paging/list.rs b/crates/eonix_mm/src/paging/list.rs new file mode 100644 index 00000000..a52cf947 --- /dev/null +++ b/crates/eonix_mm/src/paging/list.rs @@ -0,0 +1,19 @@ +pub trait PageList { + type Page; + + fn is_empty(&self) -> bool; + + fn peek_head(&mut self) -> Option<&mut Self::Page>; + + fn pop_head(&mut self) -> Option<&'static mut Self::Page>; + fn push_tail(&mut self, page: &'static mut Self::Page); + fn remove(&mut self, page: &mut Self::Page); +} + +pub trait PageListSized: PageList + Sized { + const NEW: Self; + + fn new() -> Self { + Self::NEW + } +} diff --git a/crates/eonix_mm/src/paging/raw_page.rs b/crates/eonix_mm/src/paging/raw_page.rs index 7951729d..789e863b 100644 --- a/crates/eonix_mm/src/paging/raw_page.rs +++ b/crates/eonix_mm/src/paging/raw_page.rs @@ -1,14 +1,13 @@ -use super::PFN; use core::sync::atomic::AtomicUsize; +use super::PFN; + /// A `RawPage` represents a page of memory in the kernel. It is a low-level /// representation of a page that is used by the kernel to manage memory. #[doc(notable_trait)] pub trait RawPage: Clone + Copy + From<PFN> + Into<PFN> { fn order(&self) -> u32; fn refcount(&self) -> &AtomicUsize; - - fn is_present(&self) -> bool; } #[derive(Clone, Copy)] @@ -45,8 +44,4 @@ impl RawPage for UnmanagedRawPage { fn refcount(&self) -> &AtomicUsize { &UNMANAGED_RAW_PAGE_CLONE_COUNT } - - fn is_present(&self) -> bool { - true - } } diff --git a/crates/eonix_mm/src/paging/zone.rs b/crates/eonix_mm/src/paging/zone.rs new file mode 100644 index 00000000..ec3ed15e --- /dev/null +++ b/crates/eonix_mm/src/paging/zone.rs @@ -0,0 +1,20 @@ +use core::cell::UnsafeCell; + +#[allow(unused_imports)] +use super::{Page, PageAlloc, RawPage, PFN}; +use crate::address::PRange; + +/// A [`Zone`] holds a lot of [`Page`]s that share the same NUMA node or +/// "physical location". +pub trait Zone: Send + Sync { + type Page; + + /// Whether the [`range`] is within this [`Zone`]. + fn contains_prange(&self, range: PRange) -> bool; + + /// Get the [`RawPage`] that [`pfn`] points to. + /// + /// # Return + /// [`None`] if [`pfn`] is not in this [`Zone`]. + fn get_page(&self, pfn: PFN) -> Option<&UnsafeCell<Self::Page>>; +} diff --git a/crates/slab_allocator/src/lib.rs b/crates/slab_allocator/src/lib.rs index 8a684edd..8597331d 100644 --- a/crates/slab_allocator/src/lib.rs +++ b/crates/slab_allocator/src/lib.rs @@ -2,6 +2,7 @@ use core::ptr::NonNull; +use eonix_mm::paging::{PageList, PageListSized}; use eonix_sync::Spin; #[repr(C)] @@ -10,19 +11,6 @@ pub union SlabSlot { data: u8, } -pub trait SlabPageList: Sized { - type Page: SlabPage; - - fn new() -> Self; - fn is_empty(&self) -> bool; - - fn peek_head(&mut self) -> Option<&mut Self::Page>; - - fn pop_head(&mut self) -> Option<&'static mut Self::Page>; - fn push_tail(&mut self, page: &'static mut Self::Page); - fn remove(&mut self, page: &mut Self::Page); -} - pub trait SlabPage: Sized + 'static { fn get_data_ptr(&self) -> NonNull<[u8]>; @@ -98,7 +86,7 @@ where pub trait SlabPageAlloc { type Page: SlabPage; - type PageList: SlabPageList<Page = Self::Page>; + type PageList: PageList<Page = Self::Page>; /// Allocate a page suitable for slab system use. The page MUST come with /// its allocation count 0 and next free slot None. @@ -110,7 +98,7 @@ pub trait SlabPageAlloc { pub(crate) struct SlabList<T> where - T: SlabPageList, + T: PageList, { empty_list: T, partial_list: T, @@ -132,6 +120,7 @@ unsafe impl<P, const COUNT: usize> Sync for SlabAlloc<P, COUNT> where P: SlabPag impl<L, const COUNT: usize> SlabAlloc<L, COUNT> where L: SlabPageAlloc, + L::PageList: PageListSized, { pub fn new_in(alloc: L) -> Self { Self { @@ -159,17 +148,23 @@ where impl<T> SlabList<T> where - T: SlabPageList, + T: PageListSized, { - fn new(object_size: usize) -> Self { + const fn new(object_size: usize) -> Self { Self { - empty_list: T::new(), - partial_list: T::new(), - full_list: T::new(), + empty_list: T::NEW, + partial_list: T::NEW, + full_list: T::NEW, object_size, } } +} +impl<T> SlabList<T> +where + T: PageList, + T::Page: SlabPage, +{ fn alloc_from_partial(&mut self) -> NonNull<u8> { let head = self.partial_list.peek_head().unwrap(); let slot = head.alloc_slot().unwrap(); diff --git a/src/kernel/mem/page_alloc.rs b/src/kernel/mem/page_alloc.rs index 1c018f37..9dce4567 100644 --- a/src/kernel/mem/page_alloc.rs +++ b/src/kernel/mem/page_alloc.rs @@ -1,19 +1,25 @@ mod raw_page; +mod zones; use core::sync::atomic::Ordering; -use buddy_allocator::{BuddyAllocator, BuddyRawPage as _}; +use buddy_allocator::BuddyAllocator; use eonix_mm::address::{AddrOps as _, PRange}; -use eonix_mm::paging::{GlobalPageAlloc as GlobalPageAllocTrait, PageAlloc, PFN}; +use eonix_mm::paging::{ + GlobalPageAlloc as GlobalPageAllocTrait, PageAlloc, PageList, PageListSized as _, PFN, +}; +use eonix_preempt::PreemptGuard; use eonix_sync::{NoContext, Spin}; -use intrusive_list::List; -use raw_page::PageFlags; +use raw_page::{PageFlags, RawPageList}; pub use raw_page::{RawPage, RawPagePtr}; +pub use zones::GlobalZone; const COSTLY_ORDER: u32 = 3; +const AREAS: usize = COSTLY_ORDER as usize + 1; const BATCH_SIZE: u32 = 64; -static BUDDY_ALLOC: Spin<BuddyAllocator<RawPagePtr>> = Spin::new(BuddyAllocator::new()); +static BUDDY_ALLOC: Spin<BuddyAllocator<GlobalZone, RawPageList>> = + Spin::new(BuddyAllocator::new(&GlobalZone())); #[eonix_percpu::define_percpu] static PERCPU_PAGE_ALLOC: PerCpuPageAlloc = PerCpuPageAlloc::new(); @@ -26,58 +32,42 @@ pub struct BuddyPageAlloc(); struct PerCpuPageAlloc { batch: u32, - // TODO: might be used in the future. - // high: u32, - free_areas: [List; COSTLY_ORDER as usize + 1], + free_areas: [RawPageList; AREAS], +} + +pub trait PerCpuPage { + fn set_local(&mut self, val: bool); } impl PerCpuPageAlloc { const fn new() -> Self { Self { batch: BATCH_SIZE, - // high: 0, - free_areas: [const { List::new() }; COSTLY_ORDER as usize + 1], + free_areas: [RawPageList::NEW; AREAS], } } - fn insert_free_pages(&mut self, pages_ptr: RawPagePtr, order: u32) { - let free_area = &mut self.free_areas[order as usize]; - free_area.insert(unsafe { pages_ptr.get_link() }); - } - - fn get_free_pages(&mut self, order: u32) -> Option<RawPagePtr> { - let free_area = &mut self.free_areas[order as usize]; - free_area.pop().map(|node| unsafe { - // SAFETY: `node` is a valid pointer to a `Link` that is not used by anyone. - RawPagePtr::from_link(node) - }) - } - - fn alloc_order(&mut self, order: u32) -> Option<RawPagePtr> { + fn alloc_order(&mut self, order: u32) -> Option<&'static mut RawPage> { assert!(order <= COSTLY_ORDER); - if let Some(pages) = self.get_free_pages(order) { + if let Some(pages) = self.free_areas[order as usize].pop_head() { return Some(pages); } let batch = self.batch >> order; for _ in 0..batch { - if let Some(pages_ptr) = BUDDY_ALLOC.lock().alloc_order(order) { - pages_ptr.flags().set(PageFlags::LOCAL); - self.insert_free_pages(pages_ptr, order); - } else { + let Some(page) = BUDDY_ALLOC.lock().alloc_order(order) else { break; }; + + page.set_local(true); + self.free_areas[order as usize].push_tail(page); } - self.get_free_pages(order) + self.free_areas[order as usize].pop_head() } - fn free_pages(&mut self, pages_ptr: RawPagePtr, order: u32) { - assert_eq!(pages_ptr.order(), order); - assert_eq!(pages_ptr.refcount().load(Ordering::Relaxed), 0); - - pages_ptr.refcount().store(1, Ordering::Relaxed); - self.insert_free_pages(pages_ptr, order); + fn free_pages(&mut self, page: &'static mut RawPage, order: u32) { + self.free_areas[order as usize].push_tail(page); } } @@ -87,16 +77,6 @@ impl GlobalPageAlloc { BuddyPageAlloc() } - pub fn mark_present(range: PRange) { - let mut pfn = PFN::from(range.start().ceil()); - let end_pfn = PFN::from(range.end().floor()); - - while pfn < end_pfn { - RawPagePtr::from(pfn).flags().set(PageFlags::PRESENT); - pfn = pfn + 1; - } - } - /// Add the pages in the PAddr range `range` to the global allocator. /// /// This function is only to be called on system initialization when `eonix_preempt` @@ -116,34 +96,47 @@ impl PageAlloc for GlobalPageAlloc { type RawPage = RawPagePtr; fn alloc_order(&self, order: u32) -> Option<RawPagePtr> { - if order > COSTLY_ORDER { + let raw_page = if order > COSTLY_ORDER { BUDDY_ALLOC.lock().alloc_order(order) } else { unsafe { eonix_preempt::disable(); - let page_ptr = PERCPU_PAGE_ALLOC.as_mut().alloc_order(order); + let page = PERCPU_PAGE_ALLOC.as_mut().alloc_order(order); eonix_preempt::enable(); - page_ptr + + page } - } + }; + + raw_page.map(|raw_page| { + // SAFETY: Memory order here can be Relaxed is for the same reason + // as that in the copy constructor of `std::shared_ptr`. + raw_page.refcount.fetch_add(1, Ordering::Relaxed); + + RawPagePtr::from_ref(raw_page) + }) } unsafe fn dealloc(&self, page_ptr: RawPagePtr) { + assert_eq!( + page_ptr.refcount().load(Ordering::Relaxed), + 0, + "Trying to free a page with refcount > 0" + ); + if page_ptr.order() > COSTLY_ORDER { - BUDDY_ALLOC.lock().dealloc(page_ptr); + BUDDY_ALLOC.lock().dealloc(page_ptr.as_mut()); } else { let order = page_ptr.order(); + unsafe { - eonix_preempt::disable(); - PERCPU_PAGE_ALLOC.as_mut().free_pages(page_ptr, order); - eonix_preempt::enable(); + PreemptGuard::new(PERCPU_PAGE_ALLOC.as_mut()).free_pages(page_ptr.as_mut(), order); } } } fn has_management_over(&self, page_ptr: RawPagePtr) -> bool { - BuddyAllocator::has_management_over(page_ptr) - && (page_ptr.order() > COSTLY_ORDER || page_ptr.flags().has(PageFlags::LOCAL)) + page_ptr.order() > COSTLY_ORDER || page_ptr.flags().has(PageFlags::LOCAL) } } @@ -157,14 +150,17 @@ impl PageAlloc for BuddyPageAlloc { type RawPage = RawPagePtr; fn alloc_order(&self, order: u32) -> Option<RawPagePtr> { - BUDDY_ALLOC.lock().alloc_order(order) + BUDDY_ALLOC + .lock() + .alloc_order(order) + .map(|raw_page| RawPagePtr::from_ref(raw_page)) } unsafe fn dealloc(&self, page_ptr: RawPagePtr) { - BUDDY_ALLOC.lock().dealloc(page_ptr); + BUDDY_ALLOC.lock().dealloc(page_ptr.as_mut()); } - fn has_management_over(&self, page_ptr: RawPagePtr) -> bool { - BuddyAllocator::has_management_over(page_ptr) + fn has_management_over(&self, _: RawPagePtr) -> bool { + true } } diff --git a/src/kernel/mem/page_alloc/raw_page.rs b/src/kernel/mem/page_alloc/raw_page.rs index d793ccd7..0baa7b9a 100644 --- a/src/kernel/mem/page_alloc/raw_page.rs +++ b/src/kernel/mem/page_alloc/raw_page.rs @@ -1,18 +1,18 @@ use core::ptr::NonNull; use core::sync::atomic::{AtomicU32, AtomicUsize, Ordering}; -use buddy_allocator::BuddyRawPage; +use buddy_allocator::BuddyPage; use eonix_hal::mm::ArchPhysAccess; use eonix_mm::address::{PAddr, PhysAccess as _}; -use eonix_mm::paging::{PageAlloc, RawPage as RawPageTrait, PFN}; +use eonix_mm::paging::{PageAlloc, PageList, PageListSized, RawPage as RawPageTrait, PFN}; use intrusive_list::{container_of, Link, List}; -use slab_allocator::{SlabPage, SlabPageAlloc, SlabPageList, SlabSlot}; +use slab_allocator::{SlabPage, SlabPageAlloc, SlabSlot}; -use super::GlobalPageAlloc; +use super::{GlobalPageAlloc, PerCpuPage}; use crate::kernel::mem::page_cache::PageCacheRawPage; use crate::kernel::mem::PhysAccess; -const PAGE_ARRAY: NonNull<RawPage> = +pub const PAGE_ARRAY: NonNull<RawPage> = unsafe { NonNull::new_unchecked(0xffffff8040000000 as *mut _) }; pub struct PageFlags(AtomicU32); @@ -52,21 +52,23 @@ pub struct RawPage { /// This field is only used in buddy system and is protected by the global lock. order: u32, flags: PageFlags, - refcount: AtomicUsize, + pub refcount: AtomicUsize, shared_data: PageData, } +// XXX: introduce Folio and remove this. +unsafe impl Send for RawPage {} +unsafe impl Sync for RawPage {} + #[derive(Copy, Clone, PartialEq, Eq, PartialOrd, Ord)] pub struct RawPagePtr(NonNull<RawPage>); impl PageFlags { - pub const PRESENT: u32 = 1 << 0; pub const LOCKED: u32 = 1 << 1; pub const BUDDY: u32 = 1 << 2; pub const SLAB: u32 = 1 << 3; pub const DIRTY: u32 = 1 << 4; - pub const FREE: u32 = 1 << 5; pub const LOCAL: u32 = 1 << 6; pub fn has(&self, flag: u32) -> bool { @@ -158,48 +160,31 @@ impl RawPageTrait for RawPagePtr { fn refcount(&self) -> &AtomicUsize { self.refcount() } - - fn is_present(&self) -> bool { - self.flags().has(PageFlags::PRESENT) - } } -impl BuddyRawPage for RawPagePtr { - unsafe fn from_link(link: &mut Link) -> Self { - let raw_page_ptr = container_of!(link, RawPage, link); - Self(raw_page_ptr) - } - - fn set_order(&self, order: u32) { - self.as_mut().order = order; +impl BuddyPage for RawPage { + fn pfn(&self) -> PFN { + PFN::from(RawPagePtr::from_ref(self)) } - unsafe fn get_link(&self) -> &mut Link { - &mut self.as_mut().link + fn get_order(&self) -> u32 { + self.order } fn is_buddy(&self) -> bool { - self.flags().has(PageFlags::BUDDY) - } - - fn is_free(&self) -> bool { - self.flags().has(PageFlags::FREE) - } - - fn set_buddy(&self) { - self.flags().set(PageFlags::BUDDY); - } - - fn set_free(&self) { - self.flags().set(PageFlags::FREE); + self.flags.has(PageFlags::BUDDY) } - fn clear_buddy(&self) { - self.flags().clear(PageFlags::BUDDY); + fn set_order(&mut self, order: u32) { + self.order = order; } - fn clear_free(&self) { - self.flags().clear(PageFlags::FREE); + fn set_buddy(&mut self, val: bool) { + if val { + self.flags.set(PageFlags::BUDDY); + } else { + self.flags.clear(PageFlags::BUDDY) + } } } @@ -284,15 +269,21 @@ impl PageCacheRawPage for RawPagePtr { } } -pub struct RawSlabPageList(List); +impl PerCpuPage for RawPage { + fn set_local(&mut self, val: bool) { + if val { + self.flags.set(PageFlags::LOCAL) + } else { + self.flags.clear(PageFlags::LOCAL) + } + } +} + +pub struct RawPageList(List); -impl SlabPageList for RawSlabPageList { +impl PageList for RawPageList { type Page = RawPage; - fn new() -> Self { - Self(List::new()) - } - fn is_empty(&self) -> bool { self.0.is_empty() } @@ -324,9 +315,13 @@ impl SlabPageList for RawSlabPageList { } } +impl PageListSized for RawPageList { + const NEW: Self = RawPageList(List::new()); +} + impl SlabPageAlloc for GlobalPageAlloc { type Page = RawPage; - type PageList = RawSlabPageList; + type PageList = RawPageList; unsafe fn alloc_uninit(&self) -> &'static mut RawPage { let raw_page = self.alloc().expect("Out of memory").as_mut(); diff --git a/src/kernel/mem/page_alloc/zones.rs b/src/kernel/mem/page_alloc/zones.rs new file mode 100644 index 00000000..7a2e4e33 --- /dev/null +++ b/src/kernel/mem/page_alloc/zones.rs @@ -0,0 +1,25 @@ +use core::cell::UnsafeCell; + +use eonix_mm::address::PRange; +use eonix_mm::paging::{Zone, PFN}; + +use super::RawPage; +use crate::kernel::mem::page_alloc::RawPagePtr; + +pub struct GlobalZone(); + +impl Zone for GlobalZone { + type Page = RawPage; + + fn contains_prange(&self, _: PRange) -> bool { + true + } + + fn get_page(&self, pfn: PFN) -> Option<&UnsafeCell<Self::Page>> { + unsafe { + // SAFETY: The pointer returned by [`RawPagePtr::as_ptr()`] is valid. + // And so is it wrapped with [`UnsafeCell`] + Some(&*(RawPagePtr::from(pfn).as_ptr() as *const UnsafeCell<Self::Page>)) + } + } +} diff --git a/src/kernel_init.rs b/src/kernel_init.rs index 3d8be90f..93b6da20 100644 --- a/src/kernel_init.rs +++ b/src/kernel_init.rs @@ -1,14 +1,11 @@ +use eonix_hal::bootstrap::BootStrapData; +use eonix_hal::mm::{ArchMemory, ArchPagingMode, GLOBAL_PAGE_TABLE}; +use eonix_hal::traits::mm::Memory; +use eonix_mm::address::{Addr as _, AddrOps as _, VAddr, VRange}; +use eonix_mm::page_table::{PageAttribute, PagingMode as _, PTE}; +use eonix_mm::paging::{Page as GenericPage, PAGE_SIZE, PFN}; + use crate::kernel::mem::{GlobalPageAlloc, RawPage}; -use eonix_hal::{ - bootstrap::BootStrapData, - mm::{ArchMemory, ArchPagingMode, GLOBAL_PAGE_TABLE}, - traits::mm::Memory, -}; -use eonix_mm::{ - address::{Addr as _, AddrOps as _, VAddr, VRange}, - page_table::{PageAttribute, PagingMode as _, PTE}, - paging::{Page as GenericPage, PAGE_SIZE, PFN}, -}; pub fn setup_memory(data: &mut BootStrapData) { let addr_max = ArchMemory::present_ram() @@ -50,9 +47,10 @@ pub fn setup_memory(data: &mut BootStrapData) { ); } - for range in ArchMemory::present_ram() { - GlobalPageAlloc::mark_present(range); - } + // TODO!!!: Construct the global zone with all present ram. + // for range in ArchMemory::present_ram() { + // GlobalPageAlloc::mark_present(range); + // } if let Some(early_alloc) = data.take_alloc() { for range in early_alloc.into_iter() { From ef42bc6c5b970744081f4291c87402f3ad5e1c4e Mon Sep 17 00:00:00 2001 From: greatbridf <greatbridf@icloud.com> Date: Wed, 14 Jan 2026 01:09:49 +0800 Subject: [PATCH 22/36] vfs: rework of inode and page cache system - Bump rust compiler version to nightly-2026-01-09. - Inode rework: add a generic Inode struct. - Add a macro to help function tweaks. - PageCache rework: reduce complexity and try to decouple. - Adapt fat32, tmpfs to the new page cache system. - Change the way we process mapped pages and load ELF executables. - Refine handling flags in `MMArea::handle_mmap`. Signed-off-by: greatbridf <greatbridf@icloud.com> --- .../eonix_hal/src/arch/riscv64/bootstrap.rs | 62 +- src/fs/fat32.rs | 276 ++++----- src/fs/procfs.rs | 155 +++-- src/fs/tmpfs/dir.rs | 287 +++++----- src/fs/tmpfs/file.rs | 331 +++++------ src/fs/tmpfs/mod.rs | 23 +- src/kernel/mem.rs | 2 +- src/kernel/mem/mm_area.rs | 97 ++-- src/kernel/mem/mm_list/mapping.rs | 15 +- src/kernel/mem/mm_list/page_fault.rs | 5 +- src/kernel/mem/page_alloc.rs | 4 +- src/kernel/mem/page_alloc/raw_page.rs | 29 +- src/kernel/mem/page_cache.rs | 416 +++++--------- src/kernel/syscall/mm.rs | 21 +- src/kernel/task/loader/elf.rs | 70 +-- src/kernel/vfs/dentry.rs | 66 +-- src/kernel/vfs/dentry/walk.rs | 52 +- src/kernel/vfs/file/inode_file.rs | 36 +- src/kernel/vfs/filearray.rs | 46 +- src/kernel/vfs/inode/inode.rs | 531 +++++++++--------- src/kernel/vfs/inode/mod.rs | 2 +- src/kernel/vfs/inode/ops.rs | 5 +- src/kernel/vfs/inode/statx.rs | 32 +- src/kernel/vfs/mount.rs | 28 +- src/kernel/vfs/superblock.rs | 45 +- src/lib.rs | 1 - 26 files changed, 1151 insertions(+), 1486 deletions(-) diff --git a/crates/eonix_hal/src/arch/riscv64/bootstrap.rs b/crates/eonix_hal/src/arch/riscv64/bootstrap.rs index 0f1dff63..7b3dc043 100644 --- a/crates/eonix_hal/src/arch/riscv64/bootstrap.rs +++ b/crates/eonix_hal/src/arch/riscv64/bootstrap.rs @@ -1,39 +1,31 @@ -use super::{ - config::{self, mm::*}, - console::write_str, - cpu::{CPUID, CPU_COUNT}, - time::set_next_timer, -}; -use crate::{ - arch::{ - cpu::CPU, - fdt::{init_dtb_and_fdt, FdtExt, FDT}, - mm::{ArchPhysAccess, FreeRam, PageAttribute64, GLOBAL_PAGE_TABLE}, - }, - bootstrap::BootStrapData, - mm::{ArchMemory, ArchPagingMode, BasicPageAlloc, BasicPageAllocRef, ScopedAllocator}, -}; -use core::{ - alloc::Allocator, - arch::asm, - cell::RefCell, - sync::atomic::{AtomicBool, AtomicUsize}, -}; -use core::{ - arch::{global_asm, naked_asm}, - hint::spin_loop, - sync::atomic::{AtomicPtr, Ordering}, -}; +use core::alloc::Allocator; +use core::arch::{asm, global_asm, naked_asm}; +use core::cell::RefCell; +use core::hint::spin_loop; +use core::sync::atomic::{AtomicBool, AtomicPtr, AtomicUsize, Ordering}; + use eonix_hal_traits::mm::Memory; -use eonix_mm::{ - address::{Addr as _, PAddr, PRange, PhysAccess, VAddr, VRange}, - page_table::{PageAttribute, PagingMode, PTE as _}, - paging::{Page, PageAccess, PageAlloc, PAGE_SIZE, PFN}, -}; +use eonix_mm::address::{Addr as _, PAddr, PRange, PhysAccess, VAddr, VRange}; +use eonix_mm::page_table::{PageAttribute, PagingMode, PTE as _}; +use eonix_mm::paging::{Page, PageAccess, PageAlloc, PAGE_SIZE, PFN}; use eonix_percpu::PercpuArea; use fdt::Fdt; -use riscv::{asm::sfence_vma_all, register::satp}; -use sbi::{hsm::hart_start, legacy::console_putchar, PhysicalAddress}; +use riscv::asm::sfence_vma_all; +use riscv::register::satp; +use sbi::hsm::hart_start; +use sbi::legacy::console_putchar; +use sbi::PhysicalAddress; + +use super::config::mm::*; +use super::config::{self}; +use super::console::write_str; +use super::cpu::{CPUID, CPU_COUNT}; +use super::time::set_next_timer; +use crate::arch::cpu::CPU; +use crate::arch::fdt::{init_dtb_and_fdt, FdtExt, FDT}; +use crate::arch::mm::{ArchPhysAccess, FreeRam, PageAttribute64, GLOBAL_PAGE_TABLE}; +use crate::bootstrap::BootStrapData; +use crate::mm::{ArchMemory, ArchPagingMode, BasicPageAlloc, BasicPageAllocRef, ScopedAllocator}; #[unsafe(link_section = ".bootstrap.stack")] static BOOT_STACK: [u8; 4096 * 16] = [0; 4096 * 16]; @@ -78,7 +70,7 @@ static AP_SEM: AtomicBool = AtomicBool::new(false); #[unsafe(naked)] #[unsafe(no_mangle)] #[unsafe(link_section = ".bootstrap.entry")] -unsafe extern "C" fn _start(hart_id: usize, dtb_addr: usize) -> ! { +unsafe extern "C" fn _start(hart_id: usize, dtb_addr: usize) { naked_asm!( " ld sp, 2f @@ -289,7 +281,7 @@ fn bootstrap_smp(alloc: impl Allocator, page_alloc: &RefCell<BasicPageAlloc>) { #[unsafe(naked)] #[unsafe(no_mangle)] #[unsafe(link_section = ".bootstrap.apentry")] -unsafe extern "C" fn _ap_start(hart_id: usize) -> ! { +unsafe extern "C" fn _ap_start(hart_id: usize) { naked_asm!( " la sp, 1f // set temp stack diff --git a/src/fs/fat32.rs b/src/fs/fat32.rs index 9a4e03ec..b19c9908 100644 --- a/src/fs/fat32.rs +++ b/src/fs/fat32.rs @@ -1,26 +1,22 @@ mod dir; mod file; -use alloc::sync::{Arc, Weak}; -use core::future::Future; +use alloc::sync::Arc; use core::ops::Deref; use async_trait::async_trait; use dir::{as_raw_dirents, ParseDirent}; +use eonix_mm::paging::PAGE_SIZE; use eonix_sync::RwLock; use itertools::Itertools; use crate::io::{Buffer, ByteBuffer, UninitBuffer}; use crate::kernel::block::{BlockDevice, BlockDeviceRequest}; use crate::kernel::constants::{EINVAL, EIO}; -use crate::kernel::mem::{ - CachePage, CachePageStream, Page, PageCache, PageCacheBackendOps, PageExcl, PageExt, -}; +use crate::kernel::mem::{CachePage, Page, PageExcl, PageExt, PageOffset}; use crate::kernel::timer::Instant; use crate::kernel::vfs::dentry::Dentry; -use crate::kernel::vfs::inode::{ - Ino, Inode, InodeDirOps, InodeFileOps, InodeInfo, InodeOps, InodeUse, -}; +use crate::kernel::vfs::inode::{Ino, InodeInfo, InodeOps, InodeUse}; use crate::kernel::vfs::mount::{register_filesystem, Mount, MountCreator}; use crate::kernel::vfs::types::{DeviceId, Format, Permission}; use crate::kernel::vfs::{SbRef, SbUse, SuperBlock, SuperBlockInfo}; @@ -56,6 +52,10 @@ impl Cluster { Ino::new(self.0 as _) } + pub fn from_ino(ino: Ino) -> Self { + Self(ino.as_raw() as u32) + } + fn normalized(self) -> Self { Self(self.0 - 2) } @@ -130,7 +130,7 @@ impl FatFs { } impl FatFs { - pub async fn create(device: DeviceId) -> KResult<(SbUse<Self>, InodeUse<dyn Inode>)> { + pub async fn create(device: DeviceId) -> KResult<(SbUse<Self>, InodeUse)> { let device = BlockDevice::get(device)?; let mut info = UninitBuffer::<Bootsector>::new(); @@ -217,18 +217,15 @@ impl<'fat> Iterator for ClusterIterator<'fat> { } } -struct FileInode { - cluster: Cluster, - info: Spin<InodeInfo>, - sb: SbRef<FatFs>, - page_cache: PageCache, -} +struct FileInode; impl FileInode { - fn new(cluster: Cluster, sb: SbRef<FatFs>, size: u32) -> InodeUse<FileInode> { - InodeUse::new_cyclic(|weak: &Weak<FileInode>| Self { - cluster, - info: Spin::new(InodeInfo { + fn new(cluster: Cluster, sb: SbRef<FatFs>, size: u32) -> InodeUse { + InodeUse::new( + sb, + cluster.as_ino(), + Format::REG, + InodeInfo { size: size as u64, nlink: 1, uid: 0, @@ -237,108 +234,75 @@ impl FileInode { atime: Instant::UNIX_EPOCH, ctime: Instant::UNIX_EPOCH, mtime: Instant::UNIX_EPOCH, - }), - sb, - page_cache: PageCache::new(weak.clone()), - }) + }, + Self, + ) } } impl InodeOps for FileInode { type SuperBlock = FatFs; - fn ino(&self) -> Ino { - self.cluster.as_ino() - } - - fn format(&self) -> Format { - Format::REG - } - - fn info(&self) -> &Spin<InodeInfo> { - &self.info - } - - fn super_block(&self) -> &SbRef<Self::SuperBlock> { - &self.sb - } - - fn page_cache(&self) -> Option<&PageCache> { - Some(&self.page_cache) - } -} - -impl InodeDirOps for FileInode {} -impl InodeFileOps for FileInode { - async fn read(&self, buffer: &mut dyn Buffer, offset: usize) -> KResult<usize> { - self.page_cache.read(buffer, offset).await + async fn read( + &self, + _: SbUse<Self::SuperBlock>, + inode: &InodeUse, + buffer: &mut dyn Buffer, + offset: usize, + ) -> KResult<usize> { + inode.get_page_cache().read(buffer, offset).await } - async fn read_direct(&self, buffer: &mut dyn Buffer, offset: usize) -> KResult<usize> { - let sb = self.sb.get()?; + async fn read_page( + &self, + sb: SbUse<Self::SuperBlock>, + inode: &InodeUse, + page: &mut CachePage, + offset: PageOffset, + ) -> KResult<()> { let fs = &sb.backend; let fat = sb.backend.fat.read().await; - if offset >= self.info.lock().size as usize { - return Ok(0); + if offset >= PageOffset::from_byte_ceil(inode.info.lock().size as usize) { + unreachable!("read_page called with offset beyond file size"); } let cluster_size = fs.sectors_per_cluster as usize * SECTOR_SIZE; - assert!(cluster_size <= 0x1000, "Cluster size is too large"); - - let skip_clusters = offset / cluster_size; - let inner_offset = offset % cluster_size; - - let cluster_iter = ClusterIterator::new(fat.as_ref(), self.cluster).skip(skip_clusters); - - let buffer_page = Page::alloc(); - for cluster in cluster_iter { - fs.read_cluster(cluster, &buffer_page).await?; - - let pg = buffer_page.lock(); - let data = &pg.as_bytes()[inner_offset..]; - - let end = offset + data.len(); - let real_end = end.min(self.info.lock().size as usize); - let real_size = real_end - offset; - - if buffer.fill(&data[..real_size])?.should_stop() { - break; - } + if cluster_size != PAGE_SIZE { + unimplemented!("cluster size != PAGE_SIZE"); } - Ok(buffer.wrote()) - } -} + // XXX: Ugly and inefficient O(n^2) algorithm for sequential file read. + let cluster = ClusterIterator::new(fat.as_ref(), Cluster::from_ino(inode.ino)) + .skip(offset.page_count()) + .next() + .ok_or(EIO)?; -impl PageCacheBackendOps for FileInode { - async fn read_page(&self, page: &mut CachePage, offset: usize) -> KResult<usize> { - self.read_direct(page, offset).await - } + let page = page.get_page(); + fs.read_cluster(cluster, &page).await?; - async fn write_page(&self, _page: &mut CachePageStream, _offset: usize) -> KResult<usize> { - todo!() - } + let real_len = (inode.info.lock().size as usize) - offset.byte_count(); + if real_len < PAGE_SIZE { + let mut page = page.lock(); + page.as_bytes_mut()[real_len..].fill(0); + } - fn size(&self) -> usize { - self.info.lock().size as usize + Ok(()) } } struct DirInode { - cluster: Cluster, - info: Spin<InodeInfo>, - sb: SbRef<FatFs>, - // TODO: Use the new PageCache... dir_pages: RwLock<Vec<PageExcl>>, } impl DirInode { - fn new(cluster: Cluster, sb: SbRef<FatFs>, size: u32) -> InodeUse<Self> { - InodeUse::new(Self { - cluster, - info: Spin::new(InodeInfo { + fn new(cluster: Cluster, sb: SbRef<FatFs>, size: u32) -> InodeUse { + InodeUse::new( + sb, + cluster.as_ino(), + Format::DIR, + InodeInfo { size: size as u64, nlink: 2, // '.' and '..' uid: 0, @@ -347,23 +311,23 @@ impl DirInode { atime: Instant::UNIX_EPOCH, ctime: Instant::UNIX_EPOCH, mtime: Instant::UNIX_EPOCH, - }), - sb, - dir_pages: RwLock::new(Vec::new()), - }) + }, + Self { + dir_pages: RwLock::new(Vec::new()), + }, + ) } - async fn read_dir_pages(&self) -> KResult<()> { + async fn read_dir_pages(&self, sb: &SbUse<FatFs>, inode: &InodeUse) -> KResult<()> { let mut dir_pages = self.dir_pages.write().await; if !dir_pages.is_empty() { return Ok(()); } - let sb = self.sb.get()?; let fs = &sb.backend; let fat = fs.fat.read().await; - let clusters = ClusterIterator::new(fat.as_ref(), self.cluster); + let clusters = ClusterIterator::new(fat.as_ref(), Cluster::from_ino(inode.ino)); for cluster in clusters { let page = PageExcl::alloc(); @@ -375,7 +339,11 @@ impl DirInode { Ok(()) } - async fn get_dir_pages(&self) -> KResult<impl Deref<Target = Vec<PageExcl>> + use<'_>> { + async fn get_dir_pages( + &self, + sb: &SbUse<FatFs>, + inode: &InodeUse, + ) -> KResult<impl Deref<Target = Vec<PageExcl>> + use<'_>> { { let dir_pages = self.dir_pages.read().await; if !dir_pages.is_empty() { @@ -383,7 +351,7 @@ impl DirInode { } } - self.read_dir_pages().await?; + self.read_dir_pages(sb, inode).await?; if let Some(dir_pages) = self.dir_pages.try_read() { return Ok(dir_pages); @@ -396,32 +364,13 @@ impl DirInode { impl InodeOps for DirInode { type SuperBlock = FatFs; - fn ino(&self) -> Ino { - self.cluster.as_ino() - } - - fn format(&self) -> Format { - Format::DIR - } - - fn info(&self) -> &Spin<InodeInfo> { - &self.info - } - - fn super_block(&self) -> &SbRef<Self::SuperBlock> { - &self.sb - } - - fn page_cache(&self) -> Option<&PageCache> { - None - } -} - -impl InodeFileOps for DirInode {} -impl InodeDirOps for DirInode { - async fn lookup(&self, dentry: &Arc<Dentry>) -> KResult<Option<InodeUse<dyn Inode>>> { - let sb = self.sb.get()?; - let dir_pages = self.get_dir_pages().await?; + async fn lookup( + &self, + sb: SbUse<Self::SuperBlock>, + inode: &InodeUse, + dentry: &Arc<Dentry>, + ) -> KResult<Option<InodeUse>> { + let dir_pages = self.get_dir_pages(&sb, inode).await?; let dir_data = dir_pages.iter().map(|pg| pg.as_bytes()); @@ -451,48 +400,47 @@ impl InodeDirOps for DirInode { Ok(None) } - fn readdir<'r, 'a: 'r, 'b: 'r>( - &'a self, + async fn readdir( + &self, + sb: SbUse<Self::SuperBlock>, + inode: &InodeUse, offset: usize, - callback: &'b mut (dyn FnMut(&[u8], Ino) -> KResult<bool> + Send), - ) -> impl Future<Output = KResult<KResult<usize>>> + Send + 'r { - async move { - let sb = self.sb.get()?; - let fs = &sb.backend; - let dir_pages = self.get_dir_pages().await?; + callback: &mut (dyn FnMut(&[u8], Ino) -> KResult<bool> + Send), + ) -> KResult<KResult<usize>> { + let fs = &sb.backend; + let dir_pages = self.get_dir_pages(&sb, inode).await?; - let cluster_size = fs.sectors_per_cluster as usize * SECTOR_SIZE; + let cluster_size = fs.sectors_per_cluster as usize * SECTOR_SIZE; - let cluster_offset = offset / cluster_size; - let inner_offset = offset % cluster_size; - let inner_raw_dirent_offset = inner_offset / core::mem::size_of::<dir::RawDirEntry>(); + let cluster_offset = offset / cluster_size; + let inner_offset = offset % cluster_size; + let inner_raw_dirent_offset = inner_offset / core::mem::size_of::<dir::RawDirEntry>(); - let dir_data = dir_pages - .iter() - .skip(cluster_offset) - .map(|pg| pg.as_bytes()); - - let raw_dirents = dir_data - .map(as_raw_dirents) - .take_while_inclusive(Result::is_ok) - .flatten_ok() - .skip(inner_raw_dirent_offset); - - let mut dirents = futures::stream::iter(raw_dirents); - - let mut nread = 0; - while let Some(result) = dirents.next_dirent().await { - let entry = result?; - - match callback(&entry.filename, entry.cluster.as_ino()) { - Err(err) => return Ok(Err(err)), - Ok(true) => nread += entry.entry_offset as usize, - Ok(false) => break, - } - } + let dir_data = dir_pages + .iter() + .skip(cluster_offset) + .map(|pg| pg.as_bytes()); - Ok(Ok(nread)) + let raw_dirents = dir_data + .map(as_raw_dirents) + .take_while_inclusive(Result::is_ok) + .flatten_ok() + .skip(inner_raw_dirent_offset); + + let mut dirents = futures::stream::iter(raw_dirents); + + let mut nread = 0; + while let Some(result) = dirents.next_dirent().await { + let entry = result?; + + match callback(&entry.filename, entry.cluster.as_ino()) { + Err(err) => return Ok(Err(err)), + Ok(true) => nread += entry.entry_offset as usize, + Ok(false) => break, + } } + + Ok(Ok(nread)) } } diff --git a/src/fs/procfs.rs b/src/fs/procfs.rs index 57b881df..32ede420 100644 --- a/src/fs/procfs.rs +++ b/src/fs/procfs.rs @@ -1,30 +1,21 @@ +use alloc::sync::Arc; +use core::sync::atomic::{AtomicU64, Ordering}; + +use async_trait::async_trait; +use eonix_sync::{LazyLock, RwLock}; + +use crate::io::Buffer; use crate::kernel::constants::{EACCES, EISDIR, ENOTDIR}; +use crate::kernel::mem::paging::PageBuffer; use crate::kernel::timer::Instant; -use crate::kernel::vfs::inode::{InodeDirOps, InodeFileOps, InodeInfo, InodeOps, InodeUse}; +use crate::kernel::vfs::dentry::Dentry; +use crate::kernel::vfs::inode::{Ino, InodeInfo, InodeOps, InodeUse}; +use crate::kernel::vfs::mount::{dump_mounts, register_filesystem, Mount, MountCreator}; use crate::kernel::vfs::types::{DeviceId, Format, Permission}; use crate::kernel::vfs::{SbRef, SbUse, SuperBlock, SuperBlockInfo}; -use crate::{ - io::Buffer, - kernel::{ - mem::paging::PageBuffer, - vfs::{ - dentry::Dentry, - inode::{Ino, Inode}, - mount::{dump_mounts, register_filesystem, Mount, MountCreator}, - }, - }, - prelude::*, -}; -use alloc::sync::Arc; -use async_trait::async_trait; -use core::future::Future; -use core::sync::atomic::{AtomicU64, Ordering}; -use eonix_sync::{LazyLock, RwLock}; +use crate::prelude::*; struct Node { - ino: Ino, - sb: SbRef<ProcFs>, - info: Spin<InodeInfo>, kind: NodeKind, } @@ -39,38 +30,19 @@ struct FileInode { } struct DirInode { - entries: RwLock<Vec<(Arc<[u8]>, InodeUse<Node>)>>, + entries: RwLock<Vec<(Arc<[u8]>, InodeUse)>>, } impl InodeOps for Node { type SuperBlock = ProcFs; - fn ino(&self) -> Ino { - self.ino - } - - fn format(&self) -> Format { - match &self.kind { - NodeKind::File(_) => Format::REG, - NodeKind::Dir(_) => Format::DIR, - } - } - - fn info(&self) -> &Spin<InodeInfo> { - &self.info - } - - fn super_block(&self) -> &SbRef<Self::SuperBlock> { - &self.sb - } - - fn page_cache(&self) -> Option<&crate::kernel::mem::PageCache> { - None - } -} - -impl InodeFileOps for Node { - async fn read(&self, buffer: &mut dyn Buffer, offset: usize) -> KResult<usize> { + async fn read( + &self, + _: SbUse<Self::SuperBlock>, + _: &InodeUse, + buffer: &mut dyn Buffer, + offset: usize, + ) -> KResult<usize> { let NodeKind::File(file_inode) = &self.kind else { return Err(EISDIR); }; @@ -88,10 +60,13 @@ impl InodeFileOps for Node { Ok(buffer.fill(data)?.allow_partial()) } -} -impl InodeDirOps for Node { - async fn lookup(&self, dentry: &Arc<Dentry>) -> KResult<Option<InodeUse<dyn Inode>>> { + async fn lookup( + &self, + _: SbUse<Self::SuperBlock>, + _: &InodeUse, + dentry: &Arc<Dentry>, + ) -> KResult<Option<InodeUse>> { let NodeKind::Dir(dir) = &self.kind else { return Err(ENOTDIR); }; @@ -108,29 +83,29 @@ impl InodeDirOps for Node { Ok(None) } - fn readdir<'r, 'a: 'r, 'b: 'r>( - &'a self, + async fn readdir( + &self, + _: SbUse<Self::SuperBlock>, + _: &InodeUse, offset: usize, - callback: &'b mut (dyn FnMut(&[u8], Ino) -> KResult<bool> + Send), - ) -> impl Future<Output = KResult<KResult<usize>>> + Send + 'r { - Box::pin(async move { - let NodeKind::Dir(dir) = &self.kind else { - return Err(ENOTDIR); - }; - - let entries = dir.entries.read().await; - - let mut count = 0; - for (name, node) in entries.iter().skip(offset) { - match callback(name.as_ref(), node.ino) { - Err(err) => return Ok(Err(err)), - Ok(true) => count += 1, - Ok(false) => break, - } + callback: &mut (dyn FnMut(&[u8], Ino) -> KResult<bool> + Send), + ) -> KResult<KResult<usize>> { + let NodeKind::Dir(dir) = &self.kind else { + return Err(ENOTDIR); + }; + + let entries = dir.entries.read().await; + + let mut count = 0; + for (name, node) in entries.iter().skip(offset) { + match callback(name.as_ref(), node.ino) { + Err(err) => return Ok(Err(err)), + Ok(true) => count += 1, + Ok(false) => break, } + } - Ok(Ok(count)) - }) + Ok(Ok(count)) } } @@ -139,11 +114,12 @@ impl Node { ino: Ino, sb: SbRef<ProcFs>, read: impl Fn(&mut PageBuffer) -> KResult<()> + Send + Sync + 'static, - ) -> InodeUse<Self> { - InodeUse::new(Self { - ino, + ) -> InodeUse { + InodeUse::new( sb, - info: Spin::new(InodeInfo { + ino, + Format::REG, + InodeInfo { size: 0, nlink: 1, uid: 0, @@ -152,16 +128,19 @@ impl Node { atime: Instant::UNIX_EPOCH, ctime: Instant::UNIX_EPOCH, mtime: Instant::UNIX_EPOCH, - }), - kind: NodeKind::File(FileInode::new(Box::new(read))), - }) + }, + Self { + kind: NodeKind::File(FileInode::new(Box::new(read))), + }, + ) } - fn new_dir(ino: Ino, sb: SbRef<ProcFs>) -> InodeUse<Self> { - InodeUse::new(Self { - ino, + fn new_dir(ino: Ino, sb: SbRef<ProcFs>) -> InodeUse { + InodeUse::new( sb, - info: Spin::new(InodeInfo { + ino, + Format::DIR, + InodeInfo { size: 0, nlink: 1, uid: 0, @@ -170,9 +149,11 @@ impl Node { atime: Instant::UNIX_EPOCH, ctime: Instant::UNIX_EPOCH, mtime: Instant::UNIX_EPOCH, - }), - kind: NodeKind::Dir(DirInode::new()), - }) + }, + Self { + kind: NodeKind::Dir(DirInode::new()), + }, + ) } } @@ -194,7 +175,7 @@ impl DirInode { } pub struct ProcFs { - root: InodeUse<Node>, + root: InodeUse, next_ino: AtomicU64, } @@ -240,7 +221,7 @@ where F: Send + Sync + Fn(&mut PageBuffer) -> KResult<()> + 'static, { let procfs = &GLOBAL_PROCFS.backend; - let root = &procfs.root; + let root = &procfs.root.get_priv::<Node>(); let NodeKind::Dir(root) = &root.kind else { unreachable!(); diff --git a/src/fs/tmpfs/dir.rs b/src/fs/tmpfs/dir.rs index e2be1d12..4dd64d52 100644 --- a/src/fs/tmpfs/dir.rs +++ b/src/fs/tmpfs/dir.rs @@ -1,72 +1,51 @@ -use core::{any::Any, future::Future}; +use alloc::sync::Arc; +use alloc::vec; +use alloc::vec::Vec; -use alloc::{boxed::Box, sync::Arc, vec, vec::Vec}; use eonix_log::println_warn; -use eonix_sync::{LazyLock, RwLock, Spin}; - -use crate::{ - kernel::{ - constants::{EEXIST, EINVAL, EISDIR, ENOENT, ENOSYS, ENOTDIR}, - mem::PageCache, - timer::Instant, - vfs::{ - dentry::{dcache, Dentry}, - inode::{ - Ino, Inode, InodeDirOps, InodeFileOps, InodeInfo, InodeOps, InodeUse, RenameData, - }, - types::{DeviceId, Format, Mode, Permission}, - SbRef, - }, - }, - prelude::KResult, -}; - -use super::{ - file::{DeviceInode, FileInode, SymlinkInode}, - TmpFs, -}; +use eonix_sync::{LazyLock, RwLock}; + +use super::file::{DeviceInode, FileInode, SymlinkInode}; +use super::TmpFs; +use crate::kernel::constants::{EEXIST, EINVAL, EISDIR, ENOENT, ENOSYS, ENOTDIR}; +use crate::kernel::timer::Instant; +use crate::kernel::vfs::dentry::{dcache, Dentry}; +use crate::kernel::vfs::inode::{Ino, InodeInfo, InodeOps, InodeUse, RenameData}; +use crate::kernel::vfs::types::{DeviceId, Format, Mode, Permission}; +use crate::kernel::vfs::{SbRef, SbUse}; +use crate::prelude::KResult; pub struct DirectoryInode { - sb: SbRef<TmpFs>, - ino: Ino, - info: Spin<InodeInfo>, entries: RwLock<Vec<(Arc<[u8]>, Ino)>>, } -impl InodeOps for DirectoryInode { - type SuperBlock = TmpFs; - - fn ino(&self) -> Ino { - self.ino - } +fn link(dir: &InodeUse, entries: &mut Vec<(Arc<[u8]>, Ino)>, name: Arc<[u8]>, file: &InodeUse) { + let mut dir_info = dir.info.lock(); + let mut file_info = file.info.lock(); - fn format(&self) -> Format { - Format::DIR - } + let now = Instant::now(); - fn info(&self) -> &Spin<InodeInfo> { - &self.info - } + file_info.nlink += 1; + file_info.ctime = now; - fn super_block(&self) -> &SbRef<Self::SuperBlock> { - &self.sb - } + dir_info.size += 1; + dir_info.mtime = now; + dir_info.ctime = now; - fn page_cache(&self) -> Option<&PageCache> { - None - } + entries.push((name, file.ino)); } impl DirectoryInode { - pub fn new(ino: Ino, sb: SbRef<TmpFs>, perm: Permission) -> InodeUse<Self> { + pub fn new(ino: Ino, sb: SbRef<TmpFs>, perm: Permission) -> InodeUse { static DOT: LazyLock<Arc<[u8]>> = LazyLock::new(|| Arc::from(b".".as_slice())); let now = Instant::now(); - InodeUse::new(Self { + InodeUse::new( sb, ino, - info: Spin::new(InodeInfo { + Format::DIR, + InodeInfo { size: 1, nlink: 1, // link from `.` to itself perm, @@ -75,35 +54,16 @@ impl DirectoryInode { atime: now, uid: 0, gid: 0, - }), - entries: RwLock::new(vec![(DOT.clone(), ino)]), - }) - } - - fn link( - &self, - entries: &mut Vec<(Arc<[u8]>, Ino)>, - name: Arc<[u8]>, - file: &InodeUse<dyn Inode>, - ) { - let mut self_info = self.info.lock(); - let mut file_info = file.info().lock(); - - let now = Instant::now(); - - file_info.nlink += 1; - file_info.ctime = now; - - self_info.size += 1; - self_info.mtime = now; - self_info.ctime = now; - - entries.push((name, file.ino())); + }, + Self { + entries: RwLock::new(vec![(DOT.clone(), ino)]), + }, + ) } fn do_unlink( &self, - file: &InodeUse<dyn Inode>, + file: &InodeUse, filename: &[u8], entries: &mut Vec<(Arc<[u8]>, Ino)>, now: Instant, @@ -112,11 +72,11 @@ impl DirectoryInode { file_info: &mut InodeInfo, ) -> KResult<()> { // SAFETY: `file_lock` has done the synchronization - if file.format() == Format::DIR { + if file.format == Format::DIR { return Err(EISDIR); } - let file_ino = file.ino(); + let file_ino = file.ino; entries.retain(|(name, ino)| *ino != file_ino || name.as_ref() != filename); if decrease_size { @@ -138,87 +98,114 @@ impl DirectoryInode { } } -impl InodeDirOps for DirectoryInode { - fn readdir<'r, 'a: 'r, 'b: 'r>( - &'a self, +impl InodeOps for DirectoryInode { + type SuperBlock = TmpFs; + + async fn readdir( + &self, + sb: SbUse<Self::SuperBlock>, + _: &InodeUse, offset: usize, - for_each_entry: &'b mut (dyn FnMut(&[u8], Ino) -> KResult<bool> + Send), - ) -> impl Future<Output = KResult<KResult<usize>>> + Send + 'r { - Box::pin(async move { - let _sb = self.sb.get()?; - let entries = self.entries.read().await; - - let mut count = 0; - for entry in entries.iter().skip(offset) { - match for_each_entry(&entry.0, entry.1) { - Err(err) => return Ok(Err(err)), - Ok(false) => break, - Ok(true) => count += 1, - } + for_each_entry: &mut (dyn FnMut(&[u8], Ino) -> KResult<bool> + Send), + ) -> KResult<KResult<usize>> { + let _sb = sb; + let entries = self.entries.read().await; + + let mut count = 0; + for entry in entries.iter().skip(offset) { + match for_each_entry(&entry.0, entry.1) { + Err(err) => return Ok(Err(err)), + Ok(false) => break, + Ok(true) => count += 1, } + } - Ok(Ok(count)) - }) + Ok(Ok(count)) } - async fn create(&self, at: &Arc<Dentry>, perm: Permission) -> KResult<()> { - let sb = self.sb.get()?; + async fn create( + &self, + sb: SbUse<Self::SuperBlock>, + inode: &InodeUse, + at: &Arc<Dentry>, + perm: Permission, + ) -> KResult<()> { let mut entries = self.entries.write().await; let ino = sb.backend.assign_ino(); - let file: InodeUse<dyn Inode> = FileInode::new(ino, self.sb.clone(), 0, perm); + let file = FileInode::new(ino, sb.get_ref(), 0, perm); - self.link(&mut entries, at.get_name(), &file); + link(inode, &mut entries, at.get_name(), &file); at.fill(file); Ok(()) } - async fn mknod(&self, at: &Dentry, mode: Mode, dev: DeviceId) -> KResult<()> { + async fn mknod( + &self, + sb: SbUse<Self::SuperBlock>, + inode: &InodeUse, + at: &Dentry, + mode: Mode, + dev: DeviceId, + ) -> KResult<()> { if !mode.is_chr() && !mode.is_blk() { return Err(EINVAL); } - let sb = self.sb.get()?; let mut entries = self.entries.write().await; let ino = sb.backend.assign_ino(); - let file: InodeUse<dyn Inode> = DeviceInode::new(ino, self.sb.clone(), mode, dev); + let file = DeviceInode::new(ino, sb.get_ref(), mode, dev); - self.link(&mut entries, at.get_name(), &file); + link(inode, &mut entries, at.get_name(), &file); at.fill(file); Ok(()) } - async fn symlink(&self, at: &Arc<Dentry>, target: &[u8]) -> KResult<()> { - let sb = self.sb.get()?; + async fn symlink( + &self, + sb: SbUse<Self::SuperBlock>, + inode: &InodeUse, + at: &Arc<Dentry>, + target: &[u8], + ) -> KResult<()> { let mut entries = self.entries.write().await; let ino = sb.backend.assign_ino(); - let file: InodeUse<dyn Inode> = SymlinkInode::new(ino, self.sb.clone(), target.into()); + let file = SymlinkInode::new(ino, sb.get_ref(), target.into()); - self.link(&mut entries, at.get_name(), &file); + link(inode, &mut entries, at.get_name(), &file); at.fill(file); Ok(()) } - async fn mkdir(&self, at: &Dentry, perm: Permission) -> KResult<()> { - let sb = self.sb.get()?; + async fn mkdir( + &self, + sb: SbUse<Self::SuperBlock>, + inode: &InodeUse, + at: &Dentry, + perm: Permission, + ) -> KResult<()> { let mut entries = self.entries.write().await; let ino = sb.backend.assign_ino(); - let new_dir: InodeUse<dyn Inode> = DirectoryInode::new(ino, self.sb.clone(), perm); + let new_dir = DirectoryInode::new(ino, sb.get_ref(), perm); - self.link(&mut entries, at.get_name(), &new_dir); + link(inode, &mut entries, at.get_name(), &new_dir); at.fill(new_dir); Ok(()) } - async fn unlink(&self, at: &Arc<Dentry>) -> KResult<()> { - let _sb = self.sb.get()?; + async fn unlink( + &self, + _sb: SbUse<Self::SuperBlock>, + inode: &InodeUse, + at: &Arc<Dentry>, + ) -> KResult<()> { let mut entries = self.entries.write().await; let file = at.get_inode()?; @@ -230,8 +217,8 @@ impl InodeDirOps for DirectoryInode { &mut entries, Instant::now(), true, - &mut self.info.lock(), - &mut file.info().lock(), + &mut inode.info.lock(), + &mut file.info.lock(), )?; // Remove the dentry from the dentry cache immediately @@ -241,8 +228,12 @@ impl InodeDirOps for DirectoryInode { Ok(()) } - async fn rename(&self, rename_data: RenameData<'_, '_>) -> KResult<()> { - let sb = self.sb.get()?; + async fn rename( + &self, + sb: SbUse<Self::SuperBlock>, + inode: &InodeUse, + rename_data: RenameData<'_, '_>, + ) -> KResult<()> { let _rename_lock = sb.backend.rename_lock.lock().await; let mut self_entries = self.entries.write().await; @@ -266,11 +257,11 @@ impl InodeDirOps for DirectoryInode { return Err(EEXIST); } - if new_parent.as_raw() == &raw const *self { + if inode == &new_parent { // Same directory rename // Remove from old location and add to new location - let old_ino = old_file.ino(); - let new_ino = new_file.as_ref().map(|f| f.ino()); + let old_ino = old_file.ino; + let new_ino = new_file.as_ref().map(|f| f.ino); let old_name = old_dentry.get_name(); let new_name = new_dentry.get_name(); @@ -299,7 +290,7 @@ impl InodeDirOps for DirectoryInode { // Replace existing file (i.e. rename the old and unlink the new) let new_file = new_file.unwrap(); - match (new_file.format(), old_file.format()) { + match (new_file.format, old_file.format) { (Format::DIR, _) => return Err(EISDIR), (_, Format::DIR) => return Err(ENOTDIR), _ => {} @@ -307,12 +298,12 @@ impl InodeDirOps for DirectoryInode { self_entries.remove(new_idx); - self.info.lock().size -= 1; + inode.info.lock().size -= 1; // The last reference to the inode is held by some dentry // and will be released when the dentry is released - let mut new_info = new_file.info().lock(); + let mut new_info = new_file.info.lock(); new_info.nlink -= 1; new_info.mtime = now; @@ -322,24 +313,21 @@ impl InodeDirOps for DirectoryInode { let (name, _) = &mut self_entries[old_ent_idx]; *name = new_dentry.get_name(); - let mut self_info = self.info.lock(); + let mut self_info = inode.info.lock(); self_info.mtime = now; self_info.ctime = now; } else { // Cross-directory rename - handle similar to same directory case // Get new parent directory - let new_parent_inode = new_dentry.parent().get_inode()?; - assert_eq!(new_parent_inode.format(), Format::DIR); - - let new_parent = (&new_parent_inode as &dyn Any) - .downcast_ref::<DirectoryInode>() - .expect("new parent must be a DirectoryInode"); + let new_parent = new_dentry.parent().get_inode()?; + assert_eq!(new_parent.format, Format::DIR); - let mut new_entries = new_parent.entries.write().await; + let new_parent_priv = new_parent.get_priv::<DirectoryInode>(); + let mut new_entries = new_parent_priv.entries.write().await; - let old_ino = old_file.ino(); - let new_ino = new_file.as_ref().map(|f| f.ino()); + let old_ino = old_file.ino; + let new_ino = new_file.as_ref().map(|f| f.ino); let old_name = old_dentry.get_name(); let new_name = new_dentry.get_name(); @@ -361,26 +349,28 @@ impl InodeDirOps for DirectoryInode { // Replace existing file (i.e. move the old and unlink the new) let new_file = new_file.unwrap(); - match (old_file.format(), new_file.format()) { + match (old_file.format, new_file.format) { (Format::DIR, Format::DIR) => {} (Format::DIR, _) => return Err(ENOTDIR), (_, _) => {} } // Unlink the old file that was replaced - new_parent.do_unlink( + new_parent_priv.do_unlink( &new_file, &new_name, &mut new_entries, now, false, &mut new_parent.info.lock(), - &mut new_file.info().lock(), + &mut new_file.info.lock(), )?; } else { - new_parent.info.lock().size += 1; - new_parent.info.lock().mtime = now; - new_parent.info.lock().ctime = now; + let mut info = new_parent.info.lock(); + + info.size += 1; + info.mtime = now; + info.ctime = now; } // Remove from old directory @@ -389,7 +379,7 @@ impl InodeDirOps for DirectoryInode { // Add new entry new_entries.push((new_name, old_ino)); - let mut self_info = self.info.lock(); + let mut self_info = inode.info.lock(); self_info.size -= 1; self_info.mtime = now; self_info.ctime = now; @@ -398,17 +388,16 @@ impl InodeDirOps for DirectoryInode { dcache::d_exchange(old_dentry, new_dentry).await; Ok(()) } -} - -impl InodeFileOps for DirectoryInode { - async fn chmod(&self, perm: Permission) -> KResult<()> { - let _sb = self.sb.get()?; - { - let mut info = self.info.lock(); - info.perm = perm; - info.ctime = Instant::now(); - } + async fn chmod( + &self, + _sb: SbUse<Self::SuperBlock>, + inode: &InodeUse, + perm: Permission, + ) -> KResult<()> { + let mut info = inode.info.lock(); + info.perm = perm; + info.ctime = Instant::now(); Ok(()) } diff --git a/src/fs/tmpfs/file.rs b/src/fs/tmpfs/file.rs index 624112e0..a1755908 100644 --- a/src/fs/tmpfs/file.rs +++ b/src/fs/tmpfs/file.rs @@ -1,39 +1,26 @@ +use alloc::collections::btree_map::BTreeMap; use alloc::sync::Arc; -use eonix_mm::paging::PAGE_SIZE; -use eonix_sync::{RwLock, Spin}; - -use crate::{ - io::{Buffer, Stream}, - kernel::{ - mem::{CachePage, CachePageStream, PageCache, PageCacheBackendOps}, - timer::Instant, - vfs::{ - inode::{Ino, InodeDirOps, InodeFileOps, InodeInfo, InodeOps, InodeUse, WriteOffset}, - types::{DeviceId, Format, Mode, Permission}, - SbRef, - }, - }, - prelude::KResult, -}; use super::TmpFs; +use crate::io::{Buffer, Stream}; +use crate::kernel::mem::{CachePage, PageCache, PageOffset}; +use crate::kernel::timer::Instant; +use crate::kernel::vfs::inode::{Ino, InodeInfo, InodeOps, InodeUse, WriteOffset}; +use crate::kernel::vfs::types::{DeviceId, Format, Mode, Permission}; +use crate::kernel::vfs::{SbRef, SbUse}; +use crate::prelude::KResult; -pub struct FileInode { - sb: SbRef<TmpFs>, - ino: Ino, - info: Spin<InodeInfo>, - rwsem: RwLock<()>, - pages: PageCache, -} +pub struct FileInode; impl FileInode { - pub fn new(ino: Ino, sb: SbRef<TmpFs>, size: usize, perm: Permission) -> InodeUse<Self> { + pub fn new(ino: Ino, sb: SbRef<TmpFs>, size: usize, perm: Permission) -> InodeUse { let now = Instant::now(); - InodeUse::new_cyclic(|weak| Self { + InodeUse::new( sb, ino, - info: Spin::new(InodeInfo { + Format::REG, + InodeInfo { size: size as _, nlink: 1, uid: 0, @@ -42,60 +29,34 @@ impl FileInode { atime: now, ctime: now, mtime: now, - }), - rwsem: RwLock::new(()), - pages: PageCache::new(weak.clone() as _), - }) - } -} - -impl PageCacheBackendOps for FileInode { - async fn read_page(&self, _cache_page: &mut CachePage, _offset: usize) -> KResult<usize> { - Ok(PAGE_SIZE) - } - - async fn write_page(&self, _page: &mut CachePageStream, _offset: usize) -> KResult<usize> { - Ok(PAGE_SIZE) - } - - fn size(&self) -> usize { - self.info.lock().size as usize + }, + Self, + ) } } impl InodeOps for FileInode { type SuperBlock = TmpFs; - fn ino(&self) -> Ino { - self.ino - } - - fn format(&self) -> Format { - Format::REG - } - - fn info(&self) -> &Spin<InodeInfo> { - &self.info - } - - fn super_block(&self) -> &SbRef<Self::SuperBlock> { - &self.sb - } - - fn page_cache(&self) -> Option<&PageCache> { - Some(&self.pages) - } -} - -impl InodeDirOps for FileInode {} -impl InodeFileOps for FileInode { - async fn read(&self, buffer: &mut dyn Buffer, offset: usize) -> KResult<usize> { - let _lock = self.rwsem.read().await; - self.pages.read(buffer, offset).await + async fn read( + &self, + _: SbUse<Self::SuperBlock>, + inode: &InodeUse, + buffer: &mut dyn Buffer, + offset: usize, + ) -> KResult<usize> { + let _lock = inode.rwsem.read().await; + inode.get_page_cache().read(buffer, offset).await } - async fn write(&self, stream: &mut dyn Stream, offset: WriteOffset<'_>) -> KResult<usize> { - let _lock = self.rwsem.write().await; + async fn write( + &self, + _: SbUse<Self::SuperBlock>, + inode: &InodeUse, + stream: &mut dyn Stream, + offset: WriteOffset<'_>, + ) -> KResult<usize> { + let _lock = inode.rwsem.write().await; let mut store_new_end = None; let offset = match offset { @@ -104,74 +65,131 @@ impl InodeFileOps for FileInode { store_new_end = Some(end); // `info.size` won't change since we are holding the write lock. - self.info.lock().size as usize + inode.info.lock().size as usize } }; - let wrote = self.pages.write(stream, offset).await?; + let page_cache = inode.get_page_cache(); + + if Arc::strong_count(&page_cache) == 1 { + // XXX: A temporary workaround here. Change this ASAP... + // Prevent the page cache from being dropped during the write. + let _ = Arc::into_raw(page_cache.clone()); + } + + let wrote = page_cache.write(stream, offset).await?; let cursor_end = offset + wrote; if let Some(store_end) = store_new_end { *store_end = cursor_end; } - { - let now = Instant::now(); - let mut info = self.info.lock(); - info.mtime = now; - info.ctime = now; - info.size = info.size.max(cursor_end as u64); - } - Ok(wrote) } - async fn truncate(&self, length: usize) -> KResult<()> { - let _lock = self.rwsem.write().await; + async fn truncate( + &self, + _: SbUse<Self::SuperBlock>, + inode: &InodeUse, + length: usize, + ) -> KResult<()> { + let _lock = inode.rwsem.write().await; - self.pages.resize(length).await?; + let now = Instant::now(); + let mut info = inode.info.lock(); + info.mtime = now; + info.ctime = now; + info.size = length as u64; - { - let now = Instant::now(); - let mut info = self.info.lock(); - info.mtime = now; - info.ctime = now; - info.size = length as u64; - } + Ok(()) + } + + async fn chmod( + &self, + _sb: SbUse<Self::SuperBlock>, + inode: &InodeUse, + perm: Permission, + ) -> KResult<()> { + let mut info = inode.info.lock(); + + info.perm = perm; + info.ctime = Instant::now(); + + Ok(()) + } + async fn read_page( + &self, + _: SbUse<Self::SuperBlock>, + _: &InodeUse, + page: &mut CachePage, + _: PageOffset, + ) -> KResult<()> { + page.as_bytes_mut().fill(0); Ok(()) } - async fn chmod(&self, perm: Permission) -> KResult<()> { - let _sb = self.sb.get()?; + async fn write_page( + &self, + _: SbUse<Self::SuperBlock>, + _: &InodeUse, + _: &mut CachePage, + _: PageOffset, + ) -> KResult<()> { + // XXX: actually we should refuse to do the writeback. + // think of a way to inform that of the page cache. + Ok(()) + } - { - let mut info = self.info.lock(); + async fn write_begin<'a>( + &self, + _: SbUse<Self::SuperBlock>, + _: &InodeUse, + page_cache: &PageCache, + pages: &'a mut BTreeMap<PageOffset, CachePage>, + offset: usize, + _: usize, + ) -> KResult<&'a mut CachePage> { + // TODO: Remove dependency on `page_cache`. + page_cache + .get_page_locked(pages, PageOffset::from_byte_floor(offset)) + .await + } - info.perm = perm; - info.ctime = Instant::now(); - } + async fn write_end( + &self, + _: SbUse<Self::SuperBlock>, + inode: &InodeUse, + _: &PageCache, + _: &mut BTreeMap<PageOffset, CachePage>, + offset: usize, + _: usize, + copied: usize, + ) -> KResult<()> { + let now = Instant::now(); + let mut info = inode.info.lock(); + info.mtime = now; + info.ctime = now; + info.size = info.size.max((offset + copied) as u64); Ok(()) } } pub struct DeviceInode { - sb: SbRef<TmpFs>, - ino: Ino, - info: Spin<InodeInfo>, is_block: bool, devid: DeviceId, } impl DeviceInode { - pub fn new(ino: Ino, sb: SbRef<TmpFs>, mode: Mode, devid: DeviceId) -> InodeUse<Self> { + pub fn new(ino: Ino, sb: SbRef<TmpFs>, mode: Mode, devid: DeviceId) -> InodeUse { let now = Instant::now(); - InodeUse::new(Self { + InodeUse::new( sb, ino, - info: Spin::new(InodeInfo { + mode.format(), + InodeInfo { size: 0, nlink: 1, uid: 0, @@ -180,76 +198,49 @@ impl DeviceInode { atime: now, ctime: now, mtime: now, - }), - is_block: mode.format() == Format::BLK, - devid, - }) + }, + Self { + is_block: mode.format() == Format::BLK, + devid, + }, + ) } } impl InodeOps for DeviceInode { type SuperBlock = TmpFs; - fn ino(&self) -> Ino { - self.ino - } - - fn format(&self) -> Format { - if self.is_block { - Format::BLK - } else { - Format::CHR - } - } - - fn info(&self) -> &Spin<InodeInfo> { - &self.info - } - - fn super_block(&self) -> &SbRef<Self::SuperBlock> { - &self.sb - } - - fn page_cache(&self) -> Option<&PageCache> { - None - } -} - -impl InodeDirOps for DeviceInode {} -impl InodeFileOps for DeviceInode { - async fn chmod(&self, perm: Permission) -> KResult<()> { - let _sb = self.sb.get()?; - - { - let mut info = self.info.lock(); - - info.perm = perm; - info.ctime = Instant::now(); - } + async fn chmod( + &self, + _sb: SbUse<Self::SuperBlock>, + inode: &InodeUse, + perm: Permission, + ) -> KResult<()> { + let mut info = inode.info.lock(); + info.perm = perm; + info.ctime = Instant::now(); Ok(()) } - fn devid(&self) -> KResult<DeviceId> { + fn devid(&self, _: SbUse<Self::SuperBlock>, _: &InodeUse) -> KResult<DeviceId> { Ok(self.devid) } } pub struct SymlinkInode { - sb: SbRef<TmpFs>, - ino: Ino, - info: Spin<InodeInfo>, target: Arc<[u8]>, } impl SymlinkInode { - pub fn new(ino: Ino, sb: SbRef<TmpFs>, target: Arc<[u8]>) -> InodeUse<Self> { + pub fn new(ino: Ino, sb: SbRef<TmpFs>, target: Arc<[u8]>) -> InodeUse { let now = Instant::now(); - InodeUse::new(Self { + InodeUse::new( sb, ino, - info: Spin::new(InodeInfo { + Format::LNK, + InodeInfo { size: target.len() as _, nlink: 1, uid: 0, @@ -258,39 +249,21 @@ impl SymlinkInode { atime: now, ctime: now, mtime: now, - }), - target, - }) + }, + Self { target }, + ) } } -impl InodeDirOps for SymlinkInode {} impl InodeOps for SymlinkInode { type SuperBlock = TmpFs; - fn ino(&self) -> Ino { - self.ino - } - - fn format(&self) -> Format { - Format::LNK - } - - fn info(&self) -> &Spin<InodeInfo> { - &self.info - } - - fn super_block(&self) -> &SbRef<Self::SuperBlock> { - &self.sb - } - - fn page_cache(&self) -> Option<&PageCache> { - None - } -} - -impl InodeFileOps for SymlinkInode { - async fn readlink(&self, buffer: &mut dyn Buffer) -> KResult<usize> { + async fn readlink( + &self, + _sb: SbUse<Self::SuperBlock>, + _inode: &InodeUse, + buffer: &mut dyn Buffer, + ) -> KResult<usize> { buffer .fill(self.target.as_ref()) .map(|result| result.allow_partial()) diff --git a/src/fs/tmpfs/mod.rs b/src/fs/tmpfs/mod.rs index 2bef67b6..62a0dfc2 100644 --- a/src/fs/tmpfs/mod.rs +++ b/src/fs/tmpfs/mod.rs @@ -1,23 +1,20 @@ mod dir; mod file; -use crate::kernel::vfs::inode::{Ino, InodeUse}; -use crate::kernel::vfs::types::{DeviceId, Permission}; -use crate::kernel::vfs::{SbRef, SbUse, SuperBlock, SuperBlockInfo}; -use crate::{ - kernel::vfs::{ - dentry::Dentry, - mount::{register_filesystem, Mount, MountCreator}, - }, - prelude::*, -}; use alloc::sync::Arc; +use core::sync::atomic::{AtomicU64, Ordering}; + use async_trait::async_trait; -use core::sync::atomic::AtomicU64; -use core::sync::atomic::Ordering; use dir::DirectoryInode; use eonix_sync::Mutex; +use crate::kernel::vfs::dentry::Dentry; +use crate::kernel::vfs::inode::{Ino, InodeUse}; +use crate::kernel::vfs::mount::{register_filesystem, Mount, MountCreator}; +use crate::kernel::vfs::types::{DeviceId, Permission}; +use crate::kernel::vfs::{SbRef, SbUse, SuperBlock, SuperBlockInfo}; +use crate::prelude::*; + pub struct TmpFs { next_ino: AtomicU64, rename_lock: Mutex<()>, @@ -30,7 +27,7 @@ impl TmpFs { Ino::new(self.next_ino.fetch_add(1, Ordering::Relaxed)) } - fn create() -> KResult<(SbUse<TmpFs>, InodeUse<DirectoryInode>)> { + fn create() -> KResult<(SbUse<TmpFs>, InodeUse)> { let tmpfs = SbUse::new( SuperBlockInfo { io_blksize: 4096, diff --git a/src/kernel/mem.rs b/src/kernel/mem.rs index bfc826bf..f8b5dc0b 100644 --- a/src/kernel/mem.rs +++ b/src/kernel/mem.rs @@ -12,5 +12,5 @@ pub use access::PhysAccess; pub(self) use mm_area::MMArea; pub use mm_list::{handle_kernel_page_fault, FileMapping, MMList, Mapping, Permission}; pub use page_alloc::{GlobalPageAlloc, RawPage}; -pub use page_cache::{CachePage, CachePageStream, PageCache, PageCacheBackendOps}; +pub use page_cache::{CachePage, PageCache, PageOffset}; pub use paging::{Page, PageBuffer, PageExcl, PageExt}; diff --git a/src/kernel/mem/mm_area.rs b/src/kernel/mem/mm_area.rs index dcbeeb63..2891dad8 100644 --- a/src/kernel/mem/mm_area.rs +++ b/src/kernel/mem/mm_area.rs @@ -4,12 +4,12 @@ use core::cmp; use eonix_mm::address::{AddrOps as _, VAddr, VRange}; use eonix_mm::page_table::{PageAttribute, RawAttribute, PTE}; -use eonix_mm::paging::{PAGE_SIZE, PFN}; +use eonix_mm::paging::PFN; use super::mm_list::EMPTY_PAGE; use super::{Mapping, Page, Permission}; -use crate::kernel::constants::EINVAL; -use crate::kernel::mem::{PageExcl, PageExt}; +use crate::kernel::mem::page_cache::PageOffset; +use crate::kernel::mem::{CachePage, PageExcl, PageExt}; use crate::prelude::KResult; #[derive(Debug)] @@ -141,59 +141,48 @@ impl MMArea { assert!(offset < file_mapping.length, "Offset out of range"); - let Some(page_cache) = file_mapping.file.page_cache() else { - panic!("Mapping file should have pagecache"); + let file_offset = file_mapping.offset + offset; + + let map_page = |page: &Page, cache_page: &CachePage| { + if !self.permission.write { + assert!(!write, "Write fault on read-only mapping"); + + *pfn = page.clone().into_raw(); + return; + } + + if self.is_shared { + // We don't process dirty flags in write faults. + // Simply assume that page will eventually be dirtied. + // So here we can set the dirty flag now. + cache_page.set_dirty(true); + attr.insert(PageAttribute::WRITE); + *pfn = page.clone().into_raw(); + return; + } + + if !write { + // Delay the copy-on-write until write fault happens. + attr.insert(PageAttribute::COPY_ON_WRITE); + *pfn = page.clone().into_raw(); + return; + } + + // XXX: Change this. Let's handle mapped pages before CoW pages. + // Nah, we are writing to a mapped private mapping... + let mut new_page = PageExcl::zeroed(); + new_page + .as_bytes_mut() + .copy_from_slice(page.lock().as_bytes()); + + attr.insert(PageAttribute::WRITE); + *pfn = new_page.into_page().into_raw(); }; - let file_offset = file_mapping.offset + offset; - let cnt_to_read = (file_mapping.length - offset).min(0x1000); - - page_cache - .with_page(file_offset, |page, cache_page| { - // Non-write faults: we find page in pagecache and do mapping - // Write fault: we need to care about shared or private mapping. - if !write { - // Bss is embarrassing in pagecache! - // We have to assume cnt_to_read < PAGE_SIZE all bss - if cnt_to_read < PAGE_SIZE { - let mut new_page = PageExcl::zeroed(); - - new_page.as_bytes_mut()[..cnt_to_read] - .copy_from_slice(&page.lock().as_bytes()[..cnt_to_read]); - - *pfn = new_page.into_page().into_raw(); - } else { - *pfn = page.clone().into_raw(); - } - - if self.permission.write { - if self.is_shared { - // The page may will not be written, - // But we simply assume page will be dirty - cache_page.set_dirty(); - attr.insert(PageAttribute::WRITE); - } else { - attr.insert(PageAttribute::COPY_ON_WRITE); - } - } - } else { - if self.is_shared { - cache_page.set_dirty(); - *pfn = page.clone().into_raw(); - } else { - let mut new_page = PageExcl::zeroed(); - - new_page.as_bytes_mut()[..cnt_to_read] - .copy_from_slice(&page.lock().as_bytes()[..cnt_to_read]); - - *pfn = new_page.into_page().into_raw(); - } - - attr.insert(PageAttribute::WRITE); - } - }) - .await? - .ok_or(EINVAL)?; + file_mapping + .page_cache + .with_page(PageOffset::from_byte_floor(file_offset), map_page) + .await?; attr.insert(PageAttribute::PRESENT); attr.remove(PageAttribute::MAPPED); diff --git a/src/kernel/mem/mm_list/mapping.rs b/src/kernel/mem/mm_list/mapping.rs index 5446ae42..2b837ae7 100644 --- a/src/kernel/mem/mm_list/mapping.rs +++ b/src/kernel/mem/mm_list/mapping.rs @@ -1,9 +1,12 @@ -use crate::kernel::vfs::inode::{Inode, InodeUse}; +use alloc::sync::Arc; + use eonix_mm::paging::PAGE_SIZE; +use crate::kernel::mem::PageCache; + #[derive(Debug, Clone)] pub struct FileMapping { - pub file: InodeUse<dyn Inode>, + pub page_cache: Arc<PageCache>, /// Offset in the file, aligned to 4KB boundary. pub offset: usize, /// Length of the mapping. Exceeding part will be zeroed. @@ -19,10 +22,10 @@ pub enum Mapping { } impl FileMapping { - pub fn new(file: InodeUse<dyn Inode>, offset: usize, length: usize) -> Self { + pub fn new(page_cache: Arc<PageCache>, offset: usize, length: usize) -> Self { assert_eq!(offset & (PAGE_SIZE - 1), 0); Self { - file, + page_cache, offset, length, } @@ -30,10 +33,10 @@ impl FileMapping { pub fn offset(&self, offset: usize) -> Self { if self.length <= offset { - Self::new(self.file.clone(), self.offset + self.length, 0) + Self::new(self.page_cache.clone(), self.offset + self.length, 0) } else { Self::new( - self.file.clone(), + self.page_cache.clone(), self.offset + offset, self.length - offset, ) diff --git a/src/kernel/mem/mm_list/page_fault.rs b/src/kernel/mem/mm_list/page_fault.rs index 6f14583d..7aac141d 100644 --- a/src/kernel/mem/mm_list/page_fault.rs +++ b/src/kernel/mem/mm_list/page_fault.rs @@ -1,11 +1,12 @@ -use super::{MMList, VAddr}; -use crate::kernel::task::Thread; use eonix_hal::mm::flush_tlb; use eonix_hal::traits::fault::PageFaultErrorCode; use eonix_mm::address::{Addr as _, AddrOps as _, VRange}; use eonix_mm::paging::PAGE_SIZE; use posix_types::signal::Signal; +use super::{MMList, VAddr}; +use crate::kernel::task::Thread; + #[repr(C)] struct FixEntry { start: u64, diff --git a/src/kernel/mem/page_alloc.rs b/src/kernel/mem/page_alloc.rs index 9dce4567..37344fc2 100644 --- a/src/kernel/mem/page_alloc.rs +++ b/src/kernel/mem/page_alloc.rs @@ -4,9 +4,9 @@ mod zones; use core::sync::atomic::Ordering; use buddy_allocator::BuddyAllocator; -use eonix_mm::address::{AddrOps as _, PRange}; +use eonix_mm::address::PRange; use eonix_mm::paging::{ - GlobalPageAlloc as GlobalPageAllocTrait, PageAlloc, PageList, PageListSized as _, PFN, + GlobalPageAlloc as GlobalPageAllocTrait, PageAlloc, PageList, PageListSized as _, }; use eonix_preempt::PreemptGuard; use eonix_sync::{NoContext, Spin}; diff --git a/src/kernel/mem/page_alloc/raw_page.rs b/src/kernel/mem/page_alloc/raw_page.rs index 0baa7b9a..074f82c7 100644 --- a/src/kernel/mem/page_alloc/raw_page.rs +++ b/src/kernel/mem/page_alloc/raw_page.rs @@ -32,15 +32,9 @@ impl SlabPageData { } } -#[derive(Clone, Copy)] -struct PageCacheData { - valid_size: usize, -} - #[repr(C)] union PageData { slab: SlabPageData, - page_cache: PageCacheData, } pub struct RawPage { @@ -245,27 +239,16 @@ impl SlabPage for RawPage { } impl PageCacheRawPage for RawPagePtr { - fn valid_size(&self) -> &mut usize { - unsafe { - // SAFETY: The caller ensures that the page is in some page cache. - &mut self.as_mut().shared_data.page_cache.valid_size - } - } - fn is_dirty(&self) -> bool { self.flags().has(PageFlags::DIRTY) } - fn clear_dirty(&self) { - self.flags().clear(PageFlags::DIRTY); - } - - fn set_dirty(&self) { - self.flags().set(PageFlags::DIRTY); - } - - fn cache_init(&self) { - self.as_mut().shared_data.page_cache = PageCacheData { valid_size: 0 }; + fn set_dirty(&self, dirty: bool) { + if dirty { + self.flags().set(PageFlags::DIRTY); + } else { + self.flags().clear(PageFlags::DIRTY); + } } } diff --git a/src/kernel/mem/page_cache.rs b/src/kernel/mem/page_cache.rs index 6a1c04ca..214c65a5 100644 --- a/src/kernel/mem/page_cache.rs +++ b/src/kernel/mem/page_cache.rs @@ -1,26 +1,27 @@ -use alloc::boxed::Box; -use alloc::collections::btree_map::BTreeMap; -use alloc::sync::Weak; +use alloc::collections::btree_map::{BTreeMap, Entry}; use core::future::Future; use core::mem::ManuallyDrop; -use align_ext::AlignExt; -use async_trait::async_trait; use eonix_hal::mm::ArchPhysAccess; use eonix_mm::address::{PAddr, PhysAccess}; use eonix_mm::paging::{PageAlloc, RawPage, PAGE_SIZE, PAGE_SIZE_BITS, PFN}; use eonix_sync::Mutex; -use super::paging::AllocZeroed; use super::Page; -use crate::io::{Buffer, FillResult, Stream}; +use crate::io::{Buffer, Stream}; +use crate::kernel::constants::EINVAL; use crate::kernel::mem::page_alloc::RawPagePtr; +use crate::kernel::vfs::inode::InodeUse; use crate::prelude::KResult; use crate::GlobalPageAlloc; +#[repr(transparent)] +#[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord)] +pub struct PageOffset(usize); + pub struct PageCache { - pages: Mutex<BTreeMap<usize, CachePage>>, - backend: Weak<dyn PageCacheBackend>, + pages: Mutex<BTreeMap<PageOffset, CachePage>>, + inode: InodeUse, } unsafe impl Send for PageCache {} @@ -30,70 +31,46 @@ unsafe impl Sync for PageCache {} pub struct CachePage(RawPagePtr); unsafe impl Send for CachePage {} +unsafe impl Sync for CachePage {} -impl Buffer for CachePage { - fn total(&self) -> usize { - PAGE_SIZE +impl PageOffset { + pub const fn from_byte_floor(offset: usize) -> Self { + Self(offset >> PAGE_SIZE_BITS) } - fn wrote(&self) -> usize { - self.valid_size() + pub const fn from_byte_ceil(offset: usize) -> Self { + Self((offset + PAGE_SIZE - 1) >> PAGE_SIZE_BITS) } - fn fill(&mut self, data: &[u8]) -> KResult<FillResult> { - let valid_size = self.valid_size(); - let available = &mut self.all_mut()[valid_size..]; - if available.len() == 0 { - return Ok(FillResult::Full); - } - - let len = core::cmp::min(data.len(), available.len()); - available[..len].copy_from_slice(&data[..len]); + pub fn iter_till(self, end: PageOffset) -> impl Iterator<Item = PageOffset> { + (self.0..end.0).map(PageOffset) + } - *self.0.valid_size() += len; + pub fn page_count(self) -> usize { + self.0 + } - if len < data.len() { - Ok(FillResult::Partial(len)) - } else { - Ok(FillResult::Done(len)) - } + pub fn byte_count(self) -> usize { + self.page_count() * PAGE_SIZE } } impl CachePage { pub fn new() -> Self { - let page = GlobalPageAlloc.alloc().unwrap(); - page.cache_init(); - Self(page) - } - - pub fn new_zeroed() -> Self { - let page = Page::zeroed(); - let raw_page_ptr = RawPagePtr::from(page.into_raw()); - - raw_page_ptr.cache_init(); - Self(raw_page_ptr) - } - - pub fn valid_size(&self) -> usize { - *self.0.valid_size() + Self(GlobalPageAlloc.alloc().unwrap()) } - pub fn set_valid_size(&mut self, valid_size: usize) { - *self.0.valid_size() = valid_size; - } - - pub fn all(&self) -> &[u8] { + pub fn as_bytes(&self) -> &[u8] { unsafe { core::slice::from_raw_parts( - // SAFETY: The page is exclusively owned by us, so we can safely access its data. + // SAFETY: The page is owned by us, so we can safely access its data. ArchPhysAccess::as_ptr(PAddr::from(PFN::from(self.0))).as_ptr(), PAGE_SIZE, ) } } - pub fn all_mut(&mut self) -> &mut [u8] { + pub fn as_bytes_mut(&mut self) -> &mut [u8] { unsafe { core::slice::from_raw_parts_mut( // SAFETY: The page is exclusively owned by us, so we can safely access its data. @@ -103,306 +80,171 @@ impl CachePage { } } - pub fn valid_data(&self) -> &[u8] { - &self.all()[..self.valid_size()] - } - pub fn is_dirty(&self) -> bool { self.0.is_dirty() } - pub fn set_dirty(&self) { - self.0.set_dirty(); + pub fn set_dirty(&self, dirty: bool) { + self.0.set_dirty(dirty); } - pub fn clear_dirty(&self) { - self.0.clear_dirty(); + pub fn get_page(&self) -> Page { + unsafe { Page::with_raw(PFN::from(self.0), |page| page.clone()) } } } impl PageCache { - pub fn new(backend: Weak<dyn PageCacheBackend>) -> Self { + pub fn new(inode: InodeUse) -> Self { Self { pages: Mutex::new(BTreeMap::new()), - backend: backend, + inode, } } - pub async fn read(&self, buffer: &mut dyn Buffer, mut offset: usize) -> KResult<usize> { - let mut pages = self.pages.lock().await; - let size = self.backend.upgrade().unwrap().size(); - - loop { - if offset >= size { - break; - } - let page_id = offset >> PAGE_SIZE_BITS; - let page = pages.get(&page_id); - - match page { - Some(page) => { - let inner_offset = offset % PAGE_SIZE; - let available_in_file = size.saturating_sub(offset); - - // TODO: still cause unnecessary IO if valid_size < PAGESIZE - // and fill result is Done - let page_data = &page.valid_data()[inner_offset..]; - let read_size = page_data.len().min(available_in_file); - - if read_size == 0 - || buffer.fill(&page_data[..read_size])?.should_stop() - || buffer.available() == 0 - { - break; - } - offset += read_size; - } - None => { + pub fn get_page_locked<'a>( + &self, + pages: &'a mut BTreeMap<PageOffset, CachePage>, + pgoff: PageOffset, + ) -> impl Future<Output = KResult<&'a mut CachePage>> + Send + use<'_, 'a> { + async move { + match pages.entry(pgoff) { + Entry::Occupied(ent) => Ok(ent.into_mut()), + Entry::Vacant(vacant_entry) => { let mut new_page = CachePage::new(); - self.backend - .upgrade() - .unwrap() - .read_page(&mut new_page, offset.align_down(PAGE_SIZE)) - .await?; - pages.insert(page_id, new_page); + self.inode.read_page(&mut new_page, pgoff).await?; + + Ok(vacant_entry.insert(new_page)) } } } + } - Ok(buffer.wrote()) + fn len(&self) -> usize { + self.inode.info.lock().size as usize } - pub async fn write(&self, stream: &mut dyn Stream, mut offset: usize) -> KResult<usize> { + // TODO: Remove this. + pub async fn with_page( + &self, + pgoff: PageOffset, + func: impl FnOnce(&Page, &CachePage), + ) -> KResult<()> { let mut pages = self.pages.lock().await; - let old_size = self.backend.upgrade().unwrap().size(); - let mut wrote = 0; - - loop { - let page_id = offset >> PAGE_SIZE_BITS; - let page = pages.get_mut(&page_id); - - match page { - Some(page) => { - let inner_offset = offset % PAGE_SIZE; - let cursor_end = match stream.poll_data(&mut page.all_mut()[inner_offset..])? { - Some(buf) => { - wrote += buf.len(); - inner_offset + buf.len() - } - None => { - break; - } - }; - - if page.valid_size() < cursor_end { - page.set_valid_size(cursor_end); - } - page.set_dirty(); - offset += PAGE_SIZE - inner_offset; - } - None => { - let new_page = if (offset >> PAGE_SIZE_BITS) > (old_size >> PAGE_SIZE_BITS) { - let new_page = CachePage::new_zeroed(); - new_page - } else { - let mut new_page = CachePage::new(); - self.backend - .upgrade() - .unwrap() - .read_page(&mut new_page, offset.align_down(PAGE_SIZE)) - .await?; - new_page - }; - - pages.insert(page_id, new_page); - } - } + if pgoff > PageOffset::from_byte_ceil(self.len()) { + return Err(EINVAL); } - Ok(wrote) - } + let cache_page = self.get_page_locked(&mut pages, pgoff).await?; - pub async fn fsync(&self) -> KResult<()> { - let pages = self.pages.lock().await; - for (page_id, page) in pages.iter() { - if page.is_dirty() { - self.backend - .upgrade() - .unwrap() - .write_page(&mut CachePageStream::new(*page), page_id << PAGE_SIZE_BITS) - .await?; - page.clear_dirty(); - } + unsafe { + let page = ManuallyDrop::new(Page::from_raw_unchecked(PFN::from(cache_page.0))); + + func(&page, cache_page); } + Ok(()) } - // This function is used for extend write or truncate - pub async fn resize(&self, new_size: usize) -> KResult<()> { + pub async fn read(&self, buffer: &mut dyn Buffer, mut offset: usize) -> KResult<usize> { let mut pages = self.pages.lock().await; - let old_size = self.backend.upgrade().unwrap().size(); + let total_len = self.len(); - if new_size < old_size { - let begin = new_size.align_down(PAGE_SIZE) >> PAGE_SIZE_BITS; - let end = old_size.align_up(PAGE_SIZE) >> PAGE_SIZE_BITS; + if offset >= total_len { + return Ok(0); + } - for page_id in begin..end { - pages.remove(&page_id); - } - } else if new_size > old_size { - let begin = old_size.align_down(PAGE_SIZE) >> PAGE_SIZE_BITS; - let end = new_size.align_up(PAGE_SIZE) >> PAGE_SIZE_BITS; + let pgoff_start = PageOffset::from_byte_floor(offset); + let pgoff_end = PageOffset::from_byte_ceil(total_len); - pages.remove(&begin); + for pgoff in pgoff_start.iter_till(pgoff_end) { + let page = self.get_page_locked(&mut pages, pgoff).await?; - for page_id in begin..end { - let mut new_page = CachePage::new_zeroed(); + let end_offset = (offset + PAGE_SIZE) / PAGE_SIZE * PAGE_SIZE; + let real_end = end_offset.min(total_len); - if page_id != end - 1 { - new_page.set_valid_size(PAGE_SIZE); - } else { - new_page.set_valid_size(new_size % PAGE_SIZE); - } - new_page.set_dirty(); - pages.insert(page_id, new_page); + let inner_offset = offset % PAGE_SIZE; + let data_len = real_end - offset; + + if buffer + .fill(&page.as_bytes()[inner_offset..inner_offset + data_len])? + .should_stop() + || buffer.available() == 0 + { + break; } + + offset = real_end; } - Ok(()) + Ok(buffer.wrote()) } - pub async fn with_page<F, O>(&self, offset: usize, func: F) -> KResult<Option<O>> - where - F: FnOnce(&Page, &CachePage) -> O, - { - let offset_aligin = offset.align_down(PAGE_SIZE); - let page_id = offset_aligin >> PAGE_SIZE_BITS; - let size = self.backend.upgrade().unwrap().size(); - - if offset_aligin > size { - return Ok(None); - } - + pub async fn write(&self, stream: &mut dyn Stream, mut offset: usize) -> KResult<usize> { let mut pages = self.pages.lock().await; + let mut total_written = 0; - let raw_page_ptr = match pages.get(&page_id) { - Some(CachePage(raw_page_ptr)) => *raw_page_ptr, - None => { - let mut new_page = CachePage::new(); - self.backend - .upgrade() - .unwrap() - .read_page(&mut new_page, offset_aligin) - .await?; - pages.insert(page_id, new_page); - new_page.0 + loop { + let end_offset = (offset + PAGE_SIZE) / PAGE_SIZE * PAGE_SIZE; + let len = end_offset - offset; + + // TODO: Rewrite to return a write state object. + let page = self + .inode + .write_begin(self, &mut pages, offset, len) + .await?; + + let inner_offset = offset % PAGE_SIZE; + let written = stream + .poll_data(&mut page.as_bytes_mut()[inner_offset..])? + .map(|b| b.len()) + .unwrap_or(0); + + page.set_dirty(true); + self.inode + .write_end(self, &mut pages, offset, len, written) + .await?; + + if written == 0 { + break; } - }; - - unsafe { - let page = ManuallyDrop::new(Page::from_raw_unchecked(PFN::from(raw_page_ptr))); - Ok(Some(func(&page, &CachePage(raw_page_ptr)))) + total_written += written; + offset += written; } - } -} -pub struct CachePageStream { - page: CachePage, - cur: usize, -} - -impl CachePageStream { - pub fn new(page: CachePage) -> Self { - Self { page, cur: 0 } + Ok(total_written) } -} - -impl Stream for CachePageStream { - fn poll_data<'a>(&mut self, buf: &'a mut [u8]) -> KResult<Option<&'a mut [u8]>> { - if self.cur >= self.page.valid_size() { - return Ok(None); - } - - let page_data = &self.page.all()[self.cur..self.page.valid_size()]; - let to_read = buf.len().min(page_data.len()); - buf[..to_read].copy_from_slice(&page_data[..to_read]); - self.cur += to_read; + pub async fn fsync(&self) -> KResult<()> { + let mut pages = self.pages.lock().await; - Ok(Some(&mut buf[..to_read])) - } + for (&pgoff, page) in pages.iter_mut() { + if !page.is_dirty() { + continue; + } - fn ignore(&mut self, len: usize) -> KResult<Option<usize>> { - if self.cur >= self.page.valid_size() { - return Ok(None); + self.inode.write_page(page, pgoff).await?; + page.set_dirty(false); } - let to_ignore = len.min(self.page.valid_size() - self.cur); - self.cur += to_ignore; - Ok(Some(to_ignore)) + Ok(()) } } -// with this trait, "page cache" and "block cache" are unified, -// for fs, offset is file offset (floor algin to PAGE_SIZE) -// for blkdev, offset is block idx (floor align to PAGE_SIZE / BLK_SIZE) -// Oh no, this would make unnecessary cache -pub trait PageCacheBackendOps: Sized { - fn read_page( - &self, - page: &mut CachePage, - offset: usize, - ) -> impl Future<Output = KResult<usize>> + Send; - - fn write_page( - &self, - page: &mut CachePageStream, - offset: usize, - ) -> impl Future<Output = KResult<usize>> + Send; - - fn size(&self) -> usize; -} - -#[async_trait] -pub trait PageCacheBackend: Send + Sync { - async fn read_page(&self, page: &mut CachePage, offset: usize) -> KResult<usize>; - async fn write_page(&self, page: &mut CachePageStream, offset: usize) -> KResult<usize>; - fn size(&self) -> usize; -} - -#[async_trait] -impl<T> PageCacheBackend for T -where - T: PageCacheBackendOps + Send + Sync + 'static, -{ - async fn read_page(&self, page: &mut CachePage, offset: usize) -> KResult<usize> { - self.read_page(page, offset).await - } - - async fn write_page(&self, page: &mut CachePageStream, offset: usize) -> KResult<usize> { - self.write_page(page, offset).await - } - - fn size(&self) -> usize { - self.size() +impl core::fmt::Debug for PageCache { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + f.debug_struct("PageCache").finish() } } pub trait PageCacheRawPage: RawPage { - fn valid_size(&self) -> &mut usize; - fn is_dirty(&self) -> bool; - - fn set_dirty(&self); - - fn clear_dirty(&self); - - fn cache_init(&self); + fn set_dirty(&self, dirty: bool); } impl Drop for PageCache { fn drop(&mut self) { - let _ = self.fsync(); + // TODO: Write back dirty pages... + // let _ = self.fsync(); } } diff --git a/src/kernel/syscall/mm.rs b/src/kernel/syscall/mm.rs index 4cb7908c..825440ef 100644 --- a/src/kernel/syscall/mm.rs +++ b/src/kernel/syscall/mm.rs @@ -1,20 +1,15 @@ -use super::FromSyscallArg; -use crate::kernel::constants::{EBADF, EINVAL}; -use crate::kernel::mem::FileMapping; -use crate::kernel::task::Thread; -use crate::kernel::vfs::filearray::FD; -use crate::{ - kernel::{ - constants::{UserMmapFlags, UserMmapProtocol}, - mem::{Mapping, Permission}, - }, - prelude::*, -}; use align_ext::AlignExt; use eonix_mm::address::{Addr as _, AddrOps as _, VAddr}; use eonix_mm::paging::PAGE_SIZE; use posix_types::syscall_no::*; +use super::FromSyscallArg; +use crate::kernel::constants::{UserMmapFlags, UserMmapProtocol, EBADF, EINVAL}; +use crate::kernel::mem::{FileMapping, Mapping, Permission}; +use crate::kernel::task::Thread; +use crate::kernel::vfs::filearray::FD; +use crate::prelude::*; + impl FromSyscallArg for UserMmapProtocol { fn from_arg(value: usize) -> UserMmapProtocol { UserMmapProtocol::from_bits_truncate(value as u32) @@ -74,7 +69,7 @@ async fn do_mmap2( .get_inode()? .ok_or(EBADF)?; - Mapping::File(FileMapping::new(file, pgoffset, len)) + Mapping::File(FileMapping::new(file.get_page_cache(), pgoffset, len)) }; let permission = Permission { diff --git a/src/kernel/task/loader/elf.rs b/src/kernel/task/loader/elf.rs index 36d139a8..9f8aa166 100644 --- a/src/kernel/task/loader/elf.rs +++ b/src/kernel/task/loader/elf.rs @@ -1,27 +1,22 @@ +use alloc::ffi::CString; +use alloc::sync::Arc; +use alloc::vec::Vec; + +use align_ext::AlignExt; +use eonix_mm::address::{Addr, AddrOps as _, VAddr}; +use eonix_mm::paging::PAGE_SIZE; +use xmas_elf::header::{self, Class, HeaderPt1, Machine_}; +use xmas_elf::program::{self, ProgramHeader32, ProgramHeader64}; + use super::{LoadInfo, ELF_MAGIC}; -use crate::io::UninitBuffer; +use crate::io::{ByteBuffer, UninitBuffer}; +use crate::kernel::constants::ENOEXEC; +use crate::kernel::mem::{FileMapping, MMList, Mapping, Permission}; use crate::kernel::task::loader::aux_vec::{AuxKey, AuxVec}; +use crate::kernel::vfs::dentry::Dentry; +use crate::kernel::vfs::FsContext; use crate::path::Path; -use crate::{ - io::ByteBuffer, - kernel::{ - constants::ENOEXEC, - mem::{FileMapping, MMList, Mapping, Permission}, - vfs::{dentry::Dentry, FsContext}, - }, - prelude::*, -}; -use align_ext::AlignExt; -use alloc::vec::Vec; -use alloc::{ffi::CString, sync::Arc}; -use eonix_mm::{ - address::{Addr, AddrOps as _, VAddr}, - paging::PAGE_SIZE, -}; -use xmas_elf::{ - header::{self, Class, HeaderPt1, Machine_}, - program::{self, ProgramHeader32, ProgramHeader64}, -}; +use crate::prelude::*; const INIT_STACK_SIZE: usize = 0x80_0000; @@ -366,7 +361,7 @@ impl<E: ElfArch> Elf<E> { vmap_start, file_len, Mapping::File(FileMapping::new( - self.file.get_inode()?, + self.file.get_inode()?.get_page_cache(), file_offset, real_file_length, )), @@ -376,16 +371,27 @@ impl<E: ElfArch> Elf<E> { .await?; } - if vmem_len > file_len { - mm_list - .mmap_fixed( - vmap_start + file_len, - vmem_len - file_len, - Mapping::Anonymous, - permission, - false, - ) - .await?; + if vmem_vaddr_end > load_vaddr_end { + if load_vaddr_end.page_offset() != 0 { + let mut zero_len = PAGE_SIZE - load_vaddr_end.page_offset(); + zero_len = zero_len.min(vmem_vaddr_end - load_vaddr_end); + + mm_list + .access_mut(load_vaddr_end, zero_len, |_, data| data.fill(0)) + .await?; + } + + if vmem_len - file_len > 0 { + mm_list + .mmap_fixed( + vmap_start + file_len, + vmem_len - file_len, + Mapping::Anonymous, + permission, + false, + ) + .await?; + } } Ok(vmap_start + vmem_len) diff --git a/src/kernel/vfs/dentry.rs b/src/kernel/vfs/dentry.rs index c1eb8cb8..22760de9 100644 --- a/src/kernel/vfs/dentry.rs +++ b/src/kernel/vfs/dentry.rs @@ -1,35 +1,31 @@ pub mod dcache; mod walk; -use core::{ - cell::UnsafeCell, - fmt, - hash::{BuildHasher, BuildHasherDefault, Hasher}, - sync::atomic::{AtomicPtr, AtomicU64, AtomicU8, Ordering}, -}; - use alloc::sync::Arc; +use core::cell::UnsafeCell; +use core::fmt; +use core::hash::{BuildHasher, BuildHasherDefault, Hasher}; +use core::sync::atomic::{AtomicPtr, AtomicU64, AtomicU8, Ordering}; + use arcref::AsArcRef; use eonix_sync::LazyLock; use pointers::BorrowedArc; -use posix_types::{namei::RenameFlags, open::OpenFlags, result::PosixError, stat::StatX}; - -use crate::{ - hash::KernelHasher, - io::Buffer, - io::Stream, - kernel::constants::{EEXIST, EINVAL, EISDIR, ELOOP, ENOENT, EPERM, ERANGE}, - kernel::{block::BlockDevice, CharDevice}, - path::Path, - prelude::*, - rcu::{rcu_read_lock, RCUNode, RCUPointer, RCUReadGuard}, -}; - -use super::{ - inode::{Ino, Inode, InodeUse, RenameData, WriteOffset}, - types::{DeviceId, Format, Mode, Permission}, - FsContext, -}; +use posix_types::namei::RenameFlags; +use posix_types::open::OpenFlags; +use posix_types::result::PosixError; +use posix_types::stat::StatX; + +use super::inode::{Ino, InodeUse, RenameData, WriteOffset}; +use super::types::{DeviceId, Format, Mode, Permission}; +use super::FsContext; +use crate::hash::KernelHasher; +use crate::io::{Buffer, Stream}; +use crate::kernel::block::BlockDevice; +use crate::kernel::constants::{EEXIST, EINVAL, EISDIR, ELOOP, ENOENT, EPERM, ERANGE}; +use crate::kernel::CharDevice; +use crate::path::Path; +use crate::prelude::*; +use crate::rcu::{rcu_read_lock, RCUNode, RCUPointer, RCUReadGuard}; const D_INVALID: u8 = 0; const D_REGULAR: u8 = 1; @@ -56,7 +52,7 @@ enum DentryKind { /// [lookup()]: crate::kernel::vfs::inode::InodeDirOps::lookup struct AssociatedInode { kind: UnsafeCell<Option<DentryKind>>, - inode: UnsafeCell<Option<InodeUse<dyn Inode>>>, + inode: UnsafeCell<Option<InodeUse>>, } /// # Safety @@ -181,15 +177,15 @@ impl Dentry { .map_or(core::ptr::null(), |parent| Arc::as_ptr(&parent)) } - pub fn fill(&self, file: InodeUse<dyn Inode>) { + pub fn fill(&self, file: InodeUse) { self.inode.store(file); } - pub fn inode(&self) -> Option<InodeUse<dyn Inode>> { + pub fn inode(&self) -> Option<InodeUse> { self.inode.load().map(|(_, inode)| inode.clone()) } - pub fn get_inode(&self) -> KResult<InodeUse<dyn Inode>> { + pub fn get_inode(&self) -> KResult<InodeUse> { self.inode().ok_or(ENOENT) } @@ -291,7 +287,7 @@ impl Dentry { let inode = self.get_inode()?; // Safety: Changing mode alone will have no effect on the file's contents - match inode.format() { + match inode.format { Format::DIR => Err(EISDIR), Format::REG => inode.read(buffer, offset).await, Format::BLK => { @@ -309,7 +305,7 @@ impl Dentry { pub async fn write(&self, stream: &mut dyn Stream, offset: WriteOffset<'_>) -> KResult<usize> { let inode = self.get_inode()?; // Safety: Changing mode alone will have no effect on the file's contents - match inode.format() { + match inode.format { Format::DIR => Err(EISDIR), Format::REG => inode.write(stream, offset).await, Format::BLK => Err(EINVAL), // TODO @@ -375,7 +371,7 @@ impl Dentry { } pub async fn chmod(&self, mode: Mode) -> KResult<()> { - self.get_inode()?.chmod(mode).await + self.get_inode()?.chmod(mode.perm()).await } pub async fn chown(&self, uid: u32, gid: u32) -> KResult<()> { @@ -438,8 +434,8 @@ impl AssociatedInode { } } - fn store(&self, inode: InodeUse<dyn Inode>) { - let kind = match inode.format() { + fn store(&self, inode: InodeUse) { + let kind = match inode.format { Format::REG | Format::BLK | Format::CHR => DentryKind::Regular, Format::DIR => DentryKind::Directory, Format::LNK => DentryKind::Symlink, @@ -463,7 +459,7 @@ impl AssociatedInode { DentryKind::atomic_acq(&self.kind) } - fn load(&self) -> Option<(DentryKind, &InodeUse<dyn Inode>)> { + fn load(&self) -> Option<(DentryKind, &InodeUse)> { self.kind().map(|kind| unsafe { let inode = (&*self.inode.get()) .as_ref() diff --git a/src/kernel/vfs/dentry/walk.rs b/src/kernel/vfs/dentry/walk.rs index 3e401b4b..7b1060ac 100644 --- a/src/kernel/vfs/dentry/walk.rs +++ b/src/kernel/vfs/dentry/walk.rs @@ -1,33 +1,23 @@ -use core::{ - future::Future, - hash::{BuildHasher, BuildHasherDefault, Hasher}, - ops::Deref, - pin::Pin, -}; - -use alloc::{boxed::Box, sync::Arc}; +use alloc::boxed::Box; +use alloc::sync::Arc; +use core::future::Future; +use core::hash::{BuildHasher, BuildHasherDefault, Hasher}; +use core::ops::Deref; +use core::pin::Pin; + use arcref::{ArcRef, AsArcRef}; use posix_types::result::PosixError; -use crate::{ - hash::KernelHasher, - io::ByteBuffer, - kernel::{ - constants::ELOOP, - vfs::{ - inode::{Inode, InodeUse}, - FsContext, - }, - }, - path::{Path, PathComponent, PathIterator}, - prelude::KResult, - rcu::{rcu_read_lock, RCUReadLock}, -}; - -use super::{ - dcache::{self, DCacheItem}, - Dentry, DentryKind, -}; +use super::dcache::{self, DCacheItem}; +use super::{Dentry, DentryKind}; +use crate::hash::KernelHasher; +use crate::io::ByteBuffer; +use crate::kernel::constants::ELOOP; +use crate::kernel::vfs::inode::InodeUse; +use crate::kernel::vfs::FsContext; +use crate::path::{Path, PathComponent, PathIterator}; +use crate::prelude::KResult; +use crate::rcu::{rcu_read_lock, RCUReadLock}; struct DentryFind<'a, 'b> { parent: &'a Dentry, @@ -40,7 +30,7 @@ pub enum WalkResultRcu<'rcu, 'path> { Ok(ArcRef<'rcu, Dentry>), Symlink { symlink: ArcRef<'rcu, Dentry>, - inode: InodeUse<dyn Inode>, + inode: InodeUse, }, Miss { parent: ArcRef<'rcu, Dentry>, @@ -53,7 +43,7 @@ pub enum WalkResult { Ok(Arc<Dentry>), Symlink { symlink: Arc<Dentry>, - inode: InodeUse<dyn Inode>, + inode: InodeUse, }, } @@ -270,7 +260,7 @@ impl FsContext { pub async fn follow_symlink( &self, symlink: ArcRef<'_, Dentry>, - inode: &InodeUse<dyn Inode>, + inode: &InodeUse, nr_follows: u32, ) -> KResult<Arc<Dentry>> { let mut target = [0; 256]; @@ -288,7 +278,7 @@ impl FsContext { fn follow_symlink_boxed<'r, 'a: 'r, 'b: 'r, 'c: 'r>( &'a self, symlink: ArcRef<'b, Dentry>, - inode: &'c InodeUse<dyn Inode>, + inode: &'c InodeUse, nr_follows: u32, ) -> Pin<Box<dyn Future<Output = KResult<Arc<Dentry>>> + Send + 'r>> { Box::pin(self.follow_symlink(symlink, inode, nr_follows)) diff --git a/src/kernel/vfs/file/inode_file.rs b/src/kernel/vfs/file/inode_file.rs index 96526ee9..d302079c 100644 --- a/src/kernel/vfs/file/inode_file.rs +++ b/src/kernel/vfs/file/inode_file.rs @@ -1,23 +1,17 @@ -use super::{File, FileType, SeekOption}; -use crate::{ - io::{Buffer, BufferFill, Stream}, - kernel::{ - constants::{EBADF, EFAULT, ENOTDIR, EOVERFLOW, ESPIPE}, - vfs::{ - dentry::Dentry, - inode::{Inode, InodeUse, WriteOffset}, - types::Format, - }, - }, - prelude::KResult, -}; use alloc::sync::Arc; + use eonix_sync::Mutex; -use posix_types::{ - getdent::{UserDirent, UserDirent64}, - open::OpenFlags, - stat::StatX, -}; +use posix_types::getdent::{UserDirent, UserDirent64}; +use posix_types::open::OpenFlags; +use posix_types::stat::StatX; + +use super::{File, FileType, SeekOption}; +use crate::io::{Buffer, BufferFill, Stream}; +use crate::kernel::constants::{EBADF, EFAULT, ENOTDIR, EOVERFLOW, ESPIPE}; +use crate::kernel::vfs::dentry::Dentry; +use crate::kernel::vfs::inode::{InodeUse, WriteOffset}; +use crate::kernel::vfs::types::Format; +use crate::prelude::KResult; pub struct InodeFile { pub r: bool, @@ -34,7 +28,7 @@ impl InodeFile { pub fn new(dentry: Arc<Dentry>, flags: OpenFlags) -> File { // SAFETY: `dentry` used to create `InodeFile` is valid. // SAFETY: `mode` should never change with respect to the `S_IFMT` fields. - let format = dentry.inode().expect("dentry should be invalid").format(); + let format = dentry.inode().expect("dentry should be invalid").format; let (r, w, a) = flags.as_rwa(); @@ -98,7 +92,7 @@ impl InodeFile { } impl File { - pub fn get_inode(&self) -> KResult<Option<InodeUse<dyn Inode>>> { + pub fn get_inode(&self) -> KResult<Option<InodeUse>> { if let FileType::Inode(inode_file) = &**self { Ok(Some(inode_file.dentry.get_inode()?)) } else { @@ -191,7 +185,7 @@ impl File { SeekOption::Set(n) => n, SeekOption::End(off) => { let inode = inode_file.dentry.get_inode()?; - let size = inode.info().lock().size as usize; + let size = inode.info.lock().size as usize; size.checked_add_signed(off).ok_or(EOVERFLOW)? } }; diff --git a/src/kernel/vfs/filearray.rs b/src/kernel/vfs/filearray.rs index 1862a3e1..609d969c 100644 --- a/src/kernel/vfs/filearray.rs +++ b/src/kernel/vfs/filearray.rs @@ -1,28 +1,23 @@ -use super::{ - file::{File, InodeFile, Pipe}, - types::{Format, Permission}, - Spin, TerminalFile, -}; -use crate::kernel::{ - constants::{ - EBADF, EISDIR, ENOTDIR, F_DUPFD, F_DUPFD_CLOEXEC, F_GETFD, F_GETFL, F_SETFD, F_SETFL, - }, - syscall::{FromSyscallArg, SyscallRetVal}, -}; -use crate::{ - kernel::{console::get_console, constants::ENXIO, vfs::dentry::Dentry, CharDevice}, - prelude::*, -}; use alloc::sync::Arc; -use intrusive_collections::{ - intrusive_adapter, rbtree::Entry, Bound, KeyAdapter, RBTree, RBTreeAtomicLink, -}; -use itertools::{ - FoldWhile::{Continue, Done}, - Itertools, -}; + +use intrusive_collections::rbtree::Entry; +use intrusive_collections::{intrusive_adapter, Bound, KeyAdapter, RBTree, RBTreeAtomicLink}; +use itertools::FoldWhile::{Continue, Done}; +use itertools::Itertools; use posix_types::open::{FDFlags, OpenFlags}; +use super::file::{File, InodeFile, Pipe}; +use super::types::{Format, Permission}; +use super::{Spin, TerminalFile}; +use crate::kernel::console::get_console; +use crate::kernel::constants::{ + EBADF, EISDIR, ENOTDIR, ENXIO, F_DUPFD, F_DUPFD_CLOEXEC, F_GETFD, F_GETFL, F_SETFD, F_SETFL, +}; +use crate::kernel::syscall::{FromSyscallArg, SyscallRetVal}; +use crate::kernel::vfs::dentry::Dentry; +use crate::kernel::CharDevice; +use crate::prelude::*; + #[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord)] pub struct FD(u32); @@ -291,20 +286,19 @@ impl FileArray { let fdflag = flags.as_fd_flags(); let inode = dentry.get_inode()?; - let file_format = inode.format(); - match (flags.directory(), file_format, flags.write()) { + match (flags.directory(), inode.format, flags.write()) { (true, Format::DIR, _) => {} (true, _, _) => return Err(ENOTDIR), (false, Format::DIR, true) => return Err(EISDIR), _ => {} } - if flags.truncate() && flags.write() && file_format == Format::REG { + if flags.truncate() && flags.write() && inode.format == Format::REG { inode.truncate(0).await?; } - let file = if file_format == Format::CHR { + let file = if inode.format == Format::CHR { let device = CharDevice::get(inode.devid()?).ok_or(ENXIO)?; device.open(flags)? } else { diff --git a/src/kernel/vfs/inode/inode.rs b/src/kernel/vfs/inode/inode.rs index 870a045d..5f0b98c2 100644 --- a/src/kernel/vfs/inode/inode.rs +++ b/src/kernel/vfs/inode/inode.rs @@ -1,52 +1,149 @@ use alloc::boxed::Box; -use core::{ - any::Any, - future::Future, - marker::Unsize, - ops::{CoerceUnsized, Deref}, - pin::Pin, -}; -use eonix_sync::Spin; - +use alloc::collections::btree_map::BTreeMap; use alloc::sync::{Arc, Weak}; -use async_trait::async_trait; +use core::any::Any; +use core::future::Future; +use core::ops::Deref; -use crate::{ - io::{Buffer, Stream}, - kernel::{ - constants::{EINVAL, EPERM}, - mem::PageCache, - timer::Instant, - vfs::{ - dentry::Dentry, - types::{DeviceId, Format, Mode, Permission}, - SbRef, SbUse, SuperBlock, - }, - }, - prelude::KResult, -}; +use async_trait::async_trait; +use eonix_sync::{RwLock, Spin}; use super::{Ino, RenameData, WriteOffset}; +use crate::io::{Buffer, Stream}; +use crate::kernel::constants::{EINVAL, EPERM}; +use crate::kernel::mem::{CachePage, PageCache, PageOffset}; +use crate::kernel::timer::Instant; +use crate::kernel::vfs::dentry::Dentry; +use crate::kernel::vfs::types::{DeviceId, Format, Mode, Permission}; +use crate::kernel::vfs::{SbRef, SbUse, SuperBlock}; +use crate::prelude::KResult; + +pub struct Inode { + pub ino: Ino, + pub format: Format, + pub info: Spin<InodeInfo>, + pub rwsem: RwLock<()>, + page_cache: Spin<Weak<PageCache>>, + sb: SbRef<dyn SuperBlock>, + ops: Box<dyn InodeOpsErased>, +} -pub trait InodeOps: Sized + Send + Sync + 'static { - type SuperBlock: SuperBlock + Sized; - - fn ino(&self) -> Ino; - fn format(&self) -> Format; - fn info(&self) -> &Spin<InodeInfo>; - - fn super_block(&self) -> &SbRef<Self::SuperBlock>; +macro_rules! return_type { + ($type:ty) => { + $type + }; + () => { + () + }; +} - fn page_cache(&self) -> Option<&PageCache>; +macro_rules! define_inode_ops { + { + $( + $(#[$attr:meta])* + async fn $method:ident $(<$($lt:lifetime),+>)? (&self $(,)? $($name:ident : $type:ty $(,)?)*) $(-> $ret:ty)? + $body:block + )* + + --- + + $( + $(#[$attr1:meta])* + fn $method1:ident $(<$($lt1:lifetime),+>)? (&self $(,)? $($name1:ident : $type1:ty $(,)?)*) $(-> $ret1:ty)? + $body1:block + )* + } => { + #[allow(unused_variables)] + pub trait InodeOps: Sized + Send + Sync + 'static { + type SuperBlock: SuperBlock + Sized; + + $( + $(#[$attr])* + fn $method $(<$($lt),+>)? ( + &self, + sb: SbUse<Self::SuperBlock>, + inode: &InodeUse, + $($name : $type),* + ) -> impl Future<Output = return_type!($($ret)?)> + Send { + async { $body } + })* + + $( + $(#[$attr1])* + fn $method1 $(<$($lt1),+>)? ( + &self, + sb: SbUse<Self::SuperBlock>, + inode: &InodeUse, + $($name1 : $type1),* + ) -> return_type!($($ret1)?) { + $body1 + })* + } + + #[async_trait] + trait InodeOpsErased: Any + Send + Sync + 'static { + $(async fn $method $(<$($lt),+>)? ( + &self, + sb: SbUse<dyn SuperBlock>, + inode: &InodeUse, + $($name : $type),* + ) -> return_type!($($ret)?);)* + + $(fn $method1 $(<$($lt1),+>)? ( + &self, + sb: SbUse<dyn SuperBlock>, + inode: &InodeUse, + $($name1 : $type1),* + ) -> return_type!($($ret1)?);)* + } + + #[async_trait] + impl<T> InodeOpsErased for T + where + T: InodeOps, + { + $(async fn $method $(<$($lt),+>)? ( + &self, + sb: SbUse<dyn SuperBlock>, + inode: &InodeUse, + $($name : $type),* + ) -> return_type!($($ret)?) { + self.$method(sb.downcast(), inode, $($name),*).await + })* + + $(fn $method1 $(<$($lt1),+>)? ( + &self, + sb: SbUse<dyn SuperBlock>, + inode: &InodeUse, + $($name1 : $type1),* + ) -> return_type!($($ret1)?) { + self.$method1(sb.downcast(), inode, $($name1),*) + })* + } + + impl InodeUse { + $(pub async fn $method $(<$($lt),+>)? ( + &self, + $($name : $type),* + ) -> return_type!($($ret)?) { + self.ops.$method(self.sbget()?, self, $($name),*).await + })* + + $(pub fn $method1 $(<$($lt1),+>)? ( + &self, + $($name1 : $type1),* + ) -> return_type!($($ret1)?) { + self.ops.$method1(self.sbget()?, self, $($name1),*) + })* + } + }; } -#[allow(unused_variables)] -pub trait InodeDirOps: InodeOps { - fn lookup( - &self, - dentry: &Arc<Dentry>, - ) -> impl Future<Output = KResult<Option<InodeUse<dyn Inode>>>> + Send { - async { Err(EPERM) } +define_inode_ops! { + // DIRECTORY OPERATIONS + + async fn lookup(&self, dentry: &Arc<Dentry>) -> KResult<Option<InodeUse>> { + Err(EPERM) } /// Read directory entries and call the given closure for each entry. @@ -55,255 +152,114 @@ pub trait InodeDirOps: InodeOps { /// - Ok(count): The number of entries read. /// - Ok(Err(err)): Some error occurred while calling the given closure. /// - Err(err): An error occurred while reading the directory. - fn readdir<'r, 'a: 'r, 'b: 'r>( - &'a self, + async fn readdir( + &self, offset: usize, - for_each_entry: &'b mut (dyn FnMut(&[u8], Ino) -> KResult<bool> + Send), - ) -> impl Future<Output = KResult<KResult<usize>>> + Send + 'r { - async { Err(EPERM) } + for_each_entry: &mut (dyn (for<'a> FnMut(&'a [u8], Ino) -> KResult<bool>) + Send), + ) -> KResult<KResult<usize>> { + Err(EPERM) } - fn create( - &self, - at: &Arc<Dentry>, - mode: Permission, - ) -> impl Future<Output = KResult<()>> + Send { - async { Err(EPERM) } + async fn create(&self, at: &Arc<Dentry>, mode: Permission) -> KResult<()> { + Err(EPERM) } - fn mkdir(&self, at: &Dentry, mode: Permission) -> impl Future<Output = KResult<()>> + Send { - async { Err(EPERM) } + async fn mkdir(&self, at: &Dentry, mode: Permission) -> KResult<()> { + Err(EPERM) } - fn mknod( - &self, - at: &Dentry, - mode: Mode, - dev: DeviceId, - ) -> impl Future<Output = KResult<()>> + Send { - async { Err(EPERM) } + async fn mknod(&self, at: &Dentry, mode: Mode, dev: DeviceId) -> KResult<()> { + Err(EPERM) } - fn unlink(&self, at: &Arc<Dentry>) -> impl Future<Output = KResult<()>> + Send { - async { Err(EPERM) } + async fn unlink(&self, at: &Arc<Dentry>) -> KResult<()> { + Err(EPERM) } - fn symlink(&self, at: &Arc<Dentry>, target: &[u8]) -> impl Future<Output = KResult<()>> + Send { - async { Err(EPERM) } + async fn symlink(&self, at: &Arc<Dentry>, target: &[u8]) -> KResult<()> { + Err(EPERM) } - fn rename(&self, rename_data: RenameData<'_, '_>) -> impl Future<Output = KResult<()>> + Send { - async { Err(EPERM) } + async fn rename(&self, rename_data: RenameData<'_, '_>) -> KResult<()> { + Err(EPERM) } -} -#[allow(unused_variables)] -pub trait InodeFileOps: InodeOps { - fn read( - &self, - buffer: &mut dyn Buffer, - offset: usize, - ) -> impl Future<Output = KResult<usize>> + Send { - async { Err(EINVAL) } + // FILE OPERATIONS + + async fn read(&self, buffer: &mut dyn Buffer, offset: usize) -> KResult<usize> { + Err(EINVAL) } - fn read_direct( - &self, - buffer: &mut dyn Buffer, - offset: usize, - ) -> impl Future<Output = KResult<usize>> + Send { - async { Err(EINVAL) } + async fn read_direct(&self, buffer: &mut dyn Buffer, offset: usize) -> KResult<usize> { + Err(EINVAL) } - fn write( + async fn write( &self, stream: &mut dyn Stream, - offset: WriteOffset<'_>, - ) -> impl Future<Output = KResult<usize>> + Send { - async { Err(EINVAL) } + offset: WriteOffset<'_> + ) -> KResult<usize> { + Err(EINVAL) } - fn write_direct( + async fn write_direct( &self, stream: &mut dyn Stream, offset: usize, - ) -> impl Future<Output = KResult<usize>> + Send { - async { Err(EINVAL) } - } - - fn devid(&self) -> KResult<DeviceId> { + ) -> KResult<usize> { Err(EINVAL) } - fn readlink(&self, buffer: &mut dyn Buffer) -> impl Future<Output = KResult<usize>> + Send { - async { Err(EINVAL) } - } - - fn truncate(&self, length: usize) -> impl Future<Output = KResult<()>> + Send { - async { Err(EPERM) } - } - - fn chmod(&self, perm: Permission) -> impl Future<Output = KResult<()>> + Send { - async { Err(EPERM) } - } - - fn chown(&self, uid: u32, gid: u32) -> impl Future<Output = KResult<()>> + Send { - async { Err(EPERM) } - } -} - -#[async_trait] -pub trait InodeDir { - async fn lookup(&self, dentry: &Arc<Dentry>) -> KResult<Option<InodeUse<dyn Inode>>>; - async fn create(&self, at: &Arc<Dentry>, perm: Permission) -> KResult<()>; - async fn mkdir(&self, at: &Dentry, perm: Permission) -> KResult<()>; - async fn mknod(&self, at: &Dentry, mode: Mode, dev: DeviceId) -> KResult<()>; - async fn unlink(&self, at: &Arc<Dentry>) -> KResult<()>; - async fn symlink(&self, at: &Arc<Dentry>, target: &[u8]) -> KResult<()>; - async fn rename(&self, rename_data: RenameData<'_, '_>) -> KResult<()>; - - fn readdir<'r, 'a: 'r, 'b: 'r>( - &'a self, - offset: usize, - callback: &'b mut (dyn FnMut(&[u8], Ino) -> KResult<bool> + Send), - ) -> Pin<Box<dyn Future<Output = KResult<KResult<usize>>> + Send + 'r>>; -} - -#[async_trait] -pub trait InodeFile { - async fn read(&self, buffer: &mut dyn Buffer, offset: usize) -> KResult<usize>; - async fn read_direct(&self, buffer: &mut dyn Buffer, offset: usize) -> KResult<usize>; - async fn write(&self, stream: &mut dyn Stream, offset: WriteOffset<'_>) -> KResult<usize>; - async fn write_direct(&self, stream: &mut dyn Stream, offset: usize) -> KResult<usize>; - fn devid(&self) -> KResult<DeviceId>; - async fn readlink(&self, buffer: &mut dyn Buffer) -> KResult<usize>; - async fn truncate(&self, length: usize) -> KResult<()>; - async fn chmod(&self, mode: Mode) -> KResult<()>; - async fn chown(&self, uid: u32, gid: u32) -> KResult<()>; -} - -pub trait Inode: InodeFile + InodeDir + Any + Send + Sync + 'static { - fn ino(&self) -> Ino; - fn format(&self) -> Format; - fn info(&self) -> &Spin<InodeInfo>; - - // TODO: This might should be removed... Temporary workaround for now. - fn page_cache(&self) -> Option<&PageCache>; - - fn sbref(&self) -> SbRef<dyn SuperBlock>; - fn sbget(&self) -> KResult<SbUse<dyn SuperBlock>>; -} - -#[async_trait] -impl<T> InodeFile for T -where - T: InodeFileOps, -{ - async fn read(&self, buffer: &mut dyn Buffer, offset: usize) -> KResult<usize> { - self.read(buffer, offset).await - } - - async fn read_direct(&self, buffer: &mut dyn Buffer, offset: usize) -> KResult<usize> { - self.read_direct(buffer, offset).await - } - - async fn write(&self, stream: &mut dyn Stream, offset: WriteOffset<'_>) -> KResult<usize> { - self.write(stream, offset).await - } - - async fn write_direct(&self, stream: &mut dyn Stream, offset: usize) -> KResult<usize> { - self.write_direct(stream, offset).await - } - - fn devid(&self) -> KResult<DeviceId> { - self.devid() - } - async fn readlink(&self, buffer: &mut dyn Buffer) -> KResult<usize> { - self.readlink(buffer).await + Err(EINVAL) } async fn truncate(&self, length: usize) -> KResult<()> { - self.truncate(length).await + Err(EPERM) } - async fn chmod(&self, mode: Mode) -> KResult<()> { - self.chmod(Permission::new(mode.non_format_bits())).await + async fn chmod(&self, perm: Permission) -> KResult<()> { + Err(EPERM) } async fn chown(&self, uid: u32, gid: u32) -> KResult<()> { - self.chown(uid, gid).await - } -} - -#[async_trait] -impl<T> InodeDir for T -where - T: InodeDirOps, -{ - async fn lookup(&self, dentry: &Arc<Dentry>) -> KResult<Option<InodeUse<dyn Inode>>> { - self.lookup(dentry).await - } - - async fn create(&self, at: &Arc<Dentry>, perm: Permission) -> KResult<()> { - self.create(at, perm).await - } - - async fn mkdir(&self, at: &Dentry, perm: Permission) -> KResult<()> { - self.mkdir(at, perm).await + Err(EPERM) } - async fn mknod(&self, at: &Dentry, mode: Mode, dev: DeviceId) -> KResult<()> { - self.mknod(at, mode, dev).await - } - - async fn unlink(&self, at: &Arc<Dentry>) -> KResult<()> { - self.unlink(at).await - } - - async fn symlink(&self, at: &Arc<Dentry>, target: &[u8]) -> KResult<()> { - self.symlink(at, target).await + // PAGE CACHE OPERATIONS + async fn read_page(&self, page: &mut CachePage, offset: PageOffset) -> KResult<()> { + Err(EINVAL) } - async fn rename(&self, rename_data: RenameData<'_, '_>) -> KResult<()> { - self.rename(rename_data).await + async fn write_page(&self, page: &mut CachePage, offset: PageOffset) -> KResult<()> { + Err(EINVAL) } - fn readdir<'r, 'a: 'r, 'b: 'r>( - &'a self, + async fn write_begin<'a>( + &self, + page_cache: &PageCache, + pages: &'a mut BTreeMap<PageOffset, CachePage>, offset: usize, - callback: &'b mut (dyn FnMut(&[u8], Ino) -> KResult<bool> + Send), - ) -> Pin<Box<dyn Future<Output = KResult<KResult<usize>>> + Send + 'r>> { - Box::pin(self.readdir(offset, callback)) - } -} - -impl<T> Inode for T -where - T: InodeOps + InodeFile + InodeDir, -{ - fn ino(&self) -> Ino { - self.ino() - } - - fn format(&self) -> Format { - self.format() - } - - fn info(&self) -> &Spin<InodeInfo> { - self.info() + len: usize, + ) -> KResult<&'a mut CachePage> { + Err(EINVAL) } - fn page_cache(&self) -> Option<&PageCache> { - self.page_cache() + async fn write_end( + &self, + page_cache: &PageCache, + pages: &mut BTreeMap<PageOffset, CachePage>, + offset: usize, + len: usize, + copied: usize + ) -> KResult<()> { + Err(EINVAL) } - fn sbref(&self) -> SbRef<dyn SuperBlock> { - self.super_block().clone() - } + --- - fn sbget(&self) -> KResult<SbUse<dyn SuperBlock>> { - self.super_block().get().map(|sb| sb as _) + fn devid(&self) -> KResult<DeviceId> { + Err(EINVAL) } } @@ -321,64 +277,87 @@ pub struct InodeInfo { pub mtime: Instant, } -pub struct InodeUse<I>(Arc<I>) -where - I: Inode + ?Sized; +#[repr(transparent)] +pub struct InodeUse(Arc<Inode>); + +impl InodeUse { + pub fn new( + sb: SbRef<dyn SuperBlock>, + ino: Ino, + format: Format, + info: InodeInfo, + ops: impl InodeOps, + ) -> Self { + let inode = Inode { + sb, + ino, + format, + info: Spin::new(info), + rwsem: RwLock::new(()), + page_cache: Spin::new(Weak::new()), + ops: Box::new(ops), + }; -impl<I> InodeUse<I> -where - I: Inode, -{ - pub fn new(inode: I) -> Self { Self(Arc::new(inode)) } - pub fn new_cyclic(inode_func: impl FnOnce(&Weak<I>) -> I) -> Self { - Self(Arc::new_cyclic(inode_func)) + pub fn sbref(&self) -> SbRef<dyn SuperBlock> { + self.sb.clone() } -} -impl<I> InodeUse<I> -where - I: Inode + ?Sized, -{ - pub fn as_raw(&self) -> *const I { - Arc::as_ptr(&self.0) + pub fn sbget(&self) -> KResult<SbUse<dyn SuperBlock>> { + self.sb.get().map(|sb| sb as _) + } + + pub fn get_priv<I>(&self) -> &I + where + I: InodeOps, + { + let ops = (&*self.ops) as &dyn Any; + + ops.downcast_ref() + .expect("InodeUse::private: InodeOps type mismatch") } -} -impl<T, U> CoerceUnsized<InodeUse<U>> for InodeUse<T> -where - T: Inode + Unsize<U> + ?Sized, - U: Inode + ?Sized, -{ + pub fn get_page_cache(&self) -> Arc<PageCache> { + if let Some(cache) = self.page_cache.lock().upgrade() { + return cache; + } + + // Slow path... + let cache = Arc::new(PageCache::new(self.clone())); + let mut page_cache = self.page_cache.lock(); + if let Some(cache) = page_cache.upgrade() { + return cache; + } + + *page_cache = Arc::downgrade(&cache); + cache + } } -impl<I> Clone for InodeUse<I> -where - I: Inode + ?Sized, -{ +impl Clone for InodeUse { fn clone(&self) -> Self { Self(self.0.clone()) } } -impl<I> core::fmt::Debug for InodeUse<I> -where - I: Inode + ?Sized, -{ +impl core::fmt::Debug for InodeUse { fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { - write!(f, "InodeUse(ino={})", self.ino()) + write!(f, "InodeUse(ino={})", self.ino) } } -impl<I> Deref for InodeUse<I> -where - I: Inode + ?Sized, -{ - type Target = I; +impl Deref for InodeUse { + type Target = Inode; fn deref(&self) -> &Self::Target { self.0.deref() } } + +impl PartialEq for InodeUse { + fn eq(&self, other: &Self) -> bool { + Arc::ptr_eq(&self.0, &other.0) + } +} diff --git a/src/kernel/vfs/inode/mod.rs b/src/kernel/vfs/inode/mod.rs index 08471ef3..6f4f041a 100644 --- a/src/kernel/vfs/inode/mod.rs +++ b/src/kernel/vfs/inode/mod.rs @@ -4,5 +4,5 @@ mod ops; mod statx; pub use ino::Ino; -pub use inode::{Inode, InodeDirOps, InodeFileOps, InodeInfo, InodeOps, InodeUse}; +pub use inode::{Inode, InodeInfo, InodeOps, InodeUse}; pub use ops::{RenameData, WriteOffset}; diff --git a/src/kernel/vfs/inode/ops.rs b/src/kernel/vfs/inode/ops.rs index baab1a80..7bf00ce5 100644 --- a/src/kernel/vfs/inode/ops.rs +++ b/src/kernel/vfs/inode/ops.rs @@ -1,9 +1,8 @@ use alloc::sync::Arc; +use super::inode::InodeUse; use crate::kernel::vfs::dentry::Dentry; -use super::{inode::InodeUse, Inode}; - pub enum WriteOffset<'end> { Position(usize), End(&'end mut usize), @@ -12,7 +11,7 @@ pub enum WriteOffset<'end> { pub struct RenameData<'a, 'b> { pub old_dentry: &'a Arc<Dentry>, pub new_dentry: &'b Arc<Dentry>, - pub new_parent: InodeUse<dyn Inode>, + pub new_parent: InodeUse, pub is_exchange: bool, pub no_replace: bool, } diff --git a/src/kernel/vfs/inode/statx.rs b/src/kernel/vfs/inode/statx.rs index a85ef3af..feb2a1b5 100644 --- a/src/kernel/vfs/inode/statx.rs +++ b/src/kernel/vfs/inode/statx.rs @@ -1,25 +1,17 @@ use posix_types::stat::StatX; -use crate::{ - kernel::{ - constants::{ - STATX_ATIME, STATX_BLOCKS, STATX_CTIME, STATX_GID, STATX_INO, STATX_MODE, STATX_MTIME, - STATX_NLINK, STATX_SIZE, STATX_TYPE, STATX_UID, - }, - vfs::types::Format, - }, - prelude::KResult, +use super::inode::InodeUse; +use crate::kernel::constants::{ + STATX_ATIME, STATX_BLOCKS, STATX_CTIME, STATX_GID, STATX_INO, STATX_MODE, STATX_MTIME, + STATX_NLINK, STATX_SIZE, STATX_TYPE, STATX_UID, }; +use crate::kernel::vfs::types::Format; +use crate::prelude::KResult; -use super::{inode::InodeUse, Inode}; - -impl<I> InodeUse<I> -where - I: Inode + ?Sized, -{ +impl InodeUse { pub fn statx(&self, stat: &mut StatX, mask: u32) -> KResult<()> { let sb = self.sbget()?; - let info = self.info().lock(); + let info = self.info.lock(); if mask & STATX_NLINK != 0 { stat.stx_nlink = info.nlink as _; @@ -53,10 +45,8 @@ where } if mask & STATX_TYPE != 0 { - let format = self.format(); - - stat.stx_mode |= format.as_raw() as u16; - if let Format::BLK | Format::CHR = format { + stat.stx_mode |= self.format.as_raw() as u16; + if let Format::BLK | Format::CHR = self.format { let devid = self.devid()?; stat.stx_rdev_major = devid.major as _; stat.stx_rdev_minor = devid.minor as _; @@ -65,7 +55,7 @@ where } if mask & STATX_INO != 0 { - stat.stx_ino = self.ino().as_raw(); + stat.stx_ino = self.ino.as_raw(); stat.stx_mask |= STATX_INO; } diff --git a/src/kernel/vfs/mount.rs b/src/kernel/vfs/mount.rs index 213acae9..6b171f81 100644 --- a/src/kernel/vfs/mount.rs +++ b/src/kernel/vfs/mount.rs @@ -1,17 +1,17 @@ -use super::{ - dentry::{dcache, Dentry, DROOT}, - inode::{Inode, InodeUse}, - SbUse, SuperBlock, -}; -use crate::kernel::{ - constants::{EEXIST, ENODEV, ENOTDIR}, - task::block_on, -}; -use crate::prelude::*; -use alloc::{collections::btree_map::BTreeMap, string::ToString as _, sync::Arc}; +use alloc::collections::btree_map::BTreeMap; +use alloc::string::ToString as _; +use alloc::sync::Arc; + use async_trait::async_trait; use eonix_sync::LazyLock; +use super::dentry::{dcache, Dentry, DROOT}; +use super::inode::InodeUse; +use super::{SbUse, SuperBlock}; +use crate::kernel::constants::{EEXIST, ENODEV, ENOTDIR}; +use crate::kernel::task::block_on; +use crate::prelude::*; + pub const MS_RDONLY: u64 = 1 << 0; pub const MS_NOSUID: u64 = 1 << 1; pub const MS_NODEV: u64 = 1 << 2; @@ -39,11 +39,7 @@ pub struct Mount { } impl Mount { - pub fn new( - mp: &Dentry, - sb: SbUse<dyn SuperBlock>, - root_inode: InodeUse<dyn Inode>, - ) -> KResult<Self> { + pub fn new(mp: &Dentry, sb: SbUse<dyn SuperBlock>, root_inode: InodeUse) -> KResult<Self> { let root_dentry = Dentry::create(mp.parent().clone(), &mp.get_name()); root_dentry.fill(root_inode); diff --git a/src/kernel/vfs/superblock.rs b/src/kernel/vfs/superblock.rs index 85b28c01..e3be5cef 100644 --- a/src/kernel/vfs/superblock.rs +++ b/src/kernel/vfs/superblock.rs @@ -1,16 +1,15 @@ -use core::{ - marker::Unsize, - ops::{CoerceUnsized, Deref}, -}; - use alloc::sync::{Arc, Weak}; -use eonix_sync::RwLock; +use core::any::{Any, TypeId}; +use core::marker::Unsize; +use core::ops::{CoerceUnsized, Deref}; -use crate::{kernel::constants::EIO, prelude::KResult}; +use eonix_sync::RwLock; use super::types::DeviceId; +use crate::kernel::constants::EIO; +use crate::prelude::KResult; -pub trait SuperBlock: Send + Sync + 'static {} +pub trait SuperBlock: Any + Send + Sync + 'static {} #[derive(Debug, Clone)] pub struct SuperBlockInfo { @@ -83,6 +82,36 @@ where } } +impl<S> SbUse<S> +where + S: SuperBlock + ?Sized, +{ + pub fn get_ref(&self) -> SbRef<S> { + SbRef(Arc::downgrade(&self.0)) + } +} + +impl SbUse<dyn SuperBlock> { + /// Downcast the superblock to a specific type. + /// + /// # Panics + /// Panics if the downcast fails. + pub fn downcast<U: SuperBlock>(self) -> SbUse<U> { + let Self(sb_complex) = self; + if (&sb_complex.backend as &dyn Any).type_id() != TypeId::of::<U>() { + panic!("Downcast failed: type mismatch"); + } + + unsafe { + // SAFETY: We have checked the type above and unsized coercion says + // that Arc<T> has the same layout as Arc<U> if T: Unsize<U>. + SbUse(Arc::from_raw( + Arc::into_raw(sb_complex) as *const SuperBlockComplex<U> + )) + } + } +} + impl<S> Clone for SbRef<S> where S: SuperBlock + ?Sized, diff --git a/src/lib.rs b/src/lib.rs index 959cb29f..8457169c 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -3,7 +3,6 @@ #![feature(allocator_api)] #![feature(c_size_t)] #![feature(coerce_unsized)] -#![feature(concat_idents)] #![feature(arbitrary_self_types)] #![feature(get_mut_unchecked)] #![feature(macro_metavar_expr)] From 96745750de00007a94cb52c835fa4d41bc88a5ef Mon Sep 17 00:00:00 2001 From: greatbridf <greatbridf@icloud.com> Date: Sun, 18 Jan 2026 00:56:10 +0800 Subject: [PATCH 23/36] mem: introduce new `Folio` abstraction - Remove struct `Page` and add `Folio`s to represent adjacent pages. - Introduce `Zone`s similar to that in Linux. Looking forward to removing all occurrence of `ArchPhysAccess` and so on. - Adapt existing code to new `Folio` interface in a dirty and rough way. Signed-off-by: greatbridf <greatbridf@icloud.com> --- .vscode/tasks.json | 2 +- crates/buddy_allocator/src/lib.rs | 112 +++---- .../eonix_hal/src/arch/riscv64/bootstrap.rs | 62 ++-- crates/eonix_hal/src/arch/riscv64/mm.rs | 51 +-- crates/eonix_hal/src/mm.rs | 38 ++- crates/eonix_mm/src/address/paddr.rs | 10 +- crates/eonix_mm/src/page_table.rs | 2 +- crates/eonix_mm/src/page_table/page_table.rs | 149 ++++----- crates/eonix_mm/src/page_table/pte.rs | 3 +- .../eonix_mm/src/page_table/pte_iterator.rs | 138 ++++---- crates/eonix_mm/src/page_table/walk.rs | 210 ++++++++++++ crates/eonix_mm/src/paging.rs | 8 +- crates/eonix_mm/src/paging/list.rs | 14 +- crates/eonix_mm/src/paging/page.rs | 313 +++--------------- crates/eonix_mm/src/paging/page_alloc.rs | 79 +---- crates/eonix_mm/src/paging/raw_page.rs | 47 --- crates/eonix_mm/src/paging/zone.rs | 7 +- crates/slab_allocator/src/lib.rs | 44 ++- src/driver/ahci/command.rs | 28 +- src/driver/ahci/command_table.rs | 7 +- src/driver/ahci/defs.rs | 6 +- src/driver/ahci/slot.rs | 10 +- src/driver/e1000e.rs | 19 +- src/driver/virtio/virtio_blk.rs | 8 +- src/fs/fat32.rs | 12 +- src/fs/fat32/file.rs | 24 -- src/fs/tmpfs/file.rs | 2 +- src/kernel/block.rs | 17 +- src/kernel/mem.rs | 4 +- src/kernel/mem/allocator.rs | 26 +- src/kernel/mem/folio.rs | 210 ++++++++++++ src/kernel/mem/mm_area.rs | 39 ++- src/kernel/mem/mm_list.rs | 64 ++-- src/kernel/mem/mm_list/page_table.rs | 40 +++ src/kernel/mem/page_alloc.rs | 91 ++--- src/kernel/mem/page_alloc/raw_page.rs | 136 ++------ src/kernel/mem/page_alloc/zones.rs | 22 +- src/kernel/mem/page_cache.rs | 97 +++--- src/kernel/mem/paging.rs | 122 +------ src/kernel/task/kernel_stack.rs | 25 +- src/kernel/vfs/file/mod.rs | 4 +- src/kernel_init.rs | 58 ++-- 42 files changed, 1116 insertions(+), 1244 deletions(-) create mode 100644 crates/eonix_mm/src/page_table/walk.rs delete mode 100644 crates/eonix_mm/src/paging/raw_page.rs delete mode 100644 src/fs/fat32/file.rs create mode 100644 src/kernel/mem/folio.rs create mode 100644 src/kernel/mem/mm_list/page_table.rs diff --git a/.vscode/tasks.json b/.vscode/tasks.json index a85ea0cf..e7a54791 100644 --- a/.vscode/tasks.json +++ b/.vscode/tasks.json @@ -6,7 +6,7 @@ { "label": "debug run riscv64", "type": "shell", - "command": "make srun ARCH=riscv64 IMG=/Volumes/oscomp/sdcard-rv.img", + "command": "make srun ARCH=riscv64", "isBackground": true, "problemMatcher": [ { diff --git a/crates/buddy_allocator/src/lib.rs b/crates/buddy_allocator/src/lib.rs index abe1ef7b..82a7d6c5 100644 --- a/crates/buddy_allocator/src/lib.rs +++ b/crates/buddy_allocator/src/lib.rs @@ -3,12 +3,12 @@ use core::hint::unreachable_unchecked; use eonix_mm::address::{AddrOps as _, PAddr, PRange}; -use eonix_mm::paging::{PageList, PageListSized, Zone, PFN}; +use eonix_mm::paging::{FolioList, FolioListSized, Zone, PFN}; const MAX_ORDER: u32 = 10; const AREAS: usize = const { MAX_ORDER as usize + 1 }; -pub trait BuddyPage: Sized + 'static { +pub trait BuddyFolio: Sized + 'static { fn pfn(&self) -> PFN; fn get_order(&self) -> u32; @@ -20,19 +20,19 @@ pub trait BuddyPage: Sized + 'static { struct FreeArea<L> where - L: PageList, + L: FolioList, { free_list: L, count: usize, } -unsafe impl<L> Send for FreeArea<L> where L: PageList {} -unsafe impl<L> Sync for FreeArea<L> where L: PageList {} +unsafe impl<L> Send for FreeArea<L> where L: FolioList {} +unsafe impl<L> Sync for FreeArea<L> where L: FolioList {} pub struct BuddyAllocator<Z, L> where Z: Zone + 'static, - L: PageList, + L: FolioList, { zone: &'static Z, free_areas: [FreeArea<L>; AREAS], @@ -41,8 +41,8 @@ where impl<Z, L> BuddyAllocator<Z, L> where Z: Zone + 'static, - Z::Page: BuddyPage, - L: PageListSized, + Z::Page: BuddyFolio, + L: FolioListSized, { pub const fn new(zone: &'static Z) -> Self { Self { @@ -52,13 +52,13 @@ where } } -impl<Z, L, P> BuddyAllocator<Z, L> +impl<Z, L, F> BuddyAllocator<Z, L> where - Z: Zone<Page = P>, - L: PageList<Page = P>, - P: BuddyPage + 'static, + Z: Zone<Page = F>, + L: FolioList<Folio = F>, + F: BuddyFolio + 'static, { - pub fn create_pages(&mut self, start: PAddr, end: PAddr) { + pub fn create_folios(&mut self, start: PAddr, end: PAddr) { assert!( self.zone .contains_prange(PRange::new(start.ceil(), end.floor())), @@ -82,40 +82,40 @@ where unsafe { // SAFETY: We've checked that the range is within the zone above. - self.add_page_unchecked(pfn, order) + self.add_folio_unchecked(pfn, order) }; pfn = new_end_pfn; } } - fn add_page(&mut self, pfn: PFN, order: u32) { + fn add_folio(&mut self, pfn: PFN, order: u32) { let prange = PRange::from(PAddr::from(pfn)).grow(1 << (order + 12)); assert!( self.zone.contains_prange(prange), - "The given page is not within the zone." + "The given folio is not within the zone." ); unsafe { // SAFETY: Checks above. - self.add_page_unchecked(pfn, order); + self.add_folio_unchecked(pfn, order); } } - unsafe fn add_page_unchecked(&mut self, pfn: PFN, order: u32) { - let Some(page) = self.zone.get_page(pfn) else { + unsafe fn add_folio_unchecked(&mut self, pfn: PFN, order: u32) { + let Some(mut folio) = self.zone.get_page(pfn) else { unsafe { unreachable_unchecked() } }; unsafe { // SAFETY: The caller ensures that the page is unused. - let page_mut = &mut *page.get(); - self.free_areas[order as usize].add_page(page_mut, order); + let folio_mut = folio.as_mut(); + self.free_areas[order as usize].add_folio(folio_mut, order); } } - fn break_page(&mut self, page: &mut P, order: u32, target_order: u32) { - let pfn = page.pfn(); + fn break_folio(&mut self, folio: &mut F, order: u32, target_order: u32) { + let pfn = folio.pfn(); for order in (target_order..order).rev() { let buddy_pfn = pfn + (1 << order); @@ -123,50 +123,50 @@ where unsafe { // SAFETY: We got the page from `self.free_areas`. Checks are // done when we've put the page into the buddy system. - self.add_page_unchecked(buddy_pfn, order); + self.add_folio_unchecked(buddy_pfn, order); } } - page.set_order(target_order); + folio.set_order(target_order); } pub fn alloc_order(&mut self, order: u32) -> Option<&'static mut Z::Page> { for current_order in order..AREAS as u32 { - let Some(page) = self.free_areas[current_order as usize].get_free_page() else { + let Some(folio) = self.free_areas[current_order as usize].get_free_folio() else { continue; }; if current_order > order { - self.break_page(page, current_order, order); + self.break_folio(folio, current_order, order); } - return Some(page); + return Some(folio); } None } - pub unsafe fn dealloc(&mut self, page: &'static mut Z::Page) { - let mut pfn = page.pfn(); - let mut order = page.get_order(); + pub unsafe fn dealloc(&mut self, folio: &'static mut Z::Page) { + let mut pfn = folio.pfn(); + let mut order = folio.get_order(); assert!( - !page.is_buddy(), - "Trying to free a page that is already in the buddy system: {pfn:?}", + !folio.is_buddy(), + "Trying to free a folio that is already in the buddy system: {pfn:?}", ); while order < MAX_ORDER { let buddy_pfn = pfn.buddy_pfn(order); - let Some(buddy_page) = self.try_get_buddy(buddy_pfn, order) else { + let Some(buddy) = self.try_get_buddy(buddy_pfn, order) else { break; }; - self.free_areas[order as usize].remove_page(buddy_page); + self.free_areas[order as usize].remove_folio(buddy); pfn = pfn.combined_pfn(buddy_pfn); order += 1; } - self.add_page(pfn, order); + self.add_folio(pfn, order); } /// This function checks whether the given page is within our [`Zone`] and @@ -176,32 +176,32 @@ where /// - the buddy is within the same [`Zone`] as us. /// - the buddy is a free buddy (in some [`FreeArea`]) /// - the buddy has order [`order`] - fn try_get_buddy<'a>(&mut self, buddy_pfn: PFN, order: u32) -> Option<&'a mut P> { - let buddy_page = self.zone.get_page(buddy_pfn)?; + fn try_get_buddy<'a>(&mut self, buddy_pfn: PFN, order: u32) -> Option<&'a mut F> { + let mut buddy = self.zone.get_page(buddy_pfn)?; unsafe { // SAFETY: We just test whether the page is a buddy. - let buddy_page_ref = &*buddy_page.get(); + let buddy_ref = buddy.as_ref(); - if !buddy_page_ref.is_buddy() { + if !buddy_ref.is_buddy() { return None; } // Sad... - if buddy_page_ref.get_order() != order { + if buddy_ref.get_order() != order { return None; } // SAFETY: We have the mutable reference to the buddy allocator. // So all the pages within are exclusively accessible to us. - Some(&mut *buddy_page.get()) + Some(buddy.as_mut()) } } } impl<L> FreeArea<L> where - L: PageListSized, + L: FolioListSized, { const fn new() -> Self { Self { @@ -213,34 +213,34 @@ where impl<L> FreeArea<L> where - L: PageList, - L::Page: BuddyPage + 'static, + L: FolioList, + L::Folio: BuddyFolio + 'static, { - pub fn get_free_page(&mut self) -> Option<&'static mut L::Page> { - self.free_list.pop_head().map(|page| { + pub fn get_free_folio(&mut self) -> Option<&'static mut L::Folio> { + self.free_list.pop_head().map(|folio| { assert_ne!(self.count, 0, "Oops"); - page.set_buddy(false); + folio.set_buddy(false); self.count -= 1; - page + folio }) } - pub fn add_page(&mut self, page: &'static mut L::Page, order: u32) { - page.set_order(order); - page.set_buddy(true); + pub fn add_folio(&mut self, folio: &'static mut L::Folio, order: u32) { + folio.set_order(order); + folio.set_buddy(true); self.count += 1; - self.free_list.push_tail(page); + self.free_list.push_tail(folio); } - pub fn remove_page(&mut self, page: &mut L::Page) { + pub fn remove_folio(&mut self, folio: &mut L::Folio) { assert_ne!(self.count, 0, "Oops"); - page.set_buddy(false); + folio.set_buddy(false); self.count -= 1; - self.free_list.remove(page); + self.free_list.remove(folio); } } diff --git a/crates/eonix_hal/src/arch/riscv64/bootstrap.rs b/crates/eonix_hal/src/arch/riscv64/bootstrap.rs index 7b3dc043..b2305f99 100644 --- a/crates/eonix_hal/src/arch/riscv64/bootstrap.rs +++ b/crates/eonix_hal/src/arch/riscv64/bootstrap.rs @@ -2,12 +2,13 @@ use core::alloc::Allocator; use core::arch::{asm, global_asm, naked_asm}; use core::cell::RefCell; use core::hint::spin_loop; +use core::ptr::NonNull; use core::sync::atomic::{AtomicBool, AtomicPtr, AtomicUsize, Ordering}; use eonix_hal_traits::mm::Memory; use eonix_mm::address::{Addr as _, PAddr, PRange, PhysAccess, VAddr, VRange}; -use eonix_mm::page_table::{PageAttribute, PagingMode, PTE as _}; -use eonix_mm::paging::{Page, PageAccess, PageAlloc, PAGE_SIZE, PFN}; +use eonix_mm::page_table::{PageAttribute, PageTable, PagingMode, TableAttribute, PTE as _}; +use eonix_mm::paging::{Folio, FrameAlloc, PageAccess, PageBlock, PAGE_SIZE, PFN}; use eonix_percpu::PercpuArea; use fdt::Fdt; use riscv::asm::sfence_vma_all; @@ -23,9 +24,12 @@ use super::cpu::{CPUID, CPU_COUNT}; use super::time::set_next_timer; use crate::arch::cpu::CPU; use crate::arch::fdt::{init_dtb_and_fdt, FdtExt, FDT}; -use crate::arch::mm::{ArchPhysAccess, FreeRam, PageAttribute64, GLOBAL_PAGE_TABLE}; +use crate::arch::mm::{ + ArchPagingMode, ArchPhysAccess, FreeRam, PageAccessImpl, PageAttribute64, RawPageTableSv48, + GLOBAL_PAGE_TABLE, +}; use crate::bootstrap::BootStrapData; -use crate::mm::{ArchMemory, ArchPagingMode, BasicPageAlloc, BasicPageAllocRef, ScopedAllocator}; +use crate::mm::{ArchMemory, BasicPageAlloc, BasicPageAllocRef, ScopedAllocator}; #[unsafe(link_section = ".bootstrap.stack")] static BOOT_STACK: [u8; 4096 * 16] = [0; 4096 * 16]; @@ -38,26 +42,26 @@ static TEMP_AP_STACK: [u8; 256] = [0; 256]; static TEMP_AP_STACK_START: &'static [u8; 256] = &TEMP_AP_STACK; #[repr(C, align(4096))] -struct PageTable([u64; PTES_PER_PAGE]); +struct BootPageTable([u64; PTES_PER_PAGE]); /// map 0x8000 0000 to itself and 0xffff ffff 8000 0000 #[unsafe(link_section = ".bootstrap.page_table.1")] -static BOOT_PAGE_TABLE: PageTable = { +static BOOT_PAGE_TABLE: BootPageTable = { let mut arr: [u64; PTES_PER_PAGE] = [0; PTES_PER_PAGE]; arr[0] = 0 | 0x2f; arr[510] = 0 | 0x2f; arr[511] = (0x80202 << 10) | 0x21; - PageTable(arr) + BootPageTable(arr) }; #[unsafe(link_section = ".bootstrap.page_table.2")] #[used] -static PT1: PageTable = { +static PT1: BootPageTable = { let mut arr: [u64; PTES_PER_PAGE] = [0; PTES_PER_PAGE]; arr[510] = (0x80000 << 10) | 0x2f; - PageTable(arr) + BootPageTable(arr) }; static BSP_PAGE_ALLOC: AtomicPtr<RefCell<BasicPageAlloc>> = AtomicPtr::new(core::ptr::null_mut()); @@ -111,7 +115,7 @@ pub unsafe extern "C" fn riscv64_start(hart_id: usize, dtb_addr: PAddr) -> ! { real_allocator.borrow_mut().add_range(range); } - setup_kernel_page_table(&alloc); + setup_kernel_page_table(alloc.clone()); unsafe { init_dtb_and_fdt(dtb_addr); } @@ -148,8 +152,12 @@ unsafe extern "C" { /// TODO: /// 对kernel image添加更细的控制，或者不加也行 -fn setup_kernel_page_table(alloc: impl PageAlloc) { - let global_page_table = &GLOBAL_PAGE_TABLE; +fn setup_kernel_page_table(alloc: BasicPageAllocRef) { + let global_page_table = PageTable::<ArchPagingMode, _, _>::new( + GLOBAL_PAGE_TABLE.clone(), + alloc.clone(), + PageAccessImpl, + ); let attr = PageAttribute::WRITE | PageAttribute::READ @@ -160,18 +168,11 @@ fn setup_kernel_page_table(alloc: impl PageAlloc) { const KERNEL_BSS_START: VAddr = VAddr::from(0xffffffff40000000); // Map kernel BSS - for pte in global_page_table.iter_kernel_in( - VRange::from(KERNEL_BSS_START).grow(BSS_LENGTH as usize), - ArchPagingMode::LEVELS, - &alloc, - ) { - let page = Page::alloc_in(&alloc); - - let attr = { - let mut attr = attr.clone(); - attr.remove(PageAttribute::EXECUTE); - attr - }; + let bss_range = VRange::from(KERNEL_BSS_START).grow(BSS_LENGTH as usize); + for pte in global_page_table.iter_kernel(bss_range) { + let page = alloc.alloc().unwrap(); + let attr = attr.difference(PageAttribute::EXECUTE); + pte.set(page.into_raw(), attr.into()); } @@ -189,17 +190,22 @@ fn setup_kernel_page_table(alloc: impl PageAlloc) { ); } sfence_vma_all(); + + core::mem::forget(global_page_table); } /// set up tp register to percpu -fn setup_cpu(alloc: impl PageAlloc, hart_id: usize) { +fn setup_cpu(alloc: impl FrameAlloc, hart_id: usize) { CPU_COUNT.fetch_add(1, Ordering::Relaxed); let mut percpu_area = PercpuArea::new(|layout| { let page_count = layout.size().div_ceil(PAGE_SIZE); - let page = Page::alloc_at_least_in(page_count, alloc); + let page = alloc.alloc_at_least(page_count).unwrap(); - let ptr = ArchPhysAccess::get_ptr_for_page(&page).cast(); + let ptr = unsafe { + // TODO: safety + ArchPhysAccess::as_ptr(page.start()) + }; page.into_raw(); ptr @@ -243,7 +249,7 @@ fn bootstrap_smp(alloc: impl Allocator, page_alloc: &RefCell<BasicPageAlloc>) { for hart_id in FDT.harts().filter(|&id| id != local_hart_id) { let stack_range = { let page_alloc = BasicPageAllocRef::new(&page_alloc); - let ap_stack = Page::alloc_order_in(4, page_alloc); + let ap_stack = page_alloc.alloc_order(4).unwrap(); let stack_range = ap_stack.range(); ap_stack.into_raw(); stack_range diff --git a/crates/eonix_hal/src/arch/riscv64/mm.rs b/crates/eonix_hal/src/arch/riscv64/mm.rs index 46dd9437..f67646cf 100644 --- a/crates/eonix_hal/src/arch/riscv64/mm.rs +++ b/crates/eonix_hal/src/arch/riscv64/mm.rs @@ -1,31 +1,25 @@ -use super::{ - config::mm::{PHYS_MAP_VIRT, ROOT_PAGE_TABLE_PFN}, - fdt::{FdtExt, FDT}, -}; -use crate::{arch::riscv64::config::mm::KIMAGE_OFFSET, traits::mm::Memory}; -use core::{marker::PhantomData, ptr::NonNull}; -use eonix_mm::{ - address::{Addr as _, AddrOps, PAddr, PRange, PhysAccess, VAddr}, - page_table::{ - PageAttribute, PageTable, PageTableLevel, PagingMode, RawAttribute, RawPageTable, - TableAttribute, PTE, - }, - paging::{NoAlloc, Page, PageBlock, PFN}, +use core::marker::PhantomData; +use core::ptr::NonNull; + +use eonix_hal_traits::mm::Memory; +use eonix_mm::address::{Addr as _, AddrOps, PAddr, PRange, PhysAccess, VAddr}; +use eonix_mm::page_table::{ + PageAttribute, PageTable, PageTableLevel, PagingMode, RawAttribute, RawPageTable, + TableAttribute, PTE, }; +use eonix_mm::paging::{BasicFolio, Folio, PageAccess, PageBlock, PFN}; use eonix_sync_base::LazyLock; use fdt::Fdt; -use riscv::{ - asm::{sfence_vma, sfence_vma_all}, - register::satp, -}; +use riscv::asm::{sfence_vma, sfence_vma_all}; +use riscv::register::satp; -pub const PAGE_TABLE_BASE: PFN = PFN::from_val(ROOT_PAGE_TABLE_PFN); -pub static GLOBAL_PAGE_TABLE: LazyLock<PageTable<ArchPagingMode, NoAlloc, ArchPhysAccess>> = - LazyLock::new(|| unsafe { - Page::with_raw(PAGE_TABLE_BASE, |root_table_page| { - PageTable::with_root_table(root_table_page.clone()) - }) - }); +use super::config::mm::{PHYS_MAP_VIRT, ROOT_PAGE_TABLE_PFN}; +use super::fdt::{FdtExt, FDT}; +use crate::arch::riscv64::config::mm::KIMAGE_OFFSET; +use crate::mm::BasicPageAlloc; + +const PAGE_TABLE_BASE: PFN = PFN::from_val(ROOT_PAGE_TABLE_PFN); +pub const GLOBAL_PAGE_TABLE: BasicFolio = BasicFolio::new(PAGE_TABLE_BASE, 0); pub const PA_V: u64 = 0b1 << 0; pub const PA_R: u64 = 0b1 << 1; @@ -61,6 +55,9 @@ pub struct ArchPhysAccess; pub struct ArchMemory; +#[derive(Clone)] +pub struct PageAccessImpl; + impl PTE for PTE64 { type Attr = PageAttribute64; @@ -261,6 +258,12 @@ impl PhysAccess for ArchPhysAccess { } } +impl PageAccess for PageAccessImpl { + unsafe fn get_ptr_for_pfn(&self, pfn: PFN) -> NonNull<PageBlock> { + unsafe { ArchPhysAccess::as_ptr(PAddr::from(pfn)) } + } +} + impl Memory for ArchMemory { fn present_ram() -> impl Iterator<Item = PRange> { FDT.present_ram() diff --git a/crates/eonix_hal/src/mm.rs b/crates/eonix_hal/src/mm.rs index 0a5597ac..c4b9bb74 100644 --- a/crates/eonix_hal/src/mm.rs +++ b/crates/eonix_hal/src/mm.rs @@ -1,16 +1,14 @@ -use core::{ - alloc::{AllocError, Allocator, Layout}, - cell::RefCell, - ptr::NonNull, -}; -use eonix_mm::{ - address::{AddrOps as _, PRange}, - paging::{PageAlloc, UnmanagedRawPage, PAGE_SIZE, PFN}, -}; +use core::alloc::{AllocError, Allocator, Layout}; +use core::cell::RefCell; +use core::ptr::NonNull; + +use eonix_mm::address::{AddrOps as _, PRange}; +use eonix_mm::page_table::PageTableAlloc; +use eonix_mm::paging::{BasicFolio, FrameAlloc, PAGE_SIZE, PFN}; pub use crate::arch::mm::{ flush_tlb, flush_tlb_all, get_root_page_table_pfn, set_root_page_table_pfn, ArchMemory, - ArchPagingMode, ArchPhysAccess, GLOBAL_PAGE_TABLE, + ArchPhysAccess, GLOBAL_PAGE_TABLE, }; pub struct BasicPageAlloc { @@ -118,19 +116,23 @@ impl<'a> BasicPageAllocRef<'a> { } } -impl PageAlloc for BasicPageAllocRef<'_> { - type RawPage = UnmanagedRawPage; +impl FrameAlloc for BasicPageAllocRef<'_> { + type Folio = BasicFolio; - fn alloc_order(&self, order: u32) -> Option<Self::RawPage> { - Some(Self::RawPage::new(self.0.borrow_mut().alloc(order), order)) + fn alloc_order(&self, order: u32) -> Option<Self::Folio> { + Some(BasicFolio::new(self.0.borrow_mut().alloc(order), order)) } +} + +impl PageTableAlloc for BasicPageAllocRef<'_> { + type Folio = BasicFolio; - unsafe fn dealloc(&self, _: Self::RawPage) { - panic!("Dealloc is not supported in BasicPageAlloc"); + fn alloc(&self) -> Self::Folio { + FrameAlloc::alloc(self).unwrap() } - fn has_management_over(&self, _: Self::RawPage) -> bool { - true + unsafe fn from_raw(&self, pfn: PFN) -> Self::Folio { + BasicFolio::new(pfn, 0) } } diff --git a/crates/eonix_mm/src/address/paddr.rs b/crates/eonix_mm/src/address/paddr.rs index 6fadbd2a..bbfa299e 100644 --- a/crates/eonix_mm/src/address/paddr.rs +++ b/crates/eonix_mm/src/address/paddr.rs @@ -1,11 +1,11 @@ +use core::fmt; +use core::ops::{Add, Sub}; +use core::ptr::NonNull; + use super::addr::Addr; use crate::paging::{PAGE_SIZE_BITS, PFN}; -use core::{ - fmt, - ops::{Add, Sub}, - ptr::NonNull, -}; +/// Convert PAddr to VAddr. pub trait PhysAccess { /// Translate the data that this address is pointing to into kernel /// accessible pointer. Use it with care. diff --git a/crates/eonix_mm/src/page_table.rs b/crates/eonix_mm/src/page_table.rs index 55732f72..f3528060 100644 --- a/crates/eonix_mm/src/page_table.rs +++ b/crates/eonix_mm/src/page_table.rs @@ -3,7 +3,7 @@ mod paging_mode; mod pte; mod pte_iterator; -pub use page_table::{PageTable, RawPageTable}; +pub use page_table::{PageTable, PageTableAlloc, RawPageTable}; pub use paging_mode::{PageTableLevel, PagingMode}; pub use pte::{PageAttribute, RawAttribute, TableAttribute, PTE}; pub use pte_iterator::PageTableIterator; diff --git a/crates/eonix_mm/src/page_table/page_table.rs b/crates/eonix_mm/src/page_table/page_table.rs index 8318049f..80be63b9 100644 --- a/crates/eonix_mm/src/page_table/page_table.rs +++ b/crates/eonix_mm/src/page_table/page_table.rs @@ -1,15 +1,12 @@ -use super::{ - paging_mode::PageTableLevel, - pte::{RawAttribute, TableAttribute}, - pte_iterator::{KernelIterator, UserIterator}, - PagingMode, PTE, -}; -use crate::{ - address::{PAddr, VRange}, - page_table::PageTableIterator, - paging::{GlobalPageAlloc, Page, PageAccess, PageAlloc, PageBlock}, -}; -use core::{marker::PhantomData, ptr::NonNull}; +use core::marker::PhantomData; +use core::ptr::NonNull; + +use super::paging_mode::PageTableLevel; +use super::pte::{RawAttribute, TableAttribute}; +use super::{PagingMode, PTE}; +use crate::address::{PAddr, VRange}; +use crate::page_table::PageTableIterator; +use crate::paging::{Folio, PageAccess, PageBlock, PFN}; pub trait RawPageTable<'a>: Send + 'a { type Entry: PTE + 'a; @@ -24,45 +21,60 @@ pub trait RawPageTable<'a>: Send + 'a { unsafe fn from_ptr(ptr: NonNull<PageBlock>) -> Self; } +pub trait PageTableAlloc: Clone { + type Folio: Folio; + + fn alloc(&self) -> Self::Folio; + unsafe fn from_raw(&self, pfn: PFN) -> Self::Folio; +} + +pub trait GlobalPageTableAlloc: PageTableAlloc { + const GLOBAL: Self; +} + pub struct PageTable<'a, M, A, X> where M: PagingMode, M::Entry: 'a, - A: PageAlloc, + A: PageTableAlloc, X: PageAccess, { - root_table_page: Page<A>, - phantom: PhantomData<&'a (M, X)>, + root_table_page: A::Folio, + alloc: A, + access: X, + phantom: PhantomData<&'a M>, } impl<'a, M, A, X> PageTable<'a, M, A, X> where M: PagingMode, M::Entry: 'a, - A: PageAlloc, + A: PageTableAlloc, X: PageAccess, { - pub fn with_root_table(root_table_page: Page<A>) -> Self { + pub fn new(root_table_page: A::Folio, alloc: A, access: X) -> Self { Self { root_table_page, + alloc, + access, phantom: PhantomData, } } pub fn clone_global<'b, B>(&self) -> PageTable<'b, M, B, X> where - B: GlobalPageAlloc, + B: GlobalPageTableAlloc, { - self.clone_in(B::global()) + self.clone_in(B::GLOBAL) } pub fn clone_in<'b, B>(&self, alloc: B) -> PageTable<'b, M, B, X> where - B: PageAlloc, + B: PageTableAlloc, { - let new_root_table_page = Page::alloc_in(alloc); - let new_table_data = X::get_ptr_for_page(&new_root_table_page); - let kernel_table_data = X::get_ptr_for_page(&self.root_table_page); + let new_root_table_page = alloc.alloc(); + let new_table_data = self.access.get_ptr_for_page(&new_root_table_page); + let kernel_table_data = self.access.get_ptr_for_page(&self.root_table_page); unsafe { // SAFETY: `new_table_data` and `kernel_table_data` are both valid pointers @@ -82,7 +94,7 @@ where root_page_table.index_mut(idx).take(); } - PageTable::with_root_table(new_root_table_page) + PageTable::new(new_root_table_page, alloc, self.access.clone()) } pub fn addr(&self) -> PAddr { @@ -90,100 +102,59 @@ where } pub fn iter_user(&self, range: VRange) -> impl Iterator<Item = &mut M::Entry> { - let alloc = self.root_table_page.allocator(); - let page_table_ptr = X::get_ptr_for_page(&self.root_table_page); + let page_table_ptr = self.access.get_ptr_for_page(&self.root_table_page); let root_page_table = unsafe { // SAFETY: `page_table_ptr` is a valid pointer to a page table. M::RawTable::from_ptr(page_table_ptr) }; - PageTableIterator::<M, A, X, UserIterator>::new(root_page_table, range, alloc.clone()) - } - - pub fn iter_kernel(&self, range: VRange) -> impl Iterator<Item = &mut M::Entry> { - self.iter_kernel_levels(range, M::LEVELS) + PageTableIterator::<M, _, _>::new( + root_page_table, + range, + TableAttribute::USER, + self.alloc.clone(), + self.access.clone(), + ) } - /// Iterates over the kernel space entries in the page table for the specified levels. - /// - /// # Parameters - /// - `range`: The virtual address range to iterate over. - /// - `levels`: A slice of `PageTableLevel` that specifies which levels of the page table - /// should be included in the iteration. Each level corresponds to a level in the page - /// table hierarchy, and the iterator will traverse entries at these levels. + /// Iterates over the kernel space entries in the page table. /// /// # Returns /// An iterator over mutable references to the page table entries (`M::Entry`) within the - /// specified range and levels. + /// specified range. /// /// # Example /// ``` /// let range = VRange::new(0x1234000, 0x1300000); - /// let levels = &M::LEVELS[..2]; - /// for pte in page_table.iter_kernel_levels(range, levels) { + /// for pte in page_table.iter_kernel(range) { /// // Process each entry /// } /// ``` - pub fn iter_kernel_levels( - &self, - range: VRange, - levels: &'static [PageTableLevel], - ) -> impl Iterator<Item = &mut M::Entry> { - self.iter_kernel_in(range, levels, self.root_table_page.allocator()) - } - - /// Iterates over the kernel space entries in the page table for the specified levels - /// with a given page allocator. - /// - /// # Parameters - /// - `range`: The virtual address range to iterate over. - /// - `levels`: A slice of `PageTableLevel` that specifies which levels of the page table - /// should be included in the iteration. Each level corresponds to a level in the page - /// table hierarchy, and the iterator will traverse entries at these levels. - /// - `alloc`: A page allocator that provides memory for the page table entries. - /// - /// # Returns - /// An iterator over mutable references to the page table entries (`M::Entry`) within the - /// specified range and levels. - /// - /// # Example - /// ```no_run - /// let range = VRange::new(0x1234000, 0x1300000); - /// let levels = &M::LEVELS[..2]; - /// for pte in page_table.iter_kernel_in(range, levels, NoAlloc) { - /// // Process each entry - /// } - /// ``` - pub fn iter_kernel_in<A1: PageAlloc>( - &self, - range: VRange, - levels: &'static [PageTableLevel], - alloc: A1, - ) -> impl Iterator<Item = &mut M::Entry> { - let page_table_ptr = X::get_ptr_for_page(&self.root_table_page); + pub fn iter_kernel(&self, range: VRange) -> impl Iterator<Item = &mut M::Entry> { + let page_table_ptr = self.access.get_ptr_for_page(&self.root_table_page); let root_page_table = unsafe { // SAFETY: `page_table_ptr` is a valid pointer to a page table. M::RawTable::from_ptr(page_table_ptr) }; - PageTableIterator::<M, A1, X, KernelIterator>::with_levels( + PageTableIterator::<M, _, _>::with_levels( root_page_table, range, - alloc, - levels, + TableAttribute::GLOBAL, + self.alloc.clone(), + self.access.clone(), + M::LEVELS, ) } - fn drop_page_table_recursive(page_table: &Page<A>, levels: &[PageTableLevel]) { + fn drop_page_table_recursive(&self, page_table: &A::Folio, levels: &[PageTableLevel]) { let [level, remaining_levels @ ..] = levels else { return }; if remaining_levels.is_empty() { // We reached the last level, no need to go deeper. return; } - let alloc = page_table.allocator(); - - let page_table_ptr = X::get_ptr_for_page(page_table); + let page_table_ptr = self.access.get_ptr_for_page(page_table); let mut page_table = unsafe { // SAFETY: `page_table_ptr` is a valid pointer to a page table. M::RawTable::from_ptr(page_table_ptr) @@ -201,10 +172,10 @@ where let page_table = unsafe { // SAFETY: We got the pfn from a valid page table entry, so it should be valid. - Page::from_raw_in(pfn, alloc.clone()) + self.alloc.from_raw(pfn) }; - Self::drop_page_table_recursive(&page_table, remaining_levels); + self.drop_page_table_recursive(&page_table, remaining_levels); } } } @@ -213,10 +184,10 @@ impl<'a, M, A, X> Drop for PageTable<'a, M, A, X> where M: PagingMode, M::Entry: 'a, - A: PageAlloc, + A: PageTableAlloc, X: PageAccess, { fn drop(&mut self) { - Self::drop_page_table_recursive(&self.root_table_page, M::LEVELS); + self.drop_page_table_recursive(&self.root_table_page, M::LEVELS); } } diff --git a/crates/eonix_mm/src/page_table/pte.rs b/crates/eonix_mm/src/page_table/pte.rs index e067d207..c14d5738 100644 --- a/crates/eonix_mm/src/page_table/pte.rs +++ b/crates/eonix_mm/src/page_table/pte.rs @@ -1,6 +1,7 @@ -use crate::paging::PFN; use bitflags::bitflags; +use crate::paging::PFN; + bitflags! { #[derive(Clone, Copy, PartialEq)] pub struct TableAttribute: usize { diff --git a/crates/eonix_mm/src/page_table/pte_iterator.rs b/crates/eonix_mm/src/page_table/pte_iterator.rs index 89b9fb9f..a9e4ff46 100644 --- a/crates/eonix_mm/src/page_table/pte_iterator.rs +++ b/crates/eonix_mm/src/page_table/pte_iterator.rs @@ -1,62 +1,14 @@ -use super::{ - pte::{RawAttribute, TableAttribute}, - PageTableLevel, PagingMode, RawPageTable as _, PTE, -}; -use crate::{ - address::{AddrOps as _, VRange}, - paging::{Page, PageAccess, PageAlloc}, -}; -use core::{marker::PhantomData}; - -pub struct KernelIterator; -pub struct UserIterator; - -pub trait IteratorType<M: PagingMode> { - fn page_table_attributes() -> TableAttribute; - - fn get_page_table<'a, A, X>(pte: &mut M::Entry, alloc: &A) -> M::RawTable<'a> - where - A: PageAlloc, - X: PageAccess, - { - let attr = pte.get_attr().as_table_attr().expect("Not a page table"); - - if attr.contains(TableAttribute::PRESENT) { - let pfn = pte.get_pfn(); - unsafe { - // SAFETY: We are creating a pointer to a page referenced to in - // some page table, which should be valid. - let page_table_ptr = X::get_ptr_for_pfn(pfn); - // SAFETY: `page_table_ptr` is a valid pointer to a page table. - M::RawTable::from_ptr(page_table_ptr) - } - } else { - let page = Page::alloc_in(alloc.clone()); - let page_table_ptr = X::get_ptr_for_page(&page); - - unsafe { - // SAFETY: `page_table_ptr` is good for writing and properly aligned. - page_table_ptr.write_bytes(0, 1); - } - - pte.set( - page.into_raw(), - <M::Entry as PTE>::Attr::from(Self::page_table_attributes()), - ); - - unsafe { - // SAFETY: `page_table_ptr` is a valid pointer to a page table. - M::RawTable::from_ptr(page_table_ptr) - } - } - } -} +use super::page_table::PageTableAlloc; +use super::pte::{RawAttribute, TableAttribute}; +use super::{PageTableLevel, PagingMode, RawPageTable as _, PTE}; +use crate::address::{AddrOps as _, VRange}; +use crate::paging::{Folio, PageAccess}; -pub struct PageTableIterator<'a, M, A, X, K> +pub struct PageTableIterator<'a, M, A, X> where M: PagingMode, M::Entry: 'a, - A: PageAlloc, + A: PageTableAlloc, X: PageAccess, { /// Specifies the hierarchy of page table levels to iterate over. @@ -69,19 +21,19 @@ where indicies: [u16; 8], tables: [Option<M::RawTable<'a>>; 8], + fill_entry_attr: TableAttribute, + alloc: A, - _phantom: PhantomData<&'a (X, K)>, + access: X, } -impl<'a, M, A, X, K> PageTableIterator<'a, M, A, X, K> +impl<'a, M, A, X> PageTableIterator<'a, M, A, X> where M: PagingMode, M::Entry: 'a, - A: PageAlloc, + A: PageTableAlloc, X: PageAccess, - K: IteratorType<M>, { - fn parse_tables_starting_from(&mut self, idx_level: usize) { for (idx, &pt_idx) in self .indicies @@ -98,18 +50,58 @@ where }; let parent_table = parent_table.as_mut().expect("Parent table is None"); let next_pte = parent_table.index_mut(pt_idx); - child_table.replace(K::get_page_table::<A, X>(next_pte, &self.alloc)); + + child_table.replace({ + let attr = next_pte + .get_attr() + .as_table_attr() + .expect("Not a page table"); + + if attr.contains(TableAttribute::PRESENT) { + let pfn = next_pte.get_pfn(); + unsafe { + // SAFETY: We are creating a pointer to a page referenced to in + // some page table, which should be valid. + let page_table_ptr = self.access.get_ptr_for_pfn(pfn); + // SAFETY: `page_table_ptr` is a valid pointer to a page table. + M::RawTable::from_ptr(page_table_ptr) + } + } else { + let page = self.alloc.alloc(); + let page_table_ptr = self.access.get_ptr_for_page(&page); + + unsafe { + // SAFETY: `page_table_ptr` is good for writing and properly aligned. + page_table_ptr.write_bytes(0, 1); + } + + next_pte.set(page.into_raw(), self.fill_entry_attr.into()); + + unsafe { + // SAFETY: `page_table_ptr` is a valid pointer to a page table. + M::RawTable::from_ptr(page_table_ptr) + } + } + }); } } - pub fn new(page_table: M::RawTable<'a>, range: VRange, alloc: A) -> Self { - Self::with_levels(page_table, range, alloc, M::LEVELS) + pub fn new( + page_table: M::RawTable<'a>, + range: VRange, + fill_entry_attr: TableAttribute, + alloc: A, + access: X, + ) -> Self { + Self::with_levels(page_table, range, fill_entry_attr, alloc, access, M::LEVELS) } pub fn with_levels( page_table: M::RawTable<'a>, range: VRange, + fill_entry_attr: TableAttribute, alloc: A, + access: X, levels: &'static [PageTableLevel], ) -> Self { let start = range.start().floor(); @@ -122,8 +114,9 @@ where remaining: (end - start) / last_level.page_size(), indicies: [0; 8], tables: [const { None }; 8], + fill_entry_attr: fill_entry_attr.union(TableAttribute::PRESENT), alloc, - _phantom: PhantomData, + access, }; for (i, level) in levels.iter().enumerate() { @@ -137,13 +130,12 @@ where } } -impl<'a, M, A, X, K> Iterator for PageTableIterator<'a, M, A, X, K> +impl<'a, M, A, X> Iterator for PageTableIterator<'a, M, A, X> where M: PagingMode, M::Entry: 'a, - A: PageAlloc, + A: PageTableAlloc, X: PageAccess, - K: IteratorType<M>, { type Item = &'a mut M::Entry; @@ -178,15 +170,3 @@ where Some(retval) } } - -impl<M: PagingMode> IteratorType<M> for KernelIterator { - fn page_table_attributes() -> TableAttribute { - TableAttribute::PRESENT | TableAttribute::GLOBAL - } -} - -impl<M: PagingMode> IteratorType<M> for UserIterator { - fn page_table_attributes() -> TableAttribute { - TableAttribute::PRESENT | TableAttribute::USER - } -} diff --git a/crates/eonix_mm/src/page_table/walk.rs b/crates/eonix_mm/src/page_table/walk.rs new file mode 100644 index 00000000..aba80b09 --- /dev/null +++ b/crates/eonix_mm/src/page_table/walk.rs @@ -0,0 +1,210 @@ +use super::pte::{RawAttribute, TableAttribute}; +use super::{PageTableLevel, PTE}; +use crate::address::{AddrOps, VAddr, VRange}; +use crate::paging::PFN; + +pub enum WalkState { + Next, + Skip, + Break, +} + +pub trait PageTable: Sized { + type Entry: PTE; + const LEVELS: &'static [PageTableLevel]; + + fn index(&self, index: usize) -> &Self::Entry; + fn index_mut(&mut self, index: usize) -> &mut Self::Entry; + + fn from_pfn(pfn: PFN) -> Self; + unsafe fn take_pfn(pfn: PFN) -> Self; +} + +pub struct PageTableWalk<'a, T, D> +where + T: PageTable, +{ + levels: &'a [PageTableLevel], + fill_entry: &'a [fn(&mut D, &mut T::Entry) -> Option<PFN>], + walk_entry: &'a [fn(&mut D, &mut T::Entry) -> WalkState], + data: D, +} + +fn try_get_table<T, D>( + entry: &mut T::Entry, + data: &mut D, + fill_entry: fn(&mut D, &mut T::Entry) -> Option<PFN>, +) -> Option<T> +where + T: PageTable, +{ + let (mut pfn, attr) = entry.get(); + + // Always skip huge page entries + let attr = attr.as_table_attr()?; + + // For normal entries, check present flags + if !attr.contains(TableAttribute::PRESENT) { + // Skip entries filled with nothing + pfn = fill_entry(data, entry)?; + } + + Some(T::from_pfn(pfn)) +} + +fn _walk_page_table<T, D>( + walk: &mut PageTableWalk<T, D>, + cur_level: usize, + table: &mut T, + range: VRange, +) where + T: PageTable, +{ + let level = walk.levels[cur_level]; + + let page_size = level.page_size(); + let mut addr = range.start(); + + while addr < range.end() { + let idx = level.index_of(addr); + let entry = table.index_mut(idx); + + let mut next_table = None; + if cur_level < walk.levels.len() - 1 { + next_table = try_get_table(entry, &mut walk.data, walk.fill_entry[cur_level]); + } + + match ( + walk.walk_entry[cur_level](&mut walk.data, entry), + &mut next_table, + ) { + (WalkState::Break, _) => break, + (WalkState::Next, Some(next_table)) => _walk_page_table( + walk, + cur_level + 1, + next_table, + VRange::new(addr, range.end()), + ), + // `fill_entry` says that we shouldn't continue. + (WalkState::Next, None) => {} + _ => {} + } + + addr = addr.floor_to(page_size) + page_size; + } +} + +pub fn walk_page_table<T, D>(walk: &mut PageTableWalk<T, D>, table: &mut T, range: VRange) +where + T: PageTable, +{ + _walk_page_table(walk, 0, table, range); +} + +pub fn drop_user_page_table<T>(mut root_page_table: T) +where + T: PageTable, +{ + fn walk<T: PageTable, const LEVEL: usize>(_: &mut (), entry: &mut T::Entry) -> WalkState { + let (pfn, attr) = entry.get(); + let Some(attr) = attr.as_table_attr() else { + return WalkState::Skip; + }; + + if !attr.contains(TableAttribute::USER) { + return WalkState::Skip; + } + + unsafe { + // Check `_walk_page_table`: We will and only will touch the next level of table with + // `next_table` holding a refcount. We take the table away from the parent table now. + T::take_pfn(pfn); + } + + entry.set(PFN::from_val(0), TableAttribute::empty().into()); + + if LEVEL == 2 { + WalkState::Skip + } else { + WalkState::Next + } + } + + let mut walk = PageTableWalk { + levels: T::LEVELS, + fill_entry: &[no_fill::<T, ()>, no_fill::<T, ()>, no_fill::<T, ()>], + walk_entry: &[walk::<T, 0>, walk::<T, 1>, walk::<T, 2>, skip_walk::<T, ()>], + data: (), + }; + + walk_page_table( + &mut walk, + &mut root_page_table, + VRange::new(VAddr::from(0), VAddr::from(0x0000_8000_0000_0000)), + ); +} + +pub fn iter_pte<T: PageTable>( + page_table: &mut T, + range: VRange, + fill_func: impl FnMut(&mut T::Entry) -> Option<PFN>, + for_each: impl FnMut(&mut T::Entry), +) { + let walker = (fill_func, for_each); + + fn fill_entry<T: PageTable>( + (fill, _): &mut ( + impl FnMut(&mut T::Entry) -> Option<PFN>, + impl FnMut(&mut T::Entry), + ), + entry: &mut T::Entry, + ) -> Option<PFN> { + fill(entry) + } + + fn walk_entry<T: PageTable>( + (_, for_each): &mut ( + impl FnMut(&mut T::Entry) -> Option<PFN>, + impl FnMut(&mut T::Entry), + ), + entry: &mut T::Entry, + ) -> WalkState { + for_each(entry); + WalkState::Next + } + + let mut walk = PageTableWalk { + levels: T::LEVELS, + fill_entry: &[fill_entry::<T>, fill_entry::<T>, fill_entry::<T>], + walk_entry: &[ + cont_walk::<T, _>, + cont_walk::<T, _>, + cont_walk::<T, _>, + walk_entry::<T>, + ], + data: walker, + }; + + walk_page_table(&mut walk, page_table, range); +} + +pub fn no_fill<T, D>(_: &mut D, _: &mut T::Entry) -> Option<PFN> +where + T: PageTable, +{ + None +} + +pub fn skip_walk<T, D>(_: &mut D, _: &mut T::Entry) -> WalkState +where + T: PageTable, +{ + WalkState::Skip +} + +pub fn cont_walk<T, D>(_: &mut D, _: &mut T::Entry) -> WalkState +where + T: PageTable, +{ + WalkState::Next +} diff --git a/crates/eonix_mm/src/paging.rs b/crates/eonix_mm/src/paging.rs index 0c4811f2..f0166cf3 100644 --- a/crates/eonix_mm/src/paging.rs +++ b/crates/eonix_mm/src/paging.rs @@ -2,12 +2,10 @@ mod list; mod page; mod page_alloc; mod pfn; -mod raw_page; mod zone; -pub use list::{PageList, PageListSized}; -pub use page::{Page, PageAccess, PageBlock, PAGE_SIZE, PAGE_SIZE_BITS}; -pub use page_alloc::{GlobalPageAlloc, NoAlloc, PageAlloc}; +pub use list::{FolioList, FolioListSized}; +pub use page::{BasicFolio, Folio, PageAccess, PageBlock, PAGE_SIZE, PAGE_SIZE_BITS}; +pub use page_alloc::{FrameAlloc, GlobalFrameAlloc}; pub use pfn::PFN; -pub use raw_page::{RawPage, UnmanagedRawPage}; pub use zone::Zone; diff --git a/crates/eonix_mm/src/paging/list.rs b/crates/eonix_mm/src/paging/list.rs index a52cf947..2dd557c9 100644 --- a/crates/eonix_mm/src/paging/list.rs +++ b/crates/eonix_mm/src/paging/list.rs @@ -1,16 +1,16 @@ -pub trait PageList { - type Page; +pub trait FolioList { + type Folio; fn is_empty(&self) -> bool; - fn peek_head(&mut self) -> Option<&mut Self::Page>; + fn peek_head(&mut self) -> Option<&mut Self::Folio>; - fn pop_head(&mut self) -> Option<&'static mut Self::Page>; - fn push_tail(&mut self, page: &'static mut Self::Page); - fn remove(&mut self, page: &mut Self::Page); + fn pop_head(&mut self) -> Option<&'static mut Self::Folio>; + fn push_tail(&mut self, page: &'static mut Self::Folio); + fn remove(&mut self, page: &mut Self::Folio); } -pub trait PageListSized: PageList + Sized { +pub trait FolioListSized: FolioList + Sized { const NEW: Self; fn new() -> Self { diff --git a/crates/eonix_mm/src/paging/page.rs b/crates/eonix_mm/src/paging/page.rs index c5a14b5e..8b067e43 100644 --- a/crates/eonix_mm/src/paging/page.rs +++ b/crates/eonix_mm/src/paging/page.rs @@ -1,6 +1,8 @@ -use super::{GlobalPageAlloc, PageAlloc, RawPage as _, PFN}; -use crate::address::{AddrRange, PAddr, PhysAccess}; -use core::{fmt, mem::ManuallyDrop, ptr::NonNull, sync::atomic::Ordering}; +use core::mem::ManuallyDrop; +use core::ptr::NonNull; + +use super::PFN; +use crate::address::{PAddr, PRange}; pub const PAGE_SIZE: usize = 4096; pub const PAGE_SIZE_BITS: u32 = PAGE_SIZE.trailing_zeros(); @@ -15,306 +17,81 @@ pub struct PageBlock([u8; PAGE_SIZE]); /// A trait that provides the kernel access to the page. #[doc(notable_trait)] -pub trait PageAccess { +pub trait PageAccess: Clone { /// Returns a kernel-accessible pointer to the page referenced by the given /// physical frame number. /// /// # Safety /// This function is unsafe because calling this function on some non-existing /// pfn will cause undefined behavior. - unsafe fn get_ptr_for_pfn(pfn: PFN) -> NonNull<PageBlock>; + unsafe fn get_ptr_for_pfn(&self, pfn: PFN) -> NonNull<PageBlock>; /// Returns a kernel-accessible pointer to the given page. - fn get_ptr_for_page<A: PageAlloc>(page: &Page<A>) -> NonNull<PageBlock> { + fn get_ptr_for_page<F: Folio>(&self, page: &F) -> NonNull<PageBlock> { unsafe { // SAFETY: `page.pfn()` is guaranteed to be valid. - Self::get_ptr_for_pfn(page.pfn()) + self.get_ptr_for_pfn(page.pfn()) } } } -/// A Page allocated in allocator `A`. -#[derive(PartialEq, Eq, PartialOrd, Ord)] -pub struct Page<A: PageAlloc> { - raw_page: A::RawPage, - alloc: A, -} - -unsafe impl<A: PageAlloc> Send for Page<A> {} -unsafe impl<A: PageAlloc> Sync for Page<A> {} - -impl<A> Page<A> -where - A: GlobalPageAlloc, -{ - /// Allocate a page of the given *order*. - pub fn alloc_order(order: u32) -> Self { - Self::alloc_order_in(order, A::global()) - } - - /// Allocate exactly one page. - pub fn alloc() -> Self { - Self::alloc_in(A::global()) - } +/// A [`Folio`] represents one page or a bunch of adjacent pages. +pub trait Folio { + /// Returns the physical frame number of the folio, which is aligned with + /// the folio's size and valid. + fn pfn(&self) -> PFN; - /// Allocate a contiguous block of pages that can contain at least `count` pages. - pub fn alloc_at_least(count: usize) -> Self { - Self::alloc_at_least_in(count, A::global()) - } + /// Returns the folio's *order* (log2 of the number of pages contained in + /// the folio). + fn order(&self) -> u32; - /// Acquire the ownership of the page pointed to by `pfn`, leaving `refcount` untouched. - /// - /// # Safety - /// This function is unsafe because it assumes that the caller has ensured that - /// `pfn` points to a valid page allocated through `alloc_order()` and that the - /// page have not been freed or deallocated yet. - /// - /// No checks are done. Any violation of this assumption may lead to undefined behavior. - pub unsafe fn from_raw_unchecked(pfn: PFN) -> Self { - unsafe { Self::from_raw_unchecked_in(pfn, A::global()) } - } - - /// Acquire the ownership of the page pointed to by `pfn`, leaving `refcount` untouched. - /// - /// This function is a safe wrapper around `from_paddr_unchecked()` that does **some sort - /// of** checks to ensure that the page is valid and managed by the allocator. - /// - /// # Panic - /// This function will panic if the page is not valid or if the page is not managed by - /// the allocator. - /// - /// # Safety - /// This function is unsafe because it assumes that the caller has ensured that - /// `pfn` points to an existing page (A.K.A. inside the global page array) and the - /// page will not be freed or deallocated during the call. - pub unsafe fn from_raw(pfn: PFN) -> Self { - unsafe { Self::from_raw_in(pfn, A::global()) } - } - - /// Do some work with the page without touching the reference count with the same - /// restrictions as `from_raw_in()`. - /// - /// # Safety - /// Check `from_raw()` for the safety requirements. - pub unsafe fn with_raw<F, O>(pfn: PFN, func: F) -> O - where - F: FnOnce(&Self) -> O, - { - unsafe { Self::with_raw_in(pfn, A::global(), func) } - } - - /// Do some work with the page without touching the reference count with the same - /// restrictions as `from_raw_unchecked_in()`. - /// - /// # Safety - /// Check `from_raw_unchecked()` for the safety requirements. - pub unsafe fn with_raw_unchecked<F, O>(pfn: PFN, func: F, alloc: A) -> O - where - F: FnOnce(&Self) -> O, - { - unsafe { Self::with_raw_unchecked_in(pfn, func, alloc) } - } -} - -impl<A> Page<A> -where - A: PageAlloc, -{ - /// Allocate a page of the given *order*. - pub fn alloc_order_in(order: u32, alloc: A) -> Self { - Self { - raw_page: alloc.alloc_order(order).expect("Out of memory"), - alloc, - } - } - - /// Allocate exactly one page. - pub fn alloc_in(alloc: A) -> Self { - Self { - raw_page: alloc.alloc().expect("Out of memory"), - alloc, - } - } - - /// Allocate a contiguous block of pages that can contain at least `count` pages. - pub fn alloc_at_least_in(count: usize, alloc: A) -> Self { - Self { - raw_page: alloc.alloc_at_least(count).expect("Out of memory"), - alloc, - } - } - - /// Acquire the ownership of the page pointed to by `pfn`, leaving `refcount` untouched. - /// - /// # Safety - /// This function is unsafe because it assumes that the caller has ensured that - /// `pfn` points to a valid page managed by `alloc` and that the page have not - /// been freed or deallocated yet. - /// - /// No checks are done. Any violation of this assumption may lead to undefined behavior. - pub unsafe fn from_raw_unchecked_in(pfn: PFN, alloc: A) -> Self { - Self { - raw_page: A::RawPage::from(pfn), - alloc, - } + /// Returns the total size of the folio in bytes. + fn len(&self) -> usize { + 1 << (self.order() + PAGE_SIZE_BITS) } - /// Acquire the ownership of the page pointed to by `pfn`, leaving `refcount` untouched. - /// - /// This function is a safe wrapper around `from_paddr_unchecked()` that does **some sort - /// of** checks to ensure that the page is valid and managed by the allocator. - /// - /// # Panic - /// This function will panic if the page is not valid or if the page is not managed by - /// the allocator. - /// - /// # Safety - /// This function is unsafe because it assumes that the caller has ensured that - /// `pfn` points to an existing page (A.K.A. inside the global page array) and the - /// page will not be freed or deallocated during the call. - pub unsafe fn from_raw_in(pfn: PFN, alloc: A) -> Self { - unsafe { - // SAFETY: The caller guarantees that the page is inside the global page array. - assert!(alloc.has_management_over(A::RawPage::from(pfn))); - - // SAFETY: We've checked that the validity of the page. And the caller guarantees - // that the page will not be freed or deallocated during the call. - Self::from_raw_unchecked_in(pfn, alloc) - } + /// Returns the start physical address of the folio, which is guaranteed to + /// be aligned to the folio's size and valid. + fn start(&self) -> PAddr { + PAddr::from(self.pfn()) } - /// Do some work with the page without touching the reference count with the same - /// restrictions as `from_raw_in()`. - /// - /// # Safety - /// Check `from_raw_in()` for the safety requirements. - pub unsafe fn with_raw_in<F, O>(pfn: PFN, alloc: A, func: F) -> O - where - F: FnOnce(&Self) -> O, - { - unsafe { - let me = ManuallyDrop::new(Self::from_raw_in(pfn, alloc)); - func(&me) - } + /// Returns the physical address range of the ifolio, which is guaranteed to + /// be aligned to the folio's size and valid. + fn range(&self) -> PRange { + PRange::from(self.start()).grow(self.len()) } - /// Do some work with the page without touching the reference count with the same - /// restrictions as `from_raw_unchecked_in()`. - /// - /// # Safety - /// Check `from_raw_unchecked_in()` for the safety requirements. - pub unsafe fn with_raw_unchecked_in<F, O>(pfn: PFN, func: F, alloc: A) -> O + /// Consumes the folio and returns the PFN without dropping the reference + /// count the folio holds. + fn into_raw(self) -> PFN where - F: FnOnce(&Self) -> O, + Self: Sized, { - unsafe { - let me = ManuallyDrop::new(Self::from_raw_unchecked_in(pfn, alloc)); - func(&me) - } - } - - /// Whether we are the only owner of the page. - pub fn is_exclusive(&self) -> bool { - self.raw_page.refcount().load(Ordering::Acquire) == 1 - } - - /// Returns the *order* of the page, which is the log2 of the number of pages - /// contained in the page object. - pub fn order(&self) -> u32 { - self.raw_page.order() - } - - /// Returns the total size of the page in bytes. - pub fn len(&self) -> usize { - 1 << (self.order() + PAGE_SIZE_BITS) - } - - /// Consumes the `Page` and returns the physical frame number without dropping - /// the reference count the page holds. - pub fn into_raw(self) -> PFN { let me = ManuallyDrop::new(self); me.pfn() } - - /// Returns the physical frame number of the page, which is aligned with the - /// page size and valid. - pub fn pfn(&self) -> PFN { - Into::<PFN>::into(self.raw_page) - } - - /// Returns the start physical address of the page, which is guaranteed to be - /// aligned to the page size and valid. - pub fn start(&self) -> PAddr { - PAddr::from(self.pfn()) - } - - /// Returns the physical address range of the page, which is guaranteed to be - /// aligned to the page size and valid. - pub fn range(&self) -> AddrRange<PAddr> { - AddrRange::from(self.start()).grow(self.len()) - } - - /// Get the allocator that manages this page. - pub fn allocator(&self) -> &A { - &self.alloc - } } -impl<A> Clone for Page<A> -where - A: PageAlloc, -{ - fn clone(&self) -> Self { - // SAFETY: Memory order here can be Relaxed is for the same reason as that - // in the copy constructor of `std::shared_ptr`. - self.raw_page.refcount().fetch_add(1, Ordering::Relaxed); - - Self { - raw_page: self.raw_page, - alloc: self.alloc.clone(), - } - } +/// A simple [`Folio`] with no reference counting or other ownership mechanism. +#[derive(Clone)] +pub struct BasicFolio { + pfn: PFN, + order: u32, } -impl<A> Drop for Page<A> -where - A: PageAlloc, -{ - fn drop(&mut self) { - match self.raw_page.refcount().fetch_sub(1, Ordering::AcqRel) { - 0 => panic!("Refcount for an in-use page is 0"), - 1 => unsafe { - // SAFETY: `self.raw_page` points to a valid page inside the global page array. - assert!(self.alloc.has_management_over(self.raw_page)); - - // SAFETY: `self.raw_page` is managed by the allocator and we're dropping the page. - self.alloc.dealloc(self.raw_page) - }, - _ => {} - } +impl BasicFolio { + pub const fn new(pfn: PFN, order: u32) -> Self { + Self { pfn, order } } } -impl<A: PageAlloc> fmt::Debug for Page<A> { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - write!( - f, - "Page({:?}, order={})", - Into::<PFN>::into(self.raw_page), - self.order() - ) +impl Folio for BasicFolio { + fn pfn(&self) -> PFN { + self.pfn } -} -impl<T> PageAccess for T -where - T: PhysAccess, -{ - unsafe fn get_ptr_for_pfn(pfn: PFN) -> NonNull<PageBlock> { - unsafe { - // SAFETY: The physical address of a existing page must be - // aligned to the page size. - T::as_ptr(PAddr::from(pfn)) - } + fn order(&self) -> u32 { + self.order } } diff --git a/crates/eonix_mm/src/paging/page_alloc.rs b/crates/eonix_mm/src/paging/page_alloc.rs index fe222605..267d3ccb 100644 --- a/crates/eonix_mm/src/paging/page_alloc.rs +++ b/crates/eonix_mm/src/paging/page_alloc.rs @@ -1,89 +1,44 @@ -use super::{raw_page::UnmanagedRawPage, RawPage}; +use super::Folio; -/// A trait for allocating and deallocating pages of memory. +/// A trait for allocating and deallocating folios. /// /// Note that the instances of this trait should provide pointer-like or reference-like /// behavior, meaning that the allocators are to be passed around by value and stored in /// managed data structures. This is because the allocator may be used to deallocate the /// pages it allocates. -#[doc(notable_trait)] -pub trait PageAlloc: Clone { - type RawPage: RawPage; +pub trait FrameAlloc: Clone { + type Folio: Folio; - /// Allocate a page of the given *order*. - fn alloc_order(&self, order: u32) -> Option<Self::RawPage>; + /// Allocate a folio of the given *order*. + fn alloc_order(&self, order: u32) -> Option<Self::Folio>; - /// Allocate exactly one page. - fn alloc(&self) -> Option<Self::RawPage> { + /// Allocate exactly one folio. + fn alloc(&self) -> Option<Self::Folio> { self.alloc_order(0) } - /// Allocate a contiguous block of pages that can contain at least `count` pages. - fn alloc_at_least(&self, count: usize) -> Option<Self::RawPage> { + /// Allocate a folio that can contain at least [`count`] contiguous pages. + fn alloc_at_least(&self, count: usize) -> Option<Self::Folio> { let order = count.next_power_of_two().trailing_zeros(); self.alloc_order(order) } - - /// Deallocate a page. - /// - /// # Safety - /// This function is unsafe because it assumes that the caller MUST ensure that - /// `raw_page` is allocated in this allocator and never used after this call. - unsafe fn dealloc(&self, raw_page: Self::RawPage); - - /// Check whether the page is allocated and managed by the allocator. - fn has_management_over(&self, page_ptr: Self::RawPage) -> bool; } /// A trait for global page allocators. /// /// Global means that we can get an instance of the allocator from anywhere in the kernel. -#[doc(notable_trait)] -pub trait GlobalPageAlloc: PageAlloc + 'static { - /// Get the global page allocator. - fn global() -> Self; +pub trait GlobalFrameAlloc: FrameAlloc + 'static { + /// The global page allocator. + const GLOBAL: Self; } -#[derive(Clone)] -pub struct NoAlloc; - -impl<'a, A> PageAlloc for &'a A +impl<'a, A> FrameAlloc for &'a A where - A: PageAlloc, + A: FrameAlloc, { - type RawPage = A::RawPage; + type Folio = A::Folio; - fn alloc_order(&self, order: u32) -> Option<Self::RawPage> { + fn alloc_order(&self, order: u32) -> Option<Self::Folio> { (*self).alloc_order(order) } - - unsafe fn dealloc(&self, raw_page: Self::RawPage) { - unsafe { (*self).dealloc(raw_page) } - } - - fn has_management_over(&self, raw_page: Self::RawPage) -> bool { - (*self).has_management_over(raw_page) - } -} - -impl PageAlloc for NoAlloc { - type RawPage = UnmanagedRawPage; - - fn alloc_order(&self, _: u32) -> Option<Self::RawPage> { - panic!("`NoAlloc` cannot allocate pages"); - } - - unsafe fn dealloc(&self, _: Self::RawPage) { - panic!("`NoAlloc` cannot free pages"); - } - - fn has_management_over(&self, _: Self::RawPage) -> bool { - true - } -} - -impl GlobalPageAlloc for NoAlloc { - fn global() -> Self { - Self - } } diff --git a/crates/eonix_mm/src/paging/raw_page.rs b/crates/eonix_mm/src/paging/raw_page.rs deleted file mode 100644 index 789e863b..00000000 --- a/crates/eonix_mm/src/paging/raw_page.rs +++ /dev/null @@ -1,47 +0,0 @@ -use core::sync::atomic::AtomicUsize; - -use super::PFN; - -/// A `RawPage` represents a page of memory in the kernel. It is a low-level -/// representation of a page that is used by the kernel to manage memory. -#[doc(notable_trait)] -pub trait RawPage: Clone + Copy + From<PFN> + Into<PFN> { - fn order(&self) -> u32; - fn refcount(&self) -> &AtomicUsize; -} - -#[derive(Clone, Copy)] -pub struct UnmanagedRawPage(PFN, u32); - -/// Unmanaged raw pages should always have a non-zero refcount to -/// avoid `free()` from being called. -static UNMANAGED_RAW_PAGE_CLONE_COUNT: AtomicUsize = AtomicUsize::new(1); - -impl UnmanagedRawPage { - pub const fn new(pfn: PFN, order: u32) -> Self { - Self(pfn, order) - } -} - -impl From<PFN> for UnmanagedRawPage { - fn from(value: PFN) -> Self { - Self::new(value, 0) - } -} - -impl Into<PFN> for UnmanagedRawPage { - fn into(self) -> PFN { - let Self(pfn, _) = self; - pfn - } -} - -impl RawPage for UnmanagedRawPage { - fn order(&self) -> u32 { - self.1 - } - - fn refcount(&self) -> &AtomicUsize { - &UNMANAGED_RAW_PAGE_CLONE_COUNT - } -} diff --git a/crates/eonix_mm/src/paging/zone.rs b/crates/eonix_mm/src/paging/zone.rs index ec3ed15e..a2e85343 100644 --- a/crates/eonix_mm/src/paging/zone.rs +++ b/crates/eonix_mm/src/paging/zone.rs @@ -1,7 +1,6 @@ -use core::cell::UnsafeCell; +use core::ptr::NonNull; -#[allow(unused_imports)] -use super::{Page, PageAlloc, RawPage, PFN}; +use super::PFN; use crate::address::PRange; /// A [`Zone`] holds a lot of [`Page`]s that share the same NUMA node or @@ -16,5 +15,5 @@ pub trait Zone: Send + Sync { /// /// # Return /// [`None`] if [`pfn`] is not in this [`Zone`]. - fn get_page(&self, pfn: PFN) -> Option<&UnsafeCell<Self::Page>>; + fn get_page(&self, pfn: PFN) -> Option<NonNull<Self::Page>>; } diff --git a/crates/slab_allocator/src/lib.rs b/crates/slab_allocator/src/lib.rs index 8597331d..c3e7f392 100644 --- a/crates/slab_allocator/src/lib.rs +++ b/crates/slab_allocator/src/lib.rs @@ -2,7 +2,7 @@ use core::ptr::NonNull; -use eonix_mm::paging::{PageList, PageListSized}; +use eonix_mm::paging::{FolioList, FolioListSized}; use eonix_sync::Spin; #[repr(C)] @@ -84,21 +84,21 @@ where } } -pub trait SlabPageAlloc { +/// Allocate a page suitable for slab system use. The page MUST come with +/// its allocation count 0 and next free slot None. +/// +/// # Safety +/// The page returned MUST have been properly initialized after allocation. +pub unsafe trait SlabPageAlloc { type Page: SlabPage; - type PageList: PageList<Page = Self::Page>; + type PageList: FolioList<Folio = Self::Page>; - /// Allocate a page suitable for slab system use. The page MUST come with - /// its allocation count 0 and next free slot None. - /// - /// # Safety - /// The page returned MUST be properly initialized before its usage. - unsafe fn alloc_uninit(&self) -> &'static mut Self::Page; + fn alloc_slab_page(&self) -> &'static mut Self::Page; } pub(crate) struct SlabList<T> where - T: PageList, + T: FolioList, { empty_list: T, partial_list: T, @@ -120,7 +120,7 @@ unsafe impl<P, const COUNT: usize> Sync for SlabAlloc<P, COUNT> where P: SlabPag impl<L, const COUNT: usize> SlabAlloc<L, COUNT> where L: SlabPageAlloc, - L::PageList: PageListSized, + L::PageList: FolioListSized, { pub fn new_in(alloc: L) -> Self { Self { @@ -148,7 +148,7 @@ where impl<T> SlabList<T> where - T: PageListSized, + T: FolioListSized, { const fn new(object_size: usize) -> Self { Self { @@ -162,8 +162,8 @@ where impl<T> SlabList<T> where - T: PageList, - T::Page: SlabPage, + T: FolioList, + T::Folio: SlabPage, { fn alloc_from_partial(&mut self) -> NonNull<u8> { let head = self.partial_list.peek_head().unwrap(); @@ -190,18 +190,16 @@ where slot } - fn charge(&mut self, alloc: &impl SlabPageAlloc<Page = T::Page>) { - unsafe { - let slab = alloc.alloc_uninit(); - let free_slot = make_slab_page(slab.get_data_ptr(), self.object_size); + fn charge(&mut self, alloc: &impl SlabPageAlloc<Page = T::Folio>) { + let slab = alloc.alloc_slab_page(); + let free_slot = make_slab_page(slab.get_data_ptr(), self.object_size); - slab.set_free_slot(Some(free_slot)); + slab.set_free_slot(Some(free_slot)); - self.empty_list.push_tail(slab); - } + self.empty_list.push_tail(slab); } - fn alloc(&mut self, alloc: &impl SlabPageAlloc<Page = T::Page>) -> NonNull<u8> { + fn alloc(&mut self, alloc: &impl SlabPageAlloc<Page = T::Folio>) -> NonNull<u8> { if !self.partial_list.is_empty() { return self.alloc_from_partial(); } @@ -216,7 +214,7 @@ where unsafe fn dealloc(&mut self, ptr: NonNull<u8>, _alloc: &impl SlabPageAlloc) { let slab_page = unsafe { // SAFETY: - <T::Page>::from_allocated(ptr) + <T::Folio>::from_allocated(ptr) }; let (was_full, is_empty); diff --git a/src/driver/ahci/command.rs b/src/driver/ahci/command.rs index c83339b7..4609d38d 100644 --- a/src/driver/ahci/command.rs +++ b/src/driver/ahci/command.rs @@ -1,9 +1,11 @@ +use eonix_mm::paging::Folio as _; + use crate::kernel::constants::EINVAL; -use crate::kernel::mem::paging::Page; +use crate::kernel::mem::Folio; use crate::prelude::*; pub trait Command { - fn pages(&self) -> &[Page]; + fn pages(&self) -> &[Folio]; fn lba(&self) -> u64; // in sectors @@ -14,19 +16,19 @@ pub trait Command { } pub struct IdentifyCommand { - page: Page, + page: Folio, } impl IdentifyCommand { pub fn new() -> Self { Self { - page: Page::alloc(), + page: Folio::alloc(), } } } impl Command for IdentifyCommand { - fn pages(&self) -> &[Page] { + fn pages(&self) -> &[Folio] { core::slice::from_ref(&self.page) } @@ -47,14 +49,14 @@ impl Command for IdentifyCommand { } } -pub struct ReadLBACommand<'lt> { - pages: &'lt [Page], +pub struct ReadLBACommand<'a> { + pages: &'a [Folio], lba: u64, count: u16, } -impl<'lt> ReadLBACommand<'lt> { - pub fn new(pages: &'lt [Page], lba: u64, count: u16) -> KResult<Self> { +impl<'a> ReadLBACommand<'a> { + pub fn new(pages: &'a [Folio], lba: u64, count: u16) -> KResult<Self> { if pages.len() > 248 { return Err(EINVAL); } @@ -69,7 +71,7 @@ impl<'lt> ReadLBACommand<'lt> { } impl Command for ReadLBACommand<'_> { - fn pages(&self) -> &[Page] { + fn pages(&self) -> &[Folio] { self.pages } @@ -91,13 +93,13 @@ impl Command for ReadLBACommand<'_> { } pub struct WriteLBACommand<'a> { - pages: &'a [Page], + pages: &'a [Folio], lba: u64, count: u16, } impl<'a> WriteLBACommand<'a> { - pub fn new(pages: &'a [Page], lba: u64, count: u16) -> KResult<Self> { + pub fn new(pages: &'a [Folio], lba: u64, count: u16) -> KResult<Self> { if pages.len() > 248 { return Err(EINVAL); } @@ -112,7 +114,7 @@ impl<'a> WriteLBACommand<'a> { } impl Command for WriteLBACommand<'_> { - fn pages(&self) -> &[Page] { + fn pages(&self) -> &[Folio] { self.pages } diff --git a/src/driver/ahci/command_table.rs b/src/driver/ahci/command_table.rs index 7b78d26f..00fc8a0b 100644 --- a/src/driver/ahci/command_table.rs +++ b/src/driver/ahci/command_table.rs @@ -1,13 +1,14 @@ use core::ptr::NonNull; use eonix_mm::address::PAddr; +use eonix_mm::paging::Folio as _; use super::command::Command; use super::{PRDTEntry, FISH2D}; -use crate::kernel::mem::{Page, PageExt}; +use crate::kernel::mem::FolioOwned; pub struct CommandTable { - page: Page, + page: FolioOwned, cmd_fis: NonNull<FISH2D>, prdt: NonNull<[PRDTEntry; 248]>, prdt_entries: usize, @@ -18,7 +19,7 @@ unsafe impl Sync for CommandTable {} impl CommandTable { pub fn new() -> Self { - let page = Page::alloc(); + let page = FolioOwned::alloc(); let base = page.get_ptr(); unsafe { diff --git a/src/driver/ahci/defs.rs b/src/driver/ahci/defs.rs index c5440246..66841da8 100644 --- a/src/driver/ahci/defs.rs +++ b/src/driver/ahci/defs.rs @@ -1,7 +1,9 @@ #![allow(dead_code)] -use crate::kernel::mem::paging::Page; use eonix_mm::address::Addr as _; +use eonix_mm::paging::Folio as _; + +use crate::kernel::mem::Folio; pub const VENDOR_INTEL: u16 = 0x8086; pub const DEVICE_AHCI: u16 = 0x2922; @@ -239,7 +241,7 @@ pub struct PRDTEntry { } impl PRDTEntry { - pub fn setup(&mut self, page: &Page) { + pub fn setup(&mut self, page: &Folio) { self.base = page.start().addr() as u64; self._reserved1 = 0; diff --git a/src/driver/ahci/slot.rs b/src/driver/ahci/slot.rs index 06c6f2ec..dd096f57 100644 --- a/src/driver/ahci/slot.rs +++ b/src/driver/ahci/slot.rs @@ -3,18 +3,18 @@ use core::ptr::NonNull; use core::task::{Poll, Waker}; use eonix_mm::address::{Addr as _, PAddr}; +use eonix_mm::paging::Folio as _; use eonix_sync::{Spin, SpinIrq as _}; use super::command_table::CommandTable; use super::CommandHeader; use crate::kernel::constants::EIO; -use crate::kernel::mem::paging::AllocZeroed; -use crate::kernel::mem::{Page, PageExt}; +use crate::kernel::mem::FolioOwned; use crate::KResult; pub struct CommandList { base: NonNull<u8>, - _page: Page, + _page: FolioOwned, } unsafe impl Send for CommandList {} @@ -75,7 +75,9 @@ impl CommandList { } pub fn new() -> Self { - let page = Page::zeroed(); + let mut page = FolioOwned::alloc(); + page.as_bytes_mut().fill(0); + let base = page.get_ptr(); let controls_ptr = Self::controls_ptr(base); diff --git a/src/driver/e1000e.rs b/src/driver/e1000e.rs index 73143c2c..6d6ca353 100644 --- a/src/driver/e1000e.rs +++ b/src/driver/e1000e.rs @@ -5,11 +5,12 @@ use core::ptr::NonNull; use async_trait::async_trait; use eonix_hal::fence::memory_barrier; use eonix_mm::address::{Addr, PAddr}; +use eonix_mm::paging::Folio as _; use eonix_sync::SpinIrq; use crate::kernel::constants::{EAGAIN, EFAULT, EINVAL, EIO}; use crate::kernel::interrupt::register_irq_handler; -use crate::kernel::mem::{PageExcl, PageExt, PhysAccess}; +use crate::kernel::mem::{FolioOwned, PhysAccess}; use crate::kernel::pcie::{self, Header, PCIDevice, PCIDriver, PciError}; use crate::net::netdev; use crate::prelude::*; @@ -54,13 +55,13 @@ struct E1000eDev { id: u32, regs: Registers, - rt_desc_page: PageExcl, + rt_desc_page: FolioOwned, rx_head: Option<u32>, rx_tail: Option<u32>, tx_tail: Option<u32>, - rx_buffers: Box<[PageExcl; RX_DESC_SIZE]>, - tx_buffers: Box<[Option<PageExcl>; TX_DESC_SIZE]>, + rx_buffers: Box<[FolioOwned; RX_DESC_SIZE]>, + tx_buffers: Box<[Option<FolioOwned>; TX_DESC_SIZE]>, } fn test(val: u32, bit: u32) -> bool { @@ -227,7 +228,7 @@ impl netdev::Netdev for E1000eDev { return Err(EIO); } - let mut buffer_page = PageExcl::alloc(); + let mut buffer_page = FolioOwned::alloc(); if buf.len() > buffer_page.len() { return Err(EFAULT); } @@ -363,11 +364,15 @@ impl E1000eDev { speed: netdev::LinkSpeed::SpeedUnknown, id: netdev::alloc_id(), regs, - rt_desc_page: PageExcl::zeroed(), + rt_desc_page: { + let mut folio = FolioOwned::alloc(); + folio.as_bytes_mut().fill(0); + folio + }, rx_head: None, rx_tail: None, tx_tail: None, - rx_buffers: Box::new(core::array::from_fn(|_| PageExcl::alloc_order(2))), + rx_buffers: Box::new(core::array::from_fn(|_| FolioOwned::alloc_order(2))), tx_buffers: Box::new([const { None }; 32]), }; diff --git a/src/driver/virtio/virtio_blk.rs b/src/driver/virtio/virtio_blk.rs index c5a3c3d2..5dfed88a 100644 --- a/src/driver/virtio/virtio_blk.rs +++ b/src/driver/virtio/virtio_blk.rs @@ -3,7 +3,7 @@ use alloc::boxed::Box; use async_trait::async_trait; use eonix_hal::mm::ArchPhysAccess; use eonix_mm::address::{Addr, PAddr, PhysAccess}; -use eonix_mm::paging::PFN; +use eonix_mm::paging::{Folio as _, PFN}; use eonix_sync::Spin; use virtio_drivers::device::blk::VirtIOBlk; use virtio_drivers::transport::Transport; @@ -12,7 +12,7 @@ use virtio_drivers::Hal; use crate::io::Chunks; use crate::kernel::block::{BlockDeviceRequest, BlockRequestQueue}; use crate::kernel::constants::EIO; -use crate::kernel::mem::{Page, PageExt}; +use crate::kernel::mem::Folio; use crate::prelude::KResult; pub struct HAL; @@ -22,7 +22,7 @@ unsafe impl Hal for HAL { pages: usize, _direction: virtio_drivers::BufferDirection, ) -> (virtio_drivers::PhysAddr, core::ptr::NonNull<u8>) { - let page = Page::alloc_at_least(pages); + let page = Folio::alloc_at_least(pages); let ptr = page.get_ptr(); let pfn = page.into_raw(); @@ -40,7 +40,7 @@ unsafe impl Hal for HAL { unsafe { // SAFETY: The caller ensures that the pfn corresponds to a valid // page allocated by `dma_alloc`. - Page::from_raw(pfn); + Folio::from_raw(pfn); } 0 diff --git a/src/fs/fat32.rs b/src/fs/fat32.rs index b19c9908..c1feebdf 100644 --- a/src/fs/fat32.rs +++ b/src/fs/fat32.rs @@ -1,5 +1,4 @@ mod dir; -mod file; use alloc::sync::Arc; use core::ops::Deref; @@ -13,7 +12,7 @@ use itertools::Itertools; use crate::io::{Buffer, ByteBuffer, UninitBuffer}; use crate::kernel::block::{BlockDevice, BlockDeviceRequest}; use crate::kernel::constants::{EINVAL, EIO}; -use crate::kernel::mem::{CachePage, Page, PageExcl, PageExt, PageOffset}; +use crate::kernel::mem::{CachePage, Folio, FolioOwned, PageOffset}; use crate::kernel::timer::Instant; use crate::kernel::vfs::dentry::Dentry; use crate::kernel::vfs::inode::{Ino, InodeInfo, InodeOps, InodeUse}; @@ -114,7 +113,7 @@ struct FatFs { impl SuperBlock for FatFs {} impl FatFs { - async fn read_cluster(&self, mut cluster: Cluster, buf: &Page) -> KResult<()> { + async fn read_cluster(&self, mut cluster: Cluster, buf: &Folio) -> KResult<()> { cluster = cluster.normalized(); let rq = BlockDeviceRequest::Read { @@ -278,7 +277,6 @@ impl InodeOps for FileInode { .next() .ok_or(EIO)?; - let page = page.get_page(); fs.read_cluster(cluster, &page).await?; let real_len = (inode.info.lock().size as usize) - offset.byte_count(); @@ -293,7 +291,7 @@ impl InodeOps for FileInode { struct DirInode { // TODO: Use the new PageCache... - dir_pages: RwLock<Vec<PageExcl>>, + dir_pages: RwLock<Vec<FolioOwned>>, } impl DirInode { @@ -330,7 +328,7 @@ impl DirInode { let clusters = ClusterIterator::new(fat.as_ref(), Cluster::from_ino(inode.ino)); for cluster in clusters { - let page = PageExcl::alloc(); + let page = FolioOwned::alloc(); fs.read_cluster(cluster, &page).await?; dir_pages.push(page); @@ -343,7 +341,7 @@ impl DirInode { &self, sb: &SbUse<FatFs>, inode: &InodeUse, - ) -> KResult<impl Deref<Target = Vec<PageExcl>> + use<'_>> { + ) -> KResult<impl Deref<Target = Vec<FolioOwned>> + use<'_>> { { let dir_pages = self.dir_pages.read().await; if !dir_pages.is_empty() { diff --git a/src/fs/fat32/file.rs b/src/fs/fat32/file.rs deleted file mode 100644 index 2df69728..00000000 --- a/src/fs/fat32/file.rs +++ /dev/null @@ -1,24 +0,0 @@ -use futures::Stream; - -use crate::{kernel::mem::Page, prelude::KResult}; - -use super::{ClusterIterator, FatFs}; - -pub trait ReadClusters { - fn read_clusters(self, fs: &FatFs) -> impl Stream<Item = KResult<Page>> + Send; -} - -impl ReadClusters for ClusterIterator<'_> { - fn read_clusters(self, fs: &FatFs) -> impl Stream<Item = KResult<Page>> + Send { - futures::stream::unfold(self, move |mut me| async { - let cluster = me.next()?; - let page = Page::alloc(); - - if let Err(err) = fs.read_cluster(cluster, &page).await { - return Some((Err(err), me)); - } - - Some((Ok(page), me)) - }) - } -} diff --git a/src/fs/tmpfs/file.rs b/src/fs/tmpfs/file.rs index a1755908..d560a672 100644 --- a/src/fs/tmpfs/file.rs +++ b/src/fs/tmpfs/file.rs @@ -125,7 +125,7 @@ impl InodeOps for FileInode { page: &mut CachePage, _: PageOffset, ) -> KResult<()> { - page.as_bytes_mut().fill(0); + page.lock().as_bytes_mut().fill(0); Ok(()) } diff --git a/src/kernel/block.rs b/src/kernel/block.rs index 8e017336..be2146f8 100644 --- a/src/kernel/block.rs +++ b/src/kernel/block.rs @@ -8,8 +8,7 @@ use async_trait::async_trait; use mbr::MBRPartTable; use super::constants::ENOENT; -use super::mem::paging::Page; -use super::mem::PageExt; +use super::mem::Folio; use super::vfs::types::DeviceId; use crate::io::{Buffer, Chunks, FillResult}; use crate::kernel::constants::{EEXIST, EINVAL}; @@ -202,15 +201,15 @@ impl BlockDevice { let (page_slice, page, mut page_vec); match nr_batch { ..=8 => { - page = Page::alloc(); + page = Folio::alloc(); page_slice = core::slice::from_ref(&page); } ..=16 => { - page = Page::alloc_order(1); + page = Folio::alloc_order(1); page_slice = core::slice::from_ref(&page); } ..=32 => { - page = Page::alloc_order(2); + page = Folio::alloc_order(2); page_slice = core::slice::from_ref(&page); } count => { @@ -220,8 +219,8 @@ impl BlockDevice { let nr_pages = nr_huge_pages + nr_small_pages; page_vec = Vec::with_capacity(nr_pages); - page_vec.resize_with(nr_huge_pages, || Page::alloc_order(2)); - page_vec.resize_with(nr_pages, || Page::alloc()); + page_vec.resize_with(nr_huge_pages, || Folio::alloc_order(2)); + page_vec.resize_with(nr_pages, || Folio::alloc()); page_slice = &page_vec; } } @@ -266,7 +265,7 @@ pub enum BlockDeviceRequest<'lt> { /// Number of sectors to read count: u64, /// Buffer pages to read into - buffer: &'lt [Page], + buffer: &'lt [Folio], }, Write { /// Sector to write to, in 512-byte blocks @@ -274,6 +273,6 @@ pub enum BlockDeviceRequest<'lt> { /// Number of sectors to write count: u64, /// Buffer pages to write from - buffer: &'lt [Page], + buffer: &'lt [Folio], }, } diff --git a/src/kernel/mem.rs b/src/kernel/mem.rs index f8b5dc0b..47b864bb 100644 --- a/src/kernel/mem.rs +++ b/src/kernel/mem.rs @@ -3,14 +3,16 @@ pub mod paging; mod access; mod address; mod allocator; +mod folio; mod mm_area; mod mm_list; mod page_alloc; mod page_cache; pub use access::PhysAccess; +pub use folio::{Folio, FolioOwned, LockedFolio}; pub(self) use mm_area::MMArea; pub use mm_list::{handle_kernel_page_fault, FileMapping, MMList, Mapping, Permission}; pub use page_alloc::{GlobalPageAlloc, RawPage}; pub use page_cache::{CachePage, PageCache, PageOffset}; -pub use paging::{Page, PageBuffer, PageExcl, PageExt}; +pub use paging::PageBuffer; diff --git a/src/kernel/mem/allocator.rs b/src/kernel/mem/allocator.rs index a3676ce0..3a70a8c2 100644 --- a/src/kernel/mem/allocator.rs +++ b/src/kernel/mem/allocator.rs @@ -3,11 +3,12 @@ use core::ptr::NonNull; use eonix_hal::mm::ArchPhysAccess; use eonix_mm::address::PhysAccess; -use eonix_mm::paging::{PAGE_SIZE_BITS, PFN}; +use eonix_mm::paging::{Folio as _, PAGE_SIZE_BITS, PFN}; use eonix_sync::LazyLock; use slab_allocator::SlabAlloc; -use super::{GlobalPageAlloc, Page, PageExt}; +use super::folio::Folio; +use super::GlobalPageAlloc; static SLAB_ALLOCATOR: LazyLock<SlabAlloc<GlobalPageAlloc, 9>> = LazyLock::new(|| SlabAlloc::new_in(GlobalPageAlloc)); @@ -18,19 +19,15 @@ unsafe impl GlobalAlloc for Allocator { unsafe fn alloc(&self, layout: Layout) -> *mut u8 { let size = layout.size().next_power_of_two(); - let result = if size <= 2048 { - SLAB_ALLOCATOR.alloc(size) + if size <= 2048 { + SLAB_ALLOCATOR.alloc(size).as_ptr() } else { - let page_count = size >> PAGE_SIZE_BITS; - let page = Page::alloc_at_least(page_count); - - let ptr = page.get_ptr(); - page.into_raw(); + let folio = Folio::alloc_at_least(size >> PAGE_SIZE_BITS); + let ptr = folio.get_ptr(); + folio.into_raw(); - ptr - }; - - result.as_ptr() + ptr.as_ptr() + } } unsafe fn dealloc(&self, ptr: *mut u8, layout: Layout) { @@ -45,7 +42,8 @@ unsafe impl GlobalAlloc for Allocator { } else { let paddr = ArchPhysAccess::from_ptr(ptr); let pfn = PFN::from(paddr); - Page::from_raw(pfn); + + Folio::from_raw(pfn); }; } } diff --git a/src/kernel/mem/folio.rs b/src/kernel/mem/folio.rs new file mode 100644 index 00000000..6647e1af --- /dev/null +++ b/src/kernel/mem/folio.rs @@ -0,0 +1,210 @@ +use core::fmt; +use core::mem::ManuallyDrop; +use core::ops::Deref; +use core::ptr::NonNull; +use core::sync::atomic::Ordering; + +use eonix_mm::paging::{Folio as FolioTrait, FrameAlloc, GlobalFrameAlloc, Zone, PFN}; + +use super::page_alloc::ZONE; +use super::{GlobalPageAlloc, PhysAccess as _, RawPage}; + +#[repr(transparent)] +pub struct Folio(NonNull<RawPage>); + +#[derive(Debug)] +#[repr(transparent)] +pub struct FolioOwned(Folio); + +#[repr(transparent)] +pub struct LockedFolio<'a>(&'a Folio); + +unsafe impl Send for Folio {} +unsafe impl Sync for Folio {} + +impl Folio { + pub(super) const fn from_mut_page(raw_page: &'static mut RawPage) -> Self { + Self(NonNull::new(raw_page).unwrap()) + } + + /// Allocate a folio of the given *order*. + pub fn alloc_order(order: u32) -> Self { + GlobalPageAlloc::GLOBAL + .alloc_order(order) + .expect("Out of memory") + } + + /// Allocate a folio of order 0 + pub fn alloc() -> Self { + Self::alloc_order(0) + } + + /// Allocate a folio consisting of at least [`count`] pages. + pub fn alloc_at_least(count: usize) -> Self { + GlobalPageAlloc::GLOBAL + .alloc_at_least(count) + .expect("Out of memory") + } + + /// Acquire the ownership of the folio pointed to by [`pfn`], leaving + /// [`refcount`] untouched. + /// + /// # Panic + /// This function will panic if the folio is not within the global zone. + /// + /// # Safety + /// This function is unsafe because it assumes that the caller has to ensure + /// that [`pfn`] points to a valid folio allocated through [`Self::alloc()`] + /// and that the folio have not been freed or deallocated yet. + pub unsafe fn from_raw(pfn: PFN) -> Self { + unsafe { + // SAFETY: The caller ensures that [`pfn`] points to a folio within + // the global zone. + Self(ZONE.get_page(pfn).unwrap_unchecked()) + } + } + + /// Do some work with the folio without touching the reference count with + /// the same restrictions as [`Self::from_raw()`]. + /// + /// # Safety + /// Check [`Self::from_raw()`] for safety requirements. + pub unsafe fn with_raw<F, O>(pfn: PFN, func: F) -> O + where + F: FnOnce(&Self) -> O, + { + unsafe { + let me = ManuallyDrop::new(Self::from_raw(pfn)); + func(&me) + } + } + + pub fn lock(&self) -> LockedFolio { + // TODO: actually perform the lock... + LockedFolio(self) + } + + /// Get a vmem pointer to the folio data as a byte slice. + pub fn get_bytes_ptr(&self) -> NonNull<[u8]> { + unsafe { + // SAFETY: `self.start()` can't be null. + NonNull::slice_from_raw_parts(self.start().as_ptr(), self.len()) + } + } + + /// Get a vmem pointer to the start of the folio. + pub fn get_ptr(&self) -> NonNull<u8> { + self.get_bytes_ptr().cast() + } +} + +impl Deref for Folio { + type Target = RawPage; + + fn deref(&self) -> &Self::Target { + unsafe { + // SAFETY: We don't expose mutable references to the folio. + self.0.as_ref() + } + } +} + +impl Clone for Folio { + fn clone(&self) -> Self { + // SAFETY: Memory order here can be Relaxed is for the same reason as + // that in the copy constructor of `std::shared_ptr`. + self.refcount.fetch_add(1, Ordering::Relaxed); + + Self(self.0) + } +} + +impl Drop for Folio { + fn drop(&mut self) { + match self.refcount.fetch_sub(1, Ordering::AcqRel) { + 0 => unreachable!("Refcount for an in-use page is 0"), + 1 => unsafe { GlobalPageAlloc::GLOBAL.dealloc_raw(self.0.as_mut()) }, + _ => {} + } + } +} + +impl fmt::Debug for Folio { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "Page({:?}, order={})", self.pfn(), self.order) + } +} + +impl FolioTrait for Folio { + fn pfn(&self) -> PFN { + ZONE.get_pfn(self.0.as_ptr()) + } + + fn order(&self) -> u32 { + self.order + } +} + +impl LockedFolio<'_> { + pub fn as_bytes(&self) -> &[u8] { + unsafe { + // SAFETY: `self.start()` points to valid memory of length `self.len()`. + core::slice::from_raw_parts(self.start().as_ptr().as_ptr(), self.len()) + } + } + + pub fn as_bytes_mut(&mut self) -> &mut [u8] { + unsafe { + // SAFETY: `self.start()` points to valid memory of length `self.len()`. + core::slice::from_raw_parts_mut(self.start().as_ptr().as_ptr(), self.len()) + } + } +} + +impl Deref for LockedFolio<'_> { + type Target = Folio; + + fn deref(&self) -> &Self::Target { + self.0 + } +} + +impl FolioOwned { + pub fn alloc() -> Self { + Self(Folio::alloc()) + } + + pub fn alloc_order(order: u32) -> Self { + Self(Folio::alloc_order(order)) + } + + pub fn alloc_at_least(count: usize) -> Self { + Self(Folio::alloc_at_least(count)) + } + + pub fn as_bytes(&self) -> &[u8] { + unsafe { + // SAFETY: The page is exclusively owned by us. + self.get_bytes_ptr().as_ref() + } + } + + pub fn as_bytes_mut(&mut self) -> &mut [u8] { + unsafe { + // SAFETY: The page is exclusively owned by us. + self.get_bytes_ptr().as_mut() + } + } + + pub fn share(self) -> Folio { + self.0 + } +} + +impl Deref for FolioOwned { + type Target = Folio; + + fn deref(&self) -> &Self::Target { + &self.0 + } +} diff --git a/src/kernel/mem/mm_area.rs b/src/kernel/mem/mm_area.rs index 2891dad8..782c5ef7 100644 --- a/src/kernel/mem/mm_area.rs +++ b/src/kernel/mem/mm_area.rs @@ -1,15 +1,16 @@ use core::borrow::Borrow; use core::cell::UnsafeCell; use core::cmp; +use core::sync::atomic::Ordering; use eonix_mm::address::{AddrOps as _, VAddr, VRange}; use eonix_mm::page_table::{PageAttribute, RawAttribute, PTE}; -use eonix_mm::paging::PFN; +use eonix_mm::paging::{Folio as _, PFN}; use super::mm_list::EMPTY_PAGE; -use super::{Mapping, Page, Permission}; -use crate::kernel::mem::page_cache::PageOffset; -use crate::kernel::mem::{CachePage, PageExcl, PageExt}; +use super::{Mapping, Permission}; +use crate::kernel::mem::folio::Folio; +use crate::kernel::mem::{CachePage, FolioOwned, PageOffset}; use crate::prelude::KResult; #[derive(Debug)] @@ -98,8 +99,10 @@ impl MMArea { attr.remove(PageAttribute::COPY_ON_WRITE); attr.set(PageAttribute::WRITE, self.permission.write); - let page = unsafe { Page::from_raw(*pfn) }; - if page.is_exclusive() { + let page = unsafe { Folio::from_raw(*pfn) }; + + // XXX: Change me!!! + if page.refcount.load(Ordering::Relaxed) == 1 { // SAFETY: This is actually safe. If we read `1` here and we have `MMList` lock // held, there couldn't be neither other processes sharing the page, nor other // threads making the page COW at the same time. @@ -109,9 +112,13 @@ impl MMArea { let mut new_page; if *pfn == EMPTY_PAGE.pfn() { - new_page = PageExcl::zeroed(); + new_page = { + let mut folio = FolioOwned::alloc(); + folio.as_bytes_mut().fill(0); + folio + }; } else { - new_page = PageExcl::alloc(); + new_page = FolioOwned::alloc(); unsafe { // SAFETY: `page` is CoW, which means that others won't write to it. @@ -123,7 +130,7 @@ impl MMArea { } attr.remove(PageAttribute::ACCESSED); - *pfn = new_page.into_page().into_raw(); + *pfn = new_page.share().into_raw(); } /// # Arguments @@ -143,11 +150,11 @@ impl MMArea { let file_offset = file_mapping.offset + offset; - let map_page = |page: &Page, cache_page: &CachePage| { + let map_page = |cache_page: &CachePage| { if !self.permission.write { assert!(!write, "Write fault on read-only mapping"); - *pfn = page.clone().into_raw(); + *pfn = cache_page.add_mapping(); return; } @@ -157,26 +164,26 @@ impl MMArea { // So here we can set the dirty flag now. cache_page.set_dirty(true); attr.insert(PageAttribute::WRITE); - *pfn = page.clone().into_raw(); + *pfn = cache_page.add_mapping(); return; } if !write { // Delay the copy-on-write until write fault happens. attr.insert(PageAttribute::COPY_ON_WRITE); - *pfn = page.clone().into_raw(); + *pfn = cache_page.add_mapping(); return; } // XXX: Change this. Let's handle mapped pages before CoW pages. // Nah, we are writing to a mapped private mapping... - let mut new_page = PageExcl::zeroed(); + let mut new_page = FolioOwned::alloc(); new_page .as_bytes_mut() - .copy_from_slice(page.lock().as_bytes()); + .copy_from_slice(cache_page.lock().as_bytes()); attr.insert(PageAttribute::WRITE); - *pfn = new_page.into_page().into_raw(); + *pfn = new_page.share().into_raw(); }; file_mapping diff --git a/src/kernel/mem/mm_list.rs b/src/kernel/mem/mm_list.rs index 17dc1b05..5221c73b 100644 --- a/src/kernel/mem/mm_list.rs +++ b/src/kernel/mem/mm_list.rs @@ -1,31 +1,33 @@ mod mapping; mod page_fault; +mod page_table; use alloc::collections::btree_set::BTreeSet; use core::fmt; use core::sync::atomic::{AtomicUsize, Ordering}; use eonix_hal::mm::{ - flush_tlb_all, get_root_page_table_pfn, set_root_page_table_pfn, ArchPagingMode, - ArchPhysAccess, GLOBAL_PAGE_TABLE, + flush_tlb_all, get_root_page_table_pfn, set_root_page_table_pfn, GLOBAL_PAGE_TABLE, }; use eonix_mm::address::{Addr as _, AddrOps as _, PAddr, VAddr, VRange}; -use eonix_mm::page_table::{PageAttribute, PageTable, RawAttribute, PTE}; -use eonix_mm::paging::{PAGE_SIZE, PFN}; +use eonix_mm::page_table::{PageAttribute, RawAttribute, PTE}; +use eonix_mm::paging::{Folio as _, PAGE_SIZE, PFN}; use eonix_sync::{LazyLock, Mutex}; pub use mapping::{FileMapping, Mapping}; pub use page_fault::handle_kernel_page_fault; +use page_table::KernelPageTable; use super::address::{VAddrExt as _, VRangeExt as _}; -use super::page_alloc::GlobalPageAlloc; -use super::paging::AllocZeroed as _; -use super::{MMArea, Page, PageExt}; +use super::{Folio, FolioOwned, MMArea}; use crate::kernel::constants::{EEXIST, EFAULT, EINVAL, ENOMEM}; -use crate::kernel::mem::page_alloc::RawPagePtr; use crate::prelude::*; use crate::sync::ArcSwap; -pub static EMPTY_PAGE: LazyLock<Page> = LazyLock::new(|| Page::zeroed()); +pub static EMPTY_PAGE: LazyLock<Folio> = LazyLock::new(|| { + let mut folio = FolioOwned::alloc(); + folio.as_bytes_mut().fill(0); + folio.share() +}); #[derive(Debug, Clone, Copy)] pub struct Permission { @@ -34,23 +36,21 @@ pub struct Permission { pub execute: bool, } -pub type KernelPageTable<'a> = PageTable<'a, ArchPagingMode, GlobalPageAlloc, ArchPhysAccess>; - -struct MMListInner<'a> { +struct MMListInner { areas: BTreeSet<MMArea>, - page_table: KernelPageTable<'a>, + page_table: KernelPageTable, break_start: Option<VRange>, break_pos: Option<VAddr>, } pub struct MMList { - inner: ArcSwap<Mutex<MMListInner<'static>>>, + inner: ArcSwap<Mutex<MMListInner>>, user_count: AtomicUsize, /// Only used in kernel space to switch page tables on context switch. root_page_table: AtomicUsize, } -impl MMListInner<'_> { +impl MMListInner { fn overlapping_addr(&self, addr: VAddr) -> Option<&MMArea> { self.areas.get(&VRange::from(addr)) } @@ -96,7 +96,7 @@ impl MMListInner<'_> { } } - fn unmap(&mut self, start: VAddr, len: usize) -> KResult<Vec<Page>> { + fn unmap(&mut self, start: VAddr, len: usize) -> KResult<Vec<Folio>> { assert_eq!(start.floor(), start); let end = (start + len).ceil(); let range_to_unmap = VRange::new(start, end); @@ -120,7 +120,7 @@ impl MMListInner<'_> { let (pfn, _) = pte.take(); pages_to_free.push(unsafe { // SAFETY: We got the pfn from a valid page table entry, so it should be valid. - Page::from_raw(pfn) + Folio::from_raw(pfn) }); } @@ -275,23 +275,23 @@ impl MMListInner<'_> { } } -impl Drop for MMListInner<'_> { +impl Drop for MMListInner { fn drop(&mut self) { // May buggy for area in &self.areas { if area.is_shared { for pte in self.page_table.iter_user(area.range()) { - let (pfn, _) = pte.take(); - let raw_page = RawPagePtr::from(pfn); - if raw_page.refcount().fetch_sub(1, Ordering::Relaxed) == 1 { - // Wrong here - // unsafe { Page::from_raw(pfn) }; - } + // XXX: Fix me + let _ = pte.take(); + // let raw_page = RawPagePtr::from(pfn); + // if raw_page.refcount().fetch_sub(1, Ordering::Relaxed) == 1 { + // unsafe { Page::from_raw(pfn) }; + // } } } else { for pte in self.page_table.iter_user(area.range()) { let (pfn, _) = pte.take(); - unsafe { Page::from_raw(pfn) }; + unsafe { Folio::from_raw(pfn) }; } } } @@ -327,7 +327,7 @@ impl MMList { } pub fn new() -> Self { - let page_table = GLOBAL_PAGE_TABLE.clone_global(); + let page_table = KernelPageTable::new(); Self { root_page_table: AtomicUsize::from(page_table.addr().addr()), user_count: AtomicUsize::new(0), @@ -344,7 +344,7 @@ impl MMList { let inner = self.inner.borrow(); let mut inner = inner.lock().await; - let page_table = GLOBAL_PAGE_TABLE.clone_global(); + let page_table = KernelPageTable::new(); let list = Self { root_page_table: AtomicUsize::from(page_table.addr().addr()), user_count: AtomicUsize::new(0), @@ -392,7 +392,7 @@ impl MMList { } pub fn deactivate(&self) { - set_root_page_table_pfn(PFN::from(GLOBAL_PAGE_TABLE.addr())); + set_root_page_table_pfn(PFN::from(GLOBAL_PAGE_TABLE.start())); let old_user_count = self.user_count.fetch_sub(1, Ordering::Release); assert_ne!(old_user_count, 0); @@ -444,7 +444,7 @@ impl MMList { let new_root_page_table = match &new { Some(new_mm) => new_mm.root_page_table.load(Ordering::Relaxed), - None => GLOBAL_PAGE_TABLE.addr().addr(), + None => GLOBAL_PAGE_TABLE.start().addr(), }; set_root_page_table_pfn(PFN::from(PAddr::from(new_root_page_table))); @@ -693,7 +693,7 @@ impl MMList { unsafe { // SAFETY: We are sure that the page is valid and we have the right to access it. - Page::with_raw(pte.get_pfn(), |page| { + Folio::with_raw(pte.get_pfn(), |page| { let mut pg = page.lock(); let page_data = &mut pg.as_bytes_mut()[start_offset..end_offset]; @@ -724,7 +724,7 @@ trait PageTableExt { fn set_copied(&self, from: &Self, range: VRange); } -impl PageTableExt for KernelPageTable<'_> { +impl PageTableExt for KernelPageTable { fn set_anonymous(&self, range: VRange, permission: Permission) { for pte in self.iter_user(range) { pte.set_anonymous(permission.execute); @@ -805,7 +805,7 @@ where let pfn = unsafe { // SAFETY: We get the pfn from a valid page table entry, so it should be valid as well. - Page::with_raw(from.get_pfn(), |page| page.clone().into_raw()) + Folio::with_raw(from.get_pfn(), |page| page.clone().into_raw()) }; self.set(pfn, T::Attr::from(from_attr & !PageAttribute::ACCESSED)); diff --git a/src/kernel/mem/mm_list/page_table.rs b/src/kernel/mem/mm_list/page_table.rs new file mode 100644 index 00000000..8a2acc13 --- /dev/null +++ b/src/kernel/mem/mm_list/page_table.rs @@ -0,0 +1,40 @@ +use core::ops::Deref; + +use eonix_hal::arch_exported::mm::{ArchPagingMode, PageAccessImpl}; +use eonix_hal::mm::GLOBAL_PAGE_TABLE; +use eonix_mm::page_table::PageTable; +use eonix_mm::paging::{Folio, GlobalFrameAlloc}; + +use crate::kernel::mem::{FolioOwned, GlobalPageAlloc, PhysAccess}; + +#[repr(transparent)] +pub struct KernelPageTable(PageTable<'static, ArchPagingMode, GlobalPageAlloc, PageAccessImpl>); + +impl KernelPageTable { + pub fn new() -> Self { + let global_page_table = unsafe { + // SAFETY: The region is valid and read only after initialization. + GLOBAL_PAGE_TABLE.start().as_ptr::<[u8; 4096]>().as_ref() + }; + + let mut table_page = FolioOwned::alloc(); + let entries = table_page.as_bytes_mut().len(); + table_page.as_bytes_mut()[..(entries / 2)].fill(0); + table_page.as_bytes_mut()[(entries / 2)..] + .copy_from_slice(&global_page_table[(entries / 2)..]); + + Self(PageTable::new( + table_page.share(), + GlobalPageAlloc::GLOBAL, + PageAccessImpl, + )) + } +} + +impl Deref for KernelPageTable { + type Target = PageTable<'static, ArchPagingMode, GlobalPageAlloc, PageAccessImpl>; + + fn deref(&self) -> &Self::Target { + &self.0 + } +} diff --git a/src/kernel/mem/page_alloc.rs b/src/kernel/mem/page_alloc.rs index 37344fc2..ac2485da 100644 --- a/src/kernel/mem/page_alloc.rs +++ b/src/kernel/mem/page_alloc.rs @@ -5,14 +5,14 @@ use core::sync::atomic::Ordering; use buddy_allocator::BuddyAllocator; use eonix_mm::address::PRange; -use eonix_mm::paging::{ - GlobalPageAlloc as GlobalPageAllocTrait, PageAlloc, PageList, PageListSized as _, -}; +use eonix_mm::page_table::PageTableAlloc; +use eonix_mm::paging::{FolioList, FolioListSized as _, FrameAlloc, GlobalFrameAlloc, PFN}; use eonix_preempt::PreemptGuard; use eonix_sync::{NoContext, Spin}; -use raw_page::{PageFlags, RawPageList}; -pub use raw_page::{RawPage, RawPagePtr}; -pub use zones::GlobalZone; +pub use raw_page::{PageFlags, RawPage, RawPageList}; +pub use zones::{GlobalZone, ZONE}; + +use super::folio::Folio; const COSTLY_ORDER: u32 = 3; const AREAS: usize = COSTLY_ORDER as usize + 1; @@ -27,9 +27,6 @@ static PERCPU_PAGE_ALLOC: PerCpuPageAlloc = PerCpuPageAlloc::new(); #[derive(Clone)] pub struct GlobalPageAlloc; -#[derive(Clone)] -pub struct BuddyPageAlloc(); - struct PerCpuPageAlloc { batch: u32, free_areas: [RawPageList; AREAS], @@ -72,11 +69,6 @@ impl PerCpuPageAlloc { } impl GlobalPageAlloc { - #[allow(dead_code)] - pub const fn buddy_alloc() -> BuddyPageAlloc { - BuddyPageAlloc() - } - /// Add the pages in the PAddr range `range` to the global allocator. /// /// This function is only to be called on system initialization when `eonix_preempt` @@ -88,15 +80,11 @@ impl GlobalPageAlloc { pub unsafe fn add_pages(range: PRange) { BUDDY_ALLOC .lock_with_context(NoContext) - .create_pages(range.start(), range.end()) + .create_folios(range.start(), range.end()) } -} - -impl PageAlloc for GlobalPageAlloc { - type RawPage = RawPagePtr; - fn alloc_order(&self, order: u32) -> Option<RawPagePtr> { - let raw_page = if order > COSTLY_ORDER { + pub fn alloc_raw_order(&self, order: u32) -> Option<&'static mut RawPage> { + if order > COSTLY_ORDER { BUDDY_ALLOC.lock().alloc_order(order) } else { unsafe { @@ -106,61 +94,54 @@ impl PageAlloc for GlobalPageAlloc { page } - }; - - raw_page.map(|raw_page| { - // SAFETY: Memory order here can be Relaxed is for the same reason - // as that in the copy constructor of `std::shared_ptr`. - raw_page.refcount.fetch_add(1, Ordering::Relaxed); - - RawPagePtr::from_ref(raw_page) - }) + } } - unsafe fn dealloc(&self, page_ptr: RawPagePtr) { + pub unsafe fn dealloc_raw(&self, raw_page: &'static mut RawPage) { assert_eq!( - page_ptr.refcount().load(Ordering::Relaxed), + raw_page.refcount.load(Ordering::Relaxed), 0, "Trying to free a page with refcount > 0" ); - if page_ptr.order() > COSTLY_ORDER { - BUDDY_ALLOC.lock().dealloc(page_ptr.as_mut()); + if raw_page.order > COSTLY_ORDER { + BUDDY_ALLOC.lock().dealloc(raw_page); } else { - let order = page_ptr.order(); + let order = raw_page.order; unsafe { - PreemptGuard::new(PERCPU_PAGE_ALLOC.as_mut()).free_pages(page_ptr.as_mut(), order); + PreemptGuard::new(PERCPU_PAGE_ALLOC.as_mut()).free_pages(raw_page, order); } } } - - fn has_management_over(&self, page_ptr: RawPagePtr) -> bool { - page_ptr.order() > COSTLY_ORDER || page_ptr.flags().has(PageFlags::LOCAL) - } } -impl GlobalPageAllocTrait for GlobalPageAlloc { - fn global() -> Self { - GlobalPageAlloc +impl FrameAlloc for GlobalPageAlloc { + type Folio = Folio; + + fn alloc_order(&self, order: u32) -> Option<Self::Folio> { + self.alloc_raw_order(order).map(|raw_page| { + // SAFETY: Memory order here can be Relaxed is for the same reason + // as that in the copy constructor of `std::shared_ptr`. + + raw_page.refcount.fetch_add(1, Ordering::Relaxed); + Folio::from_mut_page(raw_page) + }) } } -impl PageAlloc for BuddyPageAlloc { - type RawPage = RawPagePtr; +impl GlobalFrameAlloc for GlobalPageAlloc { + const GLOBAL: Self = GlobalPageAlloc; +} - fn alloc_order(&self, order: u32) -> Option<RawPagePtr> { - BUDDY_ALLOC - .lock() - .alloc_order(order) - .map(|raw_page| RawPagePtr::from_ref(raw_page)) - } +impl PageTableAlloc for GlobalPageAlloc { + type Folio = Folio; - unsafe fn dealloc(&self, page_ptr: RawPagePtr) { - BUDDY_ALLOC.lock().dealloc(page_ptr.as_mut()); + fn alloc(&self) -> Self::Folio { + FrameAlloc::alloc(self).unwrap() } - fn has_management_over(&self, _: RawPagePtr) -> bool { - true + unsafe fn from_raw(&self, pfn: PFN) -> Self::Folio { + unsafe { Folio::from_raw(pfn) } } } diff --git a/src/kernel/mem/page_alloc/raw_page.rs b/src/kernel/mem/page_alloc/raw_page.rs index 074f82c7..0d775245 100644 --- a/src/kernel/mem/page_alloc/raw_page.rs +++ b/src/kernel/mem/page_alloc/raw_page.rs @@ -1,20 +1,17 @@ use core::ptr::NonNull; use core::sync::atomic::{AtomicU32, AtomicUsize, Ordering}; -use buddy_allocator::BuddyPage; +use buddy_allocator::BuddyFolio; use eonix_hal::mm::ArchPhysAccess; use eonix_mm::address::{PAddr, PhysAccess as _}; -use eonix_mm::paging::{PageAlloc, PageList, PageListSized, RawPage as RawPageTrait, PFN}; +use eonix_mm::paging::{FolioList, FolioListSized, Zone, PFN}; use intrusive_list::{container_of, Link, List}; use slab_allocator::{SlabPage, SlabPageAlloc, SlabSlot}; +use super::zones::ZONE; use super::{GlobalPageAlloc, PerCpuPage}; -use crate::kernel::mem::page_cache::PageCacheRawPage; use crate::kernel::mem::PhysAccess; -pub const PAGE_ARRAY: NonNull<RawPage> = - unsafe { NonNull::new_unchecked(0xffffff8040000000 as *mut _) }; - pub struct PageFlags(AtomicU32); #[derive(Clone, Copy)] @@ -41,11 +38,11 @@ pub struct RawPage { /// This can be used for LRU page swap in the future. /// /// Now only used for free page links in the buddy system. - link: Link, + pub link: Link, /// # Safety /// This field is only used in buddy system and is protected by the global lock. - order: u32, - flags: PageFlags, + pub order: u32, + pub flags: PageFlags, pub refcount: AtomicUsize, shared_data: PageData, @@ -55,9 +52,6 @@ pub struct RawPage { unsafe impl Send for RawPage {} unsafe impl Sync for RawPage {} -#[derive(Copy, Clone, PartialEq, Eq, PartialOrd, Ord)] -pub struct RawPagePtr(NonNull<RawPage>); - impl PageFlags { pub const LOCKED: u32 = 1 << 1; pub const BUDDY: u32 = 1 << 2; @@ -85,80 +79,9 @@ impl PageFlags { } } -impl RawPagePtr { - pub const fn from_ref(raw_page_ref: &RawPage) -> Self { - Self::new(unsafe { - // SAFETY: Rust references always points to non-null addresses. - NonNull::new_unchecked(&raw const *raw_page_ref as *mut _) - }) - } - - pub const fn new(ptr: NonNull<RawPage>) -> Self { - Self(ptr) - } - - /// Get a raw pointer to the underlying `RawPage` struct. - /// - /// # Safety - /// Doing arithmetic on the pointer returned will cause immediate undefined behavior. - pub const unsafe fn as_ptr(self) -> *mut RawPage { - self.0.as_ptr() - } - - pub const fn as_ref<'a>(self) -> &'a RawPage { - unsafe { &*self.as_ptr() } - } - - pub const fn as_mut<'a>(self) -> &'a mut RawPage { - unsafe { &mut *self.as_ptr() } - } - - pub const fn order(&self) -> u32 { - self.as_ref().order - } - - pub const fn flags(&self) -> &PageFlags { - &self.as_ref().flags - } - - pub const fn refcount(&self) -> &AtomicUsize { - &self.as_ref().refcount - } - - // return the ptr point to the actually raw page - pub fn real_ptr<T>(&self) -> NonNull<T> { - let pfn = unsafe { PFN::from(RawPagePtr(NonNull::new_unchecked(self.as_ptr()))) }; - unsafe { PAddr::from(pfn).as_ptr::<T>() } - } -} - -impl From<RawPagePtr> for PFN { - fn from(value: RawPagePtr) -> Self { - let idx = unsafe { value.as_ptr().offset_from(PAGE_ARRAY.as_ptr()) as usize }; - Self::from(idx) - } -} - -impl From<PFN> for RawPagePtr { - fn from(pfn: PFN) -> Self { - let raw_page_ptr = unsafe { PAGE_ARRAY.add(usize::from(pfn)) }; - Self::new(raw_page_ptr) - } -} - -impl RawPageTrait for RawPagePtr { - fn order(&self) -> u32 { - self.order() - } - - fn refcount(&self) -> &AtomicUsize { - self.refcount() - } -} - -impl BuddyPage for RawPage { +impl BuddyFolio for RawPage { fn pfn(&self) -> PFN { - PFN::from(RawPagePtr::from_ref(self)) + ZONE.get_pfn(self) } fn get_order(&self) -> u32 { @@ -184,8 +107,7 @@ impl BuddyPage for RawPage { impl SlabPage for RawPage { fn get_data_ptr(&self) -> NonNull<[u8]> { - let raw_page_ptr = RawPagePtr::from_ref(self); - let paddr_start = PAddr::from(PFN::from(raw_page_ptr)); + let paddr_start = PAddr::from(ZONE.get_pfn(self)); let page_data_ptr = unsafe { paddr_start.as_ptr() }; NonNull::slice_from_raw_parts(page_data_ptr, 1 << (self.order + 12)) @@ -233,21 +155,9 @@ impl SlabPage for RawPage { let paddr = ArchPhysAccess::from_ptr(ptr); let pfn = PFN::from(paddr); - RawPagePtr::from(pfn).as_mut() - } - } -} - -impl PageCacheRawPage for RawPagePtr { - fn is_dirty(&self) -> bool { - self.flags().has(PageFlags::DIRTY) - } - - fn set_dirty(&self, dirty: bool) { - if dirty { - self.flags().set(PageFlags::DIRTY); - } else { - self.flags().clear(PageFlags::DIRTY); + ZONE.get_page(pfn) + .expect("Page outside of the global zone") + .as_mut() } } } @@ -264,14 +174,16 @@ impl PerCpuPage for RawPage { pub struct RawPageList(List); -impl PageList for RawPageList { - type Page = RawPage; +unsafe impl Send for RawPageList {} + +impl FolioList for RawPageList { + type Folio = RawPage; fn is_empty(&self) -> bool { self.0.is_empty() } - fn peek_head(&mut self) -> Option<&mut Self::Page> { + fn peek_head(&mut self) -> Option<&mut Self::Folio> { unsafe { let link = self.0.head()?; let mut raw_page_ptr = container_of!(link, RawPage, link); @@ -280,7 +192,7 @@ impl PageList for RawPageList { } } - fn pop_head(&mut self) -> Option<&'static mut Self::Page> { + fn pop_head(&mut self) -> Option<&'static mut Self::Folio> { unsafe { let link = self.0.pop()?; let mut raw_page_ptr = container_of!(link, RawPage, link); @@ -289,25 +201,25 @@ impl PageList for RawPageList { } } - fn push_tail(&mut self, page: &'static mut Self::Page) { + fn push_tail(&mut self, page: &'static mut Self::Folio) { self.0.insert(&mut page.link); } - fn remove(&mut self, page: &mut Self::Page) { + fn remove(&mut self, page: &mut Self::Folio) { self.0.remove(&mut page.link) } } -impl PageListSized for RawPageList { +impl FolioListSized for RawPageList { const NEW: Self = RawPageList(List::new()); } -impl SlabPageAlloc for GlobalPageAlloc { +unsafe impl SlabPageAlloc for GlobalPageAlloc { type Page = RawPage; type PageList = RawPageList; - unsafe fn alloc_uninit(&self) -> &'static mut RawPage { - let raw_page = self.alloc().expect("Out of memory").as_mut(); + fn alloc_slab_page(&self) -> &'static mut RawPage { + let raw_page = self.alloc_raw_order(0).expect("Out of memory"); raw_page.flags.set(PageFlags::SLAB); raw_page.shared_data.slab = SlabPageData::new(); diff --git a/src/kernel/mem/page_alloc/zones.rs b/src/kernel/mem/page_alloc/zones.rs index 7a2e4e33..032b9cd0 100644 --- a/src/kernel/mem/page_alloc/zones.rs +++ b/src/kernel/mem/page_alloc/zones.rs @@ -1,13 +1,23 @@ -use core::cell::UnsafeCell; +use core::ptr::NonNull; use eonix_mm::address::PRange; use eonix_mm::paging::{Zone, PFN}; use super::RawPage; -use crate::kernel::mem::page_alloc::RawPagePtr; + +pub static ZONE: GlobalZone = GlobalZone(); + +const PAGE_ARRAY: NonNull<RawPage> = + unsafe { NonNull::new_unchecked(0xffffff8040000000 as *mut _) }; pub struct GlobalZone(); +impl GlobalZone { + pub fn get_pfn(&self, page_ptr: *const RawPage) -> PFN { + PFN::from(unsafe { page_ptr.offset_from(PAGE_ARRAY.as_ptr()) as usize }) + } +} + impl Zone for GlobalZone { type Page = RawPage; @@ -15,11 +25,7 @@ impl Zone for GlobalZone { true } - fn get_page(&self, pfn: PFN) -> Option<&UnsafeCell<Self::Page>> { - unsafe { - // SAFETY: The pointer returned by [`RawPagePtr::as_ptr()`] is valid. - // And so is it wrapped with [`UnsafeCell`] - Some(&*(RawPagePtr::from(pfn).as_ptr() as *const UnsafeCell<Self::Page>)) - } + fn get_page(&self, pfn: PFN) -> Option<NonNull<RawPage>> { + Some(unsafe { PAGE_ARRAY.add(usize::from(pfn)) }) } } diff --git a/src/kernel/mem/page_cache.rs b/src/kernel/mem/page_cache.rs index 214c65a5..3fe33d5b 100644 --- a/src/kernel/mem/page_cache.rs +++ b/src/kernel/mem/page_cache.rs @@ -1,19 +1,16 @@ use alloc::collections::btree_map::{BTreeMap, Entry}; use core::future::Future; -use core::mem::ManuallyDrop; +use core::ops::{Deref, DerefMut}; -use eonix_hal::mm::ArchPhysAccess; -use eonix_mm::address::{PAddr, PhysAccess}; -use eonix_mm::paging::{PageAlloc, RawPage, PAGE_SIZE, PAGE_SIZE_BITS, PFN}; +use eonix_mm::paging::{Folio as _, PAGE_SIZE, PAGE_SIZE_BITS, PFN}; use eonix_sync::Mutex; -use super::Page; +use super::page_alloc::PageFlags; +use super::{Folio, FolioOwned}; use crate::io::{Buffer, Stream}; use crate::kernel::constants::EINVAL; -use crate::kernel::mem::page_alloc::RawPagePtr; use crate::kernel::vfs::inode::InodeUse; use crate::prelude::KResult; -use crate::GlobalPageAlloc; #[repr(transparent)] #[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord)] @@ -24,14 +21,7 @@ pub struct PageCache { inode: InodeUse, } -unsafe impl Send for PageCache {} -unsafe impl Sync for PageCache {} - -#[derive(Clone, Copy)] -pub struct CachePage(RawPagePtr); - -unsafe impl Send for CachePage {} -unsafe impl Sync for CachePage {} +pub struct CachePage(Folio); impl PageOffset { pub const fn from_byte_floor(offset: usize) -> Self { @@ -57,39 +47,47 @@ impl PageOffset { impl CachePage { pub fn new() -> Self { - Self(GlobalPageAlloc.alloc().unwrap()) + CachePage(Folio::alloc()) } - pub fn as_bytes(&self) -> &[u8] { - unsafe { - core::slice::from_raw_parts( - // SAFETY: The page is owned by us, so we can safely access its data. - ArchPhysAccess::as_ptr(PAddr::from(PFN::from(self.0))).as_ptr(), - PAGE_SIZE, - ) - } - } + pub fn new_zeroed() -> Self { + CachePage({ + let mut folio = FolioOwned::alloc(); + folio.as_bytes_mut().fill(0); - pub fn as_bytes_mut(&mut self) -> &mut [u8] { - unsafe { - core::slice::from_raw_parts_mut( - // SAFETY: The page is exclusively owned by us, so we can safely access its data. - ArchPhysAccess::as_ptr(PAddr::from(PFN::from(self.0))).as_ptr(), - PAGE_SIZE, - ) - } + folio.share() + }) } pub fn is_dirty(&self) -> bool { - self.0.is_dirty() + self.flags.has(PageFlags::DIRTY) } pub fn set_dirty(&self, dirty: bool) { - self.0.set_dirty(dirty); + if dirty { + self.flags.set(PageFlags::DIRTY); + } else { + self.flags.clear(PageFlags::DIRTY); + } + } + + pub fn add_mapping(&self) -> PFN { + // TODO: Increase map_count + self.0.clone().into_raw() } +} + +impl Deref for CachePage { + type Target = Folio; - pub fn get_page(&self) -> Page { - unsafe { Page::with_raw(PFN::from(self.0), |page| page.clone()) } + fn deref(&self) -> &Self::Target { + &self.0 + } +} + +impl DerefMut for CachePage { + fn deref_mut(&mut self) -> &mut Self::Target { + &mut self.0 } } @@ -124,11 +122,7 @@ impl PageCache { } // TODO: Remove this. - pub async fn with_page( - &self, - pgoff: PageOffset, - func: impl FnOnce(&Page, &CachePage), - ) -> KResult<()> { + pub async fn with_page(&self, pgoff: PageOffset, func: impl FnOnce(&CachePage)) -> KResult<()> { let mut pages = self.pages.lock().await; if pgoff > PageOffset::from_byte_ceil(self.len()) { return Err(EINVAL); @@ -136,11 +130,7 @@ impl PageCache { let cache_page = self.get_page_locked(&mut pages, pgoff).await?; - unsafe { - let page = ManuallyDrop::new(Page::from_raw_unchecked(PFN::from(cache_page.0))); - - func(&page, cache_page); - } + func(cache_page); Ok(()) } @@ -166,7 +156,7 @@ impl PageCache { let data_len = real_end - offset; if buffer - .fill(&page.as_bytes()[inner_offset..inner_offset + data_len])? + .fill(&page.lock().as_bytes()[inner_offset..inner_offset + data_len])? .should_stop() || buffer.available() == 0 { @@ -195,7 +185,7 @@ impl PageCache { let inner_offset = offset % PAGE_SIZE; let written = stream - .poll_data(&mut page.as_bytes_mut()[inner_offset..])? + .poll_data(&mut page.lock().as_bytes_mut()[inner_offset..])? .map(|b| b.len()) .unwrap_or(0); @@ -237,14 +227,9 @@ impl core::fmt::Debug for PageCache { } } -pub trait PageCacheRawPage: RawPage { - fn is_dirty(&self) -> bool; - fn set_dirty(&self, dirty: bool); -} - impl Drop for PageCache { fn drop(&mut self) { - // TODO: Write back dirty pages... - // let _ = self.fsync(); + // XXX: Send the PageCache to some flusher worker. + let _ = self.fsync(); } } diff --git a/src/kernel/mem/paging.rs b/src/kernel/mem/paging.rs index bca573fb..1b95ce79 100644 --- a/src/kernel/mem/paging.rs +++ b/src/kernel/mem/paging.rs @@ -1,48 +1,22 @@ -use core::ops::Deref; -use core::ptr::NonNull; +use eonix_mm::paging::Folio as _; -use eonix_mm::paging::Page as GenericPage; - -use super::page_alloc::GlobalPageAlloc; -use super::PhysAccess; +use super::folio::FolioOwned; use crate::io::{Buffer, FillResult}; -pub type Page = GenericPage<GlobalPageAlloc>; - /// A buffer that wraps a page and provides a `Buffer` interface. pub struct PageBuffer { - page: PageExcl, + page: FolioOwned, offset: usize, } -pub struct PageLocked<'a> { - page: &'a Page, -} - -/// A page that is exclusively owned. -#[repr(transparent)] -pub struct PageExcl(Page); - pub trait AllocZeroed { fn zeroed() -> Self; } -pub trait PageExt { - fn lock(&self) -> PageLocked; - - /// Get a vmem pointer to the page data as a byte slice. - fn get_bytes_ptr(&self) -> NonNull<[u8]>; - - /// Get a vmem pointer to the start of the page. - fn get_ptr(&self) -> NonNull<u8> { - self.get_bytes_ptr().cast() - } -} - impl PageBuffer { pub fn new() -> Self { Self { - page: PageExcl::alloc(), + page: FolioOwned::alloc(), offset: 0, } } @@ -86,91 +60,3 @@ impl Buffer for PageBuffer { } } } - -impl AllocZeroed for Page { - fn zeroed() -> Self { - let page = Self::alloc(); - - page.lock().as_bytes_mut().fill(0); - - page - } -} - -impl PageExt for Page { - fn lock(&self) -> PageLocked { - // TODO: Actually perform the lock. - PageLocked { page: self } - } - - fn get_bytes_ptr(&self) -> NonNull<[u8]> { - unsafe { - // SAFETY: `self.start()` can't be null. - NonNull::slice_from_raw_parts(self.start().as_ptr(), self.len()) - } - } -} - -impl PageLocked<'_> { - pub fn as_bytes(&self) -> &[u8] { - unsafe { - // SAFETY: `self.start()` points to valid memory of length `self.len()`. - core::slice::from_raw_parts(self.start().as_ptr().as_ptr(), self.len()) - } - } - - pub fn as_bytes_mut(&mut self) -> &mut [u8] { - unsafe { - // SAFETY: `self.start()` points to valid memory of length `self.len()`. - core::slice::from_raw_parts_mut(self.start().as_ptr().as_ptr(), self.len()) - } - } -} - -impl Deref for PageLocked<'_> { - type Target = Page; - - fn deref(&self) -> &Self::Target { - self.page - } -} - -impl PageExcl { - pub fn alloc() -> Self { - Self(Page::alloc()) - } - - pub fn alloc_order(order: u32) -> Self { - Self(Page::alloc_order(order)) - } - - pub fn zeroed() -> Self { - Self(Page::zeroed()) - } - - pub fn as_bytes(&self) -> &[u8] { - unsafe { - // SAFETY: The page is exclusively owned by us. - self.get_bytes_ptr().as_ref() - } - } - - pub fn as_bytes_mut(&mut self) -> &mut [u8] { - unsafe { - // SAFETY: The page is exclusively owned by us. - self.get_bytes_ptr().as_mut() - } - } - - pub fn into_page(self) -> Page { - self.0 - } -} - -impl Deref for PageExcl { - type Target = Page; - - fn deref(&self) -> &Self::Target { - &self.0 - } -} diff --git a/src/kernel/task/kernel_stack.rs b/src/kernel/task/kernel_stack.rs index d3e9de2f..f00b91bd 100644 --- a/src/kernel/task/kernel_stack.rs +++ b/src/kernel/task/kernel_stack.rs @@ -1,11 +1,12 @@ -use crate::kernel::mem::{paging::Page, PhysAccess as _}; -use core::{num::NonZero, ptr::NonNull}; +use core::ptr::NonNull; + use eonix_runtime::executor::Stack; +use crate::kernel::mem::FolioOwned; + #[derive(Debug)] pub struct KernelStack { - _pages: Page, - bottom: NonZero<usize>, + folio: FolioOwned, } impl KernelStack { @@ -14,15 +15,8 @@ impl KernelStack { const KERNEL_STACK_ORDER: u32 = 7; pub fn new() -> Self { - let pages = Page::alloc_order(Self::KERNEL_STACK_ORDER); - let bottom = unsafe { - // SAFETY: The paddr is from a page, which should be valid. - pages.range().end().as_ptr::<u8>().addr() - }; - Self { - _pages: pages, - bottom, + folio: FolioOwned::alloc_order(Self::KERNEL_STACK_ORDER), } } } @@ -33,7 +27,10 @@ impl Stack for KernelStack { } fn get_bottom(&self) -> NonNull<()> { - // SAFETY: The stack is allocated and `bottom` is non-zero. - unsafe { NonNull::new_unchecked(self.bottom.get() as *mut _) } + let ptr = self.folio.get_bytes_ptr(); + let len = ptr.len(); + + // SAFETY: The vaddr of the folio is guaranteed to be non-zero. + unsafe { ptr.cast().byte_add(len) } } } diff --git a/src/kernel/vfs/file/mod.rs b/src/kernel/vfs/file/mod.rs index eb00cc4c..799b9848 100644 --- a/src/kernel/vfs/file/mod.rs +++ b/src/kernel/vfs/file/mod.rs @@ -15,7 +15,7 @@ pub use terminal_file::TerminalFile; use crate::io::{Buffer, ByteBuffer, Chunks, IntoStream, Stream}; use crate::kernel::constants::{EBADF, EINTR, EINVAL, ENOTTY}; -use crate::kernel::mem::PageExcl; +use crate::kernel::mem::FolioOwned; use crate::kernel::task::Thread; use crate::kernel::CharDevice; use crate::prelude::KResult; @@ -94,7 +94,7 @@ impl FileType { } pub async fn sendfile(&self, dest_file: &Self, count: usize) -> KResult<usize> { - let mut buffer_page = PageExcl::alloc(); + let mut buffer_page = FolioOwned::alloc(); let buffer = buffer_page.as_bytes_mut(); self.sendfile_check()?; diff --git a/src/kernel_init.rs b/src/kernel_init.rs index 93b6da20..65af41e4 100644 --- a/src/kernel_init.rs +++ b/src/kernel_init.rs @@ -1,32 +1,26 @@ +use eonix_hal::arch_exported::mm::{ArchPagingMode, PageAccessImpl}; use eonix_hal::bootstrap::BootStrapData; -use eonix_hal::mm::{ArchMemory, ArchPagingMode, GLOBAL_PAGE_TABLE}; +use eonix_hal::mm::{ArchMemory, BasicPageAllocRef, GLOBAL_PAGE_TABLE}; use eonix_hal::traits::mm::Memory; use eonix_mm::address::{Addr as _, AddrOps as _, VAddr, VRange}; -use eonix_mm::page_table::{PageAttribute, PagingMode as _, PTE}; -use eonix_mm::paging::{Page as GenericPage, PAGE_SIZE, PFN}; +use eonix_mm::page_table::{PageAttribute, PageTable, PTE}; +use eonix_mm::paging::{Folio as _, FrameAlloc, PAGE_SIZE, PFN}; use crate::kernel::mem::{GlobalPageAlloc, RawPage}; -pub fn setup_memory(data: &mut BootStrapData) { - let addr_max = ArchMemory::present_ram() - .map(|range| range.end()) - .max() - .expect("No free memory"); - - let pfn_max = PFN::from(addr_max.ceil()); - let len_bytes_page_array = usize::from(pfn_max) * size_of::<RawPage>(); - let count_pages = len_bytes_page_array.div_ceil(PAGE_SIZE); - - let alloc = data.get_alloc().unwrap(); +fn setup_kernel_page_array(alloc: BasicPageAllocRef, count_pages: usize) { + // TODO: This should be done by the global Zone + let global_page_table = PageTable::<ArchPagingMode, _, _>::new( + GLOBAL_PAGE_TABLE.clone(), + alloc.clone(), + PageAccessImpl, + ); // Map kernel page array. const V_KERNEL_PAGE_ARRAY_START: VAddr = VAddr::from(0xffffff8040000000); - for pte in GLOBAL_PAGE_TABLE.iter_kernel_in( - VRange::from(V_KERNEL_PAGE_ARRAY_START).grow(PAGE_SIZE * count_pages), - ArchPagingMode::LEVELS, - &alloc, - ) { + let range = VRange::from(V_KERNEL_PAGE_ARRAY_START).grow(PAGE_SIZE * count_pages); + for pte in global_page_table.iter_kernel(range) { let attr = PageAttribute::PRESENT | PageAttribute::WRITE | PageAttribute::READ @@ -34,10 +28,15 @@ pub fn setup_memory(data: &mut BootStrapData) { | PageAttribute::ACCESSED | PageAttribute::DIRTY; - let page = GenericPage::alloc_in(&alloc); + let page = alloc.alloc().unwrap(); pte.set(page.into_raw(), attr.into()); } + // TODO!!!: Construct the global zone with all present ram. + // for range in ArchMemory::present_ram() { + // GlobalPageAlloc::mark_present(range); + // } + unsafe { // SAFETY: We've just mapped the area with sufficient length. core::ptr::write_bytes( @@ -47,10 +46,21 @@ pub fn setup_memory(data: &mut BootStrapData) { ); } - // TODO!!!: Construct the global zone with all present ram. - // for range in ArchMemory::present_ram() { - // GlobalPageAlloc::mark_present(range); - // } + core::mem::forget(global_page_table); +} + +pub fn setup_memory(data: &mut BootStrapData) { + let addr_max = ArchMemory::present_ram() + .map(|range| range.end()) + .max() + .expect("No free memory"); + + let pfn_max = PFN::from(addr_max.ceil()); + let len_bytes_page_array = usize::from(pfn_max) * size_of::<RawPage>(); + let count_pages = len_bytes_page_array.div_ceil(PAGE_SIZE); + + let alloc = data.get_alloc().unwrap(); + setup_kernel_page_array(alloc, count_pages); if let Some(early_alloc) = data.take_alloc() { for range in early_alloc.into_iter() { From 6bdab59e428169c70029718403442ad149d62b0d Mon Sep 17 00:00:00 2001 From: greatbridf <greatbridf@icloud.com> Date: Sun, 18 Jan 2026 02:35:05 +0800 Subject: [PATCH 24/36] style: fix or suppress warnings No functional changes - Add `{extern_,}symbol_addr` macro to retrieve symbol address. - Remove manual impl Send and Sync for RawPage - Make elided '_ lifetimes in return types explicit - Suppress unused warnings by allowing them - Remove really unused functions - Refactor `println_trace` macro to suppress unused variable warnings Signed-off-by: greatbridf <greatbridf@icloud.com> --- crates/eonix_hal/src/lib.rs | 26 +++++++++ crates/eonix_log/src/lib.rs | 22 ++++++-- crates/eonix_percpu/src/lib.rs | 31 +++++------ crates/eonix_runtime/src/scheduler.rs | 35 +++++------- crates/eonix_sync/eonix_spin/src/lib.rs | 14 +++-- .../eonix_sync/eonix_sync_rt/src/spin_irq.rs | 4 +- crates/posix_types/src/poll.rs | 2 + macros/src/lib.rs | 4 +- src/driver/ahci/slot.rs | 4 +- src/driver/e1000e.rs | 2 + src/driver/serial/io.rs | 15 ++++-- src/fs/procfs.rs | 2 + src/fs/tmpfs/file.rs | 6 +-- src/kernel/constants.rs | 2 +- src/kernel/mem/folio.rs | 2 +- src/kernel/mem/mm_list/page_fault.rs | 19 +++---- src/kernel/mem/page_alloc/raw_page.rs | 4 -- src/kernel/pcie/device.rs | 24 +++++---- src/kernel/pcie/header.rs | 17 +++--- src/kernel/pcie/init.rs | 5 +- src/kernel/syscall.rs | 23 ++++---- src/kernel/task.rs | 3 +- src/kernel/task/process.rs | 54 +++++++++---------- src/kernel/vfs/dentry.rs | 6 ++- src/kernel/vfs/types/device_id.rs | 11 ---- src/lib.rs | 5 +- src/path.rs | 6 ++- src/sync/arcswap.rs | 11 ++-- 28 files changed, 188 insertions(+), 171 deletions(-) diff --git a/crates/eonix_hal/src/lib.rs b/crates/eonix_hal/src/lib.rs index b9c7d053..d3bf7825 100644 --- a/crates/eonix_hal/src/lib.rs +++ b/crates/eonix_hal/src/lib.rs @@ -43,3 +43,29 @@ pub mod arch_exported { pub use eonix_hal_macros::{ap_main, default_trap_handler, main}; pub use eonix_hal_traits as traits; + +#[macro_export] +macro_rules! symbol_addr { + ($sym:expr) => {{ + ($sym) as *const () as usize + }}; + ($sym:expr, $type:ty) => {{ + ($sym) as *const () as *const $type + }}; +} + +#[macro_export] +macro_rules! extern_symbol_addr { + ($sym:ident) => {{ + unsafe extern "C" { + fn $sym(); + } + $crate::symbol_addr!($sym) + }}; + ($sym:ident, $type:ty) => {{ + unsafe extern "C" { + fn $sym(); + } + $crate::symbol_addr!($sym, $type) + }}; +} diff --git a/crates/eonix_log/src/lib.rs b/crates/eonix_log/src/lib.rs index 92b1639f..01b6a587 100644 --- a/crates/eonix_log/src/lib.rs +++ b/crates/eonix_log/src/lib.rs @@ -2,6 +2,7 @@ use alloc::sync::Arc; use core::fmt::{self, Write}; + use eonix_sync::{Spin, SpinIrq as _}; extern crate alloc; @@ -91,18 +92,31 @@ macro_rules! println_fatal { #[macro_export] macro_rules! println_trace { - ($feat:literal) => { + (feat:$feat:literal) => { #[deny(unexpected_cfgs)] { #[cfg(feature = $feat)] - $crate::println!("[kernel:trace] ") + $crate::println!("[kernel:trace]") } }; - ($feat:literal, $($arg:tt)*) => {{ + (feat:$feat:literal, $fmt:literal) => {{ #[deny(unexpected_cfgs)] { #[cfg(feature = $feat)] - $crate::println!("[kernel:trace] {}", format_args!($($arg)*)) + $crate::println!(concat!("[kernel:trace] ", $feat)) } }}; + (feat:$feat:literal, $fmt:literal, $($arg:expr $(,)?)*) => { + #[deny(unexpected_cfgs)] + { + // Suppress unused variables warning + #[cfg(not(feature = $feat))] + { + $(let _ = $arg;)* + } + + #[cfg(feature = $feat)] + $crate::println!("[kernel:trace] {}", format_args!($fmt, $($arg,)*)) + } + }; } diff --git a/crates/eonix_percpu/src/lib.rs b/crates/eonix_percpu/src/lib.rs index 1fc7ffb8..a00b5c05 100644 --- a/crates/eonix_percpu/src/lib.rs +++ b/crates/eonix_percpu/src/lib.rs @@ -1,28 +1,21 @@ #![no_std] use core::alloc::Layout; -use core::ptr::null_mut; -use core::ptr::NonNull; -use core::sync::atomic::AtomicPtr; -use core::sync::atomic::Ordering; - -#[cfg(target_arch = "x86_64")] -pub use eonix_percpu_macros::define_percpu_x86_64 as define_percpu; - -#[cfg(target_arch = "x86_64")] -pub use eonix_percpu_macros::define_percpu_shared_x86_64 as define_percpu_shared; - -#[cfg(target_arch = "riscv64")] -pub use eonix_percpu_macros::define_percpu_riscv64 as define_percpu; - -#[cfg(target_arch = "riscv64")] -pub use eonix_percpu_macros::define_percpu_shared_riscv64 as define_percpu_shared; +use core::ptr::{null_mut, NonNull}; +use core::sync::atomic::{AtomicPtr, Ordering}; #[cfg(target_arch = "loongarch64")] pub use eonix_percpu_macros::define_percpu_loongarch64 as define_percpu; - +#[cfg(target_arch = "riscv64")] +pub use eonix_percpu_macros::define_percpu_riscv64 as define_percpu; #[cfg(target_arch = "loongarch64")] pub use eonix_percpu_macros::define_percpu_shared_loongarch64 as define_percpu_shared; +#[cfg(target_arch = "riscv64")] +pub use eonix_percpu_macros::define_percpu_shared_riscv64 as define_percpu_shared; +#[cfg(target_arch = "x86_64")] +pub use eonix_percpu_macros::define_percpu_shared_x86_64 as define_percpu_shared; +#[cfg(target_arch = "x86_64")] +pub use eonix_percpu_macros::define_percpu_x86_64 as define_percpu; const MAX_CPUS: usize = 256; @@ -41,7 +34,7 @@ impl PercpuArea { unsafe extern "C" { fn PERCPU_LENGTH(); } - let len = PERCPU_LENGTH as usize; + let len = PERCPU_LENGTH as *const () as usize; assert_ne!(len, 0, "Percpu length should not be zero."); len @@ -52,7 +45,7 @@ impl PercpuArea { fn PERCPU_DATA_START(); } - let addr = PERCPU_DATA_START as usize; + let addr = PERCPU_DATA_START as *const () as usize; NonNull::new(addr as *mut _).expect("Percpu data should not be null.") } diff --git a/crates/eonix_runtime/src/scheduler.rs b/crates/eonix_runtime/src/scheduler.rs index 3f72fbf4..b4b7960d 100644 --- a/crates/eonix_runtime/src/scheduler.rs +++ b/crates/eonix_runtime/src/scheduler.rs @@ -1,20 +1,19 @@ -use crate::{ - executor::OutputHandle, - ready_queue::{local_rq, ReadyQueue}, - task::{Task, TaskAdapter, TaskHandle, TaskState}, -}; -use alloc::{sync::Arc, task::Wake}; -use core::{ - ops::{Deref, DerefMut}, - ptr::NonNull, - task::{Context, Poll, Waker}, -}; +use alloc::sync::Arc; +use alloc::task::Wake; +use core::ops::{Deref, DerefMut}; +use core::ptr::NonNull; +use core::task::{Context, Poll, Waker}; + use eonix_hal::processor::halt; use eonix_log::println_trace; use eonix_sync::{LazyLock, Spin, SpinIrq as _}; use intrusive_collections::RBTree; use pointers::BorrowedArc; +use crate::executor::OutputHandle; +use crate::ready_queue::{local_rq, ReadyQueue}; +use crate::task::{Task, TaskAdapter, TaskHandle, TaskState}; + #[eonix_percpu::define_percpu] static CURRENT_TASK: Option<NonNull<Task>> = None; @@ -93,12 +92,6 @@ impl Runtime { } } - fn current(&self) -> Option<BorrowedArc<Task>> { - CURRENT_TASK - .get() - .map(|ptr| unsafe { BorrowedArc::from_raw(ptr) }) - } - fn remove_and_enqueue_current(&self, rq: &mut impl DerefMut<Target = dyn ReadyQueue>) { let Some(current) = CURRENT_TASK .swap(None) @@ -116,7 +109,7 @@ impl Runtime { }) { Ok(TaskState::READY_RUNNING) => { println_trace!( - "trace_scheduler", + feat: "trace_scheduler", "Re-enqueueing task {:?} (CPU{})", current.id, eonix_hal::processor::CPU::local().cpuid(), @@ -126,7 +119,7 @@ impl Runtime { } Ok(_) => { println_trace!( - "trace_scheduler", + feat: "trace_scheduler", "Current task {:?} (CPU{}) is blocked, not re-enqueueing", current.id, eonix_hal::processor::CPU::local().cpuid(), @@ -184,7 +177,7 @@ impl Runtime { }; println_trace!( - "trace_scheduler", + feat: "trace_scheduler", "Switching to task {:?} (CPU{})", next.id, eonix_hal::processor::CPU::local().cpuid(), @@ -212,7 +205,7 @@ impl Runtime { ); println_trace!( - "trace_scheduler", + feat: "trace_scheduler", "Task {:?} finished execution, removing...", Task::current().id, ); diff --git a/crates/eonix_sync/eonix_spin/src/lib.rs b/crates/eonix_sync/eonix_spin/src/lib.rs index 4718b867..7225aceb 100644 --- a/crates/eonix_sync/eonix_spin/src/lib.rs +++ b/crates/eonix_sync/eonix_spin/src/lib.rs @@ -2,13 +2,11 @@ mod guard; -use core::{ - cell::UnsafeCell, - marker::PhantomData, - sync::atomic::{AtomicBool, Ordering}, -}; -use eonix_sync_base::{Relax, SpinRelax}; +use core::cell::UnsafeCell; +use core::marker::PhantomData; +use core::sync::atomic::{AtomicBool, Ordering}; +use eonix_sync_base::{Relax, SpinRelax}; pub use guard::{SpinGuard, UnlockedSpinGuard}; pub trait SpinContext { @@ -84,7 +82,7 @@ where T: ?Sized, R: Relax, { - pub fn lock_with_context<C>(&self, context: C) -> SpinGuard<T, C, R> + pub fn lock_with_context<C>(&self, context: C) -> SpinGuard<'_, T, C, R> where C: SpinContext, { @@ -100,7 +98,7 @@ where ) } - pub fn lock(&self) -> SpinGuard<T, DisablePreemption, R> { + pub fn lock(&self) -> SpinGuard<'_, T, DisablePreemption, R> { self.lock_with_context(DisablePreemption::save()) } diff --git a/crates/eonix_sync/eonix_sync_rt/src/spin_irq.rs b/crates/eonix_sync/eonix_sync_rt/src/spin_irq.rs index 76a28682..b70cdc3d 100644 --- a/crates/eonix_sync/eonix_sync_rt/src/spin_irq.rs +++ b/crates/eonix_sync/eonix_sync_rt/src/spin_irq.rs @@ -12,7 +12,7 @@ pub trait SpinIrq { type Context: SpinContext; type Relax; - fn lock_irq(&self) -> SpinGuard<Self::Value, Self::Context, Self::Relax>; + fn lock_irq(&self) -> SpinGuard<'_, Self::Value, Self::Context, Self::Relax>; } impl SpinContext for IrqContext { @@ -50,7 +50,7 @@ where type Context = IrqContext; type Relax = R; - fn lock_irq(&self) -> SpinGuard<Self::Value, Self::Context, Self::Relax> { + fn lock_irq(&self) -> SpinGuard<'_, Self::Value, Self::Context, Self::Relax> { self.lock_with_context(IrqContext::save()) } } diff --git a/crates/posix_types/src/poll.rs b/crates/posix_types/src/poll.rs index 781f589f..dcf5f9b2 100644 --- a/crates/posix_types/src/poll.rs +++ b/crates/posix_types/src/poll.rs @@ -1,5 +1,7 @@ pub const FDSET_LENGTH: usize = 1024 / (8 * size_of::<usize>()); +// TODO: Implement syscall pselect +#[allow(unused)] pub struct FDSet { fds_bits: [usize; FDSET_LENGTH], } diff --git a/macros/src/lib.rs b/macros/src/lib.rs index 722fa5da..09e12f99 100644 --- a/macros/src/lib.rs +++ b/macros/src/lib.rs @@ -123,7 +123,7 @@ fn define_syscall_impl(attrs: TokenStream, item: TokenStream) -> TokenStream { Box::new_in( async move { eonix_log::println_trace!( - "trace_syscall", + feat: "trace_syscall", "tid{}: {}({}) => {{", thd.tid, #syscall_name_str, @@ -133,7 +133,7 @@ fn define_syscall_impl(attrs: TokenStream, item: TokenStream) -> TokenStream { let retval = #real_fn(thd, #(#args_call),*).await.into_retval(); eonix_log::println_trace!( - "trace_syscall", + feat: "trace_syscall", "}} => {:x?}", retval, ); diff --git a/src/driver/ahci/slot.rs b/src/driver/ahci/slot.rs index dd096f57..fdb61f96 100644 --- a/src/driver/ahci/slot.rs +++ b/src/driver/ahci/slot.rs @@ -39,6 +39,8 @@ enum SlotState { Idle, Working, Finished, + // TODO: Implement AHCI error handling + #[allow(unused)] Error, } @@ -67,7 +69,7 @@ impl CommandList { + (size_of::<UnsafeCell<CommandHeader>>() + size_of::<Spin<SlotControl>>()) * 32 } - pub fn get(&self, index: usize) -> CommandSlot { + pub fn get(&self, index: usize) -> CommandSlot<'_> { CommandSlot { cmdheader: &self.cmdheaders()[index], control: &self.controls()[index], diff --git a/src/driver/e1000e.rs b/src/driver/e1000e.rs index 6d6ca353..923a4594 100644 --- a/src/driver/e1000e.rs +++ b/src/driver/e1000e.rs @@ -61,6 +61,8 @@ struct E1000eDev { tx_tail: Option<u32>, rx_buffers: Box<[FolioOwned; RX_DESC_SIZE]>, + // TODO: Implement E1000e send + #[allow(unused)] tx_buffers: Box<[Option<FolioOwned>; TX_DESC_SIZE]>, } diff --git a/src/driver/serial/io.rs b/src/driver/serial/io.rs index aec18f20..57e61c56 100644 --- a/src/driver/serial/io.rs +++ b/src/driver/serial/io.rs @@ -1,10 +1,11 @@ -use super::SerialRegister; use core::ptr::NonNull; -use eonix_hal::{fence::memory_barrier, mm::ArchPhysAccess}; -use eonix_mm::address::{PAddr, PhysAccess}; #[cfg(target_arch = "x86_64")] use eonix_hal::arch_exported::io::Port8; +use eonix_hal::mm::ArchPhysAccess; +use eonix_mm::address::{PAddr, PhysAccess}; + +use super::SerialRegister; #[cfg(target_arch = "x86_64")] pub struct SerialIO { @@ -73,10 +74,12 @@ impl SerialIO { self.line_status } + #[allow(unused)] pub fn modem_status(&self) -> impl SerialRegister { self.modem_status } + #[allow(unused)] pub fn scratch(&self) -> impl SerialRegister { self.scratch } @@ -100,7 +103,7 @@ impl SerialRegister for NonNull<u8> { let retval = unsafe { self.as_ptr().read_volatile() }; #[cfg(target_arch = "loongarch64")] - memory_barrier(); + eonix_hal::fence::memory_barrier(); retval } @@ -110,7 +113,7 @@ impl SerialRegister for NonNull<u8> { unsafe { self.as_ptr().write_volatile(data) }; #[cfg(target_arch = "loongarch64")] - memory_barrier(); + eonix_hal::fence::memory_barrier(); } } @@ -155,10 +158,12 @@ impl SerialIO { unsafe { self.base_addr.add(5) } } + #[allow(unused)] pub fn modem_status(&self) -> impl SerialRegister { unsafe { self.base_addr.add(6) } } + #[allow(unused)] pub fn scratch(&self) -> impl SerialRegister { unsafe { self.base_addr.add(7) } } diff --git a/src/fs/procfs.rs b/src/fs/procfs.rs index 32ede420..9a3933bb 100644 --- a/src/fs/procfs.rs +++ b/src/fs/procfs.rs @@ -26,6 +26,8 @@ enum NodeKind { struct FileInode { read: Option<Box<dyn Fn(&mut PageBuffer) -> KResult<()> + Send + Sync>>, + // TODO: Implement writes to procfs files + #[allow(unused)] write: Option<()>, } diff --git a/src/fs/tmpfs/file.rs b/src/fs/tmpfs/file.rs index d560a672..aafae539 100644 --- a/src/fs/tmpfs/file.rs +++ b/src/fs/tmpfs/file.rs @@ -177,7 +177,6 @@ impl InodeOps for FileInode { } pub struct DeviceInode { - is_block: bool, devid: DeviceId, } @@ -199,10 +198,7 @@ impl DeviceInode { ctime: now, mtime: now, }, - Self { - is_block: mode.format() == Format::BLK, - devid, - }, + Self { devid }, ) } } diff --git a/src/kernel/constants.rs b/src/kernel/constants.rs index 4e11d66e..b96387b0 100644 --- a/src/kernel/constants.rs +++ b/src/kernel/constants.rs @@ -36,7 +36,7 @@ pub const ENOTDIR: u32 = 20; pub const EISDIR: u32 = 21; pub const EINVAL: u32 = 22; pub const ENOTTY: u32 = 25; -pub const ENOSPC: u32 = 28; +// pub const ENOSPC: u32 = 28; pub const ESPIPE: u32 = 29; // pub const EROFS: u32 = 30; pub const EPIPE: u32 = 32; diff --git a/src/kernel/mem/folio.rs b/src/kernel/mem/folio.rs index 6647e1af..8ab4d6be 100644 --- a/src/kernel/mem/folio.rs +++ b/src/kernel/mem/folio.rs @@ -79,7 +79,7 @@ impl Folio { } } - pub fn lock(&self) -> LockedFolio { + pub fn lock(&self) -> LockedFolio<'_> { // TODO: actually perform the lock... LockedFolio(self) } diff --git a/src/kernel/mem/mm_list/page_fault.rs b/src/kernel/mem/mm_list/page_fault.rs index 7aac141d..5a56efbc 100644 --- a/src/kernel/mem/mm_list/page_fault.rs +++ b/src/kernel/mem/mm_list/page_fault.rs @@ -1,3 +1,4 @@ +use eonix_hal::extern_symbol_addr; use eonix_hal::mm::flush_tlb; use eonix_hal::traits::fault::PageFaultErrorCode; use eonix_mm::address::{Addr as _, AddrOps as _, VRange}; @@ -24,27 +25,19 @@ impl FixEntry { VAddr::from((self.start + self.length) as usize) } - #[allow(dead_code)] - fn range(&self) -> VRange { - VRange::new(self.start(), self.end()) - } - fn jump_address(&self) -> VAddr { VAddr::from(self.jump_address as usize) } fn entries() -> &'static [FixEntry] { - extern "C" { - fn FIX_START(); - fn FIX_END(); - } + let fix_seg_len_bytes = extern_symbol_addr!(FIX_END) - extern_symbol_addr!(FIX_START); unsafe { - // SAFETY: `FIX_START` and `FIX_END` are defined in the - // linker script in `.rodata` section. + // SAFETY: `FIX_START` and `FIX_END` are defined in the linker script + // in `.rodata` section. core::slice::from_raw_parts( - FIX_START as usize as *const FixEntry, - (FIX_END as usize - FIX_START as usize) / size_of::<FixEntry>(), + extern_symbol_addr!(FIX_START, FixEntry), + fix_seg_len_bytes / size_of::<FixEntry>(), ) } } diff --git a/src/kernel/mem/page_alloc/raw_page.rs b/src/kernel/mem/page_alloc/raw_page.rs index 0d775245..16d57714 100644 --- a/src/kernel/mem/page_alloc/raw_page.rs +++ b/src/kernel/mem/page_alloc/raw_page.rs @@ -48,10 +48,6 @@ pub struct RawPage { shared_data: PageData, } -// XXX: introduce Folio and remove this. -unsafe impl Send for RawPage {} -unsafe impl Sync for RawPage {} - impl PageFlags { pub const LOCKED: u32 = 1 << 1; pub const BUDDY: u32 = 1 << 2; diff --git a/src/kernel/pcie/device.rs b/src/kernel/pcie/device.rs index 085e7b9a..2a8f150d 100644 --- a/src/kernel/pcie/device.rs +++ b/src/kernel/pcie/device.rs @@ -1,14 +1,17 @@ -use super::{ - header::{Bar, Command}, - CommonHeader, Header, -}; -use crate::kernel::mem::PhysAccess as _; +use alloc::collections::btree_map::BTreeMap; +use alloc::sync::Arc; +use alloc::vec::Vec; +use core::num::NonZero; +use core::ops::RangeInclusive; + use align_ext::AlignExt; -use alloc::{collections::btree_map::BTreeMap, sync::Arc, vec::Vec}; -use core::{num::NonZero, ops::RangeInclusive}; use eonix_mm::address::{Addr, PAddr, PRange}; use eonix_sync::Spin; +use super::header::{Bar, Command}; +use super::{CommonHeader, Header}; +use crate::kernel::mem::PhysAccess as _; + pub(super) static PCIE_DEVICES: Spin<BTreeMap<u32, Vec<Arc<PCIDevice>>>> = Spin::new(BTreeMap::new()); @@ -20,7 +23,7 @@ pub struct PCIDevice<'a> { pub device_id: u16, } -#[allow(dead_code)] +#[allow(unused)] #[derive(Clone)] pub struct SegmentGroup { id: usize, @@ -28,6 +31,7 @@ pub struct SegmentGroup { base_address: PAddr, } +#[allow(unused)] #[derive(Clone)] pub struct ConfigSpace { pub bus: u8, @@ -180,10 +184,12 @@ impl PCIDevice<'_> { ); } + #[allow(unused)] pub fn config_space(&self) -> &ConfigSpace { &self.config_space } + #[allow(unused)] pub fn segment_group(&self) -> &SegmentGroup { &self.segment_group } @@ -209,7 +215,7 @@ impl PciMemoryAllocator { self.start += size; eonix_log::println_trace!( - "trace_pci", + feat: "trace_pci", "PciMemoryAllocator: Allocated {} bytes at {:#x}", size, base diff --git a/src/kernel/pcie/header.rs b/src/kernel/pcie/header.rs index 889795d3..0a44ea28 100644 --- a/src/kernel/pcie/header.rs +++ b/src/kernel/pcie/header.rs @@ -1,10 +1,9 @@ +use core::marker::PhantomData; +use core::num::NonZero; +use core::ops::{BitAnd, BitOr, Deref, Not}; +use core::sync::atomic::{AtomicU16, AtomicU32, Ordering}; + use bitflags::bitflags; -use core::{ - marker::PhantomData, - num::NonZero, - ops::{BitAnd, BitOr, Deref, Not}, - sync::atomic::{AtomicU16, AtomicU32, Ordering}, -}; use eonix_hal::fence::memory_barrier; pub trait BitFlag: Sized + Copy { @@ -215,14 +214,14 @@ where } impl CommonHeader { - pub fn command(&self) -> Register<Command> { + pub fn command(&self) -> Register<'_, Command> { Register { register: unsafe { AtomicU16::from_ptr((&raw const self._command) as *mut u16) }, _phantom: PhantomData, } } - pub fn status(&self) -> Register<Status> { + pub fn status(&self) -> Register<'_, Status> { Register { register: unsafe { AtomicU16::from_ptr((&raw const self._status) as *mut u16) }, _phantom: PhantomData, @@ -231,7 +230,7 @@ impl CommonHeader { } impl Bars<'_> { - pub fn iter(&self) -> impl Iterator<Item = BarsEntry> + '_ { + pub fn iter(&self) -> impl Iterator<Item = BarsEntry<'_>> + use<'_> { struct BarsIterator<'a> { bars: &'a [AtomicU32], pos: usize, diff --git a/src/kernel/pcie/init.rs b/src/kernel/pcie/init.rs index 4c183bc5..74a490b4 100644 --- a/src/kernel/pcie/init.rs +++ b/src/kernel/pcie/init.rs @@ -10,6 +10,7 @@ use super::error::PciError; use crate::kernel::mem::PhysAccess as _; use crate::kernel::pcie::device::PciMemoryAllocator; +#[allow(unused)] #[derive(Clone)] struct AcpiHandlerImpl; @@ -34,7 +35,6 @@ pub fn init_pcie() -> Result<(), PciError> { #[cfg(target_arch = "x86_64")] { use acpi::{AcpiTables, PciConfigRegions}; - use eonix_mm::address::PAddr; let acpi_tables = unsafe { // SAFETY: Our impl should be correct. @@ -69,7 +69,6 @@ pub fn init_pcie() -> Result<(), PciError> { #[cfg(any(target_arch = "riscv64", target_arch = "loongarch64"))] { use eonix_hal::arch_exported::fdt::FDT; - use eonix_mm::address::PRange; use crate::kernel::constants::{EINVAL, EIO, ENOENT}; @@ -88,7 +87,7 @@ pub fn init_pcie() -> Result<(), PciError> { let size = u64::from_be_bytes(entry[20..28].try_into().unwrap()); println_trace!( - "trace_pci", + feat: "trace_pci", "PCIe range: PCI address = {:#x}, CPU address = {:#x}, size = {:#x}", pci_address, cpu_address, diff --git a/src/kernel/syscall.rs b/src/kernel/syscall.rs index 78ddcd1c..d06c5d88 100644 --- a/src/kernel/syscall.rs +++ b/src/kernel/syscall.rs @@ -1,11 +1,17 @@ -use super::task::ThreadAlloc; -use crate::kernel::task::Thread; use alloc::boxed::Box; -use core::{future::Future, marker::PhantomData, ops::Deref, pin::Pin}; +use core::future::Future; +use core::marker::PhantomData; +use core::ops::Deref; +use core::pin::Pin; + +use eonix_hal::extern_symbol_addr; use eonix_mm::address::{Addr, VAddr}; use eonix_sync::LazyLock; use posix_types::ctypes::PtrT; +use super::task::ThreadAlloc; +use crate::kernel::task::Thread; + pub mod file_rw; pub mod mm; pub mod net; @@ -280,12 +286,6 @@ impl<T> core::fmt::Debug for UserMut<T> { } static SYSCALL_HANDLERS: LazyLock<[Option<SyscallHandler>; MAX_SYSCALL_NO]> = LazyLock::new(|| { - extern "C" { - // SAFETY: `SYSCALL_HANDLERS` is defined in linker script. - fn RAW_SYSCALL_HANDLERS(); - fn RAW_SYSCALL_HANDLERS_SIZE(); - } - // DO NOT TOUCH THESE FUNCTIONS!!! // THEY ARE USED FOR KEEPING THE OBJECTS NOT STRIPPED BY THE LINKER!!! file_rw::keep_alive(); @@ -294,15 +294,14 @@ static SYSCALL_HANDLERS: LazyLock<[Option<SyscallHandler>; MAX_SYSCALL_NO]> = La procops::keep_alive(); sysinfo::keep_alive(); - let raw_handlers_addr = RAW_SYSCALL_HANDLERS as *const (); - let raw_handlers_size_byte = RAW_SYSCALL_HANDLERS_SIZE as usize; + let raw_handlers_size_byte = extern_symbol_addr!(RAW_SYSCALL_HANDLERS_SIZE); assert!(raw_handlers_size_byte % size_of::<RawSyscallHandler>() == 0); let raw_handlers_count = raw_handlers_size_byte / size_of::<RawSyscallHandler>(); let raw_handlers = unsafe { core::slice::from_raw_parts( - raw_handlers_addr as *const RawSyscallHandler, + extern_symbol_addr!(RAW_SYSCALL_HANDLERS, RawSyscallHandler), raw_handlers_count, ) }; diff --git a/src/kernel/task.rs b/src/kernel/task.rs index 3fe6fe97..b0966046 100644 --- a/src/kernel/task.rs +++ b/src/kernel/task.rs @@ -11,6 +11,7 @@ mod thread; mod user_tls; pub use clone::{do_clone, CloneArgs, CloneFlags}; +use eonix_hal::symbol_addr; pub use futex::{futex_wait, futex_wake, parse_futexop, FutexFlags, FutexOp, RobustListHead}; pub use kernel_stack::KernelStack; pub use loader::ProgramLoader; @@ -185,7 +186,7 @@ where trap_ctx.set_user_mode(false); trap_ctx.set_interrupt_enabled(true); let _ = trap_ctx.set_user_call_frame( - execute::<F> as usize, + symbol_addr!(execute::<F>), Some(sp.addr().get()), None, &[(&raw mut future) as usize, output.get() as usize], diff --git a/src/kernel/task/process.rs b/src/kernel/task/process.rs index 421e4b8b..3eff5949 100644 --- a/src/kernel/task/process.rs +++ b/src/kernel/task/process.rs @@ -1,34 +1,30 @@ -use super::{ - process_group::ProcessGroupBuilder, signal::RaiseResult, thread::ThreadBuilder, ProcessGroup, - ProcessList, Session, Thread, -}; -use crate::kernel::constants::{ECHILD, EINTR, EINVAL, EPERM, ESRCH}; -use crate::kernel::task::{CloneArgs, CloneFlags}; -use crate::rcu::call_rcu; -use crate::{ - kernel::mem::MMList, - prelude::*, - rcu::{RCUPointer, RCUReadGuard}, - sync::CondVar, -}; -use alloc::{ - collections::{btree_map::BTreeMap, vec_deque::VecDeque}, - sync::{Arc, Weak}, -}; +use alloc::collections::btree_map::BTreeMap; +use alloc::collections::vec_deque::VecDeque; +use alloc::sync::{Arc, Weak}; use core::sync::atomic::{AtomicU32, Ordering}; -use eonix_mm::address::VAddr; + use eonix_sync::{ AsProof as _, AsProofMut as _, Locked, Proof, ProofMut, RwLockReadGuard, SpinGuard, UnlockableGuard as _, UnlockedGuard as _, }; use pointers::BorrowedArc; use posix_types::constants::{ - CLD_CONTINUED, CLD_DUMPED, CLD_EXITED, CLD_KILLED, CLD_STOPPED, P_PGID, P_PIDFD, + CLD_CONTINUED, CLD_DUMPED, CLD_EXITED, CLD_KILLED, CLD_STOPPED, P_ALL, P_PGID, P_PID, P_PIDFD, }; -use posix_types::constants::{P_ALL, P_PID}; use posix_types::signal::Signal; use posix_types::SIGNAL_COREDUMP; +use super::process_group::ProcessGroupBuilder; +use super::signal::RaiseResult; +use super::thread::ThreadBuilder; +use super::{ProcessGroup, ProcessList, Session, Thread}; +use crate::kernel::constants::{ECHILD, EINTR, EINVAL, EPERM, ESRCH}; +use crate::kernel::mem::MMList; +use crate::kernel::task::{CloneArgs, CloneFlags}; +use crate::prelude::*; +use crate::rcu::{call_rcu, RCUPointer, RCUReadGuard}; +use crate::sync::CondVar; + pub struct ProcessBuilder { mm_list: Option<MMList>, exit_signal: Option<Signal>, @@ -51,8 +47,6 @@ pub struct Process { pub exit_signal: Option<Signal>, - pub shm_areas: Spin<BTreeMap<VAddr, usize>>, - /// Parent process /// /// `parent` must be valid during the whole life of the process. @@ -256,7 +250,6 @@ impl ProcessBuilder { pid: self.pid.expect("should set pid before building"), wait_list: WaitList::new(), mm_list, - shm_areas: Spin::new(BTreeMap::new()), exit_signal: self.exit_signal, parent: RCUPointer::empty(), pgroup: RCUPointer::empty(), @@ -522,17 +515,17 @@ impl Process { } /// Provide RCU locked (maybe inconsistent) access to the session. - pub fn session_rcu(&self) -> RCUReadGuard<'_, BorrowedArc<Session>> { + pub fn session_rcu(&self) -> RCUReadGuard<'_, BorrowedArc<'_, Session>> { self.session.load().unwrap() } /// Provide RCU locked (maybe inconsistent) access to the process group. - pub fn pgroup_rcu(&self) -> RCUReadGuard<'_, BorrowedArc<ProcessGroup>> { + pub fn pgroup_rcu(&self) -> RCUReadGuard<'_, BorrowedArc<'_, ProcessGroup>> { self.pgroup.load().unwrap() } /// Provide RCU locked (maybe inconsistent) access to the parent process. - pub fn parent_rcu(&self) -> Option<RCUReadGuard<'_, BorrowedArc<Process>>> { + pub fn parent_rcu(&self) -> Option<RCUReadGuard<'_, BorrowedArc<'_, Process>>> { self.parent.load() } @@ -569,7 +562,7 @@ impl WaitList { self.cv_wait_procs.notify_all(); } - pub fn drain_exited(&self) -> DrainExited { + pub fn drain_exited(&self) -> DrainExited<'_> { DrainExited { wait_procs: self.wait_procs.lock(), } @@ -578,7 +571,12 @@ impl WaitList { /// # Safety /// Locks `ProcessList` and `WaitList` at the same time. When `wait` is called, /// releases the lock on `ProcessList` and `WaitList` and waits on `cv_wait_procs`. - pub async fn entry(&self, wait_id: WaitId, want_stop: bool, want_continue: bool) -> Entry { + pub async fn entry( + &self, + wait_id: WaitId, + want_stop: bool, + want_continue: bool, + ) -> Entry<'_, '_, '_> { Entry { process_list: ProcessList::get().read().await, wait_procs: self.wait_procs.lock(), diff --git a/src/kernel/vfs/dentry.rs b/src/kernel/vfs/dentry.rs index 22760de9..a401f4f7 100644 --- a/src/kernel/vfs/dentry.rs +++ b/src/kernel/vfs/dentry.rs @@ -27,6 +27,8 @@ use crate::path::Path; use crate::prelude::*; use crate::rcu::{rcu_read_lock, RCUNode, RCUPointer, RCUReadGuard}; +// TODO: Implement slab reclaim +#[allow(unused)] const D_INVALID: u8 = 0; const D_REGULAR: u8 = 1; const D_DIRECTORY: u8 = 2; @@ -159,7 +161,7 @@ impl Dentry { && &***self.name() == &***other.name() } - pub fn name(&self) -> RCUReadGuard<BorrowedArc<Arc<[u8]>>> { + pub fn name(&self) -> RCUReadGuard<'_, BorrowedArc<'_, Arc<[u8]>>> { self.name.load().expect("Dentry has no name") } @@ -167,7 +169,7 @@ impl Dentry { (***self.name()).clone() } - pub fn parent<'a>(&self) -> RCUReadGuard<'a, BorrowedArc<Dentry>> { + pub fn parent<'a>(&self) -> RCUReadGuard<'a, BorrowedArc<'_, Dentry>> { self.parent.load().expect("Dentry has no parent") } diff --git a/src/kernel/vfs/types/device_id.rs b/src/kernel/vfs/types/device_id.rs index cf3ea886..6dd128ee 100644 --- a/src/kernel/vfs/types/device_id.rs +++ b/src/kernel/vfs/types/device_id.rs @@ -10,17 +10,6 @@ impl DeviceId { pub const fn new(major: u16, minor: u16) -> Self { Self { major, minor } } - - pub const fn from_raw(raw: u32) -> Self { - Self { - major: (raw >> 16) as u16, - minor: (raw & 0xFFFF) as u16, - } - } - - pub const fn to_raw(self) -> u32 { - ((self.major as u32) << 16) | (self.minor as u32) - } } impl Debug for DeviceId { diff --git a/src/lib.rs b/src/lib.rs index 8457169c..4f7fb262 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -38,6 +38,7 @@ use core::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; use eonix_hal::arch_exported::bootstrap::shutdown; use eonix_hal::context::TaskContext; use eonix_hal::processor::{halt, CPU, CPU_COUNT}; +use eonix_hal::symbol_addr; use eonix_hal::traits::context::RawTaskContext; use eonix_hal::traits::trap::IrqState; use eonix_hal::trap::disable_irqs_save; @@ -136,7 +137,7 @@ fn kernel_init(mut data: eonix_hal::bootstrap::BootStrapData) -> ! { bottom }; ctx.set_interrupt_enabled(true); - ctx.set_program_counter(standard_main as usize); + ctx.set_program_counter(symbol_addr!(standard_main)); ctx.set_stack_pointer(stack_bottom); unsafe { @@ -162,7 +163,7 @@ fn kernel_ap_main(_stack_range: PRange) -> ! { bottom }; ctx.set_interrupt_enabled(true); - ctx.set_program_counter(standard_main as usize); + ctx.set_program_counter(symbol_addr!(standard_main)); ctx.set_stack_pointer(stack_bottom); unsafe { diff --git a/src/path.rs b/src/path.rs index b342ef5f..47b9a4b6 100644 --- a/src/path.rs +++ b/src/path.rs @@ -1,6 +1,8 @@ -use crate::{kernel::constants::ENOENT, prelude::*}; use core::fmt::{self, Debug, Formatter}; +use crate::kernel::constants::ENOENT; +use crate::prelude::*; + #[repr(transparent)] pub struct Path { all: [u8], @@ -23,7 +25,7 @@ impl Path { self.all.starts_with(&['/' as u8]) } - pub fn iter(&self) -> PathIterator { + pub fn iter(&self) -> PathIterator<'_> { PathIterator::new(&self.all) } } diff --git a/src/sync/arcswap.rs b/src/sync/arcswap.rs index fb8219b2..7421659f 100644 --- a/src/sync/arcswap.rs +++ b/src/sync/arcswap.rs @@ -1,9 +1,8 @@ use alloc::sync::Arc; -use core::{ - fmt::{self, Debug, Formatter}, - ptr::NonNull, - sync::atomic::{AtomicPtr, Ordering}, -}; +use core::fmt::{self, Debug, Formatter}; +use core::ptr::NonNull; +use core::sync::atomic::{AtomicPtr, Ordering}; + use pointers::BorrowedArc; unsafe impl<T> Send for ArcSwap<T> where T: Send + Sync {} @@ -33,7 +32,7 @@ impl<T> ArcSwap<T> { } } - pub fn borrow(&self) -> BorrowedArc<T> { + pub fn borrow(&self) -> BorrowedArc<'_, T> { unsafe { BorrowedArc::from_raw( NonNull::new(self.pointer.load(Ordering::Acquire)) From e0cb58da5d3331df3ba3288e5dde642838b3205a Mon Sep 17 00:00:00 2001 From: greatbridf <greatbridf@icloud.com> Date: Sun, 18 Jan 2026 02:56:56 +0800 Subject: [PATCH 25/36] user, init: update riscv64 init script Make print message prettier. Also panic when we have an error. Signed-off-by: greatbridf <greatbridf@icloud.com> --- user-programs/init_script_riscv64-virt.sh | 88 +++++++++++++---------- 1 file changed, 52 insertions(+), 36 deletions(-) diff --git a/user-programs/init_script_riscv64-virt.sh b/user-programs/init_script_riscv64-virt.sh index 52b2628c..b5ce95d7 100644 --- a/user-programs/init_script_riscv64-virt.sh +++ b/user-programs/init_script_riscv64-virt.sh @@ -1,60 +1,75 @@ #!/mnt/busybox sh BUSYBOX=/mnt/busybox +TERMINAL=/dev/ttyS0 +VERBOSE= -freeze() { - echo "an error occurred while executing '''$@''', freezing..." >&2 +error() { + printf "\033[91merror: \033[0m%s\n" "$1" >&2 +} + +warn() { + printf "\033[93mwarn : \033[0m%s\n" "$1" >&2 +} + +info() { + printf "\033[92minfo : \033[0m%s\n" "$1" >&2 +} + +die() { + error "$1" && freeze +} +freeze() { + info "freezing..." >&2 while true; do - true + : done + + exit 1 } -do_or_freeze() { - if $@; then - return - fi +unrecoverable() { + die "unrecoverable error occurred. check the message above." +} - freeze $@ +busybox() { + $BUSYBOX "$@" } -do_or_freeze $BUSYBOX mkdir -p /dev +trap unrecoverable EXIT + +set -euo pipefail -do_or_freeze $BUSYBOX mknod -m 666 /dev/console c 5 1 -do_or_freeze $BUSYBOX mknod -m 666 /dev/null c 1 3 -do_or_freeze $BUSYBOX mknod -m 666 /dev/zero c 1 5 -do_or_freeze $BUSYBOX mknod -m 666 /dev/vda b 8 0 -do_or_freeze $BUSYBOX mknod -m 666 /dev/vda1 b 8 1 -do_or_freeze $BUSYBOX mknod -m 666 /dev/vdb b 8 16 -do_or_freeze $BUSYBOX mknod -m 666 /dev/ttyS0 c 4 64 -do_or_freeze $BUSYBOX mknod -m 666 /dev/ttyS1 c 4 65 +if [ -n "$VERBOSE" ]; then + set -x +fi -echo -n -e "deploying busybox... " >&2 +busybox mkdir -p /dev -do_or_freeze $BUSYBOX mkdir -p /bin -do_or_freeze $BUSYBOX --install -s /bin -do_or_freeze $BUSYBOX mkdir -p /lib +busybox mknod -m 666 /dev/console c 5 1 +busybox mknod -m 666 /dev/null c 1 3 +busybox mknod -m 666 /dev/zero c 1 5 +busybox mknod -m 666 /dev/vda b 8 0 +busybox mknod -m 666 /dev/vda1 b 8 1 +busybox mknod -m 666 /dev/vdb b 8 16 +busybox mknod -m 666 /dev/ttyS0 c 4 64 +busybox mknod -m 666 /dev/ttyS1 c 4 65 -export PATH="/bin" +info "deploying busybox..." -echo ok >&2 +busybox mkdir -p /bin /lib +busybox --install -s /bin -do_or_freeze mkdir -p /etc /root /proc -do_or_freeze mount -t procfs proc proc +info "done" -# Check if the device /dev/vdb is available and can be read -if dd if=/dev/vdb of=/dev/null bs=512 count=1; then - echo -n -e "Mounting the ext4 image... " >&2 - do_or_freeze mkdir -p /mnt1 - do_or_freeze mount -t ext4 /dev/vdb /mnt1 - echo ok >&2 -fi +export PATH="/bin" -cp /mnt/ld-musl-i386.so.1 /lib/ld-musl-i386.so.1 -ln -s /lib/ld-musl-i386.so.1 /lib/libc.so +mkdir -p /etc /root /proc +mount -t procfs proc proc cat > /etc/passwd <<EOF -root:x:0:0:root:/root:/mnt/busybox sh +root:x:0:0:root:/root:/bin/sh EOF cat > /etc/group <<EOF @@ -91,7 +106,8 @@ int main() { } EOF -exec $BUSYBOX sh -l < /dev/ttyS0 > /dev/ttyS0 2> /dev/ttyS0 +# shellcheck disable=SC2094 +exec sh -l < "$TERMINAL" > "$TERMINAL" 2> "$TERMINAL" # We don't have a working init yet, so we use busybox sh directly for now. # exec /mnt/init /bin/sh -c 'exec sh -l < /dev/ttyS0 > /dev/ttyS0 2> /dev/ttyS0' From fed6f4acbd0a4a8a6ba911c3c73937610910ee7b Mon Sep 17 00:00:00 2001 From: greatbridf <greatbridf@icloud.com> Date: Sun, 18 Jan 2026 21:35:39 +0800 Subject: [PATCH 26/36] mm, proc: add an exited thread reaper - thd.exit() will set thd.dead and send it to the reaper - delay the release of process mm until we reap it - extract futex logic out of exit and exec routine Signed-off-by: greatbridf <greatbridf@icloud.com> --- src/kernel/mem/mm_list.rs | 30 +++---- src/kernel/syscall/procops.rs | 26 ++---- src/kernel/task.rs | 5 +- src/kernel/task/futex.rs | 53 +++++++++--- src/kernel/task/process_list.rs | 144 ++++++++++++++++++++------------ src/kernel/task/signal.rs | 20 ++--- src/kernel/task/thread.rs | 49 ++++++++--- src/lib.rs | 23 +---- 8 files changed, 209 insertions(+), 141 deletions(-) diff --git a/src/kernel/mem/mm_list.rs b/src/kernel/mem/mm_list.rs index 5221c73b..f073025b 100644 --- a/src/kernel/mem/mm_list.rs +++ b/src/kernel/mem/mm_list.rs @@ -398,20 +398,6 @@ impl MMList { assert_ne!(old_user_count, 0); } - /// Deactivate `self` and activate `to` with root page table changed only once. - /// This might reduce the overhead of switching page tables twice. - #[allow(dead_code)] - pub fn switch(&self, to: &Self) { - self.user_count.fetch_add(1, Ordering::Acquire); - - let root_page_table = self.root_page_table.load(Ordering::Relaxed); - assert_ne!(root_page_table, 0); - set_root_page_table_pfn(PFN::from(PAddr::from(root_page_table))); - - let old_user_count = to.user_count.fetch_sub(1, Ordering::Release); - assert_ne!(old_user_count, 0); - } - /// Replace the current page table with a new one. /// /// # Safety @@ -454,10 +440,24 @@ impl MMList { // TODO: Check whether we should wake someone up if they've been put // to sleep when calling `vfork`. - self.inner + let old_mm = self + .inner .swap(new.map(|new_mm| new_mm.inner.swap(None)).flatten()); eonix_preempt::enable(); + + // This could take long... + drop(old_mm); + } + + pub fn release(&self) { + let old_mm = self.inner.swap(None); + let old_table = self.root_page_table.swap(0, Ordering::Relaxed); + + // TODO: Remove this completely... + // XXX: `ArcSwap` is broken and never safe to use. Check `replace` above. + assert_ne!(old_table, 0, "Already released?"); + assert!(old_mm.is_some(), "Already released?"); } /// No need to do invalidation manually, `PageTable` already does it. diff --git a/src/kernel/syscall/procops.rs b/src/kernel/syscall/procops.rs index 1359d0ab..3e815f25 100644 --- a/src/kernel/syscall/procops.rs +++ b/src/kernel/syscall/procops.rs @@ -22,8 +22,8 @@ use crate::kernel::constants::{ use crate::kernel::mem::PageBuffer; use crate::kernel::syscall::{User, UserMut}; use crate::kernel::task::{ - do_clone, futex_wait, futex_wake, parse_futexop, yield_now, CloneArgs, FutexFlags, FutexOp, - ProcessList, ProgramLoader, RobustListHead, SignalAction, Thread, WaitId, WaitType, + do_clone, futex_exec, futex_wait, futex_wake, parse_futexop, yield_now, CloneArgs, FutexFlags, + FutexOp, ProcessList, ProgramLoader, RobustListHead, SignalAction, Thread, WaitId, WaitType, }; use crate::kernel::timer::sleep; use crate::kernel::user::{UserBuffer, UserPointer, UserPointerMut, UserString}; @@ -213,10 +213,7 @@ async fn execve(exec: User<u8>, argv: User<PtrT>, envp: User<PtrT>) -> KResult<S .load() .await?; - if let Some(robust_list) = thread.get_robust_list() { - let _ = robust_list.wake_all().await; - thread.set_robust_list(None); - } + futex_exec(thread).await; unsafe { // SAFETY: We are doing execve, all other threads are terminated. @@ -240,24 +237,15 @@ async fn execve(exec: User<u8>, argv: User<PtrT>, envp: User<PtrT>) -> KResult<S #[eonix_macros::define_syscall(SYS_EXIT)] async fn exit(status: u32) -> SyscallNoReturn { - let mut procs = ProcessList::get().write().await; - - unsafe { - procs - .do_exit(&thread, WaitType::Exited(status), false) - .await; - } + thread.exit(WaitType::Exited(status)); SyscallNoReturn } #[eonix_macros::define_syscall(SYS_EXIT_GROUP)] async fn exit_group(status: u32) -> SyscallNoReturn { - let mut procs = ProcessList::get().write().await; - - unsafe { - procs.do_exit(&thread, WaitType::Exited(status), true).await; - } + // XXX: Send SIGKILL to our sibling threads. + thread.exit(WaitType::Exited(status)); SyscallNoReturn } @@ -856,7 +844,7 @@ async fn rt_sigreturn() -> KResult<SyscallNoReturn> { "`rt_sigreturn` failed in thread {} with error {err}!", thread.tid ); - thread.force_kill(Signal::SIGSEGV).await; + thread.force_kill(Signal::SIGSEGV); return Err(err); } diff --git a/src/kernel/task.rs b/src/kernel/task.rs index b0966046..6505666c 100644 --- a/src/kernel/task.rs +++ b/src/kernel/task.rs @@ -12,7 +12,10 @@ mod user_tls; pub use clone::{do_clone, CloneArgs, CloneFlags}; use eonix_hal::symbol_addr; -pub use futex::{futex_wait, futex_wake, parse_futexop, FutexFlags, FutexOp, RobustListHead}; +pub use futex::{ + futex_exec, futex_exit, futex_wait, futex_wake, parse_futexop, FutexFlags, FutexOp, + RobustListHead, +}; pub use kernel_stack::KernelStack; pub use loader::ProgramLoader; pub use process::{alloc_pid, Process, ProcessBuilder, WaitId, WaitObject, WaitType}; diff --git a/src/kernel/task/futex.rs b/src/kernel/task/futex.rs index a04d7091..4dd57615 100644 --- a/src/kernel/task/futex.rs +++ b/src/kernel/task/futex.rs @@ -1,19 +1,17 @@ -use core::pin::pin; - use alloc::sync::Arc; use alloc::vec::Vec; +use core::pin::pin; + use bitflags::bitflags; +use eonix_mm::address::Addr; use eonix_sync::{LazyLock, Mutex, MutexGuard, WaitList}; use intrusive_collections::{intrusive_adapter, KeyAdapter, RBTree, RBTreeAtomicLink}; -use crate::{ - kernel::{ - constants::{EAGAIN, EINVAL}, - syscall::User, - user::UserPointer, - }, - prelude::KResult, -}; +use super::Thread; +use crate::kernel::constants::{EAGAIN, EINVAL}; +use crate::kernel::syscall::User; +use crate::kernel::user::{UserPointer, UserPointerMut}; +use crate::prelude::KResult; #[derive(PartialEq, Debug, Clone, Copy)] #[repr(u32)] @@ -318,3 +316,38 @@ impl RobustListHead { Ok(()) } } + +async fn do_futex_exit(thread: &Thread) -> KResult<()> { + if let Some(clear_ctid) = thread.get_clear_ctid() { + UserPointerMut::new(clear_ctid)?.write(0u32)?; + + futex_wake(clear_ctid.addr(), None, 1).await?; + } + + if let Some(robust_list) = thread.get_robust_list() { + robust_list.wake_all().await?; + } + + Ok(()) +} + +pub async fn futex_exit(thread: &Thread) { + // We don't care about any error happened inside. + // If they've set up a wrong pointer, good luck to them... + let _ = do_futex_exit(thread); +} + +async fn do_futex_exec(thread: &Thread) -> KResult<()> { + if let Some(robust_list) = thread.get_robust_list() { + robust_list.wake_all().await?; + thread.set_robust_list(None); + } + + Ok(()) +} + +pub async fn futex_exec(thread: &Thread) { + // We don't care about any error happened inside. + // If they've set up a wrong pointer, good luck to them... + let _ = do_futex_exec(thread); +} diff --git a/src/kernel/task/process_list.rs b/src/kernel/task/process_list.rs index af073e84..c676d22e 100644 --- a/src/kernel/task/process_list.rs +++ b/src/kernel/task/process_list.rs @@ -1,16 +1,17 @@ +use alloc::collections::btree_map::BTreeMap; +use alloc::collections::vec_deque::VecDeque; +use alloc::sync::{Arc, Weak}; +use core::pin::pin; use core::sync::atomic::Ordering; -use super::{Process, ProcessGroup, Session, Thread, WaitObject, WaitType}; -use crate::{ - kernel::{task::futex_wake, user::UserPointerMut}, - rcu::rcu_sync, -}; -use alloc::{ - collections::btree_map::BTreeMap, - sync::{Arc, Weak}, +use eonix_runtime::scheduler::RUNTIME; +use eonix_sync::{AsProof as _, AsProofMut as _, RwLock, Spin, WaitList}; + +use super::loader::LoadInfo; +use super::{ + alloc_pid, Process, ProcessBuilder, ProcessGroup, Session, Thread, ThreadBuilder, WaitObject, }; -use eonix_mm::address::Addr; -use eonix_sync::{AsProof as _, AsProofMut as _, RwLock}; +use crate::rcu::rcu_sync; pub struct ProcessList { /// The init process. @@ -78,7 +79,7 @@ impl ProcessList { } } - pub fn set_init_process(&mut self, init: Arc<Process>) { + fn set_init_process(&mut self, init: Arc<Process>) { let old_init = self.init.replace(init); assert!(old_init.is_none(), "Init process already set"); } @@ -103,45 +104,66 @@ impl ProcessList { self.sessions.get(&sid).and_then(Weak::upgrade) } - /// Make the process a zombie and notify the parent. - /// # Safety - /// This function will destroy the process and all its threads. - /// It is the caller's responsibility to ensure that the process is not - /// running or will not run after this function is called. - pub async unsafe fn do_exit( - &mut self, - thread: &Thread, - exit_status: WaitType, - is_exiting_group: bool, - ) { - let process = thread.process.clone(); - - if process.pid == 1 { - panic!("init exited"); - } + pub async fn sys_init(load_info: LoadInfo) { + let thread_builder = ThreadBuilder::new() + .name(Arc::from(&b"busybox"[..])) + .entry(load_info.entry_ip, load_info.sp); - let inner = process.inner.access_mut(self.prove_mut()); + let mut process_list = ProcessList::get().write().await; + let (thread, process) = ProcessBuilder::new() + .pid(alloc_pid()) + .mm_list(load_info.mm_list) + .thread_builder(thread_builder) + .build(&mut process_list); - thread.dead.store(true, Ordering::SeqCst); + process_list.set_init_process(process); - if is_exiting_group { - // TODO: Send SIGKILL to all threads. - // todo!() - } + // TODO!!!: Remove this. + thread.files.open_console(); - if thread.tid != process.pid { - self.threads.remove(&thread.tid); - inner.threads.remove(&thread.tid).unwrap(); - } + RUNTIME.spawn(Reaper::daemon()); + RUNTIME.spawn(thread.run()); + } + + pub fn send_to_reaper(thread: Arc<Thread>) { + GLOBAL_REAPER.reap_list.lock().push_back(thread); + GLOBAL_REAPER.wait.notify_one(); + } +} + +struct Reaper { + reap_list: Spin<VecDeque<Arc<Thread>>>, + wait: WaitList, +} + +static GLOBAL_REAPER: Reaper = Reaper { + reap_list: Spin::new(VecDeque::new()), + wait: WaitList::new(), +}; - if let Some(clear_ctid) = thread.get_clear_ctid() { - let _ = UserPointerMut::new(clear_ctid).unwrap().write(0u32); +impl Reaper { + async fn reap(&self, thread: Arc<Thread>) { + let exit_status = thread + .exit_status + .lock() + .take() + .expect("Exited thread with no exit status"); - let _ = futex_wake(clear_ctid.addr(), None, 1).await; + let process = &thread.process; + + if process.pid == 1 && thread.tid == process.pid { + panic!("init exited"); } - if let Some(robust_list) = thread.get_robust_list() { - let _ = robust_list.wake_all().await; + let mut procs = ProcessList::get().write().await; + + let inner = process.inner.access_mut(procs.prove_mut()); + + thread.dead.store(true, Ordering::SeqCst); + + if thread.tid != process.pid { + procs.threads.remove(&thread.tid); + inner.threads.remove(&thread.tid).unwrap(); } // main thread exit @@ -151,48 +173,62 @@ impl ProcessList { thread.files.close_all().await; // If we are the session leader, we should drop the control terminal. - if process.session(self.prove()).sid == process.pid { - if let Some(terminal) = process.session(self.prove()).drop_control_terminal().await + if process.session(procs.prove()).sid == process.pid { + if let Some(terminal) = process.session(procs.prove()).drop_control_terminal().await { terminal.drop_session().await; } } // Release the MMList as well as the page table. - unsafe { - // SAFETY: We are exiting the process, so no one might be using it. - process.mm_list.replace(None); - } + process.mm_list.release(); // Make children orphans (adopted by init) { - let init = self.init_process(); + let init = procs.init_process(); inner.children.retain(|_, child| { let child = child.upgrade().unwrap(); // SAFETY: `child.parent` must be ourself. So we don't need to free it. unsafe { child.parent.swap(Some(init.clone())) }; - init.add_child(&child, self.prove_mut()); + init.add_child(&child, procs.prove_mut()); false }); } - let mut init_notify = self.init_process().notify_batch(); + let mut init_notify = procs.init_process().notify_batch(); process .wait_list .drain_exited() .into_iter() .for_each(|item| init_notify.notify(item)); - init_notify.finish(self.prove()); + init_notify.finish(procs.prove()); - process.parent(self.prove()).notify( + process.parent(procs.prove()).notify( process.exit_signal, WaitObject { pid: process.pid, code: exit_status, }, - self.prove(), + procs.prove(), ); } } + + async fn daemon() { + let me = &GLOBAL_REAPER; + + loop { + let mut wait = pin!(me.wait.prepare_to_wait()); + wait.as_mut().add_to_wait_list(); + + let thd_to_reap = me.reap_list.lock().pop_front(); + if let Some(thd_to_reap) = thd_to_reap { + me.reap(thd_to_reap).await; + continue; + } + + wait.await; + } + } } diff --git a/src/kernel/task/signal.rs b/src/kernel/task/signal.rs index d9970cad..0a7b580d 100644 --- a/src/kernel/task/signal.rs +++ b/src/kernel/task/signal.rs @@ -1,11 +1,10 @@ mod signal_action; -use super::{ProcessList, Thread, WaitObject, WaitType}; -use crate::kernel::constants::{EFAULT, EINVAL}; -use crate::{kernel::user::UserPointer, prelude::*}; use alloc::collections::binary_heap::BinaryHeap; use alloc::sync::Arc; -use core::{cmp::Reverse, task::Waker}; +use core::cmp::Reverse; +use core::task::Waker; + use eonix_hal::fpu::FpuState; use eonix_hal::traits::trap::RawTrapContext; use eonix_hal::trap::TrapContext; @@ -14,9 +13,13 @@ use eonix_sync::AsProof as _; use intrusive_collections::UnsafeRef; use posix_types::signal::{SigSet, Signal}; use posix_types::{SIGNAL_IGNORE, SIGNAL_NOW, SIGNAL_STOP}; +pub use signal_action::SignalAction; use signal_action::SignalActionList; -pub use signal_action::SignalAction; +use super::{ProcessList, Thread, WaitObject, WaitType}; +use crate::kernel::constants::{EFAULT, EINVAL}; +use crate::kernel::user::UserPointer; +use crate::prelude::*; pub(self) const SAVED_DATA_SIZE: usize = size_of::<TrapContext>() + size_of::<FpuState>() + size_of::<SigSet>(); @@ -168,10 +171,7 @@ impl SignalList { pub async fn handle(&self, trap_ctx: &mut TrapContext, fpu_state: &mut FpuState) { loop { let signal = { - let signal = match self.inner.lock().pop() { - Some(signal) => signal, - None => return, - }; + let Some(signal) = self.inner.lock().pop() else { return }; let handler = self.inner.lock().actions.get(signal); if let SignalAction::SimpleHandler { mask, .. } = &handler { @@ -246,7 +246,7 @@ impl SignalList { } signal => { // Default to terminate the thread. - Thread::current().force_kill(signal).await; + Thread::current().force_kill(signal); return; } } diff --git a/src/kernel/task/thread.rs b/src/kernel/task/thread.rs index 77e8e618..7e005875 100644 --- a/src/kernel/task/thread.rs +++ b/src/kernel/task/thread.rs @@ -24,8 +24,7 @@ use super::{stackful, Process, ProcessList, WaitType}; use crate::kernel::interrupt::default_irq_handler; use crate::kernel::syscall::{syscall_handlers, SyscallHandler, User, UserMut}; use crate::kernel::task::clone::CloneArgs; -use crate::kernel::task::futex::RobustListHead; -use crate::kernel::task::CloneFlags; +use crate::kernel::task::{futex_exit, CloneFlags, RobustListHead}; use crate::kernel::timer::{should_reschedule, timer_interrupt}; use crate::kernel::user::{UserPointer, UserPointerMut}; use crate::kernel::vfs::filearray::FileArray; @@ -83,6 +82,7 @@ pub struct Thread { pub fpu_state: AtomicUniqueRefCell<FpuState>, pub dead: AtomicBool, + pub exit_status: Spin<Option<WaitType>>, inner: Spin<ThreadInner>, } @@ -240,6 +240,7 @@ impl ThreadBuilder { trap_ctx: AtomicUniqueRefCell::new(trap_ctx), fpu_state: AtomicUniqueRefCell::new(fpu_state), dead: AtomicBool::new(false), + exit_status: Spin::new(None), inner: Spin::new(ThreadInner { name, tls: self.tls, @@ -331,18 +332,26 @@ impl Thread { } } - pub async fn force_kill(&self, signal: Signal) { - let mut proc_list = ProcessList::get().write().await; - unsafe { - // SAFETY: Preemption is disabled. - proc_list - .do_exit(self, WaitType::Signaled(signal), false) - .await; + pub fn exit(&self, exit_status: WaitType) { + { + let mut self_status = self.exit_status.lock(); + if self_status.is_some() { + // Someone has got here before us. + return; + } + + *self_status = Some(exit_status); } + + self.dead.store(true, Ordering::Release); + } + + pub fn force_kill(&self, signal: Signal) { + self.exit(WaitType::Signaled(signal)); } pub fn is_dead(&self) -> bool { - self.dead.load(Ordering::SeqCst) + self.dead.load(Ordering::Acquire) } async fn real_run(&self) { @@ -385,6 +394,10 @@ impl Thread { error_code, address: addr, }) => { + if self.is_dead() { + return; + } + let mms = &self.process.mm_list; if let Err(signal) = mms.handle_user_page_fault(addr, error_code).await { self.signal_list.raise(signal); @@ -407,6 +420,10 @@ impl Thread { } } TrapType::Syscall { no, args } => { + if self.is_dead() { + return; + } + if let Some(retval) = self.handle_syscall(thd_alloc, no, args).await { let mut trap_ctx = self.trap_ctx.borrow(); trap_ctx.set_user_return_value(retval); @@ -447,7 +464,17 @@ impl Thread { } pub fn run(self: Arc<Thread>) -> impl Future<Output = ()> + Send + 'static { - async move { self.contexted(stackful(self.real_run())).await } + async move { + self.contexted(async { + stackful(self.real_run()).await; + + futex_exit(&self).await; + }) + .await; + + assert!(self.is_dead(), "`real_run` returned before the thread die?"); + ProcessList::send_to_reaper(self); + } } } diff --git a/src/lib.rs b/src/lib.rs index 4f7fb262..2e28db24 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -31,7 +31,6 @@ mod rcu; mod sync; use alloc::ffi::CString; -use alloc::sync::Arc; use core::hint::spin_loop; use core::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; @@ -46,7 +45,7 @@ use eonix_mm::address::PRange; use eonix_runtime::executor::Stack; use eonix_runtime::scheduler::RUNTIME; use kernel::mem::GlobalPageAlloc; -use kernel::task::{KernelStack, ProcessBuilder, ProcessList, ProgramLoader, ThreadBuilder}; +use kernel::task::{KernelStack, ProcessList, ProgramLoader}; use kernel::vfs::dentry::Dentry; use kernel::vfs::mount::{do_mount, MS_NOATIME, MS_NODEV, MS_NOSUID, MS_RDONLY}; use kernel::vfs::types::Permission; @@ -56,8 +55,6 @@ use kernel_init::setup_memory; use path::Path; use prelude::*; -use crate::kernel::task::alloc_pid; - #[cfg(any(target_arch = "riscv64", target_arch = "loongarch64"))] fn do_panic() -> ! { #[cfg(target_arch = "riscv64")] @@ -276,21 +273,5 @@ async fn init_process(early_kstack: PRange) { .expect("Failed to load init program") }; - let thread_builder = ThreadBuilder::new() - .name(Arc::from(&b"busybox"[..])) - .entry(load_info.entry_ip, load_info.sp); - - let mut process_list = ProcessList::get().write().await; - let (thread, process) = ProcessBuilder::new() - .pid(alloc_pid()) - .mm_list(load_info.mm_list) - .thread_builder(thread_builder) - .build(&mut process_list); - - process_list.set_init_process(process); - - // TODO!!!: Remove this. - thread.files.open_console(); - - RUNTIME.spawn(thread.run()); + ProcessList::sys_init(load_info).await; } From 96803aab444caa3c9b211cc91241642148e3a499 Mon Sep 17 00:00:00 2001 From: greatbridf <greatbridf@icloud.com> Date: Wed, 21 Jan 2026 00:41:58 +0800 Subject: [PATCH 27/36] proc: rewrite process list organization - Use intrusive lists to store and organize the process hierarchy. - Remove `FileArray::open_console()`. Do it in the init script instead. - Fix open logic: acquire controlling terminals only if O_NOCTTY is not set. Put this into TerminalFile::open(). - Send SIGHUP and then SIGCONT to foreground pgroup procs when the controlling terminal is dropped. - Set the controlling terminal of sessions in Terminal. - Limit max line width to 80. Format some codes. Signed-off-by: greatbridf <greatbridf@icloud.com> --- .rustfmt.toml | 4 +- Cargo.toml | 4 +- crates/posix_types/src/open.rs | 6 +- src/kernel/chardev.rs | 84 +++---- src/kernel/syscall/file_rw.rs | 144 +++++++++--- src/kernel/task/process.rs | 270 ++++++++++++++-------- src/kernel/task/process_group.rs | 146 +++++++----- src/kernel/task/process_list.rs | 151 ++++++------ src/kernel/task/session.rs | 187 +++++++++------ src/kernel/task/thread.rs | 106 +++++++-- src/kernel/terminal.rs | 156 ++++++++----- src/kernel/vfs/file/terminal_file.rs | 68 ++++-- src/kernel/vfs/filearray.rs | 88 +++---- user-programs/init_script_riscv64-virt.sh | 6 +- 14 files changed, 927 insertions(+), 493 deletions(-) diff --git a/.rustfmt.toml b/.rustfmt.toml index 17b2bbc5..85b1cfc7 100644 --- a/.rustfmt.toml +++ b/.rustfmt.toml @@ -1,4 +1,4 @@ -max_width = 100 +max_width = 80 hard_tabs = false tab_spaces = 4 newline_style = "Auto" @@ -14,7 +14,7 @@ single_line_if_else_max_width = 60 single_line_let_else_max_width = 60 wrap_comments = false format_code_in_doc_comments = false -doc_comment_code_block_width = 100 +doc_comment_code_block_width = 80 comment_width = 80 normalize_comments = false normalize_doc_attributes = false diff --git a/Cargo.toml b/Cargo.toml index 4fcb6f70..214e5941 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -27,8 +27,10 @@ pointers = { path = "./crates/pointers" } posix_types = { path = "./crates/posix_types" } slab_allocator = { path = "./crates/slab_allocator" } +intrusive-collections = { version = "0.9.8", features = [ + "nightly", +], git = "https://github.com/greatbridf/intrusive-rs" } bitflags = "2.6.0" -intrusive-collections = { version = "0.9.8", git = "https://github.com/greatbridf/intrusive-rs" } itertools = { version = "0.13.0", default-features = false } acpi = "5.2.0" align_ext = "0.1.0" diff --git a/crates/posix_types/src/open.rs b/crates/posix_types/src/open.rs index 758ea331..7135e5b8 100644 --- a/crates/posix_types/src/open.rs +++ b/crates/posix_types/src/open.rs @@ -11,6 +11,8 @@ bitflags! { const O_CREAT = 0x40; /// Exclusive access, fail if file exists const O_EXCL = 0x80; + /// Don't set controlling terminal. + const O_NOCTTY = 0x100; /// Truncate file to zero length if it exists const O_TRUNC = 0x200; /// Open file in append mode @@ -116,6 +118,8 @@ impl AtFlags { } pub fn statx_default_sync(&self) -> bool { - !self.intersects(AtFlags::AT_STATX_FORCE_SYNC | AtFlags::AT_STATX_DONT_SYNC) + !self.intersects( + AtFlags::AT_STATX_FORCE_SYNC | AtFlags::AT_STATX_DONT_SYNC, + ) } } diff --git a/src/kernel/chardev.rs b/src/kernel/chardev.rs index 4e01d83a..e4a6e1b3 100644 --- a/src/kernel/chardev.rs +++ b/src/kernel/chardev.rs @@ -1,22 +1,18 @@ -use super::{ - console::get_console, - constants::{EEXIST, EIO}, - task::{block_on, ProcessList, Thread}, - terminal::Terminal, - vfs::{types::DeviceId, File, FileType, TerminalFile}, -}; -use crate::{ - io::{Buffer, Stream, StreamRead}, - prelude::*, -}; -use alloc::{ - boxed::Box, - collections::btree_map::{BTreeMap, Entry}, - sync::Arc, -}; -use eonix_sync::AsProof as _; +use alloc::boxed::Box; +use alloc::collections::btree_map::{BTreeMap, Entry}; +use alloc::sync::Arc; + use posix_types::open::OpenFlags; +use super::console::get_console; +use super::constants::{EEXIST, EIO}; +use super::task::{block_on, Thread}; +use super::terminal::Terminal; +use super::vfs::types::DeviceId; +use super::vfs::{File, FileType, TerminalFile}; +use crate::io::{Buffer, Stream, StreamRead}; +use crate::prelude::*; + pub trait VirtualCharDevice: Send + Sync { fn read(&self, buffer: &mut dyn Buffer) -> KResult<usize>; fn write(&self, stream: &mut dyn Stream) -> KResult<usize>; @@ -33,12 +29,15 @@ pub struct CharDevice { device: CharDeviceType, } -static CHAR_DEVICES: Spin<BTreeMap<DeviceId, Arc<CharDevice>>> = Spin::new(BTreeMap::new()); +static CHAR_DEVICES: Spin<BTreeMap<DeviceId, Arc<CharDevice>>> = + Spin::new(BTreeMap::new()); impl CharDevice { pub fn read(&self, buffer: &mut dyn Buffer) -> KResult<usize> { match &self.device { - CharDeviceType::Terminal(terminal) => block_on(terminal.read(buffer)), + CharDeviceType::Terminal(terminal) => { + block_on(terminal.read(buffer)) + } CharDeviceType::Virtual(device) => device.read(buffer), } } @@ -46,10 +45,12 @@ impl CharDevice { pub fn write(&self, stream: &mut dyn Stream) -> KResult<usize> { match &self.device { CharDeviceType::Virtual(device) => device.write(stream), - CharDeviceType::Terminal(terminal) => stream.read_till_end(&mut [0; 128], |data| { - terminal.write(data); - Ok(()) - }), + CharDeviceType::Terminal(terminal) => { + stream.read_till_end(&mut [0; 128], |data| { + terminal.write(data); + Ok(()) + }) + } } } @@ -57,7 +58,11 @@ impl CharDevice { CHAR_DEVICES.lock().get(&devid).cloned() } - pub fn register(devid: DeviceId, name: Arc<str>, device: CharDeviceType) -> KResult<()> { + pub fn register( + devid: DeviceId, + name: Arc<str>, + device: CharDeviceType, + ) -> KResult<()> { match CHAR_DEVICES.lock().entry(devid) { Entry::Vacant(entry) => { entry.insert(Arc::new(CharDevice { name, device })); @@ -67,26 +72,21 @@ impl CharDevice { } } - pub fn open(self: &Arc<Self>, flags: OpenFlags) -> KResult<File> { - Ok(match &self.device { + pub async fn open( + self: &Arc<Self>, + thread: &Thread, + flags: OpenFlags, + ) -> KResult<File> { + let file = match &self.device { + CharDeviceType::Virtual(_) => { + File::new(flags, FileType::CharDev(self.clone())) + } CharDeviceType::Terminal(terminal) => { - let procs = block_on(ProcessList::get().read()); - let current = Thread::current(); - let session = current.process.session(procs.prove()); - // We only set the control terminal if the process is the session leader. - if session.sid == Thread::current().process.pid { - // Silently fail if we can't set the control terminal. - dont_check!(block_on(session.set_control_terminal( - &terminal, - false, - procs.prove() - ))); - } - - TerminalFile::new(terminal.clone(), flags) + TerminalFile::open(thread, terminal, flags).await } - CharDeviceType::Virtual(_) => File::new(flags, FileType::CharDev(self.clone())), - }) + }; + + Ok(file) } } diff --git a/src/kernel/syscall/file_rw.rs b/src/kernel/syscall/file_rw.rs index 93a543d7..8ac9c22a 100644 --- a/src/kernel/syscall/file_rw.rs +++ b/src/kernel/syscall/file_rw.rs @@ -12,7 +12,8 @@ use posix_types::syscall_no::*; use super::{FromSyscallArg, User}; use crate::io::{Buffer, BufferFill, IntoStream}; use crate::kernel::constants::{ - EBADF, EFAULT, EINVAL, ENOENT, ENOSYS, ENOTDIR, SEEK_CUR, SEEK_END, SEEK_SET, + EBADF, EFAULT, EINVAL, ENOENT, ENOSYS, ENOTDIR, SEEK_CUR, SEEK_END, + SEEK_SET, }; use crate::kernel::syscall::UserMut; use crate::kernel::task::Thread; @@ -61,7 +62,13 @@ async fn dentry_from( let dir_file = thread.files.get(dirfd).ok_or(EBADF)?; let dir_dentry = dir_file.as_path().ok_or(ENOTDIR)?; - Dentry::open_at(&thread.fs_context, dir_dentry, path, follow_symlink).await + Dentry::open_at( + &thread.fs_context, + dir_dentry, + path, + follow_symlink, + ) + .await } } } @@ -79,7 +86,12 @@ async fn read(fd: FD, buffer: UserMut<u8>, bufsize: usize) -> KResult<usize> { } #[eonix_macros::define_syscall(SYS_PREAD64)] -async fn pread64(fd: FD, buffer: UserMut<u8>, bufsize: usize, offset: usize) -> KResult<usize> { +async fn pread64( + fd: FD, + buffer: UserMut<u8>, + bufsize: usize, + offset: usize, +) -> KResult<usize> { let mut buffer = UserBuffer::new(buffer, bufsize)?; thread @@ -104,7 +116,12 @@ async fn write(fd: FD, buffer: User<u8>, count: usize) -> KResult<usize> { } #[eonix_macros::define_syscall(SYS_PWRITE64)] -async fn pwrite64(fd: FD, buffer: User<u8>, count: usize, offset: usize) -> KResult<usize> { +async fn pwrite64( + fd: FD, + buffer: User<u8>, + count: usize, + offset: usize, +) -> KResult<usize> { let buffer = CheckedUserPointer::new(buffer, count)?; let mut stream = buffer.into_stream(); @@ -117,11 +134,17 @@ async fn pwrite64(fd: FD, buffer: User<u8>, count: usize, offset: usize) -> KRes } #[eonix_macros::define_syscall(SYS_OPENAT)] -async fn openat(dirfd: FD, pathname: User<u8>, flags: OpenFlags, mode: Mode) -> KResult<FD> { - let dentry = dentry_from(thread, dirfd, pathname, flags.follow_symlink()).await?; +async fn openat( + dirfd: FD, + pathname: User<u8>, + flags: OpenFlags, + mode: Mode, +) -> KResult<FD> { + let dentry = + dentry_from(thread, dirfd, pathname, flags.follow_symlink()).await?; let perm = mode.perm().mask_with(*thread.fs_context.umask.lock()); - thread.files.open(&dentry, flags, perm).await + thread.files.open(thread, &dentry, flags, perm).await } #[cfg(target_arch = "x86_64")] @@ -156,7 +179,8 @@ async fn dup3(old_fd: FD, new_fd: FD, flags: OpenFlags) -> KResult<FD> { #[eonix_macros::define_syscall(SYS_PIPE2)] async fn pipe2(pipe_fd: UserMut<[FD; 2]>, flags: OpenFlags) -> KResult<()> { - let mut buffer = UserBuffer::new(pipe_fd.cast(), core::mem::size_of::<[FD; 2]>())?; + let mut buffer = + UserBuffer::new(pipe_fd.cast(), core::mem::size_of::<[FD; 2]>())?; let (read_fd, write_fd) = thread.files.pipe(flags)?; buffer.copy(&[read_fd, write_fd])?.ok_or(EFAULT) @@ -170,7 +194,11 @@ async fn pipe(pipe_fd: UserMut<[FD; 2]>) -> KResult<()> { #[cfg(target_arch = "x86_64")] #[eonix_macros::define_syscall(SYS_GETDENTS)] -async fn getdents(fd: FD, buffer: UserMut<u8>, bufsize: usize) -> KResult<usize> { +async fn getdents( + fd: FD, + buffer: UserMut<u8>, + bufsize: usize, +) -> KResult<usize> { let mut buffer = UserBuffer::new(buffer, bufsize)?; thread @@ -184,7 +212,11 @@ async fn getdents(fd: FD, buffer: UserMut<u8>, bufsize: usize) -> KResult<usize> } #[eonix_macros::define_syscall(SYS_GETDENTS64)] -async fn getdents64(fd: FD, buffer: UserMut<u8>, bufsize: usize) -> KResult<usize> { +async fn getdents64( + fd: FD, + buffer: UserMut<u8>, + bufsize: usize, +) -> KResult<usize> { let mut buffer = UserBuffer::new(buffer, bufsize)?; thread @@ -230,7 +262,8 @@ async fn newfstatat( )] #[cfg_attr(target_arch = "x86_64", eonix_macros::define_syscall(SYS_FSTAT64))] async fn newfstat(fd: FD, statbuf: UserMut<Stat>) -> KResult<()> { - sys_newfstatat(thread, fd, User::null(), statbuf, AtFlags::AT_EMPTY_PATH).await + sys_newfstatat(thread, fd, User::null(), statbuf, AtFlags::AT_EMPTY_PATH) + .await } #[eonix_macros::define_syscall(SYS_STATX)] @@ -307,7 +340,11 @@ async fn unlink(pathname: User<u8>) -> KResult<()> { } #[eonix_macros::define_syscall(SYS_SYMLINKAT)] -async fn symlinkat(target: User<u8>, dirfd: FD, linkpath: User<u8>) -> KResult<()> { +async fn symlinkat( + target: User<u8>, + dirfd: FD, + linkpath: User<u8>, +) -> KResult<()> { let target = UserString::new(target)?; let dentry = dentry_from(thread, dirfd, linkpath, false).await?; @@ -341,7 +378,12 @@ impl UserDeviceId { } #[eonix_macros::define_syscall(SYS_MKNODAT)] -async fn mknodat(dirfd: FD, pathname: User<u8>, mut mode: Mode, dev: UserDeviceId) -> KResult<()> { +async fn mknodat( + dirfd: FD, + pathname: User<u8>, + mut mode: Mode, + dev: UserDeviceId, +) -> KResult<()> { if !mode.is_blk() && !mode.is_chr() { return Err(EINVAL); } @@ -354,7 +396,11 @@ async fn mknodat(dirfd: FD, pathname: User<u8>, mut mode: Mode, dev: UserDeviceI #[cfg(target_arch = "x86_64")] #[eonix_macros::define_syscall(SYS_MKNOD)] -async fn mknod(pathname: User<u8>, mode: Mode, dev: UserDeviceId) -> KResult<()> { +async fn mknod( + pathname: User<u8>, + mode: Mode, + dev: UserDeviceId, +) -> KResult<()> { sys_mknodat(thread, FD::AT_FDCWD, pathname, mode, dev).await } @@ -373,11 +419,20 @@ async fn readlinkat( #[cfg(target_arch = "x86_64")] #[eonix_macros::define_syscall(SYS_READLINK)] -async fn readlink(pathname: User<u8>, buffer: UserMut<u8>, bufsize: usize) -> KResult<usize> { +async fn readlink( + pathname: User<u8>, + buffer: UserMut<u8>, + bufsize: usize, +) -> KResult<usize> { sys_readlinkat(thread, FD::AT_FDCWD, pathname, buffer, bufsize).await } -async fn do_lseek(thread: &Thread, fd: FD, offset: u64, whence: u32) -> KResult<u64> { +async fn do_lseek( + thread: &Thread, + fd: FD, + offset: u64, + whence: u32, +) -> KResult<u64> { let file = thread.files.get(fd).ok_or(EBADF)?; Ok(match whence { @@ -403,7 +458,8 @@ async fn llseek( result: UserMut<u64>, whence: u32, ) -> KResult<()> { - let mut result = UserBuffer::new(result.cast(), core::mem::size_of::<u64>())?; + let mut result = + UserBuffer::new(result.cast(), core::mem::size_of::<u64>())?; let offset = ((offset_high as u64) << 32) | (offset_low as u64); let new_offset = do_lseek(thread, fd, offset, whence).await?; @@ -434,9 +490,10 @@ async fn readv(fd: FD, iov_user: User<IoVec>, iovcnt: u32) -> KResult<usize> { Ok(IoVec { len: Long::ZERO, .. }) => None, - Ok(IoVec { base, len }) => { - Some(UserBuffer::new(UserMut::with_addr(base.addr()), len.get())) - } + Ok(IoVec { base, len }) => Some(UserBuffer::new( + UserMut::with_addr(base.addr()), + len.get(), + )), }) .collect::<KResult<Vec<_>>>()?; @@ -471,8 +528,11 @@ async fn writev(fd: FD, iov_user: User<IoVec>, iovcnt: u32) -> KResult<usize> { len: Long::ZERO, .. }) => None, Ok(IoVec { base, len }) => Some( - CheckedUserPointer::new(User::with_addr(base.addr()), len.get()) - .map(|ptr| ptr.into_stream()), + CheckedUserPointer::new( + User::with_addr(base.addr()), + len.get(), + ) + .map(|ptr| ptr.into_stream()), ), }) .collect::<KResult<Vec<_>>>()?; @@ -491,7 +551,12 @@ async fn writev(fd: FD, iov_user: User<IoVec>, iovcnt: u32) -> KResult<usize> { } #[eonix_macros::define_syscall(SYS_FACCESSAT)] -async fn faccessat(dirfd: FD, pathname: User<u8>, _mode: u32, flags: AtFlags) -> KResult<()> { +async fn faccessat( + dirfd: FD, + pathname: User<u8>, + _mode: u32, + flags: AtFlags, +) -> KResult<()> { let dentry = if flags.at_empty_path() { let file = thread.files.get(dirfd).ok_or(EBADF)?; file.as_path().ok_or(EBADF)?.clone() @@ -522,7 +587,12 @@ async fn access(pathname: User<u8>, mode: u32) -> KResult<()> { } #[eonix_macros::define_syscall(SYS_SENDFILE64)] -async fn sendfile64(out_fd: FD, in_fd: FD, offset: UserMut<u8>, count: usize) -> KResult<usize> { +async fn sendfile64( + out_fd: FD, + in_fd: FD, + offset: UserMut<u8>, + count: usize, +) -> KResult<usize> { let in_file = thread.files.get(in_fd).ok_or(EBADF)?; let out_file = thread.files.get(out_fd).ok_or(EBADF)?; @@ -627,7 +697,11 @@ async fn pselect6( #[cfg(target_arch = "x86_64")] #[eonix_macros::define_syscall(SYS_POLL)] -async fn poll(fds: UserMut<UserPollFd>, nfds: u32, timeout: u32) -> KResult<u32> { +async fn poll( + fds: UserMut<UserPollFd>, + nfds: u32, + timeout: u32, +) -> KResult<u32> { do_poll(thread, fds, nfds, timeout).await } @@ -639,7 +713,8 @@ async fn fchownat( gid: u32, flags: AtFlags, ) -> KResult<()> { - let dentry = dentry_from(thread, dirfd, pathname, !flags.no_follow()).await?; + let dentry = + dentry_from(thread, dirfd, pathname, !flags.no_follow()).await?; if !dentry.is_valid() { return Err(ENOENT); } @@ -648,7 +723,12 @@ async fn fchownat( } #[eonix_macros::define_syscall(SYS_FCHMODAT)] -async fn fchmodat(dirfd: FD, pathname: User<u8>, mode: Mode, flags: AtFlags) -> KResult<()> { +async fn fchmodat( + dirfd: FD, + pathname: User<u8>, + mode: Mode, + flags: AtFlags, +) -> KResult<()> { let dentry = if flags.at_empty_path() { let file = thread.files.get(dirfd).ok_or(EBADF)?; file.as_path().ok_or(EBADF)?.clone() @@ -709,12 +789,16 @@ async fn renameat2( let flags = RenameFlags::from_bits(flags).ok_or(EINVAL)?; // The two flags RENAME_NOREPLACE and RENAME_EXCHANGE are mutually exclusive. - if flags.contains(RenameFlags::RENAME_NOREPLACE | RenameFlags::RENAME_EXCHANGE) { + if flags + .contains(RenameFlags::RENAME_NOREPLACE | RenameFlags::RENAME_EXCHANGE) + { Err(EINVAL)?; } - let old_dentry = dentry_from(thread, old_dirfd, old_pathname, false).await?; - let new_dentry = dentry_from(thread, new_dirfd, new_pathname, false).await?; + let old_dentry = + dentry_from(thread, old_dirfd, old_pathname, false).await?; + let new_dentry = + dentry_from(thread, new_dirfd, new_pathname, false).await?; old_dentry.rename(&new_dentry, flags).await } diff --git a/src/kernel/task/process.rs b/src/kernel/task/process.rs index 3eff5949..1385235d 100644 --- a/src/kernel/task/process.rs +++ b/src/kernel/task/process.rs @@ -1,22 +1,24 @@ -use alloc::collections::btree_map::BTreeMap; use alloc::collections::vec_deque::VecDeque; -use alloc::sync::{Arc, Weak}; +use alloc::sync::Arc; use core::sync::atomic::{AtomicU32, Ordering}; use eonix_sync::{ - AsProof as _, AsProofMut as _, Locked, Proof, ProofMut, RwLockReadGuard, SpinGuard, - UnlockableGuard as _, UnlockedGuard as _, + AsProof as _, AsProofMut as _, Locked, Proof, ProofMut, RwLockReadGuard, + SpinGuard, UnlockableGuard as _, UnlockedGuard as _, +}; +use intrusive_collections::{ + intrusive_adapter, KeyAdapter, RBTree, RBTreeAtomicLink, }; use pointers::BorrowedArc; use posix_types::constants::{ - CLD_CONTINUED, CLD_DUMPED, CLD_EXITED, CLD_KILLED, CLD_STOPPED, P_ALL, P_PGID, P_PID, P_PIDFD, + CLD_CONTINUED, CLD_DUMPED, CLD_EXITED, CLD_KILLED, CLD_STOPPED, P_ALL, + P_PGID, P_PID, P_PIDFD, }; use posix_types::signal::Signal; use posix_types::SIGNAL_COREDUMP; -use super::process_group::ProcessGroupBuilder; use super::signal::RaiseResult; -use super::thread::ThreadBuilder; +use super::thread::{ProcessThreads, ThreadBuilder}; use super::{ProcessGroup, ProcessList, Session, Thread}; use crate::kernel::constants::{ECHILD, EINTR, EINVAL, EPERM, ESRCH}; use crate::kernel::mem::MMList; @@ -35,7 +37,6 @@ pub struct ProcessBuilder { pid: Option<u32>, } -#[derive(Debug)] pub struct Process { /// Process id /// @@ -66,14 +67,55 @@ pub struct Process { /// The only case where it may be `None` is when the process is kernel thread. pub(super) session: RCUPointer<Session>, - /// All things related to the process list. - pub(super) inner: Locked<ProcessInner, ProcessList>, + pub children: Locked<RBTree<ProcessChildren>, ProcessList>, + pub threads: Locked<RBTree<ProcessThreads>, ProcessList>, + + all_procs_link: RBTreeAtomicLink, + group_procs_link: RBTreeAtomicLink, + siblings_link: RBTreeAtomicLink, } -#[derive(Debug)] -pub(super) struct ProcessInner { - pub(super) children: BTreeMap<u32, Weak<Process>>, - pub(super) threads: BTreeMap<u32, Weak<Thread>>, +intrusive_adapter!(pub AllProcs = Arc<Process>: Process { + all_procs_link: RBTreeAtomicLink +}); +intrusive_adapter!(pub GroupProcs = Arc<Process>: Process { + group_procs_link: RBTreeAtomicLink +}); +intrusive_adapter!(pub ProcessChildren = Arc<Process>: Process { + siblings_link: RBTreeAtomicLink +}); + +impl KeyAdapter<'_> for AllProcs { + type Key = u32; + + fn get_key( + &self, + value: &'_ <Self::PointerOps as intrusive_collections::PointerOps>::Value, + ) -> Self::Key { + value.pid + } +} + +impl KeyAdapter<'_> for GroupProcs { + type Key = u32; + + fn get_key( + &self, + value: &'_ <Self::PointerOps as intrusive_collections::PointerOps>::Value, + ) -> Self::Key { + value.pid + } +} + +impl KeyAdapter<'_> for ProcessChildren { + type Key = u32; + + fn get_key( + &self, + value: &'_ <Self::PointerOps as intrusive_collections::PointerOps>::Value, + ) -> Self::Key { + value.pid + } } #[derive(Debug)] @@ -148,7 +190,9 @@ impl WaitType { pub fn to_wstatus(self) -> u32 { match self { WaitType::Exited(status) => (status & 0xff) << 8, - WaitType::Signaled(signal @ SIGNAL_COREDUMP!()) => signal.into_raw() | 0x80, + WaitType::Signaled(signal @ SIGNAL_COREDUMP!()) => { + signal.into_raw() | 0x80 + } WaitType::Signaled(signal) => signal.into_raw(), WaitType::Stopped(signal) => 0x7f | (signal.into_raw() << 8), WaitType::Continued => 0xffff, @@ -159,7 +203,9 @@ impl WaitType { // TODO: CLD_TRAPPED match self { WaitType::Exited(status) => (status, CLD_EXITED), - WaitType::Signaled(signal @ SIGNAL_COREDUMP!()) => (signal.into_raw(), CLD_DUMPED), + WaitType::Signaled(signal @ SIGNAL_COREDUMP!()) => { + (signal.into_raw(), CLD_DUMPED) + } WaitType::Signaled(signal) => (signal.into_raw(), CLD_KILLED), WaitType::Stopped(signal) => (signal.into_raw(), CLD_STOPPED), WaitType::Continued => (Signal::SIGCONT.into_raw(), CLD_CONTINUED), @@ -194,7 +240,11 @@ impl ProcessBuilder { } } - pub async fn clone_from(mut self, process: Arc<Process>, clone_args: &CloneArgs) -> Self { + pub async fn clone_from( + mut self, + process: Arc<Process>, + clone_args: &CloneArgs, + ) -> Self { let mm_list = if clone_args.flags.contains(CloneFlags::CLONE_VM) { process.mm_list.new_shared().await } else { @@ -243,7 +293,10 @@ impl ProcessBuilder { self } - pub fn build(self, process_list: &mut ProcessList) -> (Arc<Thread>, Arc<Process>) { + pub fn build( + self, + process_list: &mut ProcessList, + ) -> (Arc<Thread>, Arc<Process>) { let mm_list = self.mm_list.unwrap_or_else(|| MMList::new()); let process = Arc::new(Process { @@ -254,18 +307,23 @@ impl ProcessBuilder { parent: RCUPointer::empty(), pgroup: RCUPointer::empty(), session: RCUPointer::empty(), - inner: Locked::new( - ProcessInner { - children: BTreeMap::new(), - threads: BTreeMap::new(), - }, + children: Locked::new( + RBTree::new(ProcessChildren::NEW), + process_list, + ), + threads: Locked::new( + RBTree::new(ProcessThreads::NEW), process_list, ), + all_procs_link: RBTreeAtomicLink::new(), + group_procs_link: RBTreeAtomicLink::new(), + siblings_link: RBTreeAtomicLink::new(), }); process_list.add_process(&process); - let thread_builder = self.thread_builder.expect("Thread builder is not set"); + let thread_builder = + self.thread_builder.expect("Thread builder is not set"); let thread = thread_builder .process(process.clone()) .tid(process.pid) @@ -281,10 +339,7 @@ impl ProcessBuilder { pgroup.add_member(&process, process_list.prove_mut()); pgroup } - None => ProcessGroupBuilder::new() - .leader(&process) - .session(session.clone()) - .build(process_list), + None => ProcessGroup::new(&process, &session, process_list), }; if let Some(parent) = &self.parent { @@ -304,30 +359,30 @@ impl ProcessBuilder { impl Process { pub fn raise(&self, signal: Signal, procs: Proof<'_, ProcessList>) { - let inner = self.inner.access(procs); - for thread in inner.threads.values().map(|t| t.upgrade().unwrap()) { + let threads = self.threads.access(procs); + for thread in threads.iter() { if let RaiseResult::Finished = thread.raise(signal) { break; } } } - pub(super) fn add_child(&self, child: &Arc<Process>, procs: ProofMut<'_, ProcessList>) { - assert!(self - .inner - .access_mut(procs) - .children - .insert(child.pid, Arc::downgrade(child)) - .is_none()); + pub fn add_child( + &self, + child: &Arc<Process>, + procs: ProofMut<'_, ProcessList>, + ) { + assert!(self.all_procs_link.is_linked(), "Dead process"); + self.children.access_mut(procs).insert(child.clone()); } - pub(super) fn add_thread(&self, thread: &Arc<Thread>, procs: ProofMut<'_, ProcessList>) { - assert!(self - .inner - .access_mut(procs) - .threads - .insert(thread.tid, Arc::downgrade(thread)) - .is_none()); + pub fn add_thread( + &self, + thread: &Arc<Thread>, + procs: ProofMut<'_, ProcessList>, + ) { + assert!(self.all_procs_link.is_linked(), "Dead process"); + self.threads.access_mut(procs).insert(thread.clone()); } pub async fn wait( @@ -354,12 +409,7 @@ impl Process { break object; } - if self - .inner - .access(waits.process_list.prove()) - .children - .is_empty() - { + if self.children.access(waits.process_list.prove()).is_empty() { return Err(ECHILD); } @@ -375,12 +425,12 @@ impl Process { Ok(Some(wait_object)) } else { let mut procs = ProcessList::get().write().await; - procs.remove_process(wait_object.pid).await; + procs.remove_process(wait_object.pid); assert!(self - .inner - .access_mut(procs.prove_mut()) .children - .remove(&wait_object.pid) + .access_mut(procs.prove_mut()) + .find_mut(&wait_object.pid) + .remove() .is_some()); Ok(Some(wait_object)) @@ -396,15 +446,17 @@ impl Process { if process_list.try_find_session(self.pid).is_some() { return Err(EPERM); } + + self.pgroup(process_list.prove()) + .remove_member(self, &mut process_list); + let session = Session::new(self, &mut process_list); - let pgroup = ProcessGroupBuilder::new() - .leader(self) - .session(session.clone()) - .build(&mut process_list); + let pgroup = ProcessGroup::new(self, &session, &mut process_list); - let old_session = unsafe { self.session.swap(Some(session.clone())) }.unwrap(); - let old_pgroup = unsafe { self.pgroup.swap(Some(pgroup.clone())) }.unwrap(); - old_pgroup.remove_member(self.pid, process_list.prove_mut()); + let old_session = + unsafe { self.session.swap(Some(session.clone())) }.unwrap(); + let old_pgroup = + unsafe { self.pgroup.swap(Some(pgroup.clone())) }.unwrap(); call_rcu(move || { drop(old_session); @@ -417,47 +469,56 @@ impl Process { /// Set the process group id of the process to `pgid`. /// /// This function does the actual work. - fn do_setpgid(self: &Arc<Self>, pgid: u32, procs: &mut ProcessList) -> KResult<()> { + fn do_setpgid( + self: &Arc<Self>, + pgid: u32, + procs: &mut ProcessList, + ) -> KResult<()> { // SAFETY: We are holding the process list lock. let session = unsafe { self.session.load_locked().unwrap() }; - let pgroup = unsafe { self.pgroup.load_locked().unwrap() }; // Changing the process group of a session leader is not allowed. if session.sid == self.pid { return Err(EPERM); } - let new_pgroup = if let Some(new_pgroup) = procs.try_find_pgroup(pgid) { + let cur_pgroup = self.pgroup(procs.prove()).clone(); + let existing_pgroup = procs.try_find_pgroup(pgid); + + if let Some(new_pgroup) = &existing_pgroup { // Move us to an existing process group. // Check that the two groups are in the same session. - if new_pgroup.session.upgrade().unwrap().sid != session.sid { + if new_pgroup.session.sid != session.sid { return Err(EPERM); } // If we are already in the process group, we are done. - if new_pgroup.pgid == pgroup.pgid { + if new_pgroup.pgid == cur_pgroup.pgid { return Ok(()); } - - new_pgroup.add_member(self, procs.prove_mut()); - - new_pgroup } else { // Create a new process group only if `pgid` matches our `pid`. if pgid != self.pid { return Err(EPERM); } + } - ProcessGroupBuilder::new() - .leader(self) - .session(session.clone()) - .build(procs) - }; + // Permission checks done. Let's do the actual work. + cur_pgroup.remove_member(self, procs); - pgroup.remove_member(self.pid, procs.prove_mut()); + let new_pgroup; + if let Some(pgroup) = existing_pgroup { + pgroup.add_member(self, procs.prove_mut()); + new_pgroup = pgroup; + } else { + new_pgroup = ProcessGroup::new(self, &session, procs); + } - let old_pgroup = unsafe { self.pgroup.swap(Some(new_pgroup)) }.unwrap(); - call_rcu(move || drop(old_pgroup)); + unsafe { + // SAFETY: `cur_pgroup` held above. + self.pgroup.swap(Some(new_pgroup)); + } + call_rcu(move || drop(cur_pgroup)); Ok(()) } @@ -475,15 +536,14 @@ impl Process { let child = { // If `pid` refers to one of our children, the thread leaders must be // in out children list. - let children = &self.inner.access(procs.prove()).children; - let child = { - let child = children.get(&pid); - child.and_then(Weak::upgrade).ok_or(ESRCH)? - }; + let children = self.children.access(procs.prove()); + let child = children.find(&pid).clone_pointer().ok_or(ESRCH)?; // Changing the process group of a child is only allowed // if we are in the same session. - if child.session(procs.prove()).sid != self.session(procs.prove()).sid { + if child.session(procs.prove()).sid + != self.session(procs.prove()).sid + { return Err(EPERM); } @@ -497,19 +557,28 @@ impl Process { } /// Provide locked (consistent) access to the session. - pub fn session<'r>(&'r self, _procs: Proof<'r, ProcessList>) -> BorrowedArc<'r, Session> { + pub fn session<'r>( + &'r self, + _procs: Proof<'r, ProcessList>, + ) -> BorrowedArc<'r, Session> { // SAFETY: We are holding the process list lock. unsafe { self.session.load_locked() }.unwrap() } /// Provide locked (consistent) access to the process group. - pub fn pgroup<'r>(&'r self, _procs: Proof<'r, ProcessList>) -> BorrowedArc<'r, ProcessGroup> { + pub fn pgroup<'r>( + &'r self, + _procs: Proof<'r, ProcessList>, + ) -> BorrowedArc<'r, ProcessGroup> { // SAFETY: We are holding the process list lock. unsafe { self.pgroup.load_locked() }.unwrap() } /// Provide locked (consistent) access to the parent process. - pub fn parent<'r>(&'r self, _procs: Proof<'r, ProcessList>) -> BorrowedArc<'r, Process> { + pub fn parent<'r>( + &'r self, + _procs: Proof<'r, ProcessList>, + ) -> BorrowedArc<'r, Process> { // SAFETY: We are holding the process list lock. unsafe { self.parent.load_locked() }.unwrap() } @@ -520,16 +589,25 @@ impl Process { } /// Provide RCU locked (maybe inconsistent) access to the process group. - pub fn pgroup_rcu(&self) -> RCUReadGuard<'_, BorrowedArc<'_, ProcessGroup>> { + pub fn pgroup_rcu( + &self, + ) -> RCUReadGuard<'_, BorrowedArc<'_, ProcessGroup>> { self.pgroup.load().unwrap() } /// Provide RCU locked (maybe inconsistent) access to the parent process. - pub fn parent_rcu(&self) -> Option<RCUReadGuard<'_, BorrowedArc<'_, Process>>> { + pub fn parent_rcu( + &self, + ) -> Option<RCUReadGuard<'_, BorrowedArc<'_, Process>>> { self.parent.load() } - pub fn notify(&self, signal: Option<Signal>, wait: WaitObject, procs: Proof<'_, ProcessList>) { + pub fn notify( + &self, + signal: Option<Signal>, + wait: WaitObject, + procs: Proof<'_, ProcessList>, + ) { self.wait_list.notify(wait); if let Some(signal) = signal { @@ -607,8 +685,11 @@ impl Entry<'_, '_, '_> { WaitId::Any => true, WaitId::Pid(pid) => item.pid == pid, WaitId::Pgid(pgid) => { - if let Some(process) = self.process_list.try_find_process(item.pid) { - return process.pgroup(self.process_list.prove()).pgid == pgid; + if let Some(process) = + self.process_list.try_find_process(item.pid) + { + return process.pgroup(self.process_list.prove()).pgid + == pgid; } false } @@ -622,7 +703,10 @@ impl Entry<'_, '_, '_> { } } - pub fn wait(self, no_block: bool) -> impl core::future::Future<Output = KResult<Self>> + Send { + pub fn wait( + self, + no_block: bool, + ) -> impl core::future::Future<Output = KResult<Self>> + Send { let wait_procs = self.wait_procs.unlock(); async move { diff --git a/src/kernel/task/process_group.rs b/src/kernel/task/process_group.rs index 137c5191..8c708b5c 100644 --- a/src/kernel/task/process_group.rs +++ b/src/kernel/task/process_group.rs @@ -1,87 +1,121 @@ -use super::{Process, ProcessList, Session}; -use alloc::{ - collections::btree_map::BTreeMap, - sync::{Arc, Weak}, +use alloc::sync::{Arc, Weak}; + +use eonix_sync::{AsProofMut, Locked, Proof, ProofMut}; +use intrusive_collections::{ + intrusive_adapter, KeyAdapter, RBTree, RBTreeAtomicLink, }; -use eonix_sync::{Locked, Proof, ProofMut}; use posix_types::signal::Signal; -pub struct ProcessGroupBuilder { - pgid: Option<u32>, - leader: Option<Weak<Process>>, - session: Option<Arc<Session>>, -} +use super::process::GroupProcs; +use super::{Process, ProcessList, Session}; -#[derive(Debug)] pub struct ProcessGroup { pub pgid: u32, - pub _leader: Weak<Process>, - pub session: Weak<Session>, + pub leader: Weak<Process>, + pub session: Arc<Session>, - pub processes: Locked<BTreeMap<u32, Weak<Process>>, ProcessList>, + pub procs: Locked<RBTree<GroupProcs>, ProcessList>, + + all_groups_link: RBTreeAtomicLink, + session_groups_link: RBTreeAtomicLink, } -impl ProcessGroupBuilder { - pub const fn new() -> Self { - Self { - pgid: None, - leader: None, - session: None, - } - } +intrusive_adapter!(pub AllGroups = Arc<ProcessGroup>: ProcessGroup { + all_groups_link: RBTreeAtomicLink +}); +intrusive_adapter!(pub SessionGroups = Arc<ProcessGroup>: ProcessGroup { + session_groups_link: RBTreeAtomicLink +}); + +impl KeyAdapter<'_> for AllGroups { + type Key = u32; - pub fn leader(mut self, leader: &Arc<Process>) -> Self { - self.pgid = Some(leader.pid); - self.leader = Some(Arc::downgrade(leader)); - self + fn get_key( + &self, + value: &'_ <Self::PointerOps as intrusive_collections::PointerOps>::Value, + ) -> Self::Key { + value.pgid } +} + +impl KeyAdapter<'_> for SessionGroups { + type Key = u32; - pub fn session(mut self, session: Arc<Session>) -> Self { - self.session = Some(session); - self + fn get_key( + &self, + value: &'_ <Self::PointerOps as intrusive_collections::PointerOps>::Value, + ) -> Self::Key { + value.pgid } +} - pub fn build(self, process_list: &mut ProcessList) -> Arc<ProcessGroup> { - let pgid = self.pgid.expect("PGID is not set"); - let leader = self.leader.expect("Leader is not set"); - let session = self.session.expect("Session is not set"); +impl ProcessGroup { + /// Create a pgroup and add it to the global pgroup list. + /// Add the pgroup to the session. + /// + /// # Panics + /// Panics if `leader` is already in some pgroup. + pub fn new( + leader: &Arc<Process>, + session: &Arc<Session>, + procs: &mut ProcessList, + ) -> Arc<Self> { + let pgid = leader.pid; + let pgroup_procs = { + let mut list = RBTree::new(GroupProcs::new()); + list.insert(leader.clone()); + list + }; let pgroup = Arc::new(ProcessGroup { pgid, - session: Arc::downgrade(&session), - processes: Locked::new(BTreeMap::from([(pgid, leader.clone())]), process_list), - _leader: leader, + session: session.clone(), + procs: Locked::new(pgroup_procs, procs), + leader: Arc::downgrade(leader), + all_groups_link: RBTreeAtomicLink::new(), + session_groups_link: RBTreeAtomicLink::new(), }); - process_list.add_pgroup(&pgroup); - session.add_member(process_list, &pgroup); + procs.add_pgroup(&pgroup); + session.add_member(&pgroup, procs.prove_mut()); pgroup } -} -impl ProcessGroup { - pub(super) fn add_member(&self, process: &Arc<Process>, procs: ProofMut<'_, ProcessList>) { - assert!(self - .processes - .access_mut(procs) - .insert(process.pid, Arc::downgrade(process)) - .is_none()); + /// Add `process` to the pgroup. + /// + /// # Panics + /// Panics if `process` is already in some pgroup or the pgroup is dead. + pub fn add_member( + &self, + process: &Arc<Process>, + procs: ProofMut<'_, ProcessList>, + ) { + assert!(self.all_groups_link.is_linked(), "Dead pgroup"); + self.procs.access_mut(procs).insert(process.clone()); } - pub(super) fn remove_member(&self, pid: u32, procs: ProofMut<'_, ProcessList>) { - let processes = self.processes.access_mut(procs); - assert!(processes.remove(&pid).is_some()); - if processes.is_empty() { - self.session - .upgrade() - .unwrap() - .remove_member(self.pgid, procs); + pub fn remove_member( + self: &Arc<Self>, + process: &Arc<Process>, + procs: &mut ProcessList, + ) { + let members = self.procs.access_mut(procs.prove_mut()); + assert!( + members.find_mut(&process.pid).remove().is_some(), + "Not a member" + ); + + if !members.is_empty() { + return; } + + self.session.remove_member(self, procs); + procs.remove_pgroup(self); } pub fn raise(&self, signal: Signal, procs: Proof<'_, ProcessList>) { - let processes = self.processes.access(procs); - for process in processes.values().map(|p| p.upgrade().unwrap()) { + let members = self.procs.access(procs); + for process in members.iter() { process.raise(signal, procs); } } diff --git a/src/kernel/task/process_list.rs b/src/kernel/task/process_list.rs index c676d22e..f3371f25 100644 --- a/src/kernel/task/process_list.rs +++ b/src/kernel/task/process_list.rs @@ -1,37 +1,41 @@ -use alloc::collections::btree_map::BTreeMap; use alloc::collections::vec_deque::VecDeque; -use alloc::sync::{Arc, Weak}; +use alloc::sync::Arc; use core::pin::pin; -use core::sync::atomic::Ordering; use eonix_runtime::scheduler::RUNTIME; use eonix_sync::{AsProof as _, AsProofMut as _, RwLock, Spin, WaitList}; +use intrusive_collections::RBTree; use super::loader::LoadInfo; +use super::process::AllProcs; +use super::process_group::AllGroups; +use super::session::AllSessions; +use super::thread::AllThreads; use super::{ - alloc_pid, Process, ProcessBuilder, ProcessGroup, Session, Thread, ThreadBuilder, WaitObject, + alloc_pid, Process, ProcessBuilder, ProcessGroup, Session, Thread, + ThreadBuilder, WaitObject, }; -use crate::rcu::rcu_sync; +use crate::rcu::call_rcu; pub struct ProcessList { /// The init process. init: Option<Arc<Process>>, /// All threads. - threads: BTreeMap<u32, Arc<Thread>>, + threads: RBTree<AllThreads>, /// All processes. - processes: BTreeMap<u32, Weak<Process>>, + procs: RBTree<AllProcs>, /// All process groups. - pgroups: BTreeMap<u32, Weak<ProcessGroup>>, + pgroups: RBTree<AllGroups>, /// All sessions. - sessions: BTreeMap<u32, Weak<Session>>, + sessions: RBTree<AllSessions>, } static GLOBAL_PROC_LIST: RwLock<ProcessList> = RwLock::new(ProcessList { init: None, - threads: BTreeMap::new(), - processes: BTreeMap::new(), - pgroups: BTreeMap::new(), - sessions: BTreeMap::new(), + threads: RBTree::new(AllThreads::NEW), + procs: RBTree::new(AllProcs::NEW), + pgroups: RBTree::new(AllGroups::NEW), + sessions: RBTree::new(AllSessions::NEW), }); impl ProcessList { @@ -40,43 +44,64 @@ impl ProcessList { } pub fn add_session(&mut self, session: &Arc<Session>) { - self.sessions.insert(session.sid, Arc::downgrade(session)); + self.sessions.insert(session.clone()); } pub fn add_pgroup(&mut self, pgroup: &Arc<ProcessGroup>) { - self.pgroups.insert(pgroup.pgid, Arc::downgrade(pgroup)); + self.pgroups.insert(pgroup.clone()); } pub fn add_process(&mut self, process: &Arc<Process>) { - self.processes.insert(process.pid, Arc::downgrade(process)); + self.procs.insert(process.clone()); } pub fn add_thread(&mut self, thread: &Arc<Thread>) { - self.threads.insert(thread.tid, thread.clone()); + self.threads.insert(thread.clone()); } - pub async fn remove_process(&mut self, pid: u32) { + pub fn remove_process(&mut self, pid: u32) { // Thread group leader has the same tid as the pid. - if let Some(thread) = self.threads.remove(&pid) { - self.processes.remove(&pid); - - // SAFETY: We wait until all references are dropped below with `rcu_sync()`. - let session = unsafe { thread.process.session.swap(None) }.unwrap(); - let pgroup = unsafe { thread.process.pgroup.swap(None) }.unwrap(); - let _parent = unsafe { thread.process.parent.swap(None) }.unwrap(); - pgroup.remove_member(pid, self.prove_mut()); - rcu_sync().await; - - if Arc::strong_count(&pgroup) == 1 { - self.pgroups.remove(&pgroup.pgid); - } + let Some(_) = self.threads.find_mut(&pid).remove() else { + panic!("Thread {} not found", pid); + }; - if Arc::strong_count(&session) == 1 { - self.sessions.remove(&session.sid); - } - } else { + let Some(proc) = self.procs.find_mut(&pid).remove() else { panic!("Process {} not found", pid); - } + }; + + // SAFETY: `call_rcu` below. + let session = unsafe { proc.session.swap(None) }.unwrap(); + let pgroup = unsafe { proc.pgroup.swap(None) }.unwrap(); + let parent = unsafe { proc.parent.swap(None) }.unwrap(); + + pgroup.remove_member(&proc, self); + + call_rcu(move || { + drop(session); + drop(pgroup); + drop(parent); + }); + } + + pub fn remove_thread(&mut self, thread: &Arc<Thread>) { + assert!( + self.threads.find_mut(&thread.tid).remove().is_some(), + "Double remove" + ); + } + + pub fn remove_session(&mut self, session: &Arc<Session>) { + assert!( + self.sessions.find_mut(&session.sid).remove().is_some(), + "Double remove" + ); + } + + pub fn remove_pgroup(&mut self, pgroup: &Arc<ProcessGroup>) { + assert!( + self.pgroups.find_mut(&pgroup.pgid).remove().is_some(), + "Double remove" + ); } fn set_init_process(&mut self, init: Arc<Process>) { @@ -88,20 +113,20 @@ impl ProcessList { self.init.as_ref().unwrap() } - pub fn try_find_thread(&self, tid: u32) -> Option<&Arc<Thread>> { - self.threads.get(&tid) + pub fn try_find_thread(&self, tid: u32) -> Option<Arc<Thread>> { + self.threads.find(&tid).clone_pointer() } pub fn try_find_process(&self, pid: u32) -> Option<Arc<Process>> { - self.processes.get(&pid).and_then(Weak::upgrade) + self.procs.find(&pid).clone_pointer() } pub fn try_find_pgroup(&self, pgid: u32) -> Option<Arc<ProcessGroup>> { - self.pgroups.get(&pgid).and_then(Weak::upgrade) + self.pgroups.find(&pgid).clone_pointer() } pub fn try_find_session(&self, sid: u32) -> Option<Arc<Session>> { - self.sessions.get(&sid).and_then(Weak::upgrade) + self.sessions.find(&sid).clone_pointer() } pub async fn sys_init(load_info: LoadInfo) { @@ -118,9 +143,6 @@ impl ProcessList { process_list.set_init_process(process); - // TODO!!!: Remove this. - thread.files.open_console(); - RUNTIME.spawn(Reaper::daemon()); RUNTIME.spawn(thread.run()); } @@ -152,18 +174,19 @@ impl Reaper { let process = &thread.process; if process.pid == 1 && thread.tid == process.pid { - panic!("init exited"); + panic!("init exited: {}", alloc_pid()); } let mut procs = ProcessList::get().write().await; - let inner = process.inner.access_mut(procs.prove_mut()); - - thread.dead.store(true, Ordering::SeqCst); - if thread.tid != process.pid { - procs.threads.remove(&thread.tid); - inner.threads.remove(&thread.tid).unwrap(); + let threads = process.threads.access_mut(procs.prove_mut()); + assert!( + threads.find_mut(&thread.tid).remove().is_some(), + "Thread gone?" + ); + + procs.remove_thread(&thread); } // main thread exit @@ -172,11 +195,11 @@ impl Reaper { thread.files.close_all().await; + let session = process.session(procs.prove()).clone(); // If we are the session leader, we should drop the control terminal. - if process.session(procs.prove()).sid == process.pid { - if let Some(terminal) = process.session(procs.prove()).drop_control_terminal().await - { - terminal.drop_session().await; + if session.sid == process.pid { + if let Some(terminal) = session.control_terminal() { + terminal.drop_session(procs.prove()); } } @@ -184,16 +207,14 @@ impl Reaper { process.mm_list.release(); // Make children orphans (adopted by init) - { - let init = procs.init_process(); - inner.children.retain(|_, child| { - let child = child.upgrade().unwrap(); - // SAFETY: `child.parent` must be ourself. So we don't need to free it. - unsafe { child.parent.swap(Some(init.clone())) }; - init.add_child(&child, procs.prove_mut()); - - false - }); + let init = procs.init_process(); + let children = process.children.access_mut(procs.prove_mut()); + for child in children.take() { + // XXX: May buggy. Check here again. + // SAFETY: `child.parent` must be ourself. + // So we don't need to free it. + unsafe { child.parent.swap(Some(init.clone())) }; + init.add_child(&child, procs.prove_mut()); } let mut init_notify = procs.init_process().notify_batch(); diff --git a/src/kernel/task/session.rs b/src/kernel/task/session.rs index a7b57afd..899aa395 100644 --- a/src/kernel/task/session.rs +++ b/src/kernel/task/session.rs @@ -1,117 +1,170 @@ -use super::{Process, ProcessGroup, ProcessList, Thread}; -use crate::kernel::constants::EPERM; -use crate::{kernel::Terminal, prelude::*}; -use alloc::{ - collections::btree_map::BTreeMap, - sync::{Arc, Weak}, +use alloc::sync::{Arc, Weak}; + +use eonix_sync::{AsProof as _, AsProofMut, Locked, Proof, ProofMut}; +use intrusive_collections::{ + intrusive_adapter, KeyAdapter, RBTree, RBTreeAtomicLink, }; -use eonix_sync::{AsProof as _, AsProofMut as _, Locked, Proof, ProofMut, RwLock}; use posix_types::signal::Signal; -#[derive(Debug)] +use super::process_group::SessionGroups; +use super::{Process, ProcessGroup, ProcessList}; +use crate::kernel::constants::EPERM; +use crate::kernel::Terminal; +use crate::prelude::*; + struct SessionJobControl { - /// Foreground process group - foreground: Weak<ProcessGroup>, + foreground: Option<Arc<ProcessGroup>>, control_terminal: Option<Arc<Terminal>>, } -#[allow(dead_code)] -#[derive(Debug)] pub struct Session { pub sid: u32, pub leader: Weak<Process>, - job_control: RwLock<SessionJobControl>, + job_control: Spin<SessionJobControl>, + groups: Locked<RBTree<SessionGroups>, ProcessList>, + all_sessions_link: RBTreeAtomicLink, +} - groups: Locked<BTreeMap<u32, Weak<ProcessGroup>>, ProcessList>, +intrusive_adapter!(pub AllSessions = Arc<Session>: Session { + all_sessions_link: RBTreeAtomicLink +}); + +impl KeyAdapter<'_> for AllSessions { + type Key = u32; + + fn get_key( + &self, + value: &'_ <Self::PointerOps as intrusive_collections::PointerOps>::Value, + ) -> Self::Key { + value.sid + } } impl Session { /// Create a session and add it to the global session list. - pub fn new(leader: &Arc<Process>, process_list: &mut ProcessList) -> Arc<Self> { + pub fn new(leader: &Arc<Process>, proclist: &mut ProcessList) -> Arc<Self> { let session = Arc::new(Self { sid: leader.pid, leader: Arc::downgrade(leader), - job_control: RwLock::new(SessionJobControl { - foreground: Weak::new(), + job_control: Spin::new(SessionJobControl { + foreground: None, control_terminal: None, }), - groups: Locked::new( - BTreeMap::new(), - // SAFETY: `procs` must be the global process list, which won't be moved. - process_list, - ), + groups: Locked::new(RBTree::new(SessionGroups::NEW), proclist), + all_sessions_link: RBTreeAtomicLink::new(), }); - process_list.add_session(&session); + proclist.add_session(&session); session } - pub(super) fn add_member(&self, procs: &mut ProcessList, pgroup: &Arc<ProcessGroup>) { - let groups = self.groups.access_mut(procs.prove_mut()); - let old = groups.insert(pgroup.pgid, Arc::downgrade(pgroup)); - assert!(old.is_none(), "Process group already exists"); + pub fn add_member( + &self, + pgroup: &Arc<ProcessGroup>, + procs: ProofMut<'_, ProcessList>, + ) { + assert!(self.all_sessions_link.is_linked(), "Dead session"); + self.groups.access_mut(procs).insert(pgroup.clone()); } - pub(super) fn remove_member(&self, pgid: u32, procs: ProofMut<'_, ProcessList>) { - assert!(self.groups.access_mut(procs).remove(&pgid).is_some()); + pub fn remove_member( + self: &Arc<Self>, + pgroup: &Arc<ProcessGroup>, + procs: &mut ProcessList, + ) { + let members = self.groups.access_mut(procs.prove_mut()); + assert!( + members.find_mut(&pgroup.pgid).remove().is_some(), + "Not a member" + ); + + if let Some(fg_pgroup) = self.foreground_pgroup() { + if fg_pgroup.pgid == pgroup.pgid { + let _ = self.set_foreground_pgroup(None); + } + } + + if !members.is_empty() { + return; + } + + // Recycle dead session. + procs.remove_session(self); } - pub async fn foreground(&self) -> Option<Arc<ProcessGroup>> { - self.job_control.read().await.foreground.upgrade() + pub fn leader(&self) -> Option<Arc<Process>> { + self.leader.upgrade() + } + + pub fn foreground_pgroup(&self) -> Option<Arc<ProcessGroup>> { + self.job_control.lock().foreground.clone() + } + + pub fn control_terminal(&self) -> Option<Arc<Terminal>> { + self.job_control.lock().control_terminal.clone() } /// Set the foreground process group identified by `pgid`. /// The process group must belong to the session. - pub async fn set_foreground_pgid( + pub fn set_foreground_pgroup( &self, - pgid: u32, - procs: Proof<'_, ProcessList>, + pgroup: Option<&Arc<ProcessGroup>>, ) -> KResult<()> { - if let Some(group) = self.groups.access(procs).get(&pgid) { - self.job_control.write().await.foreground = group.clone(); - Ok(()) - } else { - // TODO: Check if the process group refers to an existing process group. - // That's not a problem though, the operation will fail anyway. - Err(EPERM) + if let Some(pgroup) = pgroup { + if pgroup.session.sid != self.sid { + return Err(EPERM); + } } + + self.job_control.lock().foreground = pgroup.cloned(); + Ok(()) } - /// Only session leaders can set the control terminal. - /// Make sure we've checked that before calling this function. - pub async fn set_control_terminal( + /// Set our controlling terminal to `terminal`. Only meant to be called by + /// the session leader. The pgroup that the session leader is in becomes the + /// new foreground pgroup. + /// + /// # Panics + /// Panics if we have a controlling terminal already + /// or the session leader is gone. + pub fn _set_control_terminal( self: &Arc<Self>, terminal: &Arc<Terminal>, - forced: bool, procs: Proof<'_, ProcessList>, - ) -> KResult<()> { - let mut job_control = self.job_control.write().await; - if let Some(_) = job_control.control_terminal.as_ref() { - if let Some(session) = terminal.session().await.as_ref() { - if session.sid == self.sid { - return Ok(()); - } - } - return Err(EPERM); - } - terminal.set_session(self, forced).await?; + ) { + let mut job_control = self.job_control.lock(); + let leader = self.leader().expect("Leader is gone?"); + + assert!( + job_control.control_terminal.is_none(), + "We have a controlling terminal already" + ); + job_control.control_terminal = Some(terminal.clone()); - job_control.foreground = Arc::downgrade(&Thread::current().process.pgroup(procs)); - Ok(()) + job_control.foreground = Some(leader.pgroup(procs).clone()); } /// Drop the control terminal reference inside the session. - /// DO NOT TOUCH THE TERMINAL'S SESSION FIELD. - pub async fn drop_control_terminal(&self) -> Option<Arc<Terminal>> { - let mut inner = self.job_control.write().await; - inner.foreground = Weak::new(); - inner.control_terminal.take() + /// Send SIGHUP and then SIGCONT to our foreground pgroup. + pub fn _drop_control_terminal(&self, procs: Proof<'_, ProcessList>) { + let foreground = { + let mut inner = self.job_control.lock(); + inner.control_terminal = None; + inner.foreground.take() + }; + + if let Some(foreground) = foreground { + foreground.raise(Signal::SIGHUP, procs); + foreground.raise(Signal::SIGCHLD, procs); + } } pub async fn raise_foreground(&self, signal: Signal) { - if let Some(fg) = self.foreground().await { - let procs = ProcessList::get().read().await; - fg.raise(signal, procs.prove()); - } + let Some(fg) = self.foreground_pgroup() else { + return; + }; + + let procs = ProcessList::get().read().await; + fg.raise(signal, procs.prove()); } } diff --git a/src/kernel/task/thread.rs b/src/kernel/task/thread.rs index 7e005875..76c56dcc 100644 --- a/src/kernel/task/thread.rs +++ b/src/kernel/task/thread.rs @@ -14,6 +14,7 @@ use eonix_hal::traits::trap::{RawTrapContext, TrapReturn, TrapType}; use eonix_hal::trap::TrapContext; use eonix_mm::address::{Addr as _, VAddr}; use eonix_sync::AsProofMut as _; +use intrusive_collections::{intrusive_adapter, KeyAdapter, RBTreeAtomicLink}; use pointers::BorrowedArc; use posix_types::signal::Signal; use stalloc::UnsafeStalloc; @@ -84,9 +85,44 @@ pub struct Thread { pub dead: AtomicBool, pub exit_status: Spin<Option<WaitType>>, + /// Link in the global thread list. + all_threads_link: RBTreeAtomicLink, + + /// Link in the process's thread list. + process_threads_link: RBTreeAtomicLink, + inner: Spin<ThreadInner>, } +intrusive_adapter!(pub AllThreads = Arc<Thread>: Thread { + all_threads_link: RBTreeAtomicLink +}); +intrusive_adapter!(pub ProcessThreads = Arc<Thread>: Thread { + process_threads_link: RBTreeAtomicLink +}); + +impl KeyAdapter<'_> for AllThreads { + type Key = u32; + + fn get_key( + &self, + value: &'_ <Self::PointerOps as intrusive_collections::PointerOps>::Value, + ) -> Self::Key { + value.tid + } +} + +impl KeyAdapter<'_> for ProcessThreads { + type Key = u32; + + fn get_key( + &self, + value: &'_ <Self::PointerOps as intrusive_collections::PointerOps>::Value, + ) -> Self::Key { + value.tid + } +} + impl ThreadBuilder { pub fn new() -> Self { Self { @@ -139,12 +175,18 @@ impl ThreadBuilder { self } - pub fn set_child_tid(mut self, set_child_tid: Option<UserMut<u32>>) -> Self { + pub fn set_child_tid( + mut self, + set_child_tid: Option<UserMut<u32>>, + ) -> Self { self.set_child_tid = set_child_tid; self } - pub fn clear_child_tid(mut self, clear_child_tid: Option<UserMut<u32>>) -> Self { + pub fn clear_child_tid( + mut self, + clear_child_tid: Option<UserMut<u32>>, + ) -> Self { self.clear_child_tid = clear_child_tid; self } @@ -171,7 +213,11 @@ impl ThreadBuilder { } /// Clone the thread from another thread. - pub fn clone_from(self, thread: &Thread, clone_args: &CloneArgs) -> KResult<Self> { + pub fn clone_from( + self, + thread: &Thread, + clone_args: &CloneArgs, + ) -> KResult<Self> { let inner = thread.inner.lock(); let mut trap_ctx = thread.trap_ctx.borrow().clone(); @@ -199,11 +245,12 @@ impl ThreadBuilder { FileArray::new_cloned(&thread.files) }; - let signal_list = if clone_args.flags.contains(CloneFlags::CLONE_SIGHAND) { - SignalList::new_shared(&thread.signal_list) - } else { - SignalList::new_cloned(&thread.signal_list) - }; + let signal_list = + if clone_args.flags.contains(CloneFlags::CLONE_SIGHAND) { + SignalList::new_shared(&thread.signal_list) + } else { + SignalList::new_cloned(&thread.signal_list) + }; Ok(self .files(files) @@ -241,6 +288,8 @@ impl ThreadBuilder { fpu_state: AtomicUniqueRefCell::new(fpu_state), dead: AtomicBool::new(false), exit_status: Spin::new(None), + all_threads_link: RBTreeAtomicLink::new(), + process_threads_link: RBTreeAtomicLink::new(), inner: Spin::new(ThreadInner { name, tls: self.tls, @@ -281,7 +330,10 @@ impl Thread { Ok(()) } - pub fn set_robust_list(&self, robust_list_address: Option<User<RobustListHead>>) { + pub fn set_robust_list( + &self, + robust_list_address: Option<User<RobustListHead>>, + ) { self.inner.lock().robust_list_address = robust_list_address; } @@ -371,7 +423,10 @@ impl Thread { while !self.is_dead() { if self.signal_list.has_pending_signal() { self.signal_list - .handle(&mut self.trap_ctx.borrow(), &mut self.fpu_state.borrow()) + .handle( + &mut self.trap_ctx.borrow(), + &mut self.fpu_state.borrow(), + ) .await; } @@ -399,7 +454,9 @@ impl Thread { } let mms = &self.process.mm_list; - if let Err(signal) = mms.handle_user_page_fault(addr, error_code).await { + if let Err(signal) = + mms.handle_user_page_fault(addr, error_code).await + { self.signal_list.raise(signal); } } @@ -409,8 +466,12 @@ impl Thread { TrapType::Fault(Fault::InvalidOp) => { self.signal_list.raise(Signal::SIGILL); } - TrapType::Fault(Fault::Unknown(_)) => unimplemented!("Unhandled fault"), - TrapType::Breakpoint => unimplemented!("Breakpoint in user space"), + TrapType::Fault(Fault::Unknown(_)) => { + unimplemented!("Unhandled fault") + } + TrapType::Breakpoint => { + unimplemented!("Breakpoint in user space") + } TrapType::Irq { callback } => callback(default_irq_handler), TrapType::Timer { callback } => { callback(timer_interrupt); @@ -424,11 +485,16 @@ impl Thread { return; } - if let Some(retval) = self.handle_syscall(thd_alloc, no, args).await { + if let Some(retval) = + self.handle_syscall(thd_alloc, no, args).await + { let mut trap_ctx = self.trap_ctx.borrow(); trap_ctx.set_user_return_value(retval); - #[cfg(any(target_arch = "riscv64", target_arch = "loongarch64"))] + #[cfg(any( + target_arch = "riscv64", + target_arch = "loongarch64" + ))] { let pc = trap_ctx.get_program_counter(); trap_ctx.set_program_counter(pc + 4); @@ -472,7 +538,10 @@ impl Thread { }) .await; - assert!(self.is_dead(), "`real_run` returned before the thread die?"); + assert!( + self.is_dead(), + "`real_run` returned before the thread die?" + ); ProcessList::send_to_reaper(self); } } @@ -499,7 +568,10 @@ pub async fn yield_now() { impl Future for Yield { type Output = (); - fn poll(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Self::Output> { + fn poll( + mut self: Pin<&mut Self>, + cx: &mut Context<'_>, + ) -> Poll<Self::Output> { if self.as_mut().yielded { Poll::Ready(()) } else { diff --git a/src/kernel/terminal.rs b/src/kernel/terminal.rs index 734655b7..ddc3cc1f 100644 --- a/src/kernel/terminal.rs +++ b/src/kernel/terminal.rs @@ -1,18 +1,19 @@ -use super::{ - task::{ProcessList, Session, Thread}, - user::{UserPointer, UserPointerMut}, -}; -use crate::kernel::constants::{EINTR, ENOTTY, EPERM}; -use crate::{io::Buffer, prelude::*, sync::CondVar}; -use alloc::{ - collections::vec_deque::VecDeque, - sync::{Arc, Weak}, -}; +use alloc::collections::vec_deque::VecDeque; +use alloc::sync::{Arc, Weak}; + use bitflags::bitflags; use eonix_log::ConsoleWrite; -use eonix_sync::{AsProof as _, Mutex}; +use eonix_sync::{Mutex, Proof}; use posix_types::signal::Signal; +use super::constants::ESRCH; +use super::task::{ProcessList, Session, Thread}; +use super::user::{UserPointer, UserPointerMut}; +use crate::io::Buffer; +use crate::kernel::constants::{EINTR, ENOTTY, EPERM}; +use crate::prelude::*; +use crate::sync::CondVar; + const BUFFER_SIZE: usize = 4096; const NCCS: usize = 19; @@ -351,12 +352,12 @@ pub trait TerminalDevice: Send + Sync { struct TerminalInner { termio: Termios, - session: Weak<Session>, buffer: VecDeque<u8>, } pub struct Terminal { inner: Mutex<TerminalInner>, + session: Spin<Weak<Session>>, device: Arc<dyn TerminalDevice>, cv: CondVar, } @@ -400,9 +401,9 @@ impl Terminal { Arc::new(Self { inner: Mutex::new(TerminalInner { termio: Termios::new_standard(), - session: Weak::new(), buffer: VecDeque::with_capacity(BUFFER_SIZE), }), + session: Spin::new(Weak::new()), cv: CondVar::new(), device, }) @@ -447,15 +448,21 @@ impl Terminal { } async fn signal(&self, inner: &mut TerminalInner, signal: Signal) { - if let Some(session) = inner.session.upgrade() { + if let Some(session) = self.session() { session.raise_foreground(signal).await; } + if !inner.termio.noflsh() { self.clear_read_buffer(inner); } } - async fn echo_and_signal(&self, inner: &mut TerminalInner, ch: u8, signal: Signal) { + async fn echo_and_signal( + &self, + inner: &mut TerminalInner, + ch: u8, + signal: Signal, + ) { self.echo_char(inner, ch); self.signal(inner, signal).await; } @@ -481,13 +488,19 @@ impl Terminal { match ch { 0xff => {} ch if ch == inner.termio.vintr() => { - return self.echo_and_signal(&mut inner, ch, Signal::SIGINT).await + return self + .echo_and_signal(&mut inner, ch, Signal::SIGINT) + .await } ch if ch == inner.termio.vquit() => { - return self.echo_and_signal(&mut inner, ch, Signal::SIGQUIT).await + return self + .echo_and_signal(&mut inner, ch, Signal::SIGQUIT) + .await } ch if ch == inner.termio.vsusp() => { - return self.echo_and_signal(&mut inner, ch, Signal::SIGTSTP).await + return self + .echo_and_signal(&mut inner, ch, Signal::SIGTSTP) + .await } _ => {} } @@ -517,8 +530,12 @@ impl Terminal { match ch { b'\r' if inner.termio.igncr() => {} - b'\r' if inner.termio.icrnl() => return self.do_commit_char(&mut inner, b'\n'), - b'\n' if inner.termio.inlcr() => return self.do_commit_char(&mut inner, b'\r'), + b'\r' if inner.termio.icrnl() => { + return self.do_commit_char(&mut inner, b'\n') + } + b'\n' if inner.termio.inlcr() => { + return self.do_commit_char(&mut inner, b'\r') + } _ => self.do_commit_char(&mut inner, ch), } } @@ -589,26 +606,30 @@ impl Terminal { pub async fn ioctl(&self, request: TerminalIORequest<'_>) -> KResult<()> { match request { TerminalIORequest::GetProcessGroup(pgid_pointer) => { - if let Some(session) = self.inner.lock().await.session.upgrade() { - if let Some(pgroup) = session.foreground().await { - return pgid_pointer.write(pgroup.pgid); - } - } + let Some(session) = self.session() else { + return Err(ENOTTY); + }; + + let Some(pgroup) = session.foreground_pgroup() else { + return Err(ENOTTY); + }; - Err(ENOTTY) + pgid_pointer.write(pgroup.pgid) } TerminalIORequest::SetProcessGroup(pgid) => { let pgid = pgid.read()?; let procs = ProcessList::get().read().await; - let inner = self.inner.lock().await; - let session = inner.session.upgrade(); + let Some(session) = self.session() else { + return Err(ENOTTY); + }; - if let Some(session) = session { - session.set_foreground_pgid(pgid, procs.prove()).await - } else { - Err(ENOTTY) - } + let Some(pgroup) = procs.try_find_pgroup(pgid) else { + return Err(ESRCH); + }; + + session.set_foreground_pgroup(Some(&pgroup))?; + Ok(()) } TerminalIORequest::GetWindowSize(ptr) => { // TODO: Get the actual window size @@ -630,9 +651,12 @@ impl Terminal { let mut inner = self.inner.lock().await; // TODO: We ignore unknown bits for now. - inner.termio.iflag = TermioIFlags::from_bits_truncate(user_termios.iflag as u16); - inner.termio.oflag = TermioOFlags::from_bits_truncate(user_termios.oflag as u16); - inner.termio.lflag = TermioLFlags::from_bits_truncate(user_termios.lflag as u16); + inner.termio.iflag = + TermioIFlags::from_bits_truncate(user_termios.iflag as u16); + inner.termio.oflag = + TermioOFlags::from_bits_truncate(user_termios.oflag as u16); + inner.termio.lflag = + TermioLFlags::from_bits_truncate(user_termios.lflag as u16); inner.termio.cflag = user_termios.cflag; inner.termio.line = user_termios.line; inner.termio.cc = user_termios.cc; @@ -642,30 +666,52 @@ impl Terminal { } } - /// Assign the `session` to this terminal. Drop the previous session if `forced` is true. - pub async fn set_session(&self, session: &Arc<Session>, forced: bool) -> KResult<()> { - let mut inner = self.inner.lock().await; - if let Some(session) = inner.session.upgrade() { + pub fn session(&self) -> Option<Arc<Session>> { + self.session.lock().upgrade() + } + + /// Drop our current controlled session. The old session lose its controlling + /// terminal and all processes in it will receive a SIGHUP and then SIGCONT. + pub fn drop_session(&self, procs: Proof<'_, ProcessList>) { + let session = + core::mem::replace(&mut *self.session.lock(), Weak::new()); + let Some(old_session) = session.upgrade() else { + return; + }; + + old_session._drop_control_terminal(procs); + } + + /// Assign the `session` to this terminal. + /// Drop the previous session if `forced` is true. + pub async fn set_session( + self: &Arc<Self>, + session: &Arc<Session>, + forced: bool, + procs: Proof<'_, ProcessList>, + ) -> KResult<()> { + let mut cur_session = self.session.lock(); + + // XXX: Holding spinlock for too long? + if let Some(old_session) = cur_session.upgrade() { + if old_session.sid == session.sid { + return Ok(()); + } + if !forced { - Err(EPERM) - } else { - session.drop_control_terminal().await; - inner.session = Arc::downgrade(&session); - Ok(()) + return Err(EPERM); } - } else { - // Sessions should set their `control_terminal` field. - inner.session = Arc::downgrade(&session); - Ok(()) + + // TODO: Check whether the caller has the CAP_SYS_ADMIN capability. + + // We've stolen the terminal from the old session. + old_session._drop_control_terminal(procs); } - } - pub async fn drop_session(&self) { - self.inner.lock().await.session = Weak::new(); - } + *cur_session = Arc::downgrade(session); + session._set_control_terminal(self, procs); - pub async fn session(&self) -> Option<Arc<Session>> { - self.inner.lock().await.session.upgrade() + Ok(()) } } diff --git a/src/kernel/vfs/file/terminal_file.rs b/src/kernel/vfs/file/terminal_file.rs index f318c5b2..04a022b5 100644 --- a/src/kernel/vfs/file/terminal_file.rs +++ b/src/kernel/vfs/file/terminal_file.rs @@ -1,24 +1,46 @@ -use super::{File, FileType, PollEvent}; -use crate::{ - io::{Buffer, Stream, StreamRead}, - kernel::{ - constants::{EINVAL, TCGETS, TCSETS, TIOCGPGRP, TIOCGWINSZ, TIOCSPGRP}, - terminal::TerminalIORequest, - user::{UserPointer, UserPointerMut}, - Terminal, - }, - prelude::KResult, -}; use alloc::sync::Arc; + +use eonix_sync::AsProof; use posix_types::open::OpenFlags; +use super::{File, FileType, PollEvent}; +use crate::io::{Buffer, Stream, StreamRead}; +use crate::kernel::constants::{ + EINVAL, TCGETS, TCSETS, TIOCGPGRP, TIOCGWINSZ, TIOCSPGRP, +}; +use crate::kernel::task::{ProcessList, Thread}; +use crate::kernel::terminal::TerminalIORequest; +use crate::kernel::user::{UserPointer, UserPointerMut}; +use crate::kernel::Terminal; +use crate::prelude::KResult; + pub struct TerminalFile { terminal: Arc<Terminal>, } impl TerminalFile { - pub fn new(tty: Arc<Terminal>, flags: OpenFlags) -> File { - File::new(flags, FileType::Terminal(TerminalFile { terminal: tty })) + pub async fn open( + thread: &Thread, + terminal: &Arc<Terminal>, + flags: OpenFlags, + ) -> File { + let set_control_tty = !flags.contains(OpenFlags::O_NOCTTY); + + let procs = ProcessList::get().read().await; + let session = thread.process.session(procs.prove()); + + // We only set the control terminal if the process is the session leader. + if set_control_tty && session.sid == thread.process.pid { + // Silently fail if we can't set the control terminal. + let _ = terminal.set_session(&session, false, procs.prove()).await; + } + + File::new( + flags, + FileType::Terminal(TerminalFile { + terminal: terminal.clone(), + }), + ) } pub async fn read(&self, buffer: &mut dyn Buffer) -> KResult<usize> { @@ -43,11 +65,21 @@ impl TerminalFile { pub async fn ioctl(&self, request: usize, arg3: usize) -> KResult<()> { self.terminal .ioctl(match request as u32 { - TCGETS => TerminalIORequest::GetTermios(UserPointerMut::with_addr(arg3)?), - TCSETS => TerminalIORequest::SetTermios(UserPointer::with_addr(arg3)?), - TIOCGPGRP => TerminalIORequest::GetProcessGroup(UserPointerMut::with_addr(arg3)?), - TIOCSPGRP => TerminalIORequest::SetProcessGroup(UserPointer::with_addr(arg3)?), - TIOCGWINSZ => TerminalIORequest::GetWindowSize(UserPointerMut::with_addr(arg3)?), + TCGETS => TerminalIORequest::GetTermios( + UserPointerMut::with_addr(arg3)?, + ), + TCSETS => { + TerminalIORequest::SetTermios(UserPointer::with_addr(arg3)?) + } + TIOCGPGRP => TerminalIORequest::GetProcessGroup( + UserPointerMut::with_addr(arg3)?, + ), + TIOCSPGRP => TerminalIORequest::SetProcessGroup( + UserPointer::with_addr(arg3)?, + ), + TIOCGWINSZ => TerminalIORequest::GetWindowSize( + UserPointerMut::with_addr(arg3)?, + ), _ => return Err(EINVAL), }) .await diff --git a/src/kernel/vfs/filearray.rs b/src/kernel/vfs/filearray.rs index 609d969c..c0b6a49e 100644 --- a/src/kernel/vfs/filearray.rs +++ b/src/kernel/vfs/filearray.rs @@ -1,19 +1,22 @@ use alloc::sync::Arc; use intrusive_collections::rbtree::Entry; -use intrusive_collections::{intrusive_adapter, Bound, KeyAdapter, RBTree, RBTreeAtomicLink}; +use intrusive_collections::{ + intrusive_adapter, Bound, KeyAdapter, RBTree, RBTreeAtomicLink, +}; use itertools::FoldWhile::{Continue, Done}; use itertools::Itertools; use posix_types::open::{FDFlags, OpenFlags}; use super::file::{File, InodeFile, Pipe}; use super::types::{Format, Permission}; -use super::{Spin, TerminalFile}; -use crate::kernel::console::get_console; +use super::Spin; use crate::kernel::constants::{ - EBADF, EISDIR, ENOTDIR, ENXIO, F_DUPFD, F_DUPFD_CLOEXEC, F_GETFD, F_GETFL, F_SETFD, F_SETFL, + EBADF, EISDIR, ENOTDIR, ENXIO, F_DUPFD, F_DUPFD_CLOEXEC, F_GETFD, F_GETFL, + F_SETFD, F_SETFL, }; use crate::kernel::syscall::{FromSyscallArg, SyscallRetVal}; +use crate::kernel::task::Thread; use crate::kernel::vfs::dentry::Dentry; use crate::kernel::CharDevice; use crate::prelude::*; @@ -80,7 +83,11 @@ impl FDAllocator { self.min_avail = FD(0); } - fn find_available(&mut self, from: FD, files: &RBTree<OpenFileAdapter>) -> FD { + fn find_available( + &mut self, + from: FD, + files: &RBTree<OpenFileAdapter>, + ) -> FD { files .range(Bound::Included(&from), Bound::Unbounded) .fold_while(from, |current, OpenFile { fd, .. }| { @@ -143,7 +150,8 @@ impl FileArray { let other_inner = other.inner.lock(); for file in other_inner.files.iter() { - let new_file = OpenFile::new(file.fd, file.flags, file.file.dup()); + let new_file = + OpenFile::new(file.fd, file.flags, file.file.dup()); new_files.insert(new_file); } (new_files, other_inner.fd_alloc.clone()) @@ -223,7 +231,12 @@ impl FileArray { /// Duplicates the file to a new file descriptor, returning the old file /// description to be dropped. - fn dup_to_no_close(&self, old_fd: FD, new_fd: FD, fd_flags: FDFlags) -> KResult<Option<File>> { + fn dup_to_no_close( + &self, + old_fd: FD, + new_fd: FD, + fd_flags: FDFlags, + ) -> KResult<Option<File>> { let mut inner = self.inner.lock(); let (files, fd_alloc) = inner.split_borrow(); @@ -240,7 +253,8 @@ impl FileArray { Entry::Occupied(mut entry) => { let mut file = entry.remove().unwrap(); file.flags = fd_flags; - let old_file = core::mem::replace(&mut file.file, new_file_data); + let old_file = + core::mem::replace(&mut file.file, new_file_data); entry.insert(file); @@ -249,8 +263,15 @@ impl FileArray { } } - pub async fn dup_to(&self, old_fd: FD, new_fd: FD, flags: OpenFlags) -> KResult<FD> { - if let Some(old_file) = self.dup_to_no_close(old_fd, new_fd, flags.as_fd_flags())? { + pub async fn dup_to( + &self, + old_fd: FD, + new_fd: FD, + flags: OpenFlags, + ) -> KResult<FD> { + if let Some(old_file) = + self.dup_to_no_close(old_fd, new_fd, flags.as_fd_flags())? + { old_file.close().await; } @@ -277,6 +298,7 @@ impl FileArray { pub async fn open( &self, + thread: &Thread, dentry: &Arc<Dentry>, flags: OpenFlags, perm: Permission, @@ -300,7 +322,7 @@ impl FileArray { let file = if inode.format == Format::CHR { let device = CharDevice::get(inode.devid()?).ok_or(ENXIO)?; - device.open(flags)? + device.open(thread, flags).await? } else { InodeFile::new(dentry.clone(), flags) }; @@ -323,7 +345,8 @@ impl FileArray { F_DUPFD | F_DUPFD_CLOEXEC => { let ofile = cursor.get().ok_or(EBADF)?; - let cloexec = cmd == F_DUPFD_CLOEXEC || ofile.flags.close_on_exec(); + let cloexec = + cmd == F_DUPFD_CLOEXEC || ofile.flags.close_on_exec(); let flags = cloexec .then_some(FDFlags::FD_CLOEXEC) .unwrap_or(FDFlags::empty()); @@ -342,7 +365,9 @@ impl FileArray { cursor.insert(ofile); 0 } - F_GETFL => cursor.get().ok_or(EBADF)?.file.get_flags().bits() as usize, + F_GETFL => { + cursor.get().ok_or(EBADF)?.file.get_flags().bits() as usize + } F_SETFL => { cursor .get() @@ -357,35 +382,6 @@ impl FileArray { Ok(ret) } - - /// Only used for init process. - pub fn open_console(&self) { - let mut inner = self.inner.lock(); - let (files, fd_alloc) = inner.split_borrow(); - - let (stdin, stdout, stderr) = ( - fd_alloc.next_fd(files), - fd_alloc.next_fd(files), - fd_alloc.next_fd(files), - ); - let console_terminal = get_console().expect("No console terminal"); - - inner.do_insert( - stdin, - FDFlags::FD_CLOEXEC, - TerminalFile::new(console_terminal.clone(), OpenFlags::empty()), - ); - inner.do_insert( - stdout, - FDFlags::FD_CLOEXEC, - TerminalFile::new(console_terminal.clone(), OpenFlags::empty()), - ); - inner.do_insert( - stderr, - FDFlags::FD_CLOEXEC, - TerminalFile::new(console_terminal.clone(), OpenFlags::empty()), - ); - } } impl FileArrayInner { @@ -397,7 +393,9 @@ impl FileArrayInner { fn do_insert(&mut self, fd: FD, flags: FDFlags, file: File) { match self.files.entry(&fd) { Entry::Occupied(_) => { - panic!("File descriptor {fd:?} already exists in the file array."); + panic!( + "File descriptor {fd:?} already exists in the file array." + ); } Entry::Vacant(insert_cursor) => { insert_cursor.insert(OpenFile::new(fd, flags, file)); @@ -405,7 +403,9 @@ impl FileArrayInner { } } - fn split_borrow(&mut self) -> (&mut RBTree<OpenFileAdapter>, &mut FDAllocator) { + fn split_borrow( + &mut self, + ) -> (&mut RBTree<OpenFileAdapter>, &mut FDAllocator) { let Self { files, fd_alloc } = self; (files, fd_alloc) } diff --git a/user-programs/init_script_riscv64-virt.sh b/user-programs/init_script_riscv64-virt.sh index b5ce95d7..f67e2a27 100644 --- a/user-programs/init_script_riscv64-virt.sh +++ b/user-programs/init_script_riscv64-virt.sh @@ -56,6 +56,9 @@ busybox mknod -m 666 /dev/vdb b 8 16 busybox mknod -m 666 /dev/ttyS0 c 4 64 busybox mknod -m 666 /dev/ttyS1 c 4 65 +exec < "$TERMINAL" +exec > "$TERMINAL" 2>&1 + info "deploying busybox..." busybox mkdir -p /bin /lib @@ -106,8 +109,7 @@ int main() { } EOF -# shellcheck disable=SC2094 -exec sh -l < "$TERMINAL" > "$TERMINAL" 2> "$TERMINAL" +exec sh -l # We don't have a working init yet, so we use busybox sh directly for now. # exec /mnt/init /bin/sh -c 'exec sh -l < /dev/ttyS0 > /dev/ttyS0 2> /dev/ttyS0' From 2c1038e79a77defab0f020fe5c951b32a2ca3d7e Mon Sep 17 00:00:00 2001 From: greatbridf <greatbridf@icloud.com> Date: Sat, 24 Jan 2026 01:31:13 +0800 Subject: [PATCH 28/36] style: reformat the files related to next patches Reformat the files with new the format style to make the real changes clearer. Signed-off-by: greatbridf <greatbridf@icloud.com> --- .../eonix_hal/src/arch/riscv64/bootstrap.rs | 50 +++++++++++++----- crates/eonix_hal/src/arch/riscv64/mm.rs | 52 ++++++++++++------- src/kernel_init.rs | 3 +- 3 files changed, 70 insertions(+), 35 deletions(-) diff --git a/crates/eonix_hal/src/arch/riscv64/bootstrap.rs b/crates/eonix_hal/src/arch/riscv64/bootstrap.rs index b2305f99..4e5afcfb 100644 --- a/crates/eonix_hal/src/arch/riscv64/bootstrap.rs +++ b/crates/eonix_hal/src/arch/riscv64/bootstrap.rs @@ -7,8 +7,12 @@ use core::sync::atomic::{AtomicBool, AtomicPtr, AtomicUsize, Ordering}; use eonix_hal_traits::mm::Memory; use eonix_mm::address::{Addr as _, PAddr, PRange, PhysAccess, VAddr, VRange}; -use eonix_mm::page_table::{PageAttribute, PageTable, PagingMode, TableAttribute, PTE as _}; -use eonix_mm::paging::{Folio, FrameAlloc, PageAccess, PageBlock, PAGE_SIZE, PFN}; +use eonix_mm::page_table::{ + PageAttribute, PageTable, PagingMode, TableAttribute, PTE as _, +}; +use eonix_mm::paging::{ + Folio, FrameAlloc, PageAccess, PageBlock, PAGE_SIZE, PFN, +}; use eonix_percpu::PercpuArea; use fdt::Fdt; use riscv::asm::sfence_vma_all; @@ -25,11 +29,13 @@ use super::time::set_next_timer; use crate::arch::cpu::CPU; use crate::arch::fdt::{init_dtb_and_fdt, FdtExt, FDT}; use crate::arch::mm::{ - ArchPagingMode, ArchPhysAccess, FreeRam, PageAccessImpl, PageAttribute64, RawPageTableSv48, - GLOBAL_PAGE_TABLE, + ArchPagingMode, ArchPhysAccess, FreeRam, PageAccessImpl, PageAttribute64, + RawPageTableSv48, GLOBAL_PAGE_TABLE, }; use crate::bootstrap::BootStrapData; -use crate::mm::{ArchMemory, BasicPageAlloc, BasicPageAllocRef, ScopedAllocator}; +use crate::mm::{ + ArchMemory, BasicPageAlloc, BasicPageAllocRef, ScopedAllocator, +}; #[unsafe(link_section = ".bootstrap.stack")] static BOOT_STACK: [u8; 4096 * 16] = [0; 4096 * 16]; @@ -64,7 +70,8 @@ static PT1: BootPageTable = { BootPageTable(arr) }; -static BSP_PAGE_ALLOC: AtomicPtr<RefCell<BasicPageAlloc>> = AtomicPtr::new(core::ptr::null_mut()); +static BSP_PAGE_ALLOC: AtomicPtr<RefCell<BasicPageAlloc>> = + AtomicPtr::new(core::ptr::null_mut()); static AP_COUNT: AtomicUsize = AtomicUsize::new(0); static AP_STACK: AtomicUsize = AtomicUsize::new(0); @@ -130,11 +137,14 @@ pub unsafe extern "C" fn riscv64_start(hart_id: usize, dtb_addr: PAddr) -> ! { } let start = unsafe { - ((&BOOT_STACK_START) as *const &'static [u8; 4096 * 16]).read_volatile() as *const _ - as usize + ((&BOOT_STACK_START) as *const &'static [u8; 4096 * 16]).read_volatile() + as *const _ as usize }; let bootstrap_data = BootStrapData { - early_stack: PRange::new(PAddr::from(start), PAddr::from(start + 4096 * 16)), + early_stack: PRange::new( + PAddr::from(start), + PAddr::from(start + 4096 * 16), + ), allocator: Some(real_allocator), }; @@ -179,7 +189,11 @@ fn setup_kernel_page_table(alloc: BasicPageAllocRef) { sfence_vma_all(); unsafe { - core::ptr::write_bytes(KERNEL_BSS_START.addr() as *mut (), 0, BSS_LENGTH as usize); + core::ptr::write_bytes( + KERNEL_BSS_START.addr() as *mut (), + 0, + BSS_LENGTH as usize, + ); } unsafe { @@ -255,7 +269,8 @@ fn bootstrap_smp(alloc: impl Allocator, page_alloc: &RefCell<BasicPageAlloc>) { stack_range }; - let old = BSP_PAGE_ALLOC.swap((&raw const *page_alloc) as *mut _, Ordering::Release); + let old = BSP_PAGE_ALLOC + .swap((&raw const *page_alloc) as *mut _, Ordering::Release); assert!(old.is_null()); while AP_STACK @@ -324,7 +339,12 @@ unsafe extern "C" fn _ap_start(hart_id: usize) { fn get_ap_stack() -> usize { while AP_SEM - .compare_exchange_weak(false, true, Ordering::Acquire, Ordering::Relaxed) + .compare_exchange_weak( + false, + true, + Ordering::Acquire, + Ordering::Relaxed, + ) .is_err() { core::hint::spin_loop(); @@ -344,12 +364,14 @@ fn get_ap_stack() -> usize { } fn ap_entry(hart_id: usize, stack_bottom: PAddr) -> ! { - let stack_range = PRange::new(stack_bottom - (1 << 3) * PAGE_SIZE, stack_bottom); + let stack_range = + PRange::new(stack_bottom - (1 << 3) * PAGE_SIZE, stack_bottom); { // SAFETY: Acquire all the work done by the BSP and other APs. let alloc = loop { - let alloc = BSP_PAGE_ALLOC.swap(core::ptr::null_mut(), Ordering::AcqRel); + let alloc = + BSP_PAGE_ALLOC.swap(core::ptr::null_mut(), Ordering::AcqRel); if !alloc.is_null() { break alloc; diff --git a/crates/eonix_hal/src/arch/riscv64/mm.rs b/crates/eonix_hal/src/arch/riscv64/mm.rs index f67646cf..45d44c6f 100644 --- a/crates/eonix_hal/src/arch/riscv64/mm.rs +++ b/crates/eonix_hal/src/arch/riscv64/mm.rs @@ -4,8 +4,8 @@ use core::ptr::NonNull; use eonix_hal_traits::mm::Memory; use eonix_mm::address::{Addr as _, AddrOps, PAddr, PRange, PhysAccess, VAddr}; use eonix_mm::page_table::{ - PageAttribute, PageTable, PageTableLevel, PagingMode, RawAttribute, RawPageTable, - TableAttribute, PTE, + PageAttribute, PageTable, PageTableLevel, PagingMode, RawAttribute, + RawPageTable, TableAttribute, PTE, }; use eonix_mm::paging::{BasicFolio, Folio, PageAccess, PageBlock, PFN}; use eonix_sync_base::LazyLock; @@ -115,7 +115,9 @@ impl RawAttribute for PageAttribute64 { table_attr |= TableAttribute::PRESENT; } - if table_attr.contains(TableAttribute::PRESENT) && self.0 & (PA_R | PA_W | PA_X) != 0 { + if table_attr.contains(TableAttribute::PRESENT) + && self.0 & (PA_R | PA_W | PA_X) != 0 + { return None; } @@ -139,7 +141,9 @@ impl RawAttribute for PageAttribute64 { page_attr |= PageAttribute::PRESENT; } - if page_attr.contains(PageAttribute::PRESENT) && (self.0 & (PA_R | PA_W | PA_X) == 0) { + if page_attr.contains(PageAttribute::PRESENT) + && (self.0 & (PA_R | PA_W | PA_X) == 0) + { return None; } @@ -278,18 +282,22 @@ impl Memory for ArchMemory { let kernel_end = PAddr::from(__kernel_end as usize - KIMAGE_OFFSET); let paddr_after_kimage_aligned = kernel_end.ceil_to(0x200000); - core::iter::once(PRange::new(kernel_end, paddr_after_kimage_aligned)).chain( - Self::present_ram() - .filter(move |range| range.end() > paddr_after_kimage_aligned) - .map(move |range| { - if range.start() < paddr_after_kimage_aligned { - let (_, right) = range.split_at(paddr_after_kimage_aligned); - right - } else { - range - } - }), - ) + core::iter::once(PRange::new(kernel_end, paddr_after_kimage_aligned)) + .chain( + Self::present_ram() + .filter(move |range| { + range.end() > paddr_after_kimage_aligned + }) + .map(move |range| { + if range.start() < paddr_after_kimage_aligned { + let (_, right) = + range.split_at(paddr_after_kimage_aligned); + right + } else { + range + } + }), + ) } } @@ -314,17 +322,21 @@ where let kernel_end = PAddr::from(__kernel_end as usize - KIMAGE_OFFSET); let paddr_after_kimage_aligned = kernel_end.ceil_to(0x200000); - core::iter::once(PRange::new(kernel_end, paddr_after_kimage_aligned)).chain( - self.filter(move |range| range.end() > paddr_after_kimage_aligned) + core::iter::once(PRange::new(kernel_end, paddr_after_kimage_aligned)) + .chain( + self.filter(move |range| { + range.end() > paddr_after_kimage_aligned + }) .map(move |range| { if range.start() < paddr_after_kimage_aligned { - let (_, right) = range.split_at(paddr_after_kimage_aligned); + let (_, right) = + range.split_at(paddr_after_kimage_aligned); right } else { range } }), - ) + ) } } diff --git a/src/kernel_init.rs b/src/kernel_init.rs index 65af41e4..64c220b2 100644 --- a/src/kernel_init.rs +++ b/src/kernel_init.rs @@ -19,7 +19,8 @@ fn setup_kernel_page_array(alloc: BasicPageAllocRef, count_pages: usize) { // Map kernel page array. const V_KERNEL_PAGE_ARRAY_START: VAddr = VAddr::from(0xffffff8040000000); - let range = VRange::from(V_KERNEL_PAGE_ARRAY_START).grow(PAGE_SIZE * count_pages); + let range = + VRange::from(V_KERNEL_PAGE_ARRAY_START).grow(PAGE_SIZE * count_pages); for pte in global_page_table.iter_kernel(range) { let attr = PageAttribute::PRESENT | PageAttribute::WRITE From 379696553466c0b8e248e2cb9bc3b1cf1a821431 Mon Sep 17 00:00:00 2001 From: greatbridf <greatbridf@icloud.com> Date: Sat, 24 Jan 2026 01:32:55 +0800 Subject: [PATCH 29/36] riscv64, hal: simplify ArchMemory::free_ram implementation We already have `FDT.present_ram().free_ram()`. Remove the impl in `ArchMemory` to avoid confusion. Signed-off-by: greatbridf <greatbridf@icloud.com> --- crates/eonix_hal/src/arch/riscv64/mm.rs | 25 +------------------------ 1 file changed, 1 insertion(+), 24 deletions(-) diff --git a/crates/eonix_hal/src/arch/riscv64/mm.rs b/crates/eonix_hal/src/arch/riscv64/mm.rs index 45d44c6f..3b6df07c 100644 --- a/crates/eonix_hal/src/arch/riscv64/mm.rs +++ b/crates/eonix_hal/src/arch/riscv64/mm.rs @@ -274,30 +274,7 @@ impl Memory for ArchMemory { } fn free_ram() -> impl Iterator<Item = PRange> { - unsafe extern "C" { - fn __kernel_start(); - fn __kernel_end(); - } - - let kernel_end = PAddr::from(__kernel_end as usize - KIMAGE_OFFSET); - let paddr_after_kimage_aligned = kernel_end.ceil_to(0x200000); - - core::iter::once(PRange::new(kernel_end, paddr_after_kimage_aligned)) - .chain( - Self::present_ram() - .filter(move |range| { - range.end() > paddr_after_kimage_aligned - }) - .map(move |range| { - if range.start() < paddr_after_kimage_aligned { - let (_, right) = - range.split_at(paddr_after_kimage_aligned); - right - } else { - range - } - }), - ) + FDT.present_ram().free_ram() } } From fe45e192b3a191e658b15c72ce44b324df71acde Mon Sep 17 00:00:00 2001 From: greatbridf <greatbridf@icloud.com> Date: Sat, 24 Jan 2026 01:51:32 +0800 Subject: [PATCH 30/36] style: add helper macros to retrieve symbol constants - Add `extern_symbol_value` to retrieve far relative symbol values. - Get `BSS_LENGTH` and `__kernel_end` using `extern_symbol_addr`. - Get `_ap_start` using `extern_symbol_value`. Signed-off-by: greatbridf <greatbridf@icloud.com> --- .../eonix_hal/src/arch/riscv64/bootstrap.rs | 28 +++++++------------ crates/eonix_hal/src/arch/riscv64/mm.rs | 8 ++---- crates/eonix_hal/src/lib.rs | 14 ++++++++++ 3 files changed, 27 insertions(+), 23 deletions(-) diff --git a/crates/eonix_hal/src/arch/riscv64/bootstrap.rs b/crates/eonix_hal/src/arch/riscv64/bootstrap.rs index 4e5afcfb..d317e448 100644 --- a/crates/eonix_hal/src/arch/riscv64/bootstrap.rs +++ b/crates/eonix_hal/src/arch/riscv64/bootstrap.rs @@ -36,6 +36,7 @@ use crate::bootstrap::BootStrapData; use crate::mm::{ ArchMemory, BasicPageAlloc, BasicPageAllocRef, ScopedAllocator, }; +use crate::{extern_symbol_addr, extern_symbol_value}; #[unsafe(link_section = ".bootstrap.stack")] static BOOT_STACK: [u8; 4096 * 16] = [0; 4096 * 16]; @@ -156,10 +157,6 @@ pub unsafe extern "C" fn riscv64_start(hart_id: usize, dtb_addr: PAddr) -> ! { } } -unsafe extern "C" { - fn BSS_LENGTH(); -} - /// TODO: /// 对kernel image添加更细的控制，或者不加也行 fn setup_kernel_page_table(alloc: BasicPageAllocRef) { @@ -171,17 +168,17 @@ fn setup_kernel_page_table(alloc: BasicPageAllocRef) { let attr = PageAttribute::WRITE | PageAttribute::READ - | PageAttribute::EXECUTE | PageAttribute::GLOBAL | PageAttribute::PRESENT; const KERNEL_BSS_START: VAddr = VAddr::from(0xffffffff40000000); + let bss_length = extern_symbol_addr!(BSS_LENGTH); + // Map kernel BSS - let bss_range = VRange::from(KERNEL_BSS_START).grow(BSS_LENGTH as usize); + let bss_range = VRange::from(KERNEL_BSS_START).grow(bss_length); for pte in global_page_table.iter_kernel(bss_range) { let page = alloc.alloc().unwrap(); - let attr = attr.difference(PageAttribute::EXECUTE); pte.set(page.into_raw(), attr.into()); } @@ -192,7 +189,7 @@ fn setup_kernel_page_table(alloc: BasicPageAllocRef) { core::ptr::write_bytes( KERNEL_BSS_START.addr() as *mut (), 0, - BSS_LENGTH as usize, + bss_length, ); } @@ -247,15 +244,6 @@ fn setup_cpu(alloc: impl FrameAlloc, hart_id: usize) { percpu_area.register(cpu.cpuid()); } -fn get_ap_start_addr() -> usize { - unsafe extern "C" { - fn _ap_start(); - } - static AP_START_VALUE: &'static unsafe extern "C" fn() = - &(_ap_start as unsafe extern "C" fn()); - unsafe { (AP_START_VALUE as *const _ as *const usize).read_volatile() } -} - fn bootstrap_smp(alloc: impl Allocator, page_alloc: &RefCell<BasicPageAlloc>) { let local_hart_id = CPU::local().cpuid(); let mut ap_count = 0; @@ -286,7 +274,11 @@ fn bootstrap_smp(alloc: impl Allocator, page_alloc: &RefCell<BasicPageAlloc>) { } unsafe { - hart_start(hart_id, PhysicalAddress::new(get_ap_start_addr()), 0); + hart_start( + hart_id, + PhysicalAddress::new(extern_symbol_value!(_ap_start)), + 0, + ); } while AP_COUNT.load(Ordering::Acquire) == ap_count { diff --git a/crates/eonix_hal/src/arch/riscv64/mm.rs b/crates/eonix_hal/src/arch/riscv64/mm.rs index 3b6df07c..6362cdca 100644 --- a/crates/eonix_hal/src/arch/riscv64/mm.rs +++ b/crates/eonix_hal/src/arch/riscv64/mm.rs @@ -16,6 +16,7 @@ use riscv::register::satp; use super::config::mm::{PHYS_MAP_VIRT, ROOT_PAGE_TABLE_PFN}; use super::fdt::{FdtExt, FDT}; use crate::arch::riscv64::config::mm::KIMAGE_OFFSET; +use crate::extern_symbol_addr; use crate::mm::BasicPageAlloc; const PAGE_TABLE_BASE: PFN = PFN::from_val(ROOT_PAGE_TABLE_PFN); @@ -291,12 +292,9 @@ where T: PresentRam, { fn free_ram(self) -> impl Iterator<Item = PRange> { - unsafe extern "C" { - fn __kernel_start(); - fn __kernel_end(); - } + let kernel_end = extern_symbol_addr!(__kernel_end) - KIMAGE_OFFSET; + let kernel_end = PAddr::from(kernel_end).ceil(); - let kernel_end = PAddr::from(__kernel_end as usize - KIMAGE_OFFSET); let paddr_after_kimage_aligned = kernel_end.ceil_to(0x200000); core::iter::once(PRange::new(kernel_end, paddr_after_kimage_aligned)) diff --git a/crates/eonix_hal/src/lib.rs b/crates/eonix_hal/src/lib.rs index d3bf7825..3f49a326 100644 --- a/crates/eonix_hal/src/lib.rs +++ b/crates/eonix_hal/src/lib.rs @@ -69,3 +69,17 @@ macro_rules! extern_symbol_addr { $crate::symbol_addr!($sym, $type) }}; } + +#[macro_export] +macro_rules! extern_symbol_value { + ($sym:ident) => {{ + unsafe extern "C" { + fn $sym(); + } + + static SYMBOL_ADDR: &'static unsafe extern "C" fn() = + &($sym as unsafe extern "C" fn()); + + unsafe { (SYMBOL_ADDR as *const _ as *const usize).read_volatile() } + }}; +} From 74156daa1e7dfe264e0ad2e049471ed3b4f42928 Mon Sep 17 00:00:00 2001 From: greatbridf <greatbridf@icloud.com> Date: Sat, 24 Jan 2026 01:53:22 +0800 Subject: [PATCH 31/36] style: reformat files Signed-off-by: greatbridf <greatbridf@icloud.com> --- crates/eonix_hal/src/lib.rs | 4 +- src/kernel/task/signal/signal_action.rs | 52 +++++++++++++++---------- 2 files changed, 34 insertions(+), 22 deletions(-) diff --git a/crates/eonix_hal/src/lib.rs b/crates/eonix_hal/src/lib.rs index 3f49a326..d8696994 100644 --- a/crates/eonix_hal/src/lib.rs +++ b/crates/eonix_hal/src/lib.rs @@ -11,7 +11,9 @@ pub mod mm; pub mod trap; pub mod fence { - pub use crate::arch::fence::{memory_barrier, read_memory_barrier, write_memory_barrier}; + pub use crate::arch::fence::{ + memory_barrier, read_memory_barrier, write_memory_barrier, + }; } pub mod fpu { diff --git a/src/kernel/task/signal/signal_action.rs b/src/kernel/task/signal/signal_action.rs index 708f9802..cbf81935 100644 --- a/src/kernel/task/signal/signal_action.rs +++ b/src/kernel/task/signal/signal_action.rs @@ -1,22 +1,24 @@ -use super::{KResult, SAVED_DATA_SIZE}; -use crate::{ - io::BufferFill as _, - kernel::{ - constants::{EFAULT, EINVAL}, - syscall::UserMut, - user::UserBuffer, - }, -}; -use alloc::{collections::btree_map::BTreeMap, sync::Arc}; +use alloc::collections::btree_map::BTreeMap; +use alloc::sync::Arc; use core::arch::naked_asm; -use eonix_hal::{fpu::FpuState, traits::trap::RawTrapContext, trap::TrapContext}; + +use eonix_hal::fpu::FpuState; +use eonix_hal::traits::trap::RawTrapContext; +use eonix_hal::trap::TrapContext; use eonix_mm::address::{Addr as _, AddrOps as _, VAddr}; use eonix_sync::Spin; -use posix_types::{ - ctypes::Long, - signal::{SigAction, SigActionHandler, SigActionRestorer, SigSet, Signal, TryFromSigAction}, - SIGNAL_NOW, +use posix_types::ctypes::Long; +use posix_types::signal::{ + SigAction, SigActionHandler, SigActionRestorer, SigSet, Signal, + TryFromSigAction, }; +use posix_types::SIGNAL_NOW; + +use super::{KResult, SAVED_DATA_SIZE}; +use crate::io::BufferFill as _; +use crate::kernel::constants::{EFAULT, EINVAL}; +use crate::kernel::syscall::UserMut; +use crate::kernel::user::UserBuffer; #[cfg(target_arch = "x86_64")] #[unsafe(naked)] @@ -139,7 +141,9 @@ impl SignalAction { handler, restorer, .. } = self else { - unreachable!("Default and Ignore actions should not be handled here"); + unreachable!( + "Default and Ignore actions should not be handled here" + ); }; let current_sp = VAddr::from(trap_ctx.get_stack_pointer()); @@ -167,7 +171,9 @@ impl SignalAction { target_arch = "riscv64", target_arch = "loongarch64" )))] - compile_error!("`vdso_sigreturn` is not implemented for this architecture"); + compile_error!( + "`vdso_sigreturn` is not implemented for this architecture" + ); #[cfg(target_arch = "x86_64")] { @@ -178,19 +184,22 @@ impl SignalAction { unsafe { // SAFETY: To prevent the compiler from optimizing this into `la` instructions // and causing a linking error. - (VDSO_SIGRETURN_ADDR as *const _ as *const usize).read_volatile() + (VDSO_SIGRETURN_ADDR as *const _ as *const usize) + .read_volatile() } } #[cfg(any(target_arch = "riscv64", target_arch = "loongarch64"))] { - static VDSO_RT_SIGRETURN_ADDR: &'static unsafe extern "C" fn() = + static VDSO_RT_SIGRETURN_ADDR: + &'static unsafe extern "C" fn() = &(vdso_rt_sigreturn as unsafe extern "C" fn()); unsafe { // SAFETY: To prevent the compiler from optimizing this into `la` instructions // and causing a linking error. - (VDSO_RT_SIGRETURN_ADDR as *const _ as *const usize).read_volatile() + (VDSO_RT_SIGRETURN_ADDR as *const _ as *const usize) + .read_volatile() } } }; @@ -201,7 +210,8 @@ impl SignalAction { Some(return_address), &[Long::new_val(signal.into_raw() as _).get()], |vaddr, data| -> Result<(), u32> { - let mut buffer = UserBuffer::new(UserMut::new(vaddr), data.len())?; + let mut buffer = + UserBuffer::new(UserMut::new(vaddr), data.len())?; for ch in data.iter() { buffer.copy(&ch)?.ok_or(EFAULT)?; } From 77176c2614a0063d55a82afed8abfbaa86052812 Mon Sep 17 00:00:00 2001 From: greatbridf <greatbridf@icloud.com> Date: Sat, 24 Jan 2026 02:07:08 +0800 Subject: [PATCH 32/36] riscv64, linker: make sure vdso lies inside .data With current ldscript, linkers will put vdso data after `__kernel_end`, which is buggy since we use the symbol to indicate the end of our kernel image and newly allocated pages may overwrite those positions. Change by place the vdso inside REGION_DATA. Remove old VDSO memory region. Align .data section end to page size border. Add a helper macro to retrieve .vdso section symbol addresses. Signed-off-by: greatbridf <greatbridf@icloud.com> --- crates/eonix_hal/src/arch/riscv64/link.x | 4 ++- crates/eonix_hal/src/arch/riscv64/memory.x | 1 - crates/eonix_hal/src/link.x.in | 4 +-- src/kernel/task/signal/signal_action.rs | 31 +++++++++------------- 4 files changed, 17 insertions(+), 23 deletions(-) diff --git a/crates/eonix_hal/src/arch/riscv64/link.x b/crates/eonix_hal/src/arch/riscv64/link.x index e348e1be..a74f0d0d 100644 --- a/crates/eonix_hal/src/arch/riscv64/link.x +++ b/crates/eonix_hal/src/arch/riscv64/link.x @@ -81,10 +81,12 @@ INSERT AFTER .rodata; SECTIONS { .vdso ALIGN(0x1000) : ALIGN(0x1000) { + VDSO_START = ABSOLUTE(.); + KEEP(*(.vdso .vdso.*)); . = ALIGN(0x1000); - } > VDSO AT> RAM + } > REGION_DATA AT> RAM VDSO_PADDR = LOADADDR(.vdso); } diff --git a/crates/eonix_hal/src/arch/riscv64/memory.x b/crates/eonix_hal/src/arch/riscv64/memory.x index 0dc7c4ff..f2029c9a 100644 --- a/crates/eonix_hal/src/arch/riscv64/memory.x +++ b/crates/eonix_hal/src/arch/riscv64/memory.x @@ -3,7 +3,6 @@ ENTRY(_start) MEMORY { RAM : org = 0x0000000080200000, len = 8M - VDSO : org = 0x00007f0000000000, len = 4K KBSS : org = 0xffffffff40000000, len = 2M KIMAGE : org = 0xffffffff80200000, len = 8M } diff --git a/crates/eonix_hal/src/link.x.in b/crates/eonix_hal/src/link.x.in index 81c269c2..eaabdfda 100644 --- a/crates/eonix_hal/src/link.x.in +++ b/crates/eonix_hal/src/link.x.in @@ -18,7 +18,7 @@ SECTIONS { __srodata = .; *(.rodata .rodata.*); - + . = ALIGN(8); PROVIDE(__eh_frame = .); @@ -41,7 +41,7 @@ SECTIONS { } > REGION_DATA AT> LINK_REGION_DATA - .data.after : + .data.after : ALIGN(0x1000) { __data_after = .; } > REGION_DATA AT> LINK_REGION_DATA diff --git a/src/kernel/task/signal/signal_action.rs b/src/kernel/task/signal/signal_action.rs index cbf81935..18348c32 100644 --- a/src/kernel/task/signal/signal_action.rs +++ b/src/kernel/task/signal/signal_action.rs @@ -20,6 +20,16 @@ use crate::kernel::constants::{EFAULT, EINVAL}; use crate::kernel::syscall::UserMut; use crate::kernel::user::UserBuffer; +macro_rules! vdso_sym_addr { + ($sym:expr) => {{ + const VDSO_START_VADDR: VAddr = VAddr::from(0x7f00_0000_0000); + let vdso_link_start = eonix_hal::extern_symbol_addr!(VDSO_START); + + eonix_hal::symbol_addr!($sym) - vdso_link_start + + VDSO_START_VADDR.addr() + }}; +} + #[cfg(target_arch = "x86_64")] #[unsafe(naked)] #[unsafe(link_section = ".vdso.sigreturn")] @@ -178,29 +188,12 @@ impl SignalAction { #[cfg(target_arch = "x86_64")] { // TODO: Check and use `vdso_rt_sigreturn` for x86 as well. - static VDSO_SIGRETURN_ADDR: &'static unsafe extern "C" fn() = - &(vdso_rt_sigreturn as unsafe extern "C" fn()); - - unsafe { - // SAFETY: To prevent the compiler from optimizing this into `la` instructions - // and causing a linking error. - (VDSO_SIGRETURN_ADDR as *const _ as *const usize) - .read_volatile() - } + vdso_sym_addr!(vdso_rt_sigreturn) } #[cfg(any(target_arch = "riscv64", target_arch = "loongarch64"))] { - static VDSO_RT_SIGRETURN_ADDR: - &'static unsafe extern "C" fn() = - &(vdso_rt_sigreturn as unsafe extern "C" fn()); - - unsafe { - // SAFETY: To prevent the compiler from optimizing this into `la` instructions - // and causing a linking error. - (VDSO_RT_SIGRETURN_ADDR as *const _ as *const usize) - .read_volatile() - } + vdso_sym_addr!(vdso_rt_sigreturn) } }; From 89f277cfd662d8a0a9ed521b3fa6ea2235eab242 Mon Sep 17 00:00:00 2001 From: greatbridf <greatbridf@icloud.com> Date: Sat, 24 Jan 2026 03:08:28 +0800 Subject: [PATCH 33/36] riscv64: rewrite FDT and present free memory parsing Strip out the memory used by the kernel and FDT data out of free memory block returned by FDT. Closes: #54 ("Random kernel freezing on process creation / exiting") Fixes: 4351cf55739f3 ("partial work: fix riscv64 bootstrap") Signed-off-by: greatbridf <greatbridf@icloud.com> --- .../eonix_hal/src/arch/riscv64/bootstrap.rs | 7 +- crates/eonix_hal/src/arch/riscv64/fdt.rs | 115 +++++++++++------- crates/eonix_hal/src/arch/riscv64/mm.rs | 36 +----- 3 files changed, 78 insertions(+), 80 deletions(-) diff --git a/crates/eonix_hal/src/arch/riscv64/bootstrap.rs b/crates/eonix_hal/src/arch/riscv64/bootstrap.rs index d317e448..ccb32527 100644 --- a/crates/eonix_hal/src/arch/riscv64/bootstrap.rs +++ b/crates/eonix_hal/src/arch/riscv64/bootstrap.rs @@ -29,7 +29,7 @@ use super::time::set_next_timer; use crate::arch::cpu::CPU; use crate::arch::fdt::{init_dtb_and_fdt, FdtExt, FDT}; use crate::arch::mm::{ - ArchPagingMode, ArchPhysAccess, FreeRam, PageAccessImpl, PageAttribute64, + ArchPagingMode, ArchPhysAccess, PageAccessImpl, PageAttribute64, RawPageTableSv48, GLOBAL_PAGE_TABLE, }; use crate::bootstrap::BootStrapData; @@ -113,13 +113,12 @@ unsafe extern "C" fn _start(hart_id: usize, dtb_addr: usize) { } pub unsafe extern "C" fn riscv64_start(hart_id: usize, dtb_addr: PAddr) -> ! { - let fdt = Fdt::from_ptr(ArchPhysAccess::as_ptr(dtb_addr).as_ptr()) - .expect("Failed to parse DTB from static memory."); + let fdt = unsafe { FdtExt::new(dtb_addr) }; let real_allocator = RefCell::new(BasicPageAlloc::new()); let alloc = BasicPageAllocRef::new(&real_allocator); - for range in fdt.present_ram().free_ram() { + for range in fdt.free_ram() { real_allocator.borrow_mut().add_range(range); } diff --git a/crates/eonix_hal/src/arch/riscv64/fdt.rs b/crates/eonix_hal/src/arch/riscv64/fdt.rs index 5efcc98d..908256c7 100644 --- a/crates/eonix_hal/src/arch/riscv64/fdt.rs +++ b/crates/eonix_hal/src/arch/riscv64/fdt.rs @@ -1,62 +1,95 @@ -use super::mm::{ArchPhysAccess, PresentRam}; -use crate::arch::riscv64::config::mm::KIMAGE_OFFSET; -use core::sync::atomic::{AtomicPtr, Ordering}; -use eonix_mm::address::{PAddr, PRange, PhysAccess}; +use core::ops::Deref; +use core::sync::atomic::{AtomicPtr, AtomicUsize, Ordering}; + +use eonix_mm::address::{Addr, AddrOps, PAddr, PRange, PhysAccess}; use eonix_sync_base::LazyLock; use fdt::Fdt; -static DTB_VIRT_PTR: AtomicPtr<u8> = AtomicPtr::new(core::ptr::null_mut()); -pub static FDT: LazyLock<Fdt<'static>> = LazyLock::new(|| unsafe { - Fdt::from_ptr(DTB_VIRT_PTR.load(Ordering::Acquire)) - .expect("Failed to parse DTB from static memory.") +use super::mm::ArchPhysAccess; +use crate::arch::riscv64::config::mm::KIMAGE_OFFSET; +use crate::extern_symbol_addr; + +static DTB_PADDR: AtomicUsize = AtomicUsize::new(0); +pub static FDT: LazyLock<FdtExt> = LazyLock::new(|| unsafe { + FdtExt::new(PAddr::from_val(DTB_PADDR.load(Ordering::Relaxed))) }); -pub trait FdtExt { - fn harts(&self) -> impl Iterator<Item = usize>; +pub struct FdtExt { + fdt: Fdt<'static>, + range: PRange, +} - fn hart_count(&self) -> usize { - self.harts().count() - } +impl FdtExt { + /// # Safety + /// The caller MUST ensure that [`addr`] points to valid FDT. + pub unsafe fn new(addr: PAddr) -> Self { + let fdt = unsafe { + Fdt::from_ptr(ArchPhysAccess::as_ptr(addr).as_ptr()) + .expect("Failed to parse DTB from static memory.") + }; - fn present_ram(&self) -> impl Iterator<Item = PRange>; -} + Self { + range: PRange::from(addr).grow(fdt.total_size()), + fdt, + } + } -impl FdtExt for Fdt<'_> { - fn harts(&self) -> impl Iterator<Item = usize> { + pub fn harts(&self) -> impl Iterator<Item = usize> { self.cpus().map(|cpu| cpu.ids().all()).flatten() } - fn present_ram(&self) -> impl Iterator<Item = PRange> + PresentRam { - struct Present<I>(I); - impl<I> PresentRam for Present<I> where I: Iterator<Item = PRange> {} - impl<I> Iterator for Present<I> - where - I: Iterator<Item = PRange>, - { - type Item = PRange; - - fn next(&mut self) -> Option<Self::Item> { - self.0.next() - } - } + pub fn hart_count(&self) -> usize { + self.harts().count() + } + pub fn present_ram(&self) -> impl Iterator<Item = PRange> { let mut index = 0; - Present(core::iter::from_fn(move || { - self.memory() + + core::iter::from_fn(move || { + let item = self + .memory() .regions() .filter_map(|region| { - region.size.map(|len| { - PRange::from(PAddr::from(region.starting_address as usize)).grow(len) - }) + let start = PAddr::from(region.starting_address as usize); + Some(start).zip(region.size) }) - .skip(index) - .next() - .inspect(|_| index += 1) - })) + .map(|(start, len)| PRange::from(start).grow(len)) + .nth(index); + + index += 1; + item + }) + } + + pub fn free_ram(&self) -> impl Iterator<Item = PRange> { + let kernel_end = extern_symbol_addr!(__kernel_end) - KIMAGE_OFFSET; + let kernel_end = PAddr::from(kernel_end).ceil(); + + // TODO: move this to some platform-specific crate + self.present_ram().map(move |mut range| { + // Strip out parts before __kernel_end + if range.overlap_with(&PRange::from(kernel_end)) { + (_, range) = range.split_at(kernel_end); + } + + // Strip out part after the FDT + if range.overlap_with(&self.range) { + (range, _) = range.split_at(self.range.start()); + } + + range + }) + } +} + +impl Deref for FdtExt { + type Target = Fdt<'static>; + + fn deref(&self) -> &Self::Target { + &self.fdt } } pub unsafe fn init_dtb_and_fdt(dtb_paddr: PAddr) { - let dtb_virt_ptr = ArchPhysAccess::as_ptr(dtb_paddr); - DTB_VIRT_PTR.store(dtb_virt_ptr.as_ptr(), Ordering::Release); + DTB_PADDR.store(dtb_paddr.addr(), Ordering::Relaxed); } diff --git a/crates/eonix_hal/src/arch/riscv64/mm.rs b/crates/eonix_hal/src/arch/riscv64/mm.rs index 6362cdca..7891f094 100644 --- a/crates/eonix_hal/src/arch/riscv64/mm.rs +++ b/crates/eonix_hal/src/arch/riscv64/mm.rs @@ -275,46 +275,12 @@ impl Memory for ArchMemory { } fn free_ram() -> impl Iterator<Item = PRange> { - FDT.present_ram().free_ram() + FDT.free_ram() } } pub type DefaultPagingMode = PagingModeSv48; -pub trait PresentRam: Iterator<Item = PRange> {} - -pub trait FreeRam: PresentRam { - fn free_ram(self) -> impl Iterator<Item = PRange>; -} - -impl<T> FreeRam for T -where - T: PresentRam, -{ - fn free_ram(self) -> impl Iterator<Item = PRange> { - let kernel_end = extern_symbol_addr!(__kernel_end) - KIMAGE_OFFSET; - let kernel_end = PAddr::from(kernel_end).ceil(); - - let paddr_after_kimage_aligned = kernel_end.ceil_to(0x200000); - - core::iter::once(PRange::new(kernel_end, paddr_after_kimage_aligned)) - .chain( - self.filter(move |range| { - range.end() > paddr_after_kimage_aligned - }) - .map(move |range| { - if range.start() < paddr_after_kimage_aligned { - let (_, right) = - range.split_at(paddr_after_kimage_aligned); - right - } else { - range - } - }), - ) - } -} - #[inline(always)] pub fn flush_tlb(vaddr: usize) { sfence_vma(0, vaddr); From 1b69f51b7c9c840b34d428d571bf4dd61282b146 Mon Sep 17 00:00:00 2001 From: greatbridf <greatbridf@icloud.com> Date: Sat, 24 Jan 2026 03:12:29 +0800 Subject: [PATCH 34/36] sysinit: pointee type should be u8 when using ptr::write_bytes Using *const (), no bytes are written to the position, which might result in uninitialized memory access. Fixes: ebd3d1224c01 ("change(x86): optimize bootstrap code, remove kinit.cpp") Fixes: 191877a3acd0 ("feat(hal): impl basic single hart bootstrap for riscv64") Signed-off-by: greatbridf <greatbridf@icloud.com> --- crates/eonix_hal/src/arch/riscv64/bootstrap.rs | 2 +- src/kernel_init.rs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/crates/eonix_hal/src/arch/riscv64/bootstrap.rs b/crates/eonix_hal/src/arch/riscv64/bootstrap.rs index ccb32527..b0c235aa 100644 --- a/crates/eonix_hal/src/arch/riscv64/bootstrap.rs +++ b/crates/eonix_hal/src/arch/riscv64/bootstrap.rs @@ -186,7 +186,7 @@ fn setup_kernel_page_table(alloc: BasicPageAllocRef) { unsafe { core::ptr::write_bytes( - KERNEL_BSS_START.addr() as *mut (), + KERNEL_BSS_START.addr() as *mut u8, 0, bss_length, ); diff --git a/src/kernel_init.rs b/src/kernel_init.rs index 64c220b2..2259f6cf 100644 --- a/src/kernel_init.rs +++ b/src/kernel_init.rs @@ -41,7 +41,7 @@ fn setup_kernel_page_array(alloc: BasicPageAllocRef, count_pages: usize) { unsafe { // SAFETY: We've just mapped the area with sufficient length. core::ptr::write_bytes( - V_KERNEL_PAGE_ARRAY_START.addr() as *mut (), + V_KERNEL_PAGE_ARRAY_START.addr() as *mut u8, 0, count_pages * PAGE_SIZE, ); From 22c4ed30dfec98c51397390b144f0e15199545cf Mon Sep 17 00:00:00 2001 From: greatbridf <greatbridf@icloud.com> Date: Sat, 24 Jan 2026 03:21:52 +0800 Subject: [PATCH 35/36] style: reformat file Signed-off-by: greatbridf <greatbridf@icloud.com> --- crates/eonix_hal/src/mm.rs | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/crates/eonix_hal/src/mm.rs b/crates/eonix_hal/src/mm.rs index c4b9bb74..c6c6a369 100644 --- a/crates/eonix_hal/src/mm.rs +++ b/crates/eonix_hal/src/mm.rs @@ -7,8 +7,8 @@ use eonix_mm::page_table::PageTableAlloc; use eonix_mm::paging::{BasicFolio, FrameAlloc, PAGE_SIZE, PFN}; pub use crate::arch::mm::{ - flush_tlb, flush_tlb_all, get_root_page_table_pfn, set_root_page_table_pfn, ArchMemory, - ArchPhysAccess, GLOBAL_PAGE_TABLE, + flush_tlb, flush_tlb_all, get_root_page_table_pfn, set_root_page_table_pfn, + ArchMemory, ArchPhysAccess, GLOBAL_PAGE_TABLE, }; pub struct BasicPageAlloc { @@ -87,7 +87,8 @@ impl BasicPageAlloc { panic!("Page allocator is full"); } - self.ranges[tail] = Some(PRange::new(range.start().ceil(), range.end().floor())); + self.ranges[tail] = + Some(PRange::new(range.start().ceil(), range.end().floor())); } pub fn alloc(&mut self, order: u32) -> PFN { @@ -147,7 +148,10 @@ impl<'a> ScopedAllocator<'a> { } } - pub fn with_alloc<'b, 'r, O>(&'r self, func: impl FnOnce(&'b ScopedAllocator<'a>) -> O) -> O + pub fn with_alloc<'b, 'r, O>( + &'r self, + func: impl FnOnce(&'b ScopedAllocator<'a>) -> O, + ) -> O where 'a: 'b, 'r: 'b, From ddfb5274e26c98638263297ea9cafdc56c24c274 Mon Sep 17 00:00:00 2001 From: greatbridf <greatbridf@icloud.com> Date: Sat, 24 Jan 2026 03:24:11 +0800 Subject: [PATCH 36/36] hal, mm: alloc basic folios from low to high addr This can shorten qemu memory map. Signed-off-by: greatbridf <greatbridf@icloud.com> --- crates/eonix_hal/src/mm.rs | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/crates/eonix_hal/src/mm.rs b/crates/eonix_hal/src/mm.rs index c6c6a369..ef006cb5 100644 --- a/crates/eonix_hal/src/mm.rs +++ b/crates/eonix_hal/src/mm.rs @@ -41,9 +41,8 @@ impl BasicPageAlloc { fn alloc_one(&mut self) -> PFN { assert_ne!(self.head, self.tail, "No free pages available"); let mut range = self.ranges[self.head].take().unwrap(); - range = range.shrink(PAGE_SIZE); - - let pfn = PFN::from(range.end()); + let pfn = PFN::from(range.start()); + range = PRange::new(range.start() + PAGE_SIZE, range.end()); if range.len() != 0 { self.ranges[self.head] = Some(range);